Compare commits
130
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f5e9a10363 | ||
|
|
8bde33804d | ||
|
|
c820d4b637 | ||
|
|
cde9fab2d2 | ||
|
|
b3b0a5262a | ||
|
|
4fc06ee233 | ||
|
|
6e20b53013 | ||
|
|
4e5f491952 | ||
|
|
a436c659e9 | ||
|
|
60a0dad94f | ||
|
|
70007a36b9 | ||
|
|
81b5258808 | ||
|
|
8d7f651ddc | ||
|
|
9f4a8c9a68 | ||
|
|
db834dfc73 | ||
|
|
39a7b2902a | ||
|
|
08bac1ad84 | ||
|
|
1b712aae2c | ||
|
|
45360f72dd | ||
|
|
dbbb495b42 | ||
|
|
0575bb4286 | ||
|
|
944ac0a03b | ||
|
|
c54d989fc7 | ||
|
|
278337dd46 | ||
|
|
0cd94a20ec | ||
|
|
cc8f4217db | ||
|
|
7650a7e7b6 | ||
|
|
d66d42d021 | ||
|
|
cf43ae6c75 | ||
|
|
b7c586cebb | ||
|
|
8e8015d3f8 | ||
|
|
14d931a8d7 | ||
|
|
9ea709af92 | ||
|
|
ebe71db564 | ||
|
|
0a27704fd4 | ||
|
|
ee9ebd6d24 | ||
|
|
02c1dadf67 | ||
|
|
95a6b0dc1e | ||
|
|
ecce953fbb | ||
|
|
6071994695 | ||
|
|
3b8d9a2e71 | ||
|
|
c60ccf6858 | ||
|
|
d30043b99b | ||
|
|
75f781ee87 | ||
|
|
b181ca6bd6 | ||
|
|
09266df638 | ||
|
|
2cae75920d | ||
|
|
e159576340 | ||
|
|
721c293382 | ||
|
|
04e5fac311 | ||
|
|
1bd1746ef2 | ||
|
|
72828c06e6 | ||
|
|
b1e2c5a948 | ||
|
|
cd9f2e17f7 | ||
|
|
41d0e3d195 | ||
|
|
2023ca5de9 | ||
|
|
d3ef4f03aa | ||
|
|
f269a39bbe | ||
|
|
c7b260adc2 | ||
|
|
e742c3543d | ||
|
|
16bcce3f04 | ||
|
|
83d3bf8309 | ||
|
|
ef0ea71d0a | ||
|
|
f39a051d9f | ||
|
|
5ff88047ff | ||
|
|
be22dd9c12 | ||
|
|
11a2b6643e | ||
|
|
d5cb46278d | ||
|
|
81c674a17e | ||
|
|
dbacbddf64 | ||
|
|
11bfb3ff6f | ||
|
|
a9b59e127f | ||
|
|
e95a25892f | ||
|
|
52adb7eab9 | ||
|
|
f5f99c9cae | ||
|
|
70f4c78905 | ||
|
|
2e201ed7fb | ||
|
|
13c548454b | ||
|
|
b575462c95 | ||
|
|
110a4952a8 | ||
|
|
4f2513c93f | ||
|
|
9b3e49c909 | ||
|
|
8e6133a8d0 | ||
|
|
83e235567f | ||
|
|
57f5819dfd | ||
|
|
bb14f2616b | ||
|
|
ad7eebee54 | ||
|
|
325ed08ed2 | ||
|
|
71918399c9 | ||
|
|
e94b6fc8ef | ||
|
|
fd1ad64692 | ||
|
|
480db0e74f | ||
|
|
40b9a69e5a | ||
|
|
3a9d5e47d4 | ||
|
|
033d31d420 | ||
|
|
a55de23fd6 | ||
|
|
743d6ef429 | ||
|
|
5c7146d245 | ||
|
|
09af862957 | ||
|
|
5061084d4c | ||
|
|
fdc4446206 | ||
|
|
68b5f36c9b | ||
|
|
474406702f | ||
|
|
7145fb3fa4 | ||
|
|
3cde198358 | ||
|
|
5d127ad4db | ||
|
|
53f9e55fd8 | ||
|
|
eef732e2ad | ||
|
|
e34ad78329 | ||
|
|
4b02cb2c84 | ||
|
|
41c3334568 | ||
|
|
6733d4e435 | ||
|
|
f576756359 | ||
|
|
445affb63d | ||
|
|
3536ef1fbb | ||
|
|
676781223d | ||
|
|
2f860a17c9 | ||
|
|
835df6b7e7 | ||
|
|
5366418881 | ||
|
|
3152d59056 | ||
|
|
234ab23155 | ||
|
|
a7b2324a3e | ||
|
|
3ed5b72459 | ||
|
|
8143a7f665 | ||
|
|
08f2db616f | ||
|
|
c8cac766c7 | ||
|
|
bc5cd12056 | ||
|
|
218fa38f8e | ||
|
|
fce7522267 | ||
|
|
3cc90e4573 |
@@ -0,0 +1,205 @@
|
|||||||
|
name: Trusted engineering checks
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [feat/single-node-delivery]
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: kf-onlinejudge-trusted-engineering
|
||||||
|
cancel-in-progress: false
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
check:
|
||||||
|
if: github.server_url == 'https://github.com' && github.repository == 'KiriAky107/KF-OnlineJudge'
|
||||||
|
runs-on: ubuntu-24.04
|
||||||
|
timeout-minutes: 30
|
||||||
|
defaults:
|
||||||
|
run:
|
||||||
|
shell: bash --noprofile --norc -euo pipefail {0}
|
||||||
|
env:
|
||||||
|
CI: "true"
|
||||||
|
RUSTUP_TOOLCHAIN: 1.99.0
|
||||||
|
CARGO_BUILD_JOBS: "2"
|
||||||
|
CARGO_NET_RETRY: "2"
|
||||||
|
UV_PYTHON: 3.13.16
|
||||||
|
UV_NO_PROGRESS: "1"
|
||||||
|
UV_HTTP_TIMEOUT: "30"
|
||||||
|
UV_HTTP_RETRIES: "2"
|
||||||
|
steps:
|
||||||
|
- name: Check out the pushed commit
|
||||||
|
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
|
||||||
|
timeout-minutes: 2
|
||||||
|
with:
|
||||||
|
persist-credentials: false
|
||||||
|
fetch-depth: 1
|
||||||
|
fetch-tags: false
|
||||||
|
submodules: false
|
||||||
|
lfs: false
|
||||||
|
|
||||||
|
- name: Set up Node.js
|
||||||
|
uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6
|
||||||
|
timeout-minutes: 3
|
||||||
|
with:
|
||||||
|
node-version: 26.11.1
|
||||||
|
check-latest: false
|
||||||
|
package-manager-cache: false
|
||||||
|
|
||||||
|
- name: Set up uv
|
||||||
|
uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7
|
||||||
|
timeout-minutes: 3
|
||||||
|
with:
|
||||||
|
version: 0.12.24
|
||||||
|
python-version: 3.13.16
|
||||||
|
enable-cache: false
|
||||||
|
|
||||||
|
- name: Install locked toolchains and dependencies
|
||||||
|
timeout-minutes: 10
|
||||||
|
run: |
|
||||||
|
test "$GITHUB_SERVER_URL" = https://github.com
|
||||||
|
test "$GITHUB_REPOSITORY" = KiriAky107/KF-OnlineJudge
|
||||||
|
test "$GITHUB_EVENT_NAME" = push
|
||||||
|
test "$GITHUB_REF" = refs/heads/feat/single-node-delivery
|
||||||
|
test "$(git rev-parse HEAD)" = "$GITHUB_SHA"
|
||||||
|
uv python install 3.13.16
|
||||||
|
rustup toolchain install 1.99.0 --profile minimal --component rustfmt --component clippy
|
||||||
|
npm install --global --ignore-scripts pnpm@11.28.4
|
||||||
|
make install
|
||||||
|
|
||||||
|
- name: Check toolchain versions and CI inputs
|
||||||
|
timeout-minutes: 1
|
||||||
|
run: |
|
||||||
|
test "$(node --version)" = v26.11.1
|
||||||
|
test "$(pnpm --version)" = 11.28.4
|
||||||
|
test "$(rustc --version | cut -d ' ' -f 2)" = 1.99.0
|
||||||
|
test "$(uv --version | cut -d ' ' -f 2)" = 0.12.24
|
||||||
|
backend/.venv/bin/python - <<'PYCODE'
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
assert sys.version_info[:3] == (3, 13, 16)
|
||||||
|
assert not any(name.startswith(('KF_', 'PGPASSWORD', 'PGSERVICE', 'DATABASE_URL', 'AMQP_URL'))
|
||||||
|
for name in os.environ), 'CI must not carry node credentials or VM flags'
|
||||||
|
assert not subprocess.check_output(['git', 'ls-files', '--', 'docs/', '.localplan/'])
|
||||||
|
PYCODE
|
||||||
|
|
||||||
|
- name: Check architecture and generated contracts
|
||||||
|
timeout-minutes: 3
|
||||||
|
run: |
|
||||||
|
uv run --project backend --locked python scripts/check_architecture.py
|
||||||
|
make contracts
|
||||||
|
git diff --exit-code -- contracts frontend/src/api/generated/schema.d.ts
|
||||||
|
|
||||||
|
- name: Check Backend and run ordinary tests
|
||||||
|
timeout-minutes: 5
|
||||||
|
working-directory: backend
|
||||||
|
run: |
|
||||||
|
uv run --locked ruff check . ../scripts
|
||||||
|
uv run --locked ruff format --check . ../scripts
|
||||||
|
uv run --locked pytest \
|
||||||
|
tests/test_contracts.py \
|
||||||
|
tests/test_judging_contracts.py \
|
||||||
|
tests/test_judging_v2_contracts.py \
|
||||||
|
tests/test_submission_performance_contract.py \
|
||||||
|
tests/test_admin_import_contracts.py \
|
||||||
|
tests/test_availability.py \
|
||||||
|
tests/test_dependency_errors.py \
|
||||||
|
tests/test_production_logging.py \
|
||||||
|
tests/test_delivery.py \
|
||||||
|
tests/test_delivery_io.py \
|
||||||
|
tests/test_delivery_proxy_startup.py \
|
||||||
|
tests/test_delivery_validation_work.py \
|
||||||
|
tests/test_delivery_bytecode.py \
|
||||||
|
tests/test_delivery_backup.py \
|
||||||
|
tests/test_delivery_restore_io.py \
|
||||||
|
tests/test_restore_io_contracts.py \
|
||||||
|
tests/test_restore_native_transport.py \
|
||||||
|
tests/test_operator_diagnostics.py \
|
||||||
|
tests/test_outbox_startup_diagnostics.py \
|
||||||
|
tests/test_delivery_role_projection.py \
|
||||||
|
tests/test_operator_imports.py \
|
||||||
|
tests/test_health.py \
|
||||||
|
tests/test_identity.py \
|
||||||
|
tests/test_coordinator_control.py \
|
||||||
|
--junitxml="$RUNNER_TEMP/kf-engineering-backend.xml"
|
||||||
|
uv run --locked python -m kf_oj.contracts.export --check
|
||||||
|
|
||||||
|
- name: Check Rust and run library and contract tests
|
||||||
|
timeout-minutes: 8
|
||||||
|
working-directory: judge
|
||||||
|
run: |
|
||||||
|
cargo fmt --all -- --check
|
||||||
|
cargo clippy --workspace --all-targets --locked -- -D warnings
|
||||||
|
cargo test --workspace --lib --locked
|
||||||
|
cargo test --package kf-contracts --locked
|
||||||
|
cargo test --package kf-sandbox --bin kf-restore-io --locked
|
||||||
|
|
||||||
|
- name: Check and build Frontend
|
||||||
|
timeout-minutes: 5
|
||||||
|
run: make check-frontend
|
||||||
|
|
||||||
|
- name: Parse development Compose configuration
|
||||||
|
timeout-minutes: 1
|
||||||
|
run: make infra-config
|
||||||
|
|
||||||
|
- name: Collect public contracts and engineering report
|
||||||
|
timeout-minutes: 1
|
||||||
|
run: |
|
||||||
|
backend/.venv/bin/python - <<'PYCODE'
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import xml.etree.ElementTree as ET
|
||||||
|
from pathlib import Path
|
||||||
|
source = subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip()
|
||||||
|
assert source == os.environ['GITHUB_SHA']
|
||||||
|
destination = Path(os.environ['RUNNER_TEMP']) / 'kf-engineering-artifacts'
|
||||||
|
destination.mkdir(mode=0o700)
|
||||||
|
names = subprocess.check_output(['git', 'ls-files', '-z', '--',
|
||||||
|
'contracts/openapi/platform.v1.json',
|
||||||
|
'contracts/schemas/*.json']).decode().split('\0')[:-1]
|
||||||
|
assert 1 <= len(names) <= 64
|
||||||
|
files, total = [], 0
|
||||||
|
for name in names:
|
||||||
|
path = Path(name)
|
||||||
|
assert not any(part.is_symlink() for part in (path, *path.parents))
|
||||||
|
assert path.is_file() and path.stat().st_size <= 2 * 1024**2
|
||||||
|
data = path.read_bytes()
|
||||||
|
json.loads(data)
|
||||||
|
total += len(data)
|
||||||
|
assert total <= 8 * 1024**2
|
||||||
|
output = destination / name
|
||||||
|
output.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
output.write_bytes(data)
|
||||||
|
files.append({'path': name, 'bytes': len(data),
|
||||||
|
'sha256': hashlib.sha256(data).hexdigest()})
|
||||||
|
junit = ET.parse(Path(os.environ['RUNNER_TEMP']) / 'kf-engineering-backend.xml')
|
||||||
|
suites = list(junit.getroot().iter('testsuite'))
|
||||||
|
backend = {name: sum(int(suite.attrib.get(name, 0)) for suite in suites)
|
||||||
|
for name in ('tests', 'failures', 'errors', 'skipped')}
|
||||||
|
assert backend['failures'] == backend['errors'] == 0
|
||||||
|
report = {'schema_version': 1, 'source_sha': source,
|
||||||
|
'scope': 'Trusted engineering checks only', 'backend': backend,
|
||||||
|
'toolchains': {'python': '3.13.16', 'rust': '1.99.0',
|
||||||
|
'node': '26.11.1', 'pnpm': '11.28.4', 'uv': '0.12.24'},
|
||||||
|
'files': files,
|
||||||
|
'not_run': ['real PostgreSQL acceptance', 'independent VM cases',
|
||||||
|
'user source execution', 'adverse archives and filesystem acceptance',
|
||||||
|
'Linux isolation acceptance', 'deployment']}
|
||||||
|
(destination / 'engineering-report.json').write_text(json.dumps(report, indent=2)+'\n')
|
||||||
|
PYCODE
|
||||||
|
|
||||||
|
- name: Upload public contracts and engineering report
|
||||||
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||||
|
timeout-minutes: 2
|
||||||
|
with:
|
||||||
|
name: engineering-checks-${{ github.sha }}
|
||||||
|
path: ${{ runner.temp }}/kf-engineering-artifacts
|
||||||
|
if-no-files-found: error
|
||||||
|
include-hidden-files: false
|
||||||
|
overwrite: false
|
||||||
|
retention-days: 1
|
||||||
|
compression-level: 6
|
||||||
@@ -16,6 +16,7 @@
|
|||||||
- 沙箱未验证隔离前拒绝执行;不得用直接运行宿主命令作为临时判题实现。
|
- 沙箱未验证隔离前拒绝执行;不得用直接运行宿主命令作为临时判题实现。
|
||||||
- 只建立当期需要的接口和实现;未实现能力明确报告,不提供模拟成绩或虚假任务成功状态。
|
- 只建立当期需要的接口和实现;未实现能力明确报告,不提供模拟成绩或虚假任务成功状态。
|
||||||
- 使用项目锁文件;新增依赖说明职责。凭据、私有代码和隐藏测试数据不进入仓库。
|
- 使用项目锁文件;新增依赖说明职责。凭据、私有代码和隐藏测试数据不进入仓库。
|
||||||
|
- 测试备份不得积累:测试结束并保存必要验收证据后,及时删除不再使用的备份、虚拟机磁盘/快照及临时工件;当前任务仍依赖的资源须列明用途和清理条件,被替代后立即清理,不能以保留日志为由长期保留大体积副本。
|
||||||
|
|
||||||
按变更运行 `make check-backend`、`make check-judge`、`make check-frontend` 和 `make architecture`。
|
按变更运行 `make check-backend`、`make check-judge`、`make check-frontend` 和 `make architecture`。
|
||||||
跨工程骨架或契约改动运行完整 `make check`,记录实际通过项和未验证的能力。
|
跨工程骨架或契约改动运行完整 `make check`,记录实际通过项和未验证的能力。
|
||||||
|
|||||||
@@ -5,14 +5,24 @@
|
|||||||
|
|
||||||
当前版本为 **0.0.1**:沙箱已有受控隔离、C/C++ 编译及 native 执行;平台已实现真实身份、
|
当前版本为 **0.0.1**:沙箱已有受控隔离、C/C++ 编译及 native 执行;平台已实现真实身份、
|
||||||
不可变题目、工件与幂等提交创建。事务 outbox、持久化 Coordinator、普通 Worker 和受控
|
不可变题目、工件与幂等提交创建。事务 outbox、持久化 Coordinator、普通 Worker 和受控
|
||||||
helper 工厂已接通 C/C++ 编译一次、多测试点评测;登录、题库、提交和个人成绩页面
|
helper 工厂已接通 C17/C++20、固定 CPython 3.13 与 OpenJDK 21 的普通题评测;登录、题库、提交和个人成绩页面
|
||||||
使用真实接口。训练、比赛、社区等扩展页面仍为开发预览。
|
使用真实接口。训练、比赛、社区等扩展页面仍为开发预览。
|
||||||
|
|
||||||
|
管理控制台使用同源 Cookie 和服务端维护权限,支持有界 ZIP 题包上传、候选预览、
|
||||||
|
不可变版本发布、题目下架/恢复及维护审计。初次授权与恢复走独立运维身份的本地 CLI;
|
||||||
|
导入 Worker 仅解析数据,不执行题包内容或学生代码。操作、限额和恢复步骤见
|
||||||
|
[题目维护运维说明](deploy/admin-problems.md)。
|
||||||
|
|
||||||
|
题目工作台支持当前版本的“本题我的提交”、右侧详情与完整个人详情。
|
||||||
|
性能展示读取受验证最终 AC 的 CPU 总和与内存峰值,只比较同版本、语言、精确工具链、
|
||||||
|
预算与可信测量环境的最近有效记录。缺测、旧环境无法验证或其他记录不足五条时明确显示
|
||||||
|
不可比较;不会填入示例成绩。口径、查询边界和权限见[提交模块说明](backend/src/kf_oj/modules/submissions/README.md)。
|
||||||
|
|
||||||
## 工程划分
|
## 工程划分
|
||||||
|
|
||||||
| 目录 | 职责 |
|
| 目录 | 职责 |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
| `backend/` | Python 模块化单体、平台 API、学习 Worker、待投递事件发布器及唯一数据库迁移入口 |
|
| `backend/` | Python 模块化单体、平台 API、题包导入 Worker、待投递事件发布器及唯一数据库迁移入口 |
|
||||||
| `judge/` | Rust workspace:协议、沙箱、判题编排、判题 Worker 和协调器 |
|
| `judge/` | Rust workspace:协议、沙箱、判题编排、判题 Worker 和协调器 |
|
||||||
| `frontend/` | 页面、路由、请求生命周期与生成的 API 类型 |
|
| `frontend/` | 页面、路由、请求生命周期与生成的 API 类型 |
|
||||||
| `contracts/` | 版本化 JSON Schema 与 OpenAPI 快照 |
|
| `contracts/` | 版本化 JSON Schema 与 OpenAPI 快照 |
|
||||||
@@ -27,12 +37,14 @@ Python 业务域分为 `identity`、`problems`、`submissions`、`community`、`
|
|||||||
|
|
||||||
Rust 分为 `kf-contracts`、`kf-sandbox`、`kf-engine`、`kf-worker`、`kf-coordinator` 五个 crate。
|
Rust 分为 `kf-contracts`、`kf-sandbox`、`kf-engine`、`kf-worker`、`kf-coordinator` 五个 crate。
|
||||||
协调器拥有执行状态写入权,判题 Worker 不直接写评测数据库。
|
协调器拥有执行状态写入权,判题 Worker 不直接写评测数据库。
|
||||||
|
Sandbox 内的 `kf-restore-io` 仅在固定离线恢复单元中复制和核验固定数据域,
|
||||||
|
通过共享有界协议接收文件记录;不启动程序、服务或写业务状态。
|
||||||
沙箱执行入口每次核验隔离与限额,任何未验证配置继续拒绝执行。
|
沙箱执行入口每次核验隔离与限额,任何未验证配置继续拒绝执行。
|
||||||
用户程序及隔离/超限验收只在独立、整体限额的无业务凭据测试 VM 内运行。
|
用户程序及隔离/超限验收只在独立、整体限额的无业务凭据测试 VM 内运行。
|
||||||
|
|
||||||
## 安装与检查
|
## 安装与检查
|
||||||
|
|
||||||
工具链基线:Python 3.13.16 + uv、Rust 1.99.0、Node 26.10.0 + pnpm 11.28.4。
|
工具链基线:Python 3.13.16 + uv、Rust 1.99.0、Node 26.11.1 + pnpm 11.28.4。
|
||||||
Python 和 Rust 版本由工程配置固定,三套工程分别提交依赖锁文件。
|
Python 和 Rust 版本由工程配置固定,三套工程分别提交依赖锁文件。
|
||||||
开发与构建使用 Linux 原生文件系统。
|
开发与构建使用 Linux 原生文件系统。
|
||||||
|
|
||||||
|
|||||||
@@ -9,6 +9,22 @@ upgrade head`,API 不在启动时创建表。迁移账号拥有该数据库对
|
|||||||
协调器组领取和更新评测、插入 attempt / 用例成绩及恢复通知,不能修改账号、题库或提交身份。
|
协调器组领取和更新评测、插入 attempt / 用例成绩及恢复通知,不能修改账号、题库或提交身份。
|
||||||
Worker 没有数据库角色,也不需要数据库连接配置。
|
Worker 没有数据库角色,也不需要数据库连接配置。
|
||||||
|
|
||||||
|
后续迁移另建 `kf_oj_identity_operator` 与 `kf_oj_problem_import` 两个受限 `NOLOGIN`
|
||||||
|
权限组。前者只通过固定账号授权函数维护管理员能力/启用状态,不能读取口令哈希、
|
||||||
|
直接修改角色字段或绕过最后一名启用管理员保护;后者仅处理已授权导入、候选/工件与
|
||||||
|
追加审计,不能写评测状态、attempt、终态或表现指标。判题 Worker 继续无数据库身份。
|
||||||
|
平台写入权按列收敛;当前会话与授权版本在每次维护事务内重新核对。
|
||||||
|
|
||||||
|
可信表现指标由 Coordinator 在有效最终报告事务中派生,平台、导入器和普通 Worker
|
||||||
|
不得自行写入。候选与发布版本/用例不可变,成功维护回执及审计只追加;清理保留所有
|
||||||
|
已发布版本、合法旧提交和未结束候选的工件引用。
|
||||||
|
|
||||||
|
升级前停止新维护写入、备份专用数据库与私有工件,并用独立迁移账号执行升级和
|
||||||
|
`uv run --locked alembic check`。既有登录角色继续绑定各自最小权限组,迁移账号不进入
|
||||||
|
运行服务配置。降级不作为丢弃候选或回滚一次发布的手段;含新语言/已引用数据时的
|
||||||
|
降级拒绝须先核对原因,不能清空历史来通过检查。账号 bootstrap/恢复步骤见
|
||||||
|
[维护运维说明](../../deploy/admin-problems.md)。
|
||||||
|
|
||||||
服务使用各自独立的普通 `LOGIN` 角色,分别加入对应权限组;登录角色不能拥有数据库/表、
|
服务使用各自独立的普通 `LOGIN` 角色,分别加入对应权限组;登录角色不能拥有数据库/表、
|
||||||
不能加入迁移账号或其他高权限组,且必须为 `NOSUPERUSER NOCREATEDB NOCREATEROLE NOBYPASSRLS`。
|
不能加入迁移账号或其他高权限组,且必须为 `NOSUPERUSER NOCREATEDB NOCREATEROLE NOBYPASSRLS`。
|
||||||
初始迁移拒绝已有的高权限同名组和继承其他角色的同名组。应用启动时不运行 migration。
|
初始迁移拒绝已有的高权限同名组和继承其他角色的同名组。应用启动时不运行 migration。
|
||||||
|
|||||||
@@ -10,6 +10,8 @@ from kf_oj.infra.db.base import Base
|
|||||||
from kf_oj.infra.db.session import build_engine
|
from kf_oj.infra.db.session import build_engine
|
||||||
from kf_oj.infra.messaging import models as messaging_models # noqa: F401
|
from kf_oj.infra.messaging import models as messaging_models # noqa: F401
|
||||||
from kf_oj.modules.identity import models as identity_models # noqa: F401
|
from kf_oj.modules.identity import models as identity_models # noqa: F401
|
||||||
|
from kf_oj.modules.operations import models as operations_models # noqa: F401
|
||||||
|
from kf_oj.modules.problems import import_models, maintenance_models # noqa: F401
|
||||||
from kf_oj.modules.problems import models as problem_models # noqa: F401
|
from kf_oj.modules.problems import models as problem_models # noqa: F401
|
||||||
from kf_oj.modules.submissions import models as submission_models # noqa: F401
|
from kf_oj.modules.submissions import models as submission_models # noqa: F401
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,119 @@
|
|||||||
|
"""Trusted final performance projection; no re-encoding or promotion of old reports."""
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision = "0003_submission_performance"
|
||||||
|
down_revision = "0002_multi_runtime"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column("judge_attempts", sa.Column("measurement_epoch", sa.String(64), nullable=True))
|
||||||
|
op.create_check_constraint(
|
||||||
|
op.f("ck_judge_attempts_measurement_epoch"),
|
||||||
|
"judge_attempts",
|
||||||
|
"measurement_epoch IS NULL OR measurement_epoch ~ '^[0-9a-f]{64}$'",
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"evaluation_performance",
|
||||||
|
sa.Column("evaluation_id", sa.Uuid(), primary_key=True),
|
||||||
|
sa.Column("submission_id", sa.Uuid(), nullable=False, unique=True),
|
||||||
|
sa.Column("attempt_id", sa.Uuid(), nullable=False, unique=True),
|
||||||
|
sa.Column("problem_version_id", sa.Uuid(), nullable=False),
|
||||||
|
sa.Column("language", sa.String(16), nullable=False),
|
||||||
|
sa.Column("comparison_group", sa.String(64), nullable=False),
|
||||||
|
sa.Column("measurement_epoch", sa.String(64), nullable=False),
|
||||||
|
sa.Column("report_digest", sa.String(64), nullable=False),
|
||||||
|
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("cpu_time_us", sa.BigInteger(), nullable=True),
|
||||||
|
sa.Column("memory_peak_bytes", sa.BigInteger(), nullable=True),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["evaluation_id", "problem_version_id"],
|
||||||
|
["evaluations.id", "evaluations.problem_version_id"],
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["submission_id", "problem_version_id"],
|
||||||
|
["submissions.id", "submissions.problem_version_id"],
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["attempt_id", "evaluation_id"], ["judge_attempts.id", "judge_attempts.evaluation_id"]
|
||||||
|
),
|
||||||
|
sa.CheckConstraint("language IN ('c17','cpp20','python313','java21')", name="language"),
|
||||||
|
sa.CheckConstraint("comparison_group ~ '^[0-9a-f]{64}$'", name="comparison_group"),
|
||||||
|
sa.CheckConstraint("measurement_epoch ~ '^[0-9a-f]{64}$'", name="measurement_epoch"),
|
||||||
|
sa.CheckConstraint("report_digest ~ '^[0-9a-f]{64}$'", name="report_digest"),
|
||||||
|
sa.CheckConstraint("cpu_time_us IS NULL OR cpu_time_us >= 0", name="cpu"),
|
||||||
|
sa.CheckConstraint("memory_peak_bytes IS NULL OR memory_peak_bytes >= 0", name="memory"),
|
||||||
|
)
|
||||||
|
for metric in ("cpu_time_us", "memory_peak_bytes"):
|
||||||
|
op.create_index(
|
||||||
|
f"ix_performance_{metric}_cohort",
|
||||||
|
"evaluation_performance",
|
||||||
|
["comparison_group", sa.text("finished_at DESC"), sa.text("evaluation_id DESC")],
|
||||||
|
postgresql_where=sa.text(f"{metric} IS NOT NULL"),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_submissions_owned_problem_history",
|
||||||
|
"submissions",
|
||||||
|
["account_id", "problem_version_id", sa.text("created_at DESC"), sa.text("id DESC")],
|
||||||
|
)
|
||||||
|
op.execute("""
|
||||||
|
CREATE FUNCTION public.kf_trusted_performance() RETURNS trigger LANGUAGE plpgsql
|
||||||
|
SET search_path = pg_catalog, public AS $$
|
||||||
|
BEGIN
|
||||||
|
IF TG_OP != 'INSERT' THEN
|
||||||
|
RAISE EXCEPTION 'final performance is immutable' USING ERRCODE = '23514';
|
||||||
|
END IF;
|
||||||
|
IF NOT EXISTS (
|
||||||
|
SELECT 1 FROM public.evaluations e
|
||||||
|
JOIN public.judge_attempts a ON a.id=e.current_attempt_id AND a.evaluation_id=e.id
|
||||||
|
JOIN public.submissions s ON s.id=e.submission_id
|
||||||
|
WHERE e.id=NEW.evaluation_id AND e.state='finished' AND e.verdict='accepted'
|
||||||
|
AND e.failure_code IS NULL AND a.cleanup_complete IS TRUE
|
||||||
|
AND e.current_attempt_id=NEW.attempt_id AND s.id=NEW.submission_id
|
||||||
|
AND s.problem_version_id=NEW.problem_version_id AND s.language=NEW.language
|
||||||
|
AND a.measurement_epoch=NEW.measurement_epoch
|
||||||
|
AND e.final_report_digest=NEW.report_digest AND a.report_digest=NEW.report_digest
|
||||||
|
AND e.updated_at=NEW.finished_at
|
||||||
|
AND e.final_report->>'cleanup_complete'='true'
|
||||||
|
AND (e.final_report->>'run_cpu_time_us')::bigint
|
||||||
|
IS NOT DISTINCT FROM NEW.cpu_time_us
|
||||||
|
AND (e.final_report->>'run_memory_peak_bytes')::bigint
|
||||||
|
IS NOT DISTINCT FROM NEW.memory_peak_bytes
|
||||||
|
) THEN
|
||||||
|
RAISE EXCEPTION 'performance requires a bound trusted final AC'
|
||||||
|
USING ERRCODE = '23514';
|
||||||
|
END IF;
|
||||||
|
RETURN NEW;
|
||||||
|
END $$
|
||||||
|
""")
|
||||||
|
op.execute("REVOKE ALL ON FUNCTION public.kf_trusted_performance() FROM PUBLIC")
|
||||||
|
op.execute("""
|
||||||
|
CREATE TRIGGER kf_trusted_performance BEFORE INSERT OR UPDATE OR DELETE
|
||||||
|
ON public.evaluation_performance FOR EACH ROW
|
||||||
|
EXECUTE FUNCTION public.kf_trusted_performance()
|
||||||
|
""")
|
||||||
|
op.execute("REVOKE ALL ON evaluation_performance FROM PUBLIC,kf_oj_platform,kf_oj_coordinator")
|
||||||
|
op.execute("GRANT SELECT ON evaluation_performance TO kf_oj_platform,kf_oj_coordinator")
|
||||||
|
op.execute("GRANT INSERT ON evaluation_performance TO kf_oj_coordinator")
|
||||||
|
# Existing attempt INSERT permission covers the new column. No UPDATE epoch permission:
|
||||||
|
# an unbound historical attempt must never be promoted by changing current configuration.
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.execute("""
|
||||||
|
DO $$ BEGIN
|
||||||
|
IF EXISTS(SELECT 1 FROM evaluation_performance) OR
|
||||||
|
EXISTS(SELECT 1 FROM judge_attempts WHERE measurement_epoch IS NOT NULL) THEN
|
||||||
|
RAISE EXCEPTION 'cannot discard trusted measurement bindings'
|
||||||
|
USING ERRCODE = '23514';
|
||||||
|
END IF;
|
||||||
|
END $$
|
||||||
|
""")
|
||||||
|
op.drop_table("evaluation_performance")
|
||||||
|
op.execute("DROP FUNCTION public.kf_trusted_performance()")
|
||||||
|
op.drop_index("ix_submissions_owned_problem_history", table_name="submissions")
|
||||||
|
op.drop_constraint(op.f("ck_judge_attempts_measurement_epoch"), "judge_attempts", type_="check")
|
||||||
|
op.drop_column("judge_attempts", "measurement_epoch")
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
"""Indexed group watermark for bounded aggregate cache invalidation."""
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision = "0004_performance_watermark"
|
||||||
|
down_revision = "0003_submission_performance"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_index(
|
||||||
|
"ix_performance_group_watermark",
|
||||||
|
"evaluation_performance",
|
||||||
|
["comparison_group", sa.text("finished_at DESC"), sa.text("evaluation_id DESC")],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index("ix_performance_group_watermark", table_name="evaluation_performance")
|
||||||
@@ -0,0 +1,150 @@
|
|||||||
|
"""Default-denied administrator authority and separate local operator/import roles."""
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision = "0005_identity_authority"
|
||||||
|
down_revision = "0004_performance_watermark"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column(
|
||||||
|
"accounts",
|
||||||
|
sa.Column("is_administrator", sa.Boolean(), server_default="false", nullable=False),
|
||||||
|
)
|
||||||
|
op.add_column(
|
||||||
|
"accounts",
|
||||||
|
sa.Column("authorization_version", sa.BigInteger(), server_default="1", nullable=False),
|
||||||
|
)
|
||||||
|
op.create_check_constraint(
|
||||||
|
op.f("ck_accounts_authorization_version"), "accounts", "authorization_version >= 1"
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_accounts_active_administrators",
|
||||||
|
"accounts",
|
||||||
|
["id"],
|
||||||
|
postgresql_where=sa.text("is_administrator AND enabled"),
|
||||||
|
)
|
||||||
|
# Logins and secrets are provisioned separately by an operator. Neither group
|
||||||
|
# may inherit an existing privileged role or quietly reuse a privileged login.
|
||||||
|
for role in ("kf_oj_identity_operator", "kf_oj_problem_import"):
|
||||||
|
op.execute(f"""
|
||||||
|
DO $$ BEGIN
|
||||||
|
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname='{role}') THEN
|
||||||
|
CREATE ROLE {role} NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE
|
||||||
|
NOREPLICATION NOBYPASSRLS;
|
||||||
|
ELSIF EXISTS (SELECT 1 FROM pg_roles WHERE rolname='{role}' AND
|
||||||
|
(rolcanlogin OR rolsuper OR rolcreatedb OR rolcreaterole
|
||||||
|
OR rolreplication OR rolbypassrls))
|
||||||
|
OR EXISTS (SELECT 1 FROM pg_auth_members m JOIN pg_roles r ON r.oid=m.member
|
||||||
|
WHERE r.rolname='{role}') THEN
|
||||||
|
RAISE EXCEPTION 'unsafe existing authority or import role';
|
||||||
|
END IF;
|
||||||
|
END $$
|
||||||
|
""")
|
||||||
|
op.execute(
|
||||||
|
"REVOKE ALL ON ALL TABLES IN SCHEMA public "
|
||||||
|
"FROM kf_oj_identity_operator,kf_oj_problem_import"
|
||||||
|
)
|
||||||
|
# Table REVOKE does not remove explicit column grants. Remove those too,
|
||||||
|
# including grants left by an earlier downgrade or manual role provisioning.
|
||||||
|
op.execute("""
|
||||||
|
DO $$ DECLARE t record; BEGIN
|
||||||
|
FOR t IN SELECT c.relname, string_agg(quote_ident(a.attname),',' ORDER BY a.attnum) cols
|
||||||
|
FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace
|
||||||
|
JOIN pg_attribute a ON a.attrelid=c.oid
|
||||||
|
WHERE n.nspname='public' AND c.relkind IN ('r','p','v','f')
|
||||||
|
AND a.attnum>0 AND NOT a.attisdropped
|
||||||
|
GROUP BY c.oid,c.relname LOOP
|
||||||
|
EXECUTE format('REVOKE SELECT (%s),INSERT (%s),UPDATE (%s),REFERENCES (%s) '
|
||||||
|
'ON TABLE public.%I FROM kf_oj_identity_operator,kf_oj_problem_import',
|
||||||
|
t.cols,t.cols,t.cols,t.cols,t.relname);
|
||||||
|
END LOOP;
|
||||||
|
END $$
|
||||||
|
""")
|
||||||
|
op.execute("GRANT USAGE ON SCHEMA public TO kf_oj_identity_operator,kf_oj_problem_import")
|
||||||
|
op.execute("REVOKE INSERT,UPDATE,DELETE ON accounts FROM kf_oj_platform")
|
||||||
|
op.execute(
|
||||||
|
"REVOKE INSERT(id,username,password_hash,enabled,created_at,"
|
||||||
|
"is_administrator,authorization_version),"
|
||||||
|
"UPDATE(id,username,password_hash,enabled,created_at,"
|
||||||
|
"is_administrator,authorization_version) "
|
||||||
|
"ON accounts FROM kf_oj_platform"
|
||||||
|
)
|
||||||
|
op.execute(
|
||||||
|
"GRANT INSERT(id,username,password_hash,enabled,created_at) ON accounts TO kf_oj_platform"
|
||||||
|
)
|
||||||
|
op.execute("GRANT UPDATE(enabled) ON accounts TO kf_oj_platform")
|
||||||
|
op.execute("GRANT SELECT ON accounts,sessions TO kf_oj_identity_operator")
|
||||||
|
op.execute("GRANT UPDATE(is_administrator,enabled) ON accounts TO kf_oj_identity_operator")
|
||||||
|
op.execute("GRANT UPDATE(revoked_at) ON sessions TO kf_oj_identity_operator")
|
||||||
|
op.execute("REVOKE ALL ON accounts,sessions FROM kf_oj_coordinator,kf_oj_problem_import")
|
||||||
|
# No import queue exists yet. Its role deliberately has no table privileges
|
||||||
|
# until the import migration grants only the queue/staging tables it needs.
|
||||||
|
op.execute("""
|
||||||
|
CREATE FUNCTION public.kf_account_authority_version() RETURNS trigger LANGUAGE plpgsql
|
||||||
|
SET search_path = pg_catalog, public AS $$
|
||||||
|
BEGIN
|
||||||
|
IF NEW.authorization_version IS DISTINCT FROM OLD.authorization_version THEN
|
||||||
|
RAISE EXCEPTION 'authorization version is server derived' USING ERRCODE='23514';
|
||||||
|
END IF;
|
||||||
|
IF OLD.is_administrator AND NEW.enabled IS DISTINCT FROM OLD.enabled
|
||||||
|
AND NOT pg_has_role(current_user,'kf_oj_identity_operator','USAGE') THEN
|
||||||
|
RAISE EXCEPTION 'administrator enablement requires the local operator'
|
||||||
|
USING ERRCODE='42501';
|
||||||
|
END IF;
|
||||||
|
IF NEW.is_administrator IS DISTINCT FROM OLD.is_administrator
|
||||||
|
OR NEW.enabled IS DISTINCT FROM OLD.enabled THEN
|
||||||
|
NEW.authorization_version := OLD.authorization_version + 1;
|
||||||
|
END IF;
|
||||||
|
RETURN NEW;
|
||||||
|
END $$
|
||||||
|
""")
|
||||||
|
op.execute("""
|
||||||
|
CREATE FUNCTION public.kf_revoke_authority_sessions() RETURNS trigger LANGUAGE plpgsql
|
||||||
|
SET search_path = pg_catalog, public AS $$
|
||||||
|
BEGIN
|
||||||
|
IF NEW.authorization_version != OLD.authorization_version THEN
|
||||||
|
UPDATE public.sessions SET revoked_at=coalesce(revoked_at,clock_timestamp())
|
||||||
|
WHERE account_id=NEW.id AND revoked_at IS NULL;
|
||||||
|
END IF;
|
||||||
|
RETURN NEW;
|
||||||
|
END $$
|
||||||
|
""")
|
||||||
|
for function in ("kf_account_authority_version", "kf_revoke_authority_sessions"):
|
||||||
|
op.execute(f"REVOKE ALL ON FUNCTION public.{function}() FROM PUBLIC")
|
||||||
|
op.execute("""
|
||||||
|
CREATE TRIGGER kf_account_authority_version BEFORE UPDATE ON public.accounts
|
||||||
|
FOR EACH ROW EXECUTE FUNCTION public.kf_account_authority_version()
|
||||||
|
""")
|
||||||
|
op.execute("""
|
||||||
|
CREATE TRIGGER kf_revoke_authority_sessions AFTER UPDATE ON public.accounts
|
||||||
|
FOR EACH ROW EXECUTE FUNCTION public.kf_revoke_authority_sessions()
|
||||||
|
""")
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.execute("""
|
||||||
|
DO $$ BEGIN
|
||||||
|
IF EXISTS(SELECT 1 FROM accounts
|
||||||
|
WHERE is_administrator OR authorization_version != 1) THEN
|
||||||
|
RAISE EXCEPTION 'cannot discard authority or revocation history'
|
||||||
|
USING ERRCODE='23514';
|
||||||
|
END IF;
|
||||||
|
END $$
|
||||||
|
""")
|
||||||
|
for name in ("kf_revoke_authority_sessions", "kf_account_authority_version"):
|
||||||
|
op.execute(f"DROP TRIGGER {name} ON public.accounts")
|
||||||
|
op.execute(f"DROP FUNCTION public.{name}()")
|
||||||
|
op.execute("REVOKE ALL ON accounts,sessions FROM kf_oj_identity_operator")
|
||||||
|
op.execute("REVOKE UPDATE(is_administrator,enabled) ON accounts FROM kf_oj_identity_operator")
|
||||||
|
op.execute("REVOKE UPDATE(revoked_at) ON sessions FROM kf_oj_identity_operator")
|
||||||
|
op.drop_index("ix_accounts_active_administrators", table_name="accounts")
|
||||||
|
op.drop_constraint(op.f("ck_accounts_authorization_version"), "accounts", type_="check")
|
||||||
|
op.drop_column("accounts", "authorization_version")
|
||||||
|
op.drop_column("accounts", "is_administrator")
|
||||||
|
op.execute("GRANT INSERT,UPDATE,DELETE ON accounts TO kf_oj_platform")
|
||||||
|
# Shared NOLOGIN groups survive downgrade, as in the foundation migration;
|
||||||
|
# removing a cluster role could break another database using the same groups.
|
||||||
@@ -0,0 +1,226 @@
|
|||||||
|
"""Fixed local authority operations and serialized last-administrator protection."""
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision = "0006_authority_operations"
|
||||||
|
down_revision = "0005_identity_authority"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
OWNER = "kf_oj_identity_authority_owner"
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.execute(f"""
|
||||||
|
DO $$ BEGIN
|
||||||
|
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname='{OWNER}') THEN
|
||||||
|
CREATE ROLE {OWNER} NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE
|
||||||
|
NOREPLICATION NOBYPASSRLS;
|
||||||
|
ELSIF EXISTS (SELECT 1 FROM pg_roles WHERE rolname='{OWNER}' AND
|
||||||
|
(rolcanlogin OR rolsuper OR rolcreatedb OR rolcreaterole
|
||||||
|
OR rolreplication OR rolbypassrls))
|
||||||
|
OR EXISTS (SELECT 1 FROM pg_auth_members m JOIN pg_roles r ON r.oid=m.member
|
||||||
|
WHERE r.rolname='{OWNER}') THEN
|
||||||
|
RAISE EXCEPTION 'unsafe existing authority function owner';
|
||||||
|
END IF;
|
||||||
|
END $$
|
||||||
|
""")
|
||||||
|
op.create_table(
|
||||||
|
"identity_authority_guard",
|
||||||
|
sa.Column("id", sa.SmallInteger(), primary_key=True),
|
||||||
|
sa.Column("generation", sa.BigInteger(), nullable=False, server_default="0"),
|
||||||
|
sa.CheckConstraint("id = 1", name="singleton"),
|
||||||
|
sa.CheckConstraint("generation >= 0", name="generation"),
|
||||||
|
)
|
||||||
|
op.execute("INSERT INTO public.identity_authority_guard(id) VALUES(1)")
|
||||||
|
for role in (OWNER, "kf_oj_identity_operator"):
|
||||||
|
op.execute(f"REVOKE ALL ON ALL TABLES IN SCHEMA public FROM {role}")
|
||||||
|
op.execute(f"""
|
||||||
|
REVOKE SELECT(id,username,password_hash,enabled,created_at,
|
||||||
|
is_administrator,authorization_version),
|
||||||
|
UPDATE(id,username,password_hash,enabled,created_at,
|
||||||
|
is_administrator,authorization_version) ON public.accounts FROM {role}
|
||||||
|
""")
|
||||||
|
op.execute(f"""
|
||||||
|
REVOKE SELECT(token_hash,account_id,csrf_hash,expires_at,revoked_at,created_at),
|
||||||
|
UPDATE(token_hash,account_id,csrf_hash,expires_at,revoked_at,created_at)
|
||||||
|
ON public.sessions FROM {role}
|
||||||
|
""")
|
||||||
|
op.execute(f"GRANT USAGE ON SCHEMA public TO {OWNER}")
|
||||||
|
op.execute(f"""
|
||||||
|
GRANT SELECT(id,username,enabled,is_administrator,authorization_version),
|
||||||
|
UPDATE(enabled,is_administrator) ON public.accounts TO {OWNER}
|
||||||
|
""")
|
||||||
|
op.execute(f"GRANT SELECT(account_id,revoked_at),UPDATE(revoked_at) ON sessions TO {OWNER}")
|
||||||
|
op.execute(f"GRANT SELECT,UPDATE(generation) ON identity_authority_guard TO {OWNER}")
|
||||||
|
# Every account write obtains the guard BEFORE it can lock an account row.
|
||||||
|
# Updating (rather than merely locking) the singleton also makes an old
|
||||||
|
# REPEATABLE READ/SERIALIZABLE snapshot fail safely with a serialization error.
|
||||||
|
op.execute("""
|
||||||
|
CREATE FUNCTION public.kf_serialize_account_authority() RETURNS trigger
|
||||||
|
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$
|
||||||
|
BEGIN
|
||||||
|
UPDATE public.identity_authority_guard SET generation=generation+1 WHERE id=1;
|
||||||
|
IF NOT FOUND THEN
|
||||||
|
RAISE EXCEPTION 'authority guard unavailable' USING ERRCODE='23514';
|
||||||
|
END IF;
|
||||||
|
RETURN NULL;
|
||||||
|
END $$
|
||||||
|
""")
|
||||||
|
op.execute("""
|
||||||
|
CREATE FUNCTION public.kf_protect_last_administrator() RETURNS trigger
|
||||||
|
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$
|
||||||
|
BEGIN
|
||||||
|
IF OLD.is_administrator AND OLD.enabled AND
|
||||||
|
(TG_OP='DELETE' OR NOT (NEW.is_administrator AND NEW.enabled)) AND
|
||||||
|
(SELECT count(*) FROM public.accounts WHERE is_administrator AND enabled) <= 1
|
||||||
|
THEN
|
||||||
|
RAISE EXCEPTION 'last enabled administrator is protected' USING ERRCODE='23514';
|
||||||
|
END IF;
|
||||||
|
IF TG_OP='DELETE' THEN RETURN OLD; END IF;
|
||||||
|
RETURN NEW;
|
||||||
|
END $$
|
||||||
|
""")
|
||||||
|
op.execute("""
|
||||||
|
CREATE OR REPLACE FUNCTION public.kf_account_authority_version() RETURNS trigger
|
||||||
|
LANGUAGE plpgsql SET search_path = pg_catalog, public AS $$
|
||||||
|
BEGIN
|
||||||
|
IF NEW.authorization_version IS DISTINCT FROM OLD.authorization_version THEN
|
||||||
|
RAISE EXCEPTION 'authorization version is server derived' USING ERRCODE='23514';
|
||||||
|
END IF;
|
||||||
|
IF OLD.is_administrator AND NEW.enabled IS DISTINCT FROM OLD.enabled
|
||||||
|
AND current_user != 'kf_oj_identity_authority_owner'
|
||||||
|
AND NOT pg_has_role(current_user,'kf_oj_identity_operator','USAGE') THEN
|
||||||
|
RAISE EXCEPTION 'administrator enablement requires the local operator'
|
||||||
|
USING ERRCODE='42501';
|
||||||
|
END IF;
|
||||||
|
IF NEW.is_administrator IS DISTINCT FROM OLD.is_administrator
|
||||||
|
OR NEW.enabled IS DISTINCT FROM OLD.enabled THEN
|
||||||
|
NEW.authorization_version := OLD.authorization_version + 1;
|
||||||
|
END IF;
|
||||||
|
RETURN NEW;
|
||||||
|
END $$
|
||||||
|
""")
|
||||||
|
op.execute("""
|
||||||
|
CREATE FUNCTION public.kf_manage_account_authority(p_username text,p_operation text)
|
||||||
|
RETURNS TABLE(account_id uuid,username text,enabled boolean,is_administrator boolean,
|
||||||
|
authorization_version bigint,changed boolean)
|
||||||
|
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$
|
||||||
|
DECLARE target record; desired_admin boolean; desired_enabled boolean; active bigint;
|
||||||
|
BEGIN
|
||||||
|
IF p_username IS NULL OR p_username !~ '^[a-z0-9_]{3,24}$' OR
|
||||||
|
p_operation IS NULL OR
|
||||||
|
p_operation NOT IN ('bootstrap','grant','revoke','enable','disable') THEN
|
||||||
|
RAISE EXCEPTION 'invalid authority operation' USING ERRCODE='22023';
|
||||||
|
END IF;
|
||||||
|
UPDATE public.identity_authority_guard g SET generation=g.generation+1 WHERE g.id=1;
|
||||||
|
IF NOT FOUND THEN
|
||||||
|
RAISE EXCEPTION 'authority guard unavailable' USING ERRCODE='23514';
|
||||||
|
END IF;
|
||||||
|
SELECT a.id,a.username,a.enabled,a.is_administrator,a.authorization_version
|
||||||
|
INTO target FROM public.accounts a WHERE a.username=p_username FOR UPDATE;
|
||||||
|
IF NOT FOUND THEN
|
||||||
|
RAISE EXCEPTION 'account not found' USING ERRCODE='P0002';
|
||||||
|
END IF;
|
||||||
|
desired_admin:=target.is_administrator; desired_enabled:=target.enabled;
|
||||||
|
SELECT count(*) INTO active FROM public.accounts a
|
||||||
|
WHERE a.is_administrator AND a.enabled;
|
||||||
|
IF p_operation='bootstrap' THEN
|
||||||
|
IF active>0 AND NOT (target.is_administrator AND target.enabled) THEN
|
||||||
|
RAISE EXCEPTION 'bootstrap requires no enabled administrator'
|
||||||
|
USING ERRCODE='23514';
|
||||||
|
END IF;
|
||||||
|
desired_admin:=true; desired_enabled:=true;
|
||||||
|
ELSIF p_operation='grant' THEN
|
||||||
|
IF active=0 THEN
|
||||||
|
RAISE EXCEPTION 'first administrator requires explicit bootstrap'
|
||||||
|
USING ERRCODE='23514';
|
||||||
|
END IF;
|
||||||
|
IF NOT target.enabled THEN
|
||||||
|
RAISE EXCEPTION 'enable the account before granting authority'
|
||||||
|
USING ERRCODE='23514';
|
||||||
|
END IF;
|
||||||
|
desired_admin:=true;
|
||||||
|
ELSIF p_operation='revoke' THEN desired_admin:=false;
|
||||||
|
ELSIF p_operation='enable' THEN desired_enabled:=true;
|
||||||
|
ELSE desired_enabled:=false;
|
||||||
|
END IF;
|
||||||
|
changed:=desired_admin IS DISTINCT FROM target.is_administrator OR
|
||||||
|
desired_enabled IS DISTINCT FROM target.enabled;
|
||||||
|
IF changed THEN
|
||||||
|
UPDATE public.accounts a SET enabled=desired_enabled,is_administrator=desired_admin
|
||||||
|
WHERE a.id=target.id;
|
||||||
|
END IF;
|
||||||
|
RETURN QUERY SELECT a.id,a.username::text,a.enabled,a.is_administrator,
|
||||||
|
a.authorization_version,changed FROM public.accounts a WHERE a.id=target.id;
|
||||||
|
END $$
|
||||||
|
""")
|
||||||
|
# A narrow NOLOGIN owner holds only these column privileges. The operator
|
||||||
|
# receives EXECUTE, never table writes or access to password/session digests.
|
||||||
|
op.execute(f"GRANT CREATE ON SCHEMA public TO {OWNER}")
|
||||||
|
for signature in (
|
||||||
|
"kf_serialize_account_authority()",
|
||||||
|
"kf_protect_last_administrator()",
|
||||||
|
"kf_manage_account_authority(text,text)",
|
||||||
|
):
|
||||||
|
op.execute(f"ALTER FUNCTION public.{signature} OWNER TO {OWNER}")
|
||||||
|
op.execute(f"REVOKE ALL ON FUNCTION public.{signature} FROM PUBLIC")
|
||||||
|
op.execute(f"REVOKE CREATE ON SCHEMA public FROM {OWNER}")
|
||||||
|
op.execute("""
|
||||||
|
GRANT EXECUTE ON FUNCTION public.kf_manage_account_authority(text,text)
|
||||||
|
TO kf_oj_identity_operator
|
||||||
|
""")
|
||||||
|
op.execute("""
|
||||||
|
CREATE TRIGGER kf_serialize_account_authority BEFORE INSERT OR UPDATE OR DELETE
|
||||||
|
ON public.accounts FOR EACH STATEMENT
|
||||||
|
EXECUTE FUNCTION public.kf_serialize_account_authority()
|
||||||
|
""")
|
||||||
|
op.execute("""
|
||||||
|
CREATE TRIGGER kf_protect_last_administrator BEFORE UPDATE OR DELETE
|
||||||
|
ON public.accounts FOR EACH ROW EXECUTE FUNCTION public.kf_protect_last_administrator()
|
||||||
|
""")
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.execute("""
|
||||||
|
DO $$ BEGIN
|
||||||
|
IF EXISTS(SELECT 1 FROM public.accounts WHERE is_administrator) THEN
|
||||||
|
RAISE EXCEPTION 'cannot remove active authority safeguards' USING ERRCODE='23514';
|
||||||
|
END IF;
|
||||||
|
END $$
|
||||||
|
""")
|
||||||
|
for name in ("kf_protect_last_administrator", "kf_serialize_account_authority"):
|
||||||
|
op.execute(f"DROP TRIGGER {name} ON public.accounts")
|
||||||
|
op.execute(f"DROP FUNCTION public.{name}()")
|
||||||
|
op.execute("DROP FUNCTION public.kf_manage_account_authority(text,text)")
|
||||||
|
op.drop_table("identity_authority_guard")
|
||||||
|
op.execute(f"REVOKE ALL ON accounts,sessions FROM {OWNER}")
|
||||||
|
op.execute(f"""
|
||||||
|
REVOKE SELECT(id,username,enabled,is_administrator,authorization_version),
|
||||||
|
UPDATE(enabled,is_administrator) ON accounts FROM {OWNER}
|
||||||
|
""")
|
||||||
|
op.execute(f"REVOKE SELECT(account_id,revoked_at),UPDATE(revoked_at) ON sessions FROM {OWNER}")
|
||||||
|
op.execute("GRANT SELECT ON accounts,sessions TO kf_oj_identity_operator")
|
||||||
|
op.execute("GRANT UPDATE(is_administrator,enabled) ON accounts TO kf_oj_identity_operator")
|
||||||
|
op.execute("GRANT UPDATE(revoked_at) ON sessions TO kf_oj_identity_operator")
|
||||||
|
# Preserve the v0005 trigger policy without referring to the function owner.
|
||||||
|
op.execute("""
|
||||||
|
CREATE OR REPLACE FUNCTION public.kf_account_authority_version() RETURNS trigger
|
||||||
|
LANGUAGE plpgsql SET search_path = pg_catalog, public AS $$
|
||||||
|
BEGIN
|
||||||
|
IF NEW.authorization_version IS DISTINCT FROM OLD.authorization_version THEN
|
||||||
|
RAISE EXCEPTION 'authorization version is server derived' USING ERRCODE='23514';
|
||||||
|
END IF;
|
||||||
|
IF OLD.is_administrator AND NEW.enabled IS DISTINCT FROM OLD.enabled
|
||||||
|
AND NOT pg_has_role(current_user,'kf_oj_identity_operator','USAGE') THEN
|
||||||
|
RAISE EXCEPTION 'administrator enablement requires the local operator'
|
||||||
|
USING ERRCODE='42501';
|
||||||
|
END IF;
|
||||||
|
IF NEW.is_administrator IS DISTINCT FROM OLD.is_administrator
|
||||||
|
OR NEW.enabled IS DISTINCT FROM OLD.enabled THEN
|
||||||
|
NEW.authorization_version := OLD.authorization_version + 1;
|
||||||
|
END IF;
|
||||||
|
RETURN NEW;
|
||||||
|
END $$
|
||||||
|
""")
|
||||||
@@ -0,0 +1,212 @@
|
|||||||
|
"""Durable imports/candidates, append-only events and accepted-authority worker port."""
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
from kf_oj.modules.operations.models import MaintenanceEvent
|
||||||
|
from kf_oj.modules.problems.import_models import (
|
||||||
|
CandidateCase,
|
||||||
|
ImportPool,
|
||||||
|
ProblemCandidate,
|
||||||
|
ProblemImportOperation,
|
||||||
|
)
|
||||||
|
|
||||||
|
revision = "0007_problem_imports"
|
||||||
|
down_revision = "0006_authority_operations"
|
||||||
|
branch_labels = depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def sql(script):
|
||||||
|
# asyncpg prepares one statement at a time; dollar-quoted bodies stay intact.
|
||||||
|
buffer, quote, dollar, i = [], False, False, 0
|
||||||
|
while i < len(script):
|
||||||
|
if not quote and script[i : i + 2] == "$$":
|
||||||
|
dollar = not dollar
|
||||||
|
buffer.append("$$")
|
||||||
|
i += 2
|
||||||
|
continue
|
||||||
|
character = script[i]
|
||||||
|
if not dollar and character == "'":
|
||||||
|
if quote and script[i : i + 2] == "''":
|
||||||
|
buffer.append("''")
|
||||||
|
i += 2
|
||||||
|
continue
|
||||||
|
quote = not quote
|
||||||
|
if character == ";" and not dollar and not quote:
|
||||||
|
if "".join(buffer).strip():
|
||||||
|
op.execute("".join(buffer))
|
||||||
|
buffer = []
|
||||||
|
else:
|
||||||
|
buffer.append(character)
|
||||||
|
i += 1
|
||||||
|
if "".join(buffer).strip():
|
||||||
|
op.execute("".join(buffer))
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade():
|
||||||
|
connection = op.get_bind()
|
||||||
|
# Cyclic candidate/operation link is installed only after both tables exist.
|
||||||
|
for model in (
|
||||||
|
ImportPool,
|
||||||
|
ProblemImportOperation,
|
||||||
|
ProblemCandidate,
|
||||||
|
CandidateCase,
|
||||||
|
):
|
||||||
|
model.__table__.create(connection)
|
||||||
|
# Freeze the historical shape. A later operations model must not cause a
|
||||||
|
# fresh database to skip ahead and conflict with a subsequent ALTER migration.
|
||||||
|
op.create_table(
|
||||||
|
"problem_maintenance_events",
|
||||||
|
sa.Column("id", sa.Uuid(), primary_key=True),
|
||||||
|
sa.Column(
|
||||||
|
"created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
|
||||||
|
),
|
||||||
|
sa.Column("actor_id", sa.Uuid(), sa.ForeignKey("accounts.id"), nullable=False),
|
||||||
|
sa.Column("authorization_version", sa.BigInteger(), nullable=False),
|
||||||
|
sa.Column("operation_id", sa.Uuid(), nullable=False),
|
||||||
|
sa.Column("action", sa.String(24), nullable=False),
|
||||||
|
sa.Column("target_id", sa.Uuid()),
|
||||||
|
sa.CheckConstraint("authorization_version > 0", name="authority"),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"action IN ('upload_created','upload_received','import_ready','import_failed',"
|
||||||
|
"'import_discarded','import_expired','published','disabled','restored')",
|
||||||
|
name="action",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_problem_maintenance_events_page", "problem_maintenance_events", ["created_at", "id"]
|
||||||
|
)
|
||||||
|
op.create_foreign_key(
|
||||||
|
"fk_problem_import_candidate",
|
||||||
|
"problem_import_operations",
|
||||||
|
"problem_candidates",
|
||||||
|
["candidate_id"],
|
||||||
|
["id"],
|
||||||
|
)
|
||||||
|
op.execute("INSERT INTO problem_import_pool(id,reserved_bytes) VALUES(1,0)")
|
||||||
|
sql("""
|
||||||
|
CREATE FUNCTION public.kf_lock_reserved_maintenance(actor uuid, epoch bigint)
|
||||||
|
RETURNS void LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$
|
||||||
|
DECLARE permitted boolean;
|
||||||
|
BEGIN
|
||||||
|
SELECT enabled AND is_administrator AND authorization_version=epoch
|
||||||
|
INTO permitted FROM public.accounts WHERE id=actor FOR SHARE;
|
||||||
|
IF permitted IS DISTINCT FROM true THEN
|
||||||
|
RAISE EXCEPTION 'problem maintenance required' USING ERRCODE='42501';
|
||||||
|
END IF;
|
||||||
|
END $$;
|
||||||
|
GRANT CREATE ON SCHEMA public TO kf_oj_identity_authority_owner;
|
||||||
|
ALTER FUNCTION public.kf_lock_reserved_maintenance(uuid,bigint)
|
||||||
|
OWNER TO kf_oj_identity_authority_owner;
|
||||||
|
REVOKE CREATE ON SCHEMA public FROM kf_oj_identity_authority_owner;
|
||||||
|
REVOKE ALL ON FUNCTION public.kf_lock_reserved_maintenance(uuid,bigint) FROM PUBLIC;
|
||||||
|
GRANT EXECUTE ON FUNCTION public.kf_lock_reserved_maintenance(uuid,bigint)
|
||||||
|
TO kf_oj_problem_import;
|
||||||
|
GRANT SELECT,INSERT,UPDATE ON problem_import_operations TO kf_oj_platform;
|
||||||
|
GRANT SELECT,UPDATE ON problem_import_pool TO kf_oj_platform,kf_oj_problem_import;
|
||||||
|
GRANT SELECT ON problem_candidates,problem_candidate_cases TO kf_oj_platform;
|
||||||
|
GRANT SELECT ON problem_import_operations TO kf_oj_problem_import;
|
||||||
|
GRANT UPDATE(state,updated_at,lease_until,attempts,fence,
|
||||||
|
reserved_bytes,failure_code,candidate_id)
|
||||||
|
ON problem_import_operations TO kf_oj_problem_import;
|
||||||
|
GRANT SELECT,INSERT,DELETE ON problem_candidates,problem_candidate_cases
|
||||||
|
TO kf_oj_problem_import;
|
||||||
|
GRANT SELECT,INSERT ON artifacts TO kf_oj_problem_import;
|
||||||
|
GRANT INSERT ON problem_maintenance_events TO kf_oj_platform,kf_oj_problem_import;
|
||||||
|
GRANT SELECT ON problem_maintenance_events TO kf_oj_platform;
|
||||||
|
""")
|
||||||
|
sql("""
|
||||||
|
CREATE FUNCTION public.kf_immutable_import_candidate() RETURNS trigger
|
||||||
|
LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$
|
||||||
|
DECLARE candidate uuid; current_state text;
|
||||||
|
BEGIN
|
||||||
|
IF TG_TABLE_NAME='problem_candidates' THEN candidate=OLD.id;
|
||||||
|
ELSE candidate=OLD.candidate_id; END IF;
|
||||||
|
SELECT o.state INTO current_state FROM public.problem_candidates c
|
||||||
|
JOIN public.problem_import_operations o ON o.id=c.operation_id WHERE c.id=candidate;
|
||||||
|
IF TG_OP='UPDATE' OR current_state NOT IN ('discarded','expired','failed') THEN
|
||||||
|
RAISE EXCEPTION 'immutable candidate' USING ERRCODE='23514';
|
||||||
|
END IF;
|
||||||
|
RETURN OLD;
|
||||||
|
END $$;
|
||||||
|
CREATE TRIGGER immutable_candidate BEFORE UPDATE OR DELETE ON problem_candidates
|
||||||
|
FOR EACH ROW EXECUTE FUNCTION public.kf_immutable_import_candidate();
|
||||||
|
CREATE TRIGGER immutable_candidate_case BEFORE UPDATE OR DELETE ON problem_candidate_cases
|
||||||
|
FOR EACH ROW EXECUTE FUNCTION public.kf_immutable_import_candidate();
|
||||||
|
CREATE FUNCTION public.kf_insert_import_candidate() RETURNS trigger
|
||||||
|
LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$
|
||||||
|
DECLARE operation uuid; current_state text;
|
||||||
|
BEGIN
|
||||||
|
IF TG_TABLE_NAME='problem_candidates' THEN operation=NEW.operation_id;
|
||||||
|
ELSE SELECT operation_id INTO operation FROM public.problem_candidates
|
||||||
|
WHERE id=NEW.candidate_id; END IF;
|
||||||
|
SELECT state INTO current_state FROM public.problem_import_operations
|
||||||
|
WHERE id=operation FOR SHARE;
|
||||||
|
IF current_state IS DISTINCT FROM 'validating' THEN
|
||||||
|
RAISE EXCEPTION 'candidate is not being validated' USING ERRCODE='23514';
|
||||||
|
END IF; RETURN NEW;
|
||||||
|
END $$;
|
||||||
|
CREATE TRIGGER insert_candidate BEFORE INSERT ON problem_candidates
|
||||||
|
FOR EACH ROW EXECUTE FUNCTION public.kf_insert_import_candidate();
|
||||||
|
CREATE TRIGGER insert_candidate_case BEFORE INSERT ON problem_candidate_cases
|
||||||
|
FOR EACH ROW EXECUTE FUNCTION public.kf_insert_import_candidate();
|
||||||
|
CREATE FUNCTION public.kf_fixed_import_receipt() RETURNS trigger
|
||||||
|
LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$ BEGIN
|
||||||
|
IF NEW.id IS DISTINCT FROM OLD.id OR NEW.owner_id IS DISTINCT FROM OLD.owner_id
|
||||||
|
OR NEW.authorization_version IS DISTINCT FROM OLD.authorization_version
|
||||||
|
OR NEW.archive_digest IS DISTINCT FROM OLD.archive_digest
|
||||||
|
OR NEW.archive_bytes IS DISTINCT FROM OLD.archive_bytes
|
||||||
|
OR NEW.created_at IS DISTINCT FROM OLD.created_at
|
||||||
|
OR NEW.expires_at IS DISTINCT FROM OLD.expires_at THEN
|
||||||
|
RAISE EXCEPTION 'immutable import receipt' USING ERRCODE='23514';
|
||||||
|
END IF;
|
||||||
|
IF OLD.state IN ('ready','published') AND NEW.state IN ('ready','published')
|
||||||
|
AND NEW.candidate_id IS DISTINCT FROM OLD.candidate_id THEN
|
||||||
|
RAISE EXCEPTION 'immutable completed candidate receipt' USING ERRCODE='23514';
|
||||||
|
END IF;
|
||||||
|
IF OLD.state='published' AND (NEW.state IS DISTINCT FROM OLD.state
|
||||||
|
OR NEW.published_version_id IS DISTINCT FROM OLD.published_version_id) THEN
|
||||||
|
RAISE EXCEPTION 'immutable published import receipt' USING ERRCODE='23514';
|
||||||
|
END IF;
|
||||||
|
IF NEW.state='ready' THEN
|
||||||
|
IF (OLD.state IS DISTINCT FROM 'ready' AND
|
||||||
|
(NOT pg_has_role(current_user,'kf_oj_problem_import','USAGE')
|
||||||
|
OR OLD.state!='validating')) OR NEW.candidate_id IS NULL
|
||||||
|
OR NOT EXISTS(SELECT 1 FROM public.problem_candidates c
|
||||||
|
WHERE c.id=NEW.candidate_id AND c.operation_id=NEW.id
|
||||||
|
AND c.case_count=(SELECT count(*) FROM public.problem_candidate_cases b
|
||||||
|
WHERE b.candidate_id=c.id)
|
||||||
|
AND c.case_count-1=(SELECT max(position) FROM public.problem_candidate_cases b
|
||||||
|
WHERE b.candidate_id=c.id)) THEN
|
||||||
|
RAISE EXCEPTION 'invalid candidate completion' USING ERRCODE='23514';
|
||||||
|
END IF;
|
||||||
|
END IF; RETURN NEW;
|
||||||
|
END $$;
|
||||||
|
CREATE TRIGGER fixed_import_receipt BEFORE UPDATE ON problem_import_operations
|
||||||
|
FOR EACH ROW EXECUTE FUNCTION public.kf_fixed_import_receipt();
|
||||||
|
""")
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade():
|
||||||
|
connection = op.get_bind()
|
||||||
|
if connection.exec_driver_sql(
|
||||||
|
"SELECT EXISTS(SELECT 1 FROM problem_import_operations)"
|
||||||
|
).scalar():
|
||||||
|
raise RuntimeError("cannot discard durable import history")
|
||||||
|
op.execute("DROP FUNCTION public.kf_lock_reserved_maintenance(uuid,bigint)")
|
||||||
|
op.drop_constraint(
|
||||||
|
"fk_problem_import_candidate", "problem_import_operations", type_="foreignkey"
|
||||||
|
)
|
||||||
|
for model in (
|
||||||
|
MaintenanceEvent,
|
||||||
|
CandidateCase,
|
||||||
|
ProblemCandidate,
|
||||||
|
ProblemImportOperation,
|
||||||
|
ImportPool,
|
||||||
|
):
|
||||||
|
model.__table__.drop(connection)
|
||||||
|
sql(
|
||||||
|
"DROP FUNCTION public.kf_immutable_import_candidate(); "
|
||||||
|
"DROP FUNCTION public.kf_fixed_import_receipt(); "
|
||||||
|
"DROP FUNCTION public.kf_insert_import_candidate()"
|
||||||
|
)
|
||||||
@@ -0,0 +1,117 @@
|
|||||||
|
"""Immutable maintenance receipts, explicit operator audit and read-only history."""
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
from kf_oj.modules.problems.maintenance_models import ProblemMaintenanceOperation
|
||||||
|
|
||||||
|
revision = "0008_problem_maintenance"
|
||||||
|
down_revision = "0007_problem_imports"
|
||||||
|
branch_labels = depends_on = None
|
||||||
|
OWNER = "kf_oj_identity_authority_owner"
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade():
|
||||||
|
ProblemMaintenanceOperation.__table__.create(op.get_bind())
|
||||||
|
op.execute("GRANT SELECT,INSERT ON problem_maintenance_operations TO kf_oj_platform")
|
||||||
|
op.add_column(
|
||||||
|
"problem_maintenance_events",
|
||||||
|
sa.Column("actor_kind", sa.String(8), nullable=False, server_default=sa.text("'account'")),
|
||||||
|
)
|
||||||
|
op.add_column("problem_maintenance_events", sa.Column("operator_name", sa.String(63)))
|
||||||
|
op.alter_column("problem_maintenance_events", "actor_id", nullable=True)
|
||||||
|
op.drop_constraint(op.f("ck_problem_maintenance_events_action"), "problem_maintenance_events")
|
||||||
|
op.create_check_constraint(
|
||||||
|
"action",
|
||||||
|
"problem_maintenance_events",
|
||||||
|
"action IN ('upload_created','upload_received','import_ready','import_failed',"
|
||||||
|
"'import_discarded','import_expired','published','disabled','restored',"
|
||||||
|
"'authority_bootstrap','authority_grant','authority_revoke','authority_enable','authority_disable')",
|
||||||
|
)
|
||||||
|
op.create_check_constraint(
|
||||||
|
"actor",
|
||||||
|
"problem_maintenance_events",
|
||||||
|
"(actor_kind='account' AND actor_id IS NOT NULL AND operator_name IS NULL) OR "
|
||||||
|
"(actor_kind='operator' AND actor_id IS NULL AND operator_name IS NOT NULL)",
|
||||||
|
)
|
||||||
|
op.execute("""CREATE FUNCTION public.kf_append_only_maintenance() RETURNS trigger
|
||||||
|
LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$ BEGIN
|
||||||
|
RAISE EXCEPTION 'append-only maintenance history' USING ERRCODE='23514'; END $$""")
|
||||||
|
for table in ("problem_maintenance_operations", "problem_maintenance_events"):
|
||||||
|
op.execute(f"""CREATE TRIGGER append_only_maintenance BEFORE UPDATE OR DELETE ON {table}
|
||||||
|
FOR EACH ROW EXECUTE FUNCTION public.kf_append_only_maintenance()""")
|
||||||
|
# Preserve the already-verified authority implementation; its only entry
|
||||||
|
# point becomes a fixed audited wrapper owned by the same narrow NOLOGIN role.
|
||||||
|
op.execute(
|
||||||
|
"ALTER FUNCTION public.kf_manage_account_authority(text,text) "
|
||||||
|
"RENAME TO kf_manage_account_authority_unlogged"
|
||||||
|
)
|
||||||
|
op.execute(
|
||||||
|
"REVOKE ALL ON FUNCTION public.kf_manage_account_authority_unlogged(text,text) "
|
||||||
|
"FROM PUBLIC,kf_oj_identity_operator"
|
||||||
|
)
|
||||||
|
op.execute(f"GRANT INSERT ON problem_maintenance_events TO {OWNER}")
|
||||||
|
op.execute("""
|
||||||
|
CREATE FUNCTION public.kf_manage_account_authority(p_username text,p_operation text)
|
||||||
|
RETURNS TABLE(account_id uuid,username text,enabled boolean,is_administrator boolean,
|
||||||
|
authorization_version bigint,changed boolean)
|
||||||
|
LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$
|
||||||
|
DECLARE result record; BEGIN
|
||||||
|
SELECT * INTO STRICT result FROM
|
||||||
|
public.kf_manage_account_authority_unlogged(p_username,p_operation);
|
||||||
|
IF result.changed THEN
|
||||||
|
INSERT INTO public.problem_maintenance_events
|
||||||
|
(id,created_at,actor_id,actor_kind,operator_name,authorization_version,
|
||||||
|
operation_id,action,target_id)
|
||||||
|
VALUES (gen_random_uuid(),clock_timestamp(),NULL,'operator',session_user,
|
||||||
|
result.authorization_version,gen_random_uuid(),'authority_' || p_operation,
|
||||||
|
result.account_id);
|
||||||
|
END IF;
|
||||||
|
RETURN QUERY SELECT result.account_id,result.username,result.enabled,
|
||||||
|
result.is_administrator,result.authorization_version,result.changed;
|
||||||
|
END $$""")
|
||||||
|
op.execute(f"GRANT CREATE ON SCHEMA public TO {OWNER}")
|
||||||
|
op.execute(f"ALTER FUNCTION public.kf_manage_account_authority(text,text) OWNER TO {OWNER}")
|
||||||
|
op.execute(f"REVOKE CREATE ON SCHEMA public FROM {OWNER}")
|
||||||
|
op.execute("REVOKE ALL ON FUNCTION public.kf_manage_account_authority(text,text) FROM PUBLIC")
|
||||||
|
op.execute(
|
||||||
|
"GRANT EXECUTE ON FUNCTION public.kf_manage_account_authority(text,text) "
|
||||||
|
"TO kf_oj_identity_operator"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade():
|
||||||
|
if (
|
||||||
|
op.get_bind()
|
||||||
|
.exec_driver_sql(
|
||||||
|
"SELECT EXISTS(SELECT 1 FROM problem_maintenance_operations) OR "
|
||||||
|
"EXISTS(SELECT 1 FROM problem_maintenance_events WHERE actor_kind='operator')"
|
||||||
|
)
|
||||||
|
.scalar()
|
||||||
|
):
|
||||||
|
raise RuntimeError("cannot discard committed maintenance history")
|
||||||
|
op.execute("DROP FUNCTION public.kf_manage_account_authority(text,text)")
|
||||||
|
op.execute(
|
||||||
|
"ALTER FUNCTION public.kf_manage_account_authority_unlogged(text,text) "
|
||||||
|
"RENAME TO kf_manage_account_authority"
|
||||||
|
)
|
||||||
|
op.execute(
|
||||||
|
"GRANT EXECUTE ON FUNCTION public.kf_manage_account_authority(text,text) "
|
||||||
|
"TO kf_oj_identity_operator"
|
||||||
|
)
|
||||||
|
op.execute(f"REVOKE INSERT ON problem_maintenance_events FROM {OWNER}")
|
||||||
|
for table in ("problem_maintenance_operations", "problem_maintenance_events"):
|
||||||
|
op.execute(f"DROP TRIGGER append_only_maintenance ON {table}")
|
||||||
|
op.execute("DROP FUNCTION public.kf_append_only_maintenance()")
|
||||||
|
ProblemMaintenanceOperation.__table__.drop(op.get_bind())
|
||||||
|
op.drop_constraint(op.f("ck_problem_maintenance_events_actor"), "problem_maintenance_events")
|
||||||
|
op.drop_constraint(op.f("ck_problem_maintenance_events_action"), "problem_maintenance_events")
|
||||||
|
op.create_check_constraint(
|
||||||
|
"action",
|
||||||
|
"problem_maintenance_events",
|
||||||
|
"action IN ('upload_created','upload_received','import_ready','import_failed',"
|
||||||
|
"'import_discarded','import_expired','published','disabled','restored')",
|
||||||
|
)
|
||||||
|
op.alter_column("problem_maintenance_events", "actor_id", nullable=False)
|
||||||
|
op.drop_column("problem_maintenance_events", "operator_name")
|
||||||
|
op.drop_column("problem_maintenance_events", "actor_kind")
|
||||||
@@ -22,6 +22,7 @@ dependencies = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
[project.scripts]
|
[project.scripts]
|
||||||
|
kf-problem-import-worker = "kf_oj.workers.problem_import:main"
|
||||||
kf-accounts = "kf_oj.modules.identity.cli:main"
|
kf-accounts = "kf_oj.modules.identity.cli:main"
|
||||||
kf-problems = "kf_oj.modules.problems.cli:main"
|
kf-problems = "kf_oj.modules.problems.cli:main"
|
||||||
kf-artifacts = "kf_oj.infra.artifacts.cli:main"
|
kf-artifacts = "kf_oj.infra.artifacts.cli:main"
|
||||||
|
|||||||
@@ -0,0 +1,26 @@
|
|||||||
|
"""Uniform live session gates; domains still lock/recheck authority in write transactions."""
|
||||||
|
|
||||||
|
from typing import Annotated
|
||||||
|
|
||||||
|
from fastapi import Depends, HTTPException
|
||||||
|
|
||||||
|
from kf_oj.api.dependencies import current_principal, unsafe_principal
|
||||||
|
from kf_oj.modules.identity.public import Principal
|
||||||
|
|
||||||
|
|
||||||
|
async def maintenance_reader(
|
||||||
|
principal: Annotated[Principal, Depends(current_principal)],
|
||||||
|
) -> Principal:
|
||||||
|
if not principal.can_maintain_problems:
|
||||||
|
raise HTTPException(403, "problem maintenance required")
|
||||||
|
return principal
|
||||||
|
|
||||||
|
|
||||||
|
async def maintenance_writer(
|
||||||
|
principal: Annotated[Principal, Depends(unsafe_principal)],
|
||||||
|
) -> Principal:
|
||||||
|
# unsafe_principal already checks an explicit Origin and the live session's
|
||||||
|
# bound CSRF token. This capability check cannot replace the domain's port.
|
||||||
|
if not principal.can_maintain_problems:
|
||||||
|
raise HTTPException(403, "problem maintenance required")
|
||||||
|
return principal
|
||||||
@@ -6,7 +6,11 @@ from fastapi.exceptions import RequestValidationError
|
|||||||
from starlette.responses import JSONResponse
|
from starlette.responses import JSONResponse
|
||||||
|
|
||||||
from kf_oj import __version__
|
from kf_oj import __version__
|
||||||
|
from kf_oj.api.availability import AvailabilityMiddleware
|
||||||
from kf_oj.api.body_limits import BodyLimitMiddleware
|
from kf_oj.api.body_limits import BodyLimitMiddleware
|
||||||
|
from kf_oj.api.dependency_errors import install_handlers
|
||||||
|
from kf_oj.api.v1.admin_imports import router as import_router
|
||||||
|
from kf_oj.api.v1.admin_problems import router as maintenance_router
|
||||||
from kf_oj.api.v1.identity import router as identity_router
|
from kf_oj.api.v1.identity import router as identity_router
|
||||||
from kf_oj.api.v1.problems import router as problems_router
|
from kf_oj.api.v1.problems import router as problems_router
|
||||||
from kf_oj.api.v1.router import router
|
from kf_oj.api.v1.router import router
|
||||||
@@ -15,11 +19,17 @@ from kf_oj.api.v1.worker_artifacts import router as worker_router
|
|||||||
from kf_oj.core.logging import configure_logging
|
from kf_oj.core.logging import configure_logging
|
||||||
from kf_oj.core.settings import Settings
|
from kf_oj.core.settings import Settings
|
||||||
from kf_oj.infra.artifacts.repository import ArtifactRepository
|
from kf_oj.infra.artifacts.repository import ArtifactRepository
|
||||||
|
from kf_oj.infra.artifacts.staging import ImportStaging
|
||||||
from kf_oj.infra.artifacts.store import ArtifactStore
|
from kf_oj.infra.artifacts.store import ArtifactStore
|
||||||
from kf_oj.infra.coordinator import CoordinatorControl
|
from kf_oj.infra.coordinator import CoordinatorControl
|
||||||
from kf_oj.infra.db.session import build_engine, build_session_factory
|
from kf_oj.infra.db.session import build_engine, build_session_factory
|
||||||
from kf_oj.modules.identity.service import IdentityService
|
from kf_oj.modules.identity.service import IdentityService
|
||||||
|
from kf_oj.modules.operations.queries import MaintenanceAuditQueries
|
||||||
|
from kf_oj.modules.operations.service import MaintenanceAudit
|
||||||
|
from kf_oj.modules.problems.imports import ProblemImports
|
||||||
|
from kf_oj.modules.problems.maintenance import ProblemMaintenance
|
||||||
from kf_oj.modules.problems.service import ProblemService
|
from kf_oj.modules.problems.service import ProblemService
|
||||||
|
from kf_oj.modules.submissions.performance import SubmissionPerformanceQueries
|
||||||
from kf_oj.modules.submissions.queries import SubmissionQueries
|
from kf_oj.modules.submissions.queries import SubmissionQueries
|
||||||
from kf_oj.modules.submissions.service import SubmissionService
|
from kf_oj.modules.submissions.service import SubmissionService
|
||||||
from kf_oj.modules.submissions.worker_access import WorkerAccess
|
from kf_oj.modules.submissions.worker_access import WorkerAccess
|
||||||
@@ -30,11 +40,12 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
|||||||
|
|
||||||
@asynccontextmanager
|
@asynccontextmanager
|
||||||
async def lifespan(_: FastAPI):
|
async def lifespan(_: FastAPI):
|
||||||
configure_logging(settings.log_level)
|
configure_logging(settings.log_level, production=settings.environment == "production")
|
||||||
logger = logging.getLogger("kf_oj.api")
|
logger = logging.getLogger("kf_oj.api")
|
||||||
logger.info("Platform API started")
|
logger.info("Platform API started")
|
||||||
engine = None
|
engine = None
|
||||||
store = None
|
store = None
|
||||||
|
staging = None
|
||||||
try:
|
try:
|
||||||
if settings.database_url is not None:
|
if settings.database_url is not None:
|
||||||
engine = build_engine(settings)
|
engine = build_engine(settings)
|
||||||
@@ -42,6 +53,7 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
|||||||
app.state.identity = IdentityService(
|
app.state.identity = IdentityService(
|
||||||
app.state.sessions, lifetime_seconds=settings.session_lifetime_seconds
|
app.state.sessions, lifetime_seconds=settings.session_lifetime_seconds
|
||||||
)
|
)
|
||||||
|
app.state.submission_performance = SubmissionPerformanceQueries(app.state.sessions)
|
||||||
artifacts = None
|
artifacts = None
|
||||||
if settings.artifact_root is not None:
|
if settings.artifact_root is not None:
|
||||||
store = ArtifactStore(
|
store = ArtifactStore(
|
||||||
@@ -50,6 +62,28 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
|||||||
max_files=settings.artifact_max_files,
|
max_files=settings.artifact_max_files,
|
||||||
)
|
)
|
||||||
artifacts = ArtifactRepository(store)
|
artifacts = ArtifactRepository(store)
|
||||||
|
if (
|
||||||
|
settings.import_staging_root is not None
|
||||||
|
and settings.judge_admission is not None
|
||||||
|
):
|
||||||
|
staging = ImportStaging(settings.import_staging_root)
|
||||||
|
app.state.problem_imports = ProblemImports(
|
||||||
|
app.state.sessions,
|
||||||
|
staging,
|
||||||
|
app.state.identity,
|
||||||
|
MaintenanceAudit(),
|
||||||
|
settings.judge_admission,
|
||||||
|
)
|
||||||
|
app.state.problem_maintenance = ProblemMaintenance(
|
||||||
|
app.state.sessions,
|
||||||
|
app.state.identity,
|
||||||
|
MaintenanceAudit(),
|
||||||
|
artifacts,
|
||||||
|
settings.judge_admission,
|
||||||
|
)
|
||||||
|
app.state.maintenance_audit = MaintenanceAuditQueries(
|
||||||
|
app.state.sessions, app.state.identity
|
||||||
|
)
|
||||||
app.state.problems = ProblemService(
|
app.state.problems = ProblemService(
|
||||||
app.state.sessions, artifacts, settings.judge_admission
|
app.state.sessions, artifacts, settings.judge_admission
|
||||||
)
|
)
|
||||||
@@ -77,10 +111,16 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
|||||||
app.state.worker_access = None
|
app.state.worker_access = None
|
||||||
app.state.submissions = None
|
app.state.submissions = None
|
||||||
app.state.submission_queries = None
|
app.state.submission_queries = None
|
||||||
|
app.state.submission_performance = None
|
||||||
|
app.state.problem_imports = None
|
||||||
|
app.state.problem_maintenance = None
|
||||||
|
app.state.maintenance_audit = None
|
||||||
if engine is not None:
|
if engine is not None:
|
||||||
await engine.dispose()
|
await engine.dispose()
|
||||||
if store is not None:
|
if store is not None:
|
||||||
store.close()
|
store.close()
|
||||||
|
if staging is not None:
|
||||||
|
staging.close()
|
||||||
logger.info("Platform API stopped")
|
logger.info("Platform API stopped")
|
||||||
|
|
||||||
app = FastAPI(
|
app = FastAPI(
|
||||||
@@ -94,11 +134,14 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
|||||||
)
|
)
|
||||||
app.state.settings = settings
|
app.state.settings = settings
|
||||||
app.add_middleware(BodyLimitMiddleware)
|
app.add_middleware(BodyLimitMiddleware)
|
||||||
|
app.add_middleware(AvailabilityMiddleware, settings=settings)
|
||||||
|
|
||||||
@app.middleware("http")
|
@app.middleware("http")
|
||||||
async def private_responses(request, call_next):
|
async def private_responses(request, call_next):
|
||||||
response = await call_next(request)
|
response = await call_next(request)
|
||||||
if request.url.path.startswith(("/api/v1/auth", "/api/v1/submissions", "/api/v1/internal")):
|
if request.url.path.startswith(
|
||||||
|
("/api/v1/auth", "/api/v1/submissions", "/api/v1/internal", "/api/v1/admin")
|
||||||
|
):
|
||||||
response.headers["Cache-Control"] = "no-store"
|
response.headers["Cache-Control"] = "no-store"
|
||||||
return response
|
return response
|
||||||
|
|
||||||
@@ -106,9 +149,13 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
|||||||
async def invalid_request(_, __):
|
async def invalid_request(_, __):
|
||||||
return JSONResponse({"detail": "invalid request"}, status_code=422)
|
return JSONResponse({"detail": "invalid request"}, status_code=422)
|
||||||
|
|
||||||
|
install_handlers(app)
|
||||||
|
|
||||||
app.include_router(router, prefix="/api/v1")
|
app.include_router(router, prefix="/api/v1")
|
||||||
app.include_router(identity_router, prefix="/api/v1")
|
app.include_router(identity_router, prefix="/api/v1")
|
||||||
app.include_router(problems_router, prefix="/api/v1")
|
app.include_router(problems_router, prefix="/api/v1")
|
||||||
app.include_router(worker_router, prefix="/api/v1")
|
app.include_router(worker_router, prefix="/api/v1")
|
||||||
app.include_router(submission_router, prefix="/api/v1")
|
app.include_router(submission_router, prefix="/api/v1")
|
||||||
|
app.include_router(import_router, prefix="/api/v1")
|
||||||
|
app.include_router(maintenance_router, prefix="/api/v1")
|
||||||
return app
|
return app
|
||||||
|
|||||||
@@ -0,0 +1,50 @@
|
|||||||
|
"""Admission barrier while already accepted work retains its read/cancel paths."""
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
|
from starlette.responses import JSONResponse
|
||||||
|
|
||||||
|
from kf_oj.infra.availability import current
|
||||||
|
|
||||||
|
|
||||||
|
def needs_open(method: str, path: str) -> bool:
|
||||||
|
if method in {"GET", "HEAD", "OPTIONS"}:
|
||||||
|
return False
|
||||||
|
if method == "POST" and path == "/api/v1/auth/logout":
|
||||||
|
return False
|
||||||
|
if method == "POST" and re.fullmatch(r"/api/v1/submissions/[0-9a-f-]{36}/cancel", path):
|
||||||
|
return False
|
||||||
|
if method == "PUT" and re.fullmatch(
|
||||||
|
r"/api/v1/admin/problem-imports/[0-9a-f-]{36}/archive", path
|
||||||
|
):
|
||||||
|
# The domain route still requires a previously authorized immutable
|
||||||
|
# receipt. This cannot create a new upload or bypass its lease/epoch.
|
||||||
|
return False
|
||||||
|
return path.startswith("/api/v1/")
|
||||||
|
|
||||||
|
|
||||||
|
class AvailabilityMiddleware:
|
||||||
|
def __init__(self, app, *, settings):
|
||||||
|
self.app, self.settings = app, settings
|
||||||
|
|
||||||
|
async def __call__(self, scope, receive, send):
|
||||||
|
if (
|
||||||
|
scope["type"] == "http"
|
||||||
|
and self.settings.environment == "production"
|
||||||
|
and needs_open(scope["method"], scope.get("path", ""))
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
state = current(
|
||||||
|
self.settings.delivery_release_id, self.settings.delivery_configuration_id
|
||||||
|
)
|
||||||
|
allowed = state.state == "open"
|
||||||
|
except (OSError, ValueError, RecursionError):
|
||||||
|
allowed = False
|
||||||
|
if not allowed:
|
||||||
|
await JSONResponse(
|
||||||
|
{"detail": "platform mutations unavailable"},
|
||||||
|
status_code=503,
|
||||||
|
headers={"Cache-Control": "no-store", "Retry-After": "10"},
|
||||||
|
)(scope, receive, send)
|
||||||
|
return
|
||||||
|
await self.app(scope, receive, send)
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
"""Bound request bytes and upload time before schema parsing can allocate large inputs."""
|
"""Bound request bytes and upload time before schema parsing can allocate large inputs."""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import re
|
||||||
|
|
||||||
from starlette.responses import JSONResponse
|
from starlette.responses import JSONResponse
|
||||||
|
|
||||||
@@ -13,6 +14,14 @@ class BodyLimitMiddleware:
|
|||||||
if scope["type"] != "http" or scope["method"] in {"GET", "HEAD", "OPTIONS"}:
|
if scope["type"] != "http" or scope["method"] in {"GET", "HEAD", "OPTIONS"}:
|
||||||
await self.app(scope, receive, send)
|
await self.app(scope, receive, send)
|
||||||
return
|
return
|
||||||
|
if scope["method"] == "PUT" and re.fullmatch(
|
||||||
|
r"/api/v1/admin/problem-imports/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/archive",
|
||||||
|
scope["path"],
|
||||||
|
):
|
||||||
|
# Only this exact endpoint can stream. Its live administrator gate runs
|
||||||
|
# before receive(), then its service enforces actual bytes/chunks/deadline.
|
||||||
|
await self.app(scope, receive, send)
|
||||||
|
return
|
||||||
limit = 8192 if scope["path"] == "/api/v1/auth/login" else 1024 * 1024
|
limit = 8192 if scope["path"] == "/api/v1/auth/login" else 1024 * 1024
|
||||||
if scope["path"] == "/api/v1/submissions":
|
if scope["path"] == "/api/v1/submissions":
|
||||||
# Max256KiB UTF-8 source, worst-case JSON escaping plus bounded metadata.
|
# Max256KiB UTF-8 source, worst-case JSON escaping plus bounded metadata.
|
||||||
|
|||||||
@@ -0,0 +1,42 @@
|
|||||||
|
"""Bounded, non-secret API responses for transient dependency/capacity failures."""
|
||||||
|
|
||||||
|
import errno
|
||||||
|
|
||||||
|
from asyncpg import CannotConnectNowError, PostgresConnectionError
|
||||||
|
from fastapi import FastAPI
|
||||||
|
from sqlalchemy.exc import OperationalError
|
||||||
|
from sqlalchemy.exc import TimeoutError as PoolTimeoutError
|
||||||
|
from starlette.responses import JSONResponse
|
||||||
|
|
||||||
|
|
||||||
|
async def temporarily_unavailable(_, __):
|
||||||
|
# Write outcomes may be uncertain. Existing operation/submission UUIDs, not
|
||||||
|
# automatic server retries, own recovery from a lost response.
|
||||||
|
return JSONResponse(
|
||||||
|
{"detail": "service temporarily unavailable"},
|
||||||
|
status_code=503,
|
||||||
|
headers={"Cache-Control": "no-store", "Retry-After": "5"},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def filesystem_failure(_, error: OSError):
|
||||||
|
if error.errno in {errno.ENOSPC, errno.EDQUOT, errno.EMFILE, errno.ENFILE}:
|
||||||
|
return await temporarily_unavailable(_, error)
|
||||||
|
# Unrelated filesystem/programming failures retain server-error semantics.
|
||||||
|
# Never serialize path/source/query/credential text from an exception.
|
||||||
|
return JSONResponse(
|
||||||
|
{"detail": "internal server error"}, status_code=500, headers={"Cache-Control": "no-store"}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def install_handlers(app: FastAPI) -> None:
|
||||||
|
for error in (
|
||||||
|
ConnectionError,
|
||||||
|
TimeoutError,
|
||||||
|
OperationalError,
|
||||||
|
PoolTimeoutError,
|
||||||
|
PostgresConnectionError,
|
||||||
|
CannotConnectNowError,
|
||||||
|
):
|
||||||
|
app.add_exception_handler(error, temporarily_unavailable)
|
||||||
|
app.add_exception_handler(OSError, filesystem_failure)
|
||||||
@@ -0,0 +1,139 @@
|
|||||||
|
"""Administrator import contract; configured service availability is explicit."""
|
||||||
|
|
||||||
|
from typing import Annotated
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||||
|
from sqlalchemy.exc import DBAPIError
|
||||||
|
from starlette.requests import ClientDisconnect
|
||||||
|
|
||||||
|
from kf_oj.api.admin_dependencies import maintenance_reader, maintenance_writer
|
||||||
|
from kf_oj.contracts.admin_problems import (
|
||||||
|
CreateProblemImport,
|
||||||
|
ProblemImportCapabilities,
|
||||||
|
ProblemImportDetail,
|
||||||
|
ProblemImportPage,
|
||||||
|
ProblemImportSummary,
|
||||||
|
)
|
||||||
|
from kf_oj.infra.artifacts.store import ArtifactError
|
||||||
|
from kf_oj.modules.identity.public import MaintenanceDenied, Principal
|
||||||
|
from kf_oj.modules.problems.imports import ImportConflict, ImportMissing, ImportUnavailable
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/admin/problem-imports", tags=["problem-maintenance"])
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/capabilities", response_model=ProblemImportCapabilities)
|
||||||
|
async def capabilities(
|
||||||
|
request: Request, _: Annotated[Principal, Depends(maintenance_reader)]
|
||||||
|
) -> ProblemImportCapabilities:
|
||||||
|
service = getattr(request.app.state, "problem_imports", None)
|
||||||
|
if service is None:
|
||||||
|
return ProblemImportCapabilities(uploads_available=False, validation_available=False)
|
||||||
|
return await service.capabilities()
|
||||||
|
|
||||||
|
|
||||||
|
def service(request):
|
||||||
|
result = getattr(request.app.state, "problem_imports", None)
|
||||||
|
if result is None:
|
||||||
|
raise HTTPException(503, "problem imports unavailable")
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
async def operation(awaitable):
|
||||||
|
try:
|
||||||
|
return await awaitable
|
||||||
|
except MaintenanceDenied:
|
||||||
|
raise HTTPException(403, "problem maintenance required") from None
|
||||||
|
except ImportConflict:
|
||||||
|
raise HTTPException(409, "import request conflict") from None
|
||||||
|
except ImportMissing:
|
||||||
|
raise HTTPException(404, "import unavailable") from None
|
||||||
|
except TimeoutError:
|
||||||
|
raise HTTPException(408, "upload deadline exceeded") from None
|
||||||
|
except (ImportUnavailable, ArtifactError, OSError, DBAPIError):
|
||||||
|
raise HTTPException(503, "problem imports unavailable") from None
|
||||||
|
except ClientDisconnect:
|
||||||
|
raise HTTPException(400, "upload interrupted") from None
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("", status_code=201, response_model=ProblemImportSummary)
|
||||||
|
async def create(
|
||||||
|
body: CreateProblemImport,
|
||||||
|
request: Request,
|
||||||
|
principal: Annotated[Principal, Depends(maintenance_writer)],
|
||||||
|
):
|
||||||
|
if request.headers.get("content-type", "").split(";", 1)[0] != "application/json":
|
||||||
|
raise HTTPException(415, "JSON request required")
|
||||||
|
return await operation(service(request).create(principal, body))
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("", response_model=ProblemImportPage)
|
||||||
|
async def page(
|
||||||
|
request: Request,
|
||||||
|
principal: Annotated[Principal, Depends(maintenance_reader)],
|
||||||
|
after: UUID | None = None,
|
||||||
|
limit: Annotated[int, Query(ge=1, le=100)] = 20,
|
||||||
|
):
|
||||||
|
return await operation(service(request).list(principal, after, limit))
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{operation_id}", response_model=ProblemImportDetail)
|
||||||
|
async def detail(
|
||||||
|
operation_id: UUID,
|
||||||
|
request: Request,
|
||||||
|
principal: Annotated[Principal, Depends(maintenance_reader)],
|
||||||
|
):
|
||||||
|
return await operation(service(request).detail(principal, operation_id))
|
||||||
|
|
||||||
|
|
||||||
|
@router.put(
|
||||||
|
"/{operation_id}/archive",
|
||||||
|
response_model=ProblemImportSummary,
|
||||||
|
openapi_extra={
|
||||||
|
"requestBody": {
|
||||||
|
"required": True,
|
||||||
|
"content": {"application/zip": {"schema": {"type": "string", "format": "binary"}}},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
async def upload(
|
||||||
|
operation_id: UUID,
|
||||||
|
request: Request,
|
||||||
|
principal: Annotated[Principal, Depends(maintenance_writer)],
|
||||||
|
):
|
||||||
|
if request.url.path != f"/api/v1/admin/problem-imports/{operation_id}/archive":
|
||||||
|
raise HTTPException(422, "canonical operation UUID required")
|
||||||
|
if request.headers.get("content-type", "").split(";", 1)[0] != "application/zip":
|
||||||
|
raise HTTPException(415, "ZIP request required")
|
||||||
|
if request.headers.get("content-encoding") not in (None, "identity"):
|
||||||
|
raise HTTPException(415, "encoded upload unavailable")
|
||||||
|
length = request.headers.get("content-length")
|
||||||
|
if length is not None:
|
||||||
|
try:
|
||||||
|
if not length.isascii() or not length.isdecimal() or len(length) > 12:
|
||||||
|
raise ValueError
|
||||||
|
length = int(length)
|
||||||
|
except ValueError:
|
||||||
|
raise HTTPException(413, "invalid archive length") from None
|
||||||
|
|
||||||
|
async def chunks():
|
||||||
|
while True:
|
||||||
|
message = await request.receive()
|
||||||
|
if message["type"] == "http.disconnect":
|
||||||
|
raise ClientDisconnect()
|
||||||
|
if message["type"] != "http.request":
|
||||||
|
raise ImportConflict("invalid upload frame")
|
||||||
|
yield message.get("body", b"")
|
||||||
|
if not message.get("more_body", False):
|
||||||
|
return
|
||||||
|
|
||||||
|
return await operation(service(request).upload(principal, operation_id, chunks(), length))
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/{operation_id}/discard", response_model=ProblemImportSummary)
|
||||||
|
async def discard(
|
||||||
|
operation_id: UUID,
|
||||||
|
request: Request,
|
||||||
|
principal: Annotated[Principal, Depends(maintenance_writer)],
|
||||||
|
):
|
||||||
|
return await operation(service(request).discard(principal, operation_id))
|
||||||
@@ -0,0 +1,150 @@
|
|||||||
|
"""Live administrator projections and actor-bound immutable mutations."""
|
||||||
|
|
||||||
|
from typing import Annotated
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||||
|
from sqlalchemy.exc import DBAPIError
|
||||||
|
|
||||||
|
from kf_oj.api.admin_dependencies import maintenance_reader, maintenance_writer
|
||||||
|
from kf_oj.contracts.problem_maintenance import (
|
||||||
|
AuditAction,
|
||||||
|
ChangeProblemAvailability,
|
||||||
|
MaintainedProblem,
|
||||||
|
MaintainedProblemPage,
|
||||||
|
MaintainedVersionPage,
|
||||||
|
MaintenanceAuditPage,
|
||||||
|
MaintenanceReceipt,
|
||||||
|
PublishCandidate,
|
||||||
|
)
|
||||||
|
from kf_oj.infra.artifacts.store import ArtifactError
|
||||||
|
from kf_oj.modules.identity.public import MaintenanceDenied, Principal
|
||||||
|
from kf_oj.modules.problems.imports import ImportConflict, ImportMissing, ImportUnavailable
|
||||||
|
from kf_oj.modules.problems.public import ProblemUnavailable
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/admin", tags=["problem-maintenance"])
|
||||||
|
|
||||||
|
|
||||||
|
def service(request, name="problem_maintenance"):
|
||||||
|
result = getattr(request.app.state, name, None)
|
||||||
|
if result is None:
|
||||||
|
raise HTTPException(503, "problem maintenance unavailable")
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
async def operation(awaitable):
|
||||||
|
try:
|
||||||
|
return await awaitable
|
||||||
|
except MaintenanceDenied:
|
||||||
|
raise HTTPException(403, "problem maintenance required") from None
|
||||||
|
except ImportConflict:
|
||||||
|
raise HTTPException(409, "maintenance precondition conflict") from None
|
||||||
|
except ImportMissing:
|
||||||
|
raise HTTPException(404, "maintenance object unavailable") from None
|
||||||
|
except (ImportUnavailable, ArtifactError, ProblemUnavailable, OSError, DBAPIError):
|
||||||
|
raise HTTPException(503, "problem maintenance unavailable") from None
|
||||||
|
except ValueError:
|
||||||
|
raise HTTPException(422, "invalid maintenance request") from None
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/problems", response_model=MaintainedProblemPage)
|
||||||
|
async def page(
|
||||||
|
request: Request,
|
||||||
|
principal: Annotated[Principal, Depends(maintenance_reader)],
|
||||||
|
after: Annotated[str | None, Query(pattern=r"^[a-z0-9][a-z0-9_.-]{0,63}$")] = None,
|
||||||
|
limit: Annotated[int, Query(ge=1, le=100)] = 20,
|
||||||
|
enabled: bool | None = None,
|
||||||
|
):
|
||||||
|
return await operation(
|
||||||
|
service(request).page(principal, after=after, limit=limit, enabled=enabled)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/problem-slugs/{slug}", response_model=MaintainedProblem)
|
||||||
|
async def by_slug(
|
||||||
|
slug: str, request: Request, principal: Annotated[Principal, Depends(maintenance_reader)]
|
||||||
|
):
|
||||||
|
from pydantic import TypeAdapter
|
||||||
|
|
||||||
|
from kf_oj.contracts.judging import Label
|
||||||
|
|
||||||
|
try:
|
||||||
|
TypeAdapter(Label).validate_python(slug)
|
||||||
|
except ValueError:
|
||||||
|
raise HTTPException(422, "invalid problem slug") from None
|
||||||
|
return await operation(service(request).by_slug(principal, slug))
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/problems/{problem_id}", response_model=MaintainedProblem)
|
||||||
|
async def detail(
|
||||||
|
problem_id: UUID, request: Request, principal: Annotated[Principal, Depends(maintenance_reader)]
|
||||||
|
):
|
||||||
|
return await operation(service(request).detail(principal, problem_id))
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/problems/{problem_id}/versions", response_model=MaintainedVersionPage)
|
||||||
|
async def versions(
|
||||||
|
problem_id: UUID,
|
||||||
|
request: Request,
|
||||||
|
principal: Annotated[Principal, Depends(maintenance_reader)],
|
||||||
|
before: Annotated[int | None, Query(ge=1)] = None,
|
||||||
|
limit: Annotated[int, Query(ge=1, le=100)] = 20,
|
||||||
|
):
|
||||||
|
return await operation(
|
||||||
|
service(request).versions(principal, problem_id, before=before, limit=limit)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/problem-maintenance/{operation_id}", response_model=MaintenanceReceipt)
|
||||||
|
async def receipt(
|
||||||
|
operation_id: UUID,
|
||||||
|
request: Request,
|
||||||
|
principal: Annotated[Principal, Depends(maintenance_reader)],
|
||||||
|
):
|
||||||
|
return await operation(service(request).receipt(principal, operation_id))
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/problem-publications", response_model=MaintenanceReceipt, status_code=201)
|
||||||
|
async def publish(
|
||||||
|
body: PublishCandidate,
|
||||||
|
request: Request,
|
||||||
|
principal: Annotated[Principal, Depends(maintenance_writer)],
|
||||||
|
):
|
||||||
|
return await operation(service(request).publish(principal, body))
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/problems/{problem_id}/disable", response_model=MaintenanceReceipt)
|
||||||
|
async def disable(
|
||||||
|
problem_id: UUID,
|
||||||
|
body: ChangeProblemAvailability,
|
||||||
|
request: Request,
|
||||||
|
principal: Annotated[Principal, Depends(maintenance_writer)],
|
||||||
|
):
|
||||||
|
return await operation(
|
||||||
|
service(request).availability(principal, problem_id, body, enabled=False)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/problems/{problem_id}/restore", response_model=MaintenanceReceipt)
|
||||||
|
async def restore(
|
||||||
|
problem_id: UUID,
|
||||||
|
body: ChangeProblemAvailability,
|
||||||
|
request: Request,
|
||||||
|
principal: Annotated[Principal, Depends(maintenance_writer)],
|
||||||
|
):
|
||||||
|
return await operation(service(request).availability(principal, problem_id, body, enabled=True))
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/maintenance-audit", response_model=MaintenanceAuditPage)
|
||||||
|
async def audit(
|
||||||
|
request: Request,
|
||||||
|
principal: Annotated[Principal, Depends(maintenance_reader)],
|
||||||
|
before: Annotated[str | None, Query(max_length=100)] = None,
|
||||||
|
limit: Annotated[int, Query(ge=1, le=100)] = 20,
|
||||||
|
action: AuditAction | None = None,
|
||||||
|
):
|
||||||
|
return await operation(
|
||||||
|
service(request, "maintenance_audit").page(
|
||||||
|
principal, before=before, limit=limit, action=action
|
||||||
|
)
|
||||||
|
)
|
||||||
@@ -9,13 +9,24 @@ from kf_oj.api.dependencies import (
|
|||||||
trusted_origin,
|
trusted_origin,
|
||||||
unsafe_principal,
|
unsafe_principal,
|
||||||
)
|
)
|
||||||
from kf_oj.contracts.identity import CurrentUser, LoginRequest, LoginResponse
|
from kf_oj.contracts.identity import AccountCapabilities, CurrentUser, LoginRequest, LoginResponse
|
||||||
from kf_oj.modules.identity.public import AuthenticationFailed, LoginThrottled, Principal
|
from kf_oj.modules.identity.public import AuthenticationFailed, LoginThrottled, Principal
|
||||||
from kf_oj.modules.identity.service import IdentityService
|
from kf_oj.modules.identity.service import IdentityService
|
||||||
|
|
||||||
router = APIRouter(prefix="/auth", tags=["identity"])
|
router = APIRouter(prefix="/auth", tags=["identity"])
|
||||||
|
|
||||||
|
|
||||||
|
def current_user(principal: Principal) -> CurrentUser:
|
||||||
|
return CurrentUser(
|
||||||
|
id=principal.account_id,
|
||||||
|
username=principal.username,
|
||||||
|
capabilities=AccountCapabilities(
|
||||||
|
problem_maintenance=principal.can_maintain_problems,
|
||||||
|
authorization_version=str(principal.authorization_version),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.post("/login", response_model=LoginResponse, dependencies=[Depends(trusted_origin)])
|
@router.post("/login", response_model=LoginResponse, dependencies=[Depends(trusted_origin)])
|
||||||
async def login(
|
async def login(
|
||||||
body: LoginRequest,
|
body: LoginRequest,
|
||||||
@@ -58,7 +69,7 @@ async def login(
|
|||||||
)
|
)
|
||||||
response.headers["Cache-Control"] = "no-store"
|
response.headers["Cache-Control"] = "no-store"
|
||||||
return LoginResponse(
|
return LoginResponse(
|
||||||
user=CurrentUser(id=issued.principal.account_id, username=issued.principal.username),
|
user=current_user(issued.principal),
|
||||||
expires_at=issued.expires_at,
|
expires_at=issued.expires_at,
|
||||||
csrf_token=issued.csrf_token,
|
csrf_token=issued.csrf_token,
|
||||||
)
|
)
|
||||||
@@ -67,7 +78,7 @@ async def login(
|
|||||||
@router.get("/me", response_model=CurrentUser)
|
@router.get("/me", response_model=CurrentUser)
|
||||||
async def me(response: Response, principal: Annotated[Principal, Depends(current_principal)]):
|
async def me(response: Response, principal: Annotated[Principal, Depends(current_principal)]):
|
||||||
response.headers["Cache-Control"] = "no-store"
|
response.headers["Cache-Control"] = "no-store"
|
||||||
return CurrentUser(id=principal.account_id, username=principal.username)
|
return current_user(principal)
|
||||||
|
|
||||||
|
|
||||||
@router.post("/logout", status_code=204, dependencies=[Depends(unsafe_principal)])
|
@router.post("/logout", status_code=204, dependencies=[Depends(unsafe_principal)])
|
||||||
|
|||||||
@@ -4,6 +4,8 @@ from uuid import UUID
|
|||||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response
|
from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response
|
||||||
|
|
||||||
from kf_oj.api.dependencies import current_principal, unsafe_principal
|
from kf_oj.api.dependencies import current_principal, unsafe_principal
|
||||||
|
from kf_oj.contracts.judging_v2 import Language
|
||||||
|
from kf_oj.contracts.submission_performance import SubmissionPerformance
|
||||||
from kf_oj.contracts.submissions import (
|
from kf_oj.contracts.submissions import (
|
||||||
CancellationAccepted,
|
CancellationAccepted,
|
||||||
CreateSubmission,
|
CreateSubmission,
|
||||||
@@ -65,9 +67,11 @@ async def history(
|
|||||||
principal: Annotated[Principal, Depends(current_principal)],
|
principal: Annotated[Principal, Depends(current_principal)],
|
||||||
after: Annotated[str | None, Query(max_length=128)] = None,
|
after: Annotated[str | None, Query(max_length=128)] = None,
|
||||||
limit: Annotated[int, Query(ge=1, le=100)] = 20,
|
limit: Annotated[int, Query(ge=1, le=100)] = 20,
|
||||||
|
problem_version_id: UUID | None = None,
|
||||||
|
language: Language | None = None,
|
||||||
):
|
):
|
||||||
try:
|
try:
|
||||||
return await queries(request).page(principal, after, limit)
|
return await queries(request).page(principal, after, limit, problem_version_id, language)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
raise HTTPException(422, "invalid history request") from None
|
raise HTTPException(422, "invalid history request") from None
|
||||||
|
|
||||||
@@ -86,6 +90,23 @@ async def detail(
|
|||||||
raise HTTPException(503, "stored result unavailable") from None
|
raise HTTPException(503, "stored result unavailable") from None
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{submission_id}/performance", response_model=SubmissionPerformance)
|
||||||
|
async def performance(
|
||||||
|
submission_id: UUID,
|
||||||
|
request: Request,
|
||||||
|
principal: Annotated[Principal, Depends(current_principal)],
|
||||||
|
):
|
||||||
|
service = getattr(request.app.state, "submission_performance", None)
|
||||||
|
if service is None:
|
||||||
|
raise HTTPException(503, "performance service unavailable")
|
||||||
|
try:
|
||||||
|
return await service.detail(principal, submission_id)
|
||||||
|
except SubmissionUnavailable:
|
||||||
|
raise HTTPException(404, "submission unavailable") from None
|
||||||
|
except ValueError:
|
||||||
|
raise HTTPException(503, "performance unavailable") from None
|
||||||
|
|
||||||
|
|
||||||
@router.get("/{submission_id}/source", response_model=SubmissionSource)
|
@router.get("/{submission_id}/source", response_model=SubmissionSource)
|
||||||
async def source(
|
async def source(
|
||||||
submission_id: UUID,
|
submission_id: UUID,
|
||||||
|
|||||||
@@ -0,0 +1,180 @@
|
|||||||
|
"""Versioned problem-import DTOs; hidden bytes, paths and toolchain selection stay private."""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from enum import StrEnum
|
||||||
|
from typing import Annotated, Literal
|
||||||
|
|
||||||
|
from pydantic import Field, StrictBool, model_validator
|
||||||
|
|
||||||
|
from kf_oj.contracts.judging import Digest, Identifier, Label, RunLimits, Version, WireModel
|
||||||
|
from kf_oj.contracts.judging_v2 import Language
|
||||||
|
from kf_oj.contracts.problems import PublicLanguageConfiguration, PublicSample
|
||||||
|
|
||||||
|
MAX_ARCHIVE_BYTES = 64 * 1024**2
|
||||||
|
MAX_UNPACKED_BYTES = 64 * 1024**2
|
||||||
|
MAX_ARCHIVE_FILES = 534
|
||||||
|
MAX_ARCHIVE_NAME_BYTES = 128
|
||||||
|
MAX_UPLOAD_CHUNKS = 4096
|
||||||
|
MAX_UPLOAD_CHUNK_BYTES = 1024**2
|
||||||
|
UPLOAD_DEADLINE_SECONDS = 30
|
||||||
|
VALIDATION_DEADLINE_SECONDS = 60
|
||||||
|
MAX_IMPORT_ATTEMPTS = 2
|
||||||
|
IMPORT_LIFETIME_SECONDS = 86400
|
||||||
|
MAX_OWNER_CANDIDATES = 10
|
||||||
|
MAX_TOTAL_CANDIDATES = 32
|
||||||
|
STAGING_MAX_BYTES = 256 * 1024**2
|
||||||
|
STAGING_RESERVATION_BYTES = MAX_ARCHIVE_BYTES + MAX_UNPACKED_BYTES
|
||||||
|
|
||||||
|
|
||||||
|
class ImportState(StrEnum):
|
||||||
|
UPLOADING = "uploading"
|
||||||
|
UPLOADED = "uploaded"
|
||||||
|
VALIDATING = "validating"
|
||||||
|
READY = "ready"
|
||||||
|
FAILED = "failed"
|
||||||
|
PUBLISHED = "published"
|
||||||
|
DISCARDED = "discarded"
|
||||||
|
EXPIRED = "expired"
|
||||||
|
|
||||||
|
|
||||||
|
class ImportFailure(StrEnum):
|
||||||
|
INVALID_ARCHIVE = "invalid_archive"
|
||||||
|
INVALID_MANIFEST = "invalid_manifest"
|
||||||
|
INVALID_DATA = "invalid_data"
|
||||||
|
DIGEST_MISMATCH = "digest_mismatch"
|
||||||
|
POLICY_REJECTED = "policy_rejected"
|
||||||
|
UPLOAD_INTERRUPTED = "upload_interrupted"
|
||||||
|
UPLOAD_DEADLINE = "upload_deadline"
|
||||||
|
VALIDATION_DEADLINE = "validation_deadline"
|
||||||
|
VALIDATION_UNAVAILABLE = "validation_unavailable"
|
||||||
|
STORAGE_UNAVAILABLE = "storage_unavailable"
|
||||||
|
AUTHORITY_REVOKED = "authority_revoked"
|
||||||
|
EXPIRED = "expired"
|
||||||
|
|
||||||
|
|
||||||
|
class ProblemImportLimits(WireModel):
|
||||||
|
schema_version: Version = 1
|
||||||
|
archive_format: Literal["single_level_zip"] = "single_level_zip"
|
||||||
|
manifest_name: Literal["problem.json"] = "problem.json"
|
||||||
|
manifest_schema_versions: tuple[Literal[1, 2], ...] = (1, 2)
|
||||||
|
languages: tuple[Language, ...] = tuple(Language)
|
||||||
|
archive_bytes: int = MAX_ARCHIVE_BYTES
|
||||||
|
unpacked_bytes: int = MAX_UNPACKED_BYTES
|
||||||
|
files: int = MAX_ARCHIVE_FILES
|
||||||
|
filename_ascii_bytes: int = MAX_ARCHIVE_NAME_BYTES
|
||||||
|
upload_chunks: int = MAX_UPLOAD_CHUNKS
|
||||||
|
chunk_bytes: int = MAX_UPLOAD_CHUNK_BYTES
|
||||||
|
upload_deadline_seconds: int = UPLOAD_DEADLINE_SECONDS
|
||||||
|
validation_deadline_seconds: int = VALIDATION_DEADLINE_SECONDS
|
||||||
|
global_concurrency: Literal[1] = 1
|
||||||
|
owner_concurrency: Literal[1] = 1
|
||||||
|
owner_candidates: int = MAX_OWNER_CANDIDATES
|
||||||
|
total_candidates: int = MAX_TOTAL_CANDIDATES
|
||||||
|
staging_bytes: int = STAGING_MAX_BYTES
|
||||||
|
candidate_lifetime_seconds: int = IMPORT_LIFETIME_SECONDS
|
||||||
|
|
||||||
|
|
||||||
|
class ProblemImportCapabilities(WireModel):
|
||||||
|
schema_version: Version = 1
|
||||||
|
limits: ProblemImportLimits = ProblemImportLimits()
|
||||||
|
uploads_available: StrictBool
|
||||||
|
validation_available: StrictBool
|
||||||
|
|
||||||
|
|
||||||
|
class CreateProblemImport(WireModel):
|
||||||
|
schema_version: Version = 1
|
||||||
|
operation_id: Identifier
|
||||||
|
archive_digest: Digest
|
||||||
|
archive_bytes: Annotated[int, Field(strict=True, ge=1, le=MAX_ARCHIVE_BYTES)]
|
||||||
|
|
||||||
|
|
||||||
|
class LegacyImportBudget(WireModel):
|
||||||
|
compile_limits: RunLimits
|
||||||
|
run_limits: RunLimits
|
||||||
|
job_wall_time_us: Annotated[int, Field(strict=True, ge=1000, le=3600000000)]
|
||||||
|
|
||||||
|
|
||||||
|
class CandidatePreview(WireModel):
|
||||||
|
candidate_id: Identifier
|
||||||
|
slug: Label
|
||||||
|
title: Annotated[str, Field(strict=True, min_length=1, max_length=160)]
|
||||||
|
statement: Annotated[str, Field(strict=True, min_length=1, max_length=262144)]
|
||||||
|
samples: Annotated[tuple[PublicSample, ...], Field(max_length=10)]
|
||||||
|
judging_schema_version: Annotated[int, Field(strict=True, ge=1, le=2)]
|
||||||
|
languages: Annotated[tuple[Language, ...], Field(min_length=1, max_length=4)]
|
||||||
|
legacy_budget: LegacyImportBudget | None
|
||||||
|
language_configurations: Annotated[
|
||||||
|
tuple[PublicLanguageConfiguration, ...], Field(max_length=4)
|
||||||
|
] = ()
|
||||||
|
case_count: Annotated[int, Field(strict=True, ge=1, le=256)]
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def bounded_text(self):
|
||||||
|
if len(self.statement.encode("utf-8")) > 262144 or "\x00" in self.statement + self.title:
|
||||||
|
raise ValueError("invalid public text")
|
||||||
|
languages = [c.language for c in self.language_configurations]
|
||||||
|
if len(set(languages)) != len(languages):
|
||||||
|
raise ValueError("duplicate language configuration")
|
||||||
|
if len(set(self.languages)) != len(self.languages):
|
||||||
|
raise ValueError("duplicate language")
|
||||||
|
if self.judging_schema_version == 1:
|
||||||
|
if (
|
||||||
|
self.language_configurations
|
||||||
|
or self.legacy_budget is None
|
||||||
|
or any(
|
||||||
|
language not in (Language.C17, Language.CPP20) for language in self.languages
|
||||||
|
)
|
||||||
|
):
|
||||||
|
raise ValueError("invalid legacy candidate")
|
||||||
|
elif tuple(languages) != self.languages or self.legacy_budget is not None:
|
||||||
|
raise ValueError("invalid multi-runtime candidate")
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class ProblemImportSummary(WireModel):
|
||||||
|
schema_version: Version = 1
|
||||||
|
operation_id: Identifier
|
||||||
|
state: ImportState
|
||||||
|
expected_archive_digest: Digest
|
||||||
|
archive_bytes: Annotated[int, Field(strict=True, ge=1, le=MAX_ARCHIVE_BYTES)]
|
||||||
|
created_at: datetime
|
||||||
|
expires_at: datetime
|
||||||
|
updated_at: datetime
|
||||||
|
failure_code: ImportFailure | None
|
||||||
|
candidate_id: Identifier | None
|
||||||
|
published_version_id: Identifier | None
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def consistent_state(self):
|
||||||
|
if self.expires_at <= self.created_at or self.updated_at < self.created_at:
|
||||||
|
raise ValueError("invalid import time range")
|
||||||
|
if (self.state in (ImportState.READY, ImportState.PUBLISHED)) != (
|
||||||
|
self.candidate_id is not None
|
||||||
|
):
|
||||||
|
raise ValueError("candidate state mismatch")
|
||||||
|
if (self.state == ImportState.PUBLISHED) != (self.published_version_id is not None):
|
||||||
|
raise ValueError("published state mismatch")
|
||||||
|
if self.state in (ImportState.FAILED, ImportState.EXPIRED):
|
||||||
|
if self.failure_code is None:
|
||||||
|
raise ValueError("failure code required")
|
||||||
|
elif self.failure_code is not None:
|
||||||
|
raise ValueError("unexpected failure code")
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class ProblemImportDetail(ProblemImportSummary):
|
||||||
|
candidate: CandidatePreview | None
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def candidate_matches(self):
|
||||||
|
if (self.candidate_id is None) != (self.candidate is None):
|
||||||
|
raise ValueError("candidate preview missing")
|
||||||
|
if self.candidate and self.candidate.candidate_id != self.candidate_id:
|
||||||
|
raise ValueError("candidate preview mismatch")
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class ProblemImportPage(WireModel):
|
||||||
|
schema_version: Version = 1
|
||||||
|
items: Annotated[tuple[ProblemImportSummary, ...], Field(max_length=100)]
|
||||||
|
next_after: Identifier | None
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
"""Root-to-API availability binding; independent from immutable judging protocols."""
|
||||||
|
|
||||||
|
from typing import Annotated, Literal
|
||||||
|
|
||||||
|
from pydantic import BaseModel, BeforeValidator, ConfigDict, Field
|
||||||
|
|
||||||
|
from kf_oj.contracts.judging import integer_version
|
||||||
|
|
||||||
|
AVAILABILITY_PATH = "/etc/kf-admission/status.json"
|
||||||
|
|
||||||
|
|
||||||
|
class AvailabilityRecord(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
|
||||||
|
|
||||||
|
schema_version: Annotated[Literal[1], BeforeValidator(integer_version)]
|
||||||
|
release_id: Annotated[str, Field(pattern=r"^sd-[0-9a-f]{12}$")]
|
||||||
|
configuration_id: Annotated[str, Field(pattern=r"^cfg-[0-9a-f]{24}$")]
|
||||||
|
state: Literal["open", "draining", "quiescent"]
|
||||||
@@ -5,9 +5,23 @@ import json
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from kf_oj.api.app import create_app
|
from kf_oj.api.app import create_app
|
||||||
|
from kf_oj.contracts.admin_problems import (
|
||||||
|
CreateProblemImport,
|
||||||
|
ProblemImportCapabilities,
|
||||||
|
ProblemImportDetail,
|
||||||
|
)
|
||||||
|
from kf_oj.contracts.availability import AvailabilityRecord
|
||||||
from kf_oj.contracts.judge import JudgeTaskNotification
|
from kf_oj.contracts.judge import JudgeTaskNotification
|
||||||
from kf_oj.contracts.judging import WIRE_MODELS
|
from kf_oj.contracts.judging import WIRE_MODELS
|
||||||
from kf_oj.contracts.judging_v2 import WIRE_MODELS as WIRE_MODELS_V2
|
from kf_oj.contracts.judging_v2 import WIRE_MODELS as WIRE_MODELS_V2
|
||||||
|
from kf_oj.contracts.problem_maintenance import (
|
||||||
|
ChangeProblemAvailability,
|
||||||
|
MaintenanceAuditPage,
|
||||||
|
MaintenanceReceipt,
|
||||||
|
PublishCandidate,
|
||||||
|
)
|
||||||
|
from kf_oj.contracts.restore_io import RestoreIOReply, RestoreIORequest
|
||||||
|
from kf_oj.contracts.submission_performance import SubmissionPerformance
|
||||||
from kf_oj.core.settings import Settings
|
from kf_oj.core.settings import Settings
|
||||||
|
|
||||||
|
|
||||||
@@ -34,6 +48,30 @@ def main() -> None:
|
|||||||
"budgets, result consistency, UUID canonical form and UTF-8 byte limits."
|
"budgets, result consistency, UUID canonical form and UTF-8 byte limits."
|
||||||
)
|
)
|
||||||
outputs[root / f"contracts/schemas/{name}.v1.schema.json"] = document
|
outputs[root / f"contracts/schemas/{name}.v1.schema.json"] = document
|
||||||
|
performance = SubmissionPerformance.model_json_schema()
|
||||||
|
performance["$schema"] = "https://json-schema.org/draft/2020-12/schema"
|
||||||
|
performance["$id"] = "urn:kf-oj:submission-performance:v1"
|
||||||
|
performance["$comment"] = (
|
||||||
|
"Anonymous bounded aggregates, decimal raw units and strict same-snapshot "
|
||||||
|
"counts; also run model validation and server-side ownership/environment checks."
|
||||||
|
)
|
||||||
|
outputs[root / "contracts/schemas/submission-performance.v1.schema.json"] = performance
|
||||||
|
for name, model in {
|
||||||
|
"delivery-availability": AvailabilityRecord,
|
||||||
|
"restore-io-request": RestoreIORequest,
|
||||||
|
"restore-io-reply": RestoreIOReply,
|
||||||
|
"problem-publish": PublishCandidate,
|
||||||
|
"problem-availability": ChangeProblemAvailability,
|
||||||
|
"problem-maintenance": MaintenanceReceipt,
|
||||||
|
"problem-maintenance-audit": MaintenanceAuditPage,
|
||||||
|
"problem-import-create": CreateProblemImport,
|
||||||
|
"problem-import": ProblemImportDetail,
|
||||||
|
"problem-import-capabilities": ProblemImportCapabilities,
|
||||||
|
}.items():
|
||||||
|
document = model.model_json_schema()
|
||||||
|
document["$schema"] = "https://json-schema.org/draft/2020-12/schema"
|
||||||
|
document["$id"] = f"urn:kf-oj:{name}:v1"
|
||||||
|
outputs[root / f"contracts/schemas/{name}.v1.schema.json"] = document
|
||||||
for name, model in WIRE_MODELS_V2.items():
|
for name, model in WIRE_MODELS_V2.items():
|
||||||
document = model.model_json_schema()
|
document = model.model_json_schema()
|
||||||
document["$schema"] = "https://json-schema.org/draft/2020-12/schema"
|
document["$schema"] = "https://json-schema.org/draft/2020-12/schema"
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Annotated
|
from typing import Annotated, Literal
|
||||||
|
|
||||||
from pydantic import Field, SecretStr
|
from pydantic import Field, SecretStr
|
||||||
|
|
||||||
@@ -11,9 +11,16 @@ class LoginRequest(WireModel):
|
|||||||
password: Annotated[SecretStr, Field(min_length=12, max_length=128)]
|
password: Annotated[SecretStr, Field(min_length=12, max_length=128)]
|
||||||
|
|
||||||
|
|
||||||
|
class AccountCapabilities(WireModel):
|
||||||
|
schema_version: Literal[1] = 1
|
||||||
|
problem_maintenance: bool = False
|
||||||
|
authorization_version: Annotated[str, Field(pattern=r"^[1-9][0-9]{0,18}$")] = "1"
|
||||||
|
|
||||||
|
|
||||||
class CurrentUser(WireModel):
|
class CurrentUser(WireModel):
|
||||||
id: Identifier
|
id: Identifier
|
||||||
username: str
|
username: str
|
||||||
|
capabilities: AccountCapabilities = Field(default_factory=AccountCapabilities)
|
||||||
|
|
||||||
|
|
||||||
class LoginResponse(WireModel):
|
class LoginResponse(WireModel):
|
||||||
|
|||||||
@@ -0,0 +1,103 @@
|
|||||||
|
"""Bounded administrator projections and immutable operation receipts, separate from judging."""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Annotated, Literal
|
||||||
|
|
||||||
|
from pydantic import Field, StrictBool
|
||||||
|
|
||||||
|
from kf_oj.contracts.admin_problems import CandidatePreview
|
||||||
|
from kf_oj.contracts.judging import Identifier, Label, Version, WireModel
|
||||||
|
from kf_oj.contracts.problems import ProblemSummary
|
||||||
|
|
||||||
|
MaintenanceAction = Literal["publish", "disable", "restore"]
|
||||||
|
AuditAction = Literal[
|
||||||
|
"upload_created",
|
||||||
|
"upload_received",
|
||||||
|
"import_ready",
|
||||||
|
"import_failed",
|
||||||
|
"import_discarded",
|
||||||
|
"import_expired",
|
||||||
|
"published",
|
||||||
|
"disabled",
|
||||||
|
"restored",
|
||||||
|
"authority_bootstrap",
|
||||||
|
"authority_grant",
|
||||||
|
"authority_revoke",
|
||||||
|
"authority_enable",
|
||||||
|
"authority_disable",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class MaintainedProblem(WireModel):
|
||||||
|
id: Identifier
|
||||||
|
slug: Label
|
||||||
|
enabled: StrictBool
|
||||||
|
active_version_id: Identifier | None
|
||||||
|
revision: Annotated[int, Field(strict=True, ge=0)]
|
||||||
|
title: Annotated[str | None, Field(max_length=160)]
|
||||||
|
created_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class MaintainedProblemPage(WireModel):
|
||||||
|
schema_version: Version = 1
|
||||||
|
items: Annotated[tuple[MaintainedProblem, ...], Field(max_length=100)]
|
||||||
|
next_after: Label | None
|
||||||
|
|
||||||
|
|
||||||
|
class MaintainedVersion(ProblemSummary):
|
||||||
|
published_at: datetime
|
||||||
|
case_count: Annotated[int, Field(strict=True, ge=1, le=256)]
|
||||||
|
|
||||||
|
|
||||||
|
class MaintainedVersionPage(WireModel):
|
||||||
|
schema_version: Version = 1
|
||||||
|
items: Annotated[tuple[MaintainedVersion, ...], Field(max_length=100)]
|
||||||
|
next_before: Annotated[int | None, Field(strict=True, ge=1)]
|
||||||
|
|
||||||
|
|
||||||
|
class PublishCandidate(WireModel):
|
||||||
|
schema_version: Version = 1
|
||||||
|
operation_id: Identifier
|
||||||
|
import_operation_id: Identifier
|
||||||
|
candidate_id: Identifier
|
||||||
|
expected_active_version_id: Identifier | None
|
||||||
|
|
||||||
|
|
||||||
|
class ChangeProblemAvailability(WireModel):
|
||||||
|
schema_version: Version = 1
|
||||||
|
operation_id: Identifier
|
||||||
|
expected_active_version_id: Identifier
|
||||||
|
expected_enabled: StrictBool
|
||||||
|
|
||||||
|
|
||||||
|
class MaintenanceReceipt(WireModel):
|
||||||
|
schema_version: Version = 1
|
||||||
|
operation_id: Identifier
|
||||||
|
action: MaintenanceAction
|
||||||
|
completed_at: datetime
|
||||||
|
problem: MaintainedProblem
|
||||||
|
published_version_id: Identifier | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class MaintenanceAuditEvent(WireModel):
|
||||||
|
id: Identifier
|
||||||
|
created_at: datetime
|
||||||
|
actor_kind: Literal["account", "operator"]
|
||||||
|
actor_id: Identifier | None
|
||||||
|
operator_name: Annotated[str | None, Field(max_length=63)]
|
||||||
|
authorization_version: Annotated[str, Field(pattern=r"^[1-9][0-9]{0,18}$")]
|
||||||
|
operation_id: Identifier
|
||||||
|
action: AuditAction
|
||||||
|
target_id: Identifier | None
|
||||||
|
result: Literal["committed"] = "committed"
|
||||||
|
|
||||||
|
|
||||||
|
class MaintenanceAuditPage(WireModel):
|
||||||
|
schema_version: Version = 1
|
||||||
|
items: Annotated[tuple[MaintenanceAuditEvent, ...], Field(max_length=100)]
|
||||||
|
next_before: Annotated[str | None, Field(max_length=100)]
|
||||||
|
|
||||||
|
|
||||||
|
# This alias deliberately uses the existing sanitized candidate contract. No new
|
||||||
|
# endpoint exposes the private manifest, artifact digest or hidden case names.
|
||||||
|
PublicCandidate = CandidatePreview
|
||||||
@@ -0,0 +1,129 @@
|
|||||||
|
"""Private offline restore byte-copy protocol; conveys no execution authority."""
|
||||||
|
|
||||||
|
import re
|
||||||
|
from typing import Annotated, Literal
|
||||||
|
|
||||||
|
from pydantic import (
|
||||||
|
BaseModel,
|
||||||
|
BeforeValidator,
|
||||||
|
ConfigDict,
|
||||||
|
Field,
|
||||||
|
RootModel,
|
||||||
|
StrictInt,
|
||||||
|
model_validator,
|
||||||
|
)
|
||||||
|
|
||||||
|
from kf_oj.contracts.judging import integer_version
|
||||||
|
|
||||||
|
MAX_FILES = 98304
|
||||||
|
MAX_LOGICAL_BYTES = 12 * 1024**3
|
||||||
|
MAX_FILE_BYTES = 256 * 1024**2
|
||||||
|
MAX_BATCH_BYTES = 64 * 1024**2
|
||||||
|
DOMAINS = frozenset(
|
||||||
|
"postgres artifacts imports app-config judge-config factory operations releases "
|
||||||
|
"images dependencies proxy recovery".split()
|
||||||
|
)
|
||||||
|
Count = Annotated[StrictInt, Field(ge=0, le=MAX_FILES)]
|
||||||
|
ByteCount = Annotated[StrictInt, Field(ge=0, le=MAX_LOGICAL_BYTES)]
|
||||||
|
|
||||||
|
|
||||||
|
class Record(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
|
||||||
|
|
||||||
|
|
||||||
|
class RestoreFile(Record):
|
||||||
|
path: Annotated[str, Field(min_length=1, max_length=512)]
|
||||||
|
uid: Annotated[StrictInt, Field(ge=0, le=1205)]
|
||||||
|
gid: Annotated[StrictInt, Field(ge=0, le=1205)]
|
||||||
|
mode: StrictInt
|
||||||
|
bytes: Annotated[StrictInt, Field(ge=0, le=MAX_FILE_BYTES)]
|
||||||
|
sha256: Annotated[str, Field(pattern=r"^[0-9a-f]{64}$")]
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def check_member(self) -> "RestoreFile":
|
||||||
|
parts = self.path.split("/")
|
||||||
|
if (
|
||||||
|
len(parts) < 2
|
||||||
|
or parts[0] not in DOMAINS
|
||||||
|
or any(
|
||||||
|
p in {"", ".", "..", ".git", ".ssh", ".localplan", "docs"}
|
||||||
|
or re.fullmatch(r"[A-Za-z0-9_.+@-]{1,255}", p) is None
|
||||||
|
for p in parts
|
||||||
|
)
|
||||||
|
):
|
||||||
|
raise ValueError("restore file must be below a fixed backup domain")
|
||||||
|
if self.uid not in {0, 1200, 1201, 1202, 1203, 1204, 1205} or self.gid not in (
|
||||||
|
{0, 1203, 1204, 1205} if self.uid == 0 else {self.uid}
|
||||||
|
):
|
||||||
|
raise ValueError("unsupported restore identity")
|
||||||
|
if self.mode not in {0o400, 0o444, 0o600, 0o640, 0o644, 0o555, 0o755}:
|
||||||
|
raise ValueError("unsupported restore file mode")
|
||||||
|
if self.sha256 in {"a" * 64, "0" * 64}:
|
||||||
|
raise ValueError("placeholder digest refused")
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class Open(Record):
|
||||||
|
schema_version: Annotated[Literal[1], BeforeValidator(integer_version)]
|
||||||
|
kind: Literal["restore.io.open"]
|
||||||
|
backup_id: Annotated[str, Field(pattern=r"^bk-[0-9a-f]{32}$")]
|
||||||
|
recovery_id: Annotated[str, Field(pattern=r"^rr-[0-9a-f]{32}$")]
|
||||||
|
expected_files: Annotated[StrictInt, Field(ge=1, le=MAX_FILES)]
|
||||||
|
expected_logical_bytes: ByteCount
|
||||||
|
deadline_monotonic_ns: Annotated[StrictInt, Field(ge=1, le=2**63 - 1)]
|
||||||
|
|
||||||
|
|
||||||
|
class Batch(Record):
|
||||||
|
schema_version: Annotated[Literal[1], BeforeValidator(integer_version)]
|
||||||
|
kind: Literal["restore.io.batch"]
|
||||||
|
batch_id: Annotated[StrictInt, Field(ge=1, le=MAX_FILES)]
|
||||||
|
files: Annotated[list[RestoreFile], Field(min_length=1, max_length=256)]
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def check_batch(self) -> "Batch":
|
||||||
|
if len(self.files) > 1 and sum(f.bytes for f in self.files) > MAX_BATCH_BYTES:
|
||||||
|
raise ValueError("restore batch byte bound exceeded")
|
||||||
|
if len({f.path for f in self.files}) != len(self.files):
|
||||||
|
raise ValueError("duplicate restore batch member")
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class Finish(Record):
|
||||||
|
schema_version: Annotated[Literal[1], BeforeValidator(integer_version)]
|
||||||
|
kind: Literal["restore.io.finish"]
|
||||||
|
|
||||||
|
|
||||||
|
class RestoreIORequest(RootModel[Annotated[Open | Batch | Finish, Field(discriminator="kind")]]):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class RestoreIOReply(Record):
|
||||||
|
schema_version: Annotated[Literal[1], BeforeValidator(integer_version)]
|
||||||
|
kind: Literal["restore.io.reply"]
|
||||||
|
stage: Literal["opened", "verified", "finished", "refused"]
|
||||||
|
batch_id: Count
|
||||||
|
verified_files: Count
|
||||||
|
verified_logical_bytes: ByteCount
|
||||||
|
execution_verified: Annotated[
|
||||||
|
Literal[False],
|
||||||
|
BeforeValidator(lambda value: value if type(value) is bool else _bad_boolean()),
|
||||||
|
]
|
||||||
|
error: Literal["none", "protocol", "guard", "deadline", "filesystem", "bytes", "identity"]
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def check_reply(self) -> "RestoreIOReply":
|
||||||
|
if (self.stage == "refused") != (self.error != "none"):
|
||||||
|
raise ValueError("restore refusal state differs")
|
||||||
|
if self.stage == "opened" and (
|
||||||
|
self.batch_id or self.verified_files or self.verified_logical_bytes
|
||||||
|
):
|
||||||
|
raise ValueError("restore open cannot claim verified data")
|
||||||
|
if self.stage in {"verified", "finished"} and not (
|
||||||
|
1 <= self.batch_id <= self.verified_files
|
||||||
|
):
|
||||||
|
raise ValueError("restore completion has no verified batch")
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
def _bad_boolean() -> bool:
|
||||||
|
raise ValueError("execution verification must be a boolean false")
|
||||||
@@ -0,0 +1,126 @@
|
|||||||
|
"""Bounded anonymous aggregates; raw measurements never pass through JS floats."""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Annotated, Literal, Self
|
||||||
|
|
||||||
|
from pydantic import AfterValidator, Field, StrictBool, model_validator
|
||||||
|
|
||||||
|
from kf_oj.contracts.judging import I64_MAX, Digest, Identifier, Version, WireModel
|
||||||
|
from kf_oj.contracts.judging_v2 import Language
|
||||||
|
|
||||||
|
MAX_PERFORMANCE_SAMPLES = 10_000
|
||||||
|
MIN_PERFORMANCE_SAMPLES = 5
|
||||||
|
MAX_PERFORMANCE_BUCKETS = 64
|
||||||
|
|
||||||
|
|
||||||
|
def bounded_decimal(value: str) -> str:
|
||||||
|
if int(value) > I64_MAX:
|
||||||
|
raise ValueError("measurement exceeds signed storage range")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
DecimalMeasurement = Annotated[
|
||||||
|
str,
|
||||||
|
Field(strict=True, min_length=1, max_length=19, pattern=r"^(0|[1-9][0-9]*)$"),
|
||||||
|
AfterValidator(bounded_decimal),
|
||||||
|
]
|
||||||
|
SampleCount = Annotated[int, Field(strict=True, ge=0, le=MAX_PERFORMANCE_SAMPLES)]
|
||||||
|
UnavailableReason = Literal[
|
||||||
|
"not_accepted", "measurement_missing", "environment_unverified", "insufficient_samples"
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class PerformanceBucket(WireModel):
|
||||||
|
start: DecimalMeasurement
|
||||||
|
end: DecimalMeasurement
|
||||||
|
upper_inclusive: StrictBool
|
||||||
|
count: SampleCount
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def interval(self) -> Self:
|
||||||
|
if int(self.end) <= int(self.start):
|
||||||
|
raise ValueError("empty or reversed interval")
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class PerformanceMetric(WireModel):
|
||||||
|
unit: Literal["cpu_microseconds", "bytes"]
|
||||||
|
current: DecimalMeasurement | None
|
||||||
|
available: StrictBool
|
||||||
|
reason: UnavailableReason | None
|
||||||
|
sample_count: SampleCount
|
||||||
|
window_truncated: StrictBool
|
||||||
|
oldest_sample_at: datetime | None
|
||||||
|
# These counts are computed against the original integers, never rounded bins.
|
||||||
|
strictly_slower_count: SampleCount | None
|
||||||
|
tied_count: SampleCount | None
|
||||||
|
buckets: Annotated[tuple[PerformanceBucket, ...], Field(max_length=MAX_PERFORMANCE_BUCKETS)]
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def aggregate(self) -> Self:
|
||||||
|
if self.oldest_sample_at is not None and self.oldest_sample_at.utcoffset() is None:
|
||||||
|
raise ValueError("sample timestamp must have a timezone")
|
||||||
|
if self.available:
|
||||||
|
if (
|
||||||
|
self.reason is not None
|
||||||
|
or self.current is None
|
||||||
|
or self.sample_count < MIN_PERFORMANCE_SAMPLES
|
||||||
|
or self.strictly_slower_count is None
|
||||||
|
or self.tied_count is None
|
||||||
|
or self.oldest_sample_at is None
|
||||||
|
or not self.buckets
|
||||||
|
):
|
||||||
|
raise ValueError("incomplete available aggregate")
|
||||||
|
if self.strictly_slower_count + self.tied_count > self.sample_count:
|
||||||
|
raise ValueError("comparison counts exceed the cohort")
|
||||||
|
if sum(b.count for b in self.buckets) != self.sample_count:
|
||||||
|
raise ValueError("histogram loses samples")
|
||||||
|
if int(self.buckets[0].start) != 0:
|
||||||
|
raise ValueError("histogram must include zero")
|
||||||
|
for index, bucket in enumerate(self.buckets):
|
||||||
|
if bucket.upper_inclusive != (index == len(self.buckets) - 1):
|
||||||
|
raise ValueError("only the last interval includes its upper endpoint")
|
||||||
|
if index and self.buckets[index - 1].end != bucket.start:
|
||||||
|
raise ValueError("histogram intervals must be contiguous")
|
||||||
|
elif (
|
||||||
|
self.reason is None
|
||||||
|
or self.buckets
|
||||||
|
or self.strictly_slower_count is not None
|
||||||
|
or self.tied_count is not None
|
||||||
|
):
|
||||||
|
raise ValueError("unavailable aggregate cannot expose comparison data")
|
||||||
|
if self.reason == "measurement_missing" and self.current is not None:
|
||||||
|
raise ValueError("missing measurement cannot claim a value")
|
||||||
|
if self.reason == "insufficient_samples" and (
|
||||||
|
self.sample_count >= MIN_PERFORMANCE_SAMPLES or self.current is None
|
||||||
|
):
|
||||||
|
raise ValueError("inconsistent insufficient cohort")
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class SubmissionPerformance(WireModel):
|
||||||
|
schema_version: Version = 1
|
||||||
|
kind: Literal["judge.performance"] = "judge.performance"
|
||||||
|
submission_id: Identifier
|
||||||
|
problem_version_id: Identifier
|
||||||
|
language: Language
|
||||||
|
as_of: datetime
|
||||||
|
# Opaque trusted comparison identity, not a worker name, path or credential.
|
||||||
|
comparison_group: Digest | None
|
||||||
|
window_limit: Literal[10_000] = MAX_PERFORMANCE_SAMPLES
|
||||||
|
minimum_samples: Literal[5] = MIN_PERFORMANCE_SAMPLES
|
||||||
|
cpu: PerformanceMetric
|
||||||
|
memory: PerformanceMetric
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def snapshot(self) -> Self:
|
||||||
|
if self.as_of.utcoffset() is None:
|
||||||
|
raise ValueError("snapshot timestamp must have a timezone")
|
||||||
|
if self.cpu.unit != "cpu_microseconds" or self.memory.unit != "bytes":
|
||||||
|
raise ValueError("metric unit mismatch")
|
||||||
|
for metric in (self.cpu, self.memory):
|
||||||
|
if metric.available and self.comparison_group is None:
|
||||||
|
raise ValueError("comparison requires trusted environment binding")
|
||||||
|
if metric.oldest_sample_at is not None and metric.oldest_sample_at > self.as_of:
|
||||||
|
raise ValueError("sample is newer than the snapshot")
|
||||||
|
return self
|
||||||
@@ -16,7 +16,53 @@ class JsonFormatter(logging.Formatter):
|
|||||||
return json.dumps(payload, ensure_ascii=False)
|
return json.dumps(payload, ensure_ascii=False)
|
||||||
|
|
||||||
|
|
||||||
def configure_logging(level: str) -> None:
|
class ProductionFormatter(logging.Formatter):
|
||||||
|
"""Closed event vocabulary; arbitrary messages/arguments/tracebacks are private data."""
|
||||||
|
|
||||||
|
EVENTS = {
|
||||||
|
("kf_oj.api", "Platform API started"): "api_started",
|
||||||
|
("kf_oj.api", "Platform API stopped"): "api_stopped",
|
||||||
|
("kf_oj.outbox", "outbox broker unavailable"): "outbox_broker_unavailable",
|
||||||
|
("kf_oj.outbox", "outbox database unavailable"): "outbox_database_unavailable",
|
||||||
|
}
|
||||||
|
LOGGERS = {"kf_oj.api", "kf_oj.outbox", "uvicorn", "uvicorn.error", "uvicorn.access"}
|
||||||
|
LEVELS = {
|
||||||
|
logging.DEBUG: "DEBUG",
|
||||||
|
logging.INFO: "INFO",
|
||||||
|
logging.WARNING: "WARNING",
|
||||||
|
logging.ERROR: "ERROR",
|
||||||
|
logging.CRITICAL: "CRITICAL",
|
||||||
|
}
|
||||||
|
|
||||||
|
def format(self, record: logging.LogRecord) -> str:
|
||||||
|
# Do not call getMessage/formatException: library errors can contain
|
||||||
|
# connection URIs, query parameters, source or diagnostic body bytes.
|
||||||
|
event = "infrastructure_event"
|
||||||
|
if isinstance(record.msg, str) and not record.args:
|
||||||
|
event = self.EVENTS.get((record.name, record.msg), event)
|
||||||
|
return json.dumps(
|
||||||
|
{
|
||||||
|
"timestamp": datetime.fromtimestamp(record.created, UTC).isoformat(),
|
||||||
|
"level": self.LEVELS.get(record.levelno, "UNKNOWN"),
|
||||||
|
"logger": record.name if record.name in self.LOGGERS else "infrastructure",
|
||||||
|
"message": event,
|
||||||
|
"exception_present": record.exc_info is not None,
|
||||||
|
},
|
||||||
|
ensure_ascii=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def configure_logging(level: str, *, production: bool = False) -> None:
|
||||||
handler = logging.StreamHandler()
|
handler = logging.StreamHandler()
|
||||||
handler.setFormatter(JsonFormatter())
|
handler.setFormatter(ProductionFormatter() if production else JsonFormatter())
|
||||||
logging.basicConfig(level=level, handlers=[handler], force=True)
|
logging.basicConfig(level=level, handlers=[handler], force=True)
|
||||||
|
if production:
|
||||||
|
# Uvicorn installs its own handlers before lifespan, which otherwise
|
||||||
|
# bypass the root formatter when reporting an unhandled application error.
|
||||||
|
for name in ("uvicorn", "uvicorn.error"):
|
||||||
|
logger = logging.getLogger(name)
|
||||||
|
logger.handlers.clear()
|
||||||
|
logger.propagate = True
|
||||||
|
access = logging.getLogger("uvicorn.access")
|
||||||
|
access.handlers.clear()
|
||||||
|
access.propagate = False
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ class Settings(BaseSettings):
|
|||||||
environment: Literal["development", "test", "production"] = "development"
|
environment: Literal["development", "test", "production"] = "development"
|
||||||
log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = "INFO"
|
log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = "INFO"
|
||||||
database_url: SecretStr | None = None
|
database_url: SecretStr | None = None
|
||||||
|
identity_operator_database_url: SecretStr | None = None
|
||||||
amqp_url: SecretStr | None = None
|
amqp_url: SecretStr | None = None
|
||||||
browser_origins: tuple[str, ...] = (
|
browser_origins: tuple[str, ...] = (
|
||||||
"http://localhost:5173",
|
"http://localhost:5173",
|
||||||
@@ -22,11 +23,15 @@ class Settings(BaseSettings):
|
|||||||
)
|
)
|
||||||
session_lifetime_seconds: int = Field(default=43200, ge=300, le=86400)
|
session_lifetime_seconds: int = Field(default=43200, ge=300, le=86400)
|
||||||
artifact_root: Path | None = None
|
artifact_root: Path | None = None
|
||||||
|
import_staging_root: Path | None = None
|
||||||
|
import_worker_cgroup: Path | None = None
|
||||||
artifact_max_bytes: int = Field(default=536870912, ge=1, le=16 * 1024**3)
|
artifact_max_bytes: int = Field(default=536870912, ge=1, le=16 * 1024**3)
|
||||||
artifact_max_files: int = Field(default=4096, ge=1, le=65536)
|
artifact_max_files: int = Field(default=4096, ge=1, le=65536)
|
||||||
judge_admission: NodeAdmission | None = None
|
judge_admission: NodeAdmission | None = None
|
||||||
coordinator_url: str | None = None
|
coordinator_url: str | None = None
|
||||||
coordinator_token: SecretStr | None = None
|
coordinator_token: SecretStr | None = None
|
||||||
|
delivery_release_id: str | None = Field(default=None, pattern=r"^sd-[0-9a-f]{12}$")
|
||||||
|
delivery_configuration_id: str | None = Field(default=None, pattern=r"^cfg-[0-9a-f]{24}$")
|
||||||
|
|
||||||
@model_validator(mode="after")
|
@model_validator(mode="after")
|
||||||
def secure_origins(self):
|
def secure_origins(self):
|
||||||
|
|||||||
@@ -0,0 +1,340 @@
|
|||||||
|
"""Bounded single-level ZIP reader; no extractall, metadata restoration or content execution."""
|
||||||
|
|
||||||
|
import binascii
|
||||||
|
import hashlib
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import stat
|
||||||
|
import struct
|
||||||
|
import time
|
||||||
|
import zlib
|
||||||
|
from collections.abc import Iterator
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
from kf_oj.contracts.admin_problems import (
|
||||||
|
MAX_ARCHIVE_BYTES,
|
||||||
|
MAX_ARCHIVE_FILES,
|
||||||
|
MAX_UNPACKED_BYTES,
|
||||||
|
)
|
||||||
|
|
||||||
|
NAME = re.compile(rb"[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}\Z")
|
||||||
|
CHUNK = 65536
|
||||||
|
LOCAL = struct.Struct("<4s5H3I2H")
|
||||||
|
CENTRAL = struct.Struct("<4s6H3I5H2I")
|
||||||
|
END = struct.Struct("<4s4H2IH")
|
||||||
|
|
||||||
|
|
||||||
|
class ArchiveError(ValueError):
|
||||||
|
"""A fixed sanitized failure; never echo an archive name, data or local path."""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ArchiveMember:
|
||||||
|
name: str
|
||||||
|
method: int
|
||||||
|
crc32: int
|
||||||
|
compressed_bytes: int
|
||||||
|
bytes: int
|
||||||
|
offset: int
|
||||||
|
flags: int
|
||||||
|
|
||||||
|
|
||||||
|
class ArchiveReader:
|
||||||
|
def __init__(self, descriptor: int, expected_bytes: int, *, deadline: float):
|
||||||
|
self.descriptor, self.deadline = descriptor, deadline
|
||||||
|
self.before = os.fstat(descriptor)
|
||||||
|
if (
|
||||||
|
not stat.S_ISREG(self.before.st_mode)
|
||||||
|
or self.before.st_uid != os.getuid()
|
||||||
|
or self.before.st_nlink != 1
|
||||||
|
or stat.S_IMODE(self.before.st_mode) != 0o600
|
||||||
|
or not 22 <= expected_bytes <= MAX_ARCHIVE_BYTES
|
||||||
|
or self.before.st_size != expected_bytes
|
||||||
|
):
|
||||||
|
raise ArchiveError("invalid_archive")
|
||||||
|
self.size = expected_bytes
|
||||||
|
self.unpacked_bytes, self.steps = 0, 0
|
||||||
|
self.read_members: set[str] = set()
|
||||||
|
self.members = self._members()
|
||||||
|
|
||||||
|
def _tick(self) -> None:
|
||||||
|
self.steps += 1
|
||||||
|
if self.steps > 8192 or time.monotonic() >= self.deadline:
|
||||||
|
raise ArchiveError("archive_deadline")
|
||||||
|
|
||||||
|
def _read(self, offset: int, size: int) -> bytes:
|
||||||
|
self._tick()
|
||||||
|
if not 0 <= size <= 1024**2 or offset < 0 or offset + size > self.size:
|
||||||
|
raise ArchiveError("invalid_archive")
|
||||||
|
data = os.pread(self.descriptor, size, offset)
|
||||||
|
if len(data) != size:
|
||||||
|
raise ArchiveError("invalid_archive")
|
||||||
|
return data
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _extra(data: bytes) -> None:
|
||||||
|
offset, fields = 0, 0
|
||||||
|
# These bounded timestamps/UID attributes are ignored, never restored.
|
||||||
|
# ZIP64, Unicode aliases and all other extension semantics are unsupported.
|
||||||
|
while offset < len(data):
|
||||||
|
if offset + 4 > len(data) or fields >= 32:
|
||||||
|
raise ArchiveError("invalid_archive")
|
||||||
|
kind, size = struct.unpack_from("<HH", data, offset)
|
||||||
|
offset += 4
|
||||||
|
if kind not in (0x5455, 0x7875, 0x5855, 0x7855, 0x000A) or offset + size > len(data):
|
||||||
|
raise ArchiveError("invalid_archive")
|
||||||
|
offset += size
|
||||||
|
fields += 1
|
||||||
|
|
||||||
|
def _members(self) -> tuple[ArchiveMember, ...]:
|
||||||
|
tail_offset = max(0, self.size - 1046)
|
||||||
|
tail = self._read(tail_offset, self.size - tail_offset)
|
||||||
|
position = tail.rfind(b"PK\x05\x06")
|
||||||
|
if position < 0 or position + END.size > len(tail):
|
||||||
|
raise ArchiveError("invalid_archive")
|
||||||
|
sig, disk, cd_disk, disk_count, count, cd_bytes, cd_offset, comment = END.unpack_from(
|
||||||
|
tail, position
|
||||||
|
)
|
||||||
|
end_offset = tail_offset + position
|
||||||
|
if (
|
||||||
|
sig != b"PK\x05\x06"
|
||||||
|
or disk != 0
|
||||||
|
or cd_disk != 0
|
||||||
|
or disk_count != count
|
||||||
|
or not 1 <= count <= MAX_ARCHIVE_FILES
|
||||||
|
or comment > 1024
|
||||||
|
or end_offset + END.size + comment != self.size
|
||||||
|
or not 46 <= cd_bytes <= 1024**2
|
||||||
|
or cd_offset + cd_bytes != end_offset
|
||||||
|
):
|
||||||
|
raise ArchiveError("invalid_archive")
|
||||||
|
directory = self._read(cd_offset, cd_bytes)
|
||||||
|
offset, records, names = 0, [], set()
|
||||||
|
for _ in range(count):
|
||||||
|
if offset + CENTRAL.size > len(directory):
|
||||||
|
raise ArchiveError("invalid_archive")
|
||||||
|
values = CENTRAL.unpack_from(directory, offset)
|
||||||
|
(
|
||||||
|
sig,
|
||||||
|
_,
|
||||||
|
needed,
|
||||||
|
flags,
|
||||||
|
method,
|
||||||
|
_,
|
||||||
|
_,
|
||||||
|
crc,
|
||||||
|
compressed,
|
||||||
|
size,
|
||||||
|
name_size,
|
||||||
|
extra_size,
|
||||||
|
comment_size,
|
||||||
|
disk,
|
||||||
|
_,
|
||||||
|
attributes,
|
||||||
|
local_offset,
|
||||||
|
) = values
|
||||||
|
allowed_flags = 0x0808 | (0x0006 if method == 8 else 0)
|
||||||
|
if (
|
||||||
|
sig != b"PK\x01\x02"
|
||||||
|
or needed not in (10, 20)
|
||||||
|
or method not in (0, 8)
|
||||||
|
or (method == 8 and needed != 20)
|
||||||
|
or flags & ~allowed_flags
|
||||||
|
or disk != 0
|
||||||
|
or not 1 <= name_size <= 128
|
||||||
|
or extra_size > 1024
|
||||||
|
or comment_size > 256
|
||||||
|
or size > MAX_UNPACKED_BYTES
|
||||||
|
or compressed > MAX_ARCHIVE_BYTES
|
||||||
|
or local_offset + LOCAL.size > cd_offset
|
||||||
|
or attributes & 0x10
|
||||||
|
or stat.S_IFMT(attributes >> 16) not in (0, stat.S_IFREG)
|
||||||
|
):
|
||||||
|
raise ArchiveError("invalid_archive")
|
||||||
|
start = offset + CENTRAL.size
|
||||||
|
finish = start + name_size + extra_size + comment_size
|
||||||
|
if finish > len(directory):
|
||||||
|
raise ArchiveError("invalid_archive")
|
||||||
|
name_bytes = directory[start : start + name_size]
|
||||||
|
if not NAME.fullmatch(name_bytes):
|
||||||
|
raise ArchiveError("invalid_archive")
|
||||||
|
name = name_bytes.decode("ascii")
|
||||||
|
if name.lower() in names:
|
||||||
|
raise ArchiveError("invalid_archive")
|
||||||
|
names.add(name.lower())
|
||||||
|
self._extra(directory[start + name_size : start + name_size + extra_size])
|
||||||
|
records.append(ArchiveMember(name, method, crc, compressed, size, local_offset, flags))
|
||||||
|
offset = finish
|
||||||
|
if offset != len(directory) or sum(m.bytes for m in records) > MAX_UNPACKED_BYTES:
|
||||||
|
raise ArchiveError("invalid_archive")
|
||||||
|
cursor, members = 0, []
|
||||||
|
for member in sorted(records, key=lambda m: m.offset):
|
||||||
|
if member.offset != cursor:
|
||||||
|
raise ArchiveError("invalid_archive")
|
||||||
|
sig, needed, flags, method, _, _, crc, compressed, size, name_size, extra_size = (
|
||||||
|
LOCAL.unpack(self._read(cursor, LOCAL.size))
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
sig != b"PK\x03\x04"
|
||||||
|
or needed not in (10, 20)
|
||||||
|
or flags != member.flags
|
||||||
|
or method != member.method
|
||||||
|
or (method == 8 and needed != 20)
|
||||||
|
or name_size != len(member.name)
|
||||||
|
or extra_size > 1024
|
||||||
|
):
|
||||||
|
raise ArchiveError("invalid_archive")
|
||||||
|
metadata = self._read(cursor + LOCAL.size, name_size + extra_size)
|
||||||
|
if metadata[:name_size] != member.name.encode("ascii"):
|
||||||
|
raise ArchiveError("invalid_archive")
|
||||||
|
self._extra(metadata[name_size:])
|
||||||
|
data_offset = cursor + LOCAL.size + name_size + extra_size
|
||||||
|
cursor = data_offset + member.compressed_bytes
|
||||||
|
if cursor > cd_offset or (method == 0 and member.compressed_bytes != member.bytes):
|
||||||
|
raise ArchiveError("invalid_archive")
|
||||||
|
if flags & 8:
|
||||||
|
if (crc, compressed, size) not in (
|
||||||
|
(0, 0, 0),
|
||||||
|
(member.crc32, member.compressed_bytes, member.bytes),
|
||||||
|
):
|
||||||
|
raise ArchiveError("invalid_archive")
|
||||||
|
marker = self._read(cursor, 4)
|
||||||
|
descriptor_size = 16 if marker == b"PK\x07\x08" else 12
|
||||||
|
descriptor = self._read(cursor, descriptor_size)
|
||||||
|
described = struct.unpack("<III", descriptor[-12:])
|
||||||
|
if described != (member.crc32, member.compressed_bytes, member.bytes):
|
||||||
|
raise ArchiveError("invalid_archive")
|
||||||
|
cursor += descriptor_size
|
||||||
|
elif (crc, compressed, size) != (member.crc32, member.compressed_bytes, member.bytes):
|
||||||
|
raise ArchiveError("invalid_archive")
|
||||||
|
members.append(
|
||||||
|
ArchiveMember(
|
||||||
|
member.name,
|
||||||
|
method,
|
||||||
|
member.crc32,
|
||||||
|
member.compressed_bytes,
|
||||||
|
member.bytes,
|
||||||
|
data_offset,
|
||||||
|
flags,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if cursor != cd_offset:
|
||||||
|
raise ArchiveError("invalid_archive")
|
||||||
|
return tuple(members)
|
||||||
|
|
||||||
|
def verify_digest(self, expected: str) -> None:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
for offset in range(0, self.size, CHUNK):
|
||||||
|
digest.update(self._read(offset, min(CHUNK, self.size - offset)))
|
||||||
|
if digest.hexdigest() != expected:
|
||||||
|
raise ArchiveError("archive_digest_mismatch")
|
||||||
|
|
||||||
|
def verify_unchanged(self) -> None:
|
||||||
|
after = os.fstat(self.descriptor)
|
||||||
|
fields = (
|
||||||
|
"st_dev",
|
||||||
|
"st_ino",
|
||||||
|
"st_nlink",
|
||||||
|
"st_mode",
|
||||||
|
"st_uid",
|
||||||
|
"st_size",
|
||||||
|
"st_mtime_ns",
|
||||||
|
"st_ctime_ns",
|
||||||
|
)
|
||||||
|
if any(getattr(self.before, f) != getattr(after, f) for f in fields):
|
||||||
|
raise ArchiveError("archive_changed")
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _nested(prefix: bytes) -> bool:
|
||||||
|
return prefix.startswith(
|
||||||
|
(
|
||||||
|
b"PK\x03\x04",
|
||||||
|
b"PK\x05\x06",
|
||||||
|
b"PK\x07\x08",
|
||||||
|
b"\x1f\x8b",
|
||||||
|
b"BZh",
|
||||||
|
b"\xfd7zXZ\x00",
|
||||||
|
b"Rar!\x1a\x07",
|
||||||
|
b"7z\xbc\xaf\x27\x1c",
|
||||||
|
)
|
||||||
|
) or (len(prefix) >= 262 and prefix[257:262] == b"ustar")
|
||||||
|
|
||||||
|
def chunks(self, name: str, maximum: int) -> Iterator[bytes]:
|
||||||
|
member = next((m for m in self.members if m.name == name), None)
|
||||||
|
if member is None or name in self.read_members or member.bytes > maximum:
|
||||||
|
raise ArchiveError("invalid_archive")
|
||||||
|
self.read_members.add(name)
|
||||||
|
used, crc, prefix = 0, 0, bytearray()
|
||||||
|
inflater = zlib.decompressobj(-15) if member.method == 8 else None
|
||||||
|
try:
|
||||||
|
for offset in range(0, member.compressed_bytes, CHUNK):
|
||||||
|
pending = self._read(
|
||||||
|
member.offset + offset, min(CHUNK, member.compressed_bytes - offset)
|
||||||
|
)
|
||||||
|
while pending:
|
||||||
|
self._tick()
|
||||||
|
if inflater:
|
||||||
|
data = inflater.decompress(pending, min(CHUNK, member.bytes - used + 1))
|
||||||
|
pending = inflater.unconsumed_tail
|
||||||
|
if inflater.unused_data:
|
||||||
|
raise ArchiveError("invalid_archive")
|
||||||
|
else:
|
||||||
|
data, pending = pending, b""
|
||||||
|
used += len(data)
|
||||||
|
self.unpacked_bytes += len(data)
|
||||||
|
if (
|
||||||
|
used > member.bytes
|
||||||
|
or used > maximum
|
||||||
|
or self.unpacked_bytes > MAX_UNPACKED_BYTES
|
||||||
|
):
|
||||||
|
raise ArchiveError("invalid_archive")
|
||||||
|
prefix.extend(data[: max(0, 512 - len(prefix))])
|
||||||
|
if self._nested(bytes(prefix)):
|
||||||
|
raise ArchiveError("invalid_archive")
|
||||||
|
crc = binascii.crc32(data, crc)
|
||||||
|
if data:
|
||||||
|
yield data
|
||||||
|
if (inflater and not inflater.eof) or used != member.bytes or crc != member.crc32:
|
||||||
|
raise ArchiveError("invalid_archive")
|
||||||
|
except zlib.error:
|
||||||
|
raise ArchiveError("invalid_archive") from None
|
||||||
|
|
||||||
|
def read(self, name: str, maximum: int) -> bytes:
|
||||||
|
return b"".join(self.chunks(name, maximum))
|
||||||
|
|
||||||
|
def extract_remaining(self, directory: int) -> dict[str, tuple[str, int]]:
|
||||||
|
info = os.fstat(directory)
|
||||||
|
if (
|
||||||
|
not stat.S_ISDIR(info.st_mode)
|
||||||
|
or info.st_uid != os.getuid()
|
||||||
|
or stat.S_IMODE(info.st_mode) != 0o700
|
||||||
|
):
|
||||||
|
raise ArchiveError("invalid_extraction_directory")
|
||||||
|
references = {}
|
||||||
|
for member in self.members:
|
||||||
|
if member.name in self.read_members:
|
||||||
|
continue
|
||||||
|
fd = os.open(
|
||||||
|
member.name,
|
||||||
|
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC,
|
||||||
|
0o600,
|
||||||
|
dir_fd=directory,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
digest, size = hashlib.sha256(), 0
|
||||||
|
for chunk in self.chunks(member.name, member.bytes):
|
||||||
|
digest.update(chunk)
|
||||||
|
size += len(chunk)
|
||||||
|
remaining = memoryview(chunk)
|
||||||
|
while remaining:
|
||||||
|
written = os.write(fd, remaining)
|
||||||
|
if written <= 0:
|
||||||
|
raise ArchiveError("extraction_unavailable")
|
||||||
|
remaining = remaining[written:]
|
||||||
|
os.fsync(fd)
|
||||||
|
references[member.name] = (digest.hexdigest(), size)
|
||||||
|
finally:
|
||||||
|
os.close(fd)
|
||||||
|
os.fsync(directory)
|
||||||
|
self.verify_unchanged()
|
||||||
|
return references
|
||||||
@@ -0,0 +1,126 @@
|
|||||||
|
"""Read-only private backup inventory, separate from garbage collection and grading."""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import asyncio
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import stat
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from sqlalchemy import select, text
|
||||||
|
from sqlalchemy.exc import SQLAlchemyError
|
||||||
|
|
||||||
|
from kf_oj.contracts.judging import ArtifactReference
|
||||||
|
from kf_oj.core.settings import Settings
|
||||||
|
from kf_oj.infra.artifacts.models import Artifact
|
||||||
|
from kf_oj.infra.db.session import build_engine, build_session_factory
|
||||||
|
|
||||||
|
MAX_REFERENCES = 65536
|
||||||
|
|
||||||
|
|
||||||
|
def verify_existing(root: Path, reference: ArtifactReference) -> None:
|
||||||
|
"""No mkdir, lock creation or writable descriptor, even for a missing store."""
|
||||||
|
root = root.absolute()
|
||||||
|
if root.resolve(strict=True) != root:
|
||||||
|
raise ValueError("backup storage path contains links")
|
||||||
|
directory = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC)
|
||||||
|
try:
|
||||||
|
info = os.fstat(directory)
|
||||||
|
if info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) != 0o700:
|
||||||
|
raise ValueError("backup storage root identity differs")
|
||||||
|
descriptor = os.open(
|
||||||
|
reference.digest,
|
||||||
|
os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC,
|
||||||
|
dir_fd=directory,
|
||||||
|
)
|
||||||
|
with os.fdopen(descriptor, "rb") as source:
|
||||||
|
before = os.fstat(source.fileno())
|
||||||
|
if (
|
||||||
|
not stat.S_ISREG(before.st_mode)
|
||||||
|
or before.st_nlink != 1
|
||||||
|
or before.st_uid != os.getuid()
|
||||||
|
or stat.S_IMODE(before.st_mode) != 0o600
|
||||||
|
or before.st_size != reference.bytes
|
||||||
|
):
|
||||||
|
raise ValueError("backup artifact identity or length differs")
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
remaining = reference.bytes
|
||||||
|
while remaining:
|
||||||
|
block = source.read(min(65536, remaining))
|
||||||
|
if not block:
|
||||||
|
raise ValueError("backup artifact truncated")
|
||||||
|
digest.update(block)
|
||||||
|
remaining -= len(block)
|
||||||
|
after = os.fstat(source.fileno())
|
||||||
|
if (
|
||||||
|
source.read(1)
|
||||||
|
or digest.hexdigest() != reference.digest
|
||||||
|
or any(
|
||||||
|
getattr(before, key) != getattr(after, key)
|
||||||
|
for key in ("st_dev", "st_ino", "st_size", "st_mtime_ns", "st_ctime_ns")
|
||||||
|
)
|
||||||
|
):
|
||||||
|
raise ValueError("backup artifact changed or digest differs")
|
||||||
|
finally:
|
||||||
|
os.close(directory)
|
||||||
|
|
||||||
|
|
||||||
|
async def inventory() -> dict:
|
||||||
|
"""Every ready registry object covers all enforced ready-state foreign keys.
|
||||||
|
|
||||||
|
The operator separately proves a stopped-writing cut. This CLI neither
|
||||||
|
claims that cut nor reads domain ORM models or changes a lifecycle state.
|
||||||
|
"""
|
||||||
|
settings = Settings()
|
||||||
|
if settings.artifact_root is None:
|
||||||
|
raise ValueError("artifact root required")
|
||||||
|
engine = build_engine(settings)
|
||||||
|
try:
|
||||||
|
sessions = build_session_factory(engine)
|
||||||
|
async with sessions.begin() as session:
|
||||||
|
await session.execute(text("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY"))
|
||||||
|
rows = await session.execute(
|
||||||
|
select(Artifact.digest, Artifact.bytes)
|
||||||
|
.where(Artifact.state == "ready")
|
||||||
|
.order_by(Artifact.digest)
|
||||||
|
.limit(MAX_REFERENCES + 1)
|
||||||
|
)
|
||||||
|
objects = list(rows)
|
||||||
|
if len(objects) > MAX_REFERENCES:
|
||||||
|
raise ValueError("backup registry count exceeds bound")
|
||||||
|
references = []
|
||||||
|
for digest, size in objects:
|
||||||
|
reference = ArtifactReference(digest=digest, bytes=size)
|
||||||
|
# A 64-KiB buffer checks actual bytes without source/case output.
|
||||||
|
await asyncio.to_thread(verify_existing, settings.artifact_root, reference)
|
||||||
|
references.append(reference.model_dump(mode="json"))
|
||||||
|
return {
|
||||||
|
"schema_version": 1,
|
||||||
|
"kind": "kf.private.artifact-backup-inventory",
|
||||||
|
"read_only": True,
|
||||||
|
"quiescence_verified": False,
|
||||||
|
"boot_id": Path("/proc/sys/kernel/random/boot_id").read_text().strip(),
|
||||||
|
"storage_identity": {
|
||||||
|
"device": settings.artifact_root.stat().st_dev,
|
||||||
|
"inode": settings.artifact_root.stat().st_ino,
|
||||||
|
"uid": settings.artifact_root.stat().st_uid,
|
||||||
|
},
|
||||||
|
"objects": references,
|
||||||
|
}
|
||||||
|
finally:
|
||||||
|
await engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
parser = argparse.ArgumentParser(description="Private read-only artifact backup inventory")
|
||||||
|
parser.parse_args()
|
||||||
|
try:
|
||||||
|
result = asyncio.run(asyncio.wait_for(inventory(), timeout=300))
|
||||||
|
print(json.dumps(result, separators=(",", ":"), sort_keys=True))
|
||||||
|
except (ValueError, OSError, SQLAlchemyError, TimeoutError):
|
||||||
|
parser.exit(1, "Backup artifact inventory refused; no metadata or grade was modified.\n")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -50,6 +50,28 @@ class ArtifactRepository:
|
|||||||
raise ArtifactError("artifact metadata unavailable")
|
raise ArtifactError("artifact metadata unavailable")
|
||||||
return await asyncio.to_thread(self.store.read, reference)
|
return await asyncio.to_thread(self.store.read, reference)
|
||||||
|
|
||||||
|
async def put_file(
|
||||||
|
self, session: AsyncSession, descriptor: int, reference: ArtifactReference
|
||||||
|
) -> ArtifactReference:
|
||||||
|
await lock_digest(session, reference.digest)
|
||||||
|
existing = await session.get(Artifact, reference.digest)
|
||||||
|
if existing is not None and (
|
||||||
|
existing.state != "ready" or existing.bytes != reference.bytes
|
||||||
|
):
|
||||||
|
raise ArtifactError("artifact metadata unavailable")
|
||||||
|
task = asyncio.create_task(asyncio.to_thread(self.store.put_file, descriptor, reference))
|
||||||
|
try:
|
||||||
|
await asyncio.shield(task)
|
||||||
|
except BaseException:
|
||||||
|
await task
|
||||||
|
raise
|
||||||
|
await session.execute(
|
||||||
|
insert(Artifact)
|
||||||
|
.values(digest=reference.digest, bytes=reference.bytes)
|
||||||
|
.on_conflict_do_nothing(index_elements=[Artifact.digest])
|
||||||
|
)
|
||||||
|
return reference
|
||||||
|
|
||||||
async def check_metadata(self, session: AsyncSession, reference: ArtifactReference) -> None:
|
async def check_metadata(self, session: AsyncSession, reference: ArtifactReference) -> None:
|
||||||
metadata = await session.get(Artifact, reference.digest)
|
metadata = await session.get(Artifact, reference.digest)
|
||||||
if metadata is None or metadata.state != "ready" or metadata.bytes != reference.bytes:
|
if metadata is None or metadata.state != "ready" or metadata.bytes != reference.bytes:
|
||||||
|
|||||||
@@ -0,0 +1,208 @@
|
|||||||
|
"""Owned, private, bounded import staging. A kernel lock survives parent/child handoff."""
|
||||||
|
|
||||||
|
import fcntl
|
||||||
|
import os
|
||||||
|
import stat
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from pathlib import Path
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from kf_oj.contracts.admin_problems import MAX_ARCHIVE_FILES, STAGING_MAX_BYTES
|
||||||
|
from kf_oj.infra.artifacts.store import ArtifactError
|
||||||
|
|
||||||
|
|
||||||
|
class ImportStaging:
|
||||||
|
def __init__(self, root: Path):
|
||||||
|
root = root.absolute()
|
||||||
|
root.mkdir(parents=True, mode=0o700, exist_ok=True)
|
||||||
|
if root.resolve() != root:
|
||||||
|
raise ArtifactError("invalid staging root")
|
||||||
|
self.root = root
|
||||||
|
self.fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC)
|
||||||
|
self.directory(self.fd)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def directory(fd):
|
||||||
|
s = os.fstat(fd)
|
||||||
|
if (
|
||||||
|
not stat.S_ISDIR(s.st_mode)
|
||||||
|
or s.st_uid != os.getuid()
|
||||||
|
or stat.S_IMODE(s.st_mode) != 0o700
|
||||||
|
):
|
||||||
|
raise ArtifactError("invalid staging directory")
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
os.close(self.fd)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def names(descriptor, maximum):
|
||||||
|
result = []
|
||||||
|
with os.scandir(descriptor) as entries:
|
||||||
|
for entry in entries:
|
||||||
|
if len(result) >= maximum:
|
||||||
|
raise ArtifactError("staging file capacity exhausted")
|
||||||
|
result.append(entry.name)
|
||||||
|
return result
|
||||||
|
|
||||||
|
def operation_ids(self):
|
||||||
|
identifiers = []
|
||||||
|
for name in self.names(self.fd, 33):
|
||||||
|
if name == ".lock":
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
identifier = UUID(name)
|
||||||
|
if str(identifier) != name:
|
||||||
|
raise ValueError
|
||||||
|
except ValueError:
|
||||||
|
raise ArtifactError("unexpected staging entry") from None
|
||||||
|
identifiers.append(identifier)
|
||||||
|
return identifiers
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def exclusive(self):
|
||||||
|
lock = os.open(
|
||||||
|
".lock", os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600, dir_fd=self.fd
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
from kf_oj.infra.artifacts.store import ArtifactStore
|
||||||
|
|
||||||
|
ArtifactStore._regular(os.fstat(lock))
|
||||||
|
try:
|
||||||
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||||
|
except BlockingIOError:
|
||||||
|
raise ArtifactError("import capacity busy") from None
|
||||||
|
yield lock
|
||||||
|
finally:
|
||||||
|
os.close(lock)
|
||||||
|
|
||||||
|
def usage(self):
|
||||||
|
total, count = 0, 0
|
||||||
|
for name in self.names(self.fd, 33):
|
||||||
|
if name == ".lock":
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
if str(UUID(name)) != name:
|
||||||
|
raise ValueError
|
||||||
|
except ValueError:
|
||||||
|
raise ArtifactError("unexpected staging entry") from None
|
||||||
|
descriptor = self.operation(UUID(name))
|
||||||
|
try:
|
||||||
|
for entry in self.names(descriptor, 3):
|
||||||
|
count += 1
|
||||||
|
if count > 2 * (MAX_ARCHIVE_FILES + 2):
|
||||||
|
raise ArtifactError("staging file capacity exhausted")
|
||||||
|
if entry == "members":
|
||||||
|
child = self.members(descriptor)
|
||||||
|
try:
|
||||||
|
for member in self.names(child, MAX_ARCHIVE_FILES):
|
||||||
|
count += 1
|
||||||
|
info = os.stat(member, dir_fd=child, follow_symlinks=False)
|
||||||
|
from kf_oj.infra.artifacts.store import ArtifactStore
|
||||||
|
|
||||||
|
ArtifactStore._regular(info)
|
||||||
|
total += info.st_size
|
||||||
|
finally:
|
||||||
|
os.close(child)
|
||||||
|
elif entry in ("archive", "validated.json"):
|
||||||
|
info = os.stat(entry, dir_fd=descriptor, follow_symlinks=False)
|
||||||
|
from kf_oj.infra.artifacts.store import ArtifactStore
|
||||||
|
|
||||||
|
ArtifactStore._regular(info)
|
||||||
|
total += info.st_size
|
||||||
|
else:
|
||||||
|
raise ArtifactError("unexpected staging entry")
|
||||||
|
if count > 2 * (MAX_ARCHIVE_FILES + 2) or total > STAGING_MAX_BYTES:
|
||||||
|
raise ArtifactError("staging capacity exhausted")
|
||||||
|
finally:
|
||||||
|
os.close(descriptor)
|
||||||
|
return total
|
||||||
|
|
||||||
|
def operation(self, operation_id: UUID, *, create=False):
|
||||||
|
name = str(operation_id)
|
||||||
|
if create:
|
||||||
|
try:
|
||||||
|
os.mkdir(name, 0o700, dir_fd=self.fd)
|
||||||
|
except FileExistsError:
|
||||||
|
pass
|
||||||
|
descriptor = os.open(
|
||||||
|
name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC, dir_fd=self.fd
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
self.directory(descriptor)
|
||||||
|
except Exception:
|
||||||
|
os.close(descriptor)
|
||||||
|
raise
|
||||||
|
return descriptor
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def members(operation, *, create=False):
|
||||||
|
if create:
|
||||||
|
os.mkdir("members", 0o700, dir_fd=operation)
|
||||||
|
descriptor = os.open(
|
||||||
|
"members", os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC, dir_fd=operation
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
ImportStaging.directory(descriptor)
|
||||||
|
except Exception:
|
||||||
|
os.close(descriptor)
|
||||||
|
raise
|
||||||
|
return descriptor
|
||||||
|
|
||||||
|
def remove(self, operation_id):
|
||||||
|
try:
|
||||||
|
descriptor = self.operation(operation_id)
|
||||||
|
except FileNotFoundError:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
names = self.names(descriptor, 3)
|
||||||
|
if len(names) > 3:
|
||||||
|
raise ArtifactError("unexpected staging entry")
|
||||||
|
for name in names:
|
||||||
|
if name == "members":
|
||||||
|
child = self.members(descriptor)
|
||||||
|
try:
|
||||||
|
entries = self.names(child, MAX_ARCHIVE_FILES)
|
||||||
|
if len(entries) > MAX_ARCHIVE_FILES:
|
||||||
|
raise ArtifactError("unexpected staging entry")
|
||||||
|
for member in entries:
|
||||||
|
from kf_oj.infra.artifacts.store import ArtifactStore
|
||||||
|
|
||||||
|
ArtifactStore._regular(
|
||||||
|
os.stat(member, dir_fd=child, follow_symlinks=False)
|
||||||
|
)
|
||||||
|
os.unlink(member, dir_fd=child)
|
||||||
|
finally:
|
||||||
|
os.close(child)
|
||||||
|
os.rmdir("members", dir_fd=descriptor)
|
||||||
|
elif name in ("archive", "validated.json"):
|
||||||
|
from kf_oj.infra.artifacts.store import ArtifactStore
|
||||||
|
|
||||||
|
ArtifactStore._regular(os.stat(name, dir_fd=descriptor, follow_symlinks=False))
|
||||||
|
os.unlink(name, dir_fd=descriptor)
|
||||||
|
else:
|
||||||
|
raise ArtifactError("unexpected staging entry")
|
||||||
|
finally:
|
||||||
|
os.close(descriptor)
|
||||||
|
os.rmdir(str(operation_id), dir_fd=self.fd)
|
||||||
|
os.fsync(self.fd)
|
||||||
|
|
||||||
|
def reset_validation(self, operation_id):
|
||||||
|
directory = self.operation(operation_id)
|
||||||
|
try:
|
||||||
|
names = self.names(directory, 3)
|
||||||
|
if "members" in names:
|
||||||
|
members = self.members(directory)
|
||||||
|
try:
|
||||||
|
for name in self.names(members, MAX_ARCHIVE_FILES):
|
||||||
|
from kf_oj.infra.artifacts.store import ArtifactStore
|
||||||
|
|
||||||
|
ArtifactStore._regular(os.stat(name, dir_fd=members, follow_symlinks=False))
|
||||||
|
os.unlink(name, dir_fd=members)
|
||||||
|
finally:
|
||||||
|
os.close(members)
|
||||||
|
os.rmdir("members", dir_fd=directory)
|
||||||
|
if "validated.json" in names:
|
||||||
|
os.unlink("validated.json", dir_fd=directory)
|
||||||
|
os.fsync(directory)
|
||||||
|
finally:
|
||||||
|
os.close(directory)
|
||||||
@@ -168,6 +168,135 @@ class ArtifactStore:
|
|||||||
with self.exclusive():
|
with self.exclusive():
|
||||||
return self._read(reference)
|
return self._read(reference)
|
||||||
|
|
||||||
|
def verify(self, reference: ArtifactReference) -> None:
|
||||||
|
"""Verify publication inputs with a 64 KiB buffer, without exposing hidden bytes."""
|
||||||
|
ArtifactReference.model_validate(reference)
|
||||||
|
with self.exclusive():
|
||||||
|
try:
|
||||||
|
descriptor = os.open(
|
||||||
|
reference.digest,
|
||||||
|
os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC,
|
||||||
|
dir_fd=self._directory,
|
||||||
|
)
|
||||||
|
with os.fdopen(descriptor, "rb") as source:
|
||||||
|
before = os.fstat(source.fileno())
|
||||||
|
self._regular(before)
|
||||||
|
if before.st_size != reference.bytes:
|
||||||
|
raise ArtifactError("artifact digest or length mismatch")
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
remaining = reference.bytes
|
||||||
|
while remaining:
|
||||||
|
data = source.read(min(65536, remaining))
|
||||||
|
if not data:
|
||||||
|
raise ArtifactError("artifact digest or length mismatch")
|
||||||
|
digest.update(data)
|
||||||
|
remaining -= len(data)
|
||||||
|
after = os.fstat(source.fileno())
|
||||||
|
if (
|
||||||
|
source.read(1)
|
||||||
|
or digest.hexdigest() != reference.digest
|
||||||
|
or any(
|
||||||
|
getattr(before, k) != getattr(after, k)
|
||||||
|
for k in (
|
||||||
|
"st_dev",
|
||||||
|
"st_ino",
|
||||||
|
"st_nlink",
|
||||||
|
"st_mode",
|
||||||
|
"st_uid",
|
||||||
|
"st_size",
|
||||||
|
"st_mtime_ns",
|
||||||
|
"st_ctime_ns",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
):
|
||||||
|
raise ArtifactError("artifact input changed")
|
||||||
|
except OSError:
|
||||||
|
raise ArtifactError("artifact unavailable") from None
|
||||||
|
|
||||||
|
def put_file(self, source: int, expected: ArtifactReference) -> ArtifactReference:
|
||||||
|
"""Adopt a validated private FD without buffering a hidden testcase in memory."""
|
||||||
|
before = os.fstat(source)
|
||||||
|
self._regular(before)
|
||||||
|
if before.st_size != expected.bytes:
|
||||||
|
raise ArtifactError("artifact digest or length mismatch")
|
||||||
|
|
||||||
|
def unchanged(descriptor, original):
|
||||||
|
current = os.fstat(descriptor)
|
||||||
|
fields = (
|
||||||
|
"st_dev",
|
||||||
|
"st_ino",
|
||||||
|
"st_nlink",
|
||||||
|
"st_mode",
|
||||||
|
"st_uid",
|
||||||
|
"st_size",
|
||||||
|
"st_mtime_ns",
|
||||||
|
"st_ctime_ns",
|
||||||
|
)
|
||||||
|
if any(getattr(current, k) != getattr(original, k) for k in fields):
|
||||||
|
raise ArtifactError("artifact input changed")
|
||||||
|
|
||||||
|
def stream(descriptor, *, output=None):
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
for offset in range(0, expected.bytes, 65536):
|
||||||
|
data = os.pread(descriptor, min(65536, expected.bytes - offset), offset)
|
||||||
|
if len(data) != min(65536, expected.bytes - offset):
|
||||||
|
raise ArtifactError("artifact digest or length mismatch")
|
||||||
|
digest.update(data)
|
||||||
|
if output is not None:
|
||||||
|
output.write(data)
|
||||||
|
if digest.hexdigest() != expected.digest:
|
||||||
|
raise ArtifactError("artifact digest or length mismatch")
|
||||||
|
|
||||||
|
with self.exclusive():
|
||||||
|
entries = self._entries()
|
||||||
|
if any(name == expected.digest for name, _ in entries):
|
||||||
|
existing = os.open(
|
||||||
|
expected.digest,
|
||||||
|
os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC,
|
||||||
|
dir_fd=self._directory,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
self._regular(os.fstat(existing))
|
||||||
|
if os.fstat(existing).st_size != expected.bytes:
|
||||||
|
raise ArtifactError("artifact digest or length mismatch")
|
||||||
|
stream(source)
|
||||||
|
stream(existing)
|
||||||
|
finally:
|
||||||
|
os.close(existing)
|
||||||
|
else:
|
||||||
|
if (
|
||||||
|
len(entries) + 1 > self.max_files
|
||||||
|
or sum(s.st_size for _, s in entries) + expected.bytes > self.max_bytes
|
||||||
|
):
|
||||||
|
raise ArtifactError("storage capacity exhausted")
|
||||||
|
temporary = ".tmp-" + uuid4().hex
|
||||||
|
descriptor = os.open(
|
||||||
|
temporary,
|
||||||
|
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC,
|
||||||
|
0o600,
|
||||||
|
dir_fd=self._directory,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with os.fdopen(descriptor, "wb") as out:
|
||||||
|
stream(source, output=out)
|
||||||
|
out.flush()
|
||||||
|
os.fsync(out.fileno())
|
||||||
|
unchanged(source, before)
|
||||||
|
os.rename(
|
||||||
|
temporary,
|
||||||
|
expected.digest,
|
||||||
|
src_dir_fd=self._directory,
|
||||||
|
dst_dir_fd=self._directory,
|
||||||
|
)
|
||||||
|
os.fsync(self._directory)
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
os.unlink(temporary, dir_fd=self._directory)
|
||||||
|
except FileNotFoundError:
|
||||||
|
pass
|
||||||
|
unchanged(source, before)
|
||||||
|
return expected
|
||||||
|
|
||||||
def remove(self, digest: str) -> None:
|
def remove(self, digest: str) -> None:
|
||||||
if not DIGEST.fullmatch(digest):
|
if not DIGEST.fullmatch(digest):
|
||||||
raise ArtifactError("invalid artifact identifier")
|
raise ArtifactError("invalid artifact identifier")
|
||||||
|
|||||||
@@ -0,0 +1,52 @@
|
|||||||
|
"""Ordinary read-only view of the fixed protected Root admission record."""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import stat
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from kf_oj.contracts.availability import AVAILABILITY_PATH, AvailabilityRecord
|
||||||
|
|
||||||
|
GATE = Path(AVAILABILITY_PATH)
|
||||||
|
|
||||||
|
|
||||||
|
def protected_record() -> AvailabilityRecord:
|
||||||
|
for path in GATE.parents:
|
||||||
|
info = path.lstat()
|
||||||
|
if not stat.S_ISDIR(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
|
||||||
|
raise ValueError("protected availability directory required")
|
||||||
|
fd = os.open(GATE, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC)
|
||||||
|
try:
|
||||||
|
info = os.fstat(fd)
|
||||||
|
if (
|
||||||
|
not stat.S_ISREG(info.st_mode)
|
||||||
|
or info.st_nlink != 1
|
||||||
|
or info.st_uid != 0
|
||||||
|
or info.st_gid != 0
|
||||||
|
or stat.S_IMODE(info.st_mode) != 0o644
|
||||||
|
or info.st_size > 4096
|
||||||
|
):
|
||||||
|
raise ValueError("protected availability file required")
|
||||||
|
with os.fdopen(fd, "rb", closefd=False) as source:
|
||||||
|
contents = source.read(4097)
|
||||||
|
if len(contents) > 4096:
|
||||||
|
raise ValueError("availability size bound")
|
||||||
|
finally:
|
||||||
|
os.close(fd)
|
||||||
|
|
||||||
|
def unique(pairs):
|
||||||
|
value = {}
|
||||||
|
for key, item in pairs:
|
||||||
|
if key in value:
|
||||||
|
raise ValueError("duplicate availability field")
|
||||||
|
value[key] = item
|
||||||
|
return value
|
||||||
|
|
||||||
|
return AvailabilityRecord.model_validate(json.loads(contents, object_pairs_hook=unique))
|
||||||
|
|
||||||
|
|
||||||
|
def current(release: str | None, batch: str | None) -> AvailabilityRecord:
|
||||||
|
value = protected_record()
|
||||||
|
if value.release_id != release or value.configuration_id != batch:
|
||||||
|
raise ValueError("availability is not this running API binding")
|
||||||
|
return value
|
||||||
@@ -13,6 +13,7 @@ def build_engine(settings: Settings) -> AsyncEngine:
|
|||||||
max_overflow=0,
|
max_overflow=0,
|
||||||
pool_timeout=5,
|
pool_timeout=5,
|
||||||
connect_args={
|
connect_args={
|
||||||
|
"timeout": 5,
|
||||||
"command_timeout": 10,
|
"command_timeout": 10,
|
||||||
"server_settings": {
|
"server_settings": {
|
||||||
"statement_timeout": "10000",
|
"statement_timeout": "10000",
|
||||||
|
|||||||
@@ -0,0 +1,48 @@
|
|||||||
|
"""Read-only bounded outbox counts, without broker or delivery dependencies."""
|
||||||
|
|
||||||
|
from sqlalchemy import text
|
||||||
|
|
||||||
|
|
||||||
|
class OutboxQueries:
|
||||||
|
def __init__(self, sessions):
|
||||||
|
self.sessions = sessions
|
||||||
|
|
||||||
|
async def status(self):
|
||||||
|
async with self.sessions() as session, session.begin():
|
||||||
|
await session.execute(text("SET TRANSACTION READ ONLY"))
|
||||||
|
await session.execute(text("SET LOCAL statement_timeout='1500'"))
|
||||||
|
await session.execute(text("SET LOCAL lock_timeout='500'"))
|
||||||
|
safe = await session.scalar(
|
||||||
|
text("""
|
||||||
|
SELECT rolcanlogin AND NOT(rolsuper OR rolcreatedb OR rolcreaterole
|
||||||
|
OR rolreplication OR rolbypassrls)
|
||||||
|
AND pg_has_role(current_user,'kf_oj_platform','USAGE')
|
||||||
|
AND NOT pg_has_role(current_user,'kf_oj_coordinator','MEMBER')
|
||||||
|
AND NOT pg_has_role(current_user,'kf_oj_identity_authority_owner','MEMBER')
|
||||||
|
AND NOT has_column_privilege(current_user,'evaluations','state','UPDATE')
|
||||||
|
FROM pg_roles WHERE rolname=session_user
|
||||||
|
""")
|
||||||
|
)
|
||||||
|
if safe is not True:
|
||||||
|
raise ValueError("dedicated outbox projection role required")
|
||||||
|
row = (
|
||||||
|
(
|
||||||
|
await session.execute(
|
||||||
|
text("""
|
||||||
|
SELECT
|
||||||
|
(SELECT count(*) FROM (SELECT 1 FROM outbox
|
||||||
|
WHERE delivered_at IS NULL LIMIT 129) p) pending,
|
||||||
|
(SELECT count(*) FROM (SELECT 1 FROM outbox WHERE delivered_at IS NULL
|
||||||
|
AND locked_until>clock_timestamp() LIMIT 129) l) locked
|
||||||
|
""")
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.mappings()
|
||||||
|
.one()
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"schema_version": 1,
|
||||||
|
"pending_capped": min(row["pending"], 128),
|
||||||
|
"locked_capped": min(row["locked"], 128),
|
||||||
|
"truncated": row["pending"] > 128 or row["locked"] > 128,
|
||||||
|
}
|
||||||
@@ -11,6 +11,8 @@ from sqlalchemy import text
|
|||||||
from kf_oj.contracts.judge import JudgeTaskNotification
|
from kf_oj.contracts.judge import JudgeTaskNotification
|
||||||
from kf_oj.core.settings import Settings
|
from kf_oj.core.settings import Settings
|
||||||
from kf_oj.infra.db.session import build_engine, build_session_factory
|
from kf_oj.infra.db.session import build_engine, build_session_factory
|
||||||
|
from kf_oj.infra.messaging.outbox_queries import OutboxQueries
|
||||||
|
from kf_oj.infra.messaging.publisher_diagnostics import emit_failure
|
||||||
|
|
||||||
QUEUE = "kf.judge.evaluations.v1"
|
QUEUE = "kf.judge.evaluations.v1"
|
||||||
QUEUE_ARGUMENTS = {
|
QUEUE_ARGUMENTS = {
|
||||||
@@ -25,6 +27,9 @@ class OutboxRepository:
|
|||||||
def __init__(self, sessions):
|
def __init__(self, sessions):
|
||||||
self.sessions = sessions
|
self.sessions = sessions
|
||||||
|
|
||||||
|
async def status(self):
|
||||||
|
return await OutboxQueries(self.sessions).status()
|
||||||
|
|
||||||
async def claim(self):
|
async def claim(self):
|
||||||
owner = uuid4()
|
owner = uuid4()
|
||||||
async with self.sessions.begin() as s:
|
async with self.sessions.begin() as s:
|
||||||
@@ -124,6 +129,7 @@ class ConfirmedBroker:
|
|||||||
|
|
||||||
|
|
||||||
async def run(settings: Settings, stop: asyncio.Event):
|
async def run(settings: Settings, stop: asyncio.Event):
|
||||||
|
_startup_phase = "configuration"
|
||||||
if settings.amqp_url is None:
|
if settings.amqp_url is None:
|
||||||
raise ValueError("AMQP configuration required")
|
raise ValueError("AMQP configuration required")
|
||||||
engine = build_engine(settings)
|
engine = build_engine(settings)
|
||||||
@@ -134,7 +140,9 @@ async def run(settings: Settings, stop: asyncio.Event):
|
|||||||
for name in ("aio_pika", "aiormq", "pamqp"):
|
for name in ("aio_pika", "aiormq", "pamqp"):
|
||||||
logging.getLogger(name).disabled = True
|
logging.getLogger(name).disabled = True
|
||||||
try:
|
try:
|
||||||
|
_startup_phase = "database_connect"
|
||||||
async with engine.connect() as c:
|
async with engine.connect() as c:
|
||||||
|
_startup_phase = "database_authority"
|
||||||
allowed = await c.scalar(
|
allowed = await c.scalar(
|
||||||
text("""SELECT NOT r.rolsuper AND NOT r.rolbypassrls
|
text("""SELECT NOT r.rolsuper AND NOT r.rolbypassrls
|
||||||
AND pg_has_role(current_user,'kf_oj_platform','MEMBER')
|
AND pg_has_role(current_user,'kf_oj_platform','MEMBER')
|
||||||
@@ -143,6 +151,7 @@ async def run(settings: Settings, stop: asyncio.Event):
|
|||||||
)
|
)
|
||||||
if not allowed:
|
if not allowed:
|
||||||
raise ValueError("dedicated platform database role required")
|
raise ValueError("dedicated platform database role required")
|
||||||
|
_startup_phase = "running"
|
||||||
while not stop.is_set():
|
while not stop.is_set():
|
||||||
try:
|
try:
|
||||||
item = await repository.claim()
|
item = await repository.claim()
|
||||||
@@ -175,7 +184,11 @@ def main():
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
asyncio.run(entry())
|
asyncio.run(entry())
|
||||||
except Exception:
|
except Exception as error:
|
||||||
|
try:
|
||||||
|
emit_failure(error, run.__code__)
|
||||||
|
except BaseException:
|
||||||
|
pass # Diagnostics must never replace the original fatal result.
|
||||||
raise SystemExit("outbox unavailable") from None
|
raise SystemExit("outbox unavailable") from None
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,160 @@
|
|||||||
|
"""Bounded outbox failure locations; never serialize errors or connection inputs."""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from pathlib import PurePath
|
||||||
|
from types import CodeType
|
||||||
|
|
||||||
|
PREFIX = "KF_OUTBOX_START_FAILURE "
|
||||||
|
PHASES = {"configuration", "database_connect", "database_authority", "running"}
|
||||||
|
ERROR_TYPES = {
|
||||||
|
"ValueError",
|
||||||
|
"ValidationError",
|
||||||
|
"TimeoutError",
|
||||||
|
"OSError",
|
||||||
|
"PermissionError",
|
||||||
|
"ConnectionRefusedError",
|
||||||
|
"ConnectionResetError",
|
||||||
|
"MemoryError",
|
||||||
|
"RuntimeError",
|
||||||
|
"InvalidPasswordError",
|
||||||
|
"InvalidAuthorizationSpecificationError",
|
||||||
|
"CannotConnectNowError",
|
||||||
|
"InterfaceError",
|
||||||
|
"OperationalError",
|
||||||
|
"DBAPIError",
|
||||||
|
"ProgrammingError",
|
||||||
|
"DataError",
|
||||||
|
"InternalServerError",
|
||||||
|
"PostgresError",
|
||||||
|
"InvalidCatalogNameError",
|
||||||
|
"InsufficientPrivilegeError",
|
||||||
|
"TooManyConnectionsError",
|
||||||
|
"QueryCanceledError",
|
||||||
|
"ConnectionDoesNotExistError",
|
||||||
|
"InternalClientError",
|
||||||
|
"ReadOnlySQLTransactionError",
|
||||||
|
}
|
||||||
|
FILES = {
|
||||||
|
"publisher.py",
|
||||||
|
"session.py",
|
||||||
|
"settings.py",
|
||||||
|
"base_events.py",
|
||||||
|
"runners.py",
|
||||||
|
"tasks.py",
|
||||||
|
"engine.py",
|
||||||
|
"base.py",
|
||||||
|
"default.py",
|
||||||
|
"connection.py",
|
||||||
|
"transaction.py",
|
||||||
|
"connect_utils.py",
|
||||||
|
"asyncpg.py",
|
||||||
|
"_concurrency_py3k.py",
|
||||||
|
"impl.py",
|
||||||
|
"selector_events.py",
|
||||||
|
"main.py",
|
||||||
|
}
|
||||||
|
FUNCTIONS = {
|
||||||
|
"main",
|
||||||
|
"entry",
|
||||||
|
"run",
|
||||||
|
"build_engine",
|
||||||
|
"connect",
|
||||||
|
"execute",
|
||||||
|
"scalar",
|
||||||
|
"_execute",
|
||||||
|
"__execute",
|
||||||
|
"_do_execute",
|
||||||
|
"_execute_context",
|
||||||
|
"_exec_single_context",
|
||||||
|
"_handle_dbapi_exception",
|
||||||
|
"_wrap_pool_connect",
|
||||||
|
"raw_connection",
|
||||||
|
"run_until_complete",
|
||||||
|
"wait_for",
|
||||||
|
"__aenter__",
|
||||||
|
"__aexit__",
|
||||||
|
"_connect",
|
||||||
|
"_connect_addr",
|
||||||
|
"__connect_addr",
|
||||||
|
"_get_pool_impl",
|
||||||
|
"_create_connection",
|
||||||
|
"_do_get",
|
||||||
|
"_invoke_creator",
|
||||||
|
"await_only",
|
||||||
|
"greenlet_spawn",
|
||||||
|
"_connection_for_bind",
|
||||||
|
"start",
|
||||||
|
"_async_ping",
|
||||||
|
"_do_ping_w_event",
|
||||||
|
"__init__",
|
||||||
|
"_sock_connect_cb",
|
||||||
|
"sock_connect",
|
||||||
|
"_ensure_connected",
|
||||||
|
"_get_statement",
|
||||||
|
"_execute_internal",
|
||||||
|
"close",
|
||||||
|
"dispose",
|
||||||
|
"_cleanup_stmts",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def failure_record(error: Exception, run_code: CodeType) -> dict:
|
||||||
|
"""Read existing exception/frame metadata, without altering the failed operation."""
|
||||||
|
pending: list[tuple[str, BaseException]] = [("error", error)]
|
||||||
|
seen = set()
|
||||||
|
records = []
|
||||||
|
phase = "unknown"
|
||||||
|
while pending and len(records) < 4:
|
||||||
|
relation, item = pending.pop(0)
|
||||||
|
if id(item) in seen:
|
||||||
|
continue
|
||||||
|
seen.add(id(item))
|
||||||
|
kind = type(item).__name__
|
||||||
|
state = getattr(item, "sqlstate", None)
|
||||||
|
if type(state) is not str or re.fullmatch(r"[A-Z0-9]{5}", state) is None:
|
||||||
|
state = None
|
||||||
|
frames = []
|
||||||
|
trace = item.__traceback__
|
||||||
|
depth = 0
|
||||||
|
while trace is not None and depth < 32:
|
||||||
|
code = trace.tb_frame.f_code
|
||||||
|
if code is run_code:
|
||||||
|
actual = trace.tb_frame.f_locals.get("_startup_phase")
|
||||||
|
if type(actual) is str and actual in PHASES:
|
||||||
|
phase = actual
|
||||||
|
name = PurePath(code.co_filename).name
|
||||||
|
if name in FILES and code.co_name in FUNCTIONS and 1 <= trace.tb_lineno <= 100000:
|
||||||
|
frames.append({"file": name, "line": trace.tb_lineno, "function": code.co_name})
|
||||||
|
trace = trace.tb_next
|
||||||
|
depth += 1
|
||||||
|
records.append(
|
||||||
|
{
|
||||||
|
"relation": relation,
|
||||||
|
"type": kind if kind in ERROR_TYPES else "other",
|
||||||
|
"sqlstate": state,
|
||||||
|
"frames": frames[-4:],
|
||||||
|
"frames_truncated": len(frames) > 4 or trace is not None,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
for link in ("orig", "__cause__", "__context__"):
|
||||||
|
child = getattr(item, link, None)
|
||||||
|
if isinstance(child, BaseException) and id(child) not in seen:
|
||||||
|
pending.append(
|
||||||
|
({"orig": "orig", "__cause__": "cause", "__context__": "context"}[link], child)
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"schema_version": 1,
|
||||||
|
"component": "outbox",
|
||||||
|
"phase": phase,
|
||||||
|
"errors": records,
|
||||||
|
"chain_truncated": any(id(item) not in seen for _, item in pending),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def emit_failure(error: Exception, run_code: CodeType) -> None:
|
||||||
|
encoded = json.dumps(failure_record(error, run_code), separators=(",", ":"), ensure_ascii=True)
|
||||||
|
if len(encoded) > 3072:
|
||||||
|
raise ValueError("outbox diagnostic bound exceeded")
|
||||||
|
print(PREFIX + encoded, file=sys.stderr, flush=True)
|
||||||
@@ -4,7 +4,7 @@
|
|||||||
`public.py` 提供已验证主体,`service.py` 管理账号和可撤销会话,`repository.py` 封装持久化。
|
`public.py` 提供已验证主体,`service.py` 管理账号和可撤销会话,`repository.py` 封装持久化。
|
||||||
用户身份不能来自前端自报字段或模型参数。
|
用户身份不能来自前端自报字段或模型参数。
|
||||||
|
|
||||||
本地管理命令为 `uv run --locked kf-accounts create <username>` / `disable <username>`。
|
本地管理命令为 `uv run --locked kf-accounts create <username>`。
|
||||||
创建口令由隐藏终端提示读取和确认,不接受命令行参数、环境中的默认口令或自助注册。
|
创建口令由隐藏终端提示读取和确认,不接受命令行参数、环境中的默认口令或自助注册。
|
||||||
用户名为 3–24 个小写 ASCII 字母、数字或下划线;口令 12–128 字符,最多 512 UTF-8 字节。
|
用户名为 3–24 个小写 ASCII 字母、数字或下划线;口令 12–128 字符,最多 512 UTF-8 字节。
|
||||||
密码摘要由新增的 `argon2-cffi` 依赖提供 Argon2id,固定 19 MiB/2 次/1 并行度。
|
密码摘要由新增的 `argon2-cffi` 依赖提供 Argon2id,固定 19 MiB/2 次/1 并行度。
|
||||||
@@ -14,6 +14,39 @@
|
|||||||
登录接口为 `/api/v1/auth/login`,退出 `/logout`,当前主体 `/me`。会话使用随机 256-bit token,
|
登录接口为 `/api/v1/auth/login`,退出 `/logout`,当前主体 `/me`。会话使用随机 256-bit token,
|
||||||
数据库只保存 token / CSRF 摘要;默认 12 小时有效,单账号保留至多 10 个有效会话。
|
数据库只保存 token / CSRF 摘要;默认 12 小时有效,单账号保留至多 10 个有效会话。
|
||||||
账号禁用立即拒绝登录和旧会话。请求只采用 ASGI 连接 peer,不读取客户端转发地址。
|
账号禁用立即拒绝登录和旧会话。请求只采用 ASGI 连接 peer,不读取客户端转发地址。
|
||||||
|
|
||||||
|
`0005_identity_authority` 为旧账号保留普通用户默认值,并加入管理员标记与服务端派生的授权版本。
|
||||||
|
启用状态或管理员标记实际变化时,同一事务递增授权版本并撤销旧会话;事务回滚也回滚撤销。
|
||||||
|
平台账号仅可插入普通账号列、更新启用状态,不能写管理员标记或授权版本;管理员的启用状态
|
||||||
|
需要独立本地运维身份。`kf_oj_identity_operator` 与 `kf_oj_problem_import` 为无登录的独立权限组,
|
||||||
|
口令与登录身份由运维另行配置,平台不继承运维组。导入组目前没有表权限,后续导入迁移只授予
|
||||||
|
队列与候选所需权限,不授予评测状态写入权。迁移不创建默认管理员。
|
||||||
|
保留授权或撤销历史后拒绝回退这次迁移。
|
||||||
|
|
||||||
|
`0006_authority_operations` 将运维组进一步收敛为固定函数的 EXECUTE 权限;不能直接读取
|
||||||
|
口令/会话摘要或修改账号/会话。函数由独立 NOLOGIN 角色持有,仅有授权所需列权限,固定
|
||||||
|
`search_path` 并逐项限定操作。所有账号写操作在行锁前更新同一个事务守卫;撤销/禁用/删除
|
||||||
|
最后一名启用管理员由数据库拒绝。重复授予/撤销不递增授权版本,实际变更仍原子撤销会话。
|
||||||
|
旧可重复读快照的并发变更产生序列化失败,调用方须结束整个事务后重新操作;不吞掉失败。
|
||||||
|
并发语义参见[PostgreSQL事务隔离](https://www.postgresql.org/docs/18/transaction-iso.html),
|
||||||
|
固定函数的权限与搜索路径参见[CREATE FUNCTION](https://www.postgresql.org/docs/18/sql-createfunction.html)。
|
||||||
|
|
||||||
|
在单独的本地运维环境配置 `KF_OJ_IDENTITY_OPERATOR_DATABASE_URL`,使用具有
|
||||||
|
`kf_oj_identity_operator` 权限组的普通 LOGIN,禁止超级用户、建库/建角色/复制/绕过 RLS
|
||||||
|
以及平台/协调器/导入组成员。API、Worker 和常规服务不配置此凭据,CLI 不回退平台身份。
|
||||||
|
先显式创建普通账号,再运行 `uv run --locked kf-accounts bootstrap-admin <username>`。
|
||||||
|
没有启用管理员时只有明确 bootstrap 可首次授权;不创建默认账号或口令。已有管理员后使用
|
||||||
|
`grant-admin` / `revoke-admin` / `disable` / `enable`。恢复已禁用管理员使用 `enable`;恢复失去
|
||||||
|
全部管理员的受控数据库备份时可再次显式 bootstrap。最后一名启用管理员不能被撤销或禁用,
|
||||||
|
应先授予另一名账号。操作的锁等待最多 2 秒、SQL 最多 5 秒,CLI 只输出账号/能力/授权版本和
|
||||||
|
是否变化;错误不输出连接串、SQL 参数或驱动诊断。网页管理 API 与题包导入仍待后续实现。
|
||||||
|
|
||||||
|
当前主体与登录响应附带版本为 1 的 `capabilities`,其中维护能力和十进制授权版本来自当前
|
||||||
|
有效会话关联的账号;浏览器只展示这些信息,不能提交它们来获得授权。模块公开端口
|
||||||
|
`ProblemMaintenancePort.require_problem_maintenance` 在调用方写事务中重新检查账号、授权版本
|
||||||
|
与实时会话,并按账号→会话的顺序持有 SHARE 锁直到提交/回滚,防止验证之后撤权的竞态。
|
||||||
|
普通构造的 `Principal` 默认没有维护能力;会话摘要仅在服务端主体内传递,不进入 DTO 或 repr。
|
||||||
|
|
||||||
受控启动使用 `--no-proxy-headers`,避免服务器中间件预先替换 peer;反向代理后的用户共享
|
受控启动使用 `--no-proxy-headers`,避免服务器中间件预先替换 peer;反向代理后的用户共享
|
||||||
该连接预算,引入可信代理的独立限流策略之前不能直接打开客户端转发头信任。
|
该连接预算,引入可信代理的独立限流策略之前不能直接打开客户端转发头信任。
|
||||||
每 peer 至多 10 次登录/15 分钟,预算通过真实数据库串行更新,失败尝试也消耗预算。
|
每 peer 至多 10 次登录/15 分钟,预算通过真实数据库串行更新,失败尝试也消耗预算。
|
||||||
|
|||||||
@@ -0,0 +1,83 @@
|
|||||||
|
"""Local-only composition for fixed database authority operations."""
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Literal
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from sqlalchemy import text
|
||||||
|
from sqlalchemy.exc import DBAPIError
|
||||||
|
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||||
|
|
||||||
|
from kf_oj.modules.identity.passwords import USERNAME
|
||||||
|
|
||||||
|
AuthorityOperation = Literal["bootstrap", "grant", "revoke", "enable", "disable"]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class AuthorityResult:
|
||||||
|
account_id: UUID
|
||||||
|
username: str
|
||||||
|
enabled: bool
|
||||||
|
is_administrator: bool
|
||||||
|
authorization_version: int
|
||||||
|
changed: bool
|
||||||
|
|
||||||
|
|
||||||
|
async def manage_authority(
|
||||||
|
sessions: async_sessionmaker, username: str, operation: AuthorityOperation
|
||||||
|
) -> AuthorityResult:
|
||||||
|
if not USERNAME.fullmatch(username) or operation not in (
|
||||||
|
"bootstrap",
|
||||||
|
"grant",
|
||||||
|
"revoke",
|
||||||
|
"enable",
|
||||||
|
"disable",
|
||||||
|
):
|
||||||
|
raise ValueError("invalid authority operation")
|
||||||
|
try:
|
||||||
|
async with sessions.begin() as session:
|
||||||
|
await session.execute(text("SET LOCAL statement_timeout='5s'"))
|
||||||
|
await session.execute(text("SET LOCAL lock_timeout='2s'"))
|
||||||
|
# The CLI must use a dedicated, unprivileged local operator login.
|
||||||
|
# Web/platform, import and coordinator credentials are never reused.
|
||||||
|
safe = await session.scalar(
|
||||||
|
text("""
|
||||||
|
SELECT NOT (rolsuper OR rolcreatedb OR rolcreaterole OR rolreplication
|
||||||
|
OR rolbypassrls)
|
||||||
|
AND rolcanlogin AND pg_has_role(current_user,'kf_oj_identity_operator','USAGE')
|
||||||
|
AND NOT pg_has_role(current_user,'kf_oj_platform','MEMBER')
|
||||||
|
AND NOT pg_has_role(current_user,'kf_oj_coordinator','MEMBER')
|
||||||
|
AND NOT pg_has_role(current_user,'kf_oj_problem_import','MEMBER')
|
||||||
|
FROM pg_roles WHERE rolname=current_user
|
||||||
|
""")
|
||||||
|
)
|
||||||
|
if not safe:
|
||||||
|
raise ValueError("a dedicated unprivileged identity operator login is required")
|
||||||
|
row = (
|
||||||
|
(
|
||||||
|
await session.execute(
|
||||||
|
text(
|
||||||
|
"SELECT * FROM public.kf_manage_account_authority(:username,:operation)"
|
||||||
|
),
|
||||||
|
{"username": username, "operation": operation},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.mappings()
|
||||||
|
.one()
|
||||||
|
)
|
||||||
|
return AuthorityResult(**row)
|
||||||
|
except DBAPIError as error:
|
||||||
|
code = getattr(error.orig, "sqlstate", None)
|
||||||
|
messages = {
|
||||||
|
"P0002": "account not found",
|
||||||
|
"23514": (
|
||||||
|
"authority change rejected; preserve an enabled administrator "
|
||||||
|
"and use explicit bootstrap"
|
||||||
|
),
|
||||||
|
"40001": "concurrent authority change rejected; retry the whole operation",
|
||||||
|
"55P03": "authority operation busy; retry later",
|
||||||
|
"57014": "authority operation timed out; verify current state before retrying",
|
||||||
|
"42501": "identity operator permissions unavailable",
|
||||||
|
}
|
||||||
|
# Never print a connection URL, SQL bind values or driver diagnostics.
|
||||||
|
raise ValueError(messages.get(code, "authority database operation unavailable")) from None
|
||||||
@@ -8,28 +8,58 @@ from sqlalchemy.exc import IntegrityError
|
|||||||
|
|
||||||
from kf_oj.core.settings import Settings
|
from kf_oj.core.settings import Settings
|
||||||
from kf_oj.infra.db.session import build_engine, build_session_factory
|
from kf_oj.infra.db.session import build_engine, build_session_factory
|
||||||
|
from kf_oj.modules.identity.authority import manage_authority
|
||||||
from kf_oj.modules.identity.service import IdentityService
|
from kf_oj.modules.identity.service import IdentityService
|
||||||
|
|
||||||
|
|
||||||
async def run(command: str, username: str, password: str | None = None) -> None:
|
async def run(command: str, username: str, password: str | None = None) -> None:
|
||||||
engine = build_engine(Settings())
|
settings = Settings()
|
||||||
|
authority_commands = {
|
||||||
|
"bootstrap-admin": "bootstrap",
|
||||||
|
"grant-admin": "grant",
|
||||||
|
"revoke-admin": "revoke",
|
||||||
|
"enable": "enable",
|
||||||
|
"disable": "disable",
|
||||||
|
}
|
||||||
|
if command != "create" and command not in authority_commands:
|
||||||
|
raise ValueError("unknown account command")
|
||||||
|
# Ordinary account creation uses the platform identity; authority management
|
||||||
|
# always requires a separately configured login and never falls back to it.
|
||||||
|
if command in authority_commands:
|
||||||
|
if settings.identity_operator_database_url is None:
|
||||||
|
raise ValueError("KF_OJ_IDENTITY_OPERATOR_DATABASE_URL is required")
|
||||||
|
if settings.identity_operator_database_url == settings.database_url:
|
||||||
|
raise ValueError("operator and platform database identities must be separate")
|
||||||
|
settings = settings.model_copy(
|
||||||
|
update={"database_url": settings.identity_operator_database_url}
|
||||||
|
)
|
||||||
|
engine = build_engine(settings)
|
||||||
try:
|
try:
|
||||||
|
if command in authority_commands:
|
||||||
|
result = await manage_authority(
|
||||||
|
build_session_factory(engine), username, authority_commands[command]
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
f"Account {result.username}: enabled={result.enabled}, "
|
||||||
|
f"administrator={result.is_administrator}, "
|
||||||
|
f"authorization_version={result.authorization_version}, changed={result.changed}."
|
||||||
|
)
|
||||||
|
return
|
||||||
service = IdentityService(build_session_factory(engine))
|
service = IdentityService(build_session_factory(engine))
|
||||||
if command == "create":
|
if command == "create":
|
||||||
assert password is not None
|
assert password is not None
|
||||||
await service.create_account(username, password)
|
await service.create_account(username, password)
|
||||||
print("Account created.")
|
print("Account created.")
|
||||||
else:
|
|
||||||
if not await service.disable_account(username):
|
|
||||||
raise ValueError("account not found")
|
|
||||||
print("Account disabled; sessions revoked.")
|
|
||||||
finally:
|
finally:
|
||||||
await engine.dispose()
|
await engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
parser = argparse.ArgumentParser(description="Manage local OJ accounts")
|
parser = argparse.ArgumentParser(description="Manage local OJ accounts")
|
||||||
parser.add_argument("command", choices=("create", "disable"))
|
parser.add_argument(
|
||||||
|
"command",
|
||||||
|
choices=("create", "disable", "enable", "bootstrap-admin", "grant-admin", "revoke-admin"),
|
||||||
|
)
|
||||||
parser.add_argument("username")
|
parser.add_argument("username")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
password = None
|
password = None
|
||||||
|
|||||||
@@ -3,24 +3,56 @@
|
|||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
from sqlalchemy import Boolean, CheckConstraint, DateTime, ForeignKey, Index, String, Text
|
from sqlalchemy import (
|
||||||
|
BigInteger,
|
||||||
|
Boolean,
|
||||||
|
CheckConstraint,
|
||||||
|
DateTime,
|
||||||
|
ForeignKey,
|
||||||
|
Index,
|
||||||
|
SmallInteger,
|
||||||
|
String,
|
||||||
|
Text,
|
||||||
|
text,
|
||||||
|
)
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
from kf_oj.infra.db.base import Base
|
from kf_oj.infra.db.base import Base
|
||||||
from kf_oj.infra.db.columns import CreatedAt
|
from kf_oj.infra.db.columns import CreatedAt
|
||||||
|
|
||||||
|
|
||||||
|
class AuthorityGuard(Base):
|
||||||
|
"""Migration-owned singleton; runtime roles have no access."""
|
||||||
|
|
||||||
|
__tablename__ = "identity_authority_guard"
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint("id = 1", name="singleton"),
|
||||||
|
CheckConstraint("generation >= 0", name="generation"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(SmallInteger, primary_key=True)
|
||||||
|
generation: Mapped[int] = mapped_column(BigInteger, server_default="0")
|
||||||
|
|
||||||
|
|
||||||
class Account(CreatedAt, Base):
|
class Account(CreatedAt, Base):
|
||||||
__tablename__ = "accounts"
|
__tablename__ = "accounts"
|
||||||
__table_args__ = (
|
__table_args__ = (
|
||||||
CheckConstraint("username ~ '^[a-z0-9_]{3,24}$'", name="canonical_username"),
|
CheckConstraint("username ~ '^[a-z0-9_]{3,24}$'", name="canonical_username"),
|
||||||
CheckConstraint("char_length(password_hash) BETWEEN 40 AND 256", name="password_hash_size"),
|
CheckConstraint("char_length(password_hash) BETWEEN 40 AND 256", name="password_hash_size"),
|
||||||
|
CheckConstraint("authorization_version >= 1", name="authorization_version"),
|
||||||
|
Index(
|
||||||
|
"ix_accounts_active_administrators",
|
||||||
|
"id",
|
||||||
|
postgresql_where=text("is_administrator AND enabled"),
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
id: Mapped[UUID] = mapped_column(primary_key=True)
|
id: Mapped[UUID] = mapped_column(primary_key=True)
|
||||||
username: Mapped[str] = mapped_column(String(24), unique=True)
|
username: Mapped[str] = mapped_column(String(24), unique=True)
|
||||||
password_hash: Mapped[str] = mapped_column(Text)
|
password_hash: Mapped[str] = mapped_column(Text)
|
||||||
enabled: Mapped[bool] = mapped_column(Boolean, server_default="true")
|
enabled: Mapped[bool] = mapped_column(Boolean, server_default="true")
|
||||||
|
is_administrator: Mapped[bool] = mapped_column(Boolean, server_default="false")
|
||||||
|
authorization_version: Mapped[int] = mapped_column(BigInteger, server_default="1")
|
||||||
|
|
||||||
|
|
||||||
class Session(CreatedAt, Base):
|
class Session(CreatedAt, Base):
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
"""Only verified principals cross business module boundaries."""
|
"""Only verified principals cross business module boundaries."""
|
||||||
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass, field
|
||||||
|
from typing import Protocol
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
|
|
||||||
@@ -8,11 +9,30 @@ from uuid import UUID
|
|||||||
class Principal:
|
class Principal:
|
||||||
account_id: UUID
|
account_id: UUID
|
||||||
username: str
|
username: str
|
||||||
|
can_maintain_problems: bool = False
|
||||||
|
authorization_version: int = 1
|
||||||
|
session_token_hash: str | None = field(default=None, repr=False, compare=False)
|
||||||
|
|
||||||
|
|
||||||
|
class MaintenanceDenied(ValueError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class ProblemMaintenancePort(Protocol):
|
||||||
|
async def require_problem_maintenance(self, session, principal: Principal) -> None:
|
||||||
|
"""Recheck and lock live authority/session inside the caller's write transaction."""
|
||||||
|
...
|
||||||
|
|
||||||
|
|
||||||
class AuthenticationFailed(ValueError):
|
class AuthenticationFailed(ValueError):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class ReservedMaintenancePort(Protocol):
|
||||||
|
async def require_reserved_maintenance(self, session, actor_id: UUID, epoch: int) -> None:
|
||||||
|
"""Recheck/lock an immutable accepted receipt, with no login/session secret access."""
|
||||||
|
...
|
||||||
|
|
||||||
|
|
||||||
class LoginThrottled(ValueError):
|
class LoginThrottled(ValueError):
|
||||||
pass
|
pass
|
||||||
|
|||||||
@@ -51,19 +51,31 @@ async def account_by_username(session: AsyncSession, username: str) -> Account |
|
|||||||
return await session.scalar(select(Account).where(Account.username == username))
|
return await session.scalar(select(Account).where(Account.username == username))
|
||||||
|
|
||||||
|
|
||||||
async def current_session(session: AsyncSession, token_hash: str):
|
async def lock_authority(session: AsyncSession, account_id) -> Account | None:
|
||||||
return (
|
return await session.scalar(
|
||||||
await session.execute(
|
select(Account)
|
||||||
select(Session, Account)
|
.where(Account.id == account_id)
|
||||||
.join(Account)
|
.with_for_update(read=True)
|
||||||
.where(
|
.execution_options(populate_existing=True)
|
||||||
Session.token_hash == token_hash,
|
)
|
||||||
Session.revoked_at.is_(None),
|
|
||||||
Session.expires_at > func.now(),
|
|
||||||
Account.enabled.is_(True),
|
async def current_session(session: AsyncSession, token_hash: str, *, lock_session: bool = False):
|
||||||
)
|
query = (
|
||||||
|
select(Session, Account)
|
||||||
|
.join(Account)
|
||||||
|
.where(
|
||||||
|
Session.token_hash == token_hash,
|
||||||
|
Session.revoked_at.is_(None),
|
||||||
|
Session.expires_at > func.clock_timestamp(),
|
||||||
|
Account.enabled.is_(True),
|
||||||
)
|
)
|
||||||
).one_or_none()
|
)
|
||||||
|
if lock_session:
|
||||||
|
query = query.with_for_update(read=True, of=Session).execution_options(
|
||||||
|
populate_existing=True
|
||||||
|
)
|
||||||
|
return (await session.execute(query)).one_or_none()
|
||||||
|
|
||||||
|
|
||||||
async def prune_sessions(session: AsyncSession, account_id) -> None:
|
async def prune_sessions(session: AsyncSession, account_id) -> None:
|
||||||
|
|||||||
@@ -7,7 +7,8 @@ from dataclasses import dataclass, field
|
|||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
from sqlalchemy import func, select, update
|
from sqlalchemy import func, select, text, update
|
||||||
|
from sqlalchemy.exc import DBAPIError
|
||||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||||
|
|
||||||
from kf_oj.modules.identity import repository
|
from kf_oj.modules.identity import repository
|
||||||
@@ -18,7 +19,12 @@ from kf_oj.modules.identity.passwords import (
|
|||||||
validate_account,
|
validate_account,
|
||||||
verify_password,
|
verify_password,
|
||||||
)
|
)
|
||||||
from kf_oj.modules.identity.public import AuthenticationFailed, LoginThrottled, Principal
|
from kf_oj.modules.identity.public import (
|
||||||
|
AuthenticationFailed,
|
||||||
|
LoginThrottled,
|
||||||
|
MaintenanceDenied,
|
||||||
|
Principal,
|
||||||
|
)
|
||||||
|
|
||||||
TOKEN = re.compile(r"[0-9a-f]{64}\Z")
|
TOKEN = re.compile(r"[0-9a-f]{64}\Z")
|
||||||
|
|
||||||
@@ -107,7 +113,7 @@ class IdentityService:
|
|||||||
expires_at=expires,
|
expires_at=expires,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
principal = Principal(account.id, account.username)
|
principal = self._principal(account, token_digest(token))
|
||||||
return IssuedSession(principal, expires, token, csrf)
|
return IssuedSession(principal, expires, token, csrf)
|
||||||
|
|
||||||
async def authenticate(self, token: str | None, *, csrf_token: str | None = None) -> Principal:
|
async def authenticate(self, token: str | None, *, csrf_token: str | None = None) -> Principal:
|
||||||
@@ -123,9 +129,62 @@ class IdentityService:
|
|||||||
record.csrf_hash, token_digest(csrf_token)
|
record.csrf_hash, token_digest(csrf_token)
|
||||||
):
|
):
|
||||||
raise AuthenticationFailed("invalid request token")
|
raise AuthenticationFailed("invalid request token")
|
||||||
return Principal(account.id, account.username)
|
return self._principal(account, record.token_hash)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _principal(account: Account, session_token_hash: str) -> Principal:
|
||||||
|
return Principal(
|
||||||
|
account.id,
|
||||||
|
account.username,
|
||||||
|
account.is_administrator,
|
||||||
|
account.authorization_version,
|
||||||
|
session_token_hash,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def require_problem_maintenance(self, session, principal: Principal) -> None:
|
||||||
|
if not session.in_transaction():
|
||||||
|
raise ValueError("maintenance authorization requires an active write transaction")
|
||||||
|
if (
|
||||||
|
not principal.can_maintain_problems
|
||||||
|
or principal.session_token_hash is None
|
||||||
|
or not TOKEN.fullmatch(principal.session_token_hash)
|
||||||
|
):
|
||||||
|
raise MaintenanceDenied("problem maintenance required")
|
||||||
|
# Account first, then session: authority changes use that same lock order.
|
||||||
|
# SHARE locks last until the caller commits/rolls back; a stale principal,
|
||||||
|
# explicit logout, revocation or expired session cannot authorize a write.
|
||||||
|
account = await repository.lock_authority(session, principal.account_id)
|
||||||
|
if (
|
||||||
|
account is None
|
||||||
|
or not account.enabled
|
||||||
|
or not account.is_administrator
|
||||||
|
or account.authorization_version != principal.authorization_version
|
||||||
|
):
|
||||||
|
raise MaintenanceDenied("problem maintenance required")
|
||||||
|
row = await repository.current_session(
|
||||||
|
session, principal.session_token_hash, lock_session=True
|
||||||
|
)
|
||||||
|
if row is None or row[1].id != account.id:
|
||||||
|
raise MaintenanceDenied("problem maintenance required")
|
||||||
|
|
||||||
async def logout(self, token: str, csrf_token: str) -> None:
|
async def logout(self, token: str, csrf_token: str) -> None:
|
||||||
await self.authenticate(token, csrf_token=csrf_token)
|
await self.authenticate(token, csrf_token=csrf_token)
|
||||||
async with self.sessions.begin() as session:
|
async with self.sessions.begin() as session:
|
||||||
await repository.revoke(session, token_digest(token))
|
await repository.revoke(session, token_digest(token))
|
||||||
|
|
||||||
|
|
||||||
|
class ReservedMaintenance:
|
||||||
|
"""Import worker uses only the fixed, narrow SQL authority port."""
|
||||||
|
|
||||||
|
async def require_reserved_maintenance(self, session, actor_id, epoch):
|
||||||
|
if not session.in_transaction() or type(epoch) is not int or epoch < 1:
|
||||||
|
raise MaintenanceDenied("invalid maintenance receipt")
|
||||||
|
try:
|
||||||
|
await session.execute(
|
||||||
|
text("SELECT public.kf_lock_reserved_maintenance(:actor,:epoch)"),
|
||||||
|
{"actor": actor_id, "epoch": epoch},
|
||||||
|
)
|
||||||
|
except DBAPIError as error:
|
||||||
|
if getattr(error.orig, "sqlstate", None) == "42501":
|
||||||
|
raise MaintenanceDenied("problem maintenance revoked") from None
|
||||||
|
raise
|
||||||
|
|||||||
@@ -1,5 +1,17 @@
|
|||||||
# operations
|
# operations
|
||||||
|
|
||||||
负责平台侧节点、任务积压、重试原因和审计查询的管理视图。
|
维护审计通过 `public.py` 的追加端口写入调用方事务;本模块不成为评测状态的写入入口。
|
||||||
对判题状态的操作必须调用协调器协议;观测模块不成为第二个状态写入入口。
|
上传、解析完成/失败、丢弃、发布、下架与恢复记录真实账号、授权版本、操作 UUID、对象与
|
||||||
当前只有目录边界,节点登记、指标聚合和管理员接口待实现。
|
数据库时间。发布回执和审计同时提交;审计失败使整个业务事务回滚,不确认半发布。
|
||||||
|
|
||||||
|
本机受控身份 CLI 使用已核验的窄权限数据库函数。授权变化由固定包装函数在同一事务
|
||||||
|
追加审计,主体类型为 operator,操作者为数据库 `session_user`,对象为实际变更账号;
|
||||||
|
不把授予/撤销冒充成该账号自己的操作。旧账号事件保留 account 主体类型。
|
||||||
|
|
||||||
|
管理员 `/api/v1/admin/maintenance-audit` 按时间和 UUID 的复合游标递减读取,每页至多
|
||||||
|
100项,可选择有限动作类型;只展示已提交事件。导入失败动作仍是失败,committed 表示
|
||||||
|
审计事务提交。响应不含口令、会话令牌、工件引用、内部路径或任意诊断正文。
|
||||||
|
|
||||||
|
应用数据库角色无审计 UPDATE/DELETE 权限,数据库触发器还阻止历史改写。
|
||||||
|
导入 Worker 只有追加权限,不能读取整个维护审计;Coordinator 无审计写入权限。
|
||||||
|
节点管理、部署控制及跨账号业务管理仍未开放。
|
||||||
|
|||||||
@@ -0,0 +1,39 @@
|
|||||||
|
"""Append-only bounded maintenance events, independent of evaluation state."""
|
||||||
|
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from sqlalchemy import BigInteger, CheckConstraint, ForeignKey, Index, String, text
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
|
from kf_oj.infra.db.base import Base
|
||||||
|
from kf_oj.infra.db.columns import CreatedAt
|
||||||
|
|
||||||
|
|
||||||
|
class MaintenanceEvent(CreatedAt, Base):
|
||||||
|
__tablename__ = "problem_maintenance_events"
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint(
|
||||||
|
"action IN ('upload_created','upload_received','import_ready',"
|
||||||
|
"'import_failed','import_discarded','import_expired','published',"
|
||||||
|
"'disabled','restored','authority_bootstrap','authority_grant',"
|
||||||
|
"'authority_revoke','authority_enable','authority_disable')",
|
||||||
|
name="action",
|
||||||
|
),
|
||||||
|
CheckConstraint("authorization_version > 0", name="authority"),
|
||||||
|
CheckConstraint(
|
||||||
|
"(actor_kind='account' AND actor_id IS NOT NULL AND operator_name IS NULL) "
|
||||||
|
"OR (actor_kind='operator' AND actor_id IS NULL AND operator_name IS NOT NULL)",
|
||||||
|
name="actor",
|
||||||
|
),
|
||||||
|
Index("ix_problem_maintenance_events_page", "created_at", "id"),
|
||||||
|
)
|
||||||
|
id: Mapped[UUID] = mapped_column(primary_key=True)
|
||||||
|
actor_id: Mapped[UUID | None] = mapped_column(ForeignKey("accounts.id"))
|
||||||
|
actor_kind: Mapped[str] = mapped_column(
|
||||||
|
String(8), default="account", server_default=text("'account'")
|
||||||
|
)
|
||||||
|
operator_name: Mapped[str | None] = mapped_column(String(63))
|
||||||
|
authorization_version: Mapped[int] = mapped_column(BigInteger)
|
||||||
|
operation_id: Mapped[UUID] = mapped_column()
|
||||||
|
action: Mapped[str] = mapped_column(String(24))
|
||||||
|
target_id: Mapped[UUID | None] = mapped_column()
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
"""Accepted actor receipts and finite maintenance events; no private data/log text."""
|
||||||
|
|
||||||
|
from typing import Literal, Protocol
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
Action = Literal[
|
||||||
|
"upload_created",
|
||||||
|
"upload_received",
|
||||||
|
"import_ready",
|
||||||
|
"import_failed",
|
||||||
|
"import_discarded",
|
||||||
|
"import_expired",
|
||||||
|
"published",
|
||||||
|
"disabled",
|
||||||
|
"restored",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class MaintenanceAuditPort(Protocol):
|
||||||
|
async def record(
|
||||||
|
self,
|
||||||
|
session,
|
||||||
|
actor_id: UUID,
|
||||||
|
authorization_version: int,
|
||||||
|
operation_id: UUID,
|
||||||
|
action: Action,
|
||||||
|
target_id: UUID | None = None,
|
||||||
|
) -> None: ...
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
"""Only authorized administrators read bounded append-only maintenance projections."""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from sqlalchemy import select, text, tuple_
|
||||||
|
|
||||||
|
from kf_oj.contracts.problem_maintenance import MaintenanceAuditEvent, MaintenanceAuditPage
|
||||||
|
from kf_oj.modules.identity.public import ProblemMaintenancePort
|
||||||
|
from kf_oj.modules.operations.models import MaintenanceEvent
|
||||||
|
|
||||||
|
|
||||||
|
class MaintenanceAuditQueries:
|
||||||
|
def __init__(self, sessions, identity: ProblemMaintenancePort):
|
||||||
|
self.sessions, self.identity = sessions, identity
|
||||||
|
|
||||||
|
async def page(self, principal, *, before=None, limit=20, action=None):
|
||||||
|
if not 1 <= limit <= 100:
|
||||||
|
raise ValueError("invalid page limit")
|
||||||
|
cursor = None
|
||||||
|
if before is not None:
|
||||||
|
stamp, identifier = before.split("|")
|
||||||
|
stamp = datetime.fromisoformat(stamp)
|
||||||
|
if stamp.tzinfo is None:
|
||||||
|
raise ValueError("invalid audit cursor")
|
||||||
|
cursor = stamp, UUID(identifier)
|
||||||
|
async with self.sessions.begin() as session:
|
||||||
|
await session.execute(text("SET LOCAL statement_timeout='5s'"))
|
||||||
|
await session.execute(text("SET LOCAL lock_timeout='2s'"))
|
||||||
|
await self.identity.require_problem_maintenance(session, principal)
|
||||||
|
q = select(MaintenanceEvent)
|
||||||
|
if cursor:
|
||||||
|
q = q.where(tuple_(MaintenanceEvent.created_at, MaintenanceEvent.id) < cursor)
|
||||||
|
if action:
|
||||||
|
q = q.where(MaintenanceEvent.action == action)
|
||||||
|
rows = list(
|
||||||
|
await session.scalars(
|
||||||
|
q.order_by(
|
||||||
|
MaintenanceEvent.created_at.desc(), MaintenanceEvent.id.desc()
|
||||||
|
).limit(limit + 1)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
visible = rows[:limit]
|
||||||
|
return MaintenanceAuditPage(
|
||||||
|
items=tuple(
|
||||||
|
MaintenanceAuditEvent(
|
||||||
|
id=r.id,
|
||||||
|
created_at=r.created_at,
|
||||||
|
actor_kind=r.actor_kind,
|
||||||
|
actor_id=r.actor_id,
|
||||||
|
operator_name=r.operator_name,
|
||||||
|
authorization_version=str(r.authorization_version),
|
||||||
|
operation_id=r.operation_id,
|
||||||
|
action=r.action,
|
||||||
|
target_id=r.target_id,
|
||||||
|
)
|
||||||
|
for r in visible
|
||||||
|
),
|
||||||
|
next_before=f"{visible[-1].created_at.isoformat()}|{visible[-1].id}"
|
||||||
|
if len(rows) > limit
|
||||||
|
else None,
|
||||||
|
)
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
|
||||||
|
from kf_oj.modules.operations.models import MaintenanceEvent
|
||||||
|
from kf_oj.modules.operations.public import Action
|
||||||
|
|
||||||
|
|
||||||
|
class MaintenanceAudit:
|
||||||
|
async def record(
|
||||||
|
self, session, actor_id, authorization_version, operation_id, action: Action, target_id=None
|
||||||
|
):
|
||||||
|
if not session.in_transaction():
|
||||||
|
raise ValueError("audit requires a write transaction")
|
||||||
|
# Supplying the trusted DB timestamp avoids ORM INSERT ... RETURNING:
|
||||||
|
# the ordinary importer has append-only authority and cannot SELECT this table.
|
||||||
|
session.add(
|
||||||
|
MaintenanceEvent(
|
||||||
|
id=uuid4(),
|
||||||
|
created_at=await session.scalar(select(func.clock_timestamp())),
|
||||||
|
actor_id=actor_id,
|
||||||
|
authorization_version=authorization_version,
|
||||||
|
operation_id=operation_id,
|
||||||
|
action=action,
|
||||||
|
target_id=target_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
@@ -31,3 +31,50 @@ v2 使用 `schema_version: 2` 与 `language_configurations`,每个配置包含
|
|||||||
导入同一 slug 的 v2 版本保持原 v1 版本与提交不变。已有提交按其保存的协议版本解析,
|
导入同一 slug 的 v2 版本保持原 v1 版本与提交不变。已有提交按其保存的协议版本解析,
|
||||||
不能重新序列化旧 canonical 数据来补入新字段。Java 使用 Main.java 与默认包 Main;
|
不能重新序列化旧 canonical 数据来补入新字段。Java 使用 Main.java 与默认包 Main;
|
||||||
Python 使用 main.py。每个新版本的语言集合独立,浏览器不得为旧版本补开新语言。
|
Python 使用 main.py。每个新版本的语言集合独立,浏览器不得为旧版本补开新语言。
|
||||||
|
|
||||||
|
|
||||||
|
Web题包导入是独立的待发布流程:`/api/v1/admin/problem-imports` 使用真实维护会话,
|
||||||
|
写入还要求明确Origin与绑定CSRF。先以稳定操作UUID、ZIP摘要和精确字节数建立容量预约,
|
||||||
|
再向该操作的 `/archive` 流式PUT;同键重试仍验证实际内容,不会确认异内容。
|
||||||
|
列表、详情和丢弃只开放给该操作的管理员本人,响应仅含公开题面、样例、语言预算与有限错误。
|
||||||
|
`ready` 尚未发布,不会加入学生题库,也不改变既有题目版本。
|
||||||
|
|
||||||
|
Web格式固定单层ZIP的stored/deflate,清单名为`problem.json`;公开文件与隐藏用例文件
|
||||||
|
必须显式分别声明且不得混用。文件名只接受有界ASCII单层普通名;未知成员、链接/设备、
|
||||||
|
加密、嵌套归档、额外前缀/尾随内容、损坏CRC与虚假大小被拒。旧受控目录CLI格式保持原义。
|
||||||
|
压缩与实际解压各64 MiB、最多534文件,上传最多4096帧/每帧1 MiB/30秒;
|
||||||
|
解析期限60秒。staging总预约256 MiB,每次128 MiB,单全局上传/解析槽位,
|
||||||
|
本人最多10个/全局32个活跃候选,操作24小时到期,解析死亡最多领取两次。
|
||||||
|
真实限额由管理capabilities返回;缺少配置或新鲜Worker资源证明时明确不可用。
|
||||||
|
|
||||||
|
`uv run --locked kf-problem-import-worker` 是普通用户、独立数据库登录的组合入口。
|
||||||
|
配置`KF_OJ_IMPORT_STAGING_ROOT`、`KF_OJ_ARTIFACT_ROOT`、可信准入和
|
||||||
|
`KF_OJ_IMPORT_WORKER_CGROUP`;staging与工件目录必须归服务用户所有且权限0700。
|
||||||
|
服务不会sudo、执行上传内容/学生源码、联网安装依赖或读写评测状态。启动及每次解析
|
||||||
|
读回128 MiB/swap0/PIDs16/CPU不超过一核及文件/FD/core上限,错误身份/无界环境拒绝服务。
|
||||||
|
解析子进程继承同一个内核文件锁,父进程死亡不能让GC在存活子进程仍持有文件时释放容量。
|
||||||
|
租约与fence防止旧结果写入;完整工件持久化、候选和引用原子落库后才返回ready。
|
||||||
|
丢弃/失败/到期先清理私有暂存字节再释放预约,保留不可变操作记录与追加式有限审计。
|
||||||
|
|
||||||
|
资源约束服务模板见`deploy/problem-import-worker.service.example`。API和Worker需可访问
|
||||||
|
同一普通用户的私有目录;数据库采用各自最小角色,导入Worker只通过固定身份公开端口
|
||||||
|
验证已受理的维护授权。PrivateNetwork模板要求Unix PostgreSQL socket;实际部署前由运维
|
||||||
|
设置外部私有配置、专用登录与目录,不包含默认口令,也不代表本轮已经生产部署。
|
||||||
|
|
||||||
|
管理员发布使用 `/api/v1/admin/problem-publications`,绑定独立操作 UUID、本人导入操作、
|
||||||
|
候选 UUID 与预期活动版本(首次发布为 null)。`publication.py` 为 CLI 与 Web 共用的
|
||||||
|
准入与不可变版本写入路径;Web 在事务内再次校验实时维护权限、候选摘要、工件实际字节
|
||||||
|
和可信节点配置。CLI 保留本机受控入口与原有导入格式,不借用网页会话。
|
||||||
|
|
||||||
|
成功维护回执只追加保存,UUID 同内容重试返回原快照,不重复创建 revision 或审计。
|
||||||
|
UUID 绑定不同内容或活动版本变化返回409;前端应核对最新状态后建立新的操作凭据。
|
||||||
|
题目状态写入同样要求预期活动版本及 enabled;下架不修改历史版本或删除任何提交。
|
||||||
|
新提交持有逻辑题目的共享锁直到提交事务结束,下架/恢复持有排他锁,二者明确排序。
|
||||||
|
|
||||||
|
管理列表按 slug、版本历史按递减 revision 分页,每页至多100条,仅返回公开元数据。
|
||||||
|
候选仍通过本人导入操作读取;维护回执仅本人可读。隐藏测试字节、摘要和宿主路径不进入
|
||||||
|
这些投影。所有管理读取也经实时维护权限检查,缺失发布配置明确503。
|
||||||
|
|
||||||
|
`for_accepted_execution` 仅在 submissions 已核对有效租约、源码归属及不可变尝试快照后
|
||||||
|
调用,读取已发布原版本时允许逻辑题目已下架。它没有外部 UUID 直读端点,不放宽未发布
|
||||||
|
版本或无有效租约的访问。公开读取与新提交仍检查 enabled。
|
||||||
|
|||||||
@@ -0,0 +1,74 @@
|
|||||||
|
"""Pure data validation shared by the bounded child and fenced candidate transaction."""
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from uuid import uuid5
|
||||||
|
|
||||||
|
from kf_oj.contracts.admin_problems import CandidatePreview, LegacyImportBudget
|
||||||
|
from kf_oj.contracts.judging import ArtifactReference, TestCase
|
||||||
|
from kf_oj.contracts.problems import PublicLanguageConfiguration
|
||||||
|
from kf_oj.modules.problems.import_format import ImportEnvelope, ProblemImport
|
||||||
|
from kf_oj.modules.problems.publication import admit_version
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ValidatedCandidate:
|
||||||
|
manifest: dict
|
||||||
|
preview: CandidatePreview
|
||||||
|
references: dict[str, ArtifactReference]
|
||||||
|
|
||||||
|
@property
|
||||||
|
def digest(self):
|
||||||
|
return hashlib.sha256(
|
||||||
|
json.dumps(
|
||||||
|
self.manifest, sort_keys=True, separators=(",", ":"), ensure_ascii=True
|
||||||
|
).encode()
|
||||||
|
).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def validate_candidate(manifest_data, preview_data, references, policy) -> ValidatedCandidate:
|
||||||
|
manifest = ImportEnvelope.model_validate(manifest_data).root
|
||||||
|
preview = CandidatePreview.model_validate(preview_data)
|
||||||
|
refs = {k: ArtifactReference.model_validate(v) for k, v in references.items()}
|
||||||
|
legacy = isinstance(manifest, ProblemImport)
|
||||||
|
if (
|
||||||
|
preview.slug != manifest.slug
|
||||||
|
or preview.title != manifest.title
|
||||||
|
or preview.case_count != len(manifest.cases)
|
||||||
|
or preview.judging_schema_version != manifest.schema_version
|
||||||
|
or tuple(v.value for v in preview.languages) != tuple(v.value for v in manifest.languages)
|
||||||
|
):
|
||||||
|
raise ValueError("invalid candidate projection")
|
||||||
|
expected_budget = (
|
||||||
|
LegacyImportBudget(
|
||||||
|
compile_limits=manifest.compile_limits,
|
||||||
|
run_limits=manifest.run_limits,
|
||||||
|
job_wall_time_us=manifest.job_wall_time_us,
|
||||||
|
)
|
||||||
|
if legacy
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
configs = (
|
||||||
|
()
|
||||||
|
if legacy
|
||||||
|
else tuple(
|
||||||
|
PublicLanguageConfiguration.from_configuration(c)
|
||||||
|
for c in manifest.language_configurations
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if preview.legacy_budget != expected_budget or preview.language_configurations != configs:
|
||||||
|
raise ValueError("invalid candidate budget")
|
||||||
|
names = {name for c in manifest.cases for name in (c.input_file, c.expected_file)}
|
||||||
|
if set(refs) != names:
|
||||||
|
raise ValueError("invalid hidden references")
|
||||||
|
cases = tuple(
|
||||||
|
TestCase(
|
||||||
|
case_id=uuid5(preview.candidate_id, c.key),
|
||||||
|
input=refs[c.input_file],
|
||||||
|
expected=refs[c.expected_file],
|
||||||
|
)
|
||||||
|
for c in manifest.cases
|
||||||
|
)
|
||||||
|
admit_version(policy, manifest, preview.candidate_id, preview.candidate_id, cases)
|
||||||
|
return ValidatedCandidate(manifest.model_dump(mode="json"), preview, refs)
|
||||||
@@ -0,0 +1,121 @@
|
|||||||
|
"""Private durable imports and immutable candidates; no grading-state write authority."""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from sqlalchemy import (
|
||||||
|
BigInteger,
|
||||||
|
CheckConstraint,
|
||||||
|
Computed,
|
||||||
|
DateTime,
|
||||||
|
ForeignKey,
|
||||||
|
ForeignKeyConstraint,
|
||||||
|
Index,
|
||||||
|
Integer,
|
||||||
|
SmallInteger,
|
||||||
|
String,
|
||||||
|
UniqueConstraint,
|
||||||
|
text,
|
||||||
|
)
|
||||||
|
from sqlalchemy.dialects.postgresql import JSONB
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
|
from kf_oj.infra.db.base import Base
|
||||||
|
from kf_oj.infra.db.columns import CreatedAt
|
||||||
|
from kf_oj.modules.problems.models import ProblemVersion
|
||||||
|
|
||||||
|
|
||||||
|
class ImportPool(Base):
|
||||||
|
__tablename__ = "problem_import_pool"
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint("id = 1", name="singleton"),
|
||||||
|
CheckConstraint("reserved_bytes BETWEEN 0 AND 268435456", name="capacity"),
|
||||||
|
)
|
||||||
|
id: Mapped[int] = mapped_column(SmallInteger, primary_key=True)
|
||||||
|
reserved_bytes: Mapped[int] = mapped_column(BigInteger, server_default="0")
|
||||||
|
worker_ready_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
worker_policy_digest: Mapped[str | None] = mapped_column(String(64))
|
||||||
|
|
||||||
|
|
||||||
|
class ProblemImportOperation(CreatedAt, Base):
|
||||||
|
__tablename__ = "problem_import_operations"
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint(
|
||||||
|
"state IN ('uploading','uploaded','validating','ready','failed',"
|
||||||
|
"'published','discarded','expired')",
|
||||||
|
name="state",
|
||||||
|
),
|
||||||
|
CheckConstraint("archive_digest ~ '^[0-9a-f]{64}$'", name="digest"),
|
||||||
|
CheckConstraint("archive_bytes BETWEEN 1 AND 67108864", name="bytes"),
|
||||||
|
CheckConstraint("authorization_version > 0", name="authority"),
|
||||||
|
CheckConstraint("attempts BETWEEN 0 AND 2 AND fence >= 0", name="attempts"),
|
||||||
|
CheckConstraint("reserved_bytes IN (0,134217728)", name="reservation"),
|
||||||
|
CheckConstraint("expires_at > created_at AND updated_at >= created_at", name="times"),
|
||||||
|
Index("ix_problem_import_owner", "owner_id", "id"),
|
||||||
|
Index("ix_problem_import_queue", "state", "created_at"),
|
||||||
|
Index("ix_problem_import_expiry", "expires_at"),
|
||||||
|
Index("ix_problem_import_reservations", "id", postgresql_where=text("reserved_bytes>0")),
|
||||||
|
)
|
||||||
|
id: Mapped[UUID] = mapped_column(primary_key=True)
|
||||||
|
owner_id: Mapped[UUID] = mapped_column(ForeignKey("accounts.id"))
|
||||||
|
authorization_version: Mapped[int] = mapped_column(BigInteger)
|
||||||
|
archive_digest: Mapped[str] = mapped_column(String(64))
|
||||||
|
archive_bytes: Mapped[int] = mapped_column(BigInteger)
|
||||||
|
state: Mapped[str] = mapped_column(String(16), server_default="uploading")
|
||||||
|
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||||
|
lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
attempts: Mapped[int] = mapped_column(Integer, server_default="0")
|
||||||
|
fence: Mapped[int] = mapped_column(BigInteger, server_default="0")
|
||||||
|
reserved_bytes: Mapped[int] = mapped_column(BigInteger, server_default="134217728")
|
||||||
|
failure_code: Mapped[str | None] = mapped_column(String(32))
|
||||||
|
candidate_id: Mapped[UUID | None] = mapped_column(
|
||||||
|
ForeignKey("problem_candidates.id", use_alter=True, name="fk_problem_import_candidate")
|
||||||
|
)
|
||||||
|
published_version_id: Mapped[UUID | None] = mapped_column(ForeignKey(ProblemVersion.id))
|
||||||
|
|
||||||
|
|
||||||
|
class ProblemCandidate(CreatedAt, Base):
|
||||||
|
__tablename__ = "problem_candidates"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("operation_id"),
|
||||||
|
CheckConstraint("manifest_digest ~ '^[0-9a-f]{64}$'", name="digest"),
|
||||||
|
CheckConstraint("case_count BETWEEN 1 AND 256", name="cases"),
|
||||||
|
CheckConstraint("octet_length(preview::text) <= 4194304", name="preview"),
|
||||||
|
CheckConstraint("octet_length(manifest::text) <= 262144", name="manifest"),
|
||||||
|
)
|
||||||
|
id: Mapped[UUID] = mapped_column(primary_key=True)
|
||||||
|
operation_id: Mapped[UUID] = mapped_column(ForeignKey("problem_import_operations.id"))
|
||||||
|
manifest_digest: Mapped[str] = mapped_column(String(64))
|
||||||
|
manifest: Mapped[dict] = mapped_column(JSONB)
|
||||||
|
preview: Mapped[dict] = mapped_column(JSONB)
|
||||||
|
case_count: Mapped[int] = mapped_column(Integer)
|
||||||
|
|
||||||
|
|
||||||
|
class CandidateCase(Base):
|
||||||
|
__tablename__ = "problem_candidate_cases"
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint("position BETWEEN 0 AND 255", name="position"),
|
||||||
|
CheckConstraint(
|
||||||
|
"input_bytes BETWEEN 0 AND 67108864 AND expected_bytes BETWEEN 0 AND 67108864",
|
||||||
|
name="bytes",
|
||||||
|
),
|
||||||
|
ForeignKeyConstraint(
|
||||||
|
["input_digest", "artifact_state"], ["artifacts.digest", "artifacts.state"]
|
||||||
|
),
|
||||||
|
ForeignKeyConstraint(
|
||||||
|
["expected_digest", "artifact_state"],
|
||||||
|
["artifacts.digest", "artifacts.state"],
|
||||||
|
name="fk_candidate_cases_expected_artifacts",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
candidate_id: Mapped[UUID] = mapped_column(
|
||||||
|
ForeignKey("problem_candidates.id"), primary_key=True
|
||||||
|
)
|
||||||
|
position: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
|
key: Mapped[str] = mapped_column(String(64))
|
||||||
|
input_digest: Mapped[str] = mapped_column(String(64))
|
||||||
|
input_bytes: Mapped[int] = mapped_column(BigInteger)
|
||||||
|
expected_digest: Mapped[str] = mapped_column(String(64))
|
||||||
|
expected_bytes: Mapped[int] = mapped_column(BigInteger)
|
||||||
|
artifact_state: Mapped[str] = mapped_column(String(8), Computed("'ready'", persisted=True))
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
"""Shared import availability error and exact policy identity; no lifecycle imports."""
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
|
||||||
|
|
||||||
|
class ImportUnavailable(ValueError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def policy_digest(policy):
|
||||||
|
return hashlib.sha256(policy.model_dump_json().encode()).hexdigest()
|
||||||
@@ -0,0 +1,430 @@
|
|||||||
|
"""Fenced ordinary-user import consumption. No student execution or evaluation DB access."""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import resource
|
||||||
|
import sys
|
||||||
|
from datetime import timedelta
|
||||||
|
from pathlib import Path
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
from sqlalchemy import delete, func, or_, select, text
|
||||||
|
from sqlalchemy.exc import DBAPIError
|
||||||
|
|
||||||
|
from kf_oj.contracts.admin_problems import MAX_IMPORT_ATTEMPTS, VALIDATION_DEADLINE_SECONDS
|
||||||
|
from kf_oj.infra.artifacts.store import ArtifactError
|
||||||
|
from kf_oj.modules.identity.public import MaintenanceDenied
|
||||||
|
from kf_oj.modules.problems.candidate_validation import validate_candidate
|
||||||
|
from kf_oj.modules.problems.import_models import (
|
||||||
|
CandidateCase,
|
||||||
|
ImportPool,
|
||||||
|
ProblemCandidate,
|
||||||
|
)
|
||||||
|
from kf_oj.modules.problems.import_models import (
|
||||||
|
ProblemImportOperation as Operation,
|
||||||
|
)
|
||||||
|
from kf_oj.modules.problems.imports import ACTIVE, ImportUnavailable, ProblemImports, policy_digest
|
||||||
|
|
||||||
|
|
||||||
|
def verify_worker_limits(configured_group: Path):
|
||||||
|
if os.getuid() == 0:
|
||||||
|
raise ImportUnavailable("import worker must be ordinary")
|
||||||
|
actual = next(
|
||||||
|
line.split(":", 2)[2]
|
||||||
|
for line in Path("/proc/self/cgroup").read_text().splitlines()
|
||||||
|
if line.startswith("0::")
|
||||||
|
)
|
||||||
|
group = Path("/sys/fs/cgroup") / actual.lstrip("/")
|
||||||
|
if group.resolve() != configured_group.absolute().resolve():
|
||||||
|
raise ImportUnavailable("import worker cgroup mismatch")
|
||||||
|
caps = {
|
||||||
|
n: (group / n).read_text().strip()
|
||||||
|
for n in ("memory.max", "memory.swap.max", "pids.max", "cpu.max")
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
caps["memory.max"] == "max"
|
||||||
|
or not 0 < int(caps["memory.max"]) <= 134217728
|
||||||
|
or caps["memory.swap.max"] != "0"
|
||||||
|
or caps["pids.max"] == "max"
|
||||||
|
or not 0 < int(caps["pids.max"]) <= 16
|
||||||
|
):
|
||||||
|
raise ImportUnavailable("import worker resource limits unavailable")
|
||||||
|
quota, period = caps["cpu.max"].split()
|
||||||
|
if quota == "max" or int(quota) > int(period):
|
||||||
|
raise ImportUnavailable("import worker CPU limit unavailable")
|
||||||
|
for name, maximum in (
|
||||||
|
(resource.RLIMIT_FSIZE, 67108864),
|
||||||
|
(resource.RLIMIT_NOFILE, 64),
|
||||||
|
(resource.RLIMIT_CORE, 0),
|
||||||
|
):
|
||||||
|
limits = resource.getrlimit(name)
|
||||||
|
if any(v == resource.RLIM_INFINITY or v > maximum for v in limits):
|
||||||
|
raise ImportUnavailable("import worker process limits unavailable")
|
||||||
|
return caps
|
||||||
|
|
||||||
|
|
||||||
|
class ImportWorker:
|
||||||
|
def __init__(self, sessions, staging, artifacts, authority, audit, policy, cgroup):
|
||||||
|
(
|
||||||
|
self.sessions,
|
||||||
|
self.staging,
|
||||||
|
self.artifacts,
|
||||||
|
self.authority,
|
||||||
|
self.audit,
|
||||||
|
self.policy,
|
||||||
|
self.cgroup,
|
||||||
|
) = sessions, staging, artifacts, authority, audit, policy, cgroup
|
||||||
|
|
||||||
|
async def heartbeat(self):
|
||||||
|
verify_worker_limits(self.cgroup)
|
||||||
|
async with self.sessions.begin() as session:
|
||||||
|
safe = await session.scalar(
|
||||||
|
text("""
|
||||||
|
SELECT rolcanlogin AND NOT(rolsuper OR rolcreatedb OR rolcreaterole
|
||||||
|
OR rolreplication OR rolbypassrls)
|
||||||
|
AND pg_has_role(current_user,'kf_oj_problem_import','USAGE')
|
||||||
|
AND NOT pg_has_role(current_user,'kf_oj_platform','MEMBER')
|
||||||
|
AND NOT pg_has_role(current_user,'kf_oj_coordinator','MEMBER')
|
||||||
|
AND NOT pg_has_role(current_user,'kf_oj_identity_operator','MEMBER')
|
||||||
|
AND NOT pg_has_role(current_user,'kf_oj_identity_authority_owner','MEMBER')
|
||||||
|
FROM pg_roles WHERE rolname=current_user
|
||||||
|
""")
|
||||||
|
)
|
||||||
|
if not safe:
|
||||||
|
raise ImportUnavailable("dedicated unprivileged import login required")
|
||||||
|
pool = await session.get(ImportPool, 1, with_for_update=True)
|
||||||
|
if pool is None:
|
||||||
|
raise ImportUnavailable("import pool unavailable")
|
||||||
|
pool.worker_ready_until = await session.scalar(
|
||||||
|
select(func.clock_timestamp())
|
||||||
|
) + timedelta(seconds=90)
|
||||||
|
pool.worker_policy_digest = policy_digest(self.policy)
|
||||||
|
|
||||||
|
async def _release(self, operation_id):
|
||||||
|
await asyncio.to_thread(self.staging.remove, operation_id)
|
||||||
|
async with self.sessions.begin() as session:
|
||||||
|
pool = await ProblemImports.pool(session)
|
||||||
|
op = await session.get(Operation, operation_id, with_for_update=True)
|
||||||
|
pool.reserved_bytes -= op.reserved_bytes
|
||||||
|
op.reserved_bytes = 0
|
||||||
|
op.lease_until = None
|
||||||
|
|
||||||
|
async def collect(self):
|
||||||
|
# Caller owns the root flock, so a live upload/parser cannot be unlinked or
|
||||||
|
# have its reservation reused, even if its DB lease expired while suspended.
|
||||||
|
async with self.sessions() as session:
|
||||||
|
now = await session.scalar(select(func.clock_timestamp()))
|
||||||
|
names = list(
|
||||||
|
await session.scalars(
|
||||||
|
select(Operation.id)
|
||||||
|
.where(
|
||||||
|
or_(
|
||||||
|
Operation.reserved_bytes > 0,
|
||||||
|
Operation.state.in_(ACTIVE) & (Operation.expires_at <= now),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.order_by(Operation.updated_at, Operation.id)
|
||||||
|
.limit(32)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
garbage = list(
|
||||||
|
await session.scalars(
|
||||||
|
select(ProblemCandidate.operation_id)
|
||||||
|
.join(Operation, Operation.id == ProblemCandidate.operation_id)
|
||||||
|
.where(Operation.state.in_(("discarded", "expired", "failed")))
|
||||||
|
.limit(32)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
names = list(dict.fromkeys(names + garbage))[:32]
|
||||||
|
for identifier in names:
|
||||||
|
async with self.sessions.begin() as session:
|
||||||
|
await ProblemImports.pool(session)
|
||||||
|
op = await session.get(Operation, identifier, with_for_update=True)
|
||||||
|
now = await session.scalar(select(func.clock_timestamp()))
|
||||||
|
if op.state in ACTIVE and op.expires_at <= now:
|
||||||
|
op.state = "expired"
|
||||||
|
op.failure_code = "expired"
|
||||||
|
op.updated_at = now
|
||||||
|
op.candidate_id = None
|
||||||
|
op.fence += 1
|
||||||
|
await self.audit.record(
|
||||||
|
session, op.owner_id, op.authorization_version, op.id, "import_expired"
|
||||||
|
)
|
||||||
|
elif op.state == "uploading" and op.lease_until and op.lease_until <= now:
|
||||||
|
op.state = "failed"
|
||||||
|
op.failure_code = "upload_interrupted"
|
||||||
|
op.updated_at = now
|
||||||
|
op.fence += 1
|
||||||
|
await self.audit.record(
|
||||||
|
session, op.owner_id, op.authorization_version, op.id, "import_failed"
|
||||||
|
)
|
||||||
|
elif op.state == "validating" and op.lease_until and op.lease_until <= now:
|
||||||
|
if op.attempts < MAX_IMPORT_ATTEMPTS:
|
||||||
|
op.state = "uploaded"
|
||||||
|
op.failure_code = None
|
||||||
|
op.lease_until = None
|
||||||
|
op.fence += 1
|
||||||
|
op.updated_at = now
|
||||||
|
await asyncio.to_thread(self.staging.reset_validation, op.id)
|
||||||
|
else:
|
||||||
|
op.state = "failed"
|
||||||
|
op.failure_code = "validation_unavailable"
|
||||||
|
op.updated_at = now
|
||||||
|
op.fence += 1
|
||||||
|
await self.audit.record(
|
||||||
|
session, op.owner_id, op.authorization_version, op.id, "import_failed"
|
||||||
|
)
|
||||||
|
if op.state in ("discarded", "expired", "failed"):
|
||||||
|
op.candidate_id = None
|
||||||
|
await session.flush()
|
||||||
|
candidate = await session.scalar(
|
||||||
|
select(ProblemCandidate).where(ProblemCandidate.operation_id == op.id)
|
||||||
|
)
|
||||||
|
if candidate:
|
||||||
|
await session.execute(
|
||||||
|
delete(CandidateCase).where(CandidateCase.candidate_id == candidate.id)
|
||||||
|
)
|
||||||
|
await session.delete(candidate)
|
||||||
|
release = (
|
||||||
|
op.state in ("ready", "published", "discarded", "expired", "failed")
|
||||||
|
and op.reserved_bytes > 0
|
||||||
|
)
|
||||||
|
if release:
|
||||||
|
await self._release(identifier)
|
||||||
|
# Uncommitted staging objects after a failed transaction are removed only under
|
||||||
|
# the same kernel lock and only from this service's private, bounded namespace.
|
||||||
|
for identifier in self.staging.operation_ids():
|
||||||
|
async with self.sessions() as session:
|
||||||
|
exists = await session.get(Operation, identifier)
|
||||||
|
if exists is None:
|
||||||
|
await asyncio.to_thread(self.staging.remove, identifier)
|
||||||
|
|
||||||
|
async def once(self):
|
||||||
|
await self.heartbeat()
|
||||||
|
try:
|
||||||
|
with self.staging.exclusive() as lock:
|
||||||
|
await self.collect()
|
||||||
|
async with self.sessions.begin() as session:
|
||||||
|
await ProblemImports.pool(session)
|
||||||
|
now = await session.scalar(select(func.clock_timestamp()))
|
||||||
|
busy = await session.scalar(
|
||||||
|
select(func.count())
|
||||||
|
.select_from(Operation)
|
||||||
|
.where(
|
||||||
|
Operation.state.in_(("uploading", "validating")),
|
||||||
|
Operation.lease_until > now,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if busy:
|
||||||
|
return False
|
||||||
|
op = await session.scalar(
|
||||||
|
select(Operation)
|
||||||
|
.where(
|
||||||
|
Operation.state == "uploaded",
|
||||||
|
Operation.attempts < MAX_IMPORT_ATTEMPTS,
|
||||||
|
Operation.expires_at > now,
|
||||||
|
)
|
||||||
|
.order_by(Operation.created_at, Operation.id)
|
||||||
|
.limit(1)
|
||||||
|
.with_for_update()
|
||||||
|
)
|
||||||
|
if op is None:
|
||||||
|
return False
|
||||||
|
op.attempts += 1
|
||||||
|
op.fence += 1
|
||||||
|
op.state = "validating"
|
||||||
|
op.updated_at = now
|
||||||
|
op.lease_until = now + timedelta(seconds=VALIDATION_DEADLINE_SECONDS + 15)
|
||||||
|
identifier, fence, size, digest, actor, epoch = (
|
||||||
|
op.id,
|
||||||
|
op.fence,
|
||||||
|
op.archive_bytes,
|
||||||
|
op.archive_digest,
|
||||||
|
op.owner_id,
|
||||||
|
op.authorization_version,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
async with self.sessions.begin() as session:
|
||||||
|
await self.authority.require_reserved_maintenance(session, actor, epoch)
|
||||||
|
candidate_id = uuid4()
|
||||||
|
result = await self.parse(identifier, size, digest, candidate_id, lock)
|
||||||
|
if "failure_code" in result:
|
||||||
|
await self.fail(identifier, fence, result["failure_code"])
|
||||||
|
return True
|
||||||
|
candidate = validate_candidate(
|
||||||
|
result["manifest"], result["preview"], result["references"], self.policy
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
candidate.digest != result["manifest_digest"]
|
||||||
|
or candidate.preview.candidate_id != candidate_id
|
||||||
|
):
|
||||||
|
raise ValueError("invalid candidate receipt")
|
||||||
|
directory = self.staging.operation(identifier)
|
||||||
|
members = self.staging.members(directory)
|
||||||
|
try:
|
||||||
|
async with self.sessions.begin() as session:
|
||||||
|
await self.authority.require_reserved_maintenance(session, actor, epoch)
|
||||||
|
await ProblemImports.pool(session)
|
||||||
|
op = await session.get(Operation, identifier, with_for_update=True)
|
||||||
|
now = await session.scalar(select(func.clock_timestamp()))
|
||||||
|
if (
|
||||||
|
op.state != "validating"
|
||||||
|
or op.fence != fence
|
||||||
|
or op.lease_until <= now
|
||||||
|
or op.expires_at <= now
|
||||||
|
):
|
||||||
|
raise ValueError("stale candidate receipt")
|
||||||
|
for name, reference in sorted(
|
||||||
|
candidate.references.items(), key=lambda item: item[1].digest
|
||||||
|
):
|
||||||
|
fd = os.open(
|
||||||
|
name,
|
||||||
|
os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC,
|
||||||
|
dir_fd=members,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
await self.artifacts.put_file(session, fd, reference)
|
||||||
|
finally:
|
||||||
|
os.close(fd)
|
||||||
|
session.add(
|
||||||
|
ProblemCandidate(
|
||||||
|
id=candidate_id,
|
||||||
|
operation_id=identifier,
|
||||||
|
manifest_digest=candidate.digest,
|
||||||
|
manifest=candidate.manifest,
|
||||||
|
preview=candidate.preview.model_dump(mode="json"),
|
||||||
|
case_count=candidate.preview.case_count,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
await session.flush()
|
||||||
|
for position, case in enumerate(candidate.manifest["cases"]):
|
||||||
|
input_ref, answer_ref = (
|
||||||
|
candidate.references[case["input_file"]],
|
||||||
|
candidate.references[case["expected_file"]],
|
||||||
|
)
|
||||||
|
session.add(
|
||||||
|
CandidateCase(
|
||||||
|
candidate_id=candidate_id,
|
||||||
|
position=position,
|
||||||
|
key=case["key"],
|
||||||
|
input_digest=input_ref.digest,
|
||||||
|
input_bytes=input_ref.bytes,
|
||||||
|
expected_digest=answer_ref.digest,
|
||||||
|
expected_bytes=answer_ref.bytes,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
await session.flush()
|
||||||
|
# Adoption must not turn an expired lease into a ready candidate.
|
||||||
|
completed_at = await session.scalar(select(func.clock_timestamp()))
|
||||||
|
if op.lease_until <= completed_at or op.expires_at <= completed_at:
|
||||||
|
raise ValueError("candidate completion lease expired")
|
||||||
|
op.state = "ready"
|
||||||
|
op.candidate_id = candidate_id
|
||||||
|
op.updated_at = completed_at
|
||||||
|
op.lease_until = None
|
||||||
|
await self.audit.record(
|
||||||
|
session, actor, epoch, identifier, "import_ready", candidate_id
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
os.close(members)
|
||||||
|
os.close(directory)
|
||||||
|
await self._release(identifier)
|
||||||
|
except MaintenanceDenied:
|
||||||
|
await self.fail(identifier, fence, "authority_revoked")
|
||||||
|
except (ArtifactError, OSError):
|
||||||
|
await self.fail(identifier, fence, "storage_unavailable")
|
||||||
|
except (ValueError, DBAPIError):
|
||||||
|
await self.fail(identifier, fence, "validation_unavailable")
|
||||||
|
return True
|
||||||
|
except ArtifactError:
|
||||||
|
return False # Another live writer holds the one global slot.
|
||||||
|
|
||||||
|
async def fail(self, identifier, fence, code):
|
||||||
|
from kf_oj.contracts.admin_problems import ImportFailure
|
||||||
|
|
||||||
|
code = ImportFailure(code).value
|
||||||
|
async with self.sessions.begin() as session:
|
||||||
|
await ProblemImports.pool(session)
|
||||||
|
op = await session.get(Operation, identifier, with_for_update=True)
|
||||||
|
if op.state != "validating" or op.fence != fence:
|
||||||
|
return
|
||||||
|
op.state = "failed"
|
||||||
|
op.failure_code = code
|
||||||
|
op.updated_at = await session.scalar(select(func.clock_timestamp()))
|
||||||
|
op.lease_until = None
|
||||||
|
await self.audit.record(
|
||||||
|
session, op.owner_id, op.authorization_version, op.id, "import_failed"
|
||||||
|
)
|
||||||
|
await self._release(identifier)
|
||||||
|
|
||||||
|
async def parse(self, identifier, size, digest, candidate, lock):
|
||||||
|
verify_worker_limits(self.cgroup)
|
||||||
|
directory = self.staging.operation(identifier)
|
||||||
|
archive = members = None
|
||||||
|
try:
|
||||||
|
archive = os.open(
|
||||||
|
"archive",
|
||||||
|
os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC,
|
||||||
|
dir_fd=directory,
|
||||||
|
)
|
||||||
|
members = self.staging.members(directory, create=True)
|
||||||
|
# Preserve only the trusted runtime import roots, without inheriting DB/
|
||||||
|
# service credentials. Curated runtimes may keep dependencies outside site-packages.
|
||||||
|
env = {
|
||||||
|
"PATH": str(Path(sys.executable).parent),
|
||||||
|
"PYTHONPATH": os.pathsep.join(
|
||||||
|
dict.fromkeys(
|
||||||
|
[str(Path(__file__).parents[3])]
|
||||||
|
+ [p for p in os.environ.get("PYTHONPATH", "").split(os.pathsep) if p]
|
||||||
|
)
|
||||||
|
),
|
||||||
|
"PYTHONUNBUFFERED": "1",
|
||||||
|
"PYTHONDONTWRITEBYTECODE": "1",
|
||||||
|
"PYTHONNOUSERSITE": "1",
|
||||||
|
"LC_ALL": "C.UTF-8",
|
||||||
|
}
|
||||||
|
# Minimal environment may require the trusted dynamic-loader path in a curated VM.
|
||||||
|
if "LD_LIBRARY_PATH" in os.environ:
|
||||||
|
env["LD_LIBRARY_PATH"] = os.environ["LD_LIBRARY_PATH"]
|
||||||
|
child = await asyncio.create_subprocess_exec(
|
||||||
|
sys.executable,
|
||||||
|
"-m",
|
||||||
|
"kf_oj.workers._archive_child",
|
||||||
|
str(archive),
|
||||||
|
str(members),
|
||||||
|
str(lock),
|
||||||
|
str(size),
|
||||||
|
digest,
|
||||||
|
str(candidate),
|
||||||
|
self.policy.model_dump_json(),
|
||||||
|
env=env,
|
||||||
|
pass_fds=(archive, members, lock),
|
||||||
|
stdout=asyncio.subprocess.PIPE,
|
||||||
|
stderr=asyncio.subprocess.DEVNULL,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
async with asyncio.timeout(VALIDATION_DEADLINE_SECONDS + 5):
|
||||||
|
output, _ = await child.communicate()
|
||||||
|
except BaseException:
|
||||||
|
child.kill()
|
||||||
|
await child.wait()
|
||||||
|
raise
|
||||||
|
if child.returncode != 0 or not 0 < len(output) <= 4 * 1024**2:
|
||||||
|
return {
|
||||||
|
"failure_code": "validation_deadline"
|
||||||
|
if child.returncode == -14
|
||||||
|
else "validation_unavailable"
|
||||||
|
}
|
||||||
|
result = json.loads(output)
|
||||||
|
if not isinstance(result, dict):
|
||||||
|
raise ValueError("invalid child result")
|
||||||
|
return result
|
||||||
|
except TimeoutError:
|
||||||
|
return {"failure_code": "validation_deadline"}
|
||||||
|
finally:
|
||||||
|
if members is not None:
|
||||||
|
os.close(members)
|
||||||
|
if archive is not None:
|
||||||
|
os.close(archive)
|
||||||
|
os.close(directory)
|
||||||
@@ -0,0 +1,432 @@
|
|||||||
|
"""Authenticated durable import lifecycle; filesystem work remains behind private adapters."""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import hashlib
|
||||||
|
import os
|
||||||
|
from contextlib import asynccontextmanager
|
||||||
|
from datetime import timedelta
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from sqlalchemy import func, select, text
|
||||||
|
|
||||||
|
from kf_oj.contracts.admin_problems import (
|
||||||
|
IMPORT_LIFETIME_SECONDS,
|
||||||
|
MAX_OWNER_CANDIDATES,
|
||||||
|
MAX_TOTAL_CANDIDATES,
|
||||||
|
MAX_UPLOAD_CHUNK_BYTES,
|
||||||
|
MAX_UPLOAD_CHUNKS,
|
||||||
|
STAGING_MAX_BYTES,
|
||||||
|
STAGING_RESERVATION_BYTES,
|
||||||
|
UPLOAD_DEADLINE_SECONDS,
|
||||||
|
CandidatePreview,
|
||||||
|
CreateProblemImport,
|
||||||
|
ProblemImportCapabilities,
|
||||||
|
ProblemImportDetail,
|
||||||
|
ProblemImportPage,
|
||||||
|
ProblemImportSummary,
|
||||||
|
)
|
||||||
|
from kf_oj.infra.artifacts.staging import ImportStaging
|
||||||
|
from kf_oj.infra.artifacts.store import ArtifactError
|
||||||
|
from kf_oj.modules.identity.public import MaintenanceDenied, ProblemMaintenancePort
|
||||||
|
from kf_oj.modules.operations.public import MaintenanceAuditPort
|
||||||
|
from kf_oj.modules.problems.import_models import ImportPool, ProblemCandidate
|
||||||
|
from kf_oj.modules.problems.import_models import ProblemImportOperation as Operation
|
||||||
|
from kf_oj.modules.problems.import_policy import ImportUnavailable as ImportUnavailable
|
||||||
|
from kf_oj.modules.problems.import_policy import policy_digest as policy_digest
|
||||||
|
|
||||||
|
ACTIVE = ("uploading", "uploaded", "validating", "ready")
|
||||||
|
|
||||||
|
|
||||||
|
class ImportConflict(ValueError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class ImportBadDigest(ImportConflict):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class ImportMissing(ValueError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def summary(operation) -> ProblemImportSummary:
|
||||||
|
return ProblemImportSummary(
|
||||||
|
operation_id=operation.id,
|
||||||
|
state=operation.state,
|
||||||
|
expected_archive_digest=operation.archive_digest,
|
||||||
|
archive_bytes=operation.archive_bytes,
|
||||||
|
created_at=operation.created_at,
|
||||||
|
expires_at=operation.expires_at,
|
||||||
|
updated_at=operation.updated_at,
|
||||||
|
failure_code=operation.failure_code,
|
||||||
|
candidate_id=operation.candidate_id,
|
||||||
|
published_version_id=operation.published_version_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ProblemImports:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
sessions,
|
||||||
|
staging: ImportStaging,
|
||||||
|
identity: ProblemMaintenancePort,
|
||||||
|
audit: MaintenanceAuditPort,
|
||||||
|
policy,
|
||||||
|
):
|
||||||
|
self.sessions, self.staging, self.identity, self.audit, self.policy = (
|
||||||
|
sessions,
|
||||||
|
staging,
|
||||||
|
identity,
|
||||||
|
audit,
|
||||||
|
policy,
|
||||||
|
)
|
||||||
|
|
||||||
|
@asynccontextmanager
|
||||||
|
async def write(self, principal):
|
||||||
|
async with self.sessions.begin() as session:
|
||||||
|
await session.execute(text("SET LOCAL statement_timeout='5s'"))
|
||||||
|
await session.execute(text("SET LOCAL lock_timeout='2s'"))
|
||||||
|
await self.identity.require_problem_maintenance(session, principal)
|
||||||
|
yield session
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def pool(session):
|
||||||
|
pool = await session.get(ImportPool, 1, with_for_update=True)
|
||||||
|
if pool is None:
|
||||||
|
raise ImportUnavailable("import capacity unavailable")
|
||||||
|
actual = await session.scalar(
|
||||||
|
select(func.coalesce(func.sum(Operation.reserved_bytes), 0)).where(
|
||||||
|
Operation.reserved_bytes > 0
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if pool.reserved_bytes != actual:
|
||||||
|
raise ImportUnavailable("import capacity inconsistent")
|
||||||
|
return pool
|
||||||
|
|
||||||
|
async def capabilities(self):
|
||||||
|
async with self.sessions() as session:
|
||||||
|
pool = await session.get(ImportPool, 1)
|
||||||
|
now = await session.scalar(select(func.clock_timestamp()))
|
||||||
|
available = bool(
|
||||||
|
pool
|
||||||
|
and pool.worker_ready_until
|
||||||
|
and pool.worker_ready_until > now
|
||||||
|
and pool.worker_policy_digest == policy_digest(self.policy)
|
||||||
|
)
|
||||||
|
return ProblemImportCapabilities(
|
||||||
|
uploads_available=available, validation_available=available
|
||||||
|
)
|
||||||
|
|
||||||
|
async def create(self, principal, request: CreateProblemImport):
|
||||||
|
# Immutable receipt retries need no filesystem slot, including while a parser/GC
|
||||||
|
# owns the root lock. Missing receipts still enter the locked creation path below.
|
||||||
|
async with self.write(principal) as session:
|
||||||
|
existing = await session.get(Operation, request.operation_id)
|
||||||
|
if existing:
|
||||||
|
if (
|
||||||
|
existing.owner_id != principal.account_id
|
||||||
|
or existing.archive_digest != request.archive_digest
|
||||||
|
or existing.archive_bytes != request.archive_bytes
|
||||||
|
):
|
||||||
|
raise ImportConflict("operation key already bound")
|
||||||
|
return summary(existing)
|
||||||
|
try:
|
||||||
|
with self.staging.exclusive():
|
||||||
|
await asyncio.to_thread(self.staging.usage)
|
||||||
|
async with self.write(principal) as session:
|
||||||
|
pool = await self.pool(session)
|
||||||
|
existing = await session.get(
|
||||||
|
Operation, request.operation_id, with_for_update=True
|
||||||
|
)
|
||||||
|
if existing:
|
||||||
|
if (
|
||||||
|
existing.owner_id != principal.account_id
|
||||||
|
or existing.archive_digest != request.archive_digest
|
||||||
|
or existing.archive_bytes != request.archive_bytes
|
||||||
|
):
|
||||||
|
raise ImportConflict("operation key already bound")
|
||||||
|
return summary(existing)
|
||||||
|
now = await session.scalar(select(func.clock_timestamp()))
|
||||||
|
if (
|
||||||
|
not pool.worker_ready_until
|
||||||
|
or pool.worker_ready_until <= now
|
||||||
|
or pool.worker_policy_digest != policy_digest(self.policy)
|
||||||
|
):
|
||||||
|
raise ImportUnavailable("import validation unavailable")
|
||||||
|
global_count = await session.scalar(
|
||||||
|
select(func.count())
|
||||||
|
.select_from(Operation)
|
||||||
|
.where(Operation.state.in_(ACTIVE))
|
||||||
|
)
|
||||||
|
owner_count = await session.scalar(
|
||||||
|
select(func.count())
|
||||||
|
.select_from(Operation)
|
||||||
|
.where(
|
||||||
|
Operation.state.in_(ACTIVE), Operation.owner_id == principal.account_id
|
||||||
|
)
|
||||||
|
)
|
||||||
|
busy = await session.scalar(
|
||||||
|
select(func.count())
|
||||||
|
.select_from(Operation)
|
||||||
|
.where(
|
||||||
|
Operation.state.in_(("uploading", "validating")),
|
||||||
|
Operation.lease_until > now,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
global_count >= MAX_TOTAL_CANDIDATES
|
||||||
|
or owner_count >= MAX_OWNER_CANDIDATES
|
||||||
|
or busy
|
||||||
|
or pool.reserved_bytes + STAGING_RESERVATION_BYTES > STAGING_MAX_BYTES
|
||||||
|
):
|
||||||
|
raise ImportUnavailable("import capacity exhausted")
|
||||||
|
operation = Operation(
|
||||||
|
id=request.operation_id,
|
||||||
|
owner_id=principal.account_id,
|
||||||
|
authorization_version=principal.authorization_version,
|
||||||
|
archive_digest=request.archive_digest,
|
||||||
|
archive_bytes=request.archive_bytes,
|
||||||
|
state="uploading",
|
||||||
|
created_at=now,
|
||||||
|
updated_at=now,
|
||||||
|
expires_at=now + timedelta(seconds=IMPORT_LIFETIME_SECONDS),
|
||||||
|
lease_until=now + timedelta(seconds=UPLOAD_DEADLINE_SECONDS + 15),
|
||||||
|
attempts=0,
|
||||||
|
fence=0,
|
||||||
|
reserved_bytes=STAGING_RESERVATION_BYTES,
|
||||||
|
)
|
||||||
|
session.add(operation)
|
||||||
|
pool.reserved_bytes += STAGING_RESERVATION_BYTES
|
||||||
|
await self.audit.record(
|
||||||
|
session,
|
||||||
|
principal.account_id,
|
||||||
|
principal.authorization_version,
|
||||||
|
operation.id,
|
||||||
|
"upload_created",
|
||||||
|
)
|
||||||
|
await session.flush()
|
||||||
|
return summary(operation)
|
||||||
|
except ArtifactError:
|
||||||
|
raise ImportUnavailable("import storage unavailable") from None
|
||||||
|
|
||||||
|
async def detail(self, principal, operation_id):
|
||||||
|
# Recheck live authority even when a principal was injected by an internal caller.
|
||||||
|
async with self.write(principal) as session:
|
||||||
|
op = await session.get(Operation, operation_id)
|
||||||
|
if op is None or op.owner_id != principal.account_id:
|
||||||
|
raise ImportMissing("import unavailable")
|
||||||
|
candidate = (
|
||||||
|
await session.get(ProblemCandidate, op.candidate_id) if op.candidate_id else None
|
||||||
|
)
|
||||||
|
return ProblemImportDetail(
|
||||||
|
**summary(op).model_dump(),
|
||||||
|
candidate=CandidatePreview.model_validate(candidate.preview) if candidate else None,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def list(self, principal, after=None, limit=20):
|
||||||
|
if not 1 <= limit <= 100:
|
||||||
|
raise ValueError("invalid page limit")
|
||||||
|
async with self.write(principal) as session:
|
||||||
|
query = select(Operation).where(Operation.owner_id == principal.account_id)
|
||||||
|
if after:
|
||||||
|
query = query.where(Operation.id > after)
|
||||||
|
rows = list(await session.scalars(query.order_by(Operation.id).limit(limit + 1)))
|
||||||
|
return ProblemImportPage(
|
||||||
|
items=tuple(summary(o) for o in rows[:limit]),
|
||||||
|
next_after=rows[limit - 1].id if len(rows) > limit else None,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def upload(self, principal, operation_id: UUID, chunks, content_length: int | None):
|
||||||
|
# Reject missing/foreign receipts before consulting shared filesystem capacity.
|
||||||
|
# The locked transaction below still rechecks authority and ownership before writes.
|
||||||
|
async with self.write(principal) as session:
|
||||||
|
op = await session.get(Operation, operation_id)
|
||||||
|
if op is None or op.owner_id != principal.account_id:
|
||||||
|
raise ImportMissing("import unavailable")
|
||||||
|
if content_length is not None and content_length != op.archive_bytes:
|
||||||
|
raise ImportConflict("archive length mismatch")
|
||||||
|
descriptor = operation_fd = None
|
||||||
|
fence = None
|
||||||
|
received = 0
|
||||||
|
chunk_count = 0
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
try:
|
||||||
|
with self.staging.exclusive():
|
||||||
|
async with self.write(principal) as session:
|
||||||
|
await self.pool(session)
|
||||||
|
op = await session.get(Operation, operation_id, with_for_update=True)
|
||||||
|
if not op or op.owner_id != principal.account_id:
|
||||||
|
raise ImportMissing("import unavailable")
|
||||||
|
accepted = op.state in ("uploaded", "validating", "ready", "published")
|
||||||
|
now = await session.scalar(select(func.clock_timestamp()))
|
||||||
|
if (
|
||||||
|
not accepted
|
||||||
|
and (
|
||||||
|
op.state != "uploading"
|
||||||
|
or op.lease_until is None
|
||||||
|
or op.lease_until <= now
|
||||||
|
)
|
||||||
|
) or op.expires_at <= now:
|
||||||
|
raise ImportConflict("upload no longer available")
|
||||||
|
if content_length is not None and content_length != op.archive_bytes:
|
||||||
|
raise ImportConflict("archive length mismatch")
|
||||||
|
if not accepted:
|
||||||
|
op.fence += 1
|
||||||
|
fence = op.fence
|
||||||
|
op.lease_until = now + timedelta(seconds=UPLOAD_DEADLINE_SECONDS + 15)
|
||||||
|
expected_bytes, expected_digest = op.archive_bytes, op.archive_digest
|
||||||
|
if not accepted:
|
||||||
|
operation_fd = self.staging.operation(operation_id, create=True)
|
||||||
|
descriptor = os.open(
|
||||||
|
"archive",
|
||||||
|
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC,
|
||||||
|
0o600,
|
||||||
|
dir_fd=operation_fd,
|
||||||
|
)
|
||||||
|
async with asyncio.timeout(UPLOAD_DEADLINE_SECONDS):
|
||||||
|
async for chunk in chunks:
|
||||||
|
chunk_count += 1
|
||||||
|
received += len(chunk)
|
||||||
|
if (
|
||||||
|
type(chunk) is not bytes
|
||||||
|
or len(chunk) > MAX_UPLOAD_CHUNK_BYTES
|
||||||
|
or chunk_count > MAX_UPLOAD_CHUNKS
|
||||||
|
or received > expected_bytes
|
||||||
|
):
|
||||||
|
raise ImportConflict("archive bytes outside policy")
|
||||||
|
digest.update(chunk)
|
||||||
|
if accepted:
|
||||||
|
continue
|
||||||
|
# Cancellation waits for the finite writer outside the event loop.
|
||||||
|
task = asyncio.create_task(asyncio.to_thread(write_all, descriptor, chunk))
|
||||||
|
try:
|
||||||
|
await asyncio.shield(task)
|
||||||
|
except BaseException:
|
||||||
|
await task
|
||||||
|
raise
|
||||||
|
if received != expected_bytes or digest.hexdigest() != expected_digest:
|
||||||
|
raise ImportBadDigest("archive digest or length mismatch")
|
||||||
|
if accepted:
|
||||||
|
async with self.write(principal) as session:
|
||||||
|
op = await session.get(Operation, operation_id)
|
||||||
|
if not op or op.state not in (
|
||||||
|
"uploaded",
|
||||||
|
"validating",
|
||||||
|
"ready",
|
||||||
|
"published",
|
||||||
|
):
|
||||||
|
raise ImportConflict("import receipt no longer available")
|
||||||
|
return summary(op)
|
||||||
|
await asyncio.to_thread(os.fsync, descriptor)
|
||||||
|
os.close(descriptor)
|
||||||
|
descriptor = None
|
||||||
|
os.fsync(operation_fd)
|
||||||
|
async with self.write(principal) as session:
|
||||||
|
await self.pool(session)
|
||||||
|
op = await session.get(Operation, operation_id, with_for_update=True)
|
||||||
|
now = await session.scalar(select(func.clock_timestamp()))
|
||||||
|
if op.state != "uploading" or op.fence != fence or op.lease_until <= now:
|
||||||
|
raise ImportConflict("upload receipt expired")
|
||||||
|
op.state = "uploaded"
|
||||||
|
op.lease_until = None
|
||||||
|
op.updated_at = now
|
||||||
|
await self.audit.record(
|
||||||
|
session, op.owner_id, op.authorization_version, op.id, "upload_received"
|
||||||
|
)
|
||||||
|
result = summary(op)
|
||||||
|
return result
|
||||||
|
except BaseException as error:
|
||||||
|
# Failure bookkeeping cannot authorize a new business write. It only fences this
|
||||||
|
# accepted upload and releases capacity after physical bytes have been removed.
|
||||||
|
if descriptor is not None:
|
||||||
|
os.close(descriptor)
|
||||||
|
descriptor = None
|
||||||
|
if operation_fd is not None:
|
||||||
|
os.close(operation_fd)
|
||||||
|
operation_fd = None
|
||||||
|
if fence is not None:
|
||||||
|
try:
|
||||||
|
with self.staging.exclusive():
|
||||||
|
await asyncio.to_thread(self.staging.remove, operation_id)
|
||||||
|
async with self.sessions.begin() as session:
|
||||||
|
pool = await self.pool(session)
|
||||||
|
op = await session.get(Operation, operation_id, with_for_update=True)
|
||||||
|
if op and op.state == "uploading" and op.fence == fence:
|
||||||
|
op.state = "failed"
|
||||||
|
op.failure_code = (
|
||||||
|
"upload_deadline"
|
||||||
|
if isinstance(error, TimeoutError)
|
||||||
|
else "digest_mismatch"
|
||||||
|
if isinstance(error, ImportBadDigest)
|
||||||
|
else "authority_revoked"
|
||||||
|
if isinstance(error, MaintenanceDenied)
|
||||||
|
else "storage_unavailable"
|
||||||
|
if isinstance(error, OSError)
|
||||||
|
else "upload_interrupted"
|
||||||
|
)
|
||||||
|
op.updated_at = await session.scalar(select(func.clock_timestamp()))
|
||||||
|
op.lease_until = None
|
||||||
|
pool.reserved_bytes -= op.reserved_bytes
|
||||||
|
op.reserved_bytes = 0
|
||||||
|
await self.audit.record(
|
||||||
|
session,
|
||||||
|
op.owner_id,
|
||||||
|
op.authorization_version,
|
||||||
|
op.id,
|
||||||
|
"import_failed",
|
||||||
|
)
|
||||||
|
except (ArtifactError, OSError):
|
||||||
|
pass # Durable reservation retained for fenced GC.
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
if descriptor is not None:
|
||||||
|
os.close(descriptor)
|
||||||
|
if operation_fd is not None:
|
||||||
|
os.close(operation_fd)
|
||||||
|
|
||||||
|
async def discard(self, principal, operation_id):
|
||||||
|
async with self.write(principal) as session:
|
||||||
|
op = await session.get(Operation, operation_id)
|
||||||
|
if op is None or op.owner_id != principal.account_id:
|
||||||
|
raise ImportMissing("import unavailable")
|
||||||
|
if op.state == "discarded":
|
||||||
|
return summary(op)
|
||||||
|
try:
|
||||||
|
with self.staging.exclusive():
|
||||||
|
async with self.write(principal) as session:
|
||||||
|
pool = await self.pool(session)
|
||||||
|
op = await session.get(Operation, operation_id, with_for_update=True)
|
||||||
|
if not op or op.owner_id != principal.account_id:
|
||||||
|
raise ImportMissing("import unavailable")
|
||||||
|
if op.state == "published":
|
||||||
|
raise ImportConflict("published import cannot be discarded")
|
||||||
|
if op.state == "discarded":
|
||||||
|
return summary(op)
|
||||||
|
await asyncio.to_thread(self.staging.remove, operation_id)
|
||||||
|
op.state = "discarded"
|
||||||
|
op.candidate_id = None
|
||||||
|
op.failure_code = None
|
||||||
|
op.lease_until = None
|
||||||
|
op.fence += 1
|
||||||
|
op.updated_at = await session.scalar(select(func.clock_timestamp()))
|
||||||
|
pool.reserved_bytes -= op.reserved_bytes
|
||||||
|
op.reserved_bytes = 0
|
||||||
|
await self.audit.record(
|
||||||
|
session,
|
||||||
|
principal.account_id,
|
||||||
|
principal.authorization_version,
|
||||||
|
op.id,
|
||||||
|
"import_discarded",
|
||||||
|
)
|
||||||
|
return summary(op)
|
||||||
|
except ArtifactError:
|
||||||
|
raise ImportUnavailable("import storage unavailable") from None
|
||||||
|
|
||||||
|
|
||||||
|
def write_all(descriptor, data):
|
||||||
|
remaining = memoryview(data)
|
||||||
|
while remaining:
|
||||||
|
size = os.write(descriptor, remaining)
|
||||||
|
if size <= 0:
|
||||||
|
raise OSError("short staging write")
|
||||||
|
remaining = remaining[size:]
|
||||||
@@ -0,0 +1,324 @@
|
|||||||
|
"""Administrator transactions; no evaluation writes or private data in projections."""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from contextlib import asynccontextmanager
|
||||||
|
from uuid import uuid4, uuid5
|
||||||
|
|
||||||
|
from sqlalchemy import func, select, text
|
||||||
|
|
||||||
|
from kf_oj.contracts.admin_problems import CandidatePreview
|
||||||
|
from kf_oj.contracts.judging import ArtifactReference, TestCase
|
||||||
|
from kf_oj.contracts.problem_maintenance import (
|
||||||
|
MaintainedProblem,
|
||||||
|
MaintainedProblemPage,
|
||||||
|
MaintainedVersion,
|
||||||
|
MaintainedVersionPage,
|
||||||
|
MaintenanceReceipt,
|
||||||
|
)
|
||||||
|
from kf_oj.modules.identity.public import ProblemMaintenancePort
|
||||||
|
from kf_oj.modules.operations.public import MaintenanceAuditPort
|
||||||
|
from kf_oj.modules.problems import repository
|
||||||
|
from kf_oj.modules.problems.candidate_validation import validate_candidate
|
||||||
|
from kf_oj.modules.problems.import_format import ImportEnvelope
|
||||||
|
from kf_oj.modules.problems.import_models import (
|
||||||
|
CandidateCase,
|
||||||
|
ProblemCandidate,
|
||||||
|
ProblemImportOperation,
|
||||||
|
)
|
||||||
|
from kf_oj.modules.problems.imports import ImportConflict, ImportMissing, ImportUnavailable
|
||||||
|
from kf_oj.modules.problems.maintenance_models import ProblemMaintenanceOperation
|
||||||
|
from kf_oj.modules.problems.models import Problem, ProblemCase, ProblemVersion
|
||||||
|
from kf_oj.modules.problems.publication import admit_version, publish_version
|
||||||
|
from kf_oj.modules.problems.service import ProblemService
|
||||||
|
|
||||||
|
|
||||||
|
def project_problem(problem, version):
|
||||||
|
return MaintainedProblem(
|
||||||
|
id=problem.id,
|
||||||
|
slug=problem.slug,
|
||||||
|
enabled=problem.enabled,
|
||||||
|
active_version_id=problem.active_version_id,
|
||||||
|
revision=version.revision if version else 0,
|
||||||
|
title=version.title if version else None,
|
||||||
|
created_at=problem.created_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ProblemMaintenance:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
sessions,
|
||||||
|
identity: ProblemMaintenancePort,
|
||||||
|
audit: MaintenanceAuditPort,
|
||||||
|
artifacts=None,
|
||||||
|
policy=None,
|
||||||
|
):
|
||||||
|
self.sessions, self.identity, self.audit = sessions, identity, audit
|
||||||
|
self.artifacts, self.policy = artifacts, policy
|
||||||
|
self._publishing = asyncio.Semaphore(2)
|
||||||
|
|
||||||
|
@asynccontextmanager
|
||||||
|
async def transaction(self, principal):
|
||||||
|
async with self.sessions.begin() as session:
|
||||||
|
await session.execute(text("SET LOCAL statement_timeout='5s'"))
|
||||||
|
await session.execute(text("SET LOCAL lock_timeout='2s'"))
|
||||||
|
await self.identity.require_problem_maintenance(session, principal)
|
||||||
|
yield session
|
||||||
|
|
||||||
|
async def page(self, principal, *, after=None, limit=20, enabled=None):
|
||||||
|
if not 1 <= limit <= 100:
|
||||||
|
raise ValueError("invalid page limit")
|
||||||
|
async with self.transaction(principal) as session:
|
||||||
|
q = select(Problem, ProblemVersion).outerjoin(
|
||||||
|
ProblemVersion, Problem.active_version_id == ProblemVersion.id
|
||||||
|
)
|
||||||
|
if after is not None:
|
||||||
|
q = q.where(Problem.slug > after)
|
||||||
|
if enabled is not None:
|
||||||
|
q = q.where(Problem.enabled == enabled)
|
||||||
|
rows = list(await session.execute(q.order_by(Problem.slug).limit(limit + 1)))
|
||||||
|
return MaintainedProblemPage(
|
||||||
|
items=tuple(project_problem(*r) for r in rows[:limit]),
|
||||||
|
next_after=rows[limit - 1][0].slug if len(rows) > limit else None,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def by_slug(self, principal, slug):
|
||||||
|
async with self.transaction(principal) as session:
|
||||||
|
row = (
|
||||||
|
await session.execute(
|
||||||
|
select(Problem, ProblemVersion)
|
||||||
|
.outerjoin(ProblemVersion, Problem.active_version_id == ProblemVersion.id)
|
||||||
|
.where(Problem.slug == slug)
|
||||||
|
)
|
||||||
|
).one_or_none()
|
||||||
|
if row is None:
|
||||||
|
raise ImportMissing("problem unavailable")
|
||||||
|
return project_problem(*row)
|
||||||
|
|
||||||
|
async def detail(self, principal, problem_id):
|
||||||
|
async with self.transaction(principal) as session:
|
||||||
|
problem = await session.get(Problem, problem_id)
|
||||||
|
if problem is None:
|
||||||
|
raise ImportMissing("problem unavailable")
|
||||||
|
version = (
|
||||||
|
await session.get(ProblemVersion, problem.active_version_id)
|
||||||
|
if problem.active_version_id
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
return project_problem(problem, version)
|
||||||
|
|
||||||
|
async def versions(self, principal, problem_id, *, before=None, limit=20):
|
||||||
|
if not 1 <= limit <= 100 or (before is not None and before < 1):
|
||||||
|
raise ValueError("invalid page limit")
|
||||||
|
async with self.transaction(principal) as session:
|
||||||
|
problem = await session.get(Problem, problem_id)
|
||||||
|
if problem is None:
|
||||||
|
raise ImportMissing("problem unavailable")
|
||||||
|
count = (
|
||||||
|
select(func.count())
|
||||||
|
.where(ProblemCase.version_id == ProblemVersion.id)
|
||||||
|
.scalar_subquery()
|
||||||
|
)
|
||||||
|
q = select(ProblemVersion, count).where(
|
||||||
|
ProblemVersion.problem_id == problem_id, ProblemVersion.published_at.is_not(None)
|
||||||
|
)
|
||||||
|
if before is not None:
|
||||||
|
q = q.where(ProblemVersion.revision < before)
|
||||||
|
rows = list(
|
||||||
|
await session.execute(q.order_by(ProblemVersion.revision.desc()).limit(limit + 1))
|
||||||
|
)
|
||||||
|
return MaintainedVersionPage(
|
||||||
|
items=tuple(
|
||||||
|
MaintainedVersion(
|
||||||
|
**ProblemService._summary(problem, v).model_dump(),
|
||||||
|
published_at=v.published_at,
|
||||||
|
case_count=n,
|
||||||
|
)
|
||||||
|
for v, n in rows[:limit]
|
||||||
|
),
|
||||||
|
next_before=rows[limit - 1][0].revision if len(rows) > limit else None,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def receipt(self, principal, operation_id):
|
||||||
|
async with self.transaction(principal) as session:
|
||||||
|
row = await session.get(ProblemMaintenanceOperation, operation_id)
|
||||||
|
if row is None or row.owner_id != principal.account_id:
|
||||||
|
raise ImportMissing("operation unavailable")
|
||||||
|
return MaintenanceReceipt.model_validate(row.receipt)
|
||||||
|
|
||||||
|
async def _replay(self, session, principal, action, body, target):
|
||||||
|
digest = hashlib.sha256(
|
||||||
|
json.dumps(
|
||||||
|
{"action": action, "target": str(target), "request": body.model_dump(mode="json")},
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
).encode()
|
||||||
|
).hexdigest()
|
||||||
|
key = int.from_bytes(hashlib.sha256(body.operation_id.bytes).digest()[:4], signed=True)
|
||||||
|
await session.execute(text("SELECT pg_advisory_xact_lock(1262907470,:key)"), {"key": key})
|
||||||
|
row = await session.get(ProblemMaintenanceOperation, body.operation_id)
|
||||||
|
if row is not None:
|
||||||
|
if row.owner_id != principal.account_id:
|
||||||
|
raise ImportMissing("operation unavailable")
|
||||||
|
if row.action != action or row.request_digest != digest:
|
||||||
|
raise ImportConflict("operation UUID already bound")
|
||||||
|
return digest, MaintenanceReceipt.model_validate(row.receipt)
|
||||||
|
return digest, None
|
||||||
|
|
||||||
|
async def _record(
|
||||||
|
self, session, principal, request, action, digest, problem, version, published=None
|
||||||
|
):
|
||||||
|
receipt = MaintenanceReceipt(
|
||||||
|
operation_id=request.operation_id,
|
||||||
|
action=action,
|
||||||
|
completed_at=await session.scalar(select(func.clock_timestamp())),
|
||||||
|
problem=project_problem(problem, version),
|
||||||
|
published_version_id=published,
|
||||||
|
)
|
||||||
|
session.add(
|
||||||
|
ProblemMaintenanceOperation(
|
||||||
|
id=request.operation_id,
|
||||||
|
owner_id=principal.account_id,
|
||||||
|
authorization_version=principal.authorization_version,
|
||||||
|
action=action,
|
||||||
|
request_digest=digest,
|
||||||
|
receipt=receipt.model_dump(mode="json"),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
await self.audit.record(
|
||||||
|
session,
|
||||||
|
principal.account_id,
|
||||||
|
principal.authorization_version,
|
||||||
|
request.operation_id,
|
||||||
|
{"publish": "published", "disable": "disabled", "restore": "restored"}[action],
|
||||||
|
published if action == "publish" else problem.id,
|
||||||
|
)
|
||||||
|
await session.flush()
|
||||||
|
return receipt
|
||||||
|
|
||||||
|
async def publish(self, principal, request):
|
||||||
|
if self.artifacts is None or self.policy is None or self._publishing.locked():
|
||||||
|
raise ImportUnavailable("publication unavailable")
|
||||||
|
async with self._publishing:
|
||||||
|
async with self.transaction(principal) as session:
|
||||||
|
digest, replay = await self._replay(
|
||||||
|
session, principal, "publish", request, request.import_operation_id
|
||||||
|
)
|
||||||
|
if replay:
|
||||||
|
return replay
|
||||||
|
op = await session.get(
|
||||||
|
ProblemImportOperation, request.import_operation_id, with_for_update=True
|
||||||
|
)
|
||||||
|
if op is None or op.owner_id != principal.account_id:
|
||||||
|
raise ImportMissing("candidate unavailable")
|
||||||
|
if (
|
||||||
|
op.state != "ready"
|
||||||
|
or op.candidate_id != request.candidate_id
|
||||||
|
or op.authorization_version != principal.authorization_version
|
||||||
|
or op.expires_at <= await session.scalar(select(func.clock_timestamp()))
|
||||||
|
):
|
||||||
|
raise ImportConflict("candidate no longer publishable")
|
||||||
|
candidate = await session.get(ProblemCandidate, op.candidate_id)
|
||||||
|
if candidate is None or candidate.operation_id != op.id:
|
||||||
|
raise ImportUnavailable("candidate unavailable")
|
||||||
|
manifest = ImportEnvelope.model_validate(candidate.manifest).root
|
||||||
|
preview = CandidatePreview.model_validate(candidate.preview)
|
||||||
|
rows = list(
|
||||||
|
await session.scalars(
|
||||||
|
select(CandidateCase)
|
||||||
|
.where(CandidateCase.candidate_id == candidate.id)
|
||||||
|
.order_by(CandidateCase.position)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if len(rows) != len(manifest.cases) or len(rows) != candidate.case_count:
|
||||||
|
raise ImportUnavailable("candidate references unavailable")
|
||||||
|
refs = {}
|
||||||
|
for position, (row, c) in enumerate(zip(rows, manifest.cases, strict=True)):
|
||||||
|
if row.position != position or row.key != c.key:
|
||||||
|
raise ImportUnavailable("candidate references unavailable")
|
||||||
|
for name, ref in (
|
||||||
|
(
|
||||||
|
c.input_file,
|
||||||
|
ArtifactReference(digest=row.input_digest, bytes=row.input_bytes),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
c.expected_file,
|
||||||
|
ArtifactReference(digest=row.expected_digest, bytes=row.expected_bytes),
|
||||||
|
),
|
||||||
|
):
|
||||||
|
if name in refs and refs[name] != ref:
|
||||||
|
raise ImportUnavailable("candidate references unavailable")
|
||||||
|
refs[name] = ref
|
||||||
|
validated = validate_candidate(
|
||||||
|
candidate.manifest, candidate.preview, refs, self.policy
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
validated.digest != candidate.manifest_digest
|
||||||
|
or preview.candidate_id != candidate.id
|
||||||
|
):
|
||||||
|
raise ImportUnavailable("candidate integrity unavailable")
|
||||||
|
await repository.lock_slug(session, manifest.slug)
|
||||||
|
problem = await repository.lock_problem(session, manifest.slug)
|
||||||
|
actual = problem.active_version_id if problem else None
|
||||||
|
if actual != request.expected_active_version_id or (
|
||||||
|
problem and not problem.enabled
|
||||||
|
):
|
||||||
|
raise ImportConflict("active version or availability changed")
|
||||||
|
if problem is None:
|
||||||
|
problem = Problem(id=uuid4(), slug=manifest.slug)
|
||||||
|
session.add(problem)
|
||||||
|
await session.flush()
|
||||||
|
version_id = uuid4()
|
||||||
|
cases = tuple(
|
||||||
|
TestCase(
|
||||||
|
case_id=uuid5(version_id, c.key),
|
||||||
|
input=refs[c.input_file],
|
||||||
|
expected=refs[c.expected_file],
|
||||||
|
)
|
||||||
|
for c in manifest.cases
|
||||||
|
)
|
||||||
|
admit_version(self.policy, manifest, version_id, problem.id, cases)
|
||||||
|
# Candidate FKs protect all referenced ready metadata from GC. The
|
||||||
|
# private adapter additionally re-hashes disk bytes without buffering them.
|
||||||
|
for ref in sorted(
|
||||||
|
{r.digest: r for r in refs.values()}.values(), key=lambda r: r.digest
|
||||||
|
):
|
||||||
|
await self.artifacts.check_metadata(session, ref)
|
||||||
|
task = asyncio.create_task(asyncio.to_thread(self.artifacts.store.verify, ref))
|
||||||
|
try:
|
||||||
|
await asyncio.shield(task)
|
||||||
|
except BaseException:
|
||||||
|
await task
|
||||||
|
raise
|
||||||
|
version = await publish_version(
|
||||||
|
session, problem, manifest, preview, version_id, cases
|
||||||
|
)
|
||||||
|
op.state, op.published_version_id = "published", version.id
|
||||||
|
op.updated_at = await session.scalar(select(func.clock_timestamp()))
|
||||||
|
return await self._record(
|
||||||
|
session, principal, request, "publish", digest, problem, version, version.id
|
||||||
|
)
|
||||||
|
|
||||||
|
async def availability(self, principal, problem_id, request, *, enabled):
|
||||||
|
action = "restore" if enabled else "disable"
|
||||||
|
async with self.transaction(principal) as session:
|
||||||
|
digest, replay = await self._replay(session, principal, action, request, problem_id)
|
||||||
|
if replay:
|
||||||
|
return replay
|
||||||
|
problem = await session.get(Problem, problem_id, with_for_update=True)
|
||||||
|
if problem is None:
|
||||||
|
raise ImportMissing("problem unavailable")
|
||||||
|
if (
|
||||||
|
problem.active_version_id != request.expected_active_version_id
|
||||||
|
or problem.enabled != request.expected_enabled
|
||||||
|
or problem.enabled == enabled
|
||||||
|
):
|
||||||
|
raise ImportConflict("active version or availability changed")
|
||||||
|
version = await session.get(ProblemVersion, problem.active_version_id)
|
||||||
|
if version is None or version.published_at is None:
|
||||||
|
raise ImportUnavailable("published version unavailable")
|
||||||
|
problem.enabled = enabled
|
||||||
|
await session.flush()
|
||||||
|
return await self._record(session, principal, request, action, digest, problem, version)
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
"""Append-only actor-bound receipts; replay never re-applies a published operation."""
|
||||||
|
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from sqlalchemy import BigInteger, CheckConstraint, ForeignKey, String
|
||||||
|
from sqlalchemy.dialects.postgresql import JSONB
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
|
from kf_oj.infra.db.base import Base
|
||||||
|
from kf_oj.infra.db.columns import CreatedAt
|
||||||
|
|
||||||
|
|
||||||
|
class ProblemMaintenanceOperation(CreatedAt, Base):
|
||||||
|
__tablename__ = "problem_maintenance_operations"
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint("action IN ('publish','disable','restore')", name="action"),
|
||||||
|
CheckConstraint("request_digest ~ '^[0-9a-f]{64}$'", name="digest"),
|
||||||
|
CheckConstraint("authorization_version > 0", name="authority"),
|
||||||
|
CheckConstraint("octet_length(receipt::text) <= 4096", name="receipt"),
|
||||||
|
)
|
||||||
|
id: Mapped[UUID] = mapped_column(primary_key=True)
|
||||||
|
owner_id: Mapped[UUID] = mapped_column(ForeignKey("accounts.id"))
|
||||||
|
authorization_version: Mapped[int] = mapped_column(BigInteger)
|
||||||
|
action: Mapped[str] = mapped_column(String(16))
|
||||||
|
request_digest: Mapped[str] = mapped_column(String(64))
|
||||||
|
receipt: Mapped[dict] = mapped_column(JSONB)
|
||||||
@@ -0,0 +1,64 @@
|
|||||||
|
"""Read-only import projection for the ordinary operator CLI; no private archive data."""
|
||||||
|
|
||||||
|
from sqlalchemy import text
|
||||||
|
|
||||||
|
from kf_oj.modules.problems.import_models import ImportPool
|
||||||
|
from kf_oj.modules.problems.import_policy import ImportUnavailable, policy_digest
|
||||||
|
|
||||||
|
|
||||||
|
class ImportOperationQueries:
|
||||||
|
def __init__(self, sessions, policy):
|
||||||
|
self.sessions, self.policy = sessions, policy
|
||||||
|
|
||||||
|
async def status(self) -> dict:
|
||||||
|
async with self.sessions() as session, session.begin():
|
||||||
|
await session.execute(text("SET TRANSACTION READ ONLY"))
|
||||||
|
await session.execute(text("SET LOCAL statement_timeout='1500'"))
|
||||||
|
await session.execute(text("SET LOCAL lock_timeout='500'"))
|
||||||
|
authorized = await session.scalar(
|
||||||
|
text("""
|
||||||
|
SELECT rolcanlogin AND NOT(rolsuper OR rolcreatedb OR rolcreaterole
|
||||||
|
OR rolreplication OR rolbypassrls)
|
||||||
|
AND pg_has_role(current_user,'kf_oj_problem_import','USAGE')
|
||||||
|
AND NOT pg_has_role(current_user,'kf_oj_platform','MEMBER')
|
||||||
|
AND NOT pg_has_role(current_user,'kf_oj_coordinator','MEMBER')
|
||||||
|
AND NOT pg_has_role(current_user,'kf_oj_identity_operator','MEMBER')
|
||||||
|
AND NOT pg_has_role(current_user,'kf_oj_identity_authority_owner','MEMBER')
|
||||||
|
FROM pg_roles WHERE rolname=session_user
|
||||||
|
""")
|
||||||
|
)
|
||||||
|
if authorized is not True:
|
||||||
|
raise ImportUnavailable("dedicated import observation role required")
|
||||||
|
pool = await session.get(ImportPool, 1)
|
||||||
|
row = (
|
||||||
|
(
|
||||||
|
await session.execute(
|
||||||
|
text("""
|
||||||
|
SELECT clock_timestamp() now,
|
||||||
|
(SELECT count(*) FROM (SELECT 1 FROM problem_import_operations
|
||||||
|
WHERE state='uploaded' LIMIT 129) q) uploaded,
|
||||||
|
(SELECT count(*) FROM (SELECT 1 FROM problem_import_operations
|
||||||
|
WHERE state='validating' LIMIT 129) q) validating,
|
||||||
|
(SELECT coalesce(sum(reserved_bytes),0) FROM (
|
||||||
|
SELECT reserved_bytes FROM problem_import_operations
|
||||||
|
WHERE reserved_bytes>0 LIMIT 129) q) actual_reserved
|
||||||
|
""")
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.mappings()
|
||||||
|
.one()
|
||||||
|
)
|
||||||
|
if pool is None or pool.reserved_bytes != row["actual_reserved"]:
|
||||||
|
raise ImportUnavailable("import reservation consistency unknown")
|
||||||
|
return {
|
||||||
|
"schema_version": 1,
|
||||||
|
"worker_ready": bool(
|
||||||
|
pool.worker_ready_until
|
||||||
|
and pool.worker_ready_until > row["now"]
|
||||||
|
and pool.worker_policy_digest == policy_digest(self.policy)
|
||||||
|
),
|
||||||
|
"reserved_bytes": pool.reserved_bytes,
|
||||||
|
"uploaded_capped": min(row["uploaded"], 128),
|
||||||
|
"validating_capped": min(row["validating"], 128),
|
||||||
|
"truncated": row["uploaded"] > 128 or row["validating"] > 128,
|
||||||
|
}
|
||||||
@@ -78,3 +78,7 @@ class JudgingVersion:
|
|||||||
|
|
||||||
class ProblemPort(Protocol):
|
class ProblemPort(Protocol):
|
||||||
async def for_submission(self, session, version_id: UUID) -> JudgingVersion: ...
|
async def for_submission(self, session, version_id: UUID) -> JudgingVersion: ...
|
||||||
|
|
||||||
|
async def for_accepted_execution(self, session, version_id: UUID) -> JudgingVersion:
|
||||||
|
"""Require an already-validated live execution lease; allow a disabled published version."""
|
||||||
|
...
|
||||||
|
|||||||
@@ -0,0 +1,82 @@
|
|||||||
|
"""One admission and immutable publication path for local and web imports."""
|
||||||
|
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
|
||||||
|
from kf_oj.contracts import judging_v2 as v2
|
||||||
|
from kf_oj.contracts.judging import MAX_SOURCE_BYTES, ArtifactReference
|
||||||
|
from kf_oj.modules.problems import repository
|
||||||
|
from kf_oj.modules.problems.import_format import ProblemImport
|
||||||
|
from kf_oj.modules.problems.models import ProblemCase, ProblemVersion
|
||||||
|
from kf_oj.modules.problems.public import JudgingVersion, ProblemUnavailable
|
||||||
|
|
||||||
|
|
||||||
|
def admit_version(policy, manifest, version_id, problem_id, cases):
|
||||||
|
legacy = isinstance(manifest, ProblemImport)
|
||||||
|
projection = JudgingVersion(
|
||||||
|
version_id,
|
||||||
|
problem_id,
|
||||||
|
manifest.languages,
|
||||||
|
manifest.toolchain if legacy else None,
|
||||||
|
manifest.compile_limits if legacy else None,
|
||||||
|
manifest.run_limits if legacy else None,
|
||||||
|
manifest.job_wall_time_us if legacy else None,
|
||||||
|
cases,
|
||||||
|
manifest.schema_version,
|
||||||
|
() if legacy else manifest.language_configurations,
|
||||||
|
)
|
||||||
|
for language in manifest.languages:
|
||||||
|
v2.admit_manifest(
|
||||||
|
policy,
|
||||||
|
projection.manifest(
|
||||||
|
uuid4(),
|
||||||
|
uuid4(),
|
||||||
|
language,
|
||||||
|
ArtifactReference(digest="0" * 64, bytes=MAX_SOURCE_BYTES),
|
||||||
|
),
|
||||||
|
1,
|
||||||
|
)
|
||||||
|
return projection
|
||||||
|
|
||||||
|
|
||||||
|
async def publish_version(session, problem, manifest, preview, version_id, cases):
|
||||||
|
"""Caller holds the logical slug/row locks and has made all artifacts durable."""
|
||||||
|
if not problem.enabled:
|
||||||
|
raise ProblemUnavailable("problem disabled")
|
||||||
|
legacy = isinstance(manifest, ProblemImport)
|
||||||
|
version = ProblemVersion(
|
||||||
|
id=version_id,
|
||||||
|
problem_id=problem.id,
|
||||||
|
revision=await repository.next_revision(session, problem.id),
|
||||||
|
title=manifest.title,
|
||||||
|
statement=preview.statement,
|
||||||
|
languages=[lang.value for lang in manifest.languages],
|
||||||
|
judging_schema_version=manifest.schema_version,
|
||||||
|
language_configurations=None
|
||||||
|
if legacy
|
||||||
|
else [c.model_dump(mode="json") for c in manifest.language_configurations],
|
||||||
|
toolchain=manifest.toolchain.model_dump(mode="json") if legacy else None,
|
||||||
|
compile_limits=manifest.compile_limits.model_dump(mode="json") if legacy else None,
|
||||||
|
run_limits=manifest.run_limits.model_dump(mode="json") if legacy else None,
|
||||||
|
job_wall_time_us=manifest.job_wall_time_us if legacy else None,
|
||||||
|
comparison=manifest.comparison,
|
||||||
|
public_samples=[s.model_dump(mode="json") for s in preview.samples],
|
||||||
|
)
|
||||||
|
session.add(version)
|
||||||
|
await session.flush()
|
||||||
|
for position, case in enumerate(cases):
|
||||||
|
session.add(
|
||||||
|
ProblemCase(
|
||||||
|
id=case.case_id,
|
||||||
|
version_id=version_id,
|
||||||
|
position=position,
|
||||||
|
input_digest=case.input.digest,
|
||||||
|
expected_digest=case.expected.digest,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
await session.flush()
|
||||||
|
version.published_at = await session.scalar(select(func.now()))
|
||||||
|
problem.active_version_id = version_id
|
||||||
|
await session.flush()
|
||||||
|
return version
|
||||||
@@ -24,7 +24,22 @@ async def next_revision(session: AsyncSession, problem_id: UUID) -> int:
|
|||||||
return (latest or 0) + 1
|
return (latest or 0) + 1
|
||||||
|
|
||||||
|
|
||||||
async def public_version(session: AsyncSession, version_id: UUID):
|
async def published_version(session: AsyncSession, version_id: UUID, *, admission=False):
|
||||||
|
query = (
|
||||||
|
select(ProblemVersion, Problem)
|
||||||
|
.join(Problem, Problem.id == ProblemVersion.problem_id)
|
||||||
|
.where(ProblemVersion.id == version_id, ProblemVersion.published_at.is_not(None))
|
||||||
|
)
|
||||||
|
if admission:
|
||||||
|
# A new submission commits while this shared problem lock is held. A
|
||||||
|
# concurrent disable either precedes admission or follows that commit.
|
||||||
|
query = query.where(Problem.enabled.is_(True)).with_for_update(read=True, of=Problem)
|
||||||
|
return (await session.execute(query)).one_or_none()
|
||||||
|
|
||||||
|
|
||||||
|
async def public_version(session: AsyncSession, version_id: UUID, *, admission=False):
|
||||||
|
if admission:
|
||||||
|
return await published_version(session, version_id, admission=True)
|
||||||
return (
|
return (
|
||||||
await session.execute(
|
await session.execute(
|
||||||
select(ProblemVersion, Problem)
|
select(ProblemVersion, Problem)
|
||||||
|
|||||||
@@ -1,8 +1,9 @@
|
|||||||
import base64
|
import base64
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
from uuid import UUID, uuid4, uuid5
|
from uuid import UUID, uuid4, uuid5
|
||||||
|
|
||||||
from sqlalchemy import func, select
|
from sqlalchemy import select
|
||||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||||
|
|
||||||
from kf_oj.contracts import judging_v2 as v2
|
from kf_oj.contracts import judging_v2 as v2
|
||||||
@@ -28,8 +29,9 @@ from kf_oj.infra.artifacts.repository import ArtifactRepository
|
|||||||
from kf_oj.modules.problems import repository
|
from kf_oj.modules.problems import repository
|
||||||
from kf_oj.modules.problems.import_format import ImportEnvelope, ProblemImport
|
from kf_oj.modules.problems.import_format import ImportEnvelope, ProblemImport
|
||||||
from kf_oj.modules.problems.local_files import LocalImportFiles
|
from kf_oj.modules.problems.local_files import LocalImportFiles
|
||||||
from kf_oj.modules.problems.models import Problem, ProblemCase, ProblemVersion
|
from kf_oj.modules.problems.models import Problem
|
||||||
from kf_oj.modules.problems.public import JudgingVersion, ProblemUnavailable
|
from kf_oj.modules.problems.public import JudgingVersion, ProblemUnavailable
|
||||||
|
from kf_oj.modules.problems.publication import admit_version, publish_version
|
||||||
|
|
||||||
|
|
||||||
class ProblemService:
|
class ProblemService:
|
||||||
@@ -83,27 +85,7 @@ class ProblemService:
|
|||||||
)
|
)
|
||||||
for case, (data, answer) in zip(manifest.cases, payloads, strict=True)
|
for case, (data, answer) in zip(manifest.cases, payloads, strict=True)
|
||||||
)
|
)
|
||||||
legacy = isinstance(manifest, ProblemImport)
|
admit_version(self.policy, manifest, version_id, uuid4(), cases)
|
||||||
projection = JudgingVersion(
|
|
||||||
version_id,
|
|
||||||
uuid4(),
|
|
||||||
manifest.languages,
|
|
||||||
manifest.toolchain if legacy else None,
|
|
||||||
manifest.compile_limits if legacy else None,
|
|
||||||
manifest.run_limits if legacy else None,
|
|
||||||
manifest.job_wall_time_us if legacy else None,
|
|
||||||
cases,
|
|
||||||
manifest.schema_version,
|
|
||||||
() if legacy else manifest.language_configurations,
|
|
||||||
)
|
|
||||||
for language in manifest.languages:
|
|
||||||
check = projection.manifest(
|
|
||||||
uuid4(),
|
|
||||||
uuid4(),
|
|
||||||
language,
|
|
||||||
ArtifactReference(digest="0" * 64, bytes=MAX_SOURCE_BYTES),
|
|
||||||
)
|
|
||||||
v2.admit_manifest(self.policy, check, 1)
|
|
||||||
async with self.sessions.begin() as session:
|
async with self.sessions.begin() as session:
|
||||||
# Lock existing and not-yet-existing logical slugs before assigning revisions.
|
# Lock existing and not-yet-existing logical slugs before assigning revisions.
|
||||||
await repository.lock_slug(session, manifest.slug)
|
await repository.lock_slug(session, manifest.slug)
|
||||||
@@ -114,32 +96,6 @@ class ProblemService:
|
|||||||
await session.flush()
|
await session.flush()
|
||||||
if not problem.enabled:
|
if not problem.enabled:
|
||||||
raise ProblemUnavailable("problem disabled")
|
raise ProblemUnavailable("problem disabled")
|
||||||
revision = await repository.next_revision(session, problem.id)
|
|
||||||
session.add(
|
|
||||||
ProblemVersion(
|
|
||||||
id=version_id,
|
|
||||||
problem_id=problem.id,
|
|
||||||
revision=revision,
|
|
||||||
title=manifest.title,
|
|
||||||
statement=statement,
|
|
||||||
languages=[lang.value for lang in manifest.languages],
|
|
||||||
judging_schema_version=manifest.schema_version,
|
|
||||||
language_configurations=(
|
|
||||||
None
|
|
||||||
if legacy
|
|
||||||
else [c.model_dump(mode="json") for c in manifest.language_configurations]
|
|
||||||
),
|
|
||||||
toolchain=manifest.toolchain.model_dump(mode="json") if legacy else None,
|
|
||||||
compile_limits=manifest.compile_limits.model_dump(mode="json")
|
|
||||||
if legacy
|
|
||||||
else None,
|
|
||||||
run_limits=manifest.run_limits.model_dump(mode="json") if legacy else None,
|
|
||||||
job_wall_time_us=manifest.job_wall_time_us if legacy else None,
|
|
||||||
comparison=manifest.comparison,
|
|
||||||
public_samples=[s.model_dump() for s in samples],
|
|
||||||
)
|
|
||||||
)
|
|
||||||
await session.flush()
|
|
||||||
# Consistent digest lock order avoids deadlocks between overlapping imports/jobs.
|
# Consistent digest lock order avoids deadlocks between overlapping imports/jobs.
|
||||||
unique = {
|
unique = {
|
||||||
ref.digest: (data, ref)
|
ref.digest: (data, ref)
|
||||||
@@ -149,20 +105,14 @@ class ProblemService:
|
|||||||
for digest in sorted(unique):
|
for digest in sorted(unique):
|
||||||
data, reference = unique[digest]
|
data, reference = unique[digest]
|
||||||
await self.artifacts.put(session, data, reference)
|
await self.artifacts.put(session, data, reference)
|
||||||
for position, case in enumerate(cases):
|
await publish_version(
|
||||||
session.add(
|
session,
|
||||||
ProblemCase(
|
problem,
|
||||||
id=case.case_id,
|
manifest,
|
||||||
version_id=version_id,
|
SimpleNamespace(statement=statement, samples=samples),
|
||||||
position=position,
|
version_id,
|
||||||
input_digest=case.input.digest,
|
cases,
|
||||||
expected_digest=case.expected.digest,
|
)
|
||||||
)
|
|
||||||
)
|
|
||||||
await session.flush()
|
|
||||||
version = await session.get(ProblemVersion, version_id)
|
|
||||||
version.published_at = await session.scalar(select(func.now()))
|
|
||||||
problem.active_version_id = version_id
|
|
||||||
return version_id
|
return version_id
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -213,7 +163,16 @@ class ProblemService:
|
|||||||
async def for_submission(self, session, version_id: UUID) -> JudgingVersion:
|
async def for_submission(self, session, version_id: UUID) -> JudgingVersion:
|
||||||
if self.artifacts is None:
|
if self.artifacts is None:
|
||||||
raise ProblemUnavailable("artifact service unavailable")
|
raise ProblemUnavailable("artifact service unavailable")
|
||||||
row = await repository.public_version(session, version_id)
|
row = await repository.public_version(session, version_id, admission=True)
|
||||||
|
return await self._judging_version(session, row)
|
||||||
|
|
||||||
|
async def for_accepted_execution(self, session, version_id: UUID) -> JudgingVersion:
|
||||||
|
# Only WorkerAccess calls this after validating the current lease and its
|
||||||
|
# immutable submission/attempt binding; this is never a public UUID lookup.
|
||||||
|
row = await repository.published_version(session, version_id)
|
||||||
|
return await self._judging_version(session, row)
|
||||||
|
|
||||||
|
async def _judging_version(self, session, row):
|
||||||
if row is None:
|
if row is None:
|
||||||
raise ProblemUnavailable("problem version unavailable")
|
raise ProblemUnavailable("problem version unavailable")
|
||||||
version, problem = row
|
version, problem = row
|
||||||
|
|||||||
@@ -0,0 +1,31 @@
|
|||||||
|
"""Stricter Web package declarations; the accepted local CLI format remains unchanged."""
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
from kf_oj.contracts.judging import decode_wire
|
||||||
|
from kf_oj.modules.problems.import_format import ImportEnvelope, ProblemImport, ProblemImportV2
|
||||||
|
|
||||||
|
ARCHIVE_SUFFIXES = (".zip", ".jar", ".war", ".tar", ".tgz", ".gz", ".bz2", ".xz", ".7z", ".rar")
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class WebDeclaration:
|
||||||
|
manifest: ProblemImport | ProblemImportV2
|
||||||
|
members: frozenset[str]
|
||||||
|
|
||||||
|
|
||||||
|
def decode_web_declaration(data: bytes) -> WebDeclaration:
|
||||||
|
# decode_wire rejects duplicate JSON keys, unknown fields and noncanonical
|
||||||
|
# protocol values before either local paths or node policy can be considered.
|
||||||
|
manifest = decode_wire(ImportEnvelope, data).root
|
||||||
|
hidden = {name for c in manifest.cases for name in (c.input_file, c.expected_file)}
|
||||||
|
samples = {name for s in manifest.samples for name in (s.input_file, s.output_file)}
|
||||||
|
metadata = {"problem.json", manifest.statement_file}
|
||||||
|
if len(metadata) != 2 or hidden & (samples | metadata) or samples & metadata:
|
||||||
|
raise ValueError("public and hidden package declarations must be separate")
|
||||||
|
members = frozenset(metadata | hidden | samples)
|
||||||
|
if len({name.lower() for name in members}) != len(members):
|
||||||
|
raise ValueError("package names must be case-insensitively unique")
|
||||||
|
if any(name.lower().endswith(ARCHIVE_SUFFIXES) for name in members):
|
||||||
|
raise ValueError("nested archive declarations are unsupported")
|
||||||
|
return WebDeclaration(manifest, members)
|
||||||
@@ -23,3 +23,28 @@ submission、queued evaluation 和 outbox;同键同内容复用既有提交,
|
|||||||
Coordinator 通过正式 claim 创建短期领取权,Worker 不写业务数据库。
|
Coordinator 通过正式 claim 创建短期领取权,Worker 不写业务数据库。
|
||||||
真实服务链与安全条件必须在独立 VM 验收;数据库 fixture 的租约或合成报告只能
|
真实服务链与安全条件必须在独立 VM 验收;数据库 fixture 的租约或合成报告只能
|
||||||
验证控制规则、授权和投影,不能作为真实程序执行/停止证明。
|
验证控制规则、授权和投影,不能作为真实程序执行/停止证明。
|
||||||
|
|
||||||
|
`performance.py` 提供本人提交的只读匿名聚合。Coordinator 在受验证最终 AC 的同一事务
|
||||||
|
写入不可变 `evaluation_performance`;平台只有 SELECT 权限。比较组绑定不可变版本、
|
||||||
|
语言、精确镜像/工具链、编译与运行预算,以及运维核验后配置的测量环境纪元。
|
||||||
|
没有原纪元的旧尝试不会通过当前配置被补造可信性,仍保留个人历史与正常成绩。
|
||||||
|
单条受控回填与纪元变更规则见[协调器说明](../../../../../judge/crates/coordinator/README.md)。
|
||||||
|
|
||||||
|
历史可按 `problem_version_id` 与可选 wire `language` 过滤,始终带当前账号条件。
|
||||||
|
`GET /api/v1/submissions/{submission_id}/performance` 每次先检查会话和所有权;
|
||||||
|
匿名拒绝、其他主体与不存在记录同为不可用,私有响应为 `no-store`。
|
||||||
|
缓存命中仍查归属。每进程最多一个聚合查询,不排等待队列;SQL 期限两秒、请求总期限
|
||||||
|
三秒、锁等待至多半秒。聚合 DTO 缓存三十秒且最多128项,新 AC 的索引水位改变即失效。
|
||||||
|
缓存不保存逐条样本数组,不向浏览器发送其他账号的身份、源码或逐条用量。
|
||||||
|
|
||||||
|
每指标在同一个只读可重复读快照内,排除当前提交,取最近最多10,000条非缺测的其他
|
||||||
|
有效 AC,响应带快照、最早样本及截断标志。少于五条时仅显示本次真实用量。
|
||||||
|
CPU 为测试点运行 CPU 微秒之和,内存为测试点运行峰值字节的最大值,不包括编译。
|
||||||
|
用量与桶边界以十进制字符串传输;零合法、缺测不补零。严格优于数按原整数的 `>`
|
||||||
|
计算,同值独立计数。响应至多24个连续桶,只有最后上界包含端点;浏览器仅格式化单位
|
||||||
|
和绘制坐标,比例以服务端严格计数除以样本数,不按人数统计。
|
||||||
|
|
||||||
|
下架只阻止新提交与公开题目读取。Worker 的版本读取通过 `ProblemPort.for_accepted_execution`,
|
||||||
|
在本模块验证当前 Coordinator 租约/尝试/源码绑定之后才调用;生成并比较原 canonical
|
||||||
|
快照,再次核对租约后才释放字节。无有效租约、外来工件、未发布版本和其他提交仍拒绝。
|
||||||
|
本人历史和源码读取保留所有权检查,不依赖题目当前 enabled 状态。
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ from sqlalchemy import (
|
|||||||
String,
|
String,
|
||||||
UniqueConstraint,
|
UniqueConstraint,
|
||||||
func,
|
func,
|
||||||
|
text,
|
||||||
)
|
)
|
||||||
from sqlalchemy.dialects.postgresql import JSONB
|
from sqlalchemy.dialects.postgresql import JSONB
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
@@ -38,6 +39,13 @@ class Submission(CreatedAt, Base):
|
|||||||
["source_digest", "artifact_state"], ["artifacts.digest", "artifacts.state"]
|
["source_digest", "artifact_state"], ["artifacts.digest", "artifacts.state"]
|
||||||
),
|
),
|
||||||
Index("ix_submissions_account_history", "account_id", "created_at", "id"),
|
Index("ix_submissions_account_history", "account_id", "created_at", "id"),
|
||||||
|
Index(
|
||||||
|
"ix_submissions_owned_problem_history",
|
||||||
|
"account_id",
|
||||||
|
"problem_version_id",
|
||||||
|
text("created_at DESC"),
|
||||||
|
text("id DESC"),
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
id: Mapped[UUID] = mapped_column(primary_key=True)
|
id: Mapped[UUID] = mapped_column(primary_key=True)
|
||||||
@@ -112,6 +120,10 @@ class JudgeAttempt(CreatedAt, Base):
|
|||||||
CheckConstraint("fencing_version > 0", name="fence"),
|
CheckConstraint("fencing_version > 0", name="fence"),
|
||||||
CheckConstraint("lease_token_hash ~ '^[0-9a-f]{64}$'", name="token_digest"),
|
CheckConstraint("lease_token_hash ~ '^[0-9a-f]{64}$'", name="token_digest"),
|
||||||
CheckConstraint("manifest_digest ~ '^[0-9a-f]{64}$'", name="manifest_digest"),
|
CheckConstraint("manifest_digest ~ '^[0-9a-f]{64}$'", name="manifest_digest"),
|
||||||
|
CheckConstraint(
|
||||||
|
"measurement_epoch IS NULL OR measurement_epoch ~ '^[0-9a-f]{64}$'",
|
||||||
|
name="measurement_epoch",
|
||||||
|
),
|
||||||
CheckConstraint("sequence >= 0", name="sequence"),
|
CheckConstraint("sequence >= 0", name="sequence"),
|
||||||
Index("ix_judge_attempts_expiry", "expires_at", "evaluation_id"),
|
Index("ix_judge_attempts_expiry", "expires_at", "evaluation_id"),
|
||||||
)
|
)
|
||||||
@@ -132,6 +144,64 @@ class JudgeAttempt(CreatedAt, Base):
|
|||||||
report: Mapped[dict | None] = mapped_column(JSONB)
|
report: Mapped[dict | None] = mapped_column(JSONB)
|
||||||
report_digest: Mapped[str | None] = mapped_column(String(64))
|
report_digest: Mapped[str | None] = mapped_column(String(64))
|
||||||
cleanup_complete: Mapped[bool | None] = mapped_column(Boolean)
|
cleanup_complete: Mapped[bool | None] = mapped_column(Boolean)
|
||||||
|
measurement_epoch: Mapped[str | None] = mapped_column(String(64))
|
||||||
|
|
||||||
|
|
||||||
|
class EvaluationPerformance(Base):
|
||||||
|
"""Coordinator-only immutable final projection; the platform only reads it."""
|
||||||
|
|
||||||
|
__tablename__ = "evaluation_performance"
|
||||||
|
__table_args__ = (
|
||||||
|
ForeignKeyConstraint(
|
||||||
|
["evaluation_id", "problem_version_id"],
|
||||||
|
["evaluations.id", "evaluations.problem_version_id"],
|
||||||
|
),
|
||||||
|
ForeignKeyConstraint(
|
||||||
|
["submission_id", "problem_version_id"],
|
||||||
|
["submissions.id", "submissions.problem_version_id"],
|
||||||
|
),
|
||||||
|
ForeignKeyConstraint(
|
||||||
|
["attempt_id", "evaluation_id"], ["judge_attempts.id", "judge_attempts.evaluation_id"]
|
||||||
|
),
|
||||||
|
CheckConstraint("language IN ('c17','cpp20','python313','java21')", name="language"),
|
||||||
|
CheckConstraint("comparison_group ~ '^[0-9a-f]{64}$'", name="comparison_group"),
|
||||||
|
CheckConstraint("measurement_epoch ~ '^[0-9a-f]{64}$'", name="measurement_epoch"),
|
||||||
|
CheckConstraint("report_digest ~ '^[0-9a-f]{64}$'", name="report_digest"),
|
||||||
|
CheckConstraint("cpu_time_us IS NULL OR cpu_time_us >= 0", name="cpu"),
|
||||||
|
CheckConstraint("memory_peak_bytes IS NULL OR memory_peak_bytes >= 0", name="memory"),
|
||||||
|
Index(
|
||||||
|
"ix_performance_group_watermark",
|
||||||
|
"comparison_group",
|
||||||
|
text("finished_at DESC"),
|
||||||
|
text("evaluation_id DESC"),
|
||||||
|
),
|
||||||
|
Index(
|
||||||
|
"ix_performance_cpu_time_us_cohort",
|
||||||
|
"comparison_group",
|
||||||
|
text("finished_at DESC"),
|
||||||
|
text("evaluation_id DESC"),
|
||||||
|
postgresql_where=text("cpu_time_us IS NOT NULL"),
|
||||||
|
),
|
||||||
|
Index(
|
||||||
|
"ix_performance_memory_peak_bytes_cohort",
|
||||||
|
"comparison_group",
|
||||||
|
text("finished_at DESC"),
|
||||||
|
text("evaluation_id DESC"),
|
||||||
|
postgresql_where=text("memory_peak_bytes IS NOT NULL"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
evaluation_id: Mapped[UUID] = mapped_column(primary_key=True)
|
||||||
|
submission_id: Mapped[UUID] = mapped_column(unique=True)
|
||||||
|
attempt_id: Mapped[UUID] = mapped_column(unique=True)
|
||||||
|
problem_version_id: Mapped[UUID]
|
||||||
|
language: Mapped[str] = mapped_column(String(16))
|
||||||
|
comparison_group: Mapped[str] = mapped_column(String(64))
|
||||||
|
measurement_epoch: Mapped[str] = mapped_column(String(64))
|
||||||
|
report_digest: Mapped[str] = mapped_column(String(64))
|
||||||
|
finished_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||||
|
cpu_time_us: Mapped[int | None] = mapped_column(BigInteger)
|
||||||
|
memory_peak_bytes: Mapped[int | None] = mapped_column(BigInteger)
|
||||||
|
|
||||||
|
|
||||||
class CaseResult(Base):
|
class CaseResult(Base):
|
||||||
|
|||||||
@@ -0,0 +1,217 @@
|
|||||||
|
"""Read-only, bounded anonymous cohorts from Coordinator's immutable projection."""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import time
|
||||||
|
from collections import OrderedDict
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from sqlalchemy import select, text
|
||||||
|
from sqlalchemy.exc import SQLAlchemyError
|
||||||
|
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||||
|
|
||||||
|
from kf_oj.contracts.judging import I64_MAX
|
||||||
|
from kf_oj.contracts.judging_versions import decode_report
|
||||||
|
from kf_oj.contracts.submission_performance import (
|
||||||
|
MAX_PERFORMANCE_SAMPLES,
|
||||||
|
MIN_PERFORMANCE_SAMPLES,
|
||||||
|
PerformanceBucket,
|
||||||
|
PerformanceMetric,
|
||||||
|
SubmissionPerformance,
|
||||||
|
)
|
||||||
|
from kf_oj.modules.identity.public import Principal
|
||||||
|
from kf_oj.modules.submissions.models import Evaluation, EvaluationPerformance, Submission
|
||||||
|
from kf_oj.modules.submissions.public import SubmissionUnavailable
|
||||||
|
|
||||||
|
CACHE_TTL_SECONDS = 30
|
||||||
|
CACHE_CAPACITY = 128
|
||||||
|
HISTOGRAM_BUCKETS = 24
|
||||||
|
|
||||||
|
|
||||||
|
def metric(unit, current, rows=(), *, reason=None) -> PerformanceMetric:
|
||||||
|
"""Integer arithmetic throughout; conversion for display belongs to the client."""
|
||||||
|
truncated = len(rows) > MAX_PERFORMANCE_SAMPLES
|
||||||
|
rows = rows[:MAX_PERFORMANCE_SAMPLES]
|
||||||
|
count = len(rows)
|
||||||
|
base = dict(
|
||||||
|
unit=unit,
|
||||||
|
current=None if current is None else str(current),
|
||||||
|
sample_count=count,
|
||||||
|
window_truncated=truncated,
|
||||||
|
oldest_sample_at=rows[-1][1] if rows else None,
|
||||||
|
)
|
||||||
|
if reason is None:
|
||||||
|
reason = (
|
||||||
|
"measurement_missing"
|
||||||
|
if current is None
|
||||||
|
else ("insufficient_samples" if count < MIN_PERFORMANCE_SAMPLES else None)
|
||||||
|
)
|
||||||
|
if reason is not None:
|
||||||
|
return PerformanceMetric(
|
||||||
|
**base,
|
||||||
|
available=False,
|
||||||
|
reason=reason,
|
||||||
|
strictly_slower_count=None,
|
||||||
|
tied_count=None,
|
||||||
|
buckets=(),
|
||||||
|
)
|
||||||
|
values = [row[0] for row in rows]
|
||||||
|
end = max(1, current, max(values))
|
||||||
|
step = (end + HISTOGRAM_BUCKETS - 1) // HISTOGRAM_BUCKETS
|
||||||
|
# Last upper bound includes the maximum, including I64_MAX. No overflow/padding.
|
||||||
|
boundaries = list(range(0, end, step)) + [end]
|
||||||
|
counts = [0] * (len(boundaries) - 1)
|
||||||
|
for value in values:
|
||||||
|
counts[min(value // step, len(counts) - 1)] += 1
|
||||||
|
return PerformanceMetric(
|
||||||
|
**base,
|
||||||
|
available=True,
|
||||||
|
reason=None,
|
||||||
|
strictly_slower_count=sum(value > current for value in values),
|
||||||
|
tied_count=sum(value == current for value in values),
|
||||||
|
buckets=tuple(
|
||||||
|
PerformanceBucket(
|
||||||
|
start=str(start),
|
||||||
|
end=str(stop),
|
||||||
|
upper_inclusive=index == len(counts) - 1,
|
||||||
|
count=counts[index],
|
||||||
|
)
|
||||||
|
for index, (start, stop) in enumerate(zip(boundaries, boundaries[1:], strict=False))
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class SubmissionPerformanceQueries:
|
||||||
|
def __init__(self, sessions: async_sessionmaker):
|
||||||
|
self.sessions = sessions
|
||||||
|
# No queued aggregate workload. At most one bounded in-memory row cohort per process.
|
||||||
|
self._slot = asyncio.Lock()
|
||||||
|
# Cache contains small immutable aggregate DTOs, never 10,000-row arrays.
|
||||||
|
self._cache: OrderedDict[tuple, tuple[float, SubmissionPerformance]] = OrderedDict()
|
||||||
|
|
||||||
|
async def detail(self, principal: Principal, submission_id: UUID) -> SubmissionPerformance:
|
||||||
|
if self._slot.locked():
|
||||||
|
raise ValueError("performance query capacity unavailable")
|
||||||
|
async with self._slot:
|
||||||
|
try:
|
||||||
|
async with asyncio.timeout(3), self.sessions() as session, session.begin():
|
||||||
|
await session.execute(
|
||||||
|
text("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY")
|
||||||
|
)
|
||||||
|
await session.execute(text("SET LOCAL statement_timeout='2s'"))
|
||||||
|
await session.execute(text("SET LOCAL lock_timeout='500ms'"))
|
||||||
|
# Ownership checked even on a cache hit. Session authentication is upstream.
|
||||||
|
row = (
|
||||||
|
await session.execute(
|
||||||
|
select(Submission, Evaluation, EvaluationPerformance)
|
||||||
|
.join(Evaluation, Evaluation.submission_id == Submission.id)
|
||||||
|
.outerjoin(
|
||||||
|
EvaluationPerformance,
|
||||||
|
EvaluationPerformance.evaluation_id == Evaluation.id,
|
||||||
|
)
|
||||||
|
.where(
|
||||||
|
Submission.id == submission_id,
|
||||||
|
Submission.account_id == principal.account_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
).one_or_none()
|
||||||
|
if row is None:
|
||||||
|
raise SubmissionUnavailable("submission unavailable")
|
||||||
|
submission, evaluation, projection = row
|
||||||
|
as_of = await session.scalar(text("SELECT clock_timestamp()"))
|
||||||
|
accepted = (
|
||||||
|
evaluation.state == "finished"
|
||||||
|
and evaluation.verdict == "accepted"
|
||||||
|
and evaluation.failure_code is None
|
||||||
|
)
|
||||||
|
if projection is not None and (
|
||||||
|
not accepted
|
||||||
|
or projection.attempt_id != evaluation.current_attempt_id
|
||||||
|
or projection.report_digest != evaluation.final_report_digest
|
||||||
|
):
|
||||||
|
raise ValueError("stored performance binding invalid")
|
||||||
|
cpu = memory = None
|
||||||
|
reason = None if accepted else "not_accepted"
|
||||||
|
if projection is not None:
|
||||||
|
cpu, memory = projection.cpu_time_us, projection.memory_peak_bytes
|
||||||
|
elif accepted:
|
||||||
|
# A single owned legacy report, never a full-table JSON scan or promotion.
|
||||||
|
report = decode_report(
|
||||||
|
json.dumps(evaluation.final_report, ensure_ascii=False).encode()
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
report.binding.evaluation_id != evaluation.id
|
||||||
|
or report.binding.attempt_id != evaluation.current_attempt_id
|
||||||
|
or report.verdict.value != "accepted"
|
||||||
|
or not report.cleanup_complete
|
||||||
|
):
|
||||||
|
raise ValueError("stored result binding invalid")
|
||||||
|
cpu, memory = report.run_cpu_time_us, report.run_memory_peak_bytes
|
||||||
|
reason = "environment_unverified"
|
||||||
|
if any(v is not None and not 0 <= v <= I64_MAX for v in (cpu, memory)):
|
||||||
|
raise ValueError("stored measurement out of range")
|
||||||
|
base = dict(
|
||||||
|
submission_id=submission.id,
|
||||||
|
problem_version_id=submission.problem_version_id,
|
||||||
|
language=submission.language,
|
||||||
|
as_of=as_of,
|
||||||
|
comparison_group=None
|
||||||
|
if projection is None
|
||||||
|
else projection.comparison_group,
|
||||||
|
)
|
||||||
|
if projection is None:
|
||||||
|
return SubmissionPerformance(
|
||||||
|
**base,
|
||||||
|
cpu=metric("cpu_microseconds", cpu, reason=reason),
|
||||||
|
memory=metric("bytes", memory, reason=reason),
|
||||||
|
)
|
||||||
|
group = projection.comparison_group
|
||||||
|
watermark = (
|
||||||
|
await session.execute(
|
||||||
|
select(
|
||||||
|
EvaluationPerformance.finished_at,
|
||||||
|
EvaluationPerformance.evaluation_id,
|
||||||
|
)
|
||||||
|
.where(EvaluationPerformance.comparison_group == group)
|
||||||
|
.order_by(
|
||||||
|
EvaluationPerformance.finished_at.desc(),
|
||||||
|
EvaluationPerformance.evaluation_id.desc(),
|
||||||
|
)
|
||||||
|
.limit(1)
|
||||||
|
)
|
||||||
|
).one()
|
||||||
|
key = (submission.id, projection.report_digest, group, tuple(watermark))
|
||||||
|
cached = self._cache.get(key)
|
||||||
|
now = time.monotonic()
|
||||||
|
if cached is not None and now - cached[0] < CACHE_TTL_SECONDS:
|
||||||
|
self._cache.move_to_end(key)
|
||||||
|
return cached[1]
|
||||||
|
metrics = []
|
||||||
|
for column, unit, current in [
|
||||||
|
(EvaluationPerformance.cpu_time_us, "cpu_microseconds", cpu),
|
||||||
|
(EvaluationPerformance.memory_peak_bytes, "bytes", memory),
|
||||||
|
]:
|
||||||
|
rows = (
|
||||||
|
await session.execute(
|
||||||
|
select(column, EvaluationPerformance.finished_at)
|
||||||
|
.where(
|
||||||
|
EvaluationPerformance.comparison_group == group,
|
||||||
|
EvaluationPerformance.evaluation_id != evaluation.id,
|
||||||
|
column.is_not(None),
|
||||||
|
)
|
||||||
|
.order_by(
|
||||||
|
EvaluationPerformance.finished_at.desc(),
|
||||||
|
EvaluationPerformance.evaluation_id.desc(),
|
||||||
|
)
|
||||||
|
.limit(MAX_PERFORMANCE_SAMPLES + 1)
|
||||||
|
)
|
||||||
|
).all()
|
||||||
|
metrics.append(metric(unit, current, rows))
|
||||||
|
result = SubmissionPerformance(**base, cpu=metrics[0], memory=metrics[1])
|
||||||
|
self._cache[key] = (now, result)
|
||||||
|
self._cache.move_to_end(key)
|
||||||
|
while len(self._cache) > CACHE_CAPACITY:
|
||||||
|
self._cache.popitem(last=False)
|
||||||
|
return result
|
||||||
|
except (SQLAlchemyError, TimeoutError) as error:
|
||||||
|
raise ValueError("performance database unavailable") from error
|
||||||
@@ -88,7 +88,14 @@ class SubmissionQueries:
|
|||||||
.where(Submission.id == submission_id, Submission.account_id == principal.account_id)
|
.where(Submission.id == submission_id, Submission.account_id == principal.account_id)
|
||||||
)
|
)
|
||||||
|
|
||||||
async def page(self, principal: Principal, after: str | None, limit: int) -> SubmissionPage:
|
async def page(
|
||||||
|
self,
|
||||||
|
principal: Principal,
|
||||||
|
after: str | None,
|
||||||
|
limit: int,
|
||||||
|
problem_version_id: UUID | None = None,
|
||||||
|
language: str | None = None,
|
||||||
|
) -> SubmissionPage:
|
||||||
if not 1 <= limit <= 100:
|
if not 1 <= limit <= 100:
|
||||||
raise ValueError("invalid history limit")
|
raise ValueError("invalid history limit")
|
||||||
query = (
|
query = (
|
||||||
@@ -98,6 +105,10 @@ class SubmissionQueries:
|
|||||||
.order_by(Submission.created_at.desc(), Submission.id.desc())
|
.order_by(Submission.created_at.desc(), Submission.id.desc())
|
||||||
.limit(limit + 1)
|
.limit(limit + 1)
|
||||||
)
|
)
|
||||||
|
if problem_version_id is not None:
|
||||||
|
query = query.where(Submission.problem_version_id == problem_version_id)
|
||||||
|
if language is not None:
|
||||||
|
query = query.where(Submission.language == language)
|
||||||
if after is not None:
|
if after is not None:
|
||||||
query = query.where(tuple_(Submission.created_at, Submission.id) < cursor_decode(after))
|
query = query.where(tuple_(Submission.created_at, Submission.id) < cursor_decode(after))
|
||||||
async with self.sessions() as session:
|
async with self.sessions() as session:
|
||||||
|
|||||||
@@ -62,7 +62,9 @@ class WorkerAccess:
|
|||||||
)
|
)
|
||||||
):
|
):
|
||||||
raise WorkerAccessDenied("worker snapshot rejected")
|
raise WorkerAccessDenied("worker snapshot rejected")
|
||||||
version = await self.problems.for_submission(session, manifest.problem_version_id)
|
version = await self.problems.for_accepted_execution(
|
||||||
|
session, manifest.problem_version_id
|
||||||
|
)
|
||||||
await self.artifacts.check_metadata(session, manifest.source)
|
await self.artifacts.check_metadata(session, manifest.source)
|
||||||
expected = version.manifest(
|
expected = version.manifest(
|
||||||
manifest.evaluation_id, manifest.submission_id, manifest.language, manifest.source
|
manifest.evaluation_id, manifest.submission_id, manifest.language, manifest.source
|
||||||
|
|||||||
@@ -0,0 +1,116 @@
|
|||||||
|
"""Fixed data-only child. It inherits no DB credentials and executes no uploaded contents."""
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import resource
|
||||||
|
import signal
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from pydantic import TypeAdapter
|
||||||
|
|
||||||
|
from kf_oj.contracts.admin_problems import (
|
||||||
|
VALIDATION_DEADLINE_SECONDS,
|
||||||
|
CandidatePreview,
|
||||||
|
LegacyImportBudget,
|
||||||
|
)
|
||||||
|
from kf_oj.contracts.judging_v2 import NodeAdmission
|
||||||
|
from kf_oj.contracts.problems import PublicLanguageConfiguration, PublicSample
|
||||||
|
from kf_oj.infra.artifacts.archive_reader import ArchiveError, ArchiveReader
|
||||||
|
from kf_oj.modules.problems.candidate_validation import validate_candidate
|
||||||
|
from kf_oj.modules.problems.import_format import ProblemImport
|
||||||
|
from kf_oj.modules.problems.web_format import decode_web_declaration
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
# These are ordinary downward-only process limits, independent of the parent's life.
|
||||||
|
resource.setrlimit(resource.RLIMIT_AS, (128 * 1024**2, 128 * 1024**2))
|
||||||
|
resource.setrlimit(resource.RLIMIT_CPU, (60, 60))
|
||||||
|
resource.setrlimit(resource.RLIMIT_FSIZE, (64 * 1024**2, 64 * 1024**2))
|
||||||
|
resource.setrlimit(resource.RLIMIT_NOFILE, (64, 64))
|
||||||
|
resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
|
||||||
|
signal.alarm(VALIDATION_DEADLINE_SECONDS)
|
||||||
|
archive, directory, lock, size, digest, candidate, policy_json = sys.argv[1:]
|
||||||
|
archive, directory, lock = int(archive), int(directory), int(lock)
|
||||||
|
os.fstat(lock) # Held inherited flock prevents GC releasing capacity while any child lives.
|
||||||
|
deadline = time.monotonic() + VALIDATION_DEADLINE_SECONDS
|
||||||
|
result = {"failure_code": "invalid_archive"}
|
||||||
|
try:
|
||||||
|
reader = ArchiveReader(archive, int(size), deadline=deadline)
|
||||||
|
reader.verify_digest(digest)
|
||||||
|
result["failure_code"] = "invalid_manifest"
|
||||||
|
declared = decode_web_declaration(reader.read("problem.json", 256 * 1024))
|
||||||
|
if {m.name for m in reader.members} != set(declared.members):
|
||||||
|
raise ValueError("unexpected package members")
|
||||||
|
manifest = declared.manifest
|
||||||
|
result["failure_code"] = "invalid_data"
|
||||||
|
statement = reader.read(manifest.statement_file, 256 * 1024).decode("utf-8")
|
||||||
|
sample_data = {
|
||||||
|
name: reader.read(name, 65536)
|
||||||
|
for name in sorted({n for s in manifest.samples for n in (s.input_file, s.output_file)})
|
||||||
|
}
|
||||||
|
samples = tuple(
|
||||||
|
PublicSample(
|
||||||
|
input=base64.b64encode(sample_data[s.input_file]).decode("ascii"),
|
||||||
|
output=base64.b64encode(sample_data[s.output_file]).decode("ascii"),
|
||||||
|
)
|
||||||
|
for s in manifest.samples
|
||||||
|
)
|
||||||
|
references = reader.extract_remaining(directory)
|
||||||
|
reader.verify_unchanged()
|
||||||
|
legacy = isinstance(manifest, ProblemImport)
|
||||||
|
preview = CandidatePreview(
|
||||||
|
candidate_id=UUID(candidate),
|
||||||
|
slug=manifest.slug,
|
||||||
|
title=manifest.title,
|
||||||
|
statement=statement,
|
||||||
|
samples=samples,
|
||||||
|
judging_schema_version=manifest.schema_version,
|
||||||
|
languages=manifest.languages,
|
||||||
|
legacy_budget=LegacyImportBudget(
|
||||||
|
compile_limits=manifest.compile_limits,
|
||||||
|
run_limits=manifest.run_limits,
|
||||||
|
job_wall_time_us=manifest.job_wall_time_us,
|
||||||
|
)
|
||||||
|
if legacy
|
||||||
|
else None,
|
||||||
|
language_configurations=()
|
||||||
|
if legacy
|
||||||
|
else tuple(
|
||||||
|
PublicLanguageConfiguration.from_configuration(c)
|
||||||
|
for c in manifest.language_configurations
|
||||||
|
),
|
||||||
|
case_count=len(manifest.cases),
|
||||||
|
)
|
||||||
|
result["failure_code"] = "policy_rejected"
|
||||||
|
validated = validate_candidate(
|
||||||
|
manifest.model_dump(mode="json"),
|
||||||
|
preview.model_dump(mode="json"),
|
||||||
|
{k: {"digest": v[0], "bytes": v[1]} for k, v in references.items()},
|
||||||
|
TypeAdapter(NodeAdmission).validate_json(policy_json),
|
||||||
|
)
|
||||||
|
result = {
|
||||||
|
"manifest": validated.manifest,
|
||||||
|
"preview": validated.preview.model_dump(mode="json"),
|
||||||
|
"references": {k: v.model_dump(mode="json") for k, v in validated.references.items()},
|
||||||
|
"manifest_digest": validated.digest,
|
||||||
|
}
|
||||||
|
except ArchiveError as error:
|
||||||
|
result = {
|
||||||
|
"failure_code": "digest_mismatch"
|
||||||
|
if str(error) == "archive_digest_mismatch"
|
||||||
|
else "invalid_archive"
|
||||||
|
}
|
||||||
|
except (ValueError, OSError, MemoryError):
|
||||||
|
result = {"failure_code": result["failure_code"]}
|
||||||
|
payload = json.dumps(result, separators=(",", ":"), ensure_ascii=True).encode()
|
||||||
|
if len(payload) > 4 * 1024**2:
|
||||||
|
payload = b'{"failure_code":"invalid_data"}'
|
||||||
|
sys.stdout.buffer.write(payload)
|
||||||
|
sys.stdout.buffer.flush()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
"""Read only publisher-owned counts; never claim, publish or acknowledge a task."""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
|
||||||
|
from kf_oj.core.settings import Settings
|
||||||
|
from kf_oj.infra.db.session import build_engine, build_session_factory
|
||||||
|
from kf_oj.infra.messaging.outbox_queries import OutboxQueries
|
||||||
|
|
||||||
|
|
||||||
|
async def run() -> dict:
|
||||||
|
engine = build_engine(Settings())
|
||||||
|
try:
|
||||||
|
return await OutboxQueries(build_session_factory(engine)).status()
|
||||||
|
finally:
|
||||||
|
await engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
try:
|
||||||
|
print(json.dumps(asyncio.run(asyncio.wait_for(run(), timeout=6)), sort_keys=True))
|
||||||
|
except Exception:
|
||||||
|
raise SystemExit("outbox status unavailable") from None
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,74 @@
|
|||||||
|
"""Ordinary import service composition. Provision its dedicated DB role and cgroup separately."""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import signal
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from sqlalchemy.exc import DBAPIError
|
||||||
|
|
||||||
|
from kf_oj.core.settings import Settings
|
||||||
|
from kf_oj.infra.artifacts.repository import ArtifactRepository
|
||||||
|
from kf_oj.infra.artifacts.staging import ImportStaging
|
||||||
|
from kf_oj.infra.artifacts.store import ArtifactStore
|
||||||
|
from kf_oj.infra.db.session import build_engine, build_session_factory
|
||||||
|
from kf_oj.modules.identity.service import ReservedMaintenance
|
||||||
|
from kf_oj.modules.operations.service import MaintenanceAudit
|
||||||
|
from kf_oj.modules.problems.import_worker import ImportWorker, verify_worker_limits
|
||||||
|
|
||||||
|
|
||||||
|
async def run():
|
||||||
|
stop = asyncio.Event()
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
for event in (signal.SIGTERM, signal.SIGINT):
|
||||||
|
loop.add_signal_handler(event, stop.set)
|
||||||
|
settings = Settings()
|
||||||
|
if (
|
||||||
|
settings.import_staging_root is None
|
||||||
|
or settings.artifact_root is None
|
||||||
|
or settings.judge_admission is None
|
||||||
|
or settings.import_worker_cgroup is None
|
||||||
|
):
|
||||||
|
raise ValueError("import service configuration required")
|
||||||
|
verify_worker_limits(settings.import_worker_cgroup)
|
||||||
|
engine = build_engine(settings)
|
||||||
|
store = ArtifactStore(
|
||||||
|
settings.artifact_root,
|
||||||
|
max_bytes=settings.artifact_max_bytes,
|
||||||
|
max_files=settings.artifact_max_files,
|
||||||
|
)
|
||||||
|
staging = ImportStaging(settings.import_staging_root)
|
||||||
|
try:
|
||||||
|
worker = ImportWorker(
|
||||||
|
build_session_factory(engine),
|
||||||
|
staging,
|
||||||
|
ArtifactRepository(store),
|
||||||
|
ReservedMaintenance(),
|
||||||
|
MaintenanceAudit(),
|
||||||
|
settings.judge_admission,
|
||||||
|
settings.import_worker_cgroup,
|
||||||
|
)
|
||||||
|
while not stop.is_set():
|
||||||
|
if not await worker.once():
|
||||||
|
try:
|
||||||
|
await asyncio.wait_for(stop.wait(), timeout=1)
|
||||||
|
except TimeoutError:
|
||||||
|
pass
|
||||||
|
finally:
|
||||||
|
staging.close()
|
||||||
|
store.close()
|
||||||
|
await engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
try:
|
||||||
|
asyncio.run(run())
|
||||||
|
except (ValueError, OSError, DBAPIError):
|
||||||
|
print(
|
||||||
|
"import service unavailable; check trusted configuration and resource limits",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
raise SystemExit(1) from None
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
"""Ordinary import-only read status; no parser, staging mutation or judging writes."""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
|
||||||
|
from kf_oj.core.settings import Settings
|
||||||
|
from kf_oj.infra.db.session import build_engine, build_session_factory
|
||||||
|
from kf_oj.modules.problems.operations import ImportOperationQueries
|
||||||
|
|
||||||
|
|
||||||
|
async def run() -> dict:
|
||||||
|
settings = Settings()
|
||||||
|
if settings.judge_admission is None:
|
||||||
|
raise ValueError("import admission required")
|
||||||
|
engine = build_engine(settings)
|
||||||
|
try:
|
||||||
|
return await ImportOperationQueries(
|
||||||
|
build_session_factory(engine), settings.judge_admission
|
||||||
|
).status()
|
||||||
|
finally:
|
||||||
|
await engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
try:
|
||||||
|
result = asyncio.run(asyncio.wait_for(run(), timeout=6))
|
||||||
|
print(json.dumps(result, sort_keys=True))
|
||||||
|
except Exception:
|
||||||
|
raise SystemExit("import status unavailable") from None
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -2,7 +2,9 @@
|
|||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
import subprocess
|
import subprocess
|
||||||
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
@@ -39,14 +41,20 @@ def postgres_url():
|
|||||||
|
|
||||||
|
|
||||||
def migrate(url: str, command: str, target: str | None = None) -> None:
|
def migrate(url: str, command: str, target: str | None = None) -> None:
|
||||||
subprocess.run(
|
try:
|
||||||
[str(Path(__file__).resolve().parents[1] / ".venv/bin/alembic"), command]
|
subprocess.run(
|
||||||
+ ([target] if target else []),
|
[sys.executable, "-m", "alembic", command] + ([target] if target else []),
|
||||||
cwd=Path(__file__).resolve().parents[1],
|
cwd=Path(__file__).resolve().parents[1],
|
||||||
env=os.environ | {"KF_OJ_DATABASE_URL": url},
|
env=os.environ | {"KF_OJ_DATABASE_URL": url},
|
||||||
check=True,
|
check=True,
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
)
|
)
|
||||||
|
except subprocess.CalledProcessError as error:
|
||||||
|
diagnostic = ((error.stdout or b"") + (error.stderr or b"")).decode(errors="replace")[
|
||||||
|
-8192:
|
||||||
|
]
|
||||||
|
diagnostic = re.sub(r"postgresql(?:\+asyncpg)?://[^\s]+", "[test database URL]", diagnostic)
|
||||||
|
raise AssertionError(f"test migration {command} failed: " + diagnostic) from None
|
||||||
|
|
||||||
|
|
||||||
async def connect(url: str):
|
async def connect(url: str):
|
||||||
|
|||||||
@@ -0,0 +1,180 @@
|
|||||||
|
"""Finite adverse archive/private-storage acceptance: independent capped VM only."""
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import stat
|
||||||
|
import struct
|
||||||
|
import time
|
||||||
|
import zipfile
|
||||||
|
from pathlib import Path
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from test_admin_import_vm import independent_import_vm as independent_import_vm
|
||||||
|
from test_admin_import_vm import package
|
||||||
|
|
||||||
|
from kf_oj.contracts.judging_v2 import AdmissionRegistry
|
||||||
|
from kf_oj.infra.artifacts.archive_reader import ArchiveError, ArchiveReader
|
||||||
|
from kf_oj.infra.artifacts.staging import ImportStaging
|
||||||
|
from kf_oj.infra.artifacts.store import ArtifactError, ArtifactStore
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
|
||||||
|
|
||||||
|
def policy():
|
||||||
|
return AdmissionRegistry.model_validate_json(
|
||||||
|
(ROOT / "tests/fixtures/contracts-v2/judge-registry.valid.json").read_bytes()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def proof(name, value):
|
||||||
|
print(
|
||||||
|
"KF_FINAL_ACCEPTANCE " + json.dumps({"name": name, "result": value}, default=str),
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_adverse_zip_and_private_storage_boundaries(independent_import_vm, tmp_path):
|
||||||
|
outside = tmp_path / "outside-sentinel"
|
||||||
|
outside.write_bytes(b"unchanged synthetic sentinel")
|
||||||
|
original = hashlib.sha256(outside.read_bytes()).hexdigest()
|
||||||
|
good, _ = package(policy())
|
||||||
|
cases = {}
|
||||||
|
for name in ("../outside-sentinel", "/outside-sentinel", "a/b", r"a\b", "a:b"):
|
||||||
|
cases["path-" + repr(name)] = package(
|
||||||
|
policy(), payload_mutation=lambda p, n=name: p.update({n: b"bad"})
|
||||||
|
)[0]
|
||||||
|
for kind in (
|
||||||
|
stat.S_IFLNK,
|
||||||
|
stat.S_IFCHR,
|
||||||
|
stat.S_IFBLK,
|
||||||
|
stat.S_IFIFO,
|
||||||
|
stat.S_IFSOCK,
|
||||||
|
stat.S_IFDIR,
|
||||||
|
):
|
||||||
|
out = io.BytesIO()
|
||||||
|
with zipfile.ZipFile(out, "w") as archive:
|
||||||
|
entry = zipfile.ZipInfo("unsafe")
|
||||||
|
entry.create_system = 3
|
||||||
|
entry.external_attr = (kind | 0o600) << 16
|
||||||
|
archive.writestr(entry, b"synthetic")
|
||||||
|
cases[f"type-{kind}"] = out.getvalue()
|
||||||
|
for duplicate in ("statement.md", "STATEMENT.MD"):
|
||||||
|
out = io.BytesIO(good)
|
||||||
|
with zipfile.ZipFile(out, "a") as archive:
|
||||||
|
archive.writestr(duplicate, b"different")
|
||||||
|
cases["duplicate-" + duplicate] = out.getvalue()
|
||||||
|
for signature in (
|
||||||
|
b"PK\x03\x04",
|
||||||
|
b"\x1f\x8b",
|
||||||
|
b"BZh",
|
||||||
|
b"\xfd7zXZ\x00",
|
||||||
|
b"Rar!\x1a\x07",
|
||||||
|
b"7z\xbc\xaf\x27\x1c",
|
||||||
|
b"X" * 257 + b"ustar",
|
||||||
|
):
|
||||||
|
cases["nested-" + signature[:6].hex()] = package(
|
||||||
|
policy(),
|
||||||
|
payload_mutation=lambda p, s=signature: p.update({"hidden.in": s + b"synthetic"}),
|
||||||
|
)[0]
|
||||||
|
central = good.index(b"PK\x01\x02")
|
||||||
|
for label, local_offset, central_offset, value, fmt in (
|
||||||
|
("encrypted", 6, 8, 1, "<H"),
|
||||||
|
("unsupported", 8, 10, 99, "<H"),
|
||||||
|
("declared-size", 22, 24, 67108865, "<I"),
|
||||||
|
("crc", 14, 16, 1, "<I"),
|
||||||
|
):
|
||||||
|
blob = bytearray(good)
|
||||||
|
struct.pack_into(fmt, blob, local_offset, value)
|
||||||
|
struct.pack_into(fmt, blob, central + central_offset, value)
|
||||||
|
cases[label] = bytes(blob)
|
||||||
|
for label, extra in (("zip64", b"\x01\x00\x00\x00"), ("unknown-extra", b"\x99\x99\x00\x00")):
|
||||||
|
out = io.BytesIO()
|
||||||
|
with zipfile.ZipFile(out, "w") as archive:
|
||||||
|
entry = zipfile.ZipInfo("data")
|
||||||
|
entry.extra = extra
|
||||||
|
archive.writestr(entry, b"synthetic")
|
||||||
|
cases[label] = out.getvalue()
|
||||||
|
out = io.BytesIO()
|
||||||
|
with zipfile.ZipFile(out, "w") as archive:
|
||||||
|
for i in range(535):
|
||||||
|
archive.writestr(f"f{i}", b"")
|
||||||
|
cases["file-count"] = out.getvalue()
|
||||||
|
# Actual bounded inflation exceeds the forged declared size. The 65 MiB stream
|
||||||
|
# is produced in chunks under the VM cap; no 65 MiB object is allocated.
|
||||||
|
out = io.BytesIO()
|
||||||
|
with zipfile.ZipFile(out, "w", compression=zipfile.ZIP_DEFLATED) as archive:
|
||||||
|
with archive.open("inflate", "w") as member:
|
||||||
|
for _ in range(1040):
|
||||||
|
member.write(b"A" * 65536)
|
||||||
|
blob = bytearray(out.getvalue())
|
||||||
|
center = blob.index(b"PK\x01\x02")
|
||||||
|
struct.pack_into("<I", blob, 22, 67108864)
|
||||||
|
struct.pack_into("<I", blob, center + 24, 67108864)
|
||||||
|
cases["actual-inflate-over-64MiB"] = bytes(blob)
|
||||||
|
refused = []
|
||||||
|
for index, (label, blob) in enumerate(cases.items()):
|
||||||
|
file = tmp_path / f"archive-{index}"
|
||||||
|
file.write_bytes(blob)
|
||||||
|
file.chmod(0o600)
|
||||||
|
fd = os.open(file, os.O_RDONLY)
|
||||||
|
try:
|
||||||
|
with pytest.raises(ArchiveError) as error:
|
||||||
|
reader = ArchiveReader(fd, len(blob), deadline=time.monotonic() + 60)
|
||||||
|
for member in reader.members:
|
||||||
|
for _ in reader.chunks(member.name, 67108864):
|
||||||
|
pass
|
||||||
|
assert str(error.value) in ("invalid_archive", "archive_deadline")
|
||||||
|
refused.append(label)
|
||||||
|
finally:
|
||||||
|
os.close(fd)
|
||||||
|
file.unlink()
|
||||||
|
file = tmp_path / "mutable"
|
||||||
|
file.write_bytes(good)
|
||||||
|
file.chmod(0o600)
|
||||||
|
fd = os.open(file, os.O_RDONLY)
|
||||||
|
try:
|
||||||
|
reader = ArchiveReader(fd, len(good), deadline=time.monotonic() + 60)
|
||||||
|
with file.open("r+b") as writer:
|
||||||
|
writer.write(b"X")
|
||||||
|
with pytest.raises(ArchiveError, match="archive_changed"):
|
||||||
|
reader.verify_unchanged()
|
||||||
|
finally:
|
||||||
|
os.close(fd)
|
||||||
|
store = ArtifactStore(tmp_path / "objects", max_bytes=4, max_files=2)
|
||||||
|
try:
|
||||||
|
ref = store.put(b"1234")
|
||||||
|
with pytest.raises(ArtifactError, match="capacity"):
|
||||||
|
store.put(b"5")
|
||||||
|
assert store.read(ref) == b"1234"
|
||||||
|
store.remove(ref.digest)
|
||||||
|
recovered = store.put(b"5")
|
||||||
|
assert store.read(recovered) == b"5"
|
||||||
|
finally:
|
||||||
|
store.close()
|
||||||
|
link_root = tmp_path / "linked-store"
|
||||||
|
link_root.symlink_to(tmp_path / "objects", target_is_directory=True)
|
||||||
|
with pytest.raises((ArtifactError, OSError)):
|
||||||
|
ArtifactStore(link_root)
|
||||||
|
staging = ImportStaging(tmp_path / "staging")
|
||||||
|
operation_id = uuid4()
|
||||||
|
(tmp_path / "staging" / str(operation_id)).symlink_to(tmp_path, target_is_directory=True)
|
||||||
|
try:
|
||||||
|
with pytest.raises((ArtifactError, OSError)):
|
||||||
|
staging.operation(operation_id)
|
||||||
|
finally:
|
||||||
|
staging.close()
|
||||||
|
assert hashlib.sha256(outside.read_bytes()).hexdigest() == original
|
||||||
|
proof(
|
||||||
|
"archive_and_storage",
|
||||||
|
{
|
||||||
|
"refused": refused,
|
||||||
|
"outside_unchanged": True,
|
||||||
|
"content_never_executed": True,
|
||||||
|
"changed_descriptor_rejected": True,
|
||||||
|
"quota_recovered": True,
|
||||||
|
"symlink_roots_rejected": True,
|
||||||
|
},
|
||||||
|
)
|
||||||
@@ -0,0 +1,319 @@
|
|||||||
|
"""Upgrade, adverse storage and cross-process admission: capped independent VM only."""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import resource
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
from conftest import connect
|
||||||
|
from sqlalchemy.engine import make_url
|
||||||
|
from test_admin_import_vm import independent_import_vm as independent_import_vm
|
||||||
|
|
||||||
|
from kf_oj.contracts.judging_v2 import AdmissionRegistry
|
||||||
|
from kf_oj.modules.identity.passwords import hash_password
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
|
||||||
|
|
||||||
|
def policy():
|
||||||
|
return AdmissionRegistry.model_validate_json(
|
||||||
|
(ROOT / "tests/fixtures/contracts-v2/judge-registry.valid.json").read_bytes()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def proof(name, value):
|
||||||
|
print(
|
||||||
|
"KF_FINAL_ACCEPTANCE " + json.dumps({"name": name, "result": value}, default=str),
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_cross_process_admission_and_worker_identity(independent_import_vm, postgres_url, tmp_path):
|
||||||
|
groups = independent_import_vm
|
||||||
|
roles = {name: "final_" + name + "_" + uuid4().hex for name in ("api", "worker")}
|
||||||
|
users = {f"capacity_admin_{i}": uuid4() for i in range(5)}
|
||||||
|
password = "Synthetic capacity acceptance only 2026!"
|
||||||
|
artifact, staging = tmp_path / "artifacts", tmp_path / "staging"
|
||||||
|
artifact.mkdir(mode=0o700)
|
||||||
|
staging.mkdir(mode=0o700)
|
||||||
|
processes, logs = {}, {}
|
||||||
|
base = {
|
||||||
|
k: v
|
||||||
|
for k, v in os.environ.items()
|
||||||
|
if k
|
||||||
|
in (
|
||||||
|
"PATH",
|
||||||
|
"PYTHONPATH",
|
||||||
|
"LD_LIBRARY_PATH",
|
||||||
|
"LC_ALL",
|
||||||
|
"PYTHONUNBUFFERED",
|
||||||
|
"PYTHONDONTWRITEBYTECODE",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
def enter(group):
|
||||||
|
def apply():
|
||||||
|
(group / "cgroup.procs").write_text(str(os.getpid()))
|
||||||
|
resource.setrlimit(resource.RLIMIT_FSIZE, (67108864, 67108864))
|
||||||
|
resource.setrlimit(resource.RLIMIT_NOFILE, (64, 64))
|
||||||
|
resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
|
||||||
|
|
||||||
|
return apply
|
||||||
|
|
||||||
|
def start(name, *, bad_identity=False, bad_group=False):
|
||||||
|
worker = name == "worker"
|
||||||
|
group = groups["import-worker" if worker else "api"]
|
||||||
|
env = base | {
|
||||||
|
"KF_OJ_DATABASE_URL": make_url(postgres_url)
|
||||||
|
.set(username=roles["api" if bad_identity or not worker else "worker"])
|
||||||
|
.render_as_string(hide_password=False),
|
||||||
|
"KF_OJ_ENVIRONMENT": "test",
|
||||||
|
"KF_OJ_ARTIFACT_ROOT": str(artifact),
|
||||||
|
"KF_OJ_IMPORT_STAGING_ROOT": str(staging),
|
||||||
|
"KF_OJ_JUDGE_ADMISSION": policy().model_dump_json(),
|
||||||
|
"KF_OJ_BROWSER_ORIGINS": '["http://127.0.0.1:58000"]',
|
||||||
|
}
|
||||||
|
if worker:
|
||||||
|
env["KF_OJ_IMPORT_WORKER_CGROUP"] = str(groups["api"] if bad_group else group)
|
||||||
|
command = [sys.executable, "-m", "kf_oj.workers.problem_import"]
|
||||||
|
else:
|
||||||
|
command = [
|
||||||
|
sys.executable,
|
||||||
|
"-m",
|
||||||
|
"uvicorn",
|
||||||
|
"kf_oj.api.app:create_app",
|
||||||
|
"--factory",
|
||||||
|
"--host",
|
||||||
|
"127.0.0.1",
|
||||||
|
"--port",
|
||||||
|
"58000" if name == "api0" else "58001",
|
||||||
|
"--no-proxy-headers",
|
||||||
|
"--limit-concurrency",
|
||||||
|
"4",
|
||||||
|
"--backlog",
|
||||||
|
"16",
|
||||||
|
]
|
||||||
|
log = (tmp_path / (name + ".log")).open("a")
|
||||||
|
logs[name] = log
|
||||||
|
processes[name] = subprocess.Popen(
|
||||||
|
command, env=env, stdout=log, stderr=log, preexec_fn=enter(group)
|
||||||
|
)
|
||||||
|
|
||||||
|
def stop(name):
|
||||||
|
process = processes.pop(name, None)
|
||||||
|
if process and process.poll() is None:
|
||||||
|
process.terminate()
|
||||||
|
try:
|
||||||
|
process.wait(timeout=20)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
process.kill()
|
||||||
|
process.wait(timeout=10)
|
||||||
|
if name in logs:
|
||||||
|
logs.pop(name).close()
|
||||||
|
|
||||||
|
async def run():
|
||||||
|
connection = await connect(postgres_url)
|
||||||
|
try:
|
||||||
|
for name, membership in (("api", "kf_oj_platform"), ("worker", "kf_oj_problem_import")):
|
||||||
|
await connection.execute(
|
||||||
|
f"CREATE ROLE {roles[name]} LOGIN NOSUPERUSER "
|
||||||
|
"NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS"
|
||||||
|
)
|
||||||
|
await connection.execute(f"GRANT {membership} TO {roles[name]}")
|
||||||
|
for name, identifier in users.items():
|
||||||
|
await connection.execute(
|
||||||
|
"INSERT INTO accounts(id,username,password_hash) VALUES($1,$2,$3)",
|
||||||
|
identifier,
|
||||||
|
name,
|
||||||
|
hash_password(name, password),
|
||||||
|
)
|
||||||
|
await connection.execute(
|
||||||
|
"SELECT * FROM kf_manage_account_authority($1,$2)",
|
||||||
|
name,
|
||||||
|
"bootstrap" if name.endswith("_0") else "grant",
|
||||||
|
)
|
||||||
|
for wrong in ("identity", "cgroup"):
|
||||||
|
start("worker", bad_identity=wrong == "identity", bad_group=wrong == "cgroup")
|
||||||
|
assert await asyncio.to_thread(processes["worker"].wait, 120) != 0
|
||||||
|
assert not await connection.fetchval(
|
||||||
|
"SELECT worker_ready_until>clock_timestamp() "
|
||||||
|
"FROM problem_import_pool WHERE id=1"
|
||||||
|
)
|
||||||
|
stop("worker")
|
||||||
|
start("worker")
|
||||||
|
deadline = time.monotonic() + 180
|
||||||
|
while not await connection.fetchval(
|
||||||
|
"SELECT worker_ready_until>clock_timestamp() FROM problem_import_pool WHERE id=1"
|
||||||
|
):
|
||||||
|
assert time.monotonic() < deadline and processes["worker"].poll() is None
|
||||||
|
await asyncio.sleep(0.1)
|
||||||
|
# Quiesce consumption. The actual verified heartbeat is retained for
|
||||||
|
# finite admission checks; synthetic capacity rows cannot be parsed.
|
||||||
|
stop("worker")
|
||||||
|
start("api0")
|
||||||
|
start("api1")
|
||||||
|
prefix = "/api/v1/admin/problem-imports"
|
||||||
|
async with (
|
||||||
|
httpx.AsyncClient(base_url="http://127.0.0.1:58000", timeout=30) as first,
|
||||||
|
httpx.AsyncClient(base_url="http://127.0.0.1:58001", timeout=30) as second,
|
||||||
|
):
|
||||||
|
for client in (first, second):
|
||||||
|
deadline = time.monotonic() + 300
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
if (await client.get("/api/v1/health/live")).status_code == 200:
|
||||||
|
break
|
||||||
|
except httpx.ConnectError:
|
||||||
|
pass
|
||||||
|
assert time.monotonic() < deadline
|
||||||
|
await asyncio.sleep(0.1)
|
||||||
|
result = await client.post(
|
||||||
|
"/api/v1/auth/login",
|
||||||
|
json={
|
||||||
|
"username": "capacity_admin_0",
|
||||||
|
"password": password,
|
||||||
|
},
|
||||||
|
headers={"Origin": "http://127.0.0.1:58000"},
|
||||||
|
)
|
||||||
|
assert result.status_code == 200, result.text
|
||||||
|
|
||||||
|
def headers(client):
|
||||||
|
return {
|
||||||
|
"Origin": "http://127.0.0.1:58000",
|
||||||
|
"X-CSRF-Token": client.cookies["kf_csrf"],
|
||||||
|
}
|
||||||
|
|
||||||
|
def body(key=None):
|
||||||
|
return {
|
||||||
|
"operation_id": str(key or uuid4()),
|
||||||
|
"archive_digest": "a" * 64,
|
||||||
|
"archive_bytes": 100,
|
||||||
|
}
|
||||||
|
|
||||||
|
async def refresh_worker():
|
||||||
|
previous = await connection.fetchval(
|
||||||
|
"SELECT worker_ready_until FROM problem_import_pool WHERE id=1"
|
||||||
|
)
|
||||||
|
start("worker")
|
||||||
|
deadline = time.monotonic() + 120
|
||||||
|
while True:
|
||||||
|
current = await connection.fetchval(
|
||||||
|
"SELECT worker_ready_until FROM problem_import_pool WHERE id=1"
|
||||||
|
)
|
||||||
|
if current and (previous is None or current > previous):
|
||||||
|
break
|
||||||
|
assert time.monotonic() < deadline
|
||||||
|
assert processes["worker"].poll() is None
|
||||||
|
await asyncio.sleep(0.1)
|
||||||
|
stop("worker")
|
||||||
|
|
||||||
|
await refresh_worker()
|
||||||
|
request = body()
|
||||||
|
replies = await asyncio.gather(
|
||||||
|
*(
|
||||||
|
client.post(prefix, json=request, headers=headers(client))
|
||||||
|
for client in (first, second)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
# Identical UUIDs converge across independent API processes. A
|
||||||
|
# transient root-lock refusal is retried once after the winner.
|
||||||
|
assert 201 in [r.status_code for r in replies]
|
||||||
|
retry = await second.post(prefix, json=request, headers=headers(second))
|
||||||
|
assert retry.status_code == 201
|
||||||
|
assert (
|
||||||
|
await connection.fetchval("SELECT count(*) FROM problem_import_operations") == 1
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
await connection.fetchval("SELECT reserved_bytes FROM problem_import_pool")
|
||||||
|
== 134217728
|
||||||
|
)
|
||||||
|
other = await first.post(prefix, json=body(), headers=headers(first))
|
||||||
|
assert other.status_code == 503
|
||||||
|
discarded = await first.post(
|
||||||
|
prefix + "/" + request["operation_id"] + "/discard", headers=headers(first)
|
||||||
|
)
|
||||||
|
assert discarded.status_code == 200, discarded.text
|
||||||
|
assert (
|
||||||
|
await connection.fetchval("SELECT reserved_bytes FROM problem_import_pool") == 0
|
||||||
|
)
|
||||||
|
|
||||||
|
async def capacity_rows(count):
|
||||||
|
# Explicit unmeasured database fixtures exercise owner/global
|
||||||
|
# admission boundaries; no ready candidate or grading is forged.
|
||||||
|
await connection.execute(
|
||||||
|
"DELETE FROM problem_import_operations WHERE state='uploaded'"
|
||||||
|
)
|
||||||
|
identifiers = list(users.values())
|
||||||
|
for i in range(count):
|
||||||
|
await connection.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO problem_import_operations(id,owner_id,authorization_version,
|
||||||
|
archive_digest,archive_bytes,state,reserved_bytes,expires_at,updated_at)
|
||||||
|
VALUES($1,$2,2,$3,100,'uploaded',0,
|
||||||
|
clock_timestamp()+interval '1 hour',clock_timestamp())
|
||||||
|
""",
|
||||||
|
uuid4(),
|
||||||
|
identifiers[0] if count == 10 else identifiers[1 + i % 4],
|
||||||
|
"b" * 64,
|
||||||
|
)
|
||||||
|
|
||||||
|
results = {}
|
||||||
|
for label, count in (("owner_limit", 10), ("global_limit", 32)):
|
||||||
|
await capacity_rows(0)
|
||||||
|
await refresh_worker()
|
||||||
|
await capacity_rows(count)
|
||||||
|
before = await connection.fetchval(
|
||||||
|
"SELECT count(*) FROM problem_import_operations"
|
||||||
|
)
|
||||||
|
response = await second.post(prefix, json=body(), headers=headers(second))
|
||||||
|
assert response.status_code == 503
|
||||||
|
assert (
|
||||||
|
await connection.fetchval("SELECT count(*) FROM problem_import_operations")
|
||||||
|
== before
|
||||||
|
)
|
||||||
|
results[label] = {
|
||||||
|
"fixture_count": count,
|
||||||
|
"actual_http_status": 503,
|
||||||
|
"no_new_receipt": True,
|
||||||
|
}
|
||||||
|
await capacity_rows(0)
|
||||||
|
await refresh_worker()
|
||||||
|
resumed = body()
|
||||||
|
result = await second.post(prefix, json=resumed, headers=headers(second))
|
||||||
|
assert result.status_code == 201, result.text
|
||||||
|
assert (
|
||||||
|
await first.post(
|
||||||
|
prefix + "/" + resumed["operation_id"] + "/discard", headers=headers(first)
|
||||||
|
)
|
||||||
|
).status_code == 200
|
||||||
|
assert (
|
||||||
|
await connection.fetchval("SELECT reserved_bytes FROM problem_import_pool") == 0
|
||||||
|
)
|
||||||
|
proof(
|
||||||
|
"cross_process_admission",
|
||||||
|
results
|
||||||
|
| {
|
||||||
|
"two_actual_api_processes": True,
|
||||||
|
"identical_uuid_one_receipt": True,
|
||||||
|
"single_slot_http_status": 503,
|
||||||
|
"wrong_worker_identity_and_group_exit_nonzero": True,
|
||||||
|
"ordinary_worker_heartbeat_verified": True,
|
||||||
|
"recovery_new_operation_201": True,
|
||||||
|
"remaining_reserved_bytes": 0,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
for name in list(processes):
|
||||||
|
stop(name)
|
||||||
|
for role in roles.values():
|
||||||
|
await connection.execute(f"DROP OWNED BY {role}")
|
||||||
|
await connection.execute(f"DROP ROLE {role}")
|
||||||
|
await connection.close()
|
||||||
|
|
||||||
|
asyncio.run(run())
|
||||||
@@ -0,0 +1,156 @@
|
|||||||
|
"""Contract checks only; upload/parser/security evidence requires the independent VM."""
|
||||||
|
|
||||||
|
import copy
|
||||||
|
import json
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
from pathlib import Path
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from pydantic import ValidationError
|
||||||
|
|
||||||
|
from kf_oj.contracts.admin_problems import (
|
||||||
|
CandidatePreview,
|
||||||
|
CreateProblemImport,
|
||||||
|
LegacyImportBudget,
|
||||||
|
ProblemImportCapabilities,
|
||||||
|
ProblemImportDetail,
|
||||||
|
)
|
||||||
|
from kf_oj.contracts.judging import AdmissionPolicy
|
||||||
|
from kf_oj.modules.problems.import_format import ImportEnvelope
|
||||||
|
from kf_oj.modules.problems.web_format import decode_web_declaration
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
|
||||||
|
|
||||||
|
def legacy_manifest():
|
||||||
|
policy = AdmissionPolicy.model_validate_json(
|
||||||
|
(ROOT / "tests/fixtures/contracts/judge-admission.valid.json").read_bytes()
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"schema_version": 1,
|
||||||
|
"slug": "web-fixture",
|
||||||
|
"title": "Public",
|
||||||
|
"statement_file": "statement.md",
|
||||||
|
"languages": ["c17", "cpp20"],
|
||||||
|
"toolchain": policy.toolchain.model_dump(mode="json"),
|
||||||
|
"comparison": "exact_bytes",
|
||||||
|
"compile_limits": policy.compile_ceiling.model_dump(),
|
||||||
|
"run_limits": policy.run_ceiling.model_dump(),
|
||||||
|
"job_wall_time_us": 60000000,
|
||||||
|
"cases": [{"key": "binary", "input_file": "hidden.in", "expected_file": "hidden.out"}],
|
||||||
|
"samples": [{"input_file": "sample.in", "output_file": "sample.out"}],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_web_declaration_separates_public_data_and_preserves_local_format():
|
||||||
|
source = legacy_manifest()
|
||||||
|
original = copy.deepcopy(source)
|
||||||
|
declared = decode_web_declaration(json.dumps(source).encode())
|
||||||
|
assert declared.members == frozenset(
|
||||||
|
("problem.json", "statement.md", "hidden.in", "hidden.out", "sample.in", "sample.out")
|
||||||
|
)
|
||||||
|
assert declared.manifest.model_dump(mode="json") == original
|
||||||
|
assert ImportEnvelope.model_validate(source).root == declared.manifest
|
||||||
|
for mutation in (
|
||||||
|
lambda x: x["samples"][0].update(input_file="hidden.in"),
|
||||||
|
lambda x: x.update(statement_file="problem.json"),
|
||||||
|
lambda x: x["cases"][0].update(input_file="Statement.md"),
|
||||||
|
lambda x: x["cases"][0].update(input_file="hidden.zip"),
|
||||||
|
lambda x: x.update(languages=["rust"]),
|
||||||
|
lambda x: x.update(download_url="https://example.invalid/private"),
|
||||||
|
):
|
||||||
|
value = copy.deepcopy(source)
|
||||||
|
mutation(value)
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
decode_web_declaration(json.dumps(value).encode())
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
decode_web_declaration(
|
||||||
|
json.dumps(source)
|
||||||
|
.replace('"schema_version": 1', '"schema_version": 1, "schema_version": 1')
|
||||||
|
.encode()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_import_operation_key_and_preview_have_no_private_payload_fields():
|
||||||
|
operation = {
|
||||||
|
"schema_version": 1,
|
||||||
|
"operation_id": str(uuid4()),
|
||||||
|
"archive_digest": "a" * 64,
|
||||||
|
"archive_bytes": 64 * 1024**2,
|
||||||
|
}
|
||||||
|
assert CreateProblemImport.model_validate(operation).archive_bytes == 64 * 1024**2
|
||||||
|
for field, value in (
|
||||||
|
("archive_bytes", True),
|
||||||
|
("archive_bytes", 64 * 1024**2 + 1),
|
||||||
|
("url", "https://example.invalid"),
|
||||||
|
("role", "admin"),
|
||||||
|
):
|
||||||
|
with pytest.raises(ValidationError):
|
||||||
|
CreateProblemImport.model_validate(operation | {field: value})
|
||||||
|
manifest = ImportEnvelope.model_validate(legacy_manifest()).root
|
||||||
|
preview = CandidatePreview(
|
||||||
|
candidate_id=uuid4(),
|
||||||
|
slug=manifest.slug,
|
||||||
|
title=manifest.title,
|
||||||
|
statement="Public only",
|
||||||
|
samples=(),
|
||||||
|
judging_schema_version=1,
|
||||||
|
languages=manifest.languages,
|
||||||
|
legacy_budget=LegacyImportBudget(
|
||||||
|
compile_limits=manifest.compile_limits,
|
||||||
|
run_limits=manifest.run_limits,
|
||||||
|
job_wall_time_us=manifest.job_wall_time_us,
|
||||||
|
),
|
||||||
|
case_count=1,
|
||||||
|
)
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
response = ProblemImportDetail(
|
||||||
|
operation_id=uuid4(),
|
||||||
|
state="ready",
|
||||||
|
expected_archive_digest="a" * 64,
|
||||||
|
archive_bytes=100,
|
||||||
|
created_at=now,
|
||||||
|
updated_at=now,
|
||||||
|
expires_at=now + timedelta(days=1),
|
||||||
|
failure_code=None,
|
||||||
|
candidate_id=preview.candidate_id,
|
||||||
|
published_version_id=None,
|
||||||
|
candidate=preview,
|
||||||
|
)
|
||||||
|
assert set(response.candidate.model_dump()) == {
|
||||||
|
"candidate_id",
|
||||||
|
"slug",
|
||||||
|
"title",
|
||||||
|
"statement",
|
||||||
|
"samples",
|
||||||
|
"judging_schema_version",
|
||||||
|
"languages",
|
||||||
|
"legacy_budget",
|
||||||
|
"language_configurations",
|
||||||
|
"case_count",
|
||||||
|
}
|
||||||
|
with pytest.raises(ValidationError):
|
||||||
|
CandidatePreview.model_validate(preview.model_dump() | {"input_digest": "a" * 64})
|
||||||
|
capabilities = ProblemImportCapabilities(uploads_available=False, validation_available=False)
|
||||||
|
assert (
|
||||||
|
capabilities.limits.global_concurrency == 1 and capabilities.limits.owner_candidates == 10
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_standalone_import_worker_resolves_all_referenced_metadata():
|
||||||
|
# A separate trusted interpreter avoids accidental model registration by API test imports.
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
script = """
|
||||||
|
import kf_oj.workers.problem_import
|
||||||
|
from kf_oj.infra.db.base import Base
|
||||||
|
for table in Base.metadata.tables.values():
|
||||||
|
for foreign in table.foreign_keys:
|
||||||
|
assert foreign.column.table is not None
|
||||||
|
"""
|
||||||
|
result = subprocess.run(
|
||||||
|
[sys.executable, "-c", script], capture_output=True, text=True, timeout=15
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,223 @@
|
|||||||
|
"""Approved baseline upgrade/retention: actual PostgreSQL in an independent capped VM."""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
from conftest import connect, migrate
|
||||||
|
from sqlalchemy.engine import make_url
|
||||||
|
from test_admin_import_vm import independent_import_vm as independent_import_vm
|
||||||
|
from test_admin_import_vm import package
|
||||||
|
|
||||||
|
from kf_oj.contracts.judging_v2 import AdmissionRegistry
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
|
||||||
|
|
||||||
|
def policy():
|
||||||
|
return AdmissionRegistry.model_validate_json(
|
||||||
|
(ROOT / "tests/fixtures/contracts-v2/judge-registry.valid.json").read_bytes()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def proof(name, value):
|
||||||
|
print(
|
||||||
|
"KF_FINAL_ACCEPTANCE " + json.dumps({"name": name, "result": value}, default=str),
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_baseline_upgrade_preserves_existing_data(independent_import_vm, postgres_url, tmp_path):
|
||||||
|
baseline = Path("/kf/baseline-platform")
|
||||||
|
assert baseline.is_dir(), "curated approved baseline required"
|
||||||
|
url = make_url(postgres_url).set(database="kf_oj_test_upgrade_" + uuid4().hex)
|
||||||
|
rendered = url.render_as_string(hide_password=False)
|
||||||
|
|
||||||
|
async def manage(command):
|
||||||
|
connection = await connect(postgres_url)
|
||||||
|
try:
|
||||||
|
await connection.execute(command)
|
||||||
|
finally:
|
||||||
|
await connection.close()
|
||||||
|
|
||||||
|
def old_run(args):
|
||||||
|
result = subprocess.run(
|
||||||
|
[sys.executable, *args],
|
||||||
|
cwd=baseline,
|
||||||
|
env=os.environ
|
||||||
|
| {
|
||||||
|
"PYTHONPATH": "/kf/baseline-platform/src:/kf/site",
|
||||||
|
"KF_OJ_DATABASE_URL": rendered,
|
||||||
|
"KF_BASELINE_FIXTURES": str(tmp_path),
|
||||||
|
},
|
||||||
|
capture_output=True,
|
||||||
|
timeout=300,
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, (result.stdout + result.stderr).decode(errors="replace")[
|
||||||
|
-2000:
|
||||||
|
]
|
||||||
|
return result
|
||||||
|
|
||||||
|
# Real old-schema data is created through the old public services. No student
|
||||||
|
# source executes here; historical report fixtures are explicitly unmeasured.
|
||||||
|
for multi in (False, True):
|
||||||
|
directory = tmp_path / ("v2" if multi else "v1")
|
||||||
|
directory.mkdir(mode=0o700)
|
||||||
|
_, payload = package(policy(), multi=multi)
|
||||||
|
for name, data in payload.items():
|
||||||
|
(directory / name).write_bytes(data)
|
||||||
|
(directory / name).chmod(0o600)
|
||||||
|
seed = """
|
||||||
|
import asyncio,hashlib,json,os,secrets
|
||||||
|
from datetime import datetime,timedelta,timezone
|
||||||
|
from pathlib import Path
|
||||||
|
from uuid import uuid4
|
||||||
|
from sqlalchemy import text,update
|
||||||
|
from kf_oj.api.app import create_app
|
||||||
|
from kf_oj.core.settings import Settings
|
||||||
|
from kf_oj.contracts.judging import ArtifactReference
|
||||||
|
from kf_oj.contracts.judging_v2 import AdmissionRegistry
|
||||||
|
from kf_oj.contracts.judging_versions import manifest_bytes
|
||||||
|
from kf_oj.contracts.submissions import CreateSubmission
|
||||||
|
from kf_oj.modules.submissions.models import Evaluation,JudgeAttempt
|
||||||
|
async def main():
|
||||||
|
root=Path(os.environ['KF_BASELINE_FIXTURES'])
|
||||||
|
fixture=Path('/kf/tests/fixtures/contracts-v2/judge-registry.valid.json')
|
||||||
|
policy=AdmissionRegistry.model_validate_json(fixture.read_bytes())
|
||||||
|
database=os.environ['KF_OJ_DATABASE_URL']
|
||||||
|
app=create_app(Settings(_env_file=None,database_url=database,
|
||||||
|
artifact_root=root/'objects',judge_admission=policy))
|
||||||
|
async with app.router.lifespan_context(app):
|
||||||
|
password='Synthetic upgrade acceptance only 2026!'
|
||||||
|
user=await app.state.identity.create_account('upgrade_student',password)
|
||||||
|
for name in ('v1','v2'):
|
||||||
|
version=await app.state.problems.import_directory(root/name)
|
||||||
|
request=CreateSubmission(problem_version_id=version,language='c17',
|
||||||
|
idempotency_key=uuid4(),source='int main(void){return 0;}')
|
||||||
|
created=await app.state.submissions.create(user,request)
|
||||||
|
# Controlled, unmeasured historical database fixtures prove preservation of
|
||||||
|
# reports and their digests. They never claim execution or trusted performance.
|
||||||
|
async with app.state.sessions.begin() as session:
|
||||||
|
projected=await app.state.problems.for_submission(session,version)
|
||||||
|
source=ArtifactReference(digest=hashlib.sha256(request.source.encode()).hexdigest(),
|
||||||
|
bytes=len(request.source.encode()))
|
||||||
|
manifest=projected.manifest(created.evaluation_id,created.submission_id,'c17',source)
|
||||||
|
md=hashlib.sha256(manifest_bytes(manifest)).hexdigest()
|
||||||
|
aid,wid=uuid4(),uuid4()
|
||||||
|
template=('contracts' if name=='v1' else 'contracts-v2')
|
||||||
|
file=('judge-result.valid.json' if name=='v1' else 'judge-result.valid-c17.json')
|
||||||
|
report=json.loads((Path('/kf/tests/fixtures')/template/file).read_text())
|
||||||
|
report['binding']={'evaluation_id':str(created.evaluation_id),'attempt_id':str(aid),
|
||||||
|
'worker_id':str(wid),'fencing_version':1}
|
||||||
|
report['manifest_digest']=md
|
||||||
|
report['cases']=[{'case_id':str(case.case_id),'verdict':'accepted',
|
||||||
|
'cpu_time_us':100,'memory_peak_bytes':262144}
|
||||||
|
for case in manifest.cases]
|
||||||
|
report['run_cpu_time_us']=100*len(manifest.cases)
|
||||||
|
report['run_memory_peak_bytes']=262144
|
||||||
|
digest=hashlib.sha256(json.dumps(report,sort_keys=True,separators=(',',':'),
|
||||||
|
ensure_ascii=False).encode()).hexdigest()
|
||||||
|
await session.execute(text('SET LOCAL ROLE kf_oj_coordinator'))
|
||||||
|
session.add(JudgeAttempt(id=aid,evaluation_id=created.evaluation_id,worker_id=wid,
|
||||||
|
number=1,fencing_version=1,lease_token_hash=hashlib.sha256(
|
||||||
|
secrets.token_hex(32).encode()).hexdigest(),
|
||||||
|
expires_at=datetime.now(timezone.utc)+timedelta(minutes=10),
|
||||||
|
manifest=manifest.model_dump(mode='json'),manifest_digest=md,
|
||||||
|
report=report,report_digest=digest,cleanup_complete=True))
|
||||||
|
await session.flush()
|
||||||
|
await session.execute(update(Evaluation).where(Evaluation.id==created.evaluation_id)
|
||||||
|
.values(current_attempt_id=aid,state='finished',attempts_count=1,
|
||||||
|
fencing_version=1,verdict='accepted',final_report=report,
|
||||||
|
final_report_digest=digest))
|
||||||
|
asyncio.run(main())
|
||||||
|
"""
|
||||||
|
asyncio.run(manage(f'CREATE DATABASE "{url.database}"'))
|
||||||
|
try:
|
||||||
|
old_run(["-m", "alembic", "upgrade", "head"])
|
||||||
|
old_run(["-c", seed])
|
||||||
|
|
||||||
|
async def snapshot(columns=None):
|
||||||
|
connection = await connect(rendered)
|
||||||
|
try:
|
||||||
|
if columns is None:
|
||||||
|
rows = await connection.fetch("""
|
||||||
|
SELECT table_name,column_name FROM information_schema.columns
|
||||||
|
WHERE table_schema='public' AND table_name<>'alembic_version'
|
||||||
|
ORDER BY table_name,ordinal_position
|
||||||
|
""")
|
||||||
|
columns = {}
|
||||||
|
for row in rows:
|
||||||
|
columns.setdefault(row["table_name"], []).append(row["column_name"])
|
||||||
|
data = {}
|
||||||
|
for table, names in columns.items():
|
||||||
|
query = (
|
||||||
|
"SELECT " + ",".join('"' + n + '"' for n in names) + ' FROM "' + table + '"'
|
||||||
|
)
|
||||||
|
records = sorted(
|
||||||
|
json.dumps(dict(r), sort_keys=True, default=str)
|
||||||
|
for r in await connection.fetch(query)
|
||||||
|
)
|
||||||
|
data[table] = {
|
||||||
|
"rows": len(records),
|
||||||
|
"sha256": hashlib.sha256("\n".join(records).encode()).hexdigest(),
|
||||||
|
}
|
||||||
|
return columns, data
|
||||||
|
finally:
|
||||||
|
await connection.close()
|
||||||
|
|
||||||
|
columns, before = asyncio.run(snapshot())
|
||||||
|
assert before["accounts"]["rows"] == 1 and before["problem_versions"]["rows"] == 2
|
||||||
|
assert before["submissions"]["rows"] == 2
|
||||||
|
assert before["judge_attempts"]["rows"] == 2
|
||||||
|
migrate(rendered, "upgrade", "head")
|
||||||
|
_, after = asyncio.run(snapshot(columns))
|
||||||
|
assert after == before
|
||||||
|
migrate(rendered, "check")
|
||||||
|
|
||||||
|
async def defaults():
|
||||||
|
connection = await connect(rendered)
|
||||||
|
try:
|
||||||
|
account = await connection.fetchrow(
|
||||||
|
"SELECT is_administrator,authorization_version FROM accounts"
|
||||||
|
)
|
||||||
|
assert not account["is_administrator"] and account["authorization_version"] == 1
|
||||||
|
assert await connection.fetchval("SELECT count(*) FROM evaluation_performance") == 0
|
||||||
|
assert (
|
||||||
|
await connection.fetchval("SELECT count(*) FROM problem_maintenance_events")
|
||||||
|
== 0
|
||||||
|
)
|
||||||
|
for role in ("kf_oj_identity_operator", "kf_oj_problem_import"):
|
||||||
|
row = await connection.fetchrow("SELECT * FROM pg_roles WHERE rolname=$1", role)
|
||||||
|
assert not any(
|
||||||
|
row[k]
|
||||||
|
for k in (
|
||||||
|
"rolcanlogin",
|
||||||
|
"rolsuper",
|
||||||
|
"rolcreatedb",
|
||||||
|
"rolcreaterole",
|
||||||
|
"rolreplication",
|
||||||
|
"rolbypassrls",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
await connection.close()
|
||||||
|
|
||||||
|
asyncio.run(defaults())
|
||||||
|
proof(
|
||||||
|
"baseline_upgrade",
|
||||||
|
{
|
||||||
|
"baseline": "5df3c370aab0b12a1c3f8670a0c066f8b77a4078",
|
||||||
|
"retained_tables": before,
|
||||||
|
"old_v1_v2_manifests_and_source_bytes": "exact",
|
||||||
|
"historical_report_retention_fixture_unmeasured": True,
|
||||||
|
"fixture_reports_not_student_execution_or_performance_acceptance": True,
|
||||||
|
"fresh_current_database": postgres_url.split("/")[-1].split("?")[0],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
asyncio.run(manage(f'DROP DATABASE "{url.database}"'))
|
||||||
@@ -0,0 +1,244 @@
|
|||||||
|
"""Actual temporary PG migration and authority storage; API/CLI acceptance is separate."""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import subprocess
|
||||||
|
from datetime import timedelta
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
import asyncpg
|
||||||
|
import pytest
|
||||||
|
from conftest import connect, migrate
|
||||||
|
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||||
|
from test_database_foundation import fixture_rows
|
||||||
|
|
||||||
|
from kf_oj.modules.identity.public import AuthenticationFailed, Principal
|
||||||
|
from kf_oj.modules.identity.service import IdentityService, token_digest
|
||||||
|
|
||||||
|
|
||||||
|
class RollBackAuthorityChange(Exception):
|
||||||
|
"""Abort the test transaction before any authority change commits."""
|
||||||
|
|
||||||
|
|
||||||
|
async def submission_snapshot(connection):
|
||||||
|
# Synthetic queued metadata only; this does not claim real program execution.
|
||||||
|
return {
|
||||||
|
table: list(
|
||||||
|
await connection.fetch(
|
||||||
|
f"SELECT to_jsonb(t)::text FROM {table} t ORDER BY to_jsonb(t)::text"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
for table in (
|
||||||
|
"artifacts",
|
||||||
|
"problems",
|
||||||
|
"problem_versions",
|
||||||
|
"problem_cases",
|
||||||
|
"submissions",
|
||||||
|
"evaluations",
|
||||||
|
"outbox",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def account_and_session(connection):
|
||||||
|
account = uuid4()
|
||||||
|
token, csrf = uuid4().hex * 2, uuid4().hex * 2
|
||||||
|
await connection.execute(
|
||||||
|
"INSERT INTO accounts(id,username,password_hash) VALUES($1,'storage_user',$2)",
|
||||||
|
account,
|
||||||
|
"synthetic-irreversible-placeholder-hash-" * 2,
|
||||||
|
)
|
||||||
|
now = await connection.fetchval("SELECT clock_timestamp()")
|
||||||
|
await connection.execute(
|
||||||
|
"INSERT INTO sessions(token_hash,csrf_hash,account_id,expires_at) VALUES($1,$2,$3,$4)",
|
||||||
|
token_digest(token),
|
||||||
|
token_digest(csrf),
|
||||||
|
account,
|
||||||
|
now + timedelta(hours=1),
|
||||||
|
)
|
||||||
|
return account, token, csrf
|
||||||
|
|
||||||
|
|
||||||
|
def test_upgrade_preserves_ordinary_account_and_current_session(postgres_url):
|
||||||
|
migrate(postgres_url, "downgrade", "0004_performance_watermark")
|
||||||
|
|
||||||
|
async def seed():
|
||||||
|
connection = await connect(postgres_url)
|
||||||
|
try:
|
||||||
|
account, token, csrf = await account_and_session(connection)
|
||||||
|
await fixture_rows(connection)
|
||||||
|
return account, token, csrf, await submission_snapshot(connection)
|
||||||
|
finally:
|
||||||
|
await connection.close()
|
||||||
|
|
||||||
|
account, token, csrf, before = asyncio.run(seed())
|
||||||
|
migrate(postgres_url, "upgrade", "head")
|
||||||
|
migrate(postgres_url, "check")
|
||||||
|
|
||||||
|
async def inspect():
|
||||||
|
connection = await connect(postgres_url)
|
||||||
|
engine = create_async_engine(postgres_url)
|
||||||
|
try:
|
||||||
|
row = await connection.fetchrow("SELECT * FROM accounts WHERE id=$1", account)
|
||||||
|
assert row["enabled"] and not row["is_administrator"]
|
||||||
|
assert row["authorization_version"] == 1
|
||||||
|
assert await submission_snapshot(connection) == before
|
||||||
|
service = IdentityService(async_sessionmaker(engine, expire_on_commit=False))
|
||||||
|
assert await service.authenticate(token, csrf_token=csrf) == Principal(
|
||||||
|
account, "storage_user"
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
await connection.close()
|
||||||
|
await engine.dispose()
|
||||||
|
|
||||||
|
asyncio.run(inspect())
|
||||||
|
|
||||||
|
|
||||||
|
def test_authority_columns_and_import_role_are_not_runtime_write_permissions(postgres_url):
|
||||||
|
async def inspect():
|
||||||
|
connection = await connect(postgres_url)
|
||||||
|
try:
|
||||||
|
for role in ("kf_oj_platform", "kf_oj_coordinator", "kf_oj_problem_import"):
|
||||||
|
for column in ("is_administrator", "authorization_version"):
|
||||||
|
assert not await connection.fetchval(
|
||||||
|
"SELECT has_column_privilege($1,'accounts',$2,'UPDATE')", role, column
|
||||||
|
)
|
||||||
|
assert not await connection.fetchval(
|
||||||
|
"SELECT has_column_privilege($1,'accounts',$2,'INSERT')", role, column
|
||||||
|
)
|
||||||
|
for table in (
|
||||||
|
"evaluations",
|
||||||
|
"judge_attempts",
|
||||||
|
"case_results",
|
||||||
|
"evaluation_performance",
|
||||||
|
):
|
||||||
|
for privilege in ("SELECT", "INSERT", "UPDATE", "DELETE"):
|
||||||
|
assert not await connection.fetchval(
|
||||||
|
"SELECT has_table_privilege('kf_oj_problem_import',$1,$2)", table, privilege
|
||||||
|
)
|
||||||
|
assert await connection.fetchval(
|
||||||
|
"SELECT has_column_privilege('kf_oj_platform','accounts','enabled','UPDATE')"
|
||||||
|
)
|
||||||
|
assert not await connection.fetchval(
|
||||||
|
"SELECT has_column_privilege('kf_oj_identity_operator',"
|
||||||
|
"'accounts','is_administrator','UPDATE')"
|
||||||
|
)
|
||||||
|
assert not await connection.fetchval(
|
||||||
|
"SELECT has_column_privilege('kf_oj_identity_operator',"
|
||||||
|
"'accounts','authorization_version','UPDATE')"
|
||||||
|
)
|
||||||
|
assert not await connection.fetchval(
|
||||||
|
"SELECT pg_has_role('kf_oj_platform','kf_oj_identity_operator','USAGE')"
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
await connection.close()
|
||||||
|
|
||||||
|
asyncio.run(inspect())
|
||||||
|
|
||||||
|
|
||||||
|
def test_authority_change_derives_version_and_revokes_old_session_atomically(postgres_url):
|
||||||
|
async def inspect():
|
||||||
|
connection = await connect(postgres_url)
|
||||||
|
engine = create_async_engine(postgres_url)
|
||||||
|
try:
|
||||||
|
account, token, csrf = await account_and_session(connection)
|
||||||
|
service = IdentityService(async_sessionmaker(engine, expire_on_commit=False))
|
||||||
|
await service.authenticate(token, csrf_token=csrf)
|
||||||
|
async with connection.transaction():
|
||||||
|
await connection.execute("SET LOCAL ROLE kf_oj_identity_operator")
|
||||||
|
await connection.execute(
|
||||||
|
"SELECT * FROM public.kf_manage_account_authority('storage_user','bootstrap')"
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
await connection.fetchval(
|
||||||
|
"SELECT authorization_version FROM accounts WHERE id=$1", account
|
||||||
|
)
|
||||||
|
== 2
|
||||||
|
)
|
||||||
|
with pytest.raises(AuthenticationFailed):
|
||||||
|
await service.authenticate(token, csrf_token=csrf)
|
||||||
|
async with connection.transaction():
|
||||||
|
await connection.execute("SET LOCAL ROLE kf_oj_identity_operator")
|
||||||
|
await connection.execute(
|
||||||
|
"SELECT * FROM public.kf_manage_account_authority('storage_user','bootstrap')"
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
await connection.fetchval(
|
||||||
|
"SELECT authorization_version FROM accounts WHERE id=$1", account
|
||||||
|
)
|
||||||
|
== 2
|
||||||
|
)
|
||||||
|
fresh_token, fresh_csrf = uuid4().hex * 2, uuid4().hex * 2
|
||||||
|
await connection.execute(
|
||||||
|
"INSERT INTO sessions(token_hash,csrf_hash,account_id,expires_at) "
|
||||||
|
"VALUES($1,$2,$3,clock_timestamp()+interval '1 hour')",
|
||||||
|
token_digest(fresh_token),
|
||||||
|
token_digest(fresh_csrf),
|
||||||
|
account,
|
||||||
|
)
|
||||||
|
await connection.execute(
|
||||||
|
"INSERT INTO accounts(id,username,password_hash) "
|
||||||
|
"VALUES(gen_random_uuid(),'storage_recovery',repeat('synthetic-only-',5))"
|
||||||
|
)
|
||||||
|
async with connection.transaction():
|
||||||
|
await connection.execute("SET LOCAL ROLE kf_oj_identity_operator")
|
||||||
|
await connection.execute(
|
||||||
|
"SELECT * FROM public.kf_manage_account_authority('storage_recovery','grant')"
|
||||||
|
)
|
||||||
|
with pytest.raises(RollBackAuthorityChange):
|
||||||
|
async with connection.transaction():
|
||||||
|
await connection.execute(
|
||||||
|
"SELECT * FROM public.kf_manage_account_authority('storage_user','revoke')"
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
await connection.fetchval(
|
||||||
|
"SELECT authorization_version FROM accounts WHERE id=$1", account
|
||||||
|
)
|
||||||
|
== 3
|
||||||
|
)
|
||||||
|
raise RollBackAuthorityChange
|
||||||
|
assert await connection.fetchval(
|
||||||
|
"SELECT is_administrator FROM accounts WHERE id=$1", account
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
await connection.fetchval(
|
||||||
|
"SELECT authorization_version FROM accounts WHERE id=$1", account
|
||||||
|
)
|
||||||
|
== 2
|
||||||
|
)
|
||||||
|
await service.authenticate(fresh_token, csrf_token=fresh_csrf)
|
||||||
|
async with connection.transaction():
|
||||||
|
await connection.execute("SET LOCAL ROLE kf_oj_identity_operator")
|
||||||
|
await connection.execute(
|
||||||
|
"SELECT * FROM public.kf_manage_account_authority('storage_user','revoke')"
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
await connection.fetchval(
|
||||||
|
"SELECT authorization_version FROM accounts WHERE id=$1", account
|
||||||
|
)
|
||||||
|
== 3
|
||||||
|
)
|
||||||
|
with pytest.raises(AuthenticationFailed):
|
||||||
|
await service.authenticate(fresh_token, csrf_token=fresh_csrf)
|
||||||
|
finally:
|
||||||
|
await connection.close()
|
||||||
|
await engine.dispose()
|
||||||
|
|
||||||
|
asyncio.run(inspect())
|
||||||
|
|
||||||
|
|
||||||
|
def test_authority_history_cannot_be_erased_by_downgrade(postgres_url):
|
||||||
|
async def change():
|
||||||
|
connection = await connect(postgres_url)
|
||||||
|
try:
|
||||||
|
account, _, _ = await account_and_session(connection)
|
||||||
|
await connection.execute("UPDATE accounts SET enabled=false WHERE id=$1", account)
|
||||||
|
with pytest.raises(asyncpg.CheckViolationError):
|
||||||
|
await connection.execute(
|
||||||
|
"UPDATE accounts SET authorization_version=1 WHERE id=$1", account
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
await connection.close()
|
||||||
|
|
||||||
|
asyncio.run(change())
|
||||||
|
with pytest.raises(subprocess.CalledProcessError):
|
||||||
|
migrate(postgres_url, "downgrade", "0004_performance_watermark")
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
"""Trusted adapter checks; real cold PG/role/permission tests remain VM-only."""
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
|
||||||
|
from kf_oj.contracts.judging import ArtifactReference
|
||||||
|
from kf_oj.infra.artifacts.backup import verify_existing
|
||||||
|
|
||||||
|
|
||||||
|
def test_backup_verifies_binary_and_empty_objects_without_creating_storage_entries(tmp_path):
|
||||||
|
tmp_path.chmod(0o700)
|
||||||
|
values = [b"", b"\0\xff\n" * 50000]
|
||||||
|
expected = {}
|
||||||
|
for value in values:
|
||||||
|
digest = hashlib.sha256(value).hexdigest()
|
||||||
|
path = tmp_path / digest
|
||||||
|
path.write_bytes(value)
|
||||||
|
path.chmod(0o600)
|
||||||
|
expected[digest] = value
|
||||||
|
before = {p.name: (p.read_bytes(), p.stat().st_mtime_ns) for p in tmp_path.iterdir()}
|
||||||
|
for digest, value in expected.items():
|
||||||
|
verify_existing(tmp_path, ArtifactReference(digest=digest, bytes=len(value)))
|
||||||
|
after = {p.name: (p.read_bytes(), p.stat().st_mtime_ns) for p in tmp_path.iterdir()}
|
||||||
|
assert after == before
|
||||||
|
assert not (tmp_path / ".lock").exists()
|
||||||
@@ -0,0 +1,219 @@
|
|||||||
|
"""Real local authority/CLI transactions; run privilege faults in the independent VM."""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
import asyncpg
|
||||||
|
import pytest
|
||||||
|
from conftest import connect
|
||||||
|
from sqlalchemy.engine import make_url
|
||||||
|
|
||||||
|
from kf_oj.modules.identity.authority import manage_authority
|
||||||
|
|
||||||
|
|
||||||
|
async def seed(connection, *usernames):
|
||||||
|
for name in usernames:
|
||||||
|
await connection.execute(
|
||||||
|
"INSERT INTO accounts(id,username,password_hash) VALUES($1,$2,repeat('synthetic-',6))",
|
||||||
|
uuid4(),
|
||||||
|
name,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def authority(connection, username, operation):
|
||||||
|
async with connection.transaction():
|
||||||
|
await connection.execute("SET LOCAL ROLE kf_oj_identity_operator")
|
||||||
|
return await connection.fetchrow(
|
||||||
|
"SELECT * FROM public.kf_manage_account_authority($1,$2)", username, operation
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_explicit_bootstrap_repeated_grants_recovery_and_last_administrator(postgres_url):
|
||||||
|
async def run():
|
||||||
|
c = await connect(postgres_url)
|
||||||
|
try:
|
||||||
|
await seed(c, "first_admin", "second_admin")
|
||||||
|
assert await c.fetchval("SELECT count(*) FROM accounts WHERE is_administrator") == 0
|
||||||
|
with pytest.raises(asyncpg.CheckViolationError):
|
||||||
|
await authority(c, "first_admin", "grant")
|
||||||
|
first = await authority(c, "first_admin", "bootstrap")
|
||||||
|
assert first["changed"] and first["authorization_version"] == 2
|
||||||
|
assert not (await authority(c, "first_admin", "bootstrap"))["changed"]
|
||||||
|
assert not (await authority(c, "first_admin", "grant"))["changed"]
|
||||||
|
with pytest.raises(asyncpg.CheckViolationError):
|
||||||
|
await authority(c, "second_admin", "bootstrap")
|
||||||
|
for operation in ("disable", "revoke"):
|
||||||
|
with pytest.raises(asyncpg.CheckViolationError):
|
||||||
|
await authority(c, "first_admin", operation)
|
||||||
|
await authority(c, "second_admin", "grant")
|
||||||
|
disabled = await authority(c, "first_admin", "disable")
|
||||||
|
assert not disabled["enabled"] and disabled["is_administrator"]
|
||||||
|
assert disabled["authorization_version"] == 3
|
||||||
|
assert not (await authority(c, "first_admin", "disable"))["changed"]
|
||||||
|
restored = await authority(c, "first_admin", "enable")
|
||||||
|
assert restored["enabled"] and restored["authorization_version"] == 4
|
||||||
|
revoked = await authority(c, "first_admin", "revoke")
|
||||||
|
assert not revoked["is_administrator"] and revoked["authorization_version"] == 5
|
||||||
|
assert not (await authority(c, "first_admin", "revoke"))["changed"]
|
||||||
|
await authority(c, "first_admin", "grant")
|
||||||
|
with pytest.raises(asyncpg.NoDataFoundError):
|
||||||
|
await authority(c, "absent_user", "grant")
|
||||||
|
for role in ("kf_oj_platform", "kf_oj_coordinator", "kf_oj_problem_import"):
|
||||||
|
assert not await c.fetchval(
|
||||||
|
"SELECT has_function_privilege($1,"
|
||||||
|
"'kf_manage_account_authority(text,text)','EXECUTE')",
|
||||||
|
role,
|
||||||
|
)
|
||||||
|
for column in ("password_hash", "is_administrator", "enabled"):
|
||||||
|
assert not await c.fetchval(
|
||||||
|
"SELECT has_column_privilege('kf_oj_identity_operator','accounts',$1,'SELECT')",
|
||||||
|
column,
|
||||||
|
)
|
||||||
|
async with c.transaction():
|
||||||
|
await c.execute("SET LOCAL ROLE kf_oj_identity_operator")
|
||||||
|
for sql in (
|
||||||
|
"SELECT password_hash FROM accounts",
|
||||||
|
"UPDATE accounts SET is_administrator=false",
|
||||||
|
"SELECT token_hash FROM sessions",
|
||||||
|
):
|
||||||
|
with pytest.raises(asyncpg.InsufficientPrivilegeError):
|
||||||
|
async with c.transaction():
|
||||||
|
await c.execute(sql)
|
||||||
|
finally:
|
||||||
|
await c.close()
|
||||||
|
|
||||||
|
asyncio.run(run())
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("isolation", ("read_committed", "repeatable_read", "serializable"))
|
||||||
|
def test_concurrent_different_account_revocations_preserve_one_admin(postgres_url, isolation):
|
||||||
|
async def serialized_run():
|
||||||
|
c, other = await connect(postgres_url), await connect(postgres_url)
|
||||||
|
first_tx = second_tx = pending = None
|
||||||
|
try:
|
||||||
|
await seed(c, "first_admin", "second_admin")
|
||||||
|
await authority(c, "first_admin", "bootstrap")
|
||||||
|
await authority(c, "second_admin", "grant")
|
||||||
|
first_tx, second_tx = (
|
||||||
|
c.transaction(isolation=isolation),
|
||||||
|
other.transaction(isolation=isolation),
|
||||||
|
)
|
||||||
|
await first_tx.start()
|
||||||
|
await second_tx.start()
|
||||||
|
assert (
|
||||||
|
await other.fetchval(
|
||||||
|
"SELECT count(*) FROM accounts WHERE is_administrator AND enabled"
|
||||||
|
)
|
||||||
|
== 2
|
||||||
|
)
|
||||||
|
await authority(c, "first_admin", "revoke")
|
||||||
|
pending = asyncio.create_task(authority(other, "second_admin", "revoke"))
|
||||||
|
deadline = time.monotonic() + 5
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
if await c.fetchval(
|
||||||
|
"SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1 AND NOT granted)",
|
||||||
|
other.get_server_pid(),
|
||||||
|
):
|
||||||
|
break
|
||||||
|
await asyncio.sleep(0.01)
|
||||||
|
else:
|
||||||
|
raise AssertionError("second authority operation never blocked on PG lock")
|
||||||
|
assert not pending.done()
|
||||||
|
await first_tx.commit()
|
||||||
|
first_tx = None
|
||||||
|
with pytest.raises((asyncpg.CheckViolationError, asyncpg.SerializationError)):
|
||||||
|
await asyncio.wait_for(pending, 5)
|
||||||
|
await second_tx.rollback()
|
||||||
|
second_tx = None
|
||||||
|
assert (
|
||||||
|
await c.fetchval("SELECT count(*) FROM accounts WHERE is_administrator AND enabled")
|
||||||
|
== 1
|
||||||
|
)
|
||||||
|
assert await c.fetchval(
|
||||||
|
"SELECT is_administrator FROM accounts WHERE username='second_admin'"
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
if pending is not None and not pending.done():
|
||||||
|
pending.cancel()
|
||||||
|
await asyncio.gather(pending, return_exceptions=True)
|
||||||
|
if first_tx is not None:
|
||||||
|
await first_tx.rollback()
|
||||||
|
if second_tx is not None:
|
||||||
|
await second_tx.rollback()
|
||||||
|
await c.close()
|
||||||
|
await other.close()
|
||||||
|
|
||||||
|
asyncio.run(serialized_run())
|
||||||
|
|
||||||
|
|
||||||
|
def test_real_cli_uses_dedicated_login_and_never_prints_database_secret(postgres_url):
|
||||||
|
role = "authority_test_" + uuid4().hex
|
||||||
|
|
||||||
|
async def setup():
|
||||||
|
c = await connect(postgres_url)
|
||||||
|
try:
|
||||||
|
await seed(c, "cli_admin", "cli_backup")
|
||||||
|
await c.execute(
|
||||||
|
f"CREATE ROLE {role} LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE "
|
||||||
|
"NOREPLICATION NOBYPASSRLS"
|
||||||
|
)
|
||||||
|
await c.execute(f"GRANT kf_oj_identity_operator TO {role}")
|
||||||
|
finally:
|
||||||
|
await c.close()
|
||||||
|
|
||||||
|
asyncio.run(setup())
|
||||||
|
url = make_url(postgres_url).set(username=role).render_as_string(hide_password=False)
|
||||||
|
env = os.environ | {
|
||||||
|
"KF_OJ_DATABASE_URL": postgres_url,
|
||||||
|
"KF_OJ_IDENTITY_OPERATOR_DATABASE_URL": url,
|
||||||
|
}
|
||||||
|
cwd = Path(__file__).resolve().parents[1]
|
||||||
|
|
||||||
|
def invoke(command, username, expected=0):
|
||||||
|
result = subprocess.run(
|
||||||
|
[sys.executable, "-m", "kf_oj.modules.identity.cli", command, username],
|
||||||
|
cwd=cwd,
|
||||||
|
env=env,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=120,
|
||||||
|
)
|
||||||
|
assert result.returncode == expected, result.stderr
|
||||||
|
assert url not in result.stdout + result.stderr
|
||||||
|
assert "Traceback" not in result.stderr
|
||||||
|
return result.stdout
|
||||||
|
|
||||||
|
try:
|
||||||
|
assert "changed=True" in invoke("bootstrap-admin", "cli_admin")
|
||||||
|
assert "changed=False" in invoke("bootstrap-admin", "cli_admin")
|
||||||
|
invoke("revoke-admin", "cli_admin", 1)
|
||||||
|
invoke("grant-admin", "cli_backup")
|
||||||
|
invoke("disable", "cli_admin")
|
||||||
|
invoke("enable", "cli_admin")
|
||||||
|
invoke("revoke-admin", "cli_admin")
|
||||||
|
invoke("grant-admin", "cli_admin")
|
||||||
|
finally:
|
||||||
|
|
||||||
|
async def cleanup():
|
||||||
|
c = await connect(postgres_url)
|
||||||
|
try:
|
||||||
|
await c.execute(f"REVOKE kf_oj_identity_operator FROM {role}")
|
||||||
|
await c.execute(f"DROP ROLE {role}")
|
||||||
|
finally:
|
||||||
|
await c.close()
|
||||||
|
|
||||||
|
asyncio.run(cleanup())
|
||||||
|
|
||||||
|
|
||||||
|
def test_bad_cli_operation_rejected_without_connecting():
|
||||||
|
async def run():
|
||||||
|
for username, operation in (("invalid name", "grant"), ("valid_name", "unknown")):
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
await manage_authority(None, username, operation)
|
||||||
|
|
||||||
|
asyncio.run(run())
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
"""Contract/admission-route tests; Root file/atomicity and drain acceptance are VM-only."""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from jsonschema import Draft202012Validator
|
||||||
|
|
||||||
|
from kf_oj.api.availability import AvailabilityMiddleware, needs_open
|
||||||
|
from kf_oj.contracts.availability import AvailabilityRecord
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("state", ["open", "draining", "quiescent"])
|
||||||
|
def test_availability_is_exactly_bound_to_a_known_version_and_state(state):
|
||||||
|
path = (
|
||||||
|
Path(__file__).resolve().parents[2]
|
||||||
|
/ "tests/fixtures/contracts/delivery-availability.valid.json"
|
||||||
|
)
|
||||||
|
value = json.loads(path.read_text())
|
||||||
|
value["state"] = state
|
||||||
|
assert AvailabilityRecord.model_validate(value).state == state
|
||||||
|
schema = path.parents[3] / "contracts/schemas/delivery-availability.v1.schema.json"
|
||||||
|
Draft202012Validator(json.loads(schema.read_text())).validate(value)
|
||||||
|
for key, replacement in (
|
||||||
|
("schema_version", True),
|
||||||
|
("release_id", "../release"),
|
||||||
|
("state", "always_allow"),
|
||||||
|
):
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
AvailabilityRecord.model_validate({**value, key: replacement})
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"method,path,blocked",
|
||||||
|
[
|
||||||
|
("POST", "/api/v1/submissions", True),
|
||||||
|
("POST", "/api/v1/admin/problem-imports", True),
|
||||||
|
("POST", "/api/v1/admin/problems/publish", True),
|
||||||
|
("POST", "/api/v1/auth/login", True),
|
||||||
|
("GET", "/api/v1/internal/evaluations/artifacts", False),
|
||||||
|
("POST", "/api/v1/auth/logout", False),
|
||||||
|
("POST", "/api/v1/submissions/11111111-1111-4111-8111-111111111111/cancel", False),
|
||||||
|
(
|
||||||
|
"PUT",
|
||||||
|
"/api/v1/admin/problem-imports/11111111-1111-4111-8111-111111111111/archive",
|
||||||
|
False,
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_drain_blocks_new_mutations_without_cutting_accepted_reads_and_cancel(
|
||||||
|
method, path, blocked
|
||||||
|
):
|
||||||
|
assert needs_open(method, path) is blocked
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_admission_record_rejects_before_body_or_domain_execution(monkeypatch):
|
||||||
|
# Transport policy only: this stub does not verify Root ownership or Linux file safety.
|
||||||
|
calls = []
|
||||||
|
messages = []
|
||||||
|
|
||||||
|
def missing(*_):
|
||||||
|
raise FileNotFoundError("record missing")
|
||||||
|
|
||||||
|
monkeypatch.setattr("kf_oj.api.availability.current", missing)
|
||||||
|
|
||||||
|
async def receive():
|
||||||
|
calls.append("body")
|
||||||
|
return {"type": "http.request", "body": b""}
|
||||||
|
|
||||||
|
async def domain(*_):
|
||||||
|
calls.append("domain")
|
||||||
|
|
||||||
|
async def send(message):
|
||||||
|
messages.append(message)
|
||||||
|
|
||||||
|
settings = SimpleNamespace(
|
||||||
|
environment="production", delivery_release_id=None, delivery_configuration_id=None
|
||||||
|
)
|
||||||
|
asyncio.run(
|
||||||
|
AvailabilityMiddleware(domain, settings=settings)(
|
||||||
|
{"type": "http", "method": "POST", "path": "/api/v1/submissions"}, receive, send
|
||||||
|
)
|
||||||
|
)
|
||||||
|
assert calls == []
|
||||||
|
assert messages[0]["status"] == 503
|
||||||
|
assert (b"cache-control", b"no-store") in messages[0]["headers"]
|
||||||
@@ -28,7 +28,13 @@ ROOT = Path(__file__).resolve().parents[2]
|
|||||||
|
|
||||||
@asynccontextmanager
|
@asynccontextmanager
|
||||||
async def fixture(
|
async def fixture(
|
||||||
postgres_url, tmp_path, monkeypatch, *, import_builder=import_fixture, node_policy=None
|
postgres_url,
|
||||||
|
tmp_path,
|
||||||
|
monkeypatch,
|
||||||
|
*,
|
||||||
|
import_builder=import_fixture,
|
||||||
|
node_policy=None,
|
||||||
|
measurement_epochs=None,
|
||||||
):
|
):
|
||||||
_, policy = import_builder(tmp_path / "import")
|
_, policy = import_builder(tmp_path / "import")
|
||||||
engine = create_async_engine(postgres_url)
|
engine = create_async_engine(postgres_url)
|
||||||
@@ -59,6 +65,9 @@ async def fixture(
|
|||||||
reserved.bind(("127.0.0.1", 0))
|
reserved.bind(("127.0.0.1", 0))
|
||||||
port = reserved.getsockname()[1]
|
port = reserved.getsockname()[1]
|
||||||
workers = [dict(id=str(uuid4()), token=secrets.token_hex(32)) for _ in range(2)]
|
workers = [dict(id=str(uuid4()), token=secrets.token_hex(32)) for _ in range(2)]
|
||||||
|
if measurement_epochs is not None:
|
||||||
|
for worker, epoch in zip(workers, measurement_epochs, strict=True):
|
||||||
|
worker["measurement_epoch"] = epoch
|
||||||
platform = secrets.token_hex(32)
|
platform = secrets.token_hex(32)
|
||||||
config = tmp_path / "coordinator.json"
|
config = tmp_path / "coordinator.json"
|
||||||
config.write_text(
|
config.write_text(
|
||||||
@@ -152,6 +161,7 @@ async def fixture(
|
|||||||
client.fixture_version = version
|
client.fixture_version = version
|
||||||
client.coordinator_unit = unit
|
client.coordinator_unit = unit
|
||||||
client.coordinator_role = role
|
client.coordinator_role = role
|
||||||
|
client.coordinator_config = config
|
||||||
yield client, c, workers, platform, create
|
yield client, c, workers, platform, create
|
||||||
finally:
|
finally:
|
||||||
subprocess.run(["systemctl", "--user", "stop", unit], check=True, capture_output=True)
|
subprocess.run(["systemctl", "--user", "stop", unit], check=True, capture_output=True)
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
import json
|
import json
|
||||||
|
import subprocess
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
import asyncpg
|
import asyncpg
|
||||||
@@ -12,9 +13,15 @@ def test_migration_roundtrip_and_role_safety(postgres_url):
|
|||||||
c = await connect(postgres_url)
|
c = await connect(postgres_url)
|
||||||
try:
|
try:
|
||||||
tables = set(await c.fetch("SELECT tablename FROM pg_tables WHERE schemaname='public'"))
|
tables = set(await c.fetch("SELECT tablename FROM pg_tables WHERE schemaname='public'"))
|
||||||
assert len(tables) == 13 # Twelve business tables and Alembic history.
|
assert len(tables) == 21 # Twenty domain/control tables and Alembic history.
|
||||||
roles = await c.fetch("SELECT * FROM pg_roles WHERE rolname LIKE 'kf_oj_%'")
|
roles = await c.fetch("SELECT * FROM pg_roles WHERE rolname LIKE 'kf_oj_%'")
|
||||||
assert {row["rolname"] for row in roles} == {"kf_oj_platform", "kf_oj_coordinator"}
|
assert {row["rolname"] for row in roles} == {
|
||||||
|
"kf_oj_platform",
|
||||||
|
"kf_oj_coordinator",
|
||||||
|
"kf_oj_identity_operator",
|
||||||
|
"kf_oj_problem_import",
|
||||||
|
"kf_oj_identity_authority_owner",
|
||||||
|
}
|
||||||
assert all(
|
assert all(
|
||||||
not any(
|
not any(
|
||||||
r[k]
|
r[k]
|
||||||
@@ -38,7 +45,14 @@ def test_migration_roundtrip_and_role_safety(postgres_url):
|
|||||||
asyncio.run(inspect())
|
asyncio.run(inspect())
|
||||||
migrate(postgres_url, "downgrade", "base")
|
migrate(postgres_url, "downgrade", "base")
|
||||||
migrate(postgres_url, "upgrade", "head")
|
migrate(postgres_url, "upgrade", "head")
|
||||||
migrate(postgres_url, "check")
|
try:
|
||||||
|
migrate(postgres_url, "check")
|
||||||
|
except subprocess.CalledProcessError as error:
|
||||||
|
raise AssertionError(
|
||||||
|
"migration model consistency failed: "
|
||||||
|
+ (error.stdout or b"").decode(errors="replace")[-8192:]
|
||||||
|
+ (error.stderr or b"").decode(errors="replace")[-8192:]
|
||||||
|
) from None
|
||||||
asyncio.run(inspect())
|
asyncio.run(inspect())
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,349 @@
|
|||||||
|
"""Trusted byte fixtures only; no privileged backup, recovery or RTO acceptance."""
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import importlib
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
sys.path.insert(0, str(ROOT))
|
||||||
|
backup_manifest = importlib.import_module("scripts.release.backup_manifest")
|
||||||
|
backup_objects = importlib.import_module("scripts.release.backup_objects")
|
||||||
|
policy = importlib.import_module("scripts.release.policy")
|
||||||
|
|
||||||
|
|
||||||
|
def empty_drain_projection():
|
||||||
|
return {
|
||||||
|
"execution": {
|
||||||
|
"database_clock_unix_ms": 1791830000000,
|
||||||
|
**{
|
||||||
|
name: {"count_capped": 0, "truncated": False}
|
||||||
|
for name in ("queued", "in_flight", "cleanup_unconfirmed")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"outbox": {
|
||||||
|
"schema_version": 1,
|
||||||
|
"pending_capped": 0,
|
||||||
|
"locked_capped": 0,
|
||||||
|
"truncated": False,
|
||||||
|
},
|
||||||
|
"imports": {
|
||||||
|
"schema_version": 1,
|
||||||
|
"worker_ready": True,
|
||||||
|
"reserved_bytes": 0,
|
||||||
|
"uploaded_capped": 0,
|
||||||
|
"validating_capped": 0,
|
||||||
|
"truncated": False,
|
||||||
|
},
|
||||||
|
"broker": {
|
||||||
|
"schema_version": 1,
|
||||||
|
"metadata_verified": True,
|
||||||
|
"messages_ready": 0,
|
||||||
|
"messages_unacknowledged": 0,
|
||||||
|
"message_bytes": 0,
|
||||||
|
"consumers": 1,
|
||||||
|
},
|
||||||
|
"factory": {
|
||||||
|
"schema_version": 1,
|
||||||
|
"configuration_sha256": "b" * 64,
|
||||||
|
"supervisor_authorized": True,
|
||||||
|
"execution_gate": "required_per_execution",
|
||||||
|
"isolation_verified": False,
|
||||||
|
"max_concurrent_runs": 1,
|
||||||
|
"active_helpers": 0,
|
||||||
|
"cleanup_complete": True,
|
||||||
|
"remaining_requests": 4096,
|
||||||
|
"remaining_lifetime_seconds": 3600,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def snapshot():
|
||||||
|
data = b"trusted repeated bytes\n"
|
||||||
|
digest = hashlib.sha256(data).hexdigest()
|
||||||
|
files = [
|
||||||
|
{
|
||||||
|
"path": path,
|
||||||
|
"uid": uid,
|
||||||
|
"gid": uid,
|
||||||
|
"mode": mode,
|
||||||
|
"bytes": len(data),
|
||||||
|
"sha256": digest,
|
||||||
|
}
|
||||||
|
for path, uid, mode in [
|
||||||
|
("postgres/PG_VERSION", 1203, 0o600),
|
||||||
|
("postgres/global/pg_control", 1203, 0o600),
|
||||||
|
("releases/first", 0, 0o444),
|
||||||
|
("releases/second", 0, 0o555),
|
||||||
|
]
|
||||||
|
]
|
||||||
|
boot = "12345678-1234-1234-1234-123456789abc"
|
||||||
|
selected = {
|
||||||
|
"schema_version": 1,
|
||||||
|
"release_id": "sd-" + "c" * 12,
|
||||||
|
"manifest_sha256": "b" * 64,
|
||||||
|
"configuration_id": "cfg-" + "d" * 24,
|
||||||
|
"receipt_sha256": "e" * 64,
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
"schema_version": 3,
|
||||||
|
"format": backup_manifest.FORMAT,
|
||||||
|
"backup_id": "bk-" + "2" * 32,
|
||||||
|
"source_sha": "c" * 40,
|
||||||
|
"manifest_sha256": selected["manifest_sha256"],
|
||||||
|
"configuration_id": selected["configuration_id"],
|
||||||
|
"created_unix_seconds": 1793070000,
|
||||||
|
"expires_unix_seconds": 1793070000 + 7 * 86400,
|
||||||
|
"selected": selected,
|
||||||
|
"consistent_cut": {
|
||||||
|
"schema_version": 1,
|
||||||
|
"source_sha": "c" * 40,
|
||||||
|
"configuration_id": selected["configuration_id"],
|
||||||
|
"action": "stop",
|
||||||
|
"phase": "complete",
|
||||||
|
"success": True,
|
||||||
|
"events": [],
|
||||||
|
"root_cleanup_confirmed": True,
|
||||||
|
"boot_id": boot,
|
||||||
|
"drain_projection": empty_drain_projection(),
|
||||||
|
},
|
||||||
|
"artifact_references": {
|
||||||
|
"schema_version": 1,
|
||||||
|
"kind": "kf.private.artifact-backup-inventory",
|
||||||
|
"read_only": True,
|
||||||
|
"quiescence_verified": False,
|
||||||
|
"boot_id": boot,
|
||||||
|
"storage_identity": {"device": 1, "inode": 2, "uid": 1200},
|
||||||
|
"objects": [],
|
||||||
|
},
|
||||||
|
"broker_cookie_sha256": "b" * 64,
|
||||||
|
"database_bootstrap_sha256": "e" * 64,
|
||||||
|
"images": {
|
||||||
|
key: {
|
||||||
|
"toolchain_id": key,
|
||||||
|
"image_kind": key,
|
||||||
|
"runtime_version": "1",
|
||||||
|
"image_revision": 1,
|
||||||
|
"image_digest": "f" * 64,
|
||||||
|
}
|
||||||
|
for key in ("gnu", "python313", "java21")
|
||||||
|
},
|
||||||
|
"directories": [
|
||||||
|
{"path": path, "uid": 0, "gid": 0, "mode": 0o700}
|
||||||
|
for path in sorted(backup_manifest.DOMAINS | {"postgres/global"})
|
||||||
|
],
|
||||||
|
"files": files,
|
||||||
|
"payload_bytes": len(data) * len(files),
|
||||||
|
"restored": False,
|
||||||
|
"execution_verified": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def object_format(value):
|
||||||
|
return {
|
||||||
|
**value,
|
||||||
|
"schema_version": 4,
|
||||||
|
"format": backup_manifest.OBJECT_FORMAT,
|
||||||
|
"logical_bytes": value["payload_bytes"],
|
||||||
|
"payload_bytes": value["files"][0]["bytes"],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_v3_reader_and_v4_logical_permissions_survive_content_deduplication(snapshot, tmp_path):
|
||||||
|
old = backup_manifest.validate(snapshot)
|
||||||
|
new = backup_manifest.validate(object_format(snapshot))
|
||||||
|
assert len(backup_manifest.physical_files(old)) == 4
|
||||||
|
assert len(backup_manifest.physical_files(new)) == 1
|
||||||
|
assert backup_manifest.logical_bytes(old) == backup_manifest.logical_bytes(new)
|
||||||
|
assert new["payload_bytes"] * 4 == new["logical_bytes"]
|
||||||
|
assert new["files"][2]["mode"] != new["files"][3]["mode"]
|
||||||
|
for item in new["files"]:
|
||||||
|
assert backup_manifest.member_path(tmp_path, new, item) == (
|
||||||
|
tmp_path / "objects" / item["sha256"]
|
||||||
|
)
|
||||||
|
assert backup_manifest.named_member(tmp_path, old, item["path"]) == (
|
||||||
|
tmp_path / "payload" / item["path"]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"field,change",
|
||||||
|
[
|
||||||
|
("payload_bytes", 1),
|
||||||
|
("logical_bytes", 1),
|
||||||
|
("schema_version", True),
|
||||||
|
("format", backup_manifest.FORMAT),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_v4_cannot_confuse_stored_bytes_with_restored_bytes(snapshot, field, change):
|
||||||
|
value = object_format(snapshot)
|
||||||
|
value[field] = change
|
||||||
|
with pytest.raises(policy.DeliveryError):
|
||||||
|
backup_manifest.validate(value)
|
||||||
|
|
||||||
|
|
||||||
|
def test_v4_preserves_logical_versions_above_v3_limit_without_raising_physical_limit(snapshot):
|
||||||
|
value = object_format(snapshot)
|
||||||
|
size = backup_manifest.MAX_FILE_BYTES
|
||||||
|
value["files"] = [
|
||||||
|
{**value["files"][0], "path": "releases/version-" + str(index), "bytes": size}
|
||||||
|
for index in range(32)
|
||||||
|
] + [{**item, "bytes": size} for item in snapshot["files"][:2]]
|
||||||
|
value["logical_bytes"] = len(value["files"]) * size
|
||||||
|
value["payload_bytes"] = size
|
||||||
|
backup_manifest.validate(value)
|
||||||
|
assert value["logical_bytes"] > backup_manifest.MAX_BYTES
|
||||||
|
old = {
|
||||||
|
**value,
|
||||||
|
"schema_version": 3,
|
||||||
|
"format": backup_manifest.FORMAT,
|
||||||
|
"payload_bytes": value["logical_bytes"],
|
||||||
|
}
|
||||||
|
old.pop("logical_bytes")
|
||||||
|
with pytest.raises(policy.DeliveryError):
|
||||||
|
backup_manifest.validate(old)
|
||||||
|
# Distinct object digests still consume the unchanged physical budget.
|
||||||
|
for index, item in enumerate(value["files"]):
|
||||||
|
item["sha256"] = hashlib.sha256(str(index).encode()).hexdigest()
|
||||||
|
value["payload_bytes"] = value["logical_bytes"]
|
||||||
|
with pytest.raises(policy.DeliveryError):
|
||||||
|
backup_manifest.validate(value)
|
||||||
|
|
||||||
|
|
||||||
|
def test_store_reads_each_source_but_writes_one_single_link_object(tmp_path, monkeypatch):
|
||||||
|
source, repeated, objects = tmp_path / "source", tmp_path / "repeated", tmp_path / "objects"
|
||||||
|
objects.mkdir(mode=0o700)
|
||||||
|
data = b"trusted source\n" * 2000
|
||||||
|
source.write_bytes(data)
|
||||||
|
repeated.write_bytes(data)
|
||||||
|
repeated.chmod(0o444)
|
||||||
|
original = backup_objects.read_stream
|
||||||
|
observed = []
|
||||||
|
|
||||||
|
def read(incoming, before, deadline, outgoing=None):
|
||||||
|
observed.append((before.st_ino, outgoing is not None))
|
||||||
|
return original(incoming, before, deadline, outgoing)
|
||||||
|
|
||||||
|
monkeypatch.setattr(backup_objects, "read_stream", read)
|
||||||
|
seen = {}
|
||||||
|
first, added = backup_objects.store(
|
||||||
|
source, source.stat(), objects, seen, time.monotonic() + 10, remaining_bytes=len(data)
|
||||||
|
)
|
||||||
|
second, repeated_bytes = backup_objects.store(
|
||||||
|
repeated, repeated.stat(), objects, seen, time.monotonic() + 10, remaining_bytes=0
|
||||||
|
)
|
||||||
|
assert first == second == hashlib.sha256(data).hexdigest()
|
||||||
|
assert (added, repeated_bytes) == (len(data), 0)
|
||||||
|
assert observed == [
|
||||||
|
(source.stat().st_ino, False),
|
||||||
|
(source.stat().st_ino, True),
|
||||||
|
(repeated.stat().st_ino, False),
|
||||||
|
]
|
||||||
|
assert list(objects.iterdir()) == [objects / first]
|
||||||
|
assert (objects / first).read_bytes() == data
|
||||||
|
assert (objects / first).stat().st_nlink == 1
|
||||||
|
assert (objects / first).stat().st_mode & 0o777 == 0o600
|
||||||
|
|
||||||
|
|
||||||
|
def test_failed_second_source_read_retains_incomplete_private_copy(tmp_path, monkeypatch):
|
||||||
|
source, objects = tmp_path / "source", tmp_path / "objects"
|
||||||
|
source.write_bytes(b"trusted fixture")
|
||||||
|
objects.mkdir(mode=0o700)
|
||||||
|
original = backup_objects.read_stream
|
||||||
|
|
||||||
|
def read(incoming, before, deadline, outgoing=None):
|
||||||
|
if outgoing is not None:
|
||||||
|
raise OSError("fixture write failure")
|
||||||
|
return original(incoming, before, deadline)
|
||||||
|
|
||||||
|
monkeypatch.setattr(backup_objects, "read_stream", read)
|
||||||
|
seen = {}
|
||||||
|
with pytest.raises(OSError):
|
||||||
|
backup_objects.store(
|
||||||
|
source, source.stat(), objects, seen, time.monotonic() + 10, remaining_bytes=100
|
||||||
|
)
|
||||||
|
assert seen == {}
|
||||||
|
assert list(objects.iterdir()) == [objects / ".pending"]
|
||||||
|
assert not (tmp_path / "manifest.json").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_restore_expands_repeated_content_into_independent_files_with_own_modes(tmp_path):
|
||||||
|
restore = importlib.import_module("scripts.release.restore")
|
||||||
|
content = tmp_path / "object"
|
||||||
|
data = b"trusted restoration fixture\n"
|
||||||
|
content.write_bytes(data)
|
||||||
|
row = {"bytes": len(data), "sha256": hashlib.sha256(data).hexdigest()}
|
||||||
|
first, second = tmp_path / "first", tmp_path / "second"
|
||||||
|
for path, mode in ((first, 0o600), (second, 0o444)):
|
||||||
|
witness = restore.write_file(
|
||||||
|
content, path, row, os.getuid(), os.getgid(), mode, time.monotonic() + 10
|
||||||
|
)
|
||||||
|
restore.unchanged_file(path, witness, os.getuid(), os.getgid(), mode)
|
||||||
|
assert path.read_bytes() == data
|
||||||
|
assert path.stat().st_mode & 0o777 == mode
|
||||||
|
assert path.stat().st_nlink == 1
|
||||||
|
assert len({path.stat().st_ino for path in (content, first, second)}) == 3
|
||||||
|
|
||||||
|
|
||||||
|
def test_delayed_filesystem_write_error_is_not_success(tmp_path, monkeypatch):
|
||||||
|
class SyncFailure:
|
||||||
|
def __call__(self, descriptor):
|
||||||
|
assert os.fstat(descriptor).st_ino == tmp_path.stat().st_ino
|
||||||
|
return -1
|
||||||
|
|
||||||
|
class Library:
|
||||||
|
syncfs = SyncFailure()
|
||||||
|
|
||||||
|
monkeypatch.setattr(backup_objects.ctypes, "CDLL", lambda *_, **__: Library())
|
||||||
|
with pytest.raises(policy.DeliveryError, match="filesystem synchronization failed"):
|
||||||
|
backup_objects.sync_filesystem(tmp_path, time.monotonic() + 10)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("missing", ["releases/older", "releases/older/nested"])
|
||||||
|
def test_every_directory_ancestor_is_required_even_when_file_parent_exists(snapshot, missing):
|
||||||
|
value = object_format(snapshot)
|
||||||
|
value["directories"] = value["directories"] + [
|
||||||
|
{"path": path, "uid": 0, "gid": 0, "mode": 0o700}
|
||||||
|
for path in ("releases/older", "releases/older/nested")
|
||||||
|
if path != missing
|
||||||
|
]
|
||||||
|
value["files"] = [
|
||||||
|
{**row, "path": "releases/older/nested/file"} if row["path"] == "releases/first" else row
|
||||||
|
for row in value["files"]
|
||||||
|
]
|
||||||
|
with pytest.raises(policy.DeliveryError, match="private backup parent missing"):
|
||||||
|
backup_manifest.validate(value)
|
||||||
|
|
||||||
|
|
||||||
|
def test_v4_complete_version_inventory_keeps_an_explicit_logical_entry_bound(snapshot):
|
||||||
|
value = object_format(snapshot)
|
||||||
|
count = backup_manifest.MAX_LOGICAL_ENTRIES - len(value["directories"])
|
||||||
|
template = {**value["files"][0], "bytes": 1, "sha256": hashlib.sha256(b"x").hexdigest()}
|
||||||
|
value["files"] = [{**template, "path": row["path"]} for row in value["files"][:2]] + [
|
||||||
|
{**template, "path": "releases/item-" + str(i)} for i in range(count - 2)
|
||||||
|
]
|
||||||
|
value.update(logical_bytes=count, payload_bytes=1)
|
||||||
|
backup_manifest.validate(value)
|
||||||
|
assert len(value["files"]) + len(value["directories"]) == 98304
|
||||||
|
assert backup_manifest.MAX_BYTES == 7 * 1024**3
|
||||||
|
assert backup_manifest.MAX_LOGICAL_BYTES == 12 * 1024**3
|
||||||
|
old = {**value, "schema_version": 3, "format": backup_manifest.FORMAT, "payload_bytes": count}
|
||||||
|
old.pop("logical_bytes")
|
||||||
|
with pytest.raises(policy.DeliveryError, match="inventory count exceeds bound"):
|
||||||
|
backup_manifest.validate(old)
|
||||||
|
value["files"].append({**template, "path": "releases/one-too-many"})
|
||||||
|
value["logical_bytes"] += 1
|
||||||
|
with pytest.raises(policy.DeliveryError, match="inventory count exceeds bound"):
|
||||||
|
backup_manifest.validate(value)
|
||||||
|
|
||||||
|
|
||||||
|
def test_private_metadata_rejects_a_file_above_the_explicit_byte_limit(tmp_path):
|
||||||
|
path = tmp_path / "metadata.json"
|
||||||
|
path.write_bytes(b" " * (backup_manifest.MAX_METADATA_BYTES + 1))
|
||||||
|
with pytest.raises(policy.DeliveryError, match="metadata byte limit"):
|
||||||
|
backup_manifest.read_metadata(path)
|
||||||
@@ -0,0 +1,114 @@
|
|||||||
|
"""Ordinary owned source/cache checks; no service, isolation or recovery claims."""
|
||||||
|
|
||||||
|
import importlib
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
sys.path.insert(0, str(ROOT))
|
||||||
|
bytecode = importlib.import_module("scripts.release.bytecode")
|
||||||
|
manifest = importlib.import_module("scripts.release.manifest")
|
||||||
|
policy = importlib.import_module("scripts.release.policy")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def compiled(tmp_path):
|
||||||
|
source = tmp_path / "site/toy.py"
|
||||||
|
source.parent.mkdir()
|
||||||
|
source.write_text("VALUE = (17, 'ordinary trusted fixture')\n")
|
||||||
|
source.chmod(0o444)
|
||||||
|
assert bytecode.compile_payload(tmp_path, "sd-" + "b" * 12) == 1
|
||||||
|
cache = source.parent / "__pycache__/toy.cpython-313.pyc"
|
||||||
|
return tmp_path, source, cache
|
||||||
|
|
||||||
|
|
||||||
|
def test_complete_cache_inventory_and_actual_import_without_source_compilation(compiled):
|
||||||
|
payload, source, cache = compiled
|
||||||
|
records = manifest.inventory(payload)
|
||||||
|
assert {r["path"] for r in records} == {
|
||||||
|
"site/toy.py",
|
||||||
|
"site/__pycache__/toy.cpython-313.pyc",
|
||||||
|
}
|
||||||
|
assert all(r["mode"] == "0444" for r in records)
|
||||||
|
# Only this tiny trusted fixture executes. An attempted source compilation
|
||||||
|
# fails, so success demonstrates CPython actually consumes the checked cache.
|
||||||
|
code = """import builtins,sys
|
||||||
|
sys.path.insert(0,sys.argv[1])
|
||||||
|
def refuse(*args,**kwargs): raise AssertionError('unexpected source compilation')
|
||||||
|
builtins.compile=refuse
|
||||||
|
import toy
|
||||||
|
assert toy.VALUE==(17,'ordinary trusted fixture')
|
||||||
|
"""
|
||||||
|
before = {p: bytecode._stamp(p.lstat()) for p in (source, cache)}
|
||||||
|
result = subprocess.run(
|
||||||
|
[sys.executable, "-I", "-B", "-c", code, str(source.parent)],
|
||||||
|
capture_output=True,
|
||||||
|
timeout=15,
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
assert {p: bytecode._stamp(p.lstat()) for p in before} == before
|
||||||
|
|
||||||
|
|
||||||
|
def test_source_change_cannot_reuse_cache_even_with_restored_mtime(compiled):
|
||||||
|
payload, source, _ = compiled
|
||||||
|
before = source.stat()
|
||||||
|
source.chmod(0o600)
|
||||||
|
source.write_text("VALUE = (18, 'ordinary trusted fixture')\n")
|
||||||
|
source.chmod(0o444)
|
||||||
|
os.utime(source, ns=(before.st_atime_ns, before.st_mtime_ns))
|
||||||
|
with pytest.raises(policy.DeliveryError, match="bytecode source hash differs"):
|
||||||
|
manifest.inventory(payload)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("offset,value", [(0, 0), (4, 0), (4, 1), (4, 2)])
|
||||||
|
def test_wrong_magic_timestamp_and_unchecked_hash_caches_are_refused(compiled, offset, value):
|
||||||
|
payload, _, cache = compiled
|
||||||
|
data = bytearray(cache.read_bytes())
|
||||||
|
data[offset] = value
|
||||||
|
cache.chmod(0o600)
|
||||||
|
cache.write_bytes(data)
|
||||||
|
cache.chmod(0o444)
|
||||||
|
with pytest.raises(policy.DeliveryError, match="only pinned checked-hash"):
|
||||||
|
manifest.inventory(payload)
|
||||||
|
|
||||||
|
|
||||||
|
def test_builder_refuses_inherited_cache_and_does_not_write_to_external_prefix(
|
||||||
|
compiled, monkeypatch
|
||||||
|
):
|
||||||
|
payload, _, cache = compiled
|
||||||
|
before = cache.read_bytes()
|
||||||
|
monkeypatch.setattr(sys, "pycache_prefix", str(payload / "outside"))
|
||||||
|
with pytest.raises(policy.DeliveryError, match="inherited bytecode cache"):
|
||||||
|
bytecode.compile_payload(payload, "sd-" + "b" * 12)
|
||||||
|
assert cache.read_bytes() == before and not (payload / "outside").exists()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"name",
|
||||||
|
[
|
||||||
|
"site/toy.pyc",
|
||||||
|
"site/__pycache__/toy.cpython-314.pyc",
|
||||||
|
"site/__pycache__/toy.cpython-313.opt-1.pyc",
|
||||||
|
"site/__pycache__/toy.py",
|
||||||
|
"site/__pycache__/nested/__pycache__/toy.cpython-313.pyc",
|
||||||
|
"site/../__pycache__/toy.cpython-313.pyc",
|
||||||
|
"images/python313/__pycache__/toy.cpython-313.pyc",
|
||||||
|
"frontend/__pycache__/toy.cpython-313.pyc",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_cache_allowlist_keeps_wrong_versions_domains_and_paths_closed(name):
|
||||||
|
with pytest.raises(policy.DeliveryError):
|
||||||
|
policy.relative_name(name)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("prefix", policy.BYTECODE_ROOTS)
|
||||||
|
def test_only_source_adjacent_pinned_cache_paths_are_allowed(prefix):
|
||||||
|
name = prefix + "package/__pycache__/toy.cpython-313.pyc"
|
||||||
|
assert policy.relative_name(name) == name
|
||||||
|
assert policy.bytecode_source(name) == prefix + "package/toy.py"
|
||||||
|
directory = prefix + "package/__pycache__/.__directory__"
|
||||||
|
assert policy.relative_name(directory) == directory
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,140 @@
|
|||||||
|
"""Ordinary modeled lifecycle checks; actual HTTPS and service gates require the VM."""
|
||||||
|
|
||||||
|
import importlib
|
||||||
|
import ssl
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||||
|
lifecycle = importlib.import_module("scripts.release.lifecycle")
|
||||||
|
proxy_health = importlib.import_module("scripts.release.proxy_health")
|
||||||
|
DeliveryError = importlib.import_module("scripts.release.policy").DeliveryError
|
||||||
|
READY = {
|
||||||
|
"reachable": True,
|
||||||
|
"pinned_certificate_and_hostname_verified": True,
|
||||||
|
"public_browser_trust_verified": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def setup_maintenance(monkeypatch, *, active=False):
|
||||||
|
maintenance = object.__new__(lifecycle.Maintenance)
|
||||||
|
maintenance.selected = {"release_id": "sd-123456789abc", "configuration_id": "cfg-" + "1" * 24}
|
||||||
|
maintenance.record = {"events": []}
|
||||||
|
clock = [100.0]
|
||||||
|
state = {"active": active}
|
||||||
|
saves = []
|
||||||
|
monkeypatch.setattr(lifecycle.time, "monotonic", lambda: clock[0])
|
||||||
|
monkeypatch.setattr(
|
||||||
|
lifecycle.time, "sleep", lambda value: clock.__setitem__(0, clock[0] + value)
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(lifecycle.install, "selection", lambda **_: maintenance.selected.copy())
|
||||||
|
monkeypatch.setattr(lifecycle.node_health, "inspect_unit", lambda *_: None)
|
||||||
|
monkeypatch.setattr(lifecycle.proxy_install, "check_binding", lambda *_: None)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
maintenance,
|
||||||
|
"unit",
|
||||||
|
lambda _: {
|
||||||
|
"ActiveState": "active" if state["active"] else "inactive",
|
||||||
|
"SubState": "running",
|
||||||
|
"MainPID": "42" if state["active"] else "0",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
maintenance,
|
||||||
|
"save",
|
||||||
|
lambda phase: saves.append((phase, [x.copy() for x in maintenance.record["events"]])),
|
||||||
|
)
|
||||||
|
|
||||||
|
def systemctl(*args):
|
||||||
|
assert args == ("--no-block", "start", "kf-proxy.service")
|
||||||
|
state["active"] = True
|
||||||
|
return ""
|
||||||
|
|
||||||
|
monkeypatch.setattr(lifecycle.service_install, "systemctl", systemctl)
|
||||||
|
return maintenance, clock, saves
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"group,active", [(True, False), (False, False), (True, True), (False, True)]
|
||||||
|
)
|
||||||
|
def test_proxy_process_is_not_confirmed_until_actual_https_probe_succeeds(
|
||||||
|
monkeypatch, group, active
|
||||||
|
):
|
||||||
|
maintenance, clock, saves = setup_maintenance(monkeypatch, active=active)
|
||||||
|
probes = []
|
||||||
|
|
||||||
|
def probe(selected, *, timeout):
|
||||||
|
assert all(not x["confirmed"] for x in maintenance.record["events"])
|
||||||
|
probes.append(timeout)
|
||||||
|
if len(probes) == 1:
|
||||||
|
raise proxy_health.HTTPSNotReady("launcher has not served HTTPS yet")
|
||||||
|
return READY.copy()
|
||||||
|
|
||||||
|
monkeypatch.setattr(proxy_health, "observe", probe)
|
||||||
|
if group:
|
||||||
|
maintenance.start_group(("proxy",))
|
||||||
|
else:
|
||||||
|
maintenance.change("proxy", "start")
|
||||||
|
assert probes == [5.0, 5.0] and clock[0] == 100.5
|
||||||
|
if not active:
|
||||||
|
assert saves[0][1][0]["confirmed"] is False
|
||||||
|
assert saves[-1][1][0]["confirmed"] is True
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"error", [ssl.SSLCertVerificationError("bad certificate"), DeliveryError("changed TLS binding")]
|
||||||
|
)
|
||||||
|
def test_proxy_security_failure_is_fatal_without_wait_or_confirmation(monkeypatch, error):
|
||||||
|
maintenance, clock, _ = setup_maintenance(monkeypatch)
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def probe(*_, **__):
|
||||||
|
calls.append(1)
|
||||||
|
raise error
|
||||||
|
|
||||||
|
monkeypatch.setattr(proxy_health, "observe", probe)
|
||||||
|
with pytest.raises(type(error)):
|
||||||
|
maintenance.start_group(("proxy",))
|
||||||
|
assert calls == [1] and clock[0] == 100
|
||||||
|
assert maintenance.record["events"] == [
|
||||||
|
{"unit": "proxy", "operation": "start", "confirmed": False}
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("late_ready", [False, True])
|
||||||
|
def test_proxy_wait_cannot_restart_or_succeed_after_original_deadline(monkeypatch, late_ready):
|
||||||
|
maintenance, clock, _ = setup_maintenance(monkeypatch)
|
||||||
|
probes = []
|
||||||
|
|
||||||
|
def unavailable(*_, timeout):
|
||||||
|
probes.append(timeout)
|
||||||
|
clock[0] += 131
|
||||||
|
if late_ready:
|
||||||
|
return READY.copy()
|
||||||
|
raise proxy_health.HTTPSNotReady("still unavailable")
|
||||||
|
|
||||||
|
monkeypatch.setattr(proxy_health, "observe", unavailable)
|
||||||
|
with pytest.raises(DeliveryError, match="bounded HTTPS startup unconfirmed"):
|
||||||
|
maintenance.start_group(("proxy",))
|
||||||
|
assert probes == [5.0]
|
||||||
|
assert maintenance.record["events"][0]["confirmed"] is False
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"error", [ConnectionRefusedError("not listening"), TimeoutError("not ready")]
|
||||||
|
)
|
||||||
|
def test_proxy_probe_labels_only_transient_transport_failure(monkeypatch, error):
|
||||||
|
def unavailable(*_, **__):
|
||||||
|
raise error
|
||||||
|
|
||||||
|
monkeypatch.setattr(proxy_health, "_observe", unavailable)
|
||||||
|
with pytest.raises(proxy_health.HTTPSNotReady):
|
||||||
|
proxy_health.observe({})
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("timeout", [0, -1, 5.01, True, float("nan")])
|
||||||
|
def test_proxy_probe_rejects_unbounded_timeout(timeout):
|
||||||
|
with pytest.raises(DeliveryError, match="timeout exceeds bound"):
|
||||||
|
proxy_health.observe({}, timeout=timeout)
|
||||||
@@ -0,0 +1,537 @@
|
|||||||
|
"""Trusted ordinary byte fixtures; real Root/cgroup/syncfs/RTO remains VM-only."""
|
||||||
|
|
||||||
|
import errno
|
||||||
|
import hashlib
|
||||||
|
import multiprocessing
|
||||||
|
import os
|
||||||
|
import stat
|
||||||
|
import time
|
||||||
|
from contextlib import closing
|
||||||
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from scripts.release import restore, restore_io
|
||||||
|
from scripts.release.policy import DeliveryError
|
||||||
|
|
||||||
|
|
||||||
|
def fixture_rows(tmp_path, contents):
|
||||||
|
source = tmp_path / "source"
|
||||||
|
target = tmp_path / "target"
|
||||||
|
source.mkdir()
|
||||||
|
target.mkdir()
|
||||||
|
rows = []
|
||||||
|
for index, raw in enumerate(contents):
|
||||||
|
digest = hashlib.sha256(raw).hexdigest()
|
||||||
|
path = source / digest
|
||||||
|
if not path.exists():
|
||||||
|
path.write_bytes(raw)
|
||||||
|
rows.append({"path": str(index), "bytes": len(raw), "sha256": digest})
|
||||||
|
return (
|
||||||
|
rows,
|
||||||
|
lambda row: source / row["sha256"],
|
||||||
|
lambda row: (
|
||||||
|
target / row["path"],
|
||||||
|
os.getuid(),
|
||||||
|
os.getgid(),
|
||||||
|
0o600 if int(row["path"]) % 2 else 0o400,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def filesystem_model(monkeypatch, target, events, callback=None):
|
||||||
|
"""Model syncfs using only tiny owned fixture fsyncs; never host syncfs."""
|
||||||
|
|
||||||
|
class Sync:
|
||||||
|
def __call__(self, descriptor):
|
||||||
|
assert stat.S_ISDIR(os.fstat(descriptor).st_mode)
|
||||||
|
if callback is not None:
|
||||||
|
callback()
|
||||||
|
files = [p for p in target.iterdir() if p.is_file()]
|
||||||
|
for path in files:
|
||||||
|
with path.open("rb") as stream:
|
||||||
|
os.fsync(stream.fileno())
|
||||||
|
events.append(("sync", descriptor))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
restore_io.ctypes, "CDLL", lambda *_a, **_kw: SimpleNamespace(syncfs=Sync())
|
||||||
|
)
|
||||||
|
return restore_io.Filesystems([target], time.monotonic() + 120)
|
||||||
|
|
||||||
|
|
||||||
|
def assert_closed(descriptors):
|
||||||
|
for descriptor in descriptors:
|
||||||
|
with pytest.raises(OSError) as failure:
|
||||||
|
os.fstat(descriptor)
|
||||||
|
assert failure.value.errno == errno.EBADF
|
||||||
|
|
||||||
|
|
||||||
|
class ReadAfterSync:
|
||||||
|
"""Picklable observation of actual tiny fixture reads in the child processes."""
|
||||||
|
|
||||||
|
def __init__(self, synced, reads, descriptors):
|
||||||
|
self.synced, self.reads, self.descriptors = synced, reads, descriptors
|
||||||
|
|
||||||
|
def __call__(self, path, item, uid, gid, mode, deadline):
|
||||||
|
assert self.synced.read_bytes() == b"sync completed"
|
||||||
|
assert all(os.fstat(fd).st_ino > 0 for fd in self.descriptors)
|
||||||
|
(self.reads / item["path"]).touch()
|
||||||
|
return restore.readback(path, item, uid, gid, mode, deadline)
|
||||||
|
|
||||||
|
|
||||||
|
def test_bounded_batches_group_identical_objects_without_losing_paths():
|
||||||
|
rows = [
|
||||||
|
{"path": str(index), "sha256": str(index % 2), "bytes": 1}
|
||||||
|
for index in range(restore_io.BATCH_FILES + 4)
|
||||||
|
]
|
||||||
|
rows.append({"path": "large", "sha256": "2", "bytes": restore_io.BATCH_BYTES + 1})
|
||||||
|
groups = list(restore_io.batches(rows))
|
||||||
|
flattened = [row for group in groups for row in group]
|
||||||
|
assert {row["path"] for row in flattened} == {row["path"] for row in rows}
|
||||||
|
assert len(flattened) == len(rows)
|
||||||
|
assert [row["sha256"] for row in flattened] == sorted(row["sha256"] for row in rows)
|
||||||
|
assert all(len(group) <= restore_io.BATCH_FILES for group in groups)
|
||||||
|
assert all(
|
||||||
|
sum(row["bytes"] for row in group) <= restore_io.BATCH_BYTES or len(group) == 1
|
||||||
|
for group in groups
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_repeated_objects_expand_to_independent_durable_verified_destinations(
|
||||||
|
tmp_path, monkeypatch
|
||||||
|
):
|
||||||
|
rows, source, destination = fixture_rows(tmp_path, [b"first", b"other", b"first", b"first"])
|
||||||
|
events = []
|
||||||
|
target = destination(rows[0])[0].parent
|
||||||
|
synced, reads = tmp_path / "synced", tmp_path / "reads"
|
||||||
|
reads.mkdir()
|
||||||
|
with closing(filesystem_model(monkeypatch, target, events)) as filesystems:
|
||||||
|
descriptors = [descriptor for _, descriptor, _ in filesystems.handles.values()]
|
||||||
|
assert list(target.iterdir()) == []
|
||||||
|
|
||||||
|
original_sync = filesystems.sync
|
||||||
|
|
||||||
|
def sync(deadline):
|
||||||
|
original_sync(deadline)
|
||||||
|
synced.write_bytes(b"sync completed")
|
||||||
|
|
||||||
|
monkeypatch.setattr(filesystems, "sync", sync)
|
||||||
|
|
||||||
|
verified = restore_io.materialize(
|
||||||
|
rows,
|
||||||
|
source,
|
||||||
|
destination,
|
||||||
|
ReadAfterSync(synced, reads, descriptors),
|
||||||
|
filesystems,
|
||||||
|
time.monotonic() + 120,
|
||||||
|
)
|
||||||
|
assert_closed(descriptors)
|
||||||
|
assert set(verified) == {row["path"] for row in rows}
|
||||||
|
assert len(list(reads.iterdir())) == len(rows)
|
||||||
|
inodes = set()
|
||||||
|
for row in rows:
|
||||||
|
path, uid, gid, mode = destination(row)
|
||||||
|
assert path.read_bytes() == source(row).read_bytes()
|
||||||
|
assert stat.S_IMODE(path.stat().st_mode) == mode
|
||||||
|
assert path.stat().st_nlink == 1
|
||||||
|
inodes.add(path.stat().st_ino)
|
||||||
|
restore.unchanged_file(path, verified[row["path"]], uid, gid, mode)
|
||||||
|
assert len(inodes) == len(rows)
|
||||||
|
|
||||||
|
|
||||||
|
def test_existing_wrong_bytes_are_refused_without_overwrite(tmp_path, monkeypatch):
|
||||||
|
prior_children = {child.pid for child in multiprocessing.active_children()}
|
||||||
|
rows, source, destination = fixture_rows(tmp_path, [b"new"])
|
||||||
|
path, _, _, mode = destination(rows[0])
|
||||||
|
path.write_bytes(b"bad")
|
||||||
|
path.chmod(mode)
|
||||||
|
before = restore.file_witness(path.stat())
|
||||||
|
with closing(filesystem_model(monkeypatch, path.parent, [])) as filesystems:
|
||||||
|
with pytest.raises(DeliveryError, match="byte verification"):
|
||||||
|
restore_io.materialize(
|
||||||
|
rows,
|
||||||
|
source,
|
||||||
|
destination,
|
||||||
|
restore.readback,
|
||||||
|
filesystems,
|
||||||
|
time.monotonic() + 120,
|
||||||
|
)
|
||||||
|
assert path.read_bytes() == b"bad" and restore.file_witness(path.stat()) == before
|
||||||
|
assert {child.pid for child in multiprocessing.active_children()} == prior_children
|
||||||
|
|
||||||
|
|
||||||
|
def test_failed_sync_keeps_private_files_and_cannot_return_verified_witnesses(
|
||||||
|
tmp_path, monkeypatch
|
||||||
|
):
|
||||||
|
rows, source, destination = fixture_rows(tmp_path, [b"a", b"b", b"c"])
|
||||||
|
reads = []
|
||||||
|
|
||||||
|
def failed_sync():
|
||||||
|
raise DeliveryError("synthetic synchronization failure")
|
||||||
|
|
||||||
|
with closing(
|
||||||
|
filesystem_model(monkeypatch, destination(rows[0])[0].parent, [], failed_sync)
|
||||||
|
) as filesystems:
|
||||||
|
descriptors = [fd for _, fd, _ in filesystems.handles.values()]
|
||||||
|
with pytest.raises(DeliveryError, match="synthetic synchronization failure"):
|
||||||
|
restore_io.materialize(
|
||||||
|
rows,
|
||||||
|
source,
|
||||||
|
destination,
|
||||||
|
lambda *args: reads.append(args),
|
||||||
|
filesystems,
|
||||||
|
time.monotonic() + 120,
|
||||||
|
)
|
||||||
|
assert_closed(descriptors)
|
||||||
|
assert reads == []
|
||||||
|
assert all(destination(row)[0].is_file() for row in rows)
|
||||||
|
|
||||||
|
|
||||||
|
def test_same_bytes_inode_replacement_across_sync_cannot_adopt_new_file(tmp_path, monkeypatch):
|
||||||
|
rows, source, destination = fixture_rows(tmp_path, [b"same"])
|
||||||
|
|
||||||
|
def replace_inode():
|
||||||
|
path, _, _, mode = destination(rows[0])
|
||||||
|
replacement = path.parent / "replacement"
|
||||||
|
replacement.write_bytes(path.read_bytes())
|
||||||
|
replacement.chmod(mode)
|
||||||
|
replacement.replace(path)
|
||||||
|
|
||||||
|
with closing(
|
||||||
|
filesystem_model(monkeypatch, destination(rows[0])[0].parent, [], replace_inode)
|
||||||
|
) as filesystems:
|
||||||
|
with pytest.raises(DeliveryError, match="changed across synchronization"):
|
||||||
|
restore_io.materialize(
|
||||||
|
rows,
|
||||||
|
source,
|
||||||
|
destination,
|
||||||
|
restore.readback,
|
||||||
|
filesystems,
|
||||||
|
time.monotonic() + 120,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_expired_batch_deadline_copies_no_files(tmp_path, monkeypatch):
|
||||||
|
rows, source, destination = fixture_rows(tmp_path, [b"data"])
|
||||||
|
with closing(filesystem_model(monkeypatch, destination(rows[0])[0].parent, [])) as filesystems:
|
||||||
|
with pytest.raises(DeliveryError, match="deadline exceeded"):
|
||||||
|
restore_io.materialize(
|
||||||
|
rows,
|
||||||
|
source,
|
||||||
|
destination,
|
||||||
|
restore.readback,
|
||||||
|
filesystems,
|
||||||
|
time.monotonic() - 1,
|
||||||
|
)
|
||||||
|
assert not destination(rows[0])[0].exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_syncfs_failure_is_propagated_in_the_ordinary_syscall_model(tmp_path, monkeypatch):
|
||||||
|
class FailedSync:
|
||||||
|
def __call__(self, _descriptor):
|
||||||
|
return -1
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
restore_io.ctypes,
|
||||||
|
"CDLL",
|
||||||
|
lambda *_a, **_kw: SimpleNamespace(syncfs=FailedSync()),
|
||||||
|
)
|
||||||
|
with closing(restore_io.Filesystems([tmp_path], time.monotonic() + 120)) as filesystems:
|
||||||
|
descriptors = [fd for _, fd, _ in filesystems.handles.values()]
|
||||||
|
with pytest.raises(DeliveryError, match="filesystem synchronization failed"):
|
||||||
|
filesystems.sync(time.monotonic() + 120)
|
||||||
|
assert_closed(descriptors)
|
||||||
|
|
||||||
|
|
||||||
|
def test_cursor_is_open_before_first_write_and_held_across_batches(tmp_path, monkeypatch):
|
||||||
|
rows, source, destination = fixture_rows(tmp_path, [b"a", b"b", b"c", b"d", b"e"])
|
||||||
|
target = destination(rows[0])[0].parent
|
||||||
|
monkeypatch.setattr(restore_io, "BATCH_FILES", 2)
|
||||||
|
events = []
|
||||||
|
with closing(filesystem_model(monkeypatch, target, events)) as filesystems:
|
||||||
|
descriptors = [fd for _, fd, _ in filesystems.handles.values()]
|
||||||
|
before = {fd: (os.fstat(fd).st_dev, os.fstat(fd).st_ino) for fd in descriptors}
|
||||||
|
real_write = restore_io.pending_file
|
||||||
|
|
||||||
|
def write(*args):
|
||||||
|
assert {fd: (os.fstat(fd).st_dev, os.fstat(fd).st_ino) for fd in descriptors} == before
|
||||||
|
return real_write(*args)
|
||||||
|
|
||||||
|
monkeypatch.setattr(restore_io, "pending_file", write)
|
||||||
|
verified = restore_io.materialize(
|
||||||
|
rows,
|
||||||
|
source,
|
||||||
|
destination,
|
||||||
|
restore.readback,
|
||||||
|
filesystems,
|
||||||
|
time.monotonic() + 120,
|
||||||
|
)
|
||||||
|
assert len(verified) == len(rows)
|
||||||
|
assert len(events) == 3
|
||||||
|
assert {fd for event, fd in events if event == "sync"} == set(descriptors)
|
||||||
|
assert_closed(descriptors)
|
||||||
|
|
||||||
|
|
||||||
|
def test_cursor_can_anchor_before_target_directories_exist(tmp_path, monkeypatch):
|
||||||
|
target = tmp_path / "later" / "nested"
|
||||||
|
|
||||||
|
class Sync:
|
||||||
|
def __call__(self, descriptor):
|
||||||
|
assert os.fstat(descriptor).st_ino == tmp_path.stat().st_ino
|
||||||
|
return 0
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
restore_io.ctypes, "CDLL", lambda *_a, **_kw: SimpleNamespace(syncfs=Sync())
|
||||||
|
)
|
||||||
|
with closing(restore_io.Filesystems([target], time.monotonic() + 120)) as filesystems:
|
||||||
|
target.mkdir(parents=True)
|
||||||
|
filesystems.contains(target / "file")
|
||||||
|
filesystems.sync(time.monotonic() + 120)
|
||||||
|
|
||||||
|
|
||||||
|
def test_constructor_closes_earlier_cursor_when_later_anchor_is_invalid(tmp_path, monkeypatch):
|
||||||
|
valid, invalid = tmp_path / "valid", tmp_path / "invalid"
|
||||||
|
valid.mkdir()
|
||||||
|
invalid.write_bytes(b"ordinary owned fixture")
|
||||||
|
opened = []
|
||||||
|
real_open = restore_io.os.open
|
||||||
|
|
||||||
|
def record_open(*args, **kwargs):
|
||||||
|
descriptor = real_open(*args, **kwargs)
|
||||||
|
opened.append(descriptor)
|
||||||
|
return descriptor
|
||||||
|
|
||||||
|
monkeypatch.setattr(restore_io.os, "open", record_open)
|
||||||
|
with pytest.raises(DeliveryError, match="anchor differs"):
|
||||||
|
restore_io.Filesystems([valid, invalid], time.monotonic() + 120)
|
||||||
|
assert len(opened) == 1
|
||||||
|
assert_closed(opened)
|
||||||
|
|
||||||
|
|
||||||
|
def test_actual_two_copy_processes_join_before_return_and_preserve_all_bytes(tmp_path, monkeypatch):
|
||||||
|
rows, source, destination = fixture_rows(
|
||||||
|
tmp_path, [b"known first", b"known second", b"known third", b"known fourth"]
|
||||||
|
)
|
||||||
|
prior_children = {child.pid for child in multiprocessing.active_children()}
|
||||||
|
markers = tmp_path / "worker-identities"
|
||||||
|
markers.mkdir()
|
||||||
|
original = restore_io.pending_file
|
||||||
|
|
||||||
|
def record_worker(*args):
|
||||||
|
(markers / str(os.getpid())).touch()
|
||||||
|
deadline = time.monotonic() + 5
|
||||||
|
while len(list(markers.iterdir())) < 2:
|
||||||
|
assert time.monotonic() < deadline
|
||||||
|
time.sleep(0.01)
|
||||||
|
return original(*args)
|
||||||
|
|
||||||
|
monkeypatch.setattr(restore_io, "pending_file", record_worker)
|
||||||
|
with closing(filesystem_model(monkeypatch, destination(rows[0])[0].parent, [])) as fs:
|
||||||
|
verified = restore_io.materialize(
|
||||||
|
rows, source, destination, restore.readback, fs, time.monotonic() + 120
|
||||||
|
)
|
||||||
|
assert len(list(markers.iterdir())) == 2
|
||||||
|
assert {child.pid for child in multiprocessing.active_children()} == prior_children
|
||||||
|
assert all(not (restore_io.Path("/proc") / p.name).exists() for p in markers.iterdir())
|
||||||
|
assert set(verified) == {row["path"] for row in rows}
|
||||||
|
for row in rows:
|
||||||
|
path, uid, gid, mode = destination(row)
|
||||||
|
assert path.read_bytes() == source(row).read_bytes()
|
||||||
|
restore.unchanged_file(path, verified[row["path"]], uid, gid, mode)
|
||||||
|
|
||||||
|
|
||||||
|
def test_process_copy_refuses_fork_from_a_multithreaded_owner(tmp_path, monkeypatch):
|
||||||
|
rows, source, destination = fixture_rows(tmp_path, [b"known fixture"])
|
||||||
|
monkeypatch.setattr(restore_io.threading, "active_count", lambda: 2)
|
||||||
|
with closing(filesystem_model(monkeypatch, destination(rows[0])[0].parent, [])) as fs:
|
||||||
|
with pytest.raises(DeliveryError, match="one POSIX owner thread"):
|
||||||
|
restore_io.materialize(
|
||||||
|
rows, source, destination, restore.readback, fs, time.monotonic() + 120
|
||||||
|
)
|
||||||
|
assert not destination(rows[0])[0].exists()
|
||||||
|
|
||||||
|
|
||||||
|
def group_files(tmp_path, values, expected=None):
|
||||||
|
rows = []
|
||||||
|
for i, raw in enumerate(values):
|
||||||
|
path = tmp_path / str(i)
|
||||||
|
path.write_bytes(raw)
|
||||||
|
path.chmod(0o600)
|
||||||
|
wanted = raw if expected is None else expected[i]
|
||||||
|
item = {
|
||||||
|
"path": str(i),
|
||||||
|
"bytes": len(wanted),
|
||||||
|
"sha256": hashlib.sha256(wanted).hexdigest(),
|
||||||
|
}
|
||||||
|
rows.append(
|
||||||
|
(
|
||||||
|
item,
|
||||||
|
path,
|
||||||
|
os.getuid(),
|
||||||
|
os.getgid(),
|
||||||
|
0o600,
|
||||||
|
restore.file_witness(path.stat()),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def read_group(group):
|
||||||
|
return restore.readback_group(group, time.monotonic() + 120)
|
||||||
|
|
||||||
|
|
||||||
|
def test_each_repeated_destination_is_fully_read_with_one_actual_reference_hash(
|
||||||
|
tmp_path, monkeypatch
|
||||||
|
):
|
||||||
|
raw = b"a" * 1048576 + b"b" * 1048576 + b"last-byte-proof"
|
||||||
|
group = group_files(tmp_path, [raw] * 4)
|
||||||
|
original_open, original_hash = os.fdopen, hashlib.sha256
|
||||||
|
observed, hashes = {}, []
|
||||||
|
|
||||||
|
class Reader:
|
||||||
|
def __init__(self, stream, path):
|
||||||
|
self.stream, self.path = stream, path
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *args):
|
||||||
|
return self.stream.__exit__(*args)
|
||||||
|
|
||||||
|
def fileno(self):
|
||||||
|
return self.stream.fileno()
|
||||||
|
|
||||||
|
def read(self, count):
|
||||||
|
assert count == 1048576
|
||||||
|
block = self.stream.read(count)
|
||||||
|
observed[self.path] = observed.get(self.path, 0) + len(block)
|
||||||
|
return block
|
||||||
|
|
||||||
|
def opened(fd, mode):
|
||||||
|
return Reader(original_open(fd, mode), Path(os.readlink(f"/proc/self/fd/{fd}")))
|
||||||
|
|
||||||
|
def hashed(*args, **kwargs):
|
||||||
|
hashes.append(True)
|
||||||
|
return original_hash(*args, **kwargs)
|
||||||
|
|
||||||
|
monkeypatch.setattr(restore.os, "fdopen", opened)
|
||||||
|
monkeypatch.setattr(restore.hashlib, "sha256", hashed)
|
||||||
|
witnesses = read_group(group)
|
||||||
|
assert observed == {row[1]: len(raw) for row in group}
|
||||||
|
assert len(hashes) == 1
|
||||||
|
assert witnesses == [row[-1] for row in group]
|
||||||
|
|
||||||
|
|
||||||
|
def test_last_byte_difference_in_repeated_target_is_refused(tmp_path):
|
||||||
|
good = b"x" * 1048576 + b"ending"
|
||||||
|
bad = good[:-1] + b"!"
|
||||||
|
group = group_files(tmp_path, [good, bad, good], [good] * 3)
|
||||||
|
with pytest.raises(DeliveryError, match="byte verification"):
|
||||||
|
read_group(group)
|
||||||
|
assert group[1][1].read_bytes() == bad
|
||||||
|
|
||||||
|
|
||||||
|
def test_identically_wrong_reference_and_duplicates_cannot_match_expected_digest(
|
||||||
|
tmp_path,
|
||||||
|
):
|
||||||
|
group = group_files(tmp_path, [b"bad"] * 3, [b"new"] * 3)
|
||||||
|
with pytest.raises(DeliveryError, match="byte verification"):
|
||||||
|
read_group(group)
|
||||||
|
|
||||||
|
|
||||||
|
def test_unique_large_and_empty_files_keep_exact_bytes(tmp_path):
|
||||||
|
values = [b"", b"", b"one", b"other", b"z" * (8 * 1024**2 + 1)]
|
||||||
|
group = group_files(tmp_path, values)
|
||||||
|
assert read_group(group) == [row[-1] for row in group]
|
||||||
|
|
||||||
|
|
||||||
|
def test_reference_is_rebound_when_digest_changes(tmp_path):
|
||||||
|
group = group_files(tmp_path, [b"aa", b"aa", b"bb", b"bb", b"aa", b"aa"])
|
||||||
|
assert read_group(group) == [row[-1] for row in group]
|
||||||
|
|
||||||
|
|
||||||
|
def test_changed_file_after_complete_read_is_refused(tmp_path, monkeypatch):
|
||||||
|
group = group_files(tmp_path, [b"known", b"known"])
|
||||||
|
original = os.fdopen
|
||||||
|
|
||||||
|
class ChangedAtEnd:
|
||||||
|
def __init__(self, stream):
|
||||||
|
self.stream = stream
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *args):
|
||||||
|
return self.stream.__exit__(*args)
|
||||||
|
|
||||||
|
def fileno(self):
|
||||||
|
return self.stream.fileno()
|
||||||
|
|
||||||
|
def read(self, count):
|
||||||
|
value = self.stream.read(count)
|
||||||
|
if not value:
|
||||||
|
group[0][1].write_bytes(b"other")
|
||||||
|
return value
|
||||||
|
|
||||||
|
monkeypatch.setattr(restore.os, "fdopen", lambda *a, **kw: ChangedAtEnd(original(*a, **kw)))
|
||||||
|
with pytest.raises(DeliveryError, match="changed during readback"):
|
||||||
|
read_group(group)
|
||||||
|
|
||||||
|
|
||||||
|
def test_group_rejects_metadata_bounds_and_expired_deadline(tmp_path):
|
||||||
|
group = group_files(tmp_path, [b"known"])
|
||||||
|
with pytest.raises(DeliveryError, match="fixed batch bounds"):
|
||||||
|
read_group(group * (restore_io.BATCH_FILES + 1))
|
||||||
|
with pytest.raises(DeliveryError, match="deadline"):
|
||||||
|
restore.readback_group(group, time.monotonic() - 1)
|
||||||
|
|
||||||
|
|
||||||
|
def test_grouped_real_copy_workers_join_and_return_independent_verified_files(
|
||||||
|
tmp_path, monkeypatch
|
||||||
|
):
|
||||||
|
children = {c.pid for c in multiprocessing.active_children()}
|
||||||
|
rows, source, destination = fixture_rows(tmp_path, [b"shared"] * 8)
|
||||||
|
with closing(filesystem_model(monkeypatch, destination(rows[0])[0].parent, [])) as filesystems:
|
||||||
|
result = restore_io.materialize(
|
||||||
|
rows,
|
||||||
|
source,
|
||||||
|
destination,
|
||||||
|
restore.readback,
|
||||||
|
filesystems,
|
||||||
|
time.monotonic() + 120,
|
||||||
|
group_readback=restore.readback_group,
|
||||||
|
)
|
||||||
|
assert {c.pid for c in multiprocessing.active_children()} == children
|
||||||
|
assert set(result) == {row["path"] for row in rows}
|
||||||
|
assert len({destination(row)[0].stat().st_ino for row in rows}) == len(rows)
|
||||||
|
for row in rows:
|
||||||
|
path, uid, gid, mode = destination(row)
|
||||||
|
assert path.read_bytes() == b"shared"
|
||||||
|
restore.unchanged_file(path, result[row["path"]], uid, gid, mode)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"name", ["releases/../private", "factory//requests", "unknown/file", "releases/.ssh/key"]
|
||||||
|
)
|
||||||
|
def test_untrusted_restore_target_names_still_require_complete_validation(tmp_path, name):
|
||||||
|
record = {"path": name, "uid": 0, "gid": 0, "mode": 0o600, "bytes": 0}
|
||||||
|
with pytest.raises(DeliveryError, match="private backup member"):
|
||||||
|
restore.target(record, tmp_path)
|
||||||
|
|
||||||
|
|
||||||
|
def test_pending_copy_refuses_a_dangling_symlink_without_creating_its_target(tmp_path):
|
||||||
|
source = tmp_path / "source"
|
||||||
|
source.write_bytes(b"trusted bytes")
|
||||||
|
target = tmp_path / "target"
|
||||||
|
missing = tmp_path / "missing"
|
||||||
|
target.symlink_to(missing)
|
||||||
|
item = {
|
||||||
|
"bytes": source.stat().st_size,
|
||||||
|
"sha256": hashlib.sha256(source.read_bytes()).hexdigest(),
|
||||||
|
}
|
||||||
|
with pytest.raises(DeliveryError, match="existing recovery destination differs"):
|
||||||
|
restore_io.pending_file(
|
||||||
|
source, target, item, os.getuid(), os.getgid(), 0o600, time.monotonic() + 10
|
||||||
|
)
|
||||||
|
assert target.is_symlink() and not missing.exists()
|
||||||
@@ -0,0 +1,240 @@
|
|||||||
|
"""Ordinary fixed-query/authority models; no PostgreSQL or VM acceptance."""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import copy
|
||||||
|
import importlib
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
sys.path.insert(0, str(ROOT))
|
||||||
|
health = importlib.import_module("scripts.release.node_health")
|
||||||
|
|
||||||
|
# Original role policy, independent of the query builder and its result mapping.
|
||||||
|
GRANTS = {
|
||||||
|
"api": {
|
||||||
|
"accounts:SELECT": True,
|
||||||
|
"evaluations:SELECT": True,
|
||||||
|
"evaluations:state:UPDATE": False,
|
||||||
|
"accounts:is_administrator:UPDATE": False,
|
||||||
|
},
|
||||||
|
"coordinator": {
|
||||||
|
"evaluations:SELECT": True,
|
||||||
|
"evaluations:state:UPDATE": True,
|
||||||
|
"accounts:SELECT": False,
|
||||||
|
"problems:active_version_id:UPDATE": False,
|
||||||
|
},
|
||||||
|
"import": {
|
||||||
|
"problem_import_operations:SELECT": True,
|
||||||
|
"problem_import_operations:state:UPDATE": True,
|
||||||
|
"evaluations:state:UPDATE": False,
|
||||||
|
"accounts:SELECT": False,
|
||||||
|
},
|
||||||
|
"identity": {
|
||||||
|
"accounts:SELECT": False,
|
||||||
|
"accounts:is_administrator:UPDATE": False,
|
||||||
|
"accounts:password_hash:SELECT": False,
|
||||||
|
"sessions:token_hash:SELECT": False,
|
||||||
|
"evaluations:SELECT": False,
|
||||||
|
"evaluations:state:UPDATE": False,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
FUNCTIONS = (
|
||||||
|
"public.kf_manage_account_authority(text,text)",
|
||||||
|
"public.kf_manage_account_authority_unlogged(text,text)",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def session(monkeypatch, mutate=None, failure=None):
|
||||||
|
import asyncpg
|
||||||
|
|
||||||
|
calls, closed, connections = [], [], []
|
||||||
|
|
||||||
|
class Connection:
|
||||||
|
def __init__(self, role):
|
||||||
|
self.role = role
|
||||||
|
|
||||||
|
async def fetchval(self, *_):
|
||||||
|
raise AssertionError("one-row projection must not issue extra catalog statements")
|
||||||
|
|
||||||
|
async def fetchrow(self, query, *parameters):
|
||||||
|
role = self.role
|
||||||
|
calls.append((role, query, parameters))
|
||||||
|
assert query.count("FROM pg_roles WHERE rolname=session_user") == 1
|
||||||
|
assert query.startswith(
|
||||||
|
"SELECT session_user login,current_database() database,rolcanlogin canlogin,"
|
||||||
|
"rolsuper superuser,rolcreatedb createdb,rolcreaterole createrole,"
|
||||||
|
"rolreplication replication,rolbypassrls bypassrls,rolconnlimit connection_limit,"
|
||||||
|
"pg_has_role(session_user,$1,'MEMBER') member,"
|
||||||
|
"pg_has_role(session_user,$1,'USAGE') inherited,"
|
||||||
|
"pg_has_role(session_user,$1,'SET') set_role,ARRAY["
|
||||||
|
)
|
||||||
|
assert not any(
|
||||||
|
word in query.upper() for word in ("INSERT ", "UPDATE ", "DELETE ", "SET ROLE", ";")
|
||||||
|
)
|
||||||
|
assert parameters[0] == health.database.MEMBERSHIP[health.configuration.PROFILES[role]]
|
||||||
|
for capability in ("MEMBER", "USAGE", "SET"):
|
||||||
|
assert f"pg_has_role(session_user,$1,'{capability}')" in query
|
||||||
|
expressions = re.findall(
|
||||||
|
r"has_(table|column|function)_privilege\(current_user,((?:\$[0-9]+,?)+)(,'EXECUTE')?\)",
|
||||||
|
query,
|
||||||
|
)
|
||||||
|
assert len(expressions) == len(GRANTS[role]) + len(FUNCTIONS)
|
||||||
|
checked, functions, values, fvalues = [], [], [], []
|
||||||
|
for kind, slots, execute in expressions:
|
||||||
|
indices = [int(x[1:]) for x in slots.split(",")]
|
||||||
|
assert all(1 < x <= len(parameters) for x in indices)
|
||||||
|
args = tuple(parameters[x - 1] for x in indices)
|
||||||
|
if kind == "function":
|
||||||
|
assert execute == ",'EXECUTE'"
|
||||||
|
assert len(args) == 1 and args[0] in FUNCTIONS
|
||||||
|
functions.append(args[0])
|
||||||
|
fvalues.append(role == "identity" and args[0] == FUNCTIONS[0])
|
||||||
|
else:
|
||||||
|
assert execute == ""
|
||||||
|
assert len(args) == (2 if kind == "table" else 3)
|
||||||
|
key = ":".join(args)
|
||||||
|
assert key in GRANTS[role]
|
||||||
|
checked.append(key)
|
||||||
|
values.append(GRANTS[role][key])
|
||||||
|
assert checked == list(GRANTS[role]) and functions == list(FUNCTIONS)
|
||||||
|
# Every bind value is used; names/privilege tokens never enter SQL text.
|
||||||
|
used = {int(x) for x in re.findall(r"\$([0-9]+)", query)}
|
||||||
|
assert used == set(range(1, len(parameters) + 1))
|
||||||
|
assert all("'" + str(value) + "'" not in query for value in parameters)
|
||||||
|
if failure is not None:
|
||||||
|
raise failure
|
||||||
|
row = {
|
||||||
|
"login": health.configuration.PROFILES[role],
|
||||||
|
"database": "kf_onlinejudge",
|
||||||
|
"canlogin": True,
|
||||||
|
"superuser": False,
|
||||||
|
"createdb": False,
|
||||||
|
"createrole": False,
|
||||||
|
"replication": False,
|
||||||
|
"bypassrls": False,
|
||||||
|
"connection_limit": 8,
|
||||||
|
"member": True,
|
||||||
|
"inherited": True,
|
||||||
|
"set_role": False,
|
||||||
|
"privileges": values,
|
||||||
|
"functions": fvalues,
|
||||||
|
}
|
||||||
|
if mutate is not None:
|
||||||
|
mutate(role, row)
|
||||||
|
return row
|
||||||
|
|
||||||
|
async def close(self, *, timeout):
|
||||||
|
assert timeout == 1
|
||||||
|
closed.append(self.role)
|
||||||
|
|
||||||
|
async def connect(endpoint, **kwargs):
|
||||||
|
role = endpoint.removeprefix("synthetic-local-")
|
||||||
|
assert role in GRANTS
|
||||||
|
assert kwargs == {
|
||||||
|
"ssl": False,
|
||||||
|
"timeout": 5,
|
||||||
|
"command_timeout": 2,
|
||||||
|
"server_settings": {
|
||||||
|
"application_name": "kf-node-health",
|
||||||
|
"default_transaction_read_only": "on",
|
||||||
|
"statement_timeout": "2000",
|
||||||
|
"lock_timeout": "500",
|
||||||
|
"idle_in_transaction_session_timeout": "3000",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
connections.append(role)
|
||||||
|
return Connection(role)
|
||||||
|
|
||||||
|
monkeypatch.setattr(asyncpg, "connect", connect)
|
||||||
|
return (
|
||||||
|
{"database": {role: "synthetic-local-" + role for role in GRANTS}},
|
||||||
|
calls,
|
||||||
|
closed,
|
||||||
|
connections,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_all_original_authority_checks_use_one_readonly_statement_per_role(monkeypatch):
|
||||||
|
endpoints, calls, closed, connected = session(monkeypatch)
|
||||||
|
result = asyncio.run(health.roles(endpoints))
|
||||||
|
assert result == {role: {"authority_verified": True} for role in GRANTS}
|
||||||
|
assert len(calls) == 4 and [x[0] for x in calls] == list(GRANTS)
|
||||||
|
assert closed == connected == list(GRANTS)
|
||||||
|
|
||||||
|
|
||||||
|
CONDITIONS = [
|
||||||
|
(role, "privileges", index) for role, checks in GRANTS.items() for index in range(len(checks))
|
||||||
|
] + [(role, "functions", index) for role in GRANTS for index in range(2)]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("role,field,index", CONDITIONS)
|
||||||
|
def test_every_original_grant_and_function_mismatch_refuses(monkeypatch, role, field, index):
|
||||||
|
def mutate(current, row):
|
||||||
|
if current == role:
|
||||||
|
row[field][index] = not row[field][index]
|
||||||
|
|
||||||
|
endpoints, calls, closed, connected = session(monkeypatch, mutate)
|
||||||
|
with pytest.raises(health.DeliveryError, match="database role authority differs"):
|
||||||
|
asyncio.run(health.roles(endpoints))
|
||||||
|
assert calls[-1][0] == role and closed == connected
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"field,value",
|
||||||
|
[
|
||||||
|
("privileges", None),
|
||||||
|
("privileges", {}),
|
||||||
|
("privileges", []),
|
||||||
|
("privileges", [True] * 5),
|
||||||
|
("privileges", [True, True, 0, False]),
|
||||||
|
("functions", [False]),
|
||||||
|
("functions", [False, None]),
|
||||||
|
("functions", [False, 0]),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_invalid_projection_shapes_and_non_boolean_values_refuse(monkeypatch, field, value):
|
||||||
|
def mutate(_, row):
|
||||||
|
row[field] = copy.deepcopy(value)
|
||||||
|
|
||||||
|
endpoints, calls, closed, connected = session(monkeypatch, mutate)
|
||||||
|
with pytest.raises(health.DeliveryError, match="database role projection values differ"):
|
||||||
|
asyncio.run(health.roles(endpoints))
|
||||||
|
assert len(calls) == 1 and closed == connected == ["api"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"field,value",
|
||||||
|
[
|
||||||
|
("superuser", True),
|
||||||
|
("createdb", True),
|
||||||
|
("createrole", True),
|
||||||
|
("replication", True),
|
||||||
|
("bypassrls", True),
|
||||||
|
("member", False),
|
||||||
|
("inherited", False),
|
||||||
|
("set_role", True),
|
||||||
|
("connection_limit", 9),
|
||||||
|
("database", "other_database"),
|
||||||
|
("canlogin", 1),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_role_flags_and_identity_still_refuse(monkeypatch, field, value):
|
||||||
|
def mutate(_, row):
|
||||||
|
row[field] = value
|
||||||
|
|
||||||
|
endpoints, calls, closed, connected = session(monkeypatch, mutate)
|
||||||
|
with pytest.raises(health.DeliveryError, match="database role authority differs"):
|
||||||
|
asyncio.run(health.roles(endpoints))
|
||||||
|
assert len(calls) == 1 and closed == connected == ["api"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_catalog_failure_preserves_error_and_closes_connection(monkeypatch):
|
||||||
|
original = RuntimeError("synthetic catalog failure")
|
||||||
|
endpoints, calls, closed, connected = session(monkeypatch, failure=original)
|
||||||
|
with pytest.raises(RuntimeError) as caught:
|
||||||
|
asyncio.run(health.roles(endpoints))
|
||||||
|
assert caught.value is original and len(calls) == 1 and closed == connected == ["api"]
|
||||||
@@ -0,0 +1,255 @@
|
|||||||
|
"""Ordinary string/worker models, not Root, durability or recovery acceptance."""
|
||||||
|
|
||||||
|
import importlib
|
||||||
|
import random
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
sys.path.insert(0, str(ROOT))
|
||||||
|
policy = importlib.import_module("scripts.release.policy")
|
||||||
|
file_copy = importlib.import_module("scripts.release.file_copy")
|
||||||
|
|
||||||
|
|
||||||
|
def old_path_result(value):
|
||||||
|
"""Frozen pre-optimization lexical policy, independent of the new cache."""
|
||||||
|
if not isinstance(value, str) or len(value) > 512:
|
||||||
|
return "invalid payload path"
|
||||||
|
if not all(re.fullmatch(r"[A-Za-z0-9_.+@-]{1,255}", p) for p in value.split("/")):
|
||||||
|
return "noncanonical payload path"
|
||||||
|
if any(
|
||||||
|
p in {".", "..", ".git", ".localplan", "docs", "__pycache__", ".env", ".ssh"}
|
||||||
|
for p in value.split("/")
|
||||||
|
):
|
||||||
|
return "private or unsafe payload path"
|
||||||
|
if not value.startswith(policy.ROOTS) or value.endswith((".pyc", ".pyo")):
|
||||||
|
return "payload path outside delivery allowlist"
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def test_lexical_policy_preserves_boundaries_and_refusal_categories():
|
||||||
|
values = [None, True, 1, [], {}, b"backend/a", ""]
|
||||||
|
components = [
|
||||||
|
".",
|
||||||
|
"..",
|
||||||
|
".git",
|
||||||
|
".localplan",
|
||||||
|
"docs",
|
||||||
|
"__pycache__",
|
||||||
|
".env",
|
||||||
|
".ssh",
|
||||||
|
"a//b",
|
||||||
|
"a/",
|
||||||
|
"a.pyc",
|
||||||
|
"a.pyo",
|
||||||
|
"a\x00b",
|
||||||
|
"a\nb",
|
||||||
|
"é",
|
||||||
|
"a" * 255,
|
||||||
|
"a" * 256,
|
||||||
|
"a" * 250 + "/" + "b" * 250,
|
||||||
|
"a" * 250 + "/" + "b" * 254,
|
||||||
|
]
|
||||||
|
values.extend("backend/" + component for component in components)
|
||||||
|
rng = random.Random(20261028)
|
||||||
|
for _ in range(3000):
|
||||||
|
value = "".join(rng.choices("abcXYZ019_.+@-/ \\:\n\x00é", k=rng.randrange(90)))
|
||||||
|
values.append(rng.choice(["backend/", "images/gnu/", "site/x/", ""]) + value)
|
||||||
|
for value in values:
|
||||||
|
refused = old_path_result(value)
|
||||||
|
if refused is None:
|
||||||
|
assert policy.relative_name(value) == value
|
||||||
|
else:
|
||||||
|
with pytest.raises(policy.DeliveryError) as observed:
|
||||||
|
policy.relative_name(value)
|
||||||
|
assert str(observed.value) == refused
|
||||||
|
|
||||||
|
|
||||||
|
def test_lexical_cache_is_bounded_and_custom_hash_cannot_bypass_policy():
|
||||||
|
class Collision(str):
|
||||||
|
def __hash__(self):
|
||||||
|
return hash("backend/valid.py")
|
||||||
|
|
||||||
|
def __eq__(self, other):
|
||||||
|
return True
|
||||||
|
|
||||||
|
policy._cached_relative_name.cache_clear()
|
||||||
|
try:
|
||||||
|
policy.relative_name("backend/valid.py")
|
||||||
|
before = policy._cached_relative_name.cache_info()
|
||||||
|
with pytest.raises(policy.DeliveryError, match="private or unsafe"):
|
||||||
|
policy.relative_name(Collision("backend/../escape"))
|
||||||
|
assert policy._cached_relative_name.cache_info() == before
|
||||||
|
for index in range(20000):
|
||||||
|
policy.relative_name("backend/member-" + str(index))
|
||||||
|
assert policy._cached_relative_name.cache_info().currsize == 16384
|
||||||
|
finally:
|
||||||
|
policy._cached_relative_name.cache_clear()
|
||||||
|
|
||||||
|
|
||||||
|
def test_deadline_stops_dispatch_but_waits_for_running_operations():
|
||||||
|
started = threading.Barrier(5)
|
||||||
|
release = threading.Event()
|
||||||
|
finished = threading.Event()
|
||||||
|
active = set()
|
||||||
|
errors = []
|
||||||
|
lock = threading.Lock()
|
||||||
|
|
||||||
|
def operation(index):
|
||||||
|
with lock:
|
||||||
|
active.add(index)
|
||||||
|
try:
|
||||||
|
started.wait(timeout=5)
|
||||||
|
assert release.wait(5)
|
||||||
|
return index
|
||||||
|
finally:
|
||||||
|
with lock:
|
||||||
|
active.remove(index)
|
||||||
|
|
||||||
|
def collect():
|
||||||
|
try:
|
||||||
|
file_copy.bounded_copies(list(range(32)), operation, deadline=time.monotonic() + 0.2)
|
||||||
|
except BaseException as error:
|
||||||
|
errors.append(error)
|
||||||
|
finally:
|
||||||
|
finished.set()
|
||||||
|
|
||||||
|
collector = threading.Thread(target=collect)
|
||||||
|
collector.start()
|
||||||
|
try:
|
||||||
|
started.wait(timeout=5)
|
||||||
|
assert not finished.wait(0.3)
|
||||||
|
with lock:
|
||||||
|
assert active == {0, 1, 2, 3}
|
||||||
|
finally:
|
||||||
|
release.set()
|
||||||
|
collector.join(5)
|
||||||
|
assert not collector.is_alive() and not active
|
||||||
|
assert len(errors) == 1 and isinstance(errors[0], policy.DeliveryError)
|
||||||
|
assert "copy deadline exceeded" in str(errors[0])
|
||||||
|
|
||||||
|
|
||||||
|
def test_single_operation_cannot_publish_after_its_deadline(monkeypatch):
|
||||||
|
current = [10.0]
|
||||||
|
monkeypatch.setattr(file_copy, "time", SimpleNamespace(monotonic=lambda: current[0]))
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def operation(index):
|
||||||
|
calls.append(index)
|
||||||
|
current[0] = 12.0
|
||||||
|
return "completed after deadline"
|
||||||
|
|
||||||
|
with pytest.raises(policy.DeliveryError, match="copy deadline exceeded"):
|
||||||
|
file_copy.bounded_copies([0], operation, deadline=11.0)
|
||||||
|
assert calls == [0]
|
||||||
|
|
||||||
|
|
||||||
|
def test_iterator_failure_joins_in_flight_work_without_dispatching_rest():
|
||||||
|
started, release, finished = (threading.Event() for _ in range(3))
|
||||||
|
calls, errors = [], []
|
||||||
|
|
||||||
|
class Broken(Sequence):
|
||||||
|
def __len__(self):
|
||||||
|
return 32
|
||||||
|
|
||||||
|
def __getitem__(self, index):
|
||||||
|
if index == 0:
|
||||||
|
return index
|
||||||
|
assert started.wait(5)
|
||||||
|
raise ValueError("finite inventory refusal")
|
||||||
|
|
||||||
|
def operation(index):
|
||||||
|
calls.append(index)
|
||||||
|
started.set()
|
||||||
|
assert release.wait(5)
|
||||||
|
return index
|
||||||
|
|
||||||
|
def collect():
|
||||||
|
try:
|
||||||
|
file_copy.bounded_copies(Broken(), operation)
|
||||||
|
except BaseException as error:
|
||||||
|
errors.append(error)
|
||||||
|
finally:
|
||||||
|
finished.set()
|
||||||
|
|
||||||
|
collector = threading.Thread(target=collect)
|
||||||
|
collector.start()
|
||||||
|
try:
|
||||||
|
assert started.wait(5)
|
||||||
|
assert not finished.wait(0.05)
|
||||||
|
finally:
|
||||||
|
release.set()
|
||||||
|
collector.join(5)
|
||||||
|
assert not collector.is_alive() and calls == [0]
|
||||||
|
assert len(errors) == 1 and str(errors[0]) == "finite inventory refusal"
|
||||||
|
|
||||||
|
|
||||||
|
def test_readiness_projection_ports_overlap_with_at_most_two_workers(monkeypatch):
|
||||||
|
"""Live ordinary threads over modeled ports, not systemd or VM acceptance."""
|
||||||
|
node_health = importlib.import_module("scripts.release.node_health")
|
||||||
|
lock = threading.Lock()
|
||||||
|
barrier = threading.Barrier(2, timeout=10)
|
||||||
|
active = 0
|
||||||
|
peak = 0
|
||||||
|
started = []
|
||||||
|
finished = []
|
||||||
|
payload = Path("/ordinary-readiness-port-model")
|
||||||
|
|
||||||
|
def query(name, actual_payload):
|
||||||
|
nonlocal active, peak
|
||||||
|
assert actual_payload == payload
|
||||||
|
with lock:
|
||||||
|
active += 1
|
||||||
|
peak = max(peak, active)
|
||||||
|
started.append(name)
|
||||||
|
try:
|
||||||
|
barrier.wait()
|
||||||
|
return {"component": name}
|
||||||
|
finally:
|
||||||
|
with lock:
|
||||||
|
active -= 1
|
||||||
|
finished.append(name)
|
||||||
|
|
||||||
|
monkeypatch.setattr(node_health.operator_units, "run", query)
|
||||||
|
with node_health._status_helpers(payload) as pending:
|
||||||
|
results = {name: future.result(timeout=15) for name, future in pending.items()}
|
||||||
|
names = {"broker-health", "outbox-health", "import-health", "factory-health"}
|
||||||
|
assert peak == 2 and active == 0
|
||||||
|
assert set(started) == set(finished) == set(results) == names
|
||||||
|
assert len(started) == len(finished) == 4
|
||||||
|
assert all(results[name] == {"component": name} for name in names)
|
||||||
|
assert all(future.done() for future in pending.values())
|
||||||
|
|
||||||
|
|
||||||
|
def test_readiness_projection_failure_keeps_error_and_joins_all_workers(monkeypatch):
|
||||||
|
"""A consumer error cannot abandon a dispatched ordinary helper model."""
|
||||||
|
node_health = importlib.import_module("scripts.release.node_health")
|
||||||
|
lock = threading.Lock()
|
||||||
|
barrier = threading.Barrier(2, timeout=10)
|
||||||
|
finished = []
|
||||||
|
|
||||||
|
def query(name, payload):
|
||||||
|
try:
|
||||||
|
barrier.wait()
|
||||||
|
if name == "broker-health":
|
||||||
|
raise policy.DeliveryError("bounded helper refused")
|
||||||
|
return {"component": name}
|
||||||
|
finally:
|
||||||
|
with lock:
|
||||||
|
finished.append(name)
|
||||||
|
|
||||||
|
monkeypatch.setattr(node_health.operator_units, "run", query)
|
||||||
|
with pytest.raises(RuntimeError, match="consumer refused observation"):
|
||||||
|
with node_health._status_helpers(Path("/ordinary-readiness-port-model")) as pending:
|
||||||
|
raise RuntimeError("consumer refused observation")
|
||||||
|
assert len(finished) == 4 and set(finished) == set(pending)
|
||||||
|
assert all(future.done() for future in pending.values())
|
||||||
|
with pytest.raises(policy.DeliveryError, match="bounded helper refused"):
|
||||||
|
pending["broker-health"].result()
|
||||||
@@ -0,0 +1,58 @@
|
|||||||
|
"""Ordinary exception-boundary checks; real dependency failure is VM-only."""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import errno
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import pytest
|
||||||
|
from asyncpg import CannotConnectNowError
|
||||||
|
from sqlalchemy.exc import OperationalError, ProgrammingError
|
||||||
|
from sqlalchemy.exc import TimeoutError as PoolTimeoutError
|
||||||
|
|
||||||
|
from kf_oj.api.app import create_app
|
||||||
|
from kf_oj.core.settings import Settings
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"error,expected",
|
||||||
|
[
|
||||||
|
(ConnectionRefusedError("private_connection_text"), 503),
|
||||||
|
(TimeoutError("private_timeout_text"), 503),
|
||||||
|
(CannotConnectNowError("private_database_text"), 503),
|
||||||
|
(PoolTimeoutError("private_pool_text"), 503),
|
||||||
|
(
|
||||||
|
OperationalError(
|
||||||
|
"private_query", {"secret": "private_value"}, RuntimeError("private_driver")
|
||||||
|
),
|
||||||
|
503,
|
||||||
|
),
|
||||||
|
(OSError(errno.ENOSPC, "private_source_path"), 503),
|
||||||
|
(OSError(errno.EDQUOT, "private_quota_path"), 503),
|
||||||
|
(FileNotFoundError(errno.ENOENT, "private_missing_path"), 500),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_dependency_boundary_preserves_status_without_exception_text(error, expected):
|
||||||
|
app = create_app(Settings(_env_file=None, environment="test"))
|
||||||
|
|
||||||
|
@app.get("/api/v1/auth/boundary-test")
|
||||||
|
async def fail():
|
||||||
|
raise error
|
||||||
|
|
||||||
|
async def request():
|
||||||
|
async with httpx.AsyncClient(
|
||||||
|
transport=httpx.ASGITransport(app=app), base_url="http://test"
|
||||||
|
) as client:
|
||||||
|
response = await client.get("/api/v1/auth/boundary-test")
|
||||||
|
assert response.status_code == expected
|
||||||
|
assert "private_" not in response.text
|
||||||
|
assert response.headers["cache-control"] == "no-store"
|
||||||
|
if expected == 503:
|
||||||
|
assert response.headers["retry-after"] == "5"
|
||||||
|
|
||||||
|
asyncio.run(request())
|
||||||
|
|
||||||
|
|
||||||
|
def test_programming_errors_are_not_classified_as_transient_dependency_failures():
|
||||||
|
app = create_app(Settings(_env_file=None, environment="test"))
|
||||||
|
assert ProgrammingError not in app.exception_handlers
|
||||||
|
assert ValueError not in app.exception_handlers
|
||||||
@@ -219,7 +219,7 @@ def test_real_concurrent_logins_keep_bounded_active_sessions(postgres_url):
|
|||||||
asyncio.run(run())
|
asyncio.run(run())
|
||||||
|
|
||||||
|
|
||||||
def test_account_cli_hidden_password_and_disable(postgres_url):
|
def test_account_cli_hidden_password_and_session_disable(postgres_url):
|
||||||
password = secrets.token_hex(16)
|
password = secrets.token_hex(16)
|
||||||
# The CLI prompts through a real private PTY; the password is never an argument/environment.
|
# The CLI prompts through a real private PTY; the password is never an argument/environment.
|
||||||
pid, descriptor = pty.fork()
|
pid, descriptor = pty.fork()
|
||||||
@@ -264,9 +264,7 @@ def test_account_cli_hidden_password_and_disable(postgres_url):
|
|||||||
try:
|
try:
|
||||||
service = IdentityService(sessions)
|
service = IdentityService(sessions)
|
||||||
issued = await service.login("cli_user", password, peer="cli-fixture")
|
issued = await service.login("cli_user", password, peer="cli-fixture")
|
||||||
from kf_oj.modules.identity.cli import run
|
assert await service.disable_account("cli_user")
|
||||||
|
|
||||||
await run("disable", "cli_user")
|
|
||||||
with pytest.raises(AuthenticationFailed):
|
with pytest.raises(AuthenticationFailed):
|
||||||
await service.authenticate(issued.token)
|
await service.authenticate(issued.token)
|
||||||
finally:
|
finally:
|
||||||
|
|||||||
@@ -256,9 +256,18 @@ def test_operator_registry_example_preserves_old_protocol_and_binds_managed_runt
|
|||||||
ceiling = v1.RunLimits.model_validate(selected["per_task_ceiling"])
|
ceiling = v1.RunLimits.model_validate(selected["per_task_ceiling"])
|
||||||
assert cap.compile_ceiling.fits(ceiling) and cap.run_ceiling.fits(ceiling)
|
assert cap.compile_ceiling.fits(ceiling) and cap.run_ceiling.fits(ceiling)
|
||||||
manifest = fixture(f"judge-manifest.valid-{cap.language}", v2.JobManifest).model_copy(
|
manifest = fixture(f"judge-manifest.valid-{cap.language}", v2.JobManifest).model_copy(
|
||||||
update={"compile_limits": cap.compile_ceiling, "run_limits": cap.run_ceiling}
|
update={
|
||||||
|
"compile_limits": cap.compile_ceiling,
|
||||||
|
"run_limits": cap.run_ceiling,
|
||||||
|
"job_wall_time_us": policy.multi_runtime.max_job_wall_time_us,
|
||||||
|
}
|
||||||
)
|
)
|
||||||
policy.admit(manifest, 1)
|
policy.admit(manifest, 1)
|
||||||
|
with pytest.raises(ValueError, match="job deadline cannot cover execution budgets"):
|
||||||
|
policy.admit(
|
||||||
|
manifest.model_copy(update={"job_wall_time_us": cap.compile_ceiling.wall_time_us}),
|
||||||
|
1,
|
||||||
|
)
|
||||||
changed = manifest.model_copy(
|
changed = manifest.model_copy(
|
||||||
update={"toolchain": manifest.toolchain.model_copy(update={"image_digest": "f" * 64})}
|
update={"toolchain": manifest.toolchain.model_copy(update={"image_digest": "f" * 64})}
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -0,0 +1,174 @@
|
|||||||
|
"""Real trusted control-plane identity checks; no grading/security VM claim."""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import secrets
|
||||||
|
from dataclasses import replace
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import pytest
|
||||||
|
from conftest import connect
|
||||||
|
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||||
|
from test_identity import restrict_platform_role
|
||||||
|
|
||||||
|
from kf_oj.api.app import create_app
|
||||||
|
from kf_oj.contracts.identity import AccountCapabilities
|
||||||
|
from kf_oj.core.settings import Settings
|
||||||
|
from kf_oj.modules.identity.public import AuthenticationFailed, MaintenanceDenied, Principal
|
||||||
|
from kf_oj.modules.identity.service import IdentityService
|
||||||
|
|
||||||
|
|
||||||
|
def test_capability_wire_version_and_default_denial():
|
||||||
|
assert AccountCapabilities().model_dump() == {
|
||||||
|
"schema_version": 1,
|
||||||
|
"problem_maintenance": False,
|
||||||
|
"authorization_version": "1",
|
||||||
|
}
|
||||||
|
for invalid in (
|
||||||
|
{"schema_version": 2},
|
||||||
|
{"authorization_version": "0"},
|
||||||
|
{"authorization_version": 1},
|
||||||
|
{"role": "admin"},
|
||||||
|
):
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
AccountCapabilities.model_validate(invalid)
|
||||||
|
|
||||||
|
|
||||||
|
async def set_administrator(connection, account_id, enabled):
|
||||||
|
username = await connection.fetchval("SELECT username FROM accounts WHERE id=$1", account_id)
|
||||||
|
active = await connection.fetchval(
|
||||||
|
"SELECT count(*) FROM accounts WHERE enabled AND is_administrator"
|
||||||
|
)
|
||||||
|
async with connection.transaction():
|
||||||
|
await connection.execute("SET LOCAL ROLE kf_oj_identity_operator")
|
||||||
|
operation = ("grant" if active else "bootstrap") if enabled else "revoke"
|
||||||
|
await connection.execute(
|
||||||
|
"SELECT * FROM public.kf_manage_account_authority($1,$2)", username, operation
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def add_recovery_administrator(connection):
|
||||||
|
await connection.execute(
|
||||||
|
"INSERT INTO accounts(id,username,password_hash) "
|
||||||
|
"VALUES(gen_random_uuid(),'authority_recovery',repeat('synthetic-only-',5))"
|
||||||
|
)
|
||||||
|
async with connection.transaction():
|
||||||
|
await connection.execute("SET LOCAL ROLE kf_oj_identity_operator")
|
||||||
|
await connection.execute(
|
||||||
|
"SELECT * FROM public.kf_manage_account_authority('authority_recovery','grant')"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_live_http_capabilities_and_old_cookies_after_grant_and_revoke(postgres_url):
|
||||||
|
async def run():
|
||||||
|
app = create_app(Settings(_env_file=None, environment="test", database_url=postgres_url))
|
||||||
|
connection = await connect(postgres_url)
|
||||||
|
try:
|
||||||
|
async with app.router.lifespan_context(app):
|
||||||
|
restrict_platform_role(app.state.sessions.kw["bind"])
|
||||||
|
password = secrets.token_hex(16)
|
||||||
|
account = await app.state.identity.create_account("authority_user", password)
|
||||||
|
async with httpx.AsyncClient(
|
||||||
|
transport=httpx.ASGITransport(app=app), base_url="http://localhost:8000"
|
||||||
|
) as client:
|
||||||
|
headers = {"Origin": "http://localhost:8000"}
|
||||||
|
body = {"username": account.username, "password": password}
|
||||||
|
ordinary = await client.post("/api/v1/auth/login", json=body, headers=headers)
|
||||||
|
assert ordinary.status_code == 200
|
||||||
|
assert ordinary.json()["user"]["capabilities"] == {
|
||||||
|
"schema_version": 1,
|
||||||
|
"problem_maintenance": False,
|
||||||
|
"authorization_version": "1",
|
||||||
|
}
|
||||||
|
await set_administrator(connection, account.account_id, True)
|
||||||
|
assert (await client.get("/api/v1/auth/me")).status_code == 401
|
||||||
|
fresh = await client.post("/api/v1/auth/login", json=body, headers=headers)
|
||||||
|
assert fresh.status_code == 200
|
||||||
|
user = fresh.json()["user"]
|
||||||
|
assert user["capabilities"]["problem_maintenance"] is True
|
||||||
|
assert user["capabilities"]["authorization_version"] == "2"
|
||||||
|
actual = await client.get(
|
||||||
|
"/api/v1/auth/me", headers={"X-Role": "admin", "X-User-ID": "invalid"}
|
||||||
|
)
|
||||||
|
assert actual.json() == user
|
||||||
|
assert actual.headers["cache-control"] == "no-store"
|
||||||
|
await add_recovery_administrator(connection)
|
||||||
|
await set_administrator(connection, account.account_id, False)
|
||||||
|
assert (await client.get("/api/v1/auth/me")).status_code == 401
|
||||||
|
refused = await client.post(
|
||||||
|
"/api/v1/auth/login", json=body | {"role": "admin"}, headers=headers
|
||||||
|
)
|
||||||
|
assert refused.status_code == 422
|
||||||
|
ordinary = await client.post("/api/v1/auth/login", json=body, headers=headers)
|
||||||
|
assert ordinary.json()["user"]["capabilities"]["problem_maintenance"] is False
|
||||||
|
assert ordinary.json()["user"]["capabilities"]["authorization_version"] == "3"
|
||||||
|
finally:
|
||||||
|
await connection.close()
|
||||||
|
|
||||||
|
asyncio.run(run())
|
||||||
|
|
||||||
|
|
||||||
|
def test_write_transaction_rechecks_session_and_serializes_authority_change(postgres_url):
|
||||||
|
async def run():
|
||||||
|
engine = create_async_engine(postgres_url)
|
||||||
|
restrict_platform_role(engine)
|
||||||
|
sessions = async_sessionmaker(engine, expire_on_commit=False)
|
||||||
|
identity = IdentityService(sessions)
|
||||||
|
connection, other = await connect(postgres_url), await connect(postgres_url)
|
||||||
|
pending = None
|
||||||
|
try:
|
||||||
|
password = secrets.token_hex(16)
|
||||||
|
account = await identity.create_account("authority_user", password)
|
||||||
|
await set_administrator(connection, account.account_id, True)
|
||||||
|
issued = await identity.login(account.username, password, peer="authority-fixture")
|
||||||
|
principal = await identity.authenticate(issued.token)
|
||||||
|
assert principal.can_maintain_problems and principal.authorization_version == 2
|
||||||
|
assert principal.session_token_hash not in repr(principal)
|
||||||
|
await add_recovery_administrator(connection)
|
||||||
|
async with sessions.begin() as session:
|
||||||
|
await identity.require_problem_maintenance(session, principal)
|
||||||
|
pending = asyncio.create_task(set_administrator(other, account.account_id, False))
|
||||||
|
await asyncio.sleep(0.1)
|
||||||
|
assert not pending.done() # Real PG SHARE lock holds through the write transaction.
|
||||||
|
await asyncio.wait_for(pending, timeout=5)
|
||||||
|
with pytest.raises(AuthenticationFailed):
|
||||||
|
await identity.authenticate(issued.token)
|
||||||
|
async with sessions.begin() as session:
|
||||||
|
with pytest.raises(MaintenanceDenied):
|
||||||
|
await identity.require_problem_maintenance(session, principal)
|
||||||
|
await set_administrator(connection, account.account_id, True)
|
||||||
|
issued = await identity.login(account.username, password, peer="authority-fresh")
|
||||||
|
principal = await identity.authenticate(issued.token)
|
||||||
|
async with sessions.begin() as session:
|
||||||
|
for forged in (
|
||||||
|
Principal(account.account_id, account.username, True),
|
||||||
|
replace(principal, session_token_hash="0" * 64),
|
||||||
|
replace(principal, authorization_version=1),
|
||||||
|
):
|
||||||
|
with pytest.raises(MaintenanceDenied):
|
||||||
|
await identity.require_problem_maintenance(session, forged)
|
||||||
|
await identity.logout(issued.token, issued.csrf_token)
|
||||||
|
async with sessions.begin() as session:
|
||||||
|
with pytest.raises(MaintenanceDenied):
|
||||||
|
await identity.require_problem_maintenance(session, principal)
|
||||||
|
issued = await identity.login(account.username, password, peer="authority-expiry")
|
||||||
|
principal = await identity.authenticate(issued.token)
|
||||||
|
# Use actual wall-clock expiry even inside an already-open transaction.
|
||||||
|
async with sessions.begin() as session:
|
||||||
|
await session.connection()
|
||||||
|
await connection.execute(
|
||||||
|
"UPDATE sessions SET expires_at=clock_timestamp()+interval '50 milliseconds' "
|
||||||
|
"WHERE token_hash=$1",
|
||||||
|
principal.session_token_hash,
|
||||||
|
)
|
||||||
|
await asyncio.sleep(0.1)
|
||||||
|
with pytest.raises(MaintenanceDenied):
|
||||||
|
await identity.require_problem_maintenance(session, principal)
|
||||||
|
finally:
|
||||||
|
if pending is not None and not pending.done():
|
||||||
|
pending.cancel()
|
||||||
|
await asyncio.gather(pending, return_exceptions=True)
|
||||||
|
await connection.close()
|
||||||
|
await other.close()
|
||||||
|
await engine.dispose()
|
||||||
|
|
||||||
|
asyncio.run(run())
|
||||||
@@ -0,0 +1,227 @@
|
|||||||
|
"""Ordinary controller/codec models; no service or isolation acceptance."""
|
||||||
|
|
||||||
|
import copy
|
||||||
|
import hashlib
|
||||||
|
import importlib
|
||||||
|
import json
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||||
|
units = importlib.import_module("scripts.release.operator_units")
|
||||||
|
install = importlib.import_module("scripts.release.install")
|
||||||
|
service_install = importlib.import_module("scripts.release.service_install")
|
||||||
|
|
||||||
|
|
||||||
|
def record():
|
||||||
|
return units.failure_record(
|
||||||
|
"factory-health",
|
||||||
|
"exit",
|
||||||
|
subprocess.CompletedProcess([], 1, b"private-output", b"private-error"),
|
||||||
|
units.DeliveryError("operator helper unavailable"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def dispatched(monkeypatch):
|
||||||
|
selected = {"release_id": "sd-" + "a" * 12, "configuration_id": "cfg-" + "b" * 24}
|
||||||
|
payload = install.RELEASES / selected["release_id"] / "payload"
|
||||||
|
monkeypatch.setattr(units.os, "getuid", lambda: 0)
|
||||||
|
monkeypatch.setattr(units.os, "geteuid", lambda: 0)
|
||||||
|
monkeypatch.setattr(install, "selection", lambda **_: selected)
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def manager(*args):
|
||||||
|
calls.append(args)
|
||||||
|
return "LoadState=not-found\nActiveState=inactive\nMainPID=0\n"
|
||||||
|
|
||||||
|
monkeypatch.setattr(service_install, "systemctl", manager)
|
||||||
|
return payload, calls
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("failure", ["exit", "timeout", "decode", "stderr_bound"])
|
||||||
|
def test_dispatch_diagnostics_keep_limits_cleanup_and_only_output_digests(
|
||||||
|
monkeypatch, dispatched, failure
|
||||||
|
):
|
||||||
|
payload, calls = dispatched
|
||||||
|
|
||||||
|
def execute(command, **kwargs):
|
||||||
|
assert kwargs["timeout"] == 915
|
||||||
|
assert "--property=MemoryMax=256M" in command
|
||||||
|
assert "--property=RuntimeMaxSec=900" in command
|
||||||
|
if failure == "timeout":
|
||||||
|
raise subprocess.TimeoutExpired(
|
||||||
|
command, 915, output=b"private-output", stderr=b"private-error"
|
||||||
|
)
|
||||||
|
return subprocess.CompletedProcess(
|
||||||
|
command,
|
||||||
|
1 if failure == "exit" else 0,
|
||||||
|
b"private-output",
|
||||||
|
b"private-error" * (400 if failure == "stderr_bound" else 1),
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr(units.subprocess, "run", execute)
|
||||||
|
with pytest.raises(units.OperatorExecutionError) as caught:
|
||||||
|
units.run("node-maintenance", payload, action="start")
|
||||||
|
error = caught.value
|
||||||
|
assert len(calls) == 2
|
||||||
|
row = error.failures[0]
|
||||||
|
assert row["helper"] == "node-maintenance"
|
||||||
|
assert row["outcome"] == ("exit" if failure == "stderr_bound" else failure)
|
||||||
|
assert row["stdout_bytes"] == 14
|
||||||
|
assert row["stdout_sha256"] == hashlib.sha256(b"private-output").hexdigest()
|
||||||
|
text = units.diagnostic_suffix(error)
|
||||||
|
assert "private-" not in text and len(text) <= 3072
|
||||||
|
assert units.diagnostic_chain(text.encode()) == error.failures
|
||||||
|
|
||||||
|
|
||||||
|
def test_nested_failure_retains_inner_helper_and_outer_actual_exit(monkeypatch, dispatched):
|
||||||
|
payload, calls = dispatched
|
||||||
|
child = units.OperatorExecutionError("private-message", [record()])
|
||||||
|
stderr = b"private-error\n" + units.diagnostic_suffix(child).encode()
|
||||||
|
monkeypatch.setattr(
|
||||||
|
units.subprocess,
|
||||||
|
"run",
|
||||||
|
lambda command, **_: subprocess.CompletedProcess(command, 1, b"", stderr),
|
||||||
|
)
|
||||||
|
with pytest.raises(units.OperatorExecutionError) as caught:
|
||||||
|
units.run("node-maintenance", payload, action="start")
|
||||||
|
rows = caught.value.failures
|
||||||
|
assert [r["helper"] for r in rows] == ["factory-health", "node-maintenance"]
|
||||||
|
assert (
|
||||||
|
rows[1]["returncode"] == 1
|
||||||
|
and rows[1]["stderr_sha256"] == hashlib.sha256(stderr).hexdigest()
|
||||||
|
)
|
||||||
|
assert len(calls) == 2 and "private-" not in units.diagnostic_suffix(caught.value)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"mutation",
|
||||||
|
[
|
||||||
|
"unknown_field",
|
||||||
|
"helper",
|
||||||
|
"outcome",
|
||||||
|
"boolean_code",
|
||||||
|
"boolean_size",
|
||||||
|
"digest",
|
||||||
|
"unknown_readiness",
|
||||||
|
"deep",
|
||||||
|
"schema",
|
||||||
|
"duplicate",
|
||||||
|
"oversize",
|
||||||
|
"invalid_json",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_untrusted_child_diagnostic_shape_is_discarded(mutation):
|
||||||
|
value = {"schema_version": 1, "failures": [record()]}
|
||||||
|
row = value["failures"][0]
|
||||||
|
if mutation == "unknown_field":
|
||||||
|
row["private"] = "must-not-escape"
|
||||||
|
elif mutation == "helper":
|
||||||
|
row["helper"] = "/private/command"
|
||||||
|
elif mutation == "outcome":
|
||||||
|
row["outcome"] = ["private-error"]
|
||||||
|
elif mutation == "boolean_code":
|
||||||
|
row["returncode"] = True
|
||||||
|
elif mutation == "boolean_size":
|
||||||
|
row["stdout_bytes"] = True
|
||||||
|
elif mutation == "digest":
|
||||||
|
row["stderr_sha256"] = "private-error"
|
||||||
|
elif mutation == "unknown_readiness":
|
||||||
|
row["readiness_errors"] = ["private-error"]
|
||||||
|
elif mutation == "deep":
|
||||||
|
value["failures"] *= 4
|
||||||
|
elif mutation == "schema":
|
||||||
|
value["schema_version"] = True
|
||||||
|
text = (units.DIAGNOSTIC_PREFIX + json.dumps(value) + "\n").encode()
|
||||||
|
if mutation == "duplicate":
|
||||||
|
text *= 2
|
||||||
|
elif mutation == "oversize":
|
||||||
|
text += b"x" * 4097
|
||||||
|
elif mutation == "invalid_json":
|
||||||
|
text = units.DIAGNOSTIC_PREFIX.encode() + b"{broken"
|
||||||
|
assert units.diagnostic_chain(text) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_readiness_preserves_two_inner_failures_and_only_fixed_codes():
|
||||||
|
rows = [record(), copy.deepcopy(record()), copy.deepcopy(record())]
|
||||||
|
error = units.readiness_failure(
|
||||||
|
"preconditions unavailable",
|
||||||
|
{
|
||||||
|
"errors": ["factory_status_unavailable"],
|
||||||
|
"operator_failures": rows,
|
||||||
|
"password": "private-value",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert len(error.failures) == 3
|
||||||
|
assert error.failures[-1]["readiness_errors"] == ["factory_status_unavailable"]
|
||||||
|
assert "private-" not in units.diagnostic_suffix(error)
|
||||||
|
refused = units.readiness_failure(
|
||||||
|
"admission unavailable", {"errors": ["private-value"]}, admission=True
|
||||||
|
)
|
||||||
|
assert refused.failures[0]["outcome"] == "admission"
|
||||||
|
assert "readiness_errors" not in refused.failures[0]
|
||||||
|
|
||||||
|
|
||||||
|
def test_readonly_health_keeps_failed_components_and_bounded_nested_causes(monkeypatch, tmp_path):
|
||||||
|
from concurrent.futures import Future
|
||||||
|
from contextlib import nullcontext
|
||||||
|
|
||||||
|
health = importlib.import_module("scripts.release.node_health")
|
||||||
|
selected = {"release_id": "sd-" + "a" * 12, "configuration_id": "cfg-" + "b" * 24}
|
||||||
|
payload = tmp_path / "payload"
|
||||||
|
monkeypatch.setattr(units, "guarded", lambda _: None)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
health, "binding", lambda: (selected, {"coordinator.json": Path("coordinator")}, payload)
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
health,
|
||||||
|
"read_json",
|
||||||
|
lambda path, *args: (
|
||||||
|
{"source_sha": "a" * 40}
|
||||||
|
if path.name == "manifest.json"
|
||||||
|
else {"platform_token": "private-token", "admission": {}}
|
||||||
|
),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(health.journal, "check_ready", lambda: {})
|
||||||
|
monkeypatch.setattr(health.proxy_install, "check_binding", lambda *args: None)
|
||||||
|
monkeypatch.setattr(health.proxy_health, "observe", lambda _: {})
|
||||||
|
monkeypatch.setattr(health, "inspect_unit", lambda *args: {})
|
||||||
|
monkeypatch.setattr(health.dependency_install, "installed_endpoints", lambda _: {})
|
||||||
|
|
||||||
|
async def roles(_):
|
||||||
|
return {}
|
||||||
|
|
||||||
|
monkeypatch.setattr(health, "roles", roles)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
health, "http", lambda *args, **kwargs: {"status": "ok", "service": "platform"}
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(health, "validate_execution", lambda *args: {})
|
||||||
|
monkeypatch.setattr(install, "selection", lambda **kwargs: selected)
|
||||||
|
pending = {}
|
||||||
|
for name in ("broker-health", "outbox-health", "import-health", "factory-health"):
|
||||||
|
future = Future()
|
||||||
|
row = record()
|
||||||
|
row["helper"] = name
|
||||||
|
future.set_exception(units.OperatorExecutionError("operator helper unavailable", [row]))
|
||||||
|
pending[name] = future
|
||||||
|
monkeypatch.setattr(health, "_status_helpers", lambda _: nullcontext(pending))
|
||||||
|
observed = health.observe(probe=False)
|
||||||
|
assert observed["ready_for_admission"] is False
|
||||||
|
assert observed["service_preconditions_met"] is False
|
||||||
|
assert observed["errors"] == [
|
||||||
|
"broker_metadata_unavailable",
|
||||||
|
"outbox_pipeline_unavailable",
|
||||||
|
"import_pipeline_unavailable",
|
||||||
|
"factory_status_unavailable",
|
||||||
|
]
|
||||||
|
error = units.readiness_failure("service preconditions unavailable", observed)
|
||||||
|
assert [r["helper"] for r in error.failures] == [
|
||||||
|
"broker-health",
|
||||||
|
"outbox-health",
|
||||||
|
"node-health",
|
||||||
|
]
|
||||||
|
assert "private-" not in units.diagnostic_suffix(error)
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user