feat(problems): add bounded administrator package imports

Use live maintenance sessions, durable capacity reservations and streamed SHA-bound ZIP uploads. An ordinary capped import worker validates stored/deflated v1/v2 packages, persists immutable unpublished candidates and finite audit events, and recovers or releases capacity after interruption and expiry.

Validate actual PG/HTTP/worker/storage fault paths in an isolated capped VM (2 passed, 0 skipped), reuse unchanged actual migration roundtrip evidence, and pass affected backend/architecture checks plus the prior full engineering checks.
This commit is contained in:
2026-10-11 22:19:38 +08:00
parent 234ab23155
commit 3152d59056
35 changed files with 5959 additions and 4 deletions
+2
View File
@@ -10,6 +10,8 @@ from kf_oj.infra.db.base import Base
from kf_oj.infra.db.session import build_engine
from kf_oj.infra.messaging import models as messaging_models # noqa: F401
from kf_oj.modules.identity import models as identity_models # noqa: F401
from kf_oj.modules.operations import models as operations_models # noqa: F401
from kf_oj.modules.problems import import_models # noqa: F401
from kf_oj.modules.problems import models as problem_models # noqa: F401
from kf_oj.modules.submissions import models as submission_models # noqa: F401
@@ -0,0 +1,189 @@
"""Durable imports/candidates, append-only events and accepted-authority worker port."""
from alembic import op
from kf_oj.modules.operations.models import MaintenanceEvent
from kf_oj.modules.problems.import_models import (
CandidateCase,
ImportPool,
ProblemCandidate,
ProblemImportOperation,
)
revision = "0007_problem_imports"
down_revision = "0006_authority_operations"
branch_labels = depends_on = None
def sql(script):
# asyncpg prepares one statement at a time; dollar-quoted bodies stay intact.
buffer, quote, dollar, i = [], False, False, 0
while i < len(script):
if not quote and script[i : i + 2] == "$$":
dollar = not dollar
buffer.append("$$")
i += 2
continue
character = script[i]
if not dollar and character == "'":
if quote and script[i : i + 2] == "''":
buffer.append("''")
i += 2
continue
quote = not quote
if character == ";" and not dollar and not quote:
if "".join(buffer).strip():
op.execute("".join(buffer))
buffer = []
else:
buffer.append(character)
i += 1
if "".join(buffer).strip():
op.execute("".join(buffer))
def upgrade():
connection = op.get_bind()
# Cyclic candidate/operation link is installed only after both tables exist.
for model in (
ImportPool,
ProblemImportOperation,
ProblemCandidate,
CandidateCase,
MaintenanceEvent,
):
model.__table__.create(connection)
op.create_foreign_key(
"fk_problem_import_candidate",
"problem_import_operations",
"problem_candidates",
["candidate_id"],
["id"],
)
op.execute("INSERT INTO problem_import_pool(id,reserved_bytes) VALUES(1,0)")
sql("""
CREATE FUNCTION public.kf_lock_reserved_maintenance(actor uuid, epoch bigint)
RETURNS void LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$
DECLARE permitted boolean;
BEGIN
SELECT enabled AND is_administrator AND authorization_version=epoch
INTO permitted FROM public.accounts WHERE id=actor FOR SHARE;
IF permitted IS DISTINCT FROM true THEN
RAISE EXCEPTION 'problem maintenance required' USING ERRCODE='42501';
END IF;
END $$;
GRANT CREATE ON SCHEMA public TO kf_oj_identity_authority_owner;
ALTER FUNCTION public.kf_lock_reserved_maintenance(uuid,bigint)
OWNER TO kf_oj_identity_authority_owner;
REVOKE CREATE ON SCHEMA public FROM kf_oj_identity_authority_owner;
REVOKE ALL ON FUNCTION public.kf_lock_reserved_maintenance(uuid,bigint) FROM PUBLIC;
GRANT EXECUTE ON FUNCTION public.kf_lock_reserved_maintenance(uuid,bigint)
TO kf_oj_problem_import;
GRANT SELECT,INSERT,UPDATE ON problem_import_operations TO kf_oj_platform;
GRANT SELECT,UPDATE ON problem_import_pool TO kf_oj_platform,kf_oj_problem_import;
GRANT SELECT ON problem_candidates,problem_candidate_cases TO kf_oj_platform;
GRANT SELECT ON problem_import_operations TO kf_oj_problem_import;
GRANT UPDATE(state,updated_at,lease_until,attempts,fence,
reserved_bytes,failure_code,candidate_id)
ON problem_import_operations TO kf_oj_problem_import;
GRANT SELECT,INSERT,DELETE ON problem_candidates,problem_candidate_cases
TO kf_oj_problem_import;
GRANT SELECT,INSERT ON artifacts TO kf_oj_problem_import;
GRANT INSERT ON problem_maintenance_events TO kf_oj_platform,kf_oj_problem_import;
GRANT SELECT ON problem_maintenance_events TO kf_oj_platform;
""")
sql("""
CREATE FUNCTION public.kf_immutable_import_candidate() RETURNS trigger
LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$
DECLARE candidate uuid; current_state text;
BEGIN
IF TG_TABLE_NAME='problem_candidates' THEN candidate=OLD.id;
ELSE candidate=OLD.candidate_id; END IF;
SELECT o.state INTO current_state FROM public.problem_candidates c
JOIN public.problem_import_operations o ON o.id=c.operation_id WHERE c.id=candidate;
IF TG_OP='UPDATE' OR current_state NOT IN ('discarded','expired','failed') THEN
RAISE EXCEPTION 'immutable candidate' USING ERRCODE='23514';
END IF;
RETURN OLD;
END $$;
CREATE TRIGGER immutable_candidate BEFORE UPDATE OR DELETE ON problem_candidates
FOR EACH ROW EXECUTE FUNCTION public.kf_immutable_import_candidate();
CREATE TRIGGER immutable_candidate_case BEFORE UPDATE OR DELETE ON problem_candidate_cases
FOR EACH ROW EXECUTE FUNCTION public.kf_immutable_import_candidate();
CREATE FUNCTION public.kf_insert_import_candidate() RETURNS trigger
LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$
DECLARE operation uuid; current_state text;
BEGIN
IF TG_TABLE_NAME='problem_candidates' THEN operation=NEW.operation_id;
ELSE SELECT operation_id INTO operation FROM public.problem_candidates
WHERE id=NEW.candidate_id; END IF;
SELECT state INTO current_state FROM public.problem_import_operations
WHERE id=operation FOR SHARE;
IF current_state IS DISTINCT FROM 'validating' THEN
RAISE EXCEPTION 'candidate is not being validated' USING ERRCODE='23514';
END IF; RETURN NEW;
END $$;
CREATE TRIGGER insert_candidate BEFORE INSERT ON problem_candidates
FOR EACH ROW EXECUTE FUNCTION public.kf_insert_import_candidate();
CREATE TRIGGER insert_candidate_case BEFORE INSERT ON problem_candidate_cases
FOR EACH ROW EXECUTE FUNCTION public.kf_insert_import_candidate();
CREATE FUNCTION public.kf_fixed_import_receipt() RETURNS trigger
LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$ BEGIN
IF NEW.id IS DISTINCT FROM OLD.id OR NEW.owner_id IS DISTINCT FROM OLD.owner_id
OR NEW.authorization_version IS DISTINCT FROM OLD.authorization_version
OR NEW.archive_digest IS DISTINCT FROM OLD.archive_digest
OR NEW.archive_bytes IS DISTINCT FROM OLD.archive_bytes
OR NEW.created_at IS DISTINCT FROM OLD.created_at
OR NEW.expires_at IS DISTINCT FROM OLD.expires_at THEN
RAISE EXCEPTION 'immutable import receipt' USING ERRCODE='23514';
END IF;
IF OLD.state IN ('ready','published') AND NEW.state IN ('ready','published')
AND NEW.candidate_id IS DISTINCT FROM OLD.candidate_id THEN
RAISE EXCEPTION 'immutable completed candidate receipt' USING ERRCODE='23514';
END IF;
IF OLD.state='published' AND (NEW.state IS DISTINCT FROM OLD.state
OR NEW.published_version_id IS DISTINCT FROM OLD.published_version_id) THEN
RAISE EXCEPTION 'immutable published import receipt' USING ERRCODE='23514';
END IF;
IF NEW.state='ready' THEN
IF (OLD.state IS DISTINCT FROM 'ready' AND
(NOT pg_has_role(current_user,'kf_oj_problem_import','USAGE')
OR OLD.state!='validating')) OR NEW.candidate_id IS NULL
OR NOT EXISTS(SELECT 1 FROM public.problem_candidates c
WHERE c.id=NEW.candidate_id AND c.operation_id=NEW.id
AND c.case_count=(SELECT count(*) FROM public.problem_candidate_cases b
WHERE b.candidate_id=c.id)
AND c.case_count-1=(SELECT max(position) FROM public.problem_candidate_cases b
WHERE b.candidate_id=c.id)) THEN
RAISE EXCEPTION 'invalid candidate completion' USING ERRCODE='23514';
END IF;
END IF; RETURN NEW;
END $$;
CREATE TRIGGER fixed_import_receipt BEFORE UPDATE ON problem_import_operations
FOR EACH ROW EXECUTE FUNCTION public.kf_fixed_import_receipt();
""")
def downgrade():
connection = op.get_bind()
if connection.exec_driver_sql(
"SELECT EXISTS(SELECT 1 FROM problem_import_operations)"
).scalar():
raise RuntimeError("cannot discard durable import history")
op.execute("DROP FUNCTION public.kf_lock_reserved_maintenance(uuid,bigint)")
op.drop_constraint(
"fk_problem_import_candidate", "problem_import_operations", type_="foreignkey"
)
for model in (
MaintenanceEvent,
CandidateCase,
ProblemCandidate,
ProblemImportOperation,
ImportPool,
):
model.__table__.drop(connection)
sql(
"DROP FUNCTION public.kf_immutable_import_candidate(); "
"DROP FUNCTION public.kf_fixed_import_receipt(); "
"DROP FUNCTION public.kf_insert_import_candidate()"
)
+1
View File
@@ -22,6 +22,7 @@ dependencies = [
]
[project.scripts]
kf-problem-import-worker = "kf_oj.workers.problem_import:main"
kf-accounts = "kf_oj.modules.identity.cli:main"
kf-problems = "kf_oj.modules.problems.cli:main"
kf-artifacts = "kf_oj.infra.artifacts.cli:main"
@@ -0,0 +1,26 @@
"""Uniform live session gates; domains still lock/recheck authority in write transactions."""
from typing import Annotated
from fastapi import Depends, HTTPException
from kf_oj.api.dependencies import current_principal, unsafe_principal
from kf_oj.modules.identity.public import Principal
async def maintenance_reader(
principal: Annotated[Principal, Depends(current_principal)],
) -> Principal:
if not principal.can_maintain_problems:
raise HTTPException(403, "problem maintenance required")
return principal
async def maintenance_writer(
principal: Annotated[Principal, Depends(unsafe_principal)],
) -> Principal:
# unsafe_principal already checks an explicit Origin and the live session's
# bound CSRF token. This capability check cannot replace the domain's port.
if not principal.can_maintain_problems:
raise HTTPException(403, "problem maintenance required")
return principal
+24 -1
View File
@@ -7,6 +7,7 @@ from starlette.responses import JSONResponse
from kf_oj import __version__
from kf_oj.api.body_limits import BodyLimitMiddleware
from kf_oj.api.v1.admin_imports import router as import_router
from kf_oj.api.v1.identity import router as identity_router
from kf_oj.api.v1.problems import router as problems_router
from kf_oj.api.v1.router import router
@@ -15,10 +16,13 @@ from kf_oj.api.v1.worker_artifacts import router as worker_router
from kf_oj.core.logging import configure_logging
from kf_oj.core.settings import Settings
from kf_oj.infra.artifacts.repository import ArtifactRepository
from kf_oj.infra.artifacts.staging import ImportStaging
from kf_oj.infra.artifacts.store import ArtifactStore
from kf_oj.infra.coordinator import CoordinatorControl
from kf_oj.infra.db.session import build_engine, build_session_factory
from kf_oj.modules.identity.service import IdentityService
from kf_oj.modules.operations.service import MaintenanceAudit
from kf_oj.modules.problems.imports import ProblemImports
from kf_oj.modules.problems.service import ProblemService
from kf_oj.modules.submissions.performance import SubmissionPerformanceQueries
from kf_oj.modules.submissions.queries import SubmissionQueries
@@ -36,6 +40,7 @@ def create_app(settings: Settings | None = None) -> FastAPI:
logger.info("Platform API started")
engine = None
store = None
staging = None
try:
if settings.database_url is not None:
engine = build_engine(settings)
@@ -52,6 +57,18 @@ def create_app(settings: Settings | None = None) -> FastAPI:
max_files=settings.artifact_max_files,
)
artifacts = ArtifactRepository(store)
if (
settings.import_staging_root is not None
and settings.judge_admission is not None
):
staging = ImportStaging(settings.import_staging_root)
app.state.problem_imports = ProblemImports(
app.state.sessions,
staging,
app.state.identity,
MaintenanceAudit(),
settings.judge_admission,
)
app.state.problems = ProblemService(
app.state.sessions, artifacts, settings.judge_admission
)
@@ -80,10 +97,13 @@ def create_app(settings: Settings | None = None) -> FastAPI:
app.state.submissions = None
app.state.submission_queries = None
app.state.submission_performance = None
app.state.problem_imports = None
if engine is not None:
await engine.dispose()
if store is not None:
store.close()
if staging is not None:
staging.close()
logger.info("Platform API stopped")
app = FastAPI(
@@ -101,7 +121,9 @@ def create_app(settings: Settings | None = None) -> FastAPI:
@app.middleware("http")
async def private_responses(request, call_next):
response = await call_next(request)
if request.url.path.startswith(("/api/v1/auth", "/api/v1/submissions", "/api/v1/internal")):
if request.url.path.startswith(
("/api/v1/auth", "/api/v1/submissions", "/api/v1/internal", "/api/v1/admin")
):
response.headers["Cache-Control"] = "no-store"
return response
@@ -114,4 +136,5 @@ def create_app(settings: Settings | None = None) -> FastAPI:
app.include_router(problems_router, prefix="/api/v1")
app.include_router(worker_router, prefix="/api/v1")
app.include_router(submission_router, prefix="/api/v1")
app.include_router(import_router, prefix="/api/v1")
return app
+9
View File
@@ -1,6 +1,7 @@
"""Bound request bytes and upload time before schema parsing can allocate large inputs."""
import asyncio
import re
from starlette.responses import JSONResponse
@@ -13,6 +14,14 @@ class BodyLimitMiddleware:
if scope["type"] != "http" or scope["method"] in {"GET", "HEAD", "OPTIONS"}:
await self.app(scope, receive, send)
return
if scope["method"] == "PUT" and re.fullmatch(
r"/api/v1/admin/problem-imports/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/archive",
scope["path"],
):
# Only this exact endpoint can stream. Its live administrator gate runs
# before receive(), then its service enforces actual bytes/chunks/deadline.
await self.app(scope, receive, send)
return
limit = 8192 if scope["path"] == "/api/v1/auth/login" else 1024 * 1024
if scope["path"] == "/api/v1/submissions":
# Max256KiB UTF-8 source, worst-case JSON escaping plus bounded metadata.
+139
View File
@@ -0,0 +1,139 @@
"""Administrator import contract; configured service availability is explicit."""
from typing import Annotated
from uuid import UUID
from fastapi import APIRouter, Depends, HTTPException, Query, Request
from sqlalchemy.exc import DBAPIError
from starlette.requests import ClientDisconnect
from kf_oj.api.admin_dependencies import maintenance_reader, maintenance_writer
from kf_oj.contracts.admin_problems import (
CreateProblemImport,
ProblemImportCapabilities,
ProblemImportDetail,
ProblemImportPage,
ProblemImportSummary,
)
from kf_oj.infra.artifacts.store import ArtifactError
from kf_oj.modules.identity.public import MaintenanceDenied, Principal
from kf_oj.modules.problems.imports import ImportConflict, ImportMissing, ImportUnavailable
router = APIRouter(prefix="/admin/problem-imports", tags=["problem-maintenance"])
@router.get("/capabilities", response_model=ProblemImportCapabilities)
async def capabilities(
request: Request, _: Annotated[Principal, Depends(maintenance_reader)]
) -> ProblemImportCapabilities:
service = getattr(request.app.state, "problem_imports", None)
if service is None:
return ProblemImportCapabilities(uploads_available=False, validation_available=False)
return await service.capabilities()
def service(request):
result = getattr(request.app.state, "problem_imports", None)
if result is None:
raise HTTPException(503, "problem imports unavailable")
return result
async def operation(awaitable):
try:
return await awaitable
except MaintenanceDenied:
raise HTTPException(403, "problem maintenance required") from None
except ImportConflict:
raise HTTPException(409, "import request conflict") from None
except ImportMissing:
raise HTTPException(404, "import unavailable") from None
except TimeoutError:
raise HTTPException(408, "upload deadline exceeded") from None
except (ImportUnavailable, ArtifactError, OSError, DBAPIError):
raise HTTPException(503, "problem imports unavailable") from None
except ClientDisconnect:
raise HTTPException(400, "upload interrupted") from None
@router.post("", status_code=201, response_model=ProblemImportSummary)
async def create(
body: CreateProblemImport,
request: Request,
principal: Annotated[Principal, Depends(maintenance_writer)],
):
if request.headers.get("content-type", "").split(";", 1)[0] != "application/json":
raise HTTPException(415, "JSON request required")
return await operation(service(request).create(principal, body))
@router.get("", response_model=ProblemImportPage)
async def page(
request: Request,
principal: Annotated[Principal, Depends(maintenance_reader)],
after: UUID | None = None,
limit: Annotated[int, Query(ge=1, le=100)] = 20,
):
return await operation(service(request).list(principal, after, limit))
@router.get("/{operation_id}", response_model=ProblemImportDetail)
async def detail(
operation_id: UUID,
request: Request,
principal: Annotated[Principal, Depends(maintenance_reader)],
):
return await operation(service(request).detail(principal, operation_id))
@router.put(
"/{operation_id}/archive",
response_model=ProblemImportSummary,
openapi_extra={
"requestBody": {
"required": True,
"content": {"application/zip": {"schema": {"type": "string", "format": "binary"}}},
}
},
)
async def upload(
operation_id: UUID,
request: Request,
principal: Annotated[Principal, Depends(maintenance_writer)],
):
if request.url.path != f"/api/v1/admin/problem-imports/{operation_id}/archive":
raise HTTPException(422, "canonical operation UUID required")
if request.headers.get("content-type", "").split(";", 1)[0] != "application/zip":
raise HTTPException(415, "ZIP request required")
if request.headers.get("content-encoding") not in (None, "identity"):
raise HTTPException(415, "encoded upload unavailable")
length = request.headers.get("content-length")
if length is not None:
try:
if not length.isascii() or not length.isdecimal() or len(length) > 12:
raise ValueError
length = int(length)
except ValueError:
raise HTTPException(413, "invalid archive length") from None
async def chunks():
while True:
message = await request.receive()
if message["type"] == "http.disconnect":
raise ClientDisconnect()
if message["type"] != "http.request":
raise ImportConflict("invalid upload frame")
yield message.get("body", b"")
if not message.get("more_body", False):
return
return await operation(service(request).upload(principal, operation_id, chunks(), length))
@router.post("/{operation_id}/discard", response_model=ProblemImportSummary)
async def discard(
operation_id: UUID,
request: Request,
principal: Annotated[Principal, Depends(maintenance_writer)],
):
return await operation(service(request).discard(principal, operation_id))
@@ -0,0 +1,180 @@
"""Versioned problem-import DTOs; hidden bytes, paths and toolchain selection stay private."""
from datetime import datetime
from enum import StrEnum
from typing import Annotated, Literal
from pydantic import Field, StrictBool, model_validator
from kf_oj.contracts.judging import Digest, Identifier, Label, RunLimits, Version, WireModel
from kf_oj.contracts.judging_v2 import Language
from kf_oj.contracts.problems import PublicLanguageConfiguration, PublicSample
MAX_ARCHIVE_BYTES = 64 * 1024**2
MAX_UNPACKED_BYTES = 64 * 1024**2
MAX_ARCHIVE_FILES = 534
MAX_ARCHIVE_NAME_BYTES = 128
MAX_UPLOAD_CHUNKS = 4096
MAX_UPLOAD_CHUNK_BYTES = 1024**2
UPLOAD_DEADLINE_SECONDS = 30
VALIDATION_DEADLINE_SECONDS = 60
MAX_IMPORT_ATTEMPTS = 2
IMPORT_LIFETIME_SECONDS = 86400
MAX_OWNER_CANDIDATES = 10
MAX_TOTAL_CANDIDATES = 32
STAGING_MAX_BYTES = 256 * 1024**2
STAGING_RESERVATION_BYTES = MAX_ARCHIVE_BYTES + MAX_UNPACKED_BYTES
class ImportState(StrEnum):
UPLOADING = "uploading"
UPLOADED = "uploaded"
VALIDATING = "validating"
READY = "ready"
FAILED = "failed"
PUBLISHED = "published"
DISCARDED = "discarded"
EXPIRED = "expired"
class ImportFailure(StrEnum):
INVALID_ARCHIVE = "invalid_archive"
INVALID_MANIFEST = "invalid_manifest"
INVALID_DATA = "invalid_data"
DIGEST_MISMATCH = "digest_mismatch"
POLICY_REJECTED = "policy_rejected"
UPLOAD_INTERRUPTED = "upload_interrupted"
UPLOAD_DEADLINE = "upload_deadline"
VALIDATION_DEADLINE = "validation_deadline"
VALIDATION_UNAVAILABLE = "validation_unavailable"
STORAGE_UNAVAILABLE = "storage_unavailable"
AUTHORITY_REVOKED = "authority_revoked"
EXPIRED = "expired"
class ProblemImportLimits(WireModel):
schema_version: Version = 1
archive_format: Literal["single_level_zip"] = "single_level_zip"
manifest_name: Literal["problem.json"] = "problem.json"
manifest_schema_versions: tuple[Literal[1, 2], ...] = (1, 2)
languages: tuple[Language, ...] = tuple(Language)
archive_bytes: int = MAX_ARCHIVE_BYTES
unpacked_bytes: int = MAX_UNPACKED_BYTES
files: int = MAX_ARCHIVE_FILES
filename_ascii_bytes: int = MAX_ARCHIVE_NAME_BYTES
upload_chunks: int = MAX_UPLOAD_CHUNKS
chunk_bytes: int = MAX_UPLOAD_CHUNK_BYTES
upload_deadline_seconds: int = UPLOAD_DEADLINE_SECONDS
validation_deadline_seconds: int = VALIDATION_DEADLINE_SECONDS
global_concurrency: Literal[1] = 1
owner_concurrency: Literal[1] = 1
owner_candidates: int = MAX_OWNER_CANDIDATES
total_candidates: int = MAX_TOTAL_CANDIDATES
staging_bytes: int = STAGING_MAX_BYTES
candidate_lifetime_seconds: int = IMPORT_LIFETIME_SECONDS
class ProblemImportCapabilities(WireModel):
schema_version: Version = 1
limits: ProblemImportLimits = ProblemImportLimits()
uploads_available: StrictBool
validation_available: StrictBool
class CreateProblemImport(WireModel):
schema_version: Version = 1
operation_id: Identifier
archive_digest: Digest
archive_bytes: Annotated[int, Field(strict=True, ge=1, le=MAX_ARCHIVE_BYTES)]
class LegacyImportBudget(WireModel):
compile_limits: RunLimits
run_limits: RunLimits
job_wall_time_us: Annotated[int, Field(strict=True, ge=1000, le=3600000000)]
class CandidatePreview(WireModel):
candidate_id: Identifier
slug: Label
title: Annotated[str, Field(strict=True, min_length=1, max_length=160)]
statement: Annotated[str, Field(strict=True, min_length=1, max_length=262144)]
samples: Annotated[tuple[PublicSample, ...], Field(max_length=10)]
judging_schema_version: Annotated[int, Field(strict=True, ge=1, le=2)]
languages: Annotated[tuple[Language, ...], Field(min_length=1, max_length=4)]
legacy_budget: LegacyImportBudget | None
language_configurations: Annotated[
tuple[PublicLanguageConfiguration, ...], Field(max_length=4)
] = ()
case_count: Annotated[int, Field(strict=True, ge=1, le=256)]
@model_validator(mode="after")
def bounded_text(self):
if len(self.statement.encode("utf-8")) > 262144 or "\x00" in self.statement + self.title:
raise ValueError("invalid public text")
languages = [c.language for c in self.language_configurations]
if len(set(languages)) != len(languages):
raise ValueError("duplicate language configuration")
if len(set(self.languages)) != len(self.languages):
raise ValueError("duplicate language")
if self.judging_schema_version == 1:
if (
self.language_configurations
or self.legacy_budget is None
or any(
language not in (Language.C17, Language.CPP20) for language in self.languages
)
):
raise ValueError("invalid legacy candidate")
elif tuple(languages) != self.languages or self.legacy_budget is not None:
raise ValueError("invalid multi-runtime candidate")
return self
class ProblemImportSummary(WireModel):
schema_version: Version = 1
operation_id: Identifier
state: ImportState
expected_archive_digest: Digest
archive_bytes: Annotated[int, Field(strict=True, ge=1, le=MAX_ARCHIVE_BYTES)]
created_at: datetime
expires_at: datetime
updated_at: datetime
failure_code: ImportFailure | None
candidate_id: Identifier | None
published_version_id: Identifier | None
@model_validator(mode="after")
def consistent_state(self):
if self.expires_at <= self.created_at or self.updated_at < self.created_at:
raise ValueError("invalid import time range")
if (self.state in (ImportState.READY, ImportState.PUBLISHED)) != (
self.candidate_id is not None
):
raise ValueError("candidate state mismatch")
if (self.state == ImportState.PUBLISHED) != (self.published_version_id is not None):
raise ValueError("published state mismatch")
if self.state in (ImportState.FAILED, ImportState.EXPIRED):
if self.failure_code is None:
raise ValueError("failure code required")
elif self.failure_code is not None:
raise ValueError("unexpected failure code")
return self
class ProblemImportDetail(ProblemImportSummary):
candidate: CandidatePreview | None
@model_validator(mode="after")
def candidate_matches(self):
if (self.candidate_id is None) != (self.candidate is None):
raise ValueError("candidate preview missing")
if self.candidate and self.candidate.candidate_id != self.candidate_id:
raise ValueError("candidate preview mismatch")
return self
class ProblemImportPage(WireModel):
schema_version: Version = 1
items: Annotated[tuple[ProblemImportSummary, ...], Field(max_length=100)]
next_after: Identifier | None
+14
View File
@@ -5,6 +5,11 @@ import json
from pathlib import Path
from kf_oj.api.app import create_app
from kf_oj.contracts.admin_problems import (
CreateProblemImport,
ProblemImportCapabilities,
ProblemImportDetail,
)
from kf_oj.contracts.judge import JudgeTaskNotification
from kf_oj.contracts.judging import WIRE_MODELS
from kf_oj.contracts.judging_v2 import WIRE_MODELS as WIRE_MODELS_V2
@@ -43,6 +48,15 @@ def main() -> None:
"counts; also run model validation and server-side ownership/environment checks."
)
outputs[root / "contracts/schemas/submission-performance.v1.schema.json"] = performance
for name, model in {
"problem-import-create": CreateProblemImport,
"problem-import": ProblemImportDetail,
"problem-import-capabilities": ProblemImportCapabilities,
}.items():
document = model.model_json_schema()
document["$schema"] = "https://json-schema.org/draft/2020-12/schema"
document["$id"] = f"urn:kf-oj:{name}:v1"
outputs[root / f"contracts/schemas/{name}.v1.schema.json"] = document
for name, model in WIRE_MODELS_V2.items():
document = model.model_json_schema()
document["$schema"] = "https://json-schema.org/draft/2020-12/schema"
@@ -0,0 +1,340 @@
"""Bounded single-level ZIP reader; no extractall, metadata restoration or content execution."""
import binascii
import hashlib
import os
import re
import stat
import struct
import time
import zlib
from collections.abc import Iterator
from dataclasses import dataclass
from kf_oj.contracts.admin_problems import (
MAX_ARCHIVE_BYTES,
MAX_ARCHIVE_FILES,
MAX_UNPACKED_BYTES,
)
NAME = re.compile(rb"[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}\Z")
CHUNK = 65536
LOCAL = struct.Struct("<4s5H3I2H")
CENTRAL = struct.Struct("<4s6H3I5H2I")
END = struct.Struct("<4s4H2IH")
class ArchiveError(ValueError):
"""A fixed sanitized failure; never echo an archive name, data or local path."""
@dataclass(frozen=True)
class ArchiveMember:
name: str
method: int
crc32: int
compressed_bytes: int
bytes: int
offset: int
flags: int
class ArchiveReader:
def __init__(self, descriptor: int, expected_bytes: int, *, deadline: float):
self.descriptor, self.deadline = descriptor, deadline
self.before = os.fstat(descriptor)
if (
not stat.S_ISREG(self.before.st_mode)
or self.before.st_uid != os.getuid()
or self.before.st_nlink != 1
or stat.S_IMODE(self.before.st_mode) != 0o600
or not 22 <= expected_bytes <= MAX_ARCHIVE_BYTES
or self.before.st_size != expected_bytes
):
raise ArchiveError("invalid_archive")
self.size = expected_bytes
self.unpacked_bytes, self.steps = 0, 0
self.read_members: set[str] = set()
self.members = self._members()
def _tick(self) -> None:
self.steps += 1
if self.steps > 8192 or time.monotonic() >= self.deadline:
raise ArchiveError("archive_deadline")
def _read(self, offset: int, size: int) -> bytes:
self._tick()
if not 0 <= size <= 1024**2 or offset < 0 or offset + size > self.size:
raise ArchiveError("invalid_archive")
data = os.pread(self.descriptor, size, offset)
if len(data) != size:
raise ArchiveError("invalid_archive")
return data
@staticmethod
def _extra(data: bytes) -> None:
offset, fields = 0, 0
# These bounded timestamps/UID attributes are ignored, never restored.
# ZIP64, Unicode aliases and all other extension semantics are unsupported.
while offset < len(data):
if offset + 4 > len(data) or fields >= 32:
raise ArchiveError("invalid_archive")
kind, size = struct.unpack_from("<HH", data, offset)
offset += 4
if kind not in (0x5455, 0x7875, 0x5855, 0x7855, 0x000A) or offset + size > len(data):
raise ArchiveError("invalid_archive")
offset += size
fields += 1
def _members(self) -> tuple[ArchiveMember, ...]:
tail_offset = max(0, self.size - 1046)
tail = self._read(tail_offset, self.size - tail_offset)
position = tail.rfind(b"PK\x05\x06")
if position < 0 or position + END.size > len(tail):
raise ArchiveError("invalid_archive")
sig, disk, cd_disk, disk_count, count, cd_bytes, cd_offset, comment = END.unpack_from(
tail, position
)
end_offset = tail_offset + position
if (
sig != b"PK\x05\x06"
or disk != 0
or cd_disk != 0
or disk_count != count
or not 1 <= count <= MAX_ARCHIVE_FILES
or comment > 1024
or end_offset + END.size + comment != self.size
or not 46 <= cd_bytes <= 1024**2
or cd_offset + cd_bytes != end_offset
):
raise ArchiveError("invalid_archive")
directory = self._read(cd_offset, cd_bytes)
offset, records, names = 0, [], set()
for _ in range(count):
if offset + CENTRAL.size > len(directory):
raise ArchiveError("invalid_archive")
values = CENTRAL.unpack_from(directory, offset)
(
sig,
_,
needed,
flags,
method,
_,
_,
crc,
compressed,
size,
name_size,
extra_size,
comment_size,
disk,
_,
attributes,
local_offset,
) = values
allowed_flags = 0x0808 | (0x0006 if method == 8 else 0)
if (
sig != b"PK\x01\x02"
or needed not in (10, 20)
or method not in (0, 8)
or (method == 8 and needed != 20)
or flags & ~allowed_flags
or disk != 0
or not 1 <= name_size <= 128
or extra_size > 1024
or comment_size > 256
or size > MAX_UNPACKED_BYTES
or compressed > MAX_ARCHIVE_BYTES
or local_offset + LOCAL.size > cd_offset
or attributes & 0x10
or stat.S_IFMT(attributes >> 16) not in (0, stat.S_IFREG)
):
raise ArchiveError("invalid_archive")
start = offset + CENTRAL.size
finish = start + name_size + extra_size + comment_size
if finish > len(directory):
raise ArchiveError("invalid_archive")
name_bytes = directory[start : start + name_size]
if not NAME.fullmatch(name_bytes):
raise ArchiveError("invalid_archive")
name = name_bytes.decode("ascii")
if name.lower() in names:
raise ArchiveError("invalid_archive")
names.add(name.lower())
self._extra(directory[start + name_size : start + name_size + extra_size])
records.append(ArchiveMember(name, method, crc, compressed, size, local_offset, flags))
offset = finish
if offset != len(directory) or sum(m.bytes for m in records) > MAX_UNPACKED_BYTES:
raise ArchiveError("invalid_archive")
cursor, members = 0, []
for member in sorted(records, key=lambda m: m.offset):
if member.offset != cursor:
raise ArchiveError("invalid_archive")
sig, needed, flags, method, _, _, crc, compressed, size, name_size, extra_size = (
LOCAL.unpack(self._read(cursor, LOCAL.size))
)
if (
sig != b"PK\x03\x04"
or needed not in (10, 20)
or flags != member.flags
or method != member.method
or (method == 8 and needed != 20)
or name_size != len(member.name)
or extra_size > 1024
):
raise ArchiveError("invalid_archive")
metadata = self._read(cursor + LOCAL.size, name_size + extra_size)
if metadata[:name_size] != member.name.encode("ascii"):
raise ArchiveError("invalid_archive")
self._extra(metadata[name_size:])
data_offset = cursor + LOCAL.size + name_size + extra_size
cursor = data_offset + member.compressed_bytes
if cursor > cd_offset or (method == 0 and member.compressed_bytes != member.bytes):
raise ArchiveError("invalid_archive")
if flags & 8:
if (crc, compressed, size) not in (
(0, 0, 0),
(member.crc32, member.compressed_bytes, member.bytes),
):
raise ArchiveError("invalid_archive")
marker = self._read(cursor, 4)
descriptor_size = 16 if marker == b"PK\x07\x08" else 12
descriptor = self._read(cursor, descriptor_size)
described = struct.unpack("<III", descriptor[-12:])
if described != (member.crc32, member.compressed_bytes, member.bytes):
raise ArchiveError("invalid_archive")
cursor += descriptor_size
elif (crc, compressed, size) != (member.crc32, member.compressed_bytes, member.bytes):
raise ArchiveError("invalid_archive")
members.append(
ArchiveMember(
member.name,
method,
member.crc32,
member.compressed_bytes,
member.bytes,
data_offset,
flags,
)
)
if cursor != cd_offset:
raise ArchiveError("invalid_archive")
return tuple(members)
def verify_digest(self, expected: str) -> None:
digest = hashlib.sha256()
for offset in range(0, self.size, CHUNK):
digest.update(self._read(offset, min(CHUNK, self.size - offset)))
if digest.hexdigest() != expected:
raise ArchiveError("archive_digest_mismatch")
def verify_unchanged(self) -> None:
after = os.fstat(self.descriptor)
fields = (
"st_dev",
"st_ino",
"st_nlink",
"st_mode",
"st_uid",
"st_size",
"st_mtime_ns",
"st_ctime_ns",
)
if any(getattr(self.before, f) != getattr(after, f) for f in fields):
raise ArchiveError("archive_changed")
@staticmethod
def _nested(prefix: bytes) -> bool:
return prefix.startswith(
(
b"PK\x03\x04",
b"PK\x05\x06",
b"PK\x07\x08",
b"\x1f\x8b",
b"BZh",
b"\xfd7zXZ\x00",
b"Rar!\x1a\x07",
b"7z\xbc\xaf\x27\x1c",
)
) or (len(prefix) >= 262 and prefix[257:262] == b"ustar")
def chunks(self, name: str, maximum: int) -> Iterator[bytes]:
member = next((m for m in self.members if m.name == name), None)
if member is None or name in self.read_members or member.bytes > maximum:
raise ArchiveError("invalid_archive")
self.read_members.add(name)
used, crc, prefix = 0, 0, bytearray()
inflater = zlib.decompressobj(-15) if member.method == 8 else None
try:
for offset in range(0, member.compressed_bytes, CHUNK):
pending = self._read(
member.offset + offset, min(CHUNK, member.compressed_bytes - offset)
)
while pending:
self._tick()
if inflater:
data = inflater.decompress(pending, min(CHUNK, member.bytes - used + 1))
pending = inflater.unconsumed_tail
if inflater.unused_data:
raise ArchiveError("invalid_archive")
else:
data, pending = pending, b""
used += len(data)
self.unpacked_bytes += len(data)
if (
used > member.bytes
or used > maximum
or self.unpacked_bytes > MAX_UNPACKED_BYTES
):
raise ArchiveError("invalid_archive")
prefix.extend(data[: max(0, 512 - len(prefix))])
if self._nested(bytes(prefix)):
raise ArchiveError("invalid_archive")
crc = binascii.crc32(data, crc)
if data:
yield data
if (inflater and not inflater.eof) or used != member.bytes or crc != member.crc32:
raise ArchiveError("invalid_archive")
except zlib.error:
raise ArchiveError("invalid_archive") from None
def read(self, name: str, maximum: int) -> bytes:
return b"".join(self.chunks(name, maximum))
def extract_remaining(self, directory: int) -> dict[str, tuple[str, int]]:
info = os.fstat(directory)
if (
not stat.S_ISDIR(info.st_mode)
or info.st_uid != os.getuid()
or stat.S_IMODE(info.st_mode) != 0o700
):
raise ArchiveError("invalid_extraction_directory")
references = {}
for member in self.members:
if member.name in self.read_members:
continue
fd = os.open(
member.name,
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC,
0o600,
dir_fd=directory,
)
try:
digest, size = hashlib.sha256(), 0
for chunk in self.chunks(member.name, member.bytes):
digest.update(chunk)
size += len(chunk)
remaining = memoryview(chunk)
while remaining:
written = os.write(fd, remaining)
if written <= 0:
raise ArchiveError("extraction_unavailable")
remaining = remaining[written:]
os.fsync(fd)
references[member.name] = (digest.hexdigest(), size)
finally:
os.close(fd)
os.fsync(directory)
self.verify_unchanged()
return references
@@ -50,6 +50,28 @@ class ArtifactRepository:
raise ArtifactError("artifact metadata unavailable")
return await asyncio.to_thread(self.store.read, reference)
async def put_file(
self, session: AsyncSession, descriptor: int, reference: ArtifactReference
) -> ArtifactReference:
await lock_digest(session, reference.digest)
existing = await session.get(Artifact, reference.digest)
if existing is not None and (
existing.state != "ready" or existing.bytes != reference.bytes
):
raise ArtifactError("artifact metadata unavailable")
task = asyncio.create_task(asyncio.to_thread(self.store.put_file, descriptor, reference))
try:
await asyncio.shield(task)
except BaseException:
await task
raise
await session.execute(
insert(Artifact)
.values(digest=reference.digest, bytes=reference.bytes)
.on_conflict_do_nothing(index_elements=[Artifact.digest])
)
return reference
async def check_metadata(self, session: AsyncSession, reference: ArtifactReference) -> None:
metadata = await session.get(Artifact, reference.digest)
if metadata is None or metadata.state != "ready" or metadata.bytes != reference.bytes:
@@ -0,0 +1,208 @@
"""Owned, private, bounded import staging. A kernel lock survives parent/child handoff."""
import fcntl
import os
import stat
from contextlib import contextmanager
from pathlib import Path
from uuid import UUID
from kf_oj.contracts.admin_problems import MAX_ARCHIVE_FILES, STAGING_MAX_BYTES
from kf_oj.infra.artifacts.store import ArtifactError
class ImportStaging:
def __init__(self, root: Path):
root = root.absolute()
root.mkdir(parents=True, mode=0o700, exist_ok=True)
if root.resolve() != root:
raise ArtifactError("invalid staging root")
self.root = root
self.fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC)
self.directory(self.fd)
@staticmethod
def directory(fd):
s = os.fstat(fd)
if (
not stat.S_ISDIR(s.st_mode)
or s.st_uid != os.getuid()
or stat.S_IMODE(s.st_mode) != 0o700
):
raise ArtifactError("invalid staging directory")
def close(self):
os.close(self.fd)
@staticmethod
def names(descriptor, maximum):
result = []
with os.scandir(descriptor) as entries:
for entry in entries:
if len(result) >= maximum:
raise ArtifactError("staging file capacity exhausted")
result.append(entry.name)
return result
def operation_ids(self):
identifiers = []
for name in self.names(self.fd, 33):
if name == ".lock":
continue
try:
identifier = UUID(name)
if str(identifier) != name:
raise ValueError
except ValueError:
raise ArtifactError("unexpected staging entry") from None
identifiers.append(identifier)
return identifiers
@contextmanager
def exclusive(self):
lock = os.open(
".lock", os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600, dir_fd=self.fd
)
try:
from kf_oj.infra.artifacts.store import ArtifactStore
ArtifactStore._regular(os.fstat(lock))
try:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
raise ArtifactError("import capacity busy") from None
yield lock
finally:
os.close(lock)
def usage(self):
total, count = 0, 0
for name in self.names(self.fd, 33):
if name == ".lock":
continue
try:
if str(UUID(name)) != name:
raise ValueError
except ValueError:
raise ArtifactError("unexpected staging entry") from None
descriptor = self.operation(UUID(name))
try:
for entry in self.names(descriptor, 3):
count += 1
if count > 2 * (MAX_ARCHIVE_FILES + 2):
raise ArtifactError("staging file capacity exhausted")
if entry == "members":
child = self.members(descriptor)
try:
for member in self.names(child, MAX_ARCHIVE_FILES):
count += 1
info = os.stat(member, dir_fd=child, follow_symlinks=False)
from kf_oj.infra.artifacts.store import ArtifactStore
ArtifactStore._regular(info)
total += info.st_size
finally:
os.close(child)
elif entry in ("archive", "validated.json"):
info = os.stat(entry, dir_fd=descriptor, follow_symlinks=False)
from kf_oj.infra.artifacts.store import ArtifactStore
ArtifactStore._regular(info)
total += info.st_size
else:
raise ArtifactError("unexpected staging entry")
if count > 2 * (MAX_ARCHIVE_FILES + 2) or total > STAGING_MAX_BYTES:
raise ArtifactError("staging capacity exhausted")
finally:
os.close(descriptor)
return total
def operation(self, operation_id: UUID, *, create=False):
name = str(operation_id)
if create:
try:
os.mkdir(name, 0o700, dir_fd=self.fd)
except FileExistsError:
pass
descriptor = os.open(
name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC, dir_fd=self.fd
)
try:
self.directory(descriptor)
except Exception:
os.close(descriptor)
raise
return descriptor
@staticmethod
def members(operation, *, create=False):
if create:
os.mkdir("members", 0o700, dir_fd=operation)
descriptor = os.open(
"members", os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC, dir_fd=operation
)
try:
ImportStaging.directory(descriptor)
except Exception:
os.close(descriptor)
raise
return descriptor
def remove(self, operation_id):
try:
descriptor = self.operation(operation_id)
except FileNotFoundError:
return
try:
names = self.names(descriptor, 3)
if len(names) > 3:
raise ArtifactError("unexpected staging entry")
for name in names:
if name == "members":
child = self.members(descriptor)
try:
entries = self.names(child, MAX_ARCHIVE_FILES)
if len(entries) > MAX_ARCHIVE_FILES:
raise ArtifactError("unexpected staging entry")
for member in entries:
from kf_oj.infra.artifacts.store import ArtifactStore
ArtifactStore._regular(
os.stat(member, dir_fd=child, follow_symlinks=False)
)
os.unlink(member, dir_fd=child)
finally:
os.close(child)
os.rmdir("members", dir_fd=descriptor)
elif name in ("archive", "validated.json"):
from kf_oj.infra.artifacts.store import ArtifactStore
ArtifactStore._regular(os.stat(name, dir_fd=descriptor, follow_symlinks=False))
os.unlink(name, dir_fd=descriptor)
else:
raise ArtifactError("unexpected staging entry")
finally:
os.close(descriptor)
os.rmdir(str(operation_id), dir_fd=self.fd)
os.fsync(self.fd)
def reset_validation(self, operation_id):
directory = self.operation(operation_id)
try:
names = self.names(directory, 3)
if "members" in names:
members = self.members(directory)
try:
for name in self.names(members, MAX_ARCHIVE_FILES):
from kf_oj.infra.artifacts.store import ArtifactStore
ArtifactStore._regular(os.stat(name, dir_fd=members, follow_symlinks=False))
os.unlink(name, dir_fd=members)
finally:
os.close(members)
os.rmdir("members", dir_fd=directory)
if "validated.json" in names:
os.unlink("validated.json", dir_fd=directory)
os.fsync(directory)
finally:
os.close(directory)
@@ -168,6 +168,90 @@ class ArtifactStore:
with self.exclusive():
return self._read(reference)
def put_file(self, source: int, expected: ArtifactReference) -> ArtifactReference:
"""Adopt a validated private FD without buffering a hidden testcase in memory."""
before = os.fstat(source)
self._regular(before)
if before.st_size != expected.bytes:
raise ArtifactError("artifact digest or length mismatch")
def unchanged(descriptor, original):
current = os.fstat(descriptor)
fields = (
"st_dev",
"st_ino",
"st_nlink",
"st_mode",
"st_uid",
"st_size",
"st_mtime_ns",
"st_ctime_ns",
)
if any(getattr(current, k) != getattr(original, k) for k in fields):
raise ArtifactError("artifact input changed")
def stream(descriptor, *, output=None):
digest = hashlib.sha256()
for offset in range(0, expected.bytes, 65536):
data = os.pread(descriptor, min(65536, expected.bytes - offset), offset)
if len(data) != min(65536, expected.bytes - offset):
raise ArtifactError("artifact digest or length mismatch")
digest.update(data)
if output is not None:
output.write(data)
if digest.hexdigest() != expected.digest:
raise ArtifactError("artifact digest or length mismatch")
with self.exclusive():
entries = self._entries()
if any(name == expected.digest for name, _ in entries):
existing = os.open(
expected.digest,
os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC,
dir_fd=self._directory,
)
try:
self._regular(os.fstat(existing))
if os.fstat(existing).st_size != expected.bytes:
raise ArtifactError("artifact digest or length mismatch")
stream(source)
stream(existing)
finally:
os.close(existing)
else:
if (
len(entries) + 1 > self.max_files
or sum(s.st_size for _, s in entries) + expected.bytes > self.max_bytes
):
raise ArtifactError("storage capacity exhausted")
temporary = ".tmp-" + uuid4().hex
descriptor = os.open(
temporary,
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC,
0o600,
dir_fd=self._directory,
)
try:
with os.fdopen(descriptor, "wb") as out:
stream(source, output=out)
out.flush()
os.fsync(out.fileno())
unchanged(source, before)
os.rename(
temporary,
expected.digest,
src_dir_fd=self._directory,
dst_dir_fd=self._directory,
)
os.fsync(self._directory)
finally:
try:
os.unlink(temporary, dir_fd=self._directory)
except FileNotFoundError:
pass
unchanged(source, before)
return expected
def remove(self, digest: str) -> None:
if not DIGEST.fullmatch(digest):
raise ArtifactError("invalid artifact identifier")
@@ -28,5 +28,11 @@ class AuthenticationFailed(ValueError):
pass
class ReservedMaintenancePort(Protocol):
async def require_reserved_maintenance(self, session, actor_id: UUID, epoch: int) -> None:
"""Recheck/lock an immutable accepted receipt, with no login/session secret access."""
...
class LoginThrottled(ValueError):
pass
+19 -1
View File
@@ -7,7 +7,8 @@ from dataclasses import dataclass, field
from datetime import datetime, timedelta
from uuid import uuid4
from sqlalchemy import func, select, update
from sqlalchemy import func, select, text, update
from sqlalchemy.exc import DBAPIError
from sqlalchemy.ext.asyncio import async_sessionmaker
from kf_oj.modules.identity import repository
@@ -170,3 +171,20 @@ class IdentityService:
await self.authenticate(token, csrf_token=csrf_token)
async with self.sessions.begin() as session:
await repository.revoke(session, token_digest(token))
class ReservedMaintenance:
"""Import worker uses only the fixed, narrow SQL authority port."""
async def require_reserved_maintenance(self, session, actor_id, epoch):
if not session.in_transaction() or type(epoch) is not int or epoch < 1:
raise MaintenanceDenied("invalid maintenance receipt")
try:
await session.execute(
text("SELECT public.kf_lock_reserved_maintenance(:actor,:epoch)"),
{"actor": actor_id, "epoch": epoch},
)
except DBAPIError as error:
if getattr(error.orig, "sqlstate", None) == "42501":
raise MaintenanceDenied("problem maintenance revoked") from None
raise
@@ -0,0 +1,29 @@
"""Append-only bounded maintenance events, independent of evaluation state."""
from uuid import UUID
from sqlalchemy import BigInteger, CheckConstraint, ForeignKey, Index, String
from sqlalchemy.orm import Mapped, mapped_column
from kf_oj.infra.db.base import Base
from kf_oj.infra.db.columns import CreatedAt
class MaintenanceEvent(CreatedAt, Base):
__tablename__ = "problem_maintenance_events"
__table_args__ = (
CheckConstraint(
"action IN ('upload_created','upload_received','import_ready',"
"'import_failed','import_discarded','import_expired','published',"
"'disabled','restored')",
name="action",
),
CheckConstraint("authorization_version > 0", name="authority"),
Index("ix_problem_maintenance_events_page", "created_at", "id"),
)
id: Mapped[UUID] = mapped_column(primary_key=True)
actor_id: Mapped[UUID] = mapped_column(ForeignKey("accounts.id"))
authorization_version: Mapped[int] = mapped_column(BigInteger)
operation_id: Mapped[UUID] = mapped_column()
action: Mapped[str] = mapped_column(String(24))
target_id: Mapped[UUID | None] = mapped_column()
@@ -0,0 +1,28 @@
"""Accepted actor receipts and finite maintenance events; no private data/log text."""
from typing import Literal, Protocol
from uuid import UUID
Action = Literal[
"upload_created",
"upload_received",
"import_ready",
"import_failed",
"import_discarded",
"import_expired",
"published",
"disabled",
"restored",
]
class MaintenanceAuditPort(Protocol):
async def record(
self,
session,
actor_id: UUID,
authorization_version: int,
operation_id: UUID,
action: Action,
target_id: UUID | None = None,
) -> None: ...
@@ -0,0 +1,27 @@
from uuid import uuid4
from sqlalchemy import func, select
from kf_oj.modules.operations.models import MaintenanceEvent
from kf_oj.modules.operations.public import Action
class MaintenanceAudit:
async def record(
self, session, actor_id, authorization_version, operation_id, action: Action, target_id=None
):
if not session.in_transaction():
raise ValueError("audit requires a write transaction")
# Supplying the trusted DB timestamp avoids ORM INSERT ... RETURNING:
# the ordinary importer has append-only authority and cannot SELECT this table.
session.add(
MaintenanceEvent(
id=uuid4(),
created_at=await session.scalar(select(func.clock_timestamp())),
actor_id=actor_id,
authorization_version=authorization_version,
operation_id=operation_id,
action=action,
target_id=target_id,
)
)
@@ -31,3 +31,32 @@ v2 使用 `schema_version: 2` 与 `language_configurations`,每个配置包含
导入同一 slug 的 v2 版本保持原 v1 版本与提交不变。已有提交按其保存的协议版本解析,
不能重新序列化旧 canonical 数据来补入新字段。Java 使用 Main.java 与默认包 Main;
Python 使用 main.py。每个新版本的语言集合独立,浏览器不得为旧版本补开新语言。
Web题包导入是独立的待发布流程:`/api/v1/admin/problem-imports` 使用真实维护会话,
写入还要求明确Origin与绑定CSRF。先以稳定操作UUID、ZIP摘要和精确字节数建立容量预约,
再向该操作的 `/archive` 流式PUT;同键重试仍验证实际内容,不会确认异内容。
列表、详情和丢弃只开放给该操作的管理员本人,响应仅含公开题面、样例、语言预算与有限错误。
`ready` 尚未发布,不会加入学生题库,也不改变既有题目版本。
Web格式固定单层ZIP的stored/deflate,清单名为`problem.json`;公开文件与隐藏用例文件
必须显式分别声明且不得混用。文件名只接受有界ASCII单层普通名;未知成员、链接/设备、
加密、嵌套归档、额外前缀/尾随内容、损坏CRC与虚假大小被拒。旧受控目录CLI格式保持原义。
压缩与实际解压各64 MiB、最多534文件,上传最多4096帧/每帧1 MiB/30秒;
解析期限60秒。staging总预约256 MiB,每次128 MiB,单全局上传/解析槽位,
本人最多10个/全局32个活跃候选,操作24小时到期,解析死亡最多领取两次。
真实限额由管理capabilities返回;缺少配置或新鲜Worker资源证明时明确不可用。
`uv run --locked kf-problem-import-worker` 是普通用户、独立数据库登录的组合入口。
配置`KF_OJ_IMPORT_STAGING_ROOT`、`KF_OJ_ARTIFACT_ROOT`、可信准入和
`KF_OJ_IMPORT_WORKER_CGROUP`;staging与工件目录必须归服务用户所有且权限0700。
服务不会sudo、执行上传内容/学生源码、联网安装依赖或读写评测状态。启动及每次解析
读回128 MiB/swap0/PIDs16/CPU不超过一核及文件/FD/core上限,错误身份/无界环境拒绝服务。
解析子进程继承同一个内核文件锁,父进程死亡不能让GC在存活子进程仍持有文件时释放容量。
租约与fence防止旧结果写入;完整工件持久化、候选和引用原子落库后才返回ready。
丢弃/失败/到期先清理私有暂存字节再释放预约,保留不可变操作记录与追加式有限审计。
资源约束服务模板见`deploy/problem-import-worker.service.example`。API和Worker需可访问
同一普通用户的私有目录;数据库采用各自最小角色,导入Worker只通过固定身份公开端口
验证已受理的维护授权。PrivateNetwork模板要求Unix PostgreSQL socket;实际部署前由运维
设置外部私有配置、专用登录与目录,不包含默认口令,也不代表本轮已经生产部署。
@@ -0,0 +1,97 @@
"""Pure data validation shared by the bounded child and fenced candidate transaction."""
import hashlib
import json
from dataclasses import dataclass
from uuid import uuid4, uuid5
from kf_oj.contracts import judging_v2 as v2
from kf_oj.contracts.admin_problems import CandidatePreview, LegacyImportBudget
from kf_oj.contracts.judging import MAX_SOURCE_BYTES, ArtifactReference, TestCase
from kf_oj.contracts.problems import PublicLanguageConfiguration
from kf_oj.modules.problems.import_format import ImportEnvelope, ProblemImport
from kf_oj.modules.problems.public import JudgingVersion
@dataclass(frozen=True)
class ValidatedCandidate:
manifest: dict
preview: CandidatePreview
references: dict[str, ArtifactReference]
@property
def digest(self):
return hashlib.sha256(
json.dumps(
self.manifest, sort_keys=True, separators=(",", ":"), ensure_ascii=True
).encode()
).hexdigest()
def validate_candidate(manifest_data, preview_data, references, policy) -> ValidatedCandidate:
manifest = ImportEnvelope.model_validate(manifest_data).root
preview = CandidatePreview.model_validate(preview_data)
refs = {k: ArtifactReference.model_validate(v) for k, v in references.items()}
legacy = isinstance(manifest, ProblemImport)
if (
preview.slug != manifest.slug
or preview.title != manifest.title
or preview.case_count != len(manifest.cases)
or preview.judging_schema_version != manifest.schema_version
or tuple(v.value for v in preview.languages) != tuple(v.value for v in manifest.languages)
):
raise ValueError("invalid candidate projection")
expected_budget = (
LegacyImportBudget(
compile_limits=manifest.compile_limits,
run_limits=manifest.run_limits,
job_wall_time_us=manifest.job_wall_time_us,
)
if legacy
else None
)
configs = (
()
if legacy
else tuple(
PublicLanguageConfiguration.from_configuration(c)
for c in manifest.language_configurations
)
)
if preview.legacy_budget != expected_budget or preview.language_configurations != configs:
raise ValueError("invalid candidate budget")
names = {name for c in manifest.cases for name in (c.input_file, c.expected_file)}
if set(refs) != names:
raise ValueError("invalid hidden references")
cases = tuple(
TestCase(
case_id=uuid5(preview.candidate_id, c.key),
input=refs[c.input_file],
expected=refs[c.expected_file],
)
for c in manifest.cases
)
projection = JudgingVersion(
preview.candidate_id,
uuid4(),
manifest.languages,
manifest.toolchain if legacy else None,
manifest.compile_limits if legacy else None,
manifest.run_limits if legacy else None,
manifest.job_wall_time_us if legacy else None,
cases,
manifest.schema_version,
() if legacy else manifest.language_configurations,
)
for language in manifest.languages:
v2.admit_manifest(
policy,
projection.manifest(
uuid4(),
uuid4(),
language,
ArtifactReference(digest="0" * 64, bytes=MAX_SOURCE_BYTES),
),
1,
)
return ValidatedCandidate(manifest.model_dump(mode="json"), preview, refs)
@@ -0,0 +1,121 @@
"""Private durable imports and immutable candidates; no grading-state write authority."""
from datetime import datetime
from uuid import UUID
from sqlalchemy import (
BigInteger,
CheckConstraint,
Computed,
DateTime,
ForeignKey,
ForeignKeyConstraint,
Index,
Integer,
SmallInteger,
String,
UniqueConstraint,
text,
)
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column
from kf_oj.infra.db.base import Base
from kf_oj.infra.db.columns import CreatedAt
from kf_oj.modules.problems.models import ProblemVersion
class ImportPool(Base):
__tablename__ = "problem_import_pool"
__table_args__ = (
CheckConstraint("id = 1", name="singleton"),
CheckConstraint("reserved_bytes BETWEEN 0 AND 268435456", name="capacity"),
)
id: Mapped[int] = mapped_column(SmallInteger, primary_key=True)
reserved_bytes: Mapped[int] = mapped_column(BigInteger, server_default="0")
worker_ready_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
worker_policy_digest: Mapped[str | None] = mapped_column(String(64))
class ProblemImportOperation(CreatedAt, Base):
__tablename__ = "problem_import_operations"
__table_args__ = (
CheckConstraint(
"state IN ('uploading','uploaded','validating','ready','failed',"
"'published','discarded','expired')",
name="state",
),
CheckConstraint("archive_digest ~ '^[0-9a-f]{64}$'", name="digest"),
CheckConstraint("archive_bytes BETWEEN 1 AND 67108864", name="bytes"),
CheckConstraint("authorization_version > 0", name="authority"),
CheckConstraint("attempts BETWEEN 0 AND 2 AND fence >= 0", name="attempts"),
CheckConstraint("reserved_bytes IN (0,134217728)", name="reservation"),
CheckConstraint("expires_at > created_at AND updated_at >= created_at", name="times"),
Index("ix_problem_import_owner", "owner_id", "id"),
Index("ix_problem_import_queue", "state", "created_at"),
Index("ix_problem_import_expiry", "expires_at"),
Index("ix_problem_import_reservations", "id", postgresql_where=text("reserved_bytes>0")),
)
id: Mapped[UUID] = mapped_column(primary_key=True)
owner_id: Mapped[UUID] = mapped_column(ForeignKey("accounts.id"))
authorization_version: Mapped[int] = mapped_column(BigInteger)
archive_digest: Mapped[str] = mapped_column(String(64))
archive_bytes: Mapped[int] = mapped_column(BigInteger)
state: Mapped[str] = mapped_column(String(16), server_default="uploading")
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
attempts: Mapped[int] = mapped_column(Integer, server_default="0")
fence: Mapped[int] = mapped_column(BigInteger, server_default="0")
reserved_bytes: Mapped[int] = mapped_column(BigInteger, server_default="134217728")
failure_code: Mapped[str | None] = mapped_column(String(32))
candidate_id: Mapped[UUID | None] = mapped_column(
ForeignKey("problem_candidates.id", use_alter=True, name="fk_problem_import_candidate")
)
published_version_id: Mapped[UUID | None] = mapped_column(ForeignKey(ProblemVersion.id))
class ProblemCandidate(CreatedAt, Base):
__tablename__ = "problem_candidates"
__table_args__ = (
UniqueConstraint("operation_id"),
CheckConstraint("manifest_digest ~ '^[0-9a-f]{64}$'", name="digest"),
CheckConstraint("case_count BETWEEN 1 AND 256", name="cases"),
CheckConstraint("octet_length(preview::text) <= 4194304", name="preview"),
CheckConstraint("octet_length(manifest::text) <= 262144", name="manifest"),
)
id: Mapped[UUID] = mapped_column(primary_key=True)
operation_id: Mapped[UUID] = mapped_column(ForeignKey("problem_import_operations.id"))
manifest_digest: Mapped[str] = mapped_column(String(64))
manifest: Mapped[dict] = mapped_column(JSONB)
preview: Mapped[dict] = mapped_column(JSONB)
case_count: Mapped[int] = mapped_column(Integer)
class CandidateCase(Base):
__tablename__ = "problem_candidate_cases"
__table_args__ = (
CheckConstraint("position BETWEEN 0 AND 255", name="position"),
CheckConstraint(
"input_bytes BETWEEN 0 AND 67108864 AND expected_bytes BETWEEN 0 AND 67108864",
name="bytes",
),
ForeignKeyConstraint(
["input_digest", "artifact_state"], ["artifacts.digest", "artifacts.state"]
),
ForeignKeyConstraint(
["expected_digest", "artifact_state"],
["artifacts.digest", "artifacts.state"],
name="fk_candidate_cases_expected_artifacts",
),
)
candidate_id: Mapped[UUID] = mapped_column(
ForeignKey("problem_candidates.id"), primary_key=True
)
position: Mapped[int] = mapped_column(Integer, primary_key=True)
key: Mapped[str] = mapped_column(String(64))
input_digest: Mapped[str] = mapped_column(String(64))
input_bytes: Mapped[int] = mapped_column(BigInteger)
expected_digest: Mapped[str] = mapped_column(String(64))
expected_bytes: Mapped[int] = mapped_column(BigInteger)
artifact_state: Mapped[str] = mapped_column(String(8), Computed("'ready'", persisted=True))
@@ -0,0 +1,430 @@
"""Fenced ordinary-user import consumption. No student execution or evaluation DB access."""
import asyncio
import json
import os
import resource
import sys
from datetime import timedelta
from pathlib import Path
from uuid import uuid4
from sqlalchemy import delete, func, or_, select, text
from sqlalchemy.exc import DBAPIError
from kf_oj.contracts.admin_problems import MAX_IMPORT_ATTEMPTS, VALIDATION_DEADLINE_SECONDS
from kf_oj.infra.artifacts.store import ArtifactError
from kf_oj.modules.identity.public import MaintenanceDenied
from kf_oj.modules.problems.candidate_validation import validate_candidate
from kf_oj.modules.problems.import_models import (
CandidateCase,
ImportPool,
ProblemCandidate,
)
from kf_oj.modules.problems.import_models import (
ProblemImportOperation as Operation,
)
from kf_oj.modules.problems.imports import ACTIVE, ImportUnavailable, ProblemImports, policy_digest
def verify_worker_limits(configured_group: Path):
if os.getuid() == 0:
raise ImportUnavailable("import worker must be ordinary")
actual = next(
line.split(":", 2)[2]
for line in Path("/proc/self/cgroup").read_text().splitlines()
if line.startswith("0::")
)
group = Path("/sys/fs/cgroup") / actual.lstrip("/")
if group.resolve() != configured_group.absolute().resolve():
raise ImportUnavailable("import worker cgroup mismatch")
caps = {
n: (group / n).read_text().strip()
for n in ("memory.max", "memory.swap.max", "pids.max", "cpu.max")
}
if (
caps["memory.max"] == "max"
or not 0 < int(caps["memory.max"]) <= 134217728
or caps["memory.swap.max"] != "0"
or caps["pids.max"] == "max"
or not 0 < int(caps["pids.max"]) <= 16
):
raise ImportUnavailable("import worker resource limits unavailable")
quota, period = caps["cpu.max"].split()
if quota == "max" or int(quota) > int(period):
raise ImportUnavailable("import worker CPU limit unavailable")
for name, maximum in (
(resource.RLIMIT_FSIZE, 67108864),
(resource.RLIMIT_NOFILE, 64),
(resource.RLIMIT_CORE, 0),
):
limits = resource.getrlimit(name)
if any(v == resource.RLIM_INFINITY or v > maximum for v in limits):
raise ImportUnavailable("import worker process limits unavailable")
return caps
class ImportWorker:
def __init__(self, sessions, staging, artifacts, authority, audit, policy, cgroup):
(
self.sessions,
self.staging,
self.artifacts,
self.authority,
self.audit,
self.policy,
self.cgroup,
) = sessions, staging, artifacts, authority, audit, policy, cgroup
async def heartbeat(self):
verify_worker_limits(self.cgroup)
async with self.sessions.begin() as session:
safe = await session.scalar(
text("""
SELECT rolcanlogin AND NOT(rolsuper OR rolcreatedb OR rolcreaterole
OR rolreplication OR rolbypassrls)
AND pg_has_role(current_user,'kf_oj_problem_import','USAGE')
AND NOT pg_has_role(current_user,'kf_oj_platform','MEMBER')
AND NOT pg_has_role(current_user,'kf_oj_coordinator','MEMBER')
AND NOT pg_has_role(current_user,'kf_oj_identity_operator','MEMBER')
AND NOT pg_has_role(current_user,'kf_oj_identity_authority_owner','MEMBER')
FROM pg_roles WHERE rolname=current_user
""")
)
if not safe:
raise ImportUnavailable("dedicated unprivileged import login required")
pool = await session.get(ImportPool, 1, with_for_update=True)
if pool is None:
raise ImportUnavailable("import pool unavailable")
pool.worker_ready_until = await session.scalar(
select(func.clock_timestamp())
) + timedelta(seconds=90)
pool.worker_policy_digest = policy_digest(self.policy)
async def _release(self, operation_id):
await asyncio.to_thread(self.staging.remove, operation_id)
async with self.sessions.begin() as session:
pool = await ProblemImports.pool(session)
op = await session.get(Operation, operation_id, with_for_update=True)
pool.reserved_bytes -= op.reserved_bytes
op.reserved_bytes = 0
op.lease_until = None
async def collect(self):
# Caller owns the root flock, so a live upload/parser cannot be unlinked or
# have its reservation reused, even if its DB lease expired while suspended.
async with self.sessions() as session:
now = await session.scalar(select(func.clock_timestamp()))
names = list(
await session.scalars(
select(Operation.id)
.where(
or_(
Operation.reserved_bytes > 0,
Operation.state.in_(ACTIVE) & (Operation.expires_at <= now),
)
)
.order_by(Operation.updated_at, Operation.id)
.limit(32)
)
)
garbage = list(
await session.scalars(
select(ProblemCandidate.operation_id)
.join(Operation, Operation.id == ProblemCandidate.operation_id)
.where(Operation.state.in_(("discarded", "expired", "failed")))
.limit(32)
)
)
names = list(dict.fromkeys(names + garbage))[:32]
for identifier in names:
async with self.sessions.begin() as session:
await ProblemImports.pool(session)
op = await session.get(Operation, identifier, with_for_update=True)
now = await session.scalar(select(func.clock_timestamp()))
if op.state in ACTIVE and op.expires_at <= now:
op.state = "expired"
op.failure_code = "expired"
op.updated_at = now
op.candidate_id = None
op.fence += 1
await self.audit.record(
session, op.owner_id, op.authorization_version, op.id, "import_expired"
)
elif op.state == "uploading" and op.lease_until and op.lease_until <= now:
op.state = "failed"
op.failure_code = "upload_interrupted"
op.updated_at = now
op.fence += 1
await self.audit.record(
session, op.owner_id, op.authorization_version, op.id, "import_failed"
)
elif op.state == "validating" and op.lease_until and op.lease_until <= now:
if op.attempts < MAX_IMPORT_ATTEMPTS:
op.state = "uploaded"
op.failure_code = None
op.lease_until = None
op.fence += 1
op.updated_at = now
await asyncio.to_thread(self.staging.reset_validation, op.id)
else:
op.state = "failed"
op.failure_code = "validation_unavailable"
op.updated_at = now
op.fence += 1
await self.audit.record(
session, op.owner_id, op.authorization_version, op.id, "import_failed"
)
if op.state in ("discarded", "expired", "failed"):
op.candidate_id = None
await session.flush()
candidate = await session.scalar(
select(ProblemCandidate).where(ProblemCandidate.operation_id == op.id)
)
if candidate:
await session.execute(
delete(CandidateCase).where(CandidateCase.candidate_id == candidate.id)
)
await session.delete(candidate)
release = (
op.state in ("ready", "published", "discarded", "expired", "failed")
and op.reserved_bytes > 0
)
if release:
await self._release(identifier)
# Uncommitted staging objects after a failed transaction are removed only under
# the same kernel lock and only from this service's private, bounded namespace.
for identifier in self.staging.operation_ids():
async with self.sessions() as session:
exists = await session.get(Operation, identifier)
if exists is None:
await asyncio.to_thread(self.staging.remove, identifier)
async def once(self):
await self.heartbeat()
try:
with self.staging.exclusive() as lock:
await self.collect()
async with self.sessions.begin() as session:
await ProblemImports.pool(session)
now = await session.scalar(select(func.clock_timestamp()))
busy = await session.scalar(
select(func.count())
.select_from(Operation)
.where(
Operation.state.in_(("uploading", "validating")),
Operation.lease_until > now,
)
)
if busy:
return False
op = await session.scalar(
select(Operation)
.where(
Operation.state == "uploaded",
Operation.attempts < MAX_IMPORT_ATTEMPTS,
Operation.expires_at > now,
)
.order_by(Operation.created_at, Operation.id)
.limit(1)
.with_for_update()
)
if op is None:
return False
op.attempts += 1
op.fence += 1
op.state = "validating"
op.updated_at = now
op.lease_until = now + timedelta(seconds=VALIDATION_DEADLINE_SECONDS + 15)
identifier, fence, size, digest, actor, epoch = (
op.id,
op.fence,
op.archive_bytes,
op.archive_digest,
op.owner_id,
op.authorization_version,
)
try:
async with self.sessions.begin() as session:
await self.authority.require_reserved_maintenance(session, actor, epoch)
candidate_id = uuid4()
result = await self.parse(identifier, size, digest, candidate_id, lock)
if "failure_code" in result:
await self.fail(identifier, fence, result["failure_code"])
return True
candidate = validate_candidate(
result["manifest"], result["preview"], result["references"], self.policy
)
if (
candidate.digest != result["manifest_digest"]
or candidate.preview.candidate_id != candidate_id
):
raise ValueError("invalid candidate receipt")
directory = self.staging.operation(identifier)
members = self.staging.members(directory)
try:
async with self.sessions.begin() as session:
await self.authority.require_reserved_maintenance(session, actor, epoch)
await ProblemImports.pool(session)
op = await session.get(Operation, identifier, with_for_update=True)
now = await session.scalar(select(func.clock_timestamp()))
if (
op.state != "validating"
or op.fence != fence
or op.lease_until <= now
or op.expires_at <= now
):
raise ValueError("stale candidate receipt")
for name, reference in sorted(
candidate.references.items(), key=lambda item: item[1].digest
):
fd = os.open(
name,
os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC,
dir_fd=members,
)
try:
await self.artifacts.put_file(session, fd, reference)
finally:
os.close(fd)
session.add(
ProblemCandidate(
id=candidate_id,
operation_id=identifier,
manifest_digest=candidate.digest,
manifest=candidate.manifest,
preview=candidate.preview.model_dump(mode="json"),
case_count=candidate.preview.case_count,
)
)
await session.flush()
for position, case in enumerate(candidate.manifest["cases"]):
input_ref, answer_ref = (
candidate.references[case["input_file"]],
candidate.references[case["expected_file"]],
)
session.add(
CandidateCase(
candidate_id=candidate_id,
position=position,
key=case["key"],
input_digest=input_ref.digest,
input_bytes=input_ref.bytes,
expected_digest=answer_ref.digest,
expected_bytes=answer_ref.bytes,
)
)
await session.flush()
# Adoption must not turn an expired lease into a ready candidate.
completed_at = await session.scalar(select(func.clock_timestamp()))
if op.lease_until <= completed_at or op.expires_at <= completed_at:
raise ValueError("candidate completion lease expired")
op.state = "ready"
op.candidate_id = candidate_id
op.updated_at = completed_at
op.lease_until = None
await self.audit.record(
session, actor, epoch, identifier, "import_ready", candidate_id
)
finally:
os.close(members)
os.close(directory)
await self._release(identifier)
except MaintenanceDenied:
await self.fail(identifier, fence, "authority_revoked")
except (ArtifactError, OSError):
await self.fail(identifier, fence, "storage_unavailable")
except (ValueError, DBAPIError):
await self.fail(identifier, fence, "validation_unavailable")
return True
except ArtifactError:
return False # Another live writer holds the one global slot.
async def fail(self, identifier, fence, code):
from kf_oj.contracts.admin_problems import ImportFailure
code = ImportFailure(code).value
async with self.sessions.begin() as session:
await ProblemImports.pool(session)
op = await session.get(Operation, identifier, with_for_update=True)
if op.state != "validating" or op.fence != fence:
return
op.state = "failed"
op.failure_code = code
op.updated_at = await session.scalar(select(func.clock_timestamp()))
op.lease_until = None
await self.audit.record(
session, op.owner_id, op.authorization_version, op.id, "import_failed"
)
await self._release(identifier)
async def parse(self, identifier, size, digest, candidate, lock):
verify_worker_limits(self.cgroup)
directory = self.staging.operation(identifier)
archive = members = None
try:
archive = os.open(
"archive",
os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC,
dir_fd=directory,
)
members = self.staging.members(directory, create=True)
# Preserve only the trusted runtime import roots, without inheriting DB/
# service credentials. Curated runtimes may keep dependencies outside site-packages.
env = {
"PATH": str(Path(sys.executable).parent),
"PYTHONPATH": os.pathsep.join(
dict.fromkeys(
[str(Path(__file__).parents[3])]
+ [p for p in os.environ.get("PYTHONPATH", "").split(os.pathsep) if p]
)
),
"PYTHONUNBUFFERED": "1",
"PYTHONDONTWRITEBYTECODE": "1",
"PYTHONNOUSERSITE": "1",
"LC_ALL": "C.UTF-8",
}
# Minimal environment may require the trusted dynamic-loader path in a curated VM.
if "LD_LIBRARY_PATH" in os.environ:
env["LD_LIBRARY_PATH"] = os.environ["LD_LIBRARY_PATH"]
child = await asyncio.create_subprocess_exec(
sys.executable,
"-m",
"kf_oj.workers._archive_child",
str(archive),
str(members),
str(lock),
str(size),
digest,
str(candidate),
self.policy.model_dump_json(),
env=env,
pass_fds=(archive, members, lock),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.DEVNULL,
)
try:
async with asyncio.timeout(VALIDATION_DEADLINE_SECONDS + 5):
output, _ = await child.communicate()
except BaseException:
child.kill()
await child.wait()
raise
if child.returncode != 0 or not 0 < len(output) <= 4 * 1024**2:
return {
"failure_code": "validation_deadline"
if child.returncode == -14
else "validation_unavailable"
}
result = json.loads(output)
if not isinstance(result, dict):
raise ValueError("invalid child result")
return result
except TimeoutError:
return {"failure_code": "validation_deadline"}
finally:
if members is not None:
os.close(members)
if archive is not None:
os.close(archive)
os.close(directory)
@@ -0,0 +1,438 @@
"""Authenticated durable import lifecycle; filesystem work remains behind private adapters."""
import asyncio
import hashlib
import os
from contextlib import asynccontextmanager
from datetime import timedelta
from uuid import UUID
from sqlalchemy import func, select, text
from kf_oj.contracts.admin_problems import (
IMPORT_LIFETIME_SECONDS,
MAX_OWNER_CANDIDATES,
MAX_TOTAL_CANDIDATES,
MAX_UPLOAD_CHUNK_BYTES,
MAX_UPLOAD_CHUNKS,
STAGING_MAX_BYTES,
STAGING_RESERVATION_BYTES,
UPLOAD_DEADLINE_SECONDS,
CandidatePreview,
CreateProblemImport,
ProblemImportCapabilities,
ProblemImportDetail,
ProblemImportPage,
ProblemImportSummary,
)
from kf_oj.infra.artifacts.staging import ImportStaging
from kf_oj.infra.artifacts.store import ArtifactError
from kf_oj.modules.identity.public import MaintenanceDenied, ProblemMaintenancePort
from kf_oj.modules.operations.public import MaintenanceAuditPort
from kf_oj.modules.problems.import_models import ImportPool, ProblemCandidate
from kf_oj.modules.problems.import_models import ProblemImportOperation as Operation
ACTIVE = ("uploading", "uploaded", "validating", "ready")
class ImportConflict(ValueError):
pass
class ImportBadDigest(ImportConflict):
pass
class ImportMissing(ValueError):
pass
class ImportUnavailable(ValueError):
pass
def summary(operation) -> ProblemImportSummary:
return ProblemImportSummary(
operation_id=operation.id,
state=operation.state,
expected_archive_digest=operation.archive_digest,
archive_bytes=operation.archive_bytes,
created_at=operation.created_at,
expires_at=operation.expires_at,
updated_at=operation.updated_at,
failure_code=operation.failure_code,
candidate_id=operation.candidate_id,
published_version_id=operation.published_version_id,
)
def policy_digest(policy):
return hashlib.sha256(policy.model_dump_json().encode()).hexdigest()
class ProblemImports:
def __init__(
self,
sessions,
staging: ImportStaging,
identity: ProblemMaintenancePort,
audit: MaintenanceAuditPort,
policy,
):
self.sessions, self.staging, self.identity, self.audit, self.policy = (
sessions,
staging,
identity,
audit,
policy,
)
@asynccontextmanager
async def write(self, principal):
async with self.sessions.begin() as session:
await session.execute(text("SET LOCAL statement_timeout='5s'"))
await session.execute(text("SET LOCAL lock_timeout='2s'"))
await self.identity.require_problem_maintenance(session, principal)
yield session
@staticmethod
async def pool(session):
pool = await session.get(ImportPool, 1, with_for_update=True)
if pool is None:
raise ImportUnavailable("import capacity unavailable")
actual = await session.scalar(
select(func.coalesce(func.sum(Operation.reserved_bytes), 0)).where(
Operation.reserved_bytes > 0
)
)
if pool.reserved_bytes != actual:
raise ImportUnavailable("import capacity inconsistent")
return pool
async def capabilities(self):
async with self.sessions() as session:
pool = await session.get(ImportPool, 1)
now = await session.scalar(select(func.clock_timestamp()))
available = bool(
pool
and pool.worker_ready_until
and pool.worker_ready_until > now
and pool.worker_policy_digest == policy_digest(self.policy)
)
return ProblemImportCapabilities(
uploads_available=available, validation_available=available
)
async def create(self, principal, request: CreateProblemImport):
# Immutable receipt retries need no filesystem slot, including while a parser/GC
# owns the root lock. Missing receipts still enter the locked creation path below.
async with self.write(principal) as session:
existing = await session.get(Operation, request.operation_id)
if existing:
if (
existing.owner_id != principal.account_id
or existing.archive_digest != request.archive_digest
or existing.archive_bytes != request.archive_bytes
):
raise ImportConflict("operation key already bound")
return summary(existing)
try:
with self.staging.exclusive():
await asyncio.to_thread(self.staging.usage)
async with self.write(principal) as session:
pool = await self.pool(session)
existing = await session.get(
Operation, request.operation_id, with_for_update=True
)
if existing:
if (
existing.owner_id != principal.account_id
or existing.archive_digest != request.archive_digest
or existing.archive_bytes != request.archive_bytes
):
raise ImportConflict("operation key already bound")
return summary(existing)
now = await session.scalar(select(func.clock_timestamp()))
if (
not pool.worker_ready_until
or pool.worker_ready_until <= now
or pool.worker_policy_digest != policy_digest(self.policy)
):
raise ImportUnavailable("import validation unavailable")
global_count = await session.scalar(
select(func.count())
.select_from(Operation)
.where(Operation.state.in_(ACTIVE))
)
owner_count = await session.scalar(
select(func.count())
.select_from(Operation)
.where(
Operation.state.in_(ACTIVE), Operation.owner_id == principal.account_id
)
)
busy = await session.scalar(
select(func.count())
.select_from(Operation)
.where(
Operation.state.in_(("uploading", "validating")),
Operation.lease_until > now,
)
)
if (
global_count >= MAX_TOTAL_CANDIDATES
or owner_count >= MAX_OWNER_CANDIDATES
or busy
or pool.reserved_bytes + STAGING_RESERVATION_BYTES > STAGING_MAX_BYTES
):
raise ImportUnavailable("import capacity exhausted")
operation = Operation(
id=request.operation_id,
owner_id=principal.account_id,
authorization_version=principal.authorization_version,
archive_digest=request.archive_digest,
archive_bytes=request.archive_bytes,
state="uploading",
created_at=now,
updated_at=now,
expires_at=now + timedelta(seconds=IMPORT_LIFETIME_SECONDS),
lease_until=now + timedelta(seconds=UPLOAD_DEADLINE_SECONDS + 15),
attempts=0,
fence=0,
reserved_bytes=STAGING_RESERVATION_BYTES,
)
session.add(operation)
pool.reserved_bytes += STAGING_RESERVATION_BYTES
await self.audit.record(
session,
principal.account_id,
principal.authorization_version,
operation.id,
"upload_created",
)
await session.flush()
return summary(operation)
except ArtifactError:
raise ImportUnavailable("import storage unavailable") from None
async def detail(self, principal, operation_id):
# Recheck live authority even when a principal was injected by an internal caller.
async with self.write(principal) as session:
op = await session.get(Operation, operation_id)
if op is None or op.owner_id != principal.account_id:
raise ImportMissing("import unavailable")
candidate = (
await session.get(ProblemCandidate, op.candidate_id) if op.candidate_id else None
)
return ProblemImportDetail(
**summary(op).model_dump(),
candidate=CandidatePreview.model_validate(candidate.preview) if candidate else None,
)
async def list(self, principal, after=None, limit=20):
if not 1 <= limit <= 100:
raise ValueError("invalid page limit")
async with self.write(principal) as session:
query = select(Operation).where(Operation.owner_id == principal.account_id)
if after:
query = query.where(Operation.id > after)
rows = list(await session.scalars(query.order_by(Operation.id).limit(limit + 1)))
return ProblemImportPage(
items=tuple(summary(o) for o in rows[:limit]),
next_after=rows[limit - 1].id if len(rows) > limit else None,
)
async def upload(self, principal, operation_id: UUID, chunks, content_length: int | None):
# Reject missing/foreign receipts before consulting shared filesystem capacity.
# The locked transaction below still rechecks authority and ownership before writes.
async with self.write(principal) as session:
op = await session.get(Operation, operation_id)
if op is None or op.owner_id != principal.account_id:
raise ImportMissing("import unavailable")
if content_length is not None and content_length != op.archive_bytes:
raise ImportConflict("archive length mismatch")
descriptor = operation_fd = None
fence = None
received = 0
chunk_count = 0
digest = hashlib.sha256()
try:
with self.staging.exclusive():
async with self.write(principal) as session:
await self.pool(session)
op = await session.get(Operation, operation_id, with_for_update=True)
if not op or op.owner_id != principal.account_id:
raise ImportMissing("import unavailable")
accepted = op.state in ("uploaded", "validating", "ready", "published")
now = await session.scalar(select(func.clock_timestamp()))
if (
not accepted
and (
op.state != "uploading"
or op.lease_until is None
or op.lease_until <= now
)
) or op.expires_at <= now:
raise ImportConflict("upload no longer available")
if content_length is not None and content_length != op.archive_bytes:
raise ImportConflict("archive length mismatch")
if not accepted:
op.fence += 1
fence = op.fence
op.lease_until = now + timedelta(seconds=UPLOAD_DEADLINE_SECONDS + 15)
expected_bytes, expected_digest = op.archive_bytes, op.archive_digest
if not accepted:
operation_fd = self.staging.operation(operation_id, create=True)
descriptor = os.open(
"archive",
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC,
0o600,
dir_fd=operation_fd,
)
async with asyncio.timeout(UPLOAD_DEADLINE_SECONDS):
async for chunk in chunks:
chunk_count += 1
received += len(chunk)
if (
type(chunk) is not bytes
or len(chunk) > MAX_UPLOAD_CHUNK_BYTES
or chunk_count > MAX_UPLOAD_CHUNKS
or received > expected_bytes
):
raise ImportConflict("archive bytes outside policy")
digest.update(chunk)
if accepted:
continue
# Cancellation waits for the finite writer outside the event loop.
task = asyncio.create_task(asyncio.to_thread(write_all, descriptor, chunk))
try:
await asyncio.shield(task)
except BaseException:
await task
raise
if received != expected_bytes or digest.hexdigest() != expected_digest:
raise ImportBadDigest("archive digest or length mismatch")
if accepted:
async with self.write(principal) as session:
op = await session.get(Operation, operation_id)
if not op or op.state not in (
"uploaded",
"validating",
"ready",
"published",
):
raise ImportConflict("import receipt no longer available")
return summary(op)
await asyncio.to_thread(os.fsync, descriptor)
os.close(descriptor)
descriptor = None
os.fsync(operation_fd)
async with self.write(principal) as session:
await self.pool(session)
op = await session.get(Operation, operation_id, with_for_update=True)
now = await session.scalar(select(func.clock_timestamp()))
if op.state != "uploading" or op.fence != fence or op.lease_until <= now:
raise ImportConflict("upload receipt expired")
op.state = "uploaded"
op.lease_until = None
op.updated_at = now
await self.audit.record(
session, op.owner_id, op.authorization_version, op.id, "upload_received"
)
result = summary(op)
return result
except BaseException as error:
# Failure bookkeeping cannot authorize a new business write. It only fences this
# accepted upload and releases capacity after physical bytes have been removed.
if descriptor is not None:
os.close(descriptor)
descriptor = None
if operation_fd is not None:
os.close(operation_fd)
operation_fd = None
if fence is not None:
try:
with self.staging.exclusive():
await asyncio.to_thread(self.staging.remove, operation_id)
async with self.sessions.begin() as session:
pool = await self.pool(session)
op = await session.get(Operation, operation_id, with_for_update=True)
if op and op.state == "uploading" and op.fence == fence:
op.state = "failed"
op.failure_code = (
"upload_deadline"
if isinstance(error, TimeoutError)
else "digest_mismatch"
if isinstance(error, ImportBadDigest)
else "authority_revoked"
if isinstance(error, MaintenanceDenied)
else "storage_unavailable"
if isinstance(error, OSError)
else "upload_interrupted"
)
op.updated_at = await session.scalar(select(func.clock_timestamp()))
op.lease_until = None
pool.reserved_bytes -= op.reserved_bytes
op.reserved_bytes = 0
await self.audit.record(
session,
op.owner_id,
op.authorization_version,
op.id,
"import_failed",
)
except (ArtifactError, OSError):
pass # Durable reservation retained for fenced GC.
raise
finally:
if descriptor is not None:
os.close(descriptor)
if operation_fd is not None:
os.close(operation_fd)
async def discard(self, principal, operation_id):
async with self.write(principal) as session:
op = await session.get(Operation, operation_id)
if op is None or op.owner_id != principal.account_id:
raise ImportMissing("import unavailable")
if op.state == "discarded":
return summary(op)
try:
with self.staging.exclusive():
async with self.write(principal) as session:
pool = await self.pool(session)
op = await session.get(Operation, operation_id, with_for_update=True)
if not op or op.owner_id != principal.account_id:
raise ImportMissing("import unavailable")
if op.state == "published":
raise ImportConflict("published import cannot be discarded")
if op.state == "discarded":
return summary(op)
await asyncio.to_thread(self.staging.remove, operation_id)
op.state = "discarded"
op.candidate_id = None
op.failure_code = None
op.lease_until = None
op.fence += 1
op.updated_at = await session.scalar(select(func.clock_timestamp()))
pool.reserved_bytes -= op.reserved_bytes
op.reserved_bytes = 0
await self.audit.record(
session,
principal.account_id,
principal.authorization_version,
op.id,
"import_discarded",
)
return summary(op)
except ArtifactError:
raise ImportUnavailable("import storage unavailable") from None
def write_all(descriptor, data):
remaining = memoryview(data)
while remaining:
size = os.write(descriptor, remaining)
if size <= 0:
raise OSError("short staging write")
remaining = remaining[size:]
@@ -0,0 +1,31 @@
"""Stricter Web package declarations; the accepted local CLI format remains unchanged."""
from dataclasses import dataclass
from kf_oj.contracts.judging import decode_wire
from kf_oj.modules.problems.import_format import ImportEnvelope, ProblemImport, ProblemImportV2
ARCHIVE_SUFFIXES = (".zip", ".jar", ".war", ".tar", ".tgz", ".gz", ".bz2", ".xz", ".7z", ".rar")
@dataclass(frozen=True)
class WebDeclaration:
manifest: ProblemImport | ProblemImportV2
members: frozenset[str]
def decode_web_declaration(data: bytes) -> WebDeclaration:
# decode_wire rejects duplicate JSON keys, unknown fields and noncanonical
# protocol values before either local paths or node policy can be considered.
manifest = decode_wire(ImportEnvelope, data).root
hidden = {name for c in manifest.cases for name in (c.input_file, c.expected_file)}
samples = {name for s in manifest.samples for name in (s.input_file, s.output_file)}
metadata = {"problem.json", manifest.statement_file}
if len(metadata) != 2 or hidden & (samples | metadata) or samples & metadata:
raise ValueError("public and hidden package declarations must be separate")
members = frozenset(metadata | hidden | samples)
if len({name.lower() for name in members}) != len(members):
raise ValueError("package names must be case-insensitively unique")
if any(name.lower().endswith(ARCHIVE_SUFFIXES) for name in members):
raise ValueError("nested archive declarations are unsupported")
return WebDeclaration(manifest, members)
+116
View File
@@ -0,0 +1,116 @@
"""Fixed data-only child. It inherits no DB credentials and executes no uploaded contents."""
import base64
import json
import os
import resource
import signal
import sys
import time
from uuid import UUID
from pydantic import TypeAdapter
from kf_oj.contracts.admin_problems import (
VALIDATION_DEADLINE_SECONDS,
CandidatePreview,
LegacyImportBudget,
)
from kf_oj.contracts.judging_v2 import NodeAdmission
from kf_oj.contracts.problems import PublicLanguageConfiguration, PublicSample
from kf_oj.infra.artifacts.archive_reader import ArchiveError, ArchiveReader
from kf_oj.modules.problems.candidate_validation import validate_candidate
from kf_oj.modules.problems.import_format import ProblemImport
from kf_oj.modules.problems.web_format import decode_web_declaration
def main():
# These are ordinary downward-only process limits, independent of the parent's life.
resource.setrlimit(resource.RLIMIT_AS, (128 * 1024**2, 128 * 1024**2))
resource.setrlimit(resource.RLIMIT_CPU, (60, 60))
resource.setrlimit(resource.RLIMIT_FSIZE, (64 * 1024**2, 64 * 1024**2))
resource.setrlimit(resource.RLIMIT_NOFILE, (64, 64))
resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
signal.alarm(VALIDATION_DEADLINE_SECONDS)
archive, directory, lock, size, digest, candidate, policy_json = sys.argv[1:]
archive, directory, lock = int(archive), int(directory), int(lock)
os.fstat(lock) # Held inherited flock prevents GC releasing capacity while any child lives.
deadline = time.monotonic() + VALIDATION_DEADLINE_SECONDS
result = {"failure_code": "invalid_archive"}
try:
reader = ArchiveReader(archive, int(size), deadline=deadline)
reader.verify_digest(digest)
result["failure_code"] = "invalid_manifest"
declared = decode_web_declaration(reader.read("problem.json", 256 * 1024))
if {m.name for m in reader.members} != set(declared.members):
raise ValueError("unexpected package members")
manifest = declared.manifest
result["failure_code"] = "invalid_data"
statement = reader.read(manifest.statement_file, 256 * 1024).decode("utf-8")
sample_data = {
name: reader.read(name, 65536)
for name in sorted({n for s in manifest.samples for n in (s.input_file, s.output_file)})
}
samples = tuple(
PublicSample(
input=base64.b64encode(sample_data[s.input_file]).decode("ascii"),
output=base64.b64encode(sample_data[s.output_file]).decode("ascii"),
)
for s in manifest.samples
)
references = reader.extract_remaining(directory)
reader.verify_unchanged()
legacy = isinstance(manifest, ProblemImport)
preview = CandidatePreview(
candidate_id=UUID(candidate),
slug=manifest.slug,
title=manifest.title,
statement=statement,
samples=samples,
judging_schema_version=manifest.schema_version,
languages=manifest.languages,
legacy_budget=LegacyImportBudget(
compile_limits=manifest.compile_limits,
run_limits=manifest.run_limits,
job_wall_time_us=manifest.job_wall_time_us,
)
if legacy
else None,
language_configurations=()
if legacy
else tuple(
PublicLanguageConfiguration.from_configuration(c)
for c in manifest.language_configurations
),
case_count=len(manifest.cases),
)
result["failure_code"] = "policy_rejected"
validated = validate_candidate(
manifest.model_dump(mode="json"),
preview.model_dump(mode="json"),
{k: {"digest": v[0], "bytes": v[1]} for k, v in references.items()},
TypeAdapter(NodeAdmission).validate_json(policy_json),
)
result = {
"manifest": validated.manifest,
"preview": validated.preview.model_dump(mode="json"),
"references": {k: v.model_dump(mode="json") for k, v in validated.references.items()},
"manifest_digest": validated.digest,
}
except ArchiveError as error:
result = {
"failure_code": "digest_mismatch"
if str(error) == "archive_digest_mismatch"
else "invalid_archive"
}
except (ValueError, OSError, MemoryError):
result = {"failure_code": result["failure_code"]}
payload = json.dumps(result, separators=(",", ":"), ensure_ascii=True).encode()
if len(payload) > 4 * 1024**2:
payload = b'{"failure_code":"invalid_data"}'
sys.stdout.buffer.write(payload)
sys.stdout.buffer.flush()
if __name__ == "__main__":
main()
@@ -0,0 +1,66 @@
"""Ordinary import service composition. Provision its dedicated DB role and cgroup separately."""
import asyncio
import sys
from sqlalchemy.exc import DBAPIError
from kf_oj.core.settings import Settings
from kf_oj.infra.artifacts.repository import ArtifactRepository
from kf_oj.infra.artifacts.staging import ImportStaging
from kf_oj.infra.artifacts.store import ArtifactStore
from kf_oj.infra.db.session import build_engine, build_session_factory
from kf_oj.modules.identity.service import ReservedMaintenance
from kf_oj.modules.operations.service import MaintenanceAudit
from kf_oj.modules.problems.import_worker import ImportWorker, verify_worker_limits
async def run():
settings = Settings()
if (
settings.import_staging_root is None
or settings.artifact_root is None
or settings.judge_admission is None
or settings.import_worker_cgroup is None
):
raise ValueError("import service configuration required")
verify_worker_limits(settings.import_worker_cgroup)
engine = build_engine(settings)
store = ArtifactStore(
settings.artifact_root,
max_bytes=settings.artifact_max_bytes,
max_files=settings.artifact_max_files,
)
staging = ImportStaging(settings.import_staging_root)
try:
worker = ImportWorker(
build_session_factory(engine),
staging,
ArtifactRepository(store),
ReservedMaintenance(),
MaintenanceAudit(),
settings.judge_admission,
settings.import_worker_cgroup,
)
while True:
if not await worker.once():
await asyncio.sleep(1)
finally:
staging.close()
store.close()
await engine.dispose()
def main():
try:
asyncio.run(run())
except (ValueError, OSError, DBAPIError):
print(
"import service unavailable; check trusted configuration and resource limits",
file=sys.stderr,
)
raise SystemExit(1) from None
if __name__ == "__main__":
main()
@@ -0,0 +1,156 @@
"""Contract checks only; upload/parser/security evidence requires the independent VM."""
import copy
import json
from datetime import UTC, datetime, timedelta
from pathlib import Path
from uuid import uuid4
import pytest
from pydantic import ValidationError
from kf_oj.contracts.admin_problems import (
CandidatePreview,
CreateProblemImport,
LegacyImportBudget,
ProblemImportCapabilities,
ProblemImportDetail,
)
from kf_oj.contracts.judging import AdmissionPolicy
from kf_oj.modules.problems.import_format import ImportEnvelope
from kf_oj.modules.problems.web_format import decode_web_declaration
ROOT = Path(__file__).resolve().parents[2]
def legacy_manifest():
policy = AdmissionPolicy.model_validate_json(
(ROOT / "tests/fixtures/contracts/judge-admission.valid.json").read_bytes()
)
return {
"schema_version": 1,
"slug": "web-fixture",
"title": "Public",
"statement_file": "statement.md",
"languages": ["c17", "cpp20"],
"toolchain": policy.toolchain.model_dump(mode="json"),
"comparison": "exact_bytes",
"compile_limits": policy.compile_ceiling.model_dump(),
"run_limits": policy.run_ceiling.model_dump(),
"job_wall_time_us": 60000000,
"cases": [{"key": "binary", "input_file": "hidden.in", "expected_file": "hidden.out"}],
"samples": [{"input_file": "sample.in", "output_file": "sample.out"}],
}
def test_web_declaration_separates_public_data_and_preserves_local_format():
source = legacy_manifest()
original = copy.deepcopy(source)
declared = decode_web_declaration(json.dumps(source).encode())
assert declared.members == frozenset(
("problem.json", "statement.md", "hidden.in", "hidden.out", "sample.in", "sample.out")
)
assert declared.manifest.model_dump(mode="json") == original
assert ImportEnvelope.model_validate(source).root == declared.manifest
for mutation in (
lambda x: x["samples"][0].update(input_file="hidden.in"),
lambda x: x.update(statement_file="problem.json"),
lambda x: x["cases"][0].update(input_file="Statement.md"),
lambda x: x["cases"][0].update(input_file="hidden.zip"),
lambda x: x.update(languages=["rust"]),
lambda x: x.update(download_url="https://example.invalid/private"),
):
value = copy.deepcopy(source)
mutation(value)
with pytest.raises(ValueError):
decode_web_declaration(json.dumps(value).encode())
with pytest.raises(ValueError):
decode_web_declaration(
json.dumps(source)
.replace('"schema_version": 1', '"schema_version": 1, "schema_version": 1')
.encode()
)
def test_import_operation_key_and_preview_have_no_private_payload_fields():
operation = {
"schema_version": 1,
"operation_id": str(uuid4()),
"archive_digest": "a" * 64,
"archive_bytes": 64 * 1024**2,
}
assert CreateProblemImport.model_validate(operation).archive_bytes == 64 * 1024**2
for field, value in (
("archive_bytes", True),
("archive_bytes", 64 * 1024**2 + 1),
("url", "https://example.invalid"),
("role", "admin"),
):
with pytest.raises(ValidationError):
CreateProblemImport.model_validate(operation | {field: value})
manifest = ImportEnvelope.model_validate(legacy_manifest()).root
preview = CandidatePreview(
candidate_id=uuid4(),
slug=manifest.slug,
title=manifest.title,
statement="Public only",
samples=(),
judging_schema_version=1,
languages=manifest.languages,
legacy_budget=LegacyImportBudget(
compile_limits=manifest.compile_limits,
run_limits=manifest.run_limits,
job_wall_time_us=manifest.job_wall_time_us,
),
case_count=1,
)
now = datetime.now(UTC)
response = ProblemImportDetail(
operation_id=uuid4(),
state="ready",
expected_archive_digest="a" * 64,
archive_bytes=100,
created_at=now,
updated_at=now,
expires_at=now + timedelta(days=1),
failure_code=None,
candidate_id=preview.candidate_id,
published_version_id=None,
candidate=preview,
)
assert set(response.candidate.model_dump()) == {
"candidate_id",
"slug",
"title",
"statement",
"samples",
"judging_schema_version",
"languages",
"legacy_budget",
"language_configurations",
"case_count",
}
with pytest.raises(ValidationError):
CandidatePreview.model_validate(preview.model_dump() | {"input_digest": "a" * 64})
capabilities = ProblemImportCapabilities(uploads_available=False, validation_available=False)
assert (
capabilities.limits.global_concurrency == 1 and capabilities.limits.owner_candidates == 10
)
def test_standalone_import_worker_resolves_all_referenced_metadata():
# A separate trusted interpreter avoids accidental model registration by API test imports.
import subprocess
import sys
script = """
import kf_oj.workers.problem_import
from kf_oj.infra.db.base import Base
for table in Base.metadata.tables.values():
for foreign in table.foreign_keys:
assert foreign.column.table is not None
"""
result = subprocess.run(
[sys.executable, "-c", script], capture_output=True, text=True, timeout=15
)
assert result.returncode == 0, result.stderr
+988
View File
@@ -0,0 +1,988 @@
"""Real HTTP/worker/storage and adverse ZIP cases: independent capped VM only."""
import asyncio
import hashlib
import io
import json
import os
import resource
import signal
import struct
import subprocess
import sys
import time
import zipfile
from pathlib import Path
from uuid import UUID, uuid4
import httpx
import pytest
from conftest import connect
from sqlalchemy.engine import make_url
from kf_oj.contracts.judging import ArtifactReference
from kf_oj.contracts.judging_v2 import AdmissionRegistry
from kf_oj.infra.artifacts.archive_reader import ArchiveError, ArchiveReader
from kf_oj.infra.artifacts.staging import ImportStaging
from kf_oj.infra.artifacts.store import ArtifactError, ArtifactStore
from kf_oj.modules.identity.passwords import hash_password
ROOT = Path(__file__).resolve().parents[2]
@pytest.fixture
def independent_import_vm():
marker = Path("/kf/INDEPENDENT_VM")
if not marker.exists() or marker.read_text().strip() != "isolated-import-test-guest":
pytest.skip("adverse archive/full import acceptance requires the independent capped VM")
assert [p.name for p in Path("/sys/class/net").iterdir()] == ["lo"]
assert all(
x not in Path("/proc/self/mountinfo").read_text() for x in ("9p", "virtiofs", "hgfs")
)
return {name: Path("/sys/fs/cgroup/kf-total") / name for name in ("api", "import-worker")}
def package(
policy, *, multi=False, method=zipfile.ZIP_STORED, mutation=None, payload_mutation=None
):
payload = {
"statement.md": b"Public statement only",
"hidden.in": b"synthetic-hidden-input\x00\xff",
"hidden.out": b"synthetic-hidden-output\x00\xfe",
"sample.in": b"1\n",
"sample.out": b"2\n",
}
manifest = {
"schema_version": 1,
"slug": "web-synthetic",
"title": "Public",
"statement_file": "statement.md",
"languages": ["c17", "cpp20"],
"toolchain": policy.legacy.toolchain.model_dump(mode="json"),
"comparison": "exact_bytes",
"compile_limits": policy.legacy.compile_ceiling.model_dump(),
"run_limits": policy.legacy.run_ceiling.model_dump(),
"job_wall_time_us": 60000000,
"cases": [{"key": "binary", "input_file": "hidden.in", "expected_file": "hidden.out"}],
"samples": [{"input_file": "sample.in", "output_file": "sample.out"}],
}
if multi:
for key in ("languages", "toolchain", "compile_limits", "run_limits", "job_wall_time_us"):
del manifest[key]
manifest["schema_version"] = 2
manifest["language_configurations"] = [
{
key: json.loads(
(
ROOT / f"tests/fixtures/contracts-v2/judge-manifest.valid-{language}.json"
).read_text()
)[key]
for key in (
"language",
"toolchain",
"compile_limits",
"run_limits",
"job_wall_time_us",
)
}
for language in ("c17", "cpp20", "python313", "java21")
]
if mutation:
mutation(manifest)
payload["problem.json"] = json.dumps(manifest).encode()
if payload_mutation:
payload_mutation(payload)
out = io.BytesIO()
with zipfile.ZipFile(out, "w", compression=method) as archive:
for name, data in payload.items():
archive.writestr(name, data)
return out.getvalue(), payload
def test_bounded_zip_and_streaming_adoption(independent_import_vm, tmp_path):
policy = AdmissionRegistry.model_validate_json(
(ROOT / "tests/fixtures/contracts-v2/judge-registry.valid.json").read_bytes()
)
for method in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED):
blob, payload = package(
policy,
method=method,
payload_mutation=lambda p: p.update({"hidden.in": b"A" * 1048576 + b"\x00\xff"}),
)
file = tmp_path / f"archive-{method}"
file.write_bytes(blob)
file.chmod(0o600)
output = tmp_path / f"members-{method}"
output.mkdir(mode=0o700)
fd = os.open(file, os.O_RDONLY)
directory = os.open(output, os.O_RDONLY | os.O_DIRECTORY)
try:
reader = ArchiveReader(fd, len(blob), deadline=time.monotonic() + 60)
reader.verify_digest(hashlib.sha256(blob).hexdigest())
assert reader.read("problem.json", 262144) == payload.pop("problem.json")
references = reader.extract_remaining(directory)
assert reader.unpacked_bytes == sum(len(v) for v in payload.values()) + len(
package(policy, method=method)[1]["problem.json"]
)
store = ArtifactStore(tmp_path / f"objects-{method}")
try:
for name, (digest, size) in references.items():
assert (output / name).read_bytes() == payload[name]
source = os.open(name, os.O_RDONLY, dir_fd=directory)
try:
reference = ArtifactReference(digest=digest, bytes=size)
assert store.put_file(source, reference) == reference
assert store.put_file(source, reference) == reference
finally:
os.close(source)
finally:
store.close()
finally:
os.close(fd)
os.close(directory)
good, _ = package(policy)
wrong_crc = bytearray(good)
struct.pack_into("<I", wrong_crc, 14, 1)
cases = [b"prefix" + good, good + b"trailing", bytes(wrong_crc)]
for name in ("../escape", "bad/name", "Statement.md"):
cases.append(package(policy, payload_mutation=lambda p, n=name: p.update({n: b"bad"}))[0])
symlink = io.BytesIO()
with zipfile.ZipFile(symlink, "w") as z:
info = zipfile.ZipInfo("link")
info.create_system = 3
info.external_attr = 0o120777 << 16
z.writestr(info, b"/outside")
cases.append(symlink.getvalue())
# Real damaged stored data and forged consistent local/central size/CRC/method/flags.
corrupt = bytearray(good)
name_bytes, extra_bytes = struct.unpack_from("<HH", corrupt, 26)
corrupt[30 + name_bytes + extra_bytes] ^= 1
cases.append(bytes(corrupt))
central = good.index(b"PK\x01\x02")
for local_offset, central_offset, value, fmt in (
(6, 8, 1, "<H"), # encryption
(8, 10, 99, "<H"), # unsupported method
(22, 24, 67108865, "<I"), # claimed size outside budget
):
damaged = bytearray(good)
struct.pack_into(fmt, damaged, local_offset, value)
struct.pack_into(fmt, damaged, central + central_offset, value)
cases.append(bytes(damaged))
compressed, _ = package(policy, method=zipfile.ZIP_DEFLATED)
forged = bytearray(compressed)
central = compressed.index(b"PK\x01\x02")
struct.pack_into("<I", forged, 22, 1)
struct.pack_into("<I", forged, central + 24, 1)
cases.append(bytes(forged)) # actual inflation exceeds the forged size
for index, blob in enumerate(cases):
file = tmp_path / f"bad-{index}"
file.write_bytes(blob)
file.chmod(0o600)
fd = os.open(file, os.O_RDONLY)
try:
with pytest.raises(ArchiveError):
reader = ArchiveReader(fd, len(blob), deadline=time.monotonic() + 60)
# Unknown declaration references are rejected by the child, rather than
# the raw ZIP decoder. Here invalid paths/types/CRC/header bytes are read.
for member in reader.members:
reader.read(member.name, 64 * 1024**2)
finally:
os.close(fd)
def test_real_admin_upload_worker_candidate_faults_restart_and_cleanup(
independent_import_vm, postgres_url, tmp_path
):
groups = independent_import_vm
policy = AdmissionRegistry.model_validate_json(
(ROOT / "tests/fixtures/contracts-v2/judge-registry.valid.json").read_bytes()
)
base = {
k: v
for k, v in os.environ.items()
if k
in (
"PATH",
"PYTHONPATH",
"LD_LIBRARY_PATH",
"LC_ALL",
"PYTHONUNBUFFERED",
"PYTHONDONTWRITEBYTECODE",
)
}
roles = {"api": "import_api_" + uuid4().hex, "worker": "import_worker_" + uuid4().hex}
password = "Synthetic import acceptance only 2026!"
users = {name: uuid4() for name in ("web_admin", "web_backup", "web_student")}
processes = {}
logs = {}
artifact = tmp_path / "artifacts"
staging = tmp_path / "staging"
artifact.mkdir(mode=0o700)
staging.mkdir(mode=0o700)
def enter(group):
def apply():
(group / "cgroup.procs").write_text(str(os.getpid()))
resource.setrlimit(resource.RLIMIT_FSIZE, (67108864, 67108864))
resource.setrlimit(resource.RLIMIT_NOFILE, (64, 64))
resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
return apply
def start(name, *, configured=True, pair=False):
group = groups["api" if name == "api" else "import-worker"]
env = base | {
"KF_OJ_DATABASE_URL": make_url(postgres_url)
.set(username=roles[name])
.render_as_string(hide_password=False),
"KF_OJ_ENVIRONMENT": "test",
"KF_OJ_ARTIFACT_ROOT": str(artifact),
"KF_OJ_IMPORT_STAGING_ROOT": str(staging),
"KF_OJ_JUDGE_ADMISSION": policy.model_dump_json(),
"KF_OJ_BROWSER_ORIGINS": '["http://127.0.0.1:58000"]',
}
if not configured:
env.pop("KF_OJ_IMPORT_STAGING_ROOT")
if name == "worker":
env["KF_OJ_IMPORT_WORKER_CGROUP"] = str(group)
command = (
[
sys.executable,
"-m",
"uvicorn",
"kf_oj.api.app:create_app",
"--factory",
"--host",
"127.0.0.1",
"--port",
"58000",
"--no-proxy-headers",
"--limit-concurrency",
"4",
"--backlog",
"16",
]
if name == "api"
else [sys.executable, "-m", "kf_oj.workers.problem_import"]
)
if pair:
assert name == "worker"
# Trusted fixture composition; both coroutines call the actual ordinary Worker.
code = """
import asyncio
from kf_oj.core.settings import Settings
from kf_oj.infra.db.session import build_engine,build_session_factory
from kf_oj.infra.artifacts.store import ArtifactStore
from kf_oj.infra.artifacts.staging import ImportStaging
from kf_oj.infra.artifacts.repository import ArtifactRepository
from kf_oj.modules.identity.service import ReservedMaintenance
from kf_oj.modules.operations.service import MaintenanceAudit
from kf_oj.modules.problems.import_worker import ImportWorker
async def run():
s=Settings();e=build_engine(s);store=ArtifactStore(s.artifact_root)
staging=ImportStaging(s.import_staging_root)
try:
w=ImportWorker(build_session_factory(e),staging,ArtifactRepository(store),
ReservedMaintenance(),MaintenanceAudit(),s.judge_admission,s.import_worker_cgroup)
results=await asyncio.gather(w.once(),w.once())
assert sorted(results)==[False,True],results
print('ACTUAL_CONCURRENT_IMPORT_CLAIM_ACCEPTED',flush=True)
finally:
staging.close();store.close();await e.dispose()
asyncio.run(run())
"""
command = [sys.executable, "-c", code]
log = (tmp_path / f"{name}.log").open("a")
logs[name] = log
processes[name] = subprocess.Popen(
command, env=env, stdout=log, stderr=log, preexec_fn=enter(group)
)
def stop(name):
p = processes.pop(name, None)
if p and p.poll() is None:
p.terminate()
try:
p.wait(timeout=20)
except subprocess.TimeoutExpired:
p.kill()
p.wait(timeout=10)
if name in logs:
logs.pop(name).close()
async def run():
c = await connect(postgres_url)
try:
for name, group in (("api", "kf_oj_platform"), ("worker", "kf_oj_problem_import")):
await c.execute(
f"CREATE ROLE {roles[name]} LOGIN NOSUPERUSER NOCREATEDB "
"NOCREATEROLE NOREPLICATION NOBYPASSRLS"
)
await c.execute(f"GRANT {group} TO {roles[name]}")
for name, identifier in users.items():
await c.execute(
"INSERT INTO accounts(id,username,password_hash) VALUES($1,$2,$3)",
identifier,
name,
hash_password(name, password),
)
await c.execute("SELECT * FROM kf_manage_account_authority('web_admin','bootstrap')")
await c.execute("SELECT * FROM kf_manage_account_authority('web_backup','grant')")
start("api")
start("worker")
async with httpx.AsyncClient(base_url="http://127.0.0.1:58000", timeout=45) as client:
deadline = time.monotonic() + 300
while time.monotonic() < deadline:
assert processes["api"].poll() is None, (tmp_path / "api.log").read_text()[
-1500:
]
assert processes["worker"].poll() is None, (
tmp_path / "worker.log"
).read_text()[-1500:]
try:
if (
await client.get("/api/v1/health/live")
).status_code == 200 and await c.fetchval(
"SELECT worker_ready_until>clock_timestamp() "
"FROM problem_import_pool WHERE id=1"
):
break
except httpx.ConnectError:
pass
await asyncio.sleep(0.1)
else:
raise AssertionError(
"real API/worker startup deadline; api="
+ (tmp_path / "api.log").read_text()[-1500:]
+ " worker="
+ (tmp_path / "worker.log").read_text()[-1500:]
)
prefix = "/api/v1/admin/problem-imports"
assert (await client.get(prefix + "/capabilities")).status_code == 401
async def login(name):
client.cookies.clear()
r = await client.post(
"/api/v1/auth/login",
json={"username": name, "password": password},
headers={"Origin": "http://127.0.0.1:58000"},
)
assert r.status_code == 200, r.text
return {
"Origin": "http://127.0.0.1:58000",
"X-CSRF-Token": client.cookies["kf_csrf"],
}
student_headers = await login("web_student")
assert (await client.get(prefix + "/capabilities")).status_code == 403
assert (
await client.post(
prefix,
json={
"operation_id": str(uuid4()),
"archive_digest": "a" * 64,
"archive_bytes": 100,
},
headers=student_headers,
)
).status_code == 403
headers = await login("web_admin")
assert (await client.get(prefix + "/capabilities")).json()[
"uploads_available"
] is True
for invalid in (
{},
{"Origin": "http://127.0.0.1:58000", "X-CSRF-Token": "0" * 64},
headers | {"Origin": "https://evil.invalid"},
headers
| {"Origin": "https://evil.invalid", "X-Forwarded-Host": "127.0.0.1:58000"},
):
assert (
await client.post(
prefix,
json={
"operation_id": str(uuid4()),
"archive_digest": "a" * 64,
"archive_bytes": 100,
},
headers=invalid,
)
).status_code == 403
async def create(blob, key=None):
key = key or uuid4()
body = {
"operation_id": str(key),
"archive_digest": hashlib.sha256(blob).hexdigest(),
"archive_bytes": len(blob),
}
deadline = time.monotonic() + 15
while True:
r = await client.post(prefix, json=body, headers=headers)
if r.status_code != 503 or time.monotonic() >= deadline:
break
await asyncio.sleep(0.2)
assert r.status_code == 201, r.text
assert r.headers["cache-control"] == "no-store"
assert (
await client.post(prefix, json=body, headers=headers)
).json() == r.json()
assert (
await client.post(
prefix, json=body | {"archive_digest": "f" * 64}, headers=headers
)
).status_code == 409
return str(key)
async def put_blob(blob, key):
deadline = time.monotonic() + 30
while True:
result = await client.put(
f"{prefix}/{key}/archive",
content=blob,
headers=headers | {"Content-Type": "application/zip"},
)
if result.status_code != 503 or time.monotonic() >= deadline:
return result
await asyncio.sleep(0.2)
async def discard_owned(key):
deadline = time.monotonic() + 15
while True:
result = await client.post(f"{prefix}/{key}/discard", headers=headers)
if result.status_code != 503 or time.monotonic() >= deadline:
return result
await asyncio.sleep(0.2)
async def upload(blob, key):
r = await put_blob(blob, key)
assert r.status_code == 200, r.text
deadline = time.monotonic() + 100
while time.monotonic() < deadline:
r = await client.get(f"{prefix}/{key}")
assert r.status_code == 200, r.text
if r.json()["state"] in ("ready", "failed"):
return r.json()
assert processes["worker"].poll() is None, (
tmp_path / "worker.log"
).read_text()[-2000:]
await asyncio.sleep(0.2)
raise AssertionError(
"candidate deadline; " + (tmp_path / "worker.log").read_text()[-2000:]
)
async def wait_released(key, state, code=None, seconds=120):
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
row = await c.fetchrow(
"SELECT state,failure_code,reserved_bytes,attempts FROM "
"problem_import_operations WHERE id=$1",
UUID(key),
)
if row["state"] == state and row["reserved_bytes"] == 0:
if code:
assert row["failure_code"] == code, dict(row)
return row
await asyncio.sleep(0.1)
raise AssertionError("operation cleanup deadline: " + str(dict(row)))
async def restart_ready_worker():
previous = await c.fetchval(
"SELECT worker_ready_until FROM problem_import_pool WHERE id=1"
)
assert previous is not None
start("worker")
deadline = time.monotonic() + 120
while time.monotonic() < deadline:
assert processes["worker"].poll() is None
if await c.fetchval(
"SELECT worker_ready_until>clock_timestamp()+interval '80 seconds' "
"AND worker_ready_until>$1 FROM problem_import_pool WHERE id=1",
previous,
):
return
await asyncio.sleep(0.1)
raise AssertionError("fresh worker heartbeat deadline")
async def raw_body(key, declared, data, *, delay=0, disconnect=False):
reader, writer = await asyncio.open_connection("127.0.0.1", 58000)
cookie = "; ".join(f"{k}={v}" for k, v in client.cookies.items())
request = (
f"PUT {prefix}/{key}/archive HTTP/1.1\r\n"
"Host: 127.0.0.1:58000\r\nOrigin: http://127.0.0.1:58000\r\n"
f"Cookie: {cookie}\r\nX-CSRF-Token: {headers['X-CSRF-Token']}\r\n"
f"Content-Type: application/zip\r\nContent-Length: {declared}\r\n"
"Connection: close\r\n\r\n"
).encode()
try:
writer.write(request + data)
await writer.drain()
if delay:
await asyncio.sleep(delay)
if disconnect:
return None
return await asyncio.wait_for(reader.read(8192), 45)
finally:
writer.close()
await writer.wait_closed()
for multi, method in ((False, zipfile.ZIP_STORED), (True, zipfile.ZIP_DEFLATED)):
blob, payload = package(policy, multi=multi, method=method)
key = await create(blob)
detail = await upload(blob, key)
assert detail["state"] == "ready", (detail["failure_code"], detail["state"])
print("ACTUAL_READY_IMPORT", multi, method, detail["candidate_id"], flush=True)
await wait_released(key, "ready")
retry = await put_blob(blob, key)
assert retry.status_code == 200
different = blob[:-1] + bytes([blob[-1] ^ 1])
conflict = await put_blob(different, key)
assert conflict.status_code == 409
assert (await client.get(f"{prefix}/{key}")).json()["candidate_id"] == detail[
"candidate_id"
]
assert detail["candidate"]["judging_schema_version"] == (2 if multi else 1)
assert detail["candidate"]["case_count"] == 1
assert len(detail["candidate"]["languages"]) == (4 if multi else 2)
assert "synthetic-hidden" not in json.dumps(detail) and str(
tmp_path
) not in json.dumps(detail)
# Actual database enforcement, not trigger inspection alone.
worker_db = await connect(
make_url(postgres_url)
.set(username=roles["worker"])
.render_as_string(hide_password=False)
)
try:
from asyncpg import CheckViolationError, InsufficientPrivilegeError
with pytest.raises(InsufficientPrivilegeError):
await worker_db.execute(
"UPDATE problem_candidates SET preview=preview WHERE id=$1",
UUID(detail["candidate_id"]),
)
with pytest.raises(CheckViolationError):
await worker_db.execute(
"DELETE FROM problem_candidates WHERE id=$1",
UUID(detail["candidate_id"]),
)
with pytest.raises(CheckViolationError):
await worker_db.execute(
"""INSERT INTO problem_candidate_cases
(candidate_id,position,key,input_digest,input_bytes,expected_digest,expected_bytes)
SELECT candidate_id,1,'late',input_digest,input_bytes,
expected_digest,expected_bytes
FROM problem_candidate_cases WHERE candidate_id=$1""",
UUID(detail["candidate_id"]),
)
finally:
await worker_db.close()
refs = await c.fetch(
"SELECT input_digest,input_bytes,expected_digest,expected_bytes "
"FROM problem_candidate_cases WHERE candidate_id=$1",
UUID(detail["candidate_id"]),
)
assert len(refs) == 1
for field, name in (("input", "hidden.in"), ("expected", "hidden.out")):
digest = refs[0][field + "_digest"]
data = (artifact / digest).read_bytes()
assert data == payload[name] and hashlib.sha256(data).hexdigest() == digest
assert await c.fetchval("SELECT count(*) FROM problem_versions") == 0
assert (await client.get("/api/v1/problems")).json()["items"] == []
assert (
await c.fetchval(
"SELECT reserved_bytes FROM problem_import_pool WHERE id=1"
)
== 0
)
assert sorted(p.name for p in staging.iterdir()) == [".lock"]
def reuse_public_member(payload):
declaration = json.loads(payload["problem.json"])
declaration["samples"][0]["output_file"] = "sample.in"
payload["problem.json"] = json.dumps(declaration).encode()
del payload["sample.out"]
# Actual child consumes malformed data but never executes contents.
for mutation in (
lambda p: p.update({"../escape": b"bad"}),
lambda p: p.update({"unknown.bin": b"bad"}),
lambda p: p.update({"hidden.in": b"PK\x03\x04" + b"bad"}),
lambda p: p.update({"sample.in": p["hidden.in"]}),
reuse_public_member,
):
blob, _ = package(policy, payload_mutation=mutation)
key = await create(blob)
result = await upload(blob, key)
# Publicly declared samples can happen to equal hidden bytes; declarations,
# not byte equality, decide visibility. This last finite case is legal.
if b"PK\x03\x04bad" in blob or b"../escape" in blob or b"unknown.bin" in blob:
assert result["state"] == "failed" and result["candidate"] is None, result
else:
assert result["state"] == "ready", result
await wait_released(key, result["state"])
assert (
await c.fetchval(
"SELECT reserved_bytes FROM problem_import_pool WHERE id=1"
)
== 0
)
blob, _ = package(policy)
key = await create(blob)
stop("worker")
occupied = ImportStaging(staging)
try:
with occupied.exclusive():
mismatch = await client.put(
f"{prefix}/{key}/archive",
content=blob[:-1],
headers=headers | {"Content-Type": "application/zip"},
)
assert mismatch.status_code == 409
finally:
occupied.close()
assert (await discard_owned(key)).status_code == 200
# Restart after receipt before consumption preserves the immutable operation.
stop("worker")
key = await create(blob)
r = await client.put(
f"{prefix}/{key}/archive",
content=blob,
headers=headers | {"Content-Type": "application/zip"},
)
assert r.status_code == 200 and r.json()["state"] == "uploaded"
start("worker")
deadline = time.monotonic() + 120
while time.monotonic() < deadline:
detail = (await client.get(f"{prefix}/{key}")).json()
if detail["state"] in ("ready", "failed"):
break
await asyncio.sleep(0.2)
assert detail["state"] == "ready", (detail["failure_code"], detail["state"])
assert (
await c.fetchval(
"SELECT attempts FROM problem_import_operations WHERE id=$1",
UUID(key),
)
== 1
)
await wait_released(key, "ready")
# A second real administrator cannot inspect, upload or discard another's candidate.
headers = await login("web_backup")
stop("worker")
occupied = ImportStaging(staging)
try:
with occupied.exclusive():
assert (await client.get(f"{prefix}/{key}")).status_code == 404
assert (
await client.post(f"{prefix}/{key}/discard", headers=headers)
).status_code == 404
assert (
await client.put(
f"{prefix}/{key}/archive",
content=blob,
headers=headers | {"Content-Type": "application/zip"},
)
).status_code == 404
finally:
occupied.close()
start("worker")
headers = await login("web_admin")
# Incomplete TCP body and actual 30-second streaming deadline free reservations.
interrupted = await create(blob)
stop("worker") # Exercise the upload path without a competing GC root lock.
await raw_body(interrupted, len(blob), blob[:8], disconnect=True)
await wait_released(interrupted, "failed", "upload_interrupted")
await restart_ready_worker()
slow = await create(blob)
stop("worker")
response = await raw_body(slow, len(blob), blob[:8], delay=33)
await wait_released(slow, "failed", "upload_deadline")
assert response.startswith(b"HTTP/1.1 408"), response[:200]
await restart_ready_worker()
# A full received body with the wrong SHA is never acknowledged as an upload.
wrong = await create(blob)
stop("worker")
r = await client.put(
f"{prefix}/{wrong}/archive",
content=blob[:-1] + b"!",
headers=headers | {"Content-Type": "application/zip"},
)
assert r.status_code == 409
await wait_released(wrong, "failed", "digest_mismatch")
await restart_ready_worker()
# Two actual reservations fill the global staging pool with Worker stopped.
stop("worker")
queued = []
for _ in range(2):
queued_key = await create(blob)
r = await client.put(
f"{prefix}/{queued_key}/archive",
content=blob,
headers=headers | {"Content-Type": "application/zip"},
)
assert r.status_code == 200
queued.append(queued_key)
assert (
await c.fetchval("SELECT reserved_bytes FROM problem_import_pool WHERE id=1")
== 268435456
)
r = await client.post(
prefix,
json={
"operation_id": str(uuid4()),
"archive_digest": hashlib.sha256(blob).hexdigest(),
"archive_bytes": len(blob),
},
headers=headers,
)
assert r.status_code == 503
for queued_key in queued:
assert (
await client.post(f"{prefix}/{queued_key}/discard", headers=headers)
).status_code == 200
assert (
await c.fetchval("SELECT reserved_bytes FROM problem_import_pool WHERE id=1")
== 0
)
start("worker")
# Kill parent AND parser in their already verified cgroup while a lease is held.
# This finite injection tests both retries. No uploaded code is executed.
interrupted_parser = await create(blob)
r = await put_blob(blob, interrupted_parser)
assert r.status_code == 200
for expected_attempt in (1, 2):
deadline = time.monotonic() + 120
while time.monotonic() < deadline:
row = await c.fetchrow(
"SELECT state,attempts FROM problem_import_operations WHERE id=$1",
UUID(interrupted_parser),
)
parser = []
for pid in (
groups["import-worker"].joinpath("cgroup.procs").read_text().split()
):
cmdline = Path(f"/proc/{pid}/cmdline")
if (
cmdline.exists()
and b"kf_oj.workers._archive_child" in cmdline.read_bytes()
):
parser.append(int(pid))
if (
row["state"] == "validating"
and row["attempts"] == expected_attempt
and parser
):
# Freeze parser, kill parent, then parser. A live child keeps the flock.
for pid in parser:
os.kill(pid, signal.SIGSTOP)
processes["worker"].kill()
processes["worker"].wait(timeout=10)
await c.execute(
"UPDATE problem_import_operations SET lease_until="
"clock_timestamp()-interval '1 second' WHERE id=$1",
UUID(interrupted_parser),
)
# A live stopped child retains the real kernel lock after parent death.
probe = ImportStaging(staging)
try:
with pytest.raises(ArtifactError):
with probe.exclusive():
pass
finally:
probe.close()
denied = await client.post(
prefix,
json={
"operation_id": str(uuid4()),
"archive_digest": hashlib.sha256(blob).hexdigest(),
"archive_bytes": len(blob),
},
headers=headers,
)
assert denied.status_code == 503
for pid in parser:
os.kill(pid, signal.SIGKILL)
stop("worker")
break
await asyncio.sleep(0.02)
else:
raise AssertionError("claimed parser injection deadline")
# Advance the lease under the VM test operator; requests cannot set clocks.
await c.execute(
"UPDATE problem_import_operations SET lease_until="
"clock_timestamp()-interval '1 second' WHERE id=$1",
UUID(interrupted_parser),
)
start("worker")
row = await wait_released(interrupted_parser, "failed", "validation_unavailable")
assert row["attempts"] == 2
assert (
await c.fetchval(
"SELECT count(*) FROM problem_candidates WHERE operation_id=$1",
UUID(interrupted_parser),
)
== 0
)
# Real concurrent claims share the same capped process and use separate lock FDs.
stop("worker")
paired_key = await create(blob)
r = await client.put(
f"{prefix}/{paired_key}/archive",
content=blob,
headers=headers | {"Content-Type": "application/zip"},
)
assert r.status_code == 200 and r.json()["state"] == "uploaded"
start("worker", pair=True)
row = await wait_released(paired_key, "ready")
assert row["attempts"] == 1
assert (
await c.fetchval(
"SELECT count(*) FROM problem_candidates WHERE operation_id=$1",
UUID(paired_key),
)
== 1
)
deadline = time.monotonic() + 120
while processes["worker"].poll() is None and time.monotonic() < deadline:
await asyncio.sleep(0.1)
assert processes["worker"].returncode == 0, (tmp_path / "worker.log").read_text()[
-1500:
]
assert (
"ACTUAL_CONCURRENT_IMPORT_CLAIM_ACCEPTED"
in (tmp_path / "worker.log").read_text()
)
stop("worker")
start("worker")
# Insert an old receipt with the test operator; immutable timestamps stay unchanged.
expired_id = uuid4()
async with c.transaction():
await c.execute(
"""INSERT INTO problem_import_operations
(id,owner_id,authorization_version,archive_digest,archive_bytes,created_at,
updated_at,expires_at,lease_until,attempts,fence,reserved_bytes)
SELECT $1,$2,authorization_version,$3,$4,
clock_timestamp()-interval '2 days',
clock_timestamp()-interval '2 days',clock_timestamp()-interval '1 day',
clock_timestamp()-interval '1 day',0,0,134217728 FROM accounts WHERE id=$2""",
expired_id,
users["web_admin"],
hashlib.sha256(blob).hexdigest(),
len(blob),
)
assert (
await c.fetchval(
"SELECT state FROM problem_import_operations WHERE id=$1", expired_id
)
== "uploading"
)
await c.execute(
"UPDATE problem_import_pool SET reserved_bytes="
"reserved_bytes+134217728 WHERE id=1"
)
await wait_released(str(expired_id), "expired", "expired")
print(
"ACTUAL_IMPORT_DISCONNECT_DEADLINE_CAPACITY_TWO_RETRIES_EXPIRY_ACCEPTED",
flush=True,
)
# Revocation is live and old cookies cannot read or write management APIs.
await c.execute("SELECT * FROM kf_manage_account_authority('web_admin','revoke')")
assert (await client.get(prefix + "/capabilities")).status_code == 401
assert (
await client.post(f"{prefix}/{key}/discard", headers=headers)
).status_code == 401
await c.execute("SELECT * FROM kf_manage_account_authority('web_admin','grant')")
headers = await login("web_admin")
stop("worker")
discard_after = await c.fetchval("SELECT clock_timestamp()")
page = (await client.get(prefix, params={"limit": 100})).json()
for entry in page["items"]:
assert (
await client.post(
f"{prefix}/{entry['operation_id']}/discard", headers=headers
)
).status_code == 200
await restart_ready_worker()
deadline = time.monotonic() + 30
while time.monotonic() < deadline and await c.fetchval(
"SELECT count(*) FROM problem_candidates"
):
await asyncio.sleep(0.1)
assert await c.fetchval("SELECT count(*) FROM problem_candidates") == 0
assert await c.fetchval("SELECT count(*) FROM problem_candidate_cases") == 0
assert (
await c.fetchval("SELECT reserved_bytes FROM problem_import_pool WHERE id=1")
== 0
)
assert await c.fetchval("SELECT count(*) FROM problem_maintenance_events") >= 10
assert (
await c.fetchval(
"SELECT bool_and(e.authorization_version=a.authorization_version) "
"FROM problem_maintenance_events e JOIN accounts a ON a.id=e.actor_id "
"WHERE e.action='import_discarded' AND e.actor_id=$1 "
"AND e.created_at >= $2",
users["web_admin"],
discard_after,
)
is True
)
for table in (
"evaluations",
"judge_attempts",
"evaluation_performance",
"accounts",
"sessions",
):
for privilege in ("SELECT", "INSERT", "UPDATE", "DELETE"):
assert not await c.fetchval(
"SELECT has_table_privilege($1,$2,$3)",
roles["worker"],
table,
privilege,
)
assert not await c.fetchval(
"SELECT has_table_privilege($1,'problem_maintenance_events','UPDATE')",
roles["api"],
)
for privilege in ("SELECT", "UPDATE", "DELETE"):
assert not await c.fetchval(
"SELECT has_table_privilege($1,'problem_maintenance_events',$2)",
roles["worker"],
privilege,
)
# Missing configuration is observed through the real server, never preview data.
stop("api")
start("api", configured=False)
deadline = time.monotonic() + 120
while time.monotonic() < deadline:
try:
r = await client.get(prefix + "/capabilities")
if r.status_code == 200:
break
except httpx.ConnectError:
pass
await asyncio.sleep(0.1)
assert r.status_code == 200 and not r.json()["uploads_available"]
assert not r.json()["validation_available"]
assert (
await client.post(
prefix,
json={
"operation_id": str(uuid4()),
"archive_digest": hashlib.sha256(blob).hexdigest(),
"archive_bytes": len(blob),
},
headers=headers,
)
).status_code == 503
print("ACTUAL_IMPORT_HTTP_WORKER_ACCEPTED", flush=True)
finally:
stop("api")
stop("worker")
for role in roles.values():
await c.execute(f"DROP OWNED BY {role}")
await c.execute(f"DROP ROLE {role}")
await c.close()
asyncio.run(run())
+9 -1
View File
@@ -1,5 +1,6 @@
import asyncio
import json
import subprocess
from uuid import uuid4
import asyncpg
@@ -12,7 +13,7 @@ def test_migration_roundtrip_and_role_safety(postgres_url):
c = await connect(postgres_url)
try:
tables = set(await c.fetch("SELECT tablename FROM pg_tables WHERE schemaname='public'"))
assert len(tables) == 15 # Fourteen domain/control tables and Alembic history.
assert len(tables) == 20 # Nineteen domain/control tables and Alembic history.
roles = await c.fetch("SELECT * FROM pg_roles WHERE rolname LIKE 'kf_oj_%'")
assert {row["rolname"] for row in roles} == {
"kf_oj_platform",
@@ -44,7 +45,14 @@ def test_migration_roundtrip_and_role_safety(postgres_url):
asyncio.run(inspect())
migrate(postgres_url, "downgrade", "base")
migrate(postgres_url, "upgrade", "head")
try:
migrate(postgres_url, "check")
except subprocess.CalledProcessError as error:
raise AssertionError(
"migration model consistency failed: "
+ (error.stdout or b"").decode(errors="replace")[-8192:]
+ (error.stderr or b"").decode(errors="replace")[-8192:]
) from None
asyncio.run(inspect())
+862
View File
@@ -76,6 +76,94 @@
"title": "CancellationAccepted",
"type": "object"
},
"CandidatePreview": {
"additionalProperties": false,
"properties": {
"candidate_id": {
"format": "uuid",
"title": "Candidate Id",
"type": "string"
},
"case_count": {
"maximum": 256.0,
"minimum": 1.0,
"title": "Case Count",
"type": "integer"
},
"judging_schema_version": {
"maximum": 2.0,
"minimum": 1.0,
"title": "Judging Schema Version",
"type": "integer"
},
"language_configurations": {
"default": [],
"items": {
"$ref": "#/components/schemas/PublicLanguageConfiguration"
},
"maxItems": 4,
"title": "Language Configurations",
"type": "array"
},
"languages": {
"items": {
"$ref": "#/components/schemas/kf_oj__contracts__judging_v2__Language"
},
"maxItems": 4,
"minItems": 1,
"title": "Languages",
"type": "array"
},
"legacy_budget": {
"anyOf": [
{
"$ref": "#/components/schemas/LegacyImportBudget"
},
{
"type": "null"
}
]
},
"samples": {
"items": {
"$ref": "#/components/schemas/PublicSample"
},
"maxItems": 10,
"title": "Samples",
"type": "array"
},
"slug": {
"pattern": "^[a-z0-9][a-z0-9_-]{0,63}$",
"title": "Slug",
"type": "string"
},
"statement": {
"maxLength": 262144,
"minLength": 1,
"title": "Statement",
"type": "string"
},
"title": {
"maxLength": 160,
"minLength": 1,
"title": "Title",
"type": "string"
}
},
"required": [
"candidate_id",
"slug",
"title",
"statement",
"samples",
"judging_schema_version",
"languages",
"legacy_budget",
"case_count"
],
"title": "CandidatePreview",
"type": "object"
},
"CompilationStatus": {
"enum": [
"succeeded",
@@ -87,6 +175,42 @@
"title": "CompilationStatus",
"type": "string"
},
"CreateProblemImport": {
"additionalProperties": false,
"properties": {
"archive_bytes": {
"maximum": 67108864.0,
"minimum": 1.0,
"title": "Archive Bytes",
"type": "integer"
},
"archive_digest": {
"maxLength": 64,
"minLength": 64,
"pattern": "^[0-9a-f]{64}$",
"title": "Archive Digest",
"type": "string"
},
"operation_id": {
"format": "uuid",
"title": "Operation Id",
"type": "string"
},
"schema_version": {
"const": 1,
"default": 1,
"title": "Schema Version",
"type": "integer"
}
},
"required": [
"operation_id",
"archive_digest",
"archive_bytes"
],
"title": "CreateProblemImport",
"type": "object"
},
"CreateSubmission": {
"additionalProperties": false,
"properties": {
@@ -192,6 +316,38 @@
"title": "HealthResponse",
"type": "object"
},
"ImportFailure": {
"enum": [
"invalid_archive",
"invalid_manifest",
"invalid_data",
"digest_mismatch",
"policy_rejected",
"upload_interrupted",
"upload_deadline",
"validation_deadline",
"validation_unavailable",
"storage_unavailable",
"authority_revoked",
"expired"
],
"title": "ImportFailure",
"type": "string"
},
"ImportState": {
"enum": [
"uploading",
"uploaded",
"validating",
"ready",
"failed",
"published",
"discarded",
"expired"
],
"title": "ImportState",
"type": "string"
},
"JobManifest": {
"additionalProperties": false,
"properties": {
@@ -284,6 +440,30 @@
"title": "Language",
"type": "string"
},
"LegacyImportBudget": {
"additionalProperties": false,
"properties": {
"compile_limits": {
"$ref": "#/components/schemas/RunLimits"
},
"job_wall_time_us": {
"maximum": 3600000000.0,
"minimum": 1000.0,
"title": "Job Wall Time Us",
"type": "integer"
},
"run_limits": {
"$ref": "#/components/schemas/RunLimits"
}
},
"required": [
"compile_limits",
"run_limits",
"job_wall_time_us"
],
"title": "LegacyImportBudget",
"type": "object"
},
"LoginRequest": {
"additionalProperties": false,
"properties": {
@@ -575,6 +755,426 @@
"title": "ProblemDetail",
"type": "object"
},
"ProblemImportCapabilities": {
"additionalProperties": false,
"properties": {
"limits": {
"$ref": "#/components/schemas/ProblemImportLimits",
"default": {
"archive_bytes": 67108864,
"archive_format": "single_level_zip",
"candidate_lifetime_seconds": 86400,
"chunk_bytes": 1048576,
"filename_ascii_bytes": 128,
"files": 534,
"global_concurrency": 1,
"languages": [
"c17",
"cpp20",
"python313",
"java21"
],
"manifest_name": "problem.json",
"manifest_schema_versions": [
1,
2
],
"owner_candidates": 10,
"owner_concurrency": 1,
"schema_version": 1,
"staging_bytes": 268435456,
"total_candidates": 32,
"unpacked_bytes": 67108864,
"upload_chunks": 4096,
"upload_deadline_seconds": 30,
"validation_deadline_seconds": 60
}
},
"schema_version": {
"const": 1,
"default": 1,
"title": "Schema Version",
"type": "integer"
},
"uploads_available": {
"title": "Uploads Available",
"type": "boolean"
},
"validation_available": {
"title": "Validation Available",
"type": "boolean"
}
},
"required": [
"uploads_available",
"validation_available"
],
"title": "ProblemImportCapabilities",
"type": "object"
},
"ProblemImportDetail": {
"additionalProperties": false,
"properties": {
"archive_bytes": {
"maximum": 67108864.0,
"minimum": 1.0,
"title": "Archive Bytes",
"type": "integer"
},
"candidate": {
"anyOf": [
{
"$ref": "#/components/schemas/CandidatePreview"
},
{
"type": "null"
}
]
},
"candidate_id": {
"anyOf": [
{
"format": "uuid",
"type": "string"
},
{
"type": "null"
}
],
"title": "Candidate Id"
},
"created_at": {
"format": "date-time",
"title": "Created At",
"type": "string"
},
"expected_archive_digest": {
"maxLength": 64,
"minLength": 64,
"pattern": "^[0-9a-f]{64}$",
"title": "Expected Archive Digest",
"type": "string"
},
"expires_at": {
"format": "date-time",
"title": "Expires At",
"type": "string"
},
"failure_code": {
"anyOf": [
{
"$ref": "#/components/schemas/ImportFailure"
},
{
"type": "null"
}
]
},
"operation_id": {
"format": "uuid",
"title": "Operation Id",
"type": "string"
},
"published_version_id": {
"anyOf": [
{
"format": "uuid",
"type": "string"
},
{
"type": "null"
}
],
"title": "Published Version Id"
},
"schema_version": {
"const": 1,
"default": 1,
"title": "Schema Version",
"type": "integer"
},
"state": {
"$ref": "#/components/schemas/ImportState"
},
"updated_at": {
"format": "date-time",
"title": "Updated At",
"type": "string"
}
},
"required": [
"operation_id",
"state",
"expected_archive_digest",
"archive_bytes",
"created_at",
"expires_at",
"updated_at",
"failure_code",
"candidate_id",
"published_version_id",
"candidate"
],
"title": "ProblemImportDetail",
"type": "object"
},
"ProblemImportLimits": {
"additionalProperties": false,
"properties": {
"archive_bytes": {
"default": 67108864,
"title": "Archive Bytes",
"type": "integer"
},
"archive_format": {
"const": "single_level_zip",
"default": "single_level_zip",
"title": "Archive Format",
"type": "string"
},
"candidate_lifetime_seconds": {
"default": 86400,
"title": "Candidate Lifetime Seconds",
"type": "integer"
},
"chunk_bytes": {
"default": 1048576,
"title": "Chunk Bytes",
"type": "integer"
},
"filename_ascii_bytes": {
"default": 128,
"title": "Filename Ascii Bytes",
"type": "integer"
},
"files": {
"default": 534,
"title": "Files",
"type": "integer"
},
"global_concurrency": {
"const": 1,
"default": 1,
"title": "Global Concurrency",
"type": "integer"
},
"languages": {
"default": [
"c17",
"cpp20",
"python313",
"java21"
],
"items": {
"$ref": "#/components/schemas/kf_oj__contracts__judging_v2__Language"
},
"title": "Languages",
"type": "array"
},
"manifest_name": {
"const": "problem.json",
"default": "problem.json",
"title": "Manifest Name",
"type": "string"
},
"manifest_schema_versions": {
"default": [
1,
2
],
"items": {
"enum": [
1,
2
],
"type": "integer"
},
"title": "Manifest Schema Versions",
"type": "array"
},
"owner_candidates": {
"default": 10,
"title": "Owner Candidates",
"type": "integer"
},
"owner_concurrency": {
"const": 1,
"default": 1,
"title": "Owner Concurrency",
"type": "integer"
},
"schema_version": {
"const": 1,
"default": 1,
"title": "Schema Version",
"type": "integer"
},
"staging_bytes": {
"default": 268435456,
"title": "Staging Bytes",
"type": "integer"
},
"total_candidates": {
"default": 32,
"title": "Total Candidates",
"type": "integer"
},
"unpacked_bytes": {
"default": 67108864,
"title": "Unpacked Bytes",
"type": "integer"
},
"upload_chunks": {
"default": 4096,
"title": "Upload Chunks",
"type": "integer"
},
"upload_deadline_seconds": {
"default": 30,
"title": "Upload Deadline Seconds",
"type": "integer"
},
"validation_deadline_seconds": {
"default": 60,
"title": "Validation Deadline Seconds",
"type": "integer"
}
},
"title": "ProblemImportLimits",
"type": "object"
},
"ProblemImportPage": {
"additionalProperties": false,
"properties": {
"items": {
"items": {
"$ref": "#/components/schemas/ProblemImportSummary"
},
"maxItems": 100,
"title": "Items",
"type": "array"
},
"next_after": {
"anyOf": [
{
"format": "uuid",
"type": "string"
},
{
"type": "null"
}
],
"title": "Next After"
},
"schema_version": {
"const": 1,
"default": 1,
"title": "Schema Version",
"type": "integer"
}
},
"required": [
"items",
"next_after"
],
"title": "ProblemImportPage",
"type": "object"
},
"ProblemImportSummary": {
"additionalProperties": false,
"properties": {
"archive_bytes": {
"maximum": 67108864.0,
"minimum": 1.0,
"title": "Archive Bytes",
"type": "integer"
},
"candidate_id": {
"anyOf": [
{
"format": "uuid",
"type": "string"
},
{
"type": "null"
}
],
"title": "Candidate Id"
},
"created_at": {
"format": "date-time",
"title": "Created At",
"type": "string"
},
"expected_archive_digest": {
"maxLength": 64,
"minLength": 64,
"pattern": "^[0-9a-f]{64}$",
"title": "Expected Archive Digest",
"type": "string"
},
"expires_at": {
"format": "date-time",
"title": "Expires At",
"type": "string"
},
"failure_code": {
"anyOf": [
{
"$ref": "#/components/schemas/ImportFailure"
},
{
"type": "null"
}
]
},
"operation_id": {
"format": "uuid",
"title": "Operation Id",
"type": "string"
},
"published_version_id": {
"anyOf": [
{
"format": "uuid",
"type": "string"
},
{
"type": "null"
}
],
"title": "Published Version Id"
},
"schema_version": {
"const": 1,
"default": 1,
"title": "Schema Version",
"type": "integer"
},
"state": {
"$ref": "#/components/schemas/ImportState"
},
"updated_at": {
"format": "date-time",
"title": "Updated At",
"type": "string"
}
},
"required": [
"operation_id",
"state",
"expected_archive_digest",
"archive_bytes",
"created_at",
"expires_at",
"updated_at",
"failure_code",
"candidate_id",
"published_version_id"
],
"title": "ProblemImportSummary",
"type": "object"
},
"ProblemPage": {
"additionalProperties": false,
"properties": {
@@ -1416,6 +2016,268 @@
},
"openapi": "3.1.0",
"paths": {
"/api/v1/admin/problem-imports": {
"get": {
"operationId": "page_api_v1_admin_problem_imports_get",
"parameters": [
{
"in": "query",
"name": "after",
"required": false,
"schema": {
"anyOf": [
{
"format": "uuid",
"type": "string"
},
{
"type": "null"
}
],
"title": "After"
}
},
{
"in": "query",
"name": "limit",
"required": false,
"schema": {
"default": 20,
"maximum": 100,
"minimum": 1,
"title": "Limit",
"type": "integer"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemImportPage"
}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"summary": "Page",
"tags": [
"problem-maintenance"
]
},
"post": {
"operationId": "create_api_v1_admin_problem_imports_post",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/CreateProblemImport"
}
}
},
"required": true
},
"responses": {
"201": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemImportSummary"
}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"summary": "Create",
"tags": [
"problem-maintenance"
]
}
},
"/api/v1/admin/problem-imports/capabilities": {
"get": {
"operationId": "capabilities_api_v1_admin_problem_imports_capabilities_get",
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemImportCapabilities"
}
}
},
"description": "Successful Response"
}
},
"summary": "Capabilities",
"tags": [
"problem-maintenance"
]
}
},
"/api/v1/admin/problem-imports/{operation_id}": {
"get": {
"operationId": "detail_api_v1_admin_problem_imports__operation_id__get",
"parameters": [
{
"in": "path",
"name": "operation_id",
"required": true,
"schema": {
"format": "uuid",
"title": "Operation Id",
"type": "string"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemImportDetail"
}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"summary": "Detail",
"tags": [
"problem-maintenance"
]
}
},
"/api/v1/admin/problem-imports/{operation_id}/archive": {
"put": {
"operationId": "upload_api_v1_admin_problem_imports__operation_id__archive_put",
"parameters": [
{
"in": "path",
"name": "operation_id",
"required": true,
"schema": {
"format": "uuid",
"title": "Operation Id",
"type": "string"
}
}
],
"requestBody": {
"content": {
"application/zip": {
"schema": {
"format": "binary",
"type": "string"
}
}
},
"required": true
},
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemImportSummary"
}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"summary": "Upload",
"tags": [
"problem-maintenance"
]
}
},
"/api/v1/admin/problem-imports/{operation_id}/discard": {
"post": {
"operationId": "discard_api_v1_admin_problem_imports__operation_id__discard_post",
"parameters": [
{
"in": "path",
"name": "operation_id",
"required": true,
"schema": {
"format": "uuid",
"title": "Operation Id",
"type": "string"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemImportSummary"
}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"summary": "Discard",
"tags": [
"problem-maintenance"
]
}
},
"/api/v1/auth/login": {
"post": {
"operationId": "login_api_v1_auth_login_post",
@@ -0,0 +1,196 @@
{
"$defs": {
"Language": {
"enum": [
"c17",
"cpp20",
"python313",
"java21"
],
"title": "Language",
"type": "string"
},
"ProblemImportLimits": {
"additionalProperties": false,
"properties": {
"archive_bytes": {
"default": 67108864,
"title": "Archive Bytes",
"type": "integer"
},
"archive_format": {
"const": "single_level_zip",
"default": "single_level_zip",
"title": "Archive Format",
"type": "string"
},
"candidate_lifetime_seconds": {
"default": 86400,
"title": "Candidate Lifetime Seconds",
"type": "integer"
},
"chunk_bytes": {
"default": 1048576,
"title": "Chunk Bytes",
"type": "integer"
},
"filename_ascii_bytes": {
"default": 128,
"title": "Filename Ascii Bytes",
"type": "integer"
},
"files": {
"default": 534,
"title": "Files",
"type": "integer"
},
"global_concurrency": {
"const": 1,
"default": 1,
"title": "Global Concurrency",
"type": "integer"
},
"languages": {
"default": [
"c17",
"cpp20",
"python313",
"java21"
],
"items": {
"$ref": "#/$defs/Language"
},
"title": "Languages",
"type": "array"
},
"manifest_name": {
"const": "problem.json",
"default": "problem.json",
"title": "Manifest Name",
"type": "string"
},
"manifest_schema_versions": {
"default": [
1,
2
],
"items": {
"enum": [
1,
2
],
"type": "integer"
},
"title": "Manifest Schema Versions",
"type": "array"
},
"owner_candidates": {
"default": 10,
"title": "Owner Candidates",
"type": "integer"
},
"owner_concurrency": {
"const": 1,
"default": 1,
"title": "Owner Concurrency",
"type": "integer"
},
"schema_version": {
"const": 1,
"default": 1,
"title": "Schema Version",
"type": "integer"
},
"staging_bytes": {
"default": 268435456,
"title": "Staging Bytes",
"type": "integer"
},
"total_candidates": {
"default": 32,
"title": "Total Candidates",
"type": "integer"
},
"unpacked_bytes": {
"default": 67108864,
"title": "Unpacked Bytes",
"type": "integer"
},
"upload_chunks": {
"default": 4096,
"title": "Upload Chunks",
"type": "integer"
},
"upload_deadline_seconds": {
"default": 30,
"title": "Upload Deadline Seconds",
"type": "integer"
},
"validation_deadline_seconds": {
"default": 60,
"title": "Validation Deadline Seconds",
"type": "integer"
}
},
"title": "ProblemImportLimits",
"type": "object"
}
},
"$id": "urn:kf-oj:problem-import-capabilities:v1",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"additionalProperties": false,
"properties": {
"limits": {
"$ref": "#/$defs/ProblemImportLimits",
"default": {
"archive_bytes": 67108864,
"archive_format": "single_level_zip",
"candidate_lifetime_seconds": 86400,
"chunk_bytes": 1048576,
"filename_ascii_bytes": 128,
"files": 534,
"global_concurrency": 1,
"languages": [
"c17",
"cpp20",
"python313",
"java21"
],
"manifest_name": "problem.json",
"manifest_schema_versions": [
1,
2
],
"owner_candidates": 10,
"owner_concurrency": 1,
"schema_version": 1,
"staging_bytes": 268435456,
"total_candidates": 32,
"unpacked_bytes": 67108864,
"upload_chunks": 4096,
"upload_deadline_seconds": 30,
"validation_deadline_seconds": 60
}
},
"schema_version": {
"const": 1,
"default": 1,
"title": "Schema Version",
"type": "integer"
},
"uploads_available": {
"title": "Uploads Available",
"type": "boolean"
},
"validation_available": {
"title": "Validation Available",
"type": "boolean"
}
},
"required": [
"uploads_available",
"validation_available"
],
"title": "ProblemImportCapabilities",
"type": "object"
}
@@ -0,0 +1,38 @@
{
"$id": "urn:kf-oj:problem-import-create:v1",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"additionalProperties": false,
"properties": {
"archive_bytes": {
"maximum": 67108864,
"minimum": 1,
"title": "Archive Bytes",
"type": "integer"
},
"archive_digest": {
"maxLength": 64,
"minLength": 64,
"pattern": "^[0-9a-f]{64}$",
"title": "Archive Digest",
"type": "string"
},
"operation_id": {
"format": "uuid",
"title": "Operation Id",
"type": "string"
},
"schema_version": {
"const": 1,
"default": 1,
"title": "Schema Version",
"type": "integer"
}
},
"required": [
"operation_id",
"archive_digest",
"archive_bytes"
],
"title": "CreateProblemImport",
"type": "object"
}
@@ -0,0 +1,423 @@
{
"$defs": {
"CandidatePreview": {
"additionalProperties": false,
"properties": {
"candidate_id": {
"format": "uuid",
"title": "Candidate Id",
"type": "string"
},
"case_count": {
"maximum": 256,
"minimum": 1,
"title": "Case Count",
"type": "integer"
},
"judging_schema_version": {
"maximum": 2,
"minimum": 1,
"title": "Judging Schema Version",
"type": "integer"
},
"language_configurations": {
"default": [],
"items": {
"$ref": "#/$defs/PublicLanguageConfiguration"
},
"maxItems": 4,
"title": "Language Configurations",
"type": "array"
},
"languages": {
"items": {
"$ref": "#/$defs/Language"
},
"maxItems": 4,
"minItems": 1,
"title": "Languages",
"type": "array"
},
"legacy_budget": {
"anyOf": [
{
"$ref": "#/$defs/LegacyImportBudget"
},
{
"type": "null"
}
]
},
"samples": {
"items": {
"$ref": "#/$defs/PublicSample"
},
"maxItems": 10,
"title": "Samples",
"type": "array"
},
"slug": {
"pattern": "^[a-z0-9][a-z0-9_-]{0,63}$",
"title": "Slug",
"type": "string"
},
"statement": {
"maxLength": 262144,
"minLength": 1,
"title": "Statement",
"type": "string"
},
"title": {
"maxLength": 160,
"minLength": 1,
"title": "Title",
"type": "string"
}
},
"required": [
"candidate_id",
"slug",
"title",
"statement",
"samples",
"judging_schema_version",
"languages",
"legacy_budget",
"case_count"
],
"title": "CandidatePreview",
"type": "object"
},
"ImportFailure": {
"enum": [
"invalid_archive",
"invalid_manifest",
"invalid_data",
"digest_mismatch",
"policy_rejected",
"upload_interrupted",
"upload_deadline",
"validation_deadline",
"validation_unavailable",
"storage_unavailable",
"authority_revoked",
"expired"
],
"title": "ImportFailure",
"type": "string"
},
"ImportState": {
"enum": [
"uploading",
"uploaded",
"validating",
"ready",
"failed",
"published",
"discarded",
"expired"
],
"title": "ImportState",
"type": "string"
},
"Language": {
"enum": [
"c17",
"cpp20",
"python313",
"java21"
],
"title": "Language",
"type": "string"
},
"LegacyImportBudget": {
"additionalProperties": false,
"properties": {
"compile_limits": {
"$ref": "#/$defs/RunLimits"
},
"job_wall_time_us": {
"maximum": 3600000000,
"minimum": 1000,
"title": "Job Wall Time Us",
"type": "integer"
},
"run_limits": {
"$ref": "#/$defs/RunLimits"
}
},
"required": [
"compile_limits",
"run_limits",
"job_wall_time_us"
],
"title": "LegacyImportBudget",
"type": "object"
},
"PreparationKind": {
"enum": [
"compile",
"syntax_check"
],
"title": "PreparationKind",
"type": "string"
},
"PublicLanguageConfiguration": {
"additionalProperties": false,
"properties": {
"compile_limits": {
"$ref": "#/$defs/RunLimits"
},
"entrypoint": {
"title": "Entrypoint",
"type": "string"
},
"job_wall_time_us": {
"title": "Job Wall Time Us",
"type": "integer"
},
"language": {
"$ref": "#/$defs/Language"
},
"preparation_kind": {
"$ref": "#/$defs/PreparationKind"
},
"run_limits": {
"$ref": "#/$defs/RunLimits"
},
"runtime_version": {
"title": "Runtime Version",
"type": "string"
},
"source_name": {
"title": "Source Name",
"type": "string"
}
},
"required": [
"language",
"runtime_version",
"source_name",
"entrypoint",
"preparation_kind",
"compile_limits",
"run_limits",
"job_wall_time_us"
],
"title": "PublicLanguageConfiguration",
"type": "object"
},
"PublicSample": {
"additionalProperties": false,
"properties": {
"encoding": {
"const": "base64",
"default": "base64",
"title": "Encoding",
"type": "string"
},
"input": {
"maxLength": 87384,
"title": "Input",
"type": "string"
},
"output": {
"maxLength": 87384,
"title": "Output",
"type": "string"
}
},
"required": [
"input",
"output"
],
"title": "PublicSample",
"type": "object"
},
"RunLimits": {
"additionalProperties": false,
"properties": {
"cpu_time_us": {
"maximum": 60000000,
"minimum": 1000,
"title": "Cpu Time Us",
"type": "integer"
},
"file_bytes": {
"maximum": 67108864,
"minimum": 1,
"title": "File Bytes",
"type": "integer"
},
"memory_bytes": {
"maximum": 2147483648,
"minimum": 1,
"title": "Memory Bytes",
"type": "integer"
},
"open_files": {
"maximum": 1024,
"minimum": 3,
"title": "Open Files",
"type": "integer"
},
"processes": {
"maximum": 128,
"minimum": 1,
"title": "Processes",
"type": "integer"
},
"stack_bytes": {
"maximum": 268435456,
"minimum": 1,
"title": "Stack Bytes",
"type": "integer"
},
"stderr_bytes": {
"maximum": 16777216,
"minimum": 1,
"title": "Stderr Bytes",
"type": "integer"
},
"stdout_bytes": {
"maximum": 16777216,
"minimum": 1,
"title": "Stdout Bytes",
"type": "integer"
},
"wall_time_us": {
"maximum": 120000000,
"minimum": 1000,
"title": "Wall Time Us",
"type": "integer"
},
"workdir_bytes": {
"maximum": 268435456,
"minimum": 1,
"title": "Workdir Bytes",
"type": "integer"
}
},
"required": [
"cpu_time_us",
"wall_time_us",
"memory_bytes",
"stack_bytes",
"processes",
"open_files",
"stdout_bytes",
"stderr_bytes",
"file_bytes",
"workdir_bytes"
],
"title": "RunLimits",
"type": "object"
}
},
"$id": "urn:kf-oj:problem-import:v1",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"additionalProperties": false,
"properties": {
"archive_bytes": {
"maximum": 67108864,
"minimum": 1,
"title": "Archive Bytes",
"type": "integer"
},
"candidate": {
"anyOf": [
{
"$ref": "#/$defs/CandidatePreview"
},
{
"type": "null"
}
]
},
"candidate_id": {
"anyOf": [
{
"format": "uuid",
"type": "string"
},
{
"type": "null"
}
],
"title": "Candidate Id"
},
"created_at": {
"format": "date-time",
"title": "Created At",
"type": "string"
},
"expected_archive_digest": {
"maxLength": 64,
"minLength": 64,
"pattern": "^[0-9a-f]{64}$",
"title": "Expected Archive Digest",
"type": "string"
},
"expires_at": {
"format": "date-time",
"title": "Expires At",
"type": "string"
},
"failure_code": {
"anyOf": [
{
"$ref": "#/$defs/ImportFailure"
},
{
"type": "null"
}
]
},
"operation_id": {
"format": "uuid",
"title": "Operation Id",
"type": "string"
},
"published_version_id": {
"anyOf": [
{
"format": "uuid",
"type": "string"
},
{
"type": "null"
}
],
"title": "Published Version Id"
},
"schema_version": {
"const": 1,
"default": 1,
"title": "Schema Version",
"type": "integer"
},
"state": {
"$ref": "#/$defs/ImportState"
},
"updated_at": {
"format": "date-time",
"title": "Updated At",
"type": "string"
}
},
"required": [
"operation_id",
"state",
"expected_archive_digest",
"archive_bytes",
"created_at",
"expires_at",
"updated_at",
"failure_code",
"candidate_id",
"published_version_id",
"candidate"
],
"title": "ProblemImportDetail",
"type": "object"
}
@@ -0,0 +1,32 @@
[Unit]
Description=KF-OnlineJudge bounded problem import service
After=postgresql.service
[Service]
Type=exec
User=kf-oj
Group=kf-oj
WorkingDirectory=/opt/kf-onlinejudge/backend
EnvironmentFile=/etc/kf-onlinejudge/import-worker.env
Environment=KF_OJ_IMPORT_WORKER_CGROUP=/sys/fs/cgroup/system.slice/kf-problem-import.service
ExecStart=/opt/kf-onlinejudge/backend/.venv/bin/kf-problem-import-worker
MemoryMax=128M
MemorySwapMax=0
TasksMax=16
CPUQuota=100%
LimitFSIZE=64M
LimitNOFILE=64
LimitCORE=0
NoNewPrivileges=yes
PrivateNetwork=yes
RestrictAddressFamilies=AF_UNIX
ProtectSystem=strict
ProtectHome=yes
ReadWritePaths=/var/lib/kf-onlinejudge/imports /var/lib/kf-onlinejudge/artifacts
KillMode=control-group
TimeoutStopSec=10
Restart=on-failure
RestartSec=30
[Install]
WantedBy=multi-user.target
+579
View File
@@ -4,6 +4,92 @@
*/
export interface paths {
"/api/v1/admin/problem-imports": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/** Page */
get: operations["page_api_v1_admin_problem_imports_get"];
put?: never;
/** Create */
post: operations["create_api_v1_admin_problem_imports_post"];
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/api/v1/admin/problem-imports/capabilities": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/** Capabilities */
get: operations["capabilities_api_v1_admin_problem_imports_capabilities_get"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/api/v1/admin/problem-imports/{operation_id}": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/** Detail */
get: operations["detail_api_v1_admin_problem_imports__operation_id__get"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/api/v1/admin/problem-imports/{operation_id}/archive": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
get?: never;
/** Upload */
put: operations["upload_api_v1_admin_problem_imports__operation_id__archive_put"];
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/api/v1/admin/problem-imports/{operation_id}/discard": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
get?: never;
put?: never;
/** Discard */
post: operations["discard_api_v1_admin_problem_imports__operation_id__discard_post"];
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/api/v1/auth/login": {
parameters: {
query?: never;
@@ -278,11 +364,57 @@ export interface components {
*/
submission_id: string;
};
/** CandidatePreview */
CandidatePreview: {
/**
* Candidate Id
* Format: uuid
*/
candidate_id: string;
/** Case Count */
case_count: number;
/** Judging Schema Version */
judging_schema_version: number;
/**
* Language Configurations
* @default []
*/
language_configurations: components["schemas"]["PublicLanguageConfiguration"][];
/** Languages */
languages: components["schemas"]["kf_oj__contracts__judging_v2__Language"][];
legacy_budget: components["schemas"]["LegacyImportBudget"] | null;
/** Samples */
samples: components["schemas"]["PublicSample"][];
/** Slug */
slug: string;
/** Statement */
statement: string;
/** Title */
title: string;
};
/**
* CompilationStatus
* @enum {string}
*/
CompilationStatus: "succeeded" | "compilation_error" | "compilation_limit" | "cancelled" | "system_error";
/** CreateProblemImport */
CreateProblemImport: {
/** Archive Bytes */
archive_bytes: number;
/** Archive Digest */
archive_digest: string;
/**
* Operation Id
* Format: uuid
*/
operation_id: string;
/**
* Schema Version
* @default 1
* @constant
*/
schema_version: 1;
};
/** CreateSubmission */
CreateSubmission: {
/**
@@ -335,6 +467,16 @@ export interface components {
/** Version */
version: string;
};
/**
* ImportFailure
* @enum {string}
*/
ImportFailure: "invalid_archive" | "invalid_manifest" | "invalid_data" | "digest_mismatch" | "policy_rejected" | "upload_interrupted" | "upload_deadline" | "validation_deadline" | "validation_unavailable" | "storage_unavailable" | "authority_revoked" | "expired";
/**
* ImportState
* @enum {string}
*/
ImportState: "uploading" | "uploaded" | "validating" | "ready" | "failed" | "published" | "discarded" | "expired";
/** JobManifest */
JobManifest: {
/** Cases */
@@ -382,6 +524,13 @@ export interface components {
* @enum {string}
*/
"Language-Input": "c17" | "cpp20" | "python313" | "java21";
/** LegacyImportBudget */
LegacyImportBudget: {
compile_limits: components["schemas"]["RunLimits"];
/** Job Wall Time Us */
job_wall_time_us: number;
run_limits: components["schemas"]["RunLimits"];
};
/** LoginRequest */
LoginRequest: {
/**
@@ -476,6 +625,254 @@ export interface components {
*/
version_id: string;
};
/** ProblemImportCapabilities */
ProblemImportCapabilities: {
/**
* @default {
* "archive_bytes": 67108864,
* "archive_format": "single_level_zip",
* "candidate_lifetime_seconds": 86400,
* "chunk_bytes": 1048576,
* "filename_ascii_bytes": 128,
* "files": 534,
* "global_concurrency": 1,
* "languages": [
* "c17",
* "cpp20",
* "python313",
* "java21"
* ],
* "manifest_name": "problem.json",
* "manifest_schema_versions": [
* 1,
* 2
* ],
* "owner_candidates": 10,
* "owner_concurrency": 1,
* "schema_version": 1,
* "staging_bytes": 268435456,
* "total_candidates": 32,
* "unpacked_bytes": 67108864,
* "upload_chunks": 4096,
* "upload_deadline_seconds": 30,
* "validation_deadline_seconds": 60
* }
*/
limits: components["schemas"]["ProblemImportLimits"];
/**
* Schema Version
* @default 1
* @constant
*/
schema_version: 1;
/** Uploads Available */
uploads_available: boolean;
/** Validation Available */
validation_available: boolean;
};
/** ProblemImportDetail */
ProblemImportDetail: {
/** Archive Bytes */
archive_bytes: number;
candidate: components["schemas"]["CandidatePreview"] | null;
/** Candidate Id */
candidate_id: string | null;
/**
* Created At
* Format: date-time
*/
created_at: string;
/** Expected Archive Digest */
expected_archive_digest: string;
/**
* Expires At
* Format: date-time
*/
expires_at: string;
failure_code: components["schemas"]["ImportFailure"] | null;
/**
* Operation Id
* Format: uuid
*/
operation_id: string;
/** Published Version Id */
published_version_id: string | null;
/**
* Schema Version
* @default 1
* @constant
*/
schema_version: 1;
state: components["schemas"]["ImportState"];
/**
* Updated At
* Format: date-time
*/
updated_at: string;
};
/** ProblemImportLimits */
ProblemImportLimits: {
/**
* Archive Bytes
* @default 67108864
*/
archive_bytes: number;
/**
* Archive Format
* @default single_level_zip
* @constant
*/
archive_format: "single_level_zip";
/**
* Candidate Lifetime Seconds
* @default 86400
*/
candidate_lifetime_seconds: number;
/**
* Chunk Bytes
* @default 1048576
*/
chunk_bytes: number;
/**
* Filename Ascii Bytes
* @default 128
*/
filename_ascii_bytes: number;
/**
* Files
* @default 534
*/
files: number;
/**
* Global Concurrency
* @default 1
* @constant
*/
global_concurrency: 1;
/**
* Languages
* @default [
* "c17",
* "cpp20",
* "python313",
* "java21"
* ]
*/
languages: components["schemas"]["kf_oj__contracts__judging_v2__Language"][];
/**
* Manifest Name
* @default problem.json
* @constant
*/
manifest_name: "problem.json";
/**
* Manifest Schema Versions
* @default [
* 1,
* 2
* ]
*/
manifest_schema_versions: (1 | 2)[];
/**
* Owner Candidates
* @default 10
*/
owner_candidates: number;
/**
* Owner Concurrency
* @default 1
* @constant
*/
owner_concurrency: 1;
/**
* Schema Version
* @default 1
* @constant
*/
schema_version: 1;
/**
* Staging Bytes
* @default 268435456
*/
staging_bytes: number;
/**
* Total Candidates
* @default 32
*/
total_candidates: number;
/**
* Unpacked Bytes
* @default 67108864
*/
unpacked_bytes: number;
/**
* Upload Chunks
* @default 4096
*/
upload_chunks: number;
/**
* Upload Deadline Seconds
* @default 30
*/
upload_deadline_seconds: number;
/**
* Validation Deadline Seconds
* @default 60
*/
validation_deadline_seconds: number;
};
/** ProblemImportPage */
ProblemImportPage: {
/** Items */
items: components["schemas"]["ProblemImportSummary"][];
/** Next After */
next_after: string | null;
/**
* Schema Version
* @default 1
* @constant
*/
schema_version: 1;
};
/** ProblemImportSummary */
ProblemImportSummary: {
/** Archive Bytes */
archive_bytes: number;
/** Candidate Id */
candidate_id: string | null;
/**
* Created At
* Format: date-time
*/
created_at: string;
/** Expected Archive Digest */
expected_archive_digest: string;
/**
* Expires At
* Format: date-time
*/
expires_at: string;
failure_code: components["schemas"]["ImportFailure"] | null;
/**
* Operation Id
* Format: uuid
*/
operation_id: string;
/** Published Version Id */
published_version_id: string | null;
/**
* Schema Version
* @default 1
* @constant
*/
schema_version: 1;
state: components["schemas"]["ImportState"];
/**
* Updated At
* Format: date-time
*/
updated_at: string;
};
/** ProblemPage */
ProblemPage: {
/** Items */
@@ -811,6 +1208,188 @@ export interface components {
}
export type $defs = Record<string, never>;
export interface operations {
page_api_v1_admin_problem_imports_get: {
parameters: {
query?: {
after?: string | null;
limit?: number;
};
header?: never;
path?: never;
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["ProblemImportPage"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
create_api_v1_admin_problem_imports_post: {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
requestBody: {
content: {
"application/json": components["schemas"]["CreateProblemImport"];
};
};
responses: {
/** @description Successful Response */
201: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["ProblemImportSummary"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
capabilities_api_v1_admin_problem_imports_capabilities_get: {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["ProblemImportCapabilities"];
};
};
};
};
detail_api_v1_admin_problem_imports__operation_id__get: {
parameters: {
query?: never;
header?: never;
path: {
operation_id: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["ProblemImportDetail"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
upload_api_v1_admin_problem_imports__operation_id__archive_put: {
parameters: {
query?: never;
header?: never;
path: {
operation_id: string;
};
cookie?: never;
};
requestBody: {
content: {
"application/zip": string;
};
};
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["ProblemImportSummary"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
discard_api_v1_admin_problem_imports__operation_id__discard_post: {
parameters: {
query?: never;
header?: never;
path: {
operation_id: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["ProblemImportSummary"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
login_api_v1_auth_login_post: {
parameters: {
query?: never;