diff --git a/backend/migrations/env.py b/backend/migrations/env.py index 500589d..9ac193e 100644 --- a/backend/migrations/env.py +++ b/backend/migrations/env.py @@ -10,6 +10,8 @@ from kf_oj.infra.db.base import Base from kf_oj.infra.db.session import build_engine from kf_oj.infra.messaging import models as messaging_models # noqa: F401 from kf_oj.modules.identity import models as identity_models # noqa: F401 +from kf_oj.modules.operations import models as operations_models # noqa: F401 +from kf_oj.modules.problems import import_models # noqa: F401 from kf_oj.modules.problems import models as problem_models # noqa: F401 from kf_oj.modules.submissions import models as submission_models # noqa: F401 diff --git a/backend/migrations/versions/0007_problem_imports.py b/backend/migrations/versions/0007_problem_imports.py new file mode 100644 index 0000000..4c686d6 --- /dev/null +++ b/backend/migrations/versions/0007_problem_imports.py @@ -0,0 +1,189 @@ +"""Durable imports/candidates, append-only events and accepted-authority worker port.""" + +from alembic import op + +from kf_oj.modules.operations.models import MaintenanceEvent +from kf_oj.modules.problems.import_models import ( + CandidateCase, + ImportPool, + ProblemCandidate, + ProblemImportOperation, +) + +revision = "0007_problem_imports" +down_revision = "0006_authority_operations" +branch_labels = depends_on = None + + +def sql(script): + # asyncpg prepares one statement at a time; dollar-quoted bodies stay intact. + buffer, quote, dollar, i = [], False, False, 0 + while i < len(script): + if not quote and script[i : i + 2] == "$$": + dollar = not dollar + buffer.append("$$") + i += 2 + continue + character = script[i] + if not dollar and character == "'": + if quote and script[i : i + 2] == "''": + buffer.append("''") + i += 2 + continue + quote = not quote + if character == ";" and not dollar and not quote: + if "".join(buffer).strip(): + op.execute("".join(buffer)) + buffer = [] + else: + buffer.append(character) + i += 1 + if "".join(buffer).strip(): + op.execute("".join(buffer)) + + +def upgrade(): + connection = op.get_bind() + # Cyclic candidate/operation link is installed only after both tables exist. + for model in ( + ImportPool, + ProblemImportOperation, + ProblemCandidate, + CandidateCase, + MaintenanceEvent, + ): + model.__table__.create(connection) + op.create_foreign_key( + "fk_problem_import_candidate", + "problem_import_operations", + "problem_candidates", + ["candidate_id"], + ["id"], + ) + op.execute("INSERT INTO problem_import_pool(id,reserved_bytes) VALUES(1,0)") + sql(""" + CREATE FUNCTION public.kf_lock_reserved_maintenance(actor uuid, epoch bigint) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ + DECLARE permitted boolean; + BEGIN + SELECT enabled AND is_administrator AND authorization_version=epoch + INTO permitted FROM public.accounts WHERE id=actor FOR SHARE; + IF permitted IS DISTINCT FROM true THEN + RAISE EXCEPTION 'problem maintenance required' USING ERRCODE='42501'; + END IF; + END $$; + GRANT CREATE ON SCHEMA public TO kf_oj_identity_authority_owner; + ALTER FUNCTION public.kf_lock_reserved_maintenance(uuid,bigint) + OWNER TO kf_oj_identity_authority_owner; + REVOKE CREATE ON SCHEMA public FROM kf_oj_identity_authority_owner; + REVOKE ALL ON FUNCTION public.kf_lock_reserved_maintenance(uuid,bigint) FROM PUBLIC; + GRANT EXECUTE ON FUNCTION public.kf_lock_reserved_maintenance(uuid,bigint) + TO kf_oj_problem_import; + GRANT SELECT,INSERT,UPDATE ON problem_import_operations TO kf_oj_platform; + GRANT SELECT,UPDATE ON problem_import_pool TO kf_oj_platform,kf_oj_problem_import; + GRANT SELECT ON problem_candidates,problem_candidate_cases TO kf_oj_platform; + GRANT SELECT ON problem_import_operations TO kf_oj_problem_import; + GRANT UPDATE(state,updated_at,lease_until,attempts,fence, + reserved_bytes,failure_code,candidate_id) + ON problem_import_operations TO kf_oj_problem_import; + GRANT SELECT,INSERT,DELETE ON problem_candidates,problem_candidate_cases + TO kf_oj_problem_import; + GRANT SELECT,INSERT ON artifacts TO kf_oj_problem_import; + GRANT INSERT ON problem_maintenance_events TO kf_oj_platform,kf_oj_problem_import; + GRANT SELECT ON problem_maintenance_events TO kf_oj_platform; + """) + sql(""" + CREATE FUNCTION public.kf_immutable_import_candidate() RETURNS trigger + LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$ + DECLARE candidate uuid; current_state text; + BEGIN + IF TG_TABLE_NAME='problem_candidates' THEN candidate=OLD.id; + ELSE candidate=OLD.candidate_id; END IF; + SELECT o.state INTO current_state FROM public.problem_candidates c + JOIN public.problem_import_operations o ON o.id=c.operation_id WHERE c.id=candidate; + IF TG_OP='UPDATE' OR current_state NOT IN ('discarded','expired','failed') THEN + RAISE EXCEPTION 'immutable candidate' USING ERRCODE='23514'; + END IF; + RETURN OLD; + END $$; + CREATE TRIGGER immutable_candidate BEFORE UPDATE OR DELETE ON problem_candidates + FOR EACH ROW EXECUTE FUNCTION public.kf_immutable_import_candidate(); + CREATE TRIGGER immutable_candidate_case BEFORE UPDATE OR DELETE ON problem_candidate_cases + FOR EACH ROW EXECUTE FUNCTION public.kf_immutable_import_candidate(); + CREATE FUNCTION public.kf_insert_import_candidate() RETURNS trigger + LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$ + DECLARE operation uuid; current_state text; + BEGIN + IF TG_TABLE_NAME='problem_candidates' THEN operation=NEW.operation_id; + ELSE SELECT operation_id INTO operation FROM public.problem_candidates + WHERE id=NEW.candidate_id; END IF; + SELECT state INTO current_state FROM public.problem_import_operations + WHERE id=operation FOR SHARE; + IF current_state IS DISTINCT FROM 'validating' THEN + RAISE EXCEPTION 'candidate is not being validated' USING ERRCODE='23514'; + END IF; RETURN NEW; + END $$; + CREATE TRIGGER insert_candidate BEFORE INSERT ON problem_candidates + FOR EACH ROW EXECUTE FUNCTION public.kf_insert_import_candidate(); + CREATE TRIGGER insert_candidate_case BEFORE INSERT ON problem_candidate_cases + FOR EACH ROW EXECUTE FUNCTION public.kf_insert_import_candidate(); + CREATE FUNCTION public.kf_fixed_import_receipt() RETURNS trigger + LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$ BEGIN + IF NEW.id IS DISTINCT FROM OLD.id OR NEW.owner_id IS DISTINCT FROM OLD.owner_id + OR NEW.authorization_version IS DISTINCT FROM OLD.authorization_version + OR NEW.archive_digest IS DISTINCT FROM OLD.archive_digest + OR NEW.archive_bytes IS DISTINCT FROM OLD.archive_bytes + OR NEW.created_at IS DISTINCT FROM OLD.created_at + OR NEW.expires_at IS DISTINCT FROM OLD.expires_at THEN + RAISE EXCEPTION 'immutable import receipt' USING ERRCODE='23514'; + END IF; + IF OLD.state IN ('ready','published') AND NEW.state IN ('ready','published') + AND NEW.candidate_id IS DISTINCT FROM OLD.candidate_id THEN + RAISE EXCEPTION 'immutable completed candidate receipt' USING ERRCODE='23514'; + END IF; + IF OLD.state='published' AND (NEW.state IS DISTINCT FROM OLD.state + OR NEW.published_version_id IS DISTINCT FROM OLD.published_version_id) THEN + RAISE EXCEPTION 'immutable published import receipt' USING ERRCODE='23514'; + END IF; + IF NEW.state='ready' THEN + IF (OLD.state IS DISTINCT FROM 'ready' AND + (NOT pg_has_role(current_user,'kf_oj_problem_import','USAGE') + OR OLD.state!='validating')) OR NEW.candidate_id IS NULL + OR NOT EXISTS(SELECT 1 FROM public.problem_candidates c + WHERE c.id=NEW.candidate_id AND c.operation_id=NEW.id + AND c.case_count=(SELECT count(*) FROM public.problem_candidate_cases b + WHERE b.candidate_id=c.id) + AND c.case_count-1=(SELECT max(position) FROM public.problem_candidate_cases b + WHERE b.candidate_id=c.id)) THEN + RAISE EXCEPTION 'invalid candidate completion' USING ERRCODE='23514'; + END IF; + END IF; RETURN NEW; + END $$; + CREATE TRIGGER fixed_import_receipt BEFORE UPDATE ON problem_import_operations + FOR EACH ROW EXECUTE FUNCTION public.kf_fixed_import_receipt(); + """) + + +def downgrade(): + connection = op.get_bind() + if connection.exec_driver_sql( + "SELECT EXISTS(SELECT 1 FROM problem_import_operations)" + ).scalar(): + raise RuntimeError("cannot discard durable import history") + op.execute("DROP FUNCTION public.kf_lock_reserved_maintenance(uuid,bigint)") + op.drop_constraint( + "fk_problem_import_candidate", "problem_import_operations", type_="foreignkey" + ) + for model in ( + MaintenanceEvent, + CandidateCase, + ProblemCandidate, + ProblemImportOperation, + ImportPool, + ): + model.__table__.drop(connection) + sql( + "DROP FUNCTION public.kf_immutable_import_candidate(); " + "DROP FUNCTION public.kf_fixed_import_receipt(); " + "DROP FUNCTION public.kf_insert_import_candidate()" + ) diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 343eeb9..b925a99 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -22,6 +22,7 @@ dependencies = [ ] [project.scripts] + kf-problem-import-worker = "kf_oj.workers.problem_import:main" kf-accounts = "kf_oj.modules.identity.cli:main" kf-problems = "kf_oj.modules.problems.cli:main" kf-artifacts = "kf_oj.infra.artifacts.cli:main" diff --git a/backend/src/kf_oj/api/admin_dependencies.py b/backend/src/kf_oj/api/admin_dependencies.py new file mode 100644 index 0000000..e108277 --- /dev/null +++ b/backend/src/kf_oj/api/admin_dependencies.py @@ -0,0 +1,26 @@ +"""Uniform live session gates; domains still lock/recheck authority in write transactions.""" + +from typing import Annotated + +from fastapi import Depends, HTTPException + +from kf_oj.api.dependencies import current_principal, unsafe_principal +from kf_oj.modules.identity.public import Principal + + +async def maintenance_reader( + principal: Annotated[Principal, Depends(current_principal)], +) -> Principal: + if not principal.can_maintain_problems: + raise HTTPException(403, "problem maintenance required") + return principal + + +async def maintenance_writer( + principal: Annotated[Principal, Depends(unsafe_principal)], +) -> Principal: + # unsafe_principal already checks an explicit Origin and the live session's + # bound CSRF token. This capability check cannot replace the domain's port. + if not principal.can_maintain_problems: + raise HTTPException(403, "problem maintenance required") + return principal diff --git a/backend/src/kf_oj/api/app.py b/backend/src/kf_oj/api/app.py index 7f85704..d7a01ab 100644 --- a/backend/src/kf_oj/api/app.py +++ b/backend/src/kf_oj/api/app.py @@ -7,6 +7,7 @@ from starlette.responses import JSONResponse from kf_oj import __version__ from kf_oj.api.body_limits import BodyLimitMiddleware +from kf_oj.api.v1.admin_imports import router as import_router from kf_oj.api.v1.identity import router as identity_router from kf_oj.api.v1.problems import router as problems_router from kf_oj.api.v1.router import router @@ -15,10 +16,13 @@ from kf_oj.api.v1.worker_artifacts import router as worker_router from kf_oj.core.logging import configure_logging from kf_oj.core.settings import Settings from kf_oj.infra.artifacts.repository import ArtifactRepository +from kf_oj.infra.artifacts.staging import ImportStaging from kf_oj.infra.artifacts.store import ArtifactStore from kf_oj.infra.coordinator import CoordinatorControl from kf_oj.infra.db.session import build_engine, build_session_factory from kf_oj.modules.identity.service import IdentityService +from kf_oj.modules.operations.service import MaintenanceAudit +from kf_oj.modules.problems.imports import ProblemImports from kf_oj.modules.problems.service import ProblemService from kf_oj.modules.submissions.performance import SubmissionPerformanceQueries from kf_oj.modules.submissions.queries import SubmissionQueries @@ -36,6 +40,7 @@ def create_app(settings: Settings | None = None) -> FastAPI: logger.info("Platform API started") engine = None store = None + staging = None try: if settings.database_url is not None: engine = build_engine(settings) @@ -52,6 +57,18 @@ def create_app(settings: Settings | None = None) -> FastAPI: max_files=settings.artifact_max_files, ) artifacts = ArtifactRepository(store) + if ( + settings.import_staging_root is not None + and settings.judge_admission is not None + ): + staging = ImportStaging(settings.import_staging_root) + app.state.problem_imports = ProblemImports( + app.state.sessions, + staging, + app.state.identity, + MaintenanceAudit(), + settings.judge_admission, + ) app.state.problems = ProblemService( app.state.sessions, artifacts, settings.judge_admission ) @@ -80,10 +97,13 @@ def create_app(settings: Settings | None = None) -> FastAPI: app.state.submissions = None app.state.submission_queries = None app.state.submission_performance = None + app.state.problem_imports = None if engine is not None: await engine.dispose() if store is not None: store.close() + if staging is not None: + staging.close() logger.info("Platform API stopped") app = FastAPI( @@ -101,7 +121,9 @@ def create_app(settings: Settings | None = None) -> FastAPI: @app.middleware("http") async def private_responses(request, call_next): response = await call_next(request) - if request.url.path.startswith(("/api/v1/auth", "/api/v1/submissions", "/api/v1/internal")): + if request.url.path.startswith( + ("/api/v1/auth", "/api/v1/submissions", "/api/v1/internal", "/api/v1/admin") + ): response.headers["Cache-Control"] = "no-store" return response @@ -114,4 +136,5 @@ def create_app(settings: Settings | None = None) -> FastAPI: app.include_router(problems_router, prefix="/api/v1") app.include_router(worker_router, prefix="/api/v1") app.include_router(submission_router, prefix="/api/v1") + app.include_router(import_router, prefix="/api/v1") return app diff --git a/backend/src/kf_oj/api/body_limits.py b/backend/src/kf_oj/api/body_limits.py index 5405096..89d3118 100644 --- a/backend/src/kf_oj/api/body_limits.py +++ b/backend/src/kf_oj/api/body_limits.py @@ -1,6 +1,7 @@ """Bound request bytes and upload time before schema parsing can allocate large inputs.""" import asyncio +import re from starlette.responses import JSONResponse @@ -13,6 +14,14 @@ class BodyLimitMiddleware: if scope["type"] != "http" or scope["method"] in {"GET", "HEAD", "OPTIONS"}: await self.app(scope, receive, send) return + if scope["method"] == "PUT" and re.fullmatch( + r"/api/v1/admin/problem-imports/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/archive", + scope["path"], + ): + # Only this exact endpoint can stream. Its live administrator gate runs + # before receive(), then its service enforces actual bytes/chunks/deadline. + await self.app(scope, receive, send) + return limit = 8192 if scope["path"] == "/api/v1/auth/login" else 1024 * 1024 if scope["path"] == "/api/v1/submissions": # Max256KiB UTF-8 source, worst-case JSON escaping plus bounded metadata. diff --git a/backend/src/kf_oj/api/v1/admin_imports.py b/backend/src/kf_oj/api/v1/admin_imports.py new file mode 100644 index 0000000..9001f9c --- /dev/null +++ b/backend/src/kf_oj/api/v1/admin_imports.py @@ -0,0 +1,139 @@ +"""Administrator import contract; configured service availability is explicit.""" + +from typing import Annotated +from uuid import UUID + +from fastapi import APIRouter, Depends, HTTPException, Query, Request +from sqlalchemy.exc import DBAPIError +from starlette.requests import ClientDisconnect + +from kf_oj.api.admin_dependencies import maintenance_reader, maintenance_writer +from kf_oj.contracts.admin_problems import ( + CreateProblemImport, + ProblemImportCapabilities, + ProblemImportDetail, + ProblemImportPage, + ProblemImportSummary, +) +from kf_oj.infra.artifacts.store import ArtifactError +from kf_oj.modules.identity.public import MaintenanceDenied, Principal +from kf_oj.modules.problems.imports import ImportConflict, ImportMissing, ImportUnavailable + +router = APIRouter(prefix="/admin/problem-imports", tags=["problem-maintenance"]) + + +@router.get("/capabilities", response_model=ProblemImportCapabilities) +async def capabilities( + request: Request, _: Annotated[Principal, Depends(maintenance_reader)] +) -> ProblemImportCapabilities: + service = getattr(request.app.state, "problem_imports", None) + if service is None: + return ProblemImportCapabilities(uploads_available=False, validation_available=False) + return await service.capabilities() + + +def service(request): + result = getattr(request.app.state, "problem_imports", None) + if result is None: + raise HTTPException(503, "problem imports unavailable") + return result + + +async def operation(awaitable): + try: + return await awaitable + except MaintenanceDenied: + raise HTTPException(403, "problem maintenance required") from None + except ImportConflict: + raise HTTPException(409, "import request conflict") from None + except ImportMissing: + raise HTTPException(404, "import unavailable") from None + except TimeoutError: + raise HTTPException(408, "upload deadline exceeded") from None + except (ImportUnavailable, ArtifactError, OSError, DBAPIError): + raise HTTPException(503, "problem imports unavailable") from None + except ClientDisconnect: + raise HTTPException(400, "upload interrupted") from None + + +@router.post("", status_code=201, response_model=ProblemImportSummary) +async def create( + body: CreateProblemImport, + request: Request, + principal: Annotated[Principal, Depends(maintenance_writer)], +): + if request.headers.get("content-type", "").split(";", 1)[0] != "application/json": + raise HTTPException(415, "JSON request required") + return await operation(service(request).create(principal, body)) + + +@router.get("", response_model=ProblemImportPage) +async def page( + request: Request, + principal: Annotated[Principal, Depends(maintenance_reader)], + after: UUID | None = None, + limit: Annotated[int, Query(ge=1, le=100)] = 20, +): + return await operation(service(request).list(principal, after, limit)) + + +@router.get("/{operation_id}", response_model=ProblemImportDetail) +async def detail( + operation_id: UUID, + request: Request, + principal: Annotated[Principal, Depends(maintenance_reader)], +): + return await operation(service(request).detail(principal, operation_id)) + + +@router.put( + "/{operation_id}/archive", + response_model=ProblemImportSummary, + openapi_extra={ + "requestBody": { + "required": True, + "content": {"application/zip": {"schema": {"type": "string", "format": "binary"}}}, + } + }, +) +async def upload( + operation_id: UUID, + request: Request, + principal: Annotated[Principal, Depends(maintenance_writer)], +): + if request.url.path != f"/api/v1/admin/problem-imports/{operation_id}/archive": + raise HTTPException(422, "canonical operation UUID required") + if request.headers.get("content-type", "").split(";", 1)[0] != "application/zip": + raise HTTPException(415, "ZIP request required") + if request.headers.get("content-encoding") not in (None, "identity"): + raise HTTPException(415, "encoded upload unavailable") + length = request.headers.get("content-length") + if length is not None: + try: + if not length.isascii() or not length.isdecimal() or len(length) > 12: + raise ValueError + length = int(length) + except ValueError: + raise HTTPException(413, "invalid archive length") from None + + async def chunks(): + while True: + message = await request.receive() + if message["type"] == "http.disconnect": + raise ClientDisconnect() + if message["type"] != "http.request": + raise ImportConflict("invalid upload frame") + yield message.get("body", b"") + if not message.get("more_body", False): + return + + return await operation(service(request).upload(principal, operation_id, chunks(), length)) + + +@router.post("/{operation_id}/discard", response_model=ProblemImportSummary) +async def discard( + operation_id: UUID, + request: Request, + principal: Annotated[Principal, Depends(maintenance_writer)], +): + return await operation(service(request).discard(principal, operation_id)) diff --git a/backend/src/kf_oj/contracts/admin_problems.py b/backend/src/kf_oj/contracts/admin_problems.py new file mode 100644 index 0000000..1b3b5b1 --- /dev/null +++ b/backend/src/kf_oj/contracts/admin_problems.py @@ -0,0 +1,180 @@ +"""Versioned problem-import DTOs; hidden bytes, paths and toolchain selection stay private.""" + +from datetime import datetime +from enum import StrEnum +from typing import Annotated, Literal + +from pydantic import Field, StrictBool, model_validator + +from kf_oj.contracts.judging import Digest, Identifier, Label, RunLimits, Version, WireModel +from kf_oj.contracts.judging_v2 import Language +from kf_oj.contracts.problems import PublicLanguageConfiguration, PublicSample + +MAX_ARCHIVE_BYTES = 64 * 1024**2 +MAX_UNPACKED_BYTES = 64 * 1024**2 +MAX_ARCHIVE_FILES = 534 +MAX_ARCHIVE_NAME_BYTES = 128 +MAX_UPLOAD_CHUNKS = 4096 +MAX_UPLOAD_CHUNK_BYTES = 1024**2 +UPLOAD_DEADLINE_SECONDS = 30 +VALIDATION_DEADLINE_SECONDS = 60 +MAX_IMPORT_ATTEMPTS = 2 +IMPORT_LIFETIME_SECONDS = 86400 +MAX_OWNER_CANDIDATES = 10 +MAX_TOTAL_CANDIDATES = 32 +STAGING_MAX_BYTES = 256 * 1024**2 +STAGING_RESERVATION_BYTES = MAX_ARCHIVE_BYTES + MAX_UNPACKED_BYTES + + +class ImportState(StrEnum): + UPLOADING = "uploading" + UPLOADED = "uploaded" + VALIDATING = "validating" + READY = "ready" + FAILED = "failed" + PUBLISHED = "published" + DISCARDED = "discarded" + EXPIRED = "expired" + + +class ImportFailure(StrEnum): + INVALID_ARCHIVE = "invalid_archive" + INVALID_MANIFEST = "invalid_manifest" + INVALID_DATA = "invalid_data" + DIGEST_MISMATCH = "digest_mismatch" + POLICY_REJECTED = "policy_rejected" + UPLOAD_INTERRUPTED = "upload_interrupted" + UPLOAD_DEADLINE = "upload_deadline" + VALIDATION_DEADLINE = "validation_deadline" + VALIDATION_UNAVAILABLE = "validation_unavailable" + STORAGE_UNAVAILABLE = "storage_unavailable" + AUTHORITY_REVOKED = "authority_revoked" + EXPIRED = "expired" + + +class ProblemImportLimits(WireModel): + schema_version: Version = 1 + archive_format: Literal["single_level_zip"] = "single_level_zip" + manifest_name: Literal["problem.json"] = "problem.json" + manifest_schema_versions: tuple[Literal[1, 2], ...] = (1, 2) + languages: tuple[Language, ...] = tuple(Language) + archive_bytes: int = MAX_ARCHIVE_BYTES + unpacked_bytes: int = MAX_UNPACKED_BYTES + files: int = MAX_ARCHIVE_FILES + filename_ascii_bytes: int = MAX_ARCHIVE_NAME_BYTES + upload_chunks: int = MAX_UPLOAD_CHUNKS + chunk_bytes: int = MAX_UPLOAD_CHUNK_BYTES + upload_deadline_seconds: int = UPLOAD_DEADLINE_SECONDS + validation_deadline_seconds: int = VALIDATION_DEADLINE_SECONDS + global_concurrency: Literal[1] = 1 + owner_concurrency: Literal[1] = 1 + owner_candidates: int = MAX_OWNER_CANDIDATES + total_candidates: int = MAX_TOTAL_CANDIDATES + staging_bytes: int = STAGING_MAX_BYTES + candidate_lifetime_seconds: int = IMPORT_LIFETIME_SECONDS + + +class ProblemImportCapabilities(WireModel): + schema_version: Version = 1 + limits: ProblemImportLimits = ProblemImportLimits() + uploads_available: StrictBool + validation_available: StrictBool + + +class CreateProblemImport(WireModel): + schema_version: Version = 1 + operation_id: Identifier + archive_digest: Digest + archive_bytes: Annotated[int, Field(strict=True, ge=1, le=MAX_ARCHIVE_BYTES)] + + +class LegacyImportBudget(WireModel): + compile_limits: RunLimits + run_limits: RunLimits + job_wall_time_us: Annotated[int, Field(strict=True, ge=1000, le=3600000000)] + + +class CandidatePreview(WireModel): + candidate_id: Identifier + slug: Label + title: Annotated[str, Field(strict=True, min_length=1, max_length=160)] + statement: Annotated[str, Field(strict=True, min_length=1, max_length=262144)] + samples: Annotated[tuple[PublicSample, ...], Field(max_length=10)] + judging_schema_version: Annotated[int, Field(strict=True, ge=1, le=2)] + languages: Annotated[tuple[Language, ...], Field(min_length=1, max_length=4)] + legacy_budget: LegacyImportBudget | None + language_configurations: Annotated[ + tuple[PublicLanguageConfiguration, ...], Field(max_length=4) + ] = () + case_count: Annotated[int, Field(strict=True, ge=1, le=256)] + + @model_validator(mode="after") + def bounded_text(self): + if len(self.statement.encode("utf-8")) > 262144 or "\x00" in self.statement + self.title: + raise ValueError("invalid public text") + languages = [c.language for c in self.language_configurations] + if len(set(languages)) != len(languages): + raise ValueError("duplicate language configuration") + if len(set(self.languages)) != len(self.languages): + raise ValueError("duplicate language") + if self.judging_schema_version == 1: + if ( + self.language_configurations + or self.legacy_budget is None + or any( + language not in (Language.C17, Language.CPP20) for language in self.languages + ) + ): + raise ValueError("invalid legacy candidate") + elif tuple(languages) != self.languages or self.legacy_budget is not None: + raise ValueError("invalid multi-runtime candidate") + return self + + +class ProblemImportSummary(WireModel): + schema_version: Version = 1 + operation_id: Identifier + state: ImportState + expected_archive_digest: Digest + archive_bytes: Annotated[int, Field(strict=True, ge=1, le=MAX_ARCHIVE_BYTES)] + created_at: datetime + expires_at: datetime + updated_at: datetime + failure_code: ImportFailure | None + candidate_id: Identifier | None + published_version_id: Identifier | None + + @model_validator(mode="after") + def consistent_state(self): + if self.expires_at <= self.created_at or self.updated_at < self.created_at: + raise ValueError("invalid import time range") + if (self.state in (ImportState.READY, ImportState.PUBLISHED)) != ( + self.candidate_id is not None + ): + raise ValueError("candidate state mismatch") + if (self.state == ImportState.PUBLISHED) != (self.published_version_id is not None): + raise ValueError("published state mismatch") + if self.state in (ImportState.FAILED, ImportState.EXPIRED): + if self.failure_code is None: + raise ValueError("failure code required") + elif self.failure_code is not None: + raise ValueError("unexpected failure code") + return self + + +class ProblemImportDetail(ProblemImportSummary): + candidate: CandidatePreview | None + + @model_validator(mode="after") + def candidate_matches(self): + if (self.candidate_id is None) != (self.candidate is None): + raise ValueError("candidate preview missing") + if self.candidate and self.candidate.candidate_id != self.candidate_id: + raise ValueError("candidate preview mismatch") + return self + + +class ProblemImportPage(WireModel): + schema_version: Version = 1 + items: Annotated[tuple[ProblemImportSummary, ...], Field(max_length=100)] + next_after: Identifier | None diff --git a/backend/src/kf_oj/contracts/export.py b/backend/src/kf_oj/contracts/export.py index 5a2af30..65bd6aa 100644 --- a/backend/src/kf_oj/contracts/export.py +++ b/backend/src/kf_oj/contracts/export.py @@ -5,6 +5,11 @@ import json from pathlib import Path from kf_oj.api.app import create_app +from kf_oj.contracts.admin_problems import ( + CreateProblemImport, + ProblemImportCapabilities, + ProblemImportDetail, +) from kf_oj.contracts.judge import JudgeTaskNotification from kf_oj.contracts.judging import WIRE_MODELS from kf_oj.contracts.judging_v2 import WIRE_MODELS as WIRE_MODELS_V2 @@ -43,6 +48,15 @@ def main() -> None: "counts; also run model validation and server-side ownership/environment checks." ) outputs[root / "contracts/schemas/submission-performance.v1.schema.json"] = performance + for name, model in { + "problem-import-create": CreateProblemImport, + "problem-import": ProblemImportDetail, + "problem-import-capabilities": ProblemImportCapabilities, + }.items(): + document = model.model_json_schema() + document["$schema"] = "https://json-schema.org/draft/2020-12/schema" + document["$id"] = f"urn:kf-oj:{name}:v1" + outputs[root / f"contracts/schemas/{name}.v1.schema.json"] = document for name, model in WIRE_MODELS_V2.items(): document = model.model_json_schema() document["$schema"] = "https://json-schema.org/draft/2020-12/schema" diff --git a/backend/src/kf_oj/infra/artifacts/archive_reader.py b/backend/src/kf_oj/infra/artifacts/archive_reader.py new file mode 100644 index 0000000..5b4818f --- /dev/null +++ b/backend/src/kf_oj/infra/artifacts/archive_reader.py @@ -0,0 +1,340 @@ +"""Bounded single-level ZIP reader; no extractall, metadata restoration or content execution.""" + +import binascii +import hashlib +import os +import re +import stat +import struct +import time +import zlib +from collections.abc import Iterator +from dataclasses import dataclass + +from kf_oj.contracts.admin_problems import ( + MAX_ARCHIVE_BYTES, + MAX_ARCHIVE_FILES, + MAX_UNPACKED_BYTES, +) + +NAME = re.compile(rb"[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}\Z") +CHUNK = 65536 +LOCAL = struct.Struct("<4s5H3I2H") +CENTRAL = struct.Struct("<4s6H3I5H2I") +END = struct.Struct("<4s4H2IH") + + +class ArchiveError(ValueError): + """A fixed sanitized failure; never echo an archive name, data or local path.""" + + +@dataclass(frozen=True) +class ArchiveMember: + name: str + method: int + crc32: int + compressed_bytes: int + bytes: int + offset: int + flags: int + + +class ArchiveReader: + def __init__(self, descriptor: int, expected_bytes: int, *, deadline: float): + self.descriptor, self.deadline = descriptor, deadline + self.before = os.fstat(descriptor) + if ( + not stat.S_ISREG(self.before.st_mode) + or self.before.st_uid != os.getuid() + or self.before.st_nlink != 1 + or stat.S_IMODE(self.before.st_mode) != 0o600 + or not 22 <= expected_bytes <= MAX_ARCHIVE_BYTES + or self.before.st_size != expected_bytes + ): + raise ArchiveError("invalid_archive") + self.size = expected_bytes + self.unpacked_bytes, self.steps = 0, 0 + self.read_members: set[str] = set() + self.members = self._members() + + def _tick(self) -> None: + self.steps += 1 + if self.steps > 8192 or time.monotonic() >= self.deadline: + raise ArchiveError("archive_deadline") + + def _read(self, offset: int, size: int) -> bytes: + self._tick() + if not 0 <= size <= 1024**2 or offset < 0 or offset + size > self.size: + raise ArchiveError("invalid_archive") + data = os.pread(self.descriptor, size, offset) + if len(data) != size: + raise ArchiveError("invalid_archive") + return data + + @staticmethod + def _extra(data: bytes) -> None: + offset, fields = 0, 0 + # These bounded timestamps/UID attributes are ignored, never restored. + # ZIP64, Unicode aliases and all other extension semantics are unsupported. + while offset < len(data): + if offset + 4 > len(data) or fields >= 32: + raise ArchiveError("invalid_archive") + kind, size = struct.unpack_from(" len(data): + raise ArchiveError("invalid_archive") + offset += size + fields += 1 + + def _members(self) -> tuple[ArchiveMember, ...]: + tail_offset = max(0, self.size - 1046) + tail = self._read(tail_offset, self.size - tail_offset) + position = tail.rfind(b"PK\x05\x06") + if position < 0 or position + END.size > len(tail): + raise ArchiveError("invalid_archive") + sig, disk, cd_disk, disk_count, count, cd_bytes, cd_offset, comment = END.unpack_from( + tail, position + ) + end_offset = tail_offset + position + if ( + sig != b"PK\x05\x06" + or disk != 0 + or cd_disk != 0 + or disk_count != count + or not 1 <= count <= MAX_ARCHIVE_FILES + or comment > 1024 + or end_offset + END.size + comment != self.size + or not 46 <= cd_bytes <= 1024**2 + or cd_offset + cd_bytes != end_offset + ): + raise ArchiveError("invalid_archive") + directory = self._read(cd_offset, cd_bytes) + offset, records, names = 0, [], set() + for _ in range(count): + if offset + CENTRAL.size > len(directory): + raise ArchiveError("invalid_archive") + values = CENTRAL.unpack_from(directory, offset) + ( + sig, + _, + needed, + flags, + method, + _, + _, + crc, + compressed, + size, + name_size, + extra_size, + comment_size, + disk, + _, + attributes, + local_offset, + ) = values + allowed_flags = 0x0808 | (0x0006 if method == 8 else 0) + if ( + sig != b"PK\x01\x02" + or needed not in (10, 20) + or method not in (0, 8) + or (method == 8 and needed != 20) + or flags & ~allowed_flags + or disk != 0 + or not 1 <= name_size <= 128 + or extra_size > 1024 + or comment_size > 256 + or size > MAX_UNPACKED_BYTES + or compressed > MAX_ARCHIVE_BYTES + or local_offset + LOCAL.size > cd_offset + or attributes & 0x10 + or stat.S_IFMT(attributes >> 16) not in (0, stat.S_IFREG) + ): + raise ArchiveError("invalid_archive") + start = offset + CENTRAL.size + finish = start + name_size + extra_size + comment_size + if finish > len(directory): + raise ArchiveError("invalid_archive") + name_bytes = directory[start : start + name_size] + if not NAME.fullmatch(name_bytes): + raise ArchiveError("invalid_archive") + name = name_bytes.decode("ascii") + if name.lower() in names: + raise ArchiveError("invalid_archive") + names.add(name.lower()) + self._extra(directory[start + name_size : start + name_size + extra_size]) + records.append(ArchiveMember(name, method, crc, compressed, size, local_offset, flags)) + offset = finish + if offset != len(directory) or sum(m.bytes for m in records) > MAX_UNPACKED_BYTES: + raise ArchiveError("invalid_archive") + cursor, members = 0, [] + for member in sorted(records, key=lambda m: m.offset): + if member.offset != cursor: + raise ArchiveError("invalid_archive") + sig, needed, flags, method, _, _, crc, compressed, size, name_size, extra_size = ( + LOCAL.unpack(self._read(cursor, LOCAL.size)) + ) + if ( + sig != b"PK\x03\x04" + or needed not in (10, 20) + or flags != member.flags + or method != member.method + or (method == 8 and needed != 20) + or name_size != len(member.name) + or extra_size > 1024 + ): + raise ArchiveError("invalid_archive") + metadata = self._read(cursor + LOCAL.size, name_size + extra_size) + if metadata[:name_size] != member.name.encode("ascii"): + raise ArchiveError("invalid_archive") + self._extra(metadata[name_size:]) + data_offset = cursor + LOCAL.size + name_size + extra_size + cursor = data_offset + member.compressed_bytes + if cursor > cd_offset or (method == 0 and member.compressed_bytes != member.bytes): + raise ArchiveError("invalid_archive") + if flags & 8: + if (crc, compressed, size) not in ( + (0, 0, 0), + (member.crc32, member.compressed_bytes, member.bytes), + ): + raise ArchiveError("invalid_archive") + marker = self._read(cursor, 4) + descriptor_size = 16 if marker == b"PK\x07\x08" else 12 + descriptor = self._read(cursor, descriptor_size) + described = struct.unpack(" None: + digest = hashlib.sha256() + for offset in range(0, self.size, CHUNK): + digest.update(self._read(offset, min(CHUNK, self.size - offset))) + if digest.hexdigest() != expected: + raise ArchiveError("archive_digest_mismatch") + + def verify_unchanged(self) -> None: + after = os.fstat(self.descriptor) + fields = ( + "st_dev", + "st_ino", + "st_nlink", + "st_mode", + "st_uid", + "st_size", + "st_mtime_ns", + "st_ctime_ns", + ) + if any(getattr(self.before, f) != getattr(after, f) for f in fields): + raise ArchiveError("archive_changed") + + @staticmethod + def _nested(prefix: bytes) -> bool: + return prefix.startswith( + ( + b"PK\x03\x04", + b"PK\x05\x06", + b"PK\x07\x08", + b"\x1f\x8b", + b"BZh", + b"\xfd7zXZ\x00", + b"Rar!\x1a\x07", + b"7z\xbc\xaf\x27\x1c", + ) + ) or (len(prefix) >= 262 and prefix[257:262] == b"ustar") + + def chunks(self, name: str, maximum: int) -> Iterator[bytes]: + member = next((m for m in self.members if m.name == name), None) + if member is None or name in self.read_members or member.bytes > maximum: + raise ArchiveError("invalid_archive") + self.read_members.add(name) + used, crc, prefix = 0, 0, bytearray() + inflater = zlib.decompressobj(-15) if member.method == 8 else None + try: + for offset in range(0, member.compressed_bytes, CHUNK): + pending = self._read( + member.offset + offset, min(CHUNK, member.compressed_bytes - offset) + ) + while pending: + self._tick() + if inflater: + data = inflater.decompress(pending, min(CHUNK, member.bytes - used + 1)) + pending = inflater.unconsumed_tail + if inflater.unused_data: + raise ArchiveError("invalid_archive") + else: + data, pending = pending, b"" + used += len(data) + self.unpacked_bytes += len(data) + if ( + used > member.bytes + or used > maximum + or self.unpacked_bytes > MAX_UNPACKED_BYTES + ): + raise ArchiveError("invalid_archive") + prefix.extend(data[: max(0, 512 - len(prefix))]) + if self._nested(bytes(prefix)): + raise ArchiveError("invalid_archive") + crc = binascii.crc32(data, crc) + if data: + yield data + if (inflater and not inflater.eof) or used != member.bytes or crc != member.crc32: + raise ArchiveError("invalid_archive") + except zlib.error: + raise ArchiveError("invalid_archive") from None + + def read(self, name: str, maximum: int) -> bytes: + return b"".join(self.chunks(name, maximum)) + + def extract_remaining(self, directory: int) -> dict[str, tuple[str, int]]: + info = os.fstat(directory) + if ( + not stat.S_ISDIR(info.st_mode) + or info.st_uid != os.getuid() + or stat.S_IMODE(info.st_mode) != 0o700 + ): + raise ArchiveError("invalid_extraction_directory") + references = {} + for member in self.members: + if member.name in self.read_members: + continue + fd = os.open( + member.name, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, + 0o600, + dir_fd=directory, + ) + try: + digest, size = hashlib.sha256(), 0 + for chunk in self.chunks(member.name, member.bytes): + digest.update(chunk) + size += len(chunk) + remaining = memoryview(chunk) + while remaining: + written = os.write(fd, remaining) + if written <= 0: + raise ArchiveError("extraction_unavailable") + remaining = remaining[written:] + os.fsync(fd) + references[member.name] = (digest.hexdigest(), size) + finally: + os.close(fd) + os.fsync(directory) + self.verify_unchanged() + return references diff --git a/backend/src/kf_oj/infra/artifacts/repository.py b/backend/src/kf_oj/infra/artifacts/repository.py index 1af4927..1729fa1 100644 --- a/backend/src/kf_oj/infra/artifacts/repository.py +++ b/backend/src/kf_oj/infra/artifacts/repository.py @@ -50,6 +50,28 @@ class ArtifactRepository: raise ArtifactError("artifact metadata unavailable") return await asyncio.to_thread(self.store.read, reference) + async def put_file( + self, session: AsyncSession, descriptor: int, reference: ArtifactReference + ) -> ArtifactReference: + await lock_digest(session, reference.digest) + existing = await session.get(Artifact, reference.digest) + if existing is not None and ( + existing.state != "ready" or existing.bytes != reference.bytes + ): + raise ArtifactError("artifact metadata unavailable") + task = asyncio.create_task(asyncio.to_thread(self.store.put_file, descriptor, reference)) + try: + await asyncio.shield(task) + except BaseException: + await task + raise + await session.execute( + insert(Artifact) + .values(digest=reference.digest, bytes=reference.bytes) + .on_conflict_do_nothing(index_elements=[Artifact.digest]) + ) + return reference + async def check_metadata(self, session: AsyncSession, reference: ArtifactReference) -> None: metadata = await session.get(Artifact, reference.digest) if metadata is None or metadata.state != "ready" or metadata.bytes != reference.bytes: diff --git a/backend/src/kf_oj/infra/artifacts/staging.py b/backend/src/kf_oj/infra/artifacts/staging.py new file mode 100644 index 0000000..0e477f6 --- /dev/null +++ b/backend/src/kf_oj/infra/artifacts/staging.py @@ -0,0 +1,208 @@ +"""Owned, private, bounded import staging. A kernel lock survives parent/child handoff.""" + +import fcntl +import os +import stat +from contextlib import contextmanager +from pathlib import Path +from uuid import UUID + +from kf_oj.contracts.admin_problems import MAX_ARCHIVE_FILES, STAGING_MAX_BYTES +from kf_oj.infra.artifacts.store import ArtifactError + + +class ImportStaging: + def __init__(self, root: Path): + root = root.absolute() + root.mkdir(parents=True, mode=0o700, exist_ok=True) + if root.resolve() != root: + raise ArtifactError("invalid staging root") + self.root = root + self.fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC) + self.directory(self.fd) + + @staticmethod + def directory(fd): + s = os.fstat(fd) + if ( + not stat.S_ISDIR(s.st_mode) + or s.st_uid != os.getuid() + or stat.S_IMODE(s.st_mode) != 0o700 + ): + raise ArtifactError("invalid staging directory") + + def close(self): + os.close(self.fd) + + @staticmethod + def names(descriptor, maximum): + result = [] + with os.scandir(descriptor) as entries: + for entry in entries: + if len(result) >= maximum: + raise ArtifactError("staging file capacity exhausted") + result.append(entry.name) + return result + + def operation_ids(self): + identifiers = [] + for name in self.names(self.fd, 33): + if name == ".lock": + continue + try: + identifier = UUID(name) + if str(identifier) != name: + raise ValueError + except ValueError: + raise ArtifactError("unexpected staging entry") from None + identifiers.append(identifier) + return identifiers + + @contextmanager + def exclusive(self): + lock = os.open( + ".lock", os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600, dir_fd=self.fd + ) + try: + from kf_oj.infra.artifacts.store import ArtifactStore + + ArtifactStore._regular(os.fstat(lock)) + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise ArtifactError("import capacity busy") from None + yield lock + finally: + os.close(lock) + + def usage(self): + total, count = 0, 0 + for name in self.names(self.fd, 33): + if name == ".lock": + continue + try: + if str(UUID(name)) != name: + raise ValueError + except ValueError: + raise ArtifactError("unexpected staging entry") from None + descriptor = self.operation(UUID(name)) + try: + for entry in self.names(descriptor, 3): + count += 1 + if count > 2 * (MAX_ARCHIVE_FILES + 2): + raise ArtifactError("staging file capacity exhausted") + if entry == "members": + child = self.members(descriptor) + try: + for member in self.names(child, MAX_ARCHIVE_FILES): + count += 1 + info = os.stat(member, dir_fd=child, follow_symlinks=False) + from kf_oj.infra.artifacts.store import ArtifactStore + + ArtifactStore._regular(info) + total += info.st_size + finally: + os.close(child) + elif entry in ("archive", "validated.json"): + info = os.stat(entry, dir_fd=descriptor, follow_symlinks=False) + from kf_oj.infra.artifacts.store import ArtifactStore + + ArtifactStore._regular(info) + total += info.st_size + else: + raise ArtifactError("unexpected staging entry") + if count > 2 * (MAX_ARCHIVE_FILES + 2) or total > STAGING_MAX_BYTES: + raise ArtifactError("staging capacity exhausted") + finally: + os.close(descriptor) + return total + + def operation(self, operation_id: UUID, *, create=False): + name = str(operation_id) + if create: + try: + os.mkdir(name, 0o700, dir_fd=self.fd) + except FileExistsError: + pass + descriptor = os.open( + name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC, dir_fd=self.fd + ) + try: + self.directory(descriptor) + except Exception: + os.close(descriptor) + raise + return descriptor + + @staticmethod + def members(operation, *, create=False): + if create: + os.mkdir("members", 0o700, dir_fd=operation) + descriptor = os.open( + "members", os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC, dir_fd=operation + ) + try: + ImportStaging.directory(descriptor) + except Exception: + os.close(descriptor) + raise + return descriptor + + def remove(self, operation_id): + try: + descriptor = self.operation(operation_id) + except FileNotFoundError: + return + try: + names = self.names(descriptor, 3) + if len(names) > 3: + raise ArtifactError("unexpected staging entry") + for name in names: + if name == "members": + child = self.members(descriptor) + try: + entries = self.names(child, MAX_ARCHIVE_FILES) + if len(entries) > MAX_ARCHIVE_FILES: + raise ArtifactError("unexpected staging entry") + for member in entries: + from kf_oj.infra.artifacts.store import ArtifactStore + + ArtifactStore._regular( + os.stat(member, dir_fd=child, follow_symlinks=False) + ) + os.unlink(member, dir_fd=child) + finally: + os.close(child) + os.rmdir("members", dir_fd=descriptor) + elif name in ("archive", "validated.json"): + from kf_oj.infra.artifacts.store import ArtifactStore + + ArtifactStore._regular(os.stat(name, dir_fd=descriptor, follow_symlinks=False)) + os.unlink(name, dir_fd=descriptor) + else: + raise ArtifactError("unexpected staging entry") + finally: + os.close(descriptor) + os.rmdir(str(operation_id), dir_fd=self.fd) + os.fsync(self.fd) + + def reset_validation(self, operation_id): + directory = self.operation(operation_id) + try: + names = self.names(directory, 3) + if "members" in names: + members = self.members(directory) + try: + for name in self.names(members, MAX_ARCHIVE_FILES): + from kf_oj.infra.artifacts.store import ArtifactStore + + ArtifactStore._regular(os.stat(name, dir_fd=members, follow_symlinks=False)) + os.unlink(name, dir_fd=members) + finally: + os.close(members) + os.rmdir("members", dir_fd=directory) + if "validated.json" in names: + os.unlink("validated.json", dir_fd=directory) + os.fsync(directory) + finally: + os.close(directory) diff --git a/backend/src/kf_oj/infra/artifacts/store.py b/backend/src/kf_oj/infra/artifacts/store.py index 3de80ee..6cf273b 100644 --- a/backend/src/kf_oj/infra/artifacts/store.py +++ b/backend/src/kf_oj/infra/artifacts/store.py @@ -168,6 +168,90 @@ class ArtifactStore: with self.exclusive(): return self._read(reference) + def put_file(self, source: int, expected: ArtifactReference) -> ArtifactReference: + """Adopt a validated private FD without buffering a hidden testcase in memory.""" + before = os.fstat(source) + self._regular(before) + if before.st_size != expected.bytes: + raise ArtifactError("artifact digest or length mismatch") + + def unchanged(descriptor, original): + current = os.fstat(descriptor) + fields = ( + "st_dev", + "st_ino", + "st_nlink", + "st_mode", + "st_uid", + "st_size", + "st_mtime_ns", + "st_ctime_ns", + ) + if any(getattr(current, k) != getattr(original, k) for k in fields): + raise ArtifactError("artifact input changed") + + def stream(descriptor, *, output=None): + digest = hashlib.sha256() + for offset in range(0, expected.bytes, 65536): + data = os.pread(descriptor, min(65536, expected.bytes - offset), offset) + if len(data) != min(65536, expected.bytes - offset): + raise ArtifactError("artifact digest or length mismatch") + digest.update(data) + if output is not None: + output.write(data) + if digest.hexdigest() != expected.digest: + raise ArtifactError("artifact digest or length mismatch") + + with self.exclusive(): + entries = self._entries() + if any(name == expected.digest for name, _ in entries): + existing = os.open( + expected.digest, + os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC, + dir_fd=self._directory, + ) + try: + self._regular(os.fstat(existing)) + if os.fstat(existing).st_size != expected.bytes: + raise ArtifactError("artifact digest or length mismatch") + stream(source) + stream(existing) + finally: + os.close(existing) + else: + if ( + len(entries) + 1 > self.max_files + or sum(s.st_size for _, s in entries) + expected.bytes > self.max_bytes + ): + raise ArtifactError("storage capacity exhausted") + temporary = ".tmp-" + uuid4().hex + descriptor = os.open( + temporary, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, + 0o600, + dir_fd=self._directory, + ) + try: + with os.fdopen(descriptor, "wb") as out: + stream(source, output=out) + out.flush() + os.fsync(out.fileno()) + unchanged(source, before) + os.rename( + temporary, + expected.digest, + src_dir_fd=self._directory, + dst_dir_fd=self._directory, + ) + os.fsync(self._directory) + finally: + try: + os.unlink(temporary, dir_fd=self._directory) + except FileNotFoundError: + pass + unchanged(source, before) + return expected + def remove(self, digest: str) -> None: if not DIGEST.fullmatch(digest): raise ArtifactError("invalid artifact identifier") diff --git a/backend/src/kf_oj/modules/identity/public.py b/backend/src/kf_oj/modules/identity/public.py index afd04bb..134f439 100644 --- a/backend/src/kf_oj/modules/identity/public.py +++ b/backend/src/kf_oj/modules/identity/public.py @@ -28,5 +28,11 @@ class AuthenticationFailed(ValueError): pass +class ReservedMaintenancePort(Protocol): + async def require_reserved_maintenance(self, session, actor_id: UUID, epoch: int) -> None: + """Recheck/lock an immutable accepted receipt, with no login/session secret access.""" + ... + + class LoginThrottled(ValueError): pass diff --git a/backend/src/kf_oj/modules/identity/service.py b/backend/src/kf_oj/modules/identity/service.py index 73dc1c2..be00542 100644 --- a/backend/src/kf_oj/modules/identity/service.py +++ b/backend/src/kf_oj/modules/identity/service.py @@ -7,7 +7,8 @@ from dataclasses import dataclass, field from datetime import datetime, timedelta from uuid import uuid4 -from sqlalchemy import func, select, update +from sqlalchemy import func, select, text, update +from sqlalchemy.exc import DBAPIError from sqlalchemy.ext.asyncio import async_sessionmaker from kf_oj.modules.identity import repository @@ -170,3 +171,20 @@ class IdentityService: await self.authenticate(token, csrf_token=csrf_token) async with self.sessions.begin() as session: await repository.revoke(session, token_digest(token)) + + +class ReservedMaintenance: + """Import worker uses only the fixed, narrow SQL authority port.""" + + async def require_reserved_maintenance(self, session, actor_id, epoch): + if not session.in_transaction() or type(epoch) is not int or epoch < 1: + raise MaintenanceDenied("invalid maintenance receipt") + try: + await session.execute( + text("SELECT public.kf_lock_reserved_maintenance(:actor,:epoch)"), + {"actor": actor_id, "epoch": epoch}, + ) + except DBAPIError as error: + if getattr(error.orig, "sqlstate", None) == "42501": + raise MaintenanceDenied("problem maintenance revoked") from None + raise diff --git a/backend/src/kf_oj/modules/operations/models.py b/backend/src/kf_oj/modules/operations/models.py new file mode 100644 index 0000000..68a480c --- /dev/null +++ b/backend/src/kf_oj/modules/operations/models.py @@ -0,0 +1,29 @@ +"""Append-only bounded maintenance events, independent of evaluation state.""" + +from uuid import UUID + +from sqlalchemy import BigInteger, CheckConstraint, ForeignKey, Index, String +from sqlalchemy.orm import Mapped, mapped_column + +from kf_oj.infra.db.base import Base +from kf_oj.infra.db.columns import CreatedAt + + +class MaintenanceEvent(CreatedAt, Base): + __tablename__ = "problem_maintenance_events" + __table_args__ = ( + CheckConstraint( + "action IN ('upload_created','upload_received','import_ready'," + "'import_failed','import_discarded','import_expired','published'," + "'disabled','restored')", + name="action", + ), + CheckConstraint("authorization_version > 0", name="authority"), + Index("ix_problem_maintenance_events_page", "created_at", "id"), + ) + id: Mapped[UUID] = mapped_column(primary_key=True) + actor_id: Mapped[UUID] = mapped_column(ForeignKey("accounts.id")) + authorization_version: Mapped[int] = mapped_column(BigInteger) + operation_id: Mapped[UUID] = mapped_column() + action: Mapped[str] = mapped_column(String(24)) + target_id: Mapped[UUID | None] = mapped_column() diff --git a/backend/src/kf_oj/modules/operations/public.py b/backend/src/kf_oj/modules/operations/public.py new file mode 100644 index 0000000..77ee1a9 --- /dev/null +++ b/backend/src/kf_oj/modules/operations/public.py @@ -0,0 +1,28 @@ +"""Accepted actor receipts and finite maintenance events; no private data/log text.""" + +from typing import Literal, Protocol +from uuid import UUID + +Action = Literal[ + "upload_created", + "upload_received", + "import_ready", + "import_failed", + "import_discarded", + "import_expired", + "published", + "disabled", + "restored", +] + + +class MaintenanceAuditPort(Protocol): + async def record( + self, + session, + actor_id: UUID, + authorization_version: int, + operation_id: UUID, + action: Action, + target_id: UUID | None = None, + ) -> None: ... diff --git a/backend/src/kf_oj/modules/operations/service.py b/backend/src/kf_oj/modules/operations/service.py new file mode 100644 index 0000000..514c281 --- /dev/null +++ b/backend/src/kf_oj/modules/operations/service.py @@ -0,0 +1,27 @@ +from uuid import uuid4 + +from sqlalchemy import func, select + +from kf_oj.modules.operations.models import MaintenanceEvent +from kf_oj.modules.operations.public import Action + + +class MaintenanceAudit: + async def record( + self, session, actor_id, authorization_version, operation_id, action: Action, target_id=None + ): + if not session.in_transaction(): + raise ValueError("audit requires a write transaction") + # Supplying the trusted DB timestamp avoids ORM INSERT ... RETURNING: + # the ordinary importer has append-only authority and cannot SELECT this table. + session.add( + MaintenanceEvent( + id=uuid4(), + created_at=await session.scalar(select(func.clock_timestamp())), + actor_id=actor_id, + authorization_version=authorization_version, + operation_id=operation_id, + action=action, + target_id=target_id, + ) + ) diff --git a/backend/src/kf_oj/modules/problems/README.md b/backend/src/kf_oj/modules/problems/README.md index 206ced8..fec557c 100644 --- a/backend/src/kf_oj/modules/problems/README.md +++ b/backend/src/kf_oj/modules/problems/README.md @@ -31,3 +31,32 @@ v2 使用 `schema_version: 2` 与 `language_configurations`，每个配置包含 导入同一 slug 的 v2 版本保持原 v1 版本与提交不变。已有提交按其保存的协议版本解析， 不能重新序列化旧 canonical 数据来补入新字段。Java 使用 Main.java 与默认包 Main； Python 使用 main.py。每个新版本的语言集合独立，浏览器不得为旧版本补开新语言。 + + +Web题包导入是独立的待发布流程：`/api/v1/admin/problem-imports` 使用真实维护会话， +写入还要求明确Origin与绑定CSRF。先以稳定操作UUID、ZIP摘要和精确字节数建立容量预约， +再向该操作的 `/archive` 流式PUT；同键重试仍验证实际内容，不会确认异内容。 +列表、详情和丢弃只开放给该操作的管理员本人，响应仅含公开题面、样例、语言预算与有限错误。 +`ready` 尚未发布，不会加入学生题库，也不改变既有题目版本。 + +Web格式固定单层ZIP的stored/deflate，清单名为`problem.json`；公开文件与隐藏用例文件 +必须显式分别声明且不得混用。文件名只接受有界ASCII单层普通名；未知成员、链接/设备、 +加密、嵌套归档、额外前缀/尾随内容、损坏CRC与虚假大小被拒。旧受控目录CLI格式保持原义。 +压缩与实际解压各64 MiB、最多534文件，上传最多4096帧/每帧1 MiB/30秒； +解析期限60秒。staging总预约256 MiB，每次128 MiB，单全局上传/解析槽位， +本人最多10个/全局32个活跃候选，操作24小时到期，解析死亡最多领取两次。 +真实限额由管理capabilities返回；缺少配置或新鲜Worker资源证明时明确不可用。 + +`uv run --locked kf-problem-import-worker` 是普通用户、独立数据库登录的组合入口。 +配置`KF_OJ_IMPORT_STAGING_ROOT`、`KF_OJ_ARTIFACT_ROOT`、可信准入和 +`KF_OJ_IMPORT_WORKER_CGROUP`；staging与工件目录必须归服务用户所有且权限0700。 +服务不会sudo、执行上传内容/学生源码、联网安装依赖或读写评测状态。启动及每次解析 +读回128 MiB/swap0/PIDs16/CPU不超过一核及文件/FD/core上限，错误身份/无界环境拒绝服务。 +解析子进程继承同一个内核文件锁，父进程死亡不能让GC在存活子进程仍持有文件时释放容量。 +租约与fence防止旧结果写入；完整工件持久化、候选和引用原子落库后才返回ready。 +丢弃/失败/到期先清理私有暂存字节再释放预约，保留不可变操作记录与追加式有限审计。 + +资源约束服务模板见`deploy/problem-import-worker.service.example`。API和Worker需可访问 +同一普通用户的私有目录；数据库采用各自最小角色，导入Worker只通过固定身份公开端口 +验证已受理的维护授权。PrivateNetwork模板要求Unix PostgreSQL socket；实际部署前由运维 +设置外部私有配置、专用登录与目录，不包含默认口令，也不代表本轮已经生产部署。 diff --git a/backend/src/kf_oj/modules/problems/candidate_validation.py b/backend/src/kf_oj/modules/problems/candidate_validation.py new file mode 100644 index 0000000..a464d2a --- /dev/null +++ b/backend/src/kf_oj/modules/problems/candidate_validation.py @@ -0,0 +1,97 @@ +"""Pure data validation shared by the bounded child and fenced candidate transaction.""" + +import hashlib +import json +from dataclasses import dataclass +from uuid import uuid4, uuid5 + +from kf_oj.contracts import judging_v2 as v2 +from kf_oj.contracts.admin_problems import CandidatePreview, LegacyImportBudget +from kf_oj.contracts.judging import MAX_SOURCE_BYTES, ArtifactReference, TestCase +from kf_oj.contracts.problems import PublicLanguageConfiguration +from kf_oj.modules.problems.import_format import ImportEnvelope, ProblemImport +from kf_oj.modules.problems.public import JudgingVersion + + +@dataclass(frozen=True) +class ValidatedCandidate: + manifest: dict + preview: CandidatePreview + references: dict[str, ArtifactReference] + + @property + def digest(self): + return hashlib.sha256( + json.dumps( + self.manifest, sort_keys=True, separators=(",", ":"), ensure_ascii=True + ).encode() + ).hexdigest() + + +def validate_candidate(manifest_data, preview_data, references, policy) -> ValidatedCandidate: + manifest = ImportEnvelope.model_validate(manifest_data).root + preview = CandidatePreview.model_validate(preview_data) + refs = {k: ArtifactReference.model_validate(v) for k, v in references.items()} + legacy = isinstance(manifest, ProblemImport) + if ( + preview.slug != manifest.slug + or preview.title != manifest.title + or preview.case_count != len(manifest.cases) + or preview.judging_schema_version != manifest.schema_version + or tuple(v.value for v in preview.languages) != tuple(v.value for v in manifest.languages) + ): + raise ValueError("invalid candidate projection") + expected_budget = ( + LegacyImportBudget( + compile_limits=manifest.compile_limits, + run_limits=manifest.run_limits, + job_wall_time_us=manifest.job_wall_time_us, + ) + if legacy + else None + ) + configs = ( + () + if legacy + else tuple( + PublicLanguageConfiguration.from_configuration(c) + for c in manifest.language_configurations + ) + ) + if preview.legacy_budget != expected_budget or preview.language_configurations != configs: + raise ValueError("invalid candidate budget") + names = {name for c in manifest.cases for name in (c.input_file, c.expected_file)} + if set(refs) != names: + raise ValueError("invalid hidden references") + cases = tuple( + TestCase( + case_id=uuid5(preview.candidate_id, c.key), + input=refs[c.input_file], + expected=refs[c.expected_file], + ) + for c in manifest.cases + ) + projection = JudgingVersion( + preview.candidate_id, + uuid4(), + manifest.languages, + manifest.toolchain if legacy else None, + manifest.compile_limits if legacy else None, + manifest.run_limits if legacy else None, + manifest.job_wall_time_us if legacy else None, + cases, + manifest.schema_version, + () if legacy else manifest.language_configurations, + ) + for language in manifest.languages: + v2.admit_manifest( + policy, + projection.manifest( + uuid4(), + uuid4(), + language, + ArtifactReference(digest="0" * 64, bytes=MAX_SOURCE_BYTES), + ), + 1, + ) + return ValidatedCandidate(manifest.model_dump(mode="json"), preview, refs) diff --git a/backend/src/kf_oj/modules/problems/import_models.py b/backend/src/kf_oj/modules/problems/import_models.py new file mode 100644 index 0000000..b92dc95 --- /dev/null +++ b/backend/src/kf_oj/modules/problems/import_models.py @@ -0,0 +1,121 @@ +"""Private durable imports and immutable candidates; no grading-state write authority.""" + +from datetime import datetime +from uuid import UUID + +from sqlalchemy import ( + BigInteger, + CheckConstraint, + Computed, + DateTime, + ForeignKey, + ForeignKeyConstraint, + Index, + Integer, + SmallInteger, + String, + UniqueConstraint, + text, +) +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, mapped_column + +from kf_oj.infra.db.base import Base +from kf_oj.infra.db.columns import CreatedAt +from kf_oj.modules.problems.models import ProblemVersion + + +class ImportPool(Base): + __tablename__ = "problem_import_pool" + __table_args__ = ( + CheckConstraint("id = 1", name="singleton"), + CheckConstraint("reserved_bytes BETWEEN 0 AND 268435456", name="capacity"), + ) + id: Mapped[int] = mapped_column(SmallInteger, primary_key=True) + reserved_bytes: Mapped[int] = mapped_column(BigInteger, server_default="0") + worker_ready_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + worker_policy_digest: Mapped[str | None] = mapped_column(String(64)) + + +class ProblemImportOperation(CreatedAt, Base): + __tablename__ = "problem_import_operations" + __table_args__ = ( + CheckConstraint( + "state IN ('uploading','uploaded','validating','ready','failed'," + "'published','discarded','expired')", + name="state", + ), + CheckConstraint("archive_digest ~ '^[0-9a-f]{64}$'", name="digest"), + CheckConstraint("archive_bytes BETWEEN 1 AND 67108864", name="bytes"), + CheckConstraint("authorization_version > 0", name="authority"), + CheckConstraint("attempts BETWEEN 0 AND 2 AND fence >= 0", name="attempts"), + CheckConstraint("reserved_bytes IN (0,134217728)", name="reservation"), + CheckConstraint("expires_at > created_at AND updated_at >= created_at", name="times"), + Index("ix_problem_import_owner", "owner_id", "id"), + Index("ix_problem_import_queue", "state", "created_at"), + Index("ix_problem_import_expiry", "expires_at"), + Index("ix_problem_import_reservations", "id", postgresql_where=text("reserved_bytes>0")), + ) + id: Mapped[UUID] = mapped_column(primary_key=True) + owner_id: Mapped[UUID] = mapped_column(ForeignKey("accounts.id")) + authorization_version: Mapped[int] = mapped_column(BigInteger) + archive_digest: Mapped[str] = mapped_column(String(64)) + archive_bytes: Mapped[int] = mapped_column(BigInteger) + state: Mapped[str] = mapped_column(String(16), server_default="uploading") + expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + attempts: Mapped[int] = mapped_column(Integer, server_default="0") + fence: Mapped[int] = mapped_column(BigInteger, server_default="0") + reserved_bytes: Mapped[int] = mapped_column(BigInteger, server_default="134217728") + failure_code: Mapped[str | None] = mapped_column(String(32)) + candidate_id: Mapped[UUID | None] = mapped_column( + ForeignKey("problem_candidates.id", use_alter=True, name="fk_problem_import_candidate") + ) + published_version_id: Mapped[UUID | None] = mapped_column(ForeignKey(ProblemVersion.id)) + + +class ProblemCandidate(CreatedAt, Base): + __tablename__ = "problem_candidates" + __table_args__ = ( + UniqueConstraint("operation_id"), + CheckConstraint("manifest_digest ~ '^[0-9a-f]{64}$'", name="digest"), + CheckConstraint("case_count BETWEEN 1 AND 256", name="cases"), + CheckConstraint("octet_length(preview::text) <= 4194304", name="preview"), + CheckConstraint("octet_length(manifest::text) <= 262144", name="manifest"), + ) + id: Mapped[UUID] = mapped_column(primary_key=True) + operation_id: Mapped[UUID] = mapped_column(ForeignKey("problem_import_operations.id")) + manifest_digest: Mapped[str] = mapped_column(String(64)) + manifest: Mapped[dict] = mapped_column(JSONB) + preview: Mapped[dict] = mapped_column(JSONB) + case_count: Mapped[int] = mapped_column(Integer) + + +class CandidateCase(Base): + __tablename__ = "problem_candidate_cases" + __table_args__ = ( + CheckConstraint("position BETWEEN 0 AND 255", name="position"), + CheckConstraint( + "input_bytes BETWEEN 0 AND 67108864 AND expected_bytes BETWEEN 0 AND 67108864", + name="bytes", + ), + ForeignKeyConstraint( + ["input_digest", "artifact_state"], ["artifacts.digest", "artifacts.state"] + ), + ForeignKeyConstraint( + ["expected_digest", "artifact_state"], + ["artifacts.digest", "artifacts.state"], + name="fk_candidate_cases_expected_artifacts", + ), + ) + candidate_id: Mapped[UUID] = mapped_column( + ForeignKey("problem_candidates.id"), primary_key=True + ) + position: Mapped[int] = mapped_column(Integer, primary_key=True) + key: Mapped[str] = mapped_column(String(64)) + input_digest: Mapped[str] = mapped_column(String(64)) + input_bytes: Mapped[int] = mapped_column(BigInteger) + expected_digest: Mapped[str] = mapped_column(String(64)) + expected_bytes: Mapped[int] = mapped_column(BigInteger) + artifact_state: Mapped[str] = mapped_column(String(8), Computed("'ready'", persisted=True)) diff --git a/backend/src/kf_oj/modules/problems/import_worker.py b/backend/src/kf_oj/modules/problems/import_worker.py new file mode 100644 index 0000000..604d0e9 --- /dev/null +++ b/backend/src/kf_oj/modules/problems/import_worker.py @@ -0,0 +1,430 @@ +"""Fenced ordinary-user import consumption. No student execution or evaluation DB access.""" + +import asyncio +import json +import os +import resource +import sys +from datetime import timedelta +from pathlib import Path +from uuid import uuid4 + +from sqlalchemy import delete, func, or_, select, text +from sqlalchemy.exc import DBAPIError + +from kf_oj.contracts.admin_problems import MAX_IMPORT_ATTEMPTS, VALIDATION_DEADLINE_SECONDS +from kf_oj.infra.artifacts.store import ArtifactError +from kf_oj.modules.identity.public import MaintenanceDenied +from kf_oj.modules.problems.candidate_validation import validate_candidate +from kf_oj.modules.problems.import_models import ( + CandidateCase, + ImportPool, + ProblemCandidate, +) +from kf_oj.modules.problems.import_models import ( + ProblemImportOperation as Operation, +) +from kf_oj.modules.problems.imports import ACTIVE, ImportUnavailable, ProblemImports, policy_digest + + +def verify_worker_limits(configured_group: Path): + if os.getuid() == 0: + raise ImportUnavailable("import worker must be ordinary") + actual = next( + line.split(":", 2)[2] + for line in Path("/proc/self/cgroup").read_text().splitlines() + if line.startswith("0::") + ) + group = Path("/sys/fs/cgroup") / actual.lstrip("/") + if group.resolve() != configured_group.absolute().resolve(): + raise ImportUnavailable("import worker cgroup mismatch") + caps = { + n: (group / n).read_text().strip() + for n in ("memory.max", "memory.swap.max", "pids.max", "cpu.max") + } + if ( + caps["memory.max"] == "max" + or not 0 < int(caps["memory.max"]) <= 134217728 + or caps["memory.swap.max"] != "0" + or caps["pids.max"] == "max" + or not 0 < int(caps["pids.max"]) <= 16 + ): + raise ImportUnavailable("import worker resource limits unavailable") + quota, period = caps["cpu.max"].split() + if quota == "max" or int(quota) > int(period): + raise ImportUnavailable("import worker CPU limit unavailable") + for name, maximum in ( + (resource.RLIMIT_FSIZE, 67108864), + (resource.RLIMIT_NOFILE, 64), + (resource.RLIMIT_CORE, 0), + ): + limits = resource.getrlimit(name) + if any(v == resource.RLIM_INFINITY or v > maximum for v in limits): + raise ImportUnavailable("import worker process limits unavailable") + return caps + + +class ImportWorker: + def __init__(self, sessions, staging, artifacts, authority, audit, policy, cgroup): + ( + self.sessions, + self.staging, + self.artifacts, + self.authority, + self.audit, + self.policy, + self.cgroup, + ) = sessions, staging, artifacts, authority, audit, policy, cgroup + + async def heartbeat(self): + verify_worker_limits(self.cgroup) + async with self.sessions.begin() as session: + safe = await session.scalar( + text(""" + SELECT rolcanlogin AND NOT(rolsuper OR rolcreatedb OR rolcreaterole + OR rolreplication OR rolbypassrls) + AND pg_has_role(current_user,'kf_oj_problem_import','USAGE') + AND NOT pg_has_role(current_user,'kf_oj_platform','MEMBER') + AND NOT pg_has_role(current_user,'kf_oj_coordinator','MEMBER') + AND NOT pg_has_role(current_user,'kf_oj_identity_operator','MEMBER') + AND NOT pg_has_role(current_user,'kf_oj_identity_authority_owner','MEMBER') + FROM pg_roles WHERE rolname=current_user + """) + ) + if not safe: + raise ImportUnavailable("dedicated unprivileged import login required") + pool = await session.get(ImportPool, 1, with_for_update=True) + if pool is None: + raise ImportUnavailable("import pool unavailable") + pool.worker_ready_until = await session.scalar( + select(func.clock_timestamp()) + ) + timedelta(seconds=90) + pool.worker_policy_digest = policy_digest(self.policy) + + async def _release(self, operation_id): + await asyncio.to_thread(self.staging.remove, operation_id) + async with self.sessions.begin() as session: + pool = await ProblemImports.pool(session) + op = await session.get(Operation, operation_id, with_for_update=True) + pool.reserved_bytes -= op.reserved_bytes + op.reserved_bytes = 0 + op.lease_until = None + + async def collect(self): + # Caller owns the root flock, so a live upload/parser cannot be unlinked or + # have its reservation reused, even if its DB lease expired while suspended. + async with self.sessions() as session: + now = await session.scalar(select(func.clock_timestamp())) + names = list( + await session.scalars( + select(Operation.id) + .where( + or_( + Operation.reserved_bytes > 0, + Operation.state.in_(ACTIVE) & (Operation.expires_at <= now), + ) + ) + .order_by(Operation.updated_at, Operation.id) + .limit(32) + ) + ) + garbage = list( + await session.scalars( + select(ProblemCandidate.operation_id) + .join(Operation, Operation.id == ProblemCandidate.operation_id) + .where(Operation.state.in_(("discarded", "expired", "failed"))) + .limit(32) + ) + ) + names = list(dict.fromkeys(names + garbage))[:32] + for identifier in names: + async with self.sessions.begin() as session: + await ProblemImports.pool(session) + op = await session.get(Operation, identifier, with_for_update=True) + now = await session.scalar(select(func.clock_timestamp())) + if op.state in ACTIVE and op.expires_at <= now: + op.state = "expired" + op.failure_code = "expired" + op.updated_at = now + op.candidate_id = None + op.fence += 1 + await self.audit.record( + session, op.owner_id, op.authorization_version, op.id, "import_expired" + ) + elif op.state == "uploading" and op.lease_until and op.lease_until <= now: + op.state = "failed" + op.failure_code = "upload_interrupted" + op.updated_at = now + op.fence += 1 + await self.audit.record( + session, op.owner_id, op.authorization_version, op.id, "import_failed" + ) + elif op.state == "validating" and op.lease_until and op.lease_until <= now: + if op.attempts < MAX_IMPORT_ATTEMPTS: + op.state = "uploaded" + op.failure_code = None + op.lease_until = None + op.fence += 1 + op.updated_at = now + await asyncio.to_thread(self.staging.reset_validation, op.id) + else: + op.state = "failed" + op.failure_code = "validation_unavailable" + op.updated_at = now + op.fence += 1 + await self.audit.record( + session, op.owner_id, op.authorization_version, op.id, "import_failed" + ) + if op.state in ("discarded", "expired", "failed"): + op.candidate_id = None + await session.flush() + candidate = await session.scalar( + select(ProblemCandidate).where(ProblemCandidate.operation_id == op.id) + ) + if candidate: + await session.execute( + delete(CandidateCase).where(CandidateCase.candidate_id == candidate.id) + ) + await session.delete(candidate) + release = ( + op.state in ("ready", "published", "discarded", "expired", "failed") + and op.reserved_bytes > 0 + ) + if release: + await self._release(identifier) + # Uncommitted staging objects after a failed transaction are removed only under + # the same kernel lock and only from this service's private, bounded namespace. + for identifier in self.staging.operation_ids(): + async with self.sessions() as session: + exists = await session.get(Operation, identifier) + if exists is None: + await asyncio.to_thread(self.staging.remove, identifier) + + async def once(self): + await self.heartbeat() + try: + with self.staging.exclusive() as lock: + await self.collect() + async with self.sessions.begin() as session: + await ProblemImports.pool(session) + now = await session.scalar(select(func.clock_timestamp())) + busy = await session.scalar( + select(func.count()) + .select_from(Operation) + .where( + Operation.state.in_(("uploading", "validating")), + Operation.lease_until > now, + ) + ) + if busy: + return False + op = await session.scalar( + select(Operation) + .where( + Operation.state == "uploaded", + Operation.attempts < MAX_IMPORT_ATTEMPTS, + Operation.expires_at > now, + ) + .order_by(Operation.created_at, Operation.id) + .limit(1) + .with_for_update() + ) + if op is None: + return False + op.attempts += 1 + op.fence += 1 + op.state = "validating" + op.updated_at = now + op.lease_until = now + timedelta(seconds=VALIDATION_DEADLINE_SECONDS + 15) + identifier, fence, size, digest, actor, epoch = ( + op.id, + op.fence, + op.archive_bytes, + op.archive_digest, + op.owner_id, + op.authorization_version, + ) + try: + async with self.sessions.begin() as session: + await self.authority.require_reserved_maintenance(session, actor, epoch) + candidate_id = uuid4() + result = await self.parse(identifier, size, digest, candidate_id, lock) + if "failure_code" in result: + await self.fail(identifier, fence, result["failure_code"]) + return True + candidate = validate_candidate( + result["manifest"], result["preview"], result["references"], self.policy + ) + if ( + candidate.digest != result["manifest_digest"] + or candidate.preview.candidate_id != candidate_id + ): + raise ValueError("invalid candidate receipt") + directory = self.staging.operation(identifier) + members = self.staging.members(directory) + try: + async with self.sessions.begin() as session: + await self.authority.require_reserved_maintenance(session, actor, epoch) + await ProblemImports.pool(session) + op = await session.get(Operation, identifier, with_for_update=True) + now = await session.scalar(select(func.clock_timestamp())) + if ( + op.state != "validating" + or op.fence != fence + or op.lease_until <= now + or op.expires_at <= now + ): + raise ValueError("stale candidate receipt") + for name, reference in sorted( + candidate.references.items(), key=lambda item: item[1].digest + ): + fd = os.open( + name, + os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC, + dir_fd=members, + ) + try: + await self.artifacts.put_file(session, fd, reference) + finally: + os.close(fd) + session.add( + ProblemCandidate( + id=candidate_id, + operation_id=identifier, + manifest_digest=candidate.digest, + manifest=candidate.manifest, + preview=candidate.preview.model_dump(mode="json"), + case_count=candidate.preview.case_count, + ) + ) + await session.flush() + for position, case in enumerate(candidate.manifest["cases"]): + input_ref, answer_ref = ( + candidate.references[case["input_file"]], + candidate.references[case["expected_file"]], + ) + session.add( + CandidateCase( + candidate_id=candidate_id, + position=position, + key=case["key"], + input_digest=input_ref.digest, + input_bytes=input_ref.bytes, + expected_digest=answer_ref.digest, + expected_bytes=answer_ref.bytes, + ) + ) + await session.flush() + # Adoption must not turn an expired lease into a ready candidate. + completed_at = await session.scalar(select(func.clock_timestamp())) + if op.lease_until <= completed_at or op.expires_at <= completed_at: + raise ValueError("candidate completion lease expired") + op.state = "ready" + op.candidate_id = candidate_id + op.updated_at = completed_at + op.lease_until = None + await self.audit.record( + session, actor, epoch, identifier, "import_ready", candidate_id + ) + finally: + os.close(members) + os.close(directory) + await self._release(identifier) + except MaintenanceDenied: + await self.fail(identifier, fence, "authority_revoked") + except (ArtifactError, OSError): + await self.fail(identifier, fence, "storage_unavailable") + except (ValueError, DBAPIError): + await self.fail(identifier, fence, "validation_unavailable") + return True + except ArtifactError: + return False # Another live writer holds the one global slot. + + async def fail(self, identifier, fence, code): + from kf_oj.contracts.admin_problems import ImportFailure + + code = ImportFailure(code).value + async with self.sessions.begin() as session: + await ProblemImports.pool(session) + op = await session.get(Operation, identifier, with_for_update=True) + if op.state != "validating" or op.fence != fence: + return + op.state = "failed" + op.failure_code = code + op.updated_at = await session.scalar(select(func.clock_timestamp())) + op.lease_until = None + await self.audit.record( + session, op.owner_id, op.authorization_version, op.id, "import_failed" + ) + await self._release(identifier) + + async def parse(self, identifier, size, digest, candidate, lock): + verify_worker_limits(self.cgroup) + directory = self.staging.operation(identifier) + archive = members = None + try: + archive = os.open( + "archive", + os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC, + dir_fd=directory, + ) + members = self.staging.members(directory, create=True) + # Preserve only the trusted runtime import roots, without inheriting DB/ + # service credentials. Curated runtimes may keep dependencies outside site-packages. + env = { + "PATH": str(Path(sys.executable).parent), + "PYTHONPATH": os.pathsep.join( + dict.fromkeys( + [str(Path(__file__).parents[3])] + + [p for p in os.environ.get("PYTHONPATH", "").split(os.pathsep) if p] + ) + ), + "PYTHONUNBUFFERED": "1", + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONNOUSERSITE": "1", + "LC_ALL": "C.UTF-8", + } + # Minimal environment may require the trusted dynamic-loader path in a curated VM. + if "LD_LIBRARY_PATH" in os.environ: + env["LD_LIBRARY_PATH"] = os.environ["LD_LIBRARY_PATH"] + child = await asyncio.create_subprocess_exec( + sys.executable, + "-m", + "kf_oj.workers._archive_child", + str(archive), + str(members), + str(lock), + str(size), + digest, + str(candidate), + self.policy.model_dump_json(), + env=env, + pass_fds=(archive, members, lock), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL, + ) + try: + async with asyncio.timeout(VALIDATION_DEADLINE_SECONDS + 5): + output, _ = await child.communicate() + except BaseException: + child.kill() + await child.wait() + raise + if child.returncode != 0 or not 0 < len(output) <= 4 * 1024**2: + return { + "failure_code": "validation_deadline" + if child.returncode == -14 + else "validation_unavailable" + } + result = json.loads(output) + if not isinstance(result, dict): + raise ValueError("invalid child result") + return result + except TimeoutError: + return {"failure_code": "validation_deadline"} + finally: + if members is not None: + os.close(members) + if archive is not None: + os.close(archive) + os.close(directory) diff --git a/backend/src/kf_oj/modules/problems/imports.py b/backend/src/kf_oj/modules/problems/imports.py new file mode 100644 index 0000000..d4be850 --- /dev/null +++ b/backend/src/kf_oj/modules/problems/imports.py @@ -0,0 +1,438 @@ +"""Authenticated durable import lifecycle; filesystem work remains behind private adapters.""" + +import asyncio +import hashlib +import os +from contextlib import asynccontextmanager +from datetime import timedelta +from uuid import UUID + +from sqlalchemy import func, select, text + +from kf_oj.contracts.admin_problems import ( + IMPORT_LIFETIME_SECONDS, + MAX_OWNER_CANDIDATES, + MAX_TOTAL_CANDIDATES, + MAX_UPLOAD_CHUNK_BYTES, + MAX_UPLOAD_CHUNKS, + STAGING_MAX_BYTES, + STAGING_RESERVATION_BYTES, + UPLOAD_DEADLINE_SECONDS, + CandidatePreview, + CreateProblemImport, + ProblemImportCapabilities, + ProblemImportDetail, + ProblemImportPage, + ProblemImportSummary, +) +from kf_oj.infra.artifacts.staging import ImportStaging +from kf_oj.infra.artifacts.store import ArtifactError +from kf_oj.modules.identity.public import MaintenanceDenied, ProblemMaintenancePort +from kf_oj.modules.operations.public import MaintenanceAuditPort +from kf_oj.modules.problems.import_models import ImportPool, ProblemCandidate +from kf_oj.modules.problems.import_models import ProblemImportOperation as Operation + +ACTIVE = ("uploading", "uploaded", "validating", "ready") + + +class ImportConflict(ValueError): + pass + + +class ImportBadDigest(ImportConflict): + pass + + +class ImportMissing(ValueError): + pass + + +class ImportUnavailable(ValueError): + pass + + +def summary(operation) -> ProblemImportSummary: + return ProblemImportSummary( + operation_id=operation.id, + state=operation.state, + expected_archive_digest=operation.archive_digest, + archive_bytes=operation.archive_bytes, + created_at=operation.created_at, + expires_at=operation.expires_at, + updated_at=operation.updated_at, + failure_code=operation.failure_code, + candidate_id=operation.candidate_id, + published_version_id=operation.published_version_id, + ) + + +def policy_digest(policy): + return hashlib.sha256(policy.model_dump_json().encode()).hexdigest() + + +class ProblemImports: + def __init__( + self, + sessions, + staging: ImportStaging, + identity: ProblemMaintenancePort, + audit: MaintenanceAuditPort, + policy, + ): + self.sessions, self.staging, self.identity, self.audit, self.policy = ( + sessions, + staging, + identity, + audit, + policy, + ) + + @asynccontextmanager + async def write(self, principal): + async with self.sessions.begin() as session: + await session.execute(text("SET LOCAL statement_timeout='5s'")) + await session.execute(text("SET LOCAL lock_timeout='2s'")) + await self.identity.require_problem_maintenance(session, principal) + yield session + + @staticmethod + async def pool(session): + pool = await session.get(ImportPool, 1, with_for_update=True) + if pool is None: + raise ImportUnavailable("import capacity unavailable") + actual = await session.scalar( + select(func.coalesce(func.sum(Operation.reserved_bytes), 0)).where( + Operation.reserved_bytes > 0 + ) + ) + if pool.reserved_bytes != actual: + raise ImportUnavailable("import capacity inconsistent") + return pool + + async def capabilities(self): + async with self.sessions() as session: + pool = await session.get(ImportPool, 1) + now = await session.scalar(select(func.clock_timestamp())) + available = bool( + pool + and pool.worker_ready_until + and pool.worker_ready_until > now + and pool.worker_policy_digest == policy_digest(self.policy) + ) + return ProblemImportCapabilities( + uploads_available=available, validation_available=available + ) + + async def create(self, principal, request: CreateProblemImport): + # Immutable receipt retries need no filesystem slot, including while a parser/GC + # owns the root lock. Missing receipts still enter the locked creation path below. + async with self.write(principal) as session: + existing = await session.get(Operation, request.operation_id) + if existing: + if ( + existing.owner_id != principal.account_id + or existing.archive_digest != request.archive_digest + or existing.archive_bytes != request.archive_bytes + ): + raise ImportConflict("operation key already bound") + return summary(existing) + try: + with self.staging.exclusive(): + await asyncio.to_thread(self.staging.usage) + async with self.write(principal) as session: + pool = await self.pool(session) + existing = await session.get( + Operation, request.operation_id, with_for_update=True + ) + if existing: + if ( + existing.owner_id != principal.account_id + or existing.archive_digest != request.archive_digest + or existing.archive_bytes != request.archive_bytes + ): + raise ImportConflict("operation key already bound") + return summary(existing) + now = await session.scalar(select(func.clock_timestamp())) + if ( + not pool.worker_ready_until + or pool.worker_ready_until <= now + or pool.worker_policy_digest != policy_digest(self.policy) + ): + raise ImportUnavailable("import validation unavailable") + global_count = await session.scalar( + select(func.count()) + .select_from(Operation) + .where(Operation.state.in_(ACTIVE)) + ) + owner_count = await session.scalar( + select(func.count()) + .select_from(Operation) + .where( + Operation.state.in_(ACTIVE), Operation.owner_id == principal.account_id + ) + ) + busy = await session.scalar( + select(func.count()) + .select_from(Operation) + .where( + Operation.state.in_(("uploading", "validating")), + Operation.lease_until > now, + ) + ) + if ( + global_count >= MAX_TOTAL_CANDIDATES + or owner_count >= MAX_OWNER_CANDIDATES + or busy + or pool.reserved_bytes + STAGING_RESERVATION_BYTES > STAGING_MAX_BYTES + ): + raise ImportUnavailable("import capacity exhausted") + operation = Operation( + id=request.operation_id, + owner_id=principal.account_id, + authorization_version=principal.authorization_version, + archive_digest=request.archive_digest, + archive_bytes=request.archive_bytes, + state="uploading", + created_at=now, + updated_at=now, + expires_at=now + timedelta(seconds=IMPORT_LIFETIME_SECONDS), + lease_until=now + timedelta(seconds=UPLOAD_DEADLINE_SECONDS + 15), + attempts=0, + fence=0, + reserved_bytes=STAGING_RESERVATION_BYTES, + ) + session.add(operation) + pool.reserved_bytes += STAGING_RESERVATION_BYTES + await self.audit.record( + session, + principal.account_id, + principal.authorization_version, + operation.id, + "upload_created", + ) + await session.flush() + return summary(operation) + except ArtifactError: + raise ImportUnavailable("import storage unavailable") from None + + async def detail(self, principal, operation_id): + # Recheck live authority even when a principal was injected by an internal caller. + async with self.write(principal) as session: + op = await session.get(Operation, operation_id) + if op is None or op.owner_id != principal.account_id: + raise ImportMissing("import unavailable") + candidate = ( + await session.get(ProblemCandidate, op.candidate_id) if op.candidate_id else None + ) + return ProblemImportDetail( + **summary(op).model_dump(), + candidate=CandidatePreview.model_validate(candidate.preview) if candidate else None, + ) + + async def list(self, principal, after=None, limit=20): + if not 1 <= limit <= 100: + raise ValueError("invalid page limit") + async with self.write(principal) as session: + query = select(Operation).where(Operation.owner_id == principal.account_id) + if after: + query = query.where(Operation.id > after) + rows = list(await session.scalars(query.order_by(Operation.id).limit(limit + 1))) + return ProblemImportPage( + items=tuple(summary(o) for o in rows[:limit]), + next_after=rows[limit - 1].id if len(rows) > limit else None, + ) + + async def upload(self, principal, operation_id: UUID, chunks, content_length: int | None): + # Reject missing/foreign receipts before consulting shared filesystem capacity. + # The locked transaction below still rechecks authority and ownership before writes. + async with self.write(principal) as session: + op = await session.get(Operation, operation_id) + if op is None or op.owner_id != principal.account_id: + raise ImportMissing("import unavailable") + if content_length is not None and content_length != op.archive_bytes: + raise ImportConflict("archive length mismatch") + descriptor = operation_fd = None + fence = None + received = 0 + chunk_count = 0 + digest = hashlib.sha256() + try: + with self.staging.exclusive(): + async with self.write(principal) as session: + await self.pool(session) + op = await session.get(Operation, operation_id, with_for_update=True) + if not op or op.owner_id != principal.account_id: + raise ImportMissing("import unavailable") + accepted = op.state in ("uploaded", "validating", "ready", "published") + now = await session.scalar(select(func.clock_timestamp())) + if ( + not accepted + and ( + op.state != "uploading" + or op.lease_until is None + or op.lease_until <= now + ) + ) or op.expires_at <= now: + raise ImportConflict("upload no longer available") + if content_length is not None and content_length != op.archive_bytes: + raise ImportConflict("archive length mismatch") + if not accepted: + op.fence += 1 + fence = op.fence + op.lease_until = now + timedelta(seconds=UPLOAD_DEADLINE_SECONDS + 15) + expected_bytes, expected_digest = op.archive_bytes, op.archive_digest + if not accepted: + operation_fd = self.staging.operation(operation_id, create=True) + descriptor = os.open( + "archive", + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, + 0o600, + dir_fd=operation_fd, + ) + async with asyncio.timeout(UPLOAD_DEADLINE_SECONDS): + async for chunk in chunks: + chunk_count += 1 + received += len(chunk) + if ( + type(chunk) is not bytes + or len(chunk) > MAX_UPLOAD_CHUNK_BYTES + or chunk_count > MAX_UPLOAD_CHUNKS + or received > expected_bytes + ): + raise ImportConflict("archive bytes outside policy") + digest.update(chunk) + if accepted: + continue + # Cancellation waits for the finite writer outside the event loop. + task = asyncio.create_task(asyncio.to_thread(write_all, descriptor, chunk)) + try: + await asyncio.shield(task) + except BaseException: + await task + raise + if received != expected_bytes or digest.hexdigest() != expected_digest: + raise ImportBadDigest("archive digest or length mismatch") + if accepted: + async with self.write(principal) as session: + op = await session.get(Operation, operation_id) + if not op or op.state not in ( + "uploaded", + "validating", + "ready", + "published", + ): + raise ImportConflict("import receipt no longer available") + return summary(op) + await asyncio.to_thread(os.fsync, descriptor) + os.close(descriptor) + descriptor = None + os.fsync(operation_fd) + async with self.write(principal) as session: + await self.pool(session) + op = await session.get(Operation, operation_id, with_for_update=True) + now = await session.scalar(select(func.clock_timestamp())) + if op.state != "uploading" or op.fence != fence or op.lease_until <= now: + raise ImportConflict("upload receipt expired") + op.state = "uploaded" + op.lease_until = None + op.updated_at = now + await self.audit.record( + session, op.owner_id, op.authorization_version, op.id, "upload_received" + ) + result = summary(op) + return result + except BaseException as error: + # Failure bookkeeping cannot authorize a new business write. It only fences this + # accepted upload and releases capacity after physical bytes have been removed. + if descriptor is not None: + os.close(descriptor) + descriptor = None + if operation_fd is not None: + os.close(operation_fd) + operation_fd = None + if fence is not None: + try: + with self.staging.exclusive(): + await asyncio.to_thread(self.staging.remove, operation_id) + async with self.sessions.begin() as session: + pool = await self.pool(session) + op = await session.get(Operation, operation_id, with_for_update=True) + if op and op.state == "uploading" and op.fence == fence: + op.state = "failed" + op.failure_code = ( + "upload_deadline" + if isinstance(error, TimeoutError) + else "digest_mismatch" + if isinstance(error, ImportBadDigest) + else "authority_revoked" + if isinstance(error, MaintenanceDenied) + else "storage_unavailable" + if isinstance(error, OSError) + else "upload_interrupted" + ) + op.updated_at = await session.scalar(select(func.clock_timestamp())) + op.lease_until = None + pool.reserved_bytes -= op.reserved_bytes + op.reserved_bytes = 0 + await self.audit.record( + session, + op.owner_id, + op.authorization_version, + op.id, + "import_failed", + ) + except (ArtifactError, OSError): + pass # Durable reservation retained for fenced GC. + raise + finally: + if descriptor is not None: + os.close(descriptor) + if operation_fd is not None: + os.close(operation_fd) + + async def discard(self, principal, operation_id): + async with self.write(principal) as session: + op = await session.get(Operation, operation_id) + if op is None or op.owner_id != principal.account_id: + raise ImportMissing("import unavailable") + if op.state == "discarded": + return summary(op) + try: + with self.staging.exclusive(): + async with self.write(principal) as session: + pool = await self.pool(session) + op = await session.get(Operation, operation_id, with_for_update=True) + if not op or op.owner_id != principal.account_id: + raise ImportMissing("import unavailable") + if op.state == "published": + raise ImportConflict("published import cannot be discarded") + if op.state == "discarded": + return summary(op) + await asyncio.to_thread(self.staging.remove, operation_id) + op.state = "discarded" + op.candidate_id = None + op.failure_code = None + op.lease_until = None + op.fence += 1 + op.updated_at = await session.scalar(select(func.clock_timestamp())) + pool.reserved_bytes -= op.reserved_bytes + op.reserved_bytes = 0 + await self.audit.record( + session, + principal.account_id, + principal.authorization_version, + op.id, + "import_discarded", + ) + return summary(op) + except ArtifactError: + raise ImportUnavailable("import storage unavailable") from None + + +def write_all(descriptor, data): + remaining = memoryview(data) + while remaining: + size = os.write(descriptor, remaining) + if size <= 0: + raise OSError("short staging write") + remaining = remaining[size:] diff --git a/backend/src/kf_oj/modules/problems/web_format.py b/backend/src/kf_oj/modules/problems/web_format.py new file mode 100644 index 0000000..417257d --- /dev/null +++ b/backend/src/kf_oj/modules/problems/web_format.py @@ -0,0 +1,31 @@ +"""Stricter Web package declarations; the accepted local CLI format remains unchanged.""" + +from dataclasses import dataclass + +from kf_oj.contracts.judging import decode_wire +from kf_oj.modules.problems.import_format import ImportEnvelope, ProblemImport, ProblemImportV2 + +ARCHIVE_SUFFIXES = (".zip", ".jar", ".war", ".tar", ".tgz", ".gz", ".bz2", ".xz", ".7z", ".rar") + + +@dataclass(frozen=True) +class WebDeclaration: + manifest: ProblemImport | ProblemImportV2 + members: frozenset[str] + + +def decode_web_declaration(data: bytes) -> WebDeclaration: + # decode_wire rejects duplicate JSON keys, unknown fields and noncanonical + # protocol values before either local paths or node policy can be considered. + manifest = decode_wire(ImportEnvelope, data).root + hidden = {name for c in manifest.cases for name in (c.input_file, c.expected_file)} + samples = {name for s in manifest.samples for name in (s.input_file, s.output_file)} + metadata = {"problem.json", manifest.statement_file} + if len(metadata) != 2 or hidden & (samples | metadata) or samples & metadata: + raise ValueError("public and hidden package declarations must be separate") + members = frozenset(metadata | hidden | samples) + if len({name.lower() for name in members}) != len(members): + raise ValueError("package names must be case-insensitively unique") + if any(name.lower().endswith(ARCHIVE_SUFFIXES) for name in members): + raise ValueError("nested archive declarations are unsupported") + return WebDeclaration(manifest, members) diff --git a/backend/src/kf_oj/workers/_archive_child.py b/backend/src/kf_oj/workers/_archive_child.py new file mode 100644 index 0000000..fca8444 --- /dev/null +++ b/backend/src/kf_oj/workers/_archive_child.py @@ -0,0 +1,116 @@ +"""Fixed data-only child. It inherits no DB credentials and executes no uploaded contents.""" + +import base64 +import json +import os +import resource +import signal +import sys +import time +from uuid import UUID + +from pydantic import TypeAdapter + +from kf_oj.contracts.admin_problems import ( + VALIDATION_DEADLINE_SECONDS, + CandidatePreview, + LegacyImportBudget, +) +from kf_oj.contracts.judging_v2 import NodeAdmission +from kf_oj.contracts.problems import PublicLanguageConfiguration, PublicSample +from kf_oj.infra.artifacts.archive_reader import ArchiveError, ArchiveReader +from kf_oj.modules.problems.candidate_validation import validate_candidate +from kf_oj.modules.problems.import_format import ProblemImport +from kf_oj.modules.problems.web_format import decode_web_declaration + + +def main(): + # These are ordinary downward-only process limits, independent of the parent's life. + resource.setrlimit(resource.RLIMIT_AS, (128 * 1024**2, 128 * 1024**2)) + resource.setrlimit(resource.RLIMIT_CPU, (60, 60)) + resource.setrlimit(resource.RLIMIT_FSIZE, (64 * 1024**2, 64 * 1024**2)) + resource.setrlimit(resource.RLIMIT_NOFILE, (64, 64)) + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + signal.alarm(VALIDATION_DEADLINE_SECONDS) + archive, directory, lock, size, digest, candidate, policy_json = sys.argv[1:] + archive, directory, lock = int(archive), int(directory), int(lock) + os.fstat(lock) # Held inherited flock prevents GC releasing capacity while any child lives. + deadline = time.monotonic() + VALIDATION_DEADLINE_SECONDS + result = {"failure_code": "invalid_archive"} + try: + reader = ArchiveReader(archive, int(size), deadline=deadline) + reader.verify_digest(digest) + result["failure_code"] = "invalid_manifest" + declared = decode_web_declaration(reader.read("problem.json", 256 * 1024)) + if {m.name for m in reader.members} != set(declared.members): + raise ValueError("unexpected package members") + manifest = declared.manifest + result["failure_code"] = "invalid_data" + statement = reader.read(manifest.statement_file, 256 * 1024).decode("utf-8") + sample_data = { + name: reader.read(name, 65536) + for name in sorted({n for s in manifest.samples for n in (s.input_file, s.output_file)}) + } + samples = tuple( + PublicSample( + input=base64.b64encode(sample_data[s.input_file]).decode("ascii"), + output=base64.b64encode(sample_data[s.output_file]).decode("ascii"), + ) + for s in manifest.samples + ) + references = reader.extract_remaining(directory) + reader.verify_unchanged() + legacy = isinstance(manifest, ProblemImport) + preview = CandidatePreview( + candidate_id=UUID(candidate), + slug=manifest.slug, + title=manifest.title, + statement=statement, + samples=samples, + judging_schema_version=manifest.schema_version, + languages=manifest.languages, + legacy_budget=LegacyImportBudget( + compile_limits=manifest.compile_limits, + run_limits=manifest.run_limits, + job_wall_time_us=manifest.job_wall_time_us, + ) + if legacy + else None, + language_configurations=() + if legacy + else tuple( + PublicLanguageConfiguration.from_configuration(c) + for c in manifest.language_configurations + ), + case_count=len(manifest.cases), + ) + result["failure_code"] = "policy_rejected" + validated = validate_candidate( + manifest.model_dump(mode="json"), + preview.model_dump(mode="json"), + {k: {"digest": v[0], "bytes": v[1]} for k, v in references.items()}, + TypeAdapter(NodeAdmission).validate_json(policy_json), + ) + result = { + "manifest": validated.manifest, + "preview": validated.preview.model_dump(mode="json"), + "references": {k: v.model_dump(mode="json") for k, v in validated.references.items()}, + "manifest_digest": validated.digest, + } + except ArchiveError as error: + result = { + "failure_code": "digest_mismatch" + if str(error) == "archive_digest_mismatch" + else "invalid_archive" + } + except (ValueError, OSError, MemoryError): + result = {"failure_code": result["failure_code"]} + payload = json.dumps(result, separators=(",", ":"), ensure_ascii=True).encode() + if len(payload) > 4 * 1024**2: + payload = b'{"failure_code":"invalid_data"}' + sys.stdout.buffer.write(payload) + sys.stdout.buffer.flush() + + +if __name__ == "__main__": + main() diff --git a/backend/src/kf_oj/workers/problem_import.py b/backend/src/kf_oj/workers/problem_import.py new file mode 100644 index 0000000..17d8983 --- /dev/null +++ b/backend/src/kf_oj/workers/problem_import.py @@ -0,0 +1,66 @@ +"""Ordinary import service composition. Provision its dedicated DB role and cgroup separately.""" + +import asyncio +import sys + +from sqlalchemy.exc import DBAPIError + +from kf_oj.core.settings import Settings +from kf_oj.infra.artifacts.repository import ArtifactRepository +from kf_oj.infra.artifacts.staging import ImportStaging +from kf_oj.infra.artifacts.store import ArtifactStore +from kf_oj.infra.db.session import build_engine, build_session_factory +from kf_oj.modules.identity.service import ReservedMaintenance +from kf_oj.modules.operations.service import MaintenanceAudit +from kf_oj.modules.problems.import_worker import ImportWorker, verify_worker_limits + + +async def run(): + settings = Settings() + if ( + settings.import_staging_root is None + or settings.artifact_root is None + or settings.judge_admission is None + or settings.import_worker_cgroup is None + ): + raise ValueError("import service configuration required") + verify_worker_limits(settings.import_worker_cgroup) + engine = build_engine(settings) + store = ArtifactStore( + settings.artifact_root, + max_bytes=settings.artifact_max_bytes, + max_files=settings.artifact_max_files, + ) + staging = ImportStaging(settings.import_staging_root) + try: + worker = ImportWorker( + build_session_factory(engine), + staging, + ArtifactRepository(store), + ReservedMaintenance(), + MaintenanceAudit(), + settings.judge_admission, + settings.import_worker_cgroup, + ) + while True: + if not await worker.once(): + await asyncio.sleep(1) + finally: + staging.close() + store.close() + await engine.dispose() + + +def main(): + try: + asyncio.run(run()) + except (ValueError, OSError, DBAPIError): + print( + "import service unavailable; check trusted configuration and resource limits", + file=sys.stderr, + ) + raise SystemExit(1) from None + + +if __name__ == "__main__": + main() diff --git a/backend/tests/test_admin_import_contracts.py b/backend/tests/test_admin_import_contracts.py new file mode 100644 index 0000000..3087a4b --- /dev/null +++ b/backend/tests/test_admin_import_contracts.py @@ -0,0 +1,156 @@ +"""Contract checks only; upload/parser/security evidence requires the independent VM.""" + +import copy +import json +from datetime import UTC, datetime, timedelta +from pathlib import Path +from uuid import uuid4 + +import pytest +from pydantic import ValidationError + +from kf_oj.contracts.admin_problems import ( + CandidatePreview, + CreateProblemImport, + LegacyImportBudget, + ProblemImportCapabilities, + ProblemImportDetail, +) +from kf_oj.contracts.judging import AdmissionPolicy +from kf_oj.modules.problems.import_format import ImportEnvelope +from kf_oj.modules.problems.web_format import decode_web_declaration + +ROOT = Path(__file__).resolve().parents[2] + + +def legacy_manifest(): + policy = AdmissionPolicy.model_validate_json( + (ROOT / "tests/fixtures/contracts/judge-admission.valid.json").read_bytes() + ) + return { + "schema_version": 1, + "slug": "web-fixture", + "title": "Public", + "statement_file": "statement.md", + "languages": ["c17", "cpp20"], + "toolchain": policy.toolchain.model_dump(mode="json"), + "comparison": "exact_bytes", + "compile_limits": policy.compile_ceiling.model_dump(), + "run_limits": policy.run_ceiling.model_dump(), + "job_wall_time_us": 60000000, + "cases": [{"key": "binary", "input_file": "hidden.in", "expected_file": "hidden.out"}], + "samples": [{"input_file": "sample.in", "output_file": "sample.out"}], + } + + +def test_web_declaration_separates_public_data_and_preserves_local_format(): + source = legacy_manifest() + original = copy.deepcopy(source) + declared = decode_web_declaration(json.dumps(source).encode()) + assert declared.members == frozenset( + ("problem.json", "statement.md", "hidden.in", "hidden.out", "sample.in", "sample.out") + ) + assert declared.manifest.model_dump(mode="json") == original + assert ImportEnvelope.model_validate(source).root == declared.manifest + for mutation in ( + lambda x: x["samples"][0].update(input_file="hidden.in"), + lambda x: x.update(statement_file="problem.json"), + lambda x: x["cases"][0].update(input_file="Statement.md"), + lambda x: x["cases"][0].update(input_file="hidden.zip"), + lambda x: x.update(languages=["rust"]), + lambda x: x.update(download_url="https://example.invalid/private"), + ): + value = copy.deepcopy(source) + mutation(value) + with pytest.raises(ValueError): + decode_web_declaration(json.dumps(value).encode()) + with pytest.raises(ValueError): + decode_web_declaration( + json.dumps(source) + .replace('"schema_version": 1', '"schema_version": 1, "schema_version": 1') + .encode() + ) + + +def test_import_operation_key_and_preview_have_no_private_payload_fields(): + operation = { + "schema_version": 1, + "operation_id": str(uuid4()), + "archive_digest": "a" * 64, + "archive_bytes": 64 * 1024**2, + } + assert CreateProblemImport.model_validate(operation).archive_bytes == 64 * 1024**2 + for field, value in ( + ("archive_bytes", True), + ("archive_bytes", 64 * 1024**2 + 1), + ("url", "https://example.invalid"), + ("role", "admin"), + ): + with pytest.raises(ValidationError): + CreateProblemImport.model_validate(operation | {field: value}) + manifest = ImportEnvelope.model_validate(legacy_manifest()).root + preview = CandidatePreview( + candidate_id=uuid4(), + slug=manifest.slug, + title=manifest.title, + statement="Public only", + samples=(), + judging_schema_version=1, + languages=manifest.languages, + legacy_budget=LegacyImportBudget( + compile_limits=manifest.compile_limits, + run_limits=manifest.run_limits, + job_wall_time_us=manifest.job_wall_time_us, + ), + case_count=1, + ) + now = datetime.now(UTC) + response = ProblemImportDetail( + operation_id=uuid4(), + state="ready", + expected_archive_digest="a" * 64, + archive_bytes=100, + created_at=now, + updated_at=now, + expires_at=now + timedelta(days=1), + failure_code=None, + candidate_id=preview.candidate_id, + published_version_id=None, + candidate=preview, + ) + assert set(response.candidate.model_dump()) == { + "candidate_id", + "slug", + "title", + "statement", + "samples", + "judging_schema_version", + "languages", + "legacy_budget", + "language_configurations", + "case_count", + } + with pytest.raises(ValidationError): + CandidatePreview.model_validate(preview.model_dump() | {"input_digest": "a" * 64}) + capabilities = ProblemImportCapabilities(uploads_available=False, validation_available=False) + assert ( + capabilities.limits.global_concurrency == 1 and capabilities.limits.owner_candidates == 10 + ) + + +def test_standalone_import_worker_resolves_all_referenced_metadata(): + # A separate trusted interpreter avoids accidental model registration by API test imports. + import subprocess + import sys + + script = """ +import kf_oj.workers.problem_import +from kf_oj.infra.db.base import Base +for table in Base.metadata.tables.values(): + for foreign in table.foreign_keys: + assert foreign.column.table is not None +""" + result = subprocess.run( + [sys.executable, "-c", script], capture_output=True, text=True, timeout=15 + ) + assert result.returncode == 0, result.stderr diff --git a/backend/tests/test_admin_import_vm.py b/backend/tests/test_admin_import_vm.py new file mode 100644 index 0000000..d46a88d --- /dev/null +++ b/backend/tests/test_admin_import_vm.py @@ -0,0 +1,988 @@ +"""Real HTTP/worker/storage and adverse ZIP cases: independent capped VM only.""" + +import asyncio +import hashlib +import io +import json +import os +import resource +import signal +import struct +import subprocess +import sys +import time +import zipfile +from pathlib import Path +from uuid import UUID, uuid4 + +import httpx +import pytest +from conftest import connect +from sqlalchemy.engine import make_url + +from kf_oj.contracts.judging import ArtifactReference +from kf_oj.contracts.judging_v2 import AdmissionRegistry +from kf_oj.infra.artifacts.archive_reader import ArchiveError, ArchiveReader +from kf_oj.infra.artifacts.staging import ImportStaging +from kf_oj.infra.artifacts.store import ArtifactError, ArtifactStore +from kf_oj.modules.identity.passwords import hash_password + +ROOT = Path(__file__).resolve().parents[2] + + +@pytest.fixture +def independent_import_vm(): + marker = Path("/kf/INDEPENDENT_VM") + if not marker.exists() or marker.read_text().strip() != "isolated-import-test-guest": + pytest.skip("adverse archive/full import acceptance requires the independent capped VM") + assert [p.name for p in Path("/sys/class/net").iterdir()] == ["lo"] + assert all( + x not in Path("/proc/self/mountinfo").read_text() for x in ("9p", "virtiofs", "hgfs") + ) + return {name: Path("/sys/fs/cgroup/kf-total") / name for name in ("api", "import-worker")} + + +def package( + policy, *, multi=False, method=zipfile.ZIP_STORED, mutation=None, payload_mutation=None +): + payload = { + "statement.md": b"Public statement only", + "hidden.in": b"synthetic-hidden-input\x00\xff", + "hidden.out": b"synthetic-hidden-output\x00\xfe", + "sample.in": b"1\n", + "sample.out": b"2\n", + } + manifest = { + "schema_version": 1, + "slug": "web-synthetic", + "title": "Public", + "statement_file": "statement.md", + "languages": ["c17", "cpp20"], + "toolchain": policy.legacy.toolchain.model_dump(mode="json"), + "comparison": "exact_bytes", + "compile_limits": policy.legacy.compile_ceiling.model_dump(), + "run_limits": policy.legacy.run_ceiling.model_dump(), + "job_wall_time_us": 60000000, + "cases": [{"key": "binary", "input_file": "hidden.in", "expected_file": "hidden.out"}], + "samples": [{"input_file": "sample.in", "output_file": "sample.out"}], + } + if multi: + for key in ("languages", "toolchain", "compile_limits", "run_limits", "job_wall_time_us"): + del manifest[key] + manifest["schema_version"] = 2 + manifest["language_configurations"] = [ + { + key: json.loads( + ( + ROOT / f"tests/fixtures/contracts-v2/judge-manifest.valid-{language}.json" + ).read_text() + )[key] + for key in ( + "language", + "toolchain", + "compile_limits", + "run_limits", + "job_wall_time_us", + ) + } + for language in ("c17", "cpp20", "python313", "java21") + ] + if mutation: + mutation(manifest) + payload["problem.json"] = json.dumps(manifest).encode() + if payload_mutation: + payload_mutation(payload) + out = io.BytesIO() + with zipfile.ZipFile(out, "w", compression=method) as archive: + for name, data in payload.items(): + archive.writestr(name, data) + return out.getvalue(), payload + + +def test_bounded_zip_and_streaming_adoption(independent_import_vm, tmp_path): + policy = AdmissionRegistry.model_validate_json( + (ROOT / "tests/fixtures/contracts-v2/judge-registry.valid.json").read_bytes() + ) + for method in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED): + blob, payload = package( + policy, + method=method, + payload_mutation=lambda p: p.update({"hidden.in": b"A" * 1048576 + b"\x00\xff"}), + ) + file = tmp_path / f"archive-{method}" + file.write_bytes(blob) + file.chmod(0o600) + output = tmp_path / f"members-{method}" + output.mkdir(mode=0o700) + fd = os.open(file, os.O_RDONLY) + directory = os.open(output, os.O_RDONLY | os.O_DIRECTORY) + try: + reader = ArchiveReader(fd, len(blob), deadline=time.monotonic() + 60) + reader.verify_digest(hashlib.sha256(blob).hexdigest()) + assert reader.read("problem.json", 262144) == payload.pop("problem.json") + references = reader.extract_remaining(directory) + assert reader.unpacked_bytes == sum(len(v) for v in payload.values()) + len( + package(policy, method=method)[1]["problem.json"] + ) + store = ArtifactStore(tmp_path / f"objects-{method}") + try: + for name, (digest, size) in references.items(): + assert (output / name).read_bytes() == payload[name] + source = os.open(name, os.O_RDONLY, dir_fd=directory) + try: + reference = ArtifactReference(digest=digest, bytes=size) + assert store.put_file(source, reference) == reference + assert store.put_file(source, reference) == reference + finally: + os.close(source) + finally: + store.close() + finally: + os.close(fd) + os.close(directory) + good, _ = package(policy) + wrong_crc = bytearray(good) + struct.pack_into("clock_timestamp() " + "FROM problem_import_pool WHERE id=1" + ): + break + except httpx.ConnectError: + pass + await asyncio.sleep(0.1) + else: + raise AssertionError( + "real API/worker startup deadline; api=" + + (tmp_path / "api.log").read_text()[-1500:] + + " worker=" + + (tmp_path / "worker.log").read_text()[-1500:] + ) + prefix = "/api/v1/admin/problem-imports" + assert (await client.get(prefix + "/capabilities")).status_code == 401 + + async def login(name): + client.cookies.clear() + r = await client.post( + "/api/v1/auth/login", + json={"username": name, "password": password}, + headers={"Origin": "http://127.0.0.1:58000"}, + ) + assert r.status_code == 200, r.text + return { + "Origin": "http://127.0.0.1:58000", + "X-CSRF-Token": client.cookies["kf_csrf"], + } + + student_headers = await login("web_student") + assert (await client.get(prefix + "/capabilities")).status_code == 403 + assert ( + await client.post( + prefix, + json={ + "operation_id": str(uuid4()), + "archive_digest": "a" * 64, + "archive_bytes": 100, + }, + headers=student_headers, + ) + ).status_code == 403 + headers = await login("web_admin") + assert (await client.get(prefix + "/capabilities")).json()[ + "uploads_available" + ] is True + for invalid in ( + {}, + {"Origin": "http://127.0.0.1:58000", "X-CSRF-Token": "0" * 64}, + headers | {"Origin": "https://evil.invalid"}, + headers + | {"Origin": "https://evil.invalid", "X-Forwarded-Host": "127.0.0.1:58000"}, + ): + assert ( + await client.post( + prefix, + json={ + "operation_id": str(uuid4()), + "archive_digest": "a" * 64, + "archive_bytes": 100, + }, + headers=invalid, + ) + ).status_code == 403 + + async def create(blob, key=None): + key = key or uuid4() + body = { + "operation_id": str(key), + "archive_digest": hashlib.sha256(blob).hexdigest(), + "archive_bytes": len(blob), + } + deadline = time.monotonic() + 15 + while True: + r = await client.post(prefix, json=body, headers=headers) + if r.status_code != 503 or time.monotonic() >= deadline: + break + await asyncio.sleep(0.2) + assert r.status_code == 201, r.text + assert r.headers["cache-control"] == "no-store" + assert ( + await client.post(prefix, json=body, headers=headers) + ).json() == r.json() + assert ( + await client.post( + prefix, json=body | {"archive_digest": "f" * 64}, headers=headers + ) + ).status_code == 409 + return str(key) + + async def put_blob(blob, key): + deadline = time.monotonic() + 30 + while True: + result = await client.put( + f"{prefix}/{key}/archive", + content=blob, + headers=headers | {"Content-Type": "application/zip"}, + ) + if result.status_code != 503 or time.monotonic() >= deadline: + return result + await asyncio.sleep(0.2) + + async def discard_owned(key): + deadline = time.monotonic() + 15 + while True: + result = await client.post(f"{prefix}/{key}/discard", headers=headers) + if result.status_code != 503 or time.monotonic() >= deadline: + return result + await asyncio.sleep(0.2) + + async def upload(blob, key): + r = await put_blob(blob, key) + assert r.status_code == 200, r.text + deadline = time.monotonic() + 100 + while time.monotonic() < deadline: + r = await client.get(f"{prefix}/{key}") + assert r.status_code == 200, r.text + if r.json()["state"] in ("ready", "failed"): + return r.json() + assert processes["worker"].poll() is None, ( + tmp_path / "worker.log" + ).read_text()[-2000:] + await asyncio.sleep(0.2) + raise AssertionError( + "candidate deadline; " + (tmp_path / "worker.log").read_text()[-2000:] + ) + + async def wait_released(key, state, code=None, seconds=120): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + row = await c.fetchrow( + "SELECT state,failure_code,reserved_bytes,attempts FROM " + "problem_import_operations WHERE id=$1", + UUID(key), + ) + if row["state"] == state and row["reserved_bytes"] == 0: + if code: + assert row["failure_code"] == code, dict(row) + return row + await asyncio.sleep(0.1) + raise AssertionError("operation cleanup deadline: " + str(dict(row))) + + async def restart_ready_worker(): + previous = await c.fetchval( + "SELECT worker_ready_until FROM problem_import_pool WHERE id=1" + ) + assert previous is not None + start("worker") + deadline = time.monotonic() + 120 + while time.monotonic() < deadline: + assert processes["worker"].poll() is None + if await c.fetchval( + "SELECT worker_ready_until>clock_timestamp()+interval '80 seconds' " + "AND worker_ready_until>$1 FROM problem_import_pool WHERE id=1", + previous, + ): + return + await asyncio.sleep(0.1) + raise AssertionError("fresh worker heartbeat deadline") + + async def raw_body(key, declared, data, *, delay=0, disconnect=False): + reader, writer = await asyncio.open_connection("127.0.0.1", 58000) + cookie = "; ".join(f"{k}={v}" for k, v in client.cookies.items()) + request = ( + f"PUT {prefix}/{key}/archive HTTP/1.1\r\n" + "Host: 127.0.0.1:58000\r\nOrigin: http://127.0.0.1:58000\r\n" + f"Cookie: {cookie}\r\nX-CSRF-Token: {headers['X-CSRF-Token']}\r\n" + f"Content-Type: application/zip\r\nContent-Length: {declared}\r\n" + "Connection: close\r\n\r\n" + ).encode() + try: + writer.write(request + data) + await writer.drain() + if delay: + await asyncio.sleep(delay) + if disconnect: + return None + return await asyncio.wait_for(reader.read(8192), 45) + finally: + writer.close() + await writer.wait_closed() + + for multi, method in ((False, zipfile.ZIP_STORED), (True, zipfile.ZIP_DEFLATED)): + blob, payload = package(policy, multi=multi, method=method) + key = await create(blob) + detail = await upload(blob, key) + assert detail["state"] == "ready", (detail["failure_code"], detail["state"]) + print("ACTUAL_READY_IMPORT", multi, method, detail["candidate_id"], flush=True) + await wait_released(key, "ready") + retry = await put_blob(blob, key) + assert retry.status_code == 200 + different = blob[:-1] + bytes([blob[-1] ^ 1]) + conflict = await put_blob(different, key) + assert conflict.status_code == 409 + assert (await client.get(f"{prefix}/{key}")).json()["candidate_id"] == detail[ + "candidate_id" + ] + assert detail["candidate"]["judging_schema_version"] == (2 if multi else 1) + assert detail["candidate"]["case_count"] == 1 + assert len(detail["candidate"]["languages"]) == (4 if multi else 2) + assert "synthetic-hidden" not in json.dumps(detail) and str( + tmp_path + ) not in json.dumps(detail) + # Actual database enforcement, not trigger inspection alone. + worker_db = await connect( + make_url(postgres_url) + .set(username=roles["worker"]) + .render_as_string(hide_password=False) + ) + try: + from asyncpg import CheckViolationError, InsufficientPrivilegeError + + with pytest.raises(InsufficientPrivilegeError): + await worker_db.execute( + "UPDATE problem_candidates SET preview=preview WHERE id=$1", + UUID(detail["candidate_id"]), + ) + with pytest.raises(CheckViolationError): + await worker_db.execute( + "DELETE FROM problem_candidates WHERE id=$1", + UUID(detail["candidate_id"]), + ) + with pytest.raises(CheckViolationError): + await worker_db.execute( + """INSERT INTO problem_candidate_cases + (candidate_id,position,key,input_digest,input_bytes,expected_digest,expected_bytes) + SELECT candidate_id,1,'late',input_digest,input_bytes, + expected_digest,expected_bytes + FROM problem_candidate_cases WHERE candidate_id=$1""", + UUID(detail["candidate_id"]), + ) + finally: + await worker_db.close() + refs = await c.fetch( + "SELECT input_digest,input_bytes,expected_digest,expected_bytes " + "FROM problem_candidate_cases WHERE candidate_id=$1", + UUID(detail["candidate_id"]), + ) + assert len(refs) == 1 + for field, name in (("input", "hidden.in"), ("expected", "hidden.out")): + digest = refs[0][field + "_digest"] + data = (artifact / digest).read_bytes() + assert data == payload[name] and hashlib.sha256(data).hexdigest() == digest + assert await c.fetchval("SELECT count(*) FROM problem_versions") == 0 + assert (await client.get("/api/v1/problems")).json()["items"] == [] + assert ( + await c.fetchval( + "SELECT reserved_bytes FROM problem_import_pool WHERE id=1" + ) + == 0 + ) + assert sorted(p.name for p in staging.iterdir()) == [".lock"] + + def reuse_public_member(payload): + declaration = json.loads(payload["problem.json"]) + declaration["samples"][0]["output_file"] = "sample.in" + payload["problem.json"] = json.dumps(declaration).encode() + del payload["sample.out"] + + # Actual child consumes malformed data but never executes contents. + for mutation in ( + lambda p: p.update({"../escape": b"bad"}), + lambda p: p.update({"unknown.bin": b"bad"}), + lambda p: p.update({"hidden.in": b"PK\x03\x04" + b"bad"}), + lambda p: p.update({"sample.in": p["hidden.in"]}), + reuse_public_member, + ): + blob, _ = package(policy, payload_mutation=mutation) + key = await create(blob) + result = await upload(blob, key) + # Publicly declared samples can happen to equal hidden bytes; declarations, + # not byte equality, decide visibility. This last finite case is legal. + if b"PK\x03\x04bad" in blob or b"../escape" in blob or b"unknown.bin" in blob: + assert result["state"] == "failed" and result["candidate"] is None, result + else: + assert result["state"] == "ready", result + await wait_released(key, result["state"]) + assert ( + await c.fetchval( + "SELECT reserved_bytes FROM problem_import_pool WHERE id=1" + ) + == 0 + ) + blob, _ = package(policy) + key = await create(blob) + stop("worker") + occupied = ImportStaging(staging) + try: + with occupied.exclusive(): + mismatch = await client.put( + f"{prefix}/{key}/archive", + content=blob[:-1], + headers=headers | {"Content-Type": "application/zip"}, + ) + assert mismatch.status_code == 409 + finally: + occupied.close() + assert (await discard_owned(key)).status_code == 200 + # Restart after receipt before consumption preserves the immutable operation. + stop("worker") + key = await create(blob) + r = await client.put( + f"{prefix}/{key}/archive", + content=blob, + headers=headers | {"Content-Type": "application/zip"}, + ) + assert r.status_code == 200 and r.json()["state"] == "uploaded" + start("worker") + deadline = time.monotonic() + 120 + while time.monotonic() < deadline: + detail = (await client.get(f"{prefix}/{key}")).json() + if detail["state"] in ("ready", "failed"): + break + await asyncio.sleep(0.2) + assert detail["state"] == "ready", (detail["failure_code"], detail["state"]) + assert ( + await c.fetchval( + "SELECT attempts FROM problem_import_operations WHERE id=$1", + UUID(key), + ) + == 1 + ) + await wait_released(key, "ready") + # A second real administrator cannot inspect, upload or discard another's candidate. + headers = await login("web_backup") + stop("worker") + occupied = ImportStaging(staging) + try: + with occupied.exclusive(): + assert (await client.get(f"{prefix}/{key}")).status_code == 404 + assert ( + await client.post(f"{prefix}/{key}/discard", headers=headers) + ).status_code == 404 + assert ( + await client.put( + f"{prefix}/{key}/archive", + content=blob, + headers=headers | {"Content-Type": "application/zip"}, + ) + ).status_code == 404 + finally: + occupied.close() + start("worker") + headers = await login("web_admin") + # Incomplete TCP body and actual 30-second streaming deadline free reservations. + interrupted = await create(blob) + stop("worker") # Exercise the upload path without a competing GC root lock. + await raw_body(interrupted, len(blob), blob[:8], disconnect=True) + await wait_released(interrupted, "failed", "upload_interrupted") + await restart_ready_worker() + slow = await create(blob) + stop("worker") + response = await raw_body(slow, len(blob), blob[:8], delay=33) + await wait_released(slow, "failed", "upload_deadline") + assert response.startswith(b"HTTP/1.1 408"), response[:200] + await restart_ready_worker() + # A full received body with the wrong SHA is never acknowledged as an upload. + wrong = await create(blob) + stop("worker") + r = await client.put( + f"{prefix}/{wrong}/archive", + content=blob[:-1] + b"!", + headers=headers | {"Content-Type": "application/zip"}, + ) + assert r.status_code == 409 + await wait_released(wrong, "failed", "digest_mismatch") + await restart_ready_worker() + # Two actual reservations fill the global staging pool with Worker stopped. + stop("worker") + queued = [] + for _ in range(2): + queued_key = await create(blob) + r = await client.put( + f"{prefix}/{queued_key}/archive", + content=blob, + headers=headers | {"Content-Type": "application/zip"}, + ) + assert r.status_code == 200 + queued.append(queued_key) + assert ( + await c.fetchval("SELECT reserved_bytes FROM problem_import_pool WHERE id=1") + == 268435456 + ) + r = await client.post( + prefix, + json={ + "operation_id": str(uuid4()), + "archive_digest": hashlib.sha256(blob).hexdigest(), + "archive_bytes": len(blob), + }, + headers=headers, + ) + assert r.status_code == 503 + for queued_key in queued: + assert ( + await client.post(f"{prefix}/{queued_key}/discard", headers=headers) + ).status_code == 200 + assert ( + await c.fetchval("SELECT reserved_bytes FROM problem_import_pool WHERE id=1") + == 0 + ) + start("worker") + # Kill parent AND parser in their already verified cgroup while a lease is held. + # This finite injection tests both retries. No uploaded code is executed. + interrupted_parser = await create(blob) + r = await put_blob(blob, interrupted_parser) + assert r.status_code == 200 + for expected_attempt in (1, 2): + deadline = time.monotonic() + 120 + while time.monotonic() < deadline: + row = await c.fetchrow( + "SELECT state,attempts FROM problem_import_operations WHERE id=$1", + UUID(interrupted_parser), + ) + parser = [] + for pid in ( + groups["import-worker"].joinpath("cgroup.procs").read_text().split() + ): + cmdline = Path(f"/proc/{pid}/cmdline") + if ( + cmdline.exists() + and b"kf_oj.workers._archive_child" in cmdline.read_bytes() + ): + parser.append(int(pid)) + if ( + row["state"] == "validating" + and row["attempts"] == expected_attempt + and parser + ): + # Freeze parser, kill parent, then parser. A live child keeps the flock. + for pid in parser: + os.kill(pid, signal.SIGSTOP) + processes["worker"].kill() + processes["worker"].wait(timeout=10) + await c.execute( + "UPDATE problem_import_operations SET lease_until=" + "clock_timestamp()-interval '1 second' WHERE id=$1", + UUID(interrupted_parser), + ) + # A live stopped child retains the real kernel lock after parent death. + probe = ImportStaging(staging) + try: + with pytest.raises(ArtifactError): + with probe.exclusive(): + pass + finally: + probe.close() + denied = await client.post( + prefix, + json={ + "operation_id": str(uuid4()), + "archive_digest": hashlib.sha256(blob).hexdigest(), + "archive_bytes": len(blob), + }, + headers=headers, + ) + assert denied.status_code == 503 + for pid in parser: + os.kill(pid, signal.SIGKILL) + stop("worker") + break + await asyncio.sleep(0.02) + else: + raise AssertionError("claimed parser injection deadline") + # Advance the lease under the VM test operator; requests cannot set clocks. + await c.execute( + "UPDATE problem_import_operations SET lease_until=" + "clock_timestamp()-interval '1 second' WHERE id=$1", + UUID(interrupted_parser), + ) + start("worker") + row = await wait_released(interrupted_parser, "failed", "validation_unavailable") + assert row["attempts"] == 2 + assert ( + await c.fetchval( + "SELECT count(*) FROM problem_candidates WHERE operation_id=$1", + UUID(interrupted_parser), + ) + == 0 + ) + # Real concurrent claims share the same capped process and use separate lock FDs. + stop("worker") + paired_key = await create(blob) + r = await client.put( + f"{prefix}/{paired_key}/archive", + content=blob, + headers=headers | {"Content-Type": "application/zip"}, + ) + assert r.status_code == 200 and r.json()["state"] == "uploaded" + start("worker", pair=True) + row = await wait_released(paired_key, "ready") + assert row["attempts"] == 1 + assert ( + await c.fetchval( + "SELECT count(*) FROM problem_candidates WHERE operation_id=$1", + UUID(paired_key), + ) + == 1 + ) + deadline = time.monotonic() + 120 + while processes["worker"].poll() is None and time.monotonic() < deadline: + await asyncio.sleep(0.1) + assert processes["worker"].returncode == 0, (tmp_path / "worker.log").read_text()[ + -1500: + ] + assert ( + "ACTUAL_CONCURRENT_IMPORT_CLAIM_ACCEPTED" + in (tmp_path / "worker.log").read_text() + ) + stop("worker") + start("worker") + # Insert an old receipt with the test operator; immutable timestamps stay unchanged. + expired_id = uuid4() + async with c.transaction(): + await c.execute( + """INSERT INTO problem_import_operations + (id,owner_id,authorization_version,archive_digest,archive_bytes,created_at, + updated_at,expires_at,lease_until,attempts,fence,reserved_bytes) + SELECT $1,$2,authorization_version,$3,$4, + clock_timestamp()-interval '2 days', + clock_timestamp()-interval '2 days',clock_timestamp()-interval '1 day', + clock_timestamp()-interval '1 day',0,0,134217728 FROM accounts WHERE id=$2""", + expired_id, + users["web_admin"], + hashlib.sha256(blob).hexdigest(), + len(blob), + ) + assert ( + await c.fetchval( + "SELECT state FROM problem_import_operations WHERE id=$1", expired_id + ) + == "uploading" + ) + await c.execute( + "UPDATE problem_import_pool SET reserved_bytes=" + "reserved_bytes+134217728 WHERE id=1" + ) + await wait_released(str(expired_id), "expired", "expired") + print( + "ACTUAL_IMPORT_DISCONNECT_DEADLINE_CAPACITY_TWO_RETRIES_EXPIRY_ACCEPTED", + flush=True, + ) + # Revocation is live and old cookies cannot read or write management APIs. + await c.execute("SELECT * FROM kf_manage_account_authority('web_admin','revoke')") + assert (await client.get(prefix + "/capabilities")).status_code == 401 + assert ( + await client.post(f"{prefix}/{key}/discard", headers=headers) + ).status_code == 401 + await c.execute("SELECT * FROM kf_manage_account_authority('web_admin','grant')") + headers = await login("web_admin") + stop("worker") + discard_after = await c.fetchval("SELECT clock_timestamp()") + page = (await client.get(prefix, params={"limit": 100})).json() + for entry in page["items"]: + assert ( + await client.post( + f"{prefix}/{entry['operation_id']}/discard", headers=headers + ) + ).status_code == 200 + await restart_ready_worker() + deadline = time.monotonic() + 30 + while time.monotonic() < deadline and await c.fetchval( + "SELECT count(*) FROM problem_candidates" + ): + await asyncio.sleep(0.1) + assert await c.fetchval("SELECT count(*) FROM problem_candidates") == 0 + assert await c.fetchval("SELECT count(*) FROM problem_candidate_cases") == 0 + assert ( + await c.fetchval("SELECT reserved_bytes FROM problem_import_pool WHERE id=1") + == 0 + ) + assert await c.fetchval("SELECT count(*) FROM problem_maintenance_events") >= 10 + assert ( + await c.fetchval( + "SELECT bool_and(e.authorization_version=a.authorization_version) " + "FROM problem_maintenance_events e JOIN accounts a ON a.id=e.actor_id " + "WHERE e.action='import_discarded' AND e.actor_id=$1 " + "AND e.created_at >= $2", + users["web_admin"], + discard_after, + ) + is True + ) + for table in ( + "evaluations", + "judge_attempts", + "evaluation_performance", + "accounts", + "sessions", + ): + for privilege in ("SELECT", "INSERT", "UPDATE", "DELETE"): + assert not await c.fetchval( + "SELECT has_table_privilege($1,$2,$3)", + roles["worker"], + table, + privilege, + ) + assert not await c.fetchval( + "SELECT has_table_privilege($1,'problem_maintenance_events','UPDATE')", + roles["api"], + ) + for privilege in ("SELECT", "UPDATE", "DELETE"): + assert not await c.fetchval( + "SELECT has_table_privilege($1,'problem_maintenance_events',$2)", + roles["worker"], + privilege, + ) + # Missing configuration is observed through the real server, never preview data. + stop("api") + start("api", configured=False) + deadline = time.monotonic() + 120 + while time.monotonic() < deadline: + try: + r = await client.get(prefix + "/capabilities") + if r.status_code == 200: + break + except httpx.ConnectError: + pass + await asyncio.sleep(0.1) + assert r.status_code == 200 and not r.json()["uploads_available"] + assert not r.json()["validation_available"] + assert ( + await client.post( + prefix, + json={ + "operation_id": str(uuid4()), + "archive_digest": hashlib.sha256(blob).hexdigest(), + "archive_bytes": len(blob), + }, + headers=headers, + ) + ).status_code == 503 + print("ACTUAL_IMPORT_HTTP_WORKER_ACCEPTED", flush=True) + finally: + stop("api") + stop("worker") + for role in roles.values(): + await c.execute(f"DROP OWNED BY {role}") + await c.execute(f"DROP ROLE {role}") + await c.close() + + asyncio.run(run()) diff --git a/backend/tests/test_database_foundation.py b/backend/tests/test_database_foundation.py index 5d869bc..cc34590 100644 --- a/backend/tests/test_database_foundation.py +++ b/backend/tests/test_database_foundation.py @@ -1,5 +1,6 @@ import asyncio import json +import subprocess from uuid import uuid4 import asyncpg @@ -12,7 +13,7 @@ def test_migration_roundtrip_and_role_safety(postgres_url): c = await connect(postgres_url) try: tables = set(await c.fetch("SELECT tablename FROM pg_tables WHERE schemaname='public'")) - assert len(tables) == 15 # Fourteen domain/control tables and Alembic history. + assert len(tables) == 20 # Nineteen domain/control tables and Alembic history. roles = await c.fetch("SELECT * FROM pg_roles WHERE rolname LIKE 'kf_oj_%'") assert {row["rolname"] for row in roles} == { "kf_oj_platform", @@ -44,7 +45,14 @@ def test_migration_roundtrip_and_role_safety(postgres_url): asyncio.run(inspect()) migrate(postgres_url, "downgrade", "base") migrate(postgres_url, "upgrade", "head") - migrate(postgres_url, "check") + try: + migrate(postgres_url, "check") + except subprocess.CalledProcessError as error: + raise AssertionError( + "migration model consistency failed: " + + (error.stdout or b"").decode(errors="replace")[-8192:] + + (error.stderr or b"").decode(errors="replace")[-8192:] + ) from None asyncio.run(inspect()) diff --git a/contracts/openapi/platform.v1.json b/contracts/openapi/platform.v1.json index dd879c7..d444a1d 100644 --- a/contracts/openapi/platform.v1.json +++ b/contracts/openapi/platform.v1.json @@ -76,6 +76,94 @@ "title": "CancellationAccepted", "type": "object" }, + "CandidatePreview": { + "additionalProperties": false, + "properties": { + "candidate_id": { + "format": "uuid", + "title": "Candidate Id", + "type": "string" + }, + "case_count": { + "maximum": 256.0, + "minimum": 1.0, + "title": "Case Count", + "type": "integer" + }, + "judging_schema_version": { + "maximum": 2.0, + "minimum": 1.0, + "title": "Judging Schema Version", + "type": "integer" + }, + "language_configurations": { + "default": [], + "items": { + "$ref": "#/components/schemas/PublicLanguageConfiguration" + }, + "maxItems": 4, + "title": "Language Configurations", + "type": "array" + }, + "languages": { + "items": { + "$ref": "#/components/schemas/kf_oj__contracts__judging_v2__Language" + }, + "maxItems": 4, + "minItems": 1, + "title": "Languages", + "type": "array" + }, + "legacy_budget": { + "anyOf": [ + { + "$ref": "#/components/schemas/LegacyImportBudget" + }, + { + "type": "null" + } + ] + }, + "samples": { + "items": { + "$ref": "#/components/schemas/PublicSample" + }, + "maxItems": 10, + "title": "Samples", + "type": "array" + }, + "slug": { + "pattern": "^[a-z0-9][a-z0-9_-]{0,63}$", + "title": "Slug", + "type": "string" + }, + "statement": { + "maxLength": 262144, + "minLength": 1, + "title": "Statement", + "type": "string" + }, + "title": { + "maxLength": 160, + "minLength": 1, + "title": "Title", + "type": "string" + } + }, + "required": [ + "candidate_id", + "slug", + "title", + "statement", + "samples", + "judging_schema_version", + "languages", + "legacy_budget", + "case_count" + ], + "title": "CandidatePreview", + "type": "object" + }, "CompilationStatus": { "enum": [ "succeeded", @@ -87,6 +175,42 @@ "title": "CompilationStatus", "type": "string" }, + "CreateProblemImport": { + "additionalProperties": false, + "properties": { + "archive_bytes": { + "maximum": 67108864.0, + "minimum": 1.0, + "title": "Archive Bytes", + "type": "integer" + }, + "archive_digest": { + "maxLength": 64, + "minLength": 64, + "pattern": "^[0-9a-f]{64}$", + "title": "Archive Digest", + "type": "string" + }, + "operation_id": { + "format": "uuid", + "title": "Operation Id", + "type": "string" + }, + "schema_version": { + "const": 1, + "default": 1, + "title": "Schema Version", + "type": "integer" + } + }, + "required": [ + "operation_id", + "archive_digest", + "archive_bytes" + ], + "title": "CreateProblemImport", + "type": "object" + }, "CreateSubmission": { "additionalProperties": false, "properties": { @@ -192,6 +316,38 @@ "title": "HealthResponse", "type": "object" }, + "ImportFailure": { + "enum": [ + "invalid_archive", + "invalid_manifest", + "invalid_data", + "digest_mismatch", + "policy_rejected", + "upload_interrupted", + "upload_deadline", + "validation_deadline", + "validation_unavailable", + "storage_unavailable", + "authority_revoked", + "expired" + ], + "title": "ImportFailure", + "type": "string" + }, + "ImportState": { + "enum": [ + "uploading", + "uploaded", + "validating", + "ready", + "failed", + "published", + "discarded", + "expired" + ], + "title": "ImportState", + "type": "string" + }, "JobManifest": { "additionalProperties": false, "properties": { @@ -284,6 +440,30 @@ "title": "Language", "type": "string" }, + "LegacyImportBudget": { + "additionalProperties": false, + "properties": { + "compile_limits": { + "$ref": "#/components/schemas/RunLimits" + }, + "job_wall_time_us": { + "maximum": 3600000000.0, + "minimum": 1000.0, + "title": "Job Wall Time Us", + "type": "integer" + }, + "run_limits": { + "$ref": "#/components/schemas/RunLimits" + } + }, + "required": [ + "compile_limits", + "run_limits", + "job_wall_time_us" + ], + "title": "LegacyImportBudget", + "type": "object" + }, "LoginRequest": { "additionalProperties": false, "properties": { @@ -575,6 +755,426 @@ "title": "ProblemDetail", "type": "object" }, + "ProblemImportCapabilities": { + "additionalProperties": false, + "properties": { + "limits": { + "$ref": "#/components/schemas/ProblemImportLimits", + "default": { + "archive_bytes": 67108864, + "archive_format": "single_level_zip", + "candidate_lifetime_seconds": 86400, + "chunk_bytes": 1048576, + "filename_ascii_bytes": 128, + "files": 534, + "global_concurrency": 1, + "languages": [ + "c17", + "cpp20", + "python313", + "java21" + ], + "manifest_name": "problem.json", + "manifest_schema_versions": [ + 1, + 2 + ], + "owner_candidates": 10, + "owner_concurrency": 1, + "schema_version": 1, + "staging_bytes": 268435456, + "total_candidates": 32, + "unpacked_bytes": 67108864, + "upload_chunks": 4096, + "upload_deadline_seconds": 30, + "validation_deadline_seconds": 60 + } + }, + "schema_version": { + "const": 1, + "default": 1, + "title": "Schema Version", + "type": "integer" + }, + "uploads_available": { + "title": "Uploads Available", + "type": "boolean" + }, + "validation_available": { + "title": "Validation Available", + "type": "boolean" + } + }, + "required": [ + "uploads_available", + "validation_available" + ], + "title": "ProblemImportCapabilities", + "type": "object" + }, + "ProblemImportDetail": { + "additionalProperties": false, + "properties": { + "archive_bytes": { + "maximum": 67108864.0, + "minimum": 1.0, + "title": "Archive Bytes", + "type": "integer" + }, + "candidate": { + "anyOf": [ + { + "$ref": "#/components/schemas/CandidatePreview" + }, + { + "type": "null" + } + ] + }, + "candidate_id": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Candidate Id" + }, + "created_at": { + "format": "date-time", + "title": "Created At", + "type": "string" + }, + "expected_archive_digest": { + "maxLength": 64, + "minLength": 64, + "pattern": "^[0-9a-f]{64}$", + "title": "Expected Archive Digest", + "type": "string" + }, + "expires_at": { + "format": "date-time", + "title": "Expires At", + "type": "string" + }, + "failure_code": { + "anyOf": [ + { + "$ref": "#/components/schemas/ImportFailure" + }, + { + "type": "null" + } + ] + }, + "operation_id": { + "format": "uuid", + "title": "Operation Id", + "type": "string" + }, + "published_version_id": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Published Version Id" + }, + "schema_version": { + "const": 1, + "default": 1, + "title": "Schema Version", + "type": "integer" + }, + "state": { + "$ref": "#/components/schemas/ImportState" + }, + "updated_at": { + "format": "date-time", + "title": "Updated At", + "type": "string" + } + }, + "required": [ + "operation_id", + "state", + "expected_archive_digest", + "archive_bytes", + "created_at", + "expires_at", + "updated_at", + "failure_code", + "candidate_id", + "published_version_id", + "candidate" + ], + "title": "ProblemImportDetail", + "type": "object" + }, + "ProblemImportLimits": { + "additionalProperties": false, + "properties": { + "archive_bytes": { + "default": 67108864, + "title": "Archive Bytes", + "type": "integer" + }, + "archive_format": { + "const": "single_level_zip", + "default": "single_level_zip", + "title": "Archive Format", + "type": "string" + }, + "candidate_lifetime_seconds": { + "default": 86400, + "title": "Candidate Lifetime Seconds", + "type": "integer" + }, + "chunk_bytes": { + "default": 1048576, + "title": "Chunk Bytes", + "type": "integer" + }, + "filename_ascii_bytes": { + "default": 128, + "title": "Filename Ascii Bytes", + "type": "integer" + }, + "files": { + "default": 534, + "title": "Files", + "type": "integer" + }, + "global_concurrency": { + "const": 1, + "default": 1, + "title": "Global Concurrency", + "type": "integer" + }, + "languages": { + "default": [ + "c17", + "cpp20", + "python313", + "java21" + ], + "items": { + "$ref": "#/components/schemas/kf_oj__contracts__judging_v2__Language" + }, + "title": "Languages", + "type": "array" + }, + "manifest_name": { + "const": "problem.json", + "default": "problem.json", + "title": "Manifest Name", + "type": "string" + }, + "manifest_schema_versions": { + "default": [ + 1, + 2 + ], + "items": { + "enum": [ + 1, + 2 + ], + "type": "integer" + }, + "title": "Manifest Schema Versions", + "type": "array" + }, + "owner_candidates": { + "default": 10, + "title": "Owner Candidates", + "type": "integer" + }, + "owner_concurrency": { + "const": 1, + "default": 1, + "title": "Owner Concurrency", + "type": "integer" + }, + "schema_version": { + "const": 1, + "default": 1, + "title": "Schema Version", + "type": "integer" + }, + "staging_bytes": { + "default": 268435456, + "title": "Staging Bytes", + "type": "integer" + }, + "total_candidates": { + "default": 32, + "title": "Total Candidates", + "type": "integer" + }, + "unpacked_bytes": { + "default": 67108864, + "title": "Unpacked Bytes", + "type": "integer" + }, + "upload_chunks": { + "default": 4096, + "title": "Upload Chunks", + "type": "integer" + }, + "upload_deadline_seconds": { + "default": 30, + "title": "Upload Deadline Seconds", + "type": "integer" + }, + "validation_deadline_seconds": { + "default": 60, + "title": "Validation Deadline Seconds", + "type": "integer" + } + }, + "title": "ProblemImportLimits", + "type": "object" + }, + "ProblemImportPage": { + "additionalProperties": false, + "properties": { + "items": { + "items": { + "$ref": "#/components/schemas/ProblemImportSummary" + }, + "maxItems": 100, + "title": "Items", + "type": "array" + }, + "next_after": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Next After" + }, + "schema_version": { + "const": 1, + "default": 1, + "title": "Schema Version", + "type": "integer" + } + }, + "required": [ + "items", + "next_after" + ], + "title": "ProblemImportPage", + "type": "object" + }, + "ProblemImportSummary": { + "additionalProperties": false, + "properties": { + "archive_bytes": { + "maximum": 67108864.0, + "minimum": 1.0, + "title": "Archive Bytes", + "type": "integer" + }, + "candidate_id": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Candidate Id" + }, + "created_at": { + "format": "date-time", + "title": "Created At", + "type": "string" + }, + "expected_archive_digest": { + "maxLength": 64, + "minLength": 64, + "pattern": "^[0-9a-f]{64}$", + "title": "Expected Archive Digest", + "type": "string" + }, + "expires_at": { + "format": "date-time", + "title": "Expires At", + "type": "string" + }, + "failure_code": { + "anyOf": [ + { + "$ref": "#/components/schemas/ImportFailure" + }, + { + "type": "null" + } + ] + }, + "operation_id": { + "format": "uuid", + "title": "Operation Id", + "type": "string" + }, + "published_version_id": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Published Version Id" + }, + "schema_version": { + "const": 1, + "default": 1, + "title": "Schema Version", + "type": "integer" + }, + "state": { + "$ref": "#/components/schemas/ImportState" + }, + "updated_at": { + "format": "date-time", + "title": "Updated At", + "type": "string" + } + }, + "required": [ + "operation_id", + "state", + "expected_archive_digest", + "archive_bytes", + "created_at", + "expires_at", + "updated_at", + "failure_code", + "candidate_id", + "published_version_id" + ], + "title": "ProblemImportSummary", + "type": "object" + }, "ProblemPage": { "additionalProperties": false, "properties": { @@ -1416,6 +2016,268 @@ }, "openapi": "3.1.0", "paths": { + "/api/v1/admin/problem-imports": { + "get": { + "operationId": "page_api_v1_admin_problem_imports_get", + "parameters": [ + { + "in": "query", + "name": "after", + "required": false, + "schema": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "After" + } + }, + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "default": 20, + "maximum": 100, + "minimum": 1, + "title": "Limit", + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemImportPage" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Page", + "tags": [ + "problem-maintenance" + ] + }, + "post": { + "operationId": "create_api_v1_admin_problem_imports_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateProblemImport" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemImportSummary" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Create", + "tags": [ + "problem-maintenance" + ] + } + }, + "/api/v1/admin/problem-imports/capabilities": { + "get": { + "operationId": "capabilities_api_v1_admin_problem_imports_capabilities_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemImportCapabilities" + } + } + }, + "description": "Successful Response" + } + }, + "summary": "Capabilities", + "tags": [ + "problem-maintenance" + ] + } + }, + "/api/v1/admin/problem-imports/{operation_id}": { + "get": { + "operationId": "detail_api_v1_admin_problem_imports__operation_id__get", + "parameters": [ + { + "in": "path", + "name": "operation_id", + "required": true, + "schema": { + "format": "uuid", + "title": "Operation Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemImportDetail" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Detail", + "tags": [ + "problem-maintenance" + ] + } + }, + "/api/v1/admin/problem-imports/{operation_id}/archive": { + "put": { + "operationId": "upload_api_v1_admin_problem_imports__operation_id__archive_put", + "parameters": [ + { + "in": "path", + "name": "operation_id", + "required": true, + "schema": { + "format": "uuid", + "title": "Operation Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/zip": { + "schema": { + "format": "binary", + "type": "string" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemImportSummary" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Upload", + "tags": [ + "problem-maintenance" + ] + } + }, + "/api/v1/admin/problem-imports/{operation_id}/discard": { + "post": { + "operationId": "discard_api_v1_admin_problem_imports__operation_id__discard_post", + "parameters": [ + { + "in": "path", + "name": "operation_id", + "required": true, + "schema": { + "format": "uuid", + "title": "Operation Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemImportSummary" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Discard", + "tags": [ + "problem-maintenance" + ] + } + }, "/api/v1/auth/login": { "post": { "operationId": "login_api_v1_auth_login_post", diff --git a/contracts/schemas/problem-import-capabilities.v1.schema.json b/contracts/schemas/problem-import-capabilities.v1.schema.json new file mode 100644 index 0000000..81f64da --- /dev/null +++ b/contracts/schemas/problem-import-capabilities.v1.schema.json @@ -0,0 +1,196 @@ +{ + "$defs": { + "Language": { + "enum": [ + "c17", + "cpp20", + "python313", + "java21" + ], + "title": "Language", + "type": "string" + }, + "ProblemImportLimits": { + "additionalProperties": false, + "properties": { + "archive_bytes": { + "default": 67108864, + "title": "Archive Bytes", + "type": "integer" + }, + "archive_format": { + "const": "single_level_zip", + "default": "single_level_zip", + "title": "Archive Format", + "type": "string" + }, + "candidate_lifetime_seconds": { + "default": 86400, + "title": "Candidate Lifetime Seconds", + "type": "integer" + }, + "chunk_bytes": { + "default": 1048576, + "title": "Chunk Bytes", + "type": "integer" + }, + "filename_ascii_bytes": { + "default": 128, + "title": "Filename Ascii Bytes", + "type": "integer" + }, + "files": { + "default": 534, + "title": "Files", + "type": "integer" + }, + "global_concurrency": { + "const": 1, + "default": 1, + "title": "Global Concurrency", + "type": "integer" + }, + "languages": { + "default": [ + "c17", + "cpp20", + "python313", + "java21" + ], + "items": { + "$ref": "#/$defs/Language" + }, + "title": "Languages", + "type": "array" + }, + "manifest_name": { + "const": "problem.json", + "default": "problem.json", + "title": "Manifest Name", + "type": "string" + }, + "manifest_schema_versions": { + "default": [ + 1, + 2 + ], + "items": { + "enum": [ + 1, + 2 + ], + "type": "integer" + }, + "title": "Manifest Schema Versions", + "type": "array" + }, + "owner_candidates": { + "default": 10, + "title": "Owner Candidates", + "type": "integer" + }, + "owner_concurrency": { + "const": 1, + "default": 1, + "title": "Owner Concurrency", + "type": "integer" + }, + "schema_version": { + "const": 1, + "default": 1, + "title": "Schema Version", + "type": "integer" + }, + "staging_bytes": { + "default": 268435456, + "title": "Staging Bytes", + "type": "integer" + }, + "total_candidates": { + "default": 32, + "title": "Total Candidates", + "type": "integer" + }, + "unpacked_bytes": { + "default": 67108864, + "title": "Unpacked Bytes", + "type": "integer" + }, + "upload_chunks": { + "default": 4096, + "title": "Upload Chunks", + "type": "integer" + }, + "upload_deadline_seconds": { + "default": 30, + "title": "Upload Deadline Seconds", + "type": "integer" + }, + "validation_deadline_seconds": { + "default": 60, + "title": "Validation Deadline Seconds", + "type": "integer" + } + }, + "title": "ProblemImportLimits", + "type": "object" + } + }, + "$id": "urn:kf-oj:problem-import-capabilities:v1", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "limits": { + "$ref": "#/$defs/ProblemImportLimits", + "default": { + "archive_bytes": 67108864, + "archive_format": "single_level_zip", + "candidate_lifetime_seconds": 86400, + "chunk_bytes": 1048576, + "filename_ascii_bytes": 128, + "files": 534, + "global_concurrency": 1, + "languages": [ + "c17", + "cpp20", + "python313", + "java21" + ], + "manifest_name": "problem.json", + "manifest_schema_versions": [ + 1, + 2 + ], + "owner_candidates": 10, + "owner_concurrency": 1, + "schema_version": 1, + "staging_bytes": 268435456, + "total_candidates": 32, + "unpacked_bytes": 67108864, + "upload_chunks": 4096, + "upload_deadline_seconds": 30, + "validation_deadline_seconds": 60 + } + }, + "schema_version": { + "const": 1, + "default": 1, + "title": "Schema Version", + "type": "integer" + }, + "uploads_available": { + "title": "Uploads Available", + "type": "boolean" + }, + "validation_available": { + "title": "Validation Available", + "type": "boolean" + } + }, + "required": [ + "uploads_available", + "validation_available" + ], + "title": "ProblemImportCapabilities", + "type": "object" +} diff --git a/contracts/schemas/problem-import-create.v1.schema.json b/contracts/schemas/problem-import-create.v1.schema.json new file mode 100644 index 0000000..25c706a --- /dev/null +++ b/contracts/schemas/problem-import-create.v1.schema.json @@ -0,0 +1,38 @@ +{ + "$id": "urn:kf-oj:problem-import-create:v1", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "archive_bytes": { + "maximum": 67108864, + "minimum": 1, + "title": "Archive Bytes", + "type": "integer" + }, + "archive_digest": { + "maxLength": 64, + "minLength": 64, + "pattern": "^[0-9a-f]{64}$", + "title": "Archive Digest", + "type": "string" + }, + "operation_id": { + "format": "uuid", + "title": "Operation Id", + "type": "string" + }, + "schema_version": { + "const": 1, + "default": 1, + "title": "Schema Version", + "type": "integer" + } + }, + "required": [ + "operation_id", + "archive_digest", + "archive_bytes" + ], + "title": "CreateProblemImport", + "type": "object" +} diff --git a/contracts/schemas/problem-import.v1.schema.json b/contracts/schemas/problem-import.v1.schema.json new file mode 100644 index 0000000..4196128 --- /dev/null +++ b/contracts/schemas/problem-import.v1.schema.json @@ -0,0 +1,423 @@ +{ + "$defs": { + "CandidatePreview": { + "additionalProperties": false, + "properties": { + "candidate_id": { + "format": "uuid", + "title": "Candidate Id", + "type": "string" + }, + "case_count": { + "maximum": 256, + "minimum": 1, + "title": "Case Count", + "type": "integer" + }, + "judging_schema_version": { + "maximum": 2, + "minimum": 1, + "title": "Judging Schema Version", + "type": "integer" + }, + "language_configurations": { + "default": [], + "items": { + "$ref": "#/$defs/PublicLanguageConfiguration" + }, + "maxItems": 4, + "title": "Language Configurations", + "type": "array" + }, + "languages": { + "items": { + "$ref": "#/$defs/Language" + }, + "maxItems": 4, + "minItems": 1, + "title": "Languages", + "type": "array" + }, + "legacy_budget": { + "anyOf": [ + { + "$ref": "#/$defs/LegacyImportBudget" + }, + { + "type": "null" + } + ] + }, + "samples": { + "items": { + "$ref": "#/$defs/PublicSample" + }, + "maxItems": 10, + "title": "Samples", + "type": "array" + }, + "slug": { + "pattern": "^[a-z0-9][a-z0-9_-]{0,63}$", + "title": "Slug", + "type": "string" + }, + "statement": { + "maxLength": 262144, + "minLength": 1, + "title": "Statement", + "type": "string" + }, + "title": { + "maxLength": 160, + "minLength": 1, + "title": "Title", + "type": "string" + } + }, + "required": [ + "candidate_id", + "slug", + "title", + "statement", + "samples", + "judging_schema_version", + "languages", + "legacy_budget", + "case_count" + ], + "title": "CandidatePreview", + "type": "object" + }, + "ImportFailure": { + "enum": [ + "invalid_archive", + "invalid_manifest", + "invalid_data", + "digest_mismatch", + "policy_rejected", + "upload_interrupted", + "upload_deadline", + "validation_deadline", + "validation_unavailable", + "storage_unavailable", + "authority_revoked", + "expired" + ], + "title": "ImportFailure", + "type": "string" + }, + "ImportState": { + "enum": [ + "uploading", + "uploaded", + "validating", + "ready", + "failed", + "published", + "discarded", + "expired" + ], + "title": "ImportState", + "type": "string" + }, + "Language": { + "enum": [ + "c17", + "cpp20", + "python313", + "java21" + ], + "title": "Language", + "type": "string" + }, + "LegacyImportBudget": { + "additionalProperties": false, + "properties": { + "compile_limits": { + "$ref": "#/$defs/RunLimits" + }, + "job_wall_time_us": { + "maximum": 3600000000, + "minimum": 1000, + "title": "Job Wall Time Us", + "type": "integer" + }, + "run_limits": { + "$ref": "#/$defs/RunLimits" + } + }, + "required": [ + "compile_limits", + "run_limits", + "job_wall_time_us" + ], + "title": "LegacyImportBudget", + "type": "object" + }, + "PreparationKind": { + "enum": [ + "compile", + "syntax_check" + ], + "title": "PreparationKind", + "type": "string" + }, + "PublicLanguageConfiguration": { + "additionalProperties": false, + "properties": { + "compile_limits": { + "$ref": "#/$defs/RunLimits" + }, + "entrypoint": { + "title": "Entrypoint", + "type": "string" + }, + "job_wall_time_us": { + "title": "Job Wall Time Us", + "type": "integer" + }, + "language": { + "$ref": "#/$defs/Language" + }, + "preparation_kind": { + "$ref": "#/$defs/PreparationKind" + }, + "run_limits": { + "$ref": "#/$defs/RunLimits" + }, + "runtime_version": { + "title": "Runtime Version", + "type": "string" + }, + "source_name": { + "title": "Source Name", + "type": "string" + } + }, + "required": [ + "language", + "runtime_version", + "source_name", + "entrypoint", + "preparation_kind", + "compile_limits", + "run_limits", + "job_wall_time_us" + ], + "title": "PublicLanguageConfiguration", + "type": "object" + }, + "PublicSample": { + "additionalProperties": false, + "properties": { + "encoding": { + "const": "base64", + "default": "base64", + "title": "Encoding", + "type": "string" + }, + "input": { + "maxLength": 87384, + "title": "Input", + "type": "string" + }, + "output": { + "maxLength": 87384, + "title": "Output", + "type": "string" + } + }, + "required": [ + "input", + "output" + ], + "title": "PublicSample", + "type": "object" + }, + "RunLimits": { + "additionalProperties": false, + "properties": { + "cpu_time_us": { + "maximum": 60000000, + "minimum": 1000, + "title": "Cpu Time Us", + "type": "integer" + }, + "file_bytes": { + "maximum": 67108864, + "minimum": 1, + "title": "File Bytes", + "type": "integer" + }, + "memory_bytes": { + "maximum": 2147483648, + "minimum": 1, + "title": "Memory Bytes", + "type": "integer" + }, + "open_files": { + "maximum": 1024, + "minimum": 3, + "title": "Open Files", + "type": "integer" + }, + "processes": { + "maximum": 128, + "minimum": 1, + "title": "Processes", + "type": "integer" + }, + "stack_bytes": { + "maximum": 268435456, + "minimum": 1, + "title": "Stack Bytes", + "type": "integer" + }, + "stderr_bytes": { + "maximum": 16777216, + "minimum": 1, + "title": "Stderr Bytes", + "type": "integer" + }, + "stdout_bytes": { + "maximum": 16777216, + "minimum": 1, + "title": "Stdout Bytes", + "type": "integer" + }, + "wall_time_us": { + "maximum": 120000000, + "minimum": 1000, + "title": "Wall Time Us", + "type": "integer" + }, + "workdir_bytes": { + "maximum": 268435456, + "minimum": 1, + "title": "Workdir Bytes", + "type": "integer" + } + }, + "required": [ + "cpu_time_us", + "wall_time_us", + "memory_bytes", + "stack_bytes", + "processes", + "open_files", + "stdout_bytes", + "stderr_bytes", + "file_bytes", + "workdir_bytes" + ], + "title": "RunLimits", + "type": "object" + } + }, + "$id": "urn:kf-oj:problem-import:v1", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "archive_bytes": { + "maximum": 67108864, + "minimum": 1, + "title": "Archive Bytes", + "type": "integer" + }, + "candidate": { + "anyOf": [ + { + "$ref": "#/$defs/CandidatePreview" + }, + { + "type": "null" + } + ] + }, + "candidate_id": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Candidate Id" + }, + "created_at": { + "format": "date-time", + "title": "Created At", + "type": "string" + }, + "expected_archive_digest": { + "maxLength": 64, + "minLength": 64, + "pattern": "^[0-9a-f]{64}$", + "title": "Expected Archive Digest", + "type": "string" + }, + "expires_at": { + "format": "date-time", + "title": "Expires At", + "type": "string" + }, + "failure_code": { + "anyOf": [ + { + "$ref": "#/$defs/ImportFailure" + }, + { + "type": "null" + } + ] + }, + "operation_id": { + "format": "uuid", + "title": "Operation Id", + "type": "string" + }, + "published_version_id": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Published Version Id" + }, + "schema_version": { + "const": 1, + "default": 1, + "title": "Schema Version", + "type": "integer" + }, + "state": { + "$ref": "#/$defs/ImportState" + }, + "updated_at": { + "format": "date-time", + "title": "Updated At", + "type": "string" + } + }, + "required": [ + "operation_id", + "state", + "expected_archive_digest", + "archive_bytes", + "created_at", + "expires_at", + "updated_at", + "failure_code", + "candidate_id", + "published_version_id", + "candidate" + ], + "title": "ProblemImportDetail", + "type": "object" +} diff --git a/deploy/problem-import-worker.service.example b/deploy/problem-import-worker.service.example new file mode 100644 index 0000000..41de7ba --- /dev/null +++ b/deploy/problem-import-worker.service.example @@ -0,0 +1,32 @@ +[Unit] +Description=KF-OnlineJudge bounded problem import service +After=postgresql.service + +[Service] +Type=exec +User=kf-oj +Group=kf-oj +WorkingDirectory=/opt/kf-onlinejudge/backend +EnvironmentFile=/etc/kf-onlinejudge/import-worker.env +Environment=KF_OJ_IMPORT_WORKER_CGROUP=/sys/fs/cgroup/system.slice/kf-problem-import.service +ExecStart=/opt/kf-onlinejudge/backend/.venv/bin/kf-problem-import-worker +MemoryMax=128M +MemorySwapMax=0 +TasksMax=16 +CPUQuota=100% +LimitFSIZE=64M +LimitNOFILE=64 +LimitCORE=0 +NoNewPrivileges=yes +PrivateNetwork=yes +RestrictAddressFamilies=AF_UNIX +ProtectSystem=strict +ProtectHome=yes +ReadWritePaths=/var/lib/kf-onlinejudge/imports /var/lib/kf-onlinejudge/artifacts +KillMode=control-group +TimeoutStopSec=10 +Restart=on-failure +RestartSec=30 + +[Install] +WantedBy=multi-user.target diff --git a/frontend/src/api/generated/schema.d.ts b/frontend/src/api/generated/schema.d.ts index fd50842..e38ee6c 100644 --- a/frontend/src/api/generated/schema.d.ts +++ b/frontend/src/api/generated/schema.d.ts @@ -4,6 +4,92 @@ */ export interface paths { + "/api/v1/admin/problem-imports": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Page */ + get: operations["page_api_v1_admin_problem_imports_get"]; + put?: never; + /** Create */ + post: operations["create_api_v1_admin_problem_imports_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/problem-imports/capabilities": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Capabilities */ + get: operations["capabilities_api_v1_admin_problem_imports_capabilities_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/problem-imports/{operation_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Detail */ + get: operations["detail_api_v1_admin_problem_imports__operation_id__get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/problem-imports/{operation_id}/archive": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + /** Upload */ + put: operations["upload_api_v1_admin_problem_imports__operation_id__archive_put"]; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/admin/problem-imports/{operation_id}/discard": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Discard */ + post: operations["discard_api_v1_admin_problem_imports__operation_id__discard_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/v1/auth/login": { parameters: { query?: never; @@ -278,11 +364,57 @@ export interface components { */ submission_id: string; }; + /** CandidatePreview */ + CandidatePreview: { + /** + * Candidate Id + * Format: uuid + */ + candidate_id: string; + /** Case Count */ + case_count: number; + /** Judging Schema Version */ + judging_schema_version: number; + /** + * Language Configurations + * @default [] + */ + language_configurations: components["schemas"]["PublicLanguageConfiguration"][]; + /** Languages */ + languages: components["schemas"]["kf_oj__contracts__judging_v2__Language"][]; + legacy_budget: components["schemas"]["LegacyImportBudget"] | null; + /** Samples */ + samples: components["schemas"]["PublicSample"][]; + /** Slug */ + slug: string; + /** Statement */ + statement: string; + /** Title */ + title: string; + }; /** * CompilationStatus * @enum {string} */ CompilationStatus: "succeeded" | "compilation_error" | "compilation_limit" | "cancelled" | "system_error"; + /** CreateProblemImport */ + CreateProblemImport: { + /** Archive Bytes */ + archive_bytes: number; + /** Archive Digest */ + archive_digest: string; + /** + * Operation Id + * Format: uuid + */ + operation_id: string; + /** + * Schema Version + * @default 1 + * @constant + */ + schema_version: 1; + }; /** CreateSubmission */ CreateSubmission: { /** @@ -335,6 +467,16 @@ export interface components { /** Version */ version: string; }; + /** + * ImportFailure + * @enum {string} + */ + ImportFailure: "invalid_archive" | "invalid_manifest" | "invalid_data" | "digest_mismatch" | "policy_rejected" | "upload_interrupted" | "upload_deadline" | "validation_deadline" | "validation_unavailable" | "storage_unavailable" | "authority_revoked" | "expired"; + /** + * ImportState + * @enum {string} + */ + ImportState: "uploading" | "uploaded" | "validating" | "ready" | "failed" | "published" | "discarded" | "expired"; /** JobManifest */ JobManifest: { /** Cases */ @@ -382,6 +524,13 @@ export interface components { * @enum {string} */ "Language-Input": "c17" | "cpp20" | "python313" | "java21"; + /** LegacyImportBudget */ + LegacyImportBudget: { + compile_limits: components["schemas"]["RunLimits"]; + /** Job Wall Time Us */ + job_wall_time_us: number; + run_limits: components["schemas"]["RunLimits"]; + }; /** LoginRequest */ LoginRequest: { /** @@ -476,6 +625,254 @@ export interface components { */ version_id: string; }; + /** ProblemImportCapabilities */ + ProblemImportCapabilities: { + /** + * @default { + * "archive_bytes": 67108864, + * "archive_format": "single_level_zip", + * "candidate_lifetime_seconds": 86400, + * "chunk_bytes": 1048576, + * "filename_ascii_bytes": 128, + * "files": 534, + * "global_concurrency": 1, + * "languages": [ + * "c17", + * "cpp20", + * "python313", + * "java21" + * ], + * "manifest_name": "problem.json", + * "manifest_schema_versions": [ + * 1, + * 2 + * ], + * "owner_candidates": 10, + * "owner_concurrency": 1, + * "schema_version": 1, + * "staging_bytes": 268435456, + * "total_candidates": 32, + * "unpacked_bytes": 67108864, + * "upload_chunks": 4096, + * "upload_deadline_seconds": 30, + * "validation_deadline_seconds": 60 + * } + */ + limits: components["schemas"]["ProblemImportLimits"]; + /** + * Schema Version + * @default 1 + * @constant + */ + schema_version: 1; + /** Uploads Available */ + uploads_available: boolean; + /** Validation Available */ + validation_available: boolean; + }; + /** ProblemImportDetail */ + ProblemImportDetail: { + /** Archive Bytes */ + archive_bytes: number; + candidate: components["schemas"]["CandidatePreview"] | null; + /** Candidate Id */ + candidate_id: string | null; + /** + * Created At + * Format: date-time + */ + created_at: string; + /** Expected Archive Digest */ + expected_archive_digest: string; + /** + * Expires At + * Format: date-time + */ + expires_at: string; + failure_code: components["schemas"]["ImportFailure"] | null; + /** + * Operation Id + * Format: uuid + */ + operation_id: string; + /** Published Version Id */ + published_version_id: string | null; + /** + * Schema Version + * @default 1 + * @constant + */ + schema_version: 1; + state: components["schemas"]["ImportState"]; + /** + * Updated At + * Format: date-time + */ + updated_at: string; + }; + /** ProblemImportLimits */ + ProblemImportLimits: { + /** + * Archive Bytes + * @default 67108864 + */ + archive_bytes: number; + /** + * Archive Format + * @default single_level_zip + * @constant + */ + archive_format: "single_level_zip"; + /** + * Candidate Lifetime Seconds + * @default 86400 + */ + candidate_lifetime_seconds: number; + /** + * Chunk Bytes + * @default 1048576 + */ + chunk_bytes: number; + /** + * Filename Ascii Bytes + * @default 128 + */ + filename_ascii_bytes: number; + /** + * Files + * @default 534 + */ + files: number; + /** + * Global Concurrency + * @default 1 + * @constant + */ + global_concurrency: 1; + /** + * Languages + * @default [ + * "c17", + * "cpp20", + * "python313", + * "java21" + * ] + */ + languages: components["schemas"]["kf_oj__contracts__judging_v2__Language"][]; + /** + * Manifest Name + * @default problem.json + * @constant + */ + manifest_name: "problem.json"; + /** + * Manifest Schema Versions + * @default [ + * 1, + * 2 + * ] + */ + manifest_schema_versions: (1 | 2)[]; + /** + * Owner Candidates + * @default 10 + */ + owner_candidates: number; + /** + * Owner Concurrency + * @default 1 + * @constant + */ + owner_concurrency: 1; + /** + * Schema Version + * @default 1 + * @constant + */ + schema_version: 1; + /** + * Staging Bytes + * @default 268435456 + */ + staging_bytes: number; + /** + * Total Candidates + * @default 32 + */ + total_candidates: number; + /** + * Unpacked Bytes + * @default 67108864 + */ + unpacked_bytes: number; + /** + * Upload Chunks + * @default 4096 + */ + upload_chunks: number; + /** + * Upload Deadline Seconds + * @default 30 + */ + upload_deadline_seconds: number; + /** + * Validation Deadline Seconds + * @default 60 + */ + validation_deadline_seconds: number; + }; + /** ProblemImportPage */ + ProblemImportPage: { + /** Items */ + items: components["schemas"]["ProblemImportSummary"][]; + /** Next After */ + next_after: string | null; + /** + * Schema Version + * @default 1 + * @constant + */ + schema_version: 1; + }; + /** ProblemImportSummary */ + ProblemImportSummary: { + /** Archive Bytes */ + archive_bytes: number; + /** Candidate Id */ + candidate_id: string | null; + /** + * Created At + * Format: date-time + */ + created_at: string; + /** Expected Archive Digest */ + expected_archive_digest: string; + /** + * Expires At + * Format: date-time + */ + expires_at: string; + failure_code: components["schemas"]["ImportFailure"] | null; + /** + * Operation Id + * Format: uuid + */ + operation_id: string; + /** Published Version Id */ + published_version_id: string | null; + /** + * Schema Version + * @default 1 + * @constant + */ + schema_version: 1; + state: components["schemas"]["ImportState"]; + /** + * Updated At + * Format: date-time + */ + updated_at: string; + }; /** ProblemPage */ ProblemPage: { /** Items */ @@ -811,6 +1208,188 @@ export interface components { } export type $defs = Record; export interface operations { + page_api_v1_admin_problem_imports_get: { + parameters: { + query?: { + after?: string | null; + limit?: number; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ProblemImportPage"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + create_api_v1_admin_problem_imports_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["CreateProblemImport"]; + }; + }; + responses: { + /** @description Successful Response */ + 201: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ProblemImportSummary"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + capabilities_api_v1_admin_problem_imports_capabilities_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ProblemImportCapabilities"]; + }; + }; + }; + }; + detail_api_v1_admin_problem_imports__operation_id__get: { + parameters: { + query?: never; + header?: never; + path: { + operation_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ProblemImportDetail"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + upload_api_v1_admin_problem_imports__operation_id__archive_put: { + parameters: { + query?: never; + header?: never; + path: { + operation_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/zip": string; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ProblemImportSummary"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + discard_api_v1_admin_problem_imports__operation_id__discard_post: { + parameters: { + query?: never; + header?: never; + path: { + operation_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ProblemImportSummary"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; login_api_v1_auth_login_post: { parameters: { query?: never;