feat(problems): add bounded administrator package imports
Use live maintenance sessions, durable capacity reservations and streamed SHA-bound ZIP uploads. An ordinary capped import worker validates stored/deflated v1/v2 packages, persists immutable unpublished candidates and finite audit events, and recovers or releases capacity after interruption and expiry. Validate actual PG/HTTP/worker/storage fault paths in an isolated capped VM (2 passed, 0 skipped), reuse unchanged actual migration roundtrip evidence, and pass affected backend/architecture checks plus the prior full engineering checks.
This commit is contained in:
@@ -10,6 +10,8 @@ from kf_oj.infra.db.base import Base
|
||||
from kf_oj.infra.db.session import build_engine
|
||||
from kf_oj.infra.messaging import models as messaging_models # noqa: F401
|
||||
from kf_oj.modules.identity import models as identity_models # noqa: F401
|
||||
from kf_oj.modules.operations import models as operations_models # noqa: F401
|
||||
from kf_oj.modules.problems import import_models # noqa: F401
|
||||
from kf_oj.modules.problems import models as problem_models # noqa: F401
|
||||
from kf_oj.modules.submissions import models as submission_models # noqa: F401
|
||||
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
"""Durable imports/candidates, append-only events and accepted-authority worker port."""
|
||||
|
||||
from alembic import op
|
||||
|
||||
from kf_oj.modules.operations.models import MaintenanceEvent
|
||||
from kf_oj.modules.problems.import_models import (
|
||||
CandidateCase,
|
||||
ImportPool,
|
||||
ProblemCandidate,
|
||||
ProblemImportOperation,
|
||||
)
|
||||
|
||||
revision = "0007_problem_imports"
|
||||
down_revision = "0006_authority_operations"
|
||||
branch_labels = depends_on = None
|
||||
|
||||
|
||||
def sql(script):
|
||||
# asyncpg prepares one statement at a time; dollar-quoted bodies stay intact.
|
||||
buffer, quote, dollar, i = [], False, False, 0
|
||||
while i < len(script):
|
||||
if not quote and script[i : i + 2] == "$$":
|
||||
dollar = not dollar
|
||||
buffer.append("$$")
|
||||
i += 2
|
||||
continue
|
||||
character = script[i]
|
||||
if not dollar and character == "'":
|
||||
if quote and script[i : i + 2] == "''":
|
||||
buffer.append("''")
|
||||
i += 2
|
||||
continue
|
||||
quote = not quote
|
||||
if character == ";" and not dollar and not quote:
|
||||
if "".join(buffer).strip():
|
||||
op.execute("".join(buffer))
|
||||
buffer = []
|
||||
else:
|
||||
buffer.append(character)
|
||||
i += 1
|
||||
if "".join(buffer).strip():
|
||||
op.execute("".join(buffer))
|
||||
|
||||
|
||||
def upgrade():
|
||||
connection = op.get_bind()
|
||||
# Cyclic candidate/operation link is installed only after both tables exist.
|
||||
for model in (
|
||||
ImportPool,
|
||||
ProblemImportOperation,
|
||||
ProblemCandidate,
|
||||
CandidateCase,
|
||||
MaintenanceEvent,
|
||||
):
|
||||
model.__table__.create(connection)
|
||||
op.create_foreign_key(
|
||||
"fk_problem_import_candidate",
|
||||
"problem_import_operations",
|
||||
"problem_candidates",
|
||||
["candidate_id"],
|
||||
["id"],
|
||||
)
|
||||
op.execute("INSERT INTO problem_import_pool(id,reserved_bytes) VALUES(1,0)")
|
||||
sql("""
|
||||
CREATE FUNCTION public.kf_lock_reserved_maintenance(actor uuid, epoch bigint)
|
||||
RETURNS void LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$
|
||||
DECLARE permitted boolean;
|
||||
BEGIN
|
||||
SELECT enabled AND is_administrator AND authorization_version=epoch
|
||||
INTO permitted FROM public.accounts WHERE id=actor FOR SHARE;
|
||||
IF permitted IS DISTINCT FROM true THEN
|
||||
RAISE EXCEPTION 'problem maintenance required' USING ERRCODE='42501';
|
||||
END IF;
|
||||
END $$;
|
||||
GRANT CREATE ON SCHEMA public TO kf_oj_identity_authority_owner;
|
||||
ALTER FUNCTION public.kf_lock_reserved_maintenance(uuid,bigint)
|
||||
OWNER TO kf_oj_identity_authority_owner;
|
||||
REVOKE CREATE ON SCHEMA public FROM kf_oj_identity_authority_owner;
|
||||
REVOKE ALL ON FUNCTION public.kf_lock_reserved_maintenance(uuid,bigint) FROM PUBLIC;
|
||||
GRANT EXECUTE ON FUNCTION public.kf_lock_reserved_maintenance(uuid,bigint)
|
||||
TO kf_oj_problem_import;
|
||||
GRANT SELECT,INSERT,UPDATE ON problem_import_operations TO kf_oj_platform;
|
||||
GRANT SELECT,UPDATE ON problem_import_pool TO kf_oj_platform,kf_oj_problem_import;
|
||||
GRANT SELECT ON problem_candidates,problem_candidate_cases TO kf_oj_platform;
|
||||
GRANT SELECT ON problem_import_operations TO kf_oj_problem_import;
|
||||
GRANT UPDATE(state,updated_at,lease_until,attempts,fence,
|
||||
reserved_bytes,failure_code,candidate_id)
|
||||
ON problem_import_operations TO kf_oj_problem_import;
|
||||
GRANT SELECT,INSERT,DELETE ON problem_candidates,problem_candidate_cases
|
||||
TO kf_oj_problem_import;
|
||||
GRANT SELECT,INSERT ON artifacts TO kf_oj_problem_import;
|
||||
GRANT INSERT ON problem_maintenance_events TO kf_oj_platform,kf_oj_problem_import;
|
||||
GRANT SELECT ON problem_maintenance_events TO kf_oj_platform;
|
||||
""")
|
||||
sql("""
|
||||
CREATE FUNCTION public.kf_immutable_import_candidate() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$
|
||||
DECLARE candidate uuid; current_state text;
|
||||
BEGIN
|
||||
IF TG_TABLE_NAME='problem_candidates' THEN candidate=OLD.id;
|
||||
ELSE candidate=OLD.candidate_id; END IF;
|
||||
SELECT o.state INTO current_state FROM public.problem_candidates c
|
||||
JOIN public.problem_import_operations o ON o.id=c.operation_id WHERE c.id=candidate;
|
||||
IF TG_OP='UPDATE' OR current_state NOT IN ('discarded','expired','failed') THEN
|
||||
RAISE EXCEPTION 'immutable candidate' USING ERRCODE='23514';
|
||||
END IF;
|
||||
RETURN OLD;
|
||||
END $$;
|
||||
CREATE TRIGGER immutable_candidate BEFORE UPDATE OR DELETE ON problem_candidates
|
||||
FOR EACH ROW EXECUTE FUNCTION public.kf_immutable_import_candidate();
|
||||
CREATE TRIGGER immutable_candidate_case BEFORE UPDATE OR DELETE ON problem_candidate_cases
|
||||
FOR EACH ROW EXECUTE FUNCTION public.kf_immutable_import_candidate();
|
||||
CREATE FUNCTION public.kf_insert_import_candidate() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$
|
||||
DECLARE operation uuid; current_state text;
|
||||
BEGIN
|
||||
IF TG_TABLE_NAME='problem_candidates' THEN operation=NEW.operation_id;
|
||||
ELSE SELECT operation_id INTO operation FROM public.problem_candidates
|
||||
WHERE id=NEW.candidate_id; END IF;
|
||||
SELECT state INTO current_state FROM public.problem_import_operations
|
||||
WHERE id=operation FOR SHARE;
|
||||
IF current_state IS DISTINCT FROM 'validating' THEN
|
||||
RAISE EXCEPTION 'candidate is not being validated' USING ERRCODE='23514';
|
||||
END IF; RETURN NEW;
|
||||
END $$;
|
||||
CREATE TRIGGER insert_candidate BEFORE INSERT ON problem_candidates
|
||||
FOR EACH ROW EXECUTE FUNCTION public.kf_insert_import_candidate();
|
||||
CREATE TRIGGER insert_candidate_case BEFORE INSERT ON problem_candidate_cases
|
||||
FOR EACH ROW EXECUTE FUNCTION public.kf_insert_import_candidate();
|
||||
CREATE FUNCTION public.kf_fixed_import_receipt() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$ BEGIN
|
||||
IF NEW.id IS DISTINCT FROM OLD.id OR NEW.owner_id IS DISTINCT FROM OLD.owner_id
|
||||
OR NEW.authorization_version IS DISTINCT FROM OLD.authorization_version
|
||||
OR NEW.archive_digest IS DISTINCT FROM OLD.archive_digest
|
||||
OR NEW.archive_bytes IS DISTINCT FROM OLD.archive_bytes
|
||||
OR NEW.created_at IS DISTINCT FROM OLD.created_at
|
||||
OR NEW.expires_at IS DISTINCT FROM OLD.expires_at THEN
|
||||
RAISE EXCEPTION 'immutable import receipt' USING ERRCODE='23514';
|
||||
END IF;
|
||||
IF OLD.state IN ('ready','published') AND NEW.state IN ('ready','published')
|
||||
AND NEW.candidate_id IS DISTINCT FROM OLD.candidate_id THEN
|
||||
RAISE EXCEPTION 'immutable completed candidate receipt' USING ERRCODE='23514';
|
||||
END IF;
|
||||
IF OLD.state='published' AND (NEW.state IS DISTINCT FROM OLD.state
|
||||
OR NEW.published_version_id IS DISTINCT FROM OLD.published_version_id) THEN
|
||||
RAISE EXCEPTION 'immutable published import receipt' USING ERRCODE='23514';
|
||||
END IF;
|
||||
IF NEW.state='ready' THEN
|
||||
IF (OLD.state IS DISTINCT FROM 'ready' AND
|
||||
(NOT pg_has_role(current_user,'kf_oj_problem_import','USAGE')
|
||||
OR OLD.state!='validating')) OR NEW.candidate_id IS NULL
|
||||
OR NOT EXISTS(SELECT 1 FROM public.problem_candidates c
|
||||
WHERE c.id=NEW.candidate_id AND c.operation_id=NEW.id
|
||||
AND c.case_count=(SELECT count(*) FROM public.problem_candidate_cases b
|
||||
WHERE b.candidate_id=c.id)
|
||||
AND c.case_count-1=(SELECT max(position) FROM public.problem_candidate_cases b
|
||||
WHERE b.candidate_id=c.id)) THEN
|
||||
RAISE EXCEPTION 'invalid candidate completion' USING ERRCODE='23514';
|
||||
END IF;
|
||||
END IF; RETURN NEW;
|
||||
END $$;
|
||||
CREATE TRIGGER fixed_import_receipt BEFORE UPDATE ON problem_import_operations
|
||||
FOR EACH ROW EXECUTE FUNCTION public.kf_fixed_import_receipt();
|
||||
""")
|
||||
|
||||
|
||||
def downgrade():
|
||||
connection = op.get_bind()
|
||||
if connection.exec_driver_sql(
|
||||
"SELECT EXISTS(SELECT 1 FROM problem_import_operations)"
|
||||
).scalar():
|
||||
raise RuntimeError("cannot discard durable import history")
|
||||
op.execute("DROP FUNCTION public.kf_lock_reserved_maintenance(uuid,bigint)")
|
||||
op.drop_constraint(
|
||||
"fk_problem_import_candidate", "problem_import_operations", type_="foreignkey"
|
||||
)
|
||||
for model in (
|
||||
MaintenanceEvent,
|
||||
CandidateCase,
|
||||
ProblemCandidate,
|
||||
ProblemImportOperation,
|
||||
ImportPool,
|
||||
):
|
||||
model.__table__.drop(connection)
|
||||
sql(
|
||||
"DROP FUNCTION public.kf_immutable_import_candidate(); "
|
||||
"DROP FUNCTION public.kf_fixed_import_receipt(); "
|
||||
"DROP FUNCTION public.kf_insert_import_candidate()"
|
||||
)
|
||||
@@ -22,6 +22,7 @@ dependencies = [
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
kf-problem-import-worker = "kf_oj.workers.problem_import:main"
|
||||
kf-accounts = "kf_oj.modules.identity.cli:main"
|
||||
kf-problems = "kf_oj.modules.problems.cli:main"
|
||||
kf-artifacts = "kf_oj.infra.artifacts.cli:main"
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
"""Uniform live session gates; domains still lock/recheck authority in write transactions."""
|
||||
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import Depends, HTTPException
|
||||
|
||||
from kf_oj.api.dependencies import current_principal, unsafe_principal
|
||||
from kf_oj.modules.identity.public import Principal
|
||||
|
||||
|
||||
async def maintenance_reader(
|
||||
principal: Annotated[Principal, Depends(current_principal)],
|
||||
) -> Principal:
|
||||
if not principal.can_maintain_problems:
|
||||
raise HTTPException(403, "problem maintenance required")
|
||||
return principal
|
||||
|
||||
|
||||
async def maintenance_writer(
|
||||
principal: Annotated[Principal, Depends(unsafe_principal)],
|
||||
) -> Principal:
|
||||
# unsafe_principal already checks an explicit Origin and the live session's
|
||||
# bound CSRF token. This capability check cannot replace the domain's port.
|
||||
if not principal.can_maintain_problems:
|
||||
raise HTTPException(403, "problem maintenance required")
|
||||
return principal
|
||||
@@ -7,6 +7,7 @@ from starlette.responses import JSONResponse
|
||||
|
||||
from kf_oj import __version__
|
||||
from kf_oj.api.body_limits import BodyLimitMiddleware
|
||||
from kf_oj.api.v1.admin_imports import router as import_router
|
||||
from kf_oj.api.v1.identity import router as identity_router
|
||||
from kf_oj.api.v1.problems import router as problems_router
|
||||
from kf_oj.api.v1.router import router
|
||||
@@ -15,10 +16,13 @@ from kf_oj.api.v1.worker_artifacts import router as worker_router
|
||||
from kf_oj.core.logging import configure_logging
|
||||
from kf_oj.core.settings import Settings
|
||||
from kf_oj.infra.artifacts.repository import ArtifactRepository
|
||||
from kf_oj.infra.artifacts.staging import ImportStaging
|
||||
from kf_oj.infra.artifacts.store import ArtifactStore
|
||||
from kf_oj.infra.coordinator import CoordinatorControl
|
||||
from kf_oj.infra.db.session import build_engine, build_session_factory
|
||||
from kf_oj.modules.identity.service import IdentityService
|
||||
from kf_oj.modules.operations.service import MaintenanceAudit
|
||||
from kf_oj.modules.problems.imports import ProblemImports
|
||||
from kf_oj.modules.problems.service import ProblemService
|
||||
from kf_oj.modules.submissions.performance import SubmissionPerformanceQueries
|
||||
from kf_oj.modules.submissions.queries import SubmissionQueries
|
||||
@@ -36,6 +40,7 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
||||
logger.info("Platform API started")
|
||||
engine = None
|
||||
store = None
|
||||
staging = None
|
||||
try:
|
||||
if settings.database_url is not None:
|
||||
engine = build_engine(settings)
|
||||
@@ -52,6 +57,18 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
||||
max_files=settings.artifact_max_files,
|
||||
)
|
||||
artifacts = ArtifactRepository(store)
|
||||
if (
|
||||
settings.import_staging_root is not None
|
||||
and settings.judge_admission is not None
|
||||
):
|
||||
staging = ImportStaging(settings.import_staging_root)
|
||||
app.state.problem_imports = ProblemImports(
|
||||
app.state.sessions,
|
||||
staging,
|
||||
app.state.identity,
|
||||
MaintenanceAudit(),
|
||||
settings.judge_admission,
|
||||
)
|
||||
app.state.problems = ProblemService(
|
||||
app.state.sessions, artifacts, settings.judge_admission
|
||||
)
|
||||
@@ -80,10 +97,13 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
||||
app.state.submissions = None
|
||||
app.state.submission_queries = None
|
||||
app.state.submission_performance = None
|
||||
app.state.problem_imports = None
|
||||
if engine is not None:
|
||||
await engine.dispose()
|
||||
if store is not None:
|
||||
store.close()
|
||||
if staging is not None:
|
||||
staging.close()
|
||||
logger.info("Platform API stopped")
|
||||
|
||||
app = FastAPI(
|
||||
@@ -101,7 +121,9 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
||||
@app.middleware("http")
|
||||
async def private_responses(request, call_next):
|
||||
response = await call_next(request)
|
||||
if request.url.path.startswith(("/api/v1/auth", "/api/v1/submissions", "/api/v1/internal")):
|
||||
if request.url.path.startswith(
|
||||
("/api/v1/auth", "/api/v1/submissions", "/api/v1/internal", "/api/v1/admin")
|
||||
):
|
||||
response.headers["Cache-Control"] = "no-store"
|
||||
return response
|
||||
|
||||
@@ -114,4 +136,5 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
||||
app.include_router(problems_router, prefix="/api/v1")
|
||||
app.include_router(worker_router, prefix="/api/v1")
|
||||
app.include_router(submission_router, prefix="/api/v1")
|
||||
app.include_router(import_router, prefix="/api/v1")
|
||||
return app
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Bound request bytes and upload time before schema parsing can allocate large inputs."""
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
|
||||
from starlette.responses import JSONResponse
|
||||
|
||||
@@ -13,6 +14,14 @@ class BodyLimitMiddleware:
|
||||
if scope["type"] != "http" or scope["method"] in {"GET", "HEAD", "OPTIONS"}:
|
||||
await self.app(scope, receive, send)
|
||||
return
|
||||
if scope["method"] == "PUT" and re.fullmatch(
|
||||
r"/api/v1/admin/problem-imports/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/archive",
|
||||
scope["path"],
|
||||
):
|
||||
# Only this exact endpoint can stream. Its live administrator gate runs
|
||||
# before receive(), then its service enforces actual bytes/chunks/deadline.
|
||||
await self.app(scope, receive, send)
|
||||
return
|
||||
limit = 8192 if scope["path"] == "/api/v1/auth/login" else 1024 * 1024
|
||||
if scope["path"] == "/api/v1/submissions":
|
||||
# Max256KiB UTF-8 source, worst-case JSON escaping plus bounded metadata.
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
"""Administrator import contract; configured service availability is explicit."""
|
||||
|
||||
from typing import Annotated
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
from sqlalchemy.exc import DBAPIError
|
||||
from starlette.requests import ClientDisconnect
|
||||
|
||||
from kf_oj.api.admin_dependencies import maintenance_reader, maintenance_writer
|
||||
from kf_oj.contracts.admin_problems import (
|
||||
CreateProblemImport,
|
||||
ProblemImportCapabilities,
|
||||
ProblemImportDetail,
|
||||
ProblemImportPage,
|
||||
ProblemImportSummary,
|
||||
)
|
||||
from kf_oj.infra.artifacts.store import ArtifactError
|
||||
from kf_oj.modules.identity.public import MaintenanceDenied, Principal
|
||||
from kf_oj.modules.problems.imports import ImportConflict, ImportMissing, ImportUnavailable
|
||||
|
||||
router = APIRouter(prefix="/admin/problem-imports", tags=["problem-maintenance"])
|
||||
|
||||
|
||||
@router.get("/capabilities", response_model=ProblemImportCapabilities)
|
||||
async def capabilities(
|
||||
request: Request, _: Annotated[Principal, Depends(maintenance_reader)]
|
||||
) -> ProblemImportCapabilities:
|
||||
service = getattr(request.app.state, "problem_imports", None)
|
||||
if service is None:
|
||||
return ProblemImportCapabilities(uploads_available=False, validation_available=False)
|
||||
return await service.capabilities()
|
||||
|
||||
|
||||
def service(request):
|
||||
result = getattr(request.app.state, "problem_imports", None)
|
||||
if result is None:
|
||||
raise HTTPException(503, "problem imports unavailable")
|
||||
return result
|
||||
|
||||
|
||||
async def operation(awaitable):
|
||||
try:
|
||||
return await awaitable
|
||||
except MaintenanceDenied:
|
||||
raise HTTPException(403, "problem maintenance required") from None
|
||||
except ImportConflict:
|
||||
raise HTTPException(409, "import request conflict") from None
|
||||
except ImportMissing:
|
||||
raise HTTPException(404, "import unavailable") from None
|
||||
except TimeoutError:
|
||||
raise HTTPException(408, "upload deadline exceeded") from None
|
||||
except (ImportUnavailable, ArtifactError, OSError, DBAPIError):
|
||||
raise HTTPException(503, "problem imports unavailable") from None
|
||||
except ClientDisconnect:
|
||||
raise HTTPException(400, "upload interrupted") from None
|
||||
|
||||
|
||||
@router.post("", status_code=201, response_model=ProblemImportSummary)
|
||||
async def create(
|
||||
body: CreateProblemImport,
|
||||
request: Request,
|
||||
principal: Annotated[Principal, Depends(maintenance_writer)],
|
||||
):
|
||||
if request.headers.get("content-type", "").split(";", 1)[0] != "application/json":
|
||||
raise HTTPException(415, "JSON request required")
|
||||
return await operation(service(request).create(principal, body))
|
||||
|
||||
|
||||
@router.get("", response_model=ProblemImportPage)
|
||||
async def page(
|
||||
request: Request,
|
||||
principal: Annotated[Principal, Depends(maintenance_reader)],
|
||||
after: UUID | None = None,
|
||||
limit: Annotated[int, Query(ge=1, le=100)] = 20,
|
||||
):
|
||||
return await operation(service(request).list(principal, after, limit))
|
||||
|
||||
|
||||
@router.get("/{operation_id}", response_model=ProblemImportDetail)
|
||||
async def detail(
|
||||
operation_id: UUID,
|
||||
request: Request,
|
||||
principal: Annotated[Principal, Depends(maintenance_reader)],
|
||||
):
|
||||
return await operation(service(request).detail(principal, operation_id))
|
||||
|
||||
|
||||
@router.put(
|
||||
"/{operation_id}/archive",
|
||||
response_model=ProblemImportSummary,
|
||||
openapi_extra={
|
||||
"requestBody": {
|
||||
"required": True,
|
||||
"content": {"application/zip": {"schema": {"type": "string", "format": "binary"}}},
|
||||
}
|
||||
},
|
||||
)
|
||||
async def upload(
|
||||
operation_id: UUID,
|
||||
request: Request,
|
||||
principal: Annotated[Principal, Depends(maintenance_writer)],
|
||||
):
|
||||
if request.url.path != f"/api/v1/admin/problem-imports/{operation_id}/archive":
|
||||
raise HTTPException(422, "canonical operation UUID required")
|
||||
if request.headers.get("content-type", "").split(";", 1)[0] != "application/zip":
|
||||
raise HTTPException(415, "ZIP request required")
|
||||
if request.headers.get("content-encoding") not in (None, "identity"):
|
||||
raise HTTPException(415, "encoded upload unavailable")
|
||||
length = request.headers.get("content-length")
|
||||
if length is not None:
|
||||
try:
|
||||
if not length.isascii() or not length.isdecimal() or len(length) > 12:
|
||||
raise ValueError
|
||||
length = int(length)
|
||||
except ValueError:
|
||||
raise HTTPException(413, "invalid archive length") from None
|
||||
|
||||
async def chunks():
|
||||
while True:
|
||||
message = await request.receive()
|
||||
if message["type"] == "http.disconnect":
|
||||
raise ClientDisconnect()
|
||||
if message["type"] != "http.request":
|
||||
raise ImportConflict("invalid upload frame")
|
||||
yield message.get("body", b"")
|
||||
if not message.get("more_body", False):
|
||||
return
|
||||
|
||||
return await operation(service(request).upload(principal, operation_id, chunks(), length))
|
||||
|
||||
|
||||
@router.post("/{operation_id}/discard", response_model=ProblemImportSummary)
|
||||
async def discard(
|
||||
operation_id: UUID,
|
||||
request: Request,
|
||||
principal: Annotated[Principal, Depends(maintenance_writer)],
|
||||
):
|
||||
return await operation(service(request).discard(principal, operation_id))
|
||||
@@ -0,0 +1,180 @@
|
||||
"""Versioned problem-import DTOs; hidden bytes, paths and toolchain selection stay private."""
|
||||
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
from typing import Annotated, Literal
|
||||
|
||||
from pydantic import Field, StrictBool, model_validator
|
||||
|
||||
from kf_oj.contracts.judging import Digest, Identifier, Label, RunLimits, Version, WireModel
|
||||
from kf_oj.contracts.judging_v2 import Language
|
||||
from kf_oj.contracts.problems import PublicLanguageConfiguration, PublicSample
|
||||
|
||||
MAX_ARCHIVE_BYTES = 64 * 1024**2
|
||||
MAX_UNPACKED_BYTES = 64 * 1024**2
|
||||
MAX_ARCHIVE_FILES = 534
|
||||
MAX_ARCHIVE_NAME_BYTES = 128
|
||||
MAX_UPLOAD_CHUNKS = 4096
|
||||
MAX_UPLOAD_CHUNK_BYTES = 1024**2
|
||||
UPLOAD_DEADLINE_SECONDS = 30
|
||||
VALIDATION_DEADLINE_SECONDS = 60
|
||||
MAX_IMPORT_ATTEMPTS = 2
|
||||
IMPORT_LIFETIME_SECONDS = 86400
|
||||
MAX_OWNER_CANDIDATES = 10
|
||||
MAX_TOTAL_CANDIDATES = 32
|
||||
STAGING_MAX_BYTES = 256 * 1024**2
|
||||
STAGING_RESERVATION_BYTES = MAX_ARCHIVE_BYTES + MAX_UNPACKED_BYTES
|
||||
|
||||
|
||||
class ImportState(StrEnum):
|
||||
UPLOADING = "uploading"
|
||||
UPLOADED = "uploaded"
|
||||
VALIDATING = "validating"
|
||||
READY = "ready"
|
||||
FAILED = "failed"
|
||||
PUBLISHED = "published"
|
||||
DISCARDED = "discarded"
|
||||
EXPIRED = "expired"
|
||||
|
||||
|
||||
class ImportFailure(StrEnum):
|
||||
INVALID_ARCHIVE = "invalid_archive"
|
||||
INVALID_MANIFEST = "invalid_manifest"
|
||||
INVALID_DATA = "invalid_data"
|
||||
DIGEST_MISMATCH = "digest_mismatch"
|
||||
POLICY_REJECTED = "policy_rejected"
|
||||
UPLOAD_INTERRUPTED = "upload_interrupted"
|
||||
UPLOAD_DEADLINE = "upload_deadline"
|
||||
VALIDATION_DEADLINE = "validation_deadline"
|
||||
VALIDATION_UNAVAILABLE = "validation_unavailable"
|
||||
STORAGE_UNAVAILABLE = "storage_unavailable"
|
||||
AUTHORITY_REVOKED = "authority_revoked"
|
||||
EXPIRED = "expired"
|
||||
|
||||
|
||||
class ProblemImportLimits(WireModel):
|
||||
schema_version: Version = 1
|
||||
archive_format: Literal["single_level_zip"] = "single_level_zip"
|
||||
manifest_name: Literal["problem.json"] = "problem.json"
|
||||
manifest_schema_versions: tuple[Literal[1, 2], ...] = (1, 2)
|
||||
languages: tuple[Language, ...] = tuple(Language)
|
||||
archive_bytes: int = MAX_ARCHIVE_BYTES
|
||||
unpacked_bytes: int = MAX_UNPACKED_BYTES
|
||||
files: int = MAX_ARCHIVE_FILES
|
||||
filename_ascii_bytes: int = MAX_ARCHIVE_NAME_BYTES
|
||||
upload_chunks: int = MAX_UPLOAD_CHUNKS
|
||||
chunk_bytes: int = MAX_UPLOAD_CHUNK_BYTES
|
||||
upload_deadline_seconds: int = UPLOAD_DEADLINE_SECONDS
|
||||
validation_deadline_seconds: int = VALIDATION_DEADLINE_SECONDS
|
||||
global_concurrency: Literal[1] = 1
|
||||
owner_concurrency: Literal[1] = 1
|
||||
owner_candidates: int = MAX_OWNER_CANDIDATES
|
||||
total_candidates: int = MAX_TOTAL_CANDIDATES
|
||||
staging_bytes: int = STAGING_MAX_BYTES
|
||||
candidate_lifetime_seconds: int = IMPORT_LIFETIME_SECONDS
|
||||
|
||||
|
||||
class ProblemImportCapabilities(WireModel):
|
||||
schema_version: Version = 1
|
||||
limits: ProblemImportLimits = ProblemImportLimits()
|
||||
uploads_available: StrictBool
|
||||
validation_available: StrictBool
|
||||
|
||||
|
||||
class CreateProblemImport(WireModel):
|
||||
schema_version: Version = 1
|
||||
operation_id: Identifier
|
||||
archive_digest: Digest
|
||||
archive_bytes: Annotated[int, Field(strict=True, ge=1, le=MAX_ARCHIVE_BYTES)]
|
||||
|
||||
|
||||
class LegacyImportBudget(WireModel):
|
||||
compile_limits: RunLimits
|
||||
run_limits: RunLimits
|
||||
job_wall_time_us: Annotated[int, Field(strict=True, ge=1000, le=3600000000)]
|
||||
|
||||
|
||||
class CandidatePreview(WireModel):
|
||||
candidate_id: Identifier
|
||||
slug: Label
|
||||
title: Annotated[str, Field(strict=True, min_length=1, max_length=160)]
|
||||
statement: Annotated[str, Field(strict=True, min_length=1, max_length=262144)]
|
||||
samples: Annotated[tuple[PublicSample, ...], Field(max_length=10)]
|
||||
judging_schema_version: Annotated[int, Field(strict=True, ge=1, le=2)]
|
||||
languages: Annotated[tuple[Language, ...], Field(min_length=1, max_length=4)]
|
||||
legacy_budget: LegacyImportBudget | None
|
||||
language_configurations: Annotated[
|
||||
tuple[PublicLanguageConfiguration, ...], Field(max_length=4)
|
||||
] = ()
|
||||
case_count: Annotated[int, Field(strict=True, ge=1, le=256)]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def bounded_text(self):
|
||||
if len(self.statement.encode("utf-8")) > 262144 or "\x00" in self.statement + self.title:
|
||||
raise ValueError("invalid public text")
|
||||
languages = [c.language for c in self.language_configurations]
|
||||
if len(set(languages)) != len(languages):
|
||||
raise ValueError("duplicate language configuration")
|
||||
if len(set(self.languages)) != len(self.languages):
|
||||
raise ValueError("duplicate language")
|
||||
if self.judging_schema_version == 1:
|
||||
if (
|
||||
self.language_configurations
|
||||
or self.legacy_budget is None
|
||||
or any(
|
||||
language not in (Language.C17, Language.CPP20) for language in self.languages
|
||||
)
|
||||
):
|
||||
raise ValueError("invalid legacy candidate")
|
||||
elif tuple(languages) != self.languages or self.legacy_budget is not None:
|
||||
raise ValueError("invalid multi-runtime candidate")
|
||||
return self
|
||||
|
||||
|
||||
class ProblemImportSummary(WireModel):
|
||||
schema_version: Version = 1
|
||||
operation_id: Identifier
|
||||
state: ImportState
|
||||
expected_archive_digest: Digest
|
||||
archive_bytes: Annotated[int, Field(strict=True, ge=1, le=MAX_ARCHIVE_BYTES)]
|
||||
created_at: datetime
|
||||
expires_at: datetime
|
||||
updated_at: datetime
|
||||
failure_code: ImportFailure | None
|
||||
candidate_id: Identifier | None
|
||||
published_version_id: Identifier | None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def consistent_state(self):
|
||||
if self.expires_at <= self.created_at or self.updated_at < self.created_at:
|
||||
raise ValueError("invalid import time range")
|
||||
if (self.state in (ImportState.READY, ImportState.PUBLISHED)) != (
|
||||
self.candidate_id is not None
|
||||
):
|
||||
raise ValueError("candidate state mismatch")
|
||||
if (self.state == ImportState.PUBLISHED) != (self.published_version_id is not None):
|
||||
raise ValueError("published state mismatch")
|
||||
if self.state in (ImportState.FAILED, ImportState.EXPIRED):
|
||||
if self.failure_code is None:
|
||||
raise ValueError("failure code required")
|
||||
elif self.failure_code is not None:
|
||||
raise ValueError("unexpected failure code")
|
||||
return self
|
||||
|
||||
|
||||
class ProblemImportDetail(ProblemImportSummary):
|
||||
candidate: CandidatePreview | None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def candidate_matches(self):
|
||||
if (self.candidate_id is None) != (self.candidate is None):
|
||||
raise ValueError("candidate preview missing")
|
||||
if self.candidate and self.candidate.candidate_id != self.candidate_id:
|
||||
raise ValueError("candidate preview mismatch")
|
||||
return self
|
||||
|
||||
|
||||
class ProblemImportPage(WireModel):
|
||||
schema_version: Version = 1
|
||||
items: Annotated[tuple[ProblemImportSummary, ...], Field(max_length=100)]
|
||||
next_after: Identifier | None
|
||||
@@ -5,6 +5,11 @@ import json
|
||||
from pathlib import Path
|
||||
|
||||
from kf_oj.api.app import create_app
|
||||
from kf_oj.contracts.admin_problems import (
|
||||
CreateProblemImport,
|
||||
ProblemImportCapabilities,
|
||||
ProblemImportDetail,
|
||||
)
|
||||
from kf_oj.contracts.judge import JudgeTaskNotification
|
||||
from kf_oj.contracts.judging import WIRE_MODELS
|
||||
from kf_oj.contracts.judging_v2 import WIRE_MODELS as WIRE_MODELS_V2
|
||||
@@ -43,6 +48,15 @@ def main() -> None:
|
||||
"counts; also run model validation and server-side ownership/environment checks."
|
||||
)
|
||||
outputs[root / "contracts/schemas/submission-performance.v1.schema.json"] = performance
|
||||
for name, model in {
|
||||
"problem-import-create": CreateProblemImport,
|
||||
"problem-import": ProblemImportDetail,
|
||||
"problem-import-capabilities": ProblemImportCapabilities,
|
||||
}.items():
|
||||
document = model.model_json_schema()
|
||||
document["$schema"] = "https://json-schema.org/draft/2020-12/schema"
|
||||
document["$id"] = f"urn:kf-oj:{name}:v1"
|
||||
outputs[root / f"contracts/schemas/{name}.v1.schema.json"] = document
|
||||
for name, model in WIRE_MODELS_V2.items():
|
||||
document = model.model_json_schema()
|
||||
document["$schema"] = "https://json-schema.org/draft/2020-12/schema"
|
||||
|
||||
@@ -0,0 +1,340 @@
|
||||
"""Bounded single-level ZIP reader; no extractall, metadata restoration or content execution."""
|
||||
|
||||
import binascii
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
import struct
|
||||
import time
|
||||
import zlib
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
|
||||
from kf_oj.contracts.admin_problems import (
|
||||
MAX_ARCHIVE_BYTES,
|
||||
MAX_ARCHIVE_FILES,
|
||||
MAX_UNPACKED_BYTES,
|
||||
)
|
||||
|
||||
NAME = re.compile(rb"[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}\Z")
|
||||
CHUNK = 65536
|
||||
LOCAL = struct.Struct("<4s5H3I2H")
|
||||
CENTRAL = struct.Struct("<4s6H3I5H2I")
|
||||
END = struct.Struct("<4s4H2IH")
|
||||
|
||||
|
||||
class ArchiveError(ValueError):
|
||||
"""A fixed sanitized failure; never echo an archive name, data or local path."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ArchiveMember:
|
||||
name: str
|
||||
method: int
|
||||
crc32: int
|
||||
compressed_bytes: int
|
||||
bytes: int
|
||||
offset: int
|
||||
flags: int
|
||||
|
||||
|
||||
class ArchiveReader:
|
||||
def __init__(self, descriptor: int, expected_bytes: int, *, deadline: float):
|
||||
self.descriptor, self.deadline = descriptor, deadline
|
||||
self.before = os.fstat(descriptor)
|
||||
if (
|
||||
not stat.S_ISREG(self.before.st_mode)
|
||||
or self.before.st_uid != os.getuid()
|
||||
or self.before.st_nlink != 1
|
||||
or stat.S_IMODE(self.before.st_mode) != 0o600
|
||||
or not 22 <= expected_bytes <= MAX_ARCHIVE_BYTES
|
||||
or self.before.st_size != expected_bytes
|
||||
):
|
||||
raise ArchiveError("invalid_archive")
|
||||
self.size = expected_bytes
|
||||
self.unpacked_bytes, self.steps = 0, 0
|
||||
self.read_members: set[str] = set()
|
||||
self.members = self._members()
|
||||
|
||||
def _tick(self) -> None:
|
||||
self.steps += 1
|
||||
if self.steps > 8192 or time.monotonic() >= self.deadline:
|
||||
raise ArchiveError("archive_deadline")
|
||||
|
||||
def _read(self, offset: int, size: int) -> bytes:
|
||||
self._tick()
|
||||
if not 0 <= size <= 1024**2 or offset < 0 or offset + size > self.size:
|
||||
raise ArchiveError("invalid_archive")
|
||||
data = os.pread(self.descriptor, size, offset)
|
||||
if len(data) != size:
|
||||
raise ArchiveError("invalid_archive")
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
def _extra(data: bytes) -> None:
|
||||
offset, fields = 0, 0
|
||||
# These bounded timestamps/UID attributes are ignored, never restored.
|
||||
# ZIP64, Unicode aliases and all other extension semantics are unsupported.
|
||||
while offset < len(data):
|
||||
if offset + 4 > len(data) or fields >= 32:
|
||||
raise ArchiveError("invalid_archive")
|
||||
kind, size = struct.unpack_from("<HH", data, offset)
|
||||
offset += 4
|
||||
if kind not in (0x5455, 0x7875, 0x5855, 0x7855, 0x000A) or offset + size > len(data):
|
||||
raise ArchiveError("invalid_archive")
|
||||
offset += size
|
||||
fields += 1
|
||||
|
||||
def _members(self) -> tuple[ArchiveMember, ...]:
|
||||
tail_offset = max(0, self.size - 1046)
|
||||
tail = self._read(tail_offset, self.size - tail_offset)
|
||||
position = tail.rfind(b"PK\x05\x06")
|
||||
if position < 0 or position + END.size > len(tail):
|
||||
raise ArchiveError("invalid_archive")
|
||||
sig, disk, cd_disk, disk_count, count, cd_bytes, cd_offset, comment = END.unpack_from(
|
||||
tail, position
|
||||
)
|
||||
end_offset = tail_offset + position
|
||||
if (
|
||||
sig != b"PK\x05\x06"
|
||||
or disk != 0
|
||||
or cd_disk != 0
|
||||
or disk_count != count
|
||||
or not 1 <= count <= MAX_ARCHIVE_FILES
|
||||
or comment > 1024
|
||||
or end_offset + END.size + comment != self.size
|
||||
or not 46 <= cd_bytes <= 1024**2
|
||||
or cd_offset + cd_bytes != end_offset
|
||||
):
|
||||
raise ArchiveError("invalid_archive")
|
||||
directory = self._read(cd_offset, cd_bytes)
|
||||
offset, records, names = 0, [], set()
|
||||
for _ in range(count):
|
||||
if offset + CENTRAL.size > len(directory):
|
||||
raise ArchiveError("invalid_archive")
|
||||
values = CENTRAL.unpack_from(directory, offset)
|
||||
(
|
||||
sig,
|
||||
_,
|
||||
needed,
|
||||
flags,
|
||||
method,
|
||||
_,
|
||||
_,
|
||||
crc,
|
||||
compressed,
|
||||
size,
|
||||
name_size,
|
||||
extra_size,
|
||||
comment_size,
|
||||
disk,
|
||||
_,
|
||||
attributes,
|
||||
local_offset,
|
||||
) = values
|
||||
allowed_flags = 0x0808 | (0x0006 if method == 8 else 0)
|
||||
if (
|
||||
sig != b"PK\x01\x02"
|
||||
or needed not in (10, 20)
|
||||
or method not in (0, 8)
|
||||
or (method == 8 and needed != 20)
|
||||
or flags & ~allowed_flags
|
||||
or disk != 0
|
||||
or not 1 <= name_size <= 128
|
||||
or extra_size > 1024
|
||||
or comment_size > 256
|
||||
or size > MAX_UNPACKED_BYTES
|
||||
or compressed > MAX_ARCHIVE_BYTES
|
||||
or local_offset + LOCAL.size > cd_offset
|
||||
or attributes & 0x10
|
||||
or stat.S_IFMT(attributes >> 16) not in (0, stat.S_IFREG)
|
||||
):
|
||||
raise ArchiveError("invalid_archive")
|
||||
start = offset + CENTRAL.size
|
||||
finish = start + name_size + extra_size + comment_size
|
||||
if finish > len(directory):
|
||||
raise ArchiveError("invalid_archive")
|
||||
name_bytes = directory[start : start + name_size]
|
||||
if not NAME.fullmatch(name_bytes):
|
||||
raise ArchiveError("invalid_archive")
|
||||
name = name_bytes.decode("ascii")
|
||||
if name.lower() in names:
|
||||
raise ArchiveError("invalid_archive")
|
||||
names.add(name.lower())
|
||||
self._extra(directory[start + name_size : start + name_size + extra_size])
|
||||
records.append(ArchiveMember(name, method, crc, compressed, size, local_offset, flags))
|
||||
offset = finish
|
||||
if offset != len(directory) or sum(m.bytes for m in records) > MAX_UNPACKED_BYTES:
|
||||
raise ArchiveError("invalid_archive")
|
||||
cursor, members = 0, []
|
||||
for member in sorted(records, key=lambda m: m.offset):
|
||||
if member.offset != cursor:
|
||||
raise ArchiveError("invalid_archive")
|
||||
sig, needed, flags, method, _, _, crc, compressed, size, name_size, extra_size = (
|
||||
LOCAL.unpack(self._read(cursor, LOCAL.size))
|
||||
)
|
||||
if (
|
||||
sig != b"PK\x03\x04"
|
||||
or needed not in (10, 20)
|
||||
or flags != member.flags
|
||||
or method != member.method
|
||||
or (method == 8 and needed != 20)
|
||||
or name_size != len(member.name)
|
||||
or extra_size > 1024
|
||||
):
|
||||
raise ArchiveError("invalid_archive")
|
||||
metadata = self._read(cursor + LOCAL.size, name_size + extra_size)
|
||||
if metadata[:name_size] != member.name.encode("ascii"):
|
||||
raise ArchiveError("invalid_archive")
|
||||
self._extra(metadata[name_size:])
|
||||
data_offset = cursor + LOCAL.size + name_size + extra_size
|
||||
cursor = data_offset + member.compressed_bytes
|
||||
if cursor > cd_offset or (method == 0 and member.compressed_bytes != member.bytes):
|
||||
raise ArchiveError("invalid_archive")
|
||||
if flags & 8:
|
||||
if (crc, compressed, size) not in (
|
||||
(0, 0, 0),
|
||||
(member.crc32, member.compressed_bytes, member.bytes),
|
||||
):
|
||||
raise ArchiveError("invalid_archive")
|
||||
marker = self._read(cursor, 4)
|
||||
descriptor_size = 16 if marker == b"PK\x07\x08" else 12
|
||||
descriptor = self._read(cursor, descriptor_size)
|
||||
described = struct.unpack("<III", descriptor[-12:])
|
||||
if described != (member.crc32, member.compressed_bytes, member.bytes):
|
||||
raise ArchiveError("invalid_archive")
|
||||
cursor += descriptor_size
|
||||
elif (crc, compressed, size) != (member.crc32, member.compressed_bytes, member.bytes):
|
||||
raise ArchiveError("invalid_archive")
|
||||
members.append(
|
||||
ArchiveMember(
|
||||
member.name,
|
||||
method,
|
||||
member.crc32,
|
||||
member.compressed_bytes,
|
||||
member.bytes,
|
||||
data_offset,
|
||||
flags,
|
||||
)
|
||||
)
|
||||
if cursor != cd_offset:
|
||||
raise ArchiveError("invalid_archive")
|
||||
return tuple(members)
|
||||
|
||||
def verify_digest(self, expected: str) -> None:
|
||||
digest = hashlib.sha256()
|
||||
for offset in range(0, self.size, CHUNK):
|
||||
digest.update(self._read(offset, min(CHUNK, self.size - offset)))
|
||||
if digest.hexdigest() != expected:
|
||||
raise ArchiveError("archive_digest_mismatch")
|
||||
|
||||
def verify_unchanged(self) -> None:
|
||||
after = os.fstat(self.descriptor)
|
||||
fields = (
|
||||
"st_dev",
|
||||
"st_ino",
|
||||
"st_nlink",
|
||||
"st_mode",
|
||||
"st_uid",
|
||||
"st_size",
|
||||
"st_mtime_ns",
|
||||
"st_ctime_ns",
|
||||
)
|
||||
if any(getattr(self.before, f) != getattr(after, f) for f in fields):
|
||||
raise ArchiveError("archive_changed")
|
||||
|
||||
@staticmethod
|
||||
def _nested(prefix: bytes) -> bool:
|
||||
return prefix.startswith(
|
||||
(
|
||||
b"PK\x03\x04",
|
||||
b"PK\x05\x06",
|
||||
b"PK\x07\x08",
|
||||
b"\x1f\x8b",
|
||||
b"BZh",
|
||||
b"\xfd7zXZ\x00",
|
||||
b"Rar!\x1a\x07",
|
||||
b"7z\xbc\xaf\x27\x1c",
|
||||
)
|
||||
) or (len(prefix) >= 262 and prefix[257:262] == b"ustar")
|
||||
|
||||
def chunks(self, name: str, maximum: int) -> Iterator[bytes]:
|
||||
member = next((m for m in self.members if m.name == name), None)
|
||||
if member is None or name in self.read_members or member.bytes > maximum:
|
||||
raise ArchiveError("invalid_archive")
|
||||
self.read_members.add(name)
|
||||
used, crc, prefix = 0, 0, bytearray()
|
||||
inflater = zlib.decompressobj(-15) if member.method == 8 else None
|
||||
try:
|
||||
for offset in range(0, member.compressed_bytes, CHUNK):
|
||||
pending = self._read(
|
||||
member.offset + offset, min(CHUNK, member.compressed_bytes - offset)
|
||||
)
|
||||
while pending:
|
||||
self._tick()
|
||||
if inflater:
|
||||
data = inflater.decompress(pending, min(CHUNK, member.bytes - used + 1))
|
||||
pending = inflater.unconsumed_tail
|
||||
if inflater.unused_data:
|
||||
raise ArchiveError("invalid_archive")
|
||||
else:
|
||||
data, pending = pending, b""
|
||||
used += len(data)
|
||||
self.unpacked_bytes += len(data)
|
||||
if (
|
||||
used > member.bytes
|
||||
or used > maximum
|
||||
or self.unpacked_bytes > MAX_UNPACKED_BYTES
|
||||
):
|
||||
raise ArchiveError("invalid_archive")
|
||||
prefix.extend(data[: max(0, 512 - len(prefix))])
|
||||
if self._nested(bytes(prefix)):
|
||||
raise ArchiveError("invalid_archive")
|
||||
crc = binascii.crc32(data, crc)
|
||||
if data:
|
||||
yield data
|
||||
if (inflater and not inflater.eof) or used != member.bytes or crc != member.crc32:
|
||||
raise ArchiveError("invalid_archive")
|
||||
except zlib.error:
|
||||
raise ArchiveError("invalid_archive") from None
|
||||
|
||||
def read(self, name: str, maximum: int) -> bytes:
|
||||
return b"".join(self.chunks(name, maximum))
|
||||
|
||||
def extract_remaining(self, directory: int) -> dict[str, tuple[str, int]]:
|
||||
info = os.fstat(directory)
|
||||
if (
|
||||
not stat.S_ISDIR(info.st_mode)
|
||||
or info.st_uid != os.getuid()
|
||||
or stat.S_IMODE(info.st_mode) != 0o700
|
||||
):
|
||||
raise ArchiveError("invalid_extraction_directory")
|
||||
references = {}
|
||||
for member in self.members:
|
||||
if member.name in self.read_members:
|
||||
continue
|
||||
fd = os.open(
|
||||
member.name,
|
||||
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC,
|
||||
0o600,
|
||||
dir_fd=directory,
|
||||
)
|
||||
try:
|
||||
digest, size = hashlib.sha256(), 0
|
||||
for chunk in self.chunks(member.name, member.bytes):
|
||||
digest.update(chunk)
|
||||
size += len(chunk)
|
||||
remaining = memoryview(chunk)
|
||||
while remaining:
|
||||
written = os.write(fd, remaining)
|
||||
if written <= 0:
|
||||
raise ArchiveError("extraction_unavailable")
|
||||
remaining = remaining[written:]
|
||||
os.fsync(fd)
|
||||
references[member.name] = (digest.hexdigest(), size)
|
||||
finally:
|
||||
os.close(fd)
|
||||
os.fsync(directory)
|
||||
self.verify_unchanged()
|
||||
return references
|
||||
@@ -50,6 +50,28 @@ class ArtifactRepository:
|
||||
raise ArtifactError("artifact metadata unavailable")
|
||||
return await asyncio.to_thread(self.store.read, reference)
|
||||
|
||||
async def put_file(
|
||||
self, session: AsyncSession, descriptor: int, reference: ArtifactReference
|
||||
) -> ArtifactReference:
|
||||
await lock_digest(session, reference.digest)
|
||||
existing = await session.get(Artifact, reference.digest)
|
||||
if existing is not None and (
|
||||
existing.state != "ready" or existing.bytes != reference.bytes
|
||||
):
|
||||
raise ArtifactError("artifact metadata unavailable")
|
||||
task = asyncio.create_task(asyncio.to_thread(self.store.put_file, descriptor, reference))
|
||||
try:
|
||||
await asyncio.shield(task)
|
||||
except BaseException:
|
||||
await task
|
||||
raise
|
||||
await session.execute(
|
||||
insert(Artifact)
|
||||
.values(digest=reference.digest, bytes=reference.bytes)
|
||||
.on_conflict_do_nothing(index_elements=[Artifact.digest])
|
||||
)
|
||||
return reference
|
||||
|
||||
async def check_metadata(self, session: AsyncSession, reference: ArtifactReference) -> None:
|
||||
metadata = await session.get(Artifact, reference.digest)
|
||||
if metadata is None or metadata.state != "ready" or metadata.bytes != reference.bytes:
|
||||
|
||||
@@ -0,0 +1,208 @@
|
||||
"""Owned, private, bounded import staging. A kernel lock survives parent/child handoff."""
|
||||
|
||||
import fcntl
|
||||
import os
|
||||
import stat
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from uuid import UUID
|
||||
|
||||
from kf_oj.contracts.admin_problems import MAX_ARCHIVE_FILES, STAGING_MAX_BYTES
|
||||
from kf_oj.infra.artifacts.store import ArtifactError
|
||||
|
||||
|
||||
class ImportStaging:
|
||||
def __init__(self, root: Path):
|
||||
root = root.absolute()
|
||||
root.mkdir(parents=True, mode=0o700, exist_ok=True)
|
||||
if root.resolve() != root:
|
||||
raise ArtifactError("invalid staging root")
|
||||
self.root = root
|
||||
self.fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC)
|
||||
self.directory(self.fd)
|
||||
|
||||
@staticmethod
|
||||
def directory(fd):
|
||||
s = os.fstat(fd)
|
||||
if (
|
||||
not stat.S_ISDIR(s.st_mode)
|
||||
or s.st_uid != os.getuid()
|
||||
or stat.S_IMODE(s.st_mode) != 0o700
|
||||
):
|
||||
raise ArtifactError("invalid staging directory")
|
||||
|
||||
def close(self):
|
||||
os.close(self.fd)
|
||||
|
||||
@staticmethod
|
||||
def names(descriptor, maximum):
|
||||
result = []
|
||||
with os.scandir(descriptor) as entries:
|
||||
for entry in entries:
|
||||
if len(result) >= maximum:
|
||||
raise ArtifactError("staging file capacity exhausted")
|
||||
result.append(entry.name)
|
||||
return result
|
||||
|
||||
def operation_ids(self):
|
||||
identifiers = []
|
||||
for name in self.names(self.fd, 33):
|
||||
if name == ".lock":
|
||||
continue
|
||||
try:
|
||||
identifier = UUID(name)
|
||||
if str(identifier) != name:
|
||||
raise ValueError
|
||||
except ValueError:
|
||||
raise ArtifactError("unexpected staging entry") from None
|
||||
identifiers.append(identifier)
|
||||
return identifiers
|
||||
|
||||
@contextmanager
|
||||
def exclusive(self):
|
||||
lock = os.open(
|
||||
".lock", os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600, dir_fd=self.fd
|
||||
)
|
||||
try:
|
||||
from kf_oj.infra.artifacts.store import ArtifactStore
|
||||
|
||||
ArtifactStore._regular(os.fstat(lock))
|
||||
try:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
raise ArtifactError("import capacity busy") from None
|
||||
yield lock
|
||||
finally:
|
||||
os.close(lock)
|
||||
|
||||
def usage(self):
|
||||
total, count = 0, 0
|
||||
for name in self.names(self.fd, 33):
|
||||
if name == ".lock":
|
||||
continue
|
||||
try:
|
||||
if str(UUID(name)) != name:
|
||||
raise ValueError
|
||||
except ValueError:
|
||||
raise ArtifactError("unexpected staging entry") from None
|
||||
descriptor = self.operation(UUID(name))
|
||||
try:
|
||||
for entry in self.names(descriptor, 3):
|
||||
count += 1
|
||||
if count > 2 * (MAX_ARCHIVE_FILES + 2):
|
||||
raise ArtifactError("staging file capacity exhausted")
|
||||
if entry == "members":
|
||||
child = self.members(descriptor)
|
||||
try:
|
||||
for member in self.names(child, MAX_ARCHIVE_FILES):
|
||||
count += 1
|
||||
info = os.stat(member, dir_fd=child, follow_symlinks=False)
|
||||
from kf_oj.infra.artifacts.store import ArtifactStore
|
||||
|
||||
ArtifactStore._regular(info)
|
||||
total += info.st_size
|
||||
finally:
|
||||
os.close(child)
|
||||
elif entry in ("archive", "validated.json"):
|
||||
info = os.stat(entry, dir_fd=descriptor, follow_symlinks=False)
|
||||
from kf_oj.infra.artifacts.store import ArtifactStore
|
||||
|
||||
ArtifactStore._regular(info)
|
||||
total += info.st_size
|
||||
else:
|
||||
raise ArtifactError("unexpected staging entry")
|
||||
if count > 2 * (MAX_ARCHIVE_FILES + 2) or total > STAGING_MAX_BYTES:
|
||||
raise ArtifactError("staging capacity exhausted")
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
return total
|
||||
|
||||
def operation(self, operation_id: UUID, *, create=False):
|
||||
name = str(operation_id)
|
||||
if create:
|
||||
try:
|
||||
os.mkdir(name, 0o700, dir_fd=self.fd)
|
||||
except FileExistsError:
|
||||
pass
|
||||
descriptor = os.open(
|
||||
name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC, dir_fd=self.fd
|
||||
)
|
||||
try:
|
||||
self.directory(descriptor)
|
||||
except Exception:
|
||||
os.close(descriptor)
|
||||
raise
|
||||
return descriptor
|
||||
|
||||
@staticmethod
|
||||
def members(operation, *, create=False):
|
||||
if create:
|
||||
os.mkdir("members", 0o700, dir_fd=operation)
|
||||
descriptor = os.open(
|
||||
"members", os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC, dir_fd=operation
|
||||
)
|
||||
try:
|
||||
ImportStaging.directory(descriptor)
|
||||
except Exception:
|
||||
os.close(descriptor)
|
||||
raise
|
||||
return descriptor
|
||||
|
||||
def remove(self, operation_id):
|
||||
try:
|
||||
descriptor = self.operation(operation_id)
|
||||
except FileNotFoundError:
|
||||
return
|
||||
try:
|
||||
names = self.names(descriptor, 3)
|
||||
if len(names) > 3:
|
||||
raise ArtifactError("unexpected staging entry")
|
||||
for name in names:
|
||||
if name == "members":
|
||||
child = self.members(descriptor)
|
||||
try:
|
||||
entries = self.names(child, MAX_ARCHIVE_FILES)
|
||||
if len(entries) > MAX_ARCHIVE_FILES:
|
||||
raise ArtifactError("unexpected staging entry")
|
||||
for member in entries:
|
||||
from kf_oj.infra.artifacts.store import ArtifactStore
|
||||
|
||||
ArtifactStore._regular(
|
||||
os.stat(member, dir_fd=child, follow_symlinks=False)
|
||||
)
|
||||
os.unlink(member, dir_fd=child)
|
||||
finally:
|
||||
os.close(child)
|
||||
os.rmdir("members", dir_fd=descriptor)
|
||||
elif name in ("archive", "validated.json"):
|
||||
from kf_oj.infra.artifacts.store import ArtifactStore
|
||||
|
||||
ArtifactStore._regular(os.stat(name, dir_fd=descriptor, follow_symlinks=False))
|
||||
os.unlink(name, dir_fd=descriptor)
|
||||
else:
|
||||
raise ArtifactError("unexpected staging entry")
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
os.rmdir(str(operation_id), dir_fd=self.fd)
|
||||
os.fsync(self.fd)
|
||||
|
||||
def reset_validation(self, operation_id):
|
||||
directory = self.operation(operation_id)
|
||||
try:
|
||||
names = self.names(directory, 3)
|
||||
if "members" in names:
|
||||
members = self.members(directory)
|
||||
try:
|
||||
for name in self.names(members, MAX_ARCHIVE_FILES):
|
||||
from kf_oj.infra.artifacts.store import ArtifactStore
|
||||
|
||||
ArtifactStore._regular(os.stat(name, dir_fd=members, follow_symlinks=False))
|
||||
os.unlink(name, dir_fd=members)
|
||||
finally:
|
||||
os.close(members)
|
||||
os.rmdir("members", dir_fd=directory)
|
||||
if "validated.json" in names:
|
||||
os.unlink("validated.json", dir_fd=directory)
|
||||
os.fsync(directory)
|
||||
finally:
|
||||
os.close(directory)
|
||||
@@ -168,6 +168,90 @@ class ArtifactStore:
|
||||
with self.exclusive():
|
||||
return self._read(reference)
|
||||
|
||||
def put_file(self, source: int, expected: ArtifactReference) -> ArtifactReference:
|
||||
"""Adopt a validated private FD without buffering a hidden testcase in memory."""
|
||||
before = os.fstat(source)
|
||||
self._regular(before)
|
||||
if before.st_size != expected.bytes:
|
||||
raise ArtifactError("artifact digest or length mismatch")
|
||||
|
||||
def unchanged(descriptor, original):
|
||||
current = os.fstat(descriptor)
|
||||
fields = (
|
||||
"st_dev",
|
||||
"st_ino",
|
||||
"st_nlink",
|
||||
"st_mode",
|
||||
"st_uid",
|
||||
"st_size",
|
||||
"st_mtime_ns",
|
||||
"st_ctime_ns",
|
||||
)
|
||||
if any(getattr(current, k) != getattr(original, k) for k in fields):
|
||||
raise ArtifactError("artifact input changed")
|
||||
|
||||
def stream(descriptor, *, output=None):
|
||||
digest = hashlib.sha256()
|
||||
for offset in range(0, expected.bytes, 65536):
|
||||
data = os.pread(descriptor, min(65536, expected.bytes - offset), offset)
|
||||
if len(data) != min(65536, expected.bytes - offset):
|
||||
raise ArtifactError("artifact digest or length mismatch")
|
||||
digest.update(data)
|
||||
if output is not None:
|
||||
output.write(data)
|
||||
if digest.hexdigest() != expected.digest:
|
||||
raise ArtifactError("artifact digest or length mismatch")
|
||||
|
||||
with self.exclusive():
|
||||
entries = self._entries()
|
||||
if any(name == expected.digest for name, _ in entries):
|
||||
existing = os.open(
|
||||
expected.digest,
|
||||
os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC,
|
||||
dir_fd=self._directory,
|
||||
)
|
||||
try:
|
||||
self._regular(os.fstat(existing))
|
||||
if os.fstat(existing).st_size != expected.bytes:
|
||||
raise ArtifactError("artifact digest or length mismatch")
|
||||
stream(source)
|
||||
stream(existing)
|
||||
finally:
|
||||
os.close(existing)
|
||||
else:
|
||||
if (
|
||||
len(entries) + 1 > self.max_files
|
||||
or sum(s.st_size for _, s in entries) + expected.bytes > self.max_bytes
|
||||
):
|
||||
raise ArtifactError("storage capacity exhausted")
|
||||
temporary = ".tmp-" + uuid4().hex
|
||||
descriptor = os.open(
|
||||
temporary,
|
||||
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC,
|
||||
0o600,
|
||||
dir_fd=self._directory,
|
||||
)
|
||||
try:
|
||||
with os.fdopen(descriptor, "wb") as out:
|
||||
stream(source, output=out)
|
||||
out.flush()
|
||||
os.fsync(out.fileno())
|
||||
unchanged(source, before)
|
||||
os.rename(
|
||||
temporary,
|
||||
expected.digest,
|
||||
src_dir_fd=self._directory,
|
||||
dst_dir_fd=self._directory,
|
||||
)
|
||||
os.fsync(self._directory)
|
||||
finally:
|
||||
try:
|
||||
os.unlink(temporary, dir_fd=self._directory)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
unchanged(source, before)
|
||||
return expected
|
||||
|
||||
def remove(self, digest: str) -> None:
|
||||
if not DIGEST.fullmatch(digest):
|
||||
raise ArtifactError("invalid artifact identifier")
|
||||
|
||||
@@ -28,5 +28,11 @@ class AuthenticationFailed(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class ReservedMaintenancePort(Protocol):
|
||||
async def require_reserved_maintenance(self, session, actor_id: UUID, epoch: int) -> None:
|
||||
"""Recheck/lock an immutable accepted receipt, with no login/session secret access."""
|
||||
...
|
||||
|
||||
|
||||
class LoginThrottled(ValueError):
|
||||
pass
|
||||
|
||||
@@ -7,7 +7,8 @@ from dataclasses import dataclass, field
|
||||
from datetime import datetime, timedelta
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy import func, select, text, update
|
||||
from sqlalchemy.exc import DBAPIError
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
from kf_oj.modules.identity import repository
|
||||
@@ -170,3 +171,20 @@ class IdentityService:
|
||||
await self.authenticate(token, csrf_token=csrf_token)
|
||||
async with self.sessions.begin() as session:
|
||||
await repository.revoke(session, token_digest(token))
|
||||
|
||||
|
||||
class ReservedMaintenance:
|
||||
"""Import worker uses only the fixed, narrow SQL authority port."""
|
||||
|
||||
async def require_reserved_maintenance(self, session, actor_id, epoch):
|
||||
if not session.in_transaction() or type(epoch) is not int or epoch < 1:
|
||||
raise MaintenanceDenied("invalid maintenance receipt")
|
||||
try:
|
||||
await session.execute(
|
||||
text("SELECT public.kf_lock_reserved_maintenance(:actor,:epoch)"),
|
||||
{"actor": actor_id, "epoch": epoch},
|
||||
)
|
||||
except DBAPIError as error:
|
||||
if getattr(error.orig, "sqlstate", None) == "42501":
|
||||
raise MaintenanceDenied("problem maintenance revoked") from None
|
||||
raise
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
"""Append-only bounded maintenance events, independent of evaluation state."""
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import BigInteger, CheckConstraint, ForeignKey, Index, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from kf_oj.infra.db.base import Base
|
||||
from kf_oj.infra.db.columns import CreatedAt
|
||||
|
||||
|
||||
class MaintenanceEvent(CreatedAt, Base):
|
||||
__tablename__ = "problem_maintenance_events"
|
||||
__table_args__ = (
|
||||
CheckConstraint(
|
||||
"action IN ('upload_created','upload_received','import_ready',"
|
||||
"'import_failed','import_discarded','import_expired','published',"
|
||||
"'disabled','restored')",
|
||||
name="action",
|
||||
),
|
||||
CheckConstraint("authorization_version > 0", name="authority"),
|
||||
Index("ix_problem_maintenance_events_page", "created_at", "id"),
|
||||
)
|
||||
id: Mapped[UUID] = mapped_column(primary_key=True)
|
||||
actor_id: Mapped[UUID] = mapped_column(ForeignKey("accounts.id"))
|
||||
authorization_version: Mapped[int] = mapped_column(BigInteger)
|
||||
operation_id: Mapped[UUID] = mapped_column()
|
||||
action: Mapped[str] = mapped_column(String(24))
|
||||
target_id: Mapped[UUID | None] = mapped_column()
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Accepted actor receipts and finite maintenance events; no private data/log text."""
|
||||
|
||||
from typing import Literal, Protocol
|
||||
from uuid import UUID
|
||||
|
||||
Action = Literal[
|
||||
"upload_created",
|
||||
"upload_received",
|
||||
"import_ready",
|
||||
"import_failed",
|
||||
"import_discarded",
|
||||
"import_expired",
|
||||
"published",
|
||||
"disabled",
|
||||
"restored",
|
||||
]
|
||||
|
||||
|
||||
class MaintenanceAuditPort(Protocol):
|
||||
async def record(
|
||||
self,
|
||||
session,
|
||||
actor_id: UUID,
|
||||
authorization_version: int,
|
||||
operation_id: UUID,
|
||||
action: Action,
|
||||
target_id: UUID | None = None,
|
||||
) -> None: ...
|
||||
@@ -0,0 +1,27 @@
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from kf_oj.modules.operations.models import MaintenanceEvent
|
||||
from kf_oj.modules.operations.public import Action
|
||||
|
||||
|
||||
class MaintenanceAudit:
|
||||
async def record(
|
||||
self, session, actor_id, authorization_version, operation_id, action: Action, target_id=None
|
||||
):
|
||||
if not session.in_transaction():
|
||||
raise ValueError("audit requires a write transaction")
|
||||
# Supplying the trusted DB timestamp avoids ORM INSERT ... RETURNING:
|
||||
# the ordinary importer has append-only authority and cannot SELECT this table.
|
||||
session.add(
|
||||
MaintenanceEvent(
|
||||
id=uuid4(),
|
||||
created_at=await session.scalar(select(func.clock_timestamp())),
|
||||
actor_id=actor_id,
|
||||
authorization_version=authorization_version,
|
||||
operation_id=operation_id,
|
||||
action=action,
|
||||
target_id=target_id,
|
||||
)
|
||||
)
|
||||
@@ -31,3 +31,32 @@ v2 使用 `schema_version: 2` 与 `language_configurations`,每个配置包含
|
||||
导入同一 slug 的 v2 版本保持原 v1 版本与提交不变。已有提交按其保存的协议版本解析,
|
||||
不能重新序列化旧 canonical 数据来补入新字段。Java 使用 Main.java 与默认包 Main;
|
||||
Python 使用 main.py。每个新版本的语言集合独立,浏览器不得为旧版本补开新语言。
|
||||
|
||||
|
||||
Web题包导入是独立的待发布流程:`/api/v1/admin/problem-imports` 使用真实维护会话,
|
||||
写入还要求明确Origin与绑定CSRF。先以稳定操作UUID、ZIP摘要和精确字节数建立容量预约,
|
||||
再向该操作的 `/archive` 流式PUT;同键重试仍验证实际内容,不会确认异内容。
|
||||
列表、详情和丢弃只开放给该操作的管理员本人,响应仅含公开题面、样例、语言预算与有限错误。
|
||||
`ready` 尚未发布,不会加入学生题库,也不改变既有题目版本。
|
||||
|
||||
Web格式固定单层ZIP的stored/deflate,清单名为`problem.json`;公开文件与隐藏用例文件
|
||||
必须显式分别声明且不得混用。文件名只接受有界ASCII单层普通名;未知成员、链接/设备、
|
||||
加密、嵌套归档、额外前缀/尾随内容、损坏CRC与虚假大小被拒。旧受控目录CLI格式保持原义。
|
||||
压缩与实际解压各64 MiB、最多534文件,上传最多4096帧/每帧1 MiB/30秒;
|
||||
解析期限60秒。staging总预约256 MiB,每次128 MiB,单全局上传/解析槽位,
|
||||
本人最多10个/全局32个活跃候选,操作24小时到期,解析死亡最多领取两次。
|
||||
真实限额由管理capabilities返回;缺少配置或新鲜Worker资源证明时明确不可用。
|
||||
|
||||
`uv run --locked kf-problem-import-worker` 是普通用户、独立数据库登录的组合入口。
|
||||
配置`KF_OJ_IMPORT_STAGING_ROOT`、`KF_OJ_ARTIFACT_ROOT`、可信准入和
|
||||
`KF_OJ_IMPORT_WORKER_CGROUP`;staging与工件目录必须归服务用户所有且权限0700。
|
||||
服务不会sudo、执行上传内容/学生源码、联网安装依赖或读写评测状态。启动及每次解析
|
||||
读回128 MiB/swap0/PIDs16/CPU不超过一核及文件/FD/core上限,错误身份/无界环境拒绝服务。
|
||||
解析子进程继承同一个内核文件锁,父进程死亡不能让GC在存活子进程仍持有文件时释放容量。
|
||||
租约与fence防止旧结果写入;完整工件持久化、候选和引用原子落库后才返回ready。
|
||||
丢弃/失败/到期先清理私有暂存字节再释放预约,保留不可变操作记录与追加式有限审计。
|
||||
|
||||
资源约束服务模板见`deploy/problem-import-worker.service.example`。API和Worker需可访问
|
||||
同一普通用户的私有目录;数据库采用各自最小角色,导入Worker只通过固定身份公开端口
|
||||
验证已受理的维护授权。PrivateNetwork模板要求Unix PostgreSQL socket;实际部署前由运维
|
||||
设置外部私有配置、专用登录与目录,不包含默认口令,也不代表本轮已经生产部署。
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Pure data validation shared by the bounded child and fenced candidate transaction."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from uuid import uuid4, uuid5
|
||||
|
||||
from kf_oj.contracts import judging_v2 as v2
|
||||
from kf_oj.contracts.admin_problems import CandidatePreview, LegacyImportBudget
|
||||
from kf_oj.contracts.judging import MAX_SOURCE_BYTES, ArtifactReference, TestCase
|
||||
from kf_oj.contracts.problems import PublicLanguageConfiguration
|
||||
from kf_oj.modules.problems.import_format import ImportEnvelope, ProblemImport
|
||||
from kf_oj.modules.problems.public import JudgingVersion
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ValidatedCandidate:
|
||||
manifest: dict
|
||||
preview: CandidatePreview
|
||||
references: dict[str, ArtifactReference]
|
||||
|
||||
@property
|
||||
def digest(self):
|
||||
return hashlib.sha256(
|
||||
json.dumps(
|
||||
self.manifest, sort_keys=True, separators=(",", ":"), ensure_ascii=True
|
||||
).encode()
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def validate_candidate(manifest_data, preview_data, references, policy) -> ValidatedCandidate:
|
||||
manifest = ImportEnvelope.model_validate(manifest_data).root
|
||||
preview = CandidatePreview.model_validate(preview_data)
|
||||
refs = {k: ArtifactReference.model_validate(v) for k, v in references.items()}
|
||||
legacy = isinstance(manifest, ProblemImport)
|
||||
if (
|
||||
preview.slug != manifest.slug
|
||||
or preview.title != manifest.title
|
||||
or preview.case_count != len(manifest.cases)
|
||||
or preview.judging_schema_version != manifest.schema_version
|
||||
or tuple(v.value for v in preview.languages) != tuple(v.value for v in manifest.languages)
|
||||
):
|
||||
raise ValueError("invalid candidate projection")
|
||||
expected_budget = (
|
||||
LegacyImportBudget(
|
||||
compile_limits=manifest.compile_limits,
|
||||
run_limits=manifest.run_limits,
|
||||
job_wall_time_us=manifest.job_wall_time_us,
|
||||
)
|
||||
if legacy
|
||||
else None
|
||||
)
|
||||
configs = (
|
||||
()
|
||||
if legacy
|
||||
else tuple(
|
||||
PublicLanguageConfiguration.from_configuration(c)
|
||||
for c in manifest.language_configurations
|
||||
)
|
||||
)
|
||||
if preview.legacy_budget != expected_budget or preview.language_configurations != configs:
|
||||
raise ValueError("invalid candidate budget")
|
||||
names = {name for c in manifest.cases for name in (c.input_file, c.expected_file)}
|
||||
if set(refs) != names:
|
||||
raise ValueError("invalid hidden references")
|
||||
cases = tuple(
|
||||
TestCase(
|
||||
case_id=uuid5(preview.candidate_id, c.key),
|
||||
input=refs[c.input_file],
|
||||
expected=refs[c.expected_file],
|
||||
)
|
||||
for c in manifest.cases
|
||||
)
|
||||
projection = JudgingVersion(
|
||||
preview.candidate_id,
|
||||
uuid4(),
|
||||
manifest.languages,
|
||||
manifest.toolchain if legacy else None,
|
||||
manifest.compile_limits if legacy else None,
|
||||
manifest.run_limits if legacy else None,
|
||||
manifest.job_wall_time_us if legacy else None,
|
||||
cases,
|
||||
manifest.schema_version,
|
||||
() if legacy else manifest.language_configurations,
|
||||
)
|
||||
for language in manifest.languages:
|
||||
v2.admit_manifest(
|
||||
policy,
|
||||
projection.manifest(
|
||||
uuid4(),
|
||||
uuid4(),
|
||||
language,
|
||||
ArtifactReference(digest="0" * 64, bytes=MAX_SOURCE_BYTES),
|
||||
),
|
||||
1,
|
||||
)
|
||||
return ValidatedCandidate(manifest.model_dump(mode="json"), preview, refs)
|
||||
@@ -0,0 +1,121 @@
|
||||
"""Private durable imports and immutable candidates; no grading-state write authority."""
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import (
|
||||
BigInteger,
|
||||
CheckConstraint,
|
||||
Computed,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
ForeignKeyConstraint,
|
||||
Index,
|
||||
Integer,
|
||||
SmallInteger,
|
||||
String,
|
||||
UniqueConstraint,
|
||||
text,
|
||||
)
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from kf_oj.infra.db.base import Base
|
||||
from kf_oj.infra.db.columns import CreatedAt
|
||||
from kf_oj.modules.problems.models import ProblemVersion
|
||||
|
||||
|
||||
class ImportPool(Base):
|
||||
__tablename__ = "problem_import_pool"
|
||||
__table_args__ = (
|
||||
CheckConstraint("id = 1", name="singleton"),
|
||||
CheckConstraint("reserved_bytes BETWEEN 0 AND 268435456", name="capacity"),
|
||||
)
|
||||
id: Mapped[int] = mapped_column(SmallInteger, primary_key=True)
|
||||
reserved_bytes: Mapped[int] = mapped_column(BigInteger, server_default="0")
|
||||
worker_ready_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
worker_policy_digest: Mapped[str | None] = mapped_column(String(64))
|
||||
|
||||
|
||||
class ProblemImportOperation(CreatedAt, Base):
|
||||
__tablename__ = "problem_import_operations"
|
||||
__table_args__ = (
|
||||
CheckConstraint(
|
||||
"state IN ('uploading','uploaded','validating','ready','failed',"
|
||||
"'published','discarded','expired')",
|
||||
name="state",
|
||||
),
|
||||
CheckConstraint("archive_digest ~ '^[0-9a-f]{64}$'", name="digest"),
|
||||
CheckConstraint("archive_bytes BETWEEN 1 AND 67108864", name="bytes"),
|
||||
CheckConstraint("authorization_version > 0", name="authority"),
|
||||
CheckConstraint("attempts BETWEEN 0 AND 2 AND fence >= 0", name="attempts"),
|
||||
CheckConstraint("reserved_bytes IN (0,134217728)", name="reservation"),
|
||||
CheckConstraint("expires_at > created_at AND updated_at >= created_at", name="times"),
|
||||
Index("ix_problem_import_owner", "owner_id", "id"),
|
||||
Index("ix_problem_import_queue", "state", "created_at"),
|
||||
Index("ix_problem_import_expiry", "expires_at"),
|
||||
Index("ix_problem_import_reservations", "id", postgresql_where=text("reserved_bytes>0")),
|
||||
)
|
||||
id: Mapped[UUID] = mapped_column(primary_key=True)
|
||||
owner_id: Mapped[UUID] = mapped_column(ForeignKey("accounts.id"))
|
||||
authorization_version: Mapped[int] = mapped_column(BigInteger)
|
||||
archive_digest: Mapped[str] = mapped_column(String(64))
|
||||
archive_bytes: Mapped[int] = mapped_column(BigInteger)
|
||||
state: Mapped[str] = mapped_column(String(16), server_default="uploading")
|
||||
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
attempts: Mapped[int] = mapped_column(Integer, server_default="0")
|
||||
fence: Mapped[int] = mapped_column(BigInteger, server_default="0")
|
||||
reserved_bytes: Mapped[int] = mapped_column(BigInteger, server_default="134217728")
|
||||
failure_code: Mapped[str | None] = mapped_column(String(32))
|
||||
candidate_id: Mapped[UUID | None] = mapped_column(
|
||||
ForeignKey("problem_candidates.id", use_alter=True, name="fk_problem_import_candidate")
|
||||
)
|
||||
published_version_id: Mapped[UUID | None] = mapped_column(ForeignKey(ProblemVersion.id))
|
||||
|
||||
|
||||
class ProblemCandidate(CreatedAt, Base):
|
||||
__tablename__ = "problem_candidates"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("operation_id"),
|
||||
CheckConstraint("manifest_digest ~ '^[0-9a-f]{64}$'", name="digest"),
|
||||
CheckConstraint("case_count BETWEEN 1 AND 256", name="cases"),
|
||||
CheckConstraint("octet_length(preview::text) <= 4194304", name="preview"),
|
||||
CheckConstraint("octet_length(manifest::text) <= 262144", name="manifest"),
|
||||
)
|
||||
id: Mapped[UUID] = mapped_column(primary_key=True)
|
||||
operation_id: Mapped[UUID] = mapped_column(ForeignKey("problem_import_operations.id"))
|
||||
manifest_digest: Mapped[str] = mapped_column(String(64))
|
||||
manifest: Mapped[dict] = mapped_column(JSONB)
|
||||
preview: Mapped[dict] = mapped_column(JSONB)
|
||||
case_count: Mapped[int] = mapped_column(Integer)
|
||||
|
||||
|
||||
class CandidateCase(Base):
|
||||
__tablename__ = "problem_candidate_cases"
|
||||
__table_args__ = (
|
||||
CheckConstraint("position BETWEEN 0 AND 255", name="position"),
|
||||
CheckConstraint(
|
||||
"input_bytes BETWEEN 0 AND 67108864 AND expected_bytes BETWEEN 0 AND 67108864",
|
||||
name="bytes",
|
||||
),
|
||||
ForeignKeyConstraint(
|
||||
["input_digest", "artifact_state"], ["artifacts.digest", "artifacts.state"]
|
||||
),
|
||||
ForeignKeyConstraint(
|
||||
["expected_digest", "artifact_state"],
|
||||
["artifacts.digest", "artifacts.state"],
|
||||
name="fk_candidate_cases_expected_artifacts",
|
||||
),
|
||||
)
|
||||
candidate_id: Mapped[UUID] = mapped_column(
|
||||
ForeignKey("problem_candidates.id"), primary_key=True
|
||||
)
|
||||
position: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
key: Mapped[str] = mapped_column(String(64))
|
||||
input_digest: Mapped[str] = mapped_column(String(64))
|
||||
input_bytes: Mapped[int] = mapped_column(BigInteger)
|
||||
expected_digest: Mapped[str] = mapped_column(String(64))
|
||||
expected_bytes: Mapped[int] = mapped_column(BigInteger)
|
||||
artifact_state: Mapped[str] = mapped_column(String(8), Computed("'ready'", persisted=True))
|
||||
@@ -0,0 +1,430 @@
|
||||
"""Fenced ordinary-user import consumption. No student execution or evaluation DB access."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import resource
|
||||
import sys
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import delete, func, or_, select, text
|
||||
from sqlalchemy.exc import DBAPIError
|
||||
|
||||
from kf_oj.contracts.admin_problems import MAX_IMPORT_ATTEMPTS, VALIDATION_DEADLINE_SECONDS
|
||||
from kf_oj.infra.artifacts.store import ArtifactError
|
||||
from kf_oj.modules.identity.public import MaintenanceDenied
|
||||
from kf_oj.modules.problems.candidate_validation import validate_candidate
|
||||
from kf_oj.modules.problems.import_models import (
|
||||
CandidateCase,
|
||||
ImportPool,
|
||||
ProblemCandidate,
|
||||
)
|
||||
from kf_oj.modules.problems.import_models import (
|
||||
ProblemImportOperation as Operation,
|
||||
)
|
||||
from kf_oj.modules.problems.imports import ACTIVE, ImportUnavailable, ProblemImports, policy_digest
|
||||
|
||||
|
||||
def verify_worker_limits(configured_group: Path):
|
||||
if os.getuid() == 0:
|
||||
raise ImportUnavailable("import worker must be ordinary")
|
||||
actual = next(
|
||||
line.split(":", 2)[2]
|
||||
for line in Path("/proc/self/cgroup").read_text().splitlines()
|
||||
if line.startswith("0::")
|
||||
)
|
||||
group = Path("/sys/fs/cgroup") / actual.lstrip("/")
|
||||
if group.resolve() != configured_group.absolute().resolve():
|
||||
raise ImportUnavailable("import worker cgroup mismatch")
|
||||
caps = {
|
||||
n: (group / n).read_text().strip()
|
||||
for n in ("memory.max", "memory.swap.max", "pids.max", "cpu.max")
|
||||
}
|
||||
if (
|
||||
caps["memory.max"] == "max"
|
||||
or not 0 < int(caps["memory.max"]) <= 134217728
|
||||
or caps["memory.swap.max"] != "0"
|
||||
or caps["pids.max"] == "max"
|
||||
or not 0 < int(caps["pids.max"]) <= 16
|
||||
):
|
||||
raise ImportUnavailable("import worker resource limits unavailable")
|
||||
quota, period = caps["cpu.max"].split()
|
||||
if quota == "max" or int(quota) > int(period):
|
||||
raise ImportUnavailable("import worker CPU limit unavailable")
|
||||
for name, maximum in (
|
||||
(resource.RLIMIT_FSIZE, 67108864),
|
||||
(resource.RLIMIT_NOFILE, 64),
|
||||
(resource.RLIMIT_CORE, 0),
|
||||
):
|
||||
limits = resource.getrlimit(name)
|
||||
if any(v == resource.RLIM_INFINITY or v > maximum for v in limits):
|
||||
raise ImportUnavailable("import worker process limits unavailable")
|
||||
return caps
|
||||
|
||||
|
||||
class ImportWorker:
|
||||
def __init__(self, sessions, staging, artifacts, authority, audit, policy, cgroup):
|
||||
(
|
||||
self.sessions,
|
||||
self.staging,
|
||||
self.artifacts,
|
||||
self.authority,
|
||||
self.audit,
|
||||
self.policy,
|
||||
self.cgroup,
|
||||
) = sessions, staging, artifacts, authority, audit, policy, cgroup
|
||||
|
||||
async def heartbeat(self):
|
||||
verify_worker_limits(self.cgroup)
|
||||
async with self.sessions.begin() as session:
|
||||
safe = await session.scalar(
|
||||
text("""
|
||||
SELECT rolcanlogin AND NOT(rolsuper OR rolcreatedb OR rolcreaterole
|
||||
OR rolreplication OR rolbypassrls)
|
||||
AND pg_has_role(current_user,'kf_oj_problem_import','USAGE')
|
||||
AND NOT pg_has_role(current_user,'kf_oj_platform','MEMBER')
|
||||
AND NOT pg_has_role(current_user,'kf_oj_coordinator','MEMBER')
|
||||
AND NOT pg_has_role(current_user,'kf_oj_identity_operator','MEMBER')
|
||||
AND NOT pg_has_role(current_user,'kf_oj_identity_authority_owner','MEMBER')
|
||||
FROM pg_roles WHERE rolname=current_user
|
||||
""")
|
||||
)
|
||||
if not safe:
|
||||
raise ImportUnavailable("dedicated unprivileged import login required")
|
||||
pool = await session.get(ImportPool, 1, with_for_update=True)
|
||||
if pool is None:
|
||||
raise ImportUnavailable("import pool unavailable")
|
||||
pool.worker_ready_until = await session.scalar(
|
||||
select(func.clock_timestamp())
|
||||
) + timedelta(seconds=90)
|
||||
pool.worker_policy_digest = policy_digest(self.policy)
|
||||
|
||||
async def _release(self, operation_id):
|
||||
await asyncio.to_thread(self.staging.remove, operation_id)
|
||||
async with self.sessions.begin() as session:
|
||||
pool = await ProblemImports.pool(session)
|
||||
op = await session.get(Operation, operation_id, with_for_update=True)
|
||||
pool.reserved_bytes -= op.reserved_bytes
|
||||
op.reserved_bytes = 0
|
||||
op.lease_until = None
|
||||
|
||||
async def collect(self):
|
||||
# Caller owns the root flock, so a live upload/parser cannot be unlinked or
|
||||
# have its reservation reused, even if its DB lease expired while suspended.
|
||||
async with self.sessions() as session:
|
||||
now = await session.scalar(select(func.clock_timestamp()))
|
||||
names = list(
|
||||
await session.scalars(
|
||||
select(Operation.id)
|
||||
.where(
|
||||
or_(
|
||||
Operation.reserved_bytes > 0,
|
||||
Operation.state.in_(ACTIVE) & (Operation.expires_at <= now),
|
||||
)
|
||||
)
|
||||
.order_by(Operation.updated_at, Operation.id)
|
||||
.limit(32)
|
||||
)
|
||||
)
|
||||
garbage = list(
|
||||
await session.scalars(
|
||||
select(ProblemCandidate.operation_id)
|
||||
.join(Operation, Operation.id == ProblemCandidate.operation_id)
|
||||
.where(Operation.state.in_(("discarded", "expired", "failed")))
|
||||
.limit(32)
|
||||
)
|
||||
)
|
||||
names = list(dict.fromkeys(names + garbage))[:32]
|
||||
for identifier in names:
|
||||
async with self.sessions.begin() as session:
|
||||
await ProblemImports.pool(session)
|
||||
op = await session.get(Operation, identifier, with_for_update=True)
|
||||
now = await session.scalar(select(func.clock_timestamp()))
|
||||
if op.state in ACTIVE and op.expires_at <= now:
|
||||
op.state = "expired"
|
||||
op.failure_code = "expired"
|
||||
op.updated_at = now
|
||||
op.candidate_id = None
|
||||
op.fence += 1
|
||||
await self.audit.record(
|
||||
session, op.owner_id, op.authorization_version, op.id, "import_expired"
|
||||
)
|
||||
elif op.state == "uploading" and op.lease_until and op.lease_until <= now:
|
||||
op.state = "failed"
|
||||
op.failure_code = "upload_interrupted"
|
||||
op.updated_at = now
|
||||
op.fence += 1
|
||||
await self.audit.record(
|
||||
session, op.owner_id, op.authorization_version, op.id, "import_failed"
|
||||
)
|
||||
elif op.state == "validating" and op.lease_until and op.lease_until <= now:
|
||||
if op.attempts < MAX_IMPORT_ATTEMPTS:
|
||||
op.state = "uploaded"
|
||||
op.failure_code = None
|
||||
op.lease_until = None
|
||||
op.fence += 1
|
||||
op.updated_at = now
|
||||
await asyncio.to_thread(self.staging.reset_validation, op.id)
|
||||
else:
|
||||
op.state = "failed"
|
||||
op.failure_code = "validation_unavailable"
|
||||
op.updated_at = now
|
||||
op.fence += 1
|
||||
await self.audit.record(
|
||||
session, op.owner_id, op.authorization_version, op.id, "import_failed"
|
||||
)
|
||||
if op.state in ("discarded", "expired", "failed"):
|
||||
op.candidate_id = None
|
||||
await session.flush()
|
||||
candidate = await session.scalar(
|
||||
select(ProblemCandidate).where(ProblemCandidate.operation_id == op.id)
|
||||
)
|
||||
if candidate:
|
||||
await session.execute(
|
||||
delete(CandidateCase).where(CandidateCase.candidate_id == candidate.id)
|
||||
)
|
||||
await session.delete(candidate)
|
||||
release = (
|
||||
op.state in ("ready", "published", "discarded", "expired", "failed")
|
||||
and op.reserved_bytes > 0
|
||||
)
|
||||
if release:
|
||||
await self._release(identifier)
|
||||
# Uncommitted staging objects after a failed transaction are removed only under
|
||||
# the same kernel lock and only from this service's private, bounded namespace.
|
||||
for identifier in self.staging.operation_ids():
|
||||
async with self.sessions() as session:
|
||||
exists = await session.get(Operation, identifier)
|
||||
if exists is None:
|
||||
await asyncio.to_thread(self.staging.remove, identifier)
|
||||
|
||||
async def once(self):
|
||||
await self.heartbeat()
|
||||
try:
|
||||
with self.staging.exclusive() as lock:
|
||||
await self.collect()
|
||||
async with self.sessions.begin() as session:
|
||||
await ProblemImports.pool(session)
|
||||
now = await session.scalar(select(func.clock_timestamp()))
|
||||
busy = await session.scalar(
|
||||
select(func.count())
|
||||
.select_from(Operation)
|
||||
.where(
|
||||
Operation.state.in_(("uploading", "validating")),
|
||||
Operation.lease_until > now,
|
||||
)
|
||||
)
|
||||
if busy:
|
||||
return False
|
||||
op = await session.scalar(
|
||||
select(Operation)
|
||||
.where(
|
||||
Operation.state == "uploaded",
|
||||
Operation.attempts < MAX_IMPORT_ATTEMPTS,
|
||||
Operation.expires_at > now,
|
||||
)
|
||||
.order_by(Operation.created_at, Operation.id)
|
||||
.limit(1)
|
||||
.with_for_update()
|
||||
)
|
||||
if op is None:
|
||||
return False
|
||||
op.attempts += 1
|
||||
op.fence += 1
|
||||
op.state = "validating"
|
||||
op.updated_at = now
|
||||
op.lease_until = now + timedelta(seconds=VALIDATION_DEADLINE_SECONDS + 15)
|
||||
identifier, fence, size, digest, actor, epoch = (
|
||||
op.id,
|
||||
op.fence,
|
||||
op.archive_bytes,
|
||||
op.archive_digest,
|
||||
op.owner_id,
|
||||
op.authorization_version,
|
||||
)
|
||||
try:
|
||||
async with self.sessions.begin() as session:
|
||||
await self.authority.require_reserved_maintenance(session, actor, epoch)
|
||||
candidate_id = uuid4()
|
||||
result = await self.parse(identifier, size, digest, candidate_id, lock)
|
||||
if "failure_code" in result:
|
||||
await self.fail(identifier, fence, result["failure_code"])
|
||||
return True
|
||||
candidate = validate_candidate(
|
||||
result["manifest"], result["preview"], result["references"], self.policy
|
||||
)
|
||||
if (
|
||||
candidate.digest != result["manifest_digest"]
|
||||
or candidate.preview.candidate_id != candidate_id
|
||||
):
|
||||
raise ValueError("invalid candidate receipt")
|
||||
directory = self.staging.operation(identifier)
|
||||
members = self.staging.members(directory)
|
||||
try:
|
||||
async with self.sessions.begin() as session:
|
||||
await self.authority.require_reserved_maintenance(session, actor, epoch)
|
||||
await ProblemImports.pool(session)
|
||||
op = await session.get(Operation, identifier, with_for_update=True)
|
||||
now = await session.scalar(select(func.clock_timestamp()))
|
||||
if (
|
||||
op.state != "validating"
|
||||
or op.fence != fence
|
||||
or op.lease_until <= now
|
||||
or op.expires_at <= now
|
||||
):
|
||||
raise ValueError("stale candidate receipt")
|
||||
for name, reference in sorted(
|
||||
candidate.references.items(), key=lambda item: item[1].digest
|
||||
):
|
||||
fd = os.open(
|
||||
name,
|
||||
os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC,
|
||||
dir_fd=members,
|
||||
)
|
||||
try:
|
||||
await self.artifacts.put_file(session, fd, reference)
|
||||
finally:
|
||||
os.close(fd)
|
||||
session.add(
|
||||
ProblemCandidate(
|
||||
id=candidate_id,
|
||||
operation_id=identifier,
|
||||
manifest_digest=candidate.digest,
|
||||
manifest=candidate.manifest,
|
||||
preview=candidate.preview.model_dump(mode="json"),
|
||||
case_count=candidate.preview.case_count,
|
||||
)
|
||||
)
|
||||
await session.flush()
|
||||
for position, case in enumerate(candidate.manifest["cases"]):
|
||||
input_ref, answer_ref = (
|
||||
candidate.references[case["input_file"]],
|
||||
candidate.references[case["expected_file"]],
|
||||
)
|
||||
session.add(
|
||||
CandidateCase(
|
||||
candidate_id=candidate_id,
|
||||
position=position,
|
||||
key=case["key"],
|
||||
input_digest=input_ref.digest,
|
||||
input_bytes=input_ref.bytes,
|
||||
expected_digest=answer_ref.digest,
|
||||
expected_bytes=answer_ref.bytes,
|
||||
)
|
||||
)
|
||||
await session.flush()
|
||||
# Adoption must not turn an expired lease into a ready candidate.
|
||||
completed_at = await session.scalar(select(func.clock_timestamp()))
|
||||
if op.lease_until <= completed_at or op.expires_at <= completed_at:
|
||||
raise ValueError("candidate completion lease expired")
|
||||
op.state = "ready"
|
||||
op.candidate_id = candidate_id
|
||||
op.updated_at = completed_at
|
||||
op.lease_until = None
|
||||
await self.audit.record(
|
||||
session, actor, epoch, identifier, "import_ready", candidate_id
|
||||
)
|
||||
finally:
|
||||
os.close(members)
|
||||
os.close(directory)
|
||||
await self._release(identifier)
|
||||
except MaintenanceDenied:
|
||||
await self.fail(identifier, fence, "authority_revoked")
|
||||
except (ArtifactError, OSError):
|
||||
await self.fail(identifier, fence, "storage_unavailable")
|
||||
except (ValueError, DBAPIError):
|
||||
await self.fail(identifier, fence, "validation_unavailable")
|
||||
return True
|
||||
except ArtifactError:
|
||||
return False # Another live writer holds the one global slot.
|
||||
|
||||
async def fail(self, identifier, fence, code):
|
||||
from kf_oj.contracts.admin_problems import ImportFailure
|
||||
|
||||
code = ImportFailure(code).value
|
||||
async with self.sessions.begin() as session:
|
||||
await ProblemImports.pool(session)
|
||||
op = await session.get(Operation, identifier, with_for_update=True)
|
||||
if op.state != "validating" or op.fence != fence:
|
||||
return
|
||||
op.state = "failed"
|
||||
op.failure_code = code
|
||||
op.updated_at = await session.scalar(select(func.clock_timestamp()))
|
||||
op.lease_until = None
|
||||
await self.audit.record(
|
||||
session, op.owner_id, op.authorization_version, op.id, "import_failed"
|
||||
)
|
||||
await self._release(identifier)
|
||||
|
||||
async def parse(self, identifier, size, digest, candidate, lock):
|
||||
verify_worker_limits(self.cgroup)
|
||||
directory = self.staging.operation(identifier)
|
||||
archive = members = None
|
||||
try:
|
||||
archive = os.open(
|
||||
"archive",
|
||||
os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC,
|
||||
dir_fd=directory,
|
||||
)
|
||||
members = self.staging.members(directory, create=True)
|
||||
# Preserve only the trusted runtime import roots, without inheriting DB/
|
||||
# service credentials. Curated runtimes may keep dependencies outside site-packages.
|
||||
env = {
|
||||
"PATH": str(Path(sys.executable).parent),
|
||||
"PYTHONPATH": os.pathsep.join(
|
||||
dict.fromkeys(
|
||||
[str(Path(__file__).parents[3])]
|
||||
+ [p for p in os.environ.get("PYTHONPATH", "").split(os.pathsep) if p]
|
||||
)
|
||||
),
|
||||
"PYTHONUNBUFFERED": "1",
|
||||
"PYTHONDONTWRITEBYTECODE": "1",
|
||||
"PYTHONNOUSERSITE": "1",
|
||||
"LC_ALL": "C.UTF-8",
|
||||
}
|
||||
# Minimal environment may require the trusted dynamic-loader path in a curated VM.
|
||||
if "LD_LIBRARY_PATH" in os.environ:
|
||||
env["LD_LIBRARY_PATH"] = os.environ["LD_LIBRARY_PATH"]
|
||||
child = await asyncio.create_subprocess_exec(
|
||||
sys.executable,
|
||||
"-m",
|
||||
"kf_oj.workers._archive_child",
|
||||
str(archive),
|
||||
str(members),
|
||||
str(lock),
|
||||
str(size),
|
||||
digest,
|
||||
str(candidate),
|
||||
self.policy.model_dump_json(),
|
||||
env=env,
|
||||
pass_fds=(archive, members, lock),
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.DEVNULL,
|
||||
)
|
||||
try:
|
||||
async with asyncio.timeout(VALIDATION_DEADLINE_SECONDS + 5):
|
||||
output, _ = await child.communicate()
|
||||
except BaseException:
|
||||
child.kill()
|
||||
await child.wait()
|
||||
raise
|
||||
if child.returncode != 0 or not 0 < len(output) <= 4 * 1024**2:
|
||||
return {
|
||||
"failure_code": "validation_deadline"
|
||||
if child.returncode == -14
|
||||
else "validation_unavailable"
|
||||
}
|
||||
result = json.loads(output)
|
||||
if not isinstance(result, dict):
|
||||
raise ValueError("invalid child result")
|
||||
return result
|
||||
except TimeoutError:
|
||||
return {"failure_code": "validation_deadline"}
|
||||
finally:
|
||||
if members is not None:
|
||||
os.close(members)
|
||||
if archive is not None:
|
||||
os.close(archive)
|
||||
os.close(directory)
|
||||
@@ -0,0 +1,438 @@
|
||||
"""Authenticated durable import lifecycle; filesystem work remains behind private adapters."""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import os
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import timedelta
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import func, select, text
|
||||
|
||||
from kf_oj.contracts.admin_problems import (
|
||||
IMPORT_LIFETIME_SECONDS,
|
||||
MAX_OWNER_CANDIDATES,
|
||||
MAX_TOTAL_CANDIDATES,
|
||||
MAX_UPLOAD_CHUNK_BYTES,
|
||||
MAX_UPLOAD_CHUNKS,
|
||||
STAGING_MAX_BYTES,
|
||||
STAGING_RESERVATION_BYTES,
|
||||
UPLOAD_DEADLINE_SECONDS,
|
||||
CandidatePreview,
|
||||
CreateProblemImport,
|
||||
ProblemImportCapabilities,
|
||||
ProblemImportDetail,
|
||||
ProblemImportPage,
|
||||
ProblemImportSummary,
|
||||
)
|
||||
from kf_oj.infra.artifacts.staging import ImportStaging
|
||||
from kf_oj.infra.artifacts.store import ArtifactError
|
||||
from kf_oj.modules.identity.public import MaintenanceDenied, ProblemMaintenancePort
|
||||
from kf_oj.modules.operations.public import MaintenanceAuditPort
|
||||
from kf_oj.modules.problems.import_models import ImportPool, ProblemCandidate
|
||||
from kf_oj.modules.problems.import_models import ProblemImportOperation as Operation
|
||||
|
||||
ACTIVE = ("uploading", "uploaded", "validating", "ready")
|
||||
|
||||
|
||||
class ImportConflict(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class ImportBadDigest(ImportConflict):
|
||||
pass
|
||||
|
||||
|
||||
class ImportMissing(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class ImportUnavailable(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def summary(operation) -> ProblemImportSummary:
|
||||
return ProblemImportSummary(
|
||||
operation_id=operation.id,
|
||||
state=operation.state,
|
||||
expected_archive_digest=operation.archive_digest,
|
||||
archive_bytes=operation.archive_bytes,
|
||||
created_at=operation.created_at,
|
||||
expires_at=operation.expires_at,
|
||||
updated_at=operation.updated_at,
|
||||
failure_code=operation.failure_code,
|
||||
candidate_id=operation.candidate_id,
|
||||
published_version_id=operation.published_version_id,
|
||||
)
|
||||
|
||||
|
||||
def policy_digest(policy):
|
||||
return hashlib.sha256(policy.model_dump_json().encode()).hexdigest()
|
||||
|
||||
|
||||
class ProblemImports:
|
||||
def __init__(
|
||||
self,
|
||||
sessions,
|
||||
staging: ImportStaging,
|
||||
identity: ProblemMaintenancePort,
|
||||
audit: MaintenanceAuditPort,
|
||||
policy,
|
||||
):
|
||||
self.sessions, self.staging, self.identity, self.audit, self.policy = (
|
||||
sessions,
|
||||
staging,
|
||||
identity,
|
||||
audit,
|
||||
policy,
|
||||
)
|
||||
|
||||
@asynccontextmanager
|
||||
async def write(self, principal):
|
||||
async with self.sessions.begin() as session:
|
||||
await session.execute(text("SET LOCAL statement_timeout='5s'"))
|
||||
await session.execute(text("SET LOCAL lock_timeout='2s'"))
|
||||
await self.identity.require_problem_maintenance(session, principal)
|
||||
yield session
|
||||
|
||||
@staticmethod
|
||||
async def pool(session):
|
||||
pool = await session.get(ImportPool, 1, with_for_update=True)
|
||||
if pool is None:
|
||||
raise ImportUnavailable("import capacity unavailable")
|
||||
actual = await session.scalar(
|
||||
select(func.coalesce(func.sum(Operation.reserved_bytes), 0)).where(
|
||||
Operation.reserved_bytes > 0
|
||||
)
|
||||
)
|
||||
if pool.reserved_bytes != actual:
|
||||
raise ImportUnavailable("import capacity inconsistent")
|
||||
return pool
|
||||
|
||||
async def capabilities(self):
|
||||
async with self.sessions() as session:
|
||||
pool = await session.get(ImportPool, 1)
|
||||
now = await session.scalar(select(func.clock_timestamp()))
|
||||
available = bool(
|
||||
pool
|
||||
and pool.worker_ready_until
|
||||
and pool.worker_ready_until > now
|
||||
and pool.worker_policy_digest == policy_digest(self.policy)
|
||||
)
|
||||
return ProblemImportCapabilities(
|
||||
uploads_available=available, validation_available=available
|
||||
)
|
||||
|
||||
async def create(self, principal, request: CreateProblemImport):
|
||||
# Immutable receipt retries need no filesystem slot, including while a parser/GC
|
||||
# owns the root lock. Missing receipts still enter the locked creation path below.
|
||||
async with self.write(principal) as session:
|
||||
existing = await session.get(Operation, request.operation_id)
|
||||
if existing:
|
||||
if (
|
||||
existing.owner_id != principal.account_id
|
||||
or existing.archive_digest != request.archive_digest
|
||||
or existing.archive_bytes != request.archive_bytes
|
||||
):
|
||||
raise ImportConflict("operation key already bound")
|
||||
return summary(existing)
|
||||
try:
|
||||
with self.staging.exclusive():
|
||||
await asyncio.to_thread(self.staging.usage)
|
||||
async with self.write(principal) as session:
|
||||
pool = await self.pool(session)
|
||||
existing = await session.get(
|
||||
Operation, request.operation_id, with_for_update=True
|
||||
)
|
||||
if existing:
|
||||
if (
|
||||
existing.owner_id != principal.account_id
|
||||
or existing.archive_digest != request.archive_digest
|
||||
or existing.archive_bytes != request.archive_bytes
|
||||
):
|
||||
raise ImportConflict("operation key already bound")
|
||||
return summary(existing)
|
||||
now = await session.scalar(select(func.clock_timestamp()))
|
||||
if (
|
||||
not pool.worker_ready_until
|
||||
or pool.worker_ready_until <= now
|
||||
or pool.worker_policy_digest != policy_digest(self.policy)
|
||||
):
|
||||
raise ImportUnavailable("import validation unavailable")
|
||||
global_count = await session.scalar(
|
||||
select(func.count())
|
||||
.select_from(Operation)
|
||||
.where(Operation.state.in_(ACTIVE))
|
||||
)
|
||||
owner_count = await session.scalar(
|
||||
select(func.count())
|
||||
.select_from(Operation)
|
||||
.where(
|
||||
Operation.state.in_(ACTIVE), Operation.owner_id == principal.account_id
|
||||
)
|
||||
)
|
||||
busy = await session.scalar(
|
||||
select(func.count())
|
||||
.select_from(Operation)
|
||||
.where(
|
||||
Operation.state.in_(("uploading", "validating")),
|
||||
Operation.lease_until > now,
|
||||
)
|
||||
)
|
||||
if (
|
||||
global_count >= MAX_TOTAL_CANDIDATES
|
||||
or owner_count >= MAX_OWNER_CANDIDATES
|
||||
or busy
|
||||
or pool.reserved_bytes + STAGING_RESERVATION_BYTES > STAGING_MAX_BYTES
|
||||
):
|
||||
raise ImportUnavailable("import capacity exhausted")
|
||||
operation = Operation(
|
||||
id=request.operation_id,
|
||||
owner_id=principal.account_id,
|
||||
authorization_version=principal.authorization_version,
|
||||
archive_digest=request.archive_digest,
|
||||
archive_bytes=request.archive_bytes,
|
||||
state="uploading",
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
expires_at=now + timedelta(seconds=IMPORT_LIFETIME_SECONDS),
|
||||
lease_until=now + timedelta(seconds=UPLOAD_DEADLINE_SECONDS + 15),
|
||||
attempts=0,
|
||||
fence=0,
|
||||
reserved_bytes=STAGING_RESERVATION_BYTES,
|
||||
)
|
||||
session.add(operation)
|
||||
pool.reserved_bytes += STAGING_RESERVATION_BYTES
|
||||
await self.audit.record(
|
||||
session,
|
||||
principal.account_id,
|
||||
principal.authorization_version,
|
||||
operation.id,
|
||||
"upload_created",
|
||||
)
|
||||
await session.flush()
|
||||
return summary(operation)
|
||||
except ArtifactError:
|
||||
raise ImportUnavailable("import storage unavailable") from None
|
||||
|
||||
async def detail(self, principal, operation_id):
|
||||
# Recheck live authority even when a principal was injected by an internal caller.
|
||||
async with self.write(principal) as session:
|
||||
op = await session.get(Operation, operation_id)
|
||||
if op is None or op.owner_id != principal.account_id:
|
||||
raise ImportMissing("import unavailable")
|
||||
candidate = (
|
||||
await session.get(ProblemCandidate, op.candidate_id) if op.candidate_id else None
|
||||
)
|
||||
return ProblemImportDetail(
|
||||
**summary(op).model_dump(),
|
||||
candidate=CandidatePreview.model_validate(candidate.preview) if candidate else None,
|
||||
)
|
||||
|
||||
async def list(self, principal, after=None, limit=20):
|
||||
if not 1 <= limit <= 100:
|
||||
raise ValueError("invalid page limit")
|
||||
async with self.write(principal) as session:
|
||||
query = select(Operation).where(Operation.owner_id == principal.account_id)
|
||||
if after:
|
||||
query = query.where(Operation.id > after)
|
||||
rows = list(await session.scalars(query.order_by(Operation.id).limit(limit + 1)))
|
||||
return ProblemImportPage(
|
||||
items=tuple(summary(o) for o in rows[:limit]),
|
||||
next_after=rows[limit - 1].id if len(rows) > limit else None,
|
||||
)
|
||||
|
||||
async def upload(self, principal, operation_id: UUID, chunks, content_length: int | None):
|
||||
# Reject missing/foreign receipts before consulting shared filesystem capacity.
|
||||
# The locked transaction below still rechecks authority and ownership before writes.
|
||||
async with self.write(principal) as session:
|
||||
op = await session.get(Operation, operation_id)
|
||||
if op is None or op.owner_id != principal.account_id:
|
||||
raise ImportMissing("import unavailable")
|
||||
if content_length is not None and content_length != op.archive_bytes:
|
||||
raise ImportConflict("archive length mismatch")
|
||||
descriptor = operation_fd = None
|
||||
fence = None
|
||||
received = 0
|
||||
chunk_count = 0
|
||||
digest = hashlib.sha256()
|
||||
try:
|
||||
with self.staging.exclusive():
|
||||
async with self.write(principal) as session:
|
||||
await self.pool(session)
|
||||
op = await session.get(Operation, operation_id, with_for_update=True)
|
||||
if not op or op.owner_id != principal.account_id:
|
||||
raise ImportMissing("import unavailable")
|
||||
accepted = op.state in ("uploaded", "validating", "ready", "published")
|
||||
now = await session.scalar(select(func.clock_timestamp()))
|
||||
if (
|
||||
not accepted
|
||||
and (
|
||||
op.state != "uploading"
|
||||
or op.lease_until is None
|
||||
or op.lease_until <= now
|
||||
)
|
||||
) or op.expires_at <= now:
|
||||
raise ImportConflict("upload no longer available")
|
||||
if content_length is not None and content_length != op.archive_bytes:
|
||||
raise ImportConflict("archive length mismatch")
|
||||
if not accepted:
|
||||
op.fence += 1
|
||||
fence = op.fence
|
||||
op.lease_until = now + timedelta(seconds=UPLOAD_DEADLINE_SECONDS + 15)
|
||||
expected_bytes, expected_digest = op.archive_bytes, op.archive_digest
|
||||
if not accepted:
|
||||
operation_fd = self.staging.operation(operation_id, create=True)
|
||||
descriptor = os.open(
|
||||
"archive",
|
||||
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC,
|
||||
0o600,
|
||||
dir_fd=operation_fd,
|
||||
)
|
||||
async with asyncio.timeout(UPLOAD_DEADLINE_SECONDS):
|
||||
async for chunk in chunks:
|
||||
chunk_count += 1
|
||||
received += len(chunk)
|
||||
if (
|
||||
type(chunk) is not bytes
|
||||
or len(chunk) > MAX_UPLOAD_CHUNK_BYTES
|
||||
or chunk_count > MAX_UPLOAD_CHUNKS
|
||||
or received > expected_bytes
|
||||
):
|
||||
raise ImportConflict("archive bytes outside policy")
|
||||
digest.update(chunk)
|
||||
if accepted:
|
||||
continue
|
||||
# Cancellation waits for the finite writer outside the event loop.
|
||||
task = asyncio.create_task(asyncio.to_thread(write_all, descriptor, chunk))
|
||||
try:
|
||||
await asyncio.shield(task)
|
||||
except BaseException:
|
||||
await task
|
||||
raise
|
||||
if received != expected_bytes or digest.hexdigest() != expected_digest:
|
||||
raise ImportBadDigest("archive digest or length mismatch")
|
||||
if accepted:
|
||||
async with self.write(principal) as session:
|
||||
op = await session.get(Operation, operation_id)
|
||||
if not op or op.state not in (
|
||||
"uploaded",
|
||||
"validating",
|
||||
"ready",
|
||||
"published",
|
||||
):
|
||||
raise ImportConflict("import receipt no longer available")
|
||||
return summary(op)
|
||||
await asyncio.to_thread(os.fsync, descriptor)
|
||||
os.close(descriptor)
|
||||
descriptor = None
|
||||
os.fsync(operation_fd)
|
||||
async with self.write(principal) as session:
|
||||
await self.pool(session)
|
||||
op = await session.get(Operation, operation_id, with_for_update=True)
|
||||
now = await session.scalar(select(func.clock_timestamp()))
|
||||
if op.state != "uploading" or op.fence != fence or op.lease_until <= now:
|
||||
raise ImportConflict("upload receipt expired")
|
||||
op.state = "uploaded"
|
||||
op.lease_until = None
|
||||
op.updated_at = now
|
||||
await self.audit.record(
|
||||
session, op.owner_id, op.authorization_version, op.id, "upload_received"
|
||||
)
|
||||
result = summary(op)
|
||||
return result
|
||||
except BaseException as error:
|
||||
# Failure bookkeeping cannot authorize a new business write. It only fences this
|
||||
# accepted upload and releases capacity after physical bytes have been removed.
|
||||
if descriptor is not None:
|
||||
os.close(descriptor)
|
||||
descriptor = None
|
||||
if operation_fd is not None:
|
||||
os.close(operation_fd)
|
||||
operation_fd = None
|
||||
if fence is not None:
|
||||
try:
|
||||
with self.staging.exclusive():
|
||||
await asyncio.to_thread(self.staging.remove, operation_id)
|
||||
async with self.sessions.begin() as session:
|
||||
pool = await self.pool(session)
|
||||
op = await session.get(Operation, operation_id, with_for_update=True)
|
||||
if op and op.state == "uploading" and op.fence == fence:
|
||||
op.state = "failed"
|
||||
op.failure_code = (
|
||||
"upload_deadline"
|
||||
if isinstance(error, TimeoutError)
|
||||
else "digest_mismatch"
|
||||
if isinstance(error, ImportBadDigest)
|
||||
else "authority_revoked"
|
||||
if isinstance(error, MaintenanceDenied)
|
||||
else "storage_unavailable"
|
||||
if isinstance(error, OSError)
|
||||
else "upload_interrupted"
|
||||
)
|
||||
op.updated_at = await session.scalar(select(func.clock_timestamp()))
|
||||
op.lease_until = None
|
||||
pool.reserved_bytes -= op.reserved_bytes
|
||||
op.reserved_bytes = 0
|
||||
await self.audit.record(
|
||||
session,
|
||||
op.owner_id,
|
||||
op.authorization_version,
|
||||
op.id,
|
||||
"import_failed",
|
||||
)
|
||||
except (ArtifactError, OSError):
|
||||
pass # Durable reservation retained for fenced GC.
|
||||
raise
|
||||
finally:
|
||||
if descriptor is not None:
|
||||
os.close(descriptor)
|
||||
if operation_fd is not None:
|
||||
os.close(operation_fd)
|
||||
|
||||
async def discard(self, principal, operation_id):
|
||||
async with self.write(principal) as session:
|
||||
op = await session.get(Operation, operation_id)
|
||||
if op is None or op.owner_id != principal.account_id:
|
||||
raise ImportMissing("import unavailable")
|
||||
if op.state == "discarded":
|
||||
return summary(op)
|
||||
try:
|
||||
with self.staging.exclusive():
|
||||
async with self.write(principal) as session:
|
||||
pool = await self.pool(session)
|
||||
op = await session.get(Operation, operation_id, with_for_update=True)
|
||||
if not op or op.owner_id != principal.account_id:
|
||||
raise ImportMissing("import unavailable")
|
||||
if op.state == "published":
|
||||
raise ImportConflict("published import cannot be discarded")
|
||||
if op.state == "discarded":
|
||||
return summary(op)
|
||||
await asyncio.to_thread(self.staging.remove, operation_id)
|
||||
op.state = "discarded"
|
||||
op.candidate_id = None
|
||||
op.failure_code = None
|
||||
op.lease_until = None
|
||||
op.fence += 1
|
||||
op.updated_at = await session.scalar(select(func.clock_timestamp()))
|
||||
pool.reserved_bytes -= op.reserved_bytes
|
||||
op.reserved_bytes = 0
|
||||
await self.audit.record(
|
||||
session,
|
||||
principal.account_id,
|
||||
principal.authorization_version,
|
||||
op.id,
|
||||
"import_discarded",
|
||||
)
|
||||
return summary(op)
|
||||
except ArtifactError:
|
||||
raise ImportUnavailable("import storage unavailable") from None
|
||||
|
||||
|
||||
def write_all(descriptor, data):
|
||||
remaining = memoryview(data)
|
||||
while remaining:
|
||||
size = os.write(descriptor, remaining)
|
||||
if size <= 0:
|
||||
raise OSError("short staging write")
|
||||
remaining = remaining[size:]
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Stricter Web package declarations; the accepted local CLI format remains unchanged."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from kf_oj.contracts.judging import decode_wire
|
||||
from kf_oj.modules.problems.import_format import ImportEnvelope, ProblemImport, ProblemImportV2
|
||||
|
||||
ARCHIVE_SUFFIXES = (".zip", ".jar", ".war", ".tar", ".tgz", ".gz", ".bz2", ".xz", ".7z", ".rar")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WebDeclaration:
|
||||
manifest: ProblemImport | ProblemImportV2
|
||||
members: frozenset[str]
|
||||
|
||||
|
||||
def decode_web_declaration(data: bytes) -> WebDeclaration:
|
||||
# decode_wire rejects duplicate JSON keys, unknown fields and noncanonical
|
||||
# protocol values before either local paths or node policy can be considered.
|
||||
manifest = decode_wire(ImportEnvelope, data).root
|
||||
hidden = {name for c in manifest.cases for name in (c.input_file, c.expected_file)}
|
||||
samples = {name for s in manifest.samples for name in (s.input_file, s.output_file)}
|
||||
metadata = {"problem.json", manifest.statement_file}
|
||||
if len(metadata) != 2 or hidden & (samples | metadata) or samples & metadata:
|
||||
raise ValueError("public and hidden package declarations must be separate")
|
||||
members = frozenset(metadata | hidden | samples)
|
||||
if len({name.lower() for name in members}) != len(members):
|
||||
raise ValueError("package names must be case-insensitively unique")
|
||||
if any(name.lower().endswith(ARCHIVE_SUFFIXES) for name in members):
|
||||
raise ValueError("nested archive declarations are unsupported")
|
||||
return WebDeclaration(manifest, members)
|
||||
@@ -0,0 +1,116 @@
|
||||
"""Fixed data-only child. It inherits no DB credentials and executes no uploaded contents."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import resource
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from kf_oj.contracts.admin_problems import (
|
||||
VALIDATION_DEADLINE_SECONDS,
|
||||
CandidatePreview,
|
||||
LegacyImportBudget,
|
||||
)
|
||||
from kf_oj.contracts.judging_v2 import NodeAdmission
|
||||
from kf_oj.contracts.problems import PublicLanguageConfiguration, PublicSample
|
||||
from kf_oj.infra.artifacts.archive_reader import ArchiveError, ArchiveReader
|
||||
from kf_oj.modules.problems.candidate_validation import validate_candidate
|
||||
from kf_oj.modules.problems.import_format import ProblemImport
|
||||
from kf_oj.modules.problems.web_format import decode_web_declaration
|
||||
|
||||
|
||||
def main():
|
||||
# These are ordinary downward-only process limits, independent of the parent's life.
|
||||
resource.setrlimit(resource.RLIMIT_AS, (128 * 1024**2, 128 * 1024**2))
|
||||
resource.setrlimit(resource.RLIMIT_CPU, (60, 60))
|
||||
resource.setrlimit(resource.RLIMIT_FSIZE, (64 * 1024**2, 64 * 1024**2))
|
||||
resource.setrlimit(resource.RLIMIT_NOFILE, (64, 64))
|
||||
resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
|
||||
signal.alarm(VALIDATION_DEADLINE_SECONDS)
|
||||
archive, directory, lock, size, digest, candidate, policy_json = sys.argv[1:]
|
||||
archive, directory, lock = int(archive), int(directory), int(lock)
|
||||
os.fstat(lock) # Held inherited flock prevents GC releasing capacity while any child lives.
|
||||
deadline = time.monotonic() + VALIDATION_DEADLINE_SECONDS
|
||||
result = {"failure_code": "invalid_archive"}
|
||||
try:
|
||||
reader = ArchiveReader(archive, int(size), deadline=deadline)
|
||||
reader.verify_digest(digest)
|
||||
result["failure_code"] = "invalid_manifest"
|
||||
declared = decode_web_declaration(reader.read("problem.json", 256 * 1024))
|
||||
if {m.name for m in reader.members} != set(declared.members):
|
||||
raise ValueError("unexpected package members")
|
||||
manifest = declared.manifest
|
||||
result["failure_code"] = "invalid_data"
|
||||
statement = reader.read(manifest.statement_file, 256 * 1024).decode("utf-8")
|
||||
sample_data = {
|
||||
name: reader.read(name, 65536)
|
||||
for name in sorted({n for s in manifest.samples for n in (s.input_file, s.output_file)})
|
||||
}
|
||||
samples = tuple(
|
||||
PublicSample(
|
||||
input=base64.b64encode(sample_data[s.input_file]).decode("ascii"),
|
||||
output=base64.b64encode(sample_data[s.output_file]).decode("ascii"),
|
||||
)
|
||||
for s in manifest.samples
|
||||
)
|
||||
references = reader.extract_remaining(directory)
|
||||
reader.verify_unchanged()
|
||||
legacy = isinstance(manifest, ProblemImport)
|
||||
preview = CandidatePreview(
|
||||
candidate_id=UUID(candidate),
|
||||
slug=manifest.slug,
|
||||
title=manifest.title,
|
||||
statement=statement,
|
||||
samples=samples,
|
||||
judging_schema_version=manifest.schema_version,
|
||||
languages=manifest.languages,
|
||||
legacy_budget=LegacyImportBudget(
|
||||
compile_limits=manifest.compile_limits,
|
||||
run_limits=manifest.run_limits,
|
||||
job_wall_time_us=manifest.job_wall_time_us,
|
||||
)
|
||||
if legacy
|
||||
else None,
|
||||
language_configurations=()
|
||||
if legacy
|
||||
else tuple(
|
||||
PublicLanguageConfiguration.from_configuration(c)
|
||||
for c in manifest.language_configurations
|
||||
),
|
||||
case_count=len(manifest.cases),
|
||||
)
|
||||
result["failure_code"] = "policy_rejected"
|
||||
validated = validate_candidate(
|
||||
manifest.model_dump(mode="json"),
|
||||
preview.model_dump(mode="json"),
|
||||
{k: {"digest": v[0], "bytes": v[1]} for k, v in references.items()},
|
||||
TypeAdapter(NodeAdmission).validate_json(policy_json),
|
||||
)
|
||||
result = {
|
||||
"manifest": validated.manifest,
|
||||
"preview": validated.preview.model_dump(mode="json"),
|
||||
"references": {k: v.model_dump(mode="json") for k, v in validated.references.items()},
|
||||
"manifest_digest": validated.digest,
|
||||
}
|
||||
except ArchiveError as error:
|
||||
result = {
|
||||
"failure_code": "digest_mismatch"
|
||||
if str(error) == "archive_digest_mismatch"
|
||||
else "invalid_archive"
|
||||
}
|
||||
except (ValueError, OSError, MemoryError):
|
||||
result = {"failure_code": result["failure_code"]}
|
||||
payload = json.dumps(result, separators=(",", ":"), ensure_ascii=True).encode()
|
||||
if len(payload) > 4 * 1024**2:
|
||||
payload = b'{"failure_code":"invalid_data"}'
|
||||
sys.stdout.buffer.write(payload)
|
||||
sys.stdout.buffer.flush()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Ordinary import service composition. Provision its dedicated DB role and cgroup separately."""
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
|
||||
from sqlalchemy.exc import DBAPIError
|
||||
|
||||
from kf_oj.core.settings import Settings
|
||||
from kf_oj.infra.artifacts.repository import ArtifactRepository
|
||||
from kf_oj.infra.artifacts.staging import ImportStaging
|
||||
from kf_oj.infra.artifacts.store import ArtifactStore
|
||||
from kf_oj.infra.db.session import build_engine, build_session_factory
|
||||
from kf_oj.modules.identity.service import ReservedMaintenance
|
||||
from kf_oj.modules.operations.service import MaintenanceAudit
|
||||
from kf_oj.modules.problems.import_worker import ImportWorker, verify_worker_limits
|
||||
|
||||
|
||||
async def run():
|
||||
settings = Settings()
|
||||
if (
|
||||
settings.import_staging_root is None
|
||||
or settings.artifact_root is None
|
||||
or settings.judge_admission is None
|
||||
or settings.import_worker_cgroup is None
|
||||
):
|
||||
raise ValueError("import service configuration required")
|
||||
verify_worker_limits(settings.import_worker_cgroup)
|
||||
engine = build_engine(settings)
|
||||
store = ArtifactStore(
|
||||
settings.artifact_root,
|
||||
max_bytes=settings.artifact_max_bytes,
|
||||
max_files=settings.artifact_max_files,
|
||||
)
|
||||
staging = ImportStaging(settings.import_staging_root)
|
||||
try:
|
||||
worker = ImportWorker(
|
||||
build_session_factory(engine),
|
||||
staging,
|
||||
ArtifactRepository(store),
|
||||
ReservedMaintenance(),
|
||||
MaintenanceAudit(),
|
||||
settings.judge_admission,
|
||||
settings.import_worker_cgroup,
|
||||
)
|
||||
while True:
|
||||
if not await worker.once():
|
||||
await asyncio.sleep(1)
|
||||
finally:
|
||||
staging.close()
|
||||
store.close()
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
asyncio.run(run())
|
||||
except (ValueError, OSError, DBAPIError):
|
||||
print(
|
||||
"import service unavailable; check trusted configuration and resource limits",
|
||||
file=sys.stderr,
|
||||
)
|
||||
raise SystemExit(1) from None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,156 @@
|
||||
"""Contract checks only; upload/parser/security evidence requires the independent VM."""
|
||||
|
||||
import copy
|
||||
import json
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from kf_oj.contracts.admin_problems import (
|
||||
CandidatePreview,
|
||||
CreateProblemImport,
|
||||
LegacyImportBudget,
|
||||
ProblemImportCapabilities,
|
||||
ProblemImportDetail,
|
||||
)
|
||||
from kf_oj.contracts.judging import AdmissionPolicy
|
||||
from kf_oj.modules.problems.import_format import ImportEnvelope
|
||||
from kf_oj.modules.problems.web_format import decode_web_declaration
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def legacy_manifest():
|
||||
policy = AdmissionPolicy.model_validate_json(
|
||||
(ROOT / "tests/fixtures/contracts/judge-admission.valid.json").read_bytes()
|
||||
)
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"slug": "web-fixture",
|
||||
"title": "Public",
|
||||
"statement_file": "statement.md",
|
||||
"languages": ["c17", "cpp20"],
|
||||
"toolchain": policy.toolchain.model_dump(mode="json"),
|
||||
"comparison": "exact_bytes",
|
||||
"compile_limits": policy.compile_ceiling.model_dump(),
|
||||
"run_limits": policy.run_ceiling.model_dump(),
|
||||
"job_wall_time_us": 60000000,
|
||||
"cases": [{"key": "binary", "input_file": "hidden.in", "expected_file": "hidden.out"}],
|
||||
"samples": [{"input_file": "sample.in", "output_file": "sample.out"}],
|
||||
}
|
||||
|
||||
|
||||
def test_web_declaration_separates_public_data_and_preserves_local_format():
|
||||
source = legacy_manifest()
|
||||
original = copy.deepcopy(source)
|
||||
declared = decode_web_declaration(json.dumps(source).encode())
|
||||
assert declared.members == frozenset(
|
||||
("problem.json", "statement.md", "hidden.in", "hidden.out", "sample.in", "sample.out")
|
||||
)
|
||||
assert declared.manifest.model_dump(mode="json") == original
|
||||
assert ImportEnvelope.model_validate(source).root == declared.manifest
|
||||
for mutation in (
|
||||
lambda x: x["samples"][0].update(input_file="hidden.in"),
|
||||
lambda x: x.update(statement_file="problem.json"),
|
||||
lambda x: x["cases"][0].update(input_file="Statement.md"),
|
||||
lambda x: x["cases"][0].update(input_file="hidden.zip"),
|
||||
lambda x: x.update(languages=["rust"]),
|
||||
lambda x: x.update(download_url="https://example.invalid/private"),
|
||||
):
|
||||
value = copy.deepcopy(source)
|
||||
mutation(value)
|
||||
with pytest.raises(ValueError):
|
||||
decode_web_declaration(json.dumps(value).encode())
|
||||
with pytest.raises(ValueError):
|
||||
decode_web_declaration(
|
||||
json.dumps(source)
|
||||
.replace('"schema_version": 1', '"schema_version": 1, "schema_version": 1')
|
||||
.encode()
|
||||
)
|
||||
|
||||
|
||||
def test_import_operation_key_and_preview_have_no_private_payload_fields():
|
||||
operation = {
|
||||
"schema_version": 1,
|
||||
"operation_id": str(uuid4()),
|
||||
"archive_digest": "a" * 64,
|
||||
"archive_bytes": 64 * 1024**2,
|
||||
}
|
||||
assert CreateProblemImport.model_validate(operation).archive_bytes == 64 * 1024**2
|
||||
for field, value in (
|
||||
("archive_bytes", True),
|
||||
("archive_bytes", 64 * 1024**2 + 1),
|
||||
("url", "https://example.invalid"),
|
||||
("role", "admin"),
|
||||
):
|
||||
with pytest.raises(ValidationError):
|
||||
CreateProblemImport.model_validate(operation | {field: value})
|
||||
manifest = ImportEnvelope.model_validate(legacy_manifest()).root
|
||||
preview = CandidatePreview(
|
||||
candidate_id=uuid4(),
|
||||
slug=manifest.slug,
|
||||
title=manifest.title,
|
||||
statement="Public only",
|
||||
samples=(),
|
||||
judging_schema_version=1,
|
||||
languages=manifest.languages,
|
||||
legacy_budget=LegacyImportBudget(
|
||||
compile_limits=manifest.compile_limits,
|
||||
run_limits=manifest.run_limits,
|
||||
job_wall_time_us=manifest.job_wall_time_us,
|
||||
),
|
||||
case_count=1,
|
||||
)
|
||||
now = datetime.now(UTC)
|
||||
response = ProblemImportDetail(
|
||||
operation_id=uuid4(),
|
||||
state="ready",
|
||||
expected_archive_digest="a" * 64,
|
||||
archive_bytes=100,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
expires_at=now + timedelta(days=1),
|
||||
failure_code=None,
|
||||
candidate_id=preview.candidate_id,
|
||||
published_version_id=None,
|
||||
candidate=preview,
|
||||
)
|
||||
assert set(response.candidate.model_dump()) == {
|
||||
"candidate_id",
|
||||
"slug",
|
||||
"title",
|
||||
"statement",
|
||||
"samples",
|
||||
"judging_schema_version",
|
||||
"languages",
|
||||
"legacy_budget",
|
||||
"language_configurations",
|
||||
"case_count",
|
||||
}
|
||||
with pytest.raises(ValidationError):
|
||||
CandidatePreview.model_validate(preview.model_dump() | {"input_digest": "a" * 64})
|
||||
capabilities = ProblemImportCapabilities(uploads_available=False, validation_available=False)
|
||||
assert (
|
||||
capabilities.limits.global_concurrency == 1 and capabilities.limits.owner_candidates == 10
|
||||
)
|
||||
|
||||
|
||||
def test_standalone_import_worker_resolves_all_referenced_metadata():
|
||||
# A separate trusted interpreter avoids accidental model registration by API test imports.
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
script = """
|
||||
import kf_oj.workers.problem_import
|
||||
from kf_oj.infra.db.base import Base
|
||||
for table in Base.metadata.tables.values():
|
||||
for foreign in table.foreign_keys:
|
||||
assert foreign.column.table is not None
|
||||
"""
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", script], capture_output=True, text=True, timeout=15
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
@@ -0,0 +1,988 @@
|
||||
"""Real HTTP/worker/storage and adverse ZIP cases: independent capped VM only."""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import resource
|
||||
import signal
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from conftest import connect
|
||||
from sqlalchemy.engine import make_url
|
||||
|
||||
from kf_oj.contracts.judging import ArtifactReference
|
||||
from kf_oj.contracts.judging_v2 import AdmissionRegistry
|
||||
from kf_oj.infra.artifacts.archive_reader import ArchiveError, ArchiveReader
|
||||
from kf_oj.infra.artifacts.staging import ImportStaging
|
||||
from kf_oj.infra.artifacts.store import ArtifactError, ArtifactStore
|
||||
from kf_oj.modules.identity.passwords import hash_password
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def independent_import_vm():
|
||||
marker = Path("/kf/INDEPENDENT_VM")
|
||||
if not marker.exists() or marker.read_text().strip() != "isolated-import-test-guest":
|
||||
pytest.skip("adverse archive/full import acceptance requires the independent capped VM")
|
||||
assert [p.name for p in Path("/sys/class/net").iterdir()] == ["lo"]
|
||||
assert all(
|
||||
x not in Path("/proc/self/mountinfo").read_text() for x in ("9p", "virtiofs", "hgfs")
|
||||
)
|
||||
return {name: Path("/sys/fs/cgroup/kf-total") / name for name in ("api", "import-worker")}
|
||||
|
||||
|
||||
def package(
|
||||
policy, *, multi=False, method=zipfile.ZIP_STORED, mutation=None, payload_mutation=None
|
||||
):
|
||||
payload = {
|
||||
"statement.md": b"Public statement only",
|
||||
"hidden.in": b"synthetic-hidden-input\x00\xff",
|
||||
"hidden.out": b"synthetic-hidden-output\x00\xfe",
|
||||
"sample.in": b"1\n",
|
||||
"sample.out": b"2\n",
|
||||
}
|
||||
manifest = {
|
||||
"schema_version": 1,
|
||||
"slug": "web-synthetic",
|
||||
"title": "Public",
|
||||
"statement_file": "statement.md",
|
||||
"languages": ["c17", "cpp20"],
|
||||
"toolchain": policy.legacy.toolchain.model_dump(mode="json"),
|
||||
"comparison": "exact_bytes",
|
||||
"compile_limits": policy.legacy.compile_ceiling.model_dump(),
|
||||
"run_limits": policy.legacy.run_ceiling.model_dump(),
|
||||
"job_wall_time_us": 60000000,
|
||||
"cases": [{"key": "binary", "input_file": "hidden.in", "expected_file": "hidden.out"}],
|
||||
"samples": [{"input_file": "sample.in", "output_file": "sample.out"}],
|
||||
}
|
||||
if multi:
|
||||
for key in ("languages", "toolchain", "compile_limits", "run_limits", "job_wall_time_us"):
|
||||
del manifest[key]
|
||||
manifest["schema_version"] = 2
|
||||
manifest["language_configurations"] = [
|
||||
{
|
||||
key: json.loads(
|
||||
(
|
||||
ROOT / f"tests/fixtures/contracts-v2/judge-manifest.valid-{language}.json"
|
||||
).read_text()
|
||||
)[key]
|
||||
for key in (
|
||||
"language",
|
||||
"toolchain",
|
||||
"compile_limits",
|
||||
"run_limits",
|
||||
"job_wall_time_us",
|
||||
)
|
||||
}
|
||||
for language in ("c17", "cpp20", "python313", "java21")
|
||||
]
|
||||
if mutation:
|
||||
mutation(manifest)
|
||||
payload["problem.json"] = json.dumps(manifest).encode()
|
||||
if payload_mutation:
|
||||
payload_mutation(payload)
|
||||
out = io.BytesIO()
|
||||
with zipfile.ZipFile(out, "w", compression=method) as archive:
|
||||
for name, data in payload.items():
|
||||
archive.writestr(name, data)
|
||||
return out.getvalue(), payload
|
||||
|
||||
|
||||
def test_bounded_zip_and_streaming_adoption(independent_import_vm, tmp_path):
|
||||
policy = AdmissionRegistry.model_validate_json(
|
||||
(ROOT / "tests/fixtures/contracts-v2/judge-registry.valid.json").read_bytes()
|
||||
)
|
||||
for method in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED):
|
||||
blob, payload = package(
|
||||
policy,
|
||||
method=method,
|
||||
payload_mutation=lambda p: p.update({"hidden.in": b"A" * 1048576 + b"\x00\xff"}),
|
||||
)
|
||||
file = tmp_path / f"archive-{method}"
|
||||
file.write_bytes(blob)
|
||||
file.chmod(0o600)
|
||||
output = tmp_path / f"members-{method}"
|
||||
output.mkdir(mode=0o700)
|
||||
fd = os.open(file, os.O_RDONLY)
|
||||
directory = os.open(output, os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
reader = ArchiveReader(fd, len(blob), deadline=time.monotonic() + 60)
|
||||
reader.verify_digest(hashlib.sha256(blob).hexdigest())
|
||||
assert reader.read("problem.json", 262144) == payload.pop("problem.json")
|
||||
references = reader.extract_remaining(directory)
|
||||
assert reader.unpacked_bytes == sum(len(v) for v in payload.values()) + len(
|
||||
package(policy, method=method)[1]["problem.json"]
|
||||
)
|
||||
store = ArtifactStore(tmp_path / f"objects-{method}")
|
||||
try:
|
||||
for name, (digest, size) in references.items():
|
||||
assert (output / name).read_bytes() == payload[name]
|
||||
source = os.open(name, os.O_RDONLY, dir_fd=directory)
|
||||
try:
|
||||
reference = ArtifactReference(digest=digest, bytes=size)
|
||||
assert store.put_file(source, reference) == reference
|
||||
assert store.put_file(source, reference) == reference
|
||||
finally:
|
||||
os.close(source)
|
||||
finally:
|
||||
store.close()
|
||||
finally:
|
||||
os.close(fd)
|
||||
os.close(directory)
|
||||
good, _ = package(policy)
|
||||
wrong_crc = bytearray(good)
|
||||
struct.pack_into("<I", wrong_crc, 14, 1)
|
||||
cases = [b"prefix" + good, good + b"trailing", bytes(wrong_crc)]
|
||||
for name in ("../escape", "bad/name", "Statement.md"):
|
||||
cases.append(package(policy, payload_mutation=lambda p, n=name: p.update({n: b"bad"}))[0])
|
||||
symlink = io.BytesIO()
|
||||
with zipfile.ZipFile(symlink, "w") as z:
|
||||
info = zipfile.ZipInfo("link")
|
||||
info.create_system = 3
|
||||
info.external_attr = 0o120777 << 16
|
||||
z.writestr(info, b"/outside")
|
||||
cases.append(symlink.getvalue())
|
||||
# Real damaged stored data and forged consistent local/central size/CRC/method/flags.
|
||||
corrupt = bytearray(good)
|
||||
name_bytes, extra_bytes = struct.unpack_from("<HH", corrupt, 26)
|
||||
corrupt[30 + name_bytes + extra_bytes] ^= 1
|
||||
cases.append(bytes(corrupt))
|
||||
central = good.index(b"PK\x01\x02")
|
||||
for local_offset, central_offset, value, fmt in (
|
||||
(6, 8, 1, "<H"), # encryption
|
||||
(8, 10, 99, "<H"), # unsupported method
|
||||
(22, 24, 67108865, "<I"), # claimed size outside budget
|
||||
):
|
||||
damaged = bytearray(good)
|
||||
struct.pack_into(fmt, damaged, local_offset, value)
|
||||
struct.pack_into(fmt, damaged, central + central_offset, value)
|
||||
cases.append(bytes(damaged))
|
||||
compressed, _ = package(policy, method=zipfile.ZIP_DEFLATED)
|
||||
forged = bytearray(compressed)
|
||||
central = compressed.index(b"PK\x01\x02")
|
||||
struct.pack_into("<I", forged, 22, 1)
|
||||
struct.pack_into("<I", forged, central + 24, 1)
|
||||
cases.append(bytes(forged)) # actual inflation exceeds the forged size
|
||||
for index, blob in enumerate(cases):
|
||||
file = tmp_path / f"bad-{index}"
|
||||
file.write_bytes(blob)
|
||||
file.chmod(0o600)
|
||||
fd = os.open(file, os.O_RDONLY)
|
||||
try:
|
||||
with pytest.raises(ArchiveError):
|
||||
reader = ArchiveReader(fd, len(blob), deadline=time.monotonic() + 60)
|
||||
# Unknown declaration references are rejected by the child, rather than
|
||||
# the raw ZIP decoder. Here invalid paths/types/CRC/header bytes are read.
|
||||
for member in reader.members:
|
||||
reader.read(member.name, 64 * 1024**2)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def test_real_admin_upload_worker_candidate_faults_restart_and_cleanup(
|
||||
independent_import_vm, postgres_url, tmp_path
|
||||
):
|
||||
groups = independent_import_vm
|
||||
policy = AdmissionRegistry.model_validate_json(
|
||||
(ROOT / "tests/fixtures/contracts-v2/judge-registry.valid.json").read_bytes()
|
||||
)
|
||||
base = {
|
||||
k: v
|
||||
for k, v in os.environ.items()
|
||||
if k
|
||||
in (
|
||||
"PATH",
|
||||
"PYTHONPATH",
|
||||
"LD_LIBRARY_PATH",
|
||||
"LC_ALL",
|
||||
"PYTHONUNBUFFERED",
|
||||
"PYTHONDONTWRITEBYTECODE",
|
||||
)
|
||||
}
|
||||
roles = {"api": "import_api_" + uuid4().hex, "worker": "import_worker_" + uuid4().hex}
|
||||
password = "Synthetic import acceptance only 2026!"
|
||||
users = {name: uuid4() for name in ("web_admin", "web_backup", "web_student")}
|
||||
processes = {}
|
||||
logs = {}
|
||||
artifact = tmp_path / "artifacts"
|
||||
staging = tmp_path / "staging"
|
||||
artifact.mkdir(mode=0o700)
|
||||
staging.mkdir(mode=0o700)
|
||||
|
||||
def enter(group):
|
||||
def apply():
|
||||
(group / "cgroup.procs").write_text(str(os.getpid()))
|
||||
resource.setrlimit(resource.RLIMIT_FSIZE, (67108864, 67108864))
|
||||
resource.setrlimit(resource.RLIMIT_NOFILE, (64, 64))
|
||||
resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
|
||||
|
||||
return apply
|
||||
|
||||
def start(name, *, configured=True, pair=False):
|
||||
group = groups["api" if name == "api" else "import-worker"]
|
||||
env = base | {
|
||||
"KF_OJ_DATABASE_URL": make_url(postgres_url)
|
||||
.set(username=roles[name])
|
||||
.render_as_string(hide_password=False),
|
||||
"KF_OJ_ENVIRONMENT": "test",
|
||||
"KF_OJ_ARTIFACT_ROOT": str(artifact),
|
||||
"KF_OJ_IMPORT_STAGING_ROOT": str(staging),
|
||||
"KF_OJ_JUDGE_ADMISSION": policy.model_dump_json(),
|
||||
"KF_OJ_BROWSER_ORIGINS": '["http://127.0.0.1:58000"]',
|
||||
}
|
||||
if not configured:
|
||||
env.pop("KF_OJ_IMPORT_STAGING_ROOT")
|
||||
if name == "worker":
|
||||
env["KF_OJ_IMPORT_WORKER_CGROUP"] = str(group)
|
||||
command = (
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"uvicorn",
|
||||
"kf_oj.api.app:create_app",
|
||||
"--factory",
|
||||
"--host",
|
||||
"127.0.0.1",
|
||||
"--port",
|
||||
"58000",
|
||||
"--no-proxy-headers",
|
||||
"--limit-concurrency",
|
||||
"4",
|
||||
"--backlog",
|
||||
"16",
|
||||
]
|
||||
if name == "api"
|
||||
else [sys.executable, "-m", "kf_oj.workers.problem_import"]
|
||||
)
|
||||
if pair:
|
||||
assert name == "worker"
|
||||
# Trusted fixture composition; both coroutines call the actual ordinary Worker.
|
||||
code = """
|
||||
import asyncio
|
||||
from kf_oj.core.settings import Settings
|
||||
from kf_oj.infra.db.session import build_engine,build_session_factory
|
||||
from kf_oj.infra.artifacts.store import ArtifactStore
|
||||
from kf_oj.infra.artifacts.staging import ImportStaging
|
||||
from kf_oj.infra.artifacts.repository import ArtifactRepository
|
||||
from kf_oj.modules.identity.service import ReservedMaintenance
|
||||
from kf_oj.modules.operations.service import MaintenanceAudit
|
||||
from kf_oj.modules.problems.import_worker import ImportWorker
|
||||
async def run():
|
||||
s=Settings();e=build_engine(s);store=ArtifactStore(s.artifact_root)
|
||||
staging=ImportStaging(s.import_staging_root)
|
||||
try:
|
||||
w=ImportWorker(build_session_factory(e),staging,ArtifactRepository(store),
|
||||
ReservedMaintenance(),MaintenanceAudit(),s.judge_admission,s.import_worker_cgroup)
|
||||
results=await asyncio.gather(w.once(),w.once())
|
||||
assert sorted(results)==[False,True],results
|
||||
print('ACTUAL_CONCURRENT_IMPORT_CLAIM_ACCEPTED',flush=True)
|
||||
finally:
|
||||
staging.close();store.close();await e.dispose()
|
||||
asyncio.run(run())
|
||||
"""
|
||||
command = [sys.executable, "-c", code]
|
||||
log = (tmp_path / f"{name}.log").open("a")
|
||||
logs[name] = log
|
||||
processes[name] = subprocess.Popen(
|
||||
command, env=env, stdout=log, stderr=log, preexec_fn=enter(group)
|
||||
)
|
||||
|
||||
def stop(name):
|
||||
p = processes.pop(name, None)
|
||||
if p and p.poll() is None:
|
||||
p.terminate()
|
||||
try:
|
||||
p.wait(timeout=20)
|
||||
except subprocess.TimeoutExpired:
|
||||
p.kill()
|
||||
p.wait(timeout=10)
|
||||
if name in logs:
|
||||
logs.pop(name).close()
|
||||
|
||||
async def run():
|
||||
c = await connect(postgres_url)
|
||||
try:
|
||||
for name, group in (("api", "kf_oj_platform"), ("worker", "kf_oj_problem_import")):
|
||||
await c.execute(
|
||||
f"CREATE ROLE {roles[name]} LOGIN NOSUPERUSER NOCREATEDB "
|
||||
"NOCREATEROLE NOREPLICATION NOBYPASSRLS"
|
||||
)
|
||||
await c.execute(f"GRANT {group} TO {roles[name]}")
|
||||
for name, identifier in users.items():
|
||||
await c.execute(
|
||||
"INSERT INTO accounts(id,username,password_hash) VALUES($1,$2,$3)",
|
||||
identifier,
|
||||
name,
|
||||
hash_password(name, password),
|
||||
)
|
||||
await c.execute("SELECT * FROM kf_manage_account_authority('web_admin','bootstrap')")
|
||||
await c.execute("SELECT * FROM kf_manage_account_authority('web_backup','grant')")
|
||||
start("api")
|
||||
start("worker")
|
||||
async with httpx.AsyncClient(base_url="http://127.0.0.1:58000", timeout=45) as client:
|
||||
deadline = time.monotonic() + 300
|
||||
while time.monotonic() < deadline:
|
||||
assert processes["api"].poll() is None, (tmp_path / "api.log").read_text()[
|
||||
-1500:
|
||||
]
|
||||
assert processes["worker"].poll() is None, (
|
||||
tmp_path / "worker.log"
|
||||
).read_text()[-1500:]
|
||||
try:
|
||||
if (
|
||||
await client.get("/api/v1/health/live")
|
||||
).status_code == 200 and await c.fetchval(
|
||||
"SELECT worker_ready_until>clock_timestamp() "
|
||||
"FROM problem_import_pool WHERE id=1"
|
||||
):
|
||||
break
|
||||
except httpx.ConnectError:
|
||||
pass
|
||||
await asyncio.sleep(0.1)
|
||||
else:
|
||||
raise AssertionError(
|
||||
"real API/worker startup deadline; api="
|
||||
+ (tmp_path / "api.log").read_text()[-1500:]
|
||||
+ " worker="
|
||||
+ (tmp_path / "worker.log").read_text()[-1500:]
|
||||
)
|
||||
prefix = "/api/v1/admin/problem-imports"
|
||||
assert (await client.get(prefix + "/capabilities")).status_code == 401
|
||||
|
||||
async def login(name):
|
||||
client.cookies.clear()
|
||||
r = await client.post(
|
||||
"/api/v1/auth/login",
|
||||
json={"username": name, "password": password},
|
||||
headers={"Origin": "http://127.0.0.1:58000"},
|
||||
)
|
||||
assert r.status_code == 200, r.text
|
||||
return {
|
||||
"Origin": "http://127.0.0.1:58000",
|
||||
"X-CSRF-Token": client.cookies["kf_csrf"],
|
||||
}
|
||||
|
||||
student_headers = await login("web_student")
|
||||
assert (await client.get(prefix + "/capabilities")).status_code == 403
|
||||
assert (
|
||||
await client.post(
|
||||
prefix,
|
||||
json={
|
||||
"operation_id": str(uuid4()),
|
||||
"archive_digest": "a" * 64,
|
||||
"archive_bytes": 100,
|
||||
},
|
||||
headers=student_headers,
|
||||
)
|
||||
).status_code == 403
|
||||
headers = await login("web_admin")
|
||||
assert (await client.get(prefix + "/capabilities")).json()[
|
||||
"uploads_available"
|
||||
] is True
|
||||
for invalid in (
|
||||
{},
|
||||
{"Origin": "http://127.0.0.1:58000", "X-CSRF-Token": "0" * 64},
|
||||
headers | {"Origin": "https://evil.invalid"},
|
||||
headers
|
||||
| {"Origin": "https://evil.invalid", "X-Forwarded-Host": "127.0.0.1:58000"},
|
||||
):
|
||||
assert (
|
||||
await client.post(
|
||||
prefix,
|
||||
json={
|
||||
"operation_id": str(uuid4()),
|
||||
"archive_digest": "a" * 64,
|
||||
"archive_bytes": 100,
|
||||
},
|
||||
headers=invalid,
|
||||
)
|
||||
).status_code == 403
|
||||
|
||||
async def create(blob, key=None):
|
||||
key = key or uuid4()
|
||||
body = {
|
||||
"operation_id": str(key),
|
||||
"archive_digest": hashlib.sha256(blob).hexdigest(),
|
||||
"archive_bytes": len(blob),
|
||||
}
|
||||
deadline = time.monotonic() + 15
|
||||
while True:
|
||||
r = await client.post(prefix, json=body, headers=headers)
|
||||
if r.status_code != 503 or time.monotonic() >= deadline:
|
||||
break
|
||||
await asyncio.sleep(0.2)
|
||||
assert r.status_code == 201, r.text
|
||||
assert r.headers["cache-control"] == "no-store"
|
||||
assert (
|
||||
await client.post(prefix, json=body, headers=headers)
|
||||
).json() == r.json()
|
||||
assert (
|
||||
await client.post(
|
||||
prefix, json=body | {"archive_digest": "f" * 64}, headers=headers
|
||||
)
|
||||
).status_code == 409
|
||||
return str(key)
|
||||
|
||||
async def put_blob(blob, key):
|
||||
deadline = time.monotonic() + 30
|
||||
while True:
|
||||
result = await client.put(
|
||||
f"{prefix}/{key}/archive",
|
||||
content=blob,
|
||||
headers=headers | {"Content-Type": "application/zip"},
|
||||
)
|
||||
if result.status_code != 503 or time.monotonic() >= deadline:
|
||||
return result
|
||||
await asyncio.sleep(0.2)
|
||||
|
||||
async def discard_owned(key):
|
||||
deadline = time.monotonic() + 15
|
||||
while True:
|
||||
result = await client.post(f"{prefix}/{key}/discard", headers=headers)
|
||||
if result.status_code != 503 or time.monotonic() >= deadline:
|
||||
return result
|
||||
await asyncio.sleep(0.2)
|
||||
|
||||
async def upload(blob, key):
|
||||
r = await put_blob(blob, key)
|
||||
assert r.status_code == 200, r.text
|
||||
deadline = time.monotonic() + 100
|
||||
while time.monotonic() < deadline:
|
||||
r = await client.get(f"{prefix}/{key}")
|
||||
assert r.status_code == 200, r.text
|
||||
if r.json()["state"] in ("ready", "failed"):
|
||||
return r.json()
|
||||
assert processes["worker"].poll() is None, (
|
||||
tmp_path / "worker.log"
|
||||
).read_text()[-2000:]
|
||||
await asyncio.sleep(0.2)
|
||||
raise AssertionError(
|
||||
"candidate deadline; " + (tmp_path / "worker.log").read_text()[-2000:]
|
||||
)
|
||||
|
||||
async def wait_released(key, state, code=None, seconds=120):
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
row = await c.fetchrow(
|
||||
"SELECT state,failure_code,reserved_bytes,attempts FROM "
|
||||
"problem_import_operations WHERE id=$1",
|
||||
UUID(key),
|
||||
)
|
||||
if row["state"] == state and row["reserved_bytes"] == 0:
|
||||
if code:
|
||||
assert row["failure_code"] == code, dict(row)
|
||||
return row
|
||||
await asyncio.sleep(0.1)
|
||||
raise AssertionError("operation cleanup deadline: " + str(dict(row)))
|
||||
|
||||
async def restart_ready_worker():
|
||||
previous = await c.fetchval(
|
||||
"SELECT worker_ready_until FROM problem_import_pool WHERE id=1"
|
||||
)
|
||||
assert previous is not None
|
||||
start("worker")
|
||||
deadline = time.monotonic() + 120
|
||||
while time.monotonic() < deadline:
|
||||
assert processes["worker"].poll() is None
|
||||
if await c.fetchval(
|
||||
"SELECT worker_ready_until>clock_timestamp()+interval '80 seconds' "
|
||||
"AND worker_ready_until>$1 FROM problem_import_pool WHERE id=1",
|
||||
previous,
|
||||
):
|
||||
return
|
||||
await asyncio.sleep(0.1)
|
||||
raise AssertionError("fresh worker heartbeat deadline")
|
||||
|
||||
async def raw_body(key, declared, data, *, delay=0, disconnect=False):
|
||||
reader, writer = await asyncio.open_connection("127.0.0.1", 58000)
|
||||
cookie = "; ".join(f"{k}={v}" for k, v in client.cookies.items())
|
||||
request = (
|
||||
f"PUT {prefix}/{key}/archive HTTP/1.1\r\n"
|
||||
"Host: 127.0.0.1:58000\r\nOrigin: http://127.0.0.1:58000\r\n"
|
||||
f"Cookie: {cookie}\r\nX-CSRF-Token: {headers['X-CSRF-Token']}\r\n"
|
||||
f"Content-Type: application/zip\r\nContent-Length: {declared}\r\n"
|
||||
"Connection: close\r\n\r\n"
|
||||
).encode()
|
||||
try:
|
||||
writer.write(request + data)
|
||||
await writer.drain()
|
||||
if delay:
|
||||
await asyncio.sleep(delay)
|
||||
if disconnect:
|
||||
return None
|
||||
return await asyncio.wait_for(reader.read(8192), 45)
|
||||
finally:
|
||||
writer.close()
|
||||
await writer.wait_closed()
|
||||
|
||||
for multi, method in ((False, zipfile.ZIP_STORED), (True, zipfile.ZIP_DEFLATED)):
|
||||
blob, payload = package(policy, multi=multi, method=method)
|
||||
key = await create(blob)
|
||||
detail = await upload(blob, key)
|
||||
assert detail["state"] == "ready", (detail["failure_code"], detail["state"])
|
||||
print("ACTUAL_READY_IMPORT", multi, method, detail["candidate_id"], flush=True)
|
||||
await wait_released(key, "ready")
|
||||
retry = await put_blob(blob, key)
|
||||
assert retry.status_code == 200
|
||||
different = blob[:-1] + bytes([blob[-1] ^ 1])
|
||||
conflict = await put_blob(different, key)
|
||||
assert conflict.status_code == 409
|
||||
assert (await client.get(f"{prefix}/{key}")).json()["candidate_id"] == detail[
|
||||
"candidate_id"
|
||||
]
|
||||
assert detail["candidate"]["judging_schema_version"] == (2 if multi else 1)
|
||||
assert detail["candidate"]["case_count"] == 1
|
||||
assert len(detail["candidate"]["languages"]) == (4 if multi else 2)
|
||||
assert "synthetic-hidden" not in json.dumps(detail) and str(
|
||||
tmp_path
|
||||
) not in json.dumps(detail)
|
||||
# Actual database enforcement, not trigger inspection alone.
|
||||
worker_db = await connect(
|
||||
make_url(postgres_url)
|
||||
.set(username=roles["worker"])
|
||||
.render_as_string(hide_password=False)
|
||||
)
|
||||
try:
|
||||
from asyncpg import CheckViolationError, InsufficientPrivilegeError
|
||||
|
||||
with pytest.raises(InsufficientPrivilegeError):
|
||||
await worker_db.execute(
|
||||
"UPDATE problem_candidates SET preview=preview WHERE id=$1",
|
||||
UUID(detail["candidate_id"]),
|
||||
)
|
||||
with pytest.raises(CheckViolationError):
|
||||
await worker_db.execute(
|
||||
"DELETE FROM problem_candidates WHERE id=$1",
|
||||
UUID(detail["candidate_id"]),
|
||||
)
|
||||
with pytest.raises(CheckViolationError):
|
||||
await worker_db.execute(
|
||||
"""INSERT INTO problem_candidate_cases
|
||||
(candidate_id,position,key,input_digest,input_bytes,expected_digest,expected_bytes)
|
||||
SELECT candidate_id,1,'late',input_digest,input_bytes,
|
||||
expected_digest,expected_bytes
|
||||
FROM problem_candidate_cases WHERE candidate_id=$1""",
|
||||
UUID(detail["candidate_id"]),
|
||||
)
|
||||
finally:
|
||||
await worker_db.close()
|
||||
refs = await c.fetch(
|
||||
"SELECT input_digest,input_bytes,expected_digest,expected_bytes "
|
||||
"FROM problem_candidate_cases WHERE candidate_id=$1",
|
||||
UUID(detail["candidate_id"]),
|
||||
)
|
||||
assert len(refs) == 1
|
||||
for field, name in (("input", "hidden.in"), ("expected", "hidden.out")):
|
||||
digest = refs[0][field + "_digest"]
|
||||
data = (artifact / digest).read_bytes()
|
||||
assert data == payload[name] and hashlib.sha256(data).hexdigest() == digest
|
||||
assert await c.fetchval("SELECT count(*) FROM problem_versions") == 0
|
||||
assert (await client.get("/api/v1/problems")).json()["items"] == []
|
||||
assert (
|
||||
await c.fetchval(
|
||||
"SELECT reserved_bytes FROM problem_import_pool WHERE id=1"
|
||||
)
|
||||
== 0
|
||||
)
|
||||
assert sorted(p.name for p in staging.iterdir()) == [".lock"]
|
||||
|
||||
def reuse_public_member(payload):
|
||||
declaration = json.loads(payload["problem.json"])
|
||||
declaration["samples"][0]["output_file"] = "sample.in"
|
||||
payload["problem.json"] = json.dumps(declaration).encode()
|
||||
del payload["sample.out"]
|
||||
|
||||
# Actual child consumes malformed data but never executes contents.
|
||||
for mutation in (
|
||||
lambda p: p.update({"../escape": b"bad"}),
|
||||
lambda p: p.update({"unknown.bin": b"bad"}),
|
||||
lambda p: p.update({"hidden.in": b"PK\x03\x04" + b"bad"}),
|
||||
lambda p: p.update({"sample.in": p["hidden.in"]}),
|
||||
reuse_public_member,
|
||||
):
|
||||
blob, _ = package(policy, payload_mutation=mutation)
|
||||
key = await create(blob)
|
||||
result = await upload(blob, key)
|
||||
# Publicly declared samples can happen to equal hidden bytes; declarations,
|
||||
# not byte equality, decide visibility. This last finite case is legal.
|
||||
if b"PK\x03\x04bad" in blob or b"../escape" in blob or b"unknown.bin" in blob:
|
||||
assert result["state"] == "failed" and result["candidate"] is None, result
|
||||
else:
|
||||
assert result["state"] == "ready", result
|
||||
await wait_released(key, result["state"])
|
||||
assert (
|
||||
await c.fetchval(
|
||||
"SELECT reserved_bytes FROM problem_import_pool WHERE id=1"
|
||||
)
|
||||
== 0
|
||||
)
|
||||
blob, _ = package(policy)
|
||||
key = await create(blob)
|
||||
stop("worker")
|
||||
occupied = ImportStaging(staging)
|
||||
try:
|
||||
with occupied.exclusive():
|
||||
mismatch = await client.put(
|
||||
f"{prefix}/{key}/archive",
|
||||
content=blob[:-1],
|
||||
headers=headers | {"Content-Type": "application/zip"},
|
||||
)
|
||||
assert mismatch.status_code == 409
|
||||
finally:
|
||||
occupied.close()
|
||||
assert (await discard_owned(key)).status_code == 200
|
||||
# Restart after receipt before consumption preserves the immutable operation.
|
||||
stop("worker")
|
||||
key = await create(blob)
|
||||
r = await client.put(
|
||||
f"{prefix}/{key}/archive",
|
||||
content=blob,
|
||||
headers=headers | {"Content-Type": "application/zip"},
|
||||
)
|
||||
assert r.status_code == 200 and r.json()["state"] == "uploaded"
|
||||
start("worker")
|
||||
deadline = time.monotonic() + 120
|
||||
while time.monotonic() < deadline:
|
||||
detail = (await client.get(f"{prefix}/{key}")).json()
|
||||
if detail["state"] in ("ready", "failed"):
|
||||
break
|
||||
await asyncio.sleep(0.2)
|
||||
assert detail["state"] == "ready", (detail["failure_code"], detail["state"])
|
||||
assert (
|
||||
await c.fetchval(
|
||||
"SELECT attempts FROM problem_import_operations WHERE id=$1",
|
||||
UUID(key),
|
||||
)
|
||||
== 1
|
||||
)
|
||||
await wait_released(key, "ready")
|
||||
# A second real administrator cannot inspect, upload or discard another's candidate.
|
||||
headers = await login("web_backup")
|
||||
stop("worker")
|
||||
occupied = ImportStaging(staging)
|
||||
try:
|
||||
with occupied.exclusive():
|
||||
assert (await client.get(f"{prefix}/{key}")).status_code == 404
|
||||
assert (
|
||||
await client.post(f"{prefix}/{key}/discard", headers=headers)
|
||||
).status_code == 404
|
||||
assert (
|
||||
await client.put(
|
||||
f"{prefix}/{key}/archive",
|
||||
content=blob,
|
||||
headers=headers | {"Content-Type": "application/zip"},
|
||||
)
|
||||
).status_code == 404
|
||||
finally:
|
||||
occupied.close()
|
||||
start("worker")
|
||||
headers = await login("web_admin")
|
||||
# Incomplete TCP body and actual 30-second streaming deadline free reservations.
|
||||
interrupted = await create(blob)
|
||||
stop("worker") # Exercise the upload path without a competing GC root lock.
|
||||
await raw_body(interrupted, len(blob), blob[:8], disconnect=True)
|
||||
await wait_released(interrupted, "failed", "upload_interrupted")
|
||||
await restart_ready_worker()
|
||||
slow = await create(blob)
|
||||
stop("worker")
|
||||
response = await raw_body(slow, len(blob), blob[:8], delay=33)
|
||||
await wait_released(slow, "failed", "upload_deadline")
|
||||
assert response.startswith(b"HTTP/1.1 408"), response[:200]
|
||||
await restart_ready_worker()
|
||||
# A full received body with the wrong SHA is never acknowledged as an upload.
|
||||
wrong = await create(blob)
|
||||
stop("worker")
|
||||
r = await client.put(
|
||||
f"{prefix}/{wrong}/archive",
|
||||
content=blob[:-1] + b"!",
|
||||
headers=headers | {"Content-Type": "application/zip"},
|
||||
)
|
||||
assert r.status_code == 409
|
||||
await wait_released(wrong, "failed", "digest_mismatch")
|
||||
await restart_ready_worker()
|
||||
# Two actual reservations fill the global staging pool with Worker stopped.
|
||||
stop("worker")
|
||||
queued = []
|
||||
for _ in range(2):
|
||||
queued_key = await create(blob)
|
||||
r = await client.put(
|
||||
f"{prefix}/{queued_key}/archive",
|
||||
content=blob,
|
||||
headers=headers | {"Content-Type": "application/zip"},
|
||||
)
|
||||
assert r.status_code == 200
|
||||
queued.append(queued_key)
|
||||
assert (
|
||||
await c.fetchval("SELECT reserved_bytes FROM problem_import_pool WHERE id=1")
|
||||
== 268435456
|
||||
)
|
||||
r = await client.post(
|
||||
prefix,
|
||||
json={
|
||||
"operation_id": str(uuid4()),
|
||||
"archive_digest": hashlib.sha256(blob).hexdigest(),
|
||||
"archive_bytes": len(blob),
|
||||
},
|
||||
headers=headers,
|
||||
)
|
||||
assert r.status_code == 503
|
||||
for queued_key in queued:
|
||||
assert (
|
||||
await client.post(f"{prefix}/{queued_key}/discard", headers=headers)
|
||||
).status_code == 200
|
||||
assert (
|
||||
await c.fetchval("SELECT reserved_bytes FROM problem_import_pool WHERE id=1")
|
||||
== 0
|
||||
)
|
||||
start("worker")
|
||||
# Kill parent AND parser in their already verified cgroup while a lease is held.
|
||||
# This finite injection tests both retries. No uploaded code is executed.
|
||||
interrupted_parser = await create(blob)
|
||||
r = await put_blob(blob, interrupted_parser)
|
||||
assert r.status_code == 200
|
||||
for expected_attempt in (1, 2):
|
||||
deadline = time.monotonic() + 120
|
||||
while time.monotonic() < deadline:
|
||||
row = await c.fetchrow(
|
||||
"SELECT state,attempts FROM problem_import_operations WHERE id=$1",
|
||||
UUID(interrupted_parser),
|
||||
)
|
||||
parser = []
|
||||
for pid in (
|
||||
groups["import-worker"].joinpath("cgroup.procs").read_text().split()
|
||||
):
|
||||
cmdline = Path(f"/proc/{pid}/cmdline")
|
||||
if (
|
||||
cmdline.exists()
|
||||
and b"kf_oj.workers._archive_child" in cmdline.read_bytes()
|
||||
):
|
||||
parser.append(int(pid))
|
||||
if (
|
||||
row["state"] == "validating"
|
||||
and row["attempts"] == expected_attempt
|
||||
and parser
|
||||
):
|
||||
# Freeze parser, kill parent, then parser. A live child keeps the flock.
|
||||
for pid in parser:
|
||||
os.kill(pid, signal.SIGSTOP)
|
||||
processes["worker"].kill()
|
||||
processes["worker"].wait(timeout=10)
|
||||
await c.execute(
|
||||
"UPDATE problem_import_operations SET lease_until="
|
||||
"clock_timestamp()-interval '1 second' WHERE id=$1",
|
||||
UUID(interrupted_parser),
|
||||
)
|
||||
# A live stopped child retains the real kernel lock after parent death.
|
||||
probe = ImportStaging(staging)
|
||||
try:
|
||||
with pytest.raises(ArtifactError):
|
||||
with probe.exclusive():
|
||||
pass
|
||||
finally:
|
||||
probe.close()
|
||||
denied = await client.post(
|
||||
prefix,
|
||||
json={
|
||||
"operation_id": str(uuid4()),
|
||||
"archive_digest": hashlib.sha256(blob).hexdigest(),
|
||||
"archive_bytes": len(blob),
|
||||
},
|
||||
headers=headers,
|
||||
)
|
||||
assert denied.status_code == 503
|
||||
for pid in parser:
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
stop("worker")
|
||||
break
|
||||
await asyncio.sleep(0.02)
|
||||
else:
|
||||
raise AssertionError("claimed parser injection deadline")
|
||||
# Advance the lease under the VM test operator; requests cannot set clocks.
|
||||
await c.execute(
|
||||
"UPDATE problem_import_operations SET lease_until="
|
||||
"clock_timestamp()-interval '1 second' WHERE id=$1",
|
||||
UUID(interrupted_parser),
|
||||
)
|
||||
start("worker")
|
||||
row = await wait_released(interrupted_parser, "failed", "validation_unavailable")
|
||||
assert row["attempts"] == 2
|
||||
assert (
|
||||
await c.fetchval(
|
||||
"SELECT count(*) FROM problem_candidates WHERE operation_id=$1",
|
||||
UUID(interrupted_parser),
|
||||
)
|
||||
== 0
|
||||
)
|
||||
# Real concurrent claims share the same capped process and use separate lock FDs.
|
||||
stop("worker")
|
||||
paired_key = await create(blob)
|
||||
r = await client.put(
|
||||
f"{prefix}/{paired_key}/archive",
|
||||
content=blob,
|
||||
headers=headers | {"Content-Type": "application/zip"},
|
||||
)
|
||||
assert r.status_code == 200 and r.json()["state"] == "uploaded"
|
||||
start("worker", pair=True)
|
||||
row = await wait_released(paired_key, "ready")
|
||||
assert row["attempts"] == 1
|
||||
assert (
|
||||
await c.fetchval(
|
||||
"SELECT count(*) FROM problem_candidates WHERE operation_id=$1",
|
||||
UUID(paired_key),
|
||||
)
|
||||
== 1
|
||||
)
|
||||
deadline = time.monotonic() + 120
|
||||
while processes["worker"].poll() is None and time.monotonic() < deadline:
|
||||
await asyncio.sleep(0.1)
|
||||
assert processes["worker"].returncode == 0, (tmp_path / "worker.log").read_text()[
|
||||
-1500:
|
||||
]
|
||||
assert (
|
||||
"ACTUAL_CONCURRENT_IMPORT_CLAIM_ACCEPTED"
|
||||
in (tmp_path / "worker.log").read_text()
|
||||
)
|
||||
stop("worker")
|
||||
start("worker")
|
||||
# Insert an old receipt with the test operator; immutable timestamps stay unchanged.
|
||||
expired_id = uuid4()
|
||||
async with c.transaction():
|
||||
await c.execute(
|
||||
"""INSERT INTO problem_import_operations
|
||||
(id,owner_id,authorization_version,archive_digest,archive_bytes,created_at,
|
||||
updated_at,expires_at,lease_until,attempts,fence,reserved_bytes)
|
||||
SELECT $1,$2,authorization_version,$3,$4,
|
||||
clock_timestamp()-interval '2 days',
|
||||
clock_timestamp()-interval '2 days',clock_timestamp()-interval '1 day',
|
||||
clock_timestamp()-interval '1 day',0,0,134217728 FROM accounts WHERE id=$2""",
|
||||
expired_id,
|
||||
users["web_admin"],
|
||||
hashlib.sha256(blob).hexdigest(),
|
||||
len(blob),
|
||||
)
|
||||
assert (
|
||||
await c.fetchval(
|
||||
"SELECT state FROM problem_import_operations WHERE id=$1", expired_id
|
||||
)
|
||||
== "uploading"
|
||||
)
|
||||
await c.execute(
|
||||
"UPDATE problem_import_pool SET reserved_bytes="
|
||||
"reserved_bytes+134217728 WHERE id=1"
|
||||
)
|
||||
await wait_released(str(expired_id), "expired", "expired")
|
||||
print(
|
||||
"ACTUAL_IMPORT_DISCONNECT_DEADLINE_CAPACITY_TWO_RETRIES_EXPIRY_ACCEPTED",
|
||||
flush=True,
|
||||
)
|
||||
# Revocation is live and old cookies cannot read or write management APIs.
|
||||
await c.execute("SELECT * FROM kf_manage_account_authority('web_admin','revoke')")
|
||||
assert (await client.get(prefix + "/capabilities")).status_code == 401
|
||||
assert (
|
||||
await client.post(f"{prefix}/{key}/discard", headers=headers)
|
||||
).status_code == 401
|
||||
await c.execute("SELECT * FROM kf_manage_account_authority('web_admin','grant')")
|
||||
headers = await login("web_admin")
|
||||
stop("worker")
|
||||
discard_after = await c.fetchval("SELECT clock_timestamp()")
|
||||
page = (await client.get(prefix, params={"limit": 100})).json()
|
||||
for entry in page["items"]:
|
||||
assert (
|
||||
await client.post(
|
||||
f"{prefix}/{entry['operation_id']}/discard", headers=headers
|
||||
)
|
||||
).status_code == 200
|
||||
await restart_ready_worker()
|
||||
deadline = time.monotonic() + 30
|
||||
while time.monotonic() < deadline and await c.fetchval(
|
||||
"SELECT count(*) FROM problem_candidates"
|
||||
):
|
||||
await asyncio.sleep(0.1)
|
||||
assert await c.fetchval("SELECT count(*) FROM problem_candidates") == 0
|
||||
assert await c.fetchval("SELECT count(*) FROM problem_candidate_cases") == 0
|
||||
assert (
|
||||
await c.fetchval("SELECT reserved_bytes FROM problem_import_pool WHERE id=1")
|
||||
== 0
|
||||
)
|
||||
assert await c.fetchval("SELECT count(*) FROM problem_maintenance_events") >= 10
|
||||
assert (
|
||||
await c.fetchval(
|
||||
"SELECT bool_and(e.authorization_version=a.authorization_version) "
|
||||
"FROM problem_maintenance_events e JOIN accounts a ON a.id=e.actor_id "
|
||||
"WHERE e.action='import_discarded' AND e.actor_id=$1 "
|
||||
"AND e.created_at >= $2",
|
||||
users["web_admin"],
|
||||
discard_after,
|
||||
)
|
||||
is True
|
||||
)
|
||||
for table in (
|
||||
"evaluations",
|
||||
"judge_attempts",
|
||||
"evaluation_performance",
|
||||
"accounts",
|
||||
"sessions",
|
||||
):
|
||||
for privilege in ("SELECT", "INSERT", "UPDATE", "DELETE"):
|
||||
assert not await c.fetchval(
|
||||
"SELECT has_table_privilege($1,$2,$3)",
|
||||
roles["worker"],
|
||||
table,
|
||||
privilege,
|
||||
)
|
||||
assert not await c.fetchval(
|
||||
"SELECT has_table_privilege($1,'problem_maintenance_events','UPDATE')",
|
||||
roles["api"],
|
||||
)
|
||||
for privilege in ("SELECT", "UPDATE", "DELETE"):
|
||||
assert not await c.fetchval(
|
||||
"SELECT has_table_privilege($1,'problem_maintenance_events',$2)",
|
||||
roles["worker"],
|
||||
privilege,
|
||||
)
|
||||
# Missing configuration is observed through the real server, never preview data.
|
||||
stop("api")
|
||||
start("api", configured=False)
|
||||
deadline = time.monotonic() + 120
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
r = await client.get(prefix + "/capabilities")
|
||||
if r.status_code == 200:
|
||||
break
|
||||
except httpx.ConnectError:
|
||||
pass
|
||||
await asyncio.sleep(0.1)
|
||||
assert r.status_code == 200 and not r.json()["uploads_available"]
|
||||
assert not r.json()["validation_available"]
|
||||
assert (
|
||||
await client.post(
|
||||
prefix,
|
||||
json={
|
||||
"operation_id": str(uuid4()),
|
||||
"archive_digest": hashlib.sha256(blob).hexdigest(),
|
||||
"archive_bytes": len(blob),
|
||||
},
|
||||
headers=headers,
|
||||
)
|
||||
).status_code == 503
|
||||
print("ACTUAL_IMPORT_HTTP_WORKER_ACCEPTED", flush=True)
|
||||
finally:
|
||||
stop("api")
|
||||
stop("worker")
|
||||
for role in roles.values():
|
||||
await c.execute(f"DROP OWNED BY {role}")
|
||||
await c.execute(f"DROP ROLE {role}")
|
||||
await c.close()
|
||||
|
||||
asyncio.run(run())
|
||||
@@ -1,5 +1,6 @@
|
||||
import asyncio
|
||||
import json
|
||||
import subprocess
|
||||
from uuid import uuid4
|
||||
|
||||
import asyncpg
|
||||
@@ -12,7 +13,7 @@ def test_migration_roundtrip_and_role_safety(postgres_url):
|
||||
c = await connect(postgres_url)
|
||||
try:
|
||||
tables = set(await c.fetch("SELECT tablename FROM pg_tables WHERE schemaname='public'"))
|
||||
assert len(tables) == 15 # Fourteen domain/control tables and Alembic history.
|
||||
assert len(tables) == 20 # Nineteen domain/control tables and Alembic history.
|
||||
roles = await c.fetch("SELECT * FROM pg_roles WHERE rolname LIKE 'kf_oj_%'")
|
||||
assert {row["rolname"] for row in roles} == {
|
||||
"kf_oj_platform",
|
||||
@@ -44,7 +45,14 @@ def test_migration_roundtrip_and_role_safety(postgres_url):
|
||||
asyncio.run(inspect())
|
||||
migrate(postgres_url, "downgrade", "base")
|
||||
migrate(postgres_url, "upgrade", "head")
|
||||
try:
|
||||
migrate(postgres_url, "check")
|
||||
except subprocess.CalledProcessError as error:
|
||||
raise AssertionError(
|
||||
"migration model consistency failed: "
|
||||
+ (error.stdout or b"").decode(errors="replace")[-8192:]
|
||||
+ (error.stderr or b"").decode(errors="replace")[-8192:]
|
||||
) from None
|
||||
asyncio.run(inspect())
|
||||
|
||||
|
||||
|
||||
@@ -76,6 +76,94 @@
|
||||
"title": "CancellationAccepted",
|
||||
"type": "object"
|
||||
},
|
||||
"CandidatePreview": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"candidate_id": {
|
||||
"format": "uuid",
|
||||
"title": "Candidate Id",
|
||||
"type": "string"
|
||||
},
|
||||
"case_count": {
|
||||
"maximum": 256.0,
|
||||
"minimum": 1.0,
|
||||
"title": "Case Count",
|
||||
"type": "integer"
|
||||
},
|
||||
"judging_schema_version": {
|
||||
"maximum": 2.0,
|
||||
"minimum": 1.0,
|
||||
"title": "Judging Schema Version",
|
||||
"type": "integer"
|
||||
},
|
||||
"language_configurations": {
|
||||
"default": [],
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/PublicLanguageConfiguration"
|
||||
},
|
||||
"maxItems": 4,
|
||||
"title": "Language Configurations",
|
||||
"type": "array"
|
||||
},
|
||||
"languages": {
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/kf_oj__contracts__judging_v2__Language"
|
||||
},
|
||||
"maxItems": 4,
|
||||
"minItems": 1,
|
||||
"title": "Languages",
|
||||
"type": "array"
|
||||
},
|
||||
"legacy_budget": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/LegacyImportBudget"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"samples": {
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/PublicSample"
|
||||
},
|
||||
"maxItems": 10,
|
||||
"title": "Samples",
|
||||
"type": "array"
|
||||
},
|
||||
"slug": {
|
||||
"pattern": "^[a-z0-9][a-z0-9_-]{0,63}$",
|
||||
"title": "Slug",
|
||||
"type": "string"
|
||||
},
|
||||
"statement": {
|
||||
"maxLength": 262144,
|
||||
"minLength": 1,
|
||||
"title": "Statement",
|
||||
"type": "string"
|
||||
},
|
||||
"title": {
|
||||
"maxLength": 160,
|
||||
"minLength": 1,
|
||||
"title": "Title",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"candidate_id",
|
||||
"slug",
|
||||
"title",
|
||||
"statement",
|
||||
"samples",
|
||||
"judging_schema_version",
|
||||
"languages",
|
||||
"legacy_budget",
|
||||
"case_count"
|
||||
],
|
||||
"title": "CandidatePreview",
|
||||
"type": "object"
|
||||
},
|
||||
"CompilationStatus": {
|
||||
"enum": [
|
||||
"succeeded",
|
||||
@@ -87,6 +175,42 @@
|
||||
"title": "CompilationStatus",
|
||||
"type": "string"
|
||||
},
|
||||
"CreateProblemImport": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"archive_bytes": {
|
||||
"maximum": 67108864.0,
|
||||
"minimum": 1.0,
|
||||
"title": "Archive Bytes",
|
||||
"type": "integer"
|
||||
},
|
||||
"archive_digest": {
|
||||
"maxLength": 64,
|
||||
"minLength": 64,
|
||||
"pattern": "^[0-9a-f]{64}$",
|
||||
"title": "Archive Digest",
|
||||
"type": "string"
|
||||
},
|
||||
"operation_id": {
|
||||
"format": "uuid",
|
||||
"title": "Operation Id",
|
||||
"type": "string"
|
||||
},
|
||||
"schema_version": {
|
||||
"const": 1,
|
||||
"default": 1,
|
||||
"title": "Schema Version",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"operation_id",
|
||||
"archive_digest",
|
||||
"archive_bytes"
|
||||
],
|
||||
"title": "CreateProblemImport",
|
||||
"type": "object"
|
||||
},
|
||||
"CreateSubmission": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
@@ -192,6 +316,38 @@
|
||||
"title": "HealthResponse",
|
||||
"type": "object"
|
||||
},
|
||||
"ImportFailure": {
|
||||
"enum": [
|
||||
"invalid_archive",
|
||||
"invalid_manifest",
|
||||
"invalid_data",
|
||||
"digest_mismatch",
|
||||
"policy_rejected",
|
||||
"upload_interrupted",
|
||||
"upload_deadline",
|
||||
"validation_deadline",
|
||||
"validation_unavailable",
|
||||
"storage_unavailable",
|
||||
"authority_revoked",
|
||||
"expired"
|
||||
],
|
||||
"title": "ImportFailure",
|
||||
"type": "string"
|
||||
},
|
||||
"ImportState": {
|
||||
"enum": [
|
||||
"uploading",
|
||||
"uploaded",
|
||||
"validating",
|
||||
"ready",
|
||||
"failed",
|
||||
"published",
|
||||
"discarded",
|
||||
"expired"
|
||||
],
|
||||
"title": "ImportState",
|
||||
"type": "string"
|
||||
},
|
||||
"JobManifest": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
@@ -284,6 +440,30 @@
|
||||
"title": "Language",
|
||||
"type": "string"
|
||||
},
|
||||
"LegacyImportBudget": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"compile_limits": {
|
||||
"$ref": "#/components/schemas/RunLimits"
|
||||
},
|
||||
"job_wall_time_us": {
|
||||
"maximum": 3600000000.0,
|
||||
"minimum": 1000.0,
|
||||
"title": "Job Wall Time Us",
|
||||
"type": "integer"
|
||||
},
|
||||
"run_limits": {
|
||||
"$ref": "#/components/schemas/RunLimits"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"compile_limits",
|
||||
"run_limits",
|
||||
"job_wall_time_us"
|
||||
],
|
||||
"title": "LegacyImportBudget",
|
||||
"type": "object"
|
||||
},
|
||||
"LoginRequest": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
@@ -575,6 +755,426 @@
|
||||
"title": "ProblemDetail",
|
||||
"type": "object"
|
||||
},
|
||||
"ProblemImportCapabilities": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"limits": {
|
||||
"$ref": "#/components/schemas/ProblemImportLimits",
|
||||
"default": {
|
||||
"archive_bytes": 67108864,
|
||||
"archive_format": "single_level_zip",
|
||||
"candidate_lifetime_seconds": 86400,
|
||||
"chunk_bytes": 1048576,
|
||||
"filename_ascii_bytes": 128,
|
||||
"files": 534,
|
||||
"global_concurrency": 1,
|
||||
"languages": [
|
||||
"c17",
|
||||
"cpp20",
|
||||
"python313",
|
||||
"java21"
|
||||
],
|
||||
"manifest_name": "problem.json",
|
||||
"manifest_schema_versions": [
|
||||
1,
|
||||
2
|
||||
],
|
||||
"owner_candidates": 10,
|
||||
"owner_concurrency": 1,
|
||||
"schema_version": 1,
|
||||
"staging_bytes": 268435456,
|
||||
"total_candidates": 32,
|
||||
"unpacked_bytes": 67108864,
|
||||
"upload_chunks": 4096,
|
||||
"upload_deadline_seconds": 30,
|
||||
"validation_deadline_seconds": 60
|
||||
}
|
||||
},
|
||||
"schema_version": {
|
||||
"const": 1,
|
||||
"default": 1,
|
||||
"title": "Schema Version",
|
||||
"type": "integer"
|
||||
},
|
||||
"uploads_available": {
|
||||
"title": "Uploads Available",
|
||||
"type": "boolean"
|
||||
},
|
||||
"validation_available": {
|
||||
"title": "Validation Available",
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"uploads_available",
|
||||
"validation_available"
|
||||
],
|
||||
"title": "ProblemImportCapabilities",
|
||||
"type": "object"
|
||||
},
|
||||
"ProblemImportDetail": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"archive_bytes": {
|
||||
"maximum": 67108864.0,
|
||||
"minimum": 1.0,
|
||||
"title": "Archive Bytes",
|
||||
"type": "integer"
|
||||
},
|
||||
"candidate": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/CandidatePreview"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"candidate_id": {
|
||||
"anyOf": [
|
||||
{
|
||||
"format": "uuid",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Candidate Id"
|
||||
},
|
||||
"created_at": {
|
||||
"format": "date-time",
|
||||
"title": "Created At",
|
||||
"type": "string"
|
||||
},
|
||||
"expected_archive_digest": {
|
||||
"maxLength": 64,
|
||||
"minLength": 64,
|
||||
"pattern": "^[0-9a-f]{64}$",
|
||||
"title": "Expected Archive Digest",
|
||||
"type": "string"
|
||||
},
|
||||
"expires_at": {
|
||||
"format": "date-time",
|
||||
"title": "Expires At",
|
||||
"type": "string"
|
||||
},
|
||||
"failure_code": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/ImportFailure"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"operation_id": {
|
||||
"format": "uuid",
|
||||
"title": "Operation Id",
|
||||
"type": "string"
|
||||
},
|
||||
"published_version_id": {
|
||||
"anyOf": [
|
||||
{
|
||||
"format": "uuid",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Published Version Id"
|
||||
},
|
||||
"schema_version": {
|
||||
"const": 1,
|
||||
"default": 1,
|
||||
"title": "Schema Version",
|
||||
"type": "integer"
|
||||
},
|
||||
"state": {
|
||||
"$ref": "#/components/schemas/ImportState"
|
||||
},
|
||||
"updated_at": {
|
||||
"format": "date-time",
|
||||
"title": "Updated At",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"operation_id",
|
||||
"state",
|
||||
"expected_archive_digest",
|
||||
"archive_bytes",
|
||||
"created_at",
|
||||
"expires_at",
|
||||
"updated_at",
|
||||
"failure_code",
|
||||
"candidate_id",
|
||||
"published_version_id",
|
||||
"candidate"
|
||||
],
|
||||
"title": "ProblemImportDetail",
|
||||
"type": "object"
|
||||
},
|
||||
"ProblemImportLimits": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"archive_bytes": {
|
||||
"default": 67108864,
|
||||
"title": "Archive Bytes",
|
||||
"type": "integer"
|
||||
},
|
||||
"archive_format": {
|
||||
"const": "single_level_zip",
|
||||
"default": "single_level_zip",
|
||||
"title": "Archive Format",
|
||||
"type": "string"
|
||||
},
|
||||
"candidate_lifetime_seconds": {
|
||||
"default": 86400,
|
||||
"title": "Candidate Lifetime Seconds",
|
||||
"type": "integer"
|
||||
},
|
||||
"chunk_bytes": {
|
||||
"default": 1048576,
|
||||
"title": "Chunk Bytes",
|
||||
"type": "integer"
|
||||
},
|
||||
"filename_ascii_bytes": {
|
||||
"default": 128,
|
||||
"title": "Filename Ascii Bytes",
|
||||
"type": "integer"
|
||||
},
|
||||
"files": {
|
||||
"default": 534,
|
||||
"title": "Files",
|
||||
"type": "integer"
|
||||
},
|
||||
"global_concurrency": {
|
||||
"const": 1,
|
||||
"default": 1,
|
||||
"title": "Global Concurrency",
|
||||
"type": "integer"
|
||||
},
|
||||
"languages": {
|
||||
"default": [
|
||||
"c17",
|
||||
"cpp20",
|
||||
"python313",
|
||||
"java21"
|
||||
],
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/kf_oj__contracts__judging_v2__Language"
|
||||
},
|
||||
"title": "Languages",
|
||||
"type": "array"
|
||||
},
|
||||
"manifest_name": {
|
||||
"const": "problem.json",
|
||||
"default": "problem.json",
|
||||
"title": "Manifest Name",
|
||||
"type": "string"
|
||||
},
|
||||
"manifest_schema_versions": {
|
||||
"default": [
|
||||
1,
|
||||
2
|
||||
],
|
||||
"items": {
|
||||
"enum": [
|
||||
1,
|
||||
2
|
||||
],
|
||||
"type": "integer"
|
||||
},
|
||||
"title": "Manifest Schema Versions",
|
||||
"type": "array"
|
||||
},
|
||||
"owner_candidates": {
|
||||
"default": 10,
|
||||
"title": "Owner Candidates",
|
||||
"type": "integer"
|
||||
},
|
||||
"owner_concurrency": {
|
||||
"const": 1,
|
||||
"default": 1,
|
||||
"title": "Owner Concurrency",
|
||||
"type": "integer"
|
||||
},
|
||||
"schema_version": {
|
||||
"const": 1,
|
||||
"default": 1,
|
||||
"title": "Schema Version",
|
||||
"type": "integer"
|
||||
},
|
||||
"staging_bytes": {
|
||||
"default": 268435456,
|
||||
"title": "Staging Bytes",
|
||||
"type": "integer"
|
||||
},
|
||||
"total_candidates": {
|
||||
"default": 32,
|
||||
"title": "Total Candidates",
|
||||
"type": "integer"
|
||||
},
|
||||
"unpacked_bytes": {
|
||||
"default": 67108864,
|
||||
"title": "Unpacked Bytes",
|
||||
"type": "integer"
|
||||
},
|
||||
"upload_chunks": {
|
||||
"default": 4096,
|
||||
"title": "Upload Chunks",
|
||||
"type": "integer"
|
||||
},
|
||||
"upload_deadline_seconds": {
|
||||
"default": 30,
|
||||
"title": "Upload Deadline Seconds",
|
||||
"type": "integer"
|
||||
},
|
||||
"validation_deadline_seconds": {
|
||||
"default": 60,
|
||||
"title": "Validation Deadline Seconds",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"title": "ProblemImportLimits",
|
||||
"type": "object"
|
||||
},
|
||||
"ProblemImportPage": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"items": {
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/ProblemImportSummary"
|
||||
},
|
||||
"maxItems": 100,
|
||||
"title": "Items",
|
||||
"type": "array"
|
||||
},
|
||||
"next_after": {
|
||||
"anyOf": [
|
||||
{
|
||||
"format": "uuid",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Next After"
|
||||
},
|
||||
"schema_version": {
|
||||
"const": 1,
|
||||
"default": 1,
|
||||
"title": "Schema Version",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"items",
|
||||
"next_after"
|
||||
],
|
||||
"title": "ProblemImportPage",
|
||||
"type": "object"
|
||||
},
|
||||
"ProblemImportSummary": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"archive_bytes": {
|
||||
"maximum": 67108864.0,
|
||||
"minimum": 1.0,
|
||||
"title": "Archive Bytes",
|
||||
"type": "integer"
|
||||
},
|
||||
"candidate_id": {
|
||||
"anyOf": [
|
||||
{
|
||||
"format": "uuid",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Candidate Id"
|
||||
},
|
||||
"created_at": {
|
||||
"format": "date-time",
|
||||
"title": "Created At",
|
||||
"type": "string"
|
||||
},
|
||||
"expected_archive_digest": {
|
||||
"maxLength": 64,
|
||||
"minLength": 64,
|
||||
"pattern": "^[0-9a-f]{64}$",
|
||||
"title": "Expected Archive Digest",
|
||||
"type": "string"
|
||||
},
|
||||
"expires_at": {
|
||||
"format": "date-time",
|
||||
"title": "Expires At",
|
||||
"type": "string"
|
||||
},
|
||||
"failure_code": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/ImportFailure"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"operation_id": {
|
||||
"format": "uuid",
|
||||
"title": "Operation Id",
|
||||
"type": "string"
|
||||
},
|
||||
"published_version_id": {
|
||||
"anyOf": [
|
||||
{
|
||||
"format": "uuid",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Published Version Id"
|
||||
},
|
||||
"schema_version": {
|
||||
"const": 1,
|
||||
"default": 1,
|
||||
"title": "Schema Version",
|
||||
"type": "integer"
|
||||
},
|
||||
"state": {
|
||||
"$ref": "#/components/schemas/ImportState"
|
||||
},
|
||||
"updated_at": {
|
||||
"format": "date-time",
|
||||
"title": "Updated At",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"operation_id",
|
||||
"state",
|
||||
"expected_archive_digest",
|
||||
"archive_bytes",
|
||||
"created_at",
|
||||
"expires_at",
|
||||
"updated_at",
|
||||
"failure_code",
|
||||
"candidate_id",
|
||||
"published_version_id"
|
||||
],
|
||||
"title": "ProblemImportSummary",
|
||||
"type": "object"
|
||||
},
|
||||
"ProblemPage": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
@@ -1416,6 +2016,268 @@
|
||||
},
|
||||
"openapi": "3.1.0",
|
||||
"paths": {
|
||||
"/api/v1/admin/problem-imports": {
|
||||
"get": {
|
||||
"operationId": "page_api_v1_admin_problem_imports_get",
|
||||
"parameters": [
|
||||
{
|
||||
"in": "query",
|
||||
"name": "after",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"anyOf": [
|
||||
{
|
||||
"format": "uuid",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "After"
|
||||
}
|
||||
},
|
||||
{
|
||||
"in": "query",
|
||||
"name": "limit",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"default": 20,
|
||||
"maximum": 100,
|
||||
"minimum": 1,
|
||||
"title": "Limit",
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ProblemImportPage"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Successful Response"
|
||||
},
|
||||
"422": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/HTTPValidationError"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Validation Error"
|
||||
}
|
||||
},
|
||||
"summary": "Page",
|
||||
"tags": [
|
||||
"problem-maintenance"
|
||||
]
|
||||
},
|
||||
"post": {
|
||||
"operationId": "create_api_v1_admin_problem_imports_post",
|
||||
"requestBody": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/CreateProblemImport"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": true
|
||||
},
|
||||
"responses": {
|
||||
"201": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ProblemImportSummary"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Successful Response"
|
||||
},
|
||||
"422": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/HTTPValidationError"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Validation Error"
|
||||
}
|
||||
},
|
||||
"summary": "Create",
|
||||
"tags": [
|
||||
"problem-maintenance"
|
||||
]
|
||||
}
|
||||
},
|
||||
"/api/v1/admin/problem-imports/capabilities": {
|
||||
"get": {
|
||||
"operationId": "capabilities_api_v1_admin_problem_imports_capabilities_get",
|
||||
"responses": {
|
||||
"200": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ProblemImportCapabilities"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Successful Response"
|
||||
}
|
||||
},
|
||||
"summary": "Capabilities",
|
||||
"tags": [
|
||||
"problem-maintenance"
|
||||
]
|
||||
}
|
||||
},
|
||||
"/api/v1/admin/problem-imports/{operation_id}": {
|
||||
"get": {
|
||||
"operationId": "detail_api_v1_admin_problem_imports__operation_id__get",
|
||||
"parameters": [
|
||||
{
|
||||
"in": "path",
|
||||
"name": "operation_id",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"format": "uuid",
|
||||
"title": "Operation Id",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ProblemImportDetail"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Successful Response"
|
||||
},
|
||||
"422": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/HTTPValidationError"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Validation Error"
|
||||
}
|
||||
},
|
||||
"summary": "Detail",
|
||||
"tags": [
|
||||
"problem-maintenance"
|
||||
]
|
||||
}
|
||||
},
|
||||
"/api/v1/admin/problem-imports/{operation_id}/archive": {
|
||||
"put": {
|
||||
"operationId": "upload_api_v1_admin_problem_imports__operation_id__archive_put",
|
||||
"parameters": [
|
||||
{
|
||||
"in": "path",
|
||||
"name": "operation_id",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"format": "uuid",
|
||||
"title": "Operation Id",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"requestBody": {
|
||||
"content": {
|
||||
"application/zip": {
|
||||
"schema": {
|
||||
"format": "binary",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": true
|
||||
},
|
||||
"responses": {
|
||||
"200": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ProblemImportSummary"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Successful Response"
|
||||
},
|
||||
"422": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/HTTPValidationError"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Validation Error"
|
||||
}
|
||||
},
|
||||
"summary": "Upload",
|
||||
"tags": [
|
||||
"problem-maintenance"
|
||||
]
|
||||
}
|
||||
},
|
||||
"/api/v1/admin/problem-imports/{operation_id}/discard": {
|
||||
"post": {
|
||||
"operationId": "discard_api_v1_admin_problem_imports__operation_id__discard_post",
|
||||
"parameters": [
|
||||
{
|
||||
"in": "path",
|
||||
"name": "operation_id",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"format": "uuid",
|
||||
"title": "Operation Id",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ProblemImportSummary"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Successful Response"
|
||||
},
|
||||
"422": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/HTTPValidationError"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Validation Error"
|
||||
}
|
||||
},
|
||||
"summary": "Discard",
|
||||
"tags": [
|
||||
"problem-maintenance"
|
||||
]
|
||||
}
|
||||
},
|
||||
"/api/v1/auth/login": {
|
||||
"post": {
|
||||
"operationId": "login_api_v1_auth_login_post",
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
{
|
||||
"$defs": {
|
||||
"Language": {
|
||||
"enum": [
|
||||
"c17",
|
||||
"cpp20",
|
||||
"python313",
|
||||
"java21"
|
||||
],
|
||||
"title": "Language",
|
||||
"type": "string"
|
||||
},
|
||||
"ProblemImportLimits": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"archive_bytes": {
|
||||
"default": 67108864,
|
||||
"title": "Archive Bytes",
|
||||
"type": "integer"
|
||||
},
|
||||
"archive_format": {
|
||||
"const": "single_level_zip",
|
||||
"default": "single_level_zip",
|
||||
"title": "Archive Format",
|
||||
"type": "string"
|
||||
},
|
||||
"candidate_lifetime_seconds": {
|
||||
"default": 86400,
|
||||
"title": "Candidate Lifetime Seconds",
|
||||
"type": "integer"
|
||||
},
|
||||
"chunk_bytes": {
|
||||
"default": 1048576,
|
||||
"title": "Chunk Bytes",
|
||||
"type": "integer"
|
||||
},
|
||||
"filename_ascii_bytes": {
|
||||
"default": 128,
|
||||
"title": "Filename Ascii Bytes",
|
||||
"type": "integer"
|
||||
},
|
||||
"files": {
|
||||
"default": 534,
|
||||
"title": "Files",
|
||||
"type": "integer"
|
||||
},
|
||||
"global_concurrency": {
|
||||
"const": 1,
|
||||
"default": 1,
|
||||
"title": "Global Concurrency",
|
||||
"type": "integer"
|
||||
},
|
||||
"languages": {
|
||||
"default": [
|
||||
"c17",
|
||||
"cpp20",
|
||||
"python313",
|
||||
"java21"
|
||||
],
|
||||
"items": {
|
||||
"$ref": "#/$defs/Language"
|
||||
},
|
||||
"title": "Languages",
|
||||
"type": "array"
|
||||
},
|
||||
"manifest_name": {
|
||||
"const": "problem.json",
|
||||
"default": "problem.json",
|
||||
"title": "Manifest Name",
|
||||
"type": "string"
|
||||
},
|
||||
"manifest_schema_versions": {
|
||||
"default": [
|
||||
1,
|
||||
2
|
||||
],
|
||||
"items": {
|
||||
"enum": [
|
||||
1,
|
||||
2
|
||||
],
|
||||
"type": "integer"
|
||||
},
|
||||
"title": "Manifest Schema Versions",
|
||||
"type": "array"
|
||||
},
|
||||
"owner_candidates": {
|
||||
"default": 10,
|
||||
"title": "Owner Candidates",
|
||||
"type": "integer"
|
||||
},
|
||||
"owner_concurrency": {
|
||||
"const": 1,
|
||||
"default": 1,
|
||||
"title": "Owner Concurrency",
|
||||
"type": "integer"
|
||||
},
|
||||
"schema_version": {
|
||||
"const": 1,
|
||||
"default": 1,
|
||||
"title": "Schema Version",
|
||||
"type": "integer"
|
||||
},
|
||||
"staging_bytes": {
|
||||
"default": 268435456,
|
||||
"title": "Staging Bytes",
|
||||
"type": "integer"
|
||||
},
|
||||
"total_candidates": {
|
||||
"default": 32,
|
||||
"title": "Total Candidates",
|
||||
"type": "integer"
|
||||
},
|
||||
"unpacked_bytes": {
|
||||
"default": 67108864,
|
||||
"title": "Unpacked Bytes",
|
||||
"type": "integer"
|
||||
},
|
||||
"upload_chunks": {
|
||||
"default": 4096,
|
||||
"title": "Upload Chunks",
|
||||
"type": "integer"
|
||||
},
|
||||
"upload_deadline_seconds": {
|
||||
"default": 30,
|
||||
"title": "Upload Deadline Seconds",
|
||||
"type": "integer"
|
||||
},
|
||||
"validation_deadline_seconds": {
|
||||
"default": 60,
|
||||
"title": "Validation Deadline Seconds",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"title": "ProblemImportLimits",
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"$id": "urn:kf-oj:problem-import-capabilities:v1",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"limits": {
|
||||
"$ref": "#/$defs/ProblemImportLimits",
|
||||
"default": {
|
||||
"archive_bytes": 67108864,
|
||||
"archive_format": "single_level_zip",
|
||||
"candidate_lifetime_seconds": 86400,
|
||||
"chunk_bytes": 1048576,
|
||||
"filename_ascii_bytes": 128,
|
||||
"files": 534,
|
||||
"global_concurrency": 1,
|
||||
"languages": [
|
||||
"c17",
|
||||
"cpp20",
|
||||
"python313",
|
||||
"java21"
|
||||
],
|
||||
"manifest_name": "problem.json",
|
||||
"manifest_schema_versions": [
|
||||
1,
|
||||
2
|
||||
],
|
||||
"owner_candidates": 10,
|
||||
"owner_concurrency": 1,
|
||||
"schema_version": 1,
|
||||
"staging_bytes": 268435456,
|
||||
"total_candidates": 32,
|
||||
"unpacked_bytes": 67108864,
|
||||
"upload_chunks": 4096,
|
||||
"upload_deadline_seconds": 30,
|
||||
"validation_deadline_seconds": 60
|
||||
}
|
||||
},
|
||||
"schema_version": {
|
||||
"const": 1,
|
||||
"default": 1,
|
||||
"title": "Schema Version",
|
||||
"type": "integer"
|
||||
},
|
||||
"uploads_available": {
|
||||
"title": "Uploads Available",
|
||||
"type": "boolean"
|
||||
},
|
||||
"validation_available": {
|
||||
"title": "Validation Available",
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"uploads_available",
|
||||
"validation_available"
|
||||
],
|
||||
"title": "ProblemImportCapabilities",
|
||||
"type": "object"
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"$id": "urn:kf-oj:problem-import-create:v1",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"archive_bytes": {
|
||||
"maximum": 67108864,
|
||||
"minimum": 1,
|
||||
"title": "Archive Bytes",
|
||||
"type": "integer"
|
||||
},
|
||||
"archive_digest": {
|
||||
"maxLength": 64,
|
||||
"minLength": 64,
|
||||
"pattern": "^[0-9a-f]{64}$",
|
||||
"title": "Archive Digest",
|
||||
"type": "string"
|
||||
},
|
||||
"operation_id": {
|
||||
"format": "uuid",
|
||||
"title": "Operation Id",
|
||||
"type": "string"
|
||||
},
|
||||
"schema_version": {
|
||||
"const": 1,
|
||||
"default": 1,
|
||||
"title": "Schema Version",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"operation_id",
|
||||
"archive_digest",
|
||||
"archive_bytes"
|
||||
],
|
||||
"title": "CreateProblemImport",
|
||||
"type": "object"
|
||||
}
|
||||
@@ -0,0 +1,423 @@
|
||||
{
|
||||
"$defs": {
|
||||
"CandidatePreview": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"candidate_id": {
|
||||
"format": "uuid",
|
||||
"title": "Candidate Id",
|
||||
"type": "string"
|
||||
},
|
||||
"case_count": {
|
||||
"maximum": 256,
|
||||
"minimum": 1,
|
||||
"title": "Case Count",
|
||||
"type": "integer"
|
||||
},
|
||||
"judging_schema_version": {
|
||||
"maximum": 2,
|
||||
"minimum": 1,
|
||||
"title": "Judging Schema Version",
|
||||
"type": "integer"
|
||||
},
|
||||
"language_configurations": {
|
||||
"default": [],
|
||||
"items": {
|
||||
"$ref": "#/$defs/PublicLanguageConfiguration"
|
||||
},
|
||||
"maxItems": 4,
|
||||
"title": "Language Configurations",
|
||||
"type": "array"
|
||||
},
|
||||
"languages": {
|
||||
"items": {
|
||||
"$ref": "#/$defs/Language"
|
||||
},
|
||||
"maxItems": 4,
|
||||
"minItems": 1,
|
||||
"title": "Languages",
|
||||
"type": "array"
|
||||
},
|
||||
"legacy_budget": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/$defs/LegacyImportBudget"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"samples": {
|
||||
"items": {
|
||||
"$ref": "#/$defs/PublicSample"
|
||||
},
|
||||
"maxItems": 10,
|
||||
"title": "Samples",
|
||||
"type": "array"
|
||||
},
|
||||
"slug": {
|
||||
"pattern": "^[a-z0-9][a-z0-9_-]{0,63}$",
|
||||
"title": "Slug",
|
||||
"type": "string"
|
||||
},
|
||||
"statement": {
|
||||
"maxLength": 262144,
|
||||
"minLength": 1,
|
||||
"title": "Statement",
|
||||
"type": "string"
|
||||
},
|
||||
"title": {
|
||||
"maxLength": 160,
|
||||
"minLength": 1,
|
||||
"title": "Title",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"candidate_id",
|
||||
"slug",
|
||||
"title",
|
||||
"statement",
|
||||
"samples",
|
||||
"judging_schema_version",
|
||||
"languages",
|
||||
"legacy_budget",
|
||||
"case_count"
|
||||
],
|
||||
"title": "CandidatePreview",
|
||||
"type": "object"
|
||||
},
|
||||
"ImportFailure": {
|
||||
"enum": [
|
||||
"invalid_archive",
|
||||
"invalid_manifest",
|
||||
"invalid_data",
|
||||
"digest_mismatch",
|
||||
"policy_rejected",
|
||||
"upload_interrupted",
|
||||
"upload_deadline",
|
||||
"validation_deadline",
|
||||
"validation_unavailable",
|
||||
"storage_unavailable",
|
||||
"authority_revoked",
|
||||
"expired"
|
||||
],
|
||||
"title": "ImportFailure",
|
||||
"type": "string"
|
||||
},
|
||||
"ImportState": {
|
||||
"enum": [
|
||||
"uploading",
|
||||
"uploaded",
|
||||
"validating",
|
||||
"ready",
|
||||
"failed",
|
||||
"published",
|
||||
"discarded",
|
||||
"expired"
|
||||
],
|
||||
"title": "ImportState",
|
||||
"type": "string"
|
||||
},
|
||||
"Language": {
|
||||
"enum": [
|
||||
"c17",
|
||||
"cpp20",
|
||||
"python313",
|
||||
"java21"
|
||||
],
|
||||
"title": "Language",
|
||||
"type": "string"
|
||||
},
|
||||
"LegacyImportBudget": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"compile_limits": {
|
||||
"$ref": "#/$defs/RunLimits"
|
||||
},
|
||||
"job_wall_time_us": {
|
||||
"maximum": 3600000000,
|
||||
"minimum": 1000,
|
||||
"title": "Job Wall Time Us",
|
||||
"type": "integer"
|
||||
},
|
||||
"run_limits": {
|
||||
"$ref": "#/$defs/RunLimits"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"compile_limits",
|
||||
"run_limits",
|
||||
"job_wall_time_us"
|
||||
],
|
||||
"title": "LegacyImportBudget",
|
||||
"type": "object"
|
||||
},
|
||||
"PreparationKind": {
|
||||
"enum": [
|
||||
"compile",
|
||||
"syntax_check"
|
||||
],
|
||||
"title": "PreparationKind",
|
||||
"type": "string"
|
||||
},
|
||||
"PublicLanguageConfiguration": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"compile_limits": {
|
||||
"$ref": "#/$defs/RunLimits"
|
||||
},
|
||||
"entrypoint": {
|
||||
"title": "Entrypoint",
|
||||
"type": "string"
|
||||
},
|
||||
"job_wall_time_us": {
|
||||
"title": "Job Wall Time Us",
|
||||
"type": "integer"
|
||||
},
|
||||
"language": {
|
||||
"$ref": "#/$defs/Language"
|
||||
},
|
||||
"preparation_kind": {
|
||||
"$ref": "#/$defs/PreparationKind"
|
||||
},
|
||||
"run_limits": {
|
||||
"$ref": "#/$defs/RunLimits"
|
||||
},
|
||||
"runtime_version": {
|
||||
"title": "Runtime Version",
|
||||
"type": "string"
|
||||
},
|
||||
"source_name": {
|
||||
"title": "Source Name",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"language",
|
||||
"runtime_version",
|
||||
"source_name",
|
||||
"entrypoint",
|
||||
"preparation_kind",
|
||||
"compile_limits",
|
||||
"run_limits",
|
||||
"job_wall_time_us"
|
||||
],
|
||||
"title": "PublicLanguageConfiguration",
|
||||
"type": "object"
|
||||
},
|
||||
"PublicSample": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"encoding": {
|
||||
"const": "base64",
|
||||
"default": "base64",
|
||||
"title": "Encoding",
|
||||
"type": "string"
|
||||
},
|
||||
"input": {
|
||||
"maxLength": 87384,
|
||||
"title": "Input",
|
||||
"type": "string"
|
||||
},
|
||||
"output": {
|
||||
"maxLength": 87384,
|
||||
"title": "Output",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"input",
|
||||
"output"
|
||||
],
|
||||
"title": "PublicSample",
|
||||
"type": "object"
|
||||
},
|
||||
"RunLimits": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"cpu_time_us": {
|
||||
"maximum": 60000000,
|
||||
"minimum": 1000,
|
||||
"title": "Cpu Time Us",
|
||||
"type": "integer"
|
||||
},
|
||||
"file_bytes": {
|
||||
"maximum": 67108864,
|
||||
"minimum": 1,
|
||||
"title": "File Bytes",
|
||||
"type": "integer"
|
||||
},
|
||||
"memory_bytes": {
|
||||
"maximum": 2147483648,
|
||||
"minimum": 1,
|
||||
"title": "Memory Bytes",
|
||||
"type": "integer"
|
||||
},
|
||||
"open_files": {
|
||||
"maximum": 1024,
|
||||
"minimum": 3,
|
||||
"title": "Open Files",
|
||||
"type": "integer"
|
||||
},
|
||||
"processes": {
|
||||
"maximum": 128,
|
||||
"minimum": 1,
|
||||
"title": "Processes",
|
||||
"type": "integer"
|
||||
},
|
||||
"stack_bytes": {
|
||||
"maximum": 268435456,
|
||||
"minimum": 1,
|
||||
"title": "Stack Bytes",
|
||||
"type": "integer"
|
||||
},
|
||||
"stderr_bytes": {
|
||||
"maximum": 16777216,
|
||||
"minimum": 1,
|
||||
"title": "Stderr Bytes",
|
||||
"type": "integer"
|
||||
},
|
||||
"stdout_bytes": {
|
||||
"maximum": 16777216,
|
||||
"minimum": 1,
|
||||
"title": "Stdout Bytes",
|
||||
"type": "integer"
|
||||
},
|
||||
"wall_time_us": {
|
||||
"maximum": 120000000,
|
||||
"minimum": 1000,
|
||||
"title": "Wall Time Us",
|
||||
"type": "integer"
|
||||
},
|
||||
"workdir_bytes": {
|
||||
"maximum": 268435456,
|
||||
"minimum": 1,
|
||||
"title": "Workdir Bytes",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"cpu_time_us",
|
||||
"wall_time_us",
|
||||
"memory_bytes",
|
||||
"stack_bytes",
|
||||
"processes",
|
||||
"open_files",
|
||||
"stdout_bytes",
|
||||
"stderr_bytes",
|
||||
"file_bytes",
|
||||
"workdir_bytes"
|
||||
],
|
||||
"title": "RunLimits",
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"$id": "urn:kf-oj:problem-import:v1",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"archive_bytes": {
|
||||
"maximum": 67108864,
|
||||
"minimum": 1,
|
||||
"title": "Archive Bytes",
|
||||
"type": "integer"
|
||||
},
|
||||
"candidate": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/$defs/CandidatePreview"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"candidate_id": {
|
||||
"anyOf": [
|
||||
{
|
||||
"format": "uuid",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Candidate Id"
|
||||
},
|
||||
"created_at": {
|
||||
"format": "date-time",
|
||||
"title": "Created At",
|
||||
"type": "string"
|
||||
},
|
||||
"expected_archive_digest": {
|
||||
"maxLength": 64,
|
||||
"minLength": 64,
|
||||
"pattern": "^[0-9a-f]{64}$",
|
||||
"title": "Expected Archive Digest",
|
||||
"type": "string"
|
||||
},
|
||||
"expires_at": {
|
||||
"format": "date-time",
|
||||
"title": "Expires At",
|
||||
"type": "string"
|
||||
},
|
||||
"failure_code": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/$defs/ImportFailure"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"operation_id": {
|
||||
"format": "uuid",
|
||||
"title": "Operation Id",
|
||||
"type": "string"
|
||||
},
|
||||
"published_version_id": {
|
||||
"anyOf": [
|
||||
{
|
||||
"format": "uuid",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Published Version Id"
|
||||
},
|
||||
"schema_version": {
|
||||
"const": 1,
|
||||
"default": 1,
|
||||
"title": "Schema Version",
|
||||
"type": "integer"
|
||||
},
|
||||
"state": {
|
||||
"$ref": "#/$defs/ImportState"
|
||||
},
|
||||
"updated_at": {
|
||||
"format": "date-time",
|
||||
"title": "Updated At",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"operation_id",
|
||||
"state",
|
||||
"expected_archive_digest",
|
||||
"archive_bytes",
|
||||
"created_at",
|
||||
"expires_at",
|
||||
"updated_at",
|
||||
"failure_code",
|
||||
"candidate_id",
|
||||
"published_version_id",
|
||||
"candidate"
|
||||
],
|
||||
"title": "ProblemImportDetail",
|
||||
"type": "object"
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
[Unit]
|
||||
Description=KF-OnlineJudge bounded problem import service
|
||||
After=postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=exec
|
||||
User=kf-oj
|
||||
Group=kf-oj
|
||||
WorkingDirectory=/opt/kf-onlinejudge/backend
|
||||
EnvironmentFile=/etc/kf-onlinejudge/import-worker.env
|
||||
Environment=KF_OJ_IMPORT_WORKER_CGROUP=/sys/fs/cgroup/system.slice/kf-problem-import.service
|
||||
ExecStart=/opt/kf-onlinejudge/backend/.venv/bin/kf-problem-import-worker
|
||||
MemoryMax=128M
|
||||
MemorySwapMax=0
|
||||
TasksMax=16
|
||||
CPUQuota=100%
|
||||
LimitFSIZE=64M
|
||||
LimitNOFILE=64
|
||||
LimitCORE=0
|
||||
NoNewPrivileges=yes
|
||||
PrivateNetwork=yes
|
||||
RestrictAddressFamilies=AF_UNIX
|
||||
ProtectSystem=strict
|
||||
ProtectHome=yes
|
||||
ReadWritePaths=/var/lib/kf-onlinejudge/imports /var/lib/kf-onlinejudge/artifacts
|
||||
KillMode=control-group
|
||||
TimeoutStopSec=10
|
||||
Restart=on-failure
|
||||
RestartSec=30
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
+579
@@ -4,6 +4,92 @@
|
||||
*/
|
||||
|
||||
export interface paths {
|
||||
"/api/v1/admin/problem-imports": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
/** Page */
|
||||
get: operations["page_api_v1_admin_problem_imports_get"];
|
||||
put?: never;
|
||||
/** Create */
|
||||
post: operations["create_api_v1_admin_problem_imports_post"];
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/api/v1/admin/problem-imports/capabilities": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
/** Capabilities */
|
||||
get: operations["capabilities_api_v1_admin_problem_imports_capabilities_get"];
|
||||
put?: never;
|
||||
post?: never;
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/api/v1/admin/problem-imports/{operation_id}": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
/** Detail */
|
||||
get: operations["detail_api_v1_admin_problem_imports__operation_id__get"];
|
||||
put?: never;
|
||||
post?: never;
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/api/v1/admin/problem-imports/{operation_id}/archive": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get?: never;
|
||||
/** Upload */
|
||||
put: operations["upload_api_v1_admin_problem_imports__operation_id__archive_put"];
|
||||
post?: never;
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/api/v1/admin/problem-imports/{operation_id}/discard": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
/** Discard */
|
||||
post: operations["discard_api_v1_admin_problem_imports__operation_id__discard_post"];
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/api/v1/auth/login": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
@@ -278,11 +364,57 @@ export interface components {
|
||||
*/
|
||||
submission_id: string;
|
||||
};
|
||||
/** CandidatePreview */
|
||||
CandidatePreview: {
|
||||
/**
|
||||
* Candidate Id
|
||||
* Format: uuid
|
||||
*/
|
||||
candidate_id: string;
|
||||
/** Case Count */
|
||||
case_count: number;
|
||||
/** Judging Schema Version */
|
||||
judging_schema_version: number;
|
||||
/**
|
||||
* Language Configurations
|
||||
* @default []
|
||||
*/
|
||||
language_configurations: components["schemas"]["PublicLanguageConfiguration"][];
|
||||
/** Languages */
|
||||
languages: components["schemas"]["kf_oj__contracts__judging_v2__Language"][];
|
||||
legacy_budget: components["schemas"]["LegacyImportBudget"] | null;
|
||||
/** Samples */
|
||||
samples: components["schemas"]["PublicSample"][];
|
||||
/** Slug */
|
||||
slug: string;
|
||||
/** Statement */
|
||||
statement: string;
|
||||
/** Title */
|
||||
title: string;
|
||||
};
|
||||
/**
|
||||
* CompilationStatus
|
||||
* @enum {string}
|
||||
*/
|
||||
CompilationStatus: "succeeded" | "compilation_error" | "compilation_limit" | "cancelled" | "system_error";
|
||||
/** CreateProblemImport */
|
||||
CreateProblemImport: {
|
||||
/** Archive Bytes */
|
||||
archive_bytes: number;
|
||||
/** Archive Digest */
|
||||
archive_digest: string;
|
||||
/**
|
||||
* Operation Id
|
||||
* Format: uuid
|
||||
*/
|
||||
operation_id: string;
|
||||
/**
|
||||
* Schema Version
|
||||
* @default 1
|
||||
* @constant
|
||||
*/
|
||||
schema_version: 1;
|
||||
};
|
||||
/** CreateSubmission */
|
||||
CreateSubmission: {
|
||||
/**
|
||||
@@ -335,6 +467,16 @@ export interface components {
|
||||
/** Version */
|
||||
version: string;
|
||||
};
|
||||
/**
|
||||
* ImportFailure
|
||||
* @enum {string}
|
||||
*/
|
||||
ImportFailure: "invalid_archive" | "invalid_manifest" | "invalid_data" | "digest_mismatch" | "policy_rejected" | "upload_interrupted" | "upload_deadline" | "validation_deadline" | "validation_unavailable" | "storage_unavailable" | "authority_revoked" | "expired";
|
||||
/**
|
||||
* ImportState
|
||||
* @enum {string}
|
||||
*/
|
||||
ImportState: "uploading" | "uploaded" | "validating" | "ready" | "failed" | "published" | "discarded" | "expired";
|
||||
/** JobManifest */
|
||||
JobManifest: {
|
||||
/** Cases */
|
||||
@@ -382,6 +524,13 @@ export interface components {
|
||||
* @enum {string}
|
||||
*/
|
||||
"Language-Input": "c17" | "cpp20" | "python313" | "java21";
|
||||
/** LegacyImportBudget */
|
||||
LegacyImportBudget: {
|
||||
compile_limits: components["schemas"]["RunLimits"];
|
||||
/** Job Wall Time Us */
|
||||
job_wall_time_us: number;
|
||||
run_limits: components["schemas"]["RunLimits"];
|
||||
};
|
||||
/** LoginRequest */
|
||||
LoginRequest: {
|
||||
/**
|
||||
@@ -476,6 +625,254 @@ export interface components {
|
||||
*/
|
||||
version_id: string;
|
||||
};
|
||||
/** ProblemImportCapabilities */
|
||||
ProblemImportCapabilities: {
|
||||
/**
|
||||
* @default {
|
||||
* "archive_bytes": 67108864,
|
||||
* "archive_format": "single_level_zip",
|
||||
* "candidate_lifetime_seconds": 86400,
|
||||
* "chunk_bytes": 1048576,
|
||||
* "filename_ascii_bytes": 128,
|
||||
* "files": 534,
|
||||
* "global_concurrency": 1,
|
||||
* "languages": [
|
||||
* "c17",
|
||||
* "cpp20",
|
||||
* "python313",
|
||||
* "java21"
|
||||
* ],
|
||||
* "manifest_name": "problem.json",
|
||||
* "manifest_schema_versions": [
|
||||
* 1,
|
||||
* 2
|
||||
* ],
|
||||
* "owner_candidates": 10,
|
||||
* "owner_concurrency": 1,
|
||||
* "schema_version": 1,
|
||||
* "staging_bytes": 268435456,
|
||||
* "total_candidates": 32,
|
||||
* "unpacked_bytes": 67108864,
|
||||
* "upload_chunks": 4096,
|
||||
* "upload_deadline_seconds": 30,
|
||||
* "validation_deadline_seconds": 60
|
||||
* }
|
||||
*/
|
||||
limits: components["schemas"]["ProblemImportLimits"];
|
||||
/**
|
||||
* Schema Version
|
||||
* @default 1
|
||||
* @constant
|
||||
*/
|
||||
schema_version: 1;
|
||||
/** Uploads Available */
|
||||
uploads_available: boolean;
|
||||
/** Validation Available */
|
||||
validation_available: boolean;
|
||||
};
|
||||
/** ProblemImportDetail */
|
||||
ProblemImportDetail: {
|
||||
/** Archive Bytes */
|
||||
archive_bytes: number;
|
||||
candidate: components["schemas"]["CandidatePreview"] | null;
|
||||
/** Candidate Id */
|
||||
candidate_id: string | null;
|
||||
/**
|
||||
* Created At
|
||||
* Format: date-time
|
||||
*/
|
||||
created_at: string;
|
||||
/** Expected Archive Digest */
|
||||
expected_archive_digest: string;
|
||||
/**
|
||||
* Expires At
|
||||
* Format: date-time
|
||||
*/
|
||||
expires_at: string;
|
||||
failure_code: components["schemas"]["ImportFailure"] | null;
|
||||
/**
|
||||
* Operation Id
|
||||
* Format: uuid
|
||||
*/
|
||||
operation_id: string;
|
||||
/** Published Version Id */
|
||||
published_version_id: string | null;
|
||||
/**
|
||||
* Schema Version
|
||||
* @default 1
|
||||
* @constant
|
||||
*/
|
||||
schema_version: 1;
|
||||
state: components["schemas"]["ImportState"];
|
||||
/**
|
||||
* Updated At
|
||||
* Format: date-time
|
||||
*/
|
||||
updated_at: string;
|
||||
};
|
||||
/** ProblemImportLimits */
|
||||
ProblemImportLimits: {
|
||||
/**
|
||||
* Archive Bytes
|
||||
* @default 67108864
|
||||
*/
|
||||
archive_bytes: number;
|
||||
/**
|
||||
* Archive Format
|
||||
* @default single_level_zip
|
||||
* @constant
|
||||
*/
|
||||
archive_format: "single_level_zip";
|
||||
/**
|
||||
* Candidate Lifetime Seconds
|
||||
* @default 86400
|
||||
*/
|
||||
candidate_lifetime_seconds: number;
|
||||
/**
|
||||
* Chunk Bytes
|
||||
* @default 1048576
|
||||
*/
|
||||
chunk_bytes: number;
|
||||
/**
|
||||
* Filename Ascii Bytes
|
||||
* @default 128
|
||||
*/
|
||||
filename_ascii_bytes: number;
|
||||
/**
|
||||
* Files
|
||||
* @default 534
|
||||
*/
|
||||
files: number;
|
||||
/**
|
||||
* Global Concurrency
|
||||
* @default 1
|
||||
* @constant
|
||||
*/
|
||||
global_concurrency: 1;
|
||||
/**
|
||||
* Languages
|
||||
* @default [
|
||||
* "c17",
|
||||
* "cpp20",
|
||||
* "python313",
|
||||
* "java21"
|
||||
* ]
|
||||
*/
|
||||
languages: components["schemas"]["kf_oj__contracts__judging_v2__Language"][];
|
||||
/**
|
||||
* Manifest Name
|
||||
* @default problem.json
|
||||
* @constant
|
||||
*/
|
||||
manifest_name: "problem.json";
|
||||
/**
|
||||
* Manifest Schema Versions
|
||||
* @default [
|
||||
* 1,
|
||||
* 2
|
||||
* ]
|
||||
*/
|
||||
manifest_schema_versions: (1 | 2)[];
|
||||
/**
|
||||
* Owner Candidates
|
||||
* @default 10
|
||||
*/
|
||||
owner_candidates: number;
|
||||
/**
|
||||
* Owner Concurrency
|
||||
* @default 1
|
||||
* @constant
|
||||
*/
|
||||
owner_concurrency: 1;
|
||||
/**
|
||||
* Schema Version
|
||||
* @default 1
|
||||
* @constant
|
||||
*/
|
||||
schema_version: 1;
|
||||
/**
|
||||
* Staging Bytes
|
||||
* @default 268435456
|
||||
*/
|
||||
staging_bytes: number;
|
||||
/**
|
||||
* Total Candidates
|
||||
* @default 32
|
||||
*/
|
||||
total_candidates: number;
|
||||
/**
|
||||
* Unpacked Bytes
|
||||
* @default 67108864
|
||||
*/
|
||||
unpacked_bytes: number;
|
||||
/**
|
||||
* Upload Chunks
|
||||
* @default 4096
|
||||
*/
|
||||
upload_chunks: number;
|
||||
/**
|
||||
* Upload Deadline Seconds
|
||||
* @default 30
|
||||
*/
|
||||
upload_deadline_seconds: number;
|
||||
/**
|
||||
* Validation Deadline Seconds
|
||||
* @default 60
|
||||
*/
|
||||
validation_deadline_seconds: number;
|
||||
};
|
||||
/** ProblemImportPage */
|
||||
ProblemImportPage: {
|
||||
/** Items */
|
||||
items: components["schemas"]["ProblemImportSummary"][];
|
||||
/** Next After */
|
||||
next_after: string | null;
|
||||
/**
|
||||
* Schema Version
|
||||
* @default 1
|
||||
* @constant
|
||||
*/
|
||||
schema_version: 1;
|
||||
};
|
||||
/** ProblemImportSummary */
|
||||
ProblemImportSummary: {
|
||||
/** Archive Bytes */
|
||||
archive_bytes: number;
|
||||
/** Candidate Id */
|
||||
candidate_id: string | null;
|
||||
/**
|
||||
* Created At
|
||||
* Format: date-time
|
||||
*/
|
||||
created_at: string;
|
||||
/** Expected Archive Digest */
|
||||
expected_archive_digest: string;
|
||||
/**
|
||||
* Expires At
|
||||
* Format: date-time
|
||||
*/
|
||||
expires_at: string;
|
||||
failure_code: components["schemas"]["ImportFailure"] | null;
|
||||
/**
|
||||
* Operation Id
|
||||
* Format: uuid
|
||||
*/
|
||||
operation_id: string;
|
||||
/** Published Version Id */
|
||||
published_version_id: string | null;
|
||||
/**
|
||||
* Schema Version
|
||||
* @default 1
|
||||
* @constant
|
||||
*/
|
||||
schema_version: 1;
|
||||
state: components["schemas"]["ImportState"];
|
||||
/**
|
||||
* Updated At
|
||||
* Format: date-time
|
||||
*/
|
||||
updated_at: string;
|
||||
};
|
||||
/** ProblemPage */
|
||||
ProblemPage: {
|
||||
/** Items */
|
||||
@@ -811,6 +1208,188 @@ export interface components {
|
||||
}
|
||||
export type $defs = Record<string, never>;
|
||||
export interface operations {
|
||||
page_api_v1_admin_problem_imports_get: {
|
||||
parameters: {
|
||||
query?: {
|
||||
after?: string | null;
|
||||
limit?: number;
|
||||
};
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemImportPage"];
|
||||
};
|
||||
};
|
||||
/** @description Validation Error */
|
||||
422: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["HTTPValidationError"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
create_api_v1_admin_problem_imports_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["CreateProblemImport"];
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
201: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemImportSummary"];
|
||||
};
|
||||
};
|
||||
/** @description Validation Error */
|
||||
422: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["HTTPValidationError"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
capabilities_api_v1_admin_problem_imports_capabilities_get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemImportCapabilities"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
detail_api_v1_admin_problem_imports__operation_id__get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path: {
|
||||
operation_id: string;
|
||||
};
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemImportDetail"];
|
||||
};
|
||||
};
|
||||
/** @description Validation Error */
|
||||
422: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["HTTPValidationError"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
upload_api_v1_admin_problem_imports__operation_id__archive_put: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path: {
|
||||
operation_id: string;
|
||||
};
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody: {
|
||||
content: {
|
||||
"application/zip": string;
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemImportSummary"];
|
||||
};
|
||||
};
|
||||
/** @description Validation Error */
|
||||
422: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["HTTPValidationError"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
discard_api_v1_admin_problem_imports__operation_id__discard_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path: {
|
||||
operation_id: string;
|
||||
};
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemImportSummary"];
|
||||
};
|
||||
};
|
||||
/** @description Validation Error */
|
||||
422: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["HTTPValidationError"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
login_api_v1_auth_login_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
||||
Reference in New Issue
Block a user