feat(submissions): define bounded performance and scoped history contracts
This commit is contained in:
@@ -4,6 +4,8 @@ from uuid import UUID
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response
|
||||
|
||||
from kf_oj.api.dependencies import current_principal, unsafe_principal
|
||||
from kf_oj.contracts.judging_v2 import Language
|
||||
from kf_oj.contracts.submission_performance import SubmissionPerformance
|
||||
from kf_oj.contracts.submissions import (
|
||||
CancellationAccepted,
|
||||
CreateSubmission,
|
||||
@@ -65,9 +67,11 @@ async def history(
|
||||
principal: Annotated[Principal, Depends(current_principal)],
|
||||
after: Annotated[str | None, Query(max_length=128)] = None,
|
||||
limit: Annotated[int, Query(ge=1, le=100)] = 20,
|
||||
problem_version_id: UUID | None = None,
|
||||
language: Language | None = None,
|
||||
):
|
||||
try:
|
||||
return await queries(request).page(principal, after, limit)
|
||||
return await queries(request).page(principal, after, limit, problem_version_id, language)
|
||||
except ValueError:
|
||||
raise HTTPException(422, "invalid history request") from None
|
||||
|
||||
@@ -86,6 +90,23 @@ async def detail(
|
||||
raise HTTPException(503, "stored result unavailable") from None
|
||||
|
||||
|
||||
@router.get("/{submission_id}/performance", response_model=SubmissionPerformance)
|
||||
async def performance(
|
||||
submission_id: UUID,
|
||||
request: Request,
|
||||
principal: Annotated[Principal, Depends(current_principal)],
|
||||
):
|
||||
service = getattr(request.app.state, "submission_performance", None)
|
||||
if service is None:
|
||||
raise HTTPException(503, "performance service unavailable")
|
||||
try:
|
||||
return await service.detail(principal, submission_id)
|
||||
except SubmissionUnavailable:
|
||||
raise HTTPException(404, "submission unavailable") from None
|
||||
except ValueError:
|
||||
raise HTTPException(503, "performance unavailable") from None
|
||||
|
||||
|
||||
@router.get("/{submission_id}/source", response_model=SubmissionSource)
|
||||
async def source(
|
||||
submission_id: UUID,
|
||||
|
||||
@@ -8,6 +8,7 @@ from kf_oj.api.app import create_app
|
||||
from kf_oj.contracts.judge import JudgeTaskNotification
|
||||
from kf_oj.contracts.judging import WIRE_MODELS
|
||||
from kf_oj.contracts.judging_v2 import WIRE_MODELS as WIRE_MODELS_V2
|
||||
from kf_oj.contracts.submission_performance import SubmissionPerformance
|
||||
from kf_oj.core.settings import Settings
|
||||
|
||||
|
||||
@@ -34,6 +35,14 @@ def main() -> None:
|
||||
"budgets, result consistency, UUID canonical form and UTF-8 byte limits."
|
||||
)
|
||||
outputs[root / f"contracts/schemas/{name}.v1.schema.json"] = document
|
||||
performance = SubmissionPerformance.model_json_schema()
|
||||
performance["$schema"] = "https://json-schema.org/draft/2020-12/schema"
|
||||
performance["$id"] = "urn:kf-oj:submission-performance:v1"
|
||||
performance["$comment"] = (
|
||||
"Anonymous bounded aggregates, decimal raw units and strict same-snapshot "
|
||||
"counts; also run model validation and server-side ownership/environment checks."
|
||||
)
|
||||
outputs[root / "contracts/schemas/submission-performance.v1.schema.json"] = performance
|
||||
for name, model in WIRE_MODELS_V2.items():
|
||||
document = model.model_json_schema()
|
||||
document["$schema"] = "https://json-schema.org/draft/2020-12/schema"
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
"""Bounded anonymous aggregates; raw measurements never pass through JS floats."""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Annotated, Literal, Self
|
||||
|
||||
from pydantic import AfterValidator, Field, StrictBool, model_validator
|
||||
|
||||
from kf_oj.contracts.judging import I64_MAX, Digest, Identifier, Version, WireModel
|
||||
from kf_oj.contracts.judging_v2 import Language
|
||||
|
||||
MAX_PERFORMANCE_SAMPLES = 10_000
|
||||
MIN_PERFORMANCE_SAMPLES = 5
|
||||
MAX_PERFORMANCE_BUCKETS = 64
|
||||
|
||||
|
||||
def bounded_decimal(value: str) -> str:
|
||||
if int(value) > I64_MAX:
|
||||
raise ValueError("measurement exceeds signed storage range")
|
||||
return value
|
||||
|
||||
|
||||
DecimalMeasurement = Annotated[
|
||||
str,
|
||||
Field(strict=True, min_length=1, max_length=19, pattern=r"^(0|[1-9][0-9]*)$"),
|
||||
AfterValidator(bounded_decimal),
|
||||
]
|
||||
SampleCount = Annotated[int, Field(strict=True, ge=0, le=MAX_PERFORMANCE_SAMPLES)]
|
||||
UnavailableReason = Literal[
|
||||
"not_accepted", "measurement_missing", "environment_unverified", "insufficient_samples"
|
||||
]
|
||||
|
||||
|
||||
class PerformanceBucket(WireModel):
|
||||
start: DecimalMeasurement
|
||||
end: DecimalMeasurement
|
||||
upper_inclusive: StrictBool
|
||||
count: SampleCount
|
||||
|
||||
@model_validator(mode="after")
|
||||
def interval(self) -> Self:
|
||||
if int(self.end) <= int(self.start):
|
||||
raise ValueError("empty or reversed interval")
|
||||
return self
|
||||
|
||||
|
||||
class PerformanceMetric(WireModel):
|
||||
unit: Literal["cpu_microseconds", "bytes"]
|
||||
current: DecimalMeasurement | None
|
||||
available: StrictBool
|
||||
reason: UnavailableReason | None
|
||||
sample_count: SampleCount
|
||||
window_truncated: StrictBool
|
||||
oldest_sample_at: datetime | None
|
||||
# These counts are computed against the original integers, never rounded bins.
|
||||
strictly_slower_count: SampleCount | None
|
||||
tied_count: SampleCount | None
|
||||
buckets: Annotated[tuple[PerformanceBucket, ...], Field(max_length=MAX_PERFORMANCE_BUCKETS)]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def aggregate(self) -> Self:
|
||||
if self.oldest_sample_at is not None and self.oldest_sample_at.utcoffset() is None:
|
||||
raise ValueError("sample timestamp must have a timezone")
|
||||
if self.available:
|
||||
if (
|
||||
self.reason is not None
|
||||
or self.current is None
|
||||
or self.sample_count < MIN_PERFORMANCE_SAMPLES
|
||||
or self.strictly_slower_count is None
|
||||
or self.tied_count is None
|
||||
or self.oldest_sample_at is None
|
||||
or not self.buckets
|
||||
):
|
||||
raise ValueError("incomplete available aggregate")
|
||||
if self.strictly_slower_count + self.tied_count > self.sample_count:
|
||||
raise ValueError("comparison counts exceed the cohort")
|
||||
if sum(b.count for b in self.buckets) != self.sample_count:
|
||||
raise ValueError("histogram loses samples")
|
||||
if int(self.buckets[0].start) != 0:
|
||||
raise ValueError("histogram must include zero")
|
||||
for index, bucket in enumerate(self.buckets):
|
||||
if bucket.upper_inclusive != (index == len(self.buckets) - 1):
|
||||
raise ValueError("only the last interval includes its upper endpoint")
|
||||
if index and self.buckets[index - 1].end != bucket.start:
|
||||
raise ValueError("histogram intervals must be contiguous")
|
||||
elif (
|
||||
self.reason is None
|
||||
or self.buckets
|
||||
or self.strictly_slower_count is not None
|
||||
or self.tied_count is not None
|
||||
):
|
||||
raise ValueError("unavailable aggregate cannot expose comparison data")
|
||||
if self.reason == "measurement_missing" and self.current is not None:
|
||||
raise ValueError("missing measurement cannot claim a value")
|
||||
if self.reason == "insufficient_samples" and (
|
||||
self.sample_count >= MIN_PERFORMANCE_SAMPLES or self.current is None
|
||||
):
|
||||
raise ValueError("inconsistent insufficient cohort")
|
||||
return self
|
||||
|
||||
|
||||
class SubmissionPerformance(WireModel):
|
||||
schema_version: Version = 1
|
||||
kind: Literal["judge.performance"] = "judge.performance"
|
||||
submission_id: Identifier
|
||||
problem_version_id: Identifier
|
||||
language: Language
|
||||
as_of: datetime
|
||||
# Opaque trusted comparison identity, not a worker name, path or credential.
|
||||
comparison_group: Digest | None
|
||||
window_limit: Literal[10_000] = MAX_PERFORMANCE_SAMPLES
|
||||
minimum_samples: Literal[5] = MIN_PERFORMANCE_SAMPLES
|
||||
cpu: PerformanceMetric
|
||||
memory: PerformanceMetric
|
||||
|
||||
@model_validator(mode="after")
|
||||
def snapshot(self) -> Self:
|
||||
if self.as_of.utcoffset() is None:
|
||||
raise ValueError("snapshot timestamp must have a timezone")
|
||||
if self.cpu.unit != "cpu_microseconds" or self.memory.unit != "bytes":
|
||||
raise ValueError("metric unit mismatch")
|
||||
for metric in (self.cpu, self.memory):
|
||||
if metric.available and self.comparison_group is None:
|
||||
raise ValueError("comparison requires trusted environment binding")
|
||||
if metric.oldest_sample_at is not None and metric.oldest_sample_at > self.as_of:
|
||||
raise ValueError("sample is newer than the snapshot")
|
||||
return self
|
||||
@@ -88,7 +88,14 @@ class SubmissionQueries:
|
||||
.where(Submission.id == submission_id, Submission.account_id == principal.account_id)
|
||||
)
|
||||
|
||||
async def page(self, principal: Principal, after: str | None, limit: int) -> SubmissionPage:
|
||||
async def page(
|
||||
self,
|
||||
principal: Principal,
|
||||
after: str | None,
|
||||
limit: int,
|
||||
problem_version_id: UUID | None = None,
|
||||
language: str | None = None,
|
||||
) -> SubmissionPage:
|
||||
if not 1 <= limit <= 100:
|
||||
raise ValueError("invalid history limit")
|
||||
query = (
|
||||
@@ -98,6 +105,10 @@ class SubmissionQueries:
|
||||
.order_by(Submission.created_at.desc(), Submission.id.desc())
|
||||
.limit(limit + 1)
|
||||
)
|
||||
if problem_version_id is not None:
|
||||
query = query.where(Submission.problem_version_id == problem_version_id)
|
||||
if language is not None:
|
||||
query = query.where(Submission.language == language)
|
||||
if after is not None:
|
||||
query = query.where(tuple_(Submission.created_at, Submission.id) < cursor_decode(after))
|
||||
async with self.sessions() as session:
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Protocol/route boundaries, not evidence of real measured execution."""
|
||||
|
||||
import copy
|
||||
import json
|
||||
from pathlib import Path
|
||||
from uuid import UUID
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from jsonschema import Draft202012Validator
|
||||
|
||||
from kf_oj.api.app import create_app
|
||||
from kf_oj.api.dependencies import current_principal
|
||||
from kf_oj.contracts.submission_performance import SubmissionPerformance
|
||||
from kf_oj.core.settings import Settings
|
||||
from kf_oj.modules.identity.public import Principal
|
||||
|
||||
ROOT = Path(__file__).parents[2]
|
||||
|
||||
|
||||
def fixture():
|
||||
return json.loads(
|
||||
(ROOT / "tests/fixtures/contracts/submission-performance.valid.json").read_text()
|
||||
)
|
||||
|
||||
|
||||
def test_decimal_measurements_zero_and_four_languages_roundtrip():
|
||||
data = fixture()
|
||||
for language in ("c17", "cpp20", "python313", "java21"):
|
||||
data["language"] = language
|
||||
response = SubmissionPerformance.model_validate(data)
|
||||
encoded = response.model_dump(mode="json")
|
||||
assert encoded == data
|
||||
assert encoded["cpu"]["current"] == "9007199254740993"
|
||||
assert encoded["memory"]["current"] == "0"
|
||||
Draft202012Validator(SubmissionPerformance.model_json_schema()).validate(encoded)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"change",
|
||||
[
|
||||
lambda v: v.update(language="Rust"),
|
||||
lambda v: v.update(schema_version=True),
|
||||
lambda v: v.update(username="other_user"),
|
||||
lambda v: v.update(comparison_group=None),
|
||||
lambda v: v.update(as_of="2026-10-11T00:00:00"),
|
||||
lambda v: v["cpu"].update(current=9007199254740993),
|
||||
lambda v: v["cpu"].update(current="092"),
|
||||
lambda v: v["cpu"].update(current="9223372036854775808"),
|
||||
lambda v: v["cpu"].update(sample_count=4),
|
||||
lambda v: v["cpu"].update(strictly_slower_count=6, tied_count=1),
|
||||
lambda v: v["cpu"].update(oldest_sample_at="2026-10-12T00:00:00Z"),
|
||||
lambda v: v["cpu"]["buckets"][0].update(upper_inclusive=True),
|
||||
lambda v: v["cpu"]["buckets"][0].update(count=2),
|
||||
lambda v: v["cpu"]["buckets"][1].update(start="9007199254740994"),
|
||||
lambda v: v["memory"].update(sample_count=5),
|
||||
lambda v: v["memory"].update(tied_count=1),
|
||||
lambda v: v["memory"].update(reason="measurement_missing"),
|
||||
],
|
||||
)
|
||||
def test_no_rounded_fabricated_unbounded_or_inconsistent_aggregate(change):
|
||||
data = copy.deepcopy(fixture())
|
||||
change(data)
|
||||
with pytest.raises(ValueError):
|
||||
SubmissionPerformance.model_validate(data)
|
||||
|
||||
|
||||
def test_performance_unconfigured_and_bad_history_filters_fail_closed():
|
||||
app = create_app(Settings(_env_file=None, environment="test"))
|
||||
sid = "11111111-1111-4111-8111-111111111111"
|
||||
with TestClient(app) as client:
|
||||
assert client.get(f"/api/v1/submissions/{sid}/performance").status_code == 503
|
||||
app.dependency_overrides[current_principal] = lambda: Principal(UUID(sid), "owner")
|
||||
response = client.get(f"/api/v1/submissions/{sid}/performance")
|
||||
assert response.status_code == 503
|
||||
assert response.headers["Cache-Control"] == "no-store"
|
||||
assert client.get("/api/v1/submissions?language=rust").status_code == 422
|
||||
assert client.get("/api/v1/submissions?problem_version_id=1001").status_code == 422
|
||||
Reference in New Issue
Block a user