279 lines
13 KiB
Python
279 lines
13 KiB
Python
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from jsonschema import Draft202012Validator, FormatChecker
|
|
|
|
from kf_oj.contracts import judging as v1
|
|
from kf_oj.contracts import judging_v2 as v2
|
|
from kf_oj.contracts import judging_versions as versions
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
FIXTURES = ROOT / "tests/fixtures/contracts-v2"
|
|
PATHS = sorted(FIXTURES.glob("*.json"))
|
|
|
|
|
|
def fixture(name, model):
|
|
return v1.decode_wire(model, (FIXTURES / f"{name}.json").read_bytes())
|
|
|
|
|
|
@pytest.mark.parametrize("path", PATHS, ids=lambda p: p.stem)
|
|
def test_multi_runtime_examples_match_schema_and_rust(path):
|
|
name = path.name.split(".")[0]
|
|
model = v2.WIRE_MODELS[name]
|
|
schema = json.loads((ROOT / f"contracts/schemas/{name}.v2.schema.json").read_text())
|
|
validator = Draft202012Validator(schema, format_checker=FormatChecker())
|
|
original = json.loads(path.read_text())
|
|
if ".invalid-" in path.name:
|
|
with pytest.raises(ValueError):
|
|
v1.decode_wire(model, path.read_bytes())
|
|
if ".invalid-semantic-" in path.name:
|
|
assert validator.is_valid(original)
|
|
else:
|
|
assert not validator.is_valid(original)
|
|
else:
|
|
value = v1.decode_wire(model, path.read_bytes())
|
|
validator.validate(original)
|
|
assert value.model_dump(mode="json") == original
|
|
|
|
|
|
@pytest.mark.parametrize("language", list(v2.Language))
|
|
def test_explicit_version_keeps_old_consumers_closed_and_canonical_bytes_stable(language):
|
|
path = FIXTURES / f"judge-manifest.valid-{language}.json"
|
|
manifest = v1.decode_wire(v2.JobManifest, path.read_bytes())
|
|
canonical = (FIXTURES / f"canonical-manifest.{language}.txt").read_bytes()
|
|
assert v1.manifest_bytes(manifest) == canonical
|
|
assert (
|
|
hashlib.sha256(v1.manifest_bytes(manifest)).digest() == hashlib.sha256(canonical).digest()
|
|
)
|
|
with pytest.raises(ValueError):
|
|
v1.decode_wire(v1.JobManifest, path.read_bytes())
|
|
old = ROOT / "tests/fixtures/contracts/judge-manifest.valid.json"
|
|
with pytest.raises(ValueError):
|
|
v1.decode_wire(v2.JobManifest, old.read_bytes())
|
|
old_manifest = v1.decode_wire(v1.JobManifest, old.read_bytes())
|
|
assert (
|
|
v1.manifest_bytes(old_manifest)
|
|
== (ROOT / "tests/fixtures/contracts/canonical-manifest.v1.txt").read_bytes()
|
|
)
|
|
|
|
|
|
def test_shared_capabilities_refuse_absent_changed_runtime_and_over_budget():
|
|
policy = fixture("judge-admission.valid", v2.AdmissionPolicy)
|
|
for language in v2.Language:
|
|
manifest = fixture(f"judge-manifest.valid-{language}", v2.JobManifest)
|
|
for attempt in (1, 2, 3):
|
|
policy.admit(manifest, attempt)
|
|
for attempt in (0, 4, True):
|
|
with pytest.raises(ValueError):
|
|
policy.admit(manifest, attempt)
|
|
for field, value in (("image_revision", 2), ("image_digest", "f" * 64)):
|
|
other = manifest.model_copy(
|
|
update={"toolchain": manifest.toolchain.model_copy(update={field: value})}
|
|
)
|
|
with pytest.raises(ValueError):
|
|
policy.admit(other, 1)
|
|
python = fixture("judge-manifest.valid-python313", v2.JobManifest)
|
|
without_python = policy.model_copy(
|
|
update={
|
|
"capabilities": tuple(c for c in policy.capabilities if c.language != python.language)
|
|
}
|
|
)
|
|
with pytest.raises(ValueError):
|
|
without_python.admit(python, 1)
|
|
too_small = python.model_copy(
|
|
update={"run_limits": python.run_limits.model_copy(update={"memory_bytes": 1})}
|
|
)
|
|
with pytest.raises(ValueError):
|
|
policy.admit(too_small, 1)
|
|
too_large = python.model_copy(
|
|
update={"run_limits": python.run_limits.model_copy(update={"memory_bytes": 128 * v1.MIB})}
|
|
)
|
|
with pytest.raises(ValueError):
|
|
policy.admit(too_large, 1)
|
|
|
|
|
|
@pytest.mark.parametrize("language", list(v2.Language))
|
|
def test_results_bind_exact_source_runtime_attempt_and_case_order(language):
|
|
manifest = fixture(f"judge-manifest.valid-{language}", v2.JobManifest)
|
|
result = fixture(f"judge-result.valid-{language}", v2.JudgementReport)
|
|
digest = hashlib.sha256(versions.manifest_bytes(manifest)).hexdigest()
|
|
assert digest == (FIXTURES / f"canonical-manifest.{language}.sha256").read_text().strip()
|
|
result.validate_for_manifest(manifest, digest)
|
|
for field, value in (("digest", "f" * 64), ("bytes", manifest.source.bytes + 1)):
|
|
other = manifest.model_copy(
|
|
update={"source": manifest.source.model_copy(update={field: value})}
|
|
)
|
|
with pytest.raises(ValueError):
|
|
result.validate_for_manifest(other, digest)
|
|
with pytest.raises(ValueError):
|
|
result.validate_for_manifest(manifest, "f" * 64)
|
|
changed_runtime = manifest.model_copy(
|
|
update={"toolchain": manifest.toolchain.model_copy(update={"image_digest": "e" * 64})}
|
|
)
|
|
with pytest.raises(ValueError):
|
|
result.validate_for_manifest(changed_runtime, digest)
|
|
short = result.model_dump(mode="json")
|
|
short.update(cases=short["cases"][:1], run_cpu_time_us=100, run_memory_peak_bytes=256 * 1024)
|
|
valid_partial = v2.JudgementReport.model_validate(short)
|
|
with pytest.raises(ValueError):
|
|
valid_partial.validate_for_manifest(manifest, digest)
|
|
|
|
|
|
def test_python_check_is_not_an_accepted_result_and_stage_kind_is_bound():
|
|
manifest = fixture("judge-manifest.valid-python313", v2.JobManifest)
|
|
stage = fixture("judge-stage.valid-python313", v2.StageReport)
|
|
stage.validate_for_manifest(manifest)
|
|
with pytest.raises(ValueError):
|
|
stage.model_copy(
|
|
update={"preparation_kind": v2.PreparationKind.COMPILE}
|
|
).validate_for_manifest(manifest)
|
|
result = fixture("judge-result.valid-python313", v2.JudgementReport).model_dump(mode="json")
|
|
result.update(cases=[], run_cpu_time_us=None, run_memory_peak_bytes=None)
|
|
with pytest.raises(ValueError):
|
|
v2.JudgementReport.model_validate(result)
|
|
|
|
|
|
def test_jvm_heap_is_smaller_than_total_memory_and_threads_share_the_budget():
|
|
manifest = fixture("judge-manifest.valid-java21", v2.JobManifest)
|
|
limits = manifest.run_limits
|
|
assert 32 * v1.MIB <= v2.jvm_heap_bytes(limits) < limits.memory_bytes
|
|
more_threads = limits.model_copy(update={"processes": 32})
|
|
assert v2.jvm_heap_bytes(more_threads) < v2.jvm_heap_bytes(limits)
|
|
with pytest.raises(ValueError):
|
|
v2.jvm_heap_bytes(limits.model_copy(update={"memory_bytes": 128 * v1.MIB}))
|
|
# Ordinary RuntimeError has no infrastructure failure code or trusted memory-limit claim.
|
|
result = fixture("judge-result.valid-java21", v2.JudgementReport).model_dump(mode="json")
|
|
result["cases"][0]["verdict"] = "runtime_error"
|
|
result["verdict"] = "runtime_error"
|
|
v2.JudgementReport.model_validate(result)
|
|
|
|
|
|
def test_v2_wire_rejects_ambiguous_private_values_without_echoing_them():
|
|
data = (FIXTURES / "judge-lease.valid.json").read_bytes()
|
|
duplicate = data.replace(b'"schema_version": 2', b'"schema_version": 2, "schema_version": 2')
|
|
for invalid in (duplicate, b" " * (v1.MAX_WIRE_BYTES + 1), data.decode().encode("utf-16")):
|
|
with pytest.raises(ValueError):
|
|
v1.decode_wire(v2.LeaseGrant, invalid)
|
|
secret = "private-placeholder-never-echo"
|
|
value = json.loads(data)
|
|
value["lease_token"] = secret
|
|
with pytest.raises(ValueError) as error:
|
|
v1.decode_wire(v2.LeaseGrant, json.dumps(value).encode())
|
|
assert secret not in str(error.value)
|
|
|
|
|
|
def test_storage_boundary_reads_both_versions_without_rewriting_or_cross_version_reports():
|
|
old = ROOT / "tests/fixtures/contracts"
|
|
old_manifest = versions.decode_manifest((old / "judge-manifest.valid.json").read_bytes())
|
|
old_report = versions.decode_report((old / "judge-result.valid.json").read_bytes())
|
|
versions.validate_report(old_report, old_manifest, old_report.manifest_digest)
|
|
assert versions.manifest_bytes(old_manifest) == (old / "canonical-manifest.v1.txt").read_bytes()
|
|
manifest = versions.decode_manifest(
|
|
(FIXTURES / "judge-manifest.valid-python313.json").read_bytes()
|
|
)
|
|
report = versions.decode_report((FIXTURES / "judge-result.valid-python313.json").read_bytes())
|
|
versions.validate_report(
|
|
report, manifest, hashlib.sha256(versions.manifest_bytes(manifest)).hexdigest()
|
|
)
|
|
for result, task in ((old_report, manifest), (report, old_manifest)):
|
|
with pytest.raises(ValueError):
|
|
versions.validate_report(result, task, result.manifest_digest)
|
|
payload = (FIXTURES / "judge-manifest.valid-python313.json").read_bytes()
|
|
for invalid in (
|
|
payload.replace(b'"schema_version": 2', b'"schema_version": 3'),
|
|
payload.replace(b'"schema_version": 2', b'"schema_version": true'),
|
|
payload.replace(b'"schema_version": 2', b'"schema_version": 2, "schema_version": 2'),
|
|
):
|
|
with pytest.raises(ValueError):
|
|
versions.decode_manifest(invalid)
|
|
|
|
|
|
def test_registry_is_explicit_for_each_protocol_and_retains_legacy_configuration():
|
|
from kf_oj.core.settings import Settings
|
|
|
|
registry = fixture("judge-registry.valid", v2.AdmissionRegistry)
|
|
original = ROOT / "tests/fixtures/contracts"
|
|
legacy = v1.decode_wire(v1.JobManifest, (original / "judge-manifest.valid.json").read_bytes())
|
|
registry.admit(legacy, 1)
|
|
assert registry.legacy.model_dump(mode="json") == json.loads(
|
|
(original / "judge-admission.valid.json").read_text()
|
|
)
|
|
for language in v2.Language:
|
|
manifest = fixture(f"judge-manifest.valid-{language}", v2.JobManifest)
|
|
registry.admit(manifest, 1)
|
|
with pytest.raises(ValueError, match="protocol"):
|
|
v2.admit_manifest(registry.legacy, manifest, 1)
|
|
for field, value in (("image_digest", "f" * 64), ("runtime_version", "other")):
|
|
changed = manifest.model_copy(
|
|
update={"toolchain": manifest.toolchain.model_copy(update={field: value})}
|
|
)
|
|
with pytest.raises(ValueError):
|
|
registry.admit(changed, 1)
|
|
with pytest.raises(ValueError, match="protocol"):
|
|
v2.admit_manifest(registry.multi_runtime, legacy, 1)
|
|
for policy in (registry.legacy, registry.multi_runtime, registry):
|
|
settings = Settings(_env_file=None, judge_admission=policy.model_dump(mode="json"))
|
|
assert type(settings.judge_admission) is type(policy)
|
|
raw = registry.model_dump(mode="json")
|
|
raw["multi_runtime"]["prefetch"] = 2
|
|
with pytest.raises(ValueError):
|
|
Settings(_env_file=None, judge_admission=raw)
|
|
|
|
|
|
def test_generic_admission_revalidates_constructed_values_in_both_generations():
|
|
registry = fixture("judge-registry.valid", v2.AdmissionRegistry)
|
|
old = ROOT / "tests/fixtures/contracts/judge-manifest.valid.json"
|
|
legacy = v1.decode_wire(v1.JobManifest, old.read_bytes())
|
|
for manifest in (legacy, fixture("judge-manifest.valid-python313", v2.JobManifest)):
|
|
for wrong in (
|
|
manifest.model_copy(update={"schema_version": True}),
|
|
manifest.model_copy(update={"job_wall_time_us": 1}),
|
|
manifest.model_copy(update={"cases": ()}),
|
|
):
|
|
with pytest.raises(ValueError):
|
|
v2.admit_manifest(registry, wrong, 1)
|
|
v2.admit_manifest(registry, legacy, 1)
|
|
|
|
|
|
def test_operator_registry_example_preserves_old_protocol_and_binds_managed_runtime():
|
|
policy = v2.AdmissionRegistry.model_validate_json(
|
|
(ROOT / "deploy/judge.admission.example.json").read_bytes()
|
|
)
|
|
old = v1.decode_wire(
|
|
v1.JobManifest, (ROOT / "tests/fixtures/contracts/judge-manifest.valid.json").read_bytes()
|
|
)
|
|
policy.admit(old, 1)
|
|
assert policy.multi_runtime is not None
|
|
root = json.loads((ROOT / "deploy/sandbox.managed.runtime.example.json").read_text())
|
|
for cap in policy.multi_runtime.capabilities:
|
|
selected = next(
|
|
item
|
|
for item in root["runtime_images"]
|
|
if item["runtime_identity"] == cap.toolchain.model_dump(mode="json")
|
|
)
|
|
ceiling = v1.RunLimits.model_validate(selected["per_task_ceiling"])
|
|
assert cap.compile_ceiling.fits(ceiling) and cap.run_ceiling.fits(ceiling)
|
|
manifest = fixture(f"judge-manifest.valid-{cap.language}", v2.JobManifest).model_copy(
|
|
update={
|
|
"compile_limits": cap.compile_ceiling,
|
|
"run_limits": cap.run_ceiling,
|
|
"job_wall_time_us": policy.multi_runtime.max_job_wall_time_us,
|
|
}
|
|
)
|
|
policy.admit(manifest, 1)
|
|
with pytest.raises(ValueError, match="job deadline cannot cover execution budgets"):
|
|
policy.admit(
|
|
manifest.model_copy(update={"job_wall_time_us": cap.compile_ceiling.wall_time_us}),
|
|
1,
|
|
)
|
|
changed = manifest.model_copy(
|
|
update={"toolchain": manifest.toolchain.model_copy(update={"image_digest": "f" * 64})}
|
|
)
|
|
with pytest.raises(ValueError):
|
|
policy.admit(changed, 1)
|
|
legacy_only = v2.AdmissionRegistry(schema_version=2, legacy=policy.legacy, multi_runtime=None)
|
|
with pytest.raises(ValueError):
|
|
legacy_only.admit(fixture("judge-manifest.valid-java21", v2.JobManifest), 1)
|