scripts/acceptance_cases/s05_sync_uploads.py

@@ -225,11 +225,11 @@ def main() -> int:
                 "dependencies": True,
                 "postgres_version": stack.postgres_version,
                 "minio_version": stack.minio_version,
-                "worker_count": len(stack.worker_ids()),
             }
         )
-        assert facts["worker_count"] == 2
         auth, base, vault_id = stack.create_vault("S-05 isolated")
+        facts["worker_count"] = len(stack.worker_ids())
+        assert facts["worker_count"] == 2
         committed: dict[str, bytes] = {}
         cleaned: dict[str, bytes] = {}
         expected_used = 0

scripts/acceptance_cases/sync_production_stack.py

@@ -267,7 +267,10 @@ class SyncProductionStack:
         return self
 
     def create_vault(self, label: str):
-        code, session, _ = call(
+        session = None
+        for _ in range(5):
+            try:
+                code, candidate, _ = call(
                     "POST",
                     self.origin + "/sync/v1/auth/sessions",
                     body={
@@ -275,14 +278,34 @@ class SyncProductionStack:
                         "password": self.password,
                         "device_name": label,
                     },
+                    timeout=10,
                 )
-        if code != 200:
+                if code == 200:
+                    session = candidate
+                    break
+            except (OSError, TimeoutError, URLError):
+                pass
+            time.sleep(0.2)
+        if session is None:
             raise RuntimeError("ACCEPTANCE_LOGIN_FAILED")
         auth = {"Authorization": "Bearer " + session["access_token"]}
-        code, vault, _ = call(
+        vault = None
-            "POST", self.origin + "/sync/v1/vaults", headers=auth, body={"name": label}
+        for _ in range(5):
+            try:
+                code, candidate, _ = call(
+                    "POST",
+                    self.origin + "/sync/v1/vaults",
+                    headers=auth,
+                    body={"name": label},
+                    timeout=10,
                 )
-        if code != 200:
+                if code == 200:
+                    vault = candidate
+                    break
+            except (OSError, TimeoutError, URLError):
+                pass
+            time.sleep(0.2)
+        if vault is None:
             raise RuntimeError("ACCEPTANCE_VAULT_FAILED")
         return auth, self.origin + "/sync/v1/vaults/" + vault["vault_id"], vault["vault_id"]
 
@@ -315,11 +338,19 @@ class SyncProductionStack:
         from concurrent.futures import ThreadPoolExecutor
 
         def probe(_):
+            try:
                 return call(
-                "GET", self.origin + "/health", headers={"Connection": "close"}
+                    "GET",
+                    self.origin + "/health",
+                    headers={"Connection": "close"},
+                    timeout=10,
                 )[2].get("x-opennexus-worker")
+            except (OSError, TimeoutError, URLError):
+                return None
 
-        with ThreadPoolExecutor(max_workers=40) as pool:
+        # A small pool is enough to reach both workers without exhausting the
+        # Windows ephemeral-port/backlog budget before the fault matrix starts.
+        with ThreadPoolExecutor(max_workers=8) as pool:
             return {value for value in pool.map(probe, range(attempts)) if value}
 
     def assert_running(self) -> None: