Add all-users validation backfill

docs/application-validation.md

@@ -62,5 +62,15 @@ Omit `--wait` to enqueue the sweep and return immediately. The command reads
 `ADMIN_TOKEN` and, unless `--api-url` is supplied,
 `DISCORD_CLUSTER_MANAGER_API_BASE_URL` from the environment.
 
+For a one-time backfill of every ranked user's current best submission, add
+`--all-users`:
+
+```bash
+kernelbot-admin validate-top10 cholesky B200 --all-users
+```
+
+This override applies only to the manual sweep. Scheduled sweeps remain capped
+at the top 10 users.
+
 Roll out KernelBot's migration and runner first, then the reference-kernels
 contract, then the Kernelboard badge.

src/kernelbot/admin_cli.py

@@ -20,10 +20,13 @@ def _validate_top10(args: argparse.Namespace) -> int:
         quote(args.leaderboard, safe=""),
         quote(args.gpu, safe=""),
     )
+    params = {"wait": str(args.wait).lower()}
+    if args.all_users:
+        params["all_users"] = "true"
     response = requests.post(
         f"{api_url.rstrip('/')}{path}",
         headers={"Authorization": f"Bearer {token}"},
-        params={"wait": str(args.wait).lower()},
+        params=params,
         timeout=None if args.wait else 30,
     )
     response.raise_for_status()
@@ -49,6 +52,11 @@ def _parser() -> argparse.ArgumentParser:
         action="store_true",
         help="Wait for all validation jobs and print their results",
     )
+    validate.add_argument(
+        "--all-users",
+        action="store_true",
+        help="Validate every ranked user's best submission instead of only the top 10",
+    )
     validate.set_defaults(run=_validate_top10)
     return parser
 

src/kernelbot/api/main.py

@@ -491,6 +491,7 @@ async def admin_run_application_validation(
     gpu_type: str,
     _: Annotated[None, Depends(require_admin)],
     wait: bool = Query(False),
+    all_users: bool = Query(False),
 ) -> dict:
     if application_validation_service is None:
         raise HTTPException(
@@ -502,15 +503,18 @@ async def admin_run_application_validation(
             leaderboard_name,
             gpu_type,
             scheduled_for=None,
+            all_users=all_users,
         )
     application_validation_service.enqueue_manual_sweep(
         leaderboard_name,
         gpu_type,
+        all_users=all_users,
     )
     return {
         "status": "accepted",
         "leaderboard": leaderboard_name,
         "gpu_type": gpu_type,
+        "all_users": all_users,
     }
 
 

src/libkernelbot/application_validation.py

@@ -101,9 +101,16 @@ def enqueue_manual_sweep(
         self,
         leaderboard_name: str,
         gpu_type: str,
+        *,
+        all_users: bool = False,
     ) -> None:
         task = asyncio.create_task(
-            self.run_sweep(leaderboard_name, gpu_type, scheduled_for=None),
+            self.run_sweep(
+                leaderboard_name,
+                gpu_type,
+                scheduled_for=None,
+                all_users=all_users,
+            ),
             name=f"manual-application-validation-{leaderboard_name}-{gpu_type}",
         )
         self._manual_tasks.add(task)
@@ -225,6 +232,7 @@ async def run_sweep(
         gpu_type: str,
         *,
         scheduled_for: datetime.date | None,
+        all_users: bool = False,
     ) -> dict[str, Any]:
         with self.backend.db as db:
             leaderboard = db.get_leaderboard(leaderboard_name)
@@ -255,7 +263,7 @@ async def run_sweep(
             submissions = db.get_leaderboard_submissions(
                 leaderboard_name,
                 gpu_type,
-                limit=TOP_K,
+                limit=None if all_users else TOP_K,
             )
 
         logger.info(
@@ -288,6 +296,7 @@ async def run_sweep(
                 "gpu_type": gpu_type,
                 "scheduled_for": scheduled_for,
                 "contract_version": validation.version,
+                "all_users": all_users,
                 "results": results,
             }
         except Exception as exc:

tests/test_admin_api.py

@@ -127,6 +127,7 @@ def test_manual_validation_can_wait_for_debug_result(
             "cholesky",
             "B200",
             scheduled_for=None,
+            all_users=False,
         )
 
     def test_manual_validation_is_async_by_default(
@@ -144,10 +145,35 @@ def test_manual_validation_is_async_by_default(
             "status": "accepted",
             "leaderboard": "cholesky",
             "gpu_type": "B200",
+            "all_users": False,
         }
         mock_validation_service.enqueue_manual_sweep.assert_called_once_with(
             "cholesky",
             "B200",
+            all_users=False,
+        )
+
+    def test_manual_validation_can_enqueue_all_users(
+        self,
+        test_client,
+        mock_validation_service,
+    ):
+        response = test_client.post(
+            "/admin/application-validations/cholesky/B200?all_users=true",
+            headers={"Authorization": "Bearer test_token"},
+        )
+
+        assert response.status_code == 200
+        assert response.json() == {
+            "status": "accepted",
+            "leaderboard": "cholesky",
+            "gpu_type": "B200",
+            "all_users": True,
+        }
+        mock_validation_service.enqueue_manual_sweep.assert_called_once_with(
+            "cholesky",
+            "B200",
+            all_users=True,
         )
 
 class TestRunnerQueue:

tests/test_admin_cli.py

@@ -65,6 +65,38 @@ def test_validate_top10_can_wait_and_url_encodes_names():
     )
 
 
+def test_validate_top10_can_override_scope_to_all_users():
+    response = Mock()
+    response.json.return_value = {"status": "accepted", "all_users": True}
+
+    with (
+        patch.dict(
+            "os.environ",
+            {
+                "DISCORD_CLUSTER_MANAGER_API_BASE_URL": "https://kernelbot.test",
+                "ADMIN_TOKEN": "secret",
+            },
+        ),
+        patch("kernelbot.admin_cli.requests.post", return_value=response) as post,
+        patch(
+            "sys.argv",
+            [
+                "kernelbot-admin",
+                "validate-top10",
+                "cholesky",
+                "B200",
+                "--all-users",
+            ],
+        ),
+    ):
+        assert admin_cli.main() == 0
+
+    assert post.call_args.kwargs["params"] == {
+        "wait": "false",
+        "all_users": "true",
+    }
+
+
 def test_validate_top10_requires_admin_token():
     with (
         patch.dict(

tests/test_application_validation.py

@@ -44,6 +44,7 @@ def __init__(self):
         self.claims = set()
         self.saved = []
         self.sweeps = []
+        self.requested_limits = []
 
     def __enter__(self):
         return self
@@ -73,6 +74,7 @@ def claim_validation_sweep(
         return len(self.claims)
 
     def get_leaderboard_submissions(self, _name, _gpu, limit):
+        self.requested_limits.append(limit)
         return self.submissions[:limit]
 
     def get_submission_code_for_validation(self, submission_id):
@@ -143,6 +145,29 @@ async def test_sweep_validates_top_ten_with_bounded_concurrency():
     assert database.sweeps == [(1, "completed", None)]
     assert database.saved[0]["fully_validated"] is True
     assert database.saved[0]["result"]["results"][0]["unexpected_private_field"] == "drop me"
+    assert database.requested_limits == [10]
+
+
+@pytest.mark.asyncio
+async def test_manual_sweep_can_validate_every_ranked_user():
+    database = FakeDB()
+    launcher = FakeLauncher()
+    backend = SimpleNamespace(db=database, launcher_map={"B200": launcher})
+    service = ApplicationValidationService(backend)
+
+    summary = await service.run_sweep(
+        "cholesky",
+        "B200",
+        scheduled_for=None,
+        all_users=True,
+    )
+
+    assert summary["status"] == "completed"
+    assert summary["all_users"] is True
+    assert len(summary["results"]) == 12
+    assert len(database.saved) == 12
+    assert database.requested_limits == [None]
+    assert launcher.max_active == 2
 
 
 @pytest.mark.asyncio