diff --git a/frontend/src/api/api.ts b/frontend/src/api/api.ts index e0491f0..fcb88a7 100644 --- a/frontend/src/api/api.ts +++ b/frontend/src/api/api.ts @@ -36,6 +36,13 @@ export interface LeaderboardDetail { user_name: string; submission_id: number; line_count?: number; + validation_status?: "completed" | "failed" | null; + validation_shapes_passed?: number | null; + validation_shapes_total?: number | null; + validation_fully_validated?: boolean | null; + validation_geomean_speedup?: number | null; + validation_contract_version?: string | null; + validation_checked_at?: string | null; }> >; } diff --git a/frontend/src/pages/leaderboard/Leaderboard.test.tsx b/frontend/src/pages/leaderboard/Leaderboard.test.tsx index 7ec63df..c409695 100644 --- a/frontend/src/pages/leaderboard/Leaderboard.test.tsx +++ b/frontend/src/pages/leaderboard/Leaderboard.test.tsx @@ -9,6 +9,10 @@ vi.mock("../../lib/hooks/useApi", () => ({ fetcherApiCallback: vi.fn(), })); +vi.mock("./components/UserTrendChart", () => ({ + default: () =>
, +})); + // Mutable auth state for mocking useAuthStore per test type AuthState = { me: null | { authenticated: boolean; user?: { identity?: string } }; @@ -108,6 +112,62 @@ describe("Leaderboard", () => { expect(screen.getByText(/custom_kernel/)).toBeInTheDocument(); }); + it("shows full and partial application validation badges beside submitters", () => { + const mockData = { + deadline: mockDeadline, + description: mockDescription, + name: mockName, + reference: mockReference, + starter: mockStarter, + gpu_types: ["B200"], + rankings: { + B200: [ + { + file_name: "fast.py", + prev_score: 0, + rank: 1, + score: 3.25, + user_name: "stable-user", + submission_id: 101, + validation_status: "completed", + validation_shapes_passed: 8, + validation_shapes_total: 8, + validation_fully_validated: true, + validation_geomean_speedup: 1.4, + validation_contract_version: "v1", + }, + { + file_name: "partial.py", + prev_score: 0.1, + rank: 2, + score: 3.5, + user_name: "partial-user", + submission_id: 102, + validation_status: "completed", + validation_shapes_passed: 6, + validation_shapes_total: 8, + validation_fully_validated: false, + validation_geomean_speedup: 1.1, + validation_contract_version: "v1", + }, + ], + }, + }; + + (apiHook.fetcherApiCallback as ReturnType).mockReturnValue({ + data: mockData, + loading: false, + error: null, + errorStatus: null, + call: mockCall, + }); + + renderWithRouter(); + + expect(screen.getByText("VALIDATED")).toBeInTheDocument(); + expect(screen.getByText("6/8 VALIDATED")).toBeInTheDocument(); + }); + it("shows loading state", () => { (apiHook.fetcherApiCallback as ReturnType).mockReturnValue({ data: null, @@ -118,7 +178,7 @@ describe("Leaderboard", () => { }); renderWithRouter(); - expect(screen.getByText(/Summoning/i)).toBeInTheDocument(); + expect(screen.getByRole("progressbar")).toBeInTheDocument(); }); it("shows error message", () => { @@ -264,17 +324,18 @@ describe("Leaderboard", () => { const btn = screen.queryByTestId("ranking-show-all-button-0"); expect(btn).toBeInTheDocument(); - // By default only 3 rows shown - expect(screen.queryAllByTestId("ranking-0-row")).toHaveLength(3); - - // Click to show all - fireEvent.click(btn!); + // Rankings start expanded. expect(screen.queryAllByTestId("ranking-0-row")).toHaveLength(4); expect(within(btn!).getByText(/Hide/i)).toBeInTheDocument(); - // Click to hide again + // Hide the rows after the top three. fireEvent.click(btn!); expect(screen.queryAllByTestId("ranking-0-row")).toHaveLength(3); + expect(within(btn!).getByText(/Show all/i)).toBeInTheDocument(); + + // Expand again. + fireEvent.click(btn!); + expect(screen.queryAllByTestId("ranking-0-row")).toHaveLength(4); }); // -------------------- Starter codeblock -------------------- @@ -510,9 +571,9 @@ describe("Leaderboard", () => { // Switch to the Submission tab explicitly fireEvent.click(screen.getByRole("tab", { name: /Submission/i })); - const submit_btn = screen.getByTestId("leaderboard-submit-btn"); - expect(submit_btn).toBeInTheDocument(); - expect(submit_btn).toBeDisabled(); + expect( + screen.queryByTestId("leaderboard-submit-btn"), + ).not.toBeInTheDocument(); const deadline_txt = screen.getByTestId("deadline-passed-text"); expect( diff --git a/frontend/src/pages/leaderboard/Leaderboard.tsx b/frontend/src/pages/leaderboard/Leaderboard.tsx index 205a97e..699910e 100644 --- a/frontend/src/pages/leaderboard/Leaderboard.tsx +++ b/frontend/src/pages/leaderboard/Leaderboard.tsx @@ -178,8 +178,8 @@ const LeaderboardContent = memo(function LeaderboardContent() { findTopUsers(); }, [id, data?.rankings]); - if (loading || !data) return ; if (error) return ; + if (loading || !data) return ; if (!data) return null; const toDeadlineLocal = (raw: string) => { diff --git a/frontend/src/pages/leaderboard/components/RankingLists.tsx b/frontend/src/pages/leaderboard/components/RankingLists.tsx index 24b8b2c..957851e 100644 --- a/frontend/src/pages/leaderboard/components/RankingLists.tsx +++ b/frontend/src/pages/leaderboard/components/RankingLists.tsx @@ -2,8 +2,10 @@ import { useState } from "react"; import { Box, Button, + Chip, Grid, Stack, + Tooltip, type SxProps, type Theme, Typography, @@ -30,6 +32,13 @@ interface RankingItem { submission_count?: number; submission_time?: string; line_count?: number; + validation_status?: "completed" | "failed" | null; + validation_shapes_passed?: number | null; + validation_shapes_total?: number | null; + validation_fully_validated?: boolean | null; + validation_geomean_speedup?: number | null; + validation_contract_version?: string | null; + validation_checked_at?: string | null; } interface RankingsListProps { @@ -97,6 +106,58 @@ const styles: Record> = { }, }; +function ValidationBadge({ item }: { item: RankingItem }) { + if (!item.validation_status) return null; + + const passed = item.validation_shapes_passed ?? 0; + const total = item.validation_shapes_total ?? 0; + const fullyValidated = + item.validation_status === "completed" && + item.validation_fully_validated === true && + total > 0 && + passed === total; + const failed = item.validation_status === "failed"; + const label = fullyValidated + ? "VALIDATED" + : failed + ? "VALIDATION ERROR" + : `${passed}/${total} VALIDATED`; + const speedup = + typeof item.validation_geomean_speedup === "number" + ? ` Geomean synchronized-wall speedup across measured shapes: ${item.validation_geomean_speedup.toFixed(2)}×.` + : ""; + const contract = item.validation_contract_version + ? ` Contract: ${item.validation_contract_version}.` + : ""; + const checked = item.validation_checked_at + ? ` Checked ${new Date(item.validation_checked_at).toLocaleString()}.` + : ""; + const tooltip = fullyValidated + ? `All ${passed}/${total} training shapes passed the convergence, numerical, and speed gates.${speedup}${contract}${checked}` + : failed + ? `The validation job failed before it could complete.${contract}${checked}` + : `${passed}/${total} training shapes passed the convergence, numerical, and speed gates.${speedup}${contract}${checked}`; + + return ( + + + + ); +} + export default function RankingsList({ rankings, leaderboardId, @@ -114,7 +175,7 @@ export default function RankingsList({ const toggleExpanded = (field: string) => { setExpanded((prev) => ({ ...prev, - [field]: !prev[field], + [field]: !(prev[field] ?? true), })); }; @@ -205,9 +266,19 @@ export default function RankingsList({ {item.rank}. - - {item.user_name} {getMedalIcon(item.rank)} - + + + {item.user_name} {getMedalIcon(item.rank)} + + + @@ -222,7 +293,7 @@ export default function RankingsList({ - )} - + {showLineCount && ( diff --git a/kernelboard/api/leaderboard.py b/kernelboard/api/leaderboard.py index 4f0b4ab..59b6434 100644 --- a/kernelboard/api/leaderboard.py +++ b/kernelboard/api/leaderboard.py @@ -265,11 +265,38 @@ def _get_query(): s.file_name AS file_name, r.submission_id AS submission_id, COALESCE(sc.submission_count, 0) AS submission_count, + validation.status AS validation_status, + validation.passed_shapes AS validation_shapes_passed, + validation.total_shapes AS validation_shapes_total, + validation.fully_validated AS validation_fully_validated, + validation.geomean_sync_wall_speedup AS validation_geomean_speedup, + validation.contract_version AS validation_contract_version, + validation.checked_at AS validation_checked_at, RANK() OVER (PARTITION BY r.runner, u.id ORDER BY r.score ASC) AS rank FROM leaderboard.runs r JOIN leaderboard.submission s ON r.submission_id = s.id LEFT JOIN leaderboard.user_info u ON s.user_id = u.id LEFT JOIN submission_counts sc ON s.user_id = sc.user_id AND r.runner = sc.runner + LEFT JOIN LATERAL ( + SELECT + status, + passed_shapes, + total_shapes, + fully_validated, + geomean_sync_wall_speedup, + contract_version, + checked_at + FROM leaderboard.submission_validation + WHERE submission_id = s.id + AND gpu_type = r.runner + AND contract_version = ( + SELECT task->'validation'->>'version' + FROM leaderboard.leaderboard + WHERE id = %(leaderboard_id)s + ) + ORDER BY checked_at DESC + LIMIT 1 + ) validation ON TRUE WHERE NOT r.secret AND r.score IS NOT NULL AND r.passed @@ -312,7 +339,14 @@ def _get_query(): 'file_name', r.file_name, 'submission_id', r.submission_id, 'submission_count', r.submission_count, - 'submission_time', r.submission_time + 'submission_time', r.submission_time, + 'validation_status', r.validation_status, + 'validation_shapes_passed', r.validation_shapes_passed, + 'validation_shapes_total', r.validation_shapes_total, + 'validation_fully_validated', r.validation_fully_validated, + 'validation_geomean_speedup', r.validation_geomean_speedup, + 'validation_contract_version', r.validation_contract_version, + 'validation_checked_at', r.validation_checked_at ) ORDER BY r.score ASC ) diff --git a/tests/api/test_leaderboard_api.py b/tests/api/test_leaderboard_api.py index 7bcfadc..fce1730 100644 --- a/tests/api/test_leaderboard_api.py +++ b/tests/api/test_leaderboard_api.py @@ -49,6 +49,92 @@ def test_active_leaderboard_omits_line_counts(client, app): assert all("line_count" not in item for item in ranked_items) +def test_leaderboard_includes_latest_validation_summary(client, app): + initial = client.get("/api/leaderboard/339").get_json() + initial_ids = { + gpu: [item["submission_id"] for item in items] + for gpu, items in initial["data"]["rankings"].items() + } + gpu_type, rankings = next(iter(initial["data"]["rankings"].items())) + submission_id = rankings[0]["submission_id"] + + with app.app_context(): + conn = get_db_connection() + with conn.cursor() as cur: + cur.execute( + """ + UPDATE leaderboard.leaderboard + SET task = jsonb_set( + task, + '{validation}', + '{"version": "v1"}'::jsonb + ) + WHERE id = 339 + """ + ) + cur.execute( + """ + INSERT INTO leaderboard.submission_validation ( + submission_id, gpu_type, contract_version, + status, passed_shapes, total_shapes, fully_validated, + geomean_sync_wall_speedup, result + ) + VALUES (%s, %s, 'v1', + 'completed', 8, 8, TRUE, 1.25, '{}') + """, + (submission_id, gpu_type), + ) + conn.commit() + + response = client.get("/api/leaderboard/339") + assert response.status_code == 200 + returned_rankings = response.get_json()["data"]["rankings"] + assert { + gpu: [item["submission_id"] for item in items] + for gpu, items in returned_rankings.items() + } == initial_ids + validated = next( + item + for item in returned_rankings[gpu_type] + if item["submission_id"] == submission_id + ) + assert validated["validation_status"] == "completed" + assert validated["validation_shapes_passed"] == 8 + assert validated["validation_shapes_total"] == 8 + assert validated["validation_fully_validated"] is True + assert validated["validation_geomean_speedup"] == 1.25 + assert validated["validation_contract_version"] == "v1" + + with app.app_context(): + conn = get_db_connection() + with conn.cursor() as cur: + cur.execute( + """ + UPDATE leaderboard.submission_validation + SET status = 'failed', + passed_shapes = 0, + total_shapes = 0, + fully_validated = FALSE + WHERE submission_id = %s AND gpu_type = %s + """, + (submission_id, gpu_type), + ) + conn.commit() + + failed_response = client.get("/api/leaderboard/339") + failed_rankings = failed_response.get_json()["data"]["rankings"] + assert { + gpu: [item["submission_id"] for item in items] + for gpu, items in failed_rankings.items() + } == initial_ids + failed = next( + item + for item in failed_rankings[gpu_type] + if item["submission_id"] == submission_id + ) + assert failed["validation_status"] == "failed" + + def test_leaderboard_line_counts_support_bytea_code_storage(client, app): with app.app_context(): conn = get_db_connection() diff --git a/tests/conftest.py b/tests/conftest.py index 3170a8b..d7e0d20 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,4 @@ +import os import random import secrets import string @@ -131,7 +132,7 @@ def db_server(): "-f", "tests/data.sql", ], - env={"PGPASSWORD": password}, + env={**os.environ, "PGPASSWORD": password}, ) if result.returncode != 0: diff --git a/tests/data.sql b/tests/data.sql index c99155e..94df941 100644 --- a/tests/data.sql +++ b/tests/data.sql @@ -9219,6 +9219,23 @@ CREATE TABLE IF NOT EXISTS leaderboard.submission_job_status ( CONSTRAINT uq_submission_job_status_submission_id UNIQUE (submission_id) -- one-to-one with submission ); + +CREATE TABLE IF NOT EXISTS leaderboard.submission_validation ( + id BIGSERIAL PRIMARY KEY, + submission_id INTEGER NOT NULL + REFERENCES leaderboard.submission(id) ON DELETE CASCADE, + gpu_type TEXT NOT NULL, + contract_version TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('completed', 'failed')), + passed_shapes INTEGER NOT NULL DEFAULT 0, + total_shapes INTEGER NOT NULL DEFAULT 0, + fully_validated BOOLEAN NOT NULL DEFAULT FALSE, + geomean_sync_wall_speedup DOUBLE PRECISION, + result JSONB NOT NULL DEFAULT '{}'::jsonb, + error TEXT, + checked_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (submission_id, gpu_type, contract_version) +); -- -- PostgreSQL database dump complete --