diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py index c02abcb0f..243dcd16d 100644 --- a/compute_worker/compute_worker.py +++ b/compute_worker/compute_worker.py @@ -968,7 +968,7 @@ async def _run_container_engine_cmd(self, container, kind): await websocket.wait_closed() except Exception as e: logger.error(e) - client.remove_container(container, force=True) + client.remove_container(container, v=True, force=True) logger.debug(f"Container {container.get('Id')} exited with status code : {str(return_Code['StatusCode'])}") @@ -983,7 +983,7 @@ async def _run_container_engine_cmd(self, container, kind): finally: try: # Last chance of removing container - client.remove_container(container.get("Id"), force=True) + client.remove_container(container.get("Id"), v=True, force=True) except Exception: pass @@ -1347,7 +1347,7 @@ def start(self): for container in containers_to_kill: try: - client.remove_container(str(container), force=True) + client.remove_container(str(container), v=True, force=True) except docker.errors.APIError as e: logger.error(e) except Exception as e: @@ -1398,7 +1398,7 @@ def start(self): containers_to_kill = self.scoring_program_container_name try: client.kill(containers_to_kill) - client.remove_container(containers_to_kill, force=True) + client.remove_container(containers_to_kill, v=True, force=True) except docker.errors.APIError as e: logger.error(e) except Exception as e: diff --git a/docker-compose.yml b/docker-compose.yml index f5b32b753..5b66662c7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -200,7 +200,7 @@ services: #---------------------------------------------------------------------------------------------------- site_worker: # This auto-reloads - command: ["watchmedo auto-restart -p '*.py' --recursive -- celery -A celery_config worker -B -Q site-worker -l info -n site-worker@%n --concurrency=2"] + command: ["celery -A celery_config worker -B -Q site-worker -l info -n site-worker@%n --concurrency=2"] working_dir: /app/src container_name: site_worker image: django_site-worker diff --git a/documentation/docs/Developers_and_Administrators/Automating-with-Selenium.md b/documentation/docs/Developers_and_Administrators/Automating-with-Selenium.md deleted file mode 100644 index 76482641d..000000000 --- a/documentation/docs/Developers_and_Administrators/Automating-with-Selenium.md +++ /dev/null @@ -1,73 +0,0 @@ - -## What and Why -It's useful to test various parts of the system with lots of data or many intricate actions. The selenium tests do this generally and are used as a guide for this tutorial. One problem is that Selenium needs to launch an instance of a browser to control. Our tests do this inside a docker container and uses a test database that doesn't persist as it cleans up after itself. We need to be able to control a live codabench session that is running. To do that we install a driver locally which is normally only inside the selenium docker container during tests. It is specific to your browser so keep that in mind. - -## Virtualenv -You'll need a python virtual env as you don't want to be inside Django or you won't be able to launch a browser. - -### Virtualenv -I used 3.8. -```bash -python3 -m venv codabench -source ./codabench/bin/activate -``` - -### Pyenv -```bash -pyenv install 3.8 -pyenv virtualenv 3.8 codabench -pyenv activate codabench -``` - -## Requirements -We have a couple extra things like `webdriver-manager` for getting a driver programmatically and `selenium` needs to be upgraded to use modern client interface. -```bash -pip install -r requitements.txt -pip install -r requitements.dev.txt -pip install webdriver-manager -pip install --upgrade selenium -``` - -## Automate competition creation -[Main Selenium Docs](https://selenium-python.readthedocs.io/) -[Install](https://selenium-python.readthedocs.io/installation.html) -[Getting Started](https://selenium-python.readthedocs.io/getting-started.html) - -```python -import os, time -from selenium import webdriver -from selenium.webdriver.common.by import By -from selenium.webdriver.chrome.service import Service -from webdriver_manager.chrome import ChromeDriverManager - -# Use `ChromeDriverManager` to ensure the `chromedriver` is installed and in PATH -service = Service(ChromeDriverManager().install()) -driver = webdriver.Chrome(service=service) - -# ... now use `driver` to control the local Chrome instance -driver.get("http://localhost/accounts/login") - -# Use CSS selectors to find the input fields and button -username_input = driver.find_element(By.CSS_SELECTOR, 'input[name="username"]') -password_input = driver.find_element(By.CSS_SELECTOR, 'input[name="password"]') -submit_button = driver.find_element(By.CSS_SELECTOR, '.submit.button') - -# Type the credentials into the fields -username_input.send_keys('bbearce') -password_input.send_keys('testtest') - -# Click the submit button -submit_button.click() - -comp_path = "/home/bbearce/Documents/codabench/src/tests/functional/test_files/competition_v2_multi_task.zip" -def upload_competition(competition_zip_path): - driver.get("http://localhost/competitions/upload") - file_input = driver.find_element(By.CSS_SELECTOR, 'input[ref="file_input"]') - file_input.send_keys(os.path.join(competition_zip_path)) - - -for i in range(30): - upload_competition(comp_path) - time.sleep(5) # tune for your system - -``` \ No newline at end of file diff --git a/documentation/docs/Organizers/Benchmark_Creation/Leaderboard-Functionality.md b/documentation/docs/Organizers/Benchmark_Creation/Leaderboard-Functionality.md index ded5fffa4..799e971b6 100644 --- a/documentation/docs/Organizers/Benchmark_Creation/Leaderboard-Functionality.md +++ b/documentation/docs/Organizers/Benchmark_Creation/Leaderboard-Functionality.md @@ -49,6 +49,7 @@ Computation options are: - avg - min - max + - avg_rank These are applied across the columns specified as `computation_indexes`. diff --git a/documentation/docs/Organizers/Benchmark_Creation/Yaml-Structure.md b/documentation/docs/Organizers/Benchmark_Creation/Yaml-Structure.md index 16bf7b0e3..12c8ffaa0 100644 --- a/documentation/docs/Organizers/Benchmark_Creation/Yaml-Structure.md +++ b/documentation/docs/Organizers/Benchmark_Creation/Yaml-Structure.md @@ -259,7 +259,7 @@ fact_sheet: { - Ascending: smaller scores are better - Descending: larger scores are better - **computation:** computation to be applied *must be accompanied by computation indexes* - - computation options: sum, avg, min, max + - computation options: sum, avg, min, max, avg_rank - **computation_indexes:** an array of indexes of the columns the computation should be applied to - **precision:** (*integer, default=2*) to round the score to *precision* number of digits - **hidden:** (*boolean, default=False*) to hide/unhide a column on leaderboard diff --git a/documentation/docs/Project_CodaBench_FAQ.md b/documentation/docs/Project_CodaBench_FAQ.md index 40f5ad780..e6e7f0fb8 100644 --- a/documentation/docs/Project_CodaBench_FAQ.md +++ b/documentation/docs/Project_CodaBench_FAQ.md @@ -5,8 +5,8 @@ Codabench benchmarks are aimed at researchers, scientists and other professionals who want to track algorithm performance via benchmarks or have participants participate in a competition to find the best solution to a problem. We run a free public instance at [https://www.codabench.org/](https://www.codabench.org/) and the raw code is on [Github](https://github.com/codalab/codabench). -### Can CodaLab competitions be privately hosted? -Yes, you can host your own CodaLab instance on a private or hosted server (e.g. Azure, GCP or AWS). For more information, see [how to deploy Codabench on your server](Developers_and_Administrators/How-to-deploy-Codabench-on-your-server.md) and [local installation](Developers_and_Administrators/Codabench-Installation.md) guide. However, most benchmark organizers do NOT need to run their own instance. If you run a computationally demanding competition, you can hook up your own [compute workers](Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md) in the backend very easily. +### Can Codabench be privately hosted? +Yes, you can host your own Codabench instance on a private or hosted server (e.g. Azure, GCP or AWS). For more information, see [how to deploy Codabench on your server](Developers_and_Administrators/How-to-deploy-Codabench-on-your-server.md) and [local installation](Developers_and_Administrators/Codabench-Installation.md) guide. However, most benchmark organizers do NOT need to run their own instance. If you run a computationally demanding competition, you can hook up your own [compute workers](Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md) in the backend very easily. ### How to change my username? diff --git a/documentation/uv.lock b/documentation/uv.lock index 5b7d224d3..dde83e83d 100644 --- a/documentation/uv.lock +++ b/documentation/uv.lock @@ -4,14 +4,14 @@ requires-python = ">=3.14" [[package]] name = "click" -version = "8.3.3" +version = "8.4.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bb/63/f9e1ea081ce35720d8b92acde70daaedace594dc93b693c869e0d5910718/click-8.3.3.tar.gz", hash = "sha256:398329ad4837b2ff7cbe1dd166a4c0f8900c3ca3a218de04466f38f6497f18a2", size = 328061, upload-time = "2026-04-22T15:11:27.506Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9b/98/518d8e5081007684232226f475082b30087d0f585e8457db087298259f49/click-8.4.1.tar.gz", hash = "sha256:918b5633eddf6b41c32d4f454bf0de810065c74e3f7dbf8ee5452f8be88d3e96", size = 353007, upload-time = "2026-05-22T04:08:37.769Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ae/44/c1221527f6a71a01ec6fbad7fa78f1d50dfa02217385cf0fa3eec7087d59/click-8.3.3-py3-none-any.whl", hash = "sha256:a2bf429bb3033c89fa4936ffb35d5cb471e3719e1f3c8a7c3fff0b8314305613", size = 110502, upload-time = "2026-04-22T15:11:25.044Z" }, + { url = "https://files.pythonhosted.org/packages/c7/0d/67e5b4109ea4a837e80daa87c2c696711955e40449a97e8926672534def2/click-8.4.1-py3-none-any.whl", hash = "sha256:482be17c6991b8c19c5429a1e995d9b0efdbb63172824c41f99965dc0ade8ec2", size = 116639, upload-time = "2026-05-22T04:08:35.26Z" }, ] [[package]] @@ -204,7 +204,7 @@ wheels = [ [[package]] name = "zensical" -version = "0.0.40" +version = "0.0.45" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, @@ -216,18 +216,18 @@ dependencies = [ { name = "pyyaml" }, { name = "tomli" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ba/a6/88062f7e235f58a5f05d82005fc35d9dbaed27c024fe9ffae5bce7f33661/zensical-0.0.40.tar.gz", hash = "sha256:5c294751977a664614cb84e987186ad8e282af77ce0d0d800fe48ee57791279d", size = 3920555, upload-time = "2026-05-04T16:19:07.962Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/d1/ecb1889fd2208b2d577e6ff952d9bee201302eec7966b5b61cc64adfd8f5/zensical-0.0.45.tar.gz", hash = "sha256:315bce4ab0470338dd3588add38fb325f840856c375722e6802bd58a06446266", size = 3935947, upload-time = "2026-06-09T11:23:32.349Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c1/c4/3066f4442923ca1e49269147b70ca7c84467524e8f5228724693b9ac85c2/zensical-0.0.40-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:b65a7143c9c6a460880bf3e65b777952bd2dcede9dd17a6c6bac9b4a0686ad9b", size = 12691533, upload-time = "2026-05-04T16:18:31.72Z" }, - { url = "https://files.pythonhosted.org/packages/5a/cb/03e961cbd01620ea91aeb835b0b4e8848c7bcdf5a799a620fb3e57bfc277/zensical-0.0.40-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:045bdcb6d00a11ddcab7d379d0d986cdf78dba8e9287d8e628ef11958241507d", size = 12556486, upload-time = "2026-05-04T16:18:35.278Z" }, - { url = "https://files.pythonhosted.org/packages/60/76/7dde50220808bdc5f5e63b97866a684418410b3cae9d00cdae1d449bcc20/zensical-0.0.40-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d48ec476c2e8ce3f8585a1278083aabc35ec80361f2c4fc4a53b9a525778f7fc", size = 12935602, upload-time = "2026-05-04T16:18:38.308Z" }, - { url = "https://files.pythonhosted.org/packages/51/55/6c8ef951c390b42249738f4338498e7a1fd64ff09e44d7cc19f5c948c45b/zensical-0.0.40-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:48c38e0ae314c25f2e5e64210bbad9be6e970f2d40fe9da106586ad90ce5e85e", size = 12904314, upload-time = "2026-05-04T16:18:41.007Z" }, - { url = "https://files.pythonhosted.org/packages/f4/ae/95008f5dc2ee441efcdc2fab36ff29ce24d7477e53390fc340c8add39342/zensical-0.0.40-cp310-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f25f62dcd61f6306cab890dfa34c81d2709f5db290b4c3f2675343771db28c90", size = 13269946, upload-time = "2026-05-04T16:18:44.387Z" }, - { url = "https://files.pythonhosted.org/packages/b9/96/cdbb2bf04255ccaaa07861bdda1ee8dd1630d2233fc2f09636abbd5e084c/zensical-0.0.40-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:168fe3489dd93ae92978b4db11d9300c63e10d382b81634232c2872ce9e746c2", size = 12974962, upload-time = "2026-05-04T16:18:47.462Z" }, - { url = "https://files.pythonhosted.org/packages/6f/ce/66e86f89fc15bbe667794ba67d7efc8fa72fe7a1be19e1efb4246ff55442/zensical-0.0.40-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:8652ba203bd588ebf2d66bda4457a4a7d8e193c886960859c75081c0e3b946de", size = 13111599, upload-time = "2026-05-04T16:18:50.14Z" }, - { url = "https://files.pythonhosted.org/packages/87/76/3d71ebdabb02d79a5c523b5e646141c362c9559947078c8d56a9f3bd7a30/zensical-0.0.40-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:9ffa6cf208b7ab6b771703be827d4d8c7f07f173abeffb35a8015a0b832b2a40", size = 13175406, upload-time = "2026-05-04T16:18:53.209Z" }, - { url = "https://files.pythonhosted.org/packages/e2/6a/2bb5f730786d590f02cb0fef796c148d5ac0d5c1556f2d78c987ad4e1346/zensical-0.0.40-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:7101ba0c739c78bc3a57d22130b59b9e6fdf96c21c8a6b4244070de6b34527d4", size = 13324783, upload-time = "2026-05-04T16:18:56.41Z" }, - { url = "https://files.pythonhosted.org/packages/2f/8c/1d2ba1454360ee948dd0f0807b048c076d9578d0d9ebba2a438ecfa9f82f/zensical-0.0.40-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:39bf728a68a5418feeda8f3385cd1063fdb8d896a6812c3dede4267b2868df12", size = 13260045, upload-time = "2026-05-04T16:18:59.244Z" }, - { url = "https://files.pythonhosted.org/packages/6c/61/efd51c5c5e15cfd5498d59df250f60294cc44d36d8ce4dc2a76fa3669c2f/zensical-0.0.40-cp310-abi3-win32.whl", hash = "sha256:bc750c3ba8d11833d9b9ac8fc14adc3435225b6d17314a21a91eb60209511ca5", size = 12244913, upload-time = "2026-05-04T16:19:02.219Z" }, - { url = "https://files.pythonhosted.org/packages/fe/9e/f3f2118fbcfd1c2dc705491c8864c596b1a748b67ffe2a024e512b9201ab/zensical-0.0.40-cp310-abi3-win_amd64.whl", hash = "sha256:c5c86ac468df2dfe515ff54ffa97725c38226f1e5c970059b7e88078abab89ab", size = 12475762, upload-time = "2026-05-04T16:19:05.025Z" }, + { url = "https://files.pythonhosted.org/packages/ad/fd/6b84115e3bbe6b76ebb1265e8ff2161c0bc88dcd6499eaf29c61a66421e9/zensical-0.0.45-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c4cb2e11132f02ae824e246e016e073458e12e9de1eaf86fd39f01890d41204c", size = 12698844, upload-time = "2026-06-09T11:22:56.537Z" }, + { url = "https://files.pythonhosted.org/packages/e2/dc/4ddf05d77c1455c32cb26da71f2a19d355927a45a3db5b26fb258a07ce8f/zensical-0.0.45-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:799a01de2102b5f731744ad31bdbc464d0c07d484e67ba148f6923679afa6ce6", size = 12571590, upload-time = "2026-06-09T11:23:00.192Z" }, + { url = "https://files.pythonhosted.org/packages/4c/53/60c6cc7b2ce8b1a83eb87bff3f7289447995552fd9a30ca76ffba22ca9d5/zensical-0.0.45-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6201e79ea8a64bd3ced3f05ef4b1529da0e675d67b1395987c0ba942e4e10dc4", size = 12939590, upload-time = "2026-06-09T11:23:02.721Z" }, + { url = "https://files.pythonhosted.org/packages/9f/1e/e9217ed75dba323a6f9a4eee28eb40416eff99932cd0ee6c394bf07b9ead/zensical-0.0.45-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:854aaf500e4a3ce64adea1faa7a1820c7cf9a4f66be1043e4e9ba727fe9cf2b5", size = 12911669, upload-time = "2026-06-09T11:23:05.407Z" }, + { url = "https://files.pythonhosted.org/packages/71/3c/6fc9fe2334bb4460a8a8d732e23a30d2ddc2ecf63c2eb3487d9e7405e70d/zensical-0.0.45-cp310-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a80c57fd50fc60415914388286ac10a7d8b6f70b8ca7235597d09fb12c3171b0", size = 13267643, upload-time = "2026-06-09T11:23:07.915Z" }, + { url = "https://files.pythonhosted.org/packages/be/f9/5696114af4ede5f1bd01e641a4ff24ee8ca49810bfaa28e5be12d930c0ef/zensical-0.0.45-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58c3510f69e08b6ed8bb9596fc9393e4687f90394aa0ef2d6118b1375ad97be5", size = 12972147, upload-time = "2026-06-09T11:23:12.069Z" }, + { url = "https://files.pythonhosted.org/packages/a3/9e/5c6acde480c43f8c993b13260925df8db31d51ab8a9977618e9efdd98d45/zensical-0.0.45-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:01c484bb2ee85e98e21e24b397ff52ffc31101f7485935eee5d3afa6cca6cc08", size = 13117360, upload-time = "2026-06-09T11:23:15.155Z" }, + { url = "https://files.pythonhosted.org/packages/3d/31/ea21f102049b35a8fe5218c5331857a15eeb60deb1bb21823a4c0701e274/zensical-0.0.45-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:3654b708830303759e866a58a60c483cd2a1c56a44acdaae5bbb341a3f40ebce", size = 13185593, upload-time = "2026-06-09T11:23:18.166Z" }, + { url = "https://files.pythonhosted.org/packages/b4/97/6ded39fe27fa8a292d17d9af713b018e4919315233b60fa4b4b0aca737a6/zensical-0.0.45-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:c4da1c37eca1474b487def0ef40d7ac2aff31a9d7a029cb7479ef7c354437361", size = 13326882, upload-time = "2026-06-09T11:23:21.027Z" }, + { url = "https://files.pythonhosted.org/packages/79/80/075975032a9e20f319c0134f8ca659d295ee4908f15ab212702a2728247f/zensical-0.0.45-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:f8a1966c186feebd3b795f9d420000bfd582e16eefdd9bc7a286d878faabae52", size = 13253961, upload-time = "2026-06-09T11:23:23.99Z" }, + { url = "https://files.pythonhosted.org/packages/3f/6a/0eab0eb311af6a07cde15ca58d5d720cbfa02cd509e4c7fb5fa20cda0b46/zensical-0.0.45-cp310-abi3-win32.whl", hash = "sha256:a1dd63a5efb8d0e5f2fadf862f02771a279dc5cbe9a982700194650065758f01", size = 12257083, upload-time = "2026-06-09T11:23:26.769Z" }, + { url = "https://files.pythonhosted.org/packages/1f/cd/b117e749c60b1d1e16b8450db1355f69f38376f783b8c6c8815202988933/zensical-0.0.45-cp310-abi3-win_amd64.whl", hash = "sha256:1f2c0e69839ce4274bde34d18139d3b0d96bbf02b245ada46243590c9eedebc1", size = 12498335, upload-time = "2026-06-09T11:23:29.702Z" }, ] diff --git a/documentation/zensical.toml b/documentation/zensical.toml index 9382bb149..e5f38ff24 100644 --- a/documentation/zensical.toml +++ b/documentation/zensical.toml @@ -7,6 +7,7 @@ nav = [ {"Home" = "index.md"}, {"Participants" = [ {"Participating in a Competition" = "Participants/User_Participating-in-a-Competition.md"}, + {"Robot Submissions" = "Developers_and_Administrators/Robot-submissions.md"}, {"List of Current Benchmarks and Competitions" = "https://www.codabench.org/competitions/public/?page=1"} ]}, {"Organizers" = [ @@ -37,20 +38,21 @@ nav = [ {"Server Status" = "Organizers/Running_a_benchmark/Server-status-page.md"} ]} ]}, - {"Developers and Administrators" = [ - {"Codabench Basic Installation Guide" = "Developers_and_Administrators/Codabench-Installation.md"}, - {"How to Deploy a Server" = "Developers_and_Administrators/How-to-deploy-Codabench-on-your-server.md"}, - {"Administrative Procedures" = "Developers_and_Administrators/Administrator-procedures.md"}, + {"Developers" = [ {"Codabench Docker Architecture" = "Developers_and_Administrators/Codabench-Architecture.md"}, {"Submission Docker Container Layout" = "Developers_and_Administrators/Submission-Docker-Container-Layout.md"}, - {"Backups - Automating Creation and Restoring" = "Developers_and_Administrators/Creating-and-Restoring-from-Backup.md"}, {"Submission Process Overview" = "Developers_and_Administrators/Submission-Process-Overview.md"}, - {"Robot Submissions" = "Developers_and_Administrators/Robot-submissions.md"}, {"Adding Tests" = "Developers_and_Administrators/Adding-e2e-tests.md"}, {"Running Tests" = "Developers_and_Administrators/Running-tests.md"}, - {"Automation" = "Developers_and_Administrators/Automating-with-Selenium.md"}, - {"Manual Validation" = "Developers_and_Administrators/Manual-validation.md"}, {"Validation and deployement of pull requests" = "Developers_and_Administrators/Validation-and-deployment-of-pull-requests.md"}, + {"Manual Validation" = "Developers_and_Administrators/Manual-validation.md"}, + + ]}, + {"Self-Hosters" = [ + {"Codabench Basic Installation Guide" = "Developers_and_Administrators/Codabench-Installation.md"}, + {"How to Deploy a Server" = "Developers_and_Administrators/How-to-deploy-Codabench-on-your-server.md"}, + {"Administrative Procedures" = "Developers_and_Administrators/Administrator-procedures.md"}, + {"Backups - Automating Creation and Restoring" = "Developers_and_Administrators/Creating-and-Restoring-from-Backup.md"}, {" Upgrading Codabench" = [ "Developers_and_Administrators/Upgrading_Codabench/index.md", {"Upgrade RabbitMQ (version < 1.0.0)" = "Developers_and_Administrators/Upgrading_Codabench/Upgrade-RabbitMQ.md"}, @@ -96,6 +98,7 @@ features = [ "navigation.expand", "navigation.instant", "navigation.instant.progress", + "navigation.indexes", ] font = false # Palette toggle for automatic mode @@ -154,6 +157,7 @@ configurations = [ ]} ] [project.markdown_extensions.pymdownx.smartsymbols] +[project.markdown_extensions.zensical.extensions.glightbox] # Extras [project.extra] footer_links = [ diff --git a/src/apps/announcements/admin.py b/src/apps/announcements/admin.py index 16fc499db..892fb0080 100644 --- a/src/apps/announcements/admin.py +++ b/src/apps/announcements/admin.py @@ -11,6 +11,7 @@ class NewsPostExpansion(admin.ModelAdmin): class AnnouncementExpansion(admin.ModelAdmin): list_display = ["id", "text_limited"] list_display_links = ["id", "text_limited"] + ordering = ('-id',) @admin.display(description="text", ordering="text") def text_limited(self, obj): diff --git a/src/apps/api/serializers/leaderboards.py b/src/apps/api/serializers/leaderboards.py index 5b5c5e025..b4e244e1f 100644 --- a/src/apps/api/serializers/leaderboards.py +++ b/src/apps/api/serializers/leaderboards.py @@ -99,13 +99,6 @@ class Meta: def get_submissions(self, instance): primary_col = instance.columns.get(index=instance.primary_index) - - ordering = [ - F('primary_col').desc(nulls_last=True) - if primary_col.sorting == 'desc' - else F('primary_col').asc(nulls_last=True) - ] - submissions_qs = ( Submission.objects.filter( leaderboard=instance, @@ -129,25 +122,31 @@ def get_submissions(self, instance): ), ) ) - .annotate(primary_col=Sum( - 'scores__score', - filter=Q(scores__column=primary_col) - )) ) + # AVERAGE_RANK columns have no stored scores; skip DB-level sort and re-sort in the view. + if primary_col.computation == Column.AVERAGE_RANK: + ordering = ['created_when'] + submissions = submissions_qs + else: + ordering = [f'{"-" if primary_col.sorting == "desc" else ""}primary_col'] + submissions = submissions_qs.annotate(primary_col=Sum('scores__score', filter=Q(scores__column=primary_col))) + for column in instance.columns.exclude(id=primary_col.id).order_by('index'): + if column.computation == Column.AVERAGE_RANK: + continue col_name = f'col{column.index}' ordering.append( F(col_name).desc(nulls_last=True) if column.sorting == 'desc' else F(col_name).asc(nulls_last=True) ) - submissions_qs = submissions_qs.annotate(**{ + submissions = submissions.annotate(**{ col_name: Sum('scores__score', filter=Q(scores__column__index=column.index)) }) - submissions_qs = submissions_qs.order_by(*ordering, 'created_when') - return SubmissionLeaderBoardSerializer(submissions_qs, many=True).data + submissions = submissions.order_by(*ordering, 'created_when') + return SubmissionLeaderBoardSerializer(submissions, many=True).data class LeaderboardPhaseSerializer(serializers.ModelSerializer): @@ -183,12 +182,7 @@ def get_submissions(self, instance): # desc == -colname # asc == colname primary_col = instance.leaderboard.columns.get(index=instance.leaderboard.primary_index) - ordering = [ - F('primary_col').desc(nulls_last=True) - if primary_col.sorting == 'desc' - else F('primary_col').asc(nulls_last=True) - ] - submissions = ( + submissions_qs = ( Submission.objects.filter( phase=instance, is_soft_deleted=False, @@ -198,14 +192,22 @@ def get_submissions(self, instance): ) .select_related('owner') .prefetch_related('scores', 'scores__column') - .annotate(primary_col=Sum('scores__score', filter=Q(scores__column=primary_col))) ) + # AVERAGE_RANK columns have no stored scores; skip DB-level sort and re-sort in the view. + if primary_col.computation == Column.AVERAGE_RANK: + ordering = ['created_when'] + submissions = submissions_qs + else: + ordering = [f'{"-" if primary_col.sorting == "desc" else ""}primary_col'] + submissions = submissions_qs.annotate(primary_col=Sum('scores__score', filter=Q(scores__column=primary_col))) for column in ( instance.leaderboard.columns .filter(hidden=False) .exclude(id=primary_col.id) .order_by('index') ): + if column.computation == Column.AVERAGE_RANK: + continue col_name = f'col{column.index}' ordering.append( F(col_name).desc(nulls_last=True) diff --git a/src/apps/api/views/competitions.py b/src/apps/api/views/competitions.py index 6eb994f12..a8e4c1b49 100644 --- a/src/apps/api/views/competitions.py +++ b/src/apps/api/views/competitions.py @@ -31,7 +31,8 @@ from datasets.models import Data from competitions.tasks import batch_send_email, manual_migration, create_competition_dump from competitions.utils import get_popular_competitions, get_recent_competitions -from leaderboards.models import Leaderboard +from leaderboards.models import Leaderboard, Column +from leaderboards.ranking import inject_average_ranks from utils.data import make_url_sassy from api.permissions import IsOrganizerOrCollaborator from django.db import transaction @@ -937,6 +938,12 @@ def _clean_group_label(raw_name, submission_parent_id=None): for k, v in submissions_keys.items(): response['submissions'][v]['detailed_results'] = submission_detailed_results[k] + # Compute average rank for any AVERAGE_RANK columns and inject into response. + col_by_index = {col['index']: col for col in columns} + avg_rank_cols = [col for col in columns if col.get('computation') == Column.AVERAGE_RANK] + if avg_rank_cols: + inject_average_ranks(response['submissions'], avg_rank_cols, col_by_index, response['primary_index']) + # --- pagination addition --- total_count = len(response['submissions']) paginator = DynamicChoicePagination() diff --git a/src/apps/competitions/admin.py b/src/apps/competitions/admin.py index b13b703cd..69bc44229 100644 --- a/src/apps/competitions/admin.py +++ b/src/apps/competitions/admin.py @@ -188,6 +188,7 @@ class CompetitionExpansion(admin.ModelAdmin): list_display_links = ["id", "title"] actions = [CompetitionExport_as_json, CompetitionExport_as_csv] raw_id_fields = ["created_by", "collaborators", "queue"] + ordering = ('-id',) list_filter = [ "published", "is_featured", @@ -268,6 +269,7 @@ class SubmissionExpansion(admin.ModelAdmin): "scores", ] search_fields = ["id", "owner__username", "phase__competition__title", "task__name"] + ordering = ('-id',) actions = [SubmissionsExport_as_csv] list_display = [ "id", @@ -351,6 +353,7 @@ class CompetitionCreationTaskStatusExpansion(admin.ModelAdmin): list_display = ["id", "created_by", "resulting_competition", "status"] search_fields = ["id", "created_by__username"] list_filter = ["status"] + ordering = ('-id',) class CompetitionParticipantExpansion(admin.ModelAdmin): @@ -358,18 +361,21 @@ class CompetitionParticipantExpansion(admin.ModelAdmin): list_display = ["id", "user", "competition", "status"] list_filter = ["status"] search_fields = ["id", "user__username", "competition"] + ordering = ('-id',) class PageExpansion(admin.ModelAdmin): raw_id_fields = ["competition"] list_display = ["id", "competition"] search_fields = ["id", "competition", "content"] + ordering = ('-id',) class PhaseExpansion(admin.ModelAdmin): raw_id_fields = ["competition", "leaderboard", "public_data", "starting_kit"] list_display = ["id", "competition", "name"] search_fields = ["id", "competition", "name"] + ordering = ('-id',) fieldsets = [ ( None, diff --git a/src/apps/competitions/models.py b/src/apps/competitions/models.py index 87b0b5513..e5facb4fc 100644 --- a/src/apps/competitions/models.py +++ b/src/apps/competitions/models.py @@ -12,7 +12,7 @@ from decimal import Decimal from celery_config import app, app_for_vhost -from leaderboards.models import SubmissionScore +from leaderboards.models import SubmissionScore, Column from profiles.models import CustomGroup, User, Organization from utils.data import PathWrapper from utils.storage import BundleStorage @@ -697,7 +697,7 @@ def check_child_submission_statuses(self): def calculate_scores(self): # leaderboards = self.phase.competition.leaderboards.all() # for leaderboard in leaderboards: - columns = self.phase.leaderboard.columns.exclude(computation__isnull=True) + columns = self.phase.leaderboard.columns.exclude(computation__isnull=True).exclude(computation=Column.AVERAGE_RANK) for column in columns: scores = self.scores.filter(column__index__in=column.computation_indexes.split(',')).values_list('score', flat=True) diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py index ca5a26987..71acb85c1 100644 --- a/src/apps/competitions/tasks.py +++ b/src/apps/competitions/tasks.py @@ -4,31 +4,36 @@ import traceback import zipfile from datetime import timedelta, datetime - +from django.conf import settings from io import BytesIO -from tempfile import TemporaryDirectory, NamedTemporaryFile +from tempfile import NamedTemporaryFile, TemporaryDirectory import oyaml as yaml import requests from celery._state import app_or_default -from django.conf import settings +from competitions.models import ( + Competition, + CompetitionCreationTaskStatus, + CompetitionDump, + Phase, + Submission, + SubmissionDetails, +) +from competitions.unpackers.utils import CompetitionUnpackingException +from competitions.unpackers.v1 import V15Unpacker +from competitions.unpackers.v2 import V2Unpacker +from datasets.models import Data from django.core.exceptions import ObjectDoesNotExist from django.core.files.base import ContentFile -from django.db.models import Subquery, OuterRef, Count, Case, When, Value, F from django.db import transaction +from django.db.models import Case, Count, F, OuterRef, Subquery, Value, When from django.utils.text import slugify from django.utils.timezone import now +from leaderboards.models import Leaderboard from rest_framework.exceptions import ValidationError +from tasks.models import Task from celery_config import app -from competitions.models import Submission, CompetitionCreationTaskStatus, SubmissionDetails, Competition, \ - CompetitionDump, Phase -from competitions.unpackers.utils import CompetitionUnpackingException -from competitions.unpackers.v1 import V15Unpacker -from competitions.unpackers.v2 import V2Unpacker -from leaderboards.models import Leaderboard -from tasks.models import Task -from datasets.models import Data from utils.data import make_url_sassy from utils.email import codalab_send_markdown_email @@ -53,35 +58,35 @@ "reward", "contact_email", "fact_sheet", - "forum_enabled" + "forum_enabled", ] TASK_FIELDS = [ - 'name', - 'description', - 'key', - 'is_public', + "name", + "description", + "key", + "is_public", ] SOLUTION_FIELDS = [ - 'name', - 'description', - 'tasks', - 'key', + "name", + "description", + "tasks", + "key", ] PHASE_FIELDS = [ - 'index', - 'name', - 'description', - 'start', - 'end', - 'max_submissions_per_day', - 'max_submissions_per_person', - 'execution_time_limit', - 'auto_migrate_to_this_phase', - 'hide_output', - 'hide_prediction_output', - 'hide_score_output', + "index", + "name", + "description", + "start", + "end", + "max_submissions_per_day", + "max_submissions_per_person", + "execution_time_limit", + "auto_migrate_to_this_phase", + "hide_output", + "hide_prediction_output", + "hide_score_output", ] PHASE_FILES = [ "input_data", @@ -91,15 +96,12 @@ "public_data", "starting_kit", ] -PAGE_FIELDS = [ - "title" -] +PAGE_FIELDS = ["title"] LEADERBOARD_FIELDS = [ - 'title', - 'key', - 'hidden', - 'submission_rule', - + "title", + "key", + "hidden", + "submission_rule", # For later # 'force_submission_to_leaderboard', # 'force_best_submission_to_leaderboard', @@ -107,50 +109,18 @@ ] COLUMN_FIELDS = [ - 'title', - 'key', - 'index', - 'sorting', - 'computation', - 'computation_indexes', - 'hidden', - 'precision', + "title", + "key", + "index", + "sorting", + "computation", + "computation_indexes", + "hidden", + "precision", ] -MAX_EXECUTION_TIME_LIMIT = int(os.environ.get('MAX_EXECUTION_TIME_LIMIT', 600)) # time limit of the default queue - - -def _get_user_group_queues(user, competition): - all_user_groups = list( - competition.participant_groups - .filter(user__pk=user.pk) - .select_related('queue') - .distinct() - ) - - if not all_user_groups: - return [] - - groups_with_queue = [g for g in all_user_groups if g.queue_id is not None] - has_groups_without_queue = any(g.queue_id is None for g in all_user_groups) - - if not groups_with_queue: - return [] - - seen_ids = set() - queues = [] - for group in groups_with_queue: - if group.queue_id not in seen_ids: - seen_ids.add(group.queue_id) - queues.append(group.queue) - - if has_groups_without_queue: - competition_queue = competition.queue - if competition_queue is None: - queues.append(None) - elif competition_queue.id not in seen_ids: - queues.append(competition_queue) - - return queues +MAX_EXECUTION_TIME_LIMIT = int( + os.environ.get("MAX_EXECUTION_TIME_LIMIT", 600) +) # time limit of the default queue def _send_to_compute_worker(submission, is_scoring): @@ -159,43 +129,51 @@ def _send_to_compute_worker(submission, is_scoring): "submissions_api_url": settings.SUBMISSIONS_API_URL, "secret": submission.secret, "docker_image": submission.phase.competition.docker_image, - "execution_time_limit": min(MAX_EXECUTION_TIME_LIMIT, submission.phase.execution_time_limit), + "execution_time_limit": min( + MAX_EXECUTION_TIME_LIMIT, submission.phase.execution_time_limit + ), "id": submission.pk, "is_scoring": is_scoring, } - if not submission.detailed_result.name and submission.phase.competition.enable_detailed_results: - submission.detailed_result.save('detailed_results.html', ContentFile(''.encode())) # must encode here for GCS - submission.save(update_fields=['detailed_result']) + if ( + not submission.detailed_result.name + and submission.phase.competition.enable_detailed_results + ): + submission.detailed_result.save( + "detailed_results.html", ContentFile("".encode()) + ) # must encode here for GCS + submission.save(update_fields=["detailed_result"]) if not submission.prediction_result.name: - submission.prediction_result.save('prediction_result.zip', ContentFile(''.encode())) # must encode here for GCS - submission.save(update_fields=['prediction_result']) + submission.prediction_result.save( + "prediction_result.zip", ContentFile("".encode()) + ) # must encode here for GCS + submission.save(update_fields=["prediction_result"]) if not submission.scoring_result.name: - submission.scoring_result.save('scoring_result.zip', ContentFile(''.encode())) # must encode here for GCS - submission.save(update_fields=['scoring_result']) + submission.scoring_result.save( + "scoring_result.zip", ContentFile("".encode()) + ) # must encode here for GCS + submission.save(update_fields=["scoring_result"]) submission = Submission.objects.get(id=submission.id) task = submission.task if not is_scoring: - run_args['prediction_result'] = make_url_sassy( - path=submission.prediction_result.name, - permission='w' + run_args["prediction_result"] = make_url_sassy( + path=submission.prediction_result.name, permission="w" ) else: if submission.phase.competition.enable_detailed_results: - run_args['detailed_results_url'] = make_url_sassy( + run_args["detailed_results_url"] = make_url_sassy( path=submission.detailed_result.name, - permission='w', - content_type='text/html' + permission="w", + content_type="text/html", ) - run_args['prediction_result'] = make_url_sassy( - path=submission.prediction_result.name, - permission='r' + run_args["prediction_result"] = make_url_sassy( + path=submission.prediction_result.name, permission="r" ) - run_args['scoring_result'] = make_url_sassy( - path=submission.scoring_result.name, - permission='w' + run_args["scoring_result"] = make_url_sassy( + path=submission.scoring_result.name, permission="w" ) if task.ingestion_program: @@ -203,12 +181,12 @@ def _send_to_compute_worker(submission, is_scoring): run_args['ingestion_program_data'] = make_url_sassy(task.ingestion_program.data_file.name) if task.input_data and (not is_scoring or task.ingestion_only_during_scoring): - run_args['input_data'] = make_url_sassy(task.input_data.data_file.name) + run_args["input_data"] = make_url_sassy(task.input_data.data_file.name) if is_scoring and task.reference_data: - run_args['reference_data'] = make_url_sassy(task.reference_data.data_file.name) + run_args["reference_data"] = make_url_sassy(task.reference_data.data_file.name) - run_args['ingestion_only_during_scoring'] = task.ingestion_only_during_scoring + run_args["ingestion_only_during_scoring"] = task.ingestion_only_during_scoring if is_scoring: run_args['scoring_program_data'] = make_url_sassy(path=task.scoring_program.data_file.name) @@ -234,37 +212,43 @@ def _send_to_compute_worker(submission, is_scoring): time_padding = 60 * 20 # 20 minutes time_limit = submission.phase.execution_time_limit + time_padding - effective_queue = submission.queue or submission.phase.competition.queue - - if effective_queue: - run_args['execution_time_limit'] = submission.phase.execution_time_limit - if submission.queue != effective_queue: - submission.queue = effective_queue - submission.save(update_fields=["queue"]) - + if ( + submission.phase.competition.queue + ): # if the competition is running on a custom queue, not the default queue + submission.queue = submission.phase.competition.queue + run_args["execution_time_limit"] = ( + submission.phase.execution_time_limit + ) # use the competition time limit + submission.save(update_fields=["queue"]) if submission.status == Submission.SUBMITTING: + # Don't want to mark an already-prepared submission as "submitted" again, so + # only do this if we were previously "SUBMITTING" submission.status = Submission.SUBMITTED submission.save(update_fields=["status"]) def _enqueue_after_commit(): + # priority of scoring tasks is higher, we don't want to wait around for + # many submissions to be scored while we're waiting for results priority = 10 if is_scoring else 0 - if effective_queue: + if submission.phase.competition.queue: celery_app = app_or_default() with celery_app.connection() as new_connection: - new_connection.virtual_host = str(effective_queue.vhost) + new_connection.virtual_host = str( + submission.phase.competition.queue.vhost + ) task = celery_app.send_task( - 'compute_worker_run', + "compute_worker_run", args=(run_args,), - queue='compute-worker', + queue="compute-worker", soft_time_limit=time_limit, connection=new_connection, priority=priority, ) else: task = app.send_task( - 'compute_worker_run', + "compute_worker_run", args=(run_args,), - queue='compute-worker', + queue="compute-worker", soft_time_limit=time_limit, priority=priority, ) @@ -276,8 +260,12 @@ def _enqueue_after_commit(): def create_detailed_output_file(detail_name, submission): # Detail logs like stdout/etc. - new_details = SubmissionDetails.objects.create(submission=submission, name=detail_name) - new_details.data_file.save(f'{detail_name}.txt', ContentFile(''.encode())) # must encode here for GCS + new_details = SubmissionDetails.objects.create( + submission=submission, name=detail_name + ) + new_details.data_file.save( + f"{detail_name}.txt", ContentFile("".encode()) + ) # must encode here for GCS return make_url_sassy(new_details.data_file.name, permission="w") @@ -288,74 +276,70 @@ def run_submission(submission_pk, tasks=None, is_scoring=False): def send_submission_message(submission, data): from channels.layers import get_channel_layer + channel_layer = get_channel_layer() user = submission.owner - asyncio.get_event_loop().run_until_complete(channel_layer.group_send(f"submission_listening_{user.pk}", { - 'type': 'submission.message', - 'text': data, - 'submission_id': submission.pk, - })) + asyncio.get_event_loop().run_until_complete( + channel_layer.group_send( + f"submission_listening_{user.pk}", + { + "type": "submission.message", + "text": data, + "submission_id": submission.pk, + }, + ) + ) def send_parent_status(submission): """Helper function we can mock in tests, instead of having to do async mocks""" - send_submission_message(submission, { - "kind": "status_update", - "status": "Running" - }) + send_submission_message(submission, {"kind": "status_update", "status": "Running"}) def send_child_id(submission, child_id): """Helper function we can mock in tests, instead of having to do async mocks""" - send_submission_message(submission, { - "kind": "child_update", - "child_id": child_id - }) + send_submission_message(submission, {"kind": "child_update", "child_id": child_id}) -@app.task(queue='site-worker', soft_time_limit=60) +@app.task(queue="site-worker", soft_time_limit=60) def _run_submission(submission_pk, task_pks=None, is_scoring=False): - select_models = ('phase', 'phase__competition') + """This function is wrapped so that when we run tests we can run this function not + via celery""" + select_models = ( + "phase", + "phase__competition", + ) prefetch_models = ( - 'details', - 'phase__tasks__input_data', - 'phase__tasks__reference_data', - 'phase__tasks__scoring_program', - 'phase__tasks__ingestion_program', + "details", + "phase__tasks__input_data", + "phase__tasks__reference_data", + "phase__tasks__scoring_program", + "phase__tasks__ingestion_program", + ) + qs = Submission.objects.select_related(*select_models).prefetch_related( + *prefetch_models ) - qs = Submission.objects.select_related(*select_models).prefetch_related(*prefetch_models) submission = qs.get(pk=submission_pk) if submission.is_specific_task_re_run: + # Should only be one task for a specified task submission tasks = Task.objects.filter(pk__in=task_pks) elif task_pks is None: tasks = submission.phase.tasks.all() else: tasks = submission.phase.tasks.filter(pk__in=task_pks) - tasks = list(tasks.order_by('pk')) - - if submission.parent is None and not is_scoring: - group_queues = _get_user_group_queues(submission.owner, submission.phase.competition) - else: - group_queues = [] - is_multi_task = len(tasks) > 1 - is_multi_queue = len(group_queues) > 1 - - if is_multi_task or is_multi_queue: - if is_multi_task and is_multi_queue: - combos = [(task, queue) for task in tasks for queue in group_queues] - elif is_multi_task: - override = group_queues[0] if group_queues else None - combos = [(task, override) for task in tasks] - else: - combos = [(tasks[0], queue) for queue in group_queues] + tasks = tasks.order_by("pk") + if len(tasks) > 1: + # The initial submission object becomes the parent submission and we create children for each task submission.has_children = True submission.save() + send_parent_status(submission) - for task, queue in combos: + for task in tasks: + # TODO: make a duplicate submission method and use it here child_sub = Submission( owner=submission.owner, phase=submission.phase, @@ -364,25 +348,19 @@ def _run_submission(submission_pk, task_pks=None, is_scoring=False): parent=submission, task=task, fact_sheet_answers=submission.fact_sheet_answers, - queue=queue, ) child_sub.save(ignore_submission_limit=True) _send_to_compute_worker(child_sub, is_scoring=False) send_child_id(submission, child_sub.id) - else: + # The initial submission object is the only submission if not submission.task: submission.task = tasks[0] submission.save() - - if group_queues and submission.queue != group_queues[0]: - submission.queue = group_queues[0] - submission.save(update_fields=['queue']) - _send_to_compute_worker(submission, is_scoring) -@app.task(queue='site-worker', soft_time_limit=60 * 60) # 1 hour timeout +@app.task(queue="site-worker", soft_time_limit=60 * 60) # 1 hour timeout def unpack_competition(status_pk): logger.info(f"Starting unpack with status pk = {status_pk}") status = CompetitionCreationTaskStatus.objects.get(pk=status_pk) @@ -403,8 +381,12 @@ def mark_status_as_failed_and_delete_dataset(competition_creation_status, detail # Extract bundle try: with NamedTemporaryFile(mode="w+b") as temp_file: - logger.info(f"Download competition bundle: {competition_dataset.data_file.name}") - competition_bundle_url = make_url_sassy(competition_dataset.data_file.name) + logger.info( + f"Download competition bundle: {competition_dataset.data_file.name}" + ) + competition_bundle_url = make_url_sassy( + competition_dataset.data_file.url + ) try: with requests.get(competition_bundle_url, stream=True) as r: r.raise_for_status() @@ -412,12 +394,14 @@ def mark_status_as_failed_and_delete_dataset(competition_creation_status, detail temp_file.write(chunk) r.close() except requests.exceptions.RequestException as e: - raise CompetitionUnpackingException(f"Failed to download bundle from storage: {e}") + raise CompetitionUnpackingException( + f"Failed to download bundle from storage: {e}" + ) # seek back to the start of the tempfile after writing to it.. temp_file.seek(0) - with zipfile.ZipFile(temp_file.name, 'r') as zip_pointer: + with zipfile.ZipFile(temp_file.name, "r") as zip_pointer: zip_pointer.extractall(temp_directory) except zipfile.BadZipFile: raise CompetitionUnpackingException("Bad zip file uploaded.") @@ -434,20 +418,24 @@ def mark_status_as_failed_and_delete_dataset(competition_creation_status, detail with open(yaml_path) as f: competition_yaml = yaml.safe_load(f.read()) except yaml.YAMLError as e: - raise CompetitionUnpackingException(f"Error parsing competition.yaml: {e}") + raise CompetitionUnpackingException( + f"Error parsing competition.yaml: {e}" + ) except Exception as e: - raise CompetitionUnpackingException(f"Failed to read competition.yaml: {e}") + raise CompetitionUnpackingException( + f"Failed to read competition.yaml: {e}" + ) - yaml_version = str(competition_yaml.get('version', '1')) + yaml_version = str(competition_yaml.get("version", "1")) logger.info(f"The YAML version is: {yaml_version}") - if yaml_version in ['1', '1.5']: + if yaml_version in ["1", "1.5"]: unpacker_class = V15Unpacker - elif yaml_version == '2': + elif yaml_version == "2": unpacker_class = V2Unpacker else: raise CompetitionUnpackingException( - 'A suitable version could not be found for this competition. Make sure one is supplied in the yaml.' + "A suitable version could not be found for this competition. Make sure one is supplied in the yaml." ) unpacker = unpacker_class( @@ -460,6 +448,7 @@ def mark_status_as_failed_and_delete_dataset(competition_creation_status, detail try: competition = unpacker.save() except ValidationError as e: + def _get_error_string(error_dict): """Helps us nicely print out a ValidationError""" for key, errors in error_dict.items(): @@ -501,7 +490,7 @@ def _get_error_string(error_dict): mark_status_as_failed_and_delete_dataset(status, message) -@app.task(queue='site-worker', soft_time_limit=60 * 10) +@app.task(queue="site-worker", soft_time_limit=60 * 10) def create_competition_dump(competition_pk, keys_instead_of_files=False): yaml_data = {"version": "2"} try: @@ -510,7 +499,7 @@ def create_competition_dump(competition_pk, keys_instead_of_files=False): logger.info(f"Finding competition {competition_pk}") comp = Competition.objects.get(pk=competition_pk) zip_buffer = BytesIO() - current_date_time = datetime.today().strftime('%Y-%m-%d %H:%M:%S') + current_date_time = datetime.today().strftime("%Y-%m-%d %H:%M:%S") zip_name = f"{comp.title}-{current_date_time}.zip" zip_file = zipfile.ZipFile(zip_buffer, "w") @@ -518,14 +507,14 @@ def create_competition_dump(competition_pk, keys_instead_of_files=False): for field in COMPETITION_FIELDS: if hasattr(comp, field): value = getattr(comp, field, "") - if field == 'queue' and value is not None: + if field == "queue" and value is not None: value = str(value.vhost) yaml_data[field] = value if comp.logo: logger.info("Checking logo") try: - yaml_data['image'] = re.sub(r'.*/', '', comp.logo.name) - zip_file.writestr(yaml_data['image'], comp.logo.read()) + yaml_data["image"] = re.sub(r".*/", "", comp.logo.name) + zip_file.writestr(yaml_data["image"], comp.logo.read()) logger.info(f"Logo found for competition {comp.pk}") except OSError: logger.warning( @@ -534,25 +523,25 @@ def create_competition_dump(competition_pk, keys_instead_of_files=False): # -------- Competition Terms ------- if comp.terms: - yaml_data['terms'] = 'terms.md' - zip_file.writestr('terms.md', comp.terms) + yaml_data["terms"] = "terms.md" + zip_file.writestr("terms.md", comp.terms) # -------- Competition Pages ------- - yaml_data['pages'] = [] + yaml_data["pages"] = [] for page in comp.pages.all(): temp_page_data = {} for field in PAGE_FIELDS: if hasattr(page, field): temp_page_data[field] = getattr(page, field, "") page_file_name = f"{slugify(page.title)}-{page.pk}.md" - temp_page_data['file'] = page_file_name - yaml_data['pages'].append(temp_page_data) - zip_file.writestr(temp_page_data['file'], page.content) + temp_page_data["file"] = page_file_name + yaml_data["pages"].append(temp_page_data) + zip_file.writestr(temp_page_data["file"], page.content) # -------- Competition Tasks/Solutions ------- - yaml_data['tasks'] = [] - yaml_data['solutions'] = [] + yaml_data["tasks"] = [] + yaml_data["solutions"] = [] task_solution_pairs = {} tasks = [task for phase in comp.phases.all() for task in phase.tasks.all()] @@ -563,23 +552,18 @@ def create_competition_dump(competition_pk, keys_instead_of_files=False): for index, task in enumerate(tasks): task_solution_pairs[task.id] = { - 'index': index, - 'solutions': { - 'ids': [], - 'indexes': [] - } + "index": index, + "solutions": {"ids": [], "indexes": []}, } - temp_task_data = { - 'index': index - } + temp_task_data = {"index": index} for field in TASK_FIELDS: data = getattr(task, field, "") # If keys_instead of files is not true and field is key, then skip this filed - if not keys_instead_of_files and field == 'key': + if not keys_instead_of_files and field == "key": continue - if field == 'key': + if field == "key": data = str(data) temp_task_data[field] = data @@ -592,116 +576,136 @@ def create_competition_dump(competition_pk, keys_instead_of_files=False): temp_task_data[file_type] = str(temp_dataset.key) else: try: - temp_task_data[file_type] = f"{file_type}-{task.pk}.zip" - zip_file.writestr(temp_task_data[file_type], temp_dataset.data_file.read()) + temp_task_data[file_type] = ( + f"{file_type}-{task.pk}.zip" + ) + zip_file.writestr( + temp_task_data[file_type], + temp_dataset.data_file.read(), + ) except OSError: logger.error( f"The file field is set, but no actual" f" file was found for dataset: {temp_dataset.pk} with name {temp_dataset.name}" ) else: - logger.warning(f"Could not find data file for dataset object: {temp_dataset.pk}") + logger.warning( + f"Could not find data file for dataset object: {temp_dataset.pk}" + ) # Now for all of our solutions for the tasks, write those too for solution in task.solutions.all(): # for index_two, solution in enumerate(task.solutions.all()): # temp_index = index_two # IF OUR SOLUTION WAS ALREADY ADDED - if solution.id in task_solution_pairs[task.id]['solutions']['ids']: - for solution_data in yaml_data['solutions']: - if solution_data['key'] == solution.key: - solution_data['tasks'].append(task.id) + if solution.id in task_solution_pairs[task.id]["solutions"]["ids"]: + for solution_data in yaml_data["solutions"]: + if solution_data["key"] == solution.key: + solution_data["tasks"].append(task.id) break break # Else if our index is already taken - elif index_two in task_solution_pairs[task.id]['solutions']['indexes']: + elif index_two in task_solution_pairs[task.id]["solutions"]["indexes"]: index_two += 1 - task_solution_pairs[task.id]['solutions']['indexes'].append(index_two) - task_solution_pairs[task.id]['solutions']['ids'].append(solution.id) + task_solution_pairs[task.id]["solutions"]["indexes"].append(index_two) + task_solution_pairs[task.id]["solutions"]["ids"].append(solution.id) - temp_solution_data = { - 'index': index_two - } + temp_solution_data = {"index": index_two} for field in SOLUTION_FIELDS: if hasattr(solution, field): data = getattr(solution, field, "") - if field == 'key': + if field == "key": data = str(data) temp_solution_data[field] = data if solution.data: - temp_dataset = getattr(solution, 'data') + temp_dataset = getattr(solution, "data") if temp_dataset: if temp_dataset.data_file: try: - temp_solution_data['path'] = f"solution-{solution.pk}.zip" - zip_file.writestr(temp_solution_data['path'], temp_dataset.data_file.read()) + temp_solution_data["path"] = ( + f"solution-{solution.pk}.zip" + ) + zip_file.writestr( + temp_solution_data["path"], + temp_dataset.data_file.read(), + ) except OSError: logger.error( f"The file field is set, but no actual" f" file was found for dataset: {temp_dataset.pk} with name {temp_dataset.name}" ) else: - logger.warning(f"Could not find data file for dataset object: {temp_dataset.pk}") + logger.warning( + f"Could not find data file for dataset object: {temp_dataset.pk}" + ) # TODO: Make sure logic here is right. Needs to be outputted as a list, but what others can we tie to? - temp_solution_data['tasks'] = [index] - yaml_data['solutions'].append(temp_solution_data) + temp_solution_data["tasks"] = [index] + yaml_data["solutions"].append(temp_solution_data) index_two += 1 # End for loop for solutions; Append tasks data - yaml_data['tasks'].append(temp_task_data) + yaml_data["tasks"].append(temp_task_data) # -------- Competition Phases ------- - yaml_data['phases'] = [] + yaml_data["phases"] = [] for phase in comp.phases.all(): temp_phase_data = {} for field in PHASE_FIELDS: if hasattr(phase, field): - if field == 'start' or field == 'end': + if field == "start" or field == "end": temp_date = getattr(phase, field) if not temp_date: continue temp_date = temp_date.strftime("%Y-%m-%d") temp_phase_data[field] = temp_date - elif field == 'max_submissions_per_person': - temp_phase_data['max_submissions'] = getattr(phase, field) + elif field == "max_submissions_per_person": + temp_phase_data["max_submissions"] = getattr(phase, field) else: temp_phase_data[field] = getattr(phase, field, "") - task_indexes = [task_solution_pairs[task.id]['index'] for task in phase.tasks.all()] - temp_phase_data['tasks'] = task_indexes + task_indexes = [ + task_solution_pairs[task.id]["index"] for task in phase.tasks.all() + ] + temp_phase_data["tasks"] = task_indexes temp_phase_solutions = [] for task in phase.tasks.all(): - temp_phase_solutions += task_solution_pairs[task.id]['solutions']['indexes'] - temp_phase_data['solutions'] = temp_phase_solutions - yaml_data['phases'].append(temp_phase_data) - yaml_data['phases'] = sorted(yaml_data['phases'], key=lambda phase: phase['index']) + temp_phase_solutions += task_solution_pairs[task.id]["solutions"][ + "indexes" + ] + temp_phase_data["solutions"] = temp_phase_solutions + yaml_data["phases"].append(temp_phase_data) + yaml_data["phases"] = sorted( + yaml_data["phases"], key=lambda phase: phase["index"] + ) # -------- Leaderboards ------- - yaml_data['leaderboards'] = [] + yaml_data["leaderboards"] = [] # Have to grab leaderboards from phases - leaderboards = Leaderboard.objects.filter(id__in=comp.phases.all().values_list('leaderboard', flat=True)) + leaderboards = Leaderboard.objects.filter( + id__in=comp.phases.all().values_list("leaderboard", flat=True) + ) for index, leaderboard in enumerate(leaderboards): - ldb_data = { - 'index': index - } + ldb_data = {"index": index} for field in LEADERBOARD_FIELDS: if hasattr(leaderboard, field): ldb_data[field] = getattr(leaderboard, field, "") - ldb_data['columns'] = [] + ldb_data["columns"] = [] for column in leaderboard.columns.all(): col_data = {} for field in COLUMN_FIELDS: if hasattr(column, field): value = getattr(column, field, "") - if field == 'computation_indexes' and value is not None: - value = value.split(',') + if field == "computation_indexes" and value is not None: + value = value.split(",") if value is not None: col_data[field] = value - ldb_data['columns'].append(col_data) - yaml_data['leaderboards'].append(ldb_data) + ldb_data["columns"].append(col_data) + yaml_data["leaderboards"].append(ldb_data) # ------- Finalize -------- logger.info(f"YAML data to be written is: {yaml_data}") - comp_yaml = yaml.safe_dump(yaml_data, default_flow_style=False, allow_unicode=True, encoding="utf-8") + comp_yaml = yaml.safe_dump( + yaml_data, default_flow_style=False, allow_unicode=True, encoding="utf-8" + ) logger.info(f"YAML output: {comp_yaml}") zip_file.writestr("competition.yaml", comp_yaml) zip_file.close() @@ -712,8 +716,8 @@ def create_competition_dump(competition_pk, keys_instead_of_files=False): temp_dataset_bundle = Data.objects.create( created_by=comp.created_by, name=f"{comp.title} Dump #{bundle_count} Created {current_date_time}", - type='competition_bundle', - description='Automatically created competition dump', + type="competition_bundle", + description="Automatically created competition dump", # 'data_file'=, ) logger.info("Saving zip to Competition Bundle") @@ -722,61 +726,74 @@ def create_competition_dump(competition_pk, keys_instead_of_files=False): temp_comp_dump = CompetitionDump.objects.create( dataset=temp_dataset_bundle, status="Finished", - details="Competition Bundle {0} for Competition {1}".format(temp_dataset_bundle.pk, comp.pk), - competition=comp + details="Competition Bundle {0} for Competition {1}".format( + temp_dataset_bundle.pk, comp.pk + ), + competition=comp, + ) + logger.info( + f"Finished creating competition dump: {temp_comp_dump.pk} for competition: {comp.pk}" ) - logger.info(f"Finished creating competition dump: {temp_comp_dump.pk} for competition: {comp.pk}") except ObjectDoesNotExist: - logger.error("Could not find competition with pk {} to create a competition dump".format(competition_pk)) + logger.error( + "Could not find competition with pk {} to create a competition dump".format( + competition_pk + ) + ) -@app.task(queue='site-worker', soft_time_limit=60 * 5) +@app.task(queue="site-worker", soft_time_limit=60 * 5) def do_phase_migrations(): # Update phase statuses - previous_subquery = Phase.objects.filter( - competition=OuterRef('competition'), - end__lte=now() - ).order_by('-index').values('index')[:1] + previous_subquery = ( + Phase.objects.filter(competition=OuterRef("competition"), end__lte=now()) + .order_by("-index") + .values("index")[:1] + ) current_subquery = Phase.objects.filter( - competition=OuterRef('competition'), + competition=OuterRef("competition"), start__lte=now(), end__gt=now(), - ).values('index')[:1] + ).values("index")[:1] - next_subquery = Phase.objects.filter( - competition=OuterRef('competition'), - start__gt=now() - ).order_by('index').values('index')[:1] + next_subquery = ( + Phase.objects.filter(competition=OuterRef("competition"), start__gt=now()) + .order_by("index") + .values("index")[:1] + ) Phase.objects.annotate( previous_index=Subquery(previous_subquery), current_index=Subquery(current_subquery), next_index=Subquery(next_subquery), - ).update(status=Case( - When(index=F('previous_index'), then=Value(Phase.PREVIOUS)), - When(index=F('current_index'), then=Value(Phase.CURRENT)), - When(index=F('next_index'), then=Value(Phase.NEXT)), - default=None - )) + ).update( + status=Case( + When(index=F("previous_index"), then=Value(Phase.PREVIOUS)), + When(index=F("current_index"), then=Value(Phase.CURRENT)), + When(index=F("next_index"), then=Value(Phase.NEXT)), + default=None, + ) + ) # Updating Competitions whose phases have finished migrating to `is_migrating=False` - completed_statuses = [Submission.FINISHED, Submission.FAILED, Submission.CANCELLED, Submission.NONE] - - running_subs_query = Submission.objects.filter( - created_by_migration=OuterRef('pk') - ).exclude( - status__in=completed_statuses - ).values_list('pk')[:1] + completed_statuses = [ + Submission.FINISHED, + Submission.FAILED, + Submission.CANCELLED, + Submission.NONE, + ] + + running_subs_query = ( + Submission.objects.filter(created_by_migration=OuterRef("pk")) + .exclude(status__in=completed_statuses) + .values_list("pk")[:1] + ) Competition.objects.filter( - pk__in=Phase.objects.annotate( - running_subs=Count(Subquery(running_subs_query)) - ).filter( - running_subs=0, - competition__is_migrating=True, - status=Phase.PREVIOUS - ).values_list('competition__pk', flat=True) + pk__in=Phase.objects.annotate(running_subs=Count(Subquery(running_subs_query))) + .filter(running_subs=0, competition__is_migrating=True, status=Phase.PREVIOUS) + .values_list("competition__pk", flat=True) ).update(is_migrating=False) # Checking for new phases to start migrating @@ -784,7 +801,7 @@ def do_phase_migrations(): auto_migrate_to_this_phase=True, start__lte=now(), competition__is_migrating=False, - has_been_migrated=False + has_been_migrated=False, ) logger.info(f"Checking {len(new_phases)} phases for phase migrations.") @@ -793,51 +810,70 @@ def do_phase_migrations(): p.check_future_phase_submissions() -@app.task(queue='site-worker', soft_time_limit=60 * 5) +@app.task(queue="site-worker", soft_time_limit=60 * 5) def manual_migration(phase_id): try: source_phase = Phase.objects.get(id=phase_id) except Phase.DoesNotExist: - logger.error(f'Could not manually migrate phase with id: {phase_id}. Phase could not be found.') + logger.error( + f"Could not manually migrate phase with id: {phase_id}. Phase could not be found." + ) return try: - destination_phase = source_phase.competition.phases.get(index=source_phase.index + 1) + destination_phase = source_phase.competition.phases.get( + index=source_phase.index + 1 + ) except Phase.DoesNotExist: - logger.error(f'Could not manually migrate phase with id: {phase_id}. The next phase could not be found.') + logger.error( + f"Could not manually migrate phase with id: {phase_id}. The next phase could not be found." + ) return - destination_phase.competition.apply_phase_migration(source_phase, destination_phase, force_migration=True) + destination_phase.competition.apply_phase_migration( + source_phase, destination_phase, force_migration=True + ) -@app.task(queue='site-worker', soft_time_limit=60 * 5) +@app.task(queue="site-worker", soft_time_limit=60 * 5) def batch_send_email(comp_id, content): try: - competition = Competition.objects.prefetch_related('participants__user').get(id=comp_id) + competition = Competition.objects.prefetch_related("participants__user").get( + id=comp_id + ) except Competition.DoesNotExist: - logger.error(f'Not sending emails because competition with id {comp_id} could not be found') + logger.error( + f"Not sending emails because competition with id {comp_id} could not be found" + ) return codalab_send_markdown_email( - subject=f'A message from the admins of {competition.title}', + subject=f"A message from the admins of {competition.title}", markdown_content=content, - recipient_list=[participant.user.email for participant in competition.participants.all()] + recipient_list=[ + participant.user.email for participant in competition.participants.all() + ], ) -@app.task(queue='site-worker', soft_time_limit=60 * 5) +@app.task(queue="site-worker", soft_time_limit=60 * 5) def update_phase_statuses(): - competitions = Competition.objects.exclude(phases__in=Phase.objects.filter(is_final_phase=True, end__lt=now())) + competitions = Competition.objects.exclude( + phases__in=Phase.objects.filter(is_final_phase=True, end__lt=now()) + ) for comp in competitions: comp.update_phase_statuses() -@app.task(queue='site-worker') +@app.task(queue="site-worker") def submission_status_cleanup(): - submissions = Submission.objects.filter(status=Submission.RUNNING, has_children=False).select_related('phase', 'parent') + submissions = Submission.objects.filter( + status=Submission.RUNNING, has_children=False + ).select_related("phase", "parent") for sub in submissions: - # Check if the submission has been running for 24 hours longer than execution_time_limit - if sub.started_when < now() - timedelta(milliseconds=(3600000 * 24) + sub.phase.execution_time_limit): + if sub.started_when < now() - timedelta( + milliseconds=(3600000 * 24) + sub.phase.execution_time_limit + ): if sub.parent is not None: sub.parent.cancel(status=Submission.FAILED) else: diff --git a/src/apps/datasets/admin.py b/src/apps/datasets/admin.py index 86cb7777c..864ca30fd 100644 --- a/src/apps/datasets/admin.py +++ b/src/apps/datasets/admin.py @@ -15,6 +15,7 @@ def DeactivateAccount(modeladmin, request, queryset): class DataExpansion(admin.ModelAdmin): raw_id_fields = ["created_by", "competition"] + ordering = ('-id',) list_display = [ "id", "name", diff --git a/src/apps/forums/admin.py b/src/apps/forums/admin.py index 1c9a38efc..733523be4 100644 --- a/src/apps/forums/admin.py +++ b/src/apps/forums/admin.py @@ -25,6 +25,7 @@ class ForumsExpansion(admin.ModelAdmin): raw_id_fields = ["competition"] list_display = ["id", "competition"] search_fields = ["id", "competition"] + ordering = ('-id',) class ThreadExpansion(admin.ModelAdmin): @@ -32,6 +33,7 @@ class ThreadExpansion(admin.ModelAdmin): list_display = ["id", "title", "started_by"] search_fields = ["id", "title", "started_by__username"] actions = [DeactivateAccountThread] + ordering = ('-id',) class PostExpansion(admin.ModelAdmin): @@ -39,6 +41,7 @@ class PostExpansion(admin.ModelAdmin): list_display = ["id", "content_limited", "posted_by"] search_fields = ["id", "content", "posted_by__username"] actions = [DeactivateAccountPost] + ordering = ('-id',) @admin.display(description="Content", ordering="content") def content_limited(self, obj): diff --git a/src/apps/leaderboards/admin.py b/src/apps/leaderboards/admin.py index db86c577d..ad4c8414b 100644 --- a/src/apps/leaderboards/admin.py +++ b/src/apps/leaderboards/admin.py @@ -7,6 +7,7 @@ class LeaderboardExpansion(admin.ModelAdmin): list_display = ["id", "title", "submission_rule", "hidden"] search_fields = ["id", "title"] list_filter = ["hidden"] + ordering = ('-id',) class ColumExpansion(admin.ModelAdmin): @@ -14,12 +15,14 @@ class ColumExpansion(admin.ModelAdmin): list_display = ["id", "title", "hidden"] search_fields = ["id", "title"] list_filter = ["hidden"] + ordering = ('-id',) class SubmissionScoreExpansion(admin.ModelAdmin): raw_id_fields = ["column"] list_display = ["id", "column", "score"] search_fields = ["id", "column"] + ordering = ('-id',) admin.site.register(models.Leaderboard, LeaderboardExpansion) diff --git a/src/apps/leaderboards/migrations/0010_alter_column_computation.py b/src/apps/leaderboards/migrations/0010_alter_column_computation.py new file mode 100644 index 000000000..11d39e8c3 --- /dev/null +++ b/src/apps/leaderboards/migrations/0010_alter_column_computation.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.12 on 2026-04-23 09:35 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('leaderboards', '0009_alter_column_id_alter_leaderboard_id_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='column', + name='computation', + field=models.TextField(blank=True, choices=[('avg', 'Average'), ('sum', 'Sum'), ('min', 'Min'), ('max', 'Max'), ('avg_rank', 'Average Rank')], null=True), + ), + ] diff --git a/src/apps/leaderboards/models.py b/src/apps/leaderboards/models.py index 4c9eda5a2..151d6d12a 100644 --- a/src/apps/leaderboards/models.py +++ b/src/apps/leaderboards/models.py @@ -36,11 +36,13 @@ class Column(models.Model): SUM = 'sum' MIN = 'min' MAX = 'max' + AVERAGE_RANK = 'avg_rank' COMPUTATION_CHOICES = ( (AVERAGE, 'Average'), (SUM, 'Sum'), (MIN, 'Min'), (MAX, 'Max'), + (AVERAGE_RANK, 'Average Rank'), ) SORTING = ( ('desc', 'Descending'), diff --git a/src/apps/leaderboards/ranking.py b/src/apps/leaderboards/ranking.py new file mode 100644 index 000000000..ab0ef8698 --- /dev/null +++ b/src/apps/leaderboards/ranking.py @@ -0,0 +1,108 @@ +from leaderboards.models import Column + + +def fractional_rank(values): + """ + Fractional (average) ranking: tied values receive the mean of the ranks they + would occupy, identical to scipy.stats.rankdata(method='average'). + Rank 1 is assigned to the smallest value. + """ + sorted_vals = sorted(values) + rank_sum = {} + rank_count = {} + for rank, val in enumerate(sorted_vals, start=1): + rank_sum[val] = rank_sum.get(val, 0) + rank + rank_count[val] = rank_count.get(val, 0) + 1 + return [rank_sum[v] / rank_count[v] for v in values] + + +def inject_average_ranks(submissions, avg_rank_cols, col_by_index, primary_index): + """ + For each AVERAGE_RANK column, rank submissions on each referenced sub-column + using fractional (average) ranking, compute the mean rank per submission, and + append it as a synthetic score entry. + If the primary column is AVERAGE_RANK, re-sort the list in-place afterward. + + Fractional ranking: tied submissions share the mean of the ranks they occupy + (e.g. two entries tying for positions 2 and 3 both receive rank 2.5). + Submissions missing a score for a sub-column are placed last (rank = n). + When a submission has multiple scores for the same column (multi-task), they are + summed before ranking, consistent with the ORM annotation in the serializer. + """ + # Pre-aggregate scores per submission per column (sum across tasks). + submission_col_scores = [] + for sub in submissions: + col_scores = {} + for s in sub['scores']: + idx = s['index'] + try: + val = float(s['score']) + except (ValueError, TypeError): + val = None + if idx not in col_scores: + col_scores[idx] = val + elif val is not None: + col_scores[idx] = (col_scores[idx] or 0) + val + submission_col_scores.append(col_scores) + + n = len(submissions) + + for col in avg_rank_cols: + if not col.get('computation_indexes'): + continue + sub_indices = [int(i) for i in col['computation_indexes']] + + per_column_ranks = [] + for sub_idx in sub_indices: + sub_col = col_by_index.get(sub_idx) + if sub_col is None: + continue + + valid_indices = [i for i in range(n) if submission_col_scores[i].get(sub_idx) is not None] + valid_scores = [submission_col_scores[i][sub_idx] for i in valid_indices] + + if not valid_scores: + continue + + # Negate descending columns so rank 1 = highest score. + scores_for_rank = [-s for s in valid_scores] if sub_col['sorting'] == 'desc' else valid_scores + fractions = fractional_rank(scores_for_rank) + + ranks = {i: float(n) for i in range(n)} # default: worst rank for unscored + for pos, sub_i in enumerate(valid_indices): + ranks[sub_i] = fractions[pos] + per_column_ranks.append(ranks) + + if not per_column_ranks: + continue + + is_primary = col['index'] == primary_index + for i, sub in enumerate(submissions): + sub_ranks = [r[i] for r in per_column_ranks] + avg_rank = sum(sub_ranks) / len(sub_ranks) + score_entry = { + 'index': col['index'], + 'column_key': col['key'], + 'score': str(round(avg_rank, col.get('precision', 2))), + 'is_primary': is_primary, + } + # The frontend matches scores by (task_id, column_key). Average rank is + # cross-task, so inject one copy per task that already has scores here. + task_ids = {s['task_id'] for s in sub['scores'] if s.get('task_id') is not None} + for task_id in task_ids: + sub['scores'].append({**score_entry, 'task_id': task_id}) + + primary_col = col_by_index.get(primary_index) + if primary_col and primary_col.get('computation') == Column.AVERAGE_RANK: + reverse = primary_col['sorting'] == 'desc' + + def _sort_key(sub): + for s in sub['scores']: + if s['index'] == primary_index: + try: + return float(s['score']) + except (ValueError, TypeError): + pass + return float('inf') if not reverse else float('-inf') + + submissions.sort(key=_sort_key, reverse=reverse) diff --git a/src/apps/leaderboards/tests/__init__.py b/src/apps/leaderboards/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/apps/leaderboards/tests/test_ranking.py b/src/apps/leaderboards/tests/test_ranking.py new file mode 100644 index 000000000..3056eb0af --- /dev/null +++ b/src/apps/leaderboards/tests/test_ranking.py @@ -0,0 +1,259 @@ +from leaderboards.ranking import fractional_rank, inject_average_ranks + + +# --------------------------------------------------------------------------- +# fractional_rank +# --------------------------------------------------------------------------- + +def test_fractional_rank_no_ties(): + # 3 distinct values → straight 1, 2, 3 + assert fractional_rank([3.0, 1.0, 2.0]) == [3.0, 1.0, 2.0] + + +def test_fractional_rank_two_way_tie(): + # Positions 1 and 2 are tied → both get 1.5 + assert fractional_rank([1.0, 1.0, 2.0]) == [1.5, 1.5, 3.0] + + +def test_fractional_rank_three_way_tie(): + # Positions 1, 2, 3 are all tied → mean(1,2,3) = 2.0 + assert fractional_rank([5.0, 5.0, 5.0]) == [2.0, 2.0, 2.0] + + +def test_fractional_rank_tie_at_end(): + # Two tied at the last positions + assert fractional_rank([1.0, 2.0, 2.0]) == [1.0, 2.5, 2.5] + + +def test_fractional_rank_single_element(): + assert fractional_rank([42.0]) == [1.0] + + +# --------------------------------------------------------------------------- +# Helpers for inject_average_ranks +# --------------------------------------------------------------------------- + +def _make_col(index, key, sorting='desc', computation_indexes=None, precision=2, computation='avg_rank'): + return { + 'id': index, + 'index': index, + 'key': key, + 'title': key, + 'sorting': sorting, + 'computation': computation, + 'computation_indexes': [str(i) for i in (computation_indexes or [])], + 'precision': precision, + 'hidden': False, + } + + +def _make_submission(scores): + """ + scores: list of (column_index, column_key, score_value, task_id) + """ + return { + 'scores': [ + {'index': idx, 'column_key': key, 'score': str(val), 'task_id': tid, 'is_primary': False} + for idx, key, val, tid in scores + ] + } + + +# --------------------------------------------------------------------------- +# inject_average_ranks +# --------------------------------------------------------------------------- + +def test_inject_average_ranks_basic_values(): + # 3 submissions, 1 descending sub-column (higher = better = rank 1). + # Scores 0.9, 0.6, 0.3 → ranks 1, 2, 3 → avg_rank 1.0, 2.0, 3.0 + col0 = _make_col(0, 'col0', sorting='desc') + avg_col = _make_col(2, 'avg_rank', computation_indexes=[0]) + + submissions = [ + _make_submission([(0, 'col0', 0.9, 1)]), + _make_submission([(0, 'col0', 0.6, 1)]), + _make_submission([(0, 'col0', 0.3, 1)]), + ] + + col_by_index = {0: col0, 2: avg_col} + inject_average_ranks(submissions, [avg_col], col_by_index, primary_index=0) + + avg_scores = [ + next(s for s in sub['scores'] if s['column_key'] == 'avg_rank') + for sub in submissions + ] + assert float(avg_scores[0]['score']) == 1.0 + assert float(avg_scores[1]['score']) == 2.0 + assert float(avg_scores[2]['score']) == 3.0 + + +def test_inject_average_ranks_with_ties(): + # Two submissions tied on the sub-column → both get fractional rank 1.5 + col0 = _make_col(0, 'col0', sorting='desc') + avg_col = _make_col(1, 'avg_rank', computation_indexes=[0]) + + submissions = [ + _make_submission([(0, 'col0', 0.8, 1)]), + _make_submission([(0, 'col0', 0.8, 1)]), + _make_submission([(0, 'col0', 0.5, 1)]), + ] + + col_by_index = {0: col0, 1: avg_col} + inject_average_ranks(submissions, [avg_col], col_by_index, primary_index=0) + + avg_scores = [ + float(next(s for s in sub['scores'] if s['column_key'] == 'avg_rank')['score']) + for sub in submissions + ] + assert avg_scores[0] == 1.5 + assert avg_scores[1] == 1.5 + assert avg_scores[2] == 3.0 + + +def test_inject_average_ranks_ascending_column(): + # Ascending column: lower score = better = rank 1 + col0 = _make_col(0, 'col0', sorting='asc') + avg_col = _make_col(1, 'avg_rank', computation_indexes=[0]) + + submissions = [ + _make_submission([(0, 'col0', 0.1, 1)]), # lowest → rank 1 + _make_submission([(0, 'col0', 0.5, 1)]), # rank 2 + _make_submission([(0, 'col0', 0.9, 1)]), # highest → rank 3 + ] + + col_by_index = {0: col0, 1: avg_col} + inject_average_ranks(submissions, [avg_col], col_by_index, primary_index=0) + + avg_scores = [ + float(next(s for s in sub['scores'] if s['column_key'] == 'avg_rank')['score']) + for sub in submissions + ] + assert avg_scores[0] == 1.0 + assert avg_scores[1] == 2.0 + assert avg_scores[2] == 3.0 + + +def test_inject_average_ranks_missing_score_gets_worst_rank(): + # Submission that has scores for other columns but not the avg_rank sub-column + # gets worst rank (= n = 3). It still has a task_id from its other scores so + # the injected entry can be matched by the frontend. + col0 = _make_col(0, 'col0', sorting='desc') + avg_col = _make_col(1, 'avg_rank', computation_indexes=[0]) + + submissions = [ + _make_submission([(0, 'col0', 0.9, 1)]), # rank 1 + _make_submission([(0, 'col0', 0.5, 1)]), # rank 2 + _make_submission([(99, 'other_col', 0.7, 1)]), # no col0 score → rank 3 + ] + + col_by_index = {0: col0, 1: avg_col} + inject_average_ranks(submissions, [avg_col], col_by_index, primary_index=0) + + avg_scores = [ + float(next(s for s in sub['scores'] if s['column_key'] == 'avg_rank')['score']) + for sub in submissions + ] + assert avg_scores[0] == 1.0 + assert avg_scores[1] == 2.0 + assert avg_scores[2] == 3.0 # worst rank = n = 3 + + +def test_inject_average_ranks_two_subcolumns(): + # Average over two sub-columns + col0 = _make_col(0, 'col0', sorting='desc') + col1 = _make_col(1, 'col1', sorting='desc') + avg_col = _make_col(2, 'avg_rank', computation_indexes=[0, 1]) + + # Sub0: col0=0.9 (rank1), col1=0.3 (rank3) → avg 2.0 + # Sub1: col0=0.6 (rank2), col1=0.6 (rank2) → avg 2.0 + # Sub2: col0=0.3 (rank3), col1=0.9 (rank1) → avg 2.0 + submissions = [ + _make_submission([(0, 'col0', 0.9, 1), (1, 'col1', 0.3, 1)]), + _make_submission([(0, 'col0', 0.6, 1), (1, 'col1', 0.6, 1)]), + _make_submission([(0, 'col0', 0.3, 1), (1, 'col1', 0.9, 1)]), + ] + + col_by_index = {0: col0, 1: col1, 2: avg_col} + inject_average_ranks(submissions, [avg_col], col_by_index, primary_index=0) + + avg_scores = [ + float(next(s for s in sub['scores'] if s['column_key'] == 'avg_rank')['score']) + for sub in submissions + ] + assert avg_scores[0] == 2.0 + assert avg_scores[1] == 2.0 + assert avg_scores[2] == 2.0 + + +def test_inject_average_ranks_task_id_propagation(): + # The injected score must carry the same task_id as existing scores so the + # frontend can match it via (task_id, column_key). + col0 = _make_col(0, 'col0', sorting='desc') + avg_col = _make_col(1, 'avg_rank', computation_indexes=[0]) + + task_id = 99 + submissions = [ + _make_submission([(0, 'col0', 0.9, task_id)]), + _make_submission([(0, 'col0', 0.5, task_id)]), + ] + + col_by_index = {0: col0, 1: avg_col} + inject_average_ranks(submissions, [avg_col], col_by_index, primary_index=0) + + for sub in submissions: + injected = [s for s in sub['scores'] if s['column_key'] == 'avg_rank'] + assert len(injected) == 1 + assert injected[0]['task_id'] == task_id + + +def test_inject_average_ranks_multi_task_injects_one_per_task(): + # Multi-task submissions have scores with different task_ids. + # One avg_rank entry must be injected per task_id. + col0 = _make_col(0, 'col0', sorting='desc') + avg_col = _make_col(1, 'avg_rank', computation_indexes=[0]) + + submissions = [ + _make_submission([(0, 'col0', 0.9, 10), (0, 'col0', 0.8, 20)]), + _make_submission([(0, 'col0', 0.5, 10), (0, 'col0', 0.4, 20)]), + ] + + col_by_index = {0: col0, 1: avg_col} + inject_average_ranks(submissions, [avg_col], col_by_index, primary_index=0) + + for sub in submissions: + injected = [s for s in sub['scores'] if s['column_key'] == 'avg_rank'] + injected_task_ids = {s['task_id'] for s in injected} + assert injected_task_ids == {10, 20} + + +def test_inject_average_ranks_sorts_by_primary_avg_rank(): + # When the avg_rank column is the primary, submissions are re-sorted + # ascending (rank 1 = best position = first row). + col0 = _make_col(0, 'col0', sorting='desc') + avg_col = _make_col(1, 'avg_rank', sorting='asc', computation_indexes=[0]) + + # Scores: 0.3 → rank3, 0.9 → rank1, 0.6 → rank2 + # After sort ascending by avg_rank: rank1 first, then rank2, then rank3 + submissions = [ + _make_submission([(0, 'col0', 0.3, 1)]), # will become rank 3 + _make_submission([(0, 'col0', 0.9, 1)]), # will become rank 1 + _make_submission([(0, 'col0', 0.6, 1)]), # will become rank 2 + ] + + col_by_index = {0: col0, 1: avg_col} + inject_average_ranks(submissions, [avg_col], col_by_index, primary_index=1) + + avg_scores = [ + float(next(s for s in sub['scores'] if s['column_key'] == 'avg_rank')['score']) + for sub in submissions + ] + assert avg_scores == [1.0, 2.0, 3.0] + + +def test_inject_average_ranks_no_avg_rank_cols_is_noop(): + submissions = [_make_submission([(0, 'col0', 0.9, 1)])] + original_scores = list(submissions[0]['scores']) + + inject_average_ranks(submissions, [], {}, primary_index=0) + + assert submissions[0]['scores'] == original_scores diff --git a/src/apps/oidc_configurations/admin.py b/src/apps/oidc_configurations/admin.py index 2b606b8a1..29d8a85f9 100644 --- a/src/apps/oidc_configurations/admin.py +++ b/src/apps/oidc_configurations/admin.py @@ -5,6 +5,7 @@ class Auth_OrganizationExpansion(admin.ModelAdmin): list_display = ["id", "name", "client_id"] search_fields = ["id", "name", "client_id"] + ordering = ('-id',) admin.site.register(Auth_Organization, Auth_OrganizationExpansion) diff --git a/src/apps/profiles/admin.py b/src/apps/profiles/admin.py index 4a9ba7ca5..b0ae3a7d7 100644 --- a/src/apps/profiles/admin.py +++ b/src/apps/profiles/admin.py @@ -72,16 +72,39 @@ def export_as_json(modeladmin, request, queryset): return HttpResponse(json.dumps(email_list), content_type="application/json") +@admin.display(description="Ban User(s)") +def ban_users(modeladmin, request, queryset): + for obj in queryset: + obj.is_banned = True + obj.save() + + +@admin.display(description="Unban User(s)") +def unban_users(modeladmin, request, queryset): + for obj in queryset: + obj.is_banned = False + obj.save() + + +@admin.display(description="Activate User(s)") +def activate_users(modeladmin, request, queryset): + for obj in queryset: + obj.is_active = True + obj.save() + + class UserExpansion(UserAdmin): # The following two lines are needed for Django-su: change_form_template = "admin/auth/user/change_form.html" change_list_template = "admin/auth/user/change_list.html" search_fields = ["id", "username", "email"] + ordering = ('-id',) list_filter = [ "is_staff", "is_superuser", "is_deleted", "is_bot", + "is_active", "is_banned", QuotaFilter, ] @@ -90,13 +113,14 @@ class UserExpansion(UserAdmin): "username", "email", "quota", + "is_active", "is_staff", "is_superuser", "is_banned", ] list_display_links = ["id", "username"] raw_id_fields = ["oidc_organization", "groups"] - actions = [export_as_csv, export_as_json] + actions = [activate_users, ban_users, unban_users, export_as_csv, export_as_json] fieldsets = [ ( None, @@ -180,18 +204,21 @@ class DeletedUserExpansion(admin.ModelAdmin): list_display = ("user_id", "username", "email", "deleted_at") search_fields = ("id", "username", "email") list_filter = ("deleted_at",) + ordering = ('-id',) class MembershipExpansion(admin.ModelAdmin): raw_id_fields = ["organization", "user"] list_display = ["id", "organization", "user", "group"] search_fields = ["id", "user__username", "token"] + ordering = ('-id',) class OrganizationExpansion(admin.ModelAdmin): raw_id_fields = ["user_record"] list_display = ["id", "name", "email", "description"] search_fields = ["name", "email", "description"] + ordering = ('-id',) admin.site.register(User, UserExpansion) diff --git a/src/apps/queues/admin.py b/src/apps/queues/admin.py index a0410d0fb..0f88de591 100644 --- a/src/apps/queues/admin.py +++ b/src/apps/queues/admin.py @@ -42,6 +42,7 @@ class QueueExpansion(admin.ModelAdmin): list_filter = ["is_public"] search_fields = ["id", "name", "owner__username", "organizers__username"] actions = [export_as_csv, export_as_json] + ordering = ('-id',) admin.site.register(models.Queue, QueueExpansion) diff --git a/src/apps/tasks/admin.py b/src/apps/tasks/admin.py index b91200eef..7c6c54fc2 100644 --- a/src/apps/tasks/admin.py +++ b/src/apps/tasks/admin.py @@ -26,6 +26,7 @@ class TaskExpansion(admin.ModelAdmin): "name", "created_by__username", ] + ordering = ('-id',) class SolutionExpansion(admin.ModelAdmin): @@ -35,6 +36,7 @@ class SolutionExpansion(admin.ModelAdmin): search_fields = [ "id", ] + ordering = ('-id',) admin.site.register(models.Task, TaskExpansion) diff --git a/src/settings/base.py b/src/settings/base.py index 47c76ad6b..d2698bb2a 100644 --- a/src/settings/base.py +++ b/src/settings/base.py @@ -276,10 +276,6 @@ 'task': 'profiles.tasks.clean_non_activated_users', 'schedule': timedelta(days=1), # Run every 24 hours }, - "refresh_compute_worker_health": { - "task": "competitions.tasks.refresh_compute_worker_health", - "schedule": 60, - }, } CELERY_TIMEZONE = 'UTC' CELERY_WORKER_PREFETCH_MULTIPLIER = 1 diff --git a/src/static/riot/competitions/detail/_header.tag b/src/static/riot/competitions/detail/_header.tag index f572fa959..b8234bb1d 100644 --- a/src/static/riot/competitions/detail/_header.tag +++ b/src/static/riot/competitions/detail/_header.tag @@ -36,13 +36,11 @@ Migrate -
diff --git a/src/static/riot/competitions/detail/detail.tag b/src/static/riot/competitions/detail/detail.tag index 7bed78b29..989a62285 100644 --- a/src/static/riot/competitions/detail/detail.tag +++ b/src/static/riot/competitions/detail/detail.tag @@ -1,648 +1,65 @@ -
-
- - - -
- - - - -
- + + + - \ No newline at end of file diff --git a/src/static/riot/competitions/detail/worker-monitor-toggle.tag b/src/static/riot/competitions/detail/worker-monitor-toggle.tag index d31b64e2e..48ea73e50 100644 --- a/src/static/riot/competitions/detail/worker-monitor-toggle.tag +++ b/src/static/riot/competitions/detail/worker-monitor-toggle.tag @@ -1,17 +1,14 @@ -