diff --git a/Cargo.toml b/Cargo.toml index 8780959..09f73f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,4 @@ urlencoding = "2.1.3" bytes = "1.11.1" futures-util = "0.3.31" zip = "2.2.2" - -[dev-dependencies] tempfile = "3.10" diff --git a/README.md b/README.md index 5047c1a..c0f6c08 100644 --- a/README.md +++ b/README.md @@ -151,10 +151,51 @@ POPCORN_BREV_PROFILER_URL=https://http--brev-profiler-proxy--dxfjds728w5v.code.r # Plain output mode (no TUI, good for CI/scripts) popcorn submit --no-tui --leaderboard grayscale_v2 --gpu A100 --mode test solution.py +# Run the public leaderboard pipeline in your own Modal account +MODAL_TOKEN_ID=... MODAL_TOKEN_SECRET=... popcorn submit --local --leaderboard eigh --gpu B200 --mode leaderboard solution.py + # Save results to a file popcorn submit --output results.json --leaderboard grayscale_v2 --gpu A100 --mode benchmark solution.py ``` +#### Local Modal mode + +`--local` runs the public `reference-kernels` task with KernelBot's evaluator +on a GPU billed to your Modal account. It bypasses Popcorn registration and the +GPU Mode API, and implies plain (non-TUI) output. The supported Modal GPU names +are `T4`, `L4`, `L4x4`, `A100`, `H100`, and `B200`; AMD runners are not available +through Modal. + +Install and authenticate Modal first: + +```bash +python3 -m pip install modal + +# Either save the token in Modal's local config... +modal token set + +# ...or supply it without changing local config. +export MODAL_TOKEN_ID=your-token-id +export MODAL_TOKEN_SECRET=your-token-secret +``` + +Then use the normal submission modes: + +```bash +popcorn submit --local --leaderboard eigh --gpu B200 --mode test submission.py +popcorn submit --local --leaderboard eigh --gpu B200 --mode benchmark submission.py +popcorn submit --local --leaderboard eigh --gpu B200 --mode leaderboard submission.py +``` + +Local `leaderboard` mode runs the same public test, benchmark, and ranked +evaluation stages and computes the task's ranking score. It does not run GPU +Mode's private seed, update gpumode.com, or create an official submission. The +output records the exact `reference-kernels` and KernelBot commits used. Set +`POPCORN_REFERENCE_KERNELS_REF` or `POPCORN_KERNELBOT_REF` to a branch, tag, or +commit when you need to pin or test another public revision. By default, the +runner resolves both `main` branches to immutable commit SHAs on every command; +if either lookup fails, it stops instead of risking a stale cached image. + **Submission modes:** - `test` - Quick test run to check correctness - `benchmark` - Benchmark your solution (no leaderboard impact) diff --git a/src/cmd/mod.rs b/src/cmd/mod.rs index f6353b4..39dfeb2 100644 --- a/src/cmd/mod.rs +++ b/src/cmd/mod.rs @@ -67,6 +67,11 @@ pub struct Cli { #[arg(long)] pub profile_brev: bool, + /// Run the public evaluation in your own Modal account instead of submitting to GPU Mode. + /// Uses Modal's configured profile or MODAL_TOKEN_ID/MODAL_TOKEN_SECRET. + #[arg(long, conflicts_with = "profile_brev")] + pub local: bool, + /// Optional: Profile a single benchmark index when using --profile-brev #[arg(long)] pub benchmark_index: Option, @@ -151,6 +156,11 @@ enum Commands { #[arg(long)] profile_brev: bool, + /// Run the public evaluation in your own Modal account instead of submitting to GPU Mode. + /// Uses Modal's configured profile or MODAL_TOKEN_ID/MODAL_TOKEN_SECRET. + #[arg(long, conflicts_with = "profile_brev")] + local: bool, + /// Optional: Profile a single benchmark index when using --profile-brev #[arg(long)] benchmark_index: Option, @@ -203,53 +213,60 @@ pub async fn execute(cli: Cli) -> Result<()> { leaderboard, mode, profile_brev, + local, benchmark_index, output, no_tui, }) => { - let config = load_config()?; - let cli_id = config.cli_id.ok_or_else(|| { - anyhow!( - "cli_id not found in config file ({}). Please run 'popcorn-cli register' first.", - get_config_path() - .map_or_else(|_| "unknown path".to_string(), |p| p.display().to_string()) - ) - })?; - // Use filepath from Submit command first, fallback to top-level filepath let final_filepath = filepath.or(cli.filepath); let final_gpu = if profile_brev { Some("B200_Brev".to_string()) } else { - gpu + gpu.clone() }; let final_mode = if profile_brev { Some("profile".to_string()) } else { - mode + mode.clone() }; - if no_tui || profile_brev { - submit::run_submit_plain( - final_filepath, // Resolved filepath - final_gpu, // From Submit command - leaderboard, // From Submit command - final_mode, // From Submit command - cli_id, - benchmark_index.or(cli.benchmark_index), - output, // From Submit command - ) - .await + if local { + submit::run_submit_local(final_filepath, gpu, leaderboard, mode, output).await } else { - submit::run_submit_tui( - final_filepath, // Resolved filepath - final_gpu, // From Submit command - leaderboard, // From Submit command - final_mode, // From Submit command - cli_id, - output, // From Submit command - ) - .await + let config = load_config()?; + let cli_id = config.cli_id.ok_or_else(|| { + anyhow!( + "cli_id not found in config file ({}). Please run 'popcorn-cli register' first.", + get_config_path().map_or_else( + |_| "unknown path".to_string(), + |p| p.display().to_string() + ) + ) + })?; + + if no_tui || profile_brev { + submit::run_submit_plain( + final_filepath, // Resolved filepath + final_gpu, // From Submit command + leaderboard, // From Submit command + final_mode, // From Submit command + cli_id, + benchmark_index.or(cli.benchmark_index), + output, // From Submit command + ) + .await + } else { + submit::run_submit_tui( + final_filepath, // Resolved filepath + final_gpu, // From Submit command + leaderboard, // From Submit command + final_mode, // From Submit command + cli_id, + output, // From Submit command + ) + .await + } } } Some(Commands::Join { code }) => { @@ -301,6 +318,7 @@ pub async fn execute(cli: Cli) -> Result<()> { None => { // Check if any of the submission-related flags were used at the top level if !cli.profile_brev + && !cli.local && (cli.gpu.is_some() || cli.leaderboard.is_some() || cli.mode.is_some()) { return Err(anyhow!( @@ -311,37 +329,50 @@ pub async fn execute(cli: Cli) -> Result<()> { // Handle the case where only a filepath is provided (for backward compatibility) if let Some(top_level_filepath) = cli.filepath { - let config = load_config()?; - let cli_id = config.cli_id.ok_or_else(|| { - anyhow!( - "cli_id not found in config file ({}). Please run `popcorn register` first.", - get_config_path() - .map_or_else(|_| "unknown path".to_string(), |p| p.display().to_string()) - ) - })?; - - if cli.profile_brev { - submit::run_submit_plain( + if cli.local { + submit::run_submit_local( Some(top_level_filepath), - Some("B200_Brev".to_string()), + cli.gpu, cli.leaderboard, - Some("profile".to_string()), - cli_id, - cli.benchmark_index, + cli.mode, cli.output, ) .await } else { - // Run TUI with only filepath, no other options - submit::run_submit_tui( - Some(top_level_filepath), - None, // No GPU option - None, // No leaderboard option - None, // No mode option - cli_id, - None, // No output option - ) - .await + let config = load_config()?; + let cli_id = config.cli_id.ok_or_else(|| { + anyhow!( + "cli_id not found in config file ({}). Please run `popcorn register` first.", + get_config_path().map_or_else( + |_| "unknown path".to_string(), + |p| p.display().to_string() + ) + ) + })?; + + if cli.profile_brev { + submit::run_submit_plain( + Some(top_level_filepath), + Some("B200_Brev".to_string()), + cli.leaderboard, + Some("profile".to_string()), + cli_id, + cli.benchmark_index, + cli.output, + ) + .await + } else { + // Run TUI with only filepath, no other options + submit::run_submit_tui( + Some(top_level_filepath), + None, // No GPU option + None, // No leaderboard option + None, // No mode option + cli_id, + None, // No output option + ) + .await + } } } else { Err(anyhow!( diff --git a/src/cmd/submit.rs b/src/cmd/submit.rs index 52d6cec..ba5093e 100644 --- a/src/cmd/submit.rs +++ b/src/cmd/submit.rs @@ -808,6 +808,74 @@ pub async fn run_submit_plain( Ok(()) } +pub async fn run_submit_local( + filepath: Option, + gpu: Option, + leaderboard: Option, + mode: Option, + output: Option, +) -> Result<()> { + let file_to_submit = filepath.ok_or_else(|| anyhow!("File path is required with --local"))?; + let submission_path = Path::new(&file_to_submit); + if !submission_path.exists() { + return Err(anyhow!("File not found: {}", file_to_submit)); + } + if utils::is_archive_file(submission_path) { + return Err(anyhow!( + "Local Modal mode currently supports single source files, not archives" + )); + } + + let (directives, has_multiple_gpus) = utils::get_popcorn_directives(submission_path)?; + if has_multiple_gpus { + return Err(anyhow!( + "Multiple GPUs are not supported yet. Please specify only one GPU." + )); + } + + let final_gpu = gpu + .or_else(|| directives.gpus.first().cloned()) + .ok_or_else(|| anyhow!("GPU not specified. Use --gpu or add a GPU directive"))?; + let final_leaderboard = leaderboard + .or_else(|| { + (!directives.leaderboard_name.is_empty()).then_some(directives.leaderboard_name.clone()) + }) + .ok_or_else(|| { + anyhow!("Leaderboard not specified. Use --leaderboard or add a leaderboard directive") + })?; + let final_mode = mode.ok_or_else(|| { + anyhow!("Submission mode not specified. Use --mode test, benchmark, or leaderboard") + })?; + + eprintln!("Running public evaluation in your Modal account"); + eprintln!("Leaderboard: {}", final_leaderboard); + eprintln!("GPU: {}", final_gpu); + eprintln!("Mode: {}", final_mode); + eprintln!("File: {}", file_to_submit); + eprintln!("\nWaiting for Modal results..."); + + let result = crate::local::run_modal_submission( + submission_path, + &final_leaderboard, + &final_gpu, + &final_mode, + ) + .await?; + + if let Some(output_path) = output { + if let Some(parent) = Path::new(&output_path).parent() { + std::fs::create_dir_all(parent) + .map_err(|e| anyhow!("Failed to create directories for {}: {}", output_path, e))?; + } + std::fs::write(&output_path, &result) + .map_err(|e| anyhow!("Failed to write result to file {}: {}", output_path, e))?; + eprintln!("\nResults written to: {}", output_path); + } + + println!("\n{}", result); + Ok(()) +} + #[derive(Debug)] struct ProfileReportLink { file_url: String, diff --git a/src/local.rs b/src/local.rs new file mode 100644 index 0000000..4a62884 --- /dev/null +++ b/src/local.rs @@ -0,0 +1,319 @@ +use std::io::ErrorKind; +use std::path::Path; +use std::process::Stdio; + +use anyhow::{anyhow, Context, Result}; +use serde_json::Value; +use tokio::io::{AsyncBufReadExt, BufReader}; +use tokio::process::Command; + +use crate::service; + +const LOCAL_RUNNER: &str = include_str!("../templates/local_modal_runner.py"); +const RESULT_MARKER: &str = "POPCORN_LOCAL_RESULT="; + +fn gpu_names(gpu: &str) -> Result<(&'static str, &'static str)> { + match gpu.to_ascii_lowercase().as_str() { + "t4" => Ok(("T4", "T4")), + "l4" => Ok(("L4", "L4")), + "l4x4" | "l4:4" => Ok(("L4x4", "L4:4")), + "a100" | "a100-80gb" => Ok(("A100", "A100-80GB")), + "h100" | "h100!" => Ok(("H100", "H100!")), + "b200" => Ok(("B200", "B200")), + _ => Err(anyhow!( + "GPU '{}' is not supported by local Modal mode. Supported GPUs: T4, L4, L4x4, A100, H100, B200", + gpu + )), + } +} + +fn score_value(value: Option<&Value>) -> Option { + match value? { + Value::Number(number) => number.as_f64(), + Value::String(text) => text.parse().ok(), + _ => None, + } +} + +fn local_score(payload: &Value) -> Option { + let run_result = payload.pointer("/result/runs/leaderboard/run/result")?; + let count = score_value(run_result.get("benchmark-count"))? as usize; + if count == 0 { + return None; + } + + let scores: Option> = (0..count) + .map(|index| score_value(run_result.get(format!("benchmark.{}.mean", index)))) + .collect(); + let scores = scores?; + let score_ns = match payload + .get("ranking_by") + .and_then(Value::as_str) + .unwrap_or("last") + { + "last" if scores.len() == 1 => scores[0], + "mean" => scores.iter().sum::() / scores.len() as f64, + "geom" => (scores.iter().map(|score| score.ln()).sum::() / scores.len() as f64).exp(), + _ => return None, + }; + Some(score_ns / 1e9) +} + +fn run_failure(run: &Value) -> Option { + let compilation = run.get("compilation").filter(|value| !value.is_null()); + if compilation + .and_then(|value| value.get("success")) + .and_then(Value::as_bool) + == Some(false) + { + return Some(format!( + "Compilation failed:\n{}", + compilation + .and_then(|value| value.get("stderr")) + .and_then(Value::as_str) + .unwrap_or("No compiler error was reported") + )); + } + + let result = run.get("run")?; + if result.get("success").and_then(Value::as_bool) == Some(false) { + return Some(format!( + "Execution failed:\n{}", + result + .get("stderr") + .and_then(Value::as_str) + .unwrap_or("No execution error was reported") + )); + } + None +} + +fn format_local_result(payload: &Value) -> Result { + let result = payload + .get("result") + .ok_or_else(|| anyhow!("Modal runner returned no result"))?; + if result.get("success").and_then(Value::as_bool) != Some(true) { + return Err(anyhow!( + "Local Modal evaluation failed:\n{}", + result + .get("error") + .and_then(Value::as_str) + .unwrap_or("No error details were returned") + )); + } + + let mut sections = vec![format!( + "Local Modal result for {} on {} (not submitted to gpumode.com)", + payload + .get("leaderboard") + .and_then(Value::as_str) + .unwrap_or("unknown leaderboard"), + payload + .get("gpu") + .and_then(Value::as_str) + .unwrap_or("unknown GPU") + )]; + + let runs = result + .get("runs") + .and_then(Value::as_object) + .ok_or_else(|| anyhow!("Modal runner returned no evaluation runs"))?; + for (key, title, formatter) in [ + ( + "test", + "Tests", + service::format_test_rows as fn(&Value) -> Vec, + ), + ( + "benchmark", + "Benchmarks", + service::format_benchmark_rows as fn(&Value) -> Vec, + ), + ( + "leaderboard", + "Ranked benchmarks", + service::format_benchmark_rows as fn(&Value) -> Vec, + ), + ] { + let Some(run) = runs.get(key) else { + continue; + }; + if let Some(failure) = run_failure(run) { + sections.push(format!("{}\n{}", title, failure)); + continue; + } + let rows = run + .pointer("/run/result") + .map(formatter) + .unwrap_or_default(); + if !rows.is_empty() { + sections.push(format!("{}\n{}", title, rows.join("\n\n"))); + } + } + + if let Some(score) = local_score(payload) { + let criterion = payload + .get("ranking_by") + .and_then(Value::as_str) + .unwrap_or("ranked"); + sections.push(format!("Local {} score: {} s", criterion, score)); + } + + sections.push(format!( + "Sources: reference-kernels {} · kernelbot {}", + payload + .get("reference_kernels_ref") + .and_then(Value::as_str) + .unwrap_or("unknown"), + payload + .get("kernelbot_ref") + .and_then(Value::as_str) + .unwrap_or("unknown") + )); + Ok(sections.join("\n\n")) +} + +pub async fn run_modal_submission( + submission_path: &Path, + leaderboard: &str, + gpu: &str, + mode: &str, +) -> Result { + if !matches!( + mode.to_ascii_lowercase().as_str(), + "test" | "benchmark" | "leaderboard" + ) { + return Err(anyhow!( + "Local Modal mode supports test, benchmark, and leaderboard; got '{}'", + mode + )); + } + let (kernelbot_gpu, modal_gpu) = gpu_names(gpu)?; + let helper = tempfile::Builder::new() + .prefix("popcorn-local-modal-") + .suffix(".py") + .tempfile() + .context("Failed to create the local Modal helper")?; + std::fs::write(helper.path(), LOCAL_RUNNER) + .context("Failed to write the local Modal helper")?; + + let submission_path = submission_path + .canonicalize() + .with_context(|| format!("Failed to resolve {}", submission_path.display()))?; + let mut child = Command::new("modal") + .arg("run") + .arg(helper.path()) + .env("POPCORN_LOCAL_SUBMISSION", &submission_path) + .env("POPCORN_LOCAL_LEADERBOARD", leaderboard) + .env("POPCORN_LOCAL_GPU", kernelbot_gpu) + .env("POPCORN_LOCAL_MODAL_GPU", modal_gpu) + .env("POPCORN_LOCAL_MODE", mode.to_ascii_lowercase()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|error| { + if error.kind() == ErrorKind::NotFound { + anyhow!( + "The Modal CLI was not found. Install it with `python3 -m pip install modal`, then configure your token with `modal token set` or MODAL_TOKEN_ID/MODAL_TOKEN_SECRET." + ) + } else { + anyhow!("Failed to start Modal: {}", error) + } + })?; + + let stdout = child.stdout.take().expect("piped stdout"); + let stderr = child.stderr.take().expect("piped stderr"); + let stdout_task = tokio::spawn(async move { + let mut lines = BufReader::new(stdout).lines(); + let mut payload = None; + while let Some(line) = lines.next_line().await? { + if let Some(index) = line.find(RESULT_MARKER) { + payload = Some(line[index + RESULT_MARKER.len()..].to_string()); + } else { + eprintln!("{}", line); + } + } + Ok::<_, std::io::Error>(payload) + }); + let stderr_task = tokio::spawn(async move { + let mut lines = BufReader::new(stderr).lines(); + while let Some(line) = lines.next_line().await? { + eprintln!("{}", line); + } + Ok::<_, std::io::Error>(()) + }); + + let status = child + .wait() + .await + .context("Failed while waiting for Modal")?; + let payload = stdout_task.await.context("Modal stdout task failed")??; + stderr_task.await.context("Modal stderr task failed")??; + if !status.success() { + return Err(anyhow!("Modal exited with status {}", status)); + } + let payload = payload.ok_or_else(|| anyhow!("Modal returned no Popcorn result"))?; + let payload: Value = serde_json::from_str(&payload).context("Modal returned invalid JSON")?; + format_local_result(&payload) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn maps_popcorn_gpu_names_to_modal() { + assert_eq!(gpu_names("B200").unwrap(), ("B200", "B200")); + assert_eq!(gpu_names("H100!").unwrap(), ("H100", "H100!")); + assert_eq!(gpu_names("A100-80GB").unwrap(), ("A100", "A100-80GB")); + assert_eq!(gpu_names("L4:4").unwrap(), ("L4x4", "L4:4")); + assert!(gpu_names("MI300").is_err()); + } + + #[test] + fn computes_geometric_mean_score_in_seconds() { + let payload = serde_json::json!({ + "ranking_by": "geom", + "result": {"runs": {"leaderboard": {"run": {"result": { + "benchmark-count": 2, + "benchmark.0.mean": 1_000_000, + "benchmark.1.mean": 4_000_000 + }}}}} + }); + let score = local_score(&payload).unwrap(); + assert!((score - 0.002).abs() < 1e-12); + } + + #[test] + fn formats_all_public_leaderboard_stages() { + let payload = serde_json::json!({ + "leaderboard": "vectoradd_py", + "gpu": "B200", + "mode": "leaderboard", + "ranking_by": "geom", + "reference_kernels_ref": "abc", + "kernelbot_ref": "def", + "result": { + "success": true, + "error": "", + "system": {}, + "runs": { + "test": {"compilation": null, "run": {"success": true, "passed": true, "result": { + "test-count": 1, "test.0.status": "pass", "test.0.spec": "size=128" + }}}, + "benchmark": {"compilation": null, "run": {"success": true, "passed": true, "result": { + "benchmark-count": 1, "benchmark.0.spec": "size=1024", "benchmark.0.mean": 1000 + }}}, + "leaderboard": {"compilation": null, "run": {"success": true, "passed": true, "result": { + "benchmark-count": 1, "benchmark.0.spec": "size=1024", "benchmark.0.mean": 1000 + }}} + } + } + }); + let output = format_local_result(&payload).unwrap(); + assert!(output.contains("Tests\n✅ size=128")); + assert!(output.contains("Benchmarks\nsize=1024")); + assert!(output.contains("Ranked benchmarks\nsize=1024")); + assert!(output.contains("not submitted to gpumode.com")); + } +} diff --git a/src/main.rs b/src/main.rs index 03e1c68..0b36a38 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,5 @@ mod cmd; +mod local; mod models; mod service; mod utils; diff --git a/src/service/mod.rs b/src/service/mod.rs index 7ac897c..d0c1ecc 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -1168,7 +1168,7 @@ fn format_time(nanoseconds: f64, error: Option) -> String { } } -fn format_test_rows(result: &Value) -> Vec { +pub(crate) fn format_test_rows(result: &Value) -> Vec { let Some(count) = result_count(result, "test-count") else { return Vec::new(); }; @@ -1196,7 +1196,7 @@ fn format_test_rows(result: &Value) -> Vec { .collect() } -fn format_benchmark_rows(result: &Value) -> Vec { +pub(crate) fn format_benchmark_rows(result: &Value) -> Vec { let Some(count) = result_count(result, "benchmark-count") else { return Vec::new(); }; diff --git a/templates/local_modal_runner.py b/templates/local_modal_runner.py new file mode 100644 index 0000000..6157d14 --- /dev/null +++ b/templates/local_modal_runner.py @@ -0,0 +1,226 @@ +"""Modal entrypoint embedded by popcorn-cli's --local mode. + +The image and evaluator intentionally track KernelBot's public Modal runner so a +local run exercises the same public task definition and evaluation sequence. +""" + +import dataclasses +import glob +import json +import os +import re +import time +import traceback +import urllib.request +from pathlib import Path + +import modal + + +RESULT_MARKER = "POPCORN_LOCAL_RESULT=" +REFERENCE_REPO = "gpu-mode/reference-kernels" +KERNELBOT_REPO = "gpu-mode/kernelbot" + + +def _github_ref(repo: str, override_env: str) -> str: + override = os.environ.get(override_env) + if override: + if not re.fullmatch(r"[A-Za-z0-9._/-]+", override): + raise ValueError(f"Invalid git ref in {override_env}") + return override + + headers = { + "Accept": "application/vnd.github+json", + "User-Agent": "popcorn-cli", + "Cache-Control": "no-cache", + } + github_token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") + if github_token: + headers["Authorization"] = f"Bearer {github_token}" + + errors = [] + for attempt in range(3): + request = urllib.request.Request( + f"https://api.github.com/repos/{repo}/git/ref/heads/main", + headers=headers, + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + sha = json.load(response)["object"]["sha"] + if re.fullmatch(r"[0-9a-f]{40}", sha): + return sha + errors.append(f"invalid SHA: {sha!r}") + except Exception as error: + errors.append(str(error)) + if attempt < 2: + time.sleep(attempt + 1) + + raise RuntimeError( + f"Could not resolve the latest {repo} main commit after 3 attempts: " + f"{errors[-1]}. Set {override_env} to an explicit commit to run pinned instead." + ) + + +reference_ref = _github_ref(REFERENCE_REPO, "POPCORN_REFERENCE_KERNELS_REF") +kernelbot_ref = _github_ref(KERNELBOT_REPO, "POPCORN_KERNELBOT_REF") + +cuda_version = "13.3.0" +mathdx_version = "26.06.0" +mathdx_archive = f"nvidia-mathdx-{mathdx_version}-cuda13.tar.gz" +mathdx_url = ( + "https://developer.download.nvidia.com/compute/cublasdx/redist/" + f"cublasdx/cuda13/{mathdx_archive}" +) +mathdx_sha256 = "042b7c57a636c271cca32dffcc0a822ed6b2abc0b8ef5703ab2445d58563a1e6" +cuda_image = ( + modal.Image.from_registry( + f"nvidia/cuda:{cuda_version}-devel-ubuntu24.04", add_python="3.13" + ) + .entrypoint([]) + .run_commands("ln -sf $(which python) /usr/local/bin/python3") + .apt_install("git", "curl", "gcc-13", "g++-13", "clang-18") + .uv_pip_install( + "ninja~=1.11", + "wheel~=0.45", + "requests~=2.32.4", + "packaging~=25.0", + "numpy~=2.3", + "pytest", + "PyYAML", + ) + .uv_pip_install( + "tinygrad~=0.10", + "helion", + ) + .uv_pip_install( + "nvidia-cutlass-dsl==4.5.2", + "cuda-core[cu13]", + "cuda-python[all]==13.0", + "cuda-tile==1.4.0", + "nvmath-python[cu13-dx]==0.9.0", + "nvidia-libmathdx-cu13==0.3.2.6", + "cuda-toolkit[cccl,nvrtc]==13.0.2", + ) + .uv_pip_install( + "torch==2.12.0", + ) + .run_commands( + "git clone --depth 1 --branch v4.5.2 https://github.com/NVIDIA/cutlass.git /opt/cutlass", + ( + f"curl -fsSL {mathdx_url} -o /tmp/{mathdx_archive} && " + f"echo '{mathdx_sha256} /tmp/{mathdx_archive}' | sha256sum -c - && " + "mkdir -p /opt/mathdx && " + f"tar -xzf /tmp/{mathdx_archive} --strip-components=4 -C /opt/mathdx && " + f"rm /tmp/{mathdx_archive}" + ), + ( + "git clone --filter=blob:none https://github.com/" + f"{KERNELBOT_REPO}.git /opt/kernelbot && " + f"git -C /opt/kernelbot checkout {kernelbot_ref}" + ), + ( + "git clone --filter=blob:none https://github.com/" + f"{REFERENCE_REPO}.git /opt/reference-kernels && " + f"git -C /opt/reference-kernels checkout {reference_ref}" + ), + ) + .env( + { + "CUTLASS_PATH": "/opt/cutlass", + "MATHDX_HOME": "/opt/mathdx", + "CPLUS_INCLUDE_PATH": ( + "/opt/mathdx/include:/opt/mathdx/external/cutlass/include:" + "/opt/cutlass/include:/opt/cutlass/tools/util/include" + ), + "PYTHONPATH": "/opt/kernelbot/src", + } + ) +) + +app = modal.App("popcorn-local-runner", image=cuda_image) +modal_gpu = os.environ["POPCORN_LOCAL_MODAL_GPU"] + + +def _find_problem(leaderboard: str) -> tuple[Path, list[str]]: + import yaml + + matches = [] + for index_path in glob.glob("/opt/reference-kernels/problems/*.y*ml"): + with open(index_path) as file: + index = yaml.safe_load(file) or {} + for problem in index.get("problems", []): + directory = problem.get("directory", "") + if problem.get("name") == leaderboard or Path(directory).name == leaderboard: + matches.append((directory, problem.get("gpus", []))) + + unique_matches = list(dict.fromkeys(directory for directory, _ in matches)) + if not unique_matches: + raise ValueError(f"Leaderboard '{leaderboard}' was not found in reference-kernels") + if len(unique_matches) > 1: + raise ValueError( + f"Leaderboard '{leaderboard}' is ambiguous: {', '.join(unique_matches)}" + ) + + directory = unique_matches[0] + supported_gpus = next(gpus for candidate, gpus in matches if candidate == directory) + return Path("/opt/reference-kernels/problems") / directory / "task.yml", supported_gpus + + +@app.function(gpu=modal_gpu, timeout=3600) +def evaluate(submission: str, leaderboard: str, gpu: str, mode: str) -> dict: + try: + from libkernelbot.consts import GPU_TO_SM, SubmissionMode + from libkernelbot.run_eval import run_config + from libkernelbot.task import build_task_config, make_task_definition + + task_path, supported_gpus = _find_problem(leaderboard) + if supported_gpus and gpu not in supported_gpus: + raise ValueError( + f"Leaderboard '{leaderboard}' does not declare GPU '{gpu}'. " + f"Supported GPUs: {', '.join(supported_gpus)}" + ) + + definition = make_task_definition(task_path) + config = build_task_config( + task=definition.task, + submission_content=submission, + arch=GPU_TO_SM[gpu], + mode=SubmissionMode(mode), + ) + result = run_config(config) + return { + "leaderboard": leaderboard, + "problem_directory": str(task_path.parent.relative_to("/opt/reference-kernels/problems")), + "gpu": gpu, + "mode": mode, + "ranking_by": definition.task.ranking_by.value, + "reference_kernels_ref": reference_ref, + "kernelbot_ref": kernelbot_ref, + "result": dataclasses.asdict(result), + } + except Exception as error: + return { + "leaderboard": leaderboard, + "gpu": gpu, + "mode": mode, + "reference_kernels_ref": reference_ref, + "kernelbot_ref": kernelbot_ref, + "result": { + "success": False, + "error": "".join(traceback.format_exception(error)), + "runs": {}, + "system": {}, + }, + } + + +@app.local_entrypoint() +def main(): + submission_path = Path(os.environ["POPCORN_LOCAL_SUBMISSION"]) + payload = evaluate.remote( + submission_path.read_text(), + os.environ["POPCORN_LOCAL_LEADERBOARD"], + os.environ["POPCORN_LOCAL_GPU"], + os.environ["POPCORN_LOCAL_MODE"], + ) + print(RESULT_MARKER + json.dumps(payload, default=str, separators=(",", ":")))