Add local Modal leaderboard mode

Cargo.toml

@@ -25,6 +25,4 @@ urlencoding = "2.1.3"
 bytes = "1.11.1"
 futures-util = "0.3.31"
 zip = "2.2.2"
-
-[dev-dependencies]
 tempfile = "3.10"

README.md

@@ -151,10 +151,51 @@ POPCORN_BREV_PROFILER_URL=https://http--brev-profiler-proxy--dxfjds728w5v.code.r
 # Plain output mode (no TUI, good for CI/scripts)
 popcorn submit --no-tui --leaderboard grayscale_v2 --gpu A100 --mode test solution.py
 
+# Run the public leaderboard pipeline in your own Modal account
+MODAL_TOKEN_ID=... MODAL_TOKEN_SECRET=... popcorn submit --local --leaderboard eigh --gpu B200 --mode leaderboard solution.py
+
 # Save results to a file
 popcorn submit --output results.json --leaderboard grayscale_v2 --gpu A100 --mode benchmark solution.py
 ```
 
+#### Local Modal mode
+
+`--local` runs the public `reference-kernels` task with KernelBot's evaluator
+on a GPU billed to your Modal account. It bypasses Popcorn registration and the
+GPU Mode API, and implies plain (non-TUI) output. The supported Modal GPU names
+are `T4`, `L4`, `L4x4`, `A100`, `H100`, and `B200`; AMD runners are not available
+through Modal.
+
+Install and authenticate Modal first:
+
+```bash
+python3 -m pip install modal
+
+# Either save the token in Modal's local config...
+modal token set
+
+# ...or supply it without changing local config.
+export MODAL_TOKEN_ID=your-token-id
+export MODAL_TOKEN_SECRET=your-token-secret
+```
+
+Then use the normal submission modes:
+
+```bash
+popcorn submit --local --leaderboard eigh --gpu B200 --mode test submission.py
+popcorn submit --local --leaderboard eigh --gpu B200 --mode benchmark submission.py
+popcorn submit --local --leaderboard eigh --gpu B200 --mode leaderboard submission.py
+```
+
+Local `leaderboard` mode runs the same public test, benchmark, and ranked
+evaluation stages and computes the task's ranking score. It does not run GPU
+Mode's private seed, update gpumode.com, or create an official submission. The
+output records the exact `reference-kernels` and KernelBot commits used. Set
+`POPCORN_REFERENCE_KERNELS_REF` or `POPCORN_KERNELBOT_REF` to a branch, tag, or
+commit when you need to pin or test another public revision. By default, the
+runner resolves both `main` branches to immutable commit SHAs on every command;
+if either lookup fails, it stops instead of risking a stale cached image.
+
 **Submission modes:**
 - `test` - Quick test run to check correctness
 - `benchmark` - Benchmark your solution (no leaderboard impact)

src/cmd/mod.rs

@@ -67,6 +67,11 @@ pub struct Cli {
     #[arg(long)]
     pub profile_brev: bool,
 
+    /// Run the public evaluation in your own Modal account instead of submitting to GPU Mode.
+    /// Uses Modal's configured profile or MODAL_TOKEN_ID/MODAL_TOKEN_SECRET.
+    #[arg(long, conflicts_with = "profile_brev")]
+    pub local: bool,
+
     /// Optional: Profile a single benchmark index when using --profile-brev
     #[arg(long)]
     pub benchmark_index: Option<usize>,
@@ -151,6 +156,11 @@ enum Commands {
         #[arg(long)]
         profile_brev: bool,
 
+        /// Run the public evaluation in your own Modal account instead of submitting to GPU Mode.
+        /// Uses Modal's configured profile or MODAL_TOKEN_ID/MODAL_TOKEN_SECRET.
+        #[arg(long, conflicts_with = "profile_brev")]
+        local: bool,
+
         /// Optional: Profile a single benchmark index when using --profile-brev
         #[arg(long)]
         benchmark_index: Option<usize>,
@@ -203,53 +213,60 @@ pub async fn execute(cli: Cli) -> Result<()> {
             leaderboard,
             mode,
             profile_brev,
+            local,
             benchmark_index,
             output,
             no_tui,
         }) => {
-            let config = load_config()?;
-            let cli_id = config.cli_id.ok_or_else(|| {
-                anyhow!(
-                    "cli_id not found in config file ({}). Please run 'popcorn-cli register' first.",
-                    get_config_path()
-                        .map_or_else(|_| "unknown path".to_string(), |p| p.display().to_string())
-                )
-            })?;
-
             // Use filepath from Submit command first, fallback to top-level filepath
             let final_filepath = filepath.or(cli.filepath);
             let final_gpu = if profile_brev {
                 Some("B200_Brev".to_string())
             } else {
-                gpu
+                gpu.clone()
             };
             let final_mode = if profile_brev {
                 Some("profile".to_string())
             } else {
-                mode
+                mode.clone()
             };
 
-            if no_tui || profile_brev {
-                submit::run_submit_plain(
-                    final_filepath, // Resolved filepath
-                    final_gpu,      // From Submit command
-                    leaderboard,    // From Submit command
-                    final_mode,     // From Submit command
-                    cli_id,
-                    benchmark_index.or(cli.benchmark_index),
-                    output, // From Submit command
-                )
-                .await
+            if local {
+                submit::run_submit_local(final_filepath, gpu, leaderboard, mode, output).await
             } else {
-                submit::run_submit_tui(
-                    final_filepath, // Resolved filepath
-                    final_gpu,      // From Submit command
-                    leaderboard,    // From Submit command
-                    final_mode,     // From Submit command
-                    cli_id,
-                    output, // From Submit command
-                )
-                .await
+                let config = load_config()?;
+                let cli_id = config.cli_id.ok_or_else(|| {
+                    anyhow!(
+                        "cli_id not found in config file ({}). Please run 'popcorn-cli register' first.",
+                        get_config_path().map_or_else(
+                            |_| "unknown path".to_string(),
+                            |p| p.display().to_string()
+                        )
+                    )
+                })?;
+
+                if no_tui || profile_brev {
+                    submit::run_submit_plain(
+                        final_filepath, // Resolved filepath
+                        final_gpu,      // From Submit command
+                        leaderboard,    // From Submit command
+                        final_mode,     // From Submit command
+                        cli_id,
+                        benchmark_index.or(cli.benchmark_index),
+                        output, // From Submit command
+                    )
+                    .await
+                } else {
+                    submit::run_submit_tui(
+                        final_filepath, // Resolved filepath
+                        final_gpu,      // From Submit command
+                        leaderboard,    // From Submit command
+                        final_mode,     // From Submit command
+                        cli_id,
+                        output, // From Submit command
+                    )
+                    .await
+                }
             }
         }
         Some(Commands::Join { code }) => {
@@ -301,6 +318,7 @@ pub async fn execute(cli: Cli) -> Result<()> {
         None => {
             // Check if any of the submission-related flags were used at the top level
             if !cli.profile_brev
+                && !cli.local
                 && (cli.gpu.is_some() || cli.leaderboard.is_some() || cli.mode.is_some())
             {
                 return Err(anyhow!(
@@ -311,37 +329,50 @@ pub async fn execute(cli: Cli) -> Result<()> {
 
             // Handle the case where only a filepath is provided (for backward compatibility)
             if let Some(top_level_filepath) = cli.filepath {
-                let config = load_config()?;
-                let cli_id = config.cli_id.ok_or_else(|| {
-                    anyhow!(
-                        "cli_id not found in config file ({}). Please run `popcorn register` first.",
-                        get_config_path()
-                            .map_or_else(|_| "unknown path".to_string(), |p| p.display().to_string())
-                    )
-                })?;
-
-                if cli.profile_brev {
-                    submit::run_submit_plain(
+                if cli.local {
+                    submit::run_submit_local(
                         Some(top_level_filepath),
-                        Some("B200_Brev".to_string()),
+                        cli.gpu,
                         cli.leaderboard,
-                        Some("profile".to_string()),
-                        cli_id,
-                        cli.benchmark_index,
+                        cli.mode,
                         cli.output,
                     )
                     .await
                 } else {
-                    // Run TUI with only filepath, no other options
-                    submit::run_submit_tui(
-                        Some(top_level_filepath),
-                        None, // No GPU option
-                        None, // No leaderboard option
-                        None, // No mode option
-                        cli_id,
-                        None, // No output option
-                    )
-                    .await
+                    let config = load_config()?;
+                    let cli_id = config.cli_id.ok_or_else(|| {
+                        anyhow!(
+                            "cli_id not found in config file ({}). Please run `popcorn register` first.",
+                            get_config_path().map_or_else(
+                                |_| "unknown path".to_string(),
+                                |p| p.display().to_string()
+                            )
+                        )
+                    })?;
+
+                    if cli.profile_brev {
+                        submit::run_submit_plain(
+                            Some(top_level_filepath),
+                            Some("B200_Brev".to_string()),
+                            cli.leaderboard,
+                            Some("profile".to_string()),
+                            cli_id,
+                            cli.benchmark_index,
+                            cli.output,
+                        )
+                        .await
+                    } else {
+                        // Run TUI with only filepath, no other options
+                        submit::run_submit_tui(
+                            Some(top_level_filepath),
+                            None, // No GPU option
+                            None, // No leaderboard option
+                            None, // No mode option
+                            cli_id,
+                            None, // No output option
+                        )
+                        .await
+                    }
                 }
             } else {
                 Err(anyhow!(

src/cmd/submit.rs

@@ -808,6 +808,74 @@ pub async fn run_submit_plain(
     Ok(())
 }
 
+pub async fn run_submit_local(
+    filepath: Option<String>,
+    gpu: Option<String>,
+    leaderboard: Option<String>,
+    mode: Option<String>,
+    output: Option<String>,
+) -> Result<()> {
+    let file_to_submit = filepath.ok_or_else(|| anyhow!("File path is required with --local"))?;
+    let submission_path = Path::new(&file_to_submit);
+    if !submission_path.exists() {
+        return Err(anyhow!("File not found: {}", file_to_submit));
+    }
+    if utils::is_archive_file(submission_path) {
+        return Err(anyhow!(
+            "Local Modal mode currently supports single source files, not archives"
+        ));
+    }
+
+    let (directives, has_multiple_gpus) = utils::get_popcorn_directives(submission_path)?;
+    if has_multiple_gpus {
+        return Err(anyhow!(
+            "Multiple GPUs are not supported yet. Please specify only one GPU."
+        ));
+    }
+
+    let final_gpu = gpu
+        .or_else(|| directives.gpus.first().cloned())
+        .ok_or_else(|| anyhow!("GPU not specified. Use --gpu or add a GPU directive"))?;
+    let final_leaderboard = leaderboard
+        .or_else(|| {
+            (!directives.leaderboard_name.is_empty()).then_some(directives.leaderboard_name.clone())
+        })
+        .ok_or_else(|| {
+            anyhow!("Leaderboard not specified. Use --leaderboard or add a leaderboard directive")
+        })?;
+    let final_mode = mode.ok_or_else(|| {
+        anyhow!("Submission mode not specified. Use --mode test, benchmark, or leaderboard")
+    })?;
+
+    eprintln!("Running public evaluation in your Modal account");
+    eprintln!("Leaderboard: {}", final_leaderboard);
+    eprintln!("GPU: {}", final_gpu);
+    eprintln!("Mode: {}", final_mode);
+    eprintln!("File: {}", file_to_submit);
+    eprintln!("\nWaiting for Modal results...");
+
+    let result = crate::local::run_modal_submission(
+        submission_path,
+        &final_leaderboard,
+        &final_gpu,
+        &final_mode,
+    )
+    .await?;
+
+    if let Some(output_path) = output {
+        if let Some(parent) = Path::new(&output_path).parent() {
+            std::fs::create_dir_all(parent)
+                .map_err(|e| anyhow!("Failed to create directories for {}: {}", output_path, e))?;
+        }
+        std::fs::write(&output_path, &result)
+            .map_err(|e| anyhow!("Failed to write result to file {}: {}", output_path, e))?;
+        eprintln!("\nResults written to: {}", output_path);
+    }
+
+    println!("\n{}", result);
+    Ok(())
+}
+
 #[derive(Debug)]
 struct ProfileReportLink {
     file_url: String,