Efficient ale_bench_eval

cloud/main.tf

@@ -1,133 +1,138 @@
 terraform {
-    required_providers {
-        aws = {
-            source  = "hashicorp/aws"
-            version = "~> 4.16"
-        }
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 4.16"
     }
+  }
 
-    required_version = ">= 1.2.0"
+  required_version = ">= 1.2.0"
 }
 
 provider "aws" {
-    region = var.region
+  region = var.region
 }
 
-data "aws_ami" "ubuntu" {
-    most_recent = true
-    owners      = ["099720109477"]  # Canonical
-
-    filter {
-        name   = "name"
-        values = ["ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-*"]
-    }
-
-    filter {
-        name   = "virtualization-type"
-        values = ["hvm"]
-    }
+locals {
+  normalized_name_prefix = trimsuffix(trimspace(var.name_prefix), "-")
+  resource_name_prefix   = local.normalized_name_prefix == "" ? "" : "${local.normalized_name_prefix}-"
+}
 
-    filter {
-        name   = "architecture"
-        values = ["x86_64"]
-    }
+data "aws_ami" "ubuntu" {
+  most_recent = true
+  owners      = ["099720109477"] # Canonical
+
+  filter {
+    name   = "name"
+    values = ["ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-*"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  filter {
+    name   = "architecture"
+    values = ["x86_64"]
+  }
 }
 
 resource "aws_key_pair" "aws_key" {
-    key_name   = var.aws_key_name
-    public_key = file(var.ssh_public_key_path)
+  key_name   = "${local.resource_name_prefix}${var.aws_key_name}"
+  public_key = file(var.ssh_public_key_path)
 }
 
 data "aws_vpc" "default" {
-    default = true
+  default = true
 }
 
 data "aws_subnets" "default" {
-    filter {
-        name   = "vpc-id"
-        values = [data.aws_vpc.default.id]
-    }
+  filter {
+    name   = "vpc-id"
+    values = [data.aws_vpc.default.id]
+  }
 }
 
 resource "aws_security_group" "ssh" {
-    name        = "allow-ssh"
-    description = "Allow SSH"
-    vpc_id      = data.aws_vpc.default.id
-
-    ingress {
-        description = "SSH from specific IP"
-        from_port   = 22
-        to_port     = 22
-        protocol    = "tcp"
-        cidr_blocks = [var.allowed_ssh_cidr]
-    }
-
-    egress {
-        description = "Allow all outbound traffic"
-        from_port   = 0
-        to_port     = 0
-        protocol    = "-1"
-        cidr_blocks = ["0.0.0.0/0"]
-    }
-
-    tags = {
-        Name = "ale-bench-security-group-ssh"
-        Project = "ale-bench"
-        ManagedBy = "terraform"
-    }
+  name        = "${local.resource_name_prefix}allow-ssh"
+  description = "Allow SSH"
+  vpc_id      = data.aws_vpc.default.id
+
+  ingress {
+    description = "SSH from specific IP"
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = [var.allowed_ssh_cidr]
+  }
+
+  egress {
+    description = "Allow all outbound traffic"
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  tags = {
+    Name      = "${local.resource_name_prefix}ale-bench-security-group-ssh"
+    Project   = "ale-bench"
+    ManagedBy = "terraform"
+  }
 }
 
 resource "aws_instance" "ale_bench_instance" {
-    count         = var.instance_count
-    ami           = data.aws_ami.ubuntu.id
-    instance_type = var.instance_type
-    key_name      = aws_key_pair.aws_key.key_name
-
-    vpc_security_group_ids = [aws_security_group.ssh.id]
-    subnet_id = tolist(data.aws_subnets.default.ids)[0]
-    associate_public_ip_address = true
-
-    root_block_device {
-        volume_type           = "gp3"
-        volume_size           = var.instance_volume_size
-        iops                  = 3000
-        throughput            = 125
-        delete_on_termination = true
-        tags = {
-            Name = "ale-bench-instance-${count.index}-storage"
-            Project = "ale-bench"
-            ManagedBy = "terraform"
-        }
+  count         = var.instance_count
+  ami           = data.aws_ami.ubuntu.id
+  instance_type = var.instance_type
+  key_name      = aws_key_pair.aws_key.key_name
+
+  vpc_security_group_ids      = [aws_security_group.ssh.id]
+  subnet_id                   = tolist(data.aws_subnets.default.ids)[0]
+  associate_public_ip_address = true
+
+  root_block_device {
+    volume_type           = "gp3"
+    volume_size           = var.instance_volume_size
+    iops                  = 3000
+    throughput            = 125
+    delete_on_termination = true
+    tags = {
+      Name      = "${local.resource_name_prefix}ale-bench-instance-${count.index}-storage"
+      Project   = "ale-bench"
+      ManagedBy = "terraform"
     }
+  }
 
-    disable_api_stop = false
-    disable_api_termination = false
-    instance_initiated_shutdown_behavior = "stop"
+  disable_api_stop                     = false
+  disable_api_termination              = false
+  instance_initiated_shutdown_behavior = "stop"
 
-    user_data = file(var.setup_file_name)
+  user_data = file(var.setup_file_name)
 
-    tags = {
-        Name = "ale-bench-instance-${count.index}"
-        Project = "ale-bench"
-        ManagedBy = "terraform"
-    }
+  tags = {
+    Name      = "${local.resource_name_prefix}ale-bench-instance-${count.index}"
+    Project   = "ale-bench"
+    ManagedBy = "terraform"
+  }
 
-    metadata_options {
-        http_endpoint = "enabled"
-        http_tokens   = "required"
-    }
+  metadata_options {
+    http_endpoint = "enabled"
+    http_tokens   = "required"
+  }
 
-    lifecycle {
-        ignore_changes = [ami]
-    }
+  lifecycle {
+    ignore_changes = [ami]
+  }
 }
 
 output "instance_public_ips" {
-    value = sort(aws_instance.ale_bench_instance[*].public_ip)
-    description = "Public IP addresses of the instances"
+  value       = sort(aws_instance.ale_bench_instance[*].public_ip)
+  description = "Public IP addresses of the instances"
 }
 
 output "ssh_connection_string" {
-    value = sort([for i in range(var.instance_count) : "ssh -i </path/to/your/secret_key> ubuntu@${aws_instance.ale_bench_instance[i].public_ip}"])
-    description = "SSH connection strings for the instances"
+  value       = sort([for i in range(var.instance_count) : "ssh -i </path/to/your/secret_key> ubuntu@${aws_instance.ale_bench_instance[i].public_ip}"])
+  description = "SSH connection strings for the instances"
 }

cloud/variables.tf

@@ -1,47 +1,53 @@
 variable "region" {
-    description = "AWS region"
-    type        = string
-    default     = "us-east-1"
+  description = "AWS region"
+  type        = string
+  default     = "us-east-1"
 }
 
 variable "instance_type" {
-    description = "EC2 instance type"
-    type        = string
-    default     = "c6i.8xlarge"
+  description = "EC2 instance type"
+  type        = string
+  default     = "c6i.8xlarge"
 }
 
 variable "instance_count" {
-    description = "The number of instances to launch"
-    type        = number
-    default     = 1
+  description = "The number of instances to launch"
+  type        = number
+  default     = 1
 }
 
 variable "instance_volume_size" {
-    description = "Size of the root volume in GB"
-    type        = number
-    default     = 100
+  description = "Size of the root volume in GB"
+  type        = number
+  default     = 100
+}
+
+variable "name_prefix" {
+  description = "Optional prefix for AWS resource names, useful when deploying multiple stacks in one account"
+  type        = string
+  default     = ""
 }
 
 variable "aws_key_name" {
-    description = "Name of the AWS key pair"
-    type        = string
-    default     = "ale-bench"
+  description = "Name of the AWS key pair"
+  type        = string
+  default     = "ale-bench"
 }
 
 variable "ssh_public_key_path" {
-    description = "Path to the public key used for SSH access"
-    type        = string
-    default     = "~/.ssh/id_rsa.pub"
+  description = "Path to the public key used for SSH access"
+  type        = string
+  default     = "~/.ssh/id_rsa.pub"
 }
 
 variable "allowed_ssh_cidr" {
-    description = "CIDR block allowed for SSH access"
-    type        = string
-    default     = "0.0.0.0/0"
+  description = "CIDR block allowed for SSH access"
+  type        = string
+  default     = "0.0.0.0/0"
 }
 
 variable "setup_file_name" {
-    description = "Name of the setup file to be copied to the instance"
-    type        = string
-    default     = "setup.sh"
+  description = "Name of the setup file to be copied to the instance"
+  type        = string
+  default     = "setup.sh"
 }

docs/evaluation.md

@@ -68,7 +68,7 @@ uv run -m ale_bench_eval --model_config_path llm_configs/gpt-5.json --num_worker
 bash scripts/run_eval.sh gpt-5
 
 # Or directly run using uv
-uv run -m ale_bench_eval --model_config_path llm_configs/gpt-5.json --n_repeated_sampling 15 --n_self_refine 16 --num_workers 10 --n_public_cases 50 --judge_version 202510 --code_language typescript --prompt_language en --max_parallel_problems 5 --problem_ids_type all --selection_method median
+uv run -m ale_bench_eval --model_config_path llm_configs/gpt-5.json --n_repeated_sampling 15 --n_self_refine 16 --num_workers 10 --n_public_cases 50 --judge_version 202510 --code_language typescript --prompt_language en --max_parallel_problems 5 --max_concurrent_llm_calls 20 --max_repeated_sampling_workers 100 --problem_ids_type all --selection_method median
 ```
 
 ### Bash Script Arguments
@@ -88,6 +88,8 @@ Script options:
 | `--judge_version`, `-j` | `202301` | Judge toolchain version (`201907`, `202301`, `202510`) |
 | `--code_language`, `-c` | depends on judge version | Programming language for generation/evaluation |
 | `--prompt_language`, `-p` | `en` | Prompt language (`en` or `ja`) |
+| `--max_concurrent_llm_calls` | Python CLI default | Maximum number of in-flight LLM calls across all problems (`none` uses the maximum possible repeated-sampling fan-out) |
+| `--max_repeated_sampling_workers` | Python CLI default | Maximum repeated-sampling LLM worker threads per problem (`none` means `n_repeated_sampling`) |
 | `--help`, `-h` | - | Show usage |
 
 Default `code_language` by `judge_version`:
@@ -105,6 +107,9 @@ bash scripts/run_eval.sh -j 202510 -c typescript -p ja -r results/gpt5-ts-ja gpt
 
 # options after config_name
 bash scripts/run_eval.sh gpt-5 --judge_version 202510 --code_language rust --root_path results/gpt5-rust
+
+# limit global LLM request concurrency while allowing repeated sampling to fan out per problem
+bash scripts/run_eval.sh gpt-5 --max_concurrent_llm_calls 20 --max_repeated_sampling_workers 100
 ```
 
 ### Command Line Arguments
@@ -120,6 +125,8 @@ bash scripts/run_eval.sh gpt-5 --judge_version 202510 --code_language rust --roo
 | `code_language` | str | `cpp20` | Target programming language (`any`, `bash`, `cpp17`, `cpp20`, `cpp23`, `csharp`, `fish`, `fortran`, `go`, `haskell`, `javascript`, `julia`, `lean`, `ocaml`, `perl`, `pypy`, `python`, `rust`, `typescript`) |
 | `prompt_language` | str | `en` | Prompt language (`en` for English, `ja` for Japanese) |
 | `max_parallel_problems` | int | `1` | Maximum number of problems to evaluate in parallel |
+| `max_concurrent_llm_calls` | int \| `None` | `None` | Maximum number of in-flight LLM calls across all problems (`None`/`none` resolves to `max_parallel_problems * effective_max_repeated_sampling_workers`) |
+| `max_repeated_sampling_workers` | int \| `None` | `None` | Maximum repeated-sampling LLM worker threads per problem (`None`/`none` resolves to `n_repeated_sampling`) |
 | `problem_ids_type` | str | `debug` | Problem ID set to evaluate (`debug`, `lite`, `all`) |
 | `selection_method` | str | `median` | Method to select solution from repeated sampling (`best`, `median`) |
 | `use_statement_image` | bool | `False` | Whether to use statement images in the evaluation process (requires a vision-capable model/provider) |
@@ -128,6 +135,8 @@ bash scripts/run_eval.sh gpt-5 --judge_version 202510 --code_language rust --roo
 
 > **Note**: Ensure that `num_workers` $\times$ `max_parallel_problems` does not exceed the number of physical CPU cores available on your machine to avoid resource contention and performance degradation.
 
+> **Note**: `max_parallel_problems` controls problem-level concurrency. `max_repeated_sampling_workers` controls only repeated-sampling LLM generation within each problem. If it is `None`, it is resolved to `n_repeated_sampling`; otherwise it is capped at `n_repeated_sampling`. `max_concurrent_llm_calls` is a global cap shared by repeated sampling and self-refinement LLM calls. If it is `None`, it is resolved to `max_parallel_problems * effective_max_repeated_sampling_workers`. Judge execution remains bounded by `num_workers` for each active problem.
+
 > **Note**: `code_language` must be supported by the selected `judge_version`.
 > If `code_language=any`, available languages are:
 > - `201907`: `cpp17`, `python`, `rust`
@@ -166,7 +175,7 @@ results/
         │   └── self_refine_results_<n>.json                # Self-refinement public result (n = number of iterations)
         ├── conversations/                                  # Conversations with LLM
         │   ├── repeated_sampling_conversations_<n>.json    # Repeated sampling conversations (n = number of iterations)
-        │   └── self_refine_conversations.json              # Self-refinement conversations
+        │   └── self_refine_conversations_<n>.json          # Self-refinement conversations (n = number of iterations)
         ├── results/
         │   ├── final_results.json                          # Private evaluation results
         │   ├── repeated_sampling_results.json              # Repeated sampling public evaluation results

llm_configs/claude-4.8-opus-high.json

@@ -0,0 +1,12 @@
+{
+    "model_name": "claude-opus-4-8",
+    "provider": "anthropic",
+    "settings": {
+        "max_tokens": 128000,
+        "anthropic_thinking": {
+            "type": "adaptive",
+            "display": "summarized"
+        },
+        "anthropic_effort": "high"
+    }
+}