FEAT: Add devcontainer setup + fix automated environment building

.devcontainer/devcontainer.json

@@ -0,0 +1,30 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/python
+{
+	"name": "Benchmark TSFM",
+	"image": "mcr.microsoft.com/devcontainers/anaconda:3",
+	"remoteUser": "root",
+
+	"runArgs": [
+		"--shm-size=50G",
+		"--memory=50g",
+		"--memory-swap=50g",
+		// only supportd on some devices
+		// "--cpus=32",
+		// Expose a GPU with specific index:
+        "--device", "nvidia.com/gpu=0",
+	],
+
+	"onCreateCommand": "bash .devcontainer/post-create.sh",
+
+	"customizations": {
+		"vscode": {
+			"extensions": [
+				"ms-python.python",
+				"charliermarsh.ruff",
+				"tamasfe.even-better-toml",
+				"Anthropic.claude-code"
+			]
+		}
+	}
+}

.devcontainer/post-create.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# VS Code runs onCreateCommand as `sh -c bash post-create.sh`, a
+# non-login, non-interactive shell that never sources the conda init
+# hooks in ~/.bashrc. Without CONDA_PREFIX set, `benchopt install`
+# crashes outright (list_conda_envs() calls Path(None)). Activate
+# explicitly so this works regardless of how the script is invoked.
+source /opt/conda/etc/profile.d/conda.sh
+conda activate base
+
+# pip install benchopt
+pip install git+https://github.com/benchopt/benchopt.git
+
+# Install AI agent skills
+benchopt sync-skills
+
+# `benchopt install` can report freshly-installed packages as missing on its
+# first pass: it checks importability in the same long-lived process that
+# ran the installs, and Python's import machinery caches negative lookups
+# from before those installs landed on disk. A second (fresh-process)
+# invocation reliably sees everything that's actually on disk.
+benchopt install -y || benchopt install -y
+
+# Installing `aeon` (Monash dataset requirement) pulls in numpy>=2, which
+# is ABI-incompatible with this image's conda-shipped scipy/pandas/
+# matplotlib/pyarrow/numexpr/bottleneck (built against numpy 1.x). Realign
+# them with pip wheels built for whatever numpy version ends up installed.
+pip install --upgrade --force-reinstall --no-deps \
+    scipy pandas matplotlib pyarrow numexpr bottleneck
+
+pip install pre-commit
+pre-commit install

.gitignore

@@ -197,9 +197,9 @@ cython_debug/
 .abstra/
 
 # Visual Studio Code
-#   Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#   Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
 #   that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
-#   and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#   and can be added to the global gitignore or merged into this file. However, if you prefer,
 #   you could uncomment the following to ignore the entire vscode folder
 .vscode/
 # Temporary file for partial code execution
@@ -225,3 +225,8 @@ __marimo__/
 /data
 /physionet*
 /__cache__
+
+# AI Tools
+.claude/
+.agents/skills/benchopt-**
+.agents/skills/.benchopt-skills-manifest.json

.pre-commit-config.yaml

@@ -5,7 +5,6 @@ repos:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: check-yaml
-      - id: check-json
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.15.16

benchmark_utils/metrics.py

@@ -541,8 +541,12 @@ def map_iou(y_true, y_pred, iou_threshold=0.5):
     "map_iou": map_iou,
 }
 
-ALL_METRICS = {**FORECASTING_METRICS, **CLASSIFICATION_METRICS, **AD_METRICS}
-
+ALL_METRICS = {
+    **FORECASTING_METRICS,
+    **CLASSIFICATION_METRICS,
+    **AD_METRICS,
+    **EVENT_METRICS,
+}
 
 # ---------------------------------------------------------------------------
 # Direction: which metrics are better when larger
@@ -560,7 +564,10 @@ def map_iou(y_true, y_pred, iou_threshold=0.5):
         "f1_weighted",  # classification
         "auc_roc",
         "auc_pr",
-        "f1_pa",  # anomaly detection
+        "f1_pa",
+        "vus_roc",
+        "vus_pr",  # anomaly detection
+        "map_iou",  # event detection
     }
 )
 

datasets/ecg.py

@@ -63,7 +63,7 @@ class Dataset(BaseDataset):
 
     name = "ECG"
 
-    requirements = ["pip::pooch", "tqdm"]
+    requirements = ["pip::pooch", "pip::tqdm"]
 
     parameters = {
         "record_ids": [

datasets/enedis.py

@@ -101,7 +101,7 @@ class Dataset(BaseDataset):
 
     name = "Enedis"
 
-    requirements = ["pip::huggingface_hub", "fsspec"]
+    requirements = ["pip::huggingface_hub", "pip::fsspec"]
 
     parameters = {
         "freq": ["D", "6h", "30min"],

datasets/mitdb.py

@@ -189,7 +189,7 @@ class Dataset(BaseDataset):
 
     name = "MITDB"
 
-    requirements = ["wfdb"]
+    requirements = ["pip::wfdb>=4.3.1"]
 
     parameters = {
         "record_ids": ["all"],

datasets/monash.py

@@ -78,8 +78,7 @@ class Dataset(BaseDataset):
 
     name = "Monash"
 
-    # aeon is already a requirement of the objective
-    requirements = []
+    requirements = ["pip::aeon"]
 
     parameters = {
         "dataset_name": ["m1_yearly_dataset"],

objective.py

@@ -63,7 +63,7 @@ class Objective(BaseObjective):
 
     # Shared requirements across ALL solvers — solvers declare model-specific
     # extras in their own ``requirements`` list.
-    requirements = ["scikit-learn", "aeon"]
+    requirements = ["pip::scikit-learn"]
 
     sampling_strategy = "run_once"
 

slides/benchmark_design.tex

@@ -234,7 +234,7 @@ \section{Implementation}
 
     \begin{lstlisting}
 class Objective(BaseObjective):
-    requirements = ["scikit-learn", "aeon"]  # shared by all solvers
+    requirements = ["scikit-learn"]  # shared by all solvers
 
     def set_data(self, X_train, y_train, X_test, y_test,
                  task, metrics, **meta):

solvers/naive.py

@@ -92,7 +92,7 @@ class Solver(BaseSolver):
 
     name = "Naive"
 
-    # No extra requirements beyond the objective's scikit-learn + aeon.
+    # No extra requirements
     requirements = []
 
     parameters = {