diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..52af0c1 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,30 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/python +{ + "name": "Benchmark TSFM", + "image": "mcr.microsoft.com/devcontainers/anaconda:3", + "remoteUser": "root", + + "runArgs": [ + "--shm-size=50G", + "--memory=50g", + "--memory-swap=50g", + // only supportd on some devices + // "--cpus=32", + // Expose a GPU with specific index: + "--device", "nvidia.com/gpu=0", + ], + + "onCreateCommand": "bash .devcontainer/post-create.sh", + + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python", + "charliermarsh.ruff", + "tamasfe.even-better-toml", + "Anthropic.claude-code" + ] + } + } +} diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh new file mode 100755 index 0000000..f41f02d --- /dev/null +++ b/.devcontainer/post-create.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -euo pipefail + +# VS Code runs onCreateCommand as `sh -c bash post-create.sh`, a +# non-login, non-interactive shell that never sources the conda init +# hooks in ~/.bashrc. Without CONDA_PREFIX set, `benchopt install` +# crashes outright (list_conda_envs() calls Path(None)). Activate +# explicitly so this works regardless of how the script is invoked. +source /opt/conda/etc/profile.d/conda.sh +conda activate base + +# pip install benchopt +pip install git+https://github.com/benchopt/benchopt.git + +# Install AI agent skills +benchopt sync-skills + +# `benchopt install` can report freshly-installed packages as missing on its +# first pass: it checks importability in the same long-lived process that +# ran the installs, and Python's import machinery caches negative lookups +# from before those installs landed on disk. A second (fresh-process) +# invocation reliably sees everything that's actually on disk. +benchopt install -y || benchopt install -y + +# Installing `aeon` (Monash dataset requirement) pulls in numpy>=2, which +# is ABI-incompatible with this image's conda-shipped scipy/pandas/ +# matplotlib/pyarrow/numexpr/bottleneck (built against numpy 1.x). Realign +# them with pip wheels built for whatever numpy version ends up installed. +pip install --upgrade --force-reinstall --no-deps \ + scipy pandas matplotlib pyarrow numexpr bottleneck + +pip install pre-commit +pre-commit install diff --git a/.gitignore b/.gitignore index bee39e8..ddd4679 100644 --- a/.gitignore +++ b/.gitignore @@ -197,9 +197,9 @@ cython_debug/ .abstra/ # Visual Studio Code -# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore +# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore # that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore -# and can be added to the global gitignore or merged into this file. However, if you prefer, +# and can be added to the global gitignore or merged into this file. However, if you prefer, # you could uncomment the following to ignore the entire vscode folder .vscode/ # Temporary file for partial code execution @@ -225,3 +225,8 @@ __marimo__/ /data /physionet* /__cache__ + +# AI Tools +.claude/ +.agents/skills/benchopt-** +.agents/skills/.benchopt-skills-manifest.json diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0c2c8ad..81c1ad4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,6 @@ repos: - id: trailing-whitespace - id: end-of-file-fixer - id: check-yaml - - id: check-json - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.15.16 diff --git a/benchmark_utils/metrics.py b/benchmark_utils/metrics.py index 4e5b8cd..7b8e9bd 100644 --- a/benchmark_utils/metrics.py +++ b/benchmark_utils/metrics.py @@ -541,8 +541,12 @@ def map_iou(y_true, y_pred, iou_threshold=0.5): "map_iou": map_iou, } -ALL_METRICS = {**FORECASTING_METRICS, **CLASSIFICATION_METRICS, **AD_METRICS} - +ALL_METRICS = { + **FORECASTING_METRICS, + **CLASSIFICATION_METRICS, + **AD_METRICS, + **EVENT_METRICS, +} # --------------------------------------------------------------------------- # Direction: which metrics are better when larger @@ -560,7 +564,10 @@ def map_iou(y_true, y_pred, iou_threshold=0.5): "f1_weighted", # classification "auc_roc", "auc_pr", - "f1_pa", # anomaly detection + "f1_pa", + "vus_roc", + "vus_pr", # anomaly detection + "map_iou", # event detection } ) diff --git a/datasets/ecg.py b/datasets/ecg.py index 42560e3..5fa6ea0 100644 --- a/datasets/ecg.py +++ b/datasets/ecg.py @@ -63,7 +63,7 @@ class Dataset(BaseDataset): name = "ECG" - requirements = ["pip::pooch", "tqdm"] + requirements = ["pip::pooch", "pip::tqdm"] parameters = { "record_ids": [ diff --git a/datasets/enedis.py b/datasets/enedis.py index 65695b3..0aa84da 100644 --- a/datasets/enedis.py +++ b/datasets/enedis.py @@ -101,7 +101,7 @@ class Dataset(BaseDataset): name = "Enedis" - requirements = ["pip::huggingface_hub", "fsspec"] + requirements = ["pip::huggingface_hub", "pip::fsspec"] parameters = { "freq": ["D", "6h", "30min"], diff --git a/datasets/mitdb.py b/datasets/mitdb.py index fe00fb9..63d80cb 100644 --- a/datasets/mitdb.py +++ b/datasets/mitdb.py @@ -189,7 +189,7 @@ class Dataset(BaseDataset): name = "MITDB" - requirements = ["wfdb"] + requirements = ["pip::wfdb>=4.3.1"] parameters = { "record_ids": ["all"], diff --git a/datasets/monash.py b/datasets/monash.py index a607f64..73db478 100644 --- a/datasets/monash.py +++ b/datasets/monash.py @@ -78,8 +78,7 @@ class Dataset(BaseDataset): name = "Monash" - # aeon is already a requirement of the objective - requirements = [] + requirements = ["pip::aeon"] parameters = { "dataset_name": ["m1_yearly_dataset"], diff --git a/objective.py b/objective.py index 1f335e7..a85fb1a 100644 --- a/objective.py +++ b/objective.py @@ -63,7 +63,7 @@ class Objective(BaseObjective): # Shared requirements across ALL solvers — solvers declare model-specific # extras in their own ``requirements`` list. - requirements = ["scikit-learn", "aeon"] + requirements = ["pip::scikit-learn"] sampling_strategy = "run_once" diff --git a/slides/benchmark_design.tex b/slides/benchmark_design.tex index e5ae9a7..97997f2 100644 --- a/slides/benchmark_design.tex +++ b/slides/benchmark_design.tex @@ -234,7 +234,7 @@ \section{Implementation} \begin{lstlisting} class Objective(BaseObjective): - requirements = ["scikit-learn", "aeon"] # shared by all solvers + requirements = ["scikit-learn"] # shared by all solvers def set_data(self, X_train, y_train, X_test, y_test, task, metrics, **meta): diff --git a/solvers/naive.py b/solvers/naive.py index b7be080..fd0fbf0 100644 --- a/solvers/naive.py +++ b/solvers/naive.py @@ -92,7 +92,7 @@ class Solver(BaseSolver): name = "Naive" - # No extra requirements beyond the objective's scikit-learn + aeon. + # No extra requirements requirements = [] parameters = {