[ENH] Remove outliers in FL-ready data

docs/source/fl_ready_dataset.md

@@ -13,11 +13,11 @@ brainflex record fl_ready_dataset start
 
 ## Pipeline installation
 
-Install the [Nipoppy-to-Fed-BioMed](https://zenodo.org/records/23753437) conversion pipeline.
+Install the [Nipoppy-to-Fed-BioMed](https://zenodo.org/records/23988781) conversion pipeline.
 This pipeline was written for this FL experiment specifically.
 
 ```bash
-nipoppy pipeline install 23753437 --study nipoppy_study
+nipoppy pipeline install 23988781 --study nipoppy_study
 ```
 
 The pipeline should now be listed when you run:

pipelines/nipoppy2fedbiomed-0.1.0/descriptor.json

@@ -3,7 +3,7 @@
     "description": "Export tabular data from a Nipoppy study for a Fed-BioMed node.",
     "tool-version": "0.1.0",
     "schema-version": "0.5",
-    "command-line": "python [SCRIPT_PATH] [NIPOPPY_ROOT] [OUTPUT_DIR] [PHENOTYPE] [DERIVATIVE] [TAG] [NAME] [DESCRIPTION] [DATASET_PATH] [HELP]",
+    "command-line": "python [SCRIPT_PATH] [NIPOPPY_ROOT] [OUTPUT_DIR] [PHENOTYPE] [DERIVATIVE] [TAG] [NAME] [DESCRIPTION] [DATASET_PATH] [OUTLIER_THRESHOLD] [HELP]",
     "inputs": [
         {
             "id": "script_path",
@@ -85,6 +85,16 @@
             "optional": true,
             "command-line-flag": "--dataset-path"
         },
+        {
+            "id": "outlier_threshold",
+            "name": "outlier_threshold",
+            "type": "Number",
+            "value-key": "[OUTLIER_THRESHOLD]",
+            "description": "Remove rows where any IDP (derivative) value is below Q1 - K*IQR or above Q3 + K*IQR. Default: no outlier removal.",
+            "optional": true,
+            "minimum": 0,
+            "command-line-flag": "--outlier-threshold"
+        },
         {
             "id": "help",
             "name": "help",

pipelines/nipoppy2fedbiomed-0.1.0/invocation-age.json

@@ -12,5 +12,6 @@
         "age"
     ],
     "name": "age",
-    "dataset_path": "$NIPOPPY_ROOT/derivatives/nipoppy2fedbiomed/0.1.0/output/age/dataset.csv"
+    "dataset_path": "$NIPOPPY_ROOT/derivatives/nipoppy2fedbiomed/0.1.0/output/age/dataset.csv",
+    "outlier_threshold": 3
 }

pipelines/nipoppy2fedbiomed-0.1.0/invocation-sex.json

@@ -12,5 +12,6 @@
         "sex"
     ],
     "name": "sex",
-    "dataset_path": "$NIPOPPY_ROOT/derivatives/nipoppy2fedbiomed/0.1.0/output/sex/dataset.csv"
+    "dataset_path": "$NIPOPPY_ROOT/derivatives/nipoppy2fedbiomed/0.1.0/output/sex/dataset.csv",
+    "outlier_threshold": 3
 }

pipelines/nipoppy2fedbiomed-0.1.0/nipoppy2fedbiomed.py

@@ -6,6 +6,10 @@
 import json
 import sys
 from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import pandas as pd
 
 MIN_NIPOPPY_VERSION = "0.4.3"
 FNAME_CSV = "dataset.csv"
@@ -90,9 +94,31 @@ def _get_parser() -> argparse.ArgumentParser:
             " Default: absolute path of the CSV file."
         ),
     )
+    parser.add_argument(
+        "--outlier-threshold",
+        type=float,
+        metavar="K",
+        help=(
+            "Remove rows where any IDP (derivative) value is below Q1 - K*IQR"
+            " or above Q3 + K*IQR. Default: no outlier removal."
+        ),
+    )
     return parser
 
 
+def _get_outlier_mask(
+    df: "pd.DataFrame", columns: list[str], threshold: float
+) -> "pd.Series":
+    """Return a boolean Series that is True for rows with any outlier value."""
+    values = df[columns]
+    q1 = values.quantile(0.25)
+    q3 = values.quantile(0.75)
+    iqr = q3 - q1
+    return ((values < q1 - threshold * iqr) | (values > q3 + threshold * iqr)).any(
+        axis="columns"
+    )
+
+
 def nipoppy2fedbiomed(
     nipoppy_root: Path,
     output_dir: Path,
@@ -102,13 +128,21 @@ def nipoppy2fedbiomed(
     name: str | None = None,
     description: str | None = None,
     dataset_path: str | None = None,
+    outlier_threshold: float | None = None,
 ) -> tuple[Path, Path]:
     """Write the CSV dataset and its Fed-BioMed description.
 
     Returns the paths to the CSV and JSON files.
     """
     if not (phenotypes or derivatives):
         raise ValueError("At least one phenotype or derivative must be specified")
+    if outlier_threshold is not None:
+        if outlier_threshold < 0:
+            raise ValueError(
+                f"Outlier threshold must be non-negative, got {outlier_threshold}"
+            )
+        if not derivatives:
+            raise ValueError("Outlier removal requires at least one derivative")
     nipoppy_root = nipoppy_root.resolve()
     name = name or nipoppy_root.name
     if description is None:
@@ -129,6 +163,17 @@ def nipoppy2fedbiomed(
         raise ValueError(f"No rows without missing values in {nipoppy_root}")
     if n_dropped := n_rows - len(df):
         print(f"Dropped {n_dropped} rows with missing values")
+    if outlier_threshold is not None:
+        idp_columns = [column for column in df.columns if column not in phenotypes]
+        n_rows = len(df)
+        df = df.loc[~_get_outlier_mask(df, idp_columns, outlier_threshold)]
+        if df.empty:
+            raise ValueError(f"No rows left after outlier removal in {nipoppy_root}")
+        if n_dropped := n_rows - len(df):
+            print(
+                f"Dropped {n_dropped} rows with outlier IDP values"
+                f" (threshold: {outlier_threshold})"
+            )
     if TERMURL_SEX in df.columns:
         df[TERMURL_SEX] = df[TERMURL_SEX].astype(int)
 
@@ -167,6 +212,7 @@ def main(argv: list[str] | None = None) -> None:
             name=args.name,
             description=args.description,
             dataset_path=args.dataset_path,
+            outlier_threshold=args.outlier_threshold,
         )
     except (ValueError, FileNotFoundError) as e:
         sys.exit(f"Error: {e}")

tests/pipelines/test_nipoppy2fedbiomed.py

@@ -117,6 +117,18 @@ def output_dir(tmp_path: Path) -> Path:
     return tmp_path / "output"
 
 
+@pytest.fixture
+def df() -> pd.DataFrame:
+    # Q1 = 2, Q3 = 4, IQR = 2 for each column
+    return pd.DataFrame(
+        {
+            "idp1": [1.0, 2.0, 3.0, 4.0, 5.0],
+            "idp2": [1.0, 2.0, 3.0, 4.0, 5.0],
+            "pheno": [1.0, 2.0, 3.0, 4.0, 5.0],
+        }
+    )
+
+
 class TestPipelineBundle:
     def test_validate(self):
         result = subprocess.run(
@@ -156,6 +168,31 @@ def test_invalid(self, module: ModuleType, value: str):
             module._parse_derivative(value)
 
 
+class TestGetOutlierMask:
+    def test_no_outliers(self, module: ModuleType, df: pd.DataFrame):
+        # fences at -4 and 10
+        mask = module._get_outlier_mask(df, ["idp1", "idp2"], 3)
+        assert mask.tolist() == [False] * 5
+
+    def test_fence_not_outlier(self, module: ModuleType, df: pd.DataFrame):
+        # fences at 1 and 5: values on the fence are kept
+        mask = module._get_outlier_mask(df, ["idp1", "idp2"], 0.5)
+        assert mask.tolist() == [False] * 5
+
+    @pytest.mark.parametrize("value", [-100.0, 100.0])
+    def test_outlier_in_any_column(
+        self, module: ModuleType, df: pd.DataFrame, value: float
+    ):
+        df.loc[2, "idp2"] = value
+        mask = module._get_outlier_mask(df, ["idp1", "idp2"], 3)
+        assert mask.tolist() == [False, False, True, False, False]
+
+    def test_ignores_other_columns(self, module: ModuleType, df: pd.DataFrame):
+        df.loc[2, "pheno"] = 100.0
+        mask = module._get_outlier_mask(df, ["idp1", "idp2"], 3)
+        assert mask.tolist() == [False] * 5
+
+
 class TestNipoppy2fedbiomed:
     def test_nipoppy2fedbiomed(
         self, module: ModuleType, nipoppy_root: Path, output_dir: Path
@@ -279,6 +316,43 @@ def test_drops_other_sex(
         assert df["nb:Sex"].tolist() == [0, 0]
         assert "Dropped 1 rows with missing values" in capsys.readouterr().out
 
+    def test_outlier_removal(
+        self,
+        module: ModuleType,
+        nipoppy_root: Path,
+        output_dir: Path,
+        capsys: pytest.CaptureFixture,
+    ):
+        # with threshold 0, only values within [Q1, Q3] are kept
+        fpath_csv, _ = module.nipoppy2fedbiomed(
+            nipoppy_root,
+            output_dir,
+            ["nb:Age", "nb:Sex"],
+            DERIVATIVES,
+            ["tag"],
+            outlier_threshold=0,
+        )
+        df = pd.read_csv(fpath_csv)
+        assert df["nb:Age"].tolist() == [30.0]
+        assert df["nb:Sex"].tolist() == [1]
+        assert "Dropped 2 rows with outlier IDP values" in capsys.readouterr().out
+
+    def test_outlier_threshold_negative(
+        self, module: ModuleType, nipoppy_root: Path, output_dir: Path
+    ):
+        with pytest.raises(ValueError, match="must be non-negative"):
+            module.nipoppy2fedbiomed(
+                nipoppy_root, output_dir, [], DERIVATIVES, ["tag"], outlier_threshold=-1
+            )
+
+    def test_outlier_threshold_without_derivatives(
+        self, module: ModuleType, nipoppy_root: Path, output_dir: Path
+    ):
+        with pytest.raises(ValueError, match="requires at least one derivative"):
+            module.nipoppy2fedbiomed(
+                nipoppy_root, output_dir, ["nb:Age"], [], ["tag"], outlier_threshold=3
+            )
+
     def test_all_missing(
         self, module: ModuleType, nipoppy_root: Path, output_dir: Path
     ):
@@ -322,6 +396,7 @@ def test_main(
                 "my-dataset",
                 "--dataset-path",
                 "$NIPOPPY_ROOT/output/dataset.csv",
+                *("--outlier-threshold", "3"),
             ]
         )
         assert "Wrote 3 rows" in capsys.readouterr().out