[ENH] Allow selection of outlier columns

docs/source/fl_ready_dataset.md

@@ -13,11 +13,11 @@ brainflex record fl_ready_dataset start
 
 ## Pipeline installation
 
-Install the [Nipoppy-to-Fed-BioMed](https://zenodo.org/records/23988781) conversion pipeline.
+Install the [Nipoppy-to-Fed-BioMed](https://zenodo.org/records/23992846) conversion pipeline.
 This pipeline was written for this FL experiment specifically.
 
 ```bash
-nipoppy pipeline install 23988781 --study nipoppy_study
+nipoppy pipeline install 23992846 --study nipoppy_study
 ```
 
 The pipeline should now be listed when you run:

pipelines/nipoppy2fedbiomed-0.1.0/descriptor.json

@@ -3,7 +3,7 @@
     "description": "Export tabular data from a Nipoppy study for a Fed-BioMed node.",
     "tool-version": "0.1.0",
     "schema-version": "0.5",
-    "command-line": "python [SCRIPT_PATH] [NIPOPPY_ROOT] [OUTPUT_DIR] [PHENOTYPE] [DERIVATIVE] [TAG] [NAME] [DESCRIPTION] [DATASET_PATH] [OUTLIER_THRESHOLD] [HELP]",
+    "command-line": "python [SCRIPT_PATH] [NIPOPPY_ROOT] [OUTPUT_DIR] [PHENOTYPE] [DERIVATIVE] [TAG] [NAME] [DESCRIPTION] [DATASET_PATH] [OUTLIER_THRESHOLD] [OUTLIER_COLUMNS] [OUTLIER_EXCLUDE_COLUMNS] [HELP]",
     "inputs": [
         {
             "id": "script_path",
@@ -90,11 +90,33 @@
             "name": "outlier_threshold",
             "type": "Number",
             "value-key": "[OUTLIER_THRESHOLD]",
-            "description": "Remove rows where any IDP (derivative) value is below Q1 - K*IQR or above Q3 + K*IQR. Default: no outlier removal.",
+            "description": "Remove rows where any IDP (derivative) value is below Q1 - K*IQR or above Q3 + K*IQR. Columns with zero IQR are ignored. Default: no outlier removal.",
             "optional": true,
             "minimum": 0,
             "command-line-flag": "--outlier-threshold"
         },
+        {
+            "id": "outlier_columns",
+            "name": "outlier_columns",
+            "type": "String",
+            "value-key": "[OUTLIER_COLUMNS]",
+            "description": "fnmatch pattern of IDP columns to check for outliers. Can be repeated. Default: all IDP columns.",
+            "optional": true,
+            "list": true,
+            "list-separator": " --outlier-column ",
+            "command-line-flag": "--outlier-column"
+        },
+        {
+            "id": "outlier_exclude_columns",
+            "name": "outlier_exclude_columns",
+            "type": "String",
+            "value-key": "[OUTLIER_EXCLUDE_COLUMNS]",
+            "description": "fnmatch pattern of IDP columns to never check for outliers, even if they match outlier_columns. Can be repeated.",
+            "optional": true,
+            "list": true,
+            "list-separator": " --outlier-exclude-column ",
+            "command-line-flag": "--outlier-exclude-column"
+        },
         {
             "id": "help",
             "name": "help",

pipelines/nipoppy2fedbiomed-0.1.0/invocation-age.json

@@ -13,5 +13,30 @@
     ],
     "name": "age",
     "dataset_path": "$NIPOPPY_ROOT/derivatives/nipoppy2fedbiomed/0.1.0/output/age/dataset.csv",
-    "outlier_threshold": 3
+    "outlier_threshold": 3,
+    "outlier_columns": [
+        "*_thickness",
+        "EstimatedTotalIntraCranialVol",
+        "Left-Lateral-Ventricle",
+        "Left-Thalamus",
+        "Left-Caudate",
+        "Left-Putamen",
+        "Left-Pallidum",
+        "Left-Hippocampus",
+        "Left-Amygdala",
+        "Left-Accumbens-area",
+        "Right-Lateral-Ventricle",
+        "Right-Thalamus",
+        "Right-Caudate",
+        "Right-Putamen",
+        "Right-Pallidum",
+        "Right-Hippocampus",
+        "Right-Amygdala",
+        "Right-Accumbens-area"
+    ],
+    "outlier_exclude_columns": [
+        "lh_MeanThickness_thickness",
+        "rh_MeanThickness_thickness",
+        "rh_temporalpole_thickness"
+    ]
 }

pipelines/nipoppy2fedbiomed-0.1.0/invocation-sex.json

@@ -13,5 +13,30 @@
     ],
     "name": "sex",
     "dataset_path": "$NIPOPPY_ROOT/derivatives/nipoppy2fedbiomed/0.1.0/output/sex/dataset.csv",
-    "outlier_threshold": 3
+    "outlier_threshold": 3,
+    "outlier_columns": [
+        "*_thickness",
+        "EstimatedTotalIntraCranialVol",
+        "Left-Lateral-Ventricle",
+        "Left-Thalamus",
+        "Left-Caudate",
+        "Left-Putamen",
+        "Left-Pallidum",
+        "Left-Hippocampus",
+        "Left-Amygdala",
+        "Left-Accumbens-area",
+        "Right-Lateral-Ventricle",
+        "Right-Thalamus",
+        "Right-Caudate",
+        "Right-Putamen",
+        "Right-Pallidum",
+        "Right-Hippocampus",
+        "Right-Amygdala",
+        "Right-Accumbens-area"
+    ],
+    "outlier_exclude_columns": [
+        "lh_MeanThickness_thickness",
+        "rh_MeanThickness_thickness",
+        "rh_temporalpole_thickness"
+    ]
 }

pipelines/nipoppy2fedbiomed-0.1.0/nipoppy2fedbiomed.py

@@ -3,6 +3,7 @@
 """Export tabular data from a Nipoppy study for a Fed-BioMed node."""
 
 import argparse
+import fnmatch
 import json
 import sys
 from pathlib import Path
@@ -100,7 +101,28 @@ def _get_parser() -> argparse.ArgumentParser:
         metavar="K",
         help=(
             "Remove rows where any IDP (derivative) value is below Q1 - K*IQR"
-            " or above Q3 + K*IQR. Default: no outlier removal."
+            " or above Q3 + K*IQR. Columns with zero IQR are ignored."
+            " Default: no outlier removal."
+        ),
+    )
+    parser.add_argument(
+        "--outlier-column",
+        dest="outlier_columns",
+        action="append",
+        metavar="PATTERN",
+        help=(
+            "fnmatch pattern of IDP columns to check for outliers. Can be repeated."
+            " Default: all IDP columns."
+        ),
+    )
+    parser.add_argument(
+        "--outlier-exclude-column",
+        dest="outlier_exclude_columns",
+        action="append",
+        metavar="PATTERN",
+        help=(
+            "fnmatch pattern of IDP columns to never check for outliers,"
+            " even if they match --outlier-column. Can be repeated."
         ),
     )
     return parser
@@ -109,16 +131,44 @@ def _get_parser() -> argparse.ArgumentParser:
 def _get_outlier_mask(
     df: "pd.DataFrame", columns: list[str], threshold: float
 ) -> "pd.Series":
-    """Return a boolean Series that is True for rows with any outlier value."""
+    """Return a boolean Series that is True for rows with any outlier value.
+
+    Columns with zero IQR are ignored.
+    """
     values = df[columns]
     q1 = values.quantile(0.25)
     q3 = values.quantile(0.75)
     iqr = q3 - q1
+    nonzero = iqr != 0
+    values, q1, q3, iqr = values.loc[:, nonzero], q1[nonzero], q3[nonzero], iqr[nonzero]
     return ((values < q1 - threshold * iqr) | (values > q3 + threshold * iqr)).any(
         axis="columns"
     )
 
 
+def _match_columns(
+    columns: list[str], patterns: list[str] | None, exclude_patterns: list[str] | None
+) -> list[str]:
+    """Return the columns matching any pattern (all if None) and no exclude pattern."""
+
+    def _matches(column: str, patterns: list[str]) -> bool:
+        return any(fnmatch.fnmatchcase(column, pattern) for pattern in patterns)
+
+    return [
+        column
+        for column in columns
+        if (patterns is None or _matches(column, patterns))
+        and not _matches(column, exclude_patterns or [])
+    ]
+
+
+def _get_zero_iqr_columns(df: "pd.DataFrame", columns: list[str]) -> list[str]:
+    """Return the columns whose IQR is zero."""
+    values = df[columns]
+    iqr = values.quantile(0.75) - values.quantile(0.25)
+    return [column for column in columns if iqr[column] == 0]
+
+
 def nipoppy2fedbiomed(
     nipoppy_root: Path,
     output_dir: Path,
@@ -129,6 +179,8 @@ def nipoppy2fedbiomed(
     description: str | None = None,
     dataset_path: str | None = None,
     outlier_threshold: float | None = None,
+    outlier_columns: list[str] | None = None,
+    outlier_exclude_columns: list[str] | None = None,
 ) -> tuple[Path, Path]:
     """Write the CSV dataset and its Fed-BioMed description.
 
@@ -143,6 +195,8 @@ def nipoppy2fedbiomed(
             )
         if not derivatives:
             raise ValueError("Outlier removal requires at least one derivative")
+    elif outlier_columns or outlier_exclude_columns:
+        raise ValueError("Outlier columns require an outlier threshold")
     nipoppy_root = nipoppy_root.resolve()
     name = name or nipoppy_root.name
     if description is None:
@@ -164,7 +218,21 @@ def nipoppy2fedbiomed(
     if n_dropped := n_rows - len(df):
         print(f"Dropped {n_dropped} rows with missing values")
     if outlier_threshold is not None:
-        idp_columns = [column for column in df.columns if column not in phenotypes]
+        idp_columns = _match_columns(
+            [column for column in df.columns if column not in phenotypes],
+            outlier_columns,
+            outlier_exclude_columns,
+        )
+        if not idp_columns:
+            raise ValueError(
+                "No IDP columns match the outlier column patterns"
+                f" {outlier_columns} (excluding {outlier_exclude_columns})"
+            )
+        if zero_iqr_columns := _get_zero_iqr_columns(df, idp_columns):
+            print(
+                f"Skipped {len(zero_iqr_columns)} columns with zero IQR in outlier"
+                f" removal: {zero_iqr_columns}"
+            )
         n_rows = len(df)
         df = df.loc[~_get_outlier_mask(df, idp_columns, outlier_threshold)]
         if df.empty:
@@ -213,6 +281,8 @@ def main(argv: list[str] | None = None) -> None:
             description=args.description,
             dataset_path=args.dataset_path,
             outlier_threshold=args.outlier_threshold,
+            outlier_columns=args.outlier_columns,
+            outlier_exclude_columns=args.outlier_exclude_columns,
         )
     except (ValueError, FileNotFoundError) as e:
         sys.exit(f"Error: {e}")