diff --git a/docs/source/fl_ready_dataset.md b/docs/source/fl_ready_dataset.md index 2e347ea..32fb394 100644 --- a/docs/source/fl_ready_dataset.md +++ b/docs/source/fl_ready_dataset.md @@ -13,11 +13,11 @@ brainflex record fl_ready_dataset start ## Pipeline installation -Install the [Nipoppy-to-Fed-BioMed](https://zenodo.org/records/23753437) conversion pipeline. +Install the [Nipoppy-to-Fed-BioMed](https://zenodo.org/records/23988781) conversion pipeline. This pipeline was written for this FL experiment specifically. ```bash -nipoppy pipeline install 23753437 --study nipoppy_study +nipoppy pipeline install 23988781 --study nipoppy_study ``` The pipeline should now be listed when you run: diff --git a/pipelines/nipoppy2fedbiomed-0.1.0/descriptor.json b/pipelines/nipoppy2fedbiomed-0.1.0/descriptor.json index a55b8d9..0d0d162 100644 --- a/pipelines/nipoppy2fedbiomed-0.1.0/descriptor.json +++ b/pipelines/nipoppy2fedbiomed-0.1.0/descriptor.json @@ -3,7 +3,7 @@ "description": "Export tabular data from a Nipoppy study for a Fed-BioMed node.", "tool-version": "0.1.0", "schema-version": "0.5", - "command-line": "python [SCRIPT_PATH] [NIPOPPY_ROOT] [OUTPUT_DIR] [PHENOTYPE] [DERIVATIVE] [TAG] [NAME] [DESCRIPTION] [DATASET_PATH] [HELP]", + "command-line": "python [SCRIPT_PATH] [NIPOPPY_ROOT] [OUTPUT_DIR] [PHENOTYPE] [DERIVATIVE] [TAG] [NAME] [DESCRIPTION] [DATASET_PATH] [OUTLIER_THRESHOLD] [HELP]", "inputs": [ { "id": "script_path", @@ -85,6 +85,16 @@ "optional": true, "command-line-flag": "--dataset-path" }, + { + "id": "outlier_threshold", + "name": "outlier_threshold", + "type": "Number", + "value-key": "[OUTLIER_THRESHOLD]", + "description": "Remove rows where any IDP (derivative) value is below Q1 - K*IQR or above Q3 + K*IQR. Default: no outlier removal.", + "optional": true, + "minimum": 0, + "command-line-flag": "--outlier-threshold" + }, { "id": "help", "name": "help", diff --git a/pipelines/nipoppy2fedbiomed-0.1.0/invocation-age.json b/pipelines/nipoppy2fedbiomed-0.1.0/invocation-age.json index 39ce62c..be8eba3 100644 --- a/pipelines/nipoppy2fedbiomed-0.1.0/invocation-age.json +++ b/pipelines/nipoppy2fedbiomed-0.1.0/invocation-age.json @@ -12,5 +12,6 @@ "age" ], "name": "age", - "dataset_path": "$NIPOPPY_ROOT/derivatives/nipoppy2fedbiomed/0.1.0/output/age/dataset.csv" + "dataset_path": "$NIPOPPY_ROOT/derivatives/nipoppy2fedbiomed/0.1.0/output/age/dataset.csv", + "outlier_threshold": 3 } diff --git a/pipelines/nipoppy2fedbiomed-0.1.0/invocation-sex.json b/pipelines/nipoppy2fedbiomed-0.1.0/invocation-sex.json index 23c85fe..d91ed20 100644 --- a/pipelines/nipoppy2fedbiomed-0.1.0/invocation-sex.json +++ b/pipelines/nipoppy2fedbiomed-0.1.0/invocation-sex.json @@ -12,5 +12,6 @@ "sex" ], "name": "sex", - "dataset_path": "$NIPOPPY_ROOT/derivatives/nipoppy2fedbiomed/0.1.0/output/sex/dataset.csv" + "dataset_path": "$NIPOPPY_ROOT/derivatives/nipoppy2fedbiomed/0.1.0/output/sex/dataset.csv", + "outlier_threshold": 3 } diff --git a/pipelines/nipoppy2fedbiomed-0.1.0/nipoppy2fedbiomed.py b/pipelines/nipoppy2fedbiomed-0.1.0/nipoppy2fedbiomed.py index 65baafc..9deebb5 100755 --- a/pipelines/nipoppy2fedbiomed-0.1.0/nipoppy2fedbiomed.py +++ b/pipelines/nipoppy2fedbiomed-0.1.0/nipoppy2fedbiomed.py @@ -6,6 +6,10 @@ import json import sys from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import pandas as pd MIN_NIPOPPY_VERSION = "0.4.3" FNAME_CSV = "dataset.csv" @@ -90,9 +94,31 @@ def _get_parser() -> argparse.ArgumentParser: " Default: absolute path of the CSV file." ), ) + parser.add_argument( + "--outlier-threshold", + type=float, + metavar="K", + help=( + "Remove rows where any IDP (derivative) value is below Q1 - K*IQR" + " or above Q3 + K*IQR. Default: no outlier removal." + ), + ) return parser +def _get_outlier_mask( + df: "pd.DataFrame", columns: list[str], threshold: float +) -> "pd.Series": + """Return a boolean Series that is True for rows with any outlier value.""" + values = df[columns] + q1 = values.quantile(0.25) + q3 = values.quantile(0.75) + iqr = q3 - q1 + return ((values < q1 - threshold * iqr) | (values > q3 + threshold * iqr)).any( + axis="columns" + ) + + def nipoppy2fedbiomed( nipoppy_root: Path, output_dir: Path, @@ -102,6 +128,7 @@ def nipoppy2fedbiomed( name: str | None = None, description: str | None = None, dataset_path: str | None = None, + outlier_threshold: float | None = None, ) -> tuple[Path, Path]: """Write the CSV dataset and its Fed-BioMed description. @@ -109,6 +136,13 @@ def nipoppy2fedbiomed( """ if not (phenotypes or derivatives): raise ValueError("At least one phenotype or derivative must be specified") + if outlier_threshold is not None: + if outlier_threshold < 0: + raise ValueError( + f"Outlier threshold must be non-negative, got {outlier_threshold}" + ) + if not derivatives: + raise ValueError("Outlier removal requires at least one derivative") nipoppy_root = nipoppy_root.resolve() name = name or nipoppy_root.name if description is None: @@ -129,6 +163,17 @@ def nipoppy2fedbiomed( raise ValueError(f"No rows without missing values in {nipoppy_root}") if n_dropped := n_rows - len(df): print(f"Dropped {n_dropped} rows with missing values") + if outlier_threshold is not None: + idp_columns = [column for column in df.columns if column not in phenotypes] + n_rows = len(df) + df = df.loc[~_get_outlier_mask(df, idp_columns, outlier_threshold)] + if df.empty: + raise ValueError(f"No rows left after outlier removal in {nipoppy_root}") + if n_dropped := n_rows - len(df): + print( + f"Dropped {n_dropped} rows with outlier IDP values" + f" (threshold: {outlier_threshold})" + ) if TERMURL_SEX in df.columns: df[TERMURL_SEX] = df[TERMURL_SEX].astype(int) @@ -167,6 +212,7 @@ def main(argv: list[str] | None = None) -> None: name=args.name, description=args.description, dataset_path=args.dataset_path, + outlier_threshold=args.outlier_threshold, ) except (ValueError, FileNotFoundError) as e: sys.exit(f"Error: {e}") diff --git a/tests/pipelines/test_nipoppy2fedbiomed.py b/tests/pipelines/test_nipoppy2fedbiomed.py index 26da01e..1ddca64 100644 --- a/tests/pipelines/test_nipoppy2fedbiomed.py +++ b/tests/pipelines/test_nipoppy2fedbiomed.py @@ -117,6 +117,18 @@ def output_dir(tmp_path: Path) -> Path: return tmp_path / "output" +@pytest.fixture +def df() -> pd.DataFrame: + # Q1 = 2, Q3 = 4, IQR = 2 for each column + return pd.DataFrame( + { + "idp1": [1.0, 2.0, 3.0, 4.0, 5.0], + "idp2": [1.0, 2.0, 3.0, 4.0, 5.0], + "pheno": [1.0, 2.0, 3.0, 4.0, 5.0], + } + ) + + class TestPipelineBundle: def test_validate(self): result = subprocess.run( @@ -156,6 +168,31 @@ def test_invalid(self, module: ModuleType, value: str): module._parse_derivative(value) +class TestGetOutlierMask: + def test_no_outliers(self, module: ModuleType, df: pd.DataFrame): + # fences at -4 and 10 + mask = module._get_outlier_mask(df, ["idp1", "idp2"], 3) + assert mask.tolist() == [False] * 5 + + def test_fence_not_outlier(self, module: ModuleType, df: pd.DataFrame): + # fences at 1 and 5: values on the fence are kept + mask = module._get_outlier_mask(df, ["idp1", "idp2"], 0.5) + assert mask.tolist() == [False] * 5 + + @pytest.mark.parametrize("value", [-100.0, 100.0]) + def test_outlier_in_any_column( + self, module: ModuleType, df: pd.DataFrame, value: float + ): + df.loc[2, "idp2"] = value + mask = module._get_outlier_mask(df, ["idp1", "idp2"], 3) + assert mask.tolist() == [False, False, True, False, False] + + def test_ignores_other_columns(self, module: ModuleType, df: pd.DataFrame): + df.loc[2, "pheno"] = 100.0 + mask = module._get_outlier_mask(df, ["idp1", "idp2"], 3) + assert mask.tolist() == [False] * 5 + + class TestNipoppy2fedbiomed: def test_nipoppy2fedbiomed( self, module: ModuleType, nipoppy_root: Path, output_dir: Path @@ -279,6 +316,43 @@ def test_drops_other_sex( assert df["nb:Sex"].tolist() == [0, 0] assert "Dropped 1 rows with missing values" in capsys.readouterr().out + def test_outlier_removal( + self, + module: ModuleType, + nipoppy_root: Path, + output_dir: Path, + capsys: pytest.CaptureFixture, + ): + # with threshold 0, only values within [Q1, Q3] are kept + fpath_csv, _ = module.nipoppy2fedbiomed( + nipoppy_root, + output_dir, + ["nb:Age", "nb:Sex"], + DERIVATIVES, + ["tag"], + outlier_threshold=0, + ) + df = pd.read_csv(fpath_csv) + assert df["nb:Age"].tolist() == [30.0] + assert df["nb:Sex"].tolist() == [1] + assert "Dropped 2 rows with outlier IDP values" in capsys.readouterr().out + + def test_outlier_threshold_negative( + self, module: ModuleType, nipoppy_root: Path, output_dir: Path + ): + with pytest.raises(ValueError, match="must be non-negative"): + module.nipoppy2fedbiomed( + nipoppy_root, output_dir, [], DERIVATIVES, ["tag"], outlier_threshold=-1 + ) + + def test_outlier_threshold_without_derivatives( + self, module: ModuleType, nipoppy_root: Path, output_dir: Path + ): + with pytest.raises(ValueError, match="requires at least one derivative"): + module.nipoppy2fedbiomed( + nipoppy_root, output_dir, ["nb:Age"], [], ["tag"], outlier_threshold=3 + ) + def test_all_missing( self, module: ModuleType, nipoppy_root: Path, output_dir: Path ): @@ -322,6 +396,7 @@ def test_main( "my-dataset", "--dataset-path", "$NIPOPPY_ROOT/output/dataset.csv", + *("--outlier-threshold", "3"), ] ) assert "Wrote 3 rows" in capsys.readouterr().out