diff --git a/bids_inventory/_plotting.py b/bids_inventory/_plotting.py index 8122cfc..f6234f2 100644 --- a/bids_inventory/_plotting.py +++ b/bids_inventory/_plotting.py @@ -10,6 +10,58 @@ from plotly.subplots import make_subplots from bids_inventory.listing import KNOWN_DATATYPES +from bids_inventory.wrangling import SOURCES + +# same color for a source in all the figures, whatever the sources shown +SOURCE_COLORS = dict( + zip( + [source for _, source in SOURCES] + ["other"], + px.colors.qualitative.Plotly + px.colors.qualitative.Dark24, + ) +) + + +def _stacked_by_source( + rows: pd.DataFrame, x: str, order: list[Any], title: str, x_title: str +) -> go.Figure: + """Plot the number of datasets for each value of ``x``, stacked by source of the datasets. + + ``rows``: one row per dataset and value of ``x``, with a ``source`` column + (and ``nb_subjects``: the number of participants is shown on hover). + ``order``: the values of ``x`` to show, in this order. + """ + counts = rows.groupby([x, "source"]).size() + participants = ( + rows.groupby([x, "source"])["nb_subjects"].sum() if "nb_subjects" in rows else None + ) + fig = go.Figure() + for source, color in SOURCE_COLORS.items(): + if source not in set(rows["source"]): + continue + y = [int(counts.get((value, source), 0)) for value in order] + hover = f"%{{x}}
{source}: %{{y}} datasets" + customdata = None + if participants is not None: + customdata = [int(participants.get((value, source), 0)) for value in order] + hover += "
%{customdata} participants" + fig.add_trace( + go.Bar( + x=order, + y=y, + name=source, + marker_color=color, + customdata=customdata, + hovertemplate=f"{hover}", + ) + ) + return fig.update_layout( + barmode="stack", + title=title, + legend_title_text="source", + xaxis={"type": "category", "title": {"text": x_title}}, + yaxis={"title": {"text": "number of datasets"}}, + ) + LABELS = { "nb_subjects": "number of participants", @@ -48,17 +100,11 @@ def scatter_subject_vs( def histogram_tasks(df: pd.DataFrame) -> go.Figure: - tasks = [] - for x in df.tasks: - tasks.extend(x) - - new_df = pd.DataFrame({"tasks": tasks}) - - count = new_df.tasks.value_counts() - order = [i[0] for i in count.items()] - return px.histogram( - new_df, x="tasks", category_orders=dict(tasks=order), title="tasks distribution" - ) + """Plot the number of datasets for each task (the most common first), by source.""" + rows = df[["name", "source", "tasks"]].explode("tasks").dropna(subset=["tasks"]) + rows = rows.drop_duplicates(subset=["name", "tasks"]) + order = list(rows["tasks"].value_counts().index) + return _stacked_by_source(rows, "tasks", order, "tasks distribution", "task") def plot_dataset_size_vs_time(df: pd.DataFrame) -> go.Figure: @@ -170,15 +216,11 @@ def plot_hours_vs_time(df: pd.DataFrame, log_y: bool = True) -> go.Figure: def datatypes_histogram(df: pd.DataFrame) -> go.Figure: - datatypes_df = df[KNOWN_DATATYPES].sum() - fig = px.bar( - datatypes_df, - labels={"index": "datatype", "value": "count"}, - hover_name=None, - hover_data=None, - title="datatypes in datasets", - ) - return fig + """Plot the number of datasets with each BIDS datatype, by source.""" + rows = df[["name", "source", "nb_subjects", "datatypes"]].explode("datatypes") + rows = rows[rows["datatypes"].isin(KNOWN_DATATYPES)] + order = [x for x in KNOWN_DATATYPES if x in set(rows["datatypes"])] + return _stacked_by_source(rows, "datatypes", order, "datatypes in datasets", "datatype") def field_strength_histogram(df: pd.DataFrame) -> go.Figure: @@ -188,27 +230,21 @@ def field_strength_histogram(df: pd.DataFrame) -> go.Figure: A dataset with data at several field strengths is counted for each one. """ rows = ( - df[["name", "nb_subjects", "field_strengths"]] + df[["name", "source", "nb_subjects", "field_strengths"]] .explode("field_strengths") .dropna(subset=["field_strengths"]) ) - counts = rows.groupby("field_strengths").agg( - datasets=("name", "nunique"), participants=("nb_subjects", "sum") - ) def order(label: str) -> float: return float(label.removesuffix("T")) if label.endswith("T") else float("inf") - counts = counts.loc[sorted(counts.index, key=order)].reset_index() - return px.bar( - counts, - x="field_strengths", - y="datasets", - hover_data=["participants"], - text="datasets", - labels={"field_strengths": "MRI field strength", "datasets": "number of datasets"}, - title="MRI field strength of the datasets", - ).update_xaxes(type="category") + return _stacked_by_source( + rows, + "field_strengths", + sorted(set(rows["field_strengths"]), key=order), + "MRI field strength of the datasets", + "MRI field strength", + ) def institutions_map(locations: pd.DataFrame) -> go.Figure: @@ -216,8 +252,11 @@ def institutions_map(locations: pd.DataFrame) -> go.Figure: ``locations``: as returned by ``bids_inventory.institutions.institutions_locations``. The size of each marker is the number of datasets of the institution. + With a ``source`` column (one row per institution and source of its datasets), + the markers are colored by source. """ located = locations.dropna(subset=["latitude", "longitude"]) + by_source = "source" in located return px.scatter_geo( located, lat="latitude", @@ -226,6 +265,9 @@ def institutions_map(locations: pd.DataFrame) -> go.Figure: hover_data=["address", "count", "nb_subjects"], projection="natural earth", size="count", + color="source" if by_source else None, + color_discrete_map=SOURCE_COLORS if by_source else None, + category_orders={"source": list(SOURCE_COLORS)} if by_source else None, title="institutions", ) @@ -388,21 +430,16 @@ def version_histogram(df: pd.DataFrame, column: str, name: str) -> go.Figure: ``column``: column of the listing with the versions (``bids_version``...). """ - versions = df[column].replace("n/a", None).dropna().astype(str) - counts = versions.value_counts() - counts = counts.loc[sorted(counts.index, key=_version_key)] - # go.Bar and not px.bar: there may be no versions at all - fig = go.Figure( - go.Bar( - x=list(counts.index), - y=list(counts.values), - hovertemplate=f"{name} %{{x}}
%{{y}} datasets", - ) - ) - return fig.update_layout( - title=f"{name} version ({len(versions)} datasets)", - xaxis={"type": "category", "title": {"text": f"{name} version"}}, - yaxis={"title": {"text": "number of datasets"}}, + rows = df[["name", "source", column]].copy() + rows[column] = rows[column].replace("n/a", None) + rows = rows.dropna(subset=[column]) + rows[column] = rows[column].astype(str) + return _stacked_by_source( + rows, + column, + sorted(set(rows[column]), key=_version_key), + f"{name} version ({len(rows)} datasets)", + f"{name} version", ) @@ -484,16 +521,15 @@ def channels_histogram(df: pd.DataFrame) -> go.Figure: def nb_datatypes_histogram(df: pd.DataFrame) -> go.Figure: - """Plot the number of datasets for each number of datatypes they have.""" - counts = df["datatypes"].apply(len).value_counts().sort_index() - fig = px.bar( - x=counts.index, - y=counts.values, - text=counts.values, - labels={"x": "number of datatypes", "y": "number of datasets"}, - title="Number of datatypes per dataset", + """Plot the number of datasets for each number of datatypes they have, by source.""" + rows = df[["name", "source"]].assign(nb_datatypes=df["datatypes"].apply(len)) + return _stacked_by_source( + rows, + "nb_datatypes", + sorted(set(rows["nb_datatypes"])), + "Number of datatypes per dataset", + "number of datatypes", ) - return fig.update_xaxes(dtick=1) # datasets with physio files whose recording labels and columns are not known @@ -508,28 +544,22 @@ def physio_types_histogram(df: pd.DataFrame) -> go.Figure: The datasets with physio files whose types are not identified are counted apart. """ with_physio = df[df["has_physio"].fillna(False).astype(bool)] - types = with_physio["physio_types"].apply( - lambda x: x if isinstance(x, list) and x else [NO_RECORDING_LABEL] + rows = with_physio[["name", "source"]].assign( + physio_types=with_physio["physio_types"].apply( + lambda x: x if isinstance(x, list) and x else [NO_RECORDING_LABEL] + ) ) - counts = types.explode().value_counts() + rows = rows.explode("physio_types") + counts = rows["physio_types"].value_counts() # the most common types first, the datasets without labels last order = [x for x in counts.index if x != NO_RECORDING_LABEL] order += [x for x in counts.index if x == NO_RECORDING_LABEL] - counts = counts.loc[order] - fig = go.Figure( - go.Bar( - x=list(counts.index), - y=list(counts.values), - text=list(counts.values), - hovertemplate="%{x}
%{y} datasets", - ) - ) - return fig.update_layout( - title=( - f"Types of physiological recordings ({len(with_physio)} datasets with physio files)" - "
from the recording labels of the physio files and the columns " - "of their sidecars: a dataset can have several types" - ), - xaxis={"type": "category", "title": {"text": "type of recording"}}, - yaxis={"title": {"text": "number of datasets"}}, + return _stacked_by_source( + rows, + "physio_types", + order, + f"Types of physiological recordings ({len(with_physio)} datasets with physio files)" + "
from the recording labels of the physio files and the columns " + "of their sidecars: a dataset can have several types", + "type of recording", ) diff --git a/bids_inventory/browse_app.py b/bids_inventory/browse_app.py index 54b3452..11e9b13 100644 --- a/bids_inventory/browse_app.py +++ b/bids_inventory/browse_app.py @@ -62,11 +62,7 @@ def _(bids_inventory_installed): version_histogram, ) from bids_inventory._version import version - from bids_inventory.institutions import ( - count_institutions, - institutions_locations, - load_institutions, - ) + from bids_inventory.institutions import institutions_locations_by_source, load_institutions from bids_inventory.listing import ( KNOWN_DATATYPES, known_datasets_df, @@ -90,12 +86,11 @@ def _(bids_inventory_installed): NOT_SPECIFIED, Path, contextlib, - count_institutions, datatypes_histogram, field_strength_histogram, filter_data, histogram_tasks, - institutions_locations, + institutions_locations_by_source, institutions_map, known_datasets_df, load_institutions, @@ -619,17 +614,16 @@ def _(channels_histogram, file_formats_histogram, filtered_df, mo): @app.cell def _( - count_institutions, filtered_df, institutions, - institutions_locations, + institutions_locations_by_source, institutions_map, mo, ): mo.vstack( [ mo.md("---"), - institutions_map(institutions_locations(institutions, count_institutions(filtered_df))), + institutions_map(institutions_locations_by_source(institutions, filtered_df)), ] ) return diff --git a/bids_inventory/institutions.py b/bids_inventory/institutions.py index 6ea084e..98d5b92 100644 --- a/bids_inventory/institutions.py +++ b/bids_inventory/institutions.py @@ -175,6 +175,25 @@ def institutions_locations( return pd.DataFrame(rows, columns=COLUMNS) +def institutions_locations_by_source( + institutions: dict[str, dict[str, Any]], df: pd.DataFrame +) -> pd.DataFrame: + """Return the locations of the institutions of the datasets, for each source of datasets. + + One row per institution and source (``source`` column), as ``institutions_locations``: + the number of datasets and subjects of the institution from this source. + """ + frames = [ + institutions_locations(institutions, count_institutions(datasets)).assign(source=source) + for source, datasets in df.groupby("source") + ] + if not frames: + return pd.DataFrame(columns=[*COLUMNS, "source"]) + locations = pd.concat(frames, ignore_index=True) + # integers, also when some sources have no located institutions + return locations.astype({"count": int, "nb_subjects": int}) + + def count_institutions(df: pd.DataFrame) -> pd.DataFrame: """Count the datasets and subjects of each institution of the inventory.""" counts: dict[str, dict[str, int]] = {} diff --git a/tests/test_institutions.py b/tests/test_institutions.py index 59d9678..c6d2de0 100644 --- a/tests/test_institutions.py +++ b/tests/test_institutions.py @@ -115,3 +115,17 @@ def test_postal_code(name, expected): from bids_inventory.institutions import postal_code assert postal_code(name) == expected + + +def test_institutions_locations_by_source(institutions): + from bids_inventory.institutions import institutions_locations_by_source + + df = pd.DataFrame( + { + "source": ["openneuro", "openneuro", "abide"], + "institutions": [["BBL_-_University_of_Geneva"], ["BBL_-_University_of_Geneva"], []], + "nb_subjects": [1, 2, 3], + } + ) + locations = institutions_locations_by_source(institutions, df) + assert locations[["source", "count", "nb_subjects"]].values.tolist() == [["openneuro", 2, 3]] diff --git a/tests/test_plotting.py b/tests/test_plotting.py index a60d182..5a8a137 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -87,14 +87,21 @@ def test_field_strength_histogram(): df = pd.DataFrame( { "name": ["ds1", "ds2", "ds3", "ds4"], + "source": ["openneuro", "abide", "openneuro", "openneuro"], "nb_subjects": [10, 20, 30, 40], - "field_strengths": [["3T", "7T"], ["unknown"], ["10.5T"], []], + "field_strengths": [["3T", "7T"], ["3T"], ["10.5T"], []], } ) fig = field_strength_histogram(df) - # sorted by field strength, unknown last; datasets without MRI not counted - assert list(fig.data[0].x) == ["3T", "7T", "10.5T", "unknown"] - assert list(fig.data[0].y) == [1, 1, 1, 1] + # one stacked bar per source; sorted by field strength; datasets without MRI not counted + bars = {trace.name: dict(zip(trace.x, trace.y)) for trace in fig.data if trace.type == "bar"} + assert bars == { + "openneuro": {"3T": 1, "7T": 1, "10.5T": 1}, + "abide": {"3T": 1, "7T": 0, "10.5T": 0}, + } + assert fig.layout.barmode == "stack" + # the number of participants on hover + assert list(fig.data[0].customdata) == [10, 10, 30] @pytest.fixture @@ -102,6 +109,7 @@ def listing(): return pd.DataFrame( { "name": ["ds1", "ds2", "ds3"], + "source": ["openneuro", "abide", "openneuro"], "datatypes": [["anat", "func"], ["eeg"], ["meg", "eeg", "anat"]], "duration": [ {"func": {"rest": [600.0, None], "nback": None}}, @@ -136,6 +144,7 @@ def test_version_histogram(listing): fig = version_histogram(listing, "bids_version", "BIDS") # sorted as versions, not as text assert list(fig.data[0].x) == ["1.9.0", "1.10.0"] + assert [trace.name for trace in fig.data if trace.type == "bar"] == ["openneuro", "abide"] assert version_histogram(listing, "mriqc_version", "MRIQC").layout.title.text.startswith( "MRIQC version (0 datasets)" ) @@ -164,7 +173,11 @@ def test_nb_datatypes_histogram(listing): from bids_inventory._plotting import nb_datatypes_histogram fig = nb_datatypes_histogram(listing) - assert dict(zip(fig.data[0].x, fig.data[0].y)) == {1: 1, 2: 1, 3: 1} + totals: dict[int, int] = {} + for trace in fig.data: + for x, y in zip(trace.x, trace.y): + totals[x] = totals.get(x, 0) + y + assert totals == {1: 1, 2: 1, 3: 1} def test_physio_types_histogram(): @@ -172,6 +185,8 @@ def test_physio_types_histogram(): df = pd.DataFrame( { + "name": ["ds1", "ds2", "ds3", "ds4"], + "source": ["openneuro"] * 4, "has_physio": [True, True, True, False], "physio_types": [["cardiac", "respiratory"], ["cardiac"], [], []], } @@ -184,3 +199,30 @@ def test_physio_types_histogram(): "not identified": 1, } assert list(fig.data[0].x)[-1] == "not identified" + + +def test_source_colors(): + """Each source has its color, the same in all the figures.""" + from bids_inventory._plotting import SOURCE_COLORS + from bids_inventory.wrangling import SOURCES + + assert set(SOURCE_COLORS) == {source for _, source in SOURCES} | {"other"} + assert len(set(SOURCE_COLORS.values())) == len(SOURCE_COLORS) + + +def test_institutions_map_by_source(): + from bids_inventory._plotting import institutions_map + + locations = pd.DataFrame( + { + "name": ["A", "A", "B"], + "count": [1, 2, 3], + "nb_subjects": [10, 20, 30], + "address": ["a", "a", "b"], + "latitude": [1.0, 1.0, 2.0], + "longitude": [1.0, 1.0, 2.0], + "source": ["openneuro", "dandi", "openneuro"], + } + ) + fig = institutions_map(locations) + assert [trace.name for trace in fig.data] == ["openneuro", "dandi"]