diff --git a/crates/ironlab-ir/src/artist.rs b/crates/ironlab-ir/src/artist.rs index a00f68e..e37f220 100644 --- a/crates/ironlab-ir/src/artist.rs +++ b/crates/ironlab-ir/src/artist.rs @@ -922,7 +922,10 @@ pub struct Binning { pub range: Option<[f64; 2]>, /// How the counts of the bins were normalised into the values drawn. pub normalization: HistNorm, - /// The number of samples counted into the bins: finite and not negative. + /// The total the values were normalised by, finite and not negative: the number of + /// samples counted into the bins, or, for one of several histograms normalised + /// together, as the members of a stack are, the number counted into the bins of all + /// of them. pub total: f64, /// The number of samples left out because they were not finite. pub excluded_non_finite: u64, diff --git a/crates/ironlab/src/artists.rs b/crates/ironlab/src/artists.rs index 0f34826..83292af 100644 --- a/crates/ironlab/src/artists.rs +++ b/crates/ironlab/src/artists.rs @@ -1059,6 +1059,11 @@ impl BarColumns { self.members.iter().map(|(id, _)| *id).collect() } + /// Returns the column of every series, in the order they were added. + pub(crate) fn columns(&self) -> impl Iterator { + self.members.iter().map(|(_, column)| column) + } + /// Returns the width of every series, as a fraction of the space of a cluster, and the /// offset of each series, in multiples of its own width, for clusters that leave the /// fraction `gap` of their space empty and columns that leave the fraction `group_gap` diff --git a/crates/ironlab/src/axes.rs b/crates/ironlab/src/axes.rs index 5dc4377..1ec904c 100644 --- a/crates/ironlab/src/axes.rs +++ b/crates/ironlab/src/axes.rs @@ -325,6 +325,33 @@ impl AxesMut<'_> { /// samples, Matplotlib's `hist` with `stacked=True` or a list of samples), each an /// ordinary bar series placed as [`bars`](Self::bars) places its series; see /// [`HistsMut`]. + /// + /// The members of a stacked column are normalised by the pooled total of the column, so + /// that the top of the stack is the normalised histogram of the pooled sample and each + /// layer is the member's share of it, as Matplotlib's `hist(stacked=True, + /// density=True)` and seaborn's default `common_norm=True` draw it. Members side by side + /// are normalised each by its own total, so that their shapes compare. + /// [`HistsMut::common_norm`] overrides either rule. + /// + /// ``` + /// use ironlab::prelude::*; + /// + /// let control = [0.2, 0.8, 1.1, 1.4, 1.9, 2.3, 2.6, 3.1]; + /// let treated = [1.2, 1.7, 2.1, 2.4, 2.8, 3.3, 3.6, 3.9]; + /// let mut fig = Figure::new(); + /// let mut ax = fig.axes(0, 0); + /// // A stack whose outline is the density of the pooled sample. + /// ax.hists(Bins::Count(8)) + /// .normalize(HistNorm::Density) + /// .stacked("all", "Control", control) + /// .stacked("all", "Treated", treated); + /// // Densities side by side, each enclosing unit area. + /// ax.hists(Bins::Count(8)) + /// .normalize(HistNorm::Density) + /// .series("Control", control) + /// .series("Treated", treated); + /// assert!(fig.validate().is_valid()); + /// ``` pub fn hists(&mut self, bins: Bins) -> HistsMut<'_> { HistsMut::new(self.fig, self.id, bins) } diff --git a/crates/ironlab/src/binning.rs b/crates/ironlab/src/binning.rs index 4e4a1c0..b10d974 100644 --- a/crates/ironlab/src/binning.rs +++ b/crates/ironlab/src/binning.rs @@ -585,6 +585,13 @@ impl Bounds { } } +/// Normalises the counts of the bins of one dimension, whose widths are given, by `total`, +/// which is the total of the included samples unless several histograms are normalised by +/// the total they pool, as the members of a stack are. +pub(crate) fn normalise_1d(counts: &[f64], widths: &[f64], total: f64, norm: HistNorm) -> Vec { + normalise(counts, widths, total, norm, cumulative_1d) +} + /// Normalises the counts of the bins, whose widths (or areas) are given, by the total of /// the included samples. The function `accumulate` turns counts into cumulative counts. fn normalise( diff --git a/crates/ironlab/src/hist.rs b/crates/ironlab/src/hist.rs index 6f421c2..99cb66c 100644 --- a/crates/ironlab/src/hist.rs +++ b/crates/ironlab/src/hist.rs @@ -18,7 +18,7 @@ use ironlab_ir::{ use crate::artists::{BarColumns, BarMut, BarsColumn, data_ids, release_data}; use crate::binning::{ BinRange, BinningError, Bins, HistNorm, HistStyle, Histogram, Histogram2d, HistogramSummary, - bin_edges, histogram, histogram2d, + bin_edges, histogram, histogram2d, normalise_1d, }; use crate::color::IntoColorSpec; use crate::color_scale; @@ -440,8 +440,8 @@ impl HistMut<'_> { /// compare bin by bin: explicit edges are used as given, and any other specification is /// applied to the pooled sample of all the members, as Matplotlib does when it is given /// several samples. Each member is an ordinary bar series with its own label, its colour -/// from the colour order, its own values and its own binning record, whose total and -/// exclusions are those of its own sample. A member added by [`series`](Self::series) +/// from the colour order, its own values and its own binning record, whose exclusions are +/// those of its own sample. A member added by [`series`](Self::series) /// stands in a column of its own, the columns splitting each bin into equal parts in the /// order they first appear, the first at the low end; a member added by /// [`stacked`](Self::stacked) spans its whole bin in the column of the name it is given and @@ -449,8 +449,18 @@ impl HistMut<'_> { /// the code that places the series of [`BarsMut`](crate::BarsMut), with no gap between /// clusters or columns, and with one width per bar, because bins may differ in width. /// -/// The normalisation and the range apply to every member, each normalised by its own -/// total, and may be set before or after the members are added; every call bins every +/// The normalisation and the range apply to every member. The members of a stacked column +/// are normalised by the pooled total of that column, as Matplotlib's +/// `hist(stacked=True, density=True)` and seaborn's default `common_norm=True` normalise +/// them, so that the top of the stack is the normalised histogram of the pooled sample and +/// each layer is the member's share of it: a stack of densities encloses unit area, and a +/// stack of probabilities, percentages or a CDF reaches 1, 100 or 1. A member standing in a +/// column of its own is normalised by its own total, because members side by side are +/// compared by shape. [`common_norm`](Self::common_norm) overrides either rule. Each +/// member's record holds the total its values were normalised by, so that a datatip +/// recovers the member's own count. +/// +/// The settings may be given before or after the members are added; every call bins every /// sample again and lays every member out again, releasing the arrays it replaces. A /// specification that cannot be applied never panics: every member is stored with no bars /// and no record, and [`error`](Self::error) returns the reason. @@ -461,10 +471,26 @@ impl HistMut<'_> { /// let control = [0.2, 0.8, 1.1, 1.4, 1.9, 2.3, 2.6, 3.1]; /// let treated = [1.2, 1.7, 2.1, 2.4, 2.8, 3.3, 3.6, 3.9]; /// let mut fig = Figure::new(); -/// fig.axes(0, 0) -/// .hists(Bins::Count(8)) +/// let mut ax = fig.axes(0, 0); +/// let mut hists = ax.hists(Bins::Count(8)); +/// hists +/// .normalize(HistNorm::Density) /// .stacked("all", "Control", control) /// .stacked("all", "Treated", treated); +/// let ids = hists.ids(); +/// // The stack, not each layer, encloses unit area. +/// let width = 3.7 / 8.0; +/// let area: f64 = ids +/// .iter() +/// .map(|id| match fig.ir().artist(*id) { +/// Some((_, ironlab::ir::Artist::Bar(bar))) => { +/// let values = fig.ir().data_array(bar.values).unwrap(); +/// values.array.as_f64().unwrap().iter().sum::() * width +/// } +/// _ => unreachable!(), +/// }) +/// .sum(); +/// assert!((area - 1.0).abs() < 1e-9); /// assert!(fig.validate().is_valid()); /// ``` #[derive(Debug)] @@ -474,6 +500,9 @@ pub struct HistsMut<'a> { bins: Bins, range: Option, norm: HistNorm, + /// Whether every member is normalised by the pooled total of all the members (`true`) + /// or each by its own (`false`), or `None` for the default that follows the layout. + common_norm: Option, /// The orientation of every member. orientation: Orientation, /// The members, in the order they were added, with the column each belongs to. @@ -493,6 +522,7 @@ impl<'a> HistsMut<'a> { bins, range: None, norm: HistNorm::default(), + common_norm: None, orientation: Orientation::Vertical, columns: BarColumns::default(), samples: Vec::new(), @@ -523,13 +553,49 @@ impl HistsMut<'_> { self.add(column, label.into(), sample.into()) } - /// Sets what the value of each bin of every member represents (see [`HistNorm`]), each - /// member normalised by its own total. + /// Sets what the value of each bin of every member represents (see [`HistNorm`]). The + /// members of a stacked column are normalised by the pooled total of the column, so + /// that the top of the stack is the normalised histogram of the pooled sample, and a + /// member in a column of its own by its own total, unless + /// [`common_norm`](Self::common_norm) says otherwise. Counts and count densities are + /// not divided by a total, and are the same either way. pub fn normalize(&mut self, norm: HistNorm) -> &mut Self { self.norm = norm; self.update() } + /// Normalises every member by the pooled total of all the members (`true`), or each + /// member by its own total (`false`), in place of the default, under which the members + /// of a stacked column share the total of their column and every other member has its + /// own (seaborn's `common_norm`). The result is the same whether this is called before + /// or after the members are added. + /// + /// ``` + /// use ironlab::prelude::*; + /// + /// let mut fig = Figure::new(); + /// let mut ax = fig.axes(0, 0); + /// let mut hists = ax.hists(Bins::Edges(vec![0.0, 1.0, 2.0])); + /// hists + /// .normalize(HistNorm::Probability) + /// .common_norm(false) + /// .stacked("all", "Small", [0.5]) + /// .stacked("all", "Large", [0.5, 1.5, 1.5]); + /// // Each layer is a distribution of its own, summing to 1, so the stack sums to 2. + /// for id in hists.ids() { + /// let Some((_, ironlab::ir::Artist::Bar(bar))) = fig.ir().artist(id) else { + /// unreachable!() + /// }; + /// let values = fig.ir().data_array(bar.values).unwrap(); + /// let sum: f64 = values.array.as_f64().unwrap().iter().sum(); + /// assert!((sum - 1.0).abs() < 1e-12); + /// } + /// ``` + pub fn common_norm(&mut self, common: bool) -> &mut Self { + self.common_norm = Some(common); + self.update() + } + /// Sets the range of values that the shared bins span, in place of the range of the /// pooled sample; the samples of each member outside it are excluded, and counted in /// that member's record. @@ -609,7 +675,7 @@ impl HistsMut<'_> { /// stacked column. The arrays it replaces are released. fn update(&mut self) -> &mut Self { let pooled = self.samples.concat(); - let (edges, binned): (Vec, Vec>) = + let (edges, binned): (Vec, Vec>) = match bin_edges(&pooled, &self.bins, self.range) { Ok(edges) => { self.error = None; @@ -620,12 +686,14 @@ impl HistsMut<'_> { } else { Bins::Edges(edges.clone()) }; - let binned = self + let counts = self .samples .iter() - .map(|sample| histogram(sample, None, &shared, self.range, self.norm).ok()) + .map(|sample| { + histogram(sample, None, &shared, self.range, HistNorm::Count).ok() + }) .collect(); - (edges, binned) + (edges, self.normalised(counts)) } Err(error) => { self.error = Some(error); @@ -642,9 +710,9 @@ impl HistsMut<'_> { let mut replaced = Vec::new(); for ((id, offset), h) in placed.into_iter().zip(binned) { let (values, binning) = match h { - Some(h) => { - let binning = - record(&self.bins, shared_edges, self.range, self.norm, h.summary); + Some((h, total)) => { + let summary = HistogramSummary { total, ..h.summary }; + let binning = record(&self.bins, shared_edges, self.range, self.norm, summary); (h.values, binning) } None => (Vec::new(), None), @@ -674,6 +742,44 @@ impl HistsMut<'_> { self.columns.stack(self.fig, edges.len().saturating_sub(1)); self } + + /// Normalises the counts of every member, binned on the shared edges, and returns each + /// member's histogram with the total it was normalised by: the pooled total of all the + /// members under common normalisation, the pooled total of the member's stacked column + /// by default, and otherwise the member's own total. A member that could not be binned + /// stays `None` and adds nothing to a pooled total. + fn normalised(&self, counts: Vec>) -> Vec> { + let own = |h: &Option| h.as_ref().map_or(0.0, |h| h.summary.total); + let all: f64 = counts.iter().map(own).sum(); + let pooled = |column: &BarsColumn| -> f64 { + self.columns + .columns() + .zip(&counts) + .filter(|(c, _)| *c == column) + .map(|(_, h)| own(h)) + .sum() + }; + let totals: Vec = self + .columns + .columns() + .zip(&counts) + .map(|(column, h)| match (self.common_norm, column) { + (Some(true), _) => all, + (None, BarsColumn::Stacked(_)) => pooled(column), + _ => own(h), + }) + .collect(); + counts + .into_iter() + .zip(totals) + .map(|(h, total)| { + h.map(|h| { + let values = normalise_1d(&h.values, &widths(&h.edges), total, self.norm); + (Histogram { values, ..h }, total) + }) + }) + .collect() + } } /// The pairs of samples that a two-dimensional handle bins, with the settings of the diff --git a/crates/ironlab/tests/hist.rs b/crates/ironlab/tests/hist.rs index 2016403..e7fcff4 100644 --- a/crates/ironlab/tests/hist.rs +++ b/crates/ironlab/tests/hist.rs @@ -15,7 +15,9 @@ //! - `AxesMut::hists(bins: Bins) -> HistsMut`, the histogram counterpart of `bars`, whose `series(label, sample)` adds //! a histogram as a column of its own and `stacked(column, label, sample)` adds one to the named stacked column. //! Every sample is binned on one shared set of edges: explicit edges as given, and any other specification applied -//! to the pooled sample of all the members. The getters are `ids() -> Vec` and +//! to the pooled sample of all the members. The members of a stacked column are normalised by the pooled total of +//! the column and every other member by its own total, unless `common_norm(bool)` normalises every member by the +//! pooled total of all the members (`true`) or each by its own (`false`). The getters are `ids() -> Vec` and //! `error() -> Option<&BinningError>`. //! - `AxesMut::hist_counts(edges, counts) -> BarMut`, which draws counts binned beforehand as touching bars. //! @@ -1441,12 +1443,12 @@ fn hists_of_a_bad_specification_draw_nothing_and_the_helper_reports_why() { assert!(has_warning(&report, IssueKind::NothingToDraw), "{report:?}"); } -// WHY: stacked and grouped density histograms compare the shapes of samples of different sizes, so the helper's -// normalisation and range must apply to every member, whether they are set before the members are added or after. -// Each member is normalised by its own total, so that its own bins enclose unit area and its record holds the total -// that a datatip needs; the range bounds the shared edges, whatever the pooled sample spans, and excludes from each -// member the values outside it, which that member's record counts. A stacked member rests on the values of the member -// below it, so that the column's height is the sum of the values drawn. +// WHY: the helper's normalisation and range must apply to every member, whether they are set before the members are +// added or after. Members side by side are compared by shape, so each is normalised by its own total and its own bins +// enclose unit area; members of one stack are normalised by the pooled total of the stack, so that the stack encloses +// unit area and each layer is its share of it. Either way each record holds the total its values were normalised by, +// which a datatip needs, and its own exclusions. The range bounds the shared edges, whatever the pooled sample spans, +// and excludes from each member the values outside it. A stacked member rests on the values of the member below it. #[test] fn hists_normalize_and_range_apply_to_every_member_whenever_they_are_set() { let first = [0.5, 1.5, 1.5, 2.5, -2.0, f64::NAN]; @@ -1477,22 +1479,21 @@ fn hists_normalize_and_range_apply_to_every_member_whenever_they_are_set() { (ids[0], first.as_slice(), 4.0, 1, 1), (ids[1], second.as_slice(), 3.0, 0, 1), ]; - for (id, sample, total, non_finite, out_of_range) in members { - let expected = histogram( + let mut areas = Vec::new(); + for (id, sample, own_total, non_finite, out_of_range) in members { + let total = if stacked { 7.0 } else { own_total }; + let counts = histogram( sample, None, &Bins::Edges(edges.clone()), Some((0.0, 4.0)), - HistNorm::Density, + HistNorm::Count, ) .unwrap(); + let expected: Vec = counts.values.iter().map(|c| c / total).collect(); let values = vector(&fig, bar(&fig, id).values); - assert_close(&values, &expected.values, &format!("{what}: densities")); - let area: f64 = values.iter().sum(); - assert!( - (area - 1.0).abs() < 1e-12, - "{what}: the member's own bins enclose unit area, not {area}" - ); + assert_close(&values, &expected, &format!("{what}: densities")); + areas.push(values.iter().sum::()); assert_eq!( record(&fig, id), Some(Binning { @@ -1504,7 +1505,7 @@ fn hists_normalize_and_range_apply_to_every_member_whenever_they_are_set() { excluded_non_finite: non_finite, excluded_out_of_range: out_of_range, }), - "{what}" + "{what}: the total normalised by, and the member's own exclusions" ); let extents = bar_extents(&fig, id); for (k, e) in extents.iter().enumerate() { @@ -1514,6 +1515,20 @@ fn hists_normalize_and_range_apply_to_every_member_whenever_they_are_set() { ); } } + if stacked { + let area: f64 = areas.iter().sum(); + assert!( + (area - 1.0).abs() < 1e-12, + "{what}: the stack encloses unit area, not {area}" + ); + } else { + for area in areas { + assert!( + (area - 1.0).abs() < 1e-12, + "{what}: each member encloses unit area, not {area}" + ); + } + } let lower = vector(&fig, bar(&fig, ids[0]).values); let bases: Vec = bar_extents(&fig, ids[1]).iter().map(|e| e.base).collect(); let expected_bases = if stacked { lower } else { vec![0.0; 4] }; @@ -1525,6 +1540,257 @@ fn hists_normalize_and_range_apply_to_every_member_whenever_they_are_set() { } } +/// The value of each bin at the top of the stack of the given members: the sum of their values. +fn stack_top(fig: &Figure, ids: &[NodeId]) -> Vec { + let mut top = vec![0.0; bar_count(fig, ids[0])]; + for id in ids { + for (t, v) in top.iter_mut().zip(vector(fig, bar(fig, *id).values)) { + *t += v; + } + } + top +} + +// WHY: a stacked histogram shows how the pooled sample divides among its members, so the top of the stack must be the +// histogram of the pooled sample under every normalisation, as Matplotlib's `hist(stacked=True, density=True)` and +// seaborn's `histplot(multiple="stack")` draw it: a density stack encloses unit area, a probability stack sums to 1, +// a percentage stack to 100, and a cumulative stack ends at the pooled total, or at 1 for a CDF. Each layer is the +// member's share of that: its own counts normalised by the pooled total. Counts and count densities need no total and +// are unchanged. Bins of unequal width check that a density is divided by the width of its own bin. +#[test] +fn the_top_of_a_stack_is_the_normalised_histogram_of_the_pooled_sample() { + let edges = vec![0.0, 1.0, 2.0, 3.0, 4.5]; + let widths = [1.0, 1.0, 1.0, 1.5]; + let pooled: Vec = STEPPED.iter().chain(&OTHER).copied().collect(); + let pooled_total = 15.0; + for norm in [ + HistNorm::Count, + HistNorm::CountDensity, + HistNorm::Density, + HistNorm::Probability, + HistNorm::Percentage, + HistNorm::CumulativeCount, + HistNorm::Cdf, + ] { + let mut fig = Figure::new(); + let ids = fig + .axes(0, 0) + .hists(Bins::Edges(edges.clone())) + .normalize(norm) + .stacked("both", "first", STEPPED) + .stacked("both", "second", OTHER) + .ids(); + let top = stack_top(&fig, &ids); + let expected = histogram(&pooled, None, &Bins::Edges(edges.clone()), None, norm).unwrap(); + assert_close( + &top, + &expected.values, + &format!("{norm:?}: the pooled histogram"), + ); + let what = format!("{norm:?}: the top of the stack"); + match norm { + HistNorm::Density => { + let area: f64 = top.iter().zip(widths).map(|(v, w)| v * w).sum(); + assert!((area - 1.0).abs() < 1e-12, "{what} encloses {area}"); + } + HistNorm::Probability => { + assert!((top.iter().sum::() - 1.0).abs() < 1e-12, "{what}") + } + HistNorm::Percentage => { + assert!((top.iter().sum::() - 100.0).abs() < 1e-10, "{what}") + } + HistNorm::CumulativeCount => assert_eq!(top.last(), Some(&pooled_total), "{what}"), + HistNorm::Cdf => assert!((top[3] - 1.0).abs() < 1e-12, "{what} ends at {}", top[3]), + _ => {} + } + for (id, sample) in ids.iter().zip([STEPPED.as_slice(), OTHER.as_slice()]) { + let counts = histogram( + sample, + None, + &Bins::Edges(edges.clone()), + None, + HistNorm::Count, + ) + .unwrap() + .values; + let cumulative: Vec = counts + .iter() + .scan(0.0, |sum, c| { + *sum += c; + Some(*sum) + }) + .collect(); + let share: Vec = match norm { + HistNorm::Count => counts, + HistNorm::CountDensity => counts.iter().zip(widths).map(|(c, w)| c / w).collect(), + HistNorm::Density => counts + .iter() + .zip(widths) + .map(|(c, w)| c / (pooled_total * w)) + .collect(), + HistNorm::Probability => counts.iter().map(|c| c / pooled_total).collect(), + HistNorm::Percentage => counts.iter().map(|c| 100.0 * c / pooled_total).collect(), + HistNorm::CumulativeCount => cumulative, + HistNorm::Cdf => cumulative.iter().map(|c| c / pooled_total).collect(), + }; + assert_close( + &vector(&fig, bar(&fig, *id).values), + &share, + &format!("{norm:?}: the layer is the member's share of the pooled histogram"), + ); + } + } +} + +// WHY: the default follows the layout, because stacked members are read as parts of a whole and side-by-side members +// are compared by shape, but some comparisons want the other: members side by side whose heights show their shares of +// the pooled sample (seaborn's `common_norm=True` with `multiple="dodge"`), or a stack of shapes normalised each on +// its own (seaborn's `common_norm=False` with `multiple="stack"`). `common_norm(true)` must normalise every member by +// the pooled total of all the members, and `common_norm(false)` each by its own, in either layout; and because the +// helper may be configured before or after its members are added, the result must not depend on when it is called. +// The record of each member holds the total its values were normalised by. +#[test] +fn common_norm_overrides_the_layout_whenever_it_is_set() { + let edges = vec![0.0, 1.0, 2.0, 3.0, 4.5]; + let samples = [STEPPED.as_slice(), OTHER.as_slice()]; + let own_totals = [10.0, 5.0]; + for stacked in [false, true] { + for common in [None, Some(true), Some(false)] { + for setter_first in [true, false] { + let what = + format!("stacked {stacked}, common {common:?}, setter first {setter_first}"); + let mut fig = Figure::new(); + let mut ax = fig.axes(0, 0); + let mut hists = ax.hists(Bins::Edges(edges.clone())); + hists.normalize(HistNorm::Probability); + if let (Some(common), true) = (common, setter_first) { + hists.common_norm(common); + } + for (k, sample) in samples.iter().enumerate() { + if stacked { + hists.stacked("both", format!("m{k}"), sample.to_vec()); + } else { + hists.series(format!("m{k}"), sample.to_vec()); + } + } + if let (Some(common), false) = (common, setter_first) { + hists.common_norm(common); + } + let pooled = common.unwrap_or(stacked); + for ((id, sample), own_total) in hists.ids().iter().zip(samples).zip(own_totals) { + let total = if pooled { 15.0 } else { own_total }; + let counts = histogram( + sample, + None, + &Bins::Edges(edges.clone()), + None, + HistNorm::Count, + ) + .unwrap() + .values; + let expected: Vec = counts.iter().map(|c| c / total).collect(); + assert_close(&vector(&fig, bar(&fig, *id).values), &expected, &what); + assert_eq!( + record(&fig, *id).map(|r| r.total), + Some(total), + "{what}: the record holds the total normalised by" + ); + } + } + } + } +} + +// WHY: each stacked column is a whole of its own, so the members of one stack are normalised by the pooled total of +// that stack alone, not of every member of the helper, and a member in a column of its own beside the stack keeps its +// own total. Each record holds the total its values were normalised by, so that a datatip recovers the member's own +// count, while the exclusions stay the member's own, because they describe what its own sample lost. +#[test] +fn the_members_of_a_stack_record_the_pooled_total_of_their_stack_and_their_own_exclusions() { + let edges = vec![0.0, 1.0, 2.0, 3.0, 4.0]; + let alone = [0.5, 1.5, 2.5]; + let lower = [0.5, 1.5, 1.5, f64::NAN]; + let upper = [2.5, 3.5, 9.0]; + let mut fig = Figure::new(); + let ids = fig + .axes(0, 0) + .hists(Bins::Edges(edges.clone())) + .normalize(HistNorm::Density) + .series("alone", alone) + .stacked("s", "lower", lower) + .stacked("s", "upper", upper) + .ids(); + let expected = [(3.0, 0, 0), (5.0, 1, 0), (5.0, 0, 1)]; + for (id, (total, non_finite, out_of_range)) in ids.iter().zip(expected) { + assert_eq!( + record(&fig, *id), + Some(Binning { + edges: NodeId::default(), + bins: Bins::Edges(edges.clone()), + range: None, + normalization: HistNorm::Density, + total, + excluded_non_finite: non_finite, + excluded_out_of_range: out_of_range, + }) + ); + } + let area: f64 = stack_top(&fig, &ids[1..]).iter().sum(); + assert!( + (area - 1.0).abs() < 1e-12, + "the stack encloses unit area, not {area}" + ); + let report = fig.validate(); + assert!(report.is_valid(), "{report:?}"); + assert!(report.warnings.is_empty(), "{report:?}"); +} + +// WHY: a layer of a stacked density histogram draws the member's share of the pooled density, but a reader pointing +// at it wants to know how many of the member's own samples lie in the bin. Because the record holds the pooled total, +// the datatip must recover the member's own count, and report as its density the member's contribution to the pooled +// density, the count divided by the pooled total and the width of the bin, so that the densities of the layers of a +// bin add up to the density of the stack there. +#[test] +fn the_datatip_of_a_stacked_member_reports_its_own_count_and_its_share_of_the_density() { + let edges = vec![0.0, 1.0, 2.0, 3.0, 4.5]; + let widths = [1.0, 1.0, 1.0, 1.5]; + let members = [ + (STEPPED.as_slice(), [1.0, 2.0, 3.0, 4.0]), + (OTHER.as_slice(), [2.0, 1.0, 1.0, 1.0]), + ]; + for norm in [HistNorm::Density, HistNorm::Probability, HistNorm::Cdf] { + let mut fig = Figure::new(); + let mut ax = fig.axes(0, 0); + let mut hists = ax.hists(Bins::Edges(edges.clone())); + hists.normalize(norm); + for (k, (sample, _)) in members.iter().enumerate() { + hists.stacked("both", format!("m{k}"), sample.to_vec()); + } + let ids = hists.ids(); + let scene = compile(&fig); + for (id, (_, counts)) in ids.iter().zip(members) { + let mut hits: Vec<_> = scene + .hit_map + .bars + .iter() + .filter(|b| b.artist == *id) + .collect(); + hits.sort_by_key(|b| b.index); + assert_eq!(hits.len(), 4, "{norm:?}: one datatip per bin"); + for (k, hit) in hits.iter().enumerate() { + let reading = hit.bin.expect("a stacked member records its binning"); + let density = counts[k] / (15.0 * widths[k]); + assert!( + (reading.count - counts[k]).abs() < 1e-9 + && (reading.density - density).abs() < 1e-9, + "{norm:?}, bin {k}: read {reading:?}, not a count of {} and a density of {density}", + counts[k] + ); + } + } + } +} + // WHY: a range that cannot bound bins is a mistake in the helper's specification, shared by every member, so it // behaves as a bad bin specification does: no panic, every member stored with no bars and no record, and the reason // on the helper until a later range fixes it, after which every member is binned and recorded again. diff --git a/docs/guides/getting-started.md b/docs/guides/getting-started.md index 0609580..88ef511 100644 --- a/docs/guides/getting-started.md +++ b/docs/guides/getting-started.md @@ -312,7 +312,7 @@ Each setter that changes the binning bins the sample again and replaces what the `hist_counts(edges, counts)` draws counts binned elsewhere, by MATLAB's `histcounts`, numpy or an instrument, as the touching bars that `hist` would draw, one fewer than the edges. It returns the `BarMut` of ordinary bars, which carry no record, because the figure cannot know how many samples the counts hold or how they were binned; `stairs(edges, counts)` draws the same counts as an outline. -`hists(bins)` draws several samples as stacked or grouped histograms, the counterpart of [`bars`](#bar-charts), and returns a `HistsMut`. `series(label, sample)` adds a histogram as a column of its own, and `stacked(column, label, sample)` adds one to the stacked column of that name. Every sample is binned on one shared set of edges, so that the samples compare bin by bin: explicit edges are used as given, and any other specification is applied to all the samples pooled together. The histograms are placed by the same code that places the series of `bars`: grouped histograms split each bin into equal parts in the order they were added, the first at the low end, and each stacked histogram spans its bin and rests on those added to its column before it. `normalize(norm)` and `range(min, max)` apply to every histogram, each normalised by its own total, and each histogram keeps a record of its own sample's total and exclusions. A stacked density histogram is therefore a stack of densities that each enclose unit area, which differs from Matplotlib's `stacked=True, density=True`, which scales the whole stack to unit area. `horizontal()` lays every histogram flat, with the bins along y. `ids()`, `last()` and `get(index)` return the histograms for styling, and `error()` returns the reason when the specification cannot be applied. A datatip on a member of a grouped histogram reports the count and density of the whole bin, although the member draws only its share of it. +`hists(bins)` draws several samples as stacked or grouped histograms, the counterpart of [`bars`](#bar-charts), and returns a `HistsMut`. `series(label, sample)` adds a histogram as a column of its own, and `stacked(column, label, sample)` adds one to the stacked column of that name. Every sample is binned on one shared set of edges, so that the samples compare bin by bin: explicit edges are used as given, and any other specification is applied to all the samples pooled together. The histograms are placed by the same code that places the series of `bars`: grouped histograms split each bin into equal parts in the order they were added, the first at the low end, and each stacked histogram spans its bin and rests on those added to its column before it. `normalize(norm)` and `range(min, max)` apply to every histogram, and how each is normalised depends on how it is placed, as [normalising stacked and side-by-side histograms](#normalising-stacked-and-side-by-side-histograms) describes. `horizontal()` lays every histogram flat, with the bins along y. `ids()`, `last()` and `get(index)` return the histograms for styling, and `error()` returns the reason when the specification cannot be applied. A datatip on a member of a grouped histogram reports the count and density of the whole bin, although the member draws only its share of it. ```rust fig.axes(0, 0) @@ -343,6 +343,33 @@ fig.axes(0, 2) The gallery's [histograms](../gallery/histogram.md) overlay two samples as step outlines of their densities with fitted normal densities, and stack the same samples as counts; its [two-dimensional histogram and density contour](../gallery/histogram_2d.md) draws a correlated sample as cells on a logarithmic colour scale and as filled contours of its density. +#### Normalising stacked and side-by-side histograms + +The histograms of a stacked column are normalised together, by the pooled total of every sample in the column, as Matplotlib's `hist(stacked=True, density=True)` and seaborn's `histplot(multiple="stack")` with its default `common_norm=True` normalise them. The top of the stack is therefore the normalised histogram of the pooled sample, and each layer is its member's share of it: a stack of densities encloses unit area, a stack of probabilities sums to 1, a stack of percentages sums to 100, and a cumulative stack ends at the pooled total, or at 1 for a CDF. Counts and count densities are not divided by a total, so they are the same however the histograms are normalised. + +Histograms that stand side by side, each in a column of its own, are normalised each by its own total, because they are placed side by side to compare their shapes: each density histogram encloses unit area, whatever the size of its sample. + +`common_norm(true)` normalises every histogram of the helper by the pooled total of all its samples, whether they are stacked or side by side, so that side-by-side histograms show how the pooled sample divides among them. `common_norm(false)` normalises every histogram by its own total, so that each layer of a stack is a distribution of its own and the stack shows the sum of their shapes. The setter gives the same result whether it is called before or after the histograms are added. + +```rust +// The outline of the stack is the density of the pooled sample. +fig.axes(0, 0) + .hists(Bins::Count(20)) + .normalize(HistNorm::Density) + .stacked("all", "Control", &control) + .stacked("all", "Treated", &treated); + +// Each layer encloses unit area of its own, so the stack encloses an area of 2. +fig.axes(0, 1) + .hists(Bins::Count(20)) + .normalize(HistNorm::Density) + .common_norm(false) + .stacked("all", "Control", &control) + .stacked("all", "Treated", &treated); +``` + +Each histogram keeps a record of the total its values were normalised by, which for a histogram normalised with others is their pooled total, and of the samples its own sample lost because they were not finite or lay outside the range. From that record, a datatip on a layer of a stack reports the count of the member's own samples in the bin, and as its density the member's contribution to the density of the stack, so that the densities of the layers of a bin add up to the density of the stack there. + ### Contours `contour(x, y, z)` draws isolines of the field `z`, `contourf(x, y, z)` fills the bands between levels, and `contour3(x, y, z)` draws each isoline at the height of its level in a three-dimensional axes. Each isoline and band is coloured by its level through the [colour scale](colour.md) of the axes. A contour without levels of its own takes them from its scale: ten levels at round values through a continuous scale, and the band edges of a discrete scale, as [discrete scales and filled contours](colour.md#discrete-scales-and-filled-contours) shows. diff --git a/docs/reference/equivalent-functions.md b/docs/reference/equivalent-functions.md index 25af35b..ce469c6 100644 --- a/docs/reference/equivalent-functions.md +++ b/docs/reference/equivalent-functions.md @@ -141,10 +141,14 @@ Bars that compare series side by side or stack them are separate series placed b | Histogram lying flat | `histogram` with `'Orientation'` set to `horizontal` | `hist` with `orientation="horizontal"` | `Histogram` with `y=` the sample | `hist` with `orientation(Orientation::Horizontal)` | | Counts binned beforehand drawn as a histogram | `histogram('BinEdges', edges, 'BinCounts', counts)` | `stairs(counts, edges, fill=True)`, or `bar` of the bin centres | `Bar` of the bin centres with `width=` the bin widths | `hist_counts(edges, counts)` | | Several histograms stacked or side by side on shared bins | no equivalent; one `histogram` per sample on the same `BinEdges` | `hist` with a list of samples, and `stacked=True` to stack them | `Histogram` traces with `barmode="stack"` or `"group"` | `hists` with one `stacked` or `series` per sample | +| Stacked histograms whose stack is normalised as a whole | no equivalent | `hist` with a list of samples, `stacked=True` and `density=True`; in seaborn, `histplot` with `multiple="stack"` and `stat="density"` | no equivalent; `histnorm` normalises each trace by its own total | `hists(bins).stacked(…).normalize(HistNorm::Density)` | +| Several histograms normalised by the total of all their samples, or each by its own | no equivalent | in seaborn, `histplot` with `common_norm=True` or `common_norm=False` | `histnorm` on each trace, which normalises it by its own total | `hists` with `common_norm(true)` or `common_norm(false)` | | Two-dimensional histogram drawn as tiles | `histogram2` with `DisplayStyle` set to `tile` | `hist2d` | `Histogram2d` | `hist2d` | | Two-dimensional histogram without its sparse bins | no equivalent | `hist2d` with `cmin=` | no equivalent | `hist2d` with `min_count` | | Contours of the density of pairs of samples | no equivalent | no equivalent; `contour` of the counts of `numpy.histogram2d` | `Histogram2dContour` | `density_contour` | +The members of a stacked column are normalised by the pooled total of the column, as Matplotlib and seaborn normalise a stack, and members side by side are normalised each by its own total, as [normalising stacked and side-by-side histograms](../guides/getting-started.md#normalising-stacked-and-side-by-side-histograms) describes; `common_norm` overrides either, as seaborn's argument of that name does. + IronLAB bins every histogram when the figure is built and stores the binned values with a record of how they were binned, rather than the sample, so a datatip reports the count and the density of each bin but the viewer cannot rebin, as [histograms](../guides/getting-started.md#histograms) describes. The binning follows `numpy.histogram`, and the normalisations take MATLAB's names. The [histograms](../gallery/histogram.md) and [two-dimensional histogram and density contour](../gallery/histogram_2d.md) gallery entries show overlaid, stacked and two-dimensional histograms and a density contour. ## Axes diff --git a/docs/reference/figure-schema.md b/docs/reference/figure-schema.md index 89c1d7d..b07f936 100644 --- a/docs/reference/figure-schema.md +++ b/docs/reference/figure-schema.md @@ -668,10 +668,12 @@ Bars and stairs carry a record of one dimension, `Binning`: | `edges` | The `NodeId` of the data array of the edges that the specification resolved to: one more than the bars or the values of the stairs, finite and strictly increasing. The record of stairs refers to the stairs' own edges; the record of bars refers to an array of its own. | | `range` | The range of sample values that was binned, as `[low, high]`, or `null` when the range of the samples was used. A range must be finite and increasing. | | `normalization` | How the counts of the bins were normalised into the values drawn, as described below. | -| `total` | The number of samples counted into the bins, a finite number from 0. | +| `total` | The total the values were normalised by, a finite number from 0: the number of samples counted into the bins or, for one of several histograms normalised together, as the members of a stack are, the number counted into the bins of all of them. A datatip divides by it to recover the histogram's own count. | | `excluded_non_finite` | The number of samples left out because they were not finite, a whole number from 0. | | `excluded_out_of_range` | The number of samples left out because they lay outside the range, a whole number from 0. | +The `total` is the total the values were normalised by, so that the count recovered from a value is the histogram's own count, whichever total divided it: a member of a stacked histogram, normalised by the pooled total of its stack as [histograms](../guides/getting-started.md#normalising-stacked-and-side-by-side-histograms) describes, records that pooled total, while its `excluded_non_finite` and `excluded_out_of_range` count only the samples its own sample lost. + Cells carry a record of two dimensions, `Binning2d`, whose `x_bins` and `y_bins` specify the bins along x and along y, whose `x_edges` and `y_edges` are the `NodeId`s of the resolved edges along x and along y in the form of `edges`, one more than the columns and one more than the rows (the record of cells on explicit edges refers to the cells' own column and row edges), whose `x_range` and `y_range` are the ranges along x and along y in the form of `range`, and whose `normalization`, `total`, `excluded_non_finite` and `excluded_out_of_range` are those of a record of one dimension. Its `min_count` is the count below which a bin was left empty, a finite number from 0, or `null` when every bin was drawn. The bins take one of four forms: