From a9048413e9ec2d71691a0faad41a95e01e8fb037 Mon Sep 17 00:00:00 2001 From: Edouard Mathieu Date: Wed, 29 Jul 2026 22:48:10 +0200 Subject: [PATCH 01/15] =?UTF-8?q?=F0=9F=93=8A=20Add=20long-run=20UK=20migr?= =?UTF-8?q?ation=20data?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From 7c0ca71a57db046f3cc45cf4d5dbc641c7d4f5cd Mon Sep 17 00:00:00 2001 From: Edouard Mathieu Date: Mon, 3 Aug 2026 15:17:41 +0200 Subject: [PATCH 02/15] =?UTF-8?q?=F0=9F=8E=89=F0=9F=A4=96=20Long-run=20mig?= =?UTF-8?q?ration=20flows=20for=20the=20United=20Kingdom,=201853-2025?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six indicators (immigration, emigration, net migration; each in people and as a share of the population) for the United Kingdom, combining the Bank of England's 'A millennium of macroeconomic data' (to 2011) with the ONS's administrative estimates of long-term international migration (from 2012). Co-Authored-By: Claude Fable 5 --- dag/migration.yml | 8 + .../2026-08-03/uk_migration.meta.yml | 156 ++++++++++++++++++ .../migration/2026-08-03/uk_migration.py | 109 ++++++++++++ .../migration/2026-08-03/uk_migration.py | 19 +++ .../migration/2026-08-03/uk_migration.py | 122 ++++++++++++++ .../millennium_of_macroeconomic_data.xlsx.dvc | 35 ++++ ...long_term_international_migration.xlsx.dvc | 31 ++++ 7 files changed, 480 insertions(+) create mode 100644 etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml create mode 100644 etl/steps/data/garden/migration/2026-08-03/uk_migration.py create mode 100644 etl/steps/data/grapher/migration/2026-08-03/uk_migration.py create mode 100644 etl/steps/data/meadow/migration/2026-08-03/uk_migration.py create mode 100644 snapshots/migration/2026-07-29/millennium_of_macroeconomic_data.xlsx.dvc create mode 100644 snapshots/migration/2026-08-03/long_term_international_migration.xlsx.dvc diff --git a/dag/migration.yml b/dag/migration.yml index 3a15d358221..37e859256ef 100644 --- a/dag/migration.yml +++ b/dag/migration.yml @@ -89,6 +89,14 @@ steps: - data://garden/demography/2024-07-15/population + # Long-run migration flows for the United Kingdom (Bank of England + ONS) + data://grapher/migration/2026-08-03/uk_migration: + - data://garden/migration/2026-08-03/uk_migration: + - data://meadow/migration/2026-08-03/uk_migration: + - snapshot://migration/2026-07-29/millennium_of_macroeconomic_data.xlsx + - snapshot://migration/2026-08-03/long_term_international_migration.xlsx + - data://garden/demography/2024-07-15/population + # UN DESA migration flows S3 JSON export (2024) export://s3/un_migration/latest/migration_stock_flows_json: - data://garden/demography/2024-07-15/population diff --git a/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml b/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml new file mode 100644 index 00000000000..12aa608686e --- /dev/null +++ b/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml @@ -0,0 +1,156 @@ +# Learn more about the available fields: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +definitions: + common: + processing_level: major + presentation: + topic_tags: + - Migration + + # The construction of the series: five sources, five periods. Shared by all six indicators. + uk_sources: |- + Our World in Data maintains this long-run series on British migration by combining five sources. The Bank of England joined the first four into a continuous series in its research dataset "A millennium of macroeconomic data". We added the fifth from the Office for National Statistics. + + - **1853–1963 — ship passenger counts.** Passengers traveling by ship between the United Kingdom and ports outside Europe and the Mediterranean. + - The UK Passenger Acts required ship captains to file lists of their passengers, and officials counted everyone on board, whatever their nationality and whatever the reason for their journey — tourists and business travelers as well as migrants. + - These statistics miss people moving to or from other European countries, and anyone traveling by air. + - Early records of arrivals are less complete than records of departures, and the Bank of England filled a few missing years of arrivals (1856, 1857, 1862, and 1868) by carrying over or averaging neighboring years. + - Before 1922, the United Kingdom included all of Ireland. From 1923 onward, the data covers the United Kingdom in its current borders — Great Britain and Northern Ireland. + - There is no data for 1939 to 1946, during and just after the Second World War. + - **1964–1974 — total migration counts.** Migrants — people intending to change their country of residence for at least a year — moving between the United Kingdom and all countries, including the rest of Europe. + - **1975–1990 — International Passenger Survey.** A sample survey of passengers at UK air and sea ports, run by the Office for National Statistics, using the same definition of a migrant. + - **1991–2011 — Long-Term International Migration estimates.** The same survey, adjusted by the Office for National Statistics to also count asylum seekers, people who end up staying longer or shorter than they first intended, and migration to and from Northern Ireland. + - **2012–2025 — estimates based on administrative records.** The Office for National Statistics' current estimates, built mainly from visa, border, and registration records rather than surveys. + - These replaced the survey-based estimates as the official measure of UK migration, and are considerably higher: the survey undercounted people moving in both directions. + - The figures for recent years are provisional. The Office for National Statistics classifies its administrative estimates as official statistics in development, and revises them — sometimes substantially — as more data becomes available. + - The source also changed how it counts British nationals in mid-2021: its earlier figures for them still drew on survey data, adjusted against the census. + + uk_joins: |- + Because these sources measure different things, the series shifts level where they join. In 1964, the older passenger count gives 245,000 arrivals while the newer migrant count gives 211,000. In 1991, the unadjusted survey gives 255,000 immigrants while the adjusted estimate gives 329,000. In 2012, the survey-based estimate gives 498,000 immigrants while the administrative estimate gives 643,000. + + uk_net_sign: |- + Positive values mean more people arrived than left; negative values mean more people left than arrived. + + uk_share_denominator: |- + This indicator divides the number of people migrating in a year by the population of the United Kingdom in the same year. Up to 2011, the Bank of England makes this calculation, using its own population estimates, which match the borders of the time — including all of Ireland before 1922. From 2012 onward, we divide the Office for National Statistics' migration estimates by [our own population estimates](https://ourworldindata.org/population-sources) for the United Kingdom. + + uk_processing_flows: |- + We combined two sources into one series: + + - Up to 2011, the data comes from the Bank of England's research dataset "A millennium of macroeconomic data" (worksheet A19a), which splices historical passenger statistics and survey-based estimates into a continuous series. We converted the figures from thousands of people to people. + - From 2012 onward, the data comes from the Office for National Statistics' administrative estimates of long-term international migration. We use the year-ending-December estimates, which cover calendar years. + + We chose 2012 as the junction because it is the first year covered by the administrative estimates, which the Office for National Statistics considers more accurate than the survey-based method used at the end of the Bank of England dataset. That dataset continues to 2016 using the older survey-based estimates; we discarded those years in favor of the administrative figures. + + uk_processing_shares: |- + We combined two sources into one series: + + - Up to 2011, the share comes from the Bank of England's research dataset "A millennium of macroeconomic data" (worksheet A19a), which calculates it from its own migration and population series. + - From 2012 onward, we calculated the share by dividing the Office for National Statistics' administrative estimates of long-term international migration (year-ending-December, covering calendar years) by [our long-run population estimates](https://ourworldindata.org/population-sources) for the United Kingdom. + + We chose 2012 as the junction because it is the first year covered by the administrative estimates, which the Office for National Statistics considers more accurate than the survey-based method used at the end of the Bank of England dataset. That dataset continues to 2016 using the older survey-based estimates; we discarded those years in favor of the administrative figures. + +dataset: + update_period_days: 365 + owners: + - Edouard Mathieu + +tables: + uk_migration_flows: + title: Migration flows, United Kingdom + variables: + immigration: + title: People immigrating to the United Kingdom + unit: people + short_unit: "" + description_short: Number of people moving to the United Kingdom each year. + description_key: |- + {definitions.uk_sources} + + {definitions.uk_joins} + description_processing: "{definitions.uk_processing_flows}" + display: + numDecimalPlaces: 0 + + emigration: + title: People emigrating from the United Kingdom + unit: people + short_unit: "" + description_short: Number of people leaving the United Kingdom each year. + description_key: |- + {definitions.uk_sources} + + {definitions.uk_joins} + description_processing: "{definitions.uk_processing_flows}" + display: + numDecimalPlaces: 0 + + net_migration: + title: Net migration to the United Kingdom + unit: people + short_unit: "" + description_short: Number of people moving to the United Kingdom each year, minus the number leaving. + description_key: |- + {definitions.uk_sources} + + {definitions.uk_joins} + description_processing: "{definitions.uk_processing_flows}" + presentation: + grapher_config: + note: "{definitions.uk_net_sign}" + display: + numDecimalPlaces: 0 + + immigration_share_of_population: + title: People immigrating to the United Kingdom as a share of the population + unit: "%" + short_unit: "%" + description_short: Number of people moving to the United Kingdom each year, as a share of the population. + description_key: |- + {definitions.uk_share_denominator} + + {definitions.uk_sources} + + {definitions.uk_joins} + description_processing: "{definitions.uk_processing_shares}" + display: + roundingMode: significantFigures + numSignificantFigures: 2 + numDecimalPlaces: 2 + + emigration_share_of_population: + title: People emigrating from the United Kingdom as a share of the population + unit: "%" + short_unit: "%" + description_short: Number of people leaving the United Kingdom each year, as a share of the population. + description_key: |- + {definitions.uk_share_denominator} + + {definitions.uk_sources} + + {definitions.uk_joins} + description_processing: "{definitions.uk_processing_shares}" + display: + roundingMode: significantFigures + numSignificantFigures: 2 + numDecimalPlaces: 2 + + net_migration_share_of_population: + title: Net migration to the United Kingdom as a share of the population + unit: "%" + short_unit: "%" + description_short: Number of people moving to the United Kingdom each year, minus the number leaving, as a share of the population. + description_key: |- + {definitions.uk_share_denominator} + + {definitions.uk_sources} + + {definitions.uk_joins} + description_processing: "{definitions.uk_processing_shares}" + presentation: + grapher_config: + note: "{definitions.uk_net_sign}" + display: + roundingMode: significantFigures + numSignificantFigures: 2 + numDecimalPlaces: 2 diff --git a/etl/steps/data/garden/migration/2026-08-03/uk_migration.py b/etl/steps/data/garden/migration/2026-08-03/uk_migration.py new file mode 100644 index 00000000000..afa8c4d0976 --- /dev/null +++ b/etl/steps/data/garden/migration/2026-08-03/uk_migration.py @@ -0,0 +1,109 @@ +"""Long-run migration flows for the United Kingdom, 1853-2025. + +This step builds an OWID-maintained series by joining two sources at 2012: + +- Up to 2011: the Bank of England's spliced historical series (sheet A19a of "A millennium of + macroeconomic data"), which itself combines passenger statistics, Mitchell's historical + statistics, and the ONS's survey-based estimates. Converted from thousands to people. + The Bank's own share-of-population figures are kept, since their denominator matches the UK's + borders of the time (including all of Ireland before 1922). +- From 2012: the ONS's administrative-data-based estimates of long-term international migration + (year-ending-December values, which cover calendar years). The ONS considers these its best + measure from 2012 onward; the Bank's 2012-2016 values came from the older survey-based method, + which undercounted both immigration and emigration, and are discarded. Shares of population for + this period are computed from OWID's long-run population estimates. +""" + +from owid.catalog import Table +from owid.catalog import processing as pr + +from etl.helpers import PathFinder + +paths = PathFinder(__file__) + +# The year from which the ONS administrative estimates replace the Bank of England series. +JOIN_YEAR = 2012 + +# Years whose net migration figure was revised by the ONS after the 2011 census. In these years the +# published net migration ("headline") figure does not equal immigration minus emigration, because +# the components were not revised alongside the balance. +CENSUS_REVISED_YEARS = set(range(2001, JOIN_YEAR)) + +FLOWS = ["immigration", "emigration", "net_migration"] + + +def sanity_check_inputs(tb_boe: Table, tb_ons: Table) -> None: + # The two sources must overlap at the join, and the administrative estimates must sit above the + # survey-based ones there (the survey undercounted both directions). + overlap = set(tb_boe["year"]) & set(tb_ons["year"]) + assert overlap == {2012, 2013, 2014, 2015, 2016}, f"Unexpected overlap between sources: {sorted(overlap)}" + boe_2012 = tb_boe.loc[tb_boe["year"] == JOIN_YEAR].iloc[0] + ons_2012 = tb_ons.loc[tb_ons["year"] == JOIN_YEAR].iloc[0] + assert 1 < ons_2012["immigration"] / (boe_2012["immigration"] * 1000) < 1.5, "Unexpected gap at the 2012 join." + + +def sanity_check_outputs(tb: Table) -> None: + assert not tb["year"].duplicated().any(), "Duplicate years." + t = tb.set_index("year") + + # Net migration must equal immigration minus emigration, except in the census-revised years. + residual = (t["net_migration"] - (t["immigration"] - t["emigration"])).dropna() + inconsistent = set(residual[residual.abs() > 1000].index) + assert inconsistent == CENSUS_REVISED_YEARS, ( + f"Net migration differs from immigration minus emigration outside 2001-2011: {sorted(inconsistent)}" + ) + + assert (t["immigration"].dropna() > 0).all() and (t["emigration"].dropna() > 0).all() + assert t[FLOWS].abs().max().max() < 3_000_000, "Flow value outside the plausible range." + for col in FLOWS: + share = t[f"{col}_share_of_population"].dropna() + assert share.abs().max() < 2.5, f"{col} share outside the plausible range." + # Every year with a flow value must also have its share (the Bank computes shares for all + # its years; we compute them for all ONS years) — except 1853-1854, which have emigration + # but no population share in the source. + missing_share = set(t[col].dropna().index) - set(share.index) + assert missing_share <= {1853, 1854}, f"Years with {col} but no share: {sorted(missing_share)}" + + +def run() -> None: + # + # Load inputs. + # + ds_meadow = paths.load_dataset("uk_migration") + tb_boe = ds_meadow.read("bank_of_england_flows") + tb_ons = ds_meadow.read("ons_flows") + + sanity_check_inputs(tb_boe, tb_ons) + + # + # Process data. + # + # Bank of England period: keep years before the join, convert flows from thousands to people. + # The Bank's share-of-population figures are kept as published; its population column only + # served as their denominator and is dropped. + tb_boe = tb_boe[tb_boe["year"] < JOIN_YEAR].copy() + for col in FLOWS: + tb_boe[col] = tb_boe[col] * 1000 + tb_boe = tb_boe.drop(columns=["population"]) + tb_boe["country"] = "United Kingdom" + + # ONS period: already in people. Compute shares of population from OWID's population estimates + # (the UK's borders have not changed in this period, so there is no boundary mismatch). + tb_ons["country"] = "United Kingdom" + tb_ons = paths.regions.add_population(tb_ons) + assert tb_ons["population"].notna().all(), "Missing population for an ONS-period year." + for col in FLOWS: + tb_ons[f"{col}_share_of_population"] = tb_ons[col] / tb_ons["population"] * 100 + tb_ons = tb_ons.drop(columns=["population"]) + + tb = pr.concat([tb_boe, tb_ons], ignore_index=True).sort_values("year") + + sanity_check_outputs(tb) + + tb = tb.format(["country", "year"], short_name="uk_migration_flows") + + # + # Save outputs. + # + ds_garden = paths.create_dataset(tables=[tb], default_metadata=ds_meadow.metadata) + ds_garden.save() diff --git a/etl/steps/data/grapher/migration/2026-08-03/uk_migration.py b/etl/steps/data/grapher/migration/2026-08-03/uk_migration.py new file mode 100644 index 00000000000..31fcc3c095b --- /dev/null +++ b/etl/steps/data/grapher/migration/2026-08-03/uk_migration.py @@ -0,0 +1,19 @@ +"""Load the garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder + +paths = PathFinder(__file__) + + +def run() -> None: + # + # Load inputs. + # + ds_garden = paths.load_dataset("uk_migration") + tables = [ds_garden.read(name, reset_index=False) for name in ds_garden.table_names] + + # + # Save outputs. + # + ds_grapher = paths.create_dataset(tables=tables, default_metadata=ds_garden.metadata) + ds_grapher.save() diff --git a/etl/steps/data/meadow/migration/2026-08-03/uk_migration.py b/etl/steps/data/meadow/migration/2026-08-03/uk_migration.py new file mode 100644 index 00000000000..54c2d794be3 --- /dev/null +++ b/etl/steps/data/meadow/migration/2026-08-03/uk_migration.py @@ -0,0 +1,122 @@ +"""Load the two sources of long-run UK migration flows and create a meadow dataset. + +- Bank of England, "A millennium of macroeconomic data", sheet A19a: the Bank's spliced annual + series of immigration (1855-2016), emigration (1853-2016) and net migration (1855-2016) for the + United Kingdom, in thousands, with the same flows as a percentage of the UK population. + There is no data for 1939-1946. +- Office for National Statistics, "Long-term international migration, provisional": estimates of + immigration, emigration and net migration built from administrative data, covering rolling + 12-month periods from the year ending June 2012 to the year ending December 2025. We keep the + year-ending-December periods, which correspond to calendar years. +""" + +import pandas as pd +from owid.catalog import Table +from owid.catalog import processing as pr + +from etl.helpers import PathFinder +from etl.snapshot import Snapshot + +paths = PathFinder(__file__) + +# Columns of sheet A19a in the Bank of England workbook (0-indexed). The spliced headline series +# sit in columns B-D, the UK population used as denominator in F, and the shares in H-J. +A19A_COLUMNS = { + 0: "year", + 1: "immigration", + 2: "emigration", + 3: "net_migration", + 5: "population", + 7: "immigration_share_of_population", + 8: "emigration_share_of_population", + 9: "net_migration_share_of_population", +} + + +def parse_bank_of_england(snap: Snapshot) -> Table: + """Parse sheet A19a: keep the spliced headline columns, drop non-data rows.""" + tb = snap.read_excel(sheet_name="A19a. UK Migration flows", header=None, skiprows=7) + tb = tb[list(A19A_COLUMNS)] + # Renaming integer column labels scrambles column-level metadata, so restore it explicitly. + metadata = {name: tb[index].metadata.copy() for index, name in A19A_COLUMNS.items()} + tb = tb.rename(columns=A19A_COLUMNS) + for name, meta in metadata.items(): + tb[name].metadata = meta + + tb = tb[pd.to_numeric(tb["year"], errors="coerce").notna()] + tb["year"] = tb["year"].astype(int) + for col in tb.columns.drop("year"): + tb[col] = pr.to_numeric(tb[col], errors="coerce") + + # Keep only years with at least one flow value: this drops 1850-1852 (population only) but + # keeps 1853-1854, which have emigration but no immigration yet. + tb = tb.dropna(subset=["immigration", "emigration", "net_migration"], how="all") + + assert tb["year"].min() == 1853 and tb["year"].max() == 2016, "Expected coverage 1853-2016." + assert set(range(1939, 1947)).isdisjoint(tb["year"]), "Expected no data during 1939-1946." + assert tb["population"].notna().all(), "Missing population in a year with flow data." + # Spot-check values against the source (in thousands): the first emigration figure, and the + # 1913 all-time emigration peak. + assert tb.loc[tb["year"] == 1853, "emigration"].item() == 330 + assert tb.loc[tb["year"] == 1913, "emigration"].item() == 702 + return tb + + +def parse_ons(snap: Snapshot) -> Table: + """Parse Table 1 of the ONS spreadsheet: keep all-nationalities flows for calendar years.""" + tb = snap.read_excel(sheet_name="1", header=None, skiprows=6) + tb = tb[[0, 1, 2]] + metadata = {index: tb[index].metadata.copy() for index in tb.columns} + tb = tb.rename(columns={0: "flow", 1: "period", 2: "value"}) + for index, name in zip(metadata, ["flow", "period", "value"]): + tb[name].metadata = metadata[index] + + tb = tb.dropna(subset=["flow", "period"]) + # Keep year-ending-December periods, which correspond to calendar years. Recent periods carry + # "P" (provisional) and "R" (revised) flags after the period label. + tb = tb[tb["period"].astype(str).str.contains("YE Dec")].copy() + tb["year"] = 2000 + tb["period"].astype(str).str.extract(r"YE Dec (\d\d)")[0].astype(int) + tb["value"] = pr.to_numeric(tb["value"], errors="raise") + + flows = {"Immigration": "immigration", "Emigration": "emigration", "Net migration": "net_migration"} + observed = set(tb["flow"]) + assert observed == set(flows), f"Unexpected flow labels in the ONS table: {observed}" + tb["flow"] = tb["flow"].map(flows) + + tb = tb.pivot(index="year", columns="flow", values="value").reset_index() + tb = tb[["year", "immigration", "emigration", "net_migration"]] + + # The series starts in 2012; the last year will move forward with future ONS releases. + assert tb["year"].min() == 2012 and tb["year"].max() >= 2025, "Expected coverage from 2012 to at least 2025." + assert tb.notna().all().all(), "Missing values in the ONS series." + residual = tb["net_migration"] - (tb["immigration"] - tb["emigration"]) + assert residual.abs().max() <= 1000, "Net migration does not match immigration minus emigration." + # Loose spot-check on a settled year (exact values shift slightly with ONS revisions). + assert 1_300_000 < tb.loc[tb["year"] == 2023, "immigration"].item() < 1_600_000 + return tb + + +def run() -> None: + # + # Load inputs. + # + snap_boe = paths.load_snapshot("millennium_of_macroeconomic_data.xlsx") + snap_ons = paths.load_snapshot("long_term_international_migration.xlsx") + + # + # Process data. + # + tb_boe = parse_bank_of_england(snap_boe) + tb_ons = parse_ons(snap_ons) + + # + # Save outputs. + # + ds_meadow = paths.create_dataset( + tables=[ + tb_boe.format(["year"], short_name="bank_of_england_flows"), + tb_ons.format(["year"], short_name="ons_flows"), + ], + default_metadata=snap_boe.metadata, + ) + ds_meadow.save() diff --git a/snapshots/migration/2026-07-29/millennium_of_macroeconomic_data.xlsx.dvc b/snapshots/migration/2026-07-29/millennium_of_macroeconomic_data.xlsx.dvc new file mode 100644 index 00000000000..146d7c20fd2 --- /dev/null +++ b/snapshots/migration/2026-07-29/millennium_of_macroeconomic_data.xlsx.dvc @@ -0,0 +1,35 @@ +# Learn more at: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +meta: + origin: + # Data product / Snapshot + title: A millennium of macroeconomic data for the UK + description: |- + A research dataset compiled by Bank of England staff, bringing together historical macroeconomic and financial statistics for the United Kingdom — national accounts, population, migration, money and credit, fiscal data, interest rates, exchange rates, trade, and labor market data. Some series reach back to the 13th century, with a few benchmark estimates for 1086. + + The dataset splices together estimates from the academic literature, historical administrative records, and official statistics into continuous long-run series, and documents the underlying sources and assumptions in each worksheet. The Bank describes it as a "best endeavours" research resource rather than official Bank of England data or National Statistics. + + Version 3.1 was finalized on 30 April 2017 and extended in September and October 2017, with corrections to the historical data made in August 2018 and August 2024. It covers data up to 2016. + date_published: "2017-04-30" + version_producer: Version 3.1 + + # Citation + producer: Bank of England + citation_full: |- + Thomas, R and Dimsdale, N (2017) "A Millennium of UK Data", Bank of England OBRA dataset, http://www.bankofengland.co.uk/research/Pages/onebank/threecenturies.aspx + + The Bank of England asks users to also cite the original sources of individual series. The migration series used by Our World in Data (worksheet A19a) draws on: Mitchell, B. R. (1988). British Historical Statistics. Cambridge University Press; and the Office for National Statistics (International Passenger Survey and Long-Term International Migration estimates). + + # Files + url_main: https://www.bankofengland.co.uk/statistics/research-datasets + url_download: https://www.bankofengland.co.uk/-/media/boe/files/statistics/research-datasets/a-millennium-of-macroeconomic-data-for-the-uk.xlsx + date_accessed: "2026-07-29" + + # License + license: + name: © Bank of England + url: https://www.bankofengland.co.uk/legal +outs: + - md5: de399fab8c99a7475e1a773e7437f004 + size: 27533690 + path: millennium_of_macroeconomic_data.xlsx diff --git a/snapshots/migration/2026-08-03/long_term_international_migration.xlsx.dvc b/snapshots/migration/2026-08-03/long_term_international_migration.xlsx.dvc new file mode 100644 index 00000000000..3c7d599ca66 --- /dev/null +++ b/snapshots/migration/2026-08-03/long_term_international_migration.xlsx.dvc @@ -0,0 +1,31 @@ +# Learn more at: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +meta: + origin: + # Data product / Snapshot + title: Long-term international migration, provisional + description: |- + Estimates of long-term international immigration, emigration, and net migration for the United Kingdom, published by the Office for National Statistics. A long-term migrant is someone who changes their country of usual residence for at least 12 months. + + The ONS builds these estimates mainly from administrative records — visa and border data from the Home Office, and National Insurance registrations — rather than surveys. They replaced the survey-based estimates as the official measure of UK migration, and cover rolling 12-month periods from the year ending June 2012 onward. The ONS classifies them as official statistics in development: recent periods are provisional and revised as more data becomes available. + date_published: "2026-05-21" + version_producer: year ending December 2025 + + # Citation + producer: UK Office for National Statistics + citation_full: |- + Office for National Statistics (2026). Long-term international migration, provisional: year ending December 2025. Released 21 May 2026. + + # Files + url_main: https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/internationalmigration/datasets/longterminternationalimmigrationemigrationandnetmigrationflowsprovisional + url_download: https://www.ons.gov.uk/file?uri=/peoplepopulationandcommunity/populationandmigration/internationalmigration/datasets/longterminternationalimmigrationemigrationandnetmigrationflowsprovisional/yearendingdecember2025/may2026publicationspreadsheet.xlsx + date_accessed: "2026-08-03" + + # License + license: + name: Open Government Licence v3.0 + url: https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/ +outs: + - md5: 39ff4f419ae283234fec147d7de81668 + size: 168193 + path: long_term_international_migration.xlsx From fe3472be84a73c3c039cf522537293ec7748b7cc Mon Sep 17 00:00:00 2001 From: Edouard Mathieu Date: Mon, 3 Aug 2026 15:23:27 +0200 Subject: [PATCH 03/15] =?UTF-8?q?=F0=9F=90=9B=F0=9F=A4=96=20Set=20the=20gr?= =?UTF-8?q?apher=20dataset=20title=20(was=20inheriting=20the=20BoE=20workb?= =?UTF-8?q?ook=20name)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml b/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml index 12aa608686e..381fa3e9f87 100644 --- a/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml +++ b/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml @@ -51,6 +51,7 @@ definitions: We chose 2012 as the junction because it is the first year covered by the administrative estimates, which the Office for National Statistics considers more accurate than the survey-based method used at the end of the Bank of England dataset. That dataset continues to 2016 using the older survey-based estimates; we discarded those years in favor of the administrative figures. dataset: + title: Long-run migration flows for the United Kingdom update_period_days: 365 owners: - Edouard Mathieu From 0738626e32e5d1c2081edc2a637fae1ac96223e2 Mon Sep 17 00:00:00 2001 From: Edouard Mathieu Date: Mon, 3 Aug 2026 15:34:11 +0200 Subject: [PATCH 04/15] =?UTF-8?q?=F0=9F=90=9B=F0=9F=A4=96=20Address=20Code?= =?UTF-8?q?x=20review:=20year=20column=20origin,=20ONS=20license=20URL?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- etl/steps/data/meadow/migration/2026-08-03/uk_migration.py | 2 ++ .../2026-08-03/long_term_international_migration.xlsx.dvc | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/etl/steps/data/meadow/migration/2026-08-03/uk_migration.py b/etl/steps/data/meadow/migration/2026-08-03/uk_migration.py index 54c2d794be3..775246b96f1 100644 --- a/etl/steps/data/meadow/migration/2026-08-03/uk_migration.py +++ b/etl/steps/data/meadow/migration/2026-08-03/uk_migration.py @@ -75,7 +75,9 @@ def parse_ons(snap: Snapshot) -> Table: # Keep year-ending-December periods, which correspond to calendar years. Recent periods carry # "P" (provisional) and "R" (revised) flags after the period label. tb = tb[tb["period"].astype(str).str.contains("YE Dec")].copy() + # .str.extract returns a plain Series, so restore the column metadata afterwards. tb["year"] = 2000 + tb["period"].astype(str).str.extract(r"YE Dec (\d\d)")[0].astype(int) + tb["year"] = tb["year"].copy_metadata(tb["period"]) tb["value"] = pr.to_numeric(tb["value"], errors="raise") flows = {"Immigration": "immigration", "Emigration": "emigration", "Net migration": "net_migration"} diff --git a/snapshots/migration/2026-08-03/long_term_international_migration.xlsx.dvc b/snapshots/migration/2026-08-03/long_term_international_migration.xlsx.dvc index 3c7d599ca66..95cb1ff060f 100644 --- a/snapshots/migration/2026-08-03/long_term_international_migration.xlsx.dvc +++ b/snapshots/migration/2026-08-03/long_term_international_migration.xlsx.dvc @@ -24,7 +24,7 @@ meta: # License license: name: Open Government Licence v3.0 - url: https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/ + url: https://www.ons.gov.uk/help/termsandconditions outs: - md5: 39ff4f419ae283234fec147d7de81668 size: 168193 From 4a677012fff76438196a359fdef8e8ce393848fe Mon Sep 17 00:00:00 2001 From: Edouard Mathieu Date: Mon, 3 Aug 2026 15:46:05 +0200 Subject: [PATCH 05/15] =?UTF-8?q?=F0=9F=92=84=F0=9F=A4=96=20First=20period?= =?UTF-8?q?=20label:=201850s=E2=80=931963?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- .../data/garden/migration/2026-08-03/uk_migration.meta.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml b/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml index 381fa3e9f87..6629ba4f9d4 100644 --- a/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml +++ b/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml @@ -11,7 +11,7 @@ definitions: uk_sources: |- Our World in Data maintains this long-run series on British migration by combining five sources. The Bank of England joined the first four into a continuous series in its research dataset "A millennium of macroeconomic data". We added the fifth from the Office for National Statistics. - - **1853–1963 — ship passenger counts.** Passengers traveling by ship between the United Kingdom and ports outside Europe and the Mediterranean. + - **1850s–1963 — ship passenger counts.** Passengers traveling by ship between the United Kingdom and ports outside Europe and the Mediterranean. - The UK Passenger Acts required ship captains to file lists of their passengers, and officials counted everyone on board, whatever their nationality and whatever the reason for their journey — tourists and business travelers as well as migrants. - These statistics miss people moving to or from other European countries, and anyone traveling by air. - Early records of arrivals are less complete than records of departures, and the Bank of England filled a few missing years of arrivals (1856, 1857, 1862, and 1868) by carrying over or averaging neighboring years. From 8311cf4a710eb833583d3dc146be021c73ccda14 Mon Sep 17 00:00:00 2001 From: Edouard Mathieu Date: Mon, 3 Aug 2026 15:59:48 +0200 Subject: [PATCH 06/15] =?UTF-8?q?=E2=9C=A8=F0=9F=A4=96=20Default=20chart?= =?UTF-8?q?=20config:=20UK=20selected,=20no=20entity=20picker,=20hidden=20?= =?UTF-8?q?entity=20annotation=20and=20series=20labels,=20origin=20URL?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- .../garden/migration/2026-08-03/uk_migration.meta.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml b/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml index 6629ba4f9d4..5ef35ea52da 100644 --- a/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml +++ b/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml @@ -6,6 +6,14 @@ definitions: presentation: topic_tags: - Migration + grapher_config: + selectedEntityNames: + - United Kingdom + addCountryMode: disabled + hideAnnotationFieldsInTitle: + entity: true + hideSeriesLabels: true + originUrl: https://ourworldindata.org/migration # The construction of the series: five sources, five periods. Shared by all six indicators. uk_sources: |- From fb2a36d0b9f795821c9ccd2aa801b874ab6d58ac Mon Sep 17 00:00:00 2001 From: Edouard Mathieu Date: Mon, 3 Aug 2026 16:13:23 +0200 Subject: [PATCH 07/15] =?UTF-8?q?=E2=9C=A8=F0=9F=A4=96=20Restrict=20defaul?= =?UTF-8?q?t=20chart=20type=20to=20line=20chart?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- .../data/garden/migration/2026-08-03/uk_migration.meta.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml b/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml index 5ef35ea52da..6d392383c9d 100644 --- a/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml +++ b/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml @@ -7,6 +7,8 @@ definitions: topic_tags: - Migration grapher_config: + chartTypes: + - LineChart selectedEntityNames: - United Kingdom addCountryMode: disabled From 380d4ad84abedbd8bd53854cf6fe97e69483bb86 Mon Sep 17 00:00:00 2001 From: Edouard Mathieu Date: Mon, 3 Aug 2026 16:17:50 +0200 Subject: [PATCH 08/15] =?UTF-8?q?=E2=9C=A8=F0=9F=A4=96=20Explicitly=20disa?= =?UTF-8?q?ble=20the=20map=20tab?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml b/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml index 6d392383c9d..19966e378ef 100644 --- a/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml +++ b/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml @@ -9,6 +9,7 @@ definitions: grapher_config: chartTypes: - LineChart + hasMapTab: false selectedEntityNames: - United Kingdom addCountryMode: disabled From a19aeb470d5f83676922aea1829deec06722dd66 Mon Sep 17 00:00:00 2001 From: Edouard Mathieu Date: Mon, 3 Aug 2026 16:27:58 +0200 Subject: [PATCH 09/15] =?UTF-8?q?=E2=9C=A8=F0=9F=A4=96=20Vertical=20annota?= =?UTF-8?q?tion=20lines=20at=20the=20method=20changes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- .../garden/migration/2026-08-03/uk_migration.meta.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml b/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml index 19966e378ef..a1d8fa5e4de 100644 --- a/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml +++ b/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml @@ -10,6 +10,15 @@ definitions: chartTypes: - LineChart hasMapTab: false + comparisonLines: + - xEquals: 1855 + label: Ships beyond Europe; no Europe, no air travel. Ireland excluded because UK. Includes travelers + - xEquals: 1923 + label: Same but Ireland now excluded because European + - xEquals: 1964 + label: Long-term migrants, all countries. Travelers now excluded + - xEquals: 1991 + label: Asylum seekers now included selectedEntityNames: - United Kingdom addCountryMode: disabled From 3ed6d8c959787a0bbe4e39b518d3f2b5f50b8efc Mon Sep 17 00:00:00 2001 From: Edouard Mathieu Date: Mon, 3 Aug 2026 16:31:38 +0200 Subject: [PATCH 10/15] =?UTF-8?q?=F0=9F=94=A8=F0=9F=A4=96=20Drop=20the=20v?= =?UTF-8?q?ertical=20annotation=20lines?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- .../garden/migration/2026-08-03/uk_migration.meta.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml b/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml index a1d8fa5e4de..19966e378ef 100644 --- a/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml +++ b/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml @@ -10,15 +10,6 @@ definitions: chartTypes: - LineChart hasMapTab: false - comparisonLines: - - xEquals: 1855 - label: Ships beyond Europe; no Europe, no air travel. Ireland excluded because UK. Includes travelers - - xEquals: 1923 - label: Same but Ireland now excluded because European - - xEquals: 1964 - label: Long-term migrants, all countries. Travelers now excluded - - xEquals: 1991 - label: Asylum seekers now included selectedEntityNames: - United Kingdom addCountryMode: disabled From f681cdc2fbb4ce2e215dc51c8c41502c34618c00 Mon Sep 17 00:00:00 2001 From: Edouard Mathieu Date: Tue, 4 Aug 2026 11:36:16 +0200 Subject: [PATCH 11/15] =?UTF-8?q?=F0=9F=8E=89=F0=9F=A4=96=20Add=20net=20mi?= =?UTF-8?q?gration=20to=20England,=201541-1870?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From the Bank of England workbook's sheet A19c (Wrigley & Schofield's parish-register reconstruction). Sign flipped from the source's net emigration to net migration, in people and as a share of the population. Co-Authored-By: Claude Fable 5 --- .../2026-08-03/uk_migration.meta.yml | 59 ++++++++++++++++++- .../migration/2026-08-03/uk_migration.py | 27 ++++++++- .../migration/2026-08-03/uk_migration.py | 29 +++++++++ 3 files changed, 112 insertions(+), 3 deletions(-) diff --git a/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml b/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml index 19966e378ef..3746633f7b3 100644 --- a/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml +++ b/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml @@ -53,6 +53,24 @@ definitions: We chose 2012 as the junction because it is the first year covered by the administrative estimates, which the Office for National Statistics considers more accurate than the survey-based method used at the end of the Bank of England dataset. That dataset continues to 2016 using the older survey-based estimates; we discarded those years in favor of the administrative figures. + england_method: |- + This data is not a direct count of migrants — no migration records exist for most of this period. Wrigley and Schofield reconstructed England's population from the registers of baptisms and burials kept by Church of England parishes, and estimated net migration as a residual: the part of the change in population that births and deaths cannot explain. + + Because it is a residual, the estimate counts everyone who crossed England's borders — including people who moved to or from Wales, Scotland, or Ireland, not only those who moved to or from other countries. It also absorbs any errors in the registration of baptisms and burials. England is defined here without Monmouthshire, which historical records grouped with Wales. + + Negative values mean more people left England than arrived. The reconstruction shows a net outflow of people in every year from 1541 to 1870. + + The original estimates are totals for five-year periods, which the Bank of England interpolated to annual values. Year-to-year changes are therefore smooth by construction, and the data is best read as showing broad levels and trends rather than what happened in any single year. + + england_producer_note: |- + Net Emigration - England (excluding Monmouthshire). Source: Wrigley and Schofield (1989), interpolated quinquennial estimates. These estimates naturally cover migration flows to and from other parts of the UK as well as to overseas countries. + + england_processing: |- + The source publishes this series as net emigration, where positive values mean more people leaving. We flipped the sign to show net migration, consistent with our United Kingdom indicators, and converted the figures from thousands of people to people. + + england_processing_share: |- + The source publishes this series as net emigration per 1,000 people, where positive values mean more people leaving. We flipped the sign to show net migration, consistent with our United Kingdom indicators, and expressed the rate as a percentage of the population. The population estimates in the denominator come from the same reconstruction by Wrigley and Schofield. + uk_processing_shares: |- We combined two sources into one series: @@ -62,7 +80,7 @@ definitions: We chose 2012 as the junction because it is the first year covered by the administrative estimates, which the Office for National Statistics considers more accurate than the survey-based method used at the end of the Bank of England dataset. That dataset continues to 2016 using the older survey-based estimates; we discarded those years in favor of the administrative figures. dataset: - title: Long-run migration flows for the United Kingdom + title: Long-run migration flows for the United Kingdom and England update_period_days: 365 owners: - Edouard Mathieu @@ -166,3 +184,42 @@ tables: roundingMode: significantFigures numSignificantFigures: 2 numDecimalPlaces: 2 + + england_net_migration: + title: Net migration to England + variables: + net_migration: + title: Net migration to England + unit: people + short_unit: "" + description_short: Estimated number of people moving to England each year, minus the number leaving. Negative values mean more people left than arrived. + description_key: |- + {definitions.england_method} + description_from_producer: "{definitions.england_producer_note}" + description_processing: "{definitions.england_processing}" + processing_level: minor + presentation: + grapher_config: + selectedEntityNames: + - England + display: + numDecimalPlaces: 0 + + net_migration_share_of_population: + title: Net migration to England as a share of the population + unit: "%" + short_unit: "%" + description_short: Estimated number of people moving to England each year, minus the number leaving, as a share of the population. Negative values mean more people left than arrived. + description_key: |- + {definitions.england_method} + description_from_producer: "{definitions.england_producer_note}" + description_processing: "{definitions.england_processing_share}" + processing_level: minor + presentation: + grapher_config: + selectedEntityNames: + - England + display: + roundingMode: significantFigures + numSignificantFigures: 2 + numDecimalPlaces: 2 diff --git a/etl/steps/data/garden/migration/2026-08-03/uk_migration.py b/etl/steps/data/garden/migration/2026-08-03/uk_migration.py index afa8c4d0976..4e11c519813 100644 --- a/etl/steps/data/garden/migration/2026-08-03/uk_migration.py +++ b/etl/steps/data/garden/migration/2026-08-03/uk_migration.py @@ -42,6 +42,14 @@ def sanity_check_inputs(tb_boe: Table, tb_ons: Table) -> None: assert 1 < ons_2012["immigration"] / (boe_2012["immigration"] * 1000) < 1.5, "Unexpected gap at the 2012 join." +def sanity_check_england(tb: Table) -> None: + assert not tb["year"].duplicated().any(), "Duplicate years in the England table." + assert len(tb) == 330, "Expected 330 years of England data." + # Wrigley and Schofield's reconstruction shows a net outflow (negative net migration) in every year. + assert (tb["net_migration"] < 0).all(), "Unexpected net inflow year in the England series." + assert tb["net_migration_share_of_population"].between(-0.5, 0).all(), "Share outside the plausible range." + + def sanity_check_outputs(tb: Table) -> None: assert not tb["year"].duplicated().any(), "Duplicate years." t = tb.set_index("year") @@ -72,6 +80,7 @@ def run() -> None: ds_meadow = paths.load_dataset("uk_migration") tb_boe = ds_meadow.read("bank_of_england_flows") tb_ons = ds_meadow.read("ons_flows") + tb_england = ds_meadow.read("england_net_migration") sanity_check_inputs(tb_boe, tb_ons) @@ -100,10 +109,24 @@ def run() -> None: sanity_check_outputs(tb) - tb = tb.format(["country", "year"], short_name="uk_migration_flows") + # England, 1541-1870: the source publishes net emigration (positive = people leaving). Flip the + # sign to net migration, consistent with the UK indicators, and express the source's per-1,000 + # rate as a percentage share of the population. + tb_england["country"] = "England" + tb_england["net_migration"] = -tb_england["net_emigration"] * 1000 + tb_england["net_migration_share_of_population"] = -tb_england["net_emigration_per_1000"] / 10 + tb_england = tb_england.drop(columns=["net_emigration", "net_emigration_per_1000"]) + + sanity_check_england(tb_england) # # Save outputs. # - ds_garden = paths.create_dataset(tables=[tb], default_metadata=ds_meadow.metadata) + ds_garden = paths.create_dataset( + tables=[ + tb.format(["country", "year"], short_name="uk_migration_flows"), + tb_england.format(["country", "year"], short_name="england_net_migration"), + ], + default_metadata=ds_meadow.metadata, + ) ds_garden.save() diff --git a/etl/steps/data/meadow/migration/2026-08-03/uk_migration.py b/etl/steps/data/meadow/migration/2026-08-03/uk_migration.py index 775246b96f1..81990fc0b94 100644 --- a/etl/steps/data/meadow/migration/2026-08-03/uk_migration.py +++ b/etl/steps/data/meadow/migration/2026-08-03/uk_migration.py @@ -8,6 +8,9 @@ immigration, emigration and net migration built from administrative data, covering rolling 12-month periods from the year ending June 2012 to the year ending December 2025. We keep the year-ending-December periods, which correspond to calendar years. +- Bank of England workbook, sheet A19c: net emigration from England (excluding Monmouthshire), + 1541-1870, in thousands and per 1,000 people, from Wrigley and Schofield's parish-register + reconstruction as interpolated to annual values by the Bank of England. """ import pandas as pd @@ -62,6 +65,30 @@ def parse_bank_of_england(snap: Snapshot) -> Table: return tb +def parse_england(snap: Snapshot) -> Table: + """Parse sheet A19c: net emigration from England, 1541-1870.""" + tb = snap.read_excel(sheet_name="A19c. English Net Migration", header=None, skiprows=7) + tb = tb[[0, 1, 2]] + # Renaming integer column labels scrambles column-level metadata, so restore it explicitly. + columns = {0: "year", 1: "net_emigration", 2: "net_emigration_per_1000"} + metadata = {name: tb[index].metadata.copy() for index, name in columns.items()} + tb = tb.rename(columns=columns) + for name, meta in metadata.items(): + tb[name].metadata = meta + + tb = tb[pd.to_numeric(tb["year"], errors="coerce").notna()] + tb["year"] = tb["year"].astype(int) + for col in ["net_emigration", "net_emigration_per_1000"]: + tb[col] = pr.to_numeric(tb[col], errors="coerce") + tb = tb.dropna(subset=["net_emigration"]) + + assert tb["year"].min() == 1541 and tb["year"].max() == 1870, "Expected coverage 1541-1870." + assert len(tb) == 330, "Expected a continuous annual series (330 years)." + # Spot-check the first value of Wrigley and Schofield's series (in thousands). + assert abs(tb.loc[tb["year"] == 1541, "net_emigration"].item() - 3.3928) < 1e-6 + return tb + + def parse_ons(snap: Snapshot) -> Table: """Parse Table 1 of the ONS spreadsheet: keep all-nationalities flows for calendar years.""" tb = snap.read_excel(sheet_name="1", header=None, skiprows=6) @@ -110,6 +137,7 @@ def run() -> None: # tb_boe = parse_bank_of_england(snap_boe) tb_ons = parse_ons(snap_ons) + tb_england = parse_england(snap_boe) # # Save outputs. @@ -118,6 +146,7 @@ def run() -> None: tables=[ tb_boe.format(["year"], short_name="bank_of_england_flows"), tb_ons.format(["year"], short_name="ons_flows"), + tb_england.format(["year"], short_name="england_net_migration"), ], default_metadata=snap_boe.metadata, ) From 141e4ea9ae54cce0b5121e0a156ed4b1eb2aa602 Mon Sep 17 00:00:00 2001 From: Edouard Mathieu Date: Tue, 4 Aug 2026 11:36:53 +0200 Subject: [PATCH 12/15] =?UTF-8?q?=F0=9F=92=84=F0=9F=A4=96=20Trim=20England?= =?UTF-8?q?=20WYSK?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- .../data/garden/migration/2026-08-03/uk_migration.meta.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml b/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml index 3746633f7b3..68ba788f22f 100644 --- a/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml +++ b/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml @@ -56,9 +56,7 @@ definitions: england_method: |- This data is not a direct count of migrants — no migration records exist for most of this period. Wrigley and Schofield reconstructed England's population from the registers of baptisms and burials kept by Church of England parishes, and estimated net migration as a residual: the part of the change in population that births and deaths cannot explain. - Because it is a residual, the estimate counts everyone who crossed England's borders — including people who moved to or from Wales, Scotland, or Ireland, not only those who moved to or from other countries. It also absorbs any errors in the registration of baptisms and burials. England is defined here without Monmouthshire, which historical records grouped with Wales. - - Negative values mean more people left England than arrived. The reconstruction shows a net outflow of people in every year from 1541 to 1870. + Because it is a residual, the estimate counts everyone who crossed England's borders — including people who moved to or from Wales, Scotland, or Ireland, not only those who moved to or from other countries. It also absorbs any errors in the registration of baptisms and burials. The original estimates are totals for five-year periods, which the Bank of England interpolated to annual values. Year-to-year changes are therefore smooth by construction, and the data is best read as showing broad levels and trends rather than what happened in any single year. From 8af81b1095287d1530791c1b84e905387afb5afc Mon Sep 17 00:00:00 2001 From: Edouard Mathieu Date: Tue, 4 Aug 2026 11:38:18 +0200 Subject: [PATCH 13/15] =?UTF-8?q?=F0=9F=92=84=F0=9F=A4=96=20Trim=20England?= =?UTF-8?q?=20processing=20wording?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- .../data/garden/migration/2026-08-03/uk_migration.meta.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml b/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml index 68ba788f22f..fa8397d0a39 100644 --- a/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml +++ b/etl/steps/data/garden/migration/2026-08-03/uk_migration.meta.yml @@ -64,10 +64,10 @@ definitions: Net Emigration - England (excluding Monmouthshire). Source: Wrigley and Schofield (1989), interpolated quinquennial estimates. These estimates naturally cover migration flows to and from other parts of the UK as well as to overseas countries. england_processing: |- - The source publishes this series as net emigration, where positive values mean more people leaving. We flipped the sign to show net migration, consistent with our United Kingdom indicators, and converted the figures from thousands of people to people. + The source publishes this series as net emigration, where positive values mean more people leaving. We flipped the sign to show net migration, and converted the figures from thousands of people to people. england_processing_share: |- - The source publishes this series as net emigration per 1,000 people, where positive values mean more people leaving. We flipped the sign to show net migration, consistent with our United Kingdom indicators, and expressed the rate as a percentage of the population. The population estimates in the denominator come from the same reconstruction by Wrigley and Schofield. + The source publishes this series as net emigration per 1,000 people, where positive values mean more people leaving. We flipped the sign to show net migration, and expressed the rate as a percentage of the population. The population estimates in the denominator come from the same reconstruction by Wrigley and Schofield. uk_processing_shares: |- We combined two sources into one series: From 847f8b4f2a02d99380dde687ee86710d78c92e12 Mon Sep 17 00:00:00 2001 From: Edouard Mathieu Date: Thu, 6 Aug 2026 10:40:42 +0200 Subject: [PATCH 14/15] =?UTF-8?q?=F0=9F=94=A8=F0=9F=A4=96=20End=20the=20En?= =?UTF-8?q?gland=20series=20in=201854,=20just=20before=20the=20UK=20series?= =?UTF-8?q?=20begins?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- .../garden/migration/2026-08-03/uk_migration.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/etl/steps/data/garden/migration/2026-08-03/uk_migration.py b/etl/steps/data/garden/migration/2026-08-03/uk_migration.py index 4e11c519813..8edae46e6f0 100644 --- a/etl/steps/data/garden/migration/2026-08-03/uk_migration.py +++ b/etl/steps/data/garden/migration/2026-08-03/uk_migration.py @@ -44,7 +44,9 @@ def sanity_check_inputs(tb_boe: Table, tb_ons: Table) -> None: def sanity_check_england(tb: Table) -> None: assert not tb["year"].duplicated().any(), "Duplicate years in the England table." - assert len(tb) == 330, "Expected 330 years of England data." + # 1541 up to 1854, the year before the UK net migration series begins. + assert tb["year"].min() == 1541 and tb["year"].max() == 1854, "Expected coverage 1541-1854." + assert len(tb) == 314, "Expected 314 years of England data." # Wrigley and Schofield's reconstruction shows a net outflow (negative net migration) in every year. assert (tb["net_migration"] < 0).all(), "Unexpected net inflow year in the England series." assert tb["net_migration_share_of_population"].between(-0.5, 0).all(), "Share outside the plausible range." @@ -109,9 +111,12 @@ def run() -> None: sanity_check_outputs(tb) - # England, 1541-1870: the source publishes net emigration (positive = people leaving). Flip the - # sign to net migration, consistent with the UK indicators, and express the source's per-1,000 - # rate as a percentage share of the population. + # England: the source publishes net emigration (positive = people leaving). Flip the sign to + # net migration, consistent with the UK indicators, and express the source's per-1,000 rate as + # a percentage share of the population. The source runs to 1870; we end the series just before + # the UK series begins, so the two do not overlap. + uk_start = int(tb.loc[tb["net_migration"].notna(), "year"].min()) + tb_england = tb_england[tb_england["year"] < uk_start].copy() tb_england["country"] = "England" tb_england["net_migration"] = -tb_england["net_emigration"] * 1000 tb_england["net_migration_share_of_population"] = -tb_england["net_emigration_per_1000"] / 10 From c2559d3df282626abc8fd594dcf1338d46ab3ab7 Mon Sep 17 00:00:00 2001 From: Edouard Mathieu Date: Mon, 10 Aug 2026 14:07:23 +0200 Subject: [PATCH 15/15] =?UTF-8?q?=F0=9F=94=A8=20Wake=20up=20staging=20serv?= =?UTF-8?q?er?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit