Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 42 additions & 0 deletions Packages/StrandImport/Sources/StrandImport/CSVParsing.swift
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,48 @@ enum HeaderNorm {
"texto_de_pergunta": "question_text",
"respondeu_sim": "answered_yes_no",
"notas": "notes",
// — Italian (cicli_fisiologici / sonno / allenamenti / voci_diario). Every header the importers read
// across cycles, sleeps, workouts and journal, from a real Italian export ("Durata (min)" and
// "GPS abilitato" are not read in any language). "FC max." and "FC media" fold to the same
// keys as the French and pt-BR aliases above, so they are deliberately NOT repeated here. The
// journal "Note" column is NOT aliased either: a global "note" key would rename the lift-program
// template's own "Note" column, so the journal parser reads "note" as a fallback instead. —
"ora_di_inizio_ciclo": "cycle_start_time",
"ora_di_fine_ciclo": "cycle_end_time",
"fuso_orario_ciclo": "cycle_timezone",
"punteggio_di_recupero_pct": "recovery_score_pct",
"frequenza_cardiaca_a_riposo_bpm": "resting_heart_rate_bpm",
"variabilita_della_frequenza_cardiaca_ms": "heart_rate_variability_ms",
"temp_cutanea_c": "skin_temp_celsius",
"ossigeno_nel_sangue_pct": "blood_oxygen_pct",
"sforzo_giornaliero": "day_strain",
"energia_bruciata_cal": "energy_burned_cal",
"inizio_del_sonno": "sleep_onset",
"inizio_del_risveglio": "wake_onset",
"andamento_del_sonno_pct": "sleep_performance_pct",
"frequenza_respiratoria_rpm": "respiratory_rate_rpm",
"durata_del_sonno_min": "asleep_duration_min",
"tempo_a_letto_min": "in_bed_duration_min",
"durata_del_sonno_leggero_min": "light_sleep_duration_min",
"durata_profondo_sws_min": "deep_sws_duration_min",
"durata_rem_min": "rem_duration_min",
"durata_del_risveglio_min": "awake_duration_min",
"sonno_richiesto_min": "sleep_need_min",
"sonno_arretrato_min": "sleep_debt_min",
"efficienza_del_sonno_pct": "sleep_efficiency_pct",
"regolarita_del_sonno_pct": "sleep_consistency_pct",
"riposo_breve": "nap",
"ora_di_inizio_allenamento": "workout_start_time",
"ora_di_fine_allenamento": "workout_end_time",
"nome_attivita": "activity_name",
"sforzo_richiesto_dall_attivita": "activity_strain",
"zona_fc_1_pct": "hr_zone_1_pct",
"zona_fc_2_pct": "hr_zone_2_pct",
"zona_fc_3_pct": "hr_zone_3_pct",
"zona_fc_4_pct": "hr_zone_4_pct",
"zona_fc_5_pct": "hr_zone_5_pct",
"testo_domanda": "question_text",
"risposta_affermativa": "answered_yes_no",
]
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,12 @@ public struct WhoopExportImporter {
case "sonos.csv": return sleepsName
case "treinos.csv": return workoutsName
case "entradas_diário.csv", "entradas_diario.csv": return journalName
// Italian: like pt-BR, all four filenames are localized. Names from a real Italian export; none of
// them carries a diacritic, so no folded variant is needed.
case "cicli_fisiologici.csv": return cyclesName
case "sonno.csv": return sleepsName
case "allenamenti.csv": return workoutsName
case "voci_diario.csv": return journalName
default: return nil
}
}
Expand Down Expand Up @@ -398,7 +404,8 @@ public struct WhoopExportImporter {
// journal import silently zeroed out to "without" because neither of the old keys ever
// matched, regardless of the account's actual answers.
r.answer = row.cell("answered_yes", "answered_yes_no", "answer", "answer_text")
r.notes = row.cell("notes")
// "note": the Italian export's "Note" column (see the Italian block in HeaderNorm.foreignAliases).
r.notes = row.cell("notes", "note")

// A journal row is only meaningful if it has a question.
if r.question == nil && r.answer == nil && r.notes == nil { continue }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -377,4 +377,55 @@ final class WhoopExportImporterTests: XCTestCase {
XCTAssertEqual(rows[0].dayStrain, 12.5)
XCTAssertEqual(rows[0].cycleStart, Fixtures.utc(2024, 3, 1, 6, 0, 0))
}

// MARK: - Localized (Italian) column headers

func testItalianHeaderNormalizationAliases() {
XCTAssertEqual(HeaderNorm.normalize("Ora di inizio ciclo"), "cycle_start_time")
XCTAssertEqual(HeaderNorm.normalize("Variabilità della frequenza cardiaca (ms)"), "heart_rate_variability_ms")
XCTAssertEqual(HeaderNorm.normalize("Temp. cutanea (C)"), "skin_temp_celsius")
XCTAssertEqual(HeaderNorm.normalize("Ossigeno nel sangue %"), "blood_oxygen_pct")
XCTAssertEqual(HeaderNorm.normalize("Durata profondo (SWS) (min)"), "deep_sws_duration_min")
XCTAssertEqual(HeaderNorm.normalize("Regolarità del sonno %"), "sleep_consistency_pct")
XCTAssertEqual(HeaderNorm.normalize("Riposo breve"), "nap")
XCTAssertEqual(HeaderNorm.normalize("Nome attività"), "activity_name")
XCTAssertEqual(HeaderNorm.normalize("Sforzo richiesto dall'attività"), "activity_strain")
XCTAssertEqual(HeaderNorm.normalize("Zona FC 3 %"), "hr_zone_3_pct")
XCTAssertEqual(HeaderNorm.normalize("Risposta affermativa"), "answered_yes_no")
// "FC max." / "FC media" share the French / pt-BR aliases and must still resolve.
XCTAssertEqual(HeaderNorm.normalize("FC max. (bpm)"), "max_hr_bpm")
XCTAssertEqual(HeaderNorm.normalize("FC media (bpm)"), "average_hr_bpm")
}

func testItalianCyclesValuesParse() throws {
// The exact cicli_fisiologici.csv header from a real Italian export + one synthetic data row.
let csv = """
Ora di inizio ciclo,Ora di fine ciclo,Fuso orario ciclo,Punteggio di recupero %,Frequenza cardiaca a riposo (bpm),Variabilità della frequenza cardiaca (ms),Temp. cutanea (C),Ossigeno nel sangue %,Sforzo giornaliero,Energia bruciata (cal),FC max. (bpm),FC media (bpm),Inizio del sonno,Inizio del risveglio,Andamento del sonno %,Frequenza respiratoria (rpm),Durata del sonno (min),Tempo a letto (min),Durata del sonno leggero (min),Durata profondo (SWS) (min),Durata REM (min),Durata del risveglio (min),Sonno richiesto (min),Sonno arretrato (min),Efficienza del sonno %,Regolarità del sonno %
2024-03-01 06:00:00,2024-03-02 06:00:00,UTC+00:00,80,52,95,33.5,96,12.5,2000,150,61,2024-03-01 23:00:00,2024-03-02 06:30:00,90,14,420,450,200,120,100,30,480,60,93,85
"""
let rows = WhoopExportImporter().parseCycles(CSVTable(text: csv))
XCTAssertEqual(rows.count, 1)
XCTAssertEqual(rows[0].recoveryScore, 80)
XCTAssertEqual(rows[0].restingHeartRate, 52)
XCTAssertEqual(rows[0].hrvMs, 95)
XCTAssertEqual(rows[0].dayStrain, 12.5)
XCTAssertEqual(rows[0].cycleStart, Fixtures.utc(2024, 3, 1, 6, 0, 0))
}

func testItalianJournalReadsQuestionAnswerAndNotes() throws {
// The exact voci_diario.csv header from a real Italian export + synthetic rows. "Note" is read
// through the parser's "note" fallback, not an alias (see HeaderNorm.foreignAliases).
let csv = """
Ora di inizio ciclo,Ora di fine ciclo,Fuso orario ciclo,Testo domanda,Risposta affermativa,Note
2024-03-01 23:00:00,2024-03-02 23:00:00,UTC+01:00,Any alcohol?,true,One glass of wine
2024-03-01 23:00:00,2024-03-02 23:00:00,UTC+01:00,Any caffeine?,false,
"""
let rows = WhoopExportImporter().parseJournal(CSVTable(text: csv))
XCTAssertEqual(rows.count, 2)
XCTAssertEqual(rows[0].question, "Any alcohol?")
XCTAssertEqual(rows[0].answer, "true")
XCTAssertEqual(rows[0].notes, "One glass of wine")
XCTAssertEqual(rows[1].answer, "false")
XCTAssertNil(rows[1].notes)
}
}
21 changes: 18 additions & 3 deletions Tools/linux-capture/correlate_ground_truth.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,12 @@

# Canonical WHOOP export filenames (English). Localized bundles rename the files; we match by these
# after aliasing, and fall back to header-sniffing, exactly like the Swift importer.
CYCLES_FILES = {"physiological_cycles.csv", "physiologische_zyklen.csv", "ciclos_fisiologicos.csv"}
SLEEP_FILES = {"sleeps.csv", "schlaf.csv", "sueno.csv", "sueños.csv"}
CYCLES_FILES = {"physiological_cycles.csv", "physiologische_zyklen.csv", "ciclos_fisiologicos.csv",
"cicli_fisiologici.csv"}
SLEEP_FILES = {"sleeps.csv", "schlaf.csv", "sueno.csv", "sueños.csv", "sonno.csv"}

# Localized column header → canonical key. Mirrors HeaderNorm.foreignAliases in CSVParsing.swift
# (German + Spanish subset that covers the physiologic/sleep numeric fields we correlate on). English
# (German, Spanish and Italian subset that covers the physiologic/sleep numeric fields we correlate on). English
# headers pass through unchanged, so an English export needs no alias.
HEADER_ALIASES = {
# English (official app.whoop.com export — headers are title case; we lowercase first)
Expand Down Expand Up @@ -76,6 +77,20 @@
"oxígeno en sangre %": "blood_oxygen_pct",
"frecuencia respiratoria (rpm)": "respiratory_rate_rpm",
"hora de inicio del ciclo": "cycle_start_time",
# Italian (cicli_fisiologici.csv / sonno.csv)
"ora di inizio ciclo": "cycle_start_time",
"ora di fine ciclo": "cycle_end_time",
"punteggio di recupero %": "recovery_score_pct",
"frequenza cardiaca a riposo (bpm)": "resting_heart_rate_bpm",
"variabilità della frequenza cardiaca (ms)": "heart_rate_variability_ms",
"temp. cutanea (c)": "skin_temp_celsius",
"ossigeno nel sangue %": "blood_oxygen_pct",
"frequenza respiratoria (rpm)": "respiratory_rate_rpm",
"durata del sonno (min)": "asleep_duration_min",
"durata profondo (sws) (min)": "deep_sws_duration_min",
"durata rem (min)": "rem_duration_min",
"inizio del sonno": "sleep_onset",
"inizio del risveglio": "wake_onset",
}

# Numeric fields we correlate on, with the scale factor the raw record is *likely* to carry the value
Expand Down
27 changes: 27 additions & 0 deletions Tools/linux-capture/test_correlate_ground_truth.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,13 @@ def test_english_headers_map_to_canonical(self):
self.assertEqual(cg._norm_header("Skin temp (celsius)"), "skin_temp_celsius")
self.assertEqual(cg._norm_header("Cycle start time"), "cycle_start_time")

def test_italian_headers_map_to_canonical(self):
self.assertEqual(cg._norm_header("Variabilità della frequenza cardiaca (ms)"),
"heart_rate_variability_ms")
self.assertEqual(cg._norm_header("Ossigeno nel sangue %"), "blood_oxygen_pct")
self.assertEqual(cg._norm_header("Temp. cutanea (C)"), "skin_temp_celsius")
self.assertEqual(cg._norm_header("Ora di inizio ciclo"), "cycle_start_time")

def test_bom_stripped(self):
self.assertEqual(cg._norm_header("Startzeit des Zyklus"), "cycle_start_time")

Expand Down Expand Up @@ -83,6 +90,26 @@ def test_merges_cycles_and_sleep_on_cycle_start(self):
self.assertEqual(truth["respiratory_rate"], [14.6])


def test_load_italian_cycles_and_sleep_from_zip(self):
# Real Italian export filenames; headers taken from a real Italian export.
cycles = ("Ora di inizio ciclo,Variabilità della frequenza cardiaca (ms),Ossigeno nel sangue %\n"
"2026-06-20 00:24:31,92,96.5\n")
sleep = "Ora di inizio ciclo,Frequenza respiratoria (rpm)\n2026-06-20 00:24:31,14.6\n"
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w") as z:
z.writestr("cicli_fisiologici.csv", cycles)
z.writestr("sonno.csv", sleep)
with tempfile.NamedTemporaryFile(suffix=".zip", delete=False) as f:
f.write(buf.getvalue())
path = f.name
rows = cg.load_ground_truth(path)
self.assertEqual(len(rows), 1)
truth = cg.truth_values(rows)
self.assertEqual(truth["hrv_ms"], [92.0])
self.assertEqual(truth["spo2_pct"], [96.5])
self.assertEqual(truth["respiratory_rate"], [14.6])


class CandidateExtractionTests(unittest.TestCase):
def test_all_encodings_present(self):
rec = struct.pack("<f", 34.93) + b"\x00" * 4
Expand Down
19 changes: 19 additions & 0 deletions Tools/linux-capture/test_validate_spo2_candidate.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,25 @@ def test_english_blood_oxygen_header(self):
self.assertEqual(vs._norm_header("Blood oxygen %"), "blood_oxygen_pct")
self.assertEqual(vs._norm_header("Cycle start time"), "cycle_start_time")

def test_italian_blood_oxygen_header(self):
self.assertEqual(vs._norm_header("Ossigeno nel sangue %"), "blood_oxygen_pct")
self.assertEqual(vs._norm_header("Ora di inizio ciclo"), "cycle_start_time")
self.assertEqual(vs._norm_header("Inizio del risveglio"), "wake_onset")

def test_load_cycles_reads_an_italian_export(self):
# Real Italian cycles filename + headers; one synthetic night.
cycles = ("Ora di inizio ciclo,Ora di fine ciclo,Ossigeno nel sangue %,Inizio del sonno,"
"Inizio del risveglio\n"
"2026-07-15 22:00:00,2026-07-16 22:00:00,96.5,2026-07-15 23:00:00,2026-07-16 07:00:00\n")
folder = tempfile.mkdtemp()
with open(os.path.join(folder, "cicli_fisiologici.csv"), "w", encoding="utf-8") as f:
f.write(cycles)
nights = vs.load_cycles(folder)
self.assertEqual(len(nights), 1)
self.assertEqual(nights[0]["spo2_export"], 96.5)
self.assertEqual(nights[0]["t0"], _utc(2026, 7, 15, 23, 0, 0)) # sleep onset, not cycle start
self.assertEqual(nights[0]["t1"], _utc(2026, 7, 16, 7, 0, 0))

def test_parse_export_ts(self):
self.assertEqual(
vs._parse_export_ts("2026-07-15 23:25:58"),
Expand Down
14 changes: 13 additions & 1 deletion Tools/linux-capture/validate_spo2_candidate.py
Original file line number Diff line number Diff line change
Expand Up @@ -173,12 +173,23 @@
"hora de inicio del ciclo": "cycle_start_time",
"oxígeno en sangre %": "blood_oxygen_pct",
"temp. cutánea (grados centígrados)": "skin_temp_celsius",
# Italian
"ora di inizio ciclo": "cycle_start_time",
"ora di fine ciclo": "cycle_end_time",
"inizio del sonno": "sleep_onset",
"inizio del risveglio": "wake_onset",
"ossigeno nel sangue %": "blood_oxygen_pct",
"temp. cutanea (c)": "skin_temp_celsius",
"frequenza cardiaca a riposo (bpm)": "resting_heart_rate_bpm",
"variabilità della frequenza cardiaca (ms)": "heart_rate_variability_ms",
"frequenza respiratoria (rpm)": "respiratory_rate_rpm",
}

CYCLES_FILES = {
"physiological_cycles.csv",
"physiologische_zyklen.csv",
"ciclos_fisiologicos.csv",
"cicli_fisiologici.csv",
}


Expand Down Expand Up @@ -244,7 +255,8 @@ def load_cycles(export_path: str) -> List[dict]:
# Header sniff: any CSV that carries blood oxygen + cycle start.
for body in files.values():
head = body.splitlines()[0].lower() if body else ""
if "blood oxygen" in head or "blutsauerstoff" in head or "oxígeno en sangre" in head:
if ("blood oxygen" in head or "blutsauerstoff" in head or "oxígeno en sangre" in head
or "ossigeno nel sangue" in head):
text = body
break
if not text:
Expand Down
42 changes: 42 additions & 0 deletions android/app/src/main/java/com/noop/ingest/CsvParser.kt
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,48 @@ internal object HeaderNorm {
"texto_de_pergunta" to "question_text",
"respondeu_sim" to "answered_yes_no",
"notas" to "notes",
// — Italian (cicli_fisiologici / sonno / allenamenti / voci_diario). Every header the importers read
// across cycles, sleeps, workouts and journal, from a real Italian export ("Durata (min)" and
// "GPS abilitato" are not read in any language). "FC max." and "FC media" fold to the same
// keys as the French and pt-BR aliases above, so they are deliberately NOT repeated here. The
// journal "Note" column is NOT aliased either: a global "note" key would rename the lift-program
// template's own "Note" column, so the journal parser reads "note" as a fallback instead. —
"ora_di_inizio_ciclo" to "cycle_start_time",
"ora_di_fine_ciclo" to "cycle_end_time",
"fuso_orario_ciclo" to "cycle_timezone",
"punteggio_di_recupero_pct" to "recovery_score_pct",
"frequenza_cardiaca_a_riposo_bpm" to "resting_heart_rate_bpm",
"variabilita_della_frequenza_cardiaca_ms" to "heart_rate_variability_ms",
"temp_cutanea_c" to "skin_temp_celsius",
"ossigeno_nel_sangue_pct" to "blood_oxygen_pct",
"sforzo_giornaliero" to "day_strain",
"energia_bruciata_cal" to "energy_burned_cal",
"inizio_del_sonno" to "sleep_onset",
"inizio_del_risveglio" to "wake_onset",
"andamento_del_sonno_pct" to "sleep_performance_pct",
"frequenza_respiratoria_rpm" to "respiratory_rate_rpm",
"durata_del_sonno_min" to "asleep_duration_min",
"tempo_a_letto_min" to "in_bed_duration_min",
"durata_del_sonno_leggero_min" to "light_sleep_duration_min",
"durata_profondo_sws_min" to "deep_sws_duration_min",
"durata_rem_min" to "rem_duration_min",
"durata_del_risveglio_min" to "awake_duration_min",
"sonno_richiesto_min" to "sleep_need_min",
"sonno_arretrato_min" to "sleep_debt_min",
"efficienza_del_sonno_pct" to "sleep_efficiency_pct",
"regolarita_del_sonno_pct" to "sleep_consistency_pct",
"riposo_breve" to "nap",
"ora_di_inizio_allenamento" to "workout_start_time",
"ora_di_fine_allenamento" to "workout_end_time",
"nome_attivita" to "activity_name",
"sforzo_richiesto_dall_attivita" to "activity_strain",
"zona_fc_1_pct" to "hr_zone_1_pct",
"zona_fc_2_pct" to "hr_zone_2_pct",
"zona_fc_3_pct" to "hr_zone_3_pct",
"zona_fc_4_pct" to "hr_zone_4_pct",
"zona_fc_5_pct" to "hr_zone_5_pct",
"testo_domanda" to "question_text",
"risposta_affermativa" to "answered_yes_no",
)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,12 @@ object WhoopCsvImporter {
"sonos.csv" -> SLEEPS_NAME
"treinos.csv" -> WORKOUTS_NAME
"entradas_diário.csv", "entradas_diario.csv" -> JOURNAL_NAME
// Italian: like pt-BR, all four filenames are localized. Names from a real Italian export; none of
// them carries a diacritic, so no folded variant is needed.
"cicli_fisiologici.csv" -> CYCLES_NAME
"sonno.csv" -> SLEEPS_NAME
"allenamenti.csv" -> WORKOUTS_NAME
"voci_diario.csv" -> JOURNAL_NAME
else -> null
}

Expand Down Expand Up @@ -597,7 +603,8 @@ object WhoopCsvImporter {
// journal import silently zeroed out to "without" because neither of the old keys ever
// matched, regardless of the account's actual answers.
val answer = row.cell("answered_yes", "answered_yes_no", "answer", "answer_text")
val notes = row.cell("notes")
// "note": the Italian export's "Note" column (see the Italian block in HeaderNorm.foreignAliases).
val notes = row.cell("notes", "note")

// Swift: a journal row is only meaningful if it has a question/answer/notes.
if (question == null && answer == null && notes == null) continue
Expand Down
Loading
Loading