diff --git a/Packages/StrandImport/Sources/StrandImport/CSVParsing.swift b/Packages/StrandImport/Sources/StrandImport/CSVParsing.swift index 22e30ec986..0058fea0ab 100644 --- a/Packages/StrandImport/Sources/StrandImport/CSVParsing.swift +++ b/Packages/StrandImport/Sources/StrandImport/CSVParsing.swift @@ -221,6 +221,48 @@ enum HeaderNorm { "texto_de_pergunta": "question_text", "respondeu_sim": "answered_yes_no", "notas": "notes", + // — Italian (cicli_fisiologici / sonno / allenamenti / voci_diario). Every header the importers read + // across cycles, sleeps, workouts and journal, from a real Italian export ("Durata (min)" and + // "GPS abilitato" are not read in any language). "FC max." and "FC media" fold to the same + // keys as the French and pt-BR aliases above, so they are deliberately NOT repeated here. The + // journal "Note" column is NOT aliased either: a global "note" key would rename the lift-program + // template's own "Note" column, so the journal parser reads "note" as a fallback instead. — + "ora_di_inizio_ciclo": "cycle_start_time", + "ora_di_fine_ciclo": "cycle_end_time", + "fuso_orario_ciclo": "cycle_timezone", + "punteggio_di_recupero_pct": "recovery_score_pct", + "frequenza_cardiaca_a_riposo_bpm": "resting_heart_rate_bpm", + "variabilita_della_frequenza_cardiaca_ms": "heart_rate_variability_ms", + "temp_cutanea_c": "skin_temp_celsius", + "ossigeno_nel_sangue_pct": "blood_oxygen_pct", + "sforzo_giornaliero": "day_strain", + "energia_bruciata_cal": "energy_burned_cal", + "inizio_del_sonno": "sleep_onset", + "inizio_del_risveglio": "wake_onset", + "andamento_del_sonno_pct": "sleep_performance_pct", + "frequenza_respiratoria_rpm": "respiratory_rate_rpm", + "durata_del_sonno_min": "asleep_duration_min", + "tempo_a_letto_min": "in_bed_duration_min", + "durata_del_sonno_leggero_min": "light_sleep_duration_min", + "durata_profondo_sws_min": "deep_sws_duration_min", + "durata_rem_min": "rem_duration_min", + "durata_del_risveglio_min": "awake_duration_min", + "sonno_richiesto_min": "sleep_need_min", + "sonno_arretrato_min": "sleep_debt_min", + "efficienza_del_sonno_pct": "sleep_efficiency_pct", + "regolarita_del_sonno_pct": "sleep_consistency_pct", + "riposo_breve": "nap", + "ora_di_inizio_allenamento": "workout_start_time", + "ora_di_fine_allenamento": "workout_end_time", + "nome_attivita": "activity_name", + "sforzo_richiesto_dall_attivita": "activity_strain", + "zona_fc_1_pct": "hr_zone_1_pct", + "zona_fc_2_pct": "hr_zone_2_pct", + "zona_fc_3_pct": "hr_zone_3_pct", + "zona_fc_4_pct": "hr_zone_4_pct", + "zona_fc_5_pct": "hr_zone_5_pct", + "testo_domanda": "question_text", + "risposta_affermativa": "answered_yes_no", ] } diff --git a/Packages/StrandImport/Sources/StrandImport/WhoopExportImporter.swift b/Packages/StrandImport/Sources/StrandImport/WhoopExportImporter.swift index b147014014..5c26c055a7 100644 --- a/Packages/StrandImport/Sources/StrandImport/WhoopExportImporter.swift +++ b/Packages/StrandImport/Sources/StrandImport/WhoopExportImporter.swift @@ -95,6 +95,12 @@ public struct WhoopExportImporter { case "sonos.csv": return sleepsName case "treinos.csv": return workoutsName case "entradas_diário.csv", "entradas_diario.csv": return journalName + // Italian: like pt-BR, all four filenames are localized. Names from a real Italian export; none of + // them carries a diacritic, so no folded variant is needed. + case "cicli_fisiologici.csv": return cyclesName + case "sonno.csv": return sleepsName + case "allenamenti.csv": return workoutsName + case "voci_diario.csv": return journalName default: return nil } } @@ -398,7 +404,8 @@ public struct WhoopExportImporter { // journal import silently zeroed out to "without" because neither of the old keys ever // matched, regardless of the account's actual answers. r.answer = row.cell("answered_yes", "answered_yes_no", "answer", "answer_text") - r.notes = row.cell("notes") + // "note": the Italian export's "Note" column (see the Italian block in HeaderNorm.foreignAliases). + r.notes = row.cell("notes", "note") // A journal row is only meaningful if it has a question. if r.question == nil && r.answer == nil && r.notes == nil { continue } diff --git a/Packages/StrandImport/Tests/StrandImportTests/WhoopExportImporterTests.swift b/Packages/StrandImport/Tests/StrandImportTests/WhoopExportImporterTests.swift index f1e86a9444..00dd683876 100644 --- a/Packages/StrandImport/Tests/StrandImportTests/WhoopExportImporterTests.swift +++ b/Packages/StrandImport/Tests/StrandImportTests/WhoopExportImporterTests.swift @@ -377,4 +377,55 @@ final class WhoopExportImporterTests: XCTestCase { XCTAssertEqual(rows[0].dayStrain, 12.5) XCTAssertEqual(rows[0].cycleStart, Fixtures.utc(2024, 3, 1, 6, 0, 0)) } + + // MARK: - Localized (Italian) column headers + + func testItalianHeaderNormalizationAliases() { + XCTAssertEqual(HeaderNorm.normalize("Ora di inizio ciclo"), "cycle_start_time") + XCTAssertEqual(HeaderNorm.normalize("Variabilità della frequenza cardiaca (ms)"), "heart_rate_variability_ms") + XCTAssertEqual(HeaderNorm.normalize("Temp. cutanea (C)"), "skin_temp_celsius") + XCTAssertEqual(HeaderNorm.normalize("Ossigeno nel sangue %"), "blood_oxygen_pct") + XCTAssertEqual(HeaderNorm.normalize("Durata profondo (SWS) (min)"), "deep_sws_duration_min") + XCTAssertEqual(HeaderNorm.normalize("Regolarità del sonno %"), "sleep_consistency_pct") + XCTAssertEqual(HeaderNorm.normalize("Riposo breve"), "nap") + XCTAssertEqual(HeaderNorm.normalize("Nome attività"), "activity_name") + XCTAssertEqual(HeaderNorm.normalize("Sforzo richiesto dall'attività"), "activity_strain") + XCTAssertEqual(HeaderNorm.normalize("Zona FC 3 %"), "hr_zone_3_pct") + XCTAssertEqual(HeaderNorm.normalize("Risposta affermativa"), "answered_yes_no") + // "FC max." / "FC media" share the French / pt-BR aliases and must still resolve. + XCTAssertEqual(HeaderNorm.normalize("FC max. (bpm)"), "max_hr_bpm") + XCTAssertEqual(HeaderNorm.normalize("FC media (bpm)"), "average_hr_bpm") + } + + func testItalianCyclesValuesParse() throws { + // The exact cicli_fisiologici.csv header from a real Italian export + one synthetic data row. + let csv = """ + Ora di inizio ciclo,Ora di fine ciclo,Fuso orario ciclo,Punteggio di recupero %,Frequenza cardiaca a riposo (bpm),Variabilità della frequenza cardiaca (ms),Temp. cutanea (C),Ossigeno nel sangue %,Sforzo giornaliero,Energia bruciata (cal),FC max. (bpm),FC media (bpm),Inizio del sonno,Inizio del risveglio,Andamento del sonno %,Frequenza respiratoria (rpm),Durata del sonno (min),Tempo a letto (min),Durata del sonno leggero (min),Durata profondo (SWS) (min),Durata REM (min),Durata del risveglio (min),Sonno richiesto (min),Sonno arretrato (min),Efficienza del sonno %,Regolarità del sonno % + 2024-03-01 06:00:00,2024-03-02 06:00:00,UTC+00:00,80,52,95,33.5,96,12.5,2000,150,61,2024-03-01 23:00:00,2024-03-02 06:30:00,90,14,420,450,200,120,100,30,480,60,93,85 + """ + let rows = WhoopExportImporter().parseCycles(CSVTable(text: csv)) + XCTAssertEqual(rows.count, 1) + XCTAssertEqual(rows[0].recoveryScore, 80) + XCTAssertEqual(rows[0].restingHeartRate, 52) + XCTAssertEqual(rows[0].hrvMs, 95) + XCTAssertEqual(rows[0].dayStrain, 12.5) + XCTAssertEqual(rows[0].cycleStart, Fixtures.utc(2024, 3, 1, 6, 0, 0)) + } + + func testItalianJournalReadsQuestionAnswerAndNotes() throws { + // The exact voci_diario.csv header from a real Italian export + synthetic rows. "Note" is read + // through the parser's "note" fallback, not an alias (see HeaderNorm.foreignAliases). + let csv = """ + Ora di inizio ciclo,Ora di fine ciclo,Fuso orario ciclo,Testo domanda,Risposta affermativa,Note + 2024-03-01 23:00:00,2024-03-02 23:00:00,UTC+01:00,Any alcohol?,true,One glass of wine + 2024-03-01 23:00:00,2024-03-02 23:00:00,UTC+01:00,Any caffeine?,false, + """ + let rows = WhoopExportImporter().parseJournal(CSVTable(text: csv)) + XCTAssertEqual(rows.count, 2) + XCTAssertEqual(rows[0].question, "Any alcohol?") + XCTAssertEqual(rows[0].answer, "true") + XCTAssertEqual(rows[0].notes, "One glass of wine") + XCTAssertEqual(rows[1].answer, "false") + XCTAssertNil(rows[1].notes) + } } diff --git a/Tools/linux-capture/correlate_ground_truth.py b/Tools/linux-capture/correlate_ground_truth.py index b5aab754b9..d8d7b59b4b 100644 --- a/Tools/linux-capture/correlate_ground_truth.py +++ b/Tools/linux-capture/correlate_ground_truth.py @@ -36,11 +36,12 @@ # Canonical WHOOP export filenames (English). Localized bundles rename the files; we match by these # after aliasing, and fall back to header-sniffing, exactly like the Swift importer. -CYCLES_FILES = {"physiological_cycles.csv", "physiologische_zyklen.csv", "ciclos_fisiologicos.csv"} -SLEEP_FILES = {"sleeps.csv", "schlaf.csv", "sueno.csv", "sueños.csv"} +CYCLES_FILES = {"physiological_cycles.csv", "physiologische_zyklen.csv", "ciclos_fisiologicos.csv", + "cicli_fisiologici.csv"} +SLEEP_FILES = {"sleeps.csv", "schlaf.csv", "sueno.csv", "sueños.csv", "sonno.csv"} # Localized column header → canonical key. Mirrors HeaderNorm.foreignAliases in CSVParsing.swift -# (German + Spanish subset that covers the physiologic/sleep numeric fields we correlate on). English +# (German, Spanish and Italian subset that covers the physiologic/sleep numeric fields we correlate on). English # headers pass through unchanged, so an English export needs no alias. HEADER_ALIASES = { # English (official app.whoop.com export — headers are title case; we lowercase first) @@ -76,6 +77,20 @@ "oxígeno en sangre %": "blood_oxygen_pct", "frecuencia respiratoria (rpm)": "respiratory_rate_rpm", "hora de inicio del ciclo": "cycle_start_time", + # Italian (cicli_fisiologici.csv / sonno.csv) + "ora di inizio ciclo": "cycle_start_time", + "ora di fine ciclo": "cycle_end_time", + "punteggio di recupero %": "recovery_score_pct", + "frequenza cardiaca a riposo (bpm)": "resting_heart_rate_bpm", + "variabilità della frequenza cardiaca (ms)": "heart_rate_variability_ms", + "temp. cutanea (c)": "skin_temp_celsius", + "ossigeno nel sangue %": "blood_oxygen_pct", + "frequenza respiratoria (rpm)": "respiratory_rate_rpm", + "durata del sonno (min)": "asleep_duration_min", + "durata profondo (sws) (min)": "deep_sws_duration_min", + "durata rem (min)": "rem_duration_min", + "inizio del sonno": "sleep_onset", + "inizio del risveglio": "wake_onset", } # Numeric fields we correlate on, with the scale factor the raw record is *likely* to carry the value diff --git a/Tools/linux-capture/test_correlate_ground_truth.py b/Tools/linux-capture/test_correlate_ground_truth.py index 9c4b6eceda..34e0431ce5 100644 --- a/Tools/linux-capture/test_correlate_ground_truth.py +++ b/Tools/linux-capture/test_correlate_ground_truth.py @@ -32,6 +32,13 @@ def test_english_headers_map_to_canonical(self): self.assertEqual(cg._norm_header("Skin temp (celsius)"), "skin_temp_celsius") self.assertEqual(cg._norm_header("Cycle start time"), "cycle_start_time") + def test_italian_headers_map_to_canonical(self): + self.assertEqual(cg._norm_header("Variabilità della frequenza cardiaca (ms)"), + "heart_rate_variability_ms") + self.assertEqual(cg._norm_header("Ossigeno nel sangue %"), "blood_oxygen_pct") + self.assertEqual(cg._norm_header("Temp. cutanea (C)"), "skin_temp_celsius") + self.assertEqual(cg._norm_header("Ora di inizio ciclo"), "cycle_start_time") + def test_bom_stripped(self): self.assertEqual(cg._norm_header("Startzeit des Zyklus"), "cycle_start_time") @@ -83,6 +90,26 @@ def test_merges_cycles_and_sleep_on_cycle_start(self): self.assertEqual(truth["respiratory_rate"], [14.6]) + def test_load_italian_cycles_and_sleep_from_zip(self): + # Real Italian export filenames; headers taken from a real Italian export. + cycles = ("Ora di inizio ciclo,Variabilità della frequenza cardiaca (ms),Ossigeno nel sangue %\n" + "2026-06-20 00:24:31,92,96.5\n") + sleep = "Ora di inizio ciclo,Frequenza respiratoria (rpm)\n2026-06-20 00:24:31,14.6\n" + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as z: + z.writestr("cicli_fisiologici.csv", cycles) + z.writestr("sonno.csv", sleep) + with tempfile.NamedTemporaryFile(suffix=".zip", delete=False) as f: + f.write(buf.getvalue()) + path = f.name + rows = cg.load_ground_truth(path) + self.assertEqual(len(rows), 1) + truth = cg.truth_values(rows) + self.assertEqual(truth["hrv_ms"], [92.0]) + self.assertEqual(truth["spo2_pct"], [96.5]) + self.assertEqual(truth["respiratory_rate"], [14.6]) + + class CandidateExtractionTests(unittest.TestCase): def test_all_encodings_present(self): rec = struct.pack(" List[dict]: # Header sniff: any CSV that carries blood oxygen + cycle start. for body in files.values(): head = body.splitlines()[0].lower() if body else "" - if "blood oxygen" in head or "blutsauerstoff" in head or "oxígeno en sangre" in head: + if ("blood oxygen" in head or "blutsauerstoff" in head or "oxígeno en sangre" in head + or "ossigeno nel sangue" in head): text = body break if not text: diff --git a/android/app/src/main/java/com/noop/ingest/CsvParser.kt b/android/app/src/main/java/com/noop/ingest/CsvParser.kt index d7045b1d75..b2b1ffd0bd 100644 --- a/android/app/src/main/java/com/noop/ingest/CsvParser.kt +++ b/android/app/src/main/java/com/noop/ingest/CsvParser.kt @@ -235,6 +235,48 @@ internal object HeaderNorm { "texto_de_pergunta" to "question_text", "respondeu_sim" to "answered_yes_no", "notas" to "notes", + // — Italian (cicli_fisiologici / sonno / allenamenti / voci_diario). Every header the importers read + // across cycles, sleeps, workouts and journal, from a real Italian export ("Durata (min)" and + // "GPS abilitato" are not read in any language). "FC max." and "FC media" fold to the same + // keys as the French and pt-BR aliases above, so they are deliberately NOT repeated here. The + // journal "Note" column is NOT aliased either: a global "note" key would rename the lift-program + // template's own "Note" column, so the journal parser reads "note" as a fallback instead. — + "ora_di_inizio_ciclo" to "cycle_start_time", + "ora_di_fine_ciclo" to "cycle_end_time", + "fuso_orario_ciclo" to "cycle_timezone", + "punteggio_di_recupero_pct" to "recovery_score_pct", + "frequenza_cardiaca_a_riposo_bpm" to "resting_heart_rate_bpm", + "variabilita_della_frequenza_cardiaca_ms" to "heart_rate_variability_ms", + "temp_cutanea_c" to "skin_temp_celsius", + "ossigeno_nel_sangue_pct" to "blood_oxygen_pct", + "sforzo_giornaliero" to "day_strain", + "energia_bruciata_cal" to "energy_burned_cal", + "inizio_del_sonno" to "sleep_onset", + "inizio_del_risveglio" to "wake_onset", + "andamento_del_sonno_pct" to "sleep_performance_pct", + "frequenza_respiratoria_rpm" to "respiratory_rate_rpm", + "durata_del_sonno_min" to "asleep_duration_min", + "tempo_a_letto_min" to "in_bed_duration_min", + "durata_del_sonno_leggero_min" to "light_sleep_duration_min", + "durata_profondo_sws_min" to "deep_sws_duration_min", + "durata_rem_min" to "rem_duration_min", + "durata_del_risveglio_min" to "awake_duration_min", + "sonno_richiesto_min" to "sleep_need_min", + "sonno_arretrato_min" to "sleep_debt_min", + "efficienza_del_sonno_pct" to "sleep_efficiency_pct", + "regolarita_del_sonno_pct" to "sleep_consistency_pct", + "riposo_breve" to "nap", + "ora_di_inizio_allenamento" to "workout_start_time", + "ora_di_fine_allenamento" to "workout_end_time", + "nome_attivita" to "activity_name", + "sforzo_richiesto_dall_attivita" to "activity_strain", + "zona_fc_1_pct" to "hr_zone_1_pct", + "zona_fc_2_pct" to "hr_zone_2_pct", + "zona_fc_3_pct" to "hr_zone_3_pct", + "zona_fc_4_pct" to "hr_zone_4_pct", + "zona_fc_5_pct" to "hr_zone_5_pct", + "testo_domanda" to "question_text", + "risposta_affermativa" to "answered_yes_no", ) } diff --git a/android/app/src/main/java/com/noop/ingest/WhoopCsvImporter.kt b/android/app/src/main/java/com/noop/ingest/WhoopCsvImporter.kt index a1011a0d01..f7e9cace88 100644 --- a/android/app/src/main/java/com/noop/ingest/WhoopCsvImporter.kt +++ b/android/app/src/main/java/com/noop/ingest/WhoopCsvImporter.kt @@ -299,6 +299,12 @@ object WhoopCsvImporter { "sonos.csv" -> SLEEPS_NAME "treinos.csv" -> WORKOUTS_NAME "entradas_diário.csv", "entradas_diario.csv" -> JOURNAL_NAME + // Italian: like pt-BR, all four filenames are localized. Names from a real Italian export; none of + // them carries a diacritic, so no folded variant is needed. + "cicli_fisiologici.csv" -> CYCLES_NAME + "sonno.csv" -> SLEEPS_NAME + "allenamenti.csv" -> WORKOUTS_NAME + "voci_diario.csv" -> JOURNAL_NAME else -> null } @@ -597,7 +603,8 @@ object WhoopCsvImporter { // journal import silently zeroed out to "without" because neither of the old keys ever // matched, regardless of the account's actual answers. val answer = row.cell("answered_yes", "answered_yes_no", "answer", "answer_text") - val notes = row.cell("notes") + // "note": the Italian export's "Note" column (see the Italian block in HeaderNorm.foreignAliases). + val notes = row.cell("notes", "note") // Swift: a journal row is only meaningful if it has a question/answer/notes. if (question == null && answer == null && notes == null) continue diff --git a/android/app/src/test/java/com/noop/ingest/WhoopCsvImporterTest.kt b/android/app/src/test/java/com/noop/ingest/WhoopCsvImporterTest.kt index faf14529f5..d378f302f2 100644 --- a/android/app/src/test/java/com/noop/ingest/WhoopCsvImporterTest.kt +++ b/android/app/src/test/java/com/noop/ingest/WhoopCsvImporterTest.kt @@ -266,4 +266,65 @@ class WhoopCsvImporterTest { // Day Strain 12.5 is rescaled onto NOOP's 0–100 Effort axis (×100/21). assertEquals(12.5 * (100.0 / 21.0), r.strain!!, 1e-9) } + + // --- Localized (Italian) headers ---------------------------------------------------------- + + /** Diacritic-folded it headers land on the canonical English keys (parity with Swift). */ + @Test + fun italianHeaderAliasesNormalize() { + assertEquals("cycle_start_time", HeaderNorm.normalize("Ora di inizio ciclo")) + assertEquals("heart_rate_variability_ms", HeaderNorm.normalize("Variabilità della frequenza cardiaca (ms)")) + assertEquals("skin_temp_celsius", HeaderNorm.normalize("Temp. cutanea (C)")) + assertEquals("blood_oxygen_pct", HeaderNorm.normalize("Ossigeno nel sangue %")) + assertEquals("deep_sws_duration_min", HeaderNorm.normalize("Durata profondo (SWS) (min)")) + assertEquals("sleep_consistency_pct", HeaderNorm.normalize("Regolarità del sonno %")) + assertEquals("nap", HeaderNorm.normalize("Riposo breve")) + assertEquals("activity_name", HeaderNorm.normalize("Nome attività")) + assertEquals("activity_strain", HeaderNorm.normalize("Sforzo richiesto dall'attività")) + assertEquals("hr_zone_3_pct", HeaderNorm.normalize("Zona FC 3 %")) + assertEquals("answered_yes_no", HeaderNorm.normalize("Risposta affermativa")) + // "FC max." / "FC media" share the French / pt-BR aliases and must still resolve. + assertEquals("max_hr_bpm", HeaderNorm.normalize("FC max. (bpm)")) + assertEquals("average_hr_bpm", HeaderNorm.normalize("FC media (bpm)")) + } + + /** A real cicli_fisiologici.csv header + one synthetic data row: values flow through the it aliases. */ + @Test + fun italianCyclesValuesParse() { + val rows = cycles( + """ + Ora di inizio ciclo,Ora di fine ciclo,Fuso orario ciclo,Punteggio di recupero %,Frequenza cardiaca a riposo (bpm),Variabilità della frequenza cardiaca (ms),Temp. cutanea (C),Ossigeno nel sangue %,Sforzo giornaliero,Energia bruciata (cal),FC max. (bpm),FC media (bpm),Inizio del sonno,Inizio del risveglio,Andamento del sonno %,Frequenza respiratoria (rpm),Durata del sonno (min),Tempo a letto (min),Durata del sonno leggero (min),Durata profondo (SWS) (min),Durata REM (min),Durata del risveglio (min),Sonno richiesto (min),Sonno arretrato (min),Efficienza del sonno %,Regolarità del sonno % + 2024-03-01 06:00:00,2024-03-02 06:00:00,UTC+00:00,80,52,95,33.5,96,12.5,2000,150,61,2024-03-01 23:00:00,2024-03-02 06:30:00,90,14,420,450,200,120,100,30,480,60,93,85 + """ + ) + assertEquals(1, rows.size) + val r = rows.single() + assertEquals(80.0, r.recovery!!, 1e-9) + assertEquals(52, r.restingHr) + assertEquals(95.0, r.avgHrv!!, 1e-9) + // Keyed off wake_onset (Inizio del risveglio 2024-03-02 06:30) = the wake day, not the onset day. + assertEquals("2024-03-02", r.day) + // Day Strain 12.5 is rescaled onto NOOP's 0–100 Effort axis (×100/21). + assertEquals(12.5 * (100.0 / 21.0), r.strain!!, 1e-9) + } + + /** A real voci_diario.csv header + synthetic rows. "Note" is read through the parser's "note" + * fallback, not an alias (see HeaderNorm.foreignAliases). */ + @Test + fun italianJournalReadsQuestionAnswerAndNotes() { + val entries = journal( + """ + Ora di inizio ciclo,Ora di fine ciclo,Fuso orario ciclo,Testo domanda,Risposta affermativa,Note + 2024-03-01 23:00:00,2024-03-02 23:00:00,UTC+01:00,Any alcohol?,true,One glass of wine + 2024-03-01 23:00:00,2024-03-02 23:00:00,UTC+01:00,Any caffeine?,false, + """, + emptyMap(), + ) + assertEquals(2, entries.size) + assertEquals("Any alcohol?", entries[0].question) + assertEquals(true, entries[0].answeredYes) + assertEquals("One glass of wine", entries[0].notes) + assertEquals(false, entries[1].answeredYes) + assertEquals(null, entries[1].notes) + } }