diff --git a/notebooks/Synthea.ipynb b/notebooks/Synthea.ipynb index 2f1b835..2770ad7 100644 --- a/notebooks/Synthea.ipynb +++ b/notebooks/Synthea.ipynb @@ -1,1692 +1,896 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Synthea Tutorial: Synthetic Patient Data Generation\n", - "\n", - "This notebook provides a comprehensive tutorial on using the Synthea wrapper module to generate synthetic healthcare patient data.\n", - "\n", - "## What is Synthea?\n", - "\n", - "Synthea™ is an open-source synthetic patient population simulator that generates realistic synthetic patient records. It creates entire lifetimes of patient data, including:\n", - "\n", - "- Demographics\n", - "- Medical history\n", - "- Medications\n", - "- Lab results\n", - "- Procedures\n", - "- Encounters\n", - "- And more...\n", - "\n", - "**Reference**: [Synthea GitHub](https://github.com/synthetichealth/synthea)\n", - "\n", - "## Prerequisites\n", - "\n", - "- Java 11 or newer installed\n", - "- Python environment with required packages\n", - "- Internet connection (for initial JAR download)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The autoreload extension is already loaded. To reload it, use:\n", - " %reload_ext autoreload\n", - "✓ Synthea module imported successfully\n" - ] - } - ], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2\n", - "%load_ext rpy2.ipython\n", - "# Import the Synthea wrapper\n", - "# If package is installed: pip install -e .\n", - "# Otherwise, add parent directory to path for development\n", - "import sys\n", - "import os\n", - "\n", - "# Add parent directory to path if package not installed\n", - "if 'AoU' not in sys.modules:\n", - " sys.path.insert(0, os.path.abspath(os.path.join(os.getcwd(), '..')))\n", - "\n", - "from AoU.phenome.synthea import SyntheaRunner, SyntheaConfig, download_synthea_jar, convert_synthea_to_omop\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 1. Initialization\n", - "\n", - "The `SyntheaRunner` class handles downloading the Synthea JAR file (if needed) and provides methods to run simulations.\n", - "\n", - "### Basic Initialization\n" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Using existing Synthea JAR: /home/schilder/.cache/synthea/synthea-with-dependencies.jar\n", - "Synthea JAR location: /home/schilder/.cache/synthea/synthea-with-dependencies.jar\n", - "Java executable: java\n" - ] - } - ], - "source": [ - "# Initialize the Synthea runner\n", - "# This will automatically download the JAR file if it doesn't exist\n", - "runner = SyntheaRunner()\n", - "\n", - "print(f\"Synthea JAR location: {runner.jar_path}\")\n", - "print(f\"Java executable: {runner.java_executable}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Custom Initialization\n", - "\n", - "You can specify custom paths and settings:\n" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "# Example: Custom cache directory\n", - "# runner = SyntheaRunner(\n", - "# cache_dir=\"./synthea_cache\", # Where to store the JAR\n", - "# java_executable=\"java\" # Path to Java (default: \"java\")\n", - "# )\n", - "\n", - "# Example: Use existing JAR file\n", - "# runner = SyntheaRunner(\n", - "# jar_path=\"/path/to/existing/synthea-with-dependencies.jar\"\n", - "# )\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 2. Quick Start: Simple Simulation\n", - "\n", - "The easiest way to generate synthetic patients is using the `run_quick()` method:\n" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "$ java -jar /home/schilder/.cache/synthea/synthea-with-dependencies.jar -s 42 -p 10 Massachusetts\n", - "\n", - "Return code: 0\n", - "Output directory: /home/schilder/projects/AoU/notebooks/output/synthea_quick\n", - "\n", - "Command executed:\n", - "java -jar /home/schilder/.cache/synthea/synthea-with-dependencies.jar -s 42 -p 10 Massachusetts\n" - ] - } - ], - "source": [ - "# Generate 10 patients in Massachusetts\n", - "result = runner.run_quick(\n", - " population_size=10,\n", - " state=\"Massachusetts\",\n", - " seed=42, # For reproducibility\n", - " output_dir=\"output/synthea_quick\"\n", - ")\n", - "\n", - "print(f\"\\nReturn code: {result['returncode']}\")\n", - "print(f\"Output directory: {result['output_dir']}\")\n", - "print(f\"\\nCommand executed:\")\n", - "print(result['command'])\n", - "# print(result[\"stdout\"])\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 3. Full Configuration with SyntheaConfig\n", - "\n", - "For more control, use the `SyntheaConfig` dataclass with the `run()` method:\n" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "$ java -jar /home/schilder/.cache/synthea/synthea-with-dependencies.jar -s 12345 -r 20200101 -p 50 -g F -a 25-65 California 'San Francisco'\n", - "\n", - "✓ Simulation completed successfully!\n" - ] - } - ], - "source": [ - "# Create a detailed configuration\n", - "config = SyntheaConfig(\n", - " # Population settings\n", - " population_size=50,\n", - " seed=12345,\n", - " reference_date=\"20200101\", # Format: YYYYMMDD\n", - " \n", - " # Demographics\n", - " gender=\"F\", # \"M\", \"F\", or None for both\n", - " min_age=25,\n", - " max_age=65,\n", - " \n", - " # Location\n", - " state=\"California\",\n", - " city=\"San Francisco\",\n", - " \n", - " # Output\n", - " output_dir=\"output/synthea_california\"\n", - ")\n", - "\n", - "# Run the simulation\n", - "result = runner.run(config, verbose=True)\n", - "\n", - "if result['returncode'] == 0:\n", - " print(\"\\n✓ Simulation completed successfully!\")\n", - "else:\n", - " print(f\"\\n✗ Simulation failed with return code {result['returncode']}\")\n", - " print(f\"Error: {result['stderr']}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 4. Convenience Methods\n", - "\n", - "The wrapper provides several convenience methods for common use cases:\n", - "\n", - "### 4.1 Custom Location\n" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "$ java -jar /home/schilder/.cache/synthea/synthea-with-dependencies.jar -s 999 -p 25 Washington Seattle\n", - "Generated patients in Seattle, WA\n", - "Output: /home/schilder/projects/AoU/notebooks/output/synthea_seattle\n" - ] - } - ], - "source": [ - "# Generate patients in a specific city\n", - "result = runner.run_custom_location(\n", - " state=\"Washington\",\n", - " city=\"Seattle\",\n", - " population_size=25,\n", - " seed=999,\n", - " output_dir=\"output/synthea_seattle\"\n", - ")\n", - "\n", - "print(f\"Generated patients in Seattle, WA\")\n", - "print(f\"Output: {result['output_dir']}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 4.2 Age-Specific Population\n" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "$ java -jar /home/schilder/.cache/synthea/synthea-with-dependencies.jar -s 456 -p 30 -a 0-18 Massachusetts\n", - "Generated pediatric population (ages 0-18)\n" - ] - } - ], - "source": [ - "# Generate only pediatric patients (ages 0-18)\n", - "result = runner.run_age_specific(\n", - " min_age=0,\n", - " max_age=18,\n", - " population_size=30,\n", - " state=\"Massachusetts\",\n", - " seed=456,\n", - " output_dir=\"output/synthea_pediatric\"\n", - ")\n", - "\n", - "print(f\"Generated pediatric population (ages 0-18)\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 5. Advanced Configuration\n", - "\n", - "### 5.1 Using Custom Config Files\n", - "\n", - "Synthea supports custom configuration files for fine-tuned control:\n" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "# Example: Using a custom config file\n", - "# config = SyntheaConfig(\n", - "# population_size=100,\n", - "# config_file=\"./custom_synthea.conf\",\n", - "# output_dir=\"output/synthea_custom\"\n", - "# )\n", - "# result = runner.run(config)\n", - "\n", - "# Note: You need to create the config file first\n", - "# See Synthea documentation for config file format\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 5.2 Custom Modules Directory\n", - "\n", - "You can use custom Synthea modules:\n" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [], - "source": [ - "# Example: Using custom modules\n", - "# config = SyntheaConfig(\n", - "# population_size=50,\n", - "# modules_dir=\"./custom_modules\",\n", - "# output_dir=\"output/synthea_custom_modules\"\n", - "# )\n", - "# result = runner.run(config)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 5.3 Exporter Flags\n", - "\n", - "Synthea supports various exporters (FHIR, CSV, etc.). You can configure them using exporter flags:\n" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Example configuration with FHIR US Core IG exporter\n" - ] - } - ], - "source": [ - "# Example: Enable FHIR US Core IG exporter\n", - "config = SyntheaConfig(\n", - " population_size=20,\n", - " state=\"Massachusetts\",\n", - " seed=789,\n", - " output_dir=\"output/synthea_fhir\",\n", - " exporter_flags={\n", - " \"exporter.fhir.use_us_core_ig\": \"true\",\n", - " \"exporter.csv.export\": \"true\"\n", - " }\n", - ")\n", - "\n", - "# result = runner.run(config)\n", - "print(\"Example configuration with FHIR US Core IG exporter\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 6. Working with Output Files\n", - "\n", - "Synthea generates various output files depending on the exporters enabled. Let's explore what gets generated:\n" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Files in output/synthea_quick:\n", - " - output (0.04 KB)\n" - ] - } - ], - "source": [ - "import os\n", - "from pathlib import Path\n", - "\n", - "# Check output directory\n", - "output_dir = \"output/synthea_quick\"\n", - "if os.path.exists(output_dir):\n", - " print(f\"Files in {output_dir}:\")\n", - " for file in sorted(os.listdir(output_dir)):\n", - " file_path = os.path.join(output_dir, file)\n", - " size = os.path.getsize(file_path) / 1024 # Size in KB\n", - " print(f\" - {file} ({size:.2f} KB)\")\n", - "else:\n", - " print(f\"Output directory {output_dir} does not exist yet.\")\n", - " print(\"Run a simulation first to generate files.\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Common Output Formats\n", - "\n", - "Synthea can generate data in multiple formats:\n", - "\n", - "- **CSV**: Tabular data files\n", - "- **FHIR**: HL7 FHIR resources (JSON)\n", - "- **JSON**: Generic JSON format\n", - "- **CCDA**: Clinical Document Architecture\n", - "\n", - "By default, Synthea generates CSV files. To enable other formats, use exporter flags.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "No CSV files found. Run a simulation to generate data.\n" - ] - } - ], - "source": [ - "# Example: Load and inspect CSV output (if available)\n", - "import pandas as pd\n", - "\n", - "csv_files = []\n", - "if os.path.exists(output_dir):\n", - " csv_files = [f for f in os.listdir(output_dir) if f.endswith('.csv')]\n", - "\n", - "if csv_files:\n", - " print(f\"Found {len(csv_files)} CSV files:\")\n", - " for csv_file in csv_files[:5]: # Show first 5\n", - " print(f\" - {csv_file}\")\n", - " \n", - " # Example: Load patients CSV\n", - " patients_file = os.path.join(output_dir, \"patients.csv\")\n", - " if os.path.exists(patients_file):\n", - " df = pd.read_csv(patients_file, nrows=5) # Load first 5 rows\n", - " print(f\"\\nSample from patients.csv:\")\n", - " print(df.head())\n", - "else:\n", - " print(\"No CSV files found. Run a simulation to generate data.\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 7. Getting Help\n", - "\n", - "You can view Synthea's built-in help:\n" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Usage: run_synthea [options] [state [city]]\n", - "Options: [-s seed] [-cs clinicianSeed] [-p populationSize]\n", - " [-ps singlePersonSeed]\n", - " [-r referenceDate as YYYYMMDD]\n", - " [-e endDate as YYYYMMDD]\n", - " [-g gender] [-a minAge-maxAge]\n", - " [-o overflowPopulation]\n", - " [-c localConfigFilePath]\n", - " [-d localModulesDirPath]\n", - " [-i initialPopulationSnapshotPath]\n", - " [-u updatedPopulationSnapshotPath]\n", - " [-t updateTimePeriodInDays]\n", - " [-f fixedRecordPath]\n", - " [-k keepMatchingPatientsPath]\n", - " [--config*=value]\n", - " * any setting from src/main/resources/synthea.properties\n", - "Examples:\n", - "run_synthea Massachusetts\n", - "run_synthea Alaska Juneau\n", - "run_synthea -s 12345\n", - "run_synthea -p 1000\n", - "run_synthea -s 987 Washington Seattle\n", - "run_synthea -s 21 -p 100 Utah \"Salt Lake City\"\n", - "run_synthea -g M -a 60-65\n", - "run_synthea -p 10 --exporter.fhir.export=true\n", - "run_synthea --exporter.baseDirectory=\"./output_tx/\" Texas\n" - ] - } - ], - "source": [ - "# Display Synthea help\n", - "help_text = runner.show_help()\n", - "print(help_text)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 8. Complete Example: Generating a Research Dataset\n", - "\n", - "Here's a complete example for generating a dataset suitable for research:\n" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Configuration created:\n", - " Population: 1000\n", - " Age range: 18-80\n", - " Location: Massachusetts\n", - " Output: output/synthea_research_dataset\n" - ] - } - ], - "source": [ - "# Generate a research-ready dataset\n", - "research_config = SyntheaConfig(\n", - " # Generate 1000 patients for statistical power\n", - " population_size=1000,\n", - " \n", - " # Use a fixed seed for reproducibility\n", - " seed=20240101,\n", - " \n", - " # Reference date for the simulation\n", - " reference_date=\"20200101\",\n", - " \n", - " # Include all genders\n", - " gender=None,\n", - " \n", - " # Adult population (18-80 years)\n", - " min_age=18,\n", - " max_age=80,\n", - " \n", - " # Geographic location\n", - " state=\"Massachusetts\",\n", - " \n", - " # Enable multiple exporters\n", - " exporter_flags={\n", - " \"exporter.csv.export\": \"true\",\n", - " \"exporter.fhir.export\": \"true\",\n", - " \"exporter.fhir.use_us_core_ig\": \"true\"\n", - " },\n", - " \n", - " # Output directory\n", - " output_dir=\"output/synthea_research_dataset\"\n", - ")\n", - "\n", - "print(\"Configuration created:\")\n", - "print(f\" Population: {research_config.population_size}\")\n", - "print(f\" Age range: {research_config.min_age}-{research_config.max_age}\")\n", - "print(f\" Location: {research_config.state}\")\n", - "print(f\" Output: {research_config.output_dir}\")\n", - "\n", - "# Uncomment to run:\n", - "# result = runner.run(research_config, verbose=True)\n", - "# if result['returncode'] == 0:\n", - "# print(\"\\n✓ Research dataset generated successfully!\")\n", - "# print(f\"Files available in: {result['output_dir']}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 9. Tips and Best Practices\n", - "\n", - "### Reproducibility\n", - "- **Always use seeds**: Set a `seed` parameter for reproducible results\n", - "- **Document your configuration**: Save your `SyntheaConfig` for future reference\n", - "\n", - "### Performance\n", - "- **Start small**: Test with small populations (10-50) before running large simulations\n", - "- **Output directory**: Use separate directories for different experiments\n", - "- **Resource usage**: Large populations (1000+) can take significant time and disk space\n", - "\n", - "### Data Quality\n", - "- **Validate output**: Check that output files are generated correctly\n", - "- **Review demographics**: Ensure the generated population matches your expectations\n", - "- **Multiple runs**: Consider running multiple simulations with different seeds for robustness\n", - "\n", - "### Common Issues\n", - "1. **Java not found**: Ensure Java 11+ is installed and in PATH\n", - "2. **Out of memory**: Large populations may require increasing Java heap size\n", - "3. **Missing output**: Check that the output directory exists and is writable\n" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Java version check:\n", - "openjdk version \"17.0.14\" 2025-01-21 LTS\n", - "OpenJDK Runtime Environment (Red_Hat-17.0.14.0.7-1) (build 17.0.14+7-LTS)\n", - "OpenJDK 64-Bit Server VM (Red_Hat-17.0.14.0.7-1) (build 17.0.14+7-LTS, mixed mode, sharing)\n", - "\n", - "\n", - "Disk space:\n", - " Total: 7151.71 GB\n", - " Free: 2026.75 GB\n", - " Used: 5124.96 GB\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[autoreload of src.phenome.synthea failed: Traceback (most recent call last):\n", - " File \"/home/schilder/.conda/envs/AoU/lib/python3.14/site-packages/IPython/extensions/autoreload.py\", line 325, in check\n", - " superreload(m, reload, self.old_objects)\n", - " ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", - " File \"/home/schilder/.conda/envs/AoU/lib/python3.14/site-packages/IPython/extensions/autoreload.py\", line 580, in superreload\n", - " module = reload(module)\n", - " File \"/home/schilder/.conda/envs/AoU/lib/python3.14/importlib/__init__.py\", line 129, in reload\n", - " _bootstrap._exec(spec, module)\n", - " ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^\n", - " File \"\", line 869, in _exec\n", - " File \"\", line 755, in exec_module\n", - " File \"\", line 893, in get_code\n", - " File \"\", line 823, in source_to_code\n", - " File \"\", line 491, in _call_with_frames_removed\n", - " File \"/home/schilder/projects/AoU/src/phenome/synthea.py\", line 974\n", - " if __name__ == \"__main__\":\n", - " ^^\n", - "IndentationError: expected an indented block after function definition on line 967\n", - "]\n" - ] - } - ], - "source": [ - "# Example: Check system requirements\n", - "import subprocess\n", - "\n", - "# Check Java version\n", - "try:\n", - " result = subprocess.run(\n", - " [\"java\", \"-version\"],\n", - " stdout=subprocess.PIPE,\n", - " stderr=subprocess.STDOUT,\n", - " text=True\n", - " )\n", - " print(\"Java version check:\")\n", - " print(result.stdout)\n", - "except FileNotFoundError:\n", - " print(\"⚠️ Java not found! Please install Java 11 or newer.\")\n", - "\n", - "# Check available disk space\n", - "import shutil\n", - "total, used, free = shutil.disk_usage(\".\")\n", - "print(f\"\\nDisk space:\")\n", - "print(f\" Total: {total / (1024**3):.2f} GB\")\n", - "print(f\" Free: {free / (1024**3):.2f} GB\")\n", - "print(f\" Used: {used / (1024**3):.2f} GB\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 10. Integration with AoU Workflow\n", - "\n", - "The Synthea-generated data can be integrated into your All of Us (AoU) research workflow:\n", - "\n", - "1. **Generate synthetic data** using this wrapper\n", - "2. **Convert to OMOP format** if needed (Synthea has OMOP export capabilities)\n", - "3. **Load into your analysis pipeline** alongside real AoU data\n", - "4. **Use for testing** algorithms and pipelines before applying to real data\n", - "\n", - "### Example: OMOP Export\n", - "\n", - "Synthea can export data in OMOP CDM format, which is compatible with AoU:\n" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "OMOP export configuration example\n", - "Note: OMOP exporter availability depends on Synthea version\n" - ] - } - ], - "source": [ - "# Example: Generate data with OMOP export\n", - "# Note: OMOP exporter may need to be enabled in Synthea config\n", - "omop_config = SyntheaConfig(\n", - " population_size=100,\n", - " state=\"Massachusetts\",\n", - " seed=42,\n", - " output_dir=\"output/synthea_omop\",\n", - " exporter_flags={\n", - " \"exporter.omop.export\": \"true\",\n", - " \"exporter.csv.export\": \"true\" # Also keep CSV for reference\n", - " }\n", - ")\n", - "\n", - "# result = runner.run(omop_config)\n", - "print(\"OMOP export configuration example\")\n", - "print(\"Note: OMOP exporter availability depends on Synthea version\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Generating Synthea data...\n", - "\n", - "$ java -jar /home/schilder/.cache/synthea/synthea-with-dependencies.jar -s 42 -p 50 --exporter.csv.export=true Massachusetts\n", - "\n", - "✓ Synthea data generated successfully!\n", - "\n", - "Checking for CSV files in: /home/schilder/projects/AoU/notebooks/output/synthea_omop\n", - " ✓ Found CSV files in: /home/schilder/projects/AoU/notebooks/output/synthea_omop/output/csv\n", - " Found 18 CSV files:\n", - " - allergies.csv (14.81 KB)\n", - " - careplans.csv (50.17 KB)\n", - " - claims.csv (4281.77 KB)\n", - " - claims_transactions.csv (30508.74 KB)\n", - " - conditions.csv (449.13 KB)\n", - " - devices.csv (100.45 KB)\n", - " - encounters.csv (1911.00 KB)\n", - " - imaging_studies.csv (175.89 KB)\n", - " - immunizations.csv (130.48 KB)\n", - " - medications.csv (1439.13 KB)\n", - " ... and 8 more\n", - "\n", - "Converting to OMOP CDM format...\n", - "Converting Synthea CSV to OMOP CDM v5.4\n", - "Input directory: /home/schilder/projects/AoU/notebooks/output/synthea_omop/output/csv\n", - "Output directory: /home/schilder/projects/AoU/notebooks/output/synthea_omop/omop\n", - " Converting person table...\n", - " ✓ Created person table: 68 rows\n", - " Converting visit_occurrence table...\n", - " ✓ Created visit_occurrence table: 5819 rows\n", - " Converting condition_occurrence table...\n", - " ✓ Created condition_occurrence table: 2918 rows\n", - " Converting drug_exposure table...\n", - " ✓ Created drug_exposure table: 5586 rows\n", - " Converting procedure_occurrence table...\n", - " ✓ Created procedure_occurrence table: 13468 rows\n", - " Converting measurement table...\n", - " ✓ Created measurement table: 46817 rows\n", - " Converting observation table...\n", - " ✓ Created observation table: 26234 rows\n", - " Converting death table...\n", - " ✓ Created death table: 18 rows\n", - " Converting observation_period table...\n", - " ✓ Created observation_period table: 68 rows\n", - " Creating cdm_source table...\n", - " ✓ Created cdm_source table\n", - "\n", - "✓ Conversion complete!\n", - " Generated 10 OMOP tables\n", - " Output directory: /home/schilder/projects/AoU/notebooks/output/synthea_omop/omop\n", - "\n", - "⚠️ Note: concept_id fields are set to 0 and require vocabulary lookup\n", - " for full OMOP compliance. Use source_value fields for concept mapping.\n", - "\n", - "💡 To get concept names:\n", - " 1. Load OMOP vocabulary/concept table\n", - " 2. Join using *_source_value fields with concept.concept_code\n", - " 3. Or use add_concept_names() helper function\n", - "\n", - "✓ Conversion complete!\n", - "\n", - "Generated 10 OMOP tables:\n", - " - cdm_source: /home/schilder/projects/AoU/notebooks/output/synthea_omop/omop/cdm_source.parquet (5.27 KB)\n", - " - condition_occurrence: /home/schilder/projects/AoU/notebooks/output/synthea_omop/omop/condition_occurrence.parquet (68.75 KB)\n", - " - death: /home/schilder/projects/AoU/notebooks/output/synthea_omop/omop/death.parquet (2.96 KB)\n", - " - drug_exposure: /home/schilder/projects/AoU/notebooks/output/synthea_omop/omop/drug_exposure.parquet (124.93 KB)\n", - " - measurement: /home/schilder/projects/AoU/notebooks/output/synthea_omop/omop/measurement.parquet (410.81 KB)\n", - " - observation: /home/schilder/projects/AoU/notebooks/output/synthea_omop/omop/observation.parquet (120.15 KB)\n", - " - observation_period: /home/schilder/projects/AoU/notebooks/output/synthea_omop/omop/observation_period.parquet (2.93 KB)\n", - " - person: /home/schilder/projects/AoU/notebooks/output/synthea_omop/omop/person.parquet (9.01 KB)\n", - " - procedure_occurrence: /home/schilder/projects/AoU/notebooks/output/synthea_omop/omop/procedure_occurrence.parquet (175.03 KB)\n", - " - visit_occurrence: /home/schilder/projects/AoU/notebooks/output/synthea_omop/omop/visit_occurrence.parquet (198.83 KB)\n" - ] - } - ], - "source": [ - "# Actually run the OMOP conversion example\n", - "# First, generate Synthea data with CSV export enabled\n", - "from pathlib import Path\n", - "\n", - "omop_config = SyntheaConfig(\n", - " population_size=50, # Smaller for demo\n", - " state=\"Massachusetts\",\n", - " seed=42,\n", - " output_dir=\"output/synthea_omop\",\n", - " exporter_flags={\n", - " \"exporter.csv.export\": \"true\" # Ensure CSV export is enabled\n", - " }\n", - ")\n", - "\n", - "# Run the simulation\n", - "print(\"Generating Synthea data...\")\n", - "result = runner.run(omop_config, force=0, verbose=True)\n", - "\n", - "if result['returncode'] == 0:\n", - " print(\"\\n✓ Synthea data generated successfully!\")\n", - " \n", - " # Check what CSV files were generated\n", - " print(f\"\\nChecking for CSV files in: {result['output_dir']}\")\n", - " synthea_output = Path(result['output_dir'])\n", - " \n", - " # Synthea may create CSV files in a 'csv' subdirectory or directly in output\n", - " csv_dirs_to_check = [\n", - " synthea_output,\n", - " synthea_output / \"csv\",\n", - " synthea_output / \"output\" / \"csv\",\n", - " ]\n", - " \n", - " csv_dir = None\n", - " for check_dir in csv_dirs_to_check:\n", - " if check_dir.exists():\n", - " csv_files = list(check_dir.glob(\"*.csv\"))\n", - " if csv_files:\n", - " csv_dir = check_dir\n", - " print(f\" ✓ Found CSV files in: {csv_dir}\")\n", - " print(f\" Found {len(csv_files)} CSV files:\")\n", - " for csv_file in sorted(csv_files)[:10]: # Show first 10\n", - " size = csv_file.stat().st_size / 1024 # KB\n", - " print(f\" - {csv_file.name} ({size:.2f} KB)\")\n", - " if len(csv_files) > 10:\n", - " print(f\" ... and {len(csv_files) - 10} more\")\n", - " break\n", - " \n", - " if csv_dir is None:\n", - " print(\" ⚠️ No CSV files found! Synthea may not have generated CSV output.\")\n", - " print(\" Check that exporter.csv.export=true is set in Synthea configuration.\")\n", - " else:\n", - " # Now convert to OMOP format\n", - " print(f\"\\nConverting to OMOP CDM format...\")\n", - " \n", - " omop_files = runner.convert_to_omop(\n", - " synthea_output_dir=str(csv_dir), # Use the directory where CSV files actually are\n", - " omop_output_dir=os.path.join(result['output_dir'], \"omop\"),\n", - " cdm_version=\"5.4\",\n", - " output_format=\"parquet\",\n", - " verbose=True\n", - " )\n", - " \n", - " print(f\"\\n✓ Conversion complete!\")\n", - " print(f\"\\nGenerated {len(omop_files)} OMOP tables:\")\n", - " for table_name, file_path in sorted(omop_files.items()):\n", - " file_size = Path(file_path).stat().st_size / 1024 # KB\n", - " print(f\" - {table_name}: {file_path} ({file_size:.2f} KB)\")\n", - "else:\n", - " print(f\"\\n✗ Simulation failed with return code {result['returncode']}\")\n", - " print(f\"Error: {result['stderr']}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 12. Synthea to OMOP Conversion Methodology\n", - "\n", - "The `convert_synthea_to_omop()` function converts Synthea CSV output to OMOP CDM format, inspired by the [OHDSI ETL-Synthea](https://github.com/OHDSI/ETL-Synthea) R package. Here's how it works:\n", - "\n", - "### Conversion Process\n", - "\n", - "The conversion follows these steps:\n", - "\n", - "1. **Read Synthea CSV Files**: Loads Synthea-generated CSV files (patients, encounters, conditions, medications, procedures, observations)\n", - "\n", - "2. **Map to OMOP CDM Tables**: Transforms Synthea data into OMOP CDM table structures:\n", - " - **person**: Patient demographics (gender, birth date, etc.)\n", - " - **visit_occurrence**: Healthcare encounters/visits\n", - " - **condition_occurrence**: Diagnoses and conditions\n", - " - **drug_exposure**: Medications and prescriptions\n", - " - **procedure_occurrence**: Medical procedures\n", - " - **measurement**: Numeric lab results and measurements\n", - " - **observation**: Non-numeric observations\n", - " - **death**: Death records\n", - " - **observation_period**: Patient observation time periods\n", - " - **cdm_source**: Metadata about the data source\n", - "\n", - "3. **Field Mapping**: Maps Synthea fields to OMOP CDM fields:\n", - " - Synthea `Id` → OMOP `person_id`\n", - " - Synthea `GENDER` → OMOP `gender_concept_id` (8507=Male, 8532=Female)\n", - " - Synthea `ENCOUNTERCLASS` → OMOP `visit_concept_id` (9201=Inpatient, 9202=Outpatient, 9203=Emergency)\n", - " - Synthea `CODE` → OMOP `*_source_value` (preserves original SNOMED codes)\n", - " - Date fields converted to OMOP date/datetime format\n", - "\n", - "4. **Visit Linking**: Creates `visit_occurrence_id` mappings to link events (conditions, medications, etc.) to their associated visits\n", - "\n", - "5. **Observation Splitting**: Separates observations into:\n", - " - **measurement**: Numeric values (lab results, vitals)\n", - " - **observation**: Non-numeric values (text observations)\n", - "\n", - "6. **Output Generation**: Saves OMOP tables in Parquet or CSV format\n", - "\n", - "### Key Features\n", - "\n", - "- **Preserves Source Values**: Original Synthea codes (typically SNOMED) are preserved in `*_source_value` fields\n", - "- **Concept ID Placeholders**: `concept_id` fields are set to 0 and require vocabulary lookup for full OMOP compliance\n", - "- **Visit Relationships**: Automatically links events to their associated visits\n", - "- **Flexible Output**: Supports both Parquet (default) and CSV formats\n", - "\n", - "### Important Notes\n", - "\n", - "⚠️ **Concept Mapping**: The function sets `concept_id` fields to 0. For full OMOP compliance, you'll need to:\n", - "- Load OMOP vocabulary tables (concept, concept_relationship, etc.)\n", - "- Map Synthea source codes (in `*_source_value` fields) to OMOP concept_ids using vocabulary lookups\n", - "- Update the `concept_id` fields with the mapped values\n", - "\n", - "⚠️ **Vocabulary Tables**: OMOP vocabulary tables (concept, vocabulary, concept_relationship, etc.) are not generated by this function and must be loaded separately from OMOP vocabulary releases.\n", - "\n", - "### Usage\n", - "\n", - "```python\n", - "# Method 1: Using the convenience method\n", - "omop_files = runner.convert_to_omop(\n", - " synthea_output_dir=\"output/synthea_data\",\n", - " omop_output_dir=\"output/omop_data\"\n", - ")\n", - "\n", - "# Method 2: Using the standalone function\n", - "from AoU.phenome.synthea import convert_synthea_to_omop\n", - "omop_files = convert_synthea_to_omop(\n", - " synthea_csv_dir=\"output/synthea_data\",\n", - " output_dir=\"output/omop_data\",\n", - " cdm_version=\"5.4\",\n", - " output_format=\"parquet\"\n", - ")\n", - "```\n", - "\n", - "### References\n", - "\n", - "- [OHDSI ETL-Synthea R Package](https://github.com/OHDSI/ETL-Synthea)\n", - "- [OMOP CDM Documentation](https://ohdsi.github.io/CommonDataModel/)\n", - "- [Synthea GitHub](https://github.com/synthetichealth/synthea)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Generate 10 patients in Massachusetts\n", - "result = runner.run_quick(\n", - " population_size=10,\n", - " state=\"Massachusetts\",\n", - " seed=42, # For reproducibility\n", - " output_dir=\"output/synthea_quick\"\n", - ")\n", - "\n", - "print(f\"\\nReturn code: {result['returncode']}\")\n", - "print(f\"Output directory: {result['output_dir']}\")\n", - "print(f\"\\nCommand executed:\")\n", - "print(result['command'])\n", - "# print(result[\"stdout\"])\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 11. Convert OMOP to Timeline\n" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Timeline configuration:\n", - " OMOP directory: output/synthea_omop/OMOP\n", - " Output timeline: output/synthea_omop/timelines_calendar_1y.parquet\n", - " Bin size: 1y\n", - " Bin mode: calendar\n", - "\n", - "✓ Found OMOP directory: output/synthea_omop/OMOP\n", - " Found 11 OMOP tables\n", - "✓ Concept table found - ready to build timelines\n", - "\n", - "============================================================\n", - "Timeline Configuration Summary\n", - "============================================================\n", - "Cohort: synthea_omop\n", - "OMOP Root: output\n", - "Time Binning: calendar mode, 1y bins\n", - "Output Format: Markdown\n", - "Event Tables: 6 tables\n", - "Era Tables: 1 tables\n" - ] - } - ], - "source": [ - "from AoU.phenome import timeline\n", - "from AoU.phenome.timeline import TimelineConfig, build_omop_timelines\n", - "import polars as pl\n", - "\n", - "# Configure timeline generation for Synthea OMOP data\n", - "# The timeline module converts OMOP CDM tables into patient timelines\n", - "# organized by time bins (e.g., yearly, monthly, or by visit)\n", - "\n", - "# Set up configuration\n", - "# Note: The timeline module expects OMOP data in: {omop_root}/{cohort}/OMOP/\n", - "# Our Synthea OMOP data is in: output/synthea_omop/omop/\n", - "# Configure timeline generation for Synthea OMOP data\n", - "# Note: Synthea only generates a subset of OMOP tables, so we exclude missing ones\n", - "timeline_cfg = TimelineConfig(\n", - " cohort=\"synthea_omop\", # Cohort name (matches directory)\n", - " omop_root=\"output\", # Root directory containing cohort folders\n", - " bin_size=\"1y\", # Time bin size: \"1y\", \"1mo\", \"1w\", etc.\n", - " bin_mode=\"calendar\", # \"calendar\" or \"visit\"\n", - " \n", - " # Output paths (auto-generated if None)\n", - " out_parquet=\"output/synthea_omop/timelines_calendar_1y.parquet\",\n", - " events_parquet=\"output/synthea_omop/events_unified.parquet\",\n", - " eras_parquet=\"output/synthea_omop/eras_periods.parquet\",\n", - " \n", - " # Only include tables that Synthea actually generates\n", - " # Available: condition_occurrence, drug_exposure, measurement, \n", - " # observation, procedure_occurrence, death\n", - " # Missing: device_exposure, specimen, survey_conduct, visit_detail\n", - " main_event_tables=(\n", - " \"condition_occurrence\",\n", - " \"drug_exposure\",\n", - " \"measurement\",\n", - " \"observation\",\n", - " \"procedure_occurrence\",\n", - " \"death\",\n", - " ),\n", - " \n", - " # Timeline formatting options\n", - " markdown=True, # Generate markdown-formatted timelines\n", - " visit_markdown_headings=True, # Use visit-level headings in markdown\n", - " include_interval_summary=True, # Add YAML-style interval summaries\n", - " include_src_name=True, # Include source concept names\n", - " \n", - " # Force rebuild (0=use cache, 1=rebuild if config differs, 2=always rebuild)\n", - " force=0\n", - ")\n", - "\n", - "print(\"Timeline configuration:\")\n", - "print(f\" OMOP directory: {timeline_cfg.omop_dir}\")\n", - "print(f\" Output timeline: {timeline_cfg.out_parquet}\")\n", - "print(f\" Bin size: {timeline_cfg.bin_size}\")\n", - "print(f\" Bin mode: {timeline_cfg.bin_mode}\")\n", - "print()\n", - "\n", - "# Check if OMOP directory exists\n", - "import os\n", - "omop_dir = timeline_cfg.omop_dir\n", - "if os.path.exists(omop_dir):\n", - " print(f\"✓ Found OMOP directory: {omop_dir}\")\n", - " omop_files = [f for f in os.listdir(omop_dir) if f.endswith('.parquet')]\n", - " print(f\" Found {len(omop_files)} OMOP tables\")\n", - " \n", - " # Check for required concept table\n", - " if 'concept.parquet' not in omop_files:\n", - " print(\"\\n⚠️ Note: 'concept.parquet' not found in OMOP directory.\")\n", - " print(\" The timeline module requires a concept table for concept name lookups.\")\n", - " print(\" For Synthea data, you may need to:\")\n", - " print(\" 1. Download OMOP vocabulary tables from OHDSI\")\n", - " print(\" 2. Or create a minimal concept table from source values\")\n", - " print(\"\\n For demonstration, we'll show the configuration setup.\")\n", - " print(\" To actually run, you'll need to add a concept table.\")\n", - " else:\n", - " print(\"✓ Concept table found - ready to build timelines\")\n", - "else:\n", - " print(f\"✗ OMOP directory not found: {omop_dir}\")\n", - " print(\" Please run the OMOP conversion example first (Section 10)\")\n", - "\n", - "print(\"\\n\" + \"=\"*60)\n", - "print(\"Timeline Configuration Summary\")\n", - "print(\"=\"*60)\n", - "print(f\"Cohort: {timeline_cfg.cohort}\")\n", - "print(f\"OMOP Root: {timeline_cfg.omop_root}\")\n", - "print(f\"Time Binning: {timeline_cfg.bin_mode} mode, {timeline_cfg.bin_size} bins\")\n", - "print(f\"Output Format: {'Markdown' if timeline_cfg.markdown else 'Plain text'}\")\n", - "print(f\"Event Tables: {len(timeline_cfg.main_event_tables)} tables\")\n", - "print(f\"Era Tables: {len(timeline_cfg.era_tables)} tables\")" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "✓ Concept table already exists: output/synthea_omop/OMOP/concept.parquet\n", - "\n", - "============================================================\n", - "Ready to build timelines!\n", - "============================================================\n", - "\n", - "To build timelines, run:\n", - " timelines_df, eras_df, survey_df = build_omop_timelines(timeline_cfg)\n", - "\n", - "This will:\n", - " 1. Load OMOP tables and concept mappings\n", - " 2. Build unified event table from all clinical domains\n", - " 3. Bin events by time (yearly in this example)\n", - " 4. Aggregate into markdown-formatted patient timelines\n", - " 5. Save results to parquet files\n" - ] - } - ], - "source": [ - "# Example: Building timelines from OMOP data\n", - "# \n", - "# To actually run the timeline conversion, you need a concept table.\n", - "# Here's how to create a minimal concept table from Synthea source values:\n", - "\n", - "import polars as pl\n", - "import os\n", - "\n", - "omop_dir = timeline_cfg.omop_dir\n", - "concept_path = os.path.join(omop_dir, \"concept.parquet\")\n", - "\n", - "# Ensure the OMOP directory exists (TimelineConfig expects uppercase OMOP)\n", - "os.makedirs(omop_dir, exist_ok=True)\n", - "\n", - "# If the expected OMOP directory doesn't have files, check for lowercase 'omop' directory\n", - "actual_omop_dir = os.path.join(timeline_cfg.omop_root, timeline_cfg.cohort, \"omop\")\n", - "if os.path.exists(actual_omop_dir):\n", - " # Check if we need to copy files from lowercase to uppercase directory\n", - " if not os.path.exists(omop_dir) or len([f for f in os.listdir(omop_dir) if f.endswith('.parquet')]) == 0:\n", - " import shutil\n", - " print(f\"Copying OMOP files from {actual_omop_dir} to {omop_dir}...\")\n", - " for file in os.listdir(actual_omop_dir):\n", - " if file.endswith('.parquet'):\n", - " src = os.path.join(actual_omop_dir, file)\n", - " dst = os.path.join(omop_dir, file)\n", - " if not os.path.exists(dst):\n", - " shutil.copy2(src, dst)\n", - " print(f\"✓ Copied OMOP files to {omop_dir}\")\n", - "\n", - "if not os.path.exists(concept_path):\n", - " print(\"Creating minimal concept table from Synthea source values...\")\n", - " print(\"(In production, use full OMOP vocabulary tables from OHDSI)\\n\")\n", - " \n", - " concept_data = []\n", - " seen_concepts = set()\n", - " \n", - " # Helper to add concepts from a table\n", - " def add_concepts_from_table(table_name, source_id_col, source_value_col, source_name_col, vocab_id, class_id):\n", - " table_path = os.path.join(omop_dir, f\"{table_name}.parquet\")\n", - " if not os.path.exists(table_path):\n", - " return\n", - " \n", - " df = pl.read_parquet(table_path)\n", - " if source_id_col not in df.columns or source_value_col not in df.columns:\n", - " return\n", - " \n", - " # Get unique combinations\n", - " unique_df = df.select([source_id_col, source_value_col, source_name_col]).unique()\n", - " \n", - " for row in unique_df.iter_rows():\n", - " concept_id, source_value, source_name = row[0], row[1], row[2] if len(row) > 2 else None\n", - " \n", - " # Skip if concept_id is 0 or we've seen this combination\n", - " if concept_id == 0 or (concept_id, source_value) in seen_concepts:\n", - " continue\n", - " \n", - " seen_concepts.add((concept_id, source_value))\n", - " concept_data.append({\n", - " \"concept_id\": concept_id,\n", - " \"concept_name\": source_name or f\"Source: {source_value}\",\n", - " \"concept_code\": str(source_value) if source_value else \"\",\n", - " \"vocabulary_id\": vocab_id,\n", - " \"concept_class_id\": class_id,\n", - " \"standard_concept\": \"S\",\n", - " \"invalid_reason\": None\n", - " })\n", - " \n", - " # Collect concepts from various tables\n", - " add_concepts_from_table(\"condition_occurrence\", \"condition_source_concept_id\", \n", - " \"condition_source_value\", \"condition_source_concept_name\", \n", - " \"Synthea\", \"Condition\")\n", - " add_concepts_from_table(\"drug_exposure\", \"drug_source_concept_id\",\n", - " \"drug_source_value\", \"drug_source_concept_name\",\n", - " \"Synthea\", \"Drug\")\n", - " add_concepts_from_table(\"procedure_occurrence\", \"procedure_source_concept_id\",\n", - " \"procedure_source_value\", \"procedure_source_concept_name\",\n", - " \"Synthea\", \"Procedure\")\n", - " add_concepts_from_table(\"measurement\", \"measurement_source_concept_id\",\n", - " \"measurement_source_value\", \"measurement_source_concept_name\",\n", - " \"Synthea\", \"Measurement\")\n", - " \n", - " # Add standard OMOP concept IDs (gender, visit types, etc.)\n", - " standard_concepts = [\n", - " (8507, \"Male\", \"M\", \"Gender\", \"Gender\"),\n", - " (8532, \"Female\", \"F\", \"Gender\", \"Gender\"),\n", - " (9201, \"Inpatient Visit\", \"IP\", \"Visit\", \"Visit\"),\n", - " (9202, \"Outpatient Visit\", \"OP\", \"Visit\", \"Visit\"),\n", - " (9203, \"Emergency Room Visit\", \"ER\", \"Visit\", \"Visit\"),\n", - " ]\n", - " \n", - " for concept_id, name, code, vocab, class_id in standard_concepts:\n", - " if concept_id not in {c[\"concept_id\"] for c in concept_data}:\n", - " concept_data.append({\n", - " \"concept_id\": concept_id,\n", - " \"concept_name\": name,\n", - " \"concept_code\": code,\n", - " \"vocabulary_id\": vocab,\n", - " \"concept_class_id\": class_id,\n", - " \"standard_concept\": \"S\",\n", - " \"invalid_reason\": None\n", - " })\n", - " \n", - " # Create minimal concept table\n", - " if concept_data:\n", - " concept_df = pl.DataFrame(concept_data)\n", - " # Add required OMOP concept table columns\n", - " concept_df = concept_df.with_columns([\n", - " pl.lit(4180186).alias(\"language_concept_id\"), # English\n", - " pl.lit(None).cast(pl.Date).alias(\"valid_start_date\"),\n", - " pl.lit(None).cast(pl.Date).alias(\"valid_end_date\"),\n", - " ])\n", - " \n", - " # Ensure proper column order\n", - " required_cols = [\n", - " \"concept_id\", \"concept_name\", \"concept_code\", \"vocabulary_id\",\n", - " \"concept_class_id\", \"standard_concept\", \"invalid_reason\",\n", - " \"language_concept_id\", \"valid_start_date\", \"valid_end_date\"\n", - " ]\n", - " concept_df = concept_df.select([c for c in required_cols if c in concept_df.columns])\n", - " \n", - " concept_df.write_parquet(concept_path)\n", - " print(f\"✓ Created minimal concept table with {len(concept_df)} concepts\")\n", - " print(f\" Saved to: {concept_path}\")\n", - " else:\n", - " print(\"⚠️ Could not create concept table - no source values found\")\n", - " print(\" Note: Synthea OMOP tables may have concept_id=0 for all concepts\")\n", - " print(\" In this case, you'll need to download OMOP vocabulary tables from OHDSI\")\n", - "else:\n", - " print(f\"✓ Concept table already exists: {concept_path}\")\n", - "\n", - "print(\"\\n\" + \"=\"*60)\n", - "print(\"Ready to build timelines!\")\n", - "print(\"=\"*60)\n", - "print(\"\\nTo build timelines, run:\")\n", - "print(\" timelines_df, eras_df, survey_df = build_omop_timelines(timeline_cfg)\")\n", - "print(\"\\nThis will:\")\n", - "print(\" 1. Load OMOP tables and concept mappings\")\n", - "print(\" 2. Build unified event table from all clinical domains\")\n", - "print(\" 3. Bin events by time (yearly in this example)\")\n", - "print(\" 4. Aggregate into markdown-formatted patient timelines\")\n", - "print(\" 5. Save results to parquet files\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[95m🚀 Starting OMOP → timeline pipeline…\u001b[0m\n", - "\u001b[95m🔹 Step 1/8: Loading OMOP dictionary from Google Sheets\u001b[0m\n", - "\u001b[95m📖 Loading OMOP dictionary from Google Sheets…\u001b[0m\n", - "\u001b[95m → https://docs.google.com/spreadsheets/d/1sPMzOod784PAR0RfjuDezoT15z7w1e0RZsMVlpT2378/export?format=csv&gid=1815943286\u001b[0m\n", - "\u001b[92m✅ Loaded dictionary with 319 rows, 9 columns.\u001b[0m\n", - "\u001b[93m⏱ Step 1 finished in 0.4 s\u001b[0m\n", - "\u001b[94m🔹 Step 2/8: Preparing unified OMOP event table\u001b[0m\n", - "\u001b[94m💡 Loading concept table (concept_id → name/code)…\u001b[0m\n", - "\u001b[94m📦 Building events from raw OMOP tables (will overwrite events cache)…\u001b[0m\n", - "\u001b[96m🧱 Building domain-specific events (lazy)…\u001b[0m\n", - "\u001b[92m🩺 Building condition events from condition_occurrence.parquet…\u001b[0m\n", - "\u001b[92m💊 Building drug events from drug_exposure.parquet…\u001b[0m\n", - "\u001b[92m🧪 Building measurement events from measurement.parquet…\u001b[0m\n", - "\u001b[92m📝 Building observation events from observation.parquet…\u001b[0m\n", - "\u001b[92m🔧 Building procedure events from procedure_occurrence.parquet…\u001b[0m\n", - "\u001b[91m💀 Building death events from death.parquet…\u001b[0m\n", - "\u001b[96m🧱 Concatenating all domain events into a single lazy frame…\u001b[0m\n", - "\u001b[93m📥 Collecting events into memory (may take a bit)…\u001b[0m\n", - "\u001b[92m✅ Events table ready with 95,041 rows and 68 unique persons.\u001b[0m\n", - "\u001b[93m💾 Saved unified events to output/synthea_omop/events_unified.parquet\u001b[0m\n", - "\u001b[93m⏱ Step 2 finished in 0.0 s\u001b[0m\n", - "\u001b[93m🔹 Step 3/8: Loading person DOBs and restricting to known persons\u001b[0m\n", - "\u001b[93m👶 Loaded DOBs for 68 individuals from person.parquet.\u001b[0m\n", - "\u001b[93m⏱ Step 3 finished in 0.0 s\u001b[0m\n", - "\u001b[93m🔹 Step 4/8: Adding age at each event_date\u001b[0m\n", - "\u001b[93m🎂 Adding age at event_date from person.birth info…\u001b[0m\n", - "\u001b[93m⏱ Step 4 finished in 0.0 s\u001b[0m\n", - "\u001b[96m🔹 Step 5/8: Adding visit metadata and binning events\u001b[0m\n", - "\u001b[94m🏥 Loading visit metadata…\u001b[0m\n" - ] - }, - { - "ename": "SchemaError", - "evalue": "datatypes of join keys don't match - `visit_concept_id`: f64 on left does not match `visit_concept_id`: i64 on right (and no other type was available to cast to)\n\nResolved plan until failure:\n\n\t---> FAILED HERE RESOLVING 'sink' <---\nSELECT [col(\"concept_id\").alias(\"visit_concept_id\"), col(\"concept_name\").alias(\"visit_type_name\"), col(\"concept_code\").alias(\"visit_type_code\")]\n DF [\"concept_id\", \"concept_name\", \"concept_code\"]; PROJECT */3 COLUMNS", - "output_type": "error", - "traceback": [ - "\u001b[31m---------------------------------------------------------------------------\u001b[39m", - "\u001b[31mSchemaError\u001b[39m Traceback (most recent call last)", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[9]\u001b[39m\u001b[32m, line 5\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;66;03m# Actually build the timelines\u001b[39;00m\n\u001b[32m 2\u001b[39m \u001b[38;5;66;03m# Uncomment the code below to run the timeline conversion\u001b[39;00m\n\u001b[32m 3\u001b[39m \n\u001b[32m 4\u001b[39m \u001b[38;5;66;03m# Build timelines from OMOP data\u001b[39;00m\n\u001b[32m----> \u001b[39m\u001b[32m5\u001b[39m timelines_df, eras_df, survey_df = \u001b[43mbuild_omop_timelines\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtimeline_cfg\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 7\u001b[39m \u001b[38;5;66;03m# The function returns:\u001b[39;00m\n\u001b[32m 8\u001b[39m \u001b[38;5;66;03m# - timelines_df: Main timeline table with one row per (person_id, time_bin)\u001b[39;00m\n\u001b[32m 9\u001b[39m \u001b[38;5;66;03m# - eras_df: Era/period table (condition_era, drug_era, observation_period)\u001b[39;00m\n\u001b[32m 10\u001b[39m \u001b[38;5;66;03m# - survey_df: Survey responses (if available)\u001b[39;00m\n\u001b[32m 11\u001b[39m \n\u001b[32m 12\u001b[39m \u001b[38;5;66;03m# Example: View a sample timeline\u001b[39;00m\n\u001b[32m 13\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m timelines_df \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(timelines_df) > \u001b[32m0\u001b[39m:\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/projects/AoU/AoU/phenome/timeline.py:2258\u001b[39m, in \u001b[36mbuild_omop_timelines\u001b[39m\u001b[34m(cfg)\u001b[39m\n\u001b[32m 2256\u001b[39m visit_meta_df = \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m 2257\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m cfg.include_visit_metadata \u001b[38;5;129;01mor\u001b[39;00m cfg.bin_mode == \u001b[33m\"\u001b[39m\u001b[33mvisit\u001b[39m\u001b[33m\"\u001b[39m:\n\u001b[32m-> \u001b[39m\u001b[32m2258\u001b[39m visit_meta_df = \u001b[43mload_visit_metadata\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcfg\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43momop_dd\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconcept_df\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 2259\u001b[39m \u001b[38;5;66;03m# Join visit metadata fields (e.g. visit_type_name) for downstream text\u001b[39;00m\n\u001b[32m 2260\u001b[39m events_df = events_df.join(\n\u001b[32m 2261\u001b[39m visit_meta_df.select(\n\u001b[32m 2262\u001b[39m [\u001b[33m\"\u001b[39m\u001b[33mperson_id\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33mvisit_occurrence_id\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33mvisit_type_name\u001b[39m\u001b[33m\"\u001b[39m]\n\u001b[32m (...)\u001b[39m\u001b[32m 2265\u001b[39m how=\u001b[33m\"\u001b[39m\u001b[33mleft\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 2266\u001b[39m )\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/projects/AoU/AoU/phenome/timeline.py:413\u001b[39m, in \u001b[36mload_visit_metadata\u001b[39m\u001b[34m(cfg, omop_dd, concept_df)\u001b[39m\n\u001b[32m 391\u001b[39m visit_concepts = concept_lf.rename(\n\u001b[32m 392\u001b[39m {\n\u001b[32m 393\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mconcept_id\u001b[39m\u001b[33m\"\u001b[39m: \u001b[33m\"\u001b[39m\u001b[33mvisit_concept_id\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m (...)\u001b[39m\u001b[32m 396\u001b[39m }\n\u001b[32m 397\u001b[39m )\n\u001b[32m 399\u001b[39m visit_lf = (\n\u001b[32m 400\u001b[39m lf\n\u001b[32m 401\u001b[39m .select(\n\u001b[32m (...)\u001b[39m\u001b[32m 410\u001b[39m .join(visit_concepts, on=\u001b[33m\"\u001b[39m\u001b[33mvisit_concept_id\u001b[39m\u001b[33m\"\u001b[39m, how=\u001b[33m\"\u001b[39m\u001b[33mleft\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m 411\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m413\u001b[39m visit_df = \u001b[43mvisit_lf\u001b[49m\u001b[43m.\u001b[49m\u001b[43mcollect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 414\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m visit_df\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/AoU/lib/python3.14/site-packages/polars/_utils/deprecation.py:97\u001b[39m, in \u001b[36mdeprecate_streaming_parameter..decorate..wrapper\u001b[39m\u001b[34m(*args, **kwargs)\u001b[39m\n\u001b[32m 93\u001b[39m kwargs[\u001b[33m\"\u001b[39m\u001b[33mengine\u001b[39m\u001b[33m\"\u001b[39m] = \u001b[33m\"\u001b[39m\u001b[33min-memory\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 95\u001b[39m \u001b[38;5;28;01mdel\u001b[39;00m kwargs[\u001b[33m\"\u001b[39m\u001b[33mstreaming\u001b[39m\u001b[33m\"\u001b[39m]\n\u001b[32m---> \u001b[39m\u001b[32m97\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunction\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/AoU/lib/python3.14/site-packages/polars/lazyframe/opt_flags.py:328\u001b[39m, in \u001b[36mforward_old_opt_flags..decorate..wrapper\u001b[39m\u001b[34m(*args, **kwargs)\u001b[39m\n\u001b[32m 325\u001b[39m optflags = cb(optflags, kwargs.pop(key)) \u001b[38;5;66;03m# type: ignore[no-untyped-call,unused-ignore]\u001b[39;00m\n\u001b[32m 327\u001b[39m kwargs[\u001b[33m\"\u001b[39m\u001b[33moptimizations\u001b[39m\u001b[33m\"\u001b[39m] = optflags\n\u001b[32m--> \u001b[39m\u001b[32m328\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunction\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/AoU/lib/python3.14/site-packages/polars/lazyframe/frame.py:2429\u001b[39m, in \u001b[36mLazyFrame.collect\u001b[39m\u001b[34m(self, type_coercion, predicate_pushdown, projection_pushdown, simplify_expression, slice_pushdown, comm_subplan_elim, comm_subexpr_elim, cluster_with_columns, collapse_joins, no_optimization, engine, background, optimizations, **_kwargs)\u001b[39m\n\u001b[32m 2427\u001b[39m \u001b[38;5;66;03m# Only for testing purposes\u001b[39;00m\n\u001b[32m 2428\u001b[39m callback = _kwargs.get(\u001b[33m\"\u001b[39m\u001b[33mpost_opt_callback\u001b[39m\u001b[33m\"\u001b[39m, callback)\n\u001b[32m-> \u001b[39m\u001b[32m2429\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m wrap_df(\u001b[43mldf\u001b[49m\u001b[43m.\u001b[49m\u001b[43mcollect\u001b[49m\u001b[43m(\u001b[49m\u001b[43mengine\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcallback\u001b[49m\u001b[43m)\u001b[49m)\n", - "\u001b[31mSchemaError\u001b[39m: datatypes of join keys don't match - `visit_concept_id`: f64 on left does not match `visit_concept_id`: i64 on right (and no other type was available to cast to)\n\nResolved plan until failure:\n\n\t---> FAILED HERE RESOLVING 'sink' <---\nSELECT [col(\"concept_id\").alias(\"visit_concept_id\"), col(\"concept_name\").alias(\"visit_type_name\"), col(\"concept_code\").alias(\"visit_type_code\")]\n DF [\"concept_id\", \"concept_name\", \"concept_code\"]; PROJECT */3 COLUMNS" - ] - } - ], - "source": [ - "# Actually build the timelines\n", - "# Uncomment the code below to run the timeline conversion\n", - "\n", - "# Build timelines from OMOP data\n", - "timelines_df, eras_df, survey_df = build_omop_timelines(timeline_cfg)\n", - "\n", - "# The function returns:\n", - "# - timelines_df: Main timeline table with one row per (person_id, time_bin)\n", - "# - eras_df: Era/period table (condition_era, drug_era, observation_period)\n", - "# - survey_df: Survey responses (if available)\n", - "\n", - "# Example: View a sample timeline\n", - "if timelines_df is not None and len(timelines_df) > 0:\n", - " print(f\"\\n✓ Built timelines for {timelines_df['person_id'].n_unique()} patients\")\n", - " print(f\" Total timeline segments: {len(timelines_df)}\")\n", - " print(f\"\\nSample timeline (first patient, first time bin):\")\n", - " print(\"=\"*60)\n", - " sample = timelines_df.head(1)\n", - " print(f\"Person ID: {sample['person_id'][0]}\")\n", - " print(f\"Time Bin: {sample['time_bin'][0]}\")\n", - " print(f\"Age Range: {sample['age_min'][0]:.1f} - {sample['age_max'][0]:.1f} years\")\n", - " print(f\"\\nTimeline Text (first 500 chars):\")\n", - " print(sample['timeline_text'][0][:500] + \"...\")\n", - " print(\"=\"*60)\n", - "\n", - "print(\"To run the timeline conversion, uncomment the code above.\")\n", - "print(\"\\nThe timeline conversion will:\")\n", - "print(\" • Load all OMOP tables from the configured directory\")\n", - "print(\" • Map concept IDs to concept names using the concept table\")\n", - "print(\" • Build unified event table from all clinical domains\")\n", - "print(\" • Add age at event for each event\")\n", - "print(\" • Bin events by time (yearly, monthly, or by visit)\")\n", - "print(\" • Aggregate into readable patient timelines\")\n", - "print(\" • Save results as parquet files for downstream use\")\n", - "print(\"\\nTimeline outputs:\")\n", - "print(f\" • Main timelines: {timeline_cfg.out_parquet}\")\n", - "print(f\" • Unified events: {timeline_cfg.events_parquet}\")\n", - "if timeline_cfg.build_eras:\n", - " print(f\" • Eras/periods: {timeline_cfg.eras_parquet}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 12. Additional Resources\n", - "\n", - "- **Synthea GitHub**: https://github.com/synthetichealth/synthea\n", - "- **Synthea Wiki**: https://github.com/synthetichealth/synthea/wiki\n", - "- **Synthea Documentation**: Comprehensive guides on modules, exporters, and configuration\n", - "- **Module Gallery**: Pre-built modules for various conditions and scenarios\n", - "\n", - "## Summary\n", - "\n", - "This tutorial covered:\n", - "- ✓ Initializing the SyntheaRunner\n", - "- ✓ Running quick simulations\n", - "- ✓ Using full configuration options\n", - "- ✓ Convenience methods for common use cases\n", - "- ✓ Advanced configuration options\n", - "- ✓ Working with output files\n", - "- ✓ Best practices and tips\n", - "\n", - "You're now ready to generate synthetic patient data for your research!\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Final example: Quick test run\n", - "print(\"Running final test simulation...\")\n", - "print(\"=\" * 50)\n", - "\n", - "test_result = runner.run_quick(\n", - " population_size=5,\n", - " state=\"Massachusetts\",\n", - " seed=999,\n", - " output_dir=\"output/synthea_tutorial_test\"\n", - ")\n", - "\n", - "if test_result['returncode'] == 0:\n", - " print(\"\\n✓ Tutorial test completed successfully!\")\n", - " print(f\"✓ Output directory: {test_result['output_dir']}\")\n", - " print(\"\\nYou can now explore the generated files in the output directory.\")\n", - "else:\n", - " print(f\"\\n✗ Test failed. Check error messages above.\")\n", - " print(f\"Return code: {test_result['returncode']}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## ETLSyntheaBuilder" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:rpy2.situation:cffi mode is CFFI_MODE.ANY\n", - "INFO:rpy2.situation:R home found: /home/schilder/.conda/envs/AoU/lib/R\n", - "INFO:rpy2.situation:R library path: /usr/local/cuda/lib64:/usr/local/cuda/lib64:\n", - "INFO:rpy2.situation:LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/lib64:\n", - "INFO:rpy2.rinterface_lib.embedded:Default options to initialize R: rpy2, --quiet, --no-save\n", - "INFO:rpy2.rinterface:Environment variable \"PWD\" redefined by R and overriding existing variable. Current: \"/home/schilder\", R: \"/home/schilder/projects/AoU/notebooks\"\n", - "INFO:rpy2.rinterface:R is already initialized. No need to initialize.\n" - ] - } - ], - "source": [ - "%load_ext rpy2.ipython" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error in `DatabaseConnector::downloadJdbcDrivers()`:\n", - "! The pathToDriver argument must be specified. Consider setting the DATABASECONNECTOR_JAR_FOLDER environment variable, for example in the .Renviron file.\n", - "Run `rlang::last_trace()` to see where the error occurred.\n", - "\n", - "Error in DatabaseConnector::downloadJdbcDrivers(\"postgresql\") :\n" - ] - }, - { - "ename": "RInterpreterError", - "evalue": "Failed to parse and evaluate line 'DatabaseConnector::downloadJdbcDrivers(\"postgresql\")\\n'.\nR error message: 'Error in DatabaseConnector::downloadJdbcDrivers(\"postgresql\") :'", - "output_type": "error", - "traceback": [ - "\u001b[31m---------------------------------------------------------------------------\u001b[39m", - "\u001b[31mRRuntimeError\u001b[39m Traceback (most recent call last)", - "\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/AoU/lib/python3.14/site-packages/rpy2/ipython/rmagic.py:768\u001b[39m, in \u001b[36mRMagics.eval\u001b[39m\u001b[34m(self, code)\u001b[39m\n\u001b[32m 767\u001b[39m r_expr = ri.parse(code)\n\u001b[32m--> \u001b[39m\u001b[32m768\u001b[39m value, visible = \u001b[43mri\u001b[49m\u001b[43m.\u001b[49m\u001b[43mevalr_expr_with_visible\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 769\u001b[39m \u001b[43m \u001b[49m\u001b[43mr_expr\u001b[49m\n\u001b[32m 770\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 771\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m (ri.embedded.RRuntimeError, \u001b[38;5;167;01mValueError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m exception:\n\u001b[32m 772\u001b[39m \u001b[38;5;66;03m# Otherwise next return seems to have copy of error.\u001b[39;00m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/AoU/lib/python3.14/site-packages/rpy2/rinterface/__init__.py:205\u001b[39m, in \u001b[36mevalr_expr_with_visible\u001b[39m\u001b[34m(expr, envir)\u001b[39m\n\u001b[32m 204\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m error_occured[\u001b[32m0\u001b[39m]:\n\u001b[32m--> \u001b[39m\u001b[32m205\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m embedded.RRuntimeError(_rinterface._geterrmessage())\n\u001b[32m 206\u001b[39m res = conversion._cdata_to_rinterface(r_res)\n", - "\u001b[31mRRuntimeError\u001b[39m: Error in DatabaseConnector::downloadJdbcDrivers(\"postgresql\") : \n", - "\nDuring handling of the above exception, another exception occurred:\n", - "\u001b[31mRInterpreterError\u001b[39m Traceback (most recent call last)", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[11]\u001b[39m\u001b[32m, line 1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m \u001b[43mget_ipython\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrun_cell_magic\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43mR\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43mDatabaseConnector::downloadJdbcDrivers(\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mpostgresql\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m)\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/AoU/lib/python3.14/site-packages/IPython/core/interactiveshell.py:2572\u001b[39m, in \u001b[36mInteractiveShell.run_cell_magic\u001b[39m\u001b[34m(self, magic_name, line, cell)\u001b[39m\n\u001b[32m 2570\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m.builtin_trap:\n\u001b[32m 2571\u001b[39m args = (magic_arg_s, cell)\n\u001b[32m-> \u001b[39m\u001b[32m2572\u001b[39m result = \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 2574\u001b[39m \u001b[38;5;66;03m# The code below prevents the output from being displayed\u001b[39;00m\n\u001b[32m 2575\u001b[39m \u001b[38;5;66;03m# when using magics with decorator @output_can_be_silenced\u001b[39;00m\n\u001b[32m 2576\u001b[39m \u001b[38;5;66;03m# when the last Python token in the expression is a ';'.\u001b[39;00m\n\u001b[32m 2577\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mgetattr\u001b[39m(fn, magic.MAGIC_OUTPUT_CAN_BE_SILENCED, \u001b[38;5;28;01mFalse\u001b[39;00m):\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/AoU/lib/python3.14/site-packages/rpy2/ipython/rmagic.py:1318\u001b[39m, in \u001b[36mRMagics.R\u001b[39m\u001b[34m(self, line, cell, local_ns)\u001b[39m\n\u001b[32m 1316\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m e.stdout.endswith(e.err):\n\u001b[32m 1317\u001b[39m \u001b[38;5;28mprint\u001b[39m(e.err)\n\u001b[32m-> \u001b[39m\u001b[32m1318\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m e\n\u001b[32m 1319\u001b[39m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[32m 1320\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(\u001b[38;5;28mself\u001b[39m.graphics_device, FileDevice):\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/AoU/lib/python3.14/site-packages/rpy2/ipython/rmagic.py:1283\u001b[39m, in \u001b[36mRMagics.R\u001b[39m\u001b[34m(self, line, cell, local_ns)\u001b[39m\n\u001b[32m 1281\u001b[39m return_output = \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[32m 1282\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m1283\u001b[39m text_result, result, visible = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43meval\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcode\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1284\u001b[39m text_output += text_result\n\u001b[32m 1285\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m visible:\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/AoU/lib/python3.14/site-packages/rpy2/ipython/rmagic.py:774\u001b[39m, in \u001b[36mRMagics.eval\u001b[39m\u001b[34m(self, code)\u001b[39m\n\u001b[32m 771\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m (ri.embedded.RRuntimeError, \u001b[38;5;167;01mValueError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m exception:\n\u001b[32m 772\u001b[39m \u001b[38;5;66;03m# Otherwise next return seems to have copy of error.\u001b[39;00m\n\u001b[32m 773\u001b[39m warning_or_other_msg = \u001b[38;5;28mself\u001b[39m.flush()\n\u001b[32m--> \u001b[39m\u001b[32m774\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m RInterpreterError(code, \u001b[38;5;28mstr\u001b[39m(exception),\n\u001b[32m 775\u001b[39m warning_or_other_msg)\n\u001b[32m 776\u001b[39m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[32m 777\u001b[39m ro._print_deferred_warnings()\n", - "\u001b[31mRInterpreterError\u001b[39m: Failed to parse and evaluate line 'DatabaseConnector::downloadJdbcDrivers(\"postgresql\")\\n'.\nR error message: 'Error in DatabaseConnector::downloadJdbcDrivers(\"postgresql\") :'" - ] - } - ], - "source": [ - "%%R\n", - "DatabaseConnector::downloadJdbcDrivers(\"postgresql\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error in `DatabaseConnector::downloadJdbcDrivers()`:\n", - "! The pathToDriver argument must be specified. Consider setting the DATABASECONNECTOR_JAR_FOLDER environment variable, for example in the .Renviron file.\n", - "Run `rlang::last_trace()` to see where the error occurred.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Error in DatabaseConnector::downloadJdbcDrivers(\"postgresql\") :\n" - ] - }, - { - "ename": "RInterpreterError", - "evalue": "Failed to parse and evaluate line '\\n\\n# devtools::install_github(\"OHDSI/ETL-Synthea\")\\n\\n library(ETLSyntheaBuilder)\\n\\n DatabaseConnector::downloadJdbcDrivers(\"postgresql\")\\n # We are loading a version 5.4 CDM into a local PostgreSQL database called \"synthea10\".\\n # The ETLSyntheaBuilder package leverages the OHDSI/CommonDataModel package for CDM creation.\\n # Valid CDM versions are determined by executing CommonDataModel::listSupportedVersions().\\n # The strings representing supported CDM versions are currently \"5.3\" and \"5.4\". \\n # The Synthea version we use in this example is 2.7.0.\\n # However, at this time we also support 3.0.0, 3.1.0, 3.2.0 and 3.3.0.\\n # Please note that Synthea\\'s MASTER branch is always active and this package will be updated to support\\n # future versions as possible.\\n # The schema to load the Synthea tables is called \"native\".\\n # The schema to load the Vocabulary and CDM tables is \"cdm_synthea10\". \\n # The username and pw are \"postgres\" and \"lollipop\".\\n # The Synthea and Vocabulary CSV files are located in /tmp/synthea/output/csv and /tmp/Vocabulary_20181119, respectively.\\n\\n # For those interested in seeing the CDM changes from 5.3 to 5.4, please see: http://ohdsi.github.io/CommonDataModel/cdm54Changes.html\\n\\ncd <- DatabaseConnector::createConnectionDetails(\\n dbms = \"postgresql\", \\n server = \"localhost/synthea10\", \\n user = \"postgres\", \\n password = \"lollipop\", \\n port = 5432, \\n# pathToDriver = \"d:/drivers\" \\n)\\n\\ncdmSchema <- \"cdm_synthea10\"\\ncdmVersion <- \"5.3\"\\nsyntheaVersion <- \"3.3.0\"\\nsyntheaSchema <- \"native\"\\nsyntheaFileLoc <- \"/home/schilder/projects/AoU/notebooks/output/synthea_omop/output/csv\"\\nvocabFileLoc <- \"/home/schilder/projects/data/OMOP\"\\n\\nETLSyntheaBuilder::CreateCDMTables(connectionDetails = cd, cdmSchema = cdmSchema, cdmVersion = cdmVersion)\\n\\nETLSyntheaBuilder::CreateSyntheaTables(connectionDetails = cd, syntheaSchema = syntheaSchema, syntheaVersion = syntheaVersion)\\n\\nETLSyntheaBuilder::LoadSyntheaTables(connectionDetails = cd, syntheaSchema = syntheaSchema, syntheaFileLoc = syntheaFileLoc)\\n\\nETLSyntheaBuilder::LoadVocabFromCsv(connectionDetails = cd, cdmSchema = cdmSchema, vocabFileLoc = vocabFileLoc)\\n\\nETLSyntheaBuilder::CreateMapAndRollupTables(connectionDetails = cd, cdmSchema = cdmSchema, syntheaSchema = syntheaSchema, cdmVersion = cdmVersion, syntheaVersion = syntheaVersion)\\n\\n## Optional Step to create extra indices\\nETLSyntheaBuilder::CreateExtraIndices(connectionDetails = cd, cdmSchema = cdmSchema, syntheaSchema = syntheaSchema, syntheaVersion = syntheaVersion)\\n\\nETLSyntheaBuilder::LoadEventTables(connectionDetails = cd, cdmSchema = cdmSchema, syntheaSchema = syntheaSchema, cdmVersion = cdmVersion, syntheaVersion = syntheaVersion)\\n'.\nR error message: 'Error in DatabaseConnector::downloadJdbcDrivers(\"postgresql\") :'", - "output_type": "error", - "traceback": [ - "\u001b[31m---------------------------------------------------------------------------\u001b[39m", - "\u001b[31mRRuntimeError\u001b[39m Traceback (most recent call last)", - "\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/AoU/lib/python3.14/site-packages/rpy2/ipython/rmagic.py:768\u001b[39m, in \u001b[36mRMagics.eval\u001b[39m\u001b[34m(self, code)\u001b[39m\n\u001b[32m 767\u001b[39m r_expr = ri.parse(code)\n\u001b[32m--> \u001b[39m\u001b[32m768\u001b[39m value, visible = \u001b[43mri\u001b[49m\u001b[43m.\u001b[49m\u001b[43mevalr_expr_with_visible\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 769\u001b[39m \u001b[43m \u001b[49m\u001b[43mr_expr\u001b[49m\n\u001b[32m 770\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 771\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m (ri.embedded.RRuntimeError, \u001b[38;5;167;01mValueError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m exception:\n\u001b[32m 772\u001b[39m \u001b[38;5;66;03m# Otherwise next return seems to have copy of error.\u001b[39;00m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/AoU/lib/python3.14/site-packages/rpy2/rinterface/__init__.py:205\u001b[39m, in \u001b[36mevalr_expr_with_visible\u001b[39m\u001b[34m(expr, envir)\u001b[39m\n\u001b[32m 204\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m error_occured[\u001b[32m0\u001b[39m]:\n\u001b[32m--> \u001b[39m\u001b[32m205\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m embedded.RRuntimeError(_rinterface._geterrmessage())\n\u001b[32m 206\u001b[39m res = conversion._cdata_to_rinterface(r_res)\n", - "\u001b[31mRRuntimeError\u001b[39m: Error in DatabaseConnector::downloadJdbcDrivers(\"postgresql\") : \n", - "\nDuring handling of the above exception, another exception occurred:\n", - "\u001b[31mRInterpreterError\u001b[39m Traceback (most recent call last)", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[10]\u001b[39m\u001b[32m, line 1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m \u001b[43mget_ipython\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrun_cell_magic\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43mR\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43m# devtools::install_github(\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mOHDSI/ETL-Synthea\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m)\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43m library(ETLSyntheaBuilder)\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43m DatabaseConnector::downloadJdbcDrivers(\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mpostgresql\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m)\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43m # We are loading a version 5.4 CDM into a local PostgreSQL database called \u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43msynthea10\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m.\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43m # The ETLSyntheaBuilder package leverages the OHDSI/CommonDataModel package for CDM creation.\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43m # Valid CDM versions are determined by executing CommonDataModel::listSupportedVersions().\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43m # The strings representing supported CDM versions are currently \u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m5.3\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m and \u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m5.4\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m. \u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43m # The Synthea version we use in this example is 2.7.0.\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43m # However, at this time we also support 3.0.0, 3.1.0, 3.2.0 and 3.3.0.\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43m # Please note that Synthea\u001b[39;49m\u001b[38;5;130;43;01m\\'\u001b[39;49;00m\u001b[33;43ms MASTER branch is always active and this package will be updated to support\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43m # future versions as possible.\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43m # The schema to load the Synthea tables is called \u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mnative\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m.\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43m # The schema to load the Vocabulary and CDM tables is \u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mcdm_synthea10\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m. \u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43m # The username and pw are \u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mpostgres\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m and \u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mlollipop\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m.\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43m # The Synthea and Vocabulary CSV files are located in /tmp/synthea/output/csv and /tmp/Vocabulary_20181119, respectively.\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43m # For those interested in seeing the CDM changes from 5.3 to 5.4, please see: http://ohdsi.github.io/CommonDataModel/cdm54Changes.html\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43mcd <- DatabaseConnector::createConnectionDetails(\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43m dbms = \u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mpostgresql\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m, \u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43m server = \u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mlocalhost/synthea10\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m, \u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43m user = \u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mpostgres\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m, \u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43m password = \u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mlollipop\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m, \u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43m port = 5432, \u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43m# pathToDriver = \u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43md:/drivers\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m \u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43m)\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43mcdmSchema <- \u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mcdm_synthea10\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43mcdmVersion <- \u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m5.3\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43msyntheaVersion <- \u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m3.3.0\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43msyntheaSchema <- \u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mnative\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43msyntheaFileLoc <- \u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m/home/schilder/projects/AoU/notebooks/output/synthea_omop/output/csv\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43mvocabFileLoc <- \u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m/home/schilder/projects/data/OMOP\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43mETLSyntheaBuilder::CreateCDMTables(connectionDetails = cd, cdmSchema = cdmSchema, cdmVersion = cdmVersion)\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43mETLSyntheaBuilder::CreateSyntheaTables(connectionDetails = cd, syntheaSchema = syntheaSchema, syntheaVersion = syntheaVersion)\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43mETLSyntheaBuilder::LoadSyntheaTables(connectionDetails = cd, syntheaSchema = syntheaSchema, syntheaFileLoc = syntheaFileLoc)\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43mETLSyntheaBuilder::LoadVocabFromCsv(connectionDetails = cd, cdmSchema = cdmSchema, vocabFileLoc = vocabFileLoc)\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43mETLSyntheaBuilder::CreateMapAndRollupTables(connectionDetails = cd, cdmSchema = cdmSchema, syntheaSchema = syntheaSchema, cdmVersion = cdmVersion, syntheaVersion = syntheaVersion)\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43m## Optional Step to create extra indices\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43mETLSyntheaBuilder::CreateExtraIndices(connectionDetails = cd, cdmSchema = cdmSchema, syntheaSchema = syntheaSchema, syntheaVersion = syntheaVersion)\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43mETLSyntheaBuilder::LoadEventTables(connectionDetails = cd, cdmSchema = cdmSchema, syntheaSchema = syntheaSchema, cdmVersion = cdmVersion, syntheaVersion = syntheaVersion)\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[33;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/AoU/lib/python3.14/site-packages/IPython/core/interactiveshell.py:2572\u001b[39m, in \u001b[36mInteractiveShell.run_cell_magic\u001b[39m\u001b[34m(self, magic_name, line, cell)\u001b[39m\n\u001b[32m 2570\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m.builtin_trap:\n\u001b[32m 2571\u001b[39m args = (magic_arg_s, cell)\n\u001b[32m-> \u001b[39m\u001b[32m2572\u001b[39m result = \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 2574\u001b[39m \u001b[38;5;66;03m# The code below prevents the output from being displayed\u001b[39;00m\n\u001b[32m 2575\u001b[39m \u001b[38;5;66;03m# when using magics with decorator @output_can_be_silenced\u001b[39;00m\n\u001b[32m 2576\u001b[39m \u001b[38;5;66;03m# when the last Python token in the expression is a ';'.\u001b[39;00m\n\u001b[32m 2577\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mgetattr\u001b[39m(fn, magic.MAGIC_OUTPUT_CAN_BE_SILENCED, \u001b[38;5;28;01mFalse\u001b[39;00m):\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/AoU/lib/python3.14/site-packages/rpy2/ipython/rmagic.py:1318\u001b[39m, in \u001b[36mRMagics.R\u001b[39m\u001b[34m(self, line, cell, local_ns)\u001b[39m\n\u001b[32m 1316\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m e.stdout.endswith(e.err):\n\u001b[32m 1317\u001b[39m \u001b[38;5;28mprint\u001b[39m(e.err)\n\u001b[32m-> \u001b[39m\u001b[32m1318\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m e\n\u001b[32m 1319\u001b[39m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[32m 1320\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(\u001b[38;5;28mself\u001b[39m.graphics_device, FileDevice):\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/AoU/lib/python3.14/site-packages/rpy2/ipython/rmagic.py:1283\u001b[39m, in \u001b[36mRMagics.R\u001b[39m\u001b[34m(self, line, cell, local_ns)\u001b[39m\n\u001b[32m 1281\u001b[39m return_output = \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[32m 1282\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m1283\u001b[39m text_result, result, visible = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43meval\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcode\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1284\u001b[39m text_output += text_result\n\u001b[32m 1285\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m visible:\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.conda/envs/AoU/lib/python3.14/site-packages/rpy2/ipython/rmagic.py:774\u001b[39m, in \u001b[36mRMagics.eval\u001b[39m\u001b[34m(self, code)\u001b[39m\n\u001b[32m 771\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m (ri.embedded.RRuntimeError, \u001b[38;5;167;01mValueError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m exception:\n\u001b[32m 772\u001b[39m \u001b[38;5;66;03m# Otherwise next return seems to have copy of error.\u001b[39;00m\n\u001b[32m 773\u001b[39m warning_or_other_msg = \u001b[38;5;28mself\u001b[39m.flush()\n\u001b[32m--> \u001b[39m\u001b[32m774\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m RInterpreterError(code, \u001b[38;5;28mstr\u001b[39m(exception),\n\u001b[32m 775\u001b[39m warning_or_other_msg)\n\u001b[32m 776\u001b[39m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[32m 777\u001b[39m ro._print_deferred_warnings()\n", - "\u001b[31mRInterpreterError\u001b[39m: Failed to parse and evaluate line '\\n\\n# devtools::install_github(\"OHDSI/ETL-Synthea\")\\n\\n library(ETLSyntheaBuilder)\\n\\n DatabaseConnector::downloadJdbcDrivers(\"postgresql\")\\n # We are loading a version 5.4 CDM into a local PostgreSQL database called \"synthea10\".\\n # The ETLSyntheaBuilder package leverages the OHDSI/CommonDataModel package for CDM creation.\\n # Valid CDM versions are determined by executing CommonDataModel::listSupportedVersions().\\n # The strings representing supported CDM versions are currently \"5.3\" and \"5.4\". \\n # The Synthea version we use in this example is 2.7.0.\\n # However, at this time we also support 3.0.0, 3.1.0, 3.2.0 and 3.3.0.\\n # Please note that Synthea\\'s MASTER branch is always active and this package will be updated to support\\n # future versions as possible.\\n # The schema to load the Synthea tables is called \"native\".\\n # The schema to load the Vocabulary and CDM tables is \"cdm_synthea10\". \\n # The username and pw are \"postgres\" and \"lollipop\".\\n # The Synthea and Vocabulary CSV files are located in /tmp/synthea/output/csv and /tmp/Vocabulary_20181119, respectively.\\n\\n # For those interested in seeing the CDM changes from 5.3 to 5.4, please see: http://ohdsi.github.io/CommonDataModel/cdm54Changes.html\\n\\ncd <- DatabaseConnector::createConnectionDetails(\\n dbms = \"postgresql\", \\n server = \"localhost/synthea10\", \\n user = \"postgres\", \\n password = \"lollipop\", \\n port = 5432, \\n# pathToDriver = \"d:/drivers\" \\n)\\n\\ncdmSchema <- \"cdm_synthea10\"\\ncdmVersion <- \"5.3\"\\nsyntheaVersion <- \"3.3.0\"\\nsyntheaSchema <- \"native\"\\nsyntheaFileLoc <- \"/home/schilder/projects/AoU/notebooks/output/synthea_omop/output/csv\"\\nvocabFileLoc <- \"/home/schilder/projects/data/OMOP\"\\n\\nETLSyntheaBuilder::CreateCDMTables(connectionDetails = cd, cdmSchema = cdmSchema, cdmVersion = cdmVersion)\\n\\nETLSyntheaBuilder::CreateSyntheaTables(connectionDetails = cd, syntheaSchema = syntheaSchema, syntheaVersion = syntheaVersion)\\n\\nETLSyntheaBuilder::LoadSyntheaTables(connectionDetails = cd, syntheaSchema = syntheaSchema, syntheaFileLoc = syntheaFileLoc)\\n\\nETLSyntheaBuilder::LoadVocabFromCsv(connectionDetails = cd, cdmSchema = cdmSchema, vocabFileLoc = vocabFileLoc)\\n\\nETLSyntheaBuilder::CreateMapAndRollupTables(connectionDetails = cd, cdmSchema = cdmSchema, syntheaSchema = syntheaSchema, cdmVersion = cdmVersion, syntheaVersion = syntheaVersion)\\n\\n## Optional Step to create extra indices\\nETLSyntheaBuilder::CreateExtraIndices(connectionDetails = cd, cdmSchema = cdmSchema, syntheaSchema = syntheaSchema, syntheaVersion = syntheaVersion)\\n\\nETLSyntheaBuilder::LoadEventTables(connectionDetails = cd, cdmSchema = cdmSchema, syntheaSchema = syntheaSchema, cdmVersion = cdmVersion, syntheaVersion = syntheaVersion)\\n'.\nR error message: 'Error in DatabaseConnector::downloadJdbcDrivers(\"postgresql\") :'" - ] - } - ], - "source": [ - "%%R\n", - "\n", - "\n", - "# devtools::install_github(\"OHDSI/ETL-Synthea\")\n", - "\n", - " library(ETLSyntheaBuilder)\n", - "\n", - "DatabaseConnector::downloadJdbcDrivers(\"postgresql\")\n", - " # We are loading a version 5.4 CDM into a local PostgreSQL database called \"synthea10\".\n", - " # The ETLSyntheaBuilder package leverages the OHDSI/CommonDataModel package for CDM creation.\n", - " # Valid CDM versions are determined by executing CommonDataModel::listSupportedVersions().\n", - " # The strings representing supported CDM versions are currently \"5.3\" and \"5.4\". \n", - " # The Synthea version we use in this example is 2.7.0.\n", - " # However, at this time we also support 3.0.0, 3.1.0, 3.2.0 and 3.3.0.\n", - " # Please note that Synthea's MASTER branch is always active and this package will be updated to support\n", - " # future versions as possible.\n", - " # The schema to load the Synthea tables is called \"native\".\n", - " # The schema to load the Vocabulary and CDM tables is \"cdm_synthea10\". \n", - " # The username and pw are \"postgres\" and \"lollipop\".\n", - " # The Synthea and Vocabulary CSV files are located in /tmp/synthea/output/csv and /tmp/Vocabulary_20181119, respectively.\n", - " \n", - " # For those interested in seeing the CDM changes from 5.3 to 5.4, please see: http://ohdsi.github.io/CommonDataModel/cdm54Changes.html\n", - " \n", - "cd <- DatabaseConnector::createConnectionDetails(\n", - " dbms = \"postgresql\", \n", - " server = \"localhost/synthea10\", \n", - " user = \"postgres\", \n", - " password = \"lollipop\", \n", - " port = 5432, \n", - "# pathToDriver = \"d:/drivers\" \n", - ")\n", - "\n", - "cdmSchema <- \"cdm_synthea_omop\"\n", - "cdmVersion <- \"5.3\"\n", - "syntheaVersion <- \"3.3.0\"\n", - "syntheaSchema <- \"native\"\n", - "syntheaFileLoc <- \"/home/schilder/projects/AoU/notebooks/output/synthea_omop/output/csv\"\n", - "vocabFileLoc <- \"/home/schilder/projects/data/OMOP\"\n", - "\n", - "ETLSyntheaBuilder::CreateCDMTables(connectionDetails = cd, cdmSchema = cdmSchema, cdmVersion = cdmVersion)\n", - " \n", - "ETLSyntheaBuilder::CreateSyntheaTables(connectionDetails = cd, syntheaSchema = syntheaSchema, syntheaVersion = syntheaVersion)\n", - " \n", - "ETLSyntheaBuilder::LoadSyntheaTables(connectionDetails = cd, syntheaSchema = syntheaSchema, syntheaFileLoc = syntheaFileLoc)\n", - " \n", - "ETLSyntheaBuilder::LoadVocabFromCsv(connectionDetails = cd, cdmSchema = cdmSchema, vocabFileLoc = vocabFileLoc)\n", - "\n", - "ETLSyntheaBuilder::CreateMapAndRollupTables(connectionDetails = cd, cdmSchema = cdmSchema, syntheaSchema = syntheaSchema, cdmVersion = cdmVersion, syntheaVersion = syntheaVersion)\n", - "\n", - "## Optional Step to create extra indices\n", - "ETLSyntheaBuilder::CreateExtraIndices(connectionDetails = cd, cdmSchema = cdmSchema, syntheaSchema = syntheaSchema, syntheaVersion = syntheaVersion)\n", - " \n", - "ETLSyntheaBuilder::LoadEventTables(connectionDetails = cd, cdmSchema = cdmSchema, syntheaSchema = syntheaSchema, cdmVersion = cdmVersion, syntheaVersion = syntheaVersion)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Synthea FHIR --> OMOP" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "# !pyomop --create --vocab ~/projects/data/OMOP/ --input /home/schilder/projects/AoU/notebooks/output/synthea_pediatric/output/fhir" - ] - }, + "cells": [ + { + "cell_type": "markdown", + "id": "0c67e543", + "metadata": {}, + "source": [ + "# Synthea Tutorial — synthetic patients → OMOP CDM\n", + "\n", + "This tutorial walks through generating synthetic patient data with\n", + "[Synthea](https://github.com/synthetichealth/synthea) and converting it to the\n", + "**OMOP Common Data Model** — the standard schema downstream EHR-foundation-model\n", + "pipelines (e.g. [`TimelineDataset`](https://github.com/bschilder/TimelineDataset))\n", + "expect.\n", + "\n", + "> **Ported from** [`AoU/notebooks/Synthea.ipynb`](https://github.com/bschilder/AoU/blob/main/notebooks/Synthea.ipynb).\n", + "> The original notebook used `AoU.phenome.synthea.*`; everything is now in the\n", + "> standalone `synthlab` package.\n", + "\n", + "## What you'll do\n", + "\n", + "1. Initialize a `SyntheaRunner` (auto-downloads the Synthea JAR if needed)\n", + "2. Configure a small simulation (10 patients) with `SyntheaConfig`\n", + "3. Run Synthea → CSV output\n", + "4. Convert the CSVs to OMOP CDM 5.4 parquet via `convert_synthea_to_omop`\n", + "5. Inspect the resulting OMOP tables\n", + "\n", + "> ⚠️ **Prereq:** Synthea requires Java (>=11). Install via `brew install openjdk@21`\n", + "> or your distro's package manager.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "896885d2", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-16T23:52:20.772932Z", + "iopub.status.busy": "2026-05-16T23:52:20.772868Z", + "iopub.status.idle": "2026-05-16T23:52:20.876129Z", + "shell.execute_reply": "2026-05-16T23:52:20.875713Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "openjdk version \"21.0.11\" 2026-04-21\n", + "OpenJDK Runtime Environment Homebrew (build 21.0.11)\n", + "OpenJDK 64-Bit Server VM Homebrew (build 21.0.11, mixed mode, sharing)\n", + "\n" + ] + } + ], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2\n", + "\n", + "import os\n", + "from pathlib import Path\n", + "\n", + "# Make Java available to subprocess (homebrew openjdk@21 is keg-only by default)\n", + "JAVA_BIN = \"/opt/homebrew/opt/openjdk@21/bin\"\n", + "if Path(JAVA_BIN).is_dir() and JAVA_BIN not in os.environ.get(\"PATH\", \"\"):\n", + " os.environ[\"PATH\"] = f\"{JAVA_BIN}:{os.environ['PATH']}\"\n", + "\n", + "import subprocess\n", + "print(subprocess.check_output([\"java\", \"-version\"], stderr=subprocess.STDOUT, text=True))" + ] + }, + { + "cell_type": "markdown", + "id": "10476d1c", + "metadata": {}, + "source": [ + "## 1. Initialize a `SyntheaRunner`\n", + "\n", + "The runner downloads the Synthea JAR (one-time, cached) and exposes a `run()`\n", + "method. The first call may take ~30 seconds for the download.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "5ea7eb23", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-16T23:52:20.877213Z", + "iopub.status.busy": "2026-05-16T23:52:20.877150Z", + "iopub.status.idle": "2026-05-16T23:52:24.565605Z", + "shell.execute_reply": "2026-05-16T23:52:24.565177Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\u001b[38;5;199m ░██████╗██╗░░░██╗███╗░░██╗████████╗██╗░░██╗ ██╗░░░░░░█████╗░██████╗░\u001b[0m\n", + "\u001b[38;5;165m ██╔════╝╚██╗░██╔╝████╗░██║╚══██╔══╝██║░░██║ ██║░░░░░██╔══██╗██╔══██╗\u001b[0m\n", + "\u001b[38;5;135m ╚█████╗░░╚████╔╝░██╔██╗██║░░░██║░░░███████║ ██║░░░░░███████║██████╦╝\u001b[0m\n", + "\u001b[38;5;99m ░╚═══██╗░░╚██╔╝░░██║╚████║░░░██║░░░██╔══██║ ██║░░░░░██╔══██║██╔══██╗\u001b[0m\n", + "\u001b[38;5;75m ██████╔╝░░░██║░░░██║░╚███║░░░██║░░░██║░░██║ ███████╗██║░░██║██████╦╝\u001b[0m\n", + "\u001b[38;5;51m ╚═════╝░░░░╚═╝░░░╚═╝░░╚══╝░░░╚═╝░░░╚═╝░░╚═╝ ╚══════╝╚═╝░░╚═╝╚═════╝░\u001b[0m\n", + "\u001b[38;5;49m ▌║█║▌│║▌│║▌║▌█║▌║█║▌│║▌│║▌║▌█║▌║█║▌│║▌│║▌║▌█║▌│║▌│║▌║▌█║\u001b[0m\n", + "\u001b[38;5;99m ════════════════════════════════════════════════════════════════════\u001b[0m\n", + "\u001b[38;5;255m \u001b[1mSynthetic Healthcare Data Toolkit\u001b[0m\n", + "\u001b[38;5;99m ────────────────────────────────────────────────────────────────────\u001b[0m\n", + "\u001b[38;5;199m ◈\u001b[0m EHR \u001b[38;5;255mSynthetic patient records (diagnoses, meds, labs)\u001b[0m\n", + "\u001b[38;5;165m ◈\u001b[0m Genomics \u001b[38;5;255mSynthetic genotypes with realistic LD structure\u001b[0m\n", + "\u001b[38;5;135m ◈\u001b[0m Imaging \u001b[38;5;255mDatasets + synthetic generation (CT, MRI, X-ray)\u001b[0m\n", + "\u001b[38;5;51m ◈\u001b[0m Multimodal \u001b[38;5;255mLinked EHR + Imaging + Genomics per patient\u001b[0m\n", + "\u001b[38;5;49m ◈\u001b[0m AI Notes \u001b[38;5;255mSOAP notes with causal graph analysis\u001b[0m\n", + "\u001b[38;5;99m ════════════════════════════════════════════════════════════════════\u001b[0m\n", + "\n", + " \u001b[38;5;51mVersion:\u001b[0m \u001b[38;5;255m0.2.0\u001b[0m\n", + " \u001b[38;5;51mCache:\u001b[0m \u001b[38;5;255m/Users/bschilder/.cache/synthlab\u001b[0m\n", + "\n", + "Using existing Synthea JAR: /Users/bschilder/Library/Caches/synthea/synthea-with-dependencies.jar\n", + "Synthea JAR: /Users/bschilder/Library/Caches/synthea/synthea-with-dependencies.jar\n" + ] + } + ], + "source": [ + "from synthlab import SyntheaRunner, SyntheaConfig, convert_synthea_to_omop\n", + "\n", + "runner = SyntheaRunner()\n", + "print(f\"Synthea JAR: {runner.jar_path}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b307ba48", + "metadata": {}, + "source": [ + "## 2. Configure a small simulation\n", + "\n", + "10 patients in Massachusetts with a fixed seed for reproducibility. The output\n", + "goes under `./output/tutorial_run/`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "1314e946", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-16T23:52:24.567050Z", + "iopub.status.busy": "2026-05-16T23:52:24.566956Z", + "iopub.status.idle": "2026-05-16T23:52:24.614254Z", + "shell.execute_reply": "2026-05-16T23:52:24.613895Z" + } + }, + "outputs": [ { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## pyhealth\n", - "\n", - "Get around the hard dep on Python>=1.14:\n", - "```\n", - "pip install --ignore-requires-python git+https://github.com/sunlabuiuc/PyHealth.git\n", - "```\n" + "data": { + "text/plain": [ + "SyntheaConfig(population_size=10, seed=42, clinician_seed=None, reference_date=None, gender=None, min_age=0, max_age=140, state='Massachusetts', city=None, config_file=None, modules_dir=None, output_dir='output/tutorial_run', exporter_flags={'exporter.csv.export': 'true', 'exporter.fhir.export': 'false'})" ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" } - ], - "metadata": { - "kernelspec": { - "display_name": "AoU", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.14.2" + ], + "source": [ + "config = SyntheaConfig(\n", + " population_size=10,\n", + " state=\"Massachusetts\",\n", + " seed=42,\n", + " output_dir=\"output/tutorial_run\",\n", + " # Enable CSV exporter (default Synthea output is FHIR JSON);\n", + " # the OMOP converter needs CSV input.\n", + " exporter_flags={\n", + " \"exporter.csv.export\": \"true\",\n", + " \"exporter.fhir.export\": \"false\",\n", + " },\n", + ")\n", + "config" + ] + }, + { + "cell_type": "markdown", + "id": "f1c65421", + "metadata": {}, + "source": [ + "## 3. Run Synthea\n", + "\n", + "This invokes the Synthea JAR via subprocess. Output: CSV files under\n", + "`output/tutorial_run/csv/` (patients, encounters, conditions, …).\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "4b9103c6", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-16T23:52:24.615424Z", + "iopub.status.busy": "2026-05-16T23:52:24.615354Z", + "iopub.status.idle": "2026-05-16T23:52:29.815269Z", + "shell.execute_reply": "2026-05-16T23:52:29.814851Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "return code: 0\n", + "output dir: /Users/bschilder/Desktop/synthlab/notebooks/output/tutorial_run\n", + "\n", + "18 CSV files generated:\n", + " allergies.csv: 713 bytes\n", + " careplans.csv: 12,037 bytes\n", + " claims.csv: 814,779 bytes\n", + " claims_transactions.csv: 6,887,243 bytes\n", + " conditions.csv: 78,651 bytes\n", + " devices.csv: 18,137 bytes\n", + " encounters.csv: 341,744 bytes\n", + " imaging_studies.csv: 5,223,972 bytes\n", + " immunizations.csv: 28,000 bytes\n", + " medications.csv: 293,452 bytes\n", + " observations.csv: 2,420,162 bytes\n", + " organizations.csv: 7,175 bytes\n", + " patients.csv: 4,428 bytes\n", + " payer_transitions.csv: 79,709 bytes\n", + " payers.csv: 1,593 bytes\n", + " procedures.csv: 600,793 bytes\n", + " providers.csv: 8,576 bytes\n", + " supplies.csv: 36,729 bytes\n" + ] } + ], + "source": [ + "result = runner.run(config, verbose=False)\n", + "print(f\"return code: {result['returncode']}\")\n", + "print(f\"output dir: {result['output_dir']}\")\n", + "\n", + "# Inspect the generated CSVs\n", + "# Synthea nests its output under `output/csv/` inside the configured output dir.\n", + "csv_dir = Path(result[\"output_dir\"]) / \"output\" / \"csv\"\n", + "if not csv_dir.is_dir(): # fallback for older layouts\n", + " csv_dir = Path(result[\"output_dir\"]) / \"csv\"\n", + "if csv_dir.is_dir():\n", + " files = sorted(csv_dir.glob(\"*.csv\"))\n", + " print(f\"\\n{len(files)} CSV files generated:\")\n", + " for f in files:\n", + " print(f\" {f.name}: {f.stat().st_size:>10,} bytes\")" + ] + }, + { + "cell_type": "markdown", + "id": "0050b2e9", + "metadata": {}, + "source": [ + "## 4. Convert to OMOP CDM 5.4\n", + "\n", + "`convert_synthea_to_omop` reads the Synthea CSVs and writes OMOP-formatted\n", + "parquet tables (`person`, `condition_occurrence`, `drug_exposure`,\n", + "`measurement`, `visit_occurrence`, etc.).\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "1e5f57b8", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-16T23:52:29.816700Z", + "iopub.status.busy": "2026-05-16T23:52:29.816619Z", + "iopub.status.idle": "2026-05-16T23:52:29.981574Z", + "shell.execute_reply": "2026-05-16T23:52:29.981232Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "generated 10 OMOP tables\n", + " cdm_source 5,456 bytes (cdm_source.parquet)\n", + " condition_occurrence 20,858 bytes (condition_occurrence.parquet)\n", + " death 2,831 bytes (death.parquet)\n", + " drug_exposure 34,869 bytes (drug_exposure.parquet)\n", + " measurement 93,791 bytes (measurement.parquet)\n", + " observation 32,973 bytes (observation.parquet)\n", + " observation_period 2,375 bytes (observation_period.parquet)\n", + " person 7,391 bytes (person.parquet)\n", + " procedure_occurrence 39,989 bytes (procedure_occurrence.parquet)\n", + " visit_occurrence 38,108 bytes (visit_occurrence.parquet)\n" + ] + } + ], + "source": [ + "omop_files = convert_synthea_to_omop(\n", + " synthea_csv_dir=str(csv_dir),\n", + " output_dir=f\"{result['output_dir']}/omop\",\n", + " cdm_version=\"5.4\",\n", + " output_format=\"parquet\",\n", + " verbose=False,\n", + ")\n", + "print(f\"generated {len(omop_files)} OMOP tables\")\n", + "for name, path in sorted(omop_files.items()):\n", + " size = Path(path).stat().st_size if Path(path).exists() else 0\n", + " print(f\" {name:<30s} {size:>10,} bytes ({Path(path).name})\")" + ] + }, + { + "cell_type": "markdown", + "id": "6841f3e0", + "metadata": {}, + "source": [ + "## 5. Inspect the OMOP output\n", + "\n", + "Load a few tables to sanity-check what was produced.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "048ff7bd", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-16T23:52:29.983015Z", + "iopub.status.busy": "2026-05-16T23:52:29.982944Z", + "iopub.status.idle": "2026-05-16T23:52:30.090553Z", + "shell.execute_reply": "2026-05-16T23:52:30.090158Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--- person: (14, 18) ---\n", + " person_id gender_concept_id year_of_birth month_of_birth day_of_birth \\\n", + "0 1 8507 2017 10 31 \n", + "1 2 8507 2010 8 6 \n", + "2 3 8507 2009 9 11 \n", + "\n", + " birth_datetime race_concept_id ethnicity_concept_id location_id \\\n", + "0 2017-10-31 0 0 NaN \n", + "1 2010-08-06 0 0 NaN \n", + "2 2009-09-11 0 0 NaN \n", + "\n", + " provider_id care_site_id person_source_value \\\n", + "0 NaN NaN 76b20010-c318-5754-8c85-983aa538522f \n", + "1 NaN NaN aee7bbe1-0c45-c028-1e62-1f4cdb30c273 \n", + "2 NaN NaN 5e688e99-61b3-5c88-3f60-21df8aaced27 \n", + "\n", + " gender_source_value gender_source_concept_id race_source_value \\\n", + "0 M 0 NaN \n", + "1 M 0 NaN \n", + "2 M 0 NaN \n", + "\n", + " race_source_concept_id ethnicity_source_value ethnicity_source_concept_id \n", + "0 0 NaN 0 \n", + "1 0 NaN 0 \n", + "2 0 NaN 0 \n", + "\n", + "--- condition_occurrence: (545, 17) ---\n", + " condition_occurrence_id person_id condition_concept_id \\\n", + "0 1 1 0 \n", + "1 2 1 0 \n", + "2 3 1 0 \n", + "\n", + " condition_start_date condition_start_datetime condition_end_date \\\n", + "0 2017-10-31 2017-10-31 2017-10-31 \n", + "1 2017-12-04 2017-12-04 2018-02-05 \n", + "2 2018-02-12 2018-02-12 2018-03-01 \n", + "\n", + " condition_end_datetime condition_type_concept_id \\\n", + "0 2017-10-31 32879 \n", + "1 2018-02-05 32879 \n", + "2 2018-03-01 32879 \n", + "\n", + " condition_status_concept_id stop_reason provider_id visit_occurrence_id \\\n", + "0 0 NaN NaN 1 \n", + "1 0 NaN NaN 2 \n", + "2 0 NaN NaN 4 \n", + "\n", + " visit_detail_id condition_source_value condition_source_concept_id \\\n", + "0 NaN 314529007 0 \n", + "1 NaN 314529007 0 \n", + "2 NaN 10509002 0 \n", + "\n", + " condition_source_concept_name condition_status_source_value \n", + "0 Medication review due (situation) NaN \n", + "1 Medication review due (situation) NaN \n", + "2 Acute bronchitis (disorder) NaN \n", + "\n", + "--- drug_exposure: (1113, 25) ---\n", + " drug_exposure_id person_id drug_concept_id drug_exposure_start_date \\\n", + "0 1 1 0 2018-02-13 \n", + "1 2 1 0 2022-03-09 \n", + "2 3 1 0 2022-03-09 \n", + "\n", + " drug_exposure_start_datetime drug_exposure_end_date \\\n", + "0 2018-02-13 05:18:16+00:00 2018-03-01 \n", + "1 2022-03-09 04:58:08+00:00 2022-03-23 \n", + "2 2022-03-09 04:58:08+00:00 2022-03-23 \n", + "\n", + " drug_exposure_end_datetime verbatim_end_date drug_type_concept_id \\\n", + "0 2018-03-01 05:18:16+00:00 NaN 38000177 \n", + "1 2022-03-23 04:58:08+00:00 NaN 38000177 \n", + "2 2022-03-23 04:58:08+00:00 NaN 38000177 \n", + "\n", + " stop_reason ... lot_number provider_id visit_occurrence_id \\\n", + "0 NaN ... NaN NaN 4 \n", + "1 NaN ... NaN NaN 16 \n", + "2 NaN ... NaN NaN 16 \n", + "\n", + " visit_detail_id drug_source_value drug_source_concept_id \\\n", + "0 NaN 313782 0 \n", + "1 NaN 308192 0 \n", + "2 NaN 198405 0 \n", + "\n", + " drug_source_concept_name route_source_value \\\n", + "0 Acetaminophen 325 MG Oral Tablet NaN \n", + "1 Amoxicillin 500 MG Oral Tablet NaN \n", + "2 Ibuprofen 100 MG Oral Tablet NaN \n", + "\n", + " dose_unit_source_value quantity_source_value \n", + "0 NaN NaN \n", + "1 NaN NaN \n", + "2 NaN NaN \n", + "\n", + "[3 rows x 25 columns]\n", + "\n", + "--- visit_occurrence: (1007, 17) ---\n", + " visit_occurrence_id person_id visit_concept_id visit_start_date \\\n", + "0 1 1 9202.0 2017-10-31 \n", + "1 2 1 9202.0 2017-12-05 \n", + "2 3 1 9202.0 2018-02-06 \n", + "\n", + " visit_start_datetime visit_end_date visit_end_datetime \\\n", + "0 2017-10-31 04:58:08+00:00 2017-10-31 2017-10-31 05:13:08+00:00 \n", + "1 2017-12-05 04:58:08+00:00 2017-12-05 2017-12-05 05:13:08+00:00 \n", + "2 2018-02-06 04:58:08+00:00 2018-02-06 2018-02-06 05:13:08+00:00 \n", + "\n", + " visit_type_concept_id provider_id care_site_id \\\n", + "0 32827 NaN NaN \n", + "1 32827 NaN NaN \n", + "2 32827 NaN NaN \n", + "\n", + " visit_source_value visit_source_concept_id \\\n", + "0 76b20010-c318-5754-57e2-651f596ddb8c 0 \n", + "1 76b20010-c318-5754-d83c-7cf85a83d079 0 \n", + "2 76b20010-c318-5754-f57b-839fffa306df 0 \n", + "\n", + " admitting_source_concept_id admitting_source_value \\\n", + "0 0 NaN \n", + "1 0 NaN \n", + "2 0 NaN \n", + "\n", + " discharge_to_concept_id discharge_to_source_value \\\n", + "0 0 NaN \n", + "1 0 NaN \n", + "2 0 NaN \n", + "\n", + " preceding_visit_occurrence_id \n", + "0 NaN \n", + "1 NaN \n", + "2 NaN \n", + "\n" + ] + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "omop_dir = Path(result[\"output_dir\"]) / \"omop\"\n", + "\n", + "for name in [\"person\", \"condition_occurrence\", \"drug_exposure\", \"visit_occurrence\"]:\n", + " candidates = list(omop_dir.glob(f\"{name}*\"))\n", + " if not candidates:\n", + " print(f\"--- {name}: (not produced) ---\\n\")\n", + " continue\n", + " path = candidates[0]\n", + " df = pd.read_parquet(path) if path.suffix == \".parquet\" else pd.read_csv(path)\n", + " print(f\"--- {name}: {df.shape} ---\")\n", + " print(df.head(3))\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "id": "08d257ee", + "metadata": {}, + "source": [ + "## Next steps\n", + "\n", + "- **Feed into `TimelineDataset`**: the OMOP parquet you just generated drops\n", + " directly into `build_omop_timelines(cfg)` from the\n", + " [TimelineDataset](https://github.com/bschilder/TimelineDataset) repo — see\n", + " its [tutorial](https://github.com/bschilder/TimelineDataset/blob/main/notebooks/tutorial.ipynb).\n", + "- **Larger cohorts**: bump `population_size` to 1k+ for realistic train/test\n", + " splits. Synthea runtime scales roughly linearly.\n", + "- **MEDS export**: synthlab also ships `meds_etl` integration — see\n", + " `notebooks/Coherent_MultimodalDataset.ipynb`.\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python (synthlab)", + "language": "python", + "name": "synthlab" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.15" }, - "nbformat": 4, - "nbformat_minor": 2 + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "state": { + "0d17c480b12a413cbe3606a912777b4e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_9fb2f5a3d31e411d8bc0be7a8a8927f8", + "IPY_MODEL_ac541923cd1c451082c3f406b25ede47", + "IPY_MODEL_5bdb82ad5f7f41d2bb237d8246afde92" + ], + "layout": "IPY_MODEL_7548129e50cf43efa203eb0d0e216e48", + "tabbable": null, + "tooltip": null + } + }, + "404f35fe51d34668b1ddd22ad541d980": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "588ec0f484ed49eab540215260f62462": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": "2", + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5bdb82ad5f7f41d2bb237d8246afde92": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_e80bafc67e194963a25bbcebe1a95244", + "placeholder": "​", + "style": "IPY_MODEL_404f35fe51d34668b1ddd22ad541d980", + "tabbable": null, + "tooltip": null, + "value": " 197M/197M [00:00<00:00, 297GB/s]" + } + }, + "7548129e50cf43efa203eb0d0e216e48": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": "inline-flex", + "flex": null, + "flex_flow": "row wrap", + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": "79px" + } + }, + "802c0c5b600b418fab025448e11aee8d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "9fb2f5a3d31e411d8bc0be7a8a8927f8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_a1c74490e7b74605b862ea439db81390", + "placeholder": "​", + "style": "IPY_MODEL_802c0c5b600b418fab025448e11aee8d", + "tabbable": null, + "tooltip": null, + "value": "100%" + } + }, + "a1c74490e7b74605b862ea439db81390": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a2d833493df94e11b132194a674fa7bb": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "ac541923cd1c451082c3f406b25ede47": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_588ec0f484ed49eab540215260f62462", + "max": 197021684.0, + "min": 0.0, + "orientation": "horizontal", + "style": "IPY_MODEL_a2d833493df94e11b132194a674fa7bb", + "tabbable": null, + "tooltip": null, + "value": 197021684.0 + } + }, + "e80bafc67e194963a25bbcebe1a95244": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + } + }, + "version_major": 2, + "version_minor": 0 + } + } + }, + "nbformat": 4, + "nbformat_minor": 5 }