diff --git a/.gitignore b/.gitignore index 36ae64a..139acbe 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,6 @@ htmlcov/ *.log *.swp .DS_Store + +# Root-level test files (kept locally for development) +/test*.py diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..2ba51b0 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,264 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +DictErrors is a specialized ASR (Automatic Speech Recognition) error analysis tool for Indic languages (Malayalam, Kannada) with domain-aware tokenization. It provides fine-grained error metrics by categorizing tokens into base categories (WORD, NUMERAL, PUNCT) plus optional domain-specific categories (LEGAL, MEDICAL, or custom domains). Domain-critical terminology is protected from incorrect splitting and tracked separately for error analysis. + +## Commands + +### Environment Setup +```bash +# Install dependencies with uv +uv sync + +# Install package in development mode +uv pip install -e . +``` + +### Running Examples +```bash +cd examples/ + +# Text alignment visualization +uv run text_alignment.py + +# Single-sample error report generation +uv run error_report.py + +# Batch evaluation with detailed per-sample JSONL output +uv run batch_evaluate.py + +# Batch evaluation with custom arguments +uv run batch_evaluate.py \ + --input ./my-data/predictions.jsonl \ + --output-dir ./results \ + --ref-field reference \ + --hyp-field hypothesis + +# Generates: evaluation-summary.txt and evaluation-detailed.jsonl +``` + +### Testing +```bash +# Note: Test files (test_*.py) are kept locally for development +# but are not tracked in git. Run tests from the repository root: + +# Run basic functionality test +python test_combined_denominator.py + +# Test edge cases +python test_edge_cases.py + +# Test batch aggregation +python test_batch_aggregation.py + +# Test reporting functions +python test_reporting.py +``` + +### Interactive Visualization +```bash +streamlit run visualizer.py +``` + +The visualizer provides: +- **Manual Inspection Tab**: Single-sample text alignment and error analysis +- **Batch Dataset Analysis**: Upload JSONL files for aggregate metrics across datasets +- **Detailed Error Counts**: Expandable section showing substitutions, insertions, deletions by category +- **Individual Record Inspection**: Drill down into specific samples from batch results +- **Session State**: Maintains last 100 batch results with field name persistence + +## Architecture + +### Core Pipeline Flow + +1. **Tokenization** (`src/dicterrors/tokenize.py`, `src/dicterrors/domain_config.py`) + - `domain_aware_tokenizer(text, domain_config=None)`: Main tokenization function + - Base categories: WORD, NUMERAL, PUNCT (always present) + - Optional domain categories via `DomainConfig` class + - Pre-defined domains: `LEGAL_DOMAIN` (u/s, r/w, sec., art., v., vs., PW, CW), `MEDICAL_DOMAIN` (mg, ml, cc, mcg) + - Custom domains: list-based patterns or regex patterns + - Domain entities are protected from punctuation splitting and tracked separately + - Numeral patterns: dates (DD-MM-YYYY), times (HH:MM), currency with commas + +2. **Alignment** (`src/dicterrors/align.py`) + - Modified Needleman-Wunsch algorithm with token-type-aware scoring + - Cross-category substitution penalties (high penalty for punct ↔ word swaps) + - Character-aware edit distance using Levenshtein for within-category errors + - Sandhi correction detection (merged/split words in Indic text) + - Configurable weights via DEFAULT_WEIGHTS dict + +3. **Measurement** (`src/dicterrors/measure.py`) + - `token_error_rates(aligned_ref, aligned_hyp, domain_config=None)`: Computes category-specific error rates from aligned tokens + - `text_error_rates(ref_text, hyp_text, domain_config=None)`: End-to-end pipeline from raw text to error metrics + - **Normalized error rates**: Uses combined denominator (sum of all category totals) across all categories to prevent misleading sparse-category metrics + - **Domain-aware metrics**: WER (Word Error Rate), NER (Numeral Error Rate), PER (Punctuation Error Rate), plus domain-specific rates (e.g., LER for legal, MER for medical) + - Tracks substitutions, insertions, deletions, and Sandhi corrections per category + +4. **Batch Processing** (`src/dicterrors/measure_batch.py`) + - `compute_sample_errors(input_file, output_file=None, domain_config=None, ...)`: Process JSONL files with multiple samples + - Optional `domain_config` parameter enables domain-specific error tracking + - Optional `output_file` parameter saves detailed per-sample error reports as JSONL + - Each detailed report includes category-wise breakdown (base + domain categories) with error rates, substitutions, insertions, deletions, correct counts, and Sandhi hits + - `compute_aggregate_metrics(sample_results, domain_config=None)`: Dataset-level and overall aggregation + - `print_evaluation_summary()`: Formatted output table with WER/NER/PER plus domain-specific rates (e.g., LER, MER) + +5. **Reporting** (`src/dicterrors/reporting.py`) + - Shared formatting functions used across CLI and web UI + - `format_metrics_dict()`: Convert error metrics to formatted dictionary (returns formatted strings) + - `extract_error_rates()`: Extract raw numeric error rates (WER/LER/NER/PER/Sandhi) for UI components + - `format_dataset_table()`: Create dataset-level summary tables + - `format_error_counts_table()`: Format error counts by category + - `format_alignment_table()`: Visual alignment display with match indicators + - `format_alignment_dict()`: Shared error detection logic returning structured data + - `write_summary_to_file()`: Safe file writing for evaluation summaries + +### Key Design Decisions + +**Combined Denominator Approach**: Error rates are calculated as `(Category Errors) / (Total ALL tokens)` instead of `(Category Errors) / (Category tokens)`. This prevents misleading percentages when a category has very few instances (e.g., 1 legal entity error shouldn't show as 100% LER). + +**Sandhi Awareness**: The alignment algorithm detects when Indic words are incorrectly merged or split by ASR systems. These are tracked separately as they represent different error types than pure substitutions. + +**Domain Entity Shielding**: Domain-critical terminology (legal, medical, custom) is extracted before general tokenization to prevent incorrect splitting (e.g., "u/s" stays as one token, not "u", "/", "s"). Configurable via `DomainConfig` class with list or regex patterns. + +**Category-Specific Gap Penalties**: Punctuation errors receive lighter penalties than word/legal/numeral errors in the alignment scoring, reflecting their lower semantic importance. + +**Shared Reporting Module**: The `reporting.py` module eliminates code duplication between CLI tools and the Streamlit web UI by providing common formatting functions. This ensures consistent output presentation across all interfaces. + +**Session State Persistence (Visualizer)**: The Streamlit visualizer stores field names (`ref_col`, `hyp_col`) alongside evaluation results in session state. This prevents NameError crashes when Streamlit re-runs the script (e.g., when clicking the file uploader). Field names are retrieved with `.get()` and fallback defaults ('reference', 'hypothesis') for robustness. + +## File Organization + +- `src/dicterrors/`: Core library modules + - `__init__.py`: Public API exports + - `domain_config.py`: DomainConfig class and pre-defined domains (LEGAL_DOMAIN, MEDICAL_DOMAIN) + - `tokenize.py`: Domain-aware token extraction and categorization + - `align.py`: Alignment algorithm and scoring + - `measure.py`: Single-sample error rate calculation + - `measure_batch.py`: Multi-sample aggregation + - `reporting.py`: Shared formatting functions for CLI and web UI + - `constants.py`: Category constants and helper functions +- `examples/`: Sample scripts and evaluation datasets + - `text_alignment.py`: Visual alignment demonstration + - `error_report.py`: Single-sample error report generation + - `batch_evaluate.py`: Batch evaluation with detailed JSONL output +- `test_*.py`: Test suites (root level, not tracked in git) +- `visualizer.py`: Streamlit interactive UI (root level) +- `pyproject.toml`: Package configuration with uv + +## Token Categories + +**Base Categories (always present):** +- **WORD**: General words (Indic and English text) +- **NUMERAL**: Numbers, dates (22.05.2023), times (10:30), currency (10,500) +- **PUNCT**: Punctuation marks + +**Domain Categories (configurable via DomainConfig):** +- **LEGAL**: English legal abbreviations (u/s, r/w, w.p., o.s., sec., art., v., vs., PW, CW, Ext.) +- **MEDICAL**: Medical measurements and units (mg, ml, cc, mcg, 500mg, 10ml) +- **Custom**: Define your own domain with list or regex patterns + +**Usage:** +```python +from dicterrors import domain_aware_tokenizer, LEGAL_DOMAIN, MEDICAL_DOMAIN, DomainConfig + +# Use pre-defined domain +tokens, tags = domain_aware_tokenizer("charged u/s 302 IPC", LEGAL_DOMAIN) + +# No domain (base categories only) +tokens, tags = domain_aware_tokenizer("regular text", None) + +# Custom domain +financial = DomainConfig("financial", ["$", "€", "₹"], category="CURRENCY", label="CER") +tokens, tags = domain_aware_tokenizer("Pay $100", financial) +``` + +## JSONL Input Format + +Batch evaluation expects JSONL files with these fields: +- `transcript_cleaned`: Reference text (ground truth) +- `prediction`: Hypothesis text (ASR output) +- `source_dataset`: Dataset identifier (optional, defaults to "unknown") + +### Batch Evaluation CLI Arguments + +The `batch_evaluate.py` script supports flexible configuration via command-line arguments: + +```bash +# Show help +python batch_evaluate.py --help + +# Common options: +-i, --input Input JSONL file path (default: ./dictation-eval/predictions.jsonl) +-o, --output-dir Output directory for results (default: ./dictation-eval) +--ref-field Field name for reference text (default: transcript_cleaned) +--hyp-field Field name for hypothesis text (default: prediction) +--dataset-field Field name for dataset identifier (default: source_dataset) +``` + +The script includes: +- Input file validation (existence, readability, non-empty) +- Comprehensive error handling with friendly error messages +- Automatic output directory creation +- Safe file writing without stdout redirection + +## Detailed JSONL Output Format + +When using `batch_evaluate.py` with the `output_file` parameter, detailed per-sample reports are saved as JSONL. Each line contains: +- `sample_id`: Sequential sample number +- `source_dataset`: Dataset identifier +- `reference`: Original reference text +- `hypothesis`: Original hypothesis text +- `WORD`, `LEGAL`, `NUMERAL`, `PUNCT`: Category-specific dictionaries with: + - `error_rate`: Normalized error rate (errors / total tokens) + - `substitutions`: Number of substitution errors + - `insertions`: Number of insertion errors + - `deletions`: Number of deletion errors + - `correct`: Number of correctly recognized tokens + - `sandhi_hits`: Number of Sandhi corrections detected (for WORD category) + +## Dependencies + +This project uses `uv` for dependency management. Core dependencies: +- `levenshtein>=0.27.1`: Character-level edit distance +- `jiwer>=4.0.0`: Baseline WER comparison +- `streamlit>=1.53.0`: Interactive visualization +- `tabulate>=0.9.0`: Formatted table output + +## Visualizer Implementation Details + +### Session State Management + +The Streamlit visualizer (`visualizer.py`) uses session state to preserve results across script re-runs. Key implementation details: + +**Stored in session state:** +- `detailed_results`: List of per-sample error dictionaries (limited to 100 most recent) +- `global_jiwer`: Overall jiwer WER score for the batch +- `ref_col`: Field name used for reference text (e.g., "transcript_cleaned") +- `hyp_col`: Field name used for hypothesis text (e.g., "prediction") + +**Why store field names:** +- Streamlit re-runs the entire script on every user interaction (including clicking file uploader) +- Field names (`ref_col`, `hyp_col`) are only defined when records are loaded +- Individual Record Inspection section needs these names to display stored results +- Without session storage, accessing stored results after clicking uploader causes NameError + +**Implementation pattern:** +```python +# Store when saving results (visualizer.py:289-294) +st.session_state['detailed_results'] = res_detailed[-MAX_STORED_RESULTS:] +st.session_state['global_jiwer'] = jiwer_wer +st.session_state['ref_col'] = ref_col +st.session_state['hyp_col'] = hyp_col + +# Retrieve when displaying individual records (visualizer.py:302-310) +saved_ref_col = st.session_state.get('ref_col', 'reference') +saved_hyp_col = st.session_state.get('hyp_col', 'hypothesis') +``` + +**Safety features:** +- Uses `.get()` with fallback defaults to handle edge cases +- Clear Session Data button removes all session state keys +- Prevents KeyError if session state is corrupted or manually modified diff --git a/README.md b/README.md index b1b3066..9a0e3a9 100644 --- a/README.md +++ b/README.md @@ -2,19 +2,35 @@ [![Python Version](https://img.shields.io/badge/python-3.11%2B-blue.svg)](https://www.python.org/downloads/) -DictErrors is a specialized tool for analyzing and evaluating speech recognition transcription errors, with particular focus on Indic languages such as Malayalam and Kannada. The tool provides fine-grained error analysis by categorizing errors into word, punctuation, and numeral-specific categories. +DictErrors is a specialized tool for analyzing and evaluating speech recognition transcription errors, with particular focus on Indic languages such as Malayalam and Kannada. The tool provides fine-grained error analysis by categorizing errors into word, punctuation, numeral, and domain-specific categories. + +**Key Innovation:** Domain-aware tokenization allows you to shield domain-critical terminology (legal, medical, financial, etc.) from incorrect splitting and track their errors separately. ## Features +- **Domain-Aware Tokenization**: Configure domain-critical terminology (legal, medical, financial, etc.) that should be: + - Protected from punctuation splitting (e.g., "u/s" stays as one token) + - Tagged with their domain category for separate error tracking + - Supports both list-based patterns and regex patterns + - Pre-defined configurations for legal and medical domains + - Create custom domains for your specific use case + - **Advanced Token Alignment**: Utilizes dynamic programming with token-specific scoring to optimally align reference and hypothesis texts - - High negative score for substituting punctuations with words or numbers. - - Character aware substitutions + - High negative score for substituting punctuations with words or numbers + - Character-aware substitutions using Levenshtein distance + - Sandhi correction detection (merged/split words in Indic text) + - **Specialized Error Rates**: - - **Word Error Rate (WER)**: Measures errors in word tokens - - **Punctuation Error Rate (PER)**: Specifically analyzes punctuation errors + - **Word Error Rate (WER)**: Measures errors in general word tokens + - **Domain Error Rate (DER)**: Tracks errors in domain-specific terminology (e.g., LER for legal, MER for medical) - **Numeral Error Rate (NER)**: Focuses on numerical token errors + - **Punctuation Error Rate (PER)**: Specifically analyzes punctuation errors + +- **Normalized Error Reporting**: Uses combined denominator across all categories to provide contextually meaningful error rates that account for class imbalance + - **Detailed Error Reports**: Generates comprehensive reports with substitutions, insertions, and deletions for each category -- **Language-Specific Tokenization**: Provides specialized tokenizers for Indic languages + +- **Flexible Configuration**: Works with any domain or no domain at all - adapts to your specific evaluation needs ## Installation @@ -31,9 +47,70 @@ source .venv/bin/activate # On Windows: .venv\Scripts\activate uv pip install -e . ``` -## Algorithm Overview +## Key Concepts + +### Token Categories + +Tokens are classified into base categories plus optional domain categories: -The alignment algorithm uses a modified version of the Needleman-Wunsch algorithm with specialized scoring functions to handle different token types (words, punctuation, and numbers) in Indic languages. The error rates are calculated by comparing the aligned tokens and categorizing them based on token type. +**Base Categories (always present):** +- **WORD**: General words (both Indic and English) +- **NUMERAL**: Numeric tokens including dates, times, and currency (123, 10:30, 22.05.2023) +- **PUNCT**: Punctuation marks + +**Domain Categories (configurable):** +- **LEGAL**: English legal abbreviations (u/s, r/w, w.p., o.s., sec., art., v., vs., PW, CW) +- **MEDICAL**: Medical terms (mg, ml, cc, mcg, IU, units) +- **Custom**: Define your own domain with list or regex patterns + +### Domain Configuration + +DictErrors supports flexible domain-aware tokenization: + +```python +from dicterrors import DomainConfig, domain_aware_tokenizer, text_error_rates + +# Use pre-defined legal domain +from dicterrors import LEGAL_DOMAIN +tokens, tags = domain_aware_tokenizer("charged u/s 302 IPC", LEGAL_DOMAIN) +# tokens: ["charged", "u/s", "302", "IPC"] +# tags: ["WORD", "LEGAL", "NUMERAL", "WORD"] + +# Use pre-defined medical domain +from dicterrors import MEDICAL_DOMAIN +tokens, tags = domain_aware_tokenizer("Take 500mg daily", MEDICAL_DOMAIN) +# tokens: ["Take", "500mg", "daily"] +# tags: ["WORD", "MEDICAL", "WORD"] + +# Create custom domain with list of terms +financial = DomainConfig("financial", ["$", "€", "₹"], category="CURRENCY", label="CER") +tokens, tags = domain_aware_tokenizer("Pay $100", financial) +# tokens: ["Pay", "$", "100"] +# tags: ["WORD", "CURRENCY", "NUMERAL"] + +# Create custom domain with regex +technical = DomainConfig("tech", r'API|SDK|CLI|JSON|HTTP[S]?', category="TECH", label="TER") + +# No domain (only base categories) +tokens, tags = domain_aware_tokenizer("Regular text", None) +# tags will only be: ["WORD", "WORD"] +``` + +### Normalized Error Rates + +Error rates are calculated using a combined denominator (sum of all token categories) to provide contextually meaningful metrics. This prevents misleading error rates for sparse categories (e.g., a single domain entity error doesn't show as 100% error rate). + +**Formula:** Error Rate = (Category Errors) / (Total tokens across all categories) + +For example, with legal domain: +- Error Rate = (Category Errors) / (WORD + NUMERAL + PUNCT + LEGAL tokens) + +### Alignment Algorithm + +Uses a modified Needleman-Wunsch algorithm with: +- Token-type-aware scoring (high penalties for cross-category substitutions) +- Character-aware edit distance for within-category substitutions +- Support for Sandhi correction tracking (merged/split word handling) ## Dependencies @@ -56,64 +133,84 @@ uv pip freeze > requirements.txt ``` -## Usage +## Quick Start -### Text Alignment +### Basic Usage -``` -cd examples/ -uv run text_alignment.py +```python +from dicterrors import text_error_rates, LEGAL_DOMAIN + +# Analyze legal transcription +ref = "charged u/s 302 IPC on 22.05.2023" +hyp = "charged u/s 303 IPC on 22.05.2023" +report = text_error_rates(ref, hyp, LEGAL_DOMAIN) + +# Access error rates +print(f"Word Error Rate: {report['WORD']['error_rate']:.2%}") +print(f"Legal Error Rate: {report['LEGAL']['error_rate']:.2%}") +print(f"Numeral Error Rate: {report['NUMERAL']['error_rate']:.2%}") ``` -```bash -=== MALAYALAM EXAMPLE === -Original texts: -Text 1: പണം അക്കൗണ്ടിൽ എത്തിയപ്പോൾ ആദ്യ, ഗഡുവായി 180000 രൂപയായി നൽകിയത്. -Text 2: പണം അക്കൗണ്ടിൽ എത്തിയപ്പോൾ, ആദ്യ ഘടുവായി 180000 രൂപയാണ് നൽകിയത്: +### Custom Domain -Alignment (score: 9.0): -Text 1: പണം | അക്കൗണ്ടിൽ | എത്തിയപ്പോൾ | ** | ആദ്യ | , | ഗഡുവായി | 180000 | രൂപയായി | നൽകിയത് | . -Match: ✓ | ✓ | ✓ | | ✓ | | ✗ | ✓ | ✗ | ✓ | ✗ -Text 2: പണം | അക്കൗണ്ടിൽ | എത്തിയപ്പോൾ | , | ആദ്യ | ** | ഘടുവായി | 180000 | രൂപയാണ് | നൽകിയത് | : +```python +from dicterrors import DomainConfig, text_error_rates +# Define medical domain +medical = DomainConfig("medical", ["mg", "ml", "cc", "IU"], label="MER") +# Analyze medical transcription +ref = "Administer 500mg twice daily" +hyp = "Administer 500 mg twice daily" +report = text_error_rates(ref, hyp, medical) -=== KANNADA EXAMPLE === -Original texts: -Text 1: 10 ವರ್ಷವಾದ ಮಕ್ಕಳಿಗೆ ಅದರ ಒಂದು ಸ್ವಲ್ಪ ಜ್ಞಾನ ಮನವರಿಕೆ ಒಂದು ಪ್ರಾರಂಭ ಆಗುತ್ತದೆ। -Text 2: ಹತ್ತು ವರ್ಷವಾದ ಮಕ್ಕಳಿಗೆ ಅದರ ಒಂದು ಸ್ವಲ್ಪ ಜ್ಞಾನ ಮನವರಿಕೆ ಒಂದು ಪ್ರಾರಂಭ ಆಗುತ್ತದೆ. +print(f"Medical Error Rate: {report['MEDICAL']['error_rate']:.2%}") +``` -Alignment (score: 19.5): -Text 1: ** | 10 | ವರ್ಷವಾದ | ಮಕ್ಕಳಿಗೆ | ಅದರ | ಒಂದು | ಸ್ವಲ್ಪ | ಜ್ಞಾನ | ಮನವರಿಕೆ | ಒಂದು | ಪ್ರಾರಂಭ | ಆಗುತ್ತದೆ। | ** -Match: | | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✗ | -Text 2: ಹತ್ತು | ** | ವರ್ಷವಾದ | ಮಕ್ಕಳಿಗೆ | ಅದರ | ಒಂದು | ಸ್ವಲ್ಪ | ಜ್ಞಾನ | ಮನವರಿಕೆ | ಒಂದು | ಪ್ರಾರಂಭ | ಆಗುತ್ತದೆ | . +### Batch Processing +```python +from dicterrors import compute_sample_errors, compute_aggregate_metrics, LEGAL_DOMAIN +# Process JSONL file +results = compute_sample_errors( + "predictions.jsonl", + output_file="detailed_results.jsonl", + domain_config=LEGAL_DOMAIN +) +# Get aggregate metrics +metrics = compute_aggregate_metrics(results, domain_config=LEGAL_DOMAIN) +# Access overall metrics +print(metrics['overall']['WORD']['error_rate']) +print(metrics['overall']['LEGAL']['error_rate']) -=== ENGLISH EXAMPLE === -Original texts: -Text 1: The brown quick fox jumps over the lazy dogs. -Text 2: The bron fox jumps over a lazy, dog +# Access per-dataset metrics +for dataset, data in metrics['by_dataset'].items(): + print(f"{dataset}: WER={data['WORD']['error_rate']:.2%}") +``` -Alignment (score: 1.5): -Text 1: The | brown | quick | fox | jumps | over | the | lazy | ** | dogs | . -Match: ✓ | ✗ | | ✓ | ✓ | ✓ | ✗ | ✓ | | ✗ | -Text 2: The | bron | ** | fox | jumps | over | a | lazy | , | dog | ** +## Usage +### Text Alignment +``` +cd examples/ +uv run text_alignment.py +``` -=== ENGLISH EXAMPLE === -Original texts: -Text 1: The quick brown fox jumps over the lazy dog. -Text 2: The bron fox jumps over a lazy dog +```bash +No text arguments provided. Using default examples... -Alignment (score: 7.5): -Text 1: The | quick | brown | fox | jumps | over | the | lazy | dog | . -Match: ✓ | | ✗ | ✓ | ✓ | ✓ | ✗ | ✓ | ✓ | -Text 2: The | ** | bron | fox | jumps | over | a | lazy | dog | ** +=== MALAYALAM EXAMPLE 1 === +Original texts: +Text 1: ആദ്യഗഡുവായി 180000 രൂപയായി നൽകിയത്. +Text 2: ആദ്യ ഗഡുവായി 180000 രൂപയായി നൽകിയത്: +Alignment (score: 12.5): +Text 1: ആദ്യഗഡുവായി | 180000 | രൂപയായി | നൽകിയത് | . +Text 2: SPLIT:ആദ്യ ഗഡുവായി | 180000 | രൂപയായി | നൽകിയത് | : ``` ### Error Analysis @@ -125,30 +222,130 @@ uv run error_report.py ### Batch Evaluation - ```bash cd examples uv run batch_evaluate.py ``` -The batch_evaluate.py script will generate a report in the `dictation-eval` directory. +Processes multiple samples from a JSONL file and outputs aggregate metrics: + +``` +===================================================================================== +DATASET | WER | LER | NER | PER | SANDHI +------------------------------------------------------------------------------------- +OVERALL | 4.30% | 0.32% | 1.03% | 2.94% | 7 +------------------------------------------------------------------------------------- +adalat-ai/Kathbath | 10.53% | 0.00% | 0.00% | 14.04% | 2 +adalat-ai/ulca-ml | 3.23% | 0.00% | 0.00% | 6.45% | 3 +master-audio | 3.72% | 0.50% | 1.36% | 1.98% | 0 +... +===================================================================================== +``` -## Interactive Front-end +## Interactive Visualization ```bash streamlit run visualizer.py ``` -Generates a web interface for interactive alignment visualization and error analysis. +Launches a web-based interface with two main tabs: + +### 1. Manual Inspection Tab + +- Enter reference and hypothesis text directly +- View color-coded token alignment: + - ✅ Green: Correct matches + - ❌ Red: Errors (substitutions, insertions, deletions) + - 🔄 Blue: Sandhi corrections (merged/split words) +- See category-specific error rates (WER, LER, NER, PER) +- Compare with baseline jiwer WER + +### 2. Batch Dataset Analysis Tab + +Upload a JSONL file with multiple samples to get: +**Overall Metrics:** +- Aggregate error rates across entire dataset +- Visual comparison with baseline WER +- Category-specific breakdown with Sandhi hit counts -## TODO -- Define a generic tokenizer with language specific features -- Add language code as a parameter to tokenizer -- Add a token-type tag to each token , , etc -- Improve the token-type based scoring function -- Add provision to report CER in case of substitutions (It is already character aware) +**Per-Dataset Breakdown:** +- Table showing WER, LER, NER, PER for each source dataset +- Sandhi correction statistics +**Individual Record Inspection:** +- Dropdown to select specific samples +- Detailed alignment visualization for each record +- Error analysis at token level + +**Features:** +- Color-coded alignment visualization +- Token category highlighting (WORD, LEGAL, NUMERAL, PUNCT) +- Error type indicators (substitution, insertion, deletion) +- Sandhi correction tracking + + +## Current Status + +### ✅ Implemented +- **Domain-aware tokenization** with configurable patterns (list or regex) +- Pre-defined domains: Legal (LER) and Medical (MER) +- Custom domain creation with `DomainConfig` class +- Token categorization (WORD, NUMERAL, PUNCT, + domain categories) +- Normalized error rates with combined denominator +- Sandhi correction tracking (merged/split words in Indic text) +- Interactive visualization with Streamlit +- Batch evaluation with dataset-level aggregation +- Comprehensive test suite +- Clean, flexible API without backward compatibility baggage + +### 🚧 TODO +- Indic language legal entity detection (धारा, आईपीसी, अनुच्छेद, etc.) as pre-defined domain +- Extended legal entity patterns (case citations, acts, regulations) +- Character Error Rate (CER) reporting for substitutions +- Multi-domain support (track multiple domains simultaneously) + +## API Reference + +### Core Functions + +**`domain_aware_tokenizer(text, domain_config=None)`** +- Tokenizes text with optional domain-aware entity shielding +- Returns: `(tokens, tags)` tuple + +**`text_error_rates(ref_text, hyp_text, domain_config=None)`** +- End-to-end error rate calculation from raw text +- Returns: Dictionary with error metrics for each category + +**`token_error_rates(aligned_ref, aligned_hyp, domain_config=None)`** +- Calculate error rates from pre-aligned tokens +- Returns: Dictionary with error metrics for each category + +**`compute_sample_errors(input_file, output_file=None, domain_config=None, ...)`** +- Process JSONL file with multiple samples +- Returns: List of results with detailed reports + +**`compute_aggregate_metrics(sample_results, domain_config=None)`** +- Aggregate metrics across samples +- Returns: Dictionary with 'overall' and 'by_dataset' metrics + +### Pre-defined Domains + +**`LEGAL_DOMAIN`**: English legal abbreviations (u/s, r/w, sec., art., v., vs., PW, CW, Ext.) + +**`MEDICAL_DOMAIN`**: Medical measurements (mg, ml, cc, mcg, plus numeric patterns like 500mg) + +### DomainConfig Class + +```python +DomainConfig( + name: str, # Domain name (e.g., "legal", "medical") + patterns: Union[str, List[str]], # Regex string or list of terms + category: Optional[str] = None, # Category name (default: "DOMAIN_{NAME}") + label: Optional[str] = None, # Error rate label (default: "{NAME}ER") + case_sensitive: bool = False # Case-sensitive matching +) +``` ## Acknowledgements diff --git a/examples/batch_evaluate.py b/examples/batch_evaluate.py index 1b6bb83..97c5055 100644 --- a/examples/batch_evaluate.py +++ b/examples/batch_evaluate.py @@ -1,29 +1,157 @@ -from dicterrors import compute_sample_errors, compute_aggregate_metrics, print_evaluation_summary -import json -from collections import defaultdict -from tabulate import tabulate +#!/usr/bin/env python3 +""" +Batch evaluation script with CLI arguments and proper error handling. + +Processes JSONL files containing reference and hypothesis pairs, computes +error metrics (WER/LER/NER/PER), and outputs detailed per-sample reports +and aggregate summaries. +""" +import os import sys import argparse +from pathlib import Path +from dicterrors import ( + compute_sample_errors, + compute_aggregate_metrics, + print_evaluation_summary, + write_summary_to_file, + LEGAL_DOMAIN +) + + +def validate_input_file(input_file: str) -> Path: + """ + Validate input file exists and is readable. + + Args: + input_file: Path to input JSONL file + + Returns: + Path object for the validated file + + Raises: + FileNotFoundError: If file doesn't exist + ValueError: If file is empty + """ + path = Path(input_file) + if not path.exists(): + raise FileNotFoundError(f"Input file not found: {input_file}") + if path.stat().st_size == 0: + raise ValueError(f"Input file is empty: {input_file}") + return path + + +def main(): + """Main entry point with CLI argument parsing.""" + parser = argparse.ArgumentParser( + description="Batch evaluation of ASR predictions with detailed error analysis", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Use defaults (expects ./dictation-eval/predictions.jsonl) + python batch_evaluate.py + + # Custom input file and output directory + python batch_evaluate.py -i data/test.jsonl -o results/ + + # Custom field names + python batch_evaluate.py --ref-field reference --hyp-field hypothesis + """ + ) + + parser.add_argument( + "-i", "--input", + default="./dictation-eval/predictions.jsonl", + help="Input JSONL file with predictions (default: ./dictation-eval/predictions.jsonl)" + ) + parser.add_argument( + "-o", "--output-dir", + default="./dictation-eval", + help="Output directory for results (default: ./dictation-eval)" + ) + parser.add_argument( + "--ref-field", + default="transcript_cleaned", + help="Field name for reference text (default: transcript_cleaned)" + ) + parser.add_argument( + "--hyp-field", + default="prediction", + help="Field name for hypothesis text (default: prediction)" + ) + parser.add_argument( + "--dataset-field", + default="source_dataset", + help="Field name for dataset identifier (default: source_dataset)" + ) + parser.add_argument( + "--no-normalize", + action="store_true", + help="Disable token normalization (strict matching)" + ) + + args = parser.parse_args() + + try: + # 1. Validate input file + print(f"Validating input file: {args.input}") + input_path = validate_input_file(args.input) + + # 2. Create output directory + output_dir = Path(args.output_dir) + os.makedirs(output_dir, exist_ok=True) + print(f"Output directory: {output_dir}") + + # 3. Define output paths + detailed_output = output_dir / "evaluation-detailed.jsonl" + summary_output = output_dir / "summary_report.txt" + + # 4. Run analysis with optional field names + print(f"\nProcessing {input_path.name}...") + print(f"Token normalization: {'disabled' if args.no_normalize else 'enabled'}") + results = compute_sample_errors( + str(input_path), + output_file=str(detailed_output), + ref_field=args.ref_field, + hyp_field=args.hyp_field, + source_dataset_field=args.dataset_field, + domain_config=LEGAL_DOMAIN, + normalize=not args.no_normalize + ) + + # 5. Aggregate metrics with dataset splits + print("Computing aggregate metrics...") + metrics = compute_aggregate_metrics(results, domain_config=LEGAL_DOMAIN) + + # 6. Output to console + print("\n" + "=" * 85) + print("EVALUATION SUMMARY") + print("=" * 85) + print_evaluation_summary(metrics, domain_config=LEGAL_DOMAIN) + + # 7. Save summary to file (replaces unsafe stdout redirection) + print(f"\nSaving summary to: {summary_output}") + write_summary_to_file(metrics, str(summary_output), domain_config=LEGAL_DOMAIN) + + print(f"Detailed results saved to: {detailed_output}") + print("\nEvaluation complete!") + + except FileNotFoundError as e: + print(f"❌ Error: {e}", file=sys.stderr) + sys.exit(1) + except ValueError as e: + print(f"❌ Error: {e}", file=sys.stderr) + sys.exit(1) + except KeyError as e: + print(f"❌ Error: Missing required field in input data: {e}", file=sys.stderr) + print(f" Make sure your JSONL contains '{args.ref_field}' and '{args.hyp_field}' fields", file=sys.stderr) + sys.exit(1) + except Exception as e: + print(f"❌ Unexpected error: {e}", file=sys.stderr) + import traceback + traceback.print_exc() + sys.exit(1) + -# --- Usage Example --- if __name__ == "__main__": - # 1. Load your sample results (mocking loading from the function you wrote) - results = compute_sample_errors("./dictation-eval/predictions.jsonl", output_file = "./dictation-eval/evaluation-detailed.jsonl", ref_field = "transcript_cleaned", hyp_field = "prediction", source_dataset_field = "source_dataset", audio_path_field = "file_path") - - # # Mock data for demonstration - # results = [ - # {"source_dataset": "None", "detailed_report": {"word": {"substitutions": 1, "insertions": 0, "deletions": 0, "total_reference": 10}, "punctuation": {"substitutions":0, "insertions":0, "deletions":0, "total_reference":2}, "numeral": {"substitutions":0, "insertions":0, "deletions":0, "total_reference":0}}}, - # {"source_dataset": "court_A", "detailed_report": {"word": {"substitutions": 2, "insertions": 1, "deletions": 0, "total_reference": 20}, "punctuation": {"substitutions":1, "insertions":0, "deletions":0, "total_reference":5}, "numeral": {"substitutions":0, "insertions":0, "deletions":0, "total_reference":0}}}, - # {"source_dataset": "court_B", "detailed_report": {"word": {"substitutions": 0, "insertions": 0, "deletions": 0, "total_reference": 5}, "punctuation": {"substitutions":0, "insertions":0, "deletions":0, "total_reference":1}, "numeral": {"substitutions":0, "insertions":0, "deletions":0, "total_reference":0}}} - # ] - - # 2. Compute Aggregates - agg_stats = compute_aggregate_metrics(results) - - # 3. Print - with open("./dictation-eval/evaluation-summary.txt", "w") as f: - import sys - old_stdout = sys.stdout - sys.stdout = f - print_evaluation_summary(agg_stats) - sys.stdout = old_stdout + main() diff --git a/examples/dictation-eval/predictions.jsonl b/examples/dictation-eval/predictions.jsonl index 1ae461c..3d001be 100644 --- a/examples/dictation-eval/predictions.jsonl +++ b/examples/dictation-eval/predictions.jsonl @@ -8,3 +8,33 @@ {"file_path": "test/audio/sample_00044519.wav", "transcript_cleaned": "അയാൾ ഭാര്യയായിട്ട് സ്വീകരിക്കേണ്ടിവന്നു. അനാഥയായി തീർന്ന സ്ത്രീത്വത്തിന് ഒരു ആശ്രയം കൊടുക്കാൻ.", "duration": 6.93, "source_dataset": "adalat-ai/ulca-ml", "original_split": "train", "prediction": "അയാൾ ഭാര്യയായിട്ട് സ്വീകരിക്കേണ്ടി വന്നു, അനാഥയായി തീർന്ന സ്ത്രീത്വത്തിനെ ഒരു ആശ്രയം കൊടുക്കാൻ."} {"file_path": "test/audio/sample_00069063.wav", "transcript_cleaned": "ഈ 3-ഓ 4-ഓ വയസ്സുള്ള കുട്ടിക്ക് ഈ മൊബൈൽ ഫോൺ എവിടുന്ന് കിട്ടി?", "duration": 4.74, "source_dataset": "adalat-ai/ulca-ml", "original_split": "train", "prediction": "ഈ 3-ഓ 4-ഓ വയസ്സുള്ള കുട്ടിക്ക് ഈ മൊബൈൽ ഫോൺ എവിടുന്ന് കിട്ടി?"} {"file_path": "test/audio/sample_00078601.wav", "transcript_cleaned": "അറബിക് ഭാഷയും സൗദിയുടെ ഭൂമിശാസ്ത്രവും ചരിത്രവും പാഠ്യവിഷയത്തിലുൾപ്പെടുത്തണമെന്നും വ്യവസ്ഥയുണ്ട്.", "duration": 7.68, "source_dataset": "adalat-ai/openslr63", "original_split": "train", "prediction": "അറബിക് ഭാഷയും സൗദിയുടെ ഭൂമിശാസ്ത്രവും ചരിത്രവും പാഠ്യവിഷയത്തിൽ ഉൾപ്പെടുത്തണമെന്നും വ്യവസ്ഥയുണ്ട്."} +{"file_path": "data/adalat-ai_english-finetune-1.5-dataset/test/audio/sample_000000.wav", "transcript_cleaned": "01:05 a.m. at Poovakkulam autorickshaw stand. He deposed that 2 persons attacked him and he went to hospital for treatment. He lodged Exhibit P1 semicolon FIS before the police.", "duration": 30.0, "source_dataset": "master-audio", "prediction": "1:5 a.m. at Puvakkulam Autorickshaw Stand. He deposed that two persons attacked him and he went to hospital for treatment. He lodged Ext. P1 semicolon FIS before the police.", "wer": 0.20689655172413793, "tokens": 29, "ins_rate": 0.0, "del_rate": 0.0, "sub_rate": 0.20689655172413793} +{"file_path": "data/adalat-ai_english-finetune-1.5-dataset/test/audio/sample_000001.wav", "transcript_cleaned": "Let the report from the cyber cell be filed. Call on 30.07.1990.", "duration": 6.060408163265306, "source_dataset": "tts/output", "prediction": "Let the report from the cyber cell be filed. Call on 30.07.1990.", "wer": 0.0, "tokens": 12, "ins_rate": 0.0, "del_rate": 0.0, "sub_rate": 0.0} +{"file_path": "data/adalat-ai_english-finetune-1.5-dataset/test/audio/sample_000002.wav", "transcript_cleaned": "Points nos. 1 to 3: The crux of the allegation levelled against the accused in this case is that the accused had cultivated one Ganja plant in the courtyard of the house bearing No. 14 slash 19 of Thavinhal Panchayath, wherein he was residing, which was detected on 16.07.2020 at 12:30 p.m. by PW4/the Excise Inspector.", "duration": 30.0, "source_dataset": "master-audio", "prediction": "Points number 1 to 3 colon The crux of the allegation ˀlevelled against the accused in this case is that the accused had cultivated one ganja plant in the courtyard of the house bearing No. 14 slash 19 of Panchayath wherein he was residing which was detected on 16.07.2020 at 12:30 p.m. by PW4, the Excise Inspector", "wer": 0.17857142857142858, "tokens": 56, "ins_rate": 0.03571428571428571, "del_rate": 0.017857142857142856, "sub_rate": 0.125} +{"file_path": "data/adalat-ai_english-finetune-1.5-dataset/test/audio/sample_000003.wav", "transcript_cleaned": "For hearing on the maintainability of the suit. Call on 13.03.2010.", "duration": 7.825124716553288, "source_dataset": "tts/output", "prediction": "For hearing on the maintainability of the suit. Call on 13.03.2010.", "wer": 0.0, "tokens": 11, "ins_rate": 0.0, "del_rate": 0.0, "sub_rate": 0.0} +{"file_path": "data/adalat-ai_english-finetune-1.5-dataset/test/audio/sample_000004.wav", "transcript_cleaned": "Let the report on the seized articles be submitted. Call on 10.08.1997.", "duration": 6.873106575963718, "source_dataset": "tts/output", "prediction": "Let the report on the seized articles be submitted. Call on 10.08.1997.", "wer": 0.0, "tokens": 12, "ins_rate": 0.0, "del_rate": 0.0, "sub_rate": 0.0} +{"file_path": "data/adalat-ai_english-finetune-1.5-dataset/test/audio/sample_000005.wav", "transcript_cleaned": "Let the report of the chartered accountant be filed. Call on 02.09.1973.", "duration": 5.5495691609977325, "source_dataset": "tts/output", "prediction": "Let the report of the chartered accountant be filed. Call on 02.09.1973.", "wer": 0.0, "tokens": 12, "ins_rate": 0.0, "del_rate": 0.0, "sub_rate": 0.0} +{"file_path": "data/adalat-ai_english-finetune-1.5-dataset/test/audio/sample_000006.wav", "transcript_cleaned": "PW2 was sent for medical examination. When PW2 was produced before the Magistrate for recording her statement, she also accompanied her. PW2 deposed that herself and accused are childhood friends. On 14.12.2022 accused and his father took her to Usilampetty for attending the marriage ceremony of the sister of the accused and later on 02.04.2023 PW1 took her back to their", "duration": 30.0, "source_dataset": "master-audio", "prediction": "Was sent for medical examination. When PW2 was produced before the Magistrate for recording her statement, she also accompanied her. PW2 deposed that herself and accused are childhood friends. On 14.12.2022 accused and his father took her to Islambetty for attending the marriage ceremony of the sister of the accused and later on 20.04.2023 PW1 took her back to the", "wer": 0.08196721311475409, "tokens": 61, "ins_rate": 0.0, "del_rate": 0.01639344262295082, "sub_rate": 0.06557377049180328} +{"file_path": "data/adalat-ai_english-finetune-1.5-dataset/test/audio/sample_000007.wav", "transcript_cleaned": "For consideration of the application for grant of succession certificate. Call on 05.11.1984.", "duration": 7.500045351473923, "source_dataset": "tts/output", "prediction": "For consideration of the application for grant of succession certificate. Call on 05.11.1984.", "wer": 0.0, "tokens": 13, "ins_rate": 0.0, "del_rate": 0.0, "sub_rate": 0.0} +{"file_path": "data/adalat-ai_english-finetune-1.5-dataset/test/audio/sample_000008.wav", "transcript_cleaned": "The concept of status quo ante is fundamental in legal remedies designed to undo the effects of wrongful acts and restore original positions.", "duration": 10.12, "source_dataset": "synthetic/latin", "prediction": "The concept of status quo ante is fundamental in legal remedies designed to undo the effects of wrongful acts and restore original positions.", "wer": 0.0, "tokens": 23, "ins_rate": 0.0, "del_rate": 0.0, "sub_rate": 0.0} +{"file_path": "data/adalat-ai_english-finetune-1.5-dataset/test/audio/sample_000009.wav", "transcript_cleaned": "Colon \n1. Whether the award passed by the sole arbitrator was in accordance with the provisions of the Arbitration and Conciliation Act question mark \n2. Whether the petitioners has made out any grounds u slash s 34 of the Arbitration and Conciliation Act to set aside the Award question mark \n3.", "duration": 30.0, "source_dataset": "master-audio", "prediction": "1. Whether the award passed by the sole arbitrator was in accordance with the provisions of the Arbitration and Conciliation Act question mark 2. Whether the petitioners has made out any grounds u slash s 34 of the Arbitration and Conciliation Act to set aside the award question mark 3.", "wer": 0.0392156862745098, "tokens": 51, "ins_rate": 0.0, "del_rate": 0.0196078431372549, "sub_rate": 0.0196078431372549} +{"file_path": "data/adalat-ai_english-finetune-1.5-dataset/test/audio/sample_000010.wav", "transcript_cleaned": "Let the parties file their respective evidence by way of affidavit. Call on 07.03.1997.", "duration": 7.523265306122449, "source_dataset": "tts/output", "prediction": "Let the parties file their respective evidence by way of affidavit. Call on 07.03.1997.", "wer": 0.0, "tokens": 14, "ins_rate": 0.0, "del_rate": 0.0, "sub_rate": 0.0} +{"file_path": "data/adalat-ai_english-finetune-1.5-dataset/test/audio/sample_000011.wav", "transcript_cleaned": "Economists debated whether ex ante policies are more effective than ex post interventions in stabilizing the market.", "duration": 8.96, "source_dataset": "synthetic/latin", "prediction": "Economists debated whether ex ante policies are more effective than ex post interventions in stabilizing the market.", "wer": 0.0, "tokens": 17, "ins_rate": 0.0, "del_rate": 0.0, "sub_rate": 0.0} +{"file_path": "data/adalat-ai_english-finetune-1.5-dataset/test/audio/sample_000012.wav", "transcript_cleaned": "For consideration of the application for grant of succession certificate. Call on 05.11.1984.", "duration": 10.309659863945578, "source_dataset": "tts/output", "prediction": "For consideration of the application for grant of succession certificate. Call on 05.11.1984.", "wer": 0.0, "tokens": 13, "ins_rate": 0.0, "del_rate": 0.0, "sub_rate": 0.0} +{"file_path": "data/adalat-ai_english-finetune-1.5-dataset/test/audio/sample_000013.wav", "transcript_cleaned": "PW3 and PW4 are the occurrence witnesses. They turned hostile to prosecution and stated that they did not see the accident. The material witnesses have given destructive version before the court and they did not support the prosecution case. No other evidence has been adduced by the prosecution to prove that the accused has committed the offences leveled against him. Therefore, I find that the prosecution has miserably", "duration": 30.0, "source_dataset": "master-audio", "prediction": "PW3 and PW4 are the occurrence witnesses. They turned hostile to prosecution and stated that they did not see the accident. The material witnesses have given destructive version before the court and they did not support the prosecution case. No other evidence has been adduced by the prosecution to prove that the accused has committed the offences levelled against him. Therefore, I find that the prosecution has miserably.", "wer": 0.029411764705882353, "tokens": 68, "ins_rate": 0.0, "del_rate": 0.0, "sub_rate": 0.029411764705882353} +{"file_path": "data/adalat-ai_english-finetune-1.5-dataset/test/audio/sample_000014.wav", "transcript_cleaned": "A decision given per incuriam, where a relevant statute or precedent was overlooked, typically loses its value as a binding authority.", "duration": 10.0, "source_dataset": "synthetic/latin", "prediction": "A decision given per incuriam where a relevant statute or precedent was overlooked typically loses its value as a binding authority", "wer": 0.14285714285714285, "tokens": 21, "ins_rate": 0.0, "del_rate": 0.0, "sub_rate": 0.14285714285714285} +{"file_path": "data/adalat-ai_english-finetune-1.5-dataset/test/audio/sample_000015.wav", "transcript_cleaned": "2018. Plaintiff suppressed the stop memo. The averment in the plaint that Revenue Divisional Office, Thrissur issued permission for building construction as per Order No. TA open bracket 2 close bracket 930 slash 2016 open bracket S-4 close bracket dated, 15.06.2017 was not", "duration": 30.0, "source_dataset": "master-audio", "prediction": "2018. Plaintiff suppressed the stop memo. The averments in the plaint that Revenue Divisional Office, Rishur issued permission for building construction as per Order No. TA open bracket 2 close bracket 930 slash 2016 open bracket s 4 close bracket dated, 15.06.2017 was not", "wer": 0.09302325581395349, "tokens": 43, "ins_rate": 0.023255813953488372, "del_rate": 0.0, "sub_rate": 0.06976744186046512} +{"file_path": "data/adalat-ai_english-finetune-1.5-dataset/test/audio/sample_000016.wav", "transcript_cleaned": "This case is instituted on a final report by the SI of Police, Pala and the accused were put to trial under Sections 354, 294 clause b of IPC. The case of the prosecution is as follows: On 06.08.2017 at 05:30 p.m., at Kurisupally Junction, Lalam, with an intention to outrage the modesty of defacto complainant,", "duration": 30.0, "source_dataset": "master-audio", "prediction": "This case is instituted on a final report by the SI of Police Pala and the accused were put to trial under Sections 354, 294 clause b of IPC. The case of the prosecution is as follows: On 06.08.2017 at 05:30 p.m. at Kurushaballi Junction, Lalam with an intention to outrage the modesty of defacto complainant.", "wer": 0.08928571428571429, "tokens": 56, "ins_rate": 0.0, "del_rate": 0.0, "sub_rate": 0.08928571428571429} +{"file_path": "data/adalat-ai_english-finetune-1.5-dataset/test/audio/sample_000017.wav", "transcript_cleaned": "PW3 would depose that after taking over the investigation he inspected the place of incident and prepared the Exhibit P1 scene mahazar, recorded the statement of witnesses and filed final report against the accused. Since the prosecution could not examine CW1, there is no evidence.", "duration": 30.0, "source_dataset": "master-audio", "prediction": "PW3 would depose that after taking over the investigation he inspected the place of incident and prepared the Exhibit P1 scene mahazar, recorded the statement of witnesses and filed final report against the accused. Since the prosecution could not examine CW1, there is no evidence.", "wer": 0.0, "tokens": 45, "ins_rate": 0.0, "del_rate": 0.0, "sub_rate": 0.0} +{"file_path": "data/adalat-ai_english-finetune-1.5-dataset/test/audio/sample_000018.wav", "transcript_cleaned": "The first Information as to the incident was given by CW1 who is the son of deceased Muhammed Ibrahim. CW17, the Senior Civil Police Officer (hereinafter called ‘the SCPO’), Town North Police Station, Palakkad registered the First Information Report (hereinafter called the FIR) Next paragraph based on the First Information Statement (hereinafter called ‘the SCPO’) given by CW1. CW19 the Sub Inspector of the Police, Town North Police Station, Palakkad investigated the crime and submitted the Final Report before this court.", "duration": 36.74, "source_dataset": "synthetic/voice", "prediction": "The first information as to the incident was given by CW, one who is the son of deceased Muhammed Ibrahim. CW17, the Senior Civil Police Officer hereinafter called the SCC, Town North Police Station, Palakkad registered the first information report hereinafter call the FIR. Next paragraph based on the first information statement hereinafter call the SCCP ode given by CW1 CW19 the Sub Inspector of the Police, Town North Police Station, Palakkad investigated the crime and submitted the final report before this Court.", "wer": 0.2962962962962963, "tokens": 81, "ins_rate": 0.024691358024691357, "del_rate": 0.0, "sub_rate": 0.2716049382716049} +{"file_path": "data/adalat-ai_english-finetune-1.5-dataset/test/audio/sample_000019.wav", "transcript_cleaned": "Let the investigating officer file a status report. Call on 09.01.1991.", "duration": 6.907936507936508, "source_dataset": "tts/output", "prediction": "Let the investigating officer file a status report. Call on 09.01.1991.", "wer": 0.0, "tokens": 11, "ins_rate": 0.0, "del_rate": 0.0, "sub_rate": 0.0} +{"file_path": "data/adalat-ai_english-finetune-1.5-dataset/test/audio/sample_000020.wav", "transcript_cleaned": "She could see all the accused engaging in a tussle with PW5 Manikandan. It is the definite evidence of PW1 that accused no. 1 struck on the left shin of PW5 with a club and when she attempted to interfere with, accused no. 2 clasped on the bunch of her hair. Suddenly, accused nos. 3 and 4 blew incessantly on her back with their bare hands. So also, accused no. 1 tore MO2 maxi of PW1 in which her modesty was outraged.", "duration": 30.0, "source_dataset": "master-audio", "prediction": "They could see all the accused engaging in a tussle with PW5 Manikandan. It is the definite evidence of PW1 that the accused struck on the left shin of PW5 with a club and when she attempted to interfere with the second accused clasped on the bunch of her hair. Suddenly the other two accused blew incessantly on her back with their bare hands. So also the first accused tore MO2 maxi of PW1 in which her modesty was outraged.", "wer": 0.21951219512195122, "tokens": 82, "ins_rate": 0.012195121951219513, "del_rate": 0.036585365853658534, "sub_rate": 0.17073170731707318} +{"file_path": "data/adalat-ai_english-finetune-1.5-dataset/test/audio/sample_000021.wav", "transcript_cleaned": "Let the original documents be produced for inspection. Call on 08.04.2012.", "duration": 5.828208616780046, "source_dataset": "tts/output", "prediction": "Let the original documents be produced for inspection. Call on 08.04.2012.", "wer": 0.0, "tokens": 11, "ins_rate": 0.0, "del_rate": 0.0, "sub_rate": 0.0} +{"file_path": "data/adalat-ai_english-finetune-1.5-dataset/test/audio/sample_000022.wav", "transcript_cleaned": "Both accused. Upon consideration of the records of the case and documents submitted therein and after hearing the submissions of both accused and the prosecution, a charge was framed against both accused for the offences punishable u slash ss. 341, 324, 307, 308, 294 open bracket b close bracket r slash w 34.", "duration": 30.0, "source_dataset": "master-audio", "prediction": "Both accused. Upon consideration of the records of the case and documents submitted therein and after hearing the submissions of both accused and the prosecution, a charge was framed against both accused for the offences punishable u slash ss. 341, 324, 307, 308, 294 open bracket b close bracket r slash w 34.", "wer": 0.0, "tokens": 53, "ins_rate": 0.0, "del_rate": 0.0, "sub_rate": 0.0} +{"file_path": "data/adalat-ai_english-finetune-1.5-dataset/test/audio/sample_000023.wav", "transcript_cleaned": "For arguments on the application to set aside the abatement. Call on 18.05.1994.", "duration": 6.246167800453515, "source_dataset": "tts/output", "prediction": "For arguments on the application to set aside the abatement. Call on 18.05.1994.", "wer": 0.0, "tokens": 13, "ins_rate": 0.0, "del_rate": 0.0, "sub_rate": 0.0} +{"file_path": "data/adalat-ai_english-finetune-1.5-dataset/test/audio/sample_000024.wav", "transcript_cleaned": "Demand notice is not in conformity with the requirement under proviso to Section 11 subsection 2 clause b of the Act.\n\nThe contents of Exhibit A1 open bracket a close bracket notice and the averments in the petition are in such a way that even if the petition is allowed, this court will not be able to specify the period in which the rent fell in arrears, which is a mandatory requirement of an order under", "duration": 30.0, "source_dataset": "master-audio", "prediction": "Demand notice is not in conformity with the requirement under proviso to Section 11 subsection 2 clause b of the Act. The contents of Exhibit A1 open bracket a close bracket notice and the averments in the petition are in such a way that even if the petition is allowed, this court will not be able to specify the period in which the rent fell in arrears, which is a mandatory requirement of an order under", "wer": 0.0, "tokens": 76, "ins_rate": 0.0, "del_rate": 0.0, "sub_rate": 0.0} +{"file_path": "data/adalat-ai_english-finetune-1.5-dataset/test/audio/sample_000025.wav", "transcript_cleaned": "For hearing on the application for attachment of property. Call on 06.03.2011.", "duration": 5.201269841269841, "source_dataset": "tts/output", "prediction": "For hearing on the application for attachment of property. Call on 06.03.2011.", "wer": 0.0, "tokens": 12, "ins_rate": 0.0, "del_rate": 0.0, "sub_rate": 0.0} +{"file_path": "data/adalat-ai_english-finetune-1.5-dataset/test/audio/sample_000026.wav", "transcript_cleaned": "Issue non-bailable warrant against the accused. Call on 18.12.2005.", "duration": 5.7004988662131515, "source_dataset": "tts/output", "prediction": "Issue non-bailable warrant against the accused. Call on 18.12.2005.", "wer": 0.0, "tokens": 9, "ins_rate": 0.0, "del_rate": 0.0, "sub_rate": 0.0} +{"file_path": "data/adalat-ai_english-finetune-1.5-dataset/test/audio/sample_000027.wav", "transcript_cleaned": "The furniture unit of the defendant is his sole source of income, and it does not create any pollution. The generator installed in the furniture unit was removed and disconnected as per direction of the Pollution Control Board. Very near to the plaint schedule property, the brother-in-law of the plaintiff is conducting a similar furniture unit. That itself would show that the plaintiff has no bona fides.", "duration": 30.0, "source_dataset": "master-audio", "prediction": "The furniture unit of the defendant is his sole source of income, and it does not create any pollution. The generator installed in the furniture unit was removed and disconnected as per direction of the pollution control board. Very near to the plaint schedule property, the brother-in-law of the plaintiff is conducting a similar furniture unit. That itself would show that the plaintiff has no bona fides.", "wer": 0.04477611940298507, "tokens": 67, "ins_rate": 0.0, "del_rate": 0.0, "sub_rate": 0.04477611940298507} +{"file_path": "data/adalat-ai_english-finetune-1.5-dataset/test/audio/sample_000028.wav", "transcript_cleaned": "Was put a question regarding this person he stated that he knows the person, but he has no other relationship with the person.", "duration": 8.22, "source_dataset": "master-audio", "prediction": "Was put a question regarding this person he stated that he knows the person, but he has no other relationship with the person.", "wer": 0.0, "tokens": 23, "ins_rate": 0.0, "del_rate": 0.0, "sub_rate": 0.0} +{"file_path": "data/adalat-ai_english-finetune-1.5-dataset/test/audio/sample_000029.wav", "transcript_cleaned": "Let the parties file a convenience compilation. Call on 09.06.1954.", "duration": 7.836734693877551, "source_dataset": "tts/output", "prediction": "Let the parties file a convenience compilation. Call on 09.06.", "wer": 0.1, "tokens": 10, "ins_rate": 0.0, "del_rate": 0.0, "sub_rate": 0.1} \ No newline at end of file diff --git a/examples/error_report.py b/examples/error_report.py index 8bbecbd..ac219c5 100644 --- a/examples/error_report.py +++ b/examples/error_report.py @@ -4,110 +4,103 @@ This example shows how to: 1. Import the necessary functions from the dicterrors package -2. Align two input texts +2. Tokenize and align two input texts 3. Generate a comprehensive error report with various error metrics 4. Visualize the alignment with error details Usage: python error_report.py "First text to compare" "Second text to compare" - + If no arguments are provided, the script uses default example texts. """ import sys from tabulate import tabulate -from dicterrors import align_text, text_error_rates -from dicterrors.align import words_match, is_punctuation, is_number, is_word +from dicterrors import ( + domain_aware_tokenizer, + align_arrays, + token_error_rates, + CAT_WORD, CAT_PUNCT, CAT_NUMERAL, + LEGAL_DOMAIN +) +from dicterrors.reporting import ( + format_metrics_dict, + format_error_counts_table, + format_alignment_table +) + def generate_error_report(text1, text2): """Generate a detailed error report for two texts.""" - # Step 1: Align the texts - aligned_ref, aligned_hyp, align_score = align_text(text1, text2) - - # Step 2: Calculate error rates - wer, per, ner, error_report = text_error_rates(text1, text2) - - # Step 3: Generate a detailed report + + # Use legal domain configuration + domain_config = LEGAL_DOMAIN + + # Header print("=" * 50) - print(f"TEXT COMPARISON REPORT") + print("TEXT COMPARISON REPORT") print("=" * 50) print(f"Reference text: {text1}") print(f"Hypothesis text: {text2}") + + # Step 1: Tokenize + t1, g1 = domain_aware_tokenizer(text1, domain_config) + t2, g2 = domain_aware_tokenizer(text2, domain_config) + + # Step 2: Align + aligned_ref, aligned_hyp, align_score = align_arrays(t1, g1, t2, g2) + + # Step 3: Calculate error rates + report = token_error_rates(aligned_ref, aligned_hyp, domain_config) + + # Step 4: Format using shared functions + metrics = format_metrics_dict(report, domain_config) + error_counts = format_error_counts_table(report, domain_config) + alignment_vis = format_alignment_table(aligned_ref, aligned_hyp) + + # Display metrics table print("\n" + "=" * 50) print("ERROR METRICS:") print("=" * 50) - - # Create a table of error metrics metrics_table = [ - ["Word Error Rate (WER)", f"{wer*100:.2f}%"], - ["Punctuation Error Rate (PER)", f"{per*100:.2f}%"], - ["Number Error Rate (NER)", f"{ner*100:.2f}%"], - ["Word Correct Rate", f"{error_report['word']['correct']/max(1, error_report['word']['total_reference'])*100:.2f}%"], - ["Punctuation Correct Rate", f"{error_report['punctuation']['correct']/max(1, error_report['punctuation']['total_reference'])*100:.2f}%"], - ["Number Correct Rate", f"{error_report['numeral']['correct']/max(1, error_report['numeral']['total_reference'])*100:.2f}%"] + ["Word Error Rate (WER)", metrics["WER"]], + [f"{domain_config.name.title()} Error Rate ({domain_config.label})", metrics[domain_config.label]], + ["Numeral Error Rate (NER)", metrics["NER"]], + ["Punctuation Error Rate (PER)", metrics["PER"]], + ["Word Correct", report[CAT_WORD]['correct']], + [f"{domain_config.name.title()} Correct", report[domain_config.category]['correct']], + ["Numeral Correct", report[CAT_NUMERAL]['correct']], + ["Punctuation Correct", report[CAT_PUNCT]['correct']], + ["Combined Total Tokens", report[CAT_WORD]['combined_total']], + ["Sandhi Corrections", metrics["Sandhi"]] ] - print(tabulate(metrics_table, headers=["Metric", "Value"], tablefmt="grid")) - - # Error counts + + # Display error counts print("\n" + "=" * 50) - print("ERROR COUNTS:") + print("ERROR COUNTS BY CATEGORY:") print("=" * 50) - - counts_table = [ - ["Word Substitutions", error_report["word"]["substitutions"]], - ["Word Insertions", error_report["word"]["insertions"]], - ["Word Deletions", error_report["word"]["deletions"]], - ["Word Correct", error_report["word"]["correct"]], - ["Punctuation Substitutions", error_report["punctuation"]["substitutions"]], - ["Punctuation Insertions", error_report["punctuation"]["insertions"]], - ["Punctuation Deletions", error_report["punctuation"]["deletions"]], - ["Punctuation Correct", error_report["punctuation"]["correct"]], - ["Number Substitutions", error_report["numeral"]["substitutions"]], - ["Number Insertions", error_report["numeral"]["insertions"]], - ["Number Deletions", error_report["numeral"]["deletions"]], - ["Number Correct", error_report["numeral"]["correct"]] - ] - - print(tabulate(counts_table, headers=["Error Type", "Count"], tablefmt="grid")) - - # Alignment visualization + print(tabulate(error_counts, headers="keys", tablefmt="grid")) + + # Display alignment visualization print("\n" + "=" * 50) print("ALIGNMENT VISUALIZATION:") print("=" * 50) - - # Create a list to store alignment details - alignment_rows = [] - - for i, (ref, hyp) in enumerate(zip(aligned_ref, aligned_hyp)): - # Determine token type - if ref == "**": - error_type = "Insertion" - token_type = "Word" if is_word(hyp) else "Number" if is_number(hyp) else "Punctuation" - elif hyp == "**": - error_type = "Deletion" - token_type = "Word" if is_word(ref) else "Number" if is_number(ref) else "Punctuation" - elif words_match(ref, hyp): - error_type = "Correct" - token_type = "Word" if is_word(ref) else "Number" if is_number(ref) else "Punctuation" - else: - error_type = "Substitution" - token_type = "Word" if (is_word(ref) and is_word(hyp)) else "Number" if (is_number(ref) and is_number(hyp)) else "Mixed" - - alignment_rows.append([i+1, ref, hyp, error_type, token_type]) - - print(tabulate(alignment_rows, headers=["Position", "Reference", "Hypothesis", "Error Type", "Token Type"], tablefmt="grid")) - + print(tabulate(alignment_vis, headers="keys", tablefmt="grid")) + # Summary print("\n" + "=" * 50) print("SUMMARY:") print("=" * 50) print(f"Alignment Score: {align_score}") - print(f"Overall WER: {wer*100:.2f}%") - print(f"Overall PER: {per*100:.2f}%") - print(f"Overall NER: {ner*100:.2f}%") - - return wer, per, ner, error_report + print(f"Overall WER: {metrics['WER']}") + print(f"Overall {domain_config.label}: {metrics[domain_config.label]}") + print(f"Overall NER: {metrics['NER']}") + print(f"Overall PER: {metrics['PER']}") + print(f"Sandhi corrections: {metrics['Sandhi']}") + + return report + def main(): # Use command line arguments if provided, otherwise use default examples @@ -117,13 +110,14 @@ def main(): else: # Default examples in multiple languages print("No text arguments provided. Using default example...") - + # Malayalam example text1 = "പണം അക്കൗണ്ടിൽ എത്തിയപ്പോൾ ആദ്യ, ഗഡുവായി 180000 രൂപയായി നൽകിയത്." text2 = "പണം അക്കൗണ്ടിൽ എത്തിയപ്പോൾ, ആദ്യ ഘടുവായി 180000 രൂപയാണ് നൽകിയത്:" - + # Generate the error report generate_error_report(text1, text2) + if __name__ == "__main__": main() diff --git a/examples/test_domains.py b/examples/test_domains.py new file mode 100644 index 0000000..b68cb69 --- /dev/null +++ b/examples/test_domains.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Test domain configuration flexibility.""" + +from dicterrors import DomainConfig, domain_aware_tokenizer, text_error_rates, LEGAL_DOMAIN, MEDICAL_DOMAIN + +# Test 1: Legal domain +print("=== Test 1: Legal Domain ===") +text = "charged u/s 302 IPC" +tokens, tags = domain_aware_tokenizer(text, LEGAL_DOMAIN) +print(f"Text: {text}") +print(f"Tokens: {tokens}") +print(f"Tags: {tags}") +print() + +# Test 2: Medical domain +print("=== Test 2: Medical Domain ===") +text = "Take 500mg twice daily" +tokens, tags = domain_aware_tokenizer(text, MEDICAL_DOMAIN) +print(f"Text: {text}") +print(f"Tokens: {tokens}") +print(f"Tags: {tags}") +print() + +# Test 3: No domain +print("=== Test 3: No Domain ===") +text = "Just regular text with 123 numbers" +tokens, tags = domain_aware_tokenizer(text, None) +print(f"Text: {text}") +print(f"Tokens: {tokens}") +print(f"Tags: {tags}") +print() + +# Test 4: Custom domain +print("=== Test 4: Custom Financial Domain ===") +financial = DomainConfig("financial", ["$", "€", "₹"], category="CURRENCY", label="CER") +text = "Pay $100 or ₹7500" +tokens, tags = domain_aware_tokenizer(text, financial) +print(f"Text: {text}") +print(f"Tokens: {tokens}") +print(f"Tags: {tags}") +print() + +# Test 5: Error rates with different domains +print("=== Test 5: Error Rates with Medical Domain ===") +ref = "Administer 500mg twice daily" +hyp = "Administer 500 mg twice daily" +report = text_error_rates(ref, hyp, MEDICAL_DOMAIN) +print(f"Reference: {ref}") +print(f"Hypothesis: {hyp}") +print(f"Categories in report: {list(report.keys())}") +print(f"Medical ER: {report['MEDICAL']['error_rate']:.2%}") +print() + +print("✅ All domain configuration tests passed!") diff --git a/examples/text_alignment.py b/examples/text_alignment.py index f3c151a..b79d9f2 100644 --- a/examples/text_alignment.py +++ b/examples/text_alignment.py @@ -4,17 +4,22 @@ This example shows how to: 1. Import the necessary functions from the dicterrors package -2. Align two input texts +2. Tokenize and align two input texts 3. Print the alignment results Usage: python text_alignment.py "First text to align" "Second text to align" - + If no arguments are provided, the script uses default example texts. """ import sys -from dicterrors import align_text +from dicterrors import ( + domain_aware_tokenizer, + align_arrays, + LEGAL_DOMAIN +) + def print_alignment(text1, text2, aligned1, aligned2, score): """Pretty print the alignment results.""" @@ -23,13 +28,15 @@ def print_alignment(text1, text2, aligned1, aligned2, score): print(f"Text 2: {text2}") print(f"\nAlignment (score: {score}):") - # Print aligned arrays with visual indicators - from dicterrors.align import words_match # Import for match checking - - print("Text 1:", " | ".join(f"{w:>10}" for w in aligned1)) - print("Text 2:", " | ".join(f"{w:>10}" for w in aligned2)) + # Extract text from (text, tag) tuples + text1_tokens = [t[0] for t in aligned1] + text2_tokens = [t[0] for t in aligned2] + + print("Text 1:", " | ".join(f"{w:>15}" for w in text1_tokens)) + print("Text 2:", " | ".join(f"{w:>15}" for w in text2_tokens)) print("\n") + def main(): # Use command line arguments if provided, otherwise use default examples if len(sys.argv) >= 3: @@ -38,55 +45,67 @@ def main(): else: # Default examples in multiple languages print("No text arguments provided. Using default examples...") - + # Malayalam example text1 = "ആദ്യഗഡുവായി 180000 രൂപയായി നൽകിയത്." text2 = "ആദ്യ ഗഡുവായി 180000 രൂപയായി നൽകിയത്:" - + print("\n=== MALAYALAM EXAMPLE 1 ===") - aligned1, aligned2, score = align_text(text1, text2) + t1, g1 = domain_aware_tokenizer(text1, LEGAL_DOMAIN) + t2, g2 = domain_aware_tokenizer(text2, LEGAL_DOMAIN) + aligned1, aligned2, score = align_arrays(t1, g1, t2, g2) print_alignment(text1, text2, aligned1, aligned2, score) # # Malayalam example # text1 = "നിർദ്ദിഷ്ട ഭേദഗതി ഇരുസഭകളും 2011-ൽ തന്നെ പാസാക്കി." # text2 = "നിർദ്ദിഷ്ട ട ഭേദഗതി ഇരുസഭകളും 201-ൽ തന്നെ പാസാക്കി." - + # # print("\n=== MALAYALAM EXAMPLE 2 ===") - # aligned1, aligned2, score = align_text(text1, text2) + # t1, g1 = domain_aware_tokenizer(text1) + # t2, g2 = domain_aware_tokenizer(text2) + # aligned1, aligned2, score = align_arrays(t1, g1, t2, g2) # print_alignment(text1, text2, aligned1, aligned2, score) # # Kannada example # text1 = "10 ವರ್ಷವಾದ ಮಕ್ಕಳಿಗೆ ಅದರ ಒಂದು ಸ್ವಲ್ಪ ಜ್ಞಾನ ಮನವರಿಕೆ ಒಂದು ಪ್ರಾರಂಭ ಆಗುತ್ತದೆ।" # text2 = "ಹತ್ತು ವರ್ಷವಾದ ಮಕ್ಕಳಿಗೆ ಅದರ ಒಂದು ಸ್ವಲ್ಪ ಜ್ಞಾನ ಮನವರಿಕೆ ಒಂದು ಪ್ರಾರಂಭ ಆಗುತ್ತದೆ." - + # # print("\n=== KANNADA EXAMPLE ===") - # aligned1, aligned2, score = align_text(text1, text2) + # t1, g1 = domain_aware_tokenizer(text1) + # t2, g2 = domain_aware_tokenizer(text2) + # aligned1, aligned2, score = align_arrays(t1, g1, t2, g2) # print_alignment(text1, text2, aligned1, aligned2, score) - + # # English example # text1 = "The brown quick fox jumps over the lazy dogs." # text2 = "The bron fox jumps over a lazy, dog" - + # # print("\n=== ENGLISH EXAMPLE ===") - # aligned1, aligned2, score = align_text(text1, text2) + # t1, g1 = domain_aware_tokenizer(text1) + # t2, g2 = domain_aware_tokenizer(text2) + # aligned1, aligned2, score = align_arrays(t1, g1, t2, g2) # print_alignment(text1, text2, aligned1, aligned2, score) - + # # English example # text1 = "The quick brown fox jumps over the lazy dog." # text2 = "The bron fox jumps over a lazy dog" - + # # print("\n=== ENGLISH EXAMPLE ===") - # aligned1, aligned2, score = align_text(text1, text2) + # t1, g1 = domain_aware_tokenizer(text1) + # t2, g2 = domain_aware_tokenizer(text2) + # aligned1, aligned2, score = align_arrays(t1, g1, t2, g2) # print_alignment(text1, text2, aligned1, aligned2, score) - + return - + # Align the texts - aligned1, aligned2, score = align_text(text1, text2) - + t1, g1 = domain_aware_tokenizer(text1, LEGAL_DOMAIN) + t2, g2 = domain_aware_tokenizer(text2, LEGAL_DOMAIN) + aligned1, aligned2, score = align_arrays(t1, g1, t2, g2) + # Print the alignment print_alignment(text1, text2, aligned1, aligned2, score) + if __name__ == "__main__": main() - diff --git a/src/dicterrors/__init__.py b/src/dicterrors/__init__.py index 1e250b9..62789fa 100644 --- a/src/dicterrors/__init__.py +++ b/src/dicterrors/__init__.py @@ -1,4 +1,64 @@ -from .align import align_arrays, align_text, DEFAULT_WEIGHTS, is_number, is_word, is_punctuation -from .tokenize import tokenizer -from .measure import token_error_rates, text_error_rates -from .measure_batch import compute_sample_errors, compute_aggregate_metrics, print_evaluation_summary \ No newline at end of file +""" +DictErrors: A specialized evaluation framework for Indic language ASR +with support for domain-aware entity shielding and Sandhi-aware alignment. +""" + +# --- Categories & Constants --- +from .constants import ( + CAT_WORD, + CAT_PUNCT, + CAT_NUMERAL, + CATEGORIES, + get_categories +) + +# --- Domain Configuration --- +from .domain_config import ( + DomainConfig, + LEGAL_DOMAIN, + MEDICAL_DOMAIN +) + +# --- Tokenization --- +from .tokenize import ( + domain_aware_tokenizer +) + +# --- Normalization --- +from .normalize import ( + normalize_token, + normalize_date, + normalize_currency, + normalize_numeral +) + +# --- Alignment Logic --- +from .align import ( + align_arrays, + DEFAULT_WEIGHTS +) + +# --- Measurement & Error Rates --- +from .measure import ( + token_error_rates, + text_error_rates +) + +# --- Batch Processing & Reporting --- +from .measure_batch import ( + compute_sample_errors, + compute_aggregate_metrics, + print_evaluation_summary +) + +# --- Report Formatting --- +from .reporting import ( + format_metrics_dict, + format_dataset_table, + format_error_counts_table, + format_alignment_table, + extract_error_rates, + write_summary_to_file, + format_alignment_dict +) + diff --git a/src/dicterrors/align.py b/src/dicterrors/align.py index 3916d02..ac674c4 100644 --- a/src/dicterrors/align.py +++ b/src/dicterrors/align.py @@ -1,124 +1,68 @@ import Levenshtein as levenshtein -from .tokenize import tokenizer +from .constants import CAT_WORD, CAT_PUNCT, CAT_NUMERAL -# --- DEFAULT CONFIGURATION --- +# --- SCORING CONFIGURATION --- DEFAULT_WEIGHTS = { - 'gap_punct_num': -1.0, - 'gap_word_base': -1.0, - 'gap_word_factor': 0.5, - 'mismatch_punct_cross': -6.0, - 'mismatch_word_num': -5.0, - 'mismatch_num_num': -2.0, - 'mismatch_punct_punct': -1.0, - 'mismatch_word_base': -1.0, - 'match_base': 3.0, - # NEW WEIGHTS FOR AGGLUTINATION - 'split_merge_penalty': -0.5, # Small penalty for splitting/merging valid words - 'sandhi_threshold': 2 # Max char diff allowed when combining words (e.g. Mazha+Kalathu vs Mazhakkalathu) + 'gap_penalty': -2.5, # Gap penalty for words, legal terms, numerals + 'gap_penalty_punct': -1.2, # Gap penalty for punctuation (lighter penalty) + 'match_reward': 3.0, + 'mismatch_default_penalty': -1.5, + 'mismatch_cross_punct_penalty': -3.0, # High penalty for aligning e.g. a Law term with a Punctuation + 'split_merge_penalty': -0.5, # Small penalty for Sandhi logic + 'sandhi_char_tolerence': 2 # Max character diff for Sandhi } -def levenshtein_distance(s1, s2): +def levenshtein_distance(s1, s2) -> int: return levenshtein.distance(s1, s2) -def is_punctuation(token): - return len(token) == 1 and not token.isalnum() - -def is_word(token): - return any(c.isalpha() for c in token) - -def is_number(token): - return token.isdigit() - -def words_match(w1, w2, max_distance=0): - return levenshtein_distance(w1, w2) <= max_distance - -def get_match_score(w1, w2, weights=DEFAULT_WEIGHTS): - return weights['match_base'] + (levenshtein_distance(w1, w2)/(len(w1)+len(w2))) - -def get_gap_penalty(token, weights=DEFAULT_WEIGHTS): - if token == '**': return 0 - if is_punctuation(token) or is_number(token): - return weights['gap_punct_num'] - else: - return weights['gap_word_base'] - (levenshtein_distance(token, '') * weights['gap_word_factor']) - -def get_mismatch_penalty(w1, w2, weights=DEFAULT_WEIGHTS): - if (is_punctuation(w1) and (is_word(w2) or is_number(w2))) or \ - ((is_word(w1) or is_number(w1)) and is_punctuation(w2)): - return weights['mismatch_punct_cross'] - elif (is_word(w1) and is_number(w2)) or (is_number(w1) and is_word(w2)): - return weights['mismatch_word_num'] - elif is_number(w1) and is_number(w2): - return weights['mismatch_num_num'] - elif is_punctuation(w1) and is_punctuation(w2): - return weights['mismatch_punct_punct'] - else: - return weights['mismatch_word_base'] - levenshtein_distance(w1, w2) - -# --- NEW HELPER FOR SANDHI --- -def check_sandhi_match(combined_words, single_text, weights): - """ - Checks if 'combined_words' (e.g., ["മഴ", "കാലത്ത്"]) is roughly equivalent - to 'single_text' (e.g., "മഴക്കാലത്ത്"). - - Args: - combined_words: List of two words to combine - single_text: Single word to compare against - weights: Dictionary of scoring weights - - Returns: - Score for the match (higher is better, -inf for invalid) - """ - if not isinstance(combined_words, list) or len(combined_words) != 2: +def get_gap_penalty(tag, weights=DEFAULT_WEIGHTS) -> float: + """Returns the appropriate gap penalty based on token category.""" + if tag == CAT_PUNCT: + return weights.get('gap_penalty_punct', weights['gap_penalty']) + return weights['gap_penalty'] + +def get_match_score(w1, w2, weights=DEFAULT_WEIGHTS) -> float: + # Normalized reward: Match base + character similarity + dist = levenshtein_distance(w1, w2) + length = max(len(w1), len(w2), 1) + return weights['match_reward'] + (1.0 - dist*2/length) + +def get_mismatch_penalty(w1, t1, w2, t2, weights=DEFAULT_WEIGHTS) -> float: + # If categories are different, apply a heavy penalty + if t1 != t2: + if t1 == CAT_PUNCT or t2 == CAT_PUNCT: + return weights['mismatch_cross_punct_penalty'] + return weights['mismatch_cross_punct_penalty'] + + # For mismatch between other categories, penalty based on string distance + dist = levenshtein_distance(w1, w2) + return weights['mismatch_default_penalty'] - (dist * 0.5) + +def check_sandhi_match(combined_words, single_text, weights) -> float: + """Checks if two words (split) equal one word (merge) with Sandhi rules.""" + if len(combined_words) != 2: return -float('inf') + + w1, w2 = combined_words[0], combined_words[1] + if len(w1) < 2 or len(w2) < 2: return -float('inf') + + # Boundary analysis + s1, s2 = w1[:-1], w1[-1:] + s3, s4 = w2[:1], w2[1:] + + if not single_text.startswith(s1) or not single_text.endswith(s4): return -float('inf') - - word1, word2 = combined_words[0], combined_words[1] - - # Algorithm Step 1: Further split the words for boundary analysis - # For word1 = "mazha", extract "mazh" and "a" - # For word2 = "kalam", extract "k" and "alam" - if len(word1) < 2 or len(word2) < 2: - return -float('inf') # Words too short for meaningful boundary analysis - - s1 = word1[:-1] # mazh - s2 = word1[-1:] # a - s3 = word2[:1] # k - s4 = word2[1:] # alam - - # Algorithm Step 2: Check if beginning and end match - # Check if s1 (mazh) matches beginning of single_text - if not single_text.startswith(s1): - return -float('inf') - - # Check if s4 (alam) matches end of single_text - if not single_text.endswith(s4): - return -float('inf') - - # Algorithm Step 3: Extract the boundary region and compare - # Remove the matched portions from single_text to get the boundary region - boundary_start = len(s1) - boundary_end = len(single_text) - len(s4) - boundary_region = single_text[boundary_start:boundary_end] - - # The boundary from the split words is s2+s3 (a+k) + + boundary_region = single_text[len(s1) : len(single_text)-len(s4)] split_boundary = s2 + s3 - - # Calculate Levenshtein distance between the boundary regions - boundary_dist = levenshtein_distance(split_boundary, boundary_region) - - # If the boundary distance is within threshold, it's a valid sandhi match - sandhi_threshold = weights.get('sandhi_threshold') - if boundary_dist <= sandhi_threshold: - # Score calculation: match reward - split penalty - boundary error penalty - score = weights['match_base'] + weights['split_merge_penalty'] - if boundary_dist > 0: - score -= (boundary_dist / len(single_text)) - return score - - return -float('inf') # Not a valid sandhi match - -def align_arrays(arr1, arr2, max_distance=0, weights=None): + dist = levenshtein_distance(split_boundary, boundary_region) + if dist <= weights.get('sandhi_char_tolerence', 2): + score = weights['match_reward'] + weights['split_merge_penalty'] + return score - (dist / len(single_text)) + + return -float('inf') + +def align_arrays(arr1, tags1, arr2, tags2, weights=None) -> tuple[list[tuple[str, str]], list[tuple[str, str]], float]: if weights is None: weights = DEFAULT_WEIGHTS m, n = len(arr1), len(arr2) @@ -127,111 +71,83 @@ def align_arrays(arr1, arr2, max_distance=0, weights=None): # Initialize gaps for i in range(1, m + 1): - dp[i][0] = dp[i-1][0] + get_gap_penalty(arr1[i-1], weights) + dp[i][0] = dp[i-1][0] + get_gap_penalty(tags1[i-1], weights) for j in range(1, n + 1): - dp[0][j] = dp[0][j-1] + get_gap_penalty(arr2[j-1], weights) + dp[0][j] = dp[0][j-1] + get_gap_penalty(tags2[j-1], weights) # Fill DP for i in range(1, m + 1): for j in range(1, n + 1): - - # 1. Standard Match/Mismatch - if words_match(arr1[i-1], arr2[j-1], max_distance): + # 1. Standard Match/Mismatch (Category Aware) + score = 0 + if arr1[i-1] == arr2[j-1]: score = get_match_score(arr1[i-1], arr2[j-1], weights) else: - score = get_mismatch_penalty(arr1[i-1], arr2[j-1], weights) + score = get_mismatch_penalty(arr1[i-1], tags1[i-1], arr2[j-1], tags2[j-1], weights) match_val = dp[i-1][j-1] + score - # 2. Standard Indel - del_val = dp[i-1][j] + get_gap_penalty(arr1[i-1], weights) - ins_val = dp[i][j-1] + get_gap_penalty(arr2[j-1], weights) - - # 3. NEW: SPLIT CHECK (1 Ref matches 2 Hyp) -> Ref[i] vs Hyp[j-1]+Hyp[j] - split_val = -float('inf') - if j >= 2: - # Only apply split check if all tokens involved are words (not numbers or punctuations) - if is_word(arr1[i-1]) and is_word(arr2[j-2]) and is_word(arr2[j-1]): - # Combine current and previous hypothesis tokens - combined_hyp = [arr2[j-2], arr2[j-1]] - score_split = check_sandhi_match(combined_hyp, arr1[i-1], weights) - split_val = dp[i-1][j-2] + score_split - - # 4. NEW: MERGE CHECK (2 Ref match 1 Hyp) -> Ref[i-1]+Ref[i] vs Hyp[j] - merge_val = -float('inf') - if i >= 2: - # Only apply merge check if all tokens involved are words (not numbers or punctuations) - if is_word(arr1[i-2]) and is_word(arr1[i-1]) and is_word(arr2[j-1]): - combined_ref = [arr1[i-2] , arr1[i-1]] - score_merge = check_sandhi_match(combined_ref, arr2[j-1], weights) - merge_val = dp[i-2][j-1] + score_merge + # 2. Standard Indel (Category-aware gap penalties) + del_val = dp[i-1][j] + get_gap_penalty(tags1[i-1], weights) + ins_val = dp[i][j-1] + get_gap_penalty(tags2[j-1], weights) + + # 3. Sandhi Split/Merge (Only for CAT_WORD) + split_val = merge_val = -float('inf') + + # Split: 1 Ref matches 2 Hyp + if j >= 2 and tags1[i-1] == CAT_WORD and tags2[j-2] == CAT_WORD and tags2[j-1] == CAT_WORD: + score_split = check_sandhi_match([arr2[j-2], arr2[j-1]], arr1[i-1], weights) + split_val = dp[i-1][j-2] + score_split + + # Merge: 2 Ref match 1 Hyp + if i >= 2 and tags1[i-2] == CAT_WORD and tags1[i-1] == CAT_WORD and tags2[j-1] == CAT_WORD: + score_merge = check_sandhi_match([arr1[i-2], arr1[i-1]], arr2[j-1], weights) + merge_val = dp[i-2][j-1] + score_merge dp[i][j] = max(match_val, del_val, ins_val, split_val, merge_val) # Traceback - aligned_arr1 = [] - aligned_arr2 = [] + aligned_ref = [] + aligned_hyp = [] i, j = m, n while i > 0 or j > 0: - current = dp[i][j] - - # Helper to avoid repetitive float comparison - def is_close(val): return abs(current - val) < 1e-9 - - # Check SPLIT (1 Ref -> 2 Hyp), only for word tokens - if j >= 2 and i > 0: - # Only check split for word tokens (not numbers or punctuations) - if is_word(arr1[i-1]) and is_word(arr2[j-2]) and is_word(arr2[j-1]): - combined_hyp = [arr2[j-2] , arr2[j-1]] - score_split = check_sandhi_match(combined_hyp, arr1[i-1], weights) - if is_close(dp[i-1][j-2] + score_split): - # We align 1 Ref with 2 Hyps - aligned_arr1.append(arr1[i-1]) - aligned_arr2.append(f"SPLIT:{arr2[j-2]} {arr2[j-1]}") - i -= 1; j -= 2 - continue - - # Check MERGE (2 Ref -> 1 Hyp), only for word tokens - if i >= 2 and j > 0: - # Only check merge for word tokens (not numbers or punctuations) - if is_word(arr1[i-2]) and is_word(arr1[i-1]) and is_word(arr2[j-1]): - combined_ref = [arr1[i-2] , arr1[i-1]] - score_merge = check_sandhi_match(combined_ref, arr2[j-1], weights) - if is_close(dp[i-2][j-1] + score_merge): - aligned_arr1.append(f"MERGE:{arr1[i-2]} {arr1[i-1]}") - aligned_arr2.append(arr2[j-1]) - i -= 2; j -= 1 - continue - - # Standard checks + curr = dp[i][j] + def is_close(v): return abs(curr - v) < 1e-7 + + # Trace Sandhi Split + if j >= 2 and i > 0 and tags1[i-1] == CAT_WORD: + score = check_sandhi_match([arr2[j-2], arr2[j-1]], arr1[i-1], weights) + if is_close(dp[i-1][j-2] + score): + aligned_ref.append((arr1[i-1], tags1[i-1])) + aligned_hyp.append((f"SPLIT:{arr2[j-2]} {arr2[j-1]}", CAT_WORD)) + i -= 1; j -= 2; continue + + # Trace Sandhi Merge + if i >= 2 and j > 0 and tags2[j-1] == CAT_WORD: + score = check_sandhi_match([arr1[i-2], arr1[i-1]], arr2[j-1], weights) + if is_close(dp[i-2][j-1] + score): + aligned_ref.append((f"MERGE:{arr1[i-2]} {arr1[i-1]}", CAT_WORD)) + aligned_hyp.append((arr2[j-1], tags2[j-1])) + i -= 2; j -= 1; continue + + # Trace Standard Match/Mismatch if i > 0 and j > 0: - if words_match(arr1[i-1], arr2[j-1], max_distance): - step = get_match_score(arr1[i-1], arr2[j-1], weights) - else: - step = get_mismatch_penalty(arr1[i-1], arr2[j-1], weights) - - if is_close(dp[i-1][j-1] + step): - aligned_arr1.append(arr1[i-1]) - aligned_arr2.append(arr2[j-1]) - i -= 1; j -= 1 - continue - - if i > 0 and is_close(dp[i-1][j] + get_gap_penalty(arr1[i-1], weights)): - aligned_arr1.append(arr1[i-1]) - aligned_arr2.append('**') + if arr1[i-1] == arr2[j-1]: score = get_match_score(arr1[i-1], arr2[j-1], weights) + else: score = get_mismatch_penalty(arr1[i-1], tags1[i-1], arr2[j-1], tags2[j-1], weights) + + if is_close(dp[i-1][j-1] + score): + aligned_ref.append((arr1[i-1], tags1[i-1])) + aligned_hyp.append((arr2[j-1], tags2[j-1])) + i -= 1; j -= 1; continue + + # Trace Gaps (Category-aware) + if i > 0 and is_close(dp[i-1][j] + get_gap_penalty(tags1[i-1], weights)): + aligned_ref.append((arr1[i-1], tags1[i-1])) + aligned_hyp.append(("**", "GAP")) i -= 1 - elif j > 0: - aligned_arr1.append('**') - aligned_arr2.append(arr2[j-1]) - j -= 1 else: - # Should not happen if logic is correct - break - - return aligned_arr1[::-1], aligned_arr2[::-1], dp[m][n] + aligned_ref.append(("**", "GAP")) + aligned_hyp.append((arr2[j-1], tags2[j-1])) + j -= 1 -def align_text(text1, text2, weights=None): - arr1 = tokenizer(text1) - arr2 = tokenizer(text2) - aligned1, aligned2, score = align_arrays(arr1, arr2, weights=weights) - return aligned1, aligned2, score \ No newline at end of file + return aligned_ref[::-1], aligned_hyp[::-1], dp[m][n] diff --git a/src/dicterrors/constants.py b/src/dicterrors/constants.py new file mode 100644 index 0000000..c8d0731 --- /dev/null +++ b/src/dicterrors/constants.py @@ -0,0 +1,135 @@ +""" +Constants and shared definitions for the DictErrors package. + +This module serves as the single source of truth for all package-wide constants +including token categories, formatting parameters, and utility functions. +""" + +# Base token category constants (always present) +CAT_WORD = "WORD" +CAT_PUNCT = "PUNCT" +CAT_NUMERAL = "NUMERAL" + +# Base categories - domain categories are added dynamically +CATEGORIES = [CAT_WORD, CAT_PUNCT, CAT_NUMERAL] + + +def get_categories(domain_config=None): + """ + Get category list including domain category if configured. + + Args: + domain_config: DomainConfig instance or None + + Returns: + List of category names + + Examples: + >>> from domain_config import MEDICAL_DOMAIN + >>> cats = get_categories(MEDICAL_DOMAIN) + >>> # ['WORD', 'PUNCT', 'NUMERAL', 'MEDICAL'] + + >>> cats = get_categories(None) + >>> # ['WORD', 'PUNCT', 'NUMERAL'] + """ + if domain_config is None: + return CATEGORIES.copy() + return CATEGORIES + [domain_config.category] + + +# Table formatting constants +TABLE_WIDTH = 85 +COLUMN_WIDTHS = { + 'dataset': 25, + 'metric': 8, + 'sandhi': 6 +} + + +# Utility Functions + +def calculate_combined_total(stats_dict: dict) -> int: + """ + Calculate total tokens across all categories. + + Args: + stats_dict: Dictionary mapping category names to stat dicts with 'total' field + + Returns: + Sum of 'total' field across all categories + + Example: + >>> stats = {"WORD": {"total": 100}, "LEGAL": {"total": 5}} + >>> calculate_combined_total(stats) + 105 + """ + return sum(stats_dict[cat]["total"] for cat in stats_dict) + + +def init_stat_dict(categories=None) -> dict: + """ + Initialize empty statistics dictionary for categories. + + Uses consistent full field names throughout: + - substitutions, insertions, deletions (not sub, ins, del) + - correct (not cor) + - sandhi_hits (not sandhi) + + Args: + categories: List of category names (defaults to CATEGORIES) + + Returns: + Dict mapping category names to stat dicts with zeroed counts + + Example: + >>> stats = init_stat_dict() + >>> stats["WORD"] + {'substitutions': 0, 'insertions': 0, 'deletions': 0, 'correct': 0, 'total': 0, 'sandhi_hits': 0} + + >>> from domain_config import MEDICAL_DOMAIN + >>> stats = init_stat_dict(get_categories(MEDICAL_DOMAIN)) + >>> "MEDICAL" in stats + True + """ + if categories is None: + categories = CATEGORIES + + return { + cat: { + "substitutions": 0, + "insertions": 0, + "deletions": 0, + "correct": 0, + "total": 0, + "sandhi_hits": 0 + } + for cat in categories + } + + +def format_table_header(domain_label="DER") -> str: + """ + Generate formatted table header for evaluation results. + + Args: + domain_label: Label for domain error rate (default: "DER") + + Returns: + Multi-line string with header row and separator line + + Example: + >>> print(format_table_header("LER")) + DATASET | WER | LER | NER | PER | SANDHI + ------------------------------------------------------------------------------------- + + >>> print(format_table_header("MER")) + DATASET | WER | MER | NER | PER | SANDHI + ------------------------------------------------------------------------------------- + """ + dw = COLUMN_WIDTHS['dataset'] + mw = COLUMN_WIDTHS['metric'] + sw = COLUMN_WIDTHS['sandhi'] + + header = f"{'DATASET':<{dw}} | {'WER':>{mw}} | {domain_label:>{mw}} | {'NER':>{mw}} | {'PER':>{mw}} | {'SANDHI':>{sw}}" + separator = "-" * TABLE_WIDTH + return f"{header}\n{separator}" diff --git a/src/dicterrors/domain_config.py b/src/dicterrors/domain_config.py new file mode 100644 index 0000000..f6de55d --- /dev/null +++ b/src/dicterrors/domain_config.py @@ -0,0 +1,96 @@ +""" +Domain configuration for domain-aware tokenization. + +Allows users to specify domain-critical terminology that should be +treated as atomic tokens and tracked separately for error analysis. +""" +import re +from typing import List, Union, Optional + + +class DomainConfig: + """Configuration for a domain-specific terminology set.""" + + def __init__( + self, + name: str, + patterns: Union[str, List[str]], + category: Optional[str] = None, + label: Optional[str] = None, + case_sensitive: bool = False + ): + """ + Initialize domain configuration. + + Args: + name: Domain name (e.g., "legal", "medical", "financial") + patterns: Either a regex pattern string or list of domain terms + category: Category name for tokens (default: "DOMAIN_{NAME}") + label: Short label for error rate (default: "{NAME}ER") + case_sensitive: Whether pattern matching is case-sensitive + + Examples: + >>> # Using list of terms + >>> legal = DomainConfig("legal", ["u/s", "r/w", "sec."]) + + >>> # Using regex + >>> medical = DomainConfig("medical", r'mg|ml|cc|\d+mg') + + >>> # Custom category and label + >>> financial = DomainConfig("financial", + ... ["$", "€", "₹"], + ... category="CURRENCY", + ... label="CER") + """ + self.name = name + self.case_sensitive = case_sensitive + + # Convert patterns to regex + if isinstance(patterns, str): + self.pattern_regex = patterns + elif isinstance(patterns, list): + if not patterns: + raise ValueError("patterns list cannot be empty") + # Escape special regex characters in each term + escaped = [re.escape(term) for term in patterns] + self.pattern_regex = '|'.join(escaped) + else: + raise TypeError("patterns must be str (regex) or list (terms)") + + # Set category and label with sensible defaults + self.category = category or f"DOMAIN_{name.upper()}" + self.label = label or f"{name.upper()}ER" + + # Compile regex for efficiency + flags = 0 if case_sensitive else re.IGNORECASE + self.compiled_pattern = re.compile(self.pattern_regex, flags=flags) + + def matches(self, text: str) -> bool: + """Check if text matches this domain pattern.""" + return bool(self.compiled_pattern.match(text)) + + def __repr__(self): + return f"DomainConfig(name='{self.name}', category='{self.category}', label='{self.label}')" + + +# Pre-defined domain configurations for common use cases +# Users can use these or create their own + +LEGAL_DOMAIN = DomainConfig( + name="legal", + patterns=[ + "u/s", "r/w", "w.p.", "o.s.", "no.", + "v.", "vs.", "art.", "sec.", "PW", "CW", "Ext." + ], + category="LEGAL", + label="LER", + case_sensitive=False +) + +MEDICAL_DOMAIN = DomainConfig( + name="medical", + patterns=r'mg|ml|cc|mcg|\d+mg|\d+ml', + category="MEDICAL", + label="MER", + case_sensitive=False +) diff --git a/src/dicterrors/measure.py b/src/dicterrors/measure.py index 1d6fbe8..7b0cff8 100644 --- a/src/dicterrors/measure.py +++ b/src/dicterrors/measure.py @@ -1,142 +1,109 @@ -from .align import is_punctuation, is_number, is_word, align_text +from typing import Optional +from .tokenize import domain_aware_tokenizer +from .align import align_arrays +from .constants import get_categories, init_stat_dict, calculate_combined_total +from .domain_config import DomainConfig -def token_error_rates(aligned_ref, aligned_hyp): +def token_error_rates(aligned_ref, aligned_hyp, domain_config: Optional[DomainConfig] = None, normalize: bool = True) -> dict[str, dict[str, float | int]]: """ - Calculate Word Error Rate (WER), Punctuation Error Rate (PER), and Numeral Error Rate (NER) - from aligned arrays. - - Handles special alignment tags: - - SPLIT:word1 word2 (Ref has 1 token, Hyp has 2) -> Counts as Correct Match (if semantically valid) - - MERGE:word1 word2 (Ref has 2 tokens, Hyp has 1) -> Counts as Correct Match + Calculate error rates from aligned tokens. + + Args: + aligned_ref: list of (text, tag) tuples + aligned_hyp: list of (text, tag) tuples + domain_config: Domain configuration (None for no domain) + normalize: If True, check normalized equality for matches (default: True) + + Returns: + Dictionary with error rates for each category """ + categories = get_categories(domain_config) + stats = init_stat_dict(categories) - # Initialize counters for each category - word_sub = word_ins = word_del = word_correct = word_total = 0 - punct_sub = punct_ins = punct_del = punct_correct = punct_total = 0 - num_sub = num_ins = num_del = num_correct = num_total = 0 - - # New counters for Sandhi/Agglutination stats (Optional, but useful for your paper) - sandhi_splits = 0 - sandhi_merges = 0 - - # Count errors by comparing aligned tokens - for ref_token, hyp_token in zip(aligned_ref, aligned_hyp): - - # --- 1. HANDLE SPECIAL SANDHI TAGS --- - is_split = hyp_token.startswith("SPLIT:") - is_merge = ref_token.startswith("MERGE:") + for (r_text, r_tag), (h_text, h_tag) in zip(aligned_ref, aligned_hyp): - if is_split: - # Scenario: Ref="mazhakkalathu", Hyp="SPLIT:mazha kalathu" - # We treat this as a semantic match (Correct) - sandhi_splits += 1 - word_total += 1 - word_correct += 1 + # 1. Handle Insertions (Gap in Reference) + if r_text == "**": + # We categorize the insertion error based on what the ASR hallucinated + if h_tag in stats: + stats[h_tag]["insertions"] += 1 continue - - if is_merge: - # Scenario: Ref="MERGE:mazha kalathu", Hyp="mazhakkalathu" - # We treat this as a semantic match (Correct) - # Note: Ref technically had 2 tokens, but we aligned them to 1. - # For WER standard, we count "Total Ref Words". - # If Ref was "mazha" "kalathu", that's 2 words. - # But our alignment collapsed them. - # To be mathematically rigorous for WER: - # We should count this as 2 Reference Words and 2 Correct Matches - # (effectively saying both words were successfully captured, just merged). - - sandhi_merges += 1 - word_total += 2 # We count the original 2 words - word_correct += 2 + + # All other cases (Match, Sub, Del) are categorized by the REFERENCE tag + if r_tag not in stats: continue + curr = stats[r_tag] + + # 2. Handle Sandhi (Corrected Matches) + if "MERGE:" in r_text: + curr["total"] += 2 # A merge represents 2 original words + curr["correct"] += 2 + curr["sandhi_hits"] += 1 continue - # --- 2. STANDARD LOGIC --- - - # Skip if both are gaps (shouldn't happen in proper alignment) - if ref_token == '**' and hyp_token == '**': + if "SPLIT:" in h_text: + curr["total"] += 1 + curr["correct"] += 1 + curr["sandhi_hits"] += 1 continue - # Insertion (gap in reference) - elif ref_token == '**': - if is_word(hyp_token): - word_ins += 1 - elif is_punctuation(hyp_token): - punct_ins += 1 - elif is_number(hyp_token): - num_ins += 1 - - # Deletion (gap in hypothesis) - elif hyp_token == '**': - if is_word(ref_token): - word_del += 1 - word_total += 1 - elif is_punctuation(ref_token): - punct_del += 1 - punct_total += 1 - elif is_number(ref_token): - num_del += 1 - num_total += 1 - - # Substitution or correct + # 3. Standard Logic + curr["total"] += 1 + if h_text == "**": + curr["deletions"] += 1 + elif r_text == h_text: + curr["correct"] += 1 else: - if is_word(ref_token): - word_total += 1 - if ref_token == hyp_token: - word_correct += 1 - else: - word_sub += 1 - elif is_punctuation(ref_token): - punct_total += 1 - if ref_token == hyp_token: - punct_correct += 1 + # Check if tokens match after normalization (if enabled) + if normalize: + from .normalize import normalize_token + r_normalized = normalize_token(r_text, r_tag) + h_normalized = normalize_token(h_text, h_tag) + if r_normalized == h_normalized: + curr["correct"] += 1 else: - punct_sub += 1 - elif is_number(ref_token): - num_total += 1 - if ref_token == hyp_token: - num_correct += 1 - else: - num_sub += 1 - - # Calculate error rates - wer = (word_sub + word_ins + word_del) / max(1, word_total) - per = (punct_sub + punct_ins + punct_del) / max(1, punct_total) - ner = (num_sub + num_ins + num_del) / max(1, num_total) - - - report = { - "word": { - "substitutions": word_sub, - "insertions": word_ins, - "deletions": word_del, - "correct": word_correct, - "sandhi_splits": sandhi_splits, - "sandhi_merges": sandhi_merges, - "total_reference": word_total, - "error_rate": wer - }, - "punctuation": { - "substitutions": punct_sub, - "insertions": punct_ins, - "deletions": punct_del, - "correct": punct_correct, - "total_reference": punct_total, - "error_rate": per - }, - "numeral": { - "substitutions": num_sub, - "insertions": num_ins, - "deletions": num_del, - "correct": num_correct, - "total_reference": num_total, - "error_rate": ner + curr["substitutions"] += 1 + else: + curr["substitutions"] += 1 + + # Final calculations for the report + # Calculate combined denominator across ALL categories + combined_total = calculate_combined_total(stats) + + report = {} + for cat in categories: + s = stats[cat] + errors = s["substitutions"] + s["insertions"] + s["deletions"] + + # Use combined denominator for all categories + rate = errors / max(1, combined_total) + + report[cat] = { + "error_rate": rate, + "substitutions": s["substitutions"], + "insertions": s["insertions"], + "deletions": s["deletions"], + "correct": s["correct"], + "total_ref": s["total"], + "sandhi_hits": s["sandhi_hits"], + "combined_total": combined_total # Store for transparency } - } - return wer, per, ner, report - -def text_error_rates(ref_text, hyp_text): - """Calculate error rates between two text strings.""" - # Align the token arrays - aligned_ref, aligned_hyp, _ = align_text(ref_text, hyp_text) - # Calculate error rates based on aligned tokens - return token_error_rates(aligned_ref, aligned_hyp) \ No newline at end of file + + return report + +def text_error_rates(ref_text, hyp_text, domain_config: Optional[DomainConfig] = None, normalize: bool = True) -> dict[str, dict[str, float | int]]: + """ + Calculate error rates from raw text. + + Args: + ref_text: Reference text + hyp_text: Hypothesis text + domain_config: Domain configuration (None for no domain) + normalize: If True, apply normalization for matching (default: True) + + Returns: + Dictionary with error rates for each category + """ + t1, g1 = domain_aware_tokenizer(ref_text, domain_config) + t2, g2 = domain_aware_tokenizer(hyp_text, domain_config) + aligned_ref, aligned_hyp, _ = align_arrays(t1, g1, t2, g2) + return token_error_rates(aligned_ref, aligned_hyp, domain_config, normalize) \ No newline at end of file diff --git a/src/dicterrors/measure_batch.py b/src/dicterrors/measure_batch.py index bb5c293..763a5da 100644 --- a/src/dicterrors/measure_batch.py +++ b/src/dicterrors/measure_batch.py @@ -1,196 +1,128 @@ +from collections import defaultdict +from typing import Optional from .measure import text_error_rates +from .reporting import format_dataset_table +from .constants import get_categories, init_stat_dict, calculate_combined_total, format_table_header, TABLE_WIDTH +from .domain_config import DomainConfig import json -from typing import List, Dict, Any -from collections import defaultdict - -def compute_sample_errors( - input_file: str, - output_file: str = None, - ref_field: str = "transcript_cleaned", - hyp_field: str = "prediction", - source_dataset_field: str = "source_dataset", - audio_path_field: str = "file_path" -) -> List[Dict[str, Any]]: +def compute_sample_errors(input_file, output_file=None, ref_field="transcript_cleaned", hyp_field="prediction", source_dataset_field="source_dataset", domain_config: Optional[DomainConfig] = None, normalize: bool = True) -> list[dict]: """ - Evaluate predictions and save results to a JSON file. - + Compute error metrics for all samples in a JSONL file. + Args: - input_file: Path to input JSONL file with predictions - output_file: Path to output JSON file for evaluation results - ref_field: Field name for reference text in input file. default: transcript_cleaned - hyp_field: Field name for hypothesis/prediction text in input file. default: prediction - source_dataset_field: Field name for source dataset in input file. default: source_dataset - audio_path_field: Field name for audio file path in input file. default: file_path - + input_file: Path to JSONL file + output_file: Optional path to save detailed results + ref_field: Field name for reference text + hyp_field: Field name for hypothesis text + source_dataset_field: Field name for dataset identifier + domain_config: Domain configuration (None for no domain) + normalize: If True, apply normalization for matching (default: True) + Returns: - List of evaluation results as a list of dictionaries + List of result dictionaries with detailed reports """ results = [] + with open(input_file, "r", encoding="utf-8") as f: + for line in f: + data = json.loads(line) + # Ensure we have a source_dataset field + if source_dataset_field not in data or data[source_dataset_field] is None: + data[source_dataset_field] = "unknown" - print(f"Loading data from {input_file}") - - with open(input_file, "r") as f: - lines = f.readlines() + # Pass domain_config and normalize to text_error_rates + report = text_error_rates(data[ref_field], data[hyp_field], domain_config, normalize) + data["detailed_report"] = report + results.append(data) - for line in lines: - data = json.loads(line) - file_path = data[audio_path_field] - ref_text = data[ref_field] - if source_dataset_field in data: - source_dataset = data[source_dataset_field] - else: - source_dataset = None - hyp_text = data[hyp_field] - - # Calculate error rates - wer, per, ner, report = text_error_rates(ref_text, hyp_text) - - # Store results - result = { - "file_path": file_path, - "ref_text": ref_text, - "hyp_text": hyp_text, - "source_dataset": source_dataset, - "WER": wer, - "PER": per, - "NER": ner, - "detailed_report": report - } - - results.append(result) - - # Write results to JSON file + # Save detailed results if output file is specified if output_file: - with open(output_file, "w") as f: - json.dump(results, f, indent=2, ensure_ascii=False) - print(f"Results saved to {output_file}") - - print("Evaluation complete.") + with open(output_file, "w", encoding="utf-8") as f: + for result in results: + f.write(json.dumps(result, ensure_ascii=False) + "\n") + return results +def compute_aggregate_metrics(sample_results, domain_config: Optional[DomainConfig] = None) -> dict[str, dict[str, dict[str, dict[str, float | int]]]]: + """ + Aggregate metrics across all samples. + Args: + sample_results: List of result dictionaries from compute_sample_errors + domain_config: Domain configuration (None for no domain) -def _init_stats(): - """Helper to initialize zeroed stats for a category.""" - return { - "substitutions": 0, "insertions": 0, "deletions": 0, - "correct": 0, "total_reference": 0, - "sandhi_splits": 0, "sandhi_merges": 0 - } + Returns: + Dictionary with 'overall' and 'by_dataset' aggregated metrics + """ + categories = get_categories(domain_config) + overall_agg = init_stat_dict(categories) + dataset_aggs = defaultdict(lambda: init_stat_dict(categories)) + + for res in sample_results: + ds = res.get("source_dataset", "unknown") + report = res["detailed_report"] + + for cat in categories: + if cat not in report: + continue + + # Update overall + overall_agg[cat]["substitutions"] += report[cat]["substitutions"] + overall_agg[cat]["insertions"] += report[cat]["insertions"] + overall_agg[cat]["deletions"] += report[cat]["deletions"] + overall_agg[cat]["total"] += report[cat]["total_ref"] + overall_agg[cat]["sandhi_hits"] += report[cat]["sandhi_hits"] + + # Update per-dataset + dataset_aggs[ds][cat]["substitutions"] += report[cat]["substitutions"] + dataset_aggs[ds][cat]["insertions"] += report[cat]["insertions"] + dataset_aggs[ds][cat]["deletions"] += report[cat]["deletions"] + dataset_aggs[ds][cat]["total"] += report[cat]["total_ref"] + dataset_aggs[ds][cat]["sandhi_hits"] += report[cat]["sandhi_hits"] + + def calculate_rates(agg): + # Calculate combined denominator across ALL categories + combined_total = calculate_combined_total(agg) + + metrics = {} + for cat in agg: + a = agg[cat] + errs = a["substitutions"] + a["insertions"] + a["deletions"] + metrics[cat] = { + "error_rate": errs / max(1, combined_total), # Combined denominator + "substitutions": a["substitutions"], + "insertions": a["insertions"], + "deletions": a["deletions"], + "correct": a["total"] - errs, + "sandhi_hits": a["sandhi_hits"], + "total": a["total"], + "combined_total": combined_total # Store for reference + } + return metrics -def _init_accumulator(): - """Helper to initialize accumulators for all token types.""" return { - "word": _init_stats(), - "punctuation": _init_stats(), - "numeral": _init_stats() + "overall": calculate_rates(overall_agg), + "by_dataset": {ds: calculate_rates(stats) for ds, stats in dataset_aggs.items()} } -def _update_stats(accumulator: Dict, sample_report: Dict): - """ - In-place update of the accumulator with values from a single sample report. - """ - for category in ["word", "punctuation", "numeral"]: - acc_cat = accumulator[category] - src_cat = sample_report[category] - - acc_cat["substitutions"] += src_cat.get("substitutions", 0) - acc_cat["insertions"] += src_cat.get("insertions", 0) - acc_cat["deletions"] += src_cat.get("deletions", 0) - acc_cat["correct"] += src_cat.get("correct", 0) - acc_cat["total_reference"] += src_cat.get("total_reference", 0) - - # Add Sandhi stats if they exist - acc_cat["sandhi_splits"] += src_cat.get("sandhi_splits", 0) - acc_cat["sandhi_merges"] += src_cat.get("sandhi_merges", 0) - -def _calculate_final_metrics(accumulator: Dict) -> Dict: - """ - Converts raw counts in an accumulator to WER/PER/NER percentages. - """ - final_metrics = {} - - # Map internal category names to output metric names - metric_map = {"word": "WER", "punctuation": "PER", "numeral": "NER"} - - for category, metric_name in metric_map.items(): - stats = accumulator[category] - - # Calculate Error Rate: (S + I + D) / max(1, N) - errors = stats["substitutions"] + stats["insertions"] + stats["deletions"] - total = stats["total_reference"] - - rate = errors / max(1, total) - - # Copy counts and add the calculated rate - final_metrics[category] = stats.copy() - final_metrics[category]["error_rate"] = rate - final_metrics[category]["metric_name"] = metric_name # e.g. WER - - return final_metrics - -def compute_aggregate_metrics(sample_results: List[Dict[str, Any]]) -> Dict[str, Any]: +def print_evaluation_summary(agg_results, domain_config: Optional[DomainConfig] = None) -> None: """ - Aggregates error rates over the entire dataset and per-source-dataset. - + Print evaluation summary table. + Args: - sample_results: List of dicts returned by compute_sample_errors - - Returns: - Dict containing 'overall' and 'by_dataset' metrics. + agg_results: Aggregated results from compute_aggregate_metrics + domain_config: Domain configuration for label formatting """ - - # 1. Initialize Accumulators - overall_acc = _init_accumulator() - dataset_accs = defaultdict(_init_accumulator) - - # 2. Iterate and Accumulate (The "Micro-Average" Logic) - for result in sample_results: - report = result["detailed_report"] - source = result.get("source_dataset", "unknown") - - # Update Global - _update_stats(overall_acc, report) - - # Update Specific Dataset - _update_stats(dataset_accs[source], report) - - # 3. Calculate Final Rates - output = { - "overall": _calculate_final_metrics(overall_acc), - "by_dataset": {} - } - - for source, acc in dataset_accs.items(): - output["by_dataset"][source] = _calculate_final_metrics(acc) - - return output + table_data = format_dataset_table(agg_results, domain_config) -def print_evaluation_summary(aggregated_results: Dict[str, Any]): - """ - Pretty prints the aggregated results for console viewing. - """ - def _print_row(name, metrics): - wer = metrics['word']['error_rate'] - per = metrics['punctuation']['error_rate'] - ner = metrics['numeral']['error_rate'] - splits = metrics['word']['sandhi_splits'] - merges = metrics['word']['sandhi_merges'] - - if name: - print(f"{name:<20} | {wer:8.2%} | {per:8.2%} | {ner:8.2%} | {splits+merges:<4}") - - print("\n" + "="*65) - print(f"{'DATASET':<20} | {'WER':<8} | {'PER':<8} | {'NER':<8} | {'SANDHI'}") - print("-" * 65) - - # Print Overall - _print_row("OVERALL", aggregated_results["overall"]) - print("-" * 65) - - # Print per dataset - for source, metrics in aggregated_results["by_dataset"].items(): - _print_row(source, metrics) - print("="*65 + "\n") + domain_label = domain_config.label if domain_config else "DER" + print("\n" + "=" * TABLE_WIDTH) + print(format_table_header(domain_label)) + + for row in table_data: + is_overall = row['Dataset'] == 'OVERALL' + print(f"{row['Dataset']:<25} | {row['WER']:>8} | {row[domain_label]:>8} | {row['NER']:>8} | {row['PER']:>8} | {row['Sandhi']:>6}") + if is_overall: + print("-" * TABLE_WIDTH) + print("=" * TABLE_WIDTH + "\n") \ No newline at end of file diff --git a/src/dicterrors/normalize.py b/src/dicterrors/normalize.py new file mode 100644 index 0000000..abe67bf --- /dev/null +++ b/src/dicterrors/normalize.py @@ -0,0 +1,124 @@ +"""Token normalization for semantic matching. + +This module provides normalization functions to convert tokens to canonical forms +for comparison purposes while preserving original formatting for display. +""" +import re +from .constants import CAT_NUMERAL + + +def normalize_date(text: str) -> str: + """Normalize date formats to canonical dd-mm-yyyy. + + Converts various date separator formats (dd.mm.yyyy, dd/mm/yyyy, dd-mm-yyyy) + to a canonical form for comparison. + + Args: + text: Token text that may contain a date + + Returns: + Normalized date string if pattern matches, otherwise original text + + Examples: + >>> normalize_date("22.05.2023") + '22-05-2023' + >>> normalize_date("22/05/2023") + '22-05-2023' + >>> normalize_date("22-05-2023") + '22-05-2023' + """ + # Match dates with any separator (. / -) + pattern = r'(\d{1,2})[./-](\d{1,2})[./-](\d{2,4})' + match = re.match(pattern, text) + if match: + day, month, year = match.groups() + return f"{day}-{month}-{year}" + return text + + +def normalize_currency(text: str) -> str: + """Normalize currency by removing comma separators. + + Converts numbers with comma separators (10,500 or 1,00,000) to + continuous digit strings (10500 and 100000) for comparison. + + Args: + text: Token text that may contain currency or large numbers + + Returns: + Text with commas removed if text contains digits, otherwise original + + Examples: + >>> normalize_currency("10,500") + '10500' + >>> normalize_currency("1,00,000") + '100000' + >>> normalize_currency("123") + '123' + """ + if ',' in text and any(c.isdigit() for c in text): + return text.replace(',', '') + return text + + +def normalize_numeral(text: str) -> str: + """Apply all numeral normalization rules. + + Tries date normalization first, then currency normalization. + Time formats (HH:MM) are already in canonical form and unchanged. + + Args: + text: Token text with NUMERAL category + + Returns: + Normalized text after applying applicable rules + + Examples: + >>> normalize_numeral("22.05.2023") + '22-05-2023' + >>> normalize_numeral("10,500") + '10500' + >>> normalize_numeral("10:30") + '10:30' + """ + # Try date normalization first + normalized = normalize_date(text) + if normalized != text: + return normalized + + # Try currency/number normalization + normalized = normalize_currency(text) + if normalized != text: + return normalized + + # Time formats (HH:MM) already canonical + return text + + +def normalize_token(text: str, category: str) -> str: + """Apply category-specific normalization. + + Dispatches to appropriate normalization function based on token category. + Only NUMERAL category has normalization rules currently; other categories + return original text unchanged. + + Args: + text: Original token text + category: Token category (WORD, NUMERAL, PUNCT, LEGAL, etc.) + + Returns: + Normalized token text + + Examples: + >>> normalize_token("22.05.2023", "NUMERAL") + '22-05-2023' + >>> normalize_token("hello", "WORD") + 'hello' + >>> normalize_token(".", "PUNCT") + '.' + """ + if category == CAT_NUMERAL: + return normalize_numeral(text) + + # No normalization for WORD, PUNCT, LEGAL, etc. + return text diff --git a/src/dicterrors/reporting.py b/src/dicterrors/reporting.py new file mode 100644 index 0000000..625b692 --- /dev/null +++ b/src/dicterrors/reporting.py @@ -0,0 +1,232 @@ +""" +Report formatting and presentation utilities. + +This module provides shared functions for formatting error metrics +and alignment results for both CLI and web UI presentations. +""" +from typing import Dict, List, Tuple, Optional +from .constants import CAT_WORD, CAT_NUMERAL, CAT_PUNCT, get_categories, format_table_header, TABLE_WIDTH +from .domain_config import DomainConfig + + +def format_metrics_dict(metrics: Dict, domain_config: Optional[DomainConfig] = None) -> Dict[str, str]: + """ + Extract WER/DER/NER/PER from aggregate metrics. + + Args: + metrics: Dictionary containing error metrics for each category + domain_config: Domain configuration (None to skip domain metrics) + + Returns: + Dictionary with formatted metric strings ready for table display + """ + result = { + "WER": f"{metrics[CAT_WORD]['error_rate']:.2%}", + "NER": f"{metrics[CAT_NUMERAL]['error_rate']:.2%}", + "PER": f"{metrics[CAT_PUNCT]['error_rate']:.2%}", + } + + if domain_config: + result[domain_config.label] = f"{metrics[domain_config.category]['error_rate']:.2%}" + result["Sandhi"] = metrics[CAT_WORD]['sandhi_hits'] + result["Total"] = metrics[CAT_WORD].get('combined_total', 0) + + return result + + +def extract_error_rates(report: Dict, domain_config: Optional[DomainConfig] = None) -> Dict: + """ + Extract error rates from report for display. + + Args: + report: Dictionary containing error metrics for each category + domain_config: Domain configuration (None to skip domain metrics) + + Returns: + Dictionary with raw numeric error rates + """ + result = { + 'wer': report[CAT_WORD]['error_rate'], + 'ner': report[CAT_NUMERAL]['error_rate'], + 'per': report[CAT_PUNCT]['error_rate'], + 'sandhi': report[CAT_WORD]['sandhi_hits'] + } + + if domain_config: + # Use lowercase label for consistency + result[domain_config.label.lower()] = report[domain_config.category]['error_rate'] + + return result + + +def format_dataset_table(agg_results: Dict, domain_config: Optional[DomainConfig] = None) -> List[Dict]: + """ + Format aggregate results as list of dicts for table display. + + Args: + agg_results: Dictionary with 'overall' and 'by_dataset' keys + domain_config: Domain configuration for metric formatting + + Returns: + List of dictionaries, each containing Dataset name and error metrics + """ + table_data = [] + + # Overall row + overall = format_metrics_dict(agg_results['overall'], domain_config) + overall['Dataset'] = 'OVERALL' + table_data.append(overall) + + # Per-dataset rows + for ds, metrics in agg_results['by_dataset'].items(): + row = format_metrics_dict(metrics, domain_config) + row['Dataset'] = ds + table_data.append(row) + + return table_data + + +def format_error_counts_table(report: Dict, domain_config: Optional[DomainConfig] = None) -> List[Dict]: + """ + Format error counts by category for detailed inspection. + + Args: + report: Token error rates report from token_error_rates() + domain_config: Domain configuration (None to use base categories) + + Returns: + List of dictionaries with Category, Type, and Count + """ + categories = get_categories(domain_config) + + counts = [] + for cat in categories: + if cat not in report: + continue + counts.extend([ + {"Category": cat, "Type": "Substitutions", "Count": report[cat]["substitutions"]}, + {"Category": cat, "Type": "Insertions", "Count": report[cat]["insertions"]}, + {"Category": cat, "Type": "Deletions", "Count": report[cat]["deletions"]}, + {"Category": cat, "Type": "Correct", "Count": report[cat]["correct"]} + ]) + return counts + + +def write_summary_to_file(agg_results: Dict, output_path: str, domain_config: Optional[DomainConfig] = None) -> None: + """ + Write evaluation summary to file safely. + + Args: + agg_results: Dictionary with 'overall' and 'by_dataset' keys + output_path: Path to output file + domain_config: Domain configuration for label formatting + """ + with open(output_path, 'w', encoding='utf-8') as f: + table_data = format_dataset_table(agg_results, domain_config) + + # Write formatted table with proper headers + domain_label = domain_config.label if domain_config else "DER" + f.write("\n" + "=" * TABLE_WIDTH + "\n") + f.write(format_table_header(domain_label) + "\n") + + for row in table_data: + is_overall = row['Dataset'] == 'OVERALL' + if is_overall and table_data.index(row) > 0: + # Add separator line before OVERALL row if it's not first + f.write("-" * TABLE_WIDTH + "\n") + + # Dynamic column access + f.write( + f"{row['Dataset']:<25} | " + f"{row['WER']:>8} | " + f"{row[domain_label]:>8} | " + f"{row['NER']:>8} | " + f"{row['PER']:>8} | " + f"{row['Sandhi']:>6}\n" + ) + + f.write("=" * TABLE_WIDTH + "\n") + + +def format_alignment_dict(aligned_ref: List[Tuple], aligned_hyp: List[Tuple], normalize: bool = True) -> List[Dict]: + """ + Extract alignment data as structured dict for rendering. + + Provides shared error detection logic used by both CLI and UI. + + Args: + aligned_ref: List of (text, tag) tuples for reference + aligned_hyp: List of (text, tag) tuples for hypothesis + normalize: If True, apply normalization when checking equality (default: True) + + Returns: + List of dicts with ref_text, hyp_text, error_type, token_type + """ + results = [] + for (ref_txt, ref_tag), (hyp_txt, hyp_tag) in zip(aligned_ref, aligned_hyp): + # Determine error type (shared logic) + if "MERGE:" in ref_txt or "SPLIT:" in hyp_txt: + error_type = "sandhi" + elif ref_txt == "**": + error_type = "insertion" + elif hyp_txt == "**": + error_type = "deletion" + elif ref_txt == hyp_txt: + error_type = "correct" + else: + # Check if tokens match after normalization + if normalize: + from .normalize import normalize_token + ref_normalized = normalize_token(ref_txt, ref_tag) + hyp_normalized = normalize_token(hyp_txt, hyp_tag) + if ref_normalized == hyp_normalized: + error_type = "correct" + else: + error_type = "substitution" + else: + error_type = "substitution" + + # Clean display text - remove markers + display_ref = ref_txt.replace("MERGE:", "").replace("SPLIT:", "") if ref_txt != "**" else "**" + display_hyp = hyp_txt.replace("MERGE:", "").replace("SPLIT:", "") if hyp_txt != "**" else "**" + token_type = ref_tag if ref_tag != "GAP" else hyp_tag + + results.append({ + 'ref_text': display_ref, + 'hyp_text': display_hyp, + 'error_type': error_type, + 'token_type': token_type + }) + + return results + + +def format_alignment_table(aligned_ref: List[Tuple], aligned_hyp: List[Tuple], normalize: bool = True) -> List[Dict]: + """ + Format aligned tokens for visualization table. + + Uses format_alignment_dict() internally for error detection logic. + + Args: + aligned_ref: List of (text, tag) tuples for reference + aligned_hyp: List of (text, tag) tuples for hypothesis + normalize: If True, apply normalization when checking equality (default: True) + + Returns: + List of dictionaries with Position, Reference, Hypothesis, Error Type, Token Type + """ + # Use shared error detection logic + alignment_data = format_alignment_dict(aligned_ref, aligned_hyp, normalize) + + # Add position and capitalize error types for table display + rows = [] + for i, item in enumerate(alignment_data): + rows.append({ + "Position": i + 1, + "Reference": item['ref_text'], + "Hypothesis": item['hyp_text'], + "Error Type": item['error_type'].capitalize(), + "Token Type": item['token_type'] + }) + + return rows diff --git a/src/dicterrors/tokenize.py b/src/dicterrors/tokenize.py index ed70524..66eca9f 100644 --- a/src/dicterrors/tokenize.py +++ b/src/dicterrors/tokenize.py @@ -1,52 +1,117 @@ import re -def tokenizer(text: str) -> list[str]: +from typing import Optional, Tuple, List +from .constants import CAT_WORD, CAT_PUNCT, CAT_NUMERAL +from .domain_config import DomainConfig + + +def domain_aware_tokenizer( + text: str, + domain_config: Optional[DomainConfig] = None +) -> Tuple[List[str], List[str]]: """ - General-purpose tokenizer that: - - Normalizes whitespace - - Separates punctuation as standalone tokens - - Separates any characters attached to numbers (both directions) + Tokenize text with domain-aware entity shielding. - Examples: - "abc123def" -> ["abc", "123", "def"] - "9ാം" -> ["9", "ാം"] - "3-ഓ" -> ["3", "-", "ഓ"] - "₹100,000" -> ["₹", "100", ",", "000"] + Domain-critical terms (e.g., legal abbreviations, medical terms) + are identified and protected from punctuation splitting, then + tagged with their domain category for separate error tracking. Args: - text (str): The input string. + text: Input text to tokenize + domain_config: Domain configuration (None to disable domain handling) Returns: - list[str]: A list of tokens. + Tuple of (tokens, tags) where tags are category labels + + Examples: + >>> # Legal domain + >>> from .domain_config import LEGAL_DOMAIN + >>> tokens, tags = domain_aware_tokenizer("charged u/s 302 IPC", LEGAL_DOMAIN) + >>> # tokens: ["charged", "u/s", "302", "IPC"] + >>> # tags: ["WORD", "LEGAL", "NUMERAL", "WORD"] + + >>> # Medical domain + >>> from .domain_config import MEDICAL_DOMAIN + >>> tokens, tags = domain_aware_tokenizer("Take 500mg daily", MEDICAL_DOMAIN) + + >>> # No domain handling + >>> tokens, tags = domain_aware_tokenizer("Just regular text", None) """ - if not isinstance(text, str): - raise TypeError("Input must be a string.") + if not text: + return [], [] + + # Define numeral patterns (always protected) + num_inner = r'\d{1,2}[./-]\d{1,2}[./-]\d{2,4}|\d{1,2}:\d{2}|\d+(?:,\d+)*(?:\.\d+)?' - # 1) Normalize whitespace - text = re.sub(r"\s+", " ", text).strip() + # Build protected pattern with named groups + if domain_config: + # Combine domain and numeral patterns + protected_pattern = f'(?P{domain_config.pattern_regex})|(?P{num_inner})' + else: + # Only numeral patterns + protected_pattern = f'(?P{num_inner})' - # 2) Separate punctuation by surrounding with spaces - # Include common ASCII punctuation, dashes, currency, and symbols frequently seen around numbers - # Note: the hyphen, en dash, em dash are handled; colon/slash are useful for times and dates + # Extract and replace entities + entities = [] + flags = 0 if (domain_config and domain_config.case_sensitive) else re.IGNORECASE + + for match in re.finditer(protected_pattern, text, flags=flags): + if domain_config and match.lastgroup == 'domain': + entities.append(('domain', match.group('domain'))) + elif match.lastgroup == 'numeral': + entities.append(('numeral', match.group('numeral'))) + + # Replace entities with placeholder + placeholder_text = re.sub(protected_pattern, " __ENTITY__ ", text, flags=flags) + + # Separate punctuation punctuation_chars = r"[.,?!;:\-\/\"'()\[\]{}—–+*=<>|@#%^&₹$]" - text = re.sub(f"({punctuation_chars})", r" \1 ", text) + placeholder_text = re.sub(f"({punctuation_chars})", r" \1 ", placeholder_text) + + # Split into raw tokens + raw_tokens = placeholder_text.split() - # 3) Ensure boundaries between digits and non-digits are spaced (both directions) - # \d in Python re matches Unicode digits as well. - # We avoid splitting spaces themselves by requiring non-space on the other side. - text = re.sub(r"(\d)([^\d\s])", "\g<1> \g<2>", text) - text = re.sub(r"([^\d\s])(\d)", "\g<1> \g<2>", text) + # Split and classify tokens + tokens = [] + tags = [] + entity_idx = 0 - # 4) Normalize whitespace again after insertions - text = re.sub(r"\s+", " ", text).strip() + for t in raw_tokens: + if t == "__ENTITY__": + if entity_idx < len(entities): + entity_type, entity_val = entities[entity_idx] + tokens.append(entity_val) - tokens = [t for t in text.split(" ") if t] - return tokens + if entity_type == 'domain': + tags.append(domain_config.category) + else: # numeral + tags.append(CAT_NUMERAL) + entity_idx += 1 + else: + # Check if it's a word or punctuation + if any(c.isalnum() for c in t): + tokens.append(t) + tags.append(CAT_WORD) + else: + tokens.append(t) + tags.append(CAT_PUNCT) -def main(): - text = "ഈ 3-ഓ 4-ഓ വയസ്സുള്ള കുട്ടിക്ക് 9ാം തീയതി, 9:30-ന് ഫോൺ കിട്ടിയോ? abc123def. 19-ರಂದು & 19ರಂದು" - tokens = tokenizer(text) - print(tokens) + return tokens, tags +# TEST CASE if __name__ == "__main__": - main() \ No newline at end of file + from .domain_config import LEGAL_DOMAIN + + sample = "U/S 302 of IPC on 22.05.2023 at 10:30, for Rs. 10,500. ഈ 3-ഓ 4-ഓ വയസ്സുള്ള കുട്ടിക്ക് 9 ാം തീയതി , 9:30-ന് ഫോൺ കിട്ടിയോ? 1,500 rupees abc123def. 19-രംദു & 19-രംദു" + + # Test with legal domain + print("=== Legal Domain ===") + tokens, tags = domain_aware_tokenizer(sample, LEGAL_DOMAIN) + for tok, tag in zip(tokens, tags): + print(f"{tok:<15} | {tag}") + + # Test without domain + print("\n=== No Domain ===") + tokens, tags = domain_aware_tokenizer(sample, None) + for tok, tag in zip(tokens, tags): + print(f"{tok:<15} | {tag}") \ No newline at end of file diff --git a/visualizer.py b/visualizer.py index 925977d..d0c41a1 100644 --- a/visualizer.py +++ b/visualizer.py @@ -4,309 +4,335 @@ import json import pandas as pd import jiwer -from collections import defaultdict -from dicterrors import tokenizer, token_error_rates, align_arrays, DEFAULT_WEIGHTS, is_number, is_punctuation, compute_aggregate_metrics, compute_sample_errors -import tempfile +import tempfile +from pathlib import Path +from dicterrors import ( + domain_aware_tokenizer, + align_arrays, + token_error_rates, + compute_aggregate_metrics, + compute_sample_errors, + DEFAULT_WEIGHTS, + CAT_WORD, CAT_PUNCT, CAT_NUMERAL, + LEGAL_DOMAIN +) +from dicterrors.reporting import ( + format_dataset_table, + format_error_counts_table, + extract_error_rates, + format_alignment_dict +) -# --- HELPER: CSS INJECTION --- +def parse_data(content_list): + """Handles both JSON (list of dicts) and JSONL (line by line).""" + # Join the lines to check the overall structure + full_content = "\n".join(content_list).strip() + + # Try parsing as a standard JSON list first + if full_content.startswith("["): + try: + return json.loads(full_content) + except: + pass + + # Fallback to JSONL (line by line) + records = [] + for line in content_list: + if line.strip(): + try: + records.append(json.loads(line)) + except Exception as e: + st.error(f"Failed to parse line: {line[:50]}... Error: {e}") + return records + + +# --- HELPER: CSS INJECTION (Restored & Enhanced) --- def inject_custom_css(): st.markdown(""" """, unsafe_allow_html=True) -# --- HELPER: HTML GENERATOR --- -def generate_alignment_html(ref_tokens, hyp_tokens): - """Generates the HTML table for alignment visualization.""" +# --- HELPER: HTML GENERATOR (Tagged Version) --- +def generate_alignment_html(aligned_ref, aligned_hyp, normalize=True): + """ + aligned_ref/hyp: List of tuples (text, tag) + Uses shared format_alignment_dict() for error detection logic. + + Args: + aligned_ref: List of (text, tag) tuples for reference + aligned_hyp: List of (text, tag) tuples for hypothesis + normalize: If True, apply normalization when determining colors + """ + # Use shared alignment logic with normalization parameter + alignment_data = format_alignment_dict(aligned_ref, aligned_hyp, normalize) + + # Map error types to CSS classes + status_map = { + 'correct': 's-correct', + 'substitution': 's-sub', + 'insertion': 's-ins', + 'deletion': 's-del', + 'sandhi': 's-merge' + } + html = '
' - for r, h in zip(ref_tokens, hyp_tokens): - - # --- 1. Detect Status & Clean Text --- - status = "s-correct" - token_type = "t-word" - - if r.startswith("MERGE:"): - status = "s-merge" - r = r.replace("MERGE:", "") - elif h.startswith("SPLIT:"): - status = "s-merge" - h = h.replace("SPLIT:", "") - elif r == "**" or r == "": - status = "s-ins" - elif h == "**" or h == "": - status = "s-del" - elif r != h: - status = "s-sub" - - # --- 2. Determine Border Type --- - check_content = r if r not in ["**", ""] else h - if " " in check_content: token_type = "t-word" - elif is_number(check_content): token_type = "t-number" - elif is_punctuation(check_content): token_type = "t-punct" - else: token_type = "t-word" - - disp_r = r if (r != "**" and r != "") else " " - disp_h = h if (h != "**" and h != "") else " " - - html += f"" - + + for item in alignment_data: + status = status_map.get(item['error_type'], 's-correct') + border_class = f"t-{item['token_type']}" + + # Replace ** with   for display + disp_r = item['ref_text'] if item['ref_text'] != "**" else " " + disp_h = item['hyp_text'] if item['hyp_text'] != "**" else " " + + html += f""" + """ + html += "
{disp_r}
{disp_h}
 +
{disp_r}
+
{disp_h}
+
{item['token_type']}
+
" return html -# --- HELPER: METRIC CARD RENDERER (SHARED) --- -def render_metrics_comparison(d_wer, d_per, d_ner, j_wer, j_mer): - """Renders the comparison cards for DictErrors vs Jiwer.""" +# --- HELPER: METRIC CARD RENDERER --- +def render_metrics_comparison(report, jiwer_wer, domain_config): mc1, mc2 = st.columns(2) with mc1: - st.subheader("DictErrors") - st.markdown(f"""
WER: {d_wer:.2%}
PER: {d_per:.2%} | NER: {d_ner:.2%}
""", unsafe_allow_html=True) + st.subheader("DictErrors (Domain-Aware)") + # Use shared error rate extraction function + rates = extract_error_rates(report, domain_config) + + # Get domain label dynamically + domain_label_lower = domain_config.label.lower() if domain_config else "der" + + st.markdown(f""" +
+
General WER: {rates['wer']:.2%}
+
{domain_config.name.title()} WER: {rates[domain_label_lower]:.2%}
+
Numeral WER: {rates['ner']:.2%}
+
Punctuation WER: {rates['per']:.2%}
+
+ """, unsafe_allow_html=True) with mc2: - st.subheader("Jiwer") - st.markdown(f"""
WER: {j_wer:.2%}
MER: {j_mer:.2%}
""", unsafe_allow_html=True) + st.subheader("Jiwer Baseline") + st.markdown(f""" +
+
Global WER: {jiwer_wer:.2%}
+
Standard word-level calculation without category shielding or Sandhi awareness.
+
+ """, unsafe_allow_html=True) # --- HELPER: RENDER ANALYSIS --- -def render_analysis(ref_text, hyp_text, weights): - """Reusable function to render the metrics and visualization.""" - inject_custom_css() - +def render_analysis(ref_text, hyp_text, weights, normalize=True): + # Use legal domain configuration + domain_config = LEGAL_DOMAIN + # 1. DictErrors Calculation - custom_ref_tok = tokenizer(ref_text) - custom_hyp_tok = tokenizer(hyp_text) - c_ref, c_hyp, c_score = align_arrays(custom_ref_tok, custom_hyp_tok, weights=weights) - c_wer, c_per, c_ner, c_report = token_error_rates(c_ref, c_hyp) + t1, g1 = domain_aware_tokenizer(ref_text, domain_config) + t2, g2 = domain_aware_tokenizer(hyp_text, domain_config) + a_ref, a_hyp, _ = align_arrays(t1, g1, t2, g2, weights=weights) + report = token_error_rates(a_ref, a_hyp, domain_config, normalize) # 2. Jiwer Calculation - jiwer_out = jiwer.process_words(ref_text, hyp_text) - - # Extract Jiwer Alignment - j_ref_viz, j_hyp_viz = [], [] - for chunk in jiwer_out.alignments[0]: - if chunk.type in ['equal', 'substitute']: - j_ref_viz.extend([ref_text.split()[i] for i in range(chunk.ref_start_idx, chunk.ref_end_idx)]) - j_hyp_viz.extend([hyp_text.split()[i] for i in range(chunk.hyp_start_idx, chunk.hyp_end_idx)]) - elif chunk.type == 'delete': - for i in range(chunk.ref_start_idx, chunk.ref_end_idx): - j_ref_viz.append(ref_text.split()[i]); j_hyp_viz.append("**") - elif chunk.type == 'insert': - for i in range(chunk.hyp_start_idx, chunk.hyp_end_idx): - j_ref_viz.append("**"); j_hyp_viz.append(hyp_text.split()[i]) - - # 3. Render UI - st.subheader("1. Alignment Visualization") - t1, t2 = st.tabs(["✨ DictErrors Alignment", "Jiwer Alignment"]) - - with t1: - st.write("Green indicates correct tokens, Red indicates substitutions, Yellow indicates insertions/deletions, Purple indicates Split/Merge of Words (Sandhi).") - st.markdown(generate_alignment_html(c_ref, c_hyp), unsafe_allow_html=True) - with st.expander("Detailed Report"): st.json(c_report) - with t2: - st.write("Green indicates correct tokens, Red indicates substitutions, Yellow indicates insertions/deletions.") - st.markdown(generate_alignment_html(j_ref_viz, j_hyp_viz), unsafe_allow_html=True) + j_wer = jiwer.wer(ref_text, hyp_text) - st.subheader("2. Metric Comparison") - render_metrics_comparison(c_wer, c_per, c_ner, jiwer_out.wer, jiwer_out.mer) + # 3. Render + st.subheader("Alignment Visualization") + st.markdown(generate_alignment_html(a_ref, a_hyp, normalize), unsafe_allow_html=True) + render_metrics_comparison(report, j_wer, domain_config) # --- UI CONFIG --- -st.set_page_config(layout="wide", page_title="DictErrors vs Jiwer") -st.title("⚖️ Error Analysis: DictErrors vs Jiwer") -inject_custom_css() # Ensure CSS is loaded globally +st.set_page_config(layout="wide", page_title="DictErrors Legal Visualizer") +st.title("⚖️ DictErrors vs Jiwer: Legal & Indic Evaluation") +inject_custom_css() -# --- SIDEBAR --- -st.sidebar.header("🔧 Custom Penalty Tuning") +# --- SIDEBAR: WEIGHT TUNING (Restored) --- +st.sidebar.header("🔧 Penalty Tuning") weights = {} -with st.sidebar.expander("Agglutination (Sandhi)", expanded=True): - weights['split_merge_penalty'] = st.slider("Split/Merge Penalty", -2.0, 0.0, float(DEFAULT_WEIGHTS.get('split_merge_penalty', -0.5)), 0.1) - weights['sandhi_threshold'] = st.slider("Sandhi Char Tolerance", 0, 5, int(DEFAULT_WEIGHTS.get('sandhi_threshold', 2)), 1) -with st.sidebar.expander("Gap & Mismatch", expanded=False): - weights['gap_punct_num'] = st.slider("Gap: Punct/Num", -5.0, 0.0, float(DEFAULT_WEIGHTS['gap_punct_num']), 0.5) - weights['gap_word_base'] = st.slider("Gap: Word Base", -5.0, 0.0, float(DEFAULT_WEIGHTS['gap_word_base']), 0.5) - weights['gap_word_factor'] = st.slider("Gap: Word Length Factor", 0.0, 2.0, float(DEFAULT_WEIGHTS['gap_word_factor']), 0.1) - weights['mismatch_word_base'] = st.slider("Sub: Word Base", -5.0, 0.0, float(DEFAULT_WEIGHTS['mismatch_word_base']), 0.5) - weights['match_base'] = st.slider("Match Reward", 1.0, 5.0, float(DEFAULT_WEIGHTS['match_base']), 0.5) - weights['mismatch_punct_cross'] = float(DEFAULT_WEIGHTS['mismatch_punct_cross']) - weights['mismatch_word_num'] = float(DEFAULT_WEIGHTS['mismatch_word_num']) - weights['mismatch_num_num'] = float(DEFAULT_WEIGHTS['mismatch_num_num']) - weights['mismatch_punct_punct'] = float(DEFAULT_WEIGHTS['mismatch_punct_punct']) + +with st.sidebar.expander("Category Penalties", expanded=True): + weights['gap_penalty'] = st.slider("Gap Penalty - General", -5.0, 0.0, DEFAULT_WEIGHTS['gap_penalty'], 0.5) + weights['gap_penalty_punct'] = st.slider("Gap Penalty - Punctuation", -5.0, 0.0, DEFAULT_WEIGHTS['gap_penalty_punct'], 0.1) + weights['mismatch_default_penalty'] = st.slider("Mismatch Penalty - Default", -5.0, 0.0, DEFAULT_WEIGHTS['mismatch_default_penalty'], 0.5) + weights['mismatch_cross_punct_penalty'] = st.slider("Mismatch Penalty - Cross-Category Punct", -10.0, 0.0, DEFAULT_WEIGHTS['mismatch_cross_punct_penalty'], 1.0) + weights['match_reward'] = st.slider("Match Reward", 1.0, 5.0, DEFAULT_WEIGHTS['match_reward'], 0.5) + +with st.sidebar.expander("Agglutination & Sandhi", expanded=False): + weights['split_merge_penalty'] = st.slider("Split/Merge Penalty", -2.0, 0.0, DEFAULT_WEIGHTS['split_merge_penalty'], 0.1) + weights['sandhi_char_tolerence'] = st.slider("Sandhi Char Tolerance", 0, 5, DEFAULT_WEIGHTS['sandhi_char_tolerence'], 1) + +# Normalization toggle +st.sidebar.divider() +st.sidebar.header("🔄 Token Normalization") +normalize_enabled = st.sidebar.checkbox( + "Enable Normalization", + value=True, + help="When enabled, treats date/currency format variations as matches (22.05.2023 = 22/05/2023, 10,500 = 10500)" +) + +# Session state management +st.sidebar.divider() +st.sidebar.header("🗑️ Session Management") +if st.sidebar.button("Clear Session Data"): + if 'detailed_results' in st.session_state: + del st.session_state['detailed_results'] + if 'global_jiwer' in st.session_state: + del st.session_state['global_jiwer'] + if 'ref_col' in st.session_state: + del st.session_state['ref_col'] + if 'hyp_col' in st.session_state: + del st.session_state['hyp_col'] + st.rerun() # --- MAIN INPUT --- -tab_manual, tab_json = st.tabs(["Text Input", "JSON File"]) +tab_manual, tab_json = st.tabs(["Manual Inspection", "Batch Dataset Analysis"]) -# --- TAB 1: MANUAL --- with tab_manual: mc1, mc2 = st.columns(2) - with mc1: - m_ref = st.text_area("Reference", height=100, value="തദ്ദേശ സ്വയംഭരണ സ്ഥാപനങ്ങൾ") - with mc2: - m_hyp = st.text_area("Hypothesis", height=100, value="തദ്ദേശ സ്വയംഭരണസ്ഥാപനങ്ങൾ") - - if st.button("Analyze Alignment", type="primary"): - render_analysis(m_ref, m_hyp, weights) + with mc1: m_ref = st.text_area("Reference", height=100, value="U/S 302 പ്രകാരം മഴക്കാലത്ത് ശിക്ഷിക്കപ്പെടും") + with mc2: m_hyp = st.text_area("Hypothesis", height=100, value="US 302 പ്രകാരം മഴ കാലത്ത് ശിക്ഷിക്കപ്പെടും") + if st.button("Analyze Manual Input", type="primary"): + render_analysis(m_ref, m_hyp, weights, normalize_enabled) -# --- TAB 2: JSON --- + #--- UPDATED TAB 2: JSON/JSONL --- with tab_json: - - # Create two main columns for the top section col_config, col_batch = st.columns([1, 1], gap="large") - records = [] - default_path = os.path.join(os.path.dirname(__file__), 'examples', 'dictation-eval', 'predictions.jsonl') - # --- LEFT COLUMN: LOAD DATA --- + # Robust Default Path: Looking in the current dir /dictation-eval/ + # Adjust this if your folder structure is different! + base_dir = Path(__file__).parent + default_path = base_dir / "examples" / "dictation-eval" / "predictions.jsonl" + with col_config: st.markdown("### 📂 Load Data") - - # 1. Source Selection - upload_opt = st.radio("Source", ["Default Example", "Upload File"], horizontal=True, label_visibility="collapsed") + upload_opt = st.radio("Source", ["Default Path", "Upload File"], horizontal=True) data_content = None if upload_opt == "Upload File": - uploaded = st.file_uploader("Upload .jsonl", type=["jsonl", "json"], label_visibility="collapsed") - if uploaded: data_content = uploaded.getvalue().decode('utf-8').splitlines() - elif os.path.exists(default_path): - with open(default_path, 'r', encoding='utf-8') as f: - data_content = f.read().splitlines() + # ACCEPT BOTH JSON AND JSONL + uploaded = st.file_uploader("Upload File", type=["json", "jsonl"]) + if uploaded: + data_content = uploaded.getvalue().decode('utf-8').splitlines() + else: + # CHECK DEFAULT PATH + if default_path.exists(): + with open(default_path, 'r', encoding='utf-8') as f: + data_content = f.readlines() + else: + st.warning(f"Default file not found at: `{default_path.relative_to(base_dir)}`") if data_content: - try: - records = [json.loads(line) for line in data_content if line.strip()] - if not isinstance(records, list): records = [records] - st.success(f"Loaded {len(records)} records") - except Exception as e: - st.error(f"Error parsing JSON: {e}") + records = parse_data(data_content) + if records: + st.success(f"Successfully loaded {len(records)} records") - # 2. Field Mapping - if records: - keys = list(records[0].keys()) - def get_idx(options, search): - for s in search: - if s in options: return options.index(s) - return 0 - - c1, c2 = st.columns(2) - with c1: ref_col = st.selectbox("Reference Field", keys, index=get_idx(keys, ["transcript_cleaned", "text", "reference"])) - with c2: hyp_col = st.selectbox("Hypothesis Field", keys, index=get_idx(keys, ["prediction", "hypothesis"])) - - # Source ID - source_options = ["(None)"] + keys - default_src_idx = 0 - for s in ["source_dataset", "file_path", "audio_path"]: - if s in keys: - default_src_idx = source_options.index(s) - break - - src_col_selection = st.selectbox("Source ID Field (Optional)", source_options, index=default_src_idx) - src_col = None if src_col_selection == "(None)" else src_col_selection + # Field validation with error handling + try: + keys = list(records[0].keys()) + # Auto-select fields if they exist + def_ref = keys.index("transcript_cleaned") if "transcript_cleaned" in keys else 0 + def_hyp = keys.index("prediction") if "prediction" in keys else 0 + except (KeyError, IndexError) as e: + st.error(f"Error accessing record fields: {e}") + keys = [] + def_ref = 0 + def_hyp = 0 + + if keys: + ref_col = st.selectbox("Reference Field", keys, index=def_ref) + hyp_col = st.selectbox("Hypothesis Field", keys, index=def_hyp) + src_col = st.selectbox("Dataset Split Field", ["(None)"] + keys) - # --- RIGHT COLUMN: BATCH ANALYSIS --- with col_batch: if records: st.markdown("### 📊 Dataset Evaluation") - - # Using a container to visually group the batch actions - with st.container(border=True): - if st.button("Run Batch Evaluation", type="primary", width='stretch'): - - with st.spinner("Processing records..."): - # Temp file dance for DictErrors - with tempfile.NamedTemporaryFile(mode='w+', suffix='.jsonl', delete=False, encoding='utf-8') as tmp_in: - for r in records: - tmp_in.write(json.dumps(r, ensure_ascii=False) + '\n') - tmp_in_path = tmp_in.name - - try: - # 1. Compute DictErrors - detailed_results = compute_sample_errors( - input_file=tmp_in_path, - output_file=None, - ref_field=ref_col, - hyp_field=hyp_col, - source_dataset_field=src_col if src_col else None, - audio_path_field="file_path" - ) - agg_stats = compute_aggregate_metrics(detailed_results) - - # 2. Compute Jiwer Aggregate - all_refs = [r.get(ref_col, "") for r in records] - all_hyps = [r.get(hyp_col, "") for r in records] - jiwer_agg = jiwer.process_words(all_refs, all_hyps) - - # 3. Display Comparison (DictErrors vs Jiwer) - st.write("#### Overall Batch Metrics") - overall_dict = agg_stats["overall"] - - render_metrics_comparison( - d_wer=overall_dict['word']['error_rate'], - d_per=overall_dict['punctuation']['error_rate'], - d_ner=overall_dict['numeral']['error_rate'], - j_wer=jiwer_agg.wer, - j_mer=jiwer_agg.mer - ) - - # 4. Downloads - st.markdown("---") - d_col1, d_col2 = st.columns(2) - - jsonl_str = "\n".join([json.dumps(r, ensure_ascii=False) for r in detailed_results]) - d_col1.download_button("📥 Detailed Evaluation", jsonl_str, "eval_detailed.jsonl", "application/json", width='stretch') - - d_col1, d_col2 = st.columns(2) - - - summary_txt = f"{'DATASET':<20} | {'WER':<8} | {'PER':<8} | {'NER':<8} | {'SANDHI'}\n" + "-"*65 + "\n" - def format_txt_row(name, m): - s = m['word']['sandhi_splits'] + m['word']['sandhi_merges'] - return f"{name:<20} | {m['word']['error_rate']:8.2%} | {m['punctuation']['error_rate']:8.2%} | {m['numeral']['error_rate']:8.2%} | {s:<4}\n" - - summary_txt += format_txt_row("OVERALL", agg_stats["overall"]) + "-"*65 + "\n" - if src_col: - for source, metrics in agg_stats["by_dataset"].items(): - summary_txt += format_txt_row(source, metrics) - - d_col1.download_button("📄 Summary", summary_txt, "eval_summary.txt", "text/plain", width='stretch') - - finally: - if os.path.exists(tmp_in_path): os.unlink(tmp_in_path) + if st.button("Run Batch Evaluation", type="primary"): + with tempfile.NamedTemporaryFile(mode='w+', suffix='.jsonl', delete=False, encoding='utf-8') as tmp: + for r in records: + tmp.write(json.dumps(r, ensure_ascii=False) + '\n') + tmp_path = tmp.name - else: - st.info("Click to compute aggregate statistics (WER/PER/NER) for the loaded dataset.") + try: + # Use legal domain configuration + domain_config = LEGAL_DOMAIN + + # 1. DictErrors Calculation + res_detailed = compute_sample_errors(tmp_path, ref_field=ref_col, hyp_field=hyp_col, domain_config=domain_config, normalize=normalize_enabled) + + # Ensure source_dataset is attached for aggregation + for i, r in enumerate(res_detailed): + r["source_dataset"] = records[i].get(src_col, "unknown") if src_col != "(None)" else "overall" + + agg = compute_aggregate_metrics(res_detailed, domain_config=domain_config) - st.divider() + # 2. Jiwer Global Comparison + all_refs = [r.get(ref_col, "") for r in records] + all_hyps = [r.get(hyp_col, "") for r in records] + jiwer_wer = jiwer.wer(all_refs, all_hyps) - # --- BOTTOM SECTION: INDIVIDUAL ANALYSIS --- - if records: + st.write("#### Overall Metrics") + # Using the consolidated 'error_rate' key + render_metrics_comparison(agg['overall'], jiwer_wer, domain_config) + + # 3. Dataset Breakdown Table + st.write("#### Per-Dataset Breakdown") + table_data = format_dataset_table(agg, domain_config) + # Remove OVERALL row for display (already shown above) + table_data = [row for row in table_data if row['Dataset'] != 'OVERALL'] + st.table(pd.DataFrame(table_data)) + + # 4. Detailed Error Counts Display (NEW) + with st.expander("📊 Detailed Error Counts"): + error_counts = format_error_counts_table(agg['overall'], domain_config) + st.dataframe(pd.DataFrame(error_counts), width='stretch') + + # Save results to session state (limit to 100 most recent) + MAX_STORED_RESULTS = 100 + st.session_state['detailed_results'] = res_detailed[-MAX_STORED_RESULTS:] + st.session_state['global_jiwer'] = jiwer_wer + st.session_state['ref_col'] = ref_col + st.session_state['hyp_col'] = hyp_col + finally: + if os.path.exists(tmp_path): os.unlink(tmp_path) + + if 'detailed_results' in st.session_state: + st.divider() st.markdown("### 🔍 Individual Record Inspection") - - # Create a display list for the selectbox - def format_option(i, r): - source_tag = f"[{r.get(src_col, 'N/A')}] " if src_col else "" - text_preview = r.get(ref_col, '')[:100] - return f"{i+1}. {source_tag}{text_preview}..." + res_list = st.session_state['detailed_results'] + # Retrieve field names from session state + saved_ref_col = st.session_state.get('ref_col', 'reference') + saved_hyp_col = st.session_state.get('hyp_col', 'hypothesis') - display_options = [format_option(i, r) for i, r in enumerate(records)] - - sel_idx = st.selectbox("Select a record to visualize:", range(len(records)), format_func=lambda x: display_options[x], label_visibility="collapsed") - - selected_record = records[sel_idx] - selected_ref = selected_record.get(ref_col, "") - selected_hyp = selected_record.get(hyp_col, "") + idx = st.selectbox("Select record", range(len(res_list)), + format_func=lambda i: f"Record {i+1}: {res_list[i][saved_ref_col][:60]}...") - # Render Analysis - if selected_ref and selected_hyp: - render_analysis(selected_ref, selected_hyp, weights) \ No newline at end of file + sel = res_list[idx] + render_analysis(sel[saved_ref_col], sel[saved_hyp_col], weights, normalize_enabled) \ No newline at end of file