FlashCards/utils.py at main · Samster101/FlashCards · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
from __future__ import annotations
from typing import Optional
import unicodedata
import re

_PUNCT_TRANSLATION_TABLE = None


def _get_punct_table() -> dict[int, str]:
    global _PUNCT_TRANSLATION_TABLE
    if _PUNCT_TRANSLATION_TABLE is None:
        _PUNCT_TRANSLATION_TABLE = str.maketrans({
            '\u2019': "'",
            '\u2018': "'",
            '\u02BC': "'",
            '\u2032': "'",
            '\u00B4': "'",
            '\u0060': "'",
            '\u201C': '"',
            '\u201D': '"',
            '\u201E': '"',
            '\u2013': '-',
            '\u2014': '-',
            '\u2212': '-',
            '\u00A0': ' ',
            '\u2009': ' ',
            '\u200A': ' ',
            '\u2002': ' ',
            '\u2003': ' ',
            '\u202F': ' ',
            '\u2007': ' ',
        })
    return _PUNCT_TRANSLATION_TABLE


def normalize_text(s: Optional[str]) -> str:
    if s is None:
        return ''

    s = unicodedata.normalize('NFKC', s)
    s = s.translate(_get_punct_table())
    s = s.lower().strip()
    s = re.sub(r'\s+', ' ', s)
    return s