Updated apply_regex_anonymization (#404)

* Updated ac_dc

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Update .gitmodules

Updating submodule config

* updated test case

Updated test case for faker address to be street_address rather than full address

* Updated apply_regex_anonymization

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

Co-authored-by: Ian Yu <ianyu@MBPC02FC7Z5MD6R.phub.net.cable.rogers.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: 3 people

ac_dc/anonymization.py

@@ -1,4 +1,5 @@
 from muliwai.pii_regexes import detect_ner_with_regex_and_context
+from muliwai.pii_regexes import regex_rulebase
 
 trannum = str.maketrans("0123456789", "1111111111")
 
@@ -184,7 +185,12 @@ def apply_regex_anonymization(
     sentence: str, lang_id: str, context_window: int = 20, anonymize_condition=None
 ) -> str:
     lang_id = lang_id.split("_")[0]
-    ner = detect_ner_with_regex_and_context(sentence, lang_id)
+    ner = detect_ner_with_regex_and_context(
+        sentence=sentence,
+        src_lang=lang_id,
+        context_window=context_window,
+        tag_type=regex_rulebase.keys(),
+    )
     if anonymize_condition:
         for (ent, start, end, tag) in ner:
             # we need to actually walk through and replace by start, end span.

ac_dc/test_anonymization.py

@@ -1,5 +1,4 @@
 import random
-
 from anonymization import apply_regex_anonymization
 from faker import Faker
 from num2words import num2words
@@ -20,7 +19,7 @@ def test_en():
     fake = Faker("en_US")
     sentences = [
         f"I am {num2words(random.randint(0,120))} years old, and she is {random.randint(0,120)} year-old",  # Age
-        f"Sherry lives at {fake.address()}",  # Address
+        f"Sherry lives at {fake.street_address()}",  # Address
         f"My dad is a cancer fighter. Her grandma is suffering from alzheimer's",  # Disease
         f"Let me tell you a secret, Mr. Nguyen's SSN is {fake.ssn() if random.choice([True, False]) else fake.ssn().replace('-', '')}.",  # Government ID
         f"Dear Ian, the payment through {fake.credit_card_number()} has been successfully executed.",  # Credit card
@@ -37,7 +36,7 @@ def test_zh():
     fake = Faker("zh_CN")
     sentences = [
         f'我今年{num2words(random.randint(0,120), lang="ja")}歲, 而她去年{random.randint(0,120)}岁',  # Age
-        f"我住在{fake.address()}",  # Address
+        f"我住在{fake.street_address()}",  # Address
         f"我爸是抗癌戰士。她奶奶有老人癡呆",  # Disease
         f"李雪妮小姐331125198402010129",  # Government ID
         f"先生，信用卡号{fake.credit_card_number()}已缴费成功",  # Credit card