Initial commit

2026-02-06 20:31:14 +08:00
parent a0b51be095
commit c589bcb837
145 changed files with 28773 additions and 0 deletions
@@ -0,0 +1,72 @@
+import re
+
+import ToJyutping
+from g2pM import G2pM
+from g2p_en import G2p as G2pE
+
+_EN_WORD_RE = re.compile(r"^[A-Za-z]+(?:'[A-Za-z]+)*$")
+_ZH_WORD_RE = re.compile(r"[\u4e00-\u9fff]")
+
+EN_FLAG = "en_"
+YUE_FLAG = "yue_"
+ZH_FLAG = "zh_"
+
+g2p_zh = G2pM()
+g2p_en = G2pE()
+
+
+def is_chinese_char(word: str) -> bool:
+    if len(word) != 1:
+        return False
+    return bool(_ZH_WORD_RE.fullmatch(word))
+
+def is_english_word(word: str) -> bool:
+    if not word:
+        return False
+    return bool(_EN_WORD_RE.fullmatch(word))
+
+def g2p_cantonese(sent):
+    return ToJyutping.get_jyutping_list(sent)       # with tone
+
+def g2p_mandarin(sent):
+    return g2p_zh(sent, tone=True, char_split=False)
+
+def g2p_english(word):
+    return g2p_en(word)
+
+def g2p_transform(words, lang):
+        
+    zh_words = []
+    transformed_words = [0] * len(words)
+
+    for idx, w in enumerate(words):
+        if w == "<SP>":
+            transformed_words[idx] = w
+            continue
+
+        w = w.replace("?", "").replace(".", "").replace("!", "").replace(",", "")
+
+        if is_chinese_char(w):
+            zh_words.append([idx, w])
+        else:
+            if is_english_word(w):
+                w = EN_FLAG + "-".join(g2p_english(w.lower()))
+            else:
+                w = "<SP>"
+        transformed_words[idx] = w
+
+    sent = "".join([k[1] for k in zh_words])
+
+    # zh (zh and yue) transformer to g2p
+    if len(sent) > 0:
+        if lang == "Cantonese":
+            g2pm_rst = g2p_cantonese(sent)       # with tone
+            g2pm_rst = [YUE_FLAG + k[1] for k in g2pm_rst]
+        else:
+            g2pm_rst = g2p_mandarin(sent)
+            g2pm_rst = [ZH_FLAG + k for k in g2pm_rst]
+        for p, w in zip([k[0] for k in zh_words], g2pm_rst):
+            transformed_words[p] = w
+
+    return transformed_words
+