65 lines
915 B
Python
65 lines
915 B
Python
# -*- coding: utf-8 -*-
|
|
"""Conservative ASR text cleanup lexicons."""
|
|
|
|
from __future__ import annotations
|
|
|
|
SAFE_DOUBLE_WORDS = {
|
|
"哥哥",
|
|
"叔叔",
|
|
"爸爸",
|
|
"妈妈",
|
|
"奶奶",
|
|
"爷爷",
|
|
"姐姐",
|
|
"弟弟",
|
|
"妹妹",
|
|
"伯伯",
|
|
"姑姑",
|
|
"舅舅",
|
|
"星星",
|
|
"猩猩",
|
|
}
|
|
|
|
NUMERIC_STUTTER_CHARS = set("零〇○Ｏ一幺二两三四五六七八九十百千万亿点")
|
|
|
|
FILLER_TOKENS = (
|
|
"啊",
|
|
"呢",
|
|
"吧",
|
|
"哦",
|
|
"嗯",
|
|
"呃",
|
|
"哈",
|
|
"哇",
|
|
"呀",
|
|
"哎",
|
|
"诶",
|
|
"欸",
|
|
"额",
|
|
)
|
|
|
|
DISCOURSE_FILLER_TOKENS = (
|
|
"那个",
|
|
"这个",
|
|
"然后",
|
|
"所以说",
|
|
"其实",
|
|
"那么",
|
|
)
|
|
|
|
REPEAT_COLLAPSIBLE_DISCOURSE_TOKENS = (
|
|
"那个",
|
|
"这个",
|
|
"然后",
|
|
"就是",
|
|
"所以",
|
|
"所以说",
|
|
"其实",
|
|
"那么",
|
|
)
|
|
|
|
TERMINAL_FILLER_TOKENS = (
|
|
"哈",
|
|
"嗯",
|
|
)
|