调整数字归一化处理逻辑
parent
3084f3b79a
commit
6b39723029
|
|
@ -5,12 +5,33 @@
|
|||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# itntext导入 - 延迟导入以避免初始化问题
|
||||
_itntext_normalizer = None
|
||||
|
||||
_CHINESE_DECIMAL_SCALE_PATTERN = re.compile(
|
||||
r"(?P<integer>[零〇○一二两三四五六七八九十百千万亿]+)点"
|
||||
r"(?P<fraction>[零〇○一二两三四五六七八九]+)(?P<scale>万|亿)"
|
||||
)
|
||||
_CHINESE_DIGITS = {
|
||||
"零": "0",
|
||||
"〇": "0",
|
||||
"○": "0",
|
||||
"一": "1",
|
||||
"二": "2",
|
||||
"两": "2",
|
||||
"三": "3",
|
||||
"四": "4",
|
||||
"五": "5",
|
||||
"六": "6",
|
||||
"七": "7",
|
||||
"八": "8",
|
||||
"九": "9",
|
||||
}
|
||||
|
||||
|
||||
def _get_normalizer():
|
||||
"""获取itntext标准化器实例(单例模式)"""
|
||||
|
|
@ -45,7 +66,29 @@ def apply_itn_to_text(text: str) -> str:
|
|||
|
||||
try:
|
||||
normalizer = _get_normalizer()
|
||||
result = normalizer.normalize(text)
|
||||
# Protect Chinese decimal quantities with 万/亿: generic ITN rules may
|
||||
# interpret the spoken fractional digits as separate magnitude groups.
|
||||
# Normalize surrounding text normally, and keep the fraction digit-by-digit.
|
||||
matches = list(_CHINESE_DECIMAL_SCALE_PATTERN.finditer(text))
|
||||
if matches:
|
||||
parts = []
|
||||
cursor = 0
|
||||
for match in matches:
|
||||
parts.append(normalizer.normalize(text[cursor:match.start()]))
|
||||
integer = str(normalizer.normalize(match.group("integer"))).strip()
|
||||
if not re.fullmatch(r"\d+", integer):
|
||||
# If this ITN version cannot normalize the integer component,
|
||||
# fall back to its regular whole-text path rather than guess.
|
||||
result = normalizer.normalize(text)
|
||||
break
|
||||
fraction = "".join(_CHINESE_DIGITS[digit] for digit in match.group("fraction"))
|
||||
parts.append(f"{integer}.{fraction}{match.group('scale')}")
|
||||
cursor = match.end()
|
||||
else:
|
||||
parts.append(normalizer.normalize(text[cursor:]))
|
||||
result = "".join(parts)
|
||||
else:
|
||||
result = normalizer.normalize(text)
|
||||
logger.debug(f"ITN处理: '{text}' -> '{result}'")
|
||||
return result
|
||||
except Exception as e:
|
||||
|
|
|
|||
Loading…
Reference in New Issue