调整数字归一化处理逻辑

main
Bifang 2026-10-12 14:30:22 +08:00
parent 3084f3b79a
commit 6b39723029
1 changed files with 44 additions and 1 deletions

View File

@ -5,12 +5,33 @@
"""
import logging
import re
logger = logging.getLogger(__name__)
# itntext导入 - 延迟导入以避免初始化问题
_itntext_normalizer = None
_CHINESE_DECIMAL_SCALE_PATTERN = re.compile(
r"(?P<integer>[零〇○一二两三四五六七八九十百千万亿]+)点"
r"(?P<fraction>[零〇○一二两三四五六七八九]+)(?P<scale>万|亿)"
)
_CHINESE_DIGITS = {
"零": "0",
"〇": "0",
"○": "0",
"一": "1",
"二": "2",
"两": "2",
"三": "3",
"四": "4",
"五": "5",
"六": "6",
"七": "7",
"八": "8",
"九": "9",
}
def _get_normalizer():
"""获取itntext标准化器实例(单例模式)"""
@ -45,7 +66,29 @@ def apply_itn_to_text(text: str) -> str:
try:
normalizer = _get_normalizer()
result = normalizer.normalize(text)
# Protect Chinese decimal quantities with 万/亿: generic ITN rules may
# interpret the spoken fractional digits as separate magnitude groups.
# Normalize surrounding text normally, and keep the fraction digit-by-digit.
matches = list(_CHINESE_DECIMAL_SCALE_PATTERN.finditer(text))
if matches:
parts = []
cursor = 0
for match in matches:
parts.append(normalizer.normalize(text[cursor:match.start()]))
integer = str(normalizer.normalize(match.group("integer"))).strip()
if not re.fullmatch(r"\d+", integer):
# If this ITN version cannot normalize the integer component,
# fall back to its regular whole-text path rather than guess.
result = normalizer.normalize(text)
break
fraction = "".join(_CHINESE_DIGITS[digit] for digit in match.group("fraction"))
parts.append(f"{integer}.{fraction}{match.group('scale')}")
cursor = match.end()
else:
parts.append(normalizer.normalize(text[cursor:]))
result = "".join(parts)
else:
result = normalizer.normalize(text)
logger.debug(f"ITN处理: '{text}' -> '{result}'")
return result
except Exception as e: