# -*- coding: utf-8 -*- """ 基于itntext的ITN(逆文本标准化)工具模块 使用itntext库提供高质量的中文ITN处理 """ import logging import re logger = logging.getLogger(__name__) # itntext导入 - 延迟导入以避免初始化问题 _itntext_normalizer = None _CHINESE_DECIMAL_SCALE_PATTERN = re.compile( r"(?P[零〇○一二两三四五六七八九十百千万亿]+)点" r"(?P[零〇○一二两三四五六七八九]+)(?P万|亿)" ) _CHINESE_DIGITS = { "零": "0", "〇": "0", "○": "0", "一": "1", "二": "2", "两": "2", "三": "3", "四": "4", "五": "5", "六": "6", "七": "7", "八": "8", "九": "9", } def _get_normalizer(): """获取itntext标准化器实例(单例模式)""" global _itntext_normalizer if _itntext_normalizer is None: try: from itntext import Normalizer _itntext_normalizer = Normalizer(lang="zh", operator="itn") logger.info("itntext ITN模块初始化成功") except ImportError as e: logger.error(f"导入itntext失败: {e}") raise ImportError("请安装itntext库: pip install itntext") except Exception as e: logger.error(f"初始化itntext失败: {e}") raise return _itntext_normalizer def apply_itn_to_text(text: str) -> str: """ 对文本应用逆文本标准化(ITN) 使用itntext库进行高质量的中文ITN处理 Args: text: 语音识别结果文本 Returns: 应用ITN后的文本 """ if not text or not text.strip(): return text try: normalizer = _get_normalizer() # Protect Chinese decimal quantities with 万/亿: generic ITN rules may # interpret the spoken fractional digits as separate magnitude groups. # Normalize surrounding text normally, and keep the fraction digit-by-digit. matches = list(_CHINESE_DECIMAL_SCALE_PATTERN.finditer(text)) if matches: parts = [] cursor = 0 for match in matches: parts.append(normalizer.normalize(text[cursor:match.start()])) integer = str(normalizer.normalize(match.group("integer"))).strip() if not re.fullmatch(r"\d+", integer): # If this ITN version cannot normalize the integer component, # fall back to its regular whole-text path rather than guess. result = normalizer.normalize(text) break fraction = "".join(_CHINESE_DIGITS[digit] for digit in match.group("fraction")) parts.append(f"{integer}.{fraction}{match.group('scale')}") cursor = match.end() else: parts.append(normalizer.normalize(text[cursor:])) result = "".join(parts) else: result = normalizer.normalize(text) logger.debug(f"ITN处理: '{text}' -> '{result}'") return result except Exception as e: logger.warning(f"ITN处理失败: {text}, 错误: {str(e)}") return text def normalize_asr_text(text: str, enable_itn: bool) -> str: if not enable_itn: return text return apply_itn_to_text(text)