UnisKB/apps/models_provider/context_policy.py

35 lines
1.4 KiB
Python

"""Deployment defaults; explicit model settings always take precedence."""
DEFAULT_CONTEXT_TOKENS = 65536
DEFAULT_SUMMARY_OUTPUT_TOKENS = 8192
DEFAULT_SAFETY_TOKENS = 4096
def model_context(params):
value = params.get('context_tokens')
if value is None:
value = params.get('num_ctx', DEFAULT_CONTEXT_TOKENS)
if isinstance(value, str) and value.isdigit():
value = int(value)
if type(value) is not int or not 1024 <= value <= 2000000:
raise ValueError('模型上下文须为 1024 至 2000000 的整数')
return value
def summary_defaults(params):
result = dict(params)
context = model_context(result)
result['context_tokens'] = context
result.setdefault('safety_tokens', min(DEFAULT_SAFETY_TOKENS, max(256, context // 16)))
safety = int(result['safety_tokens'])
output = result.get('output_tokens', result.get('max_tokens'))
if output is None:
output = min(DEFAULT_SUMMARY_OUTPUT_TOKENS, max(256, context // 8),
int(result.get('max_output_tokens') or DEFAULT_SUMMARY_OUTPUT_TOKENS))
result['output_tokens'] = int(output)
result.setdefault('max_output_tokens', max(int(output), min(DEFAULT_SUMMARY_OUTPUT_TOKENS,
context - safety - 1024)))
result.setdefault('max_input_tokens', context)
result.setdefault('input_tokens', min(int(result['max_input_tokens']), context - int(output) - safety))
return result