88 lines
2.8 KiB
Python
88 lines
2.8 KiB
Python
import logging
|
||
from typing import Union
|
||
|
||
from openai import OpenAI
|
||
from PIL import Image
|
||
|
||
from docreader.config import CONFIG
|
||
from docreader.ocr.base import OCRBackend
|
||
from docreader.utils import endecode
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
class VLMOCRBackend(OCRBackend):
|
||
"""VLM OCR backend implementation using OpenAI API format"""
|
||
|
||
def __init__(self):
|
||
"""Initialize VLM OCR backend
|
||
|
||
Args:
|
||
api_key: API key for OpenAI API
|
||
base_url: Base URL for OpenAI API
|
||
model: Model name
|
||
"""
|
||
self.model = CONFIG.ocr_model
|
||
self.client = OpenAI(
|
||
api_key=CONFIG.ocr_api_key,
|
||
base_url=CONFIG.ocr_api_base_url,
|
||
timeout=30,
|
||
)
|
||
self.temperature = 0.0
|
||
self.max_tokens = 5000
|
||
|
||
# Prompt for OCR text extraction with specific formatting requirements
|
||
self.prompt = "提取文档图片中正文的所有信息用markdown格式表示,"
|
||
"其中页眉、页脚部分忽略,"
|
||
"表格用html格式表达,"
|
||
"文档中公式用latex格式表示,"
|
||
"按照阅读顺序组织进行解析。"
|
||
|
||
def predict(self, image: Union[str, bytes, Image.Image]) -> str:
|
||
"""Extract text from an image using VLM OCR
|
||
|
||
Args:
|
||
image: Image file path, bytes, or PIL Image object
|
||
|
||
Returns:
|
||
Extracted text
|
||
"""
|
||
if self.client is None:
|
||
logger.error("VLM OCR client not initialized")
|
||
return ""
|
||
|
||
try:
|
||
# Encode image to base64 format for API transmission
|
||
img_base64 = endecode.decode_image(image)
|
||
if not img_base64:
|
||
return ""
|
||
|
||
# Call VLM OCR API using OpenAI-compatible format
|
||
logger.info(f"Calling VLM OCR API with model: {self.model}")
|
||
response = self.client.chat.completions.create(
|
||
model=self.model,
|
||
messages=[
|
||
{
|
||
"role": "user",
|
||
"content": [
|
||
{
|
||
"type": "image_url",
|
||
"image_url": {
|
||
"url": f"data:image/png;base64,{img_base64}"
|
||
},
|
||
},
|
||
{
|
||
"type": "text",
|
||
"text": self.prompt,
|
||
},
|
||
],
|
||
}
|
||
],
|
||
temperature=self.temperature,
|
||
max_tokens=self.max_tokens,
|
||
)
|
||
return response.choices[0].message.content or ""
|
||
except Exception as e:
|
||
logger.error(f"VLM OCR prediction error: {str(e)}")
|
||
return ""
|