WeKnora/docreader/parser/caption.py

361 lines
12 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

import json
import logging
import os
import time
from dataclasses import dataclass, field
from typing import List, Optional, Union
import requests
import ollama
logger = logging.getLogger(__name__)
@dataclass
class ImageUrl:
"""Image URL data structure for caption requests."""
url: Optional[str] = None
detail: Optional[str] = None
@dataclass
class Content:
"""Content data structure that can contain text or image URL."""
type: Optional[str] = None
text: Optional[str] = None
image_url: Optional[ImageUrl] = None
@dataclass
class SystemMessage:
"""System message for VLM model requests."""
role: Optional[str] = None
content: Optional[str] = None
@dataclass
class UserMessage:
"""User message for VLM model requests, can contain multiple content items."""
role: Optional[str] = None
content: List[Content] = field(default_factory=list)
@dataclass
class CompletionRequest:
"""Request structure for VLM model completion API."""
model: str
temperature: float
top_p: float
messages: List[Union[SystemMessage, UserMessage]]
user: str
@dataclass
class Model:
"""Model identifier structure."""
id: str
@dataclass
class ModelsResp:
"""Response structure for available models API."""
data: List[Model] = field(default_factory=list)
@dataclass
class Message:
"""Message structure in API response."""
role: Optional[str] = None
content: Optional[str] = None
tool_calls: Optional[str] = None
@dataclass
class Choice:
"""Choice structure in API response."""
message: Optional[Message] = None
@dataclass
class Usage:
"""Token usage information in API response."""
prompt_tokens: Optional[int] = 0
total_tokens: Optional[int] = 0
completion_tokens: Optional[int] = 0
@dataclass
class CaptionChatResp:
"""Response structure for caption chat API."""
id: Optional[str] = None
created: Optional[int] = None
model: Optional[Model] = None
object: Optional[str] = None
choices: List[Choice] = field(default_factory=list)
usage: Optional[Usage] = None
@staticmethod
def from_json(json_data: dict) -> "CaptionChatResp":
"""
Parse API response JSON into a CaptionChatResp object.
Args:
json_data: The JSON response from the API
Returns:
A parsed CaptionChatResp object
"""
logger.info("Parsing CaptionChatResp from JSON")
# Manually parse nested fields with safe field extraction
choices = []
for choice in json_data.get("choices", []):
message_data = choice.get("message", {})
message = Message(
role=message_data.get("role"),
content=message_data.get("content"),
tool_calls=message_data.get("tool_calls"),
)
choices.append(Choice(message=message))
# Handle usage with safe field extraction
usage_data = json_data.get("usage", {})
usage = None
if usage_data:
usage = Usage(
prompt_tokens=usage_data.get("prompt_tokens", 0),
total_tokens=usage_data.get("total_tokens", 0),
completion_tokens=usage_data.get("completion_tokens", 0),
)
logger.info(
f"Parsed {len(choices)} choices and usage data: {usage is not None}"
)
return CaptionChatResp(
id=json_data.get("id"),
created=json_data.get("created"),
model=json_data.get("model"),
object=json_data.get("object"),
choices=choices,
usage=usage,
)
def choice_data(self) -> str:
"""
Extract the content from the first choice in the response.
Returns:
The content string from the first choice, or empty string if no choices
"""
if self.choices:
logger.info("Retrieving content from first choice")
return self.choices[0].message.content
logger.warning("No choices available in response")
return ""
class Caption:
"""
Service for generating captions for images using a Vision Language Model.
Uses an external API to process images and return textual descriptions.
"""
def __init__(self, vlm_config=None):
"""Initialize the Caption service with configuration from parameters or environment variables."""
logger.info("Initializing Caption service")
self.prompt = """简单凝炼的描述图片的主要内容"""
# Use provided VLM config if available, otherwise fall back to environment variables
if vlm_config and vlm_config.get("base_url") and vlm_config.get("model_name"):
self.completion_url = vlm_config.get("base_url", "") + "/chat/completions"
self.model = vlm_config.get("model_name", "")
self.api_key = vlm_config.get("api_key", "")
self.interface_type = vlm_config.get("interface_type", "openai").lower()
else:
if os.getenv("VLM_MODEL_BASE_URL") == "" or os.getenv("VLM_MODEL_NAME") == "":
logger.error("VLM_MODEL_BASE_URL or VLM_MODEL_NAME is not set")
return
self.completion_url = os.getenv("VLM_MODEL_BASE_URL") + "/chat/completions"
self.model = os.getenv("VLM_MODEL_NAME")
self.api_key = os.getenv("VLM_MODEL_API_KEY")
self.interface_type = os.getenv("VLM_INTERFACE_TYPE", "openai").lower()
# 验证接口类型
if self.interface_type not in ["ollama", "openai"]:
logger.warning(f"Unknown interface type: {self.interface_type}, defaulting to openai")
self.interface_type = "openai"
logger.info(
f"Service configured with model: {self.model}, endpoint: {self.completion_url}, interface: {self.interface_type}"
)
def _call_caption_api(self, image_data: str) -> Optional[CaptionChatResp]:
"""
Call the Caption API to generate a description for the given image.
Args:
image_data: URL of the image or base64 encoded image data
Returns:
CaptionChatResp object if successful, None otherwise
"""
logger.info(f"Calling Caption API for image captioning")
logger.info(f"Processing image data: {image_data[:50] if len(image_data) > 50 else image_data}")
# 根据接口类型选择调用方式
if self.interface_type == "ollama":
return self._call_ollama_api(image_data)
else:
return self._call_openai_api(image_data)
def _call_ollama_api(self, image_base64: str) -> Optional[CaptionChatResp]:
"""Call Ollama API for image captioning using base64 encoded image data."""
host = self.completion_url.replace("/v1/chat/completions", "")
client = ollama.Client(
host=host,
)
try:
logger.info(f"Calling Ollama API with model: {self.model}")
# 调用Ollama API,使用images参数传递base64编码的图片
response = client.generate(
model=self.model,
prompt="简单凝炼的描述图片的主要内容",
images=[image_base64], # image_base64是base64编码的图片数据
options={"temperature": 0.1},
stream=False,
)
# 构造响应对象
caption_resp = CaptionChatResp(
id="ollama_response",
created=int(time.time()),
model=self.model,
object="chat.completion",
choices=[
Choice(
message=Message(
role="assistant",
content=response.response
)
)
]
)
logger.info("Successfully received response from Ollama API")
return caption_resp
except Exception as e:
logger.error(f"Error calling Ollama API: {e}")
return None
def _call_openai_api(self, image_base64: str) -> Optional[CaptionChatResp]:
"""Call OpenAI-compatible API for image captioning."""
logger.info(f"Calling OpenAI-compatible API with model: {self.model}")
user_msg = UserMessage(
role="user",
content=[
Content(type="text", text=self.prompt),
Content(
type="image_url", image_url=ImageUrl(url="data:image/png;base64," + image_base64, detail="auto")
),
],
)
gpt_req = CompletionRequest(
model=self.model,
temperature=0.3,
top_p=0.8,
messages=[user_msg],
user="abc",
)
headers = {
"Content-Type": "application/json",
"Accept": "text/event-stream",
"Cache-Control": "no-cache",
"Connection": "keep-alive",
}
if self.api_key:
headers["Authorization"] = f"Bearer {self.api_key}"
try:
logger.info(f"Sending request to OpenAI-compatible API with model: {self.model}")
response = requests.post(
self.completion_url,
data=json.dumps(gpt_req, default=lambda o: o.__dict__, indent=4),
headers=headers,
timeout=30,
)
if response.status_code != 200:
logger.error(
f"OpenAI-compatible API returned non-200 status code: {response.status_code}"
)
response.raise_for_status()
logger.info(
f"Successfully received response from OpenAI-compatible API with status: {response.status_code}"
)
logger.info(f"Converting response to CaptionChatResp object")
caption_resp = CaptionChatResp.from_json(response.json())
if caption_resp.usage:
logger.info(
f"API usage: prompt_tokens={caption_resp.usage.prompt_tokens}, "
f"completion_tokens={caption_resp.usage.completion_tokens}"
)
return caption_resp
except requests.exceptions.Timeout:
logger.error(f"Timeout while calling OpenAI-compatible API after 30 seconds")
return None
except requests.exceptions.RequestException as e:
logger.error(f"Request error calling OpenAI-compatible API: {e}")
return None
except Exception as e:
logger.error(f"Unexpected error calling OpenAI-compatible API: {e}")
return None
def get_caption(self, image_data: str) -> str:
"""
Get a caption for the provided image data.
Args:
image_data: URL of the image or base64 encoded image data
Returns:
Caption text as string, or empty string if captioning failed
"""
logger.info("Getting caption for image")
if not image_data or self.completion_url is None:
logger.error("Image data is not set")
return ""
caption_resp = self._call_caption_api(image_data)
if caption_resp:
caption = caption_resp.choice_data()
caption_length = len(caption)
logger.info(f"Successfully generated caption of length {caption_length}")
logger.info(
f"Caption: {caption[:50]}..."
if caption_length > 50
else f"Caption: {caption}"
)
return caption
logger.warning("Failed to get caption from Caption API")
return ""