cosmo/backend/app/services/rocket_event_service.py

331 lines
13 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""全球火箭发射事件同步。
数据源:The Space Devs 的 Launch Library 2(https://ll.thespacedevs.com)。
`/launch/upcoming/` 按发射时间升序返回正在排期与刚刚飞完的发射,含最终结果状态,
因此一次同步既能拿到未来的排期,也能把已飞完任务的状态改写为成功/失败。
(同源的 /launch/past/ 目前会返回 500,历史回填不可用。)
匿名调用有较严的速率限制,未配置 LAUNCH_LIBRARY_TOKEN 时按每 6 小时一次的
默认频率同步;申请令牌后填进 .env 即可放宽。
"""
import asyncio
import logging
from collections import Counter
from datetime import datetime, timedelta, timezone
from typing import Any
import httpx
from sqlalchemy import select
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.sql import func
from app.config import settings
from app.models.db import RocketConfig, RocketEvent
from app.models.db.scheduled_job import JobType, ScheduledJob
logger = logging.getLogger(__name__)
SOURCE = "thespacedevs"
USER_AGENT = "cosmo-backend/1.0 (+https://cosmo.unisspace.com)"
DEFAULT_CRON = "20 */6 * * *" # 每 6 小时一次
DEFAULT_JOB_NAME = "同步全球火箭发射事件"
# 抓取的分页是数据源的传输细节,不暴露给定时任务:只按「提前天数」自动翻页,
# 这两个常量是内部实现(PAGE_SIZE 取数据源上限,MAX_PAGES 兜底防止停不下来)。
PAGE_SIZE = 100
MAX_PAGES = 10
# 数据源型号名(小写、空格分隔)→ 本地 rocket_configs.code。
# 源里同一型号有大量变体(Falcon 9 Block 5 / Long March 5B / Soyuz 2.1b ...),
# 先试完整型号名,再退化到主干名,最后靠 code/name 直接匹配。
LAUNCHER_ALIASES: dict[str, str] = {
"falcon 9": "falcon-9",
"falcon 9 block 5": "falcon-9",
"falcon heavy": "falcon-heavy",
"starship": "starship",
"electron": "electron",
"electron hk": "electron",
"long march 2f": "long-march-2f",
"long march 7": "long-march-7",
"long march 7a": "long-march-7",
"long march 5": "long-march-5",
"long march 5b": "long-march-5",
"long march 6a": "long-march-6a",
"long march 8": "long-march-8",
"long march 8a": "long-march-8",
"long march 8b": "long-march-8",
"ariane 62": "ariane-6",
"ariane 64": "ariane-6",
"soyuz 2": "soyuz-2",
"soyuz 2.1a": "soyuz-2",
"soyuz 2.1b": "soyuz-2",
}
class LaunchSourceError(Exception):
"""数据源不可用。"""
class LaunchSourceThrottled(LaunchSourceError):
"""数据源限流(HTTP 429)。"""
def __init__(self, retry_after_seconds: int | None = None):
super().__init__(f"数据源限流,{retry_after_seconds} 秒后可重试"
if retry_after_seconds else "数据源限流")
self.retry_after_seconds = retry_after_seconds
def _normalize(text: str | None) -> str:
return " ".join((text or "").lower().replace("-", " ").replace("_", " ").split())
def _parse_net(value: str | None) -> datetime | None:
"""Launch Library 的时间是 `2026-10-08T07:54:00Z`,统一存成无时区 UTC。"""
if not value:
return None
try:
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError:
return None
if parsed.tzinfo is not None:
parsed = parsed.astimezone(timezone.utc).replace(tzinfo=None)
return parsed
def _clip(value: str | None, size: int) -> str | None:
return value[:size] if value else None
def _to_float(value: Any) -> float | None:
try:
return float(value)
except (TypeError, ValueError):
return None
async def build_rocket_lookup(db: AsyncSession) -> dict[str, int]:
"""把本地型号按 code / 英文名 / 中文名登记成查找表,再叠加内置别名。"""
rows = (await db.execute(
select(RocketConfig.id, RocketConfig.code, RocketConfig.name, RocketConfig.name_zh)
)).all()
lookup: dict[str, int] = {}
by_code: dict[str, int] = {}
for rocket_id, code, name, name_zh in rows:
for key in (code, name, name_zh):
normalized = _normalize(key)
if normalized:
lookup.setdefault(normalized, rocket_id)
by_code[_normalize(code)] = rocket_id
for alias, code in LAUNCHER_ALIASES.items():
rocket_id = by_code.get(_normalize(code))
if rocket_id is not None:
lookup.setdefault(alias, rocket_id)
return lookup
def match_rocket(lookup: dict[str, int], configuration: dict) -> int | None:
"""按 full_name → name+variant → name → family 依次尝试匹配本地型号。"""
name = (configuration.get("name") or "").strip()
variant = (configuration.get("variant") or "").strip()
candidates = [
configuration.get("full_name"),
f"{name} {variant}" if name and variant else None,
name,
configuration.get("family"),
]
for candidate in candidates:
rocket_id = lookup.get(_normalize(candidate))
if rocket_id is not None:
return rocket_id
return None
def parse_launch(item: dict, lookup: dict[str, int]) -> dict | None:
"""把源侧一条发射记录转成 rocket_events 的字段。"""
source_id = str(item.get("id") or "").strip()
net = _parse_net(item.get("net"))
if not source_id or net is None:
return None
configuration = (item.get("rocket") or {}).get("configuration") or {}
mission = item.get("mission") or {}
pad = item.get("pad") or {}
location = pad.get("location") or {}
mission_name = mission.get("name") if isinstance(mission.get("name"), str) else None
return {
"source": SOURCE,
"source_id": _clip(source_id, 64),
"rocket_config_id": match_rocket(lookup, configuration),
"name": _clip(item.get("name") or mission_name or source_id, 200),
"mission_name": _clip(mission_name, 200),
"mission_description": _clip(mission.get("description"), 4000),
"mission_type": _clip(mission.get("type"), 60),
"rocket_name": _clip(configuration.get("full_name") or configuration.get("name"), 120),
"provider": _clip((item.get("launch_service_provider") or {}).get("name"), 120),
"orbit": _clip((mission.get("orbit") or {}).get("name") if isinstance(mission.get("orbit"), dict) else None, 60),
"launch_site_name": _clip(pad.get("name"), 120),
"location_name": _clip(location.get("name"), 120),
"latitude": _to_float(pad.get("latitude")),
"longitude": _to_float(pad.get("longitude")),
"net": net,
"status": _clip((item.get("status") or {}).get("name"), 40) or "",
"image_url": _clip(item.get("image"), 500),
"source_url": _clip(item.get("url"), 300),
}
async def _get_page(client: httpx.AsyncClient, base: str, params: dict) -> dict:
response = await client.get(f"{base}/launch/upcoming/", params=params)
if response.status_code == 429:
retry_after = None
try:
retry_after = int(response.headers.get("Retry-After"))
except (TypeError, ValueError):
pass
raise LaunchSourceThrottled(retry_after)
try:
response.raise_for_status()
except httpx.HTTPStatusError as exc:
raise LaunchSourceError(f"数据源返回 {exc.response.status_code}") from exc
return response.json()
async def fetch_launches(
until: datetime,
*,
page_size: int = PAGE_SIZE,
max_pages: int = MAX_PAGES,
) -> list[dict]:
"""拉取排期(含刚飞完的),按发射时间升序,翻到覆盖 until 为止。"""
headers = {"User-Agent": USER_AGENT, "accept": "application/json"}
if settings.launch_library_token:
headers["Authorization"] = f"Token {settings.launch_library_token}"
client_kwargs: dict[str, Any] = {
"headers": headers,
"timeout": settings.launch_library_timeout,
}
if settings.proxy_dict:
client_kwargs["proxies"] = settings.proxy_dict
items: list[dict] = []
async with httpx.AsyncClient(**client_kwargs) as client:
for page in range(max_pages):
payload = await _get_page(client, settings.launch_library_api_base,
{"limit": page_size, "offset": page * page_size})
results = payload.get("results") or []
items.extend(results)
if len(results) < page_size or not payload.get("next"):
break
# 源按发射时间升序返回:本页最后一条已经晚于时间窗,再翻页只是白翻
last_net = None
for record in reversed(results):
last_net = _parse_net(record.get("net"))
if last_net:
break
if last_net and last_net > until:
break
await asyncio.sleep(1.5) # 别连续压同一个匿名 IP
return items
async def sync_launch_events(
db: AsyncSession,
*,
days_ahead: int = 90,
log: logging.Logger | None = None,
) -> dict[str, Any]:
"""同步发射事件并按型号关联 rocket_configs,返回本次统计。
重复抓取不会产生重复记录:写入走 (source, source_id) 的 ON CONFLICT DO UPDATE,
同一次发射只会更新原有那一行(状态、NET、型号关联随之刷新)。
"""
log = log or logger
horizon = datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(days=days_ahead)
items = await fetch_launches(horizon)
lookup = await build_rocket_lookup(db)
rows = []
unmapped: Counter[str] = Counter()
for item in items:
values = parse_launch(item, lookup)
if values is None or values["net"] > horizon:
continue
if values["rocket_config_id"] is None and values["rocket_name"]:
unmapped[values["rocket_name"]] += 1
rows.append(values)
known = set()
if rows:
batch = [r["source_id"] for r in rows]
for start in range(0, len(batch), 500):
result = await db.execute(
select(RocketEvent.source_id).where(
RocketEvent.source == SOURCE,
RocketEvent.source_id.in_(batch[start:start + 500]),
)
)
known.update(result.scalars().all())
mutable = (
"rocket_config_id", "name", "mission_name", "mission_description", "mission_type",
"rocket_name", "provider", "orbit", "launch_site_name", "location_name",
"latitude", "longitude", "net", "status", "image_url", "source_url",
)
for values in rows:
statement = insert(RocketEvent).values(**values)
statement = statement.on_conflict_do_update(
index_elements=["source", "source_id"],
set_={key: values[key] for key in mutable} | {"updated_at": func.now()},
)
await db.execute(statement)
await db.commit()
unmapped_ranking = [{"rocket": name, "count": count} for name, count in unmapped.most_common(10)]
summary = {
"success": True,
"source": SOURCE,
"fetched": len(items),
"stored": len(rows),
"created": len(rows) - len(known),
"updated": len(known),
"mapped": sum(1 for r in rows if r["rocket_config_id"] is not None),
"horizon": horizon.isoformat(),
"unmapped_rockets": unmapped_ranking,
}
log.info("火箭发射事件同步:拉取 %s 条,写入 %s 条(新增 %s,更新 %s),型号已关联 %s 条",
summary["fetched"], summary["stored"], summary["created"], summary["updated"], summary["mapped"])
return summary
async def ensure_launch_event_job(db: AsyncSession) -> bool:
"""保证默认同步任务存在;已存在则不动管理端改过的频率与参数。"""
job = (await db.execute(
select(ScheduledJob).where(ScheduledJob.predefined_function == "sync_rocket_launch_events")
)).scalar_one_or_none()
if job:
# 抓取页数改成按「提前天数」自动翻页之后,旧的页数参数已作废,清掉免得误导
stale = [key for key in ("max_pages", "page_size") if key in (job.function_params or {})]
if stale:
job.function_params = {k: v for k, v in job.function_params.items() if k not in stale}
logger.info("已清理定时任务「%s」的作废参数:%s", job.name, ", ".join(stale))
return False
db.add(ScheduledJob(
name=DEFAULT_JOB_NAME,
job_type=JobType.PREDEFINED,
predefined_function="sync_rocket_launch_events",
function_params={"days_ahead": 90},
cron_expression=DEFAULT_CRON,
is_active=True,
description="从 The Space Devs Launch Library 2 获取全球火箭发射排期与结果,并关联本地火箭型号",
))
await db.flush()
logger.info("已创建默认定时任务:%s(%s)", DEFAULT_JOB_NAME, DEFAULT_CRON)
return True