331 lines
13 KiB
Python
331 lines
13 KiB
Python
"""全球火箭发射事件同步。
|
||
|
||
数据源:The Space Devs 的 Launch Library 2(https://ll.thespacedevs.com)。
|
||
`/launch/upcoming/` 按发射时间升序返回正在排期与刚刚飞完的发射,含最终结果状态,
|
||
因此一次同步既能拿到未来的排期,也能把已飞完任务的状态改写为成功/失败。
|
||
(同源的 /launch/past/ 目前会返回 500,历史回填不可用。)
|
||
|
||
匿名调用有较严的速率限制,未配置 LAUNCH_LIBRARY_TOKEN 时按每 6 小时一次的
|
||
默认频率同步;申请令牌后填进 .env 即可放宽。
|
||
"""
|
||
import asyncio
|
||
import logging
|
||
from collections import Counter
|
||
from datetime import datetime, timedelta, timezone
|
||
from typing import Any
|
||
|
||
import httpx
|
||
from sqlalchemy import select
|
||
from sqlalchemy.dialects.postgresql import insert
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
from sqlalchemy.sql import func
|
||
|
||
from app.config import settings
|
||
from app.models.db import RocketConfig, RocketEvent
|
||
from app.models.db.scheduled_job import JobType, ScheduledJob
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
SOURCE = "thespacedevs"
|
||
USER_AGENT = "cosmo-backend/1.0 (+https://cosmo.unisspace.com)"
|
||
DEFAULT_CRON = "20 */6 * * *" # 每 6 小时一次
|
||
DEFAULT_JOB_NAME = "同步全球火箭发射事件"
|
||
|
||
# 抓取的分页是数据源的传输细节,不暴露给定时任务:只按「提前天数」自动翻页,
|
||
# 这两个常量是内部实现(PAGE_SIZE 取数据源上限,MAX_PAGES 兜底防止停不下来)。
|
||
PAGE_SIZE = 100
|
||
MAX_PAGES = 10
|
||
|
||
# 数据源型号名(小写、空格分隔)→ 本地 rocket_configs.code。
|
||
# 源里同一型号有大量变体(Falcon 9 Block 5 / Long March 5B / Soyuz 2.1b ...),
|
||
# 先试完整型号名,再退化到主干名,最后靠 code/name 直接匹配。
|
||
LAUNCHER_ALIASES: dict[str, str] = {
|
||
"falcon 9": "falcon-9",
|
||
"falcon 9 block 5": "falcon-9",
|
||
"falcon heavy": "falcon-heavy",
|
||
"starship": "starship",
|
||
"electron": "electron",
|
||
"electron hk": "electron",
|
||
"long march 2f": "long-march-2f",
|
||
"long march 7": "long-march-7",
|
||
"long march 7a": "long-march-7",
|
||
"long march 5": "long-march-5",
|
||
"long march 5b": "long-march-5",
|
||
"long march 6a": "long-march-6a",
|
||
"long march 8": "long-march-8",
|
||
"long march 8a": "long-march-8",
|
||
"long march 8b": "long-march-8",
|
||
"ariane 62": "ariane-6",
|
||
"ariane 64": "ariane-6",
|
||
"soyuz 2": "soyuz-2",
|
||
"soyuz 2.1a": "soyuz-2",
|
||
"soyuz 2.1b": "soyuz-2",
|
||
}
|
||
|
||
|
||
class LaunchSourceError(Exception):
|
||
"""数据源不可用。"""
|
||
|
||
|
||
class LaunchSourceThrottled(LaunchSourceError):
|
||
"""数据源限流(HTTP 429)。"""
|
||
|
||
def __init__(self, retry_after_seconds: int | None = None):
|
||
super().__init__(f"数据源限流,{retry_after_seconds} 秒后可重试"
|
||
if retry_after_seconds else "数据源限流")
|
||
self.retry_after_seconds = retry_after_seconds
|
||
|
||
|
||
def _normalize(text: str | None) -> str:
|
||
return " ".join((text or "").lower().replace("-", " ").replace("_", " ").split())
|
||
|
||
|
||
def _parse_net(value: str | None) -> datetime | None:
|
||
"""Launch Library 的时间是 `2026-10-08T07:54:00Z`,统一存成无时区 UTC。"""
|
||
if not value:
|
||
return None
|
||
try:
|
||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||
except ValueError:
|
||
return None
|
||
if parsed.tzinfo is not None:
|
||
parsed = parsed.astimezone(timezone.utc).replace(tzinfo=None)
|
||
return parsed
|
||
|
||
|
||
def _clip(value: str | None, size: int) -> str | None:
|
||
return value[:size] if value else None
|
||
|
||
|
||
def _to_float(value: Any) -> float | None:
|
||
try:
|
||
return float(value)
|
||
except (TypeError, ValueError):
|
||
return None
|
||
|
||
|
||
async def build_rocket_lookup(db: AsyncSession) -> dict[str, int]:
|
||
"""把本地型号按 code / 英文名 / 中文名登记成查找表,再叠加内置别名。"""
|
||
rows = (await db.execute(
|
||
select(RocketConfig.id, RocketConfig.code, RocketConfig.name, RocketConfig.name_zh)
|
||
)).all()
|
||
|
||
lookup: dict[str, int] = {}
|
||
by_code: dict[str, int] = {}
|
||
for rocket_id, code, name, name_zh in rows:
|
||
for key in (code, name, name_zh):
|
||
normalized = _normalize(key)
|
||
if normalized:
|
||
lookup.setdefault(normalized, rocket_id)
|
||
by_code[_normalize(code)] = rocket_id
|
||
|
||
for alias, code in LAUNCHER_ALIASES.items():
|
||
rocket_id = by_code.get(_normalize(code))
|
||
if rocket_id is not None:
|
||
lookup.setdefault(alias, rocket_id)
|
||
return lookup
|
||
|
||
|
||
def match_rocket(lookup: dict[str, int], configuration: dict) -> int | None:
|
||
"""按 full_name → name+variant → name → family 依次尝试匹配本地型号。"""
|
||
name = (configuration.get("name") or "").strip()
|
||
variant = (configuration.get("variant") or "").strip()
|
||
candidates = [
|
||
configuration.get("full_name"),
|
||
f"{name} {variant}" if name and variant else None,
|
||
name,
|
||
configuration.get("family"),
|
||
]
|
||
for candidate in candidates:
|
||
rocket_id = lookup.get(_normalize(candidate))
|
||
if rocket_id is not None:
|
||
return rocket_id
|
||
return None
|
||
|
||
|
||
def parse_launch(item: dict, lookup: dict[str, int]) -> dict | None:
|
||
"""把源侧一条发射记录转成 rocket_events 的字段。"""
|
||
source_id = str(item.get("id") or "").strip()
|
||
net = _parse_net(item.get("net"))
|
||
if not source_id or net is None:
|
||
return None
|
||
|
||
configuration = (item.get("rocket") or {}).get("configuration") or {}
|
||
mission = item.get("mission") or {}
|
||
pad = item.get("pad") or {}
|
||
location = pad.get("location") or {}
|
||
mission_name = mission.get("name") if isinstance(mission.get("name"), str) else None
|
||
|
||
return {
|
||
"source": SOURCE,
|
||
"source_id": _clip(source_id, 64),
|
||
"rocket_config_id": match_rocket(lookup, configuration),
|
||
"name": _clip(item.get("name") or mission_name or source_id, 200),
|
||
"mission_name": _clip(mission_name, 200),
|
||
"mission_description": _clip(mission.get("description"), 4000),
|
||
"mission_type": _clip(mission.get("type"), 60),
|
||
"rocket_name": _clip(configuration.get("full_name") or configuration.get("name"), 120),
|
||
"provider": _clip((item.get("launch_service_provider") or {}).get("name"), 120),
|
||
"orbit": _clip((mission.get("orbit") or {}).get("name") if isinstance(mission.get("orbit"), dict) else None, 60),
|
||
"launch_site_name": _clip(pad.get("name"), 120),
|
||
"location_name": _clip(location.get("name"), 120),
|
||
"latitude": _to_float(pad.get("latitude")),
|
||
"longitude": _to_float(pad.get("longitude")),
|
||
"net": net,
|
||
"status": _clip((item.get("status") or {}).get("name"), 40) or "",
|
||
"image_url": _clip(item.get("image"), 500),
|
||
"source_url": _clip(item.get("url"), 300),
|
||
}
|
||
|
||
|
||
async def _get_page(client: httpx.AsyncClient, base: str, params: dict) -> dict:
|
||
response = await client.get(f"{base}/launch/upcoming/", params=params)
|
||
if response.status_code == 429:
|
||
retry_after = None
|
||
try:
|
||
retry_after = int(response.headers.get("Retry-After"))
|
||
except (TypeError, ValueError):
|
||
pass
|
||
raise LaunchSourceThrottled(retry_after)
|
||
try:
|
||
response.raise_for_status()
|
||
except httpx.HTTPStatusError as exc:
|
||
raise LaunchSourceError(f"数据源返回 {exc.response.status_code}") from exc
|
||
return response.json()
|
||
|
||
|
||
async def fetch_launches(
|
||
until: datetime,
|
||
*,
|
||
page_size: int = PAGE_SIZE,
|
||
max_pages: int = MAX_PAGES,
|
||
) -> list[dict]:
|
||
"""拉取排期(含刚飞完的),按发射时间升序,翻到覆盖 until 为止。"""
|
||
headers = {"User-Agent": USER_AGENT, "accept": "application/json"}
|
||
if settings.launch_library_token:
|
||
headers["Authorization"] = f"Token {settings.launch_library_token}"
|
||
|
||
client_kwargs: dict[str, Any] = {
|
||
"headers": headers,
|
||
"timeout": settings.launch_library_timeout,
|
||
}
|
||
if settings.proxy_dict:
|
||
client_kwargs["proxies"] = settings.proxy_dict
|
||
|
||
items: list[dict] = []
|
||
async with httpx.AsyncClient(**client_kwargs) as client:
|
||
for page in range(max_pages):
|
||
payload = await _get_page(client, settings.launch_library_api_base,
|
||
{"limit": page_size, "offset": page * page_size})
|
||
results = payload.get("results") or []
|
||
items.extend(results)
|
||
if len(results) < page_size or not payload.get("next"):
|
||
break
|
||
# 源按发射时间升序返回:本页最后一条已经晚于时间窗,再翻页只是白翻
|
||
last_net = None
|
||
for record in reversed(results):
|
||
last_net = _parse_net(record.get("net"))
|
||
if last_net:
|
||
break
|
||
if last_net and last_net > until:
|
||
break
|
||
await asyncio.sleep(1.5) # 别连续压同一个匿名 IP
|
||
return items
|
||
|
||
|
||
async def sync_launch_events(
|
||
db: AsyncSession,
|
||
*,
|
||
days_ahead: int = 90,
|
||
log: logging.Logger | None = None,
|
||
) -> dict[str, Any]:
|
||
"""同步发射事件并按型号关联 rocket_configs,返回本次统计。
|
||
|
||
重复抓取不会产生重复记录:写入走 (source, source_id) 的 ON CONFLICT DO UPDATE,
|
||
同一次发射只会更新原有那一行(状态、NET、型号关联随之刷新)。
|
||
"""
|
||
log = log or logger
|
||
horizon = datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(days=days_ahead)
|
||
|
||
items = await fetch_launches(horizon)
|
||
lookup = await build_rocket_lookup(db)
|
||
|
||
rows = []
|
||
unmapped: Counter[str] = Counter()
|
||
for item in items:
|
||
values = parse_launch(item, lookup)
|
||
if values is None or values["net"] > horizon:
|
||
continue
|
||
if values["rocket_config_id"] is None and values["rocket_name"]:
|
||
unmapped[values["rocket_name"]] += 1
|
||
rows.append(values)
|
||
|
||
known = set()
|
||
if rows:
|
||
batch = [r["source_id"] for r in rows]
|
||
for start in range(0, len(batch), 500):
|
||
result = await db.execute(
|
||
select(RocketEvent.source_id).where(
|
||
RocketEvent.source == SOURCE,
|
||
RocketEvent.source_id.in_(batch[start:start + 500]),
|
||
)
|
||
)
|
||
known.update(result.scalars().all())
|
||
|
||
mutable = (
|
||
"rocket_config_id", "name", "mission_name", "mission_description", "mission_type",
|
||
"rocket_name", "provider", "orbit", "launch_site_name", "location_name",
|
||
"latitude", "longitude", "net", "status", "image_url", "source_url",
|
||
)
|
||
for values in rows:
|
||
statement = insert(RocketEvent).values(**values)
|
||
statement = statement.on_conflict_do_update(
|
||
index_elements=["source", "source_id"],
|
||
set_={key: values[key] for key in mutable} | {"updated_at": func.now()},
|
||
)
|
||
await db.execute(statement)
|
||
await db.commit()
|
||
|
||
unmapped_ranking = [{"rocket": name, "count": count} for name, count in unmapped.most_common(10)]
|
||
summary = {
|
||
"success": True,
|
||
"source": SOURCE,
|
||
"fetched": len(items),
|
||
"stored": len(rows),
|
||
"created": len(rows) - len(known),
|
||
"updated": len(known),
|
||
"mapped": sum(1 for r in rows if r["rocket_config_id"] is not None),
|
||
"horizon": horizon.isoformat(),
|
||
"unmapped_rockets": unmapped_ranking,
|
||
}
|
||
log.info("火箭发射事件同步:拉取 %s 条,写入 %s 条(新增 %s,更新 %s),型号已关联 %s 条",
|
||
summary["fetched"], summary["stored"], summary["created"], summary["updated"], summary["mapped"])
|
||
return summary
|
||
|
||
|
||
async def ensure_launch_event_job(db: AsyncSession) -> bool:
|
||
"""保证默认同步任务存在;已存在则不动管理端改过的频率与参数。"""
|
||
job = (await db.execute(
|
||
select(ScheduledJob).where(ScheduledJob.predefined_function == "sync_rocket_launch_events")
|
||
)).scalar_one_or_none()
|
||
if job:
|
||
# 抓取页数改成按「提前天数」自动翻页之后,旧的页数参数已作废,清掉免得误导
|
||
stale = [key for key in ("max_pages", "page_size") if key in (job.function_params or {})]
|
||
if stale:
|
||
job.function_params = {k: v for k, v in job.function_params.items() if k not in stale}
|
||
logger.info("已清理定时任务「%s」的作废参数:%s", job.name, ", ".join(stale))
|
||
return False
|
||
|
||
db.add(ScheduledJob(
|
||
name=DEFAULT_JOB_NAME,
|
||
job_type=JobType.PREDEFINED,
|
||
predefined_function="sync_rocket_launch_events",
|
||
function_params={"days_ahead": 90},
|
||
cron_expression=DEFAULT_CRON,
|
||
is_active=True,
|
||
description="从 The Space Devs Launch Library 2 获取全球火箭发射排期与结果,并关联本地火箭型号",
|
||
))
|
||
await db.flush()
|
||
logger.info("已创建默认定时任务:%s(%s)", DEFAULT_JOB_NAME, DEFAULT_CRON)
|
||
return True
|