107 lines
4.0 KiB
Python
107 lines
4.0 KiB
Python
"""A WeKnora parser plugin, written with the Python SDK: Jupyter notebooks
|
|
(.ipynb) become Markdown. Markdown cells are kept as they are, code cells
|
|
become fenced blocks, text outputs follow their cell and PNG/JPEG charts are
|
|
handed to WeKnora as images."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import json
|
|
|
|
from weknora_plugin import ErrorCode, ParsedImage, ParseOutput, Plugin, PluginError
|
|
|
|
plugin = Plugin("weknora-examples.notebooks", "1.0.0")
|
|
|
|
IMAGE_TYPES = {"image/png": "png", "image/jpeg": "jpg"}
|
|
|
|
|
|
def _text(value) -> str:
|
|
"""Notebook strings are either a string or a list of lines."""
|
|
if isinstance(value, list):
|
|
return "".join(value)
|
|
return value or ""
|
|
|
|
|
|
def _fence(body: str, lang: str = "") -> str:
|
|
# A fence longer than any backtick run inside, so code cannot close it.
|
|
longest, run = 0, 0
|
|
for ch in body:
|
|
run = run + 1 if ch == "`" else 0
|
|
longest = max(longest, run)
|
|
ticks = "`" * max(3, longest + 1)
|
|
return f"{ticks}{lang}\n{body.rstrip()}\n{ticks}"
|
|
|
|
|
|
def convert(nb: dict, include_outputs: bool = True, max_output_chars: int = 2000) -> ParseOutput:
|
|
lang = ((nb.get("metadata") or {}).get("kernelspec") or {}).get("language") or (
|
|
(nb.get("metadata") or {}).get("language_info") or {}
|
|
).get("name", "")
|
|
parts, images = [], []
|
|
for i, cell in enumerate(nb.get("cells") or []):
|
|
kind, source = cell.get("cell_type"), _text(cell.get("source")).strip()
|
|
if kind == "markdown" and source:
|
|
parts.append(source)
|
|
elif kind == "code":
|
|
if source:
|
|
parts.append(_fence(source, lang))
|
|
if include_outputs:
|
|
parts.extend(_outputs(i, cell.get("outputs") or [], images, max_output_chars))
|
|
elif kind == "raw" and source:
|
|
parts.append(_fence(source))
|
|
title = next((p.splitlines()[0][2:].strip() for p in parts if p.startswith("# ")), "")
|
|
meta = {"cells": str(len(nb.get("cells") or [])), "language": lang}
|
|
if title:
|
|
meta["title"] = title
|
|
return ParseOutput(markdown="\n\n".join(parts) + "\n", images=images, metadata=meta)
|
|
|
|
|
|
def _outputs(cell: int, outputs: list, images: list, limit: int) -> list:
|
|
parts = []
|
|
for j, out in enumerate(outputs):
|
|
kind = out.get("output_type")
|
|
if kind == "stream":
|
|
text = _text(out.get("text"))
|
|
elif kind == "error":
|
|
text = f"{out.get('ename', 'Error')}: {out.get('evalue', '')}"
|
|
else:
|
|
data = out.get("data") or {}
|
|
image = next((m for m in IMAGE_TYPES if m in data), None)
|
|
if image:
|
|
ref = f"images/cell{cell}-{j}.{IMAGE_TYPES[image]}"
|
|
try:
|
|
images.append(ParsedImage(original_ref=ref, data=base64.b64decode(_text(data[image])), mime_type=image))
|
|
parts.append(f"")
|
|
except ValueError:
|
|
pass
|
|
continue
|
|
if "text/markdown" in data:
|
|
parts.append(_text(data["text/markdown"]).strip())
|
|
continue
|
|
text = _text(data.get("text/plain"))
|
|
text = text.strip()
|
|
if text:
|
|
if limit and len(text) > limit:
|
|
text = text[:limit] + "\n…"
|
|
parts.append(_fence(text, "text"))
|
|
return parts
|
|
|
|
|
|
@plugin.parser("ipynb")
|
|
def parse(call, doc):
|
|
try:
|
|
nb = json.loads(doc.content.decode("utf-8-sig"))
|
|
except (UnicodeDecodeError, ValueError) as e:
|
|
raise PluginError(ErrorCode.BAD_REQUEST, f"{doc.file_name or 'file'} is not a Jupyter notebook: {e}") from None
|
|
if not isinstance(nb, dict) or "cells" not in nb:
|
|
raise PluginError(ErrorCode.BAD_REQUEST, f"{doc.file_name or 'file'} is not a Jupyter notebook (no cells)")
|
|
cfg = call.tenant
|
|
return convert(
|
|
nb,
|
|
include_outputs=cfg.get("include_outputs", True) is not False,
|
|
max_output_chars=int(cfg.get("max_output_chars", 2000) or 0),
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
plugin.serve()
|