WeKnora/internal/application/service/tenant_skill_verify_python_...

449 lines
18 KiB
Go

package service
import (
"bytes"
"os"
"os/exec"
"path/filepath"
"sort"
"strings"
"testing"
"github.com/stretchr/testify/require"
)
// The Go tests above pin which commands an install issues. These run the
// embedded checker itself, because its judgement is what decides whether a
// working skill installs — and a check that is too strict fails good skills
// just as surely as one that is too loose passes broken ones. Every case here
// is a shape a real skill ships.
func TestSkillPythonVerifier(t *testing.T) {
// A false environment marker is silent when packaging can evaluate it, and
// a note when it cannot. The install must succeed in both environments;
// only the note is conditional.
unevaluableMarkerNote := ""
if !pythonCanEvaluateMarkers(t) {
unevaluableMarkerNote = "requirements.txt declares pywin32 but it is not installed"
}
cases := []struct {
name string
files map[string]string
// optional names the files whose findings must be reported instead of
// refusing the install. The install path fills this from the naming
// conventions the ecosystems share; here it is explicit.
optional []string
// wantProblem is a substring of the expected stderr. Empty means the
// tree must verify cleanly.
wantProblem string
// wantExit is the code a failing tree must exit with: 2 when installing
// a package would satisfy everything found, 1 when it would not. The
// install flow reads it to decide whether another installer round is
// worth its minutes.
wantExit int
// wantNote is a substring of a stdout note - something the checker
// reported without refusing the install.
wantNote string
}{{
name: "a package whose __init__ imports its own submodules",
files: map[string]string{
"scripts/__init__.py": "from .chart_generator import ChartGenerator\n",
"scripts/chart_generator.py": "class ChartGenerator:\n pass\n",
},
}, {
// The shape that failed a real install: sys.path is rearranged at run
// time, so the absolute import of the skill's own package resolves
// only once the file executes. Deciding it statically is not possible,
// and guessing wrong rejects a working skill. It is reported, because
// the only directory that can satisfy it ships no script of its own.
name: "a script that puts the skill root on sys.path and imports its own package",
files: map[string]string{
"scripts/__init__.py": "",
"scripts/helper.py": "def go():\n pass\n",
"scripts/ux_regression_check.py": "import sys\n" +
"from pathlib import Path\n" +
"sys.path.insert(0, str(Path(__file__).resolve().parent.parent))\n" +
"from scripts.helper import go\n",
},
wantNote: "imports scripts, which the skill ships but no directory it " +
"executes scripts from can reach",
}, {
name: "a script importing a sibling module directly",
files: map[string]string{
"scripts/run.py": "import helper\n",
"scripts/helper.py": "x = 1\n",
"scripts/README.txt": "not python\n",
},
}, {
// scripts/ is both a package (relative imports in __init__) and a
// directory of files the runtime executes with python file.py.
// Loadability is the union of those prefixes, so a sibling import
// still resolves when the package marker is present.
name: "a packaged scripts directory whose files also import siblings",
files: map[string]string{
"scripts/__init__.py": "",
"scripts/run.py": "import helper\n",
"scripts/helper.py": "x = 1\n",
},
}, {
name: "an optional dependency behind try/except",
files: map[string]string{
"scripts/run.py": "try:\n import matplotlib\nexcept ImportError:\n" +
" matplotlib = None\n",
},
}, {
name: "a dependency imported lazily inside a function",
files: map[string]string{
"scripts/run.py": "def render():\n import matplotlib\n return matplotlib\n",
},
}, {
name: "a dependency the installer never installed",
files: map[string]string{
"scripts/run.py": "import totally_absent_package\n",
},
wantProblem: "scripts/run.py imports totally_absent_package, " +
"which is not available in this image",
wantExit: 2,
}, {
name: "a syntax error",
files: map[string]string{
"scripts/run.py": "def broken(:\n pass\n",
},
wantProblem: "scripts/run.py has a syntax error on line 1",
wantExit: 1,
}, {
// The exit code is the whole verdict, so one unfixable finding among
// fixable ones has to sink the batch: sending the installer back for a
// package it can install would only delay a failure it cannot.
name: "a syntax error alongside a missing dependency",
files: map[string]string{
"scripts/bad.py": "def broken(:\n",
"scripts/run.py": "import totally_absent_package\n",
},
wantProblem: "scripts/bad.py has a syntax error",
wantExit: 1,
}, {
name: "a relative import in a directory that is not a package",
files: map[string]string{
"scripts/run.py": "from .helper import go\n",
"scripts/helper.py": "def go():\n pass\n",
"scripts/notinit.py": "",
},
wantProblem: "has no __init__.py",
wantExit: 1,
}, {
name: "a relative import of a module the skill does not ship",
files: map[string]string{
"pkg/__init__.py": "",
"pkg/sub/__init__.py": "",
"pkg/sub/run.py": "from ..missing import go\n",
},
wantProblem: "pkg/sub/run.py imports '..missing', which does not exist in the skill",
wantExit: 1,
}, {
name: "a requirement the venv does not carry",
files: map[string]string{
"requirements.txt": "# pinned\npandas==3.0.1\n-r other.txt\n",
"scripts/run.py": "x = 1\n",
},
wantProblem: "requirements.txt declares pandas but it is not installed",
wantExit: 2,
}, {
// pip skips a line whose marker is false here, so refusing the install
// over it rejects a skill whose requirements are all present. Markers
// compare versions with version semantics, so they are evaluated by
// `packaging` or not at all. CI's system Python usually has it (the
// marker is false, the line is silent); a bare skill venv does not
// (the line is a note, not a failure). Either way the install proceeds.
name: "a requirement gated by an environment marker",
files: map[string]string{
"requirements.txt": "pywin32; sys_platform == \"win32\"\n" +
"totally_absent_package; extra == \"dev\"\n",
"scripts/run.py": "x = 1\n",
},
wantNote: unevaluableMarkerNote,
}, {
// An extras-gated dependency is never installed unless the extra is
// requested, so it is not even worth a note.
name: "an extras-gated requirement is not reported at all",
files: map[string]string{
"requirements.txt": "totally_absent_package; extra == \"dev\"\n",
"scripts/run.py": "x = 1\n",
},
}, {
// poetry tables carrying optional, markers or a python constraint are
// conditional, and poetry would not have installed them here either.
name: "a poetry dependency marked optional",
files: map[string]string{
"pyproject.toml": "[tool.poetry.dependencies]\npython = \"^3.11\"\n" +
"totally_absent_package = { version = \"^1.0\", optional = true }\n",
"scripts/run.py": "x = 1\n",
},
}, {
name: "a nested file name is not treated as a first-party import",
files: map[string]string{
"vendor/totally_absent_package.py": "x = 1\n",
"scripts/run.py": "import totally_absent_package\n",
},
wantProblem: "scripts/run.py imports totally_absent_package, " +
"which is not available in this image",
wantExit: 2,
}, {
// The official Anthropic office toolkit (xlsx/docx/pptx): a non-package
// directory holds entry scripts plus sibling packages. Library modules
// import those siblings by short name (`from helpers import ...`).
// Treating every .py as isolated __main__ rejects this layout; Python's
// package path root is scripts/office/, which is where helpers lives.
name: "a library package importing a sibling package by short name",
files: map[string]string{
"scripts/recalc.py": "from office.soffice import run_soffice\n",
"scripts/office/soffice.py": "def run_soffice():\n pass\n",
"scripts/office/validate.py": "from helpers import safe_extract\n" +
"from validators import BaseSchemaValidator\n",
"scripts/office/helpers/__init__.py": "def safe_extract():\n pass\n",
"scripts/office/validators/__init__.py": "from .base import BaseSchemaValidator\n",
"scripts/office/validators/base.py": "from helpers import safe_extract\n" +
"class BaseSchemaValidator:\n pass\n",
"scripts/office/validators/docx.py": "from helpers import safe_extract\n",
},
}, {
name: "a library package importing a dependency the installer never installed",
files: map[string]string{
"scripts/office/validate.py": "x = 1\n",
"scripts/office/helpers/__init__.py": "x = 1\n",
"scripts/office/validators/__init__.py": "",
"scripts/office/validators/base.py": "import totally_absent_package\n",
},
wantProblem: "scripts/office/validators/base.py imports totally_absent_package, " +
"which is not available in this image",
wantExit: 2,
}, {
// The same layout without __init__.py anywhere. PEP 420 makes those
// directories namespace packages, so a neighbouring entry script still
// resolves the short-name import - which is why prefixes are every
// ancestor and not only the ones a package marker chains together.
name: "a namespace package importing a sibling by short name",
files: map[string]string{
"scripts/office/validate.py": "from helpers import safe_extract\n",
"scripts/office/helpers/__init__.py": "def safe_extract():\n pass\n",
"scripts/office/validators/base.py": "from helpers import safe_extract\n",
"scripts/pkg/sub/mod.py": "from util import go\n",
"scripts/pkg/util.py": "def go():\n pass\n",
},
}, {
name: "a pyproject.toml dependency the venv does not carry",
files: map[string]string{
"pyproject.toml": "[project]\nname = \"demo\"\ndependencies = [\n" +
" \"totally_absent_package>=1.0\",\n]\n",
"scripts/run.py": "x = 1\n",
},
wantProblem: "pyproject.toml declares totally_absent_package but it is not installed",
wantExit: 2,
}, {
// Skills ship their tests. Nothing the skill offers loads them, so a
// bundled tests/ directory must not decide whether the skill installs -
// and refusing over one throws away the dependency work that succeeded.
name: "a bundled test file importing a package the image does not carry",
files: map[string]string{
"scripts/run.py": "x = 1\n",
"tests/test_run.py": "import pytest\nimport totally_absent_package\n",
"examples/demo.py": "import totally_absent_package\n",
"tests/conftest.py": "def broken(:\n",
"scripts/helper.py": "x = 1\n",
},
optional: []string{
"examples/demo.py", "tests/conftest.py", "tests/test_run.py",
},
wantNote: "auxiliary file; this does not fail the install",
}, {
// Lines that name a distribution only indirectly cannot be checked by
// name, and inventing one from the URL would fail installs whose
// requirements are all present.
name: "requirements that point at a VCS, an archive or a local path",
files: map[string]string{
"requirements.txt": "git+https://example.com/x/y.git#egg=y\n" +
"./vendor/local-wheel.whl\n" +
"https://example.com/pkg-1.0.tar.gz\n" +
"--index-url https://example.com/simple\n",
"scripts/run.py": "x = 1\n",
},
}}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
root := writeSkillTree(t, tc.files)
entry := pythonFiles(tc.files)
if len(tc.optional) > 0 {
entry = withoutPaths(entry, tc.optional)
}
stdout, stderr, err := runSkillPythonVerifier(t, root, entry, tc.optional)
if tc.wantProblem == "" {
require.NoError(t, err, "this skill must install; stderr: %s", stderr)
require.Contains(t, stdout, "verified")
} else {
require.Error(t, err, "this skill is broken and must not reach a snapshot")
require.Contains(t, stderr, tc.wantProblem)
require.Equal(t, tc.wantExit, verifierExitCode(t, err),
"the exit code is what decides whether an installer round can fix this; "+
"stderr: %s", stderr)
}
if tc.wantNote != "" {
require.Contains(t, stdout, "note: ",
"a finding that does not refuse the install must still be reported")
require.Contains(t, stdout, tc.wantNote)
}
})
}
}
func verifierExitCode(t *testing.T, err error) int {
t.Helper()
var exit *exec.ExitError
require.ErrorAs(t, err, &exit, "the checker must fail by exiting, not by crashing")
return exit.ExitCode()
}
func withoutPaths(all, drop []string) []string {
dropped := make(map[string]struct{}, len(drop))
for _, rel := range drop {
dropped[rel] = struct{}{}
}
kept := make([]string, 0, len(all))
for _, rel := range all {
if _, ok := dropped[rel]; !ok {
kept = append(kept, rel)
}
}
return kept
}
// Verification must be able to read a skill that writes files or opens sockets
// the moment it is imported, without doing either.
func TestSkillPythonVerifierNeverExecutesTheSkill(t *testing.T) {
root := writeSkillTree(t, map[string]string{
"scripts/run.py": "import os\n" +
"open(os.path.join(os.path.dirname(__file__), 'SIDE_EFFECT'), 'w').close()\n",
})
_, stderr, err := runSkillPythonVerifier(t, root, []string{"scripts/run.py"}, nil)
require.NoError(t, err, stderr)
_, statErr := os.Stat(filepath.Join(root, "scripts", "SIDE_EFFECT"))
require.True(t, os.IsNotExist(statErr),
"the checker ran the skill's module body instead of reading it")
}
// The skill tree is owned by root and readable by everyone; a file the
// execution user cannot open is an install that would fail on first use.
func TestSkillPythonVerifierReportsAnUnreadableScript(t *testing.T) {
if os.Geteuid() == 0 {
t.Skip("root can read a 000 file, so this states nothing when tests run as root")
}
root := writeSkillTree(t, map[string]string{"scripts/run.py": "x = 1\n"})
require.NoError(t, os.Chmod(filepath.Join(root, "scripts", "run.py"), 0o000))
_, stderr, err := runSkillPythonVerifier(t, root, []string{"scripts/run.py"}, nil)
require.Error(t, err)
require.Contains(t, stderr, "cannot be read by the skill execution user")
require.Equal(t, 1, verifierExitCode(t, err),
"a file the execution user cannot read is not something installing a package fixes")
}
// The layout that started this: the official office toolkit ships entry scripts
// beside sibling packages, and its library modules import those siblings by
// short name. Treating every .py as an isolated __main__ rejected it outright.
// The imports it cannot resolve here are the third-party ones it genuinely
// needs — including two its SKILL.md never mentions — and that is the list the
// installer gets handed back.
func TestSkillPythonVerifierAcceptsTheOfficeToolkitLayout(t *testing.T) {
files := map[string]string{
"SKILL.md": "# xlsx\n",
"scripts/recalc.py": "import json\nimport sys\nfrom pathlib import Path\n" +
"from office.soffice import run_soffice\n",
"scripts/office/soffice.py": "import subprocess\nimport tempfile\n" +
"def run_soffice():\n pass\n",
"scripts/office/validate.py": "import argparse\n" +
"from helpers import safe_extract\n" +
"from validators import DOCXSchemaValidator\n",
"scripts/office/helpers/__init__.py": "import zipfile\n" +
"def safe_extract():\n pass\n",
"scripts/office/validators/__init__.py": "from .docx import DOCXSchemaValidator\n",
"scripts/office/validators/base.py": "import re\n" +
"from helpers import safe_extract\n" +
"class BaseSchemaValidator:\n pass\n",
"scripts/office/validators/docx.py": "from helpers import safe_extract\n" +
"from .base import BaseSchemaValidator\n" +
"class DOCXSchemaValidator(BaseSchemaValidator):\n pass\n",
"scripts/office/helpers/pptx_chart.py": "from __future__ import annotations\n" +
"import re\nfrom . import part_text\n",
}
root := writeSkillTree(t, files)
stdout, stderr, err := runSkillPythonVerifier(t, root, pythonFiles(files), nil)
require.NoError(t, err, "the toolkit's own layout must not be a failed install; stderr: %s", stderr)
require.Contains(t, stdout, "verified")
require.NotContains(t, stderr, "helpers",
"helpers is a sibling package of validators/, reachable from scripts/office/")
}
func writeSkillTree(t *testing.T, files map[string]string) string {
t.Helper()
root := t.TempDir()
for rel, content := range files {
full := filepath.Join(root, filepath.FromSlash(rel))
require.NoError(t, os.MkdirAll(filepath.Dir(full), 0o755))
require.NoError(t, os.WriteFile(full, []byte(content), 0o644))
}
return root
}
func pythonFiles(files map[string]string) []string {
var scripts []string
for rel := range files {
if strings.HasSuffix(rel, ".py") {
scripts = append(scripts, rel)
}
}
sort.Strings(scripts)
return scripts
}
// pythonCanEvaluateMarkers reports whether this interpreter has packaging, the
// library the checker uses for PEP 508 markers. Without it a false marker is a
// note rather than a skip, which is the contract a bare venv relies on.
func pythonCanEvaluateMarkers(t *testing.T) bool {
t.Helper()
python, err := exec.LookPath("python3")
if err != nil {
return false
}
return exec.Command(python, "-c", "from packaging.markers import Marker").Run() == nil
}
// runSkillPythonVerifier feeds the embedded checker to a real interpreter the
// same way the sandbox command does: on stdin, with the tree and the files to
// check as argv, auxiliary files last behind the separator.
func runSkillPythonVerifier(
t *testing.T, root string, scripts, optional []string,
) (string, string, error) {
t.Helper()
python, err := exec.LookPath("python3")
if err != nil {
t.Skip("python3 is not on PATH")
}
argv := append([]string{"-", root}, scripts...)
if len(optional) > 0 {
argv = append(append(argv, skillVerifyOptionalFlag), optional...)
}
cmd := exec.Command(python, argv...)
cmd.Stdin = strings.NewReader(skillPythonVerifier)
var stdout, stderr bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = &stderr
runErr := cmd.Run()
return stdout.String(), stderr.String(), runErr
}