fix(skills): verify installs by loadability and repair missing deps

The gate was answering the wrong question (every .py as isolated __main__),
and the installer derived dependencies from SKILL.md prose independently.
Align import prefixes with Python's path rules, keep the gate as the only
authority, and feed its findings back for one repair round.
main
wizardchen 2026-08-31 11:42:47 +08:00 committed by lyingbug
parent 4d78f9dc12
commit faefceec15
9 changed files with 1260 additions and 232 deletions

View File

@ -44,6 +44,17 @@ const (
// files. Writing each file with MakeDir+WriteFile is two round trips per
// entry, which is why a 50-file skill crawled through "seeding 12/56".
skillSeedArchivePath = sandbox.SkillsImageRoot + "/.weknora-seed.tar"
// skillInstallVerifyRounds bounds the installer conversation.
//
// The first round works from SKILL.md. A second is driven by the gate's own
// findings, and exists because those are two different descriptions of what
// a skill needs: the official office toolkit imports defusedxml and lxml at
// module level while its SKILL.md names neither, so an installer working
// from prose alone cannot know to install them. One round of reconciliation
// closes that gap; more would only be a retry loop over an answer that is
// not going to change.
skillInstallVerifyRounds = 2
)
// InstallSkill validates an uploaded archive, records it, and kicks off the
@ -367,25 +378,16 @@ func (s *TenantSkillService) runInstall(
}
s.publishProgress(ctx, tenantID, configID, skillID, SkillProgress{Percent: 35, Stage: "seeded"})
// 3. Let the installer agent install dependencies.
if err := s.driveInstallerAgent(ctx, tenantID, skillID, sess, transcript, prompt); err != nil {
return err
}
s.publishProgress(ctx, tenantID, configID, skillID, SkillProgress{Percent: 80, Stage: "agent_done"})
// 4. Hand the tree to the execution user BEFORE verifying it. The agent
// created these files as root, and the language passes below
// deliberately run as the ordinary user: verifying first would test
// permissions that never reach the image, and a restrictive root umask
// would fail a perfectly good install because the .venv interpreter was
// unreadable.
if err := s.normalizeSkillPermissions(ctx, mgr, sess.ID, skillDir); err != nil {
return err
}
// 5. Verify ourselves. "The agent said it worked" is a sentence, not
// evidence, and this is the last gate before the image is switched.
if err := s.verifySkill(ctx, mgr, sess.ID, skillDir, bundle); err != nil {
// 3-5. Install dependencies and verify the result. "The agent said it
// worked" is a sentence, not evidence, and this is the last gate
// before the image is switched — but the gate and the agent must not
// answer "what does this skill need" separately, so a failure it can
// still fix goes back to the same agent before the run is failed.
if err := s.installDependenciesAndVerify(ctx, installerJob{
tenantID: tenantID, configID: configID, skillID: skillID,
sess: sess, mgr: mgr, transcript: transcript,
prompt: prompt, skillDir: skillDir, bundle: bundle,
}); err != nil {
return err
}
if err := s.writeManifestEntry(ctx, mgr, sess.ID, skillID, bundle); err != nil {
@ -690,19 +692,114 @@ func (s *TenantSkillService) beginInstallTranscript(
return transcript, prompt
}
// driveInstallerAgent runs one installer conversation. It calls the engine
// directly instead of going through sessionService.AgentQA, because AgentQA
// swallows engine failures (it emits an error event and returns nil) and we
// need a reliable signal before switching the image.
func (s *TenantSkillService) driveInstallerAgent(
ctx context.Context, tenantID uint64, skillID string, sess *types.Session,
transcript *installTranscript, prompt string,
) error {
// installerJob is everything the install-and-verify step needs. It is a struct
// because the step takes the union of what the agent side and the image side
// each need, and a nine-parameter call tells the reader nothing.
type installerJob struct {
tenantID uint64
configID string
skillID string
sess *types.Session
mgr sandbox.Manager
transcript *installTranscript
prompt string
skillDir string
bundle *SkillBundle
}
// installDependenciesAndVerify runs the installer conversation and the gate as
// one decision.
//
// They used to be two, and that is what let a working skill be rejected: the
// installer derived what to install from SKILL.md prose while the gate derived
// what must resolve from the imports every file executes. Those disagree on any
// skill whose library modules import something its documentation never names,
// and the install died with the answer already in hand. The gate is the only
// authority here, so its own findings become the next round's instruction, and
// nothing in this path derives that list a second time.
//
// A failure the gate marked unrepairable — a syntax error, a file the execution
// user cannot read — ends the run immediately. Another round cannot change it,
// and the bundle has to change instead.
func (s *TenantSkillService) installDependenciesAndVerify(
ctx context.Context, job installerJob,
) (err error) {
run, err := s.openInstallerRun(ctx, job.tenantID, job.sess, job.transcript)
if err != nil {
job.transcript.Finish(context.WithoutCancel(ctx), err)
return err
}
// Detached from cancellation: when the install lock's renewal fails the
// context dies, and the transcript of the run that just died is precisely
// what someone will want to read.
defer func() { job.transcript.Finish(context.WithoutCancel(ctx), err) }()
prompt := job.prompt
for round := 1; ; round++ {
if err = run.round(ctx, prompt); err != nil {
return err
}
s.publishProgress(ctx, job.tenantID, job.configID, job.skillID,
SkillProgress{Percent: 80, Stage: "agent_done"})
// The tree is handed to the execution user BEFORE it is verified. The
// agent created these files as root, and the language passes
// deliberately run as the ordinary user: verifying first would test
// permissions that never reach the image, and a restrictive root umask
// would fail a perfectly good install because the .venv interpreter
// was unreadable.
if err = s.normalizeSkillPermissions(ctx, job.mgr, job.sess.ID, job.skillDir); err != nil {
return err
}
notes, verifyErr := s.verifySkill(ctx, job.mgr, job.sess.ID, job.skillDir, job.bundle)
s.reportVerificationNotes(ctx, job, notes)
if verifyErr == nil {
return nil
}
var gate *skillVerificationError
if round >= skillInstallVerifyRounds || !errors.As(verifyErr, &gate) || !gate.Repairable {
err = verifyErr
return err
}
logger.Infof(ctx, "[skill] %s failed %s verification with %d fixable finding(s); "+
"handing them back to the installer", job.skillID, gate.Language, len(gate.Problems))
s.publishProgress(ctx, job.tenantID, job.configID, job.skillID, SkillProgress{
Percent: 82, Stage: "repairing",
Log: fmt.Sprintf("%s verification found %d missing dependency/dependencies; "+
"asking the installer to add them", gate.Language, len(gate.Problems)),
})
if err = s.reopenSkillDirForRepair(ctx, job.mgr, job.sess.ID, job.skillDir); err != nil {
return err
}
prompt = buildRepairPrompt(job.skillDir, gate)
job.transcript.RecordPrompt(prompt)
}
}
// installerRun is one installer conversation, held open across rounds. A repair
// has to reach the same root shell, in the same sandbox, and be readable in the
// same transcript as the install it is repairing.
type installerRun struct {
engine interfaces.AgentEngine
transcript *installTranscript
sessionID string
}
// openInstallerRun builds the engine the conversation runs on. It calls the
// engine directly instead of going through sessionService.AgentQA, because
// AgentQA swallows engine failures (it emits an error event and returns nil)
// and we need a reliable signal before switching the image.
func (s *TenantSkillService) openInstallerRun(
ctx context.Context, tenantID uint64, sess *types.Session, transcript *installTranscript,
) (*installerRun, error) {
if s.installerAgents == nil {
return errors.New("custom agent service is not configured")
return nil, errors.New("custom agent service is not configured")
}
if transcript == nil {
return errors.New("install transcript was not seeded")
return nil, errors.New("install transcript was not seeded")
}
// The record is tenant-writable: updateBuiltinAgent lets a tenant persist a
// Config for any built-in ID, this one included. It is therefore read for
@ -710,42 +807,129 @@ func (s *TenantSkillService) driveInstallerAgent(
// platform's own registry entry.
record, err := s.installerAgents.GetAgentByID(ctx, types.BuiltinSkillInstallerID)
if err != nil {
return fmt.Errorf("load installer agent: %w", err)
return nil, fmt.Errorf("load installer agent: %w", err)
}
agentConfig := installerAgentConfig(installerAgentDefaults(ctx, tenantID), sess.SandboxConfigID)
chatModel, err := s.resolveInstallerModel(ctx, tenantID, record)
if err != nil {
return err
return nil, err
}
engine, err := s.agents.CreateAgentEngine(
ctx, agentConfig, chatModel, nil, transcript.bus, sess.ID, transcript.assistantMessageID,
)
if err != nil {
runErr := fmt.Errorf("create installer engine: %w", err)
transcript.Finish(ctx, runErr)
return runErr
return nil, fmt.Errorf("create installer engine: %w", err)
}
return &installerRun{
engine: engine, transcript: transcript, sessionID: sess.ID,
}, nil
}
state, err := engine.Execute(ctx, sess.ID, transcript.assistantMessageID, prompt, nil)
runErr := err
if runErr == nil && (state == nil || !state.IsComplete) {
runErr = errors.New("installer agent stopped without completing")
// round runs one installer turn.
//
// No conversation history is replayed. The engine is stateless across turns, so
// a repair round could be handed the first round's transcript — but what the
// gate reported is both shorter and more exact than anything that could be
// inferred from it, and re-reading the round that already missed a dependency
// is not what makes the next one find it.
func (r *installerRun) round(ctx context.Context, prompt string) error {
state, err := r.engine.Execute(ctx, r.sessionID, r.transcript.assistantMessageID, prompt, nil)
if err != nil {
return fmt.Errorf("installer agent failed: %w", err)
}
// Detached from cancellation: when the install lock's renewal fails the
// context dies, and the transcript of the run that just died is precisely
// what someone will want to read.
transcript.Finish(context.WithoutCancel(ctx), runErr)
if runErr != nil {
if err != nil {
return fmt.Errorf("installer agent failed: %w", err)
}
return runErr
if state == nil || !state.IsComplete {
return errors.New("installer agent stopped without completing")
}
return nil
}
// reportVerificationNotes surfaces what the gate noticed but did not refuse: an
// auxiliary file that cannot load, a requirement whose environment marker
// cannot be evaluated here, an import that resolves only from a directory the
// skill ships no script in. None is evidence of a broken install, and all of
// them are what someone debugging this skill later will wish they had been
// told, so they are logged and streamed rather than discarded.
func (s *TenantSkillService) reportVerificationNotes(
ctx context.Context, job installerJob, notes []string,
) {
if len(notes) == 0 {
return
}
for _, note := range notes {
logger.Infof(ctx, "[skill] %s verification note: %s", job.skillID, note)
}
// One event, not one per note. Progress keeps only the latest value, so
// publishing them separately would leave a late subscriber holding whichever
// note happened to be last.
s.publishProgress(ctx, job.tenantID, job.configID, job.skillID, SkillProgress{
Percent: 80, Stage: "verify_note", Log: strings.Join(notes, "\n"),
})
}
// reopenSkillDirForRepair undoes normalizeSkillPermissions for another round.
//
// Verification runs as the ordinary user, so the tree is locked to 555 and root
// before it; a repair then has to install into .venv again. Install commands
// run as root and would get away with writing through those modes, but relying
// on that makes the next reader work out whether it still holds — restoring the
// state the seed left is the same two commands and needs no such argument.
func (s *TenantSkillService) reopenSkillDirForRepair(
ctx context.Context, mgr sandbox.Manager, sessionID, skillDir string,
) error {
if err := guardSkillDir(skillDir); err != nil {
return err
}
dir := sandbox.ShellQuote(skillDir)
user := sandbox.DefaultSandboxExecUser
cmd := fmt.Sprintf("chown -R %s:%s %s && chmod -R u+rwX,go+rX %s", user, user, dir, dir)
if _, err := s.execInstall(ctx, mgr, sessionID, cmd); err != nil {
return fmt.Errorf("reopen skill directory %s for a repair round: %w", skillDir, err)
}
return nil
}
// buildRepairPrompt turns the gate's own findings into the next round's brief.
//
// It carries no analysis of its own, deliberately. The reason a first round can
// fail on a working skill is that SKILL.md and the imports its files execute are
// two different descriptions of what the skill needs; a third description
// written here would reintroduce exactly that gap. The only thing added is the
// import-name-to-distribution mapping, which is a property of PyPI rather than
// of this skill.
func buildRepairPrompt(skillDir string, gate *skillVerificationError) string {
var findings strings.Builder
for _, problem := range gate.Problems {
findings.WriteString("- ")
findings.WriteString(problem)
findings.WriteString("\n")
}
return fmt.Sprintf(`Verification of the skill you just installed failed. Fix only this and stop.
The %s check reported:
%s
Every line above is a dependency this image cannot resolve. The names were read
from the imports the skill's own files execute when they load, which is why some
of them appear nowhere in SKILL.md — install them anyway.
- Python packages go into %s/.venv (`+"`uv pip install`"+`, or
%s/.venv/bin/python -m pip install). Node packages go under %s/node_modules.
- The name in each line is an IMPORT name; install the distribution that
provides it. Common cases where they differ: PIL -> pillow, yaml -> pyyaml,
docx -> python-docx, pptx -> python-pptx, cv2 -> opencv-python-headless,
bs4 -> beautifulsoup4, sklearn -> scikit-learn, fitz -> pymupdf.
- Do NOT edit, move or delete any of the skill's own files, and do NOT edit
SKILL.md or requirements.txt to make the check pass. The uploaded archive is
what read_skill serves, so a source edit here makes the installed skill differ
from what everyone else sees.
- If a package genuinely cannot be installed in this image, say so plainly in
your summary rather than working around it.
The same verification runs again as soon as you finish.
`, gate.Language, findings.String(), skillDir, skillDir, skillDir)
}
// normalizeSkillPermissions makes the skill tree readable and executable by
// the non-root execution user, and writable by nobody. Installs run as root,
// so without the mode change the user that actually runs skills could not read
@ -1526,6 +1710,13 @@ venv. It never runs the skill's code, so nothing is expected to answer --help.
Lazy imports and install_deps.py extras are invisible to that check — you still
have to install them.
SKILL.md is not the dependency list. What the check resolves is the set of
top-level imports the skill's files actually execute, and a library module
routinely imports something the documentation never mentions. Read every .py in
the tree (grep '^import\|^from' over it) and install what those lines need, not
only what the Dependencies section names. If verification does fail on a missing
package, you get its exact findings back and one round to install them.
SKILL.md:
%s
`, skillDir, uvAvailable, skillDir, skillDir, skillDir,

View File

@ -1043,9 +1043,9 @@ func TestRunInstallRequiresVenvWhenRequirementsExist(t *testing.T) {
require.Nil(t, fx.configRepo.saved)
}
// Verification used to run one guessed "entry script" with --help. Nothing in
// the runtime designates an entry script — the model names whichever path it
// likes in execute_skill_script — so the guess left every other file unchecked
// Verification used to run one guessed "entry script" with --help. A skill is
// not one entry point: it is every file the tree ships, some run as scripts
// and some imported as packages. The guess left every other file unchecked
// while failing installs over the one it happened to pick.
func TestRunInstallVerifiesEveryScriptOfEveryLanguage(t *testing.T) {
fx := newInstallFixture(t)
@ -1058,7 +1058,7 @@ func TestRunInstallVerifiesEveryScriptOfEveryLanguage(t *testing.T) {
pythonPass := skillPythonVerifyCommand(installSkillDir, []string{
"scripts/__init__.py", "scripts/extract.py", "scripts/helper.py",
})
}, nil)
require.Contains(t, fx.commands, pythonPass,
"every python file must be checked, not the first one in sort order")
require.Contains(t, fx.commands,
@ -1083,7 +1083,7 @@ func TestRunInstallDoesNotExecuteAnyScriptToVerifyIt(t *testing.T) {
}
func TestSkillPythonVerifyCommandPrefersTheSkillVenv(t *testing.T) {
command := skillPythonVerifyCommand("/opt/skills/demo", []string{"scripts/a.py"})
command := skillPythonVerifyCommand("/opt/skills/demo", []string{"scripts/a.py"}, nil)
require.Contains(t, command, "if [ -x /opt/skills/demo/.venv/bin/python ]; "+
"then py=/opt/skills/demo/.venv/bin/python; else py=python3; fi",
@ -1095,11 +1095,68 @@ func TestSkillPythonVerifyCommandPrefersTheSkillVenv(t *testing.T) {
// A skill may ship a file whose name needs quoting; the command is assembled
// by hand, so the shell must never see it as more than one word.
func TestSkillVerifyCommandsQuoteAwkwardPaths(t *testing.T) {
python := skillPythonVerifyCommand("/opt/skills/demo", []string{"scripts/a b'c.py"})
python := skillPythonVerifyCommand("/opt/skills/demo", []string{"scripts/a b'c.py"}, nil)
require.Contains(t, python, `'scripts/a b'\''c.py'`)
shell := skillShellVerifyCommand("/opt/skills/demo", []string{"a b.sh"})
require.Equal(t, `for f in '/opt/skills/demo/a b.sh'; do sh -n "$f" || exit 1; done`, shell)
require.Equal(t,
`if command -v bash >/dev/null 2>&1; then parser=bash; else parser=sh; fi; `+
`for f in '/opt/skills/demo/a b.sh'; do "$parser" -n "$f" || exit 1; done`,
shell)
}
// A skill's shell scripts carry a bash shebang almost exclusively, and
// SkillInterpreterCommand runs them with bash. Checking them with sh — dash on
// Debian — refused installs over array literals, `function f()`, C-style for
// loops and process substitution, none of which dash can parse.
func TestSkillShellVerifyCommandParsesWithBash(t *testing.T) {
command := skillShellVerifyCommand("/opt/skills/demo", []string{"bin/setup.sh"})
require.Contains(t, command, "parser=bash")
require.Contains(t, command, `"$parser" -n "$f"`)
require.Contains(t, command, "else parser=sh",
"an image without bash must still get a syntax check")
require.NotContains(t, command, `sh -n "$f"`,
"the check must not hard-code the shell that cannot parse these files")
}
// The Python pass is the only one whose verdict depends on what the image
// carries, so it is the only one that can fail an install over a file nothing
// the skill offers ever loads. Those files are named separately.
func TestSkillPythonVerifyCommandSeparatesAuxiliaryFiles(t *testing.T) {
command := skillPythonVerifyCommand("/opt/skills/demo",
[]string{"scripts/run.py"}, []string{"tests/test_run.py"})
require.True(t, strings.HasSuffix(command,
`- /opt/skills/demo scripts/run.py --optional tests/test_run.py`),
"got %q", command)
}
// The split follows the conventions the language ecosystems already share. A
// bundled tests/ directory imports pytest and an examples/ script imports
// whatever it illustrates; neither is reachable from anything the skill
// exposes, and neither may decide whether the skill installs.
func TestSkillAuxiliaryScriptFollowsEcosystemConventions(t *testing.T) {
for _, rel := range []string{
"tests/test_run.py", "test/helpers.py", "examples/demo.py",
"scripts/__tests__/a.py", "benchmarks/bench.py", "docs/conf.py",
"conftest.py", "setup.py", "scripts/extract_test.py",
"scripts/test_extract.py",
} {
require.True(t, skillAuxiliaryScript(rel), "%s must be auxiliary", rel)
}
for _, rel := range []string{
"scripts/extract.py", "scripts/office/validators/base.py",
"recalc.py", "scripts/latest.py", "scripts/contest.py",
} {
require.False(t, skillAuxiliaryScript(rel), "%s is part of what the skill offers", rel)
}
entry, auxiliary := splitAuxiliaryScripts([]string{
"scripts/a.py", "tests/test_a.py", "scripts/b.py",
})
require.Equal(t, []string{"scripts/a.py", "scripts/b.py"}, entry)
require.Equal(t, []string{"tests/test_a.py"}, auxiliary)
}
func TestSkillNodeVerifyCommandChecksDeclaredDependencies(t *testing.T) {
@ -1307,7 +1364,8 @@ func TestRunInstallKeepsOldImageWhenVerificationFails(t *testing.T) {
fx.configRepo.entity.Config.SkillImage = &types.SkillImageConfig{
SnapshotID: "snap-old", Generation: 3, OwnerFingerprint: fx.fingerprint,
}
fx.loadCheckExitCode = 1
fx.loadCheckExitCodes = []int{1}
fx.loadCheckStderr = "scripts/extract.py has a syntax error on line 1: invalid syntax"
err := fx.svc.runInstall(context.Background(), 7, "cfg-1", "sk-1", fx.bundle)
@ -1322,6 +1380,98 @@ func TestRunInstallKeepsOldImageWhenVerificationFails(t *testing.T) {
require.NotEmpty(t, skill.Error)
}
// The failure this closes: the installer derives what to install from SKILL.md
// prose, and the gate derives what must resolve from the imports every file
// executes. Those disagree on any skill whose library modules import something
// its documentation never names — the official office toolkit imports
// defusedxml and lxml at module level and mentions neither — so the install
// died with the answer already in hand, and a retry replayed the same prompt to
// the same effect. The gate's own lines are the repair round's brief.
func TestRunInstallHandsVerificationFindingsBackToTheInstaller(t *testing.T) {
fx := newInstallFixture(t)
fx.loadCheckExitCodes = []int{skillVerifyRepairableExit, 0}
fx.loadCheckStderr = "scripts/office/validators/base.py imports defusedxml, " +
"which is not available in this image\n" +
"scripts/recalc.py imports openpyxl, which is not available in this image"
require.NoError(t, fx.svc.runInstall(context.Background(), 7, "cfg-1", "sk-1", fx.bundle))
require.Len(t, fx.agentPrompts, 2, "a fixable failure must reach the installer again")
repair := fx.agentPrompts[1]
require.Contains(t, repair, "imports defusedxml",
"the repair round is driven by the gate's own findings, not by a second guess")
require.Contains(t, repair, "imports openpyxl")
require.Contains(t, repair, installSkillDir+"/.venv",
"the round has to be told where the packages belong")
require.Contains(t, repair, "Do NOT edit",
"a repair must not be allowed to edit the tree into passing")
require.Equal(t, 2, fx.loadCheckPasses, "the repair has to be verified, not trusted")
require.Contains(t, fx.events, "create-snapshot")
require.NotNil(t, fx.configRepo.saved, "a repaired install must reach the image pointer")
skill, _ := fx.skillRepo.GetSkill(context.Background(), 7, "cfg-1", "sk-1")
require.Equal(t, types.SkillStatusReady, skill.Status)
require.Empty(t, skill.Error)
}
// A repair round has to be able to write into .venv again, which the
// verification pass just locked down: the checks run as the ordinary user, so
// the tree is handed over at 555 and root-owned before every one of them.
func TestRunInstallReopensTheTreeBeforeARepairRound(t *testing.T) {
fx := newInstallFixture(t)
fx.loadCheckExitCodes = []int{skillVerifyRepairableExit, 0}
require.NoError(t, fx.svc.runInstall(context.Background(), 7, "cfg-1", "sk-1", fx.bundle))
reopen := indexOfCommandContaining(fx.commands, "chmod -R u+rwX,go+rX "+installSkillDir)
require.GreaterOrEqual(t, reopen, 0, "the tree must be writable again for the repair")
firstLock := indexOfCommandContaining(fx.commands, "chmod -R 555 "+installSkillDir)
require.GreaterOrEqual(t, firstLock, 0)
require.Greater(t, reopen, firstLock,
"the tree is reopened after the pass that locked it, not before")
// And locked again for the pass that checks the repair, so the final image
// carries the modes every verification ran against.
require.Greater(t, lastIndexOfCommandContaining(fx.commands, "chmod -R 555 "+installSkillDir),
reopen, "the repaired tree must be locked down again before it is verified")
}
// Installing a package cannot fix a file that does not parse, and the bundle
// has to change instead. Spending another agent round on it would only delay
// the same failure by minutes.
func TestRunInstallDoesNotRetryAFailureInstallingCannotFix(t *testing.T) {
fx := newInstallFixture(t)
fx.loadCheckExitCodes = []int{1}
fx.loadCheckStderr = "scripts/extract.py has a syntax error on line 1: invalid syntax"
err := fx.svc.runInstall(context.Background(), 7, "cfg-1", "sk-1", fx.bundle)
require.Error(t, err)
require.ErrorContains(t, err, "syntax error")
require.Len(t, fx.agentPrompts, 1, "an unfixable finding must not buy another round")
require.Equal(t, 1, fx.loadCheckPasses)
}
// The loop is bounded. An installer that cannot satisfy the gate in one repair
// is not going to satisfy it in ten, and a skill install must not become an
// open-ended retry against a billed sandbox.
func TestRunInstallStopsAfterOneRepairRound(t *testing.T) {
fx := newInstallFixture(t)
fx.loadCheckExitCodes = []int{skillVerifyRepairableExit}
err := fx.svc.runInstall(context.Background(), 7, "cfg-1", "sk-1", fx.bundle)
require.Error(t, err)
require.ErrorContains(t, err, "imports pandas",
"the failure must name what the gate found, not just that it failed")
require.Len(t, fx.agentPrompts, skillInstallVerifyRounds)
require.Equal(t, skillInstallVerifyRounds, fx.loadCheckPasses)
require.NotContains(t, fx.events, "create-snapshot")
require.Nil(t, fx.configRepo.saved)
}
func TestRunInstallDeletesTheSnapshotWhenSwitchFails(t *testing.T) {
fx := newInstallFixture(t)
fx.configRepo.updateErr = errUpdateBoom
@ -1549,7 +1699,7 @@ const (
// reason; the properties worth stating verbatim are asserted separately in
// TestSkillPythonVerifyCommand*.
var installPythonVerifyCommand = skillPythonVerifyCommand(
installSkillDir, []string{"scripts/extract.py"},
installSkillDir, []string{"scripts/extract.py"}, nil,
)
func indexOfEvent(events []string, needle string) int {
@ -1561,6 +1711,27 @@ func indexOfEvent(events []string, needle string) int {
return -1
}
// indexOfCommandContaining and its Last variant locate one shell command in the
// ordered transcript. A repair round issues the same commands twice, so the
// tests that care about ordering need both ends.
func indexOfCommandContaining(commands []string, needle string) int {
for i, command := range commands {
if strings.Contains(command, needle) {
return i
}
}
return -1
}
func lastIndexOfCommandContaining(commands []string, needle string) int {
for i := len(commands) - 1; i >= 0; i-- {
if strings.Contains(commands[i], needle) {
return i
}
}
return -1
}
// staleMark is one request to mark a config's bound sandboxes stale. The
// tenant is part of it because marking the right config of the wrong workspace
// would rebuild sandboxes that never carried this image.
@ -1584,10 +1755,17 @@ type installFixture struct {
events []string
commands []string
fingerprint string
// loadCheck* drive the per-language script verification pass, which is
// the last gate before the snapshot.
loadCheckExitCode int
loadCheckResult *sandbox.ExecuteResult
// loadCheck* drive the per-language script verification pass, which is the
// last gate before the snapshot. exitCodes is consumed one entry per python
// pass and its last entry repeats, so a test about a single round writes one
// value and a test about the repair round writes two. Exit 2 is the
// checker's "everything I found is a missing dependency", which is what
// earns another installer round.
loadCheckExitCodes []int
loadCheckStdout string
loadCheckStderr string
loadCheckPasses int
loadCheckResult *sandbox.ExecuteResult
// depsExitCode fails the declared-dependency check (venv / node_modules).
depsExitCode int
// execResult is scoped to execResultCommand: an unscoped stub result
@ -1597,6 +1775,10 @@ type installFixture struct {
execResult *sandbox.ExecuteResult
loadCheckRanAsRoot bool
agentErr error
// agentPrompts is every prompt the installer engine was handed, in order.
// A repair round is a second entry, and what it says is the whole point of
// having one.
agentPrompts []string
// beforeExecute runs at the moment the engine would start, so a test can
// observe the state an attaching console would see mid-install.
beforeExecute func()
@ -1657,6 +1839,10 @@ func newInstallFixture(t *testing.T) *installFixture {
fx := &installFixture{t: t}
fx.fingerprint = sandbox.SkillImageFingerprint("e2b", "key-1", "https://e2b.example")
// The checker's own wording for a missing dependency, so a test reading the
// repair prompt sees what a real run would put in it.
fx.loadCheckStderr = "scripts/extract.py imports pandas, " +
"which is not available in this image"
fx.bundle = &SkillBundle{
Name: "pdf-tools",
Version: "1.0.0",
@ -1721,6 +1907,19 @@ func (f *installFixture) record(event string) {
f.events = append(f.events, event)
}
// nextLoadCheckExit returns the code for this verification pass, repeating the
// last configured entry once they run out.
func (f *installFixture) nextLoadCheckExit() int {
f.loadCheckPasses++
if len(f.loadCheckExitCodes) == 0 {
return 0
}
if f.loadCheckPasses > len(f.loadCheckExitCodes) {
return f.loadCheckExitCodes[len(f.loadCheckExitCodes)-1]
}
return f.loadCheckExitCodes[f.loadCheckPasses-1]
}
// now is the fixture's clock, so a test can express "one heartbeat ago"
// against the same instant the service reads.
func (f *installFixture) now() time.Time { return f.svc.now() }
@ -2558,7 +2757,9 @@ func (m *installSandboxManager) ExecShellCommandWithOptions(
return m.fx.loadCheckResult, nil
}
return &sandbox.ExecuteResult{
ExitCode: m.fx.loadCheckExitCode, Stderr: "scripts/extract.py imports pandas",
ExitCode: m.fx.nextLoadCheckExit(),
Stdout: m.fx.loadCheckStdout,
Stderr: m.fx.loadCheckStderr,
}, nil
case command == removeSkillDirCommand:
m.fx.record("remove-skill-dir")
@ -2741,16 +2942,17 @@ type installAgentEngine struct {
}
func (e *installAgentEngine) Execute(
context.Context,
string,
string,
string,
[]chat.Message,
...[]string,
_ context.Context,
_ string,
_ string,
prompt string,
_ []chat.Message,
_ ...[]string,
) (*types.AgentState, error) {
if e.fx.beforeExecute != nil {
e.fx.beforeExecute()
}
e.fx.agentPrompts = append(e.fx.agentPrompts, prompt)
e.fx.record("agent-execute")
if e.fx.agentDelay > 0 {
time.Sleep(e.fx.agentDelay)

View File

@ -38,11 +38,17 @@ type installTranscript struct {
sessionID string
assistantMessageID string
mu sync.Mutex
message *types.Message
answers []*installAnswerSegment
starts map[string]time.Time
finished bool
mu sync.Mutex
message *types.Message
answers []*installAnswerSegment
starts map[string]time.Time
// closed guards the terminal event and the final write, both of which say
// "this install is over" and must be said once.
closed bool
// Totals accumulated across the run's engine turns, reported on the one
// terminal event Finish emits.
totalSteps int
totalDurationMs int64
}
// installAnswerSegment accumulates the prose streamed under one final-answer
@ -142,13 +148,20 @@ func (tr *installTranscript) Subscribe() {
tr.bus.On(event.EventAgentComplete, tr.onComplete)
}
// Finish closes the record. runErr is the engine's verdict: the engine emits
// no complete event when it fails, and a failed install is the one people
// actually come to read, so the failure is written here rather than hoped for.
// Finish closes the record. runErr is the install's verdict, not just the
// engine's: verification runs after the agent stops, and a failure there is the
// one people actually come to read, so it is written here rather than hoped for.
func (tr *installTranscript) Finish(ctx context.Context, runErr error) {
if tr == nil {
return
}
tr.mu.Lock()
alreadyClosed := tr.closed
tr.closed = true
tr.mu.Unlock()
if alreadyClosed {
return
}
if runErr != nil {
tr.append(interfaces.StreamEvent{
ID: uuid.NewString(),
@ -173,22 +186,41 @@ func (tr *installTranscript) Finish(ctx context.Context, runErr error) {
}
tr.mu.Lock()
alreadyComplete := tr.finished
tr.finished = true
steps, duration := tr.totalSteps, tr.totalDurationMs
tr.mu.Unlock()
if !alreadyComplete {
tr.append(interfaces.StreamEvent{
ID: uuid.NewString(),
Type: types.ResponseTypeComplete,
Done: true,
Timestamp: time.Now(),
Data: map[string]interface{}{},
})
}
tr.append(interfaces.StreamEvent{
ID: uuid.NewString(),
Type: types.ResponseTypeComplete,
Done: true,
Timestamp: time.Now(),
Data: map[string]interface{}{
"total_steps": steps,
"total_duration_ms": duration,
},
})
tr.save(ctx)
}
// RecordPrompt logs a follow-up instruction the installer was given mid-run.
//
// The transcript exists so one replay of the event log is the whole
// conversation. A repair round whose instruction is missing reads as the agent
// spontaneously deciding to install more packages, which is precisely the
// moment someone is reading this to find out why.
func (tr *installTranscript) RecordPrompt(prompt string) {
if tr == nil {
return
}
tr.append(interfaces.StreamEvent{
ID: uuid.NewString(),
Type: types.ResponseTypeInstallPrompt,
Content: prompt,
Done: true,
Timestamp: time.Now(),
Data: map[string]interface{}{},
})
}
func (tr *installTranscript) onThought(_ context.Context, evt event.Event) error {
data, ok := evt.Data.(event.AgentThoughtData)
if !ok {
@ -326,13 +358,16 @@ func (tr *installTranscript) onError(_ context.Context, evt event.Event) error {
return nil
}
// onComplete records what the round finished with. It deliberately emits no
// terminal event: an install may run another installer round after
// verification, and a console that saw "complete" would stop following before
// that round began. Only Finish closes the stream.
func (tr *installTranscript) onComplete(_ context.Context, evt event.Event) error {
data, ok := evt.Data.(event.AgentCompleteData)
if !ok {
return nil
}
tr.mu.Lock()
tr.finished = true
if data.MessageID == tr.assistantMessageID {
msg := tr.ensureMessageLocked()
msg.IsCompleted = true
@ -347,18 +382,11 @@ func (tr *installTranscript) onComplete(_ context.Context, evt event.Event) erro
if tr.composeAnswerLocked() == "" && data.FinalAnswer != "" {
tr.segmentLocked(evt.ID).content = data.FinalAnswer
}
// Summed rather than overwritten: an install that needed a repair round ran
// two engine turns, and its cost is both of them.
tr.totalSteps += data.TotalSteps
tr.totalDurationMs += data.TotalDurationMs
tr.mu.Unlock()
tr.append(interfaces.StreamEvent{
ID: evt.ID,
Type: types.ResponseTypeComplete,
Done: true,
Timestamp: time.Now(),
Data: map[string]interface{}{
"total_steps": data.TotalSteps,
"total_duration_ms": data.TotalDurationMs,
},
})
return nil
}

View File

@ -265,3 +265,56 @@ func TestInstallTranscriptCreateOpensTheLogWithThePrompt(t *testing.T) {
require.Equal(t, []types.ResponseType{types.ResponseTypeInstallPrompt}, streams.types())
require.Equal(t, "install pdf-tools", streams.events[0].Content)
}
// An install may run a second installer round, because verification happens
// after the agent stops and a dependency it names is something the agent can
// still fix. The engine reports "complete" at the end of every turn, so the
// transcript must not treat the first one as the end of the install — a console
// that saw it would stop following before the repair round began.
func TestInstallTranscriptStaysOpenAcrossInstallerRounds(t *testing.T) {
tr, bus, streams, messages := newTranscriptForTest(t)
ctx := context.Background()
finishRound := func(steps int, ms int64) {
require.NoError(t, bus.Emit(ctx, event.Event{
ID: "done", Type: event.EventAgentComplete,
Data: event.AgentCompleteData{
MessageID: "msg-1", TotalSteps: steps, TotalDurationMs: ms,
},
}))
}
finishRound(3, 1000)
tr.RecordPrompt("Verification failed. Install defusedxml into the venv.")
finishRound(2, 500)
tr.Finish(ctx, nil)
kinds := streams.types()
require.Equal(t, []types.ResponseType{
types.ResponseTypeInstallPrompt,
types.ResponseTypeComplete,
}, kinds, "the repair instruction has to be in the log, and the install ends once")
require.Contains(t, streams.events[0].Content, "Install defusedxml")
terminal := streams.events[len(streams.events)-1]
require.EqualValues(t, 5, terminal.Data["total_steps"],
"an install that needed a repair round cost both turns")
require.EqualValues(t, 1500, terminal.Data["total_duration_ms"])
require.Len(t, messages.updated, 1)
}
// Finish is deferred by the install path and also called on the error path that
// gives up before that defer is armed. Saying "this install is over" twice would
// duplicate the verdict in the record people read.
func TestInstallTranscriptFinishIsIdempotent(t *testing.T) {
tr, _, streams, messages := newTranscriptForTest(t)
tr.Finish(context.Background(), errors.New("python verification failed: boom"))
tr.Finish(context.Background(), errors.New("python verification failed: boom"))
require.Equal(t, []types.ResponseType{
types.ResponseTypeError,
types.ResponseTypeComplete,
}, streams.types())
require.Len(t, messages.updated, 1)
}

View File

@ -20,28 +20,66 @@ import (
//go:embed tenant_skill_verify.py
var skillPythonVerifier string
// skillVerifyRepairableExit is the exit code a verification pass uses when
// every problem it found is a dependency missing from this image. It separates
// "another installer round can fix this" from "the bundle has to change", which
// is the only distinction that decides what the install flow does next.
const skillVerifyRepairableExit = 2
// skillVerifyNotePrefix marks a line the checker reports without refusing the
// install. Notes travel on stdout so a non-zero exit stays unambiguous.
const skillVerifyNotePrefix = "note: "
// skillVerificationError is what the gate said, kept structured because it is
// also the brief for a repair round. The gate is the only authority on what has
// to resolve in this image, so handing back its own lines is what keeps the
// installer from deriving "what this skill needs" a second time.
type skillVerificationError struct {
// Language names the pass that failed, as the operator sees it.
Language string
// Repairable is true when installing a package would satisfy every line.
Repairable bool
// Problems are the checker's own lines, one per finding.
Problems []string
// Summary describes the command result itself, and is the only thing worth
// printing when the pass died without producing lines at all.
Summary string
}
func (e *skillVerificationError) Error() string {
if len(e.Problems) == 0 {
return fmt.Sprintf("%s verification failed (%s)", e.Language, e.Summary)
}
return fmt.Sprintf("%s verification failed: %s", e.Language, strings.Join(e.Problems, "; "))
}
// verifySkill is the server's own check, and the last gate before the image
// pointer moves: a broken install must leave the previous snapshot serving.
//
// It deliberately proves loadability rather than behaviour. Nothing in the
// runtime designates an entry point — the model names any script it likes in
// execute_skill_script — so there is no single command whose success would
// stand in for the skill working. What every one of those calls does need is
// that the interpreter can parse the file and resolve its imports inside this
// image, and that is checkable for all scripts at once without executing a
// line of the skill's own code.
// It proves loadability, not behaviour, and not "every file can be used as
// python this_file.py". A skill ships scripts and library packages together;
// execute_skill_script names files, but those files are also imported by each
// other. The Python pass resolves each file's imports against the prefixes
// Python itself would use — the file's own directory and every ancestor up to
// the skill root — plus the venv. Guessing a single entry script, or treating
// every .py as an isolated __main__, both fail real skills.
//
// Findings are graded rather than uniformly fatal. Refusing an install costs
// the minutes of dependency work that already succeeded, so only evidence that
// the install itself is broken may do it: a file nothing the skill offers ever
// loads, or a requirement pip would have skipped, is returned as a note.
//
// Not executing is the point, not a limitation. The previous implementation
// ran one guessed script with --help; skills that do not parse arguments
// simply ran their whole main path inside the tree about to be snapshotted.
func (s *TenantSkillService) verifySkill(
ctx context.Context, mgr sandbox.Manager, sessionID, skillDir string, bundle *SkillBundle,
) error {
) ([]string, error) {
if err := s.verifySkillTree(ctx, mgr, sessionID, skillDir, bundle); err != nil {
return err
return nil, err
}
if err := s.verifyDeclaredDependencies(ctx, mgr, sessionID, skillDir, bundle); err != nil {
return err
return nil, err
}
return s.verifyScriptsLoad(ctx, mgr, sessionID, skillDir, bundle)
}
@ -93,30 +131,43 @@ func (s *TenantSkillService) verifyDeclaredDependencies(
}
// verifyScriptsLoad runs one pass per language present in the bundle. Each
// pass covers every script of that language, because every one of them is a
// script the model may name at runtime.
// pass covers every file of that language: syntax and imports are properties
// of the file, not of a guessed entry point.
//
// Only the Python pass takes the auxiliary split. Its findings depend on what
// the image carries, so a bundled tests/ directory can fail an install that
// works; `node --check` and `bash -n` only state whether a file parses, which
// no amount of installing changes either way.
func (s *TenantSkillService) verifyScriptsLoad(
ctx context.Context, mgr sandbox.Manager, sessionID, skillDir string, bundle *SkillBundle,
) error {
) ([]string, error) {
var notes []string
if scripts := sortedScriptPaths(bundle, ".py"); len(scripts) > 0 {
if err := s.execVerify(ctx, mgr, sessionID, skillDir, "python",
skillPythonVerifyCommand(skillDir, scripts)); err != nil {
return err
entry, auxiliary := splitAuxiliaryScripts(scripts)
found, err := s.execVerify(ctx, mgr, sessionID, skillDir, "python",
skillPythonVerifyCommand(skillDir, entry, auxiliary))
notes = append(notes, found...)
if err != nil {
return notes, err
}
}
if scripts := sortedScriptPaths(bundle, ".js", ".mjs", ".cjs"); len(scripts) > 0 {
if err := s.execVerify(ctx, mgr, sessionID, skillDir, "node",
skillNodeVerifyCommand(skillDir, scripts, nodeDependencyNames(bundle))); err != nil {
return err
found, err := s.execVerify(ctx, mgr, sessionID, skillDir, "node",
skillNodeVerifyCommand(skillDir, scripts, nodeDependencyNames(bundle)))
notes = append(notes, found...)
if err != nil {
return notes, err
}
}
if scripts := sortedScriptPaths(bundle, ".sh"); len(scripts) > 0 {
if err := s.execVerify(ctx, mgr, sessionID, skillDir, "shell",
skillShellVerifyCommand(skillDir, scripts)); err != nil {
return err
found, err := s.execVerify(ctx, mgr, sessionID, skillDir, "shell",
skillShellVerifyCommand(skillDir, scripts))
notes = append(notes, found...)
if err != nil {
return notes, err
}
}
return nil
return notes, nil
}
// execVerify runs one verification pass as the ordinary sandbox user, with the
@ -124,10 +175,10 @@ func (s *TenantSkillService) verifyScriptsLoad(
// install-mode root would test permissions that never reach a session.
func (s *TenantSkillService) execVerify(
ctx context.Context, mgr sandbox.Manager, sessionID, skillDir, label, command string,
) error {
) ([]string, error) {
executor, err := installExecutor(mgr)
if err != nil {
return err
return nil, err
}
res, err := executor.ExecShellCommandWithOptions(ctx, sessionID, command,
sandbox.ShellExecOptions{
@ -139,12 +190,46 @@ func (s *TenantSkillService) execVerify(
},
})
if err != nil {
return fmt.Errorf("%s verification: %w", label, err)
return nil, fmt.Errorf("%s verification: %w", label, err)
}
// Notes are read even from a failed pass: an install refused for a missing
// package may also have noticed something about a file it did not refuse
// over, and that is exactly the context whoever reads the failure needs.
notes := verificationNotes(res.Stdout)
if res.ExitCode != 0 {
return fmt.Errorf("%s verification failed (%s)", label, describeExecFailure(res))
return notes, &skillVerificationError{
Language: label,
Repairable: res.ExitCode == skillVerifyRepairableExit,
Problems: verificationProblems(res.Stderr),
Summary: describeExecFailure(res),
}
}
return nil
return notes, nil
}
// verificationNotes pulls the reported-but-not-refused findings out of stdout.
func verificationNotes(stdout string) []string {
var notes []string
for _, line := range strings.Split(stdout, "\n") {
line = strings.TrimSpace(line)
if after, ok := strings.CutPrefix(line, skillVerifyNotePrefix); ok {
notes = append(notes, after)
}
}
return notes
}
// verificationProblems splits a failed pass's stderr into its findings. Blank
// lines are dropped; everything else is the checker's own wording, which is
// what a repair round is given.
func verificationProblems(stderr string) []string {
var problems []string
for _, line := range strings.Split(stderr, "\n") {
if line = strings.TrimSpace(line); line != "" {
problems = append(problems, line)
}
}
return problems
}
// skillPythonVerifyCommand pipes the verifier into the same interpreter a
@ -152,13 +237,23 @@ func (s *TenantSkillService) execVerify(
// line. The list is explicit rather than a directory walk so the pass covers
// exactly the uploaded sources — never .venv, node_modules or anything else
// the agent created while installing.
func skillPythonVerifyCommand(skillDir string, scripts []string) string {
//
// Auxiliary files are named after --optional. They are checked the same way,
// and what is found in them is reported rather than allowed to refuse the
// install.
func skillPythonVerifyCommand(skillDir string, entry, auxiliary []string) string {
venv := path.Join(skillDir, ".venv", "bin", "python")
quotedVenv := sandbox.ShellQuote(venv)
args := []string{sandbox.ShellQuote(skillDir)}
for _, rel := range scripts {
for _, rel := range entry {
args = append(args, sandbox.ShellQuote(rel))
}
if len(auxiliary) > 0 {
args = append(args, skillVerifyOptionalFlag)
for _, rel := range auxiliary {
args = append(args, sandbox.ShellQuote(rel))
}
}
// The script arrives base64-encoded on stdin: it contains quotes, newlines
// and backslashes, and a heredoc would still be at the mercy of whatever
// the remote exec layer does with multi-line command strings.
@ -173,6 +268,9 @@ func skillPythonVerifyCommand(skillDir string, scripts []string) string {
// skillNodeVerifyCommand parses every JavaScript file and, when the bundle
// declares runtime dependencies, checks each one resolves. `node --check` is
// parse-only: it never runs the module body.
//
// A declared dependency that is missing exits with the repairable code: it is
// the one thing here another installer round can still put in place.
func skillNodeVerifyCommand(skillDir string, scripts, deps []string) string {
parts := make([]string, 0, 2)
if len(deps) > 0 {
@ -182,18 +280,75 @@ func skillNodeVerifyCommand(skillDir string, scripts, deps []string) string {
}
parts = append(parts, fmt.Sprintf(
"for d in %s; do [ -e %s/\"$d\" ] || "+
"{ echo \"package.json declares $d but node_modules/$d is missing\" >&2; exit 1; }; done",
"{ echo \"package.json declares $d but node_modules/$d is missing\" >&2; exit %d; }; done",
strings.Join(quoted, " "), sandbox.ShellQuote(path.Join(skillDir, "node_modules")),
skillVerifyRepairableExit,
))
}
parts = append(parts, forEachScript(skillDir, scripts, "node --check \"$f\""))
return strings.Join(parts, "; ")
}
// skillShellVerifyCommand parses every shell script. `sh -n` reads the file
// and exits without executing any of it.
// skillShellVerifyCommand parses every shell script with the shell that would
// run it. `-n` reads the file and exits without executing any of it.
//
// bash, not sh: skills ship `#!/bin/bash` almost exclusively, and
// SkillInterpreterCommand runs them with bash for that reason. /bin/sh is dash
// on Debian, where an array literal, `function f()`, a C-style for loop and
// process substitution are all outright syntax errors — so checking with sh
// would refuse installs over scripts that run perfectly.
func skillShellVerifyCommand(skillDir string, scripts []string) string {
return forEachScript(skillDir, scripts, "sh -n \"$f\"")
return "if command -v bash >/dev/null 2>&1; then parser=bash; else parser=sh; fi; " +
forEachScript(skillDir, scripts, `"$parser" -n "$f"`)
}
// skillVerifyOptionalFlag separates the files whose failure refuses the install
// from the ones whose failure is only reported. It is the checker's own argv
// contract; both sides must keep using this spelling.
const skillVerifyOptionalFlag = "--optional"
// auxiliaryScriptDirs are the directory names Python and JavaScript projects
// universally use for files that ship with a project without being part of
// what it offers. A skill's tests import pytest; its examples import whatever
// they illustrate. Nothing the skill exposes loads either, so neither may
// decide whether the skill installs.
var auxiliaryScriptDirs = map[string]struct{}{
"test": {}, "tests": {}, "testing": {}, "__tests__": {},
"example": {}, "examples": {}, "sample": {}, "samples": {},
"benchmark": {}, "benchmarks": {}, "fixtures": {},
"doc": {}, "docs": {},
}
// skillAuxiliaryScript reports whether a bundled path is one of those files.
//
// The rule is the naming convention the language communities already share,
// not a list grown one skill at a time: a directory the ecosystem reserves for
// tests, examples or docs, or a file name pytest and setuptools already treat
// as theirs.
func skillAuxiliaryScript(rel string) bool {
segments := strings.Split(path.Clean(rel), "/")
for _, segment := range segments[:len(segments)-1] {
if _, ok := auxiliaryScriptDirs[strings.ToLower(segment)]; ok {
return true
}
}
base := strings.ToLower(segments[len(segments)-1])
stem := strings.TrimSuffix(base, path.Ext(base))
return stem == "conftest" || stem == "setup" ||
strings.HasPrefix(stem, "test_") || strings.HasSuffix(stem, "_test")
}
// splitAuxiliaryScripts separates the files whose failure is a failed install
// from the ones whose failure is a note, preserving the caller's order.
func splitAuxiliaryScripts(scripts []string) (entry, auxiliary []string) {
for _, rel := range scripts {
if skillAuxiliaryScript(rel) {
auxiliary = append(auxiliary, rel)
continue
}
entry = append(entry, rel)
}
return entry, auxiliary
}
func forEachScript(skillDir string, scripts []string, check string) string {

View File

@ -3,14 +3,36 @@
Run inside the sandbox, by the interpreter a real skill call would use, as the
ordinary execution user, after the tree has been given its final permissions:
python3 - <skill-dir> <relative-script> [<relative-script> ...]
python3 - <skill-dir> <script> [<script> ...] [--optional <script> ...]
Nothing here executes the skill's own code. Everything it reports is decided
from the parse tree and from what the environment can resolve, so a skill that
writes files or calls the network on import cannot do either while being
checked, and a script that never learned --help cannot be failed for it.
Exits 1 and writes one line per problem to stderr; exits 0 otherwise.
Loadability follows Python's import rules, not "this file as isolated
__main__". A file's import prefixes are its own directory plus every ancestor
up to the skill root, because any of those can be sys.path[0]: either the
bundle ships a script there (the runtime hands `python <file>` a directory),
or the file put it there itself. A sibling subtree is never an ancestor, so
vendor/requests.py still cannot make `import requests` look satisfied to a
script under scripts/.
Findings are graded, because rejecting an install is expensive: it throws away
minutes of dependency work and leaves the previous image serving.
stdout `note: ...` lines - reported, image kept
stderr problem lines - the install is refused
exit 0 the image may be kept
exit 1 a problem no installer round can fix: a syntax error, a file the
execution user cannot read, a relative import that can never
resolve
exit 2 every problem is a dependency missing from this image, so handing
these lines back to the installer is worth a round
Files named after --optional are checked identically, but their findings are
notes: nothing the skill offers loads a test or an example, and a bundled
tests/ directory must not decide whether the skill installs.
"""
import ast
@ -19,54 +41,72 @@ import os
import re
import sys
root = os.path.abspath(sys.argv[1])
relative_scripts = sys.argv[2:]
problems = []
OPTIONAL_FLAG = "--optional"
# Directories that belong to the installed environment rather than the skill's
# own sources, and would drown the scan below in third-party module names.
PRUNED = {".venv", "venv", "node_modules", "__pycache__", ".git"}
EXIT_UNREPAIRABLE = 1
EXIT_MISSING_DEPENDENCY = 2
root = os.path.abspath(sys.argv[1])
_argv_scripts = sys.argv[2:]
if OPTIONAL_FLAG in _argv_scripts:
_cut = _argv_scripts.index(OPTIONAL_FLAG)
entry_scripts = _argv_scripts[:_cut]
optional_scripts = _argv_scripts[_cut + 1 :]
else:
entry_scripts = _argv_scripts
optional_scripts = []
all_scripts = entry_scripts + optional_scripts
optional_set = set(optional_scripts)
# problems refuse the install; notes are reported and the image is kept.
problems = []
notes = []
# Whether any problem is something installing a package cannot fix. It decides
# the exit code, which is how the caller knows if another installer round could
# help or if the bundle itself has to change.
unrepairable = False
# The directories the bundle actually ships a script in. A prefix that is one
# of these is reachable on its own, because the runtime can be asked to execute
# a file there; any other prefix is only on sys.path if the skill puts it there.
script_dirs = {os.path.dirname(os.path.join(root, rel)) for rel in all_scripts}
def add_problem(message, repairable=False):
global unrepairable
if message not in problems:
problems.append(message)
if not repairable:
unrepairable = True
def add_note(message):
if message not in notes:
notes.append(message)
def note_instead_of_problem(message, repairable=False):
"""Reporter for an auxiliary file: the finding is real, the verdict is not.
repairable is accepted and ignored so this can stand in for add_problem;
a note never influences the exit code.
"""
add_note("%s (auxiliary file; this does not fail the install)" % message)
def under_root(candidate):
return candidate == root or candidate.startswith(root + os.sep)
def own_top_level_names():
"""Top-level names that refer to something the skill itself ships.
Only the skill root is consulted. A nested file such as vendor/requests.py
must not make `import requests` look first-party — that name is a
dependency the installer was asked to put in the venv, and skipping it
would snapshot a broken install.
An import of one of these is the skill reaching for its own code, and
whether it resolves depends on how the script arranges sys.path at run
time — commonly by inserting its parent before importing `scripts.x`.
That is a runtime decision this checker cannot evaluate without executing
the file, so such imports are left to the skill. Sibling modules still
resolve through find_spec, which puts the script's own directory first.
"""
names = set()
try:
entries = os.listdir(root)
except OSError:
return names
for name in entries:
if name in PRUNED or name.startswith("."):
continue
full = os.path.join(root, name)
if os.path.isdir(full):
names.add(name)
elif name.endswith(".py"):
names.add(name[:-3])
return names
own_modules = own_top_level_names()
def is_package(directory):
"""Whether directory is a regular Python package.
Only __init__.py counts, and only the relative-import check needs it: a
`from . import x` in a directory without one can never resolve when the
file is run directly. Absolute imports do not consult this, because a
PEP 420 namespace package has no __init__.py and is still importable.
"""
return os.path.isfile(os.path.join(directory, "__init__.py"))
@ -78,19 +118,45 @@ def module_exists(base, dotted):
)
def is_importable(name, script_dir):
"""Whether a top-level name is available to this script at run time.
def import_search_dirs(script_path):
"""Every directory that could be sys.path[0] when this file is loaded.
Its own directory comes first: execute_skill_script runs `python <file>`.
Each ancestor up to the skill root follows, because that is what makes a
neighbouring entry script's imports resolve - Python puts the directory of
the script it was handed on the path, and a package below that directory,
regular or namespace, is importable from there. It is also where the
`sys.path.insert(0, parent)` idiom lands, which cannot be seen without
executing the file.
Over-approximating here rejects nothing that works. Under-approximating
rejected the official office toolkit, whose library modules import their
sibling packages by short name.
"""
current = os.path.dirname(os.path.abspath(script_path))
dirs = []
while under_root(current):
if current not in dirs:
dirs.append(current)
parent = os.path.dirname(current)
if parent == current:
break
current = parent
return dirs
def find_spec_with(name, prefixes):
"""Whether find_spec resolves name with prefixes ahead of sys.path.
find_spec consults the finders without importing, so a missing dependency
is reported without the side effects of loading a present one. script_dir
leads the path because that is where the interpreter puts the directory of
the script it was handed, which is how sibling modules resolve at runtime.
is reported without the side effects of loading a present one. Only
top-level names are ever queried: find_spec on a dotted name imports the
parent package, which would run the skill's own module body.
"""
if name in own_modules:
return True
saved = sys.path[:]
sys.path.insert(0, script_dir)
try:
for directory in reversed(prefixes):
sys.path.insert(0, directory)
return importlib.util.find_spec(name) is not None
except Exception:
return False
@ -98,39 +164,68 @@ def is_importable(name, script_dir):
sys.path[:] = saved
def check_imports(rel, tree, script_dir):
"""Check the imports that running this file is guaranteed to execute.
def resolve_top_level(name, script_path):
"""How a top-level name resolves for this file.
"image" the venv, the standard library, or a directory the bundle
ships a script in - something the runtime reaches on its own.
"unattested" only a directory that ships no script of its own can provide
it, so it is on sys.path only if the file puts it there. That
is unprovable without executing, so it is reported rather
than trusted or refused.
"" nothing in this image can provide it.
"""
dirs = import_search_dirs(script_path)
attested = [directory for directory in dirs if directory in script_dirs]
if find_spec_with(name, attested):
return "image"
if len(attested) != len(dirs) and find_spec_with(name, dirs):
return "unattested"
return ""
def check_imports(rel, tree, script_path, report):
"""Check the imports that loading this file is guaranteed to execute.
Only statements at module level are checked, and only unconditionally: an
import nested in try/except, in an `if`, or inside a function is how a
skill declares an optional or lazily loaded dependency, and failing an
install over one would reject a working skill.
"""
script_dir = os.path.dirname(script_path)
for node in tree.body:
if isinstance(node, ast.Import):
for alias in node.names:
top = alias.name.split(".")[0]
if not is_importable(top, script_dir):
problems.append(
"%s imports %s, which is not available in this image"
% (rel, top)
)
check_absolute_import(rel, alias.name, script_path, report)
elif isinstance(node, ast.ImportFrom):
if node.level:
check_relative_import(rel, node, script_dir)
check_relative_import(rel, node, script_dir, report)
elif node.module:
top = node.module.split(".")[0]
if not is_importable(top, script_dir):
problems.append(
"%s imports %s, which is not available in this image"
% (rel, top)
)
check_absolute_import(rel, node.module, script_path, report)
def check_relative_import(rel, node, script_dir):
def check_absolute_import(rel, dotted, script_path, report):
name = dotted.split(".")[0]
origin = resolve_top_level(name, script_path)
if origin == "image":
return
if origin == "unattested":
add_note(
"%s imports %s, which the skill ships but no directory it executes "
"scripts from can reach; it resolves only if the file puts that "
"directory on sys.path itself" % (rel, name)
)
return
report(
"%s imports %s, which is not available in this image" % (rel, name),
repairable=True,
)
def check_relative_import(rel, node, script_dir, report):
spelling = "." * node.level + (node.module or "")
if not is_package(script_dir):
problems.append(
report(
"%s uses the relative import '%s' but its directory has no "
"__init__.py, so the import can never resolve when the script is "
"run directly" % (rel, spelling)
@ -140,7 +235,7 @@ def check_relative_import(rel, node, script_dir):
for _ in range(node.level - 1):
base = os.path.dirname(base)
if not under_root(base):
problems.append(
report(
"%s uses the relative import '%s', which reaches outside the "
"skill directory" % (rel, spelling)
)
@ -149,9 +244,7 @@ def check_relative_import(rel, node, script_dir):
# __init__ defines rather than a submodule, so only the package itself is
# checked in that case.
if node.module and not module_exists(base, node.module):
problems.append(
"%s imports '%s', which does not exist in the skill" % (rel, spelling)
)
report("%s imports '%s', which does not exist in the skill" % (rel, spelling))
def distribution_name(raw):
@ -170,6 +263,35 @@ def distribution_name(raw):
return name
def marker_of(raw):
"""The PEP 508 environment marker on a requirement line, or ''."""
_, _, marker = raw.split("#")[0].partition(";")
return marker.strip()
def marker_applies(marker):
"""Whether pip would install a line carrying this marker in this image.
None means the answer cannot be established, and callers must read that as
"not enforceable" rather than "absent". Markers compare versions with
version semantics, so they are evaluated by `packaging` or not at all: a
hand-rolled string comparison reads "3.10" as lower than "3.9" and would
fail installs whose requirements pip resolved correctly. A bare venv has
no `packaging`, so None is the common answer.
"""
if re.search(r"\bextra\b", marker):
# Gated on an extra, which pip does not install unless it is requested.
return False
try:
from packaging.markers import Marker
except Exception:
return None
try:
return bool(Marker(marker).evaluate())
except Exception:
return None
def load_pyproject(path):
try:
import tomllib
@ -200,13 +322,29 @@ def declared_requirement_lines():
yield "pyproject.toml", item
poetry = ((data.get("tool") or {}).get("poetry") or {}).get("dependencies") or {}
if isinstance(poetry, dict):
for name in poetry:
for name, spec in poetry.items():
if str(name).strip().lower() == "python":
continue
# A table-valued dependency carrying optional, markers or a python
# constraint is conditional, and poetry would not have installed it
# here unconditionally.
if isinstance(spec, dict) and (
spec.get("optional") or spec.get("markers") or spec.get("python")
):
continue
yield "pyproject.toml", str(name)
def check_declared_requirements():
"""Check the manifests name nothing the venv is missing.
This is the installer's literal instruction - "install requirements.txt" -
so a distribution it names that pip did not land is a failed install. A
line pip would have skipped is not: an environment marker that is false
here, or an extras-gated dependency, is reported instead. Refusing an
install over `pywin32; sys_platform == "win32"` on Linux rejects a skill
whose requirements are all present.
"""
try:
from importlib import metadata
except ImportError:
@ -217,39 +355,56 @@ def check_declared_requirements():
continue
try:
metadata.distribution(name)
continue
except Exception:
problems.append(
"%s declares %s but it is not installed in %s"
% (source, name, sys.prefix)
pass
missing = "%s declares %s but it is not installed in %s" % (
source,
name,
sys.prefix,
)
marker = marker_of(raw)
if not marker:
add_problem(missing, repairable=True)
continue
applies = marker_applies(marker)
if applies:
add_problem(missing, repairable=True)
elif applies is None:
add_note(
"%s, and its environment marker '%s' cannot be evaluated here, "
"so it is not enforced" % (missing, marker)
)
for relative in relative_scripts:
for relative in all_scripts:
report = note_instead_of_problem if relative in optional_set else add_problem
script = os.path.join(root, relative)
try:
with open(script, "rb") as source:
code = source.read()
except OSError as exc:
problems.append(
"%s cannot be read by the skill execution user (%s)" % (relative, exc)
)
report("%s cannot be read by the skill execution user (%s)" % (relative, exc))
continue
try:
parsed = ast.parse(code, filename=relative)
except SyntaxError as exc:
problems.append(
report(
"%s has a syntax error on line %s: %s" % (relative, exc.lineno, exc.msg)
)
continue
check_imports(relative, parsed, os.path.dirname(script))
check_imports(relative, parsed, script, report)
check_declared_requirements()
for note in notes:
sys.stdout.write("note: %s\n" % note)
if problems:
sys.stderr.write("\n".join(problems) + "\n")
sys.exit(1)
sys.exit(EXIT_UNREPAIRABLE if unrepairable else EXIT_MISSING_DEPENDENCY)
print(
"verified %d python file(s) against %s"
% (len(relative_scripts), sys.executable or "python3")
% (len(all_scripts), sys.executable or "python3")
)

View File

@ -21,9 +21,21 @@ func TestSkillPythonVerifier(t *testing.T) {
cases := []struct {
name string
files map[string]string
// optional names the files whose findings must be reported instead of
// refusing the install. The install path fills this from the naming
// conventions the ecosystems share; here it is explicit.
optional []string
// wantProblem is a substring of the expected stderr. Empty means the
// tree must verify cleanly.
wantProblem string
// wantExit is the code a failing tree must exit with: 2 when installing
// a package would satisfy everything found, 1 when it would not. The
// install flow reads it to decide whether another installer round is
// worth its minutes.
wantExit int
// wantNote is a substring of a stdout note - something the checker
// reported without refusing the install.
wantNote string
}{{
name: "a package whose __init__ imports its own submodules",
files: map[string]string{
@ -34,7 +46,8 @@ func TestSkillPythonVerifier(t *testing.T) {
// The shape that failed a real install: sys.path is rearranged at run
// time, so the absolute import of the skill's own package resolves
// only once the file executes. Deciding it statically is not possible,
// and guessing wrong rejects a working skill.
// and guessing wrong rejects a working skill. It is reported, because
// the only directory that can satisfy it ships no script of its own.
name: "a script that puts the skill root on sys.path and imports its own package",
files: map[string]string{
"scripts/__init__.py": "",
@ -44,6 +57,8 @@ func TestSkillPythonVerifier(t *testing.T) {
"sys.path.insert(0, str(Path(__file__).resolve().parent.parent))\n" +
"from scripts.helper import go\n",
},
wantNote: "imports scripts, which the skill ships but no directory it " +
"executes scripts from can reach",
}, {
name: "a script importing a sibling module directly",
files: map[string]string{
@ -51,6 +66,17 @@ func TestSkillPythonVerifier(t *testing.T) {
"scripts/helper.py": "x = 1\n",
"scripts/README.txt": "not python\n",
},
}, {
// scripts/ is both a package (relative imports in __init__) and a
// directory of files the runtime executes with python file.py.
// Loadability is the union of those prefixes, so a sibling import
// still resolves when the package marker is present.
name: "a packaged scripts directory whose files also import siblings",
files: map[string]string{
"scripts/__init__.py": "",
"scripts/run.py": "import helper\n",
"scripts/helper.py": "x = 1\n",
},
}, {
name: "an optional dependency behind try/except",
files: map[string]string{
@ -69,12 +95,25 @@ func TestSkillPythonVerifier(t *testing.T) {
},
wantProblem: "scripts/run.py imports totally_absent_package, " +
"which is not available in this image",
wantExit: 2,
}, {
name: "a syntax error",
files: map[string]string{
"scripts/run.py": "def broken(:\n pass\n",
},
wantProblem: "scripts/run.py has a syntax error on line 1",
wantExit: 1,
}, {
// The exit code is the whole verdict, so one unfixable finding among
// fixable ones has to sink the batch: sending the installer back for a
// package it can install would only delay a failure it cannot.
name: "a syntax error alongside a missing dependency",
files: map[string]string{
"scripts/bad.py": "def broken(:\n",
"scripts/run.py": "import totally_absent_package\n",
},
wantProblem: "scripts/bad.py has a syntax error",
wantExit: 1,
}, {
name: "a relative import in a directory that is not a package",
files: map[string]string{
@ -83,6 +122,7 @@ func TestSkillPythonVerifier(t *testing.T) {
"scripts/notinit.py": "",
},
wantProblem: "has no __init__.py",
wantExit: 1,
}, {
name: "a relative import of a module the skill does not ship",
files: map[string]string{
@ -91,14 +131,45 @@ func TestSkillPythonVerifier(t *testing.T) {
"pkg/sub/run.py": "from ..missing import go\n",
},
wantProblem: "pkg/sub/run.py imports '..missing', which does not exist in the skill",
wantExit: 1,
}, {
name: "a requirement the venv does not carry",
files: map[string]string{
"requirements.txt": "# pinned\npandas==3.0.1\n-r other.txt\n" +
"totally_absent_package>=1.0 ; python_version>=\"3.9\"\n",
"requirements.txt": "# pinned\npandas==3.0.1\n-r other.txt\n",
"scripts/run.py": "x = 1\n",
},
wantProblem: "requirements.txt declares pandas but it is not installed",
wantExit: 2,
}, {
// pip skips a line whose marker is false here, so refusing the install
// over it rejects a skill whose requirements are all present. Markers
// compare versions with version semantics, so they are evaluated by
// `packaging` or not at all - and a bare venv has no `packaging`, which
// is why "cannot evaluate" has to mean "report", not "enforce".
name: "a requirement gated by an environment marker",
files: map[string]string{
"requirements.txt": "pywin32; sys_platform == \"win32\"\n" +
"totally_absent_package; extra == \"dev\"\n",
"scripts/run.py": "x = 1\n",
},
wantNote: "requirements.txt declares pywin32 but it is not installed",
}, {
// An extras-gated dependency is never installed unless the extra is
// requested, so it is not even worth a note.
name: "an extras-gated requirement is not reported at all",
files: map[string]string{
"requirements.txt": "totally_absent_package; extra == \"dev\"\n",
"scripts/run.py": "x = 1\n",
},
}, {
// poetry tables carrying optional, markers or a python constraint are
// conditional, and poetry would not have installed them here either.
name: "a poetry dependency marked optional",
files: map[string]string{
"pyproject.toml": "[tool.poetry.dependencies]\npython = \"^3.11\"\n" +
"totally_absent_package = { version = \"^1.0\", optional = true }\n",
"scripts/run.py": "x = 1\n",
},
wantProblem: "requirements.txt declares totally_absent_package but it is not installed",
}, {
name: "a nested file name is not treated as a first-party import",
files: map[string]string{
@ -107,6 +178,49 @@ func TestSkillPythonVerifier(t *testing.T) {
},
wantProblem: "scripts/run.py imports totally_absent_package, " +
"which is not available in this image",
wantExit: 2,
}, {
// The official Anthropic office toolkit (xlsx/docx/pptx): a non-package
// directory holds entry scripts plus sibling packages. Library modules
// import those siblings by short name (`from helpers import ...`).
// Treating every .py as isolated __main__ rejects this layout; Python's
// package path root is scripts/office/, which is where helpers lives.
name: "a library package importing a sibling package by short name",
files: map[string]string{
"scripts/recalc.py": "from office.soffice import run_soffice\n",
"scripts/office/soffice.py": "def run_soffice():\n pass\n",
"scripts/office/validate.py": "from helpers import safe_extract\n" +
"from validators import BaseSchemaValidator\n",
"scripts/office/helpers/__init__.py": "def safe_extract():\n pass\n",
"scripts/office/validators/__init__.py": "from .base import BaseSchemaValidator\n",
"scripts/office/validators/base.py": "from helpers import safe_extract\n" +
"class BaseSchemaValidator:\n pass\n",
"scripts/office/validators/docx.py": "from helpers import safe_extract\n",
},
}, {
name: "a library package importing a dependency the installer never installed",
files: map[string]string{
"scripts/office/validate.py": "x = 1\n",
"scripts/office/helpers/__init__.py": "x = 1\n",
"scripts/office/validators/__init__.py": "",
"scripts/office/validators/base.py": "import totally_absent_package\n",
},
wantProblem: "scripts/office/validators/base.py imports totally_absent_package, " +
"which is not available in this image",
wantExit: 2,
}, {
// The same layout without __init__.py anywhere. PEP 420 makes those
// directories namespace packages, so a neighbouring entry script still
// resolves the short-name import - which is why prefixes are every
// ancestor and not only the ones a package marker chains together.
name: "a namespace package importing a sibling by short name",
files: map[string]string{
"scripts/office/validate.py": "from helpers import safe_extract\n",
"scripts/office/helpers/__init__.py": "def safe_extract():\n pass\n",
"scripts/office/validators/base.py": "from helpers import safe_extract\n",
"scripts/pkg/sub/mod.py": "from util import go\n",
"scripts/pkg/util.py": "def go():\n pass\n",
},
}, {
name: "a pyproject.toml dependency the venv does not carry",
files: map[string]string{
@ -115,6 +229,23 @@ func TestSkillPythonVerifier(t *testing.T) {
"scripts/run.py": "x = 1\n",
},
wantProblem: "pyproject.toml declares totally_absent_package but it is not installed",
wantExit: 2,
}, {
// Skills ship their tests. Nothing the skill offers loads them, so a
// bundled tests/ directory must not decide whether the skill installs -
// and refusing over one throws away the dependency work that succeeded.
name: "a bundled test file importing a package the image does not carry",
files: map[string]string{
"scripts/run.py": "x = 1\n",
"tests/test_run.py": "import pytest\nimport totally_absent_package\n",
"examples/demo.py": "import totally_absent_package\n",
"tests/conftest.py": "def broken(:\n",
"scripts/helper.py": "x = 1\n",
},
optional: []string{
"examples/demo.py", "tests/conftest.py", "tests/test_run.py",
},
wantNote: "auxiliary file; this does not fail the install",
}, {
// Lines that name a distribution only indirectly cannot be checked by
// name, and inventing one from the URL would fail installs whose
@ -132,19 +263,52 @@ func TestSkillPythonVerifier(t *testing.T) {
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
root := writeSkillTree(t, tc.files)
stdout, stderr, err := runSkillPythonVerifier(t, root, pythonFiles(tc.files))
entry := pythonFiles(tc.files)
if len(tc.optional) > 0 {
entry = withoutPaths(entry, tc.optional)
}
stdout, stderr, err := runSkillPythonVerifier(t, root, entry, tc.optional)
if tc.wantProblem == "" {
require.NoError(t, err, "this skill must install; stderr: %s", stderr)
require.Contains(t, stdout, "verified")
return
} else {
require.Error(t, err, "this skill is broken and must not reach a snapshot")
require.Contains(t, stderr, tc.wantProblem)
require.Equal(t, tc.wantExit, verifierExitCode(t, err),
"the exit code is what decides whether an installer round can fix this; "+
"stderr: %s", stderr)
}
if tc.wantNote != "" {
require.Contains(t, stdout, "note: ",
"a finding that does not refuse the install must still be reported")
require.Contains(t, stdout, tc.wantNote)
}
require.Error(t, err, "this skill is broken and must not reach a snapshot")
require.Contains(t, stderr, tc.wantProblem)
})
}
}
func verifierExitCode(t *testing.T, err error) int {
t.Helper()
var exit *exec.ExitError
require.ErrorAs(t, err, &exit, "the checker must fail by exiting, not by crashing")
return exit.ExitCode()
}
func withoutPaths(all, drop []string) []string {
dropped := make(map[string]struct{}, len(drop))
for _, rel := range drop {
dropped[rel] = struct{}{}
}
kept := make([]string, 0, len(all))
for _, rel := range all {
if _, ok := dropped[rel]; !ok {
kept = append(kept, rel)
}
}
return kept
}
// Verification must be able to read a skill that writes files or opens sockets
// the moment it is imported, without doing either.
func TestSkillPythonVerifierNeverExecutesTheSkill(t *testing.T) {
@ -153,7 +317,7 @@ func TestSkillPythonVerifierNeverExecutesTheSkill(t *testing.T) {
"open(os.path.join(os.path.dirname(__file__), 'SIDE_EFFECT'), 'w').close()\n",
})
_, stderr, err := runSkillPythonVerifier(t, root, []string{"scripts/run.py"})
_, stderr, err := runSkillPythonVerifier(t, root, []string{"scripts/run.py"}, nil)
require.NoError(t, err, stderr)
_, statErr := os.Stat(filepath.Join(root, "scripts", "SIDE_EFFECT"))
@ -170,10 +334,50 @@ func TestSkillPythonVerifierReportsAnUnreadableScript(t *testing.T) {
root := writeSkillTree(t, map[string]string{"scripts/run.py": "x = 1\n"})
require.NoError(t, os.Chmod(filepath.Join(root, "scripts", "run.py"), 0o000))
_, stderr, err := runSkillPythonVerifier(t, root, []string{"scripts/run.py"})
_, stderr, err := runSkillPythonVerifier(t, root, []string{"scripts/run.py"}, nil)
require.Error(t, err)
require.Contains(t, stderr, "cannot be read by the skill execution user")
require.Equal(t, 1, verifierExitCode(t, err),
"a file the execution user cannot read is not something installing a package fixes")
}
// The layout that started this: the official office toolkit ships entry scripts
// beside sibling packages, and its library modules import those siblings by
// short name. Treating every .py as an isolated __main__ rejected it outright.
// The imports it cannot resolve here are the third-party ones it genuinely
// needs — including two its SKILL.md never mentions — and that is the list the
// installer gets handed back.
func TestSkillPythonVerifierAcceptsTheOfficeToolkitLayout(t *testing.T) {
files := map[string]string{
"SKILL.md": "# xlsx\n",
"scripts/recalc.py": "import json\nimport sys\nfrom pathlib import Path\n" +
"from office.soffice import run_soffice\n",
"scripts/office/soffice.py": "import subprocess\nimport tempfile\n" +
"def run_soffice():\n pass\n",
"scripts/office/validate.py": "import argparse\n" +
"from helpers import safe_extract\n" +
"from validators import DOCXSchemaValidator\n",
"scripts/office/helpers/__init__.py": "import zipfile\n" +
"def safe_extract():\n pass\n",
"scripts/office/validators/__init__.py": "from .docx import DOCXSchemaValidator\n",
"scripts/office/validators/base.py": "import re\n" +
"from helpers import safe_extract\n" +
"class BaseSchemaValidator:\n pass\n",
"scripts/office/validators/docx.py": "from helpers import safe_extract\n" +
"from .base import BaseSchemaValidator\n" +
"class DOCXSchemaValidator(BaseSchemaValidator):\n pass\n",
"scripts/office/helpers/pptx_chart.py": "from __future__ import annotations\n" +
"import re\nfrom . import part_text\n",
}
root := writeSkillTree(t, files)
stdout, stderr, err := runSkillPythonVerifier(t, root, pythonFiles(files), nil)
require.NoError(t, err, "the toolkit's own layout must not be a failed install; stderr: %s", stderr)
require.Contains(t, stdout, "verified")
require.NotContains(t, stderr, "helpers",
"helpers is a sibling package of validators/, reachable from scripts/office/")
}
func writeSkillTree(t *testing.T, files map[string]string) string {
@ -200,14 +404,20 @@ func pythonFiles(files map[string]string) []string {
// runSkillPythonVerifier feeds the embedded checker to a real interpreter the
// same way the sandbox command does: on stdin, with the tree and the files to
// check as argv.
func runSkillPythonVerifier(t *testing.T, root string, scripts []string) (string, string, error) {
// check as argv, auxiliary files last behind the separator.
func runSkillPythonVerifier(
t *testing.T, root string, scripts, optional []string,
) (string, string, error) {
t.Helper()
python, err := exec.LookPath("python3")
if err != nil {
t.Skip("python3 is not on PATH")
}
cmd := exec.Command(python, append([]string{"-", root}, scripts...)...)
argv := append([]string{"-", root}, scripts...)
if len(optional) > 0 {
argv = append(append(argv, skillVerifyOptionalFlag), optional...)
}
cmd := exec.Command(python, argv...)
cmd.Stdin = strings.NewReader(skillPythonVerifier)
var stdout, stderr bytes.Buffer
cmd.Stdout = &stdout

View File

@ -189,7 +189,17 @@ func SkillInterpreterCommand(skillDir, scriptPath string) (string, []string) {
case ".js", ".mjs", ".cjs":
return "node", []string{scriptPath}
case ".sh":
return "/bin/sh", []string{scriptPath}
// bash, with sh only as a fallback. Skill shell scripts carry a
// `#!/bin/bash` shebang almost exclusively, and /bin/sh is dash on
// Debian: an array literal, `function f()`, a C-style for loop and
// process substitution are all syntax errors there, so running these
// files with sh breaks scripts that are perfectly valid. The
// install-time check parses them with the same shell.
script := ShellQuote(scriptPath)
return "/bin/sh", []string{"-c", fmt.Sprintf(
`if command -v bash >/dev/null 2>&1; then exec bash %s "$@"; else exec sh %s "$@"; fi`,
script, script,
), skillShellArgv0}
default:
return "/bin/sh", []string{scriptPath}
}

View File

@ -161,10 +161,34 @@ func TestSkillInterpreterCommand(t *testing.T) {
}
})
t.Run("shell scripts run with sh", func(t *testing.T) {
// Skills ship `#!/bin/bash` almost exclusively, and on Debian /bin/sh is
// dash: an array literal, `function f()`, a C-style for loop and process
// substitution are syntax errors there. Running these files with sh broke
// scripts that are perfectly valid, and made the install-time `sh -n` check
// refuse them on the way in.
t.Run("shell scripts prefer bash", func(t *testing.T) {
cmd, args := SkillInterpreterCommand(dir, dir+"/scripts/run.sh")
require.Equal(t, "/bin/sh", cmd)
require.Equal(t, []string{dir + "/scripts/run.sh"}, args)
require.Len(t, args, 3)
require.Equal(t, "-c", args[0])
require.Contains(t, args[1], "exec bash "+dir+"/scripts/run.sh")
require.Contains(t, args[1], "else", "there must be a fallback when bash is absent")
require.Equal(t, "weknora-skill", args[2])
})
t.Run("a shell script receives the caller's arguments", func(t *testing.T) {
if _, err := os.Stat("/bin/sh"); err != nil {
t.Skipf("shell is not available: %v", err)
}
scriptDir := t.TempDir()
script := filepath.Join(scriptDir, "echo-args.sh")
require.NoError(t, os.WriteFile(script, []byte("printf '%s\\n' \"$@\"\n"), 0o755))
cmd, baseArgs := SkillInterpreterCommand(scriptDir, script)
out, err := exec.Command(cmd, append(append([]string{}, baseArgs...),
"--first", "value")...).CombinedOutput()
require.NoError(t, err, string(out))
require.Equal(t, "--first\nvalue\n", string(out))
})
t.Run("unknown extension falls back to sh", func(t *testing.T) {