From faefceec15035c2eedc46ce465a93f535d832445 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Mon, 31 Aug 2026 11:42:47 +0800 Subject: [PATCH] fix(skills): verify installs by loadability and repair missing deps The gate was answering the wrong question (every .py as isolated __main__), and the installer derived dependencies from SKILL.md prose independently. Align import prefixes with Python's path rules, keep the gate as the only authority, and feed its findings back for one repair round. --- .../service/tenant_skill_install.go | 285 +++++++++++++--- .../service/tenant_skill_install_test.go | 242 +++++++++++-- .../service/tenant_skill_transcript.go | 92 +++-- .../service/tenant_skill_transcript_test.go | 53 +++ .../service/tenant_skill_verify.go | 223 ++++++++++-- .../service/tenant_skill_verify.py | 321 +++++++++++++----- .../tenant_skill_verify_python_test.go | 236 ++++++++++++- internal/sandbox/skill_paths.go | 12 +- internal/sandbox/skill_paths_test.go | 28 +- 9 files changed, 1260 insertions(+), 232 deletions(-) diff --git a/internal/application/service/tenant_skill_install.go b/internal/application/service/tenant_skill_install.go index 0d0bcfb0..420685fc 100644 --- a/internal/application/service/tenant_skill_install.go +++ b/internal/application/service/tenant_skill_install.go @@ -44,6 +44,17 @@ const ( // files. Writing each file with MakeDir+WriteFile is two round trips per // entry, which is why a 50-file skill crawled through "seeding 12/56". skillSeedArchivePath = sandbox.SkillsImageRoot + "/.weknora-seed.tar" + + // skillInstallVerifyRounds bounds the installer conversation. + // + // The first round works from SKILL.md. A second is driven by the gate's own + // findings, and exists because those are two different descriptions of what + // a skill needs: the official office toolkit imports defusedxml and lxml at + // module level while its SKILL.md names neither, so an installer working + // from prose alone cannot know to install them. One round of reconciliation + // closes that gap; more would only be a retry loop over an answer that is + // not going to change. + skillInstallVerifyRounds = 2 ) // InstallSkill validates an uploaded archive, records it, and kicks off the @@ -367,25 +378,16 @@ func (s *TenantSkillService) runInstall( } s.publishProgress(ctx, tenantID, configID, skillID, SkillProgress{Percent: 35, Stage: "seeded"}) - // 3. Let the installer agent install dependencies. - if err := s.driveInstallerAgent(ctx, tenantID, skillID, sess, transcript, prompt); err != nil { - return err - } - s.publishProgress(ctx, tenantID, configID, skillID, SkillProgress{Percent: 80, Stage: "agent_done"}) - - // 4. Hand the tree to the execution user BEFORE verifying it. The agent - // created these files as root, and the language passes below - // deliberately run as the ordinary user: verifying first would test - // permissions that never reach the image, and a restrictive root umask - // would fail a perfectly good install because the .venv interpreter was - // unreadable. - if err := s.normalizeSkillPermissions(ctx, mgr, sess.ID, skillDir); err != nil { - return err - } - - // 5. Verify ourselves. "The agent said it worked" is a sentence, not - // evidence, and this is the last gate before the image is switched. - if err := s.verifySkill(ctx, mgr, sess.ID, skillDir, bundle); err != nil { + // 3-5. Install dependencies and verify the result. "The agent said it + // worked" is a sentence, not evidence, and this is the last gate + // before the image is switched — but the gate and the agent must not + // answer "what does this skill need" separately, so a failure it can + // still fix goes back to the same agent before the run is failed. + if err := s.installDependenciesAndVerify(ctx, installerJob{ + tenantID: tenantID, configID: configID, skillID: skillID, + sess: sess, mgr: mgr, transcript: transcript, + prompt: prompt, skillDir: skillDir, bundle: bundle, + }); err != nil { return err } if err := s.writeManifestEntry(ctx, mgr, sess.ID, skillID, bundle); err != nil { @@ -690,19 +692,114 @@ func (s *TenantSkillService) beginInstallTranscript( return transcript, prompt } -// driveInstallerAgent runs one installer conversation. It calls the engine -// directly instead of going through sessionService.AgentQA, because AgentQA -// swallows engine failures (it emits an error event and returns nil) and we -// need a reliable signal before switching the image. -func (s *TenantSkillService) driveInstallerAgent( - ctx context.Context, tenantID uint64, skillID string, sess *types.Session, - transcript *installTranscript, prompt string, -) error { +// installerJob is everything the install-and-verify step needs. It is a struct +// because the step takes the union of what the agent side and the image side +// each need, and a nine-parameter call tells the reader nothing. +type installerJob struct { + tenantID uint64 + configID string + skillID string + sess *types.Session + mgr sandbox.Manager + transcript *installTranscript + prompt string + skillDir string + bundle *SkillBundle +} + +// installDependenciesAndVerify runs the installer conversation and the gate as +// one decision. +// +// They used to be two, and that is what let a working skill be rejected: the +// installer derived what to install from SKILL.md prose while the gate derived +// what must resolve from the imports every file executes. Those disagree on any +// skill whose library modules import something its documentation never names, +// and the install died with the answer already in hand. The gate is the only +// authority here, so its own findings become the next round's instruction, and +// nothing in this path derives that list a second time. +// +// A failure the gate marked unrepairable — a syntax error, a file the execution +// user cannot read — ends the run immediately. Another round cannot change it, +// and the bundle has to change instead. +func (s *TenantSkillService) installDependenciesAndVerify( + ctx context.Context, job installerJob, +) (err error) { + run, err := s.openInstallerRun(ctx, job.tenantID, job.sess, job.transcript) + if err != nil { + job.transcript.Finish(context.WithoutCancel(ctx), err) + return err + } + // Detached from cancellation: when the install lock's renewal fails the + // context dies, and the transcript of the run that just died is precisely + // what someone will want to read. + defer func() { job.transcript.Finish(context.WithoutCancel(ctx), err) }() + + prompt := job.prompt + for round := 1; ; round++ { + if err = run.round(ctx, prompt); err != nil { + return err + } + s.publishProgress(ctx, job.tenantID, job.configID, job.skillID, + SkillProgress{Percent: 80, Stage: "agent_done"}) + + // The tree is handed to the execution user BEFORE it is verified. The + // agent created these files as root, and the language passes + // deliberately run as the ordinary user: verifying first would test + // permissions that never reach the image, and a restrictive root umask + // would fail a perfectly good install because the .venv interpreter + // was unreadable. + if err = s.normalizeSkillPermissions(ctx, job.mgr, job.sess.ID, job.skillDir); err != nil { + return err + } + + notes, verifyErr := s.verifySkill(ctx, job.mgr, job.sess.ID, job.skillDir, job.bundle) + s.reportVerificationNotes(ctx, job, notes) + if verifyErr == nil { + return nil + } + + var gate *skillVerificationError + if round >= skillInstallVerifyRounds || !errors.As(verifyErr, &gate) || !gate.Repairable { + err = verifyErr + return err + } + + logger.Infof(ctx, "[skill] %s failed %s verification with %d fixable finding(s); "+ + "handing them back to the installer", job.skillID, gate.Language, len(gate.Problems)) + s.publishProgress(ctx, job.tenantID, job.configID, job.skillID, SkillProgress{ + Percent: 82, Stage: "repairing", + Log: fmt.Sprintf("%s verification found %d missing dependency/dependencies; "+ + "asking the installer to add them", gate.Language, len(gate.Problems)), + }) + if err = s.reopenSkillDirForRepair(ctx, job.mgr, job.sess.ID, job.skillDir); err != nil { + return err + } + prompt = buildRepairPrompt(job.skillDir, gate) + job.transcript.RecordPrompt(prompt) + } +} + +// installerRun is one installer conversation, held open across rounds. A repair +// has to reach the same root shell, in the same sandbox, and be readable in the +// same transcript as the install it is repairing. +type installerRun struct { + engine interfaces.AgentEngine + transcript *installTranscript + sessionID string +} + +// openInstallerRun builds the engine the conversation runs on. It calls the +// engine directly instead of going through sessionService.AgentQA, because +// AgentQA swallows engine failures (it emits an error event and returns nil) +// and we need a reliable signal before switching the image. +func (s *TenantSkillService) openInstallerRun( + ctx context.Context, tenantID uint64, sess *types.Session, transcript *installTranscript, +) (*installerRun, error) { if s.installerAgents == nil { - return errors.New("custom agent service is not configured") + return nil, errors.New("custom agent service is not configured") } if transcript == nil { - return errors.New("install transcript was not seeded") + return nil, errors.New("install transcript was not seeded") } // The record is tenant-writable: updateBuiltinAgent lets a tenant persist a // Config for any built-in ID, this one included. It is therefore read for @@ -710,42 +807,129 @@ func (s *TenantSkillService) driveInstallerAgent( // platform's own registry entry. record, err := s.installerAgents.GetAgentByID(ctx, types.BuiltinSkillInstallerID) if err != nil { - return fmt.Errorf("load installer agent: %w", err) + return nil, fmt.Errorf("load installer agent: %w", err) } agentConfig := installerAgentConfig(installerAgentDefaults(ctx, tenantID), sess.SandboxConfigID) chatModel, err := s.resolveInstallerModel(ctx, tenantID, record) if err != nil { - return err + return nil, err } engine, err := s.agents.CreateAgentEngine( ctx, agentConfig, chatModel, nil, transcript.bus, sess.ID, transcript.assistantMessageID, ) if err != nil { - runErr := fmt.Errorf("create installer engine: %w", err) - transcript.Finish(ctx, runErr) - return runErr + return nil, fmt.Errorf("create installer engine: %w", err) } + return &installerRun{ + engine: engine, transcript: transcript, sessionID: sess.ID, + }, nil +} - state, err := engine.Execute(ctx, sess.ID, transcript.assistantMessageID, prompt, nil) - runErr := err - if runErr == nil && (state == nil || !state.IsComplete) { - runErr = errors.New("installer agent stopped without completing") +// round runs one installer turn. +// +// No conversation history is replayed. The engine is stateless across turns, so +// a repair round could be handed the first round's transcript — but what the +// gate reported is both shorter and more exact than anything that could be +// inferred from it, and re-reading the round that already missed a dependency +// is not what makes the next one find it. +func (r *installerRun) round(ctx context.Context, prompt string) error { + state, err := r.engine.Execute(ctx, r.sessionID, r.transcript.assistantMessageID, prompt, nil) + if err != nil { + return fmt.Errorf("installer agent failed: %w", err) } - // Detached from cancellation: when the install lock's renewal fails the - // context dies, and the transcript of the run that just died is precisely - // what someone will want to read. - transcript.Finish(context.WithoutCancel(ctx), runErr) - if runErr != nil { - if err != nil { - return fmt.Errorf("installer agent failed: %w", err) - } - return runErr + if state == nil || !state.IsComplete { + return errors.New("installer agent stopped without completing") } return nil } +// reportVerificationNotes surfaces what the gate noticed but did not refuse: an +// auxiliary file that cannot load, a requirement whose environment marker +// cannot be evaluated here, an import that resolves only from a directory the +// skill ships no script in. None is evidence of a broken install, and all of +// them are what someone debugging this skill later will wish they had been +// told, so they are logged and streamed rather than discarded. +func (s *TenantSkillService) reportVerificationNotes( + ctx context.Context, job installerJob, notes []string, +) { + if len(notes) == 0 { + return + } + for _, note := range notes { + logger.Infof(ctx, "[skill] %s verification note: %s", job.skillID, note) + } + // One event, not one per note. Progress keeps only the latest value, so + // publishing them separately would leave a late subscriber holding whichever + // note happened to be last. + s.publishProgress(ctx, job.tenantID, job.configID, job.skillID, SkillProgress{ + Percent: 80, Stage: "verify_note", Log: strings.Join(notes, "\n"), + }) +} + +// reopenSkillDirForRepair undoes normalizeSkillPermissions for another round. +// +// Verification runs as the ordinary user, so the tree is locked to 555 and root +// before it; a repair then has to install into .venv again. Install commands +// run as root and would get away with writing through those modes, but relying +// on that makes the next reader work out whether it still holds — restoring the +// state the seed left is the same two commands and needs no such argument. +func (s *TenantSkillService) reopenSkillDirForRepair( + ctx context.Context, mgr sandbox.Manager, sessionID, skillDir string, +) error { + if err := guardSkillDir(skillDir); err != nil { + return err + } + dir := sandbox.ShellQuote(skillDir) + user := sandbox.DefaultSandboxExecUser + cmd := fmt.Sprintf("chown -R %s:%s %s && chmod -R u+rwX,go+rX %s", user, user, dir, dir) + if _, err := s.execInstall(ctx, mgr, sessionID, cmd); err != nil { + return fmt.Errorf("reopen skill directory %s for a repair round: %w", skillDir, err) + } + return nil +} + +// buildRepairPrompt turns the gate's own findings into the next round's brief. +// +// It carries no analysis of its own, deliberately. The reason a first round can +// fail on a working skill is that SKILL.md and the imports its files execute are +// two different descriptions of what the skill needs; a third description +// written here would reintroduce exactly that gap. The only thing added is the +// import-name-to-distribution mapping, which is a property of PyPI rather than +// of this skill. +func buildRepairPrompt(skillDir string, gate *skillVerificationError) string { + var findings strings.Builder + for _, problem := range gate.Problems { + findings.WriteString("- ") + findings.WriteString(problem) + findings.WriteString("\n") + } + return fmt.Sprintf(`Verification of the skill you just installed failed. Fix only this and stop. + +The %s check reported: +%s +Every line above is a dependency this image cannot resolve. The names were read +from the imports the skill's own files execute when they load, which is why some +of them appear nowhere in SKILL.md — install them anyway. + +- Python packages go into %s/.venv (`+"`uv pip install`"+`, or + %s/.venv/bin/python -m pip install). Node packages go under %s/node_modules. +- The name in each line is an IMPORT name; install the distribution that + provides it. Common cases where they differ: PIL -> pillow, yaml -> pyyaml, + docx -> python-docx, pptx -> python-pptx, cv2 -> opencv-python-headless, + bs4 -> beautifulsoup4, sklearn -> scikit-learn, fitz -> pymupdf. +- Do NOT edit, move or delete any of the skill's own files, and do NOT edit + SKILL.md or requirements.txt to make the check pass. The uploaded archive is + what read_skill serves, so a source edit here makes the installed skill differ + from what everyone else sees. +- If a package genuinely cannot be installed in this image, say so plainly in + your summary rather than working around it. + +The same verification runs again as soon as you finish. +`, gate.Language, findings.String(), skillDir, skillDir, skillDir) +} + // normalizeSkillPermissions makes the skill tree readable and executable by // the non-root execution user, and writable by nobody. Installs run as root, // so without the mode change the user that actually runs skills could not read @@ -1526,6 +1710,13 @@ venv. It never runs the skill's code, so nothing is expected to answer --help. Lazy imports and install_deps.py extras are invisible to that check — you still have to install them. +SKILL.md is not the dependency list. What the check resolves is the set of +top-level imports the skill's files actually execute, and a library module +routinely imports something the documentation never mentions. Read every .py in +the tree (grep '^import\|^from' over it) and install what those lines need, not +only what the Dependencies section names. If verification does fail on a missing +package, you get its exact findings back and one round to install them. + SKILL.md: %s `, skillDir, uvAvailable, skillDir, skillDir, skillDir, diff --git a/internal/application/service/tenant_skill_install_test.go b/internal/application/service/tenant_skill_install_test.go index 62b33b7e..f5f47097 100644 --- a/internal/application/service/tenant_skill_install_test.go +++ b/internal/application/service/tenant_skill_install_test.go @@ -1043,9 +1043,9 @@ func TestRunInstallRequiresVenvWhenRequirementsExist(t *testing.T) { require.Nil(t, fx.configRepo.saved) } -// Verification used to run one guessed "entry script" with --help. Nothing in -// the runtime designates an entry script — the model names whichever path it -// likes in execute_skill_script — so the guess left every other file unchecked +// Verification used to run one guessed "entry script" with --help. A skill is +// not one entry point: it is every file the tree ships, some run as scripts +// and some imported as packages. The guess left every other file unchecked // while failing installs over the one it happened to pick. func TestRunInstallVerifiesEveryScriptOfEveryLanguage(t *testing.T) { fx := newInstallFixture(t) @@ -1058,7 +1058,7 @@ func TestRunInstallVerifiesEveryScriptOfEveryLanguage(t *testing.T) { pythonPass := skillPythonVerifyCommand(installSkillDir, []string{ "scripts/__init__.py", "scripts/extract.py", "scripts/helper.py", - }) + }, nil) require.Contains(t, fx.commands, pythonPass, "every python file must be checked, not the first one in sort order") require.Contains(t, fx.commands, @@ -1083,7 +1083,7 @@ func TestRunInstallDoesNotExecuteAnyScriptToVerifyIt(t *testing.T) { } func TestSkillPythonVerifyCommandPrefersTheSkillVenv(t *testing.T) { - command := skillPythonVerifyCommand("/opt/skills/demo", []string{"scripts/a.py"}) + command := skillPythonVerifyCommand("/opt/skills/demo", []string{"scripts/a.py"}, nil) require.Contains(t, command, "if [ -x /opt/skills/demo/.venv/bin/python ]; "+ "then py=/opt/skills/demo/.venv/bin/python; else py=python3; fi", @@ -1095,11 +1095,68 @@ func TestSkillPythonVerifyCommandPrefersTheSkillVenv(t *testing.T) { // A skill may ship a file whose name needs quoting; the command is assembled // by hand, so the shell must never see it as more than one word. func TestSkillVerifyCommandsQuoteAwkwardPaths(t *testing.T) { - python := skillPythonVerifyCommand("/opt/skills/demo", []string{"scripts/a b'c.py"}) + python := skillPythonVerifyCommand("/opt/skills/demo", []string{"scripts/a b'c.py"}, nil) require.Contains(t, python, `'scripts/a b'\''c.py'`) shell := skillShellVerifyCommand("/opt/skills/demo", []string{"a b.sh"}) - require.Equal(t, `for f in '/opt/skills/demo/a b.sh'; do sh -n "$f" || exit 1; done`, shell) + require.Equal(t, + `if command -v bash >/dev/null 2>&1; then parser=bash; else parser=sh; fi; `+ + `for f in '/opt/skills/demo/a b.sh'; do "$parser" -n "$f" || exit 1; done`, + shell) +} + +// A skill's shell scripts carry a bash shebang almost exclusively, and +// SkillInterpreterCommand runs them with bash. Checking them with sh — dash on +// Debian — refused installs over array literals, `function f()`, C-style for +// loops and process substitution, none of which dash can parse. +func TestSkillShellVerifyCommandParsesWithBash(t *testing.T) { + command := skillShellVerifyCommand("/opt/skills/demo", []string{"bin/setup.sh"}) + + require.Contains(t, command, "parser=bash") + require.Contains(t, command, `"$parser" -n "$f"`) + require.Contains(t, command, "else parser=sh", + "an image without bash must still get a syntax check") + require.NotContains(t, command, `sh -n "$f"`, + "the check must not hard-code the shell that cannot parse these files") +} + +// The Python pass is the only one whose verdict depends on what the image +// carries, so it is the only one that can fail an install over a file nothing +// the skill offers ever loads. Those files are named separately. +func TestSkillPythonVerifyCommandSeparatesAuxiliaryFiles(t *testing.T) { + command := skillPythonVerifyCommand("/opt/skills/demo", + []string{"scripts/run.py"}, []string{"tests/test_run.py"}) + + require.True(t, strings.HasSuffix(command, + `- /opt/skills/demo scripts/run.py --optional tests/test_run.py`), + "got %q", command) +} + +// The split follows the conventions the language ecosystems already share. A +// bundled tests/ directory imports pytest and an examples/ script imports +// whatever it illustrates; neither is reachable from anything the skill +// exposes, and neither may decide whether the skill installs. +func TestSkillAuxiliaryScriptFollowsEcosystemConventions(t *testing.T) { + for _, rel := range []string{ + "tests/test_run.py", "test/helpers.py", "examples/demo.py", + "scripts/__tests__/a.py", "benchmarks/bench.py", "docs/conf.py", + "conftest.py", "setup.py", "scripts/extract_test.py", + "scripts/test_extract.py", + } { + require.True(t, skillAuxiliaryScript(rel), "%s must be auxiliary", rel) + } + for _, rel := range []string{ + "scripts/extract.py", "scripts/office/validators/base.py", + "recalc.py", "scripts/latest.py", "scripts/contest.py", + } { + require.False(t, skillAuxiliaryScript(rel), "%s is part of what the skill offers", rel) + } + + entry, auxiliary := splitAuxiliaryScripts([]string{ + "scripts/a.py", "tests/test_a.py", "scripts/b.py", + }) + require.Equal(t, []string{"scripts/a.py", "scripts/b.py"}, entry) + require.Equal(t, []string{"tests/test_a.py"}, auxiliary) } func TestSkillNodeVerifyCommandChecksDeclaredDependencies(t *testing.T) { @@ -1307,7 +1364,8 @@ func TestRunInstallKeepsOldImageWhenVerificationFails(t *testing.T) { fx.configRepo.entity.Config.SkillImage = &types.SkillImageConfig{ SnapshotID: "snap-old", Generation: 3, OwnerFingerprint: fx.fingerprint, } - fx.loadCheckExitCode = 1 + fx.loadCheckExitCodes = []int{1} + fx.loadCheckStderr = "scripts/extract.py has a syntax error on line 1: invalid syntax" err := fx.svc.runInstall(context.Background(), 7, "cfg-1", "sk-1", fx.bundle) @@ -1322,6 +1380,98 @@ func TestRunInstallKeepsOldImageWhenVerificationFails(t *testing.T) { require.NotEmpty(t, skill.Error) } +// The failure this closes: the installer derives what to install from SKILL.md +// prose, and the gate derives what must resolve from the imports every file +// executes. Those disagree on any skill whose library modules import something +// its documentation never names — the official office toolkit imports +// defusedxml and lxml at module level and mentions neither — so the install +// died with the answer already in hand, and a retry replayed the same prompt to +// the same effect. The gate's own lines are the repair round's brief. +func TestRunInstallHandsVerificationFindingsBackToTheInstaller(t *testing.T) { + fx := newInstallFixture(t) + fx.loadCheckExitCodes = []int{skillVerifyRepairableExit, 0} + fx.loadCheckStderr = "scripts/office/validators/base.py imports defusedxml, " + + "which is not available in this image\n" + + "scripts/recalc.py imports openpyxl, which is not available in this image" + + require.NoError(t, fx.svc.runInstall(context.Background(), 7, "cfg-1", "sk-1", fx.bundle)) + + require.Len(t, fx.agentPrompts, 2, "a fixable failure must reach the installer again") + repair := fx.agentPrompts[1] + require.Contains(t, repair, "imports defusedxml", + "the repair round is driven by the gate's own findings, not by a second guess") + require.Contains(t, repair, "imports openpyxl") + require.Contains(t, repair, installSkillDir+"/.venv", + "the round has to be told where the packages belong") + require.Contains(t, repair, "Do NOT edit", + "a repair must not be allowed to edit the tree into passing") + + require.Equal(t, 2, fx.loadCheckPasses, "the repair has to be verified, not trusted") + require.Contains(t, fx.events, "create-snapshot") + require.NotNil(t, fx.configRepo.saved, "a repaired install must reach the image pointer") + + skill, _ := fx.skillRepo.GetSkill(context.Background(), 7, "cfg-1", "sk-1") + require.Equal(t, types.SkillStatusReady, skill.Status) + require.Empty(t, skill.Error) +} + +// A repair round has to be able to write into .venv again, which the +// verification pass just locked down: the checks run as the ordinary user, so +// the tree is handed over at 555 and root-owned before every one of them. +func TestRunInstallReopensTheTreeBeforeARepairRound(t *testing.T) { + fx := newInstallFixture(t) + fx.loadCheckExitCodes = []int{skillVerifyRepairableExit, 0} + + require.NoError(t, fx.svc.runInstall(context.Background(), 7, "cfg-1", "sk-1", fx.bundle)) + + reopen := indexOfCommandContaining(fx.commands, "chmod -R u+rwX,go+rX "+installSkillDir) + require.GreaterOrEqual(t, reopen, 0, "the tree must be writable again for the repair") + + firstLock := indexOfCommandContaining(fx.commands, "chmod -R 555 "+installSkillDir) + require.GreaterOrEqual(t, firstLock, 0) + require.Greater(t, reopen, firstLock, + "the tree is reopened after the pass that locked it, not before") + + // And locked again for the pass that checks the repair, so the final image + // carries the modes every verification ran against. + require.Greater(t, lastIndexOfCommandContaining(fx.commands, "chmod -R 555 "+installSkillDir), + reopen, "the repaired tree must be locked down again before it is verified") +} + +// Installing a package cannot fix a file that does not parse, and the bundle +// has to change instead. Spending another agent round on it would only delay +// the same failure by minutes. +func TestRunInstallDoesNotRetryAFailureInstallingCannotFix(t *testing.T) { + fx := newInstallFixture(t) + fx.loadCheckExitCodes = []int{1} + fx.loadCheckStderr = "scripts/extract.py has a syntax error on line 1: invalid syntax" + + err := fx.svc.runInstall(context.Background(), 7, "cfg-1", "sk-1", fx.bundle) + + require.Error(t, err) + require.ErrorContains(t, err, "syntax error") + require.Len(t, fx.agentPrompts, 1, "an unfixable finding must not buy another round") + require.Equal(t, 1, fx.loadCheckPasses) +} + +// The loop is bounded. An installer that cannot satisfy the gate in one repair +// is not going to satisfy it in ten, and a skill install must not become an +// open-ended retry against a billed sandbox. +func TestRunInstallStopsAfterOneRepairRound(t *testing.T) { + fx := newInstallFixture(t) + fx.loadCheckExitCodes = []int{skillVerifyRepairableExit} + + err := fx.svc.runInstall(context.Background(), 7, "cfg-1", "sk-1", fx.bundle) + + require.Error(t, err) + require.ErrorContains(t, err, "imports pandas", + "the failure must name what the gate found, not just that it failed") + require.Len(t, fx.agentPrompts, skillInstallVerifyRounds) + require.Equal(t, skillInstallVerifyRounds, fx.loadCheckPasses) + require.NotContains(t, fx.events, "create-snapshot") + require.Nil(t, fx.configRepo.saved) +} + func TestRunInstallDeletesTheSnapshotWhenSwitchFails(t *testing.T) { fx := newInstallFixture(t) fx.configRepo.updateErr = errUpdateBoom @@ -1549,7 +1699,7 @@ const ( // reason; the properties worth stating verbatim are asserted separately in // TestSkillPythonVerifyCommand*. var installPythonVerifyCommand = skillPythonVerifyCommand( - installSkillDir, []string{"scripts/extract.py"}, + installSkillDir, []string{"scripts/extract.py"}, nil, ) func indexOfEvent(events []string, needle string) int { @@ -1561,6 +1711,27 @@ func indexOfEvent(events []string, needle string) int { return -1 } +// indexOfCommandContaining and its Last variant locate one shell command in the +// ordered transcript. A repair round issues the same commands twice, so the +// tests that care about ordering need both ends. +func indexOfCommandContaining(commands []string, needle string) int { + for i, command := range commands { + if strings.Contains(command, needle) { + return i + } + } + return -1 +} + +func lastIndexOfCommandContaining(commands []string, needle string) int { + for i := len(commands) - 1; i >= 0; i-- { + if strings.Contains(commands[i], needle) { + return i + } + } + return -1 +} + // staleMark is one request to mark a config's bound sandboxes stale. The // tenant is part of it because marking the right config of the wrong workspace // would rebuild sandboxes that never carried this image. @@ -1584,10 +1755,17 @@ type installFixture struct { events []string commands []string fingerprint string - // loadCheck* drive the per-language script verification pass, which is - // the last gate before the snapshot. - loadCheckExitCode int - loadCheckResult *sandbox.ExecuteResult + // loadCheck* drive the per-language script verification pass, which is the + // last gate before the snapshot. exitCodes is consumed one entry per python + // pass and its last entry repeats, so a test about a single round writes one + // value and a test about the repair round writes two. Exit 2 is the + // checker's "everything I found is a missing dependency", which is what + // earns another installer round. + loadCheckExitCodes []int + loadCheckStdout string + loadCheckStderr string + loadCheckPasses int + loadCheckResult *sandbox.ExecuteResult // depsExitCode fails the declared-dependency check (venv / node_modules). depsExitCode int // execResult is scoped to execResultCommand: an unscoped stub result @@ -1597,6 +1775,10 @@ type installFixture struct { execResult *sandbox.ExecuteResult loadCheckRanAsRoot bool agentErr error + // agentPrompts is every prompt the installer engine was handed, in order. + // A repair round is a second entry, and what it says is the whole point of + // having one. + agentPrompts []string // beforeExecute runs at the moment the engine would start, so a test can // observe the state an attaching console would see mid-install. beforeExecute func() @@ -1657,6 +1839,10 @@ func newInstallFixture(t *testing.T) *installFixture { fx := &installFixture{t: t} fx.fingerprint = sandbox.SkillImageFingerprint("e2b", "key-1", "https://e2b.example") + // The checker's own wording for a missing dependency, so a test reading the + // repair prompt sees what a real run would put in it. + fx.loadCheckStderr = "scripts/extract.py imports pandas, " + + "which is not available in this image" fx.bundle = &SkillBundle{ Name: "pdf-tools", Version: "1.0.0", @@ -1721,6 +1907,19 @@ func (f *installFixture) record(event string) { f.events = append(f.events, event) } +// nextLoadCheckExit returns the code for this verification pass, repeating the +// last configured entry once they run out. +func (f *installFixture) nextLoadCheckExit() int { + f.loadCheckPasses++ + if len(f.loadCheckExitCodes) == 0 { + return 0 + } + if f.loadCheckPasses > len(f.loadCheckExitCodes) { + return f.loadCheckExitCodes[len(f.loadCheckExitCodes)-1] + } + return f.loadCheckExitCodes[f.loadCheckPasses-1] +} + // now is the fixture's clock, so a test can express "one heartbeat ago" // against the same instant the service reads. func (f *installFixture) now() time.Time { return f.svc.now() } @@ -2558,7 +2757,9 @@ func (m *installSandboxManager) ExecShellCommandWithOptions( return m.fx.loadCheckResult, nil } return &sandbox.ExecuteResult{ - ExitCode: m.fx.loadCheckExitCode, Stderr: "scripts/extract.py imports pandas", + ExitCode: m.fx.nextLoadCheckExit(), + Stdout: m.fx.loadCheckStdout, + Stderr: m.fx.loadCheckStderr, }, nil case command == removeSkillDirCommand: m.fx.record("remove-skill-dir") @@ -2741,16 +2942,17 @@ type installAgentEngine struct { } func (e *installAgentEngine) Execute( - context.Context, - string, - string, - string, - []chat.Message, - ...[]string, + _ context.Context, + _ string, + _ string, + prompt string, + _ []chat.Message, + _ ...[]string, ) (*types.AgentState, error) { if e.fx.beforeExecute != nil { e.fx.beforeExecute() } + e.fx.agentPrompts = append(e.fx.agentPrompts, prompt) e.fx.record("agent-execute") if e.fx.agentDelay > 0 { time.Sleep(e.fx.agentDelay) diff --git a/internal/application/service/tenant_skill_transcript.go b/internal/application/service/tenant_skill_transcript.go index 883a6063..ed34d02a 100644 --- a/internal/application/service/tenant_skill_transcript.go +++ b/internal/application/service/tenant_skill_transcript.go @@ -38,11 +38,17 @@ type installTranscript struct { sessionID string assistantMessageID string - mu sync.Mutex - message *types.Message - answers []*installAnswerSegment - starts map[string]time.Time - finished bool + mu sync.Mutex + message *types.Message + answers []*installAnswerSegment + starts map[string]time.Time + // closed guards the terminal event and the final write, both of which say + // "this install is over" and must be said once. + closed bool + // Totals accumulated across the run's engine turns, reported on the one + // terminal event Finish emits. + totalSteps int + totalDurationMs int64 } // installAnswerSegment accumulates the prose streamed under one final-answer @@ -142,13 +148,20 @@ func (tr *installTranscript) Subscribe() { tr.bus.On(event.EventAgentComplete, tr.onComplete) } -// Finish closes the record. runErr is the engine's verdict: the engine emits -// no complete event when it fails, and a failed install is the one people -// actually come to read, so the failure is written here rather than hoped for. +// Finish closes the record. runErr is the install's verdict, not just the +// engine's: verification runs after the agent stops, and a failure there is the +// one people actually come to read, so it is written here rather than hoped for. func (tr *installTranscript) Finish(ctx context.Context, runErr error) { if tr == nil { return } + tr.mu.Lock() + alreadyClosed := tr.closed + tr.closed = true + tr.mu.Unlock() + if alreadyClosed { + return + } if runErr != nil { tr.append(interfaces.StreamEvent{ ID: uuid.NewString(), @@ -173,22 +186,41 @@ func (tr *installTranscript) Finish(ctx context.Context, runErr error) { } tr.mu.Lock() - alreadyComplete := tr.finished - tr.finished = true + steps, duration := tr.totalSteps, tr.totalDurationMs tr.mu.Unlock() - - if !alreadyComplete { - tr.append(interfaces.StreamEvent{ - ID: uuid.NewString(), - Type: types.ResponseTypeComplete, - Done: true, - Timestamp: time.Now(), - Data: map[string]interface{}{}, - }) - } + tr.append(interfaces.StreamEvent{ + ID: uuid.NewString(), + Type: types.ResponseTypeComplete, + Done: true, + Timestamp: time.Now(), + Data: map[string]interface{}{ + "total_steps": steps, + "total_duration_ms": duration, + }, + }) tr.save(ctx) } +// RecordPrompt logs a follow-up instruction the installer was given mid-run. +// +// The transcript exists so one replay of the event log is the whole +// conversation. A repair round whose instruction is missing reads as the agent +// spontaneously deciding to install more packages, which is precisely the +// moment someone is reading this to find out why. +func (tr *installTranscript) RecordPrompt(prompt string) { + if tr == nil { + return + } + tr.append(interfaces.StreamEvent{ + ID: uuid.NewString(), + Type: types.ResponseTypeInstallPrompt, + Content: prompt, + Done: true, + Timestamp: time.Now(), + Data: map[string]interface{}{}, + }) +} + func (tr *installTranscript) onThought(_ context.Context, evt event.Event) error { data, ok := evt.Data.(event.AgentThoughtData) if !ok { @@ -326,13 +358,16 @@ func (tr *installTranscript) onError(_ context.Context, evt event.Event) error { return nil } +// onComplete records what the round finished with. It deliberately emits no +// terminal event: an install may run another installer round after +// verification, and a console that saw "complete" would stop following before +// that round began. Only Finish closes the stream. func (tr *installTranscript) onComplete(_ context.Context, evt event.Event) error { data, ok := evt.Data.(event.AgentCompleteData) if !ok { return nil } tr.mu.Lock() - tr.finished = true if data.MessageID == tr.assistantMessageID { msg := tr.ensureMessageLocked() msg.IsCompleted = true @@ -347,18 +382,11 @@ func (tr *installTranscript) onComplete(_ context.Context, evt event.Event) erro if tr.composeAnswerLocked() == "" && data.FinalAnswer != "" { tr.segmentLocked(evt.ID).content = data.FinalAnswer } + // Summed rather than overwritten: an install that needed a repair round ran + // two engine turns, and its cost is both of them. + tr.totalSteps += data.TotalSteps + tr.totalDurationMs += data.TotalDurationMs tr.mu.Unlock() - - tr.append(interfaces.StreamEvent{ - ID: evt.ID, - Type: types.ResponseTypeComplete, - Done: true, - Timestamp: time.Now(), - Data: map[string]interface{}{ - "total_steps": data.TotalSteps, - "total_duration_ms": data.TotalDurationMs, - }, - }) return nil } diff --git a/internal/application/service/tenant_skill_transcript_test.go b/internal/application/service/tenant_skill_transcript_test.go index 32796001..ddce6858 100644 --- a/internal/application/service/tenant_skill_transcript_test.go +++ b/internal/application/service/tenant_skill_transcript_test.go @@ -265,3 +265,56 @@ func TestInstallTranscriptCreateOpensTheLogWithThePrompt(t *testing.T) { require.Equal(t, []types.ResponseType{types.ResponseTypeInstallPrompt}, streams.types()) require.Equal(t, "install pdf-tools", streams.events[0].Content) } + +// An install may run a second installer round, because verification happens +// after the agent stops and a dependency it names is something the agent can +// still fix. The engine reports "complete" at the end of every turn, so the +// transcript must not treat the first one as the end of the install — a console +// that saw it would stop following before the repair round began. +func TestInstallTranscriptStaysOpenAcrossInstallerRounds(t *testing.T) { + tr, bus, streams, messages := newTranscriptForTest(t) + ctx := context.Background() + + finishRound := func(steps int, ms int64) { + require.NoError(t, bus.Emit(ctx, event.Event{ + ID: "done", Type: event.EventAgentComplete, + Data: event.AgentCompleteData{ + MessageID: "msg-1", TotalSteps: steps, TotalDurationMs: ms, + }, + })) + } + + finishRound(3, 1000) + tr.RecordPrompt("Verification failed. Install defusedxml into the venv.") + finishRound(2, 500) + tr.Finish(ctx, nil) + + kinds := streams.types() + require.Equal(t, []types.ResponseType{ + types.ResponseTypeInstallPrompt, + types.ResponseTypeComplete, + }, kinds, "the repair instruction has to be in the log, and the install ends once") + require.Contains(t, streams.events[0].Content, "Install defusedxml") + + terminal := streams.events[len(streams.events)-1] + require.EqualValues(t, 5, terminal.Data["total_steps"], + "an install that needed a repair round cost both turns") + require.EqualValues(t, 1500, terminal.Data["total_duration_ms"]) + require.Len(t, messages.updated, 1) +} + +// Finish is deferred by the install path and also called on the error path that +// gives up before that defer is armed. Saying "this install is over" twice would +// duplicate the verdict in the record people read. +func TestInstallTranscriptFinishIsIdempotent(t *testing.T) { + tr, _, streams, messages := newTranscriptForTest(t) + + tr.Finish(context.Background(), errors.New("python verification failed: boom")) + tr.Finish(context.Background(), errors.New("python verification failed: boom")) + + require.Equal(t, []types.ResponseType{ + types.ResponseTypeError, + types.ResponseTypeComplete, + }, streams.types()) + require.Len(t, messages.updated, 1) +} diff --git a/internal/application/service/tenant_skill_verify.go b/internal/application/service/tenant_skill_verify.go index 89df23ef..1e63645f 100644 --- a/internal/application/service/tenant_skill_verify.go +++ b/internal/application/service/tenant_skill_verify.go @@ -20,28 +20,66 @@ import ( //go:embed tenant_skill_verify.py var skillPythonVerifier string +// skillVerifyRepairableExit is the exit code a verification pass uses when +// every problem it found is a dependency missing from this image. It separates +// "another installer round can fix this" from "the bundle has to change", which +// is the only distinction that decides what the install flow does next. +const skillVerifyRepairableExit = 2 + +// skillVerifyNotePrefix marks a line the checker reports without refusing the +// install. Notes travel on stdout so a non-zero exit stays unambiguous. +const skillVerifyNotePrefix = "note: " + +// skillVerificationError is what the gate said, kept structured because it is +// also the brief for a repair round. The gate is the only authority on what has +// to resolve in this image, so handing back its own lines is what keeps the +// installer from deriving "what this skill needs" a second time. +type skillVerificationError struct { + // Language names the pass that failed, as the operator sees it. + Language string + // Repairable is true when installing a package would satisfy every line. + Repairable bool + // Problems are the checker's own lines, one per finding. + Problems []string + // Summary describes the command result itself, and is the only thing worth + // printing when the pass died without producing lines at all. + Summary string +} + +func (e *skillVerificationError) Error() string { + if len(e.Problems) == 0 { + return fmt.Sprintf("%s verification failed (%s)", e.Language, e.Summary) + } + return fmt.Sprintf("%s verification failed: %s", e.Language, strings.Join(e.Problems, "; ")) +} + // verifySkill is the server's own check, and the last gate before the image // pointer moves: a broken install must leave the previous snapshot serving. // -// It deliberately proves loadability rather than behaviour. Nothing in the -// runtime designates an entry point — the model names any script it likes in -// execute_skill_script — so there is no single command whose success would -// stand in for the skill working. What every one of those calls does need is -// that the interpreter can parse the file and resolve its imports inside this -// image, and that is checkable for all scripts at once without executing a -// line of the skill's own code. +// It proves loadability, not behaviour, and not "every file can be used as +// python this_file.py". A skill ships scripts and library packages together; +// execute_skill_script names files, but those files are also imported by each +// other. The Python pass resolves each file's imports against the prefixes +// Python itself would use — the file's own directory and every ancestor up to +// the skill root — plus the venv. Guessing a single entry script, or treating +// every .py as an isolated __main__, both fail real skills. +// +// Findings are graded rather than uniformly fatal. Refusing an install costs +// the minutes of dependency work that already succeeded, so only evidence that +// the install itself is broken may do it: a file nothing the skill offers ever +// loads, or a requirement pip would have skipped, is returned as a note. // // Not executing is the point, not a limitation. The previous implementation // ran one guessed script with --help; skills that do not parse arguments // simply ran their whole main path inside the tree about to be snapshotted. func (s *TenantSkillService) verifySkill( ctx context.Context, mgr sandbox.Manager, sessionID, skillDir string, bundle *SkillBundle, -) error { +) ([]string, error) { if err := s.verifySkillTree(ctx, mgr, sessionID, skillDir, bundle); err != nil { - return err + return nil, err } if err := s.verifyDeclaredDependencies(ctx, mgr, sessionID, skillDir, bundle); err != nil { - return err + return nil, err } return s.verifyScriptsLoad(ctx, mgr, sessionID, skillDir, bundle) } @@ -93,30 +131,43 @@ func (s *TenantSkillService) verifyDeclaredDependencies( } // verifyScriptsLoad runs one pass per language present in the bundle. Each -// pass covers every script of that language, because every one of them is a -// script the model may name at runtime. +// pass covers every file of that language: syntax and imports are properties +// of the file, not of a guessed entry point. +// +// Only the Python pass takes the auxiliary split. Its findings depend on what +// the image carries, so a bundled tests/ directory can fail an install that +// works; `node --check` and `bash -n` only state whether a file parses, which +// no amount of installing changes either way. func (s *TenantSkillService) verifyScriptsLoad( ctx context.Context, mgr sandbox.Manager, sessionID, skillDir string, bundle *SkillBundle, -) error { +) ([]string, error) { + var notes []string if scripts := sortedScriptPaths(bundle, ".py"); len(scripts) > 0 { - if err := s.execVerify(ctx, mgr, sessionID, skillDir, "python", - skillPythonVerifyCommand(skillDir, scripts)); err != nil { - return err + entry, auxiliary := splitAuxiliaryScripts(scripts) + found, err := s.execVerify(ctx, mgr, sessionID, skillDir, "python", + skillPythonVerifyCommand(skillDir, entry, auxiliary)) + notes = append(notes, found...) + if err != nil { + return notes, err } } if scripts := sortedScriptPaths(bundle, ".js", ".mjs", ".cjs"); len(scripts) > 0 { - if err := s.execVerify(ctx, mgr, sessionID, skillDir, "node", - skillNodeVerifyCommand(skillDir, scripts, nodeDependencyNames(bundle))); err != nil { - return err + found, err := s.execVerify(ctx, mgr, sessionID, skillDir, "node", + skillNodeVerifyCommand(skillDir, scripts, nodeDependencyNames(bundle))) + notes = append(notes, found...) + if err != nil { + return notes, err } } if scripts := sortedScriptPaths(bundle, ".sh"); len(scripts) > 0 { - if err := s.execVerify(ctx, mgr, sessionID, skillDir, "shell", - skillShellVerifyCommand(skillDir, scripts)); err != nil { - return err + found, err := s.execVerify(ctx, mgr, sessionID, skillDir, "shell", + skillShellVerifyCommand(skillDir, scripts)) + notes = append(notes, found...) + if err != nil { + return notes, err } } - return nil + return notes, nil } // execVerify runs one verification pass as the ordinary sandbox user, with the @@ -124,10 +175,10 @@ func (s *TenantSkillService) verifyScriptsLoad( // install-mode root would test permissions that never reach a session. func (s *TenantSkillService) execVerify( ctx context.Context, mgr sandbox.Manager, sessionID, skillDir, label, command string, -) error { +) ([]string, error) { executor, err := installExecutor(mgr) if err != nil { - return err + return nil, err } res, err := executor.ExecShellCommandWithOptions(ctx, sessionID, command, sandbox.ShellExecOptions{ @@ -139,12 +190,46 @@ func (s *TenantSkillService) execVerify( }, }) if err != nil { - return fmt.Errorf("%s verification: %w", label, err) + return nil, fmt.Errorf("%s verification: %w", label, err) } + // Notes are read even from a failed pass: an install refused for a missing + // package may also have noticed something about a file it did not refuse + // over, and that is exactly the context whoever reads the failure needs. + notes := verificationNotes(res.Stdout) if res.ExitCode != 0 { - return fmt.Errorf("%s verification failed (%s)", label, describeExecFailure(res)) + return notes, &skillVerificationError{ + Language: label, + Repairable: res.ExitCode == skillVerifyRepairableExit, + Problems: verificationProblems(res.Stderr), + Summary: describeExecFailure(res), + } } - return nil + return notes, nil +} + +// verificationNotes pulls the reported-but-not-refused findings out of stdout. +func verificationNotes(stdout string) []string { + var notes []string + for _, line := range strings.Split(stdout, "\n") { + line = strings.TrimSpace(line) + if after, ok := strings.CutPrefix(line, skillVerifyNotePrefix); ok { + notes = append(notes, after) + } + } + return notes +} + +// verificationProblems splits a failed pass's stderr into its findings. Blank +// lines are dropped; everything else is the checker's own wording, which is +// what a repair round is given. +func verificationProblems(stderr string) []string { + var problems []string + for _, line := range strings.Split(stderr, "\n") { + if line = strings.TrimSpace(line); line != "" { + problems = append(problems, line) + } + } + return problems } // skillPythonVerifyCommand pipes the verifier into the same interpreter a @@ -152,13 +237,23 @@ func (s *TenantSkillService) execVerify( // line. The list is explicit rather than a directory walk so the pass covers // exactly the uploaded sources — never .venv, node_modules or anything else // the agent created while installing. -func skillPythonVerifyCommand(skillDir string, scripts []string) string { +// +// Auxiliary files are named after --optional. They are checked the same way, +// and what is found in them is reported rather than allowed to refuse the +// install. +func skillPythonVerifyCommand(skillDir string, entry, auxiliary []string) string { venv := path.Join(skillDir, ".venv", "bin", "python") quotedVenv := sandbox.ShellQuote(venv) args := []string{sandbox.ShellQuote(skillDir)} - for _, rel := range scripts { + for _, rel := range entry { args = append(args, sandbox.ShellQuote(rel)) } + if len(auxiliary) > 0 { + args = append(args, skillVerifyOptionalFlag) + for _, rel := range auxiliary { + args = append(args, sandbox.ShellQuote(rel)) + } + } // The script arrives base64-encoded on stdin: it contains quotes, newlines // and backslashes, and a heredoc would still be at the mercy of whatever // the remote exec layer does with multi-line command strings. @@ -173,6 +268,9 @@ func skillPythonVerifyCommand(skillDir string, scripts []string) string { // skillNodeVerifyCommand parses every JavaScript file and, when the bundle // declares runtime dependencies, checks each one resolves. `node --check` is // parse-only: it never runs the module body. +// +// A declared dependency that is missing exits with the repairable code: it is +// the one thing here another installer round can still put in place. func skillNodeVerifyCommand(skillDir string, scripts, deps []string) string { parts := make([]string, 0, 2) if len(deps) > 0 { @@ -182,18 +280,75 @@ func skillNodeVerifyCommand(skillDir string, scripts, deps []string) string { } parts = append(parts, fmt.Sprintf( "for d in %s; do [ -e %s/\"$d\" ] || "+ - "{ echo \"package.json declares $d but node_modules/$d is missing\" >&2; exit 1; }; done", + "{ echo \"package.json declares $d but node_modules/$d is missing\" >&2; exit %d; }; done", strings.Join(quoted, " "), sandbox.ShellQuote(path.Join(skillDir, "node_modules")), + skillVerifyRepairableExit, )) } parts = append(parts, forEachScript(skillDir, scripts, "node --check \"$f\"")) return strings.Join(parts, "; ") } -// skillShellVerifyCommand parses every shell script. `sh -n` reads the file -// and exits without executing any of it. +// skillShellVerifyCommand parses every shell script with the shell that would +// run it. `-n` reads the file and exits without executing any of it. +// +// bash, not sh: skills ship `#!/bin/bash` almost exclusively, and +// SkillInterpreterCommand runs them with bash for that reason. /bin/sh is dash +// on Debian, where an array literal, `function f()`, a C-style for loop and +// process substitution are all outright syntax errors — so checking with sh +// would refuse installs over scripts that run perfectly. func skillShellVerifyCommand(skillDir string, scripts []string) string { - return forEachScript(skillDir, scripts, "sh -n \"$f\"") + return "if command -v bash >/dev/null 2>&1; then parser=bash; else parser=sh; fi; " + + forEachScript(skillDir, scripts, `"$parser" -n "$f"`) +} + +// skillVerifyOptionalFlag separates the files whose failure refuses the install +// from the ones whose failure is only reported. It is the checker's own argv +// contract; both sides must keep using this spelling. +const skillVerifyOptionalFlag = "--optional" + +// auxiliaryScriptDirs are the directory names Python and JavaScript projects +// universally use for files that ship with a project without being part of +// what it offers. A skill's tests import pytest; its examples import whatever +// they illustrate. Nothing the skill exposes loads either, so neither may +// decide whether the skill installs. +var auxiliaryScriptDirs = map[string]struct{}{ + "test": {}, "tests": {}, "testing": {}, "__tests__": {}, + "example": {}, "examples": {}, "sample": {}, "samples": {}, + "benchmark": {}, "benchmarks": {}, "fixtures": {}, + "doc": {}, "docs": {}, +} + +// skillAuxiliaryScript reports whether a bundled path is one of those files. +// +// The rule is the naming convention the language communities already share, +// not a list grown one skill at a time: a directory the ecosystem reserves for +// tests, examples or docs, or a file name pytest and setuptools already treat +// as theirs. +func skillAuxiliaryScript(rel string) bool { + segments := strings.Split(path.Clean(rel), "/") + for _, segment := range segments[:len(segments)-1] { + if _, ok := auxiliaryScriptDirs[strings.ToLower(segment)]; ok { + return true + } + } + base := strings.ToLower(segments[len(segments)-1]) + stem := strings.TrimSuffix(base, path.Ext(base)) + return stem == "conftest" || stem == "setup" || + strings.HasPrefix(stem, "test_") || strings.HasSuffix(stem, "_test") +} + +// splitAuxiliaryScripts separates the files whose failure is a failed install +// from the ones whose failure is a note, preserving the caller's order. +func splitAuxiliaryScripts(scripts []string) (entry, auxiliary []string) { + for _, rel := range scripts { + if skillAuxiliaryScript(rel) { + auxiliary = append(auxiliary, rel) + continue + } + entry = append(entry, rel) + } + return entry, auxiliary } func forEachScript(skillDir string, scripts []string, check string) string { diff --git a/internal/application/service/tenant_skill_verify.py b/internal/application/service/tenant_skill_verify.py index 0ae0ae7b..78d6264d 100644 --- a/internal/application/service/tenant_skill_verify.py +++ b/internal/application/service/tenant_skill_verify.py @@ -3,14 +3,36 @@ Run inside the sandbox, by the interpreter a real skill call would use, as the ordinary execution user, after the tree has been given its final permissions: - python3 - [ ...] + python3 -