diff --git a/internal/application/service/tenant_skill_install.go b/internal/application/service/tenant_skill_install.go index 0d0bcfb0..420685fc 100644 --- a/internal/application/service/tenant_skill_install.go +++ b/internal/application/service/tenant_skill_install.go @@ -44,6 +44,17 @@ const ( // files. Writing each file with MakeDir+WriteFile is two round trips per // entry, which is why a 50-file skill crawled through "seeding 12/56". skillSeedArchivePath = sandbox.SkillsImageRoot + "/.weknora-seed.tar" + + // skillInstallVerifyRounds bounds the installer conversation. + // + // The first round works from SKILL.md. A second is driven by the gate's own + // findings, and exists because those are two different descriptions of what + // a skill needs: the official office toolkit imports defusedxml and lxml at + // module level while its SKILL.md names neither, so an installer working + // from prose alone cannot know to install them. One round of reconciliation + // closes that gap; more would only be a retry loop over an answer that is + // not going to change. + skillInstallVerifyRounds = 2 ) // InstallSkill validates an uploaded archive, records it, and kicks off the @@ -367,25 +378,16 @@ func (s *TenantSkillService) runInstall( } s.publishProgress(ctx, tenantID, configID, skillID, SkillProgress{Percent: 35, Stage: "seeded"}) - // 3. Let the installer agent install dependencies. - if err := s.driveInstallerAgent(ctx, tenantID, skillID, sess, transcript, prompt); err != nil { - return err - } - s.publishProgress(ctx, tenantID, configID, skillID, SkillProgress{Percent: 80, Stage: "agent_done"}) - - // 4. Hand the tree to the execution user BEFORE verifying it. The agent - // created these files as root, and the language passes below - // deliberately run as the ordinary user: verifying first would test - // permissions that never reach the image, and a restrictive root umask - // would fail a perfectly good install because the .venv interpreter was - // unreadable. - if err := s.normalizeSkillPermissions(ctx, mgr, sess.ID, skillDir); err != nil { - return err - } - - // 5. Verify ourselves. "The agent said it worked" is a sentence, not - // evidence, and this is the last gate before the image is switched. - if err := s.verifySkill(ctx, mgr, sess.ID, skillDir, bundle); err != nil { + // 3-5. Install dependencies and verify the result. "The agent said it + // worked" is a sentence, not evidence, and this is the last gate + // before the image is switched — but the gate and the agent must not + // answer "what does this skill need" separately, so a failure it can + // still fix goes back to the same agent before the run is failed. + if err := s.installDependenciesAndVerify(ctx, installerJob{ + tenantID: tenantID, configID: configID, skillID: skillID, + sess: sess, mgr: mgr, transcript: transcript, + prompt: prompt, skillDir: skillDir, bundle: bundle, + }); err != nil { return err } if err := s.writeManifestEntry(ctx, mgr, sess.ID, skillID, bundle); err != nil { @@ -690,19 +692,114 @@ func (s *TenantSkillService) beginInstallTranscript( return transcript, prompt } -// driveInstallerAgent runs one installer conversation. It calls the engine -// directly instead of going through sessionService.AgentQA, because AgentQA -// swallows engine failures (it emits an error event and returns nil) and we -// need a reliable signal before switching the image. -func (s *TenantSkillService) driveInstallerAgent( - ctx context.Context, tenantID uint64, skillID string, sess *types.Session, - transcript *installTranscript, prompt string, -) error { +// installerJob is everything the install-and-verify step needs. It is a struct +// because the step takes the union of what the agent side and the image side +// each need, and a nine-parameter call tells the reader nothing. +type installerJob struct { + tenantID uint64 + configID string + skillID string + sess *types.Session + mgr sandbox.Manager + transcript *installTranscript + prompt string + skillDir string + bundle *SkillBundle +} + +// installDependenciesAndVerify runs the installer conversation and the gate as +// one decision. +// +// They used to be two, and that is what let a working skill be rejected: the +// installer derived what to install from SKILL.md prose while the gate derived +// what must resolve from the imports every file executes. Those disagree on any +// skill whose library modules import something its documentation never names, +// and the install died with the answer already in hand. The gate is the only +// authority here, so its own findings become the next round's instruction, and +// nothing in this path derives that list a second time. +// +// A failure the gate marked unrepairable — a syntax error, a file the execution +// user cannot read — ends the run immediately. Another round cannot change it, +// and the bundle has to change instead. +func (s *TenantSkillService) installDependenciesAndVerify( + ctx context.Context, job installerJob, +) (err error) { + run, err := s.openInstallerRun(ctx, job.tenantID, job.sess, job.transcript) + if err != nil { + job.transcript.Finish(context.WithoutCancel(ctx), err) + return err + } + // Detached from cancellation: when the install lock's renewal fails the + // context dies, and the transcript of the run that just died is precisely + // what someone will want to read. + defer func() { job.transcript.Finish(context.WithoutCancel(ctx), err) }() + + prompt := job.prompt + for round := 1; ; round++ { + if err = run.round(ctx, prompt); err != nil { + return err + } + s.publishProgress(ctx, job.tenantID, job.configID, job.skillID, + SkillProgress{Percent: 80, Stage: "agent_done"}) + + // The tree is handed to the execution user BEFORE it is verified. The + // agent created these files as root, and the language passes + // deliberately run as the ordinary user: verifying first would test + // permissions that never reach the image, and a restrictive root umask + // would fail a perfectly good install because the .venv interpreter + // was unreadable. + if err = s.normalizeSkillPermissions(ctx, job.mgr, job.sess.ID, job.skillDir); err != nil { + return err + } + + notes, verifyErr := s.verifySkill(ctx, job.mgr, job.sess.ID, job.skillDir, job.bundle) + s.reportVerificationNotes(ctx, job, notes) + if verifyErr == nil { + return nil + } + + var gate *skillVerificationError + if round >= skillInstallVerifyRounds || !errors.As(verifyErr, &gate) || !gate.Repairable { + err = verifyErr + return err + } + + logger.Infof(ctx, "[skill] %s failed %s verification with %d fixable finding(s); "+ + "handing them back to the installer", job.skillID, gate.Language, len(gate.Problems)) + s.publishProgress(ctx, job.tenantID, job.configID, job.skillID, SkillProgress{ + Percent: 82, Stage: "repairing", + Log: fmt.Sprintf("%s verification found %d missing dependency/dependencies; "+ + "asking the installer to add them", gate.Language, len(gate.Problems)), + }) + if err = s.reopenSkillDirForRepair(ctx, job.mgr, job.sess.ID, job.skillDir); err != nil { + return err + } + prompt = buildRepairPrompt(job.skillDir, gate) + job.transcript.RecordPrompt(prompt) + } +} + +// installerRun is one installer conversation, held open across rounds. A repair +// has to reach the same root shell, in the same sandbox, and be readable in the +// same transcript as the install it is repairing. +type installerRun struct { + engine interfaces.AgentEngine + transcript *installTranscript + sessionID string +} + +// openInstallerRun builds the engine the conversation runs on. It calls the +// engine directly instead of going through sessionService.AgentQA, because +// AgentQA swallows engine failures (it emits an error event and returns nil) +// and we need a reliable signal before switching the image. +func (s *TenantSkillService) openInstallerRun( + ctx context.Context, tenantID uint64, sess *types.Session, transcript *installTranscript, +) (*installerRun, error) { if s.installerAgents == nil { - return errors.New("custom agent service is not configured") + return nil, errors.New("custom agent service is not configured") } if transcript == nil { - return errors.New("install transcript was not seeded") + return nil, errors.New("install transcript was not seeded") } // The record is tenant-writable: updateBuiltinAgent lets a tenant persist a // Config for any built-in ID, this one included. It is therefore read for @@ -710,42 +807,129 @@ func (s *TenantSkillService) driveInstallerAgent( // platform's own registry entry. record, err := s.installerAgents.GetAgentByID(ctx, types.BuiltinSkillInstallerID) if err != nil { - return fmt.Errorf("load installer agent: %w", err) + return nil, fmt.Errorf("load installer agent: %w", err) } agentConfig := installerAgentConfig(installerAgentDefaults(ctx, tenantID), sess.SandboxConfigID) chatModel, err := s.resolveInstallerModel(ctx, tenantID, record) if err != nil { - return err + return nil, err } engine, err := s.agents.CreateAgentEngine( ctx, agentConfig, chatModel, nil, transcript.bus, sess.ID, transcript.assistantMessageID, ) if err != nil { - runErr := fmt.Errorf("create installer engine: %w", err) - transcript.Finish(ctx, runErr) - return runErr + return nil, fmt.Errorf("create installer engine: %w", err) } + return &installerRun{ + engine: engine, transcript: transcript, sessionID: sess.ID, + }, nil +} - state, err := engine.Execute(ctx, sess.ID, transcript.assistantMessageID, prompt, nil) - runErr := err - if runErr == nil && (state == nil || !state.IsComplete) { - runErr = errors.New("installer agent stopped without completing") +// round runs one installer turn. +// +// No conversation history is replayed. The engine is stateless across turns, so +// a repair round could be handed the first round's transcript — but what the +// gate reported is both shorter and more exact than anything that could be +// inferred from it, and re-reading the round that already missed a dependency +// is not what makes the next one find it. +func (r *installerRun) round(ctx context.Context, prompt string) error { + state, err := r.engine.Execute(ctx, r.sessionID, r.transcript.assistantMessageID, prompt, nil) + if err != nil { + return fmt.Errorf("installer agent failed: %w", err) } - // Detached from cancellation: when the install lock's renewal fails the - // context dies, and the transcript of the run that just died is precisely - // what someone will want to read. - transcript.Finish(context.WithoutCancel(ctx), runErr) - if runErr != nil { - if err != nil { - return fmt.Errorf("installer agent failed: %w", err) - } - return runErr + if state == nil || !state.IsComplete { + return errors.New("installer agent stopped without completing") } return nil } +// reportVerificationNotes surfaces what the gate noticed but did not refuse: an +// auxiliary file that cannot load, a requirement whose environment marker +// cannot be evaluated here, an import that resolves only from a directory the +// skill ships no script in. None is evidence of a broken install, and all of +// them are what someone debugging this skill later will wish they had been +// told, so they are logged and streamed rather than discarded. +func (s *TenantSkillService) reportVerificationNotes( + ctx context.Context, job installerJob, notes []string, +) { + if len(notes) == 0 { + return + } + for _, note := range notes { + logger.Infof(ctx, "[skill] %s verification note: %s", job.skillID, note) + } + // One event, not one per note. Progress keeps only the latest value, so + // publishing them separately would leave a late subscriber holding whichever + // note happened to be last. + s.publishProgress(ctx, job.tenantID, job.configID, job.skillID, SkillProgress{ + Percent: 80, Stage: "verify_note", Log: strings.Join(notes, "\n"), + }) +} + +// reopenSkillDirForRepair undoes normalizeSkillPermissions for another round. +// +// Verification runs as the ordinary user, so the tree is locked to 555 and root +// before it; a repair then has to install into .venv again. Install commands +// run as root and would get away with writing through those modes, but relying +// on that makes the next reader work out whether it still holds — restoring the +// state the seed left is the same two commands and needs no such argument. +func (s *TenantSkillService) reopenSkillDirForRepair( + ctx context.Context, mgr sandbox.Manager, sessionID, skillDir string, +) error { + if err := guardSkillDir(skillDir); err != nil { + return err + } + dir := sandbox.ShellQuote(skillDir) + user := sandbox.DefaultSandboxExecUser + cmd := fmt.Sprintf("chown -R %s:%s %s && chmod -R u+rwX,go+rX %s", user, user, dir, dir) + if _, err := s.execInstall(ctx, mgr, sessionID, cmd); err != nil { + return fmt.Errorf("reopen skill directory %s for a repair round: %w", skillDir, err) + } + return nil +} + +// buildRepairPrompt turns the gate's own findings into the next round's brief. +// +// It carries no analysis of its own, deliberately. The reason a first round can +// fail on a working skill is that SKILL.md and the imports its files execute are +// two different descriptions of what the skill needs; a third description +// written here would reintroduce exactly that gap. The only thing added is the +// import-name-to-distribution mapping, which is a property of PyPI rather than +// of this skill. +func buildRepairPrompt(skillDir string, gate *skillVerificationError) string { + var findings strings.Builder + for _, problem := range gate.Problems { + findings.WriteString("- ") + findings.WriteString(problem) + findings.WriteString("\n") + } + return fmt.Sprintf(`Verification of the skill you just installed failed. Fix only this and stop. + +The %s check reported: +%s +Every line above is a dependency this image cannot resolve. The names were read +from the imports the skill's own files execute when they load, which is why some +of them appear nowhere in SKILL.md — install them anyway. + +- Python packages go into %s/.venv (`+"`uv pip install`"+`, or + %s/.venv/bin/python -m pip install). Node packages go under %s/node_modules. +- The name in each line is an IMPORT name; install the distribution that + provides it. Common cases where they differ: PIL -> pillow, yaml -> pyyaml, + docx -> python-docx, pptx -> python-pptx, cv2 -> opencv-python-headless, + bs4 -> beautifulsoup4, sklearn -> scikit-learn, fitz -> pymupdf. +- Do NOT edit, move or delete any of the skill's own files, and do NOT edit + SKILL.md or requirements.txt to make the check pass. The uploaded archive is + what read_skill serves, so a source edit here makes the installed skill differ + from what everyone else sees. +- If a package genuinely cannot be installed in this image, say so plainly in + your summary rather than working around it. + +The same verification runs again as soon as you finish. +`, gate.Language, findings.String(), skillDir, skillDir, skillDir) +} + // normalizeSkillPermissions makes the skill tree readable and executable by // the non-root execution user, and writable by nobody. Installs run as root, // so without the mode change the user that actually runs skills could not read @@ -1526,6 +1710,13 @@ venv. It never runs the skill's code, so nothing is expected to answer --help. Lazy imports and install_deps.py extras are invisible to that check — you still have to install them. +SKILL.md is not the dependency list. What the check resolves is the set of +top-level imports the skill's files actually execute, and a library module +routinely imports something the documentation never mentions. Read every .py in +the tree (grep '^import\|^from' over it) and install what those lines need, not +only what the Dependencies section names. If verification does fail on a missing +package, you get its exact findings back and one round to install them. + SKILL.md: %s `, skillDir, uvAvailable, skillDir, skillDir, skillDir, diff --git a/internal/application/service/tenant_skill_install_test.go b/internal/application/service/tenant_skill_install_test.go index 62b33b7e..f5f47097 100644 --- a/internal/application/service/tenant_skill_install_test.go +++ b/internal/application/service/tenant_skill_install_test.go @@ -1043,9 +1043,9 @@ func TestRunInstallRequiresVenvWhenRequirementsExist(t *testing.T) { require.Nil(t, fx.configRepo.saved) } -// Verification used to run one guessed "entry script" with --help. Nothing in -// the runtime designates an entry script — the model names whichever path it -// likes in execute_skill_script — so the guess left every other file unchecked +// Verification used to run one guessed "entry script" with --help. A skill is +// not one entry point: it is every file the tree ships, some run as scripts +// and some imported as packages. The guess left every other file unchecked // while failing installs over the one it happened to pick. func TestRunInstallVerifiesEveryScriptOfEveryLanguage(t *testing.T) { fx := newInstallFixture(t) @@ -1058,7 +1058,7 @@ func TestRunInstallVerifiesEveryScriptOfEveryLanguage(t *testing.T) { pythonPass := skillPythonVerifyCommand(installSkillDir, []string{ "scripts/__init__.py", "scripts/extract.py", "scripts/helper.py", - }) + }, nil) require.Contains(t, fx.commands, pythonPass, "every python file must be checked, not the first one in sort order") require.Contains(t, fx.commands, @@ -1083,7 +1083,7 @@ func TestRunInstallDoesNotExecuteAnyScriptToVerifyIt(t *testing.T) { } func TestSkillPythonVerifyCommandPrefersTheSkillVenv(t *testing.T) { - command := skillPythonVerifyCommand("/opt/skills/demo", []string{"scripts/a.py"}) + command := skillPythonVerifyCommand("/opt/skills/demo", []string{"scripts/a.py"}, nil) require.Contains(t, command, "if [ -x /opt/skills/demo/.venv/bin/python ]; "+ "then py=/opt/skills/demo/.venv/bin/python; else py=python3; fi", @@ -1095,11 +1095,68 @@ func TestSkillPythonVerifyCommandPrefersTheSkillVenv(t *testing.T) { // A skill may ship a file whose name needs quoting; the command is assembled // by hand, so the shell must never see it as more than one word. func TestSkillVerifyCommandsQuoteAwkwardPaths(t *testing.T) { - python := skillPythonVerifyCommand("/opt/skills/demo", []string{"scripts/a b'c.py"}) + python := skillPythonVerifyCommand("/opt/skills/demo", []string{"scripts/a b'c.py"}, nil) require.Contains(t, python, `'scripts/a b'\''c.py'`) shell := skillShellVerifyCommand("/opt/skills/demo", []string{"a b.sh"}) - require.Equal(t, `for f in '/opt/skills/demo/a b.sh'; do sh -n "$f" || exit 1; done`, shell) + require.Equal(t, + `if command -v bash >/dev/null 2>&1; then parser=bash; else parser=sh; fi; `+ + `for f in '/opt/skills/demo/a b.sh'; do "$parser" -n "$f" || exit 1; done`, + shell) +} + +// A skill's shell scripts carry a bash shebang almost exclusively, and +// SkillInterpreterCommand runs them with bash. Checking them with sh — dash on +// Debian — refused installs over array literals, `function f()`, C-style for +// loops and process substitution, none of which dash can parse. +func TestSkillShellVerifyCommandParsesWithBash(t *testing.T) { + command := skillShellVerifyCommand("/opt/skills/demo", []string{"bin/setup.sh"}) + + require.Contains(t, command, "parser=bash") + require.Contains(t, command, `"$parser" -n "$f"`) + require.Contains(t, command, "else parser=sh", + "an image without bash must still get a syntax check") + require.NotContains(t, command, `sh -n "$f"`, + "the check must not hard-code the shell that cannot parse these files") +} + +// The Python pass is the only one whose verdict depends on what the image +// carries, so it is the only one that can fail an install over a file nothing +// the skill offers ever loads. Those files are named separately. +func TestSkillPythonVerifyCommandSeparatesAuxiliaryFiles(t *testing.T) { + command := skillPythonVerifyCommand("/opt/skills/demo", + []string{"scripts/run.py"}, []string{"tests/test_run.py"}) + + require.True(t, strings.HasSuffix(command, + `- /opt/skills/demo scripts/run.py --optional tests/test_run.py`), + "got %q", command) +} + +// The split follows the conventions the language ecosystems already share. A +// bundled tests/ directory imports pytest and an examples/ script imports +// whatever it illustrates; neither is reachable from anything the skill +// exposes, and neither may decide whether the skill installs. +func TestSkillAuxiliaryScriptFollowsEcosystemConventions(t *testing.T) { + for _, rel := range []string{ + "tests/test_run.py", "test/helpers.py", "examples/demo.py", + "scripts/__tests__/a.py", "benchmarks/bench.py", "docs/conf.py", + "conftest.py", "setup.py", "scripts/extract_test.py", + "scripts/test_extract.py", + } { + require.True(t, skillAuxiliaryScript(rel), "%s must be auxiliary", rel) + } + for _, rel := range []string{ + "scripts/extract.py", "scripts/office/validators/base.py", + "recalc.py", "scripts/latest.py", "scripts/contest.py", + } { + require.False(t, skillAuxiliaryScript(rel), "%s is part of what the skill offers", rel) + } + + entry, auxiliary := splitAuxiliaryScripts([]string{ + "scripts/a.py", "tests/test_a.py", "scripts/b.py", + }) + require.Equal(t, []string{"scripts/a.py", "scripts/b.py"}, entry) + require.Equal(t, []string{"tests/test_a.py"}, auxiliary) } func TestSkillNodeVerifyCommandChecksDeclaredDependencies(t *testing.T) { @@ -1307,7 +1364,8 @@ func TestRunInstallKeepsOldImageWhenVerificationFails(t *testing.T) { fx.configRepo.entity.Config.SkillImage = &types.SkillImageConfig{ SnapshotID: "snap-old", Generation: 3, OwnerFingerprint: fx.fingerprint, } - fx.loadCheckExitCode = 1 + fx.loadCheckExitCodes = []int{1} + fx.loadCheckStderr = "scripts/extract.py has a syntax error on line 1: invalid syntax" err := fx.svc.runInstall(context.Background(), 7, "cfg-1", "sk-1", fx.bundle) @@ -1322,6 +1380,98 @@ func TestRunInstallKeepsOldImageWhenVerificationFails(t *testing.T) { require.NotEmpty(t, skill.Error) } +// The failure this closes: the installer derives what to install from SKILL.md +// prose, and the gate derives what must resolve from the imports every file +// executes. Those disagree on any skill whose library modules import something +// its documentation never names — the official office toolkit imports +// defusedxml and lxml at module level and mentions neither — so the install +// died with the answer already in hand, and a retry replayed the same prompt to +// the same effect. The gate's own lines are the repair round's brief. +func TestRunInstallHandsVerificationFindingsBackToTheInstaller(t *testing.T) { + fx := newInstallFixture(t) + fx.loadCheckExitCodes = []int{skillVerifyRepairableExit, 0} + fx.loadCheckStderr = "scripts/office/validators/base.py imports defusedxml, " + + "which is not available in this image\n" + + "scripts/recalc.py imports openpyxl, which is not available in this image" + + require.NoError(t, fx.svc.runInstall(context.Background(), 7, "cfg-1", "sk-1", fx.bundle)) + + require.Len(t, fx.agentPrompts, 2, "a fixable failure must reach the installer again") + repair := fx.agentPrompts[1] + require.Contains(t, repair, "imports defusedxml", + "the repair round is driven by the gate's own findings, not by a second guess") + require.Contains(t, repair, "imports openpyxl") + require.Contains(t, repair, installSkillDir+"/.venv", + "the round has to be told where the packages belong") + require.Contains(t, repair, "Do NOT edit", + "a repair must not be allowed to edit the tree into passing") + + require.Equal(t, 2, fx.loadCheckPasses, "the repair has to be verified, not trusted") + require.Contains(t, fx.events, "create-snapshot") + require.NotNil(t, fx.configRepo.saved, "a repaired install must reach the image pointer") + + skill, _ := fx.skillRepo.GetSkill(context.Background(), 7, "cfg-1", "sk-1") + require.Equal(t, types.SkillStatusReady, skill.Status) + require.Empty(t, skill.Error) +} + +// A repair round has to be able to write into .venv again, which the +// verification pass just locked down: the checks run as the ordinary user, so +// the tree is handed over at 555 and root-owned before every one of them. +func TestRunInstallReopensTheTreeBeforeARepairRound(t *testing.T) { + fx := newInstallFixture(t) + fx.loadCheckExitCodes = []int{skillVerifyRepairableExit, 0} + + require.NoError(t, fx.svc.runInstall(context.Background(), 7, "cfg-1", "sk-1", fx.bundle)) + + reopen := indexOfCommandContaining(fx.commands, "chmod -R u+rwX,go+rX "+installSkillDir) + require.GreaterOrEqual(t, reopen, 0, "the tree must be writable again for the repair") + + firstLock := indexOfCommandContaining(fx.commands, "chmod -R 555 "+installSkillDir) + require.GreaterOrEqual(t, firstLock, 0) + require.Greater(t, reopen, firstLock, + "the tree is reopened after the pass that locked it, not before") + + // And locked again for the pass that checks the repair, so the final image + // carries the modes every verification ran against. + require.Greater(t, lastIndexOfCommandContaining(fx.commands, "chmod -R 555 "+installSkillDir), + reopen, "the repaired tree must be locked down again before it is verified") +} + +// Installing a package cannot fix a file that does not parse, and the bundle +// has to change instead. Spending another agent round on it would only delay +// the same failure by minutes. +func TestRunInstallDoesNotRetryAFailureInstallingCannotFix(t *testing.T) { + fx := newInstallFixture(t) + fx.loadCheckExitCodes = []int{1} + fx.loadCheckStderr = "scripts/extract.py has a syntax error on line 1: invalid syntax" + + err := fx.svc.runInstall(context.Background(), 7, "cfg-1", "sk-1", fx.bundle) + + require.Error(t, err) + require.ErrorContains(t, err, "syntax error") + require.Len(t, fx.agentPrompts, 1, "an unfixable finding must not buy another round") + require.Equal(t, 1, fx.loadCheckPasses) +} + +// The loop is bounded. An installer that cannot satisfy the gate in one repair +// is not going to satisfy it in ten, and a skill install must not become an +// open-ended retry against a billed sandbox. +func TestRunInstallStopsAfterOneRepairRound(t *testing.T) { + fx := newInstallFixture(t) + fx.loadCheckExitCodes = []int{skillVerifyRepairableExit} + + err := fx.svc.runInstall(context.Background(), 7, "cfg-1", "sk-1", fx.bundle) + + require.Error(t, err) + require.ErrorContains(t, err, "imports pandas", + "the failure must name what the gate found, not just that it failed") + require.Len(t, fx.agentPrompts, skillInstallVerifyRounds) + require.Equal(t, skillInstallVerifyRounds, fx.loadCheckPasses) + require.NotContains(t, fx.events, "create-snapshot") + require.Nil(t, fx.configRepo.saved) +} + func TestRunInstallDeletesTheSnapshotWhenSwitchFails(t *testing.T) { fx := newInstallFixture(t) fx.configRepo.updateErr = errUpdateBoom @@ -1549,7 +1699,7 @@ const ( // reason; the properties worth stating verbatim are asserted separately in // TestSkillPythonVerifyCommand*. var installPythonVerifyCommand = skillPythonVerifyCommand( - installSkillDir, []string{"scripts/extract.py"}, + installSkillDir, []string{"scripts/extract.py"}, nil, ) func indexOfEvent(events []string, needle string) int { @@ -1561,6 +1711,27 @@ func indexOfEvent(events []string, needle string) int { return -1 } +// indexOfCommandContaining and its Last variant locate one shell command in the +// ordered transcript. A repair round issues the same commands twice, so the +// tests that care about ordering need both ends. +func indexOfCommandContaining(commands []string, needle string) int { + for i, command := range commands { + if strings.Contains(command, needle) { + return i + } + } + return -1 +} + +func lastIndexOfCommandContaining(commands []string, needle string) int { + for i := len(commands) - 1; i >= 0; i-- { + if strings.Contains(commands[i], needle) { + return i + } + } + return -1 +} + // staleMark is one request to mark a config's bound sandboxes stale. The // tenant is part of it because marking the right config of the wrong workspace // would rebuild sandboxes that never carried this image. @@ -1584,10 +1755,17 @@ type installFixture struct { events []string commands []string fingerprint string - // loadCheck* drive the per-language script verification pass, which is - // the last gate before the snapshot. - loadCheckExitCode int - loadCheckResult *sandbox.ExecuteResult + // loadCheck* drive the per-language script verification pass, which is the + // last gate before the snapshot. exitCodes is consumed one entry per python + // pass and its last entry repeats, so a test about a single round writes one + // value and a test about the repair round writes two. Exit 2 is the + // checker's "everything I found is a missing dependency", which is what + // earns another installer round. + loadCheckExitCodes []int + loadCheckStdout string + loadCheckStderr string + loadCheckPasses int + loadCheckResult *sandbox.ExecuteResult // depsExitCode fails the declared-dependency check (venv / node_modules). depsExitCode int // execResult is scoped to execResultCommand: an unscoped stub result @@ -1597,6 +1775,10 @@ type installFixture struct { execResult *sandbox.ExecuteResult loadCheckRanAsRoot bool agentErr error + // agentPrompts is every prompt the installer engine was handed, in order. + // A repair round is a second entry, and what it says is the whole point of + // having one. + agentPrompts []string // beforeExecute runs at the moment the engine would start, so a test can // observe the state an attaching console would see mid-install. beforeExecute func() @@ -1657,6 +1839,10 @@ func newInstallFixture(t *testing.T) *installFixture { fx := &installFixture{t: t} fx.fingerprint = sandbox.SkillImageFingerprint("e2b", "key-1", "https://e2b.example") + // The checker's own wording for a missing dependency, so a test reading the + // repair prompt sees what a real run would put in it. + fx.loadCheckStderr = "scripts/extract.py imports pandas, " + + "which is not available in this image" fx.bundle = &SkillBundle{ Name: "pdf-tools", Version: "1.0.0", @@ -1721,6 +1907,19 @@ func (f *installFixture) record(event string) { f.events = append(f.events, event) } +// nextLoadCheckExit returns the code for this verification pass, repeating the +// last configured entry once they run out. +func (f *installFixture) nextLoadCheckExit() int { + f.loadCheckPasses++ + if len(f.loadCheckExitCodes) == 0 { + return 0 + } + if f.loadCheckPasses > len(f.loadCheckExitCodes) { + return f.loadCheckExitCodes[len(f.loadCheckExitCodes)-1] + } + return f.loadCheckExitCodes[f.loadCheckPasses-1] +} + // now is the fixture's clock, so a test can express "one heartbeat ago" // against the same instant the service reads. func (f *installFixture) now() time.Time { return f.svc.now() } @@ -2558,7 +2757,9 @@ func (m *installSandboxManager) ExecShellCommandWithOptions( return m.fx.loadCheckResult, nil } return &sandbox.ExecuteResult{ - ExitCode: m.fx.loadCheckExitCode, Stderr: "scripts/extract.py imports pandas", + ExitCode: m.fx.nextLoadCheckExit(), + Stdout: m.fx.loadCheckStdout, + Stderr: m.fx.loadCheckStderr, }, nil case command == removeSkillDirCommand: m.fx.record("remove-skill-dir") @@ -2741,16 +2942,17 @@ type installAgentEngine struct { } func (e *installAgentEngine) Execute( - context.Context, - string, - string, - string, - []chat.Message, - ...[]string, + _ context.Context, + _ string, + _ string, + prompt string, + _ []chat.Message, + _ ...[]string, ) (*types.AgentState, error) { if e.fx.beforeExecute != nil { e.fx.beforeExecute() } + e.fx.agentPrompts = append(e.fx.agentPrompts, prompt) e.fx.record("agent-execute") if e.fx.agentDelay > 0 { time.Sleep(e.fx.agentDelay) diff --git a/internal/application/service/tenant_skill_transcript.go b/internal/application/service/tenant_skill_transcript.go index 883a6063..ed34d02a 100644 --- a/internal/application/service/tenant_skill_transcript.go +++ b/internal/application/service/tenant_skill_transcript.go @@ -38,11 +38,17 @@ type installTranscript struct { sessionID string assistantMessageID string - mu sync.Mutex - message *types.Message - answers []*installAnswerSegment - starts map[string]time.Time - finished bool + mu sync.Mutex + message *types.Message + answers []*installAnswerSegment + starts map[string]time.Time + // closed guards the terminal event and the final write, both of which say + // "this install is over" and must be said once. + closed bool + // Totals accumulated across the run's engine turns, reported on the one + // terminal event Finish emits. + totalSteps int + totalDurationMs int64 } // installAnswerSegment accumulates the prose streamed under one final-answer @@ -142,13 +148,20 @@ func (tr *installTranscript) Subscribe() { tr.bus.On(event.EventAgentComplete, tr.onComplete) } -// Finish closes the record. runErr is the engine's verdict: the engine emits -// no complete event when it fails, and a failed install is the one people -// actually come to read, so the failure is written here rather than hoped for. +// Finish closes the record. runErr is the install's verdict, not just the +// engine's: verification runs after the agent stops, and a failure there is the +// one people actually come to read, so it is written here rather than hoped for. func (tr *installTranscript) Finish(ctx context.Context, runErr error) { if tr == nil { return } + tr.mu.Lock() + alreadyClosed := tr.closed + tr.closed = true + tr.mu.Unlock() + if alreadyClosed { + return + } if runErr != nil { tr.append(interfaces.StreamEvent{ ID: uuid.NewString(), @@ -173,22 +186,41 @@ func (tr *installTranscript) Finish(ctx context.Context, runErr error) { } tr.mu.Lock() - alreadyComplete := tr.finished - tr.finished = true + steps, duration := tr.totalSteps, tr.totalDurationMs tr.mu.Unlock() - - if !alreadyComplete { - tr.append(interfaces.StreamEvent{ - ID: uuid.NewString(), - Type: types.ResponseTypeComplete, - Done: true, - Timestamp: time.Now(), - Data: map[string]interface{}{}, - }) - } + tr.append(interfaces.StreamEvent{ + ID: uuid.NewString(), + Type: types.ResponseTypeComplete, + Done: true, + Timestamp: time.Now(), + Data: map[string]interface{}{ + "total_steps": steps, + "total_duration_ms": duration, + }, + }) tr.save(ctx) } +// RecordPrompt logs a follow-up instruction the installer was given mid-run. +// +// The transcript exists so one replay of the event log is the whole +// conversation. A repair round whose instruction is missing reads as the agent +// spontaneously deciding to install more packages, which is precisely the +// moment someone is reading this to find out why. +func (tr *installTranscript) RecordPrompt(prompt string) { + if tr == nil { + return + } + tr.append(interfaces.StreamEvent{ + ID: uuid.NewString(), + Type: types.ResponseTypeInstallPrompt, + Content: prompt, + Done: true, + Timestamp: time.Now(), + Data: map[string]interface{}{}, + }) +} + func (tr *installTranscript) onThought(_ context.Context, evt event.Event) error { data, ok := evt.Data.(event.AgentThoughtData) if !ok { @@ -326,13 +358,16 @@ func (tr *installTranscript) onError(_ context.Context, evt event.Event) error { return nil } +// onComplete records what the round finished with. It deliberately emits no +// terminal event: an install may run another installer round after +// verification, and a console that saw "complete" would stop following before +// that round began. Only Finish closes the stream. func (tr *installTranscript) onComplete(_ context.Context, evt event.Event) error { data, ok := evt.Data.(event.AgentCompleteData) if !ok { return nil } tr.mu.Lock() - tr.finished = true if data.MessageID == tr.assistantMessageID { msg := tr.ensureMessageLocked() msg.IsCompleted = true @@ -347,18 +382,11 @@ func (tr *installTranscript) onComplete(_ context.Context, evt event.Event) erro if tr.composeAnswerLocked() == "" && data.FinalAnswer != "" { tr.segmentLocked(evt.ID).content = data.FinalAnswer } + // Summed rather than overwritten: an install that needed a repair round ran + // two engine turns, and its cost is both of them. + tr.totalSteps += data.TotalSteps + tr.totalDurationMs += data.TotalDurationMs tr.mu.Unlock() - - tr.append(interfaces.StreamEvent{ - ID: evt.ID, - Type: types.ResponseTypeComplete, - Done: true, - Timestamp: time.Now(), - Data: map[string]interface{}{ - "total_steps": data.TotalSteps, - "total_duration_ms": data.TotalDurationMs, - }, - }) return nil } diff --git a/internal/application/service/tenant_skill_transcript_test.go b/internal/application/service/tenant_skill_transcript_test.go index 32796001..ddce6858 100644 --- a/internal/application/service/tenant_skill_transcript_test.go +++ b/internal/application/service/tenant_skill_transcript_test.go @@ -265,3 +265,56 @@ func TestInstallTranscriptCreateOpensTheLogWithThePrompt(t *testing.T) { require.Equal(t, []types.ResponseType{types.ResponseTypeInstallPrompt}, streams.types()) require.Equal(t, "install pdf-tools", streams.events[0].Content) } + +// An install may run a second installer round, because verification happens +// after the agent stops and a dependency it names is something the agent can +// still fix. The engine reports "complete" at the end of every turn, so the +// transcript must not treat the first one as the end of the install — a console +// that saw it would stop following before the repair round began. +func TestInstallTranscriptStaysOpenAcrossInstallerRounds(t *testing.T) { + tr, bus, streams, messages := newTranscriptForTest(t) + ctx := context.Background() + + finishRound := func(steps int, ms int64) { + require.NoError(t, bus.Emit(ctx, event.Event{ + ID: "done", Type: event.EventAgentComplete, + Data: event.AgentCompleteData{ + MessageID: "msg-1", TotalSteps: steps, TotalDurationMs: ms, + }, + })) + } + + finishRound(3, 1000) + tr.RecordPrompt("Verification failed. Install defusedxml into the venv.") + finishRound(2, 500) + tr.Finish(ctx, nil) + + kinds := streams.types() + require.Equal(t, []types.ResponseType{ + types.ResponseTypeInstallPrompt, + types.ResponseTypeComplete, + }, kinds, "the repair instruction has to be in the log, and the install ends once") + require.Contains(t, streams.events[0].Content, "Install defusedxml") + + terminal := streams.events[len(streams.events)-1] + require.EqualValues(t, 5, terminal.Data["total_steps"], + "an install that needed a repair round cost both turns") + require.EqualValues(t, 1500, terminal.Data["total_duration_ms"]) + require.Len(t, messages.updated, 1) +} + +// Finish is deferred by the install path and also called on the error path that +// gives up before that defer is armed. Saying "this install is over" twice would +// duplicate the verdict in the record people read. +func TestInstallTranscriptFinishIsIdempotent(t *testing.T) { + tr, _, streams, messages := newTranscriptForTest(t) + + tr.Finish(context.Background(), errors.New("python verification failed: boom")) + tr.Finish(context.Background(), errors.New("python verification failed: boom")) + + require.Equal(t, []types.ResponseType{ + types.ResponseTypeError, + types.ResponseTypeComplete, + }, streams.types()) + require.Len(t, messages.updated, 1) +} diff --git a/internal/application/service/tenant_skill_verify.go b/internal/application/service/tenant_skill_verify.go index 89df23ef..1e63645f 100644 --- a/internal/application/service/tenant_skill_verify.go +++ b/internal/application/service/tenant_skill_verify.go @@ -20,28 +20,66 @@ import ( //go:embed tenant_skill_verify.py var skillPythonVerifier string +// skillVerifyRepairableExit is the exit code a verification pass uses when +// every problem it found is a dependency missing from this image. It separates +// "another installer round can fix this" from "the bundle has to change", which +// is the only distinction that decides what the install flow does next. +const skillVerifyRepairableExit = 2 + +// skillVerifyNotePrefix marks a line the checker reports without refusing the +// install. Notes travel on stdout so a non-zero exit stays unambiguous. +const skillVerifyNotePrefix = "note: " + +// skillVerificationError is what the gate said, kept structured because it is +// also the brief for a repair round. The gate is the only authority on what has +// to resolve in this image, so handing back its own lines is what keeps the +// installer from deriving "what this skill needs" a second time. +type skillVerificationError struct { + // Language names the pass that failed, as the operator sees it. + Language string + // Repairable is true when installing a package would satisfy every line. + Repairable bool + // Problems are the checker's own lines, one per finding. + Problems []string + // Summary describes the command result itself, and is the only thing worth + // printing when the pass died without producing lines at all. + Summary string +} + +func (e *skillVerificationError) Error() string { + if len(e.Problems) == 0 { + return fmt.Sprintf("%s verification failed (%s)", e.Language, e.Summary) + } + return fmt.Sprintf("%s verification failed: %s", e.Language, strings.Join(e.Problems, "; ")) +} + // verifySkill is the server's own check, and the last gate before the image // pointer moves: a broken install must leave the previous snapshot serving. // -// It deliberately proves loadability rather than behaviour. Nothing in the -// runtime designates an entry point — the model names any script it likes in -// execute_skill_script — so there is no single command whose success would -// stand in for the skill working. What every one of those calls does need is -// that the interpreter can parse the file and resolve its imports inside this -// image, and that is checkable for all scripts at once without executing a -// line of the skill's own code. +// It proves loadability, not behaviour, and not "every file can be used as +// python this_file.py". A skill ships scripts and library packages together; +// execute_skill_script names files, but those files are also imported by each +// other. The Python pass resolves each file's imports against the prefixes +// Python itself would use — the file's own directory and every ancestor up to +// the skill root — plus the venv. Guessing a single entry script, or treating +// every .py as an isolated __main__, both fail real skills. +// +// Findings are graded rather than uniformly fatal. Refusing an install costs +// the minutes of dependency work that already succeeded, so only evidence that +// the install itself is broken may do it: a file nothing the skill offers ever +// loads, or a requirement pip would have skipped, is returned as a note. // // Not executing is the point, not a limitation. The previous implementation // ran one guessed script with --help; skills that do not parse arguments // simply ran their whole main path inside the tree about to be snapshotted. func (s *TenantSkillService) verifySkill( ctx context.Context, mgr sandbox.Manager, sessionID, skillDir string, bundle *SkillBundle, -) error { +) ([]string, error) { if err := s.verifySkillTree(ctx, mgr, sessionID, skillDir, bundle); err != nil { - return err + return nil, err } if err := s.verifyDeclaredDependencies(ctx, mgr, sessionID, skillDir, bundle); err != nil { - return err + return nil, err } return s.verifyScriptsLoad(ctx, mgr, sessionID, skillDir, bundle) } @@ -93,30 +131,43 @@ func (s *TenantSkillService) verifyDeclaredDependencies( } // verifyScriptsLoad runs one pass per language present in the bundle. Each -// pass covers every script of that language, because every one of them is a -// script the model may name at runtime. +// pass covers every file of that language: syntax and imports are properties +// of the file, not of a guessed entry point. +// +// Only the Python pass takes the auxiliary split. Its findings depend on what +// the image carries, so a bundled tests/ directory can fail an install that +// works; `node --check` and `bash -n` only state whether a file parses, which +// no amount of installing changes either way. func (s *TenantSkillService) verifyScriptsLoad( ctx context.Context, mgr sandbox.Manager, sessionID, skillDir string, bundle *SkillBundle, -) error { +) ([]string, error) { + var notes []string if scripts := sortedScriptPaths(bundle, ".py"); len(scripts) > 0 { - if err := s.execVerify(ctx, mgr, sessionID, skillDir, "python", - skillPythonVerifyCommand(skillDir, scripts)); err != nil { - return err + entry, auxiliary := splitAuxiliaryScripts(scripts) + found, err := s.execVerify(ctx, mgr, sessionID, skillDir, "python", + skillPythonVerifyCommand(skillDir, entry, auxiliary)) + notes = append(notes, found...) + if err != nil { + return notes, err } } if scripts := sortedScriptPaths(bundle, ".js", ".mjs", ".cjs"); len(scripts) > 0 { - if err := s.execVerify(ctx, mgr, sessionID, skillDir, "node", - skillNodeVerifyCommand(skillDir, scripts, nodeDependencyNames(bundle))); err != nil { - return err + found, err := s.execVerify(ctx, mgr, sessionID, skillDir, "node", + skillNodeVerifyCommand(skillDir, scripts, nodeDependencyNames(bundle))) + notes = append(notes, found...) + if err != nil { + return notes, err } } if scripts := sortedScriptPaths(bundle, ".sh"); len(scripts) > 0 { - if err := s.execVerify(ctx, mgr, sessionID, skillDir, "shell", - skillShellVerifyCommand(skillDir, scripts)); err != nil { - return err + found, err := s.execVerify(ctx, mgr, sessionID, skillDir, "shell", + skillShellVerifyCommand(skillDir, scripts)) + notes = append(notes, found...) + if err != nil { + return notes, err } } - return nil + return notes, nil } // execVerify runs one verification pass as the ordinary sandbox user, with the @@ -124,10 +175,10 @@ func (s *TenantSkillService) verifyScriptsLoad( // install-mode root would test permissions that never reach a session. func (s *TenantSkillService) execVerify( ctx context.Context, mgr sandbox.Manager, sessionID, skillDir, label, command string, -) error { +) ([]string, error) { executor, err := installExecutor(mgr) if err != nil { - return err + return nil, err } res, err := executor.ExecShellCommandWithOptions(ctx, sessionID, command, sandbox.ShellExecOptions{ @@ -139,12 +190,46 @@ func (s *TenantSkillService) execVerify( }, }) if err != nil { - return fmt.Errorf("%s verification: %w", label, err) + return nil, fmt.Errorf("%s verification: %w", label, err) } + // Notes are read even from a failed pass: an install refused for a missing + // package may also have noticed something about a file it did not refuse + // over, and that is exactly the context whoever reads the failure needs. + notes := verificationNotes(res.Stdout) if res.ExitCode != 0 { - return fmt.Errorf("%s verification failed (%s)", label, describeExecFailure(res)) + return notes, &skillVerificationError{ + Language: label, + Repairable: res.ExitCode == skillVerifyRepairableExit, + Problems: verificationProblems(res.Stderr), + Summary: describeExecFailure(res), + } } - return nil + return notes, nil +} + +// verificationNotes pulls the reported-but-not-refused findings out of stdout. +func verificationNotes(stdout string) []string { + var notes []string + for _, line := range strings.Split(stdout, "\n") { + line = strings.TrimSpace(line) + if after, ok := strings.CutPrefix(line, skillVerifyNotePrefix); ok { + notes = append(notes, after) + } + } + return notes +} + +// verificationProblems splits a failed pass's stderr into its findings. Blank +// lines are dropped; everything else is the checker's own wording, which is +// what a repair round is given. +func verificationProblems(stderr string) []string { + var problems []string + for _, line := range strings.Split(stderr, "\n") { + if line = strings.TrimSpace(line); line != "" { + problems = append(problems, line) + } + } + return problems } // skillPythonVerifyCommand pipes the verifier into the same interpreter a @@ -152,13 +237,23 @@ func (s *TenantSkillService) execVerify( // line. The list is explicit rather than a directory walk so the pass covers // exactly the uploaded sources — never .venv, node_modules or anything else // the agent created while installing. -func skillPythonVerifyCommand(skillDir string, scripts []string) string { +// +// Auxiliary files are named after --optional. They are checked the same way, +// and what is found in them is reported rather than allowed to refuse the +// install. +func skillPythonVerifyCommand(skillDir string, entry, auxiliary []string) string { venv := path.Join(skillDir, ".venv", "bin", "python") quotedVenv := sandbox.ShellQuote(venv) args := []string{sandbox.ShellQuote(skillDir)} - for _, rel := range scripts { + for _, rel := range entry { args = append(args, sandbox.ShellQuote(rel)) } + if len(auxiliary) > 0 { + args = append(args, skillVerifyOptionalFlag) + for _, rel := range auxiliary { + args = append(args, sandbox.ShellQuote(rel)) + } + } // The script arrives base64-encoded on stdin: it contains quotes, newlines // and backslashes, and a heredoc would still be at the mercy of whatever // the remote exec layer does with multi-line command strings. @@ -173,6 +268,9 @@ func skillPythonVerifyCommand(skillDir string, scripts []string) string { // skillNodeVerifyCommand parses every JavaScript file and, when the bundle // declares runtime dependencies, checks each one resolves. `node --check` is // parse-only: it never runs the module body. +// +// A declared dependency that is missing exits with the repairable code: it is +// the one thing here another installer round can still put in place. func skillNodeVerifyCommand(skillDir string, scripts, deps []string) string { parts := make([]string, 0, 2) if len(deps) > 0 { @@ -182,18 +280,75 @@ func skillNodeVerifyCommand(skillDir string, scripts, deps []string) string { } parts = append(parts, fmt.Sprintf( "for d in %s; do [ -e %s/\"$d\" ] || "+ - "{ echo \"package.json declares $d but node_modules/$d is missing\" >&2; exit 1; }; done", + "{ echo \"package.json declares $d but node_modules/$d is missing\" >&2; exit %d; }; done", strings.Join(quoted, " "), sandbox.ShellQuote(path.Join(skillDir, "node_modules")), + skillVerifyRepairableExit, )) } parts = append(parts, forEachScript(skillDir, scripts, "node --check \"$f\"")) return strings.Join(parts, "; ") } -// skillShellVerifyCommand parses every shell script. `sh -n` reads the file -// and exits without executing any of it. +// skillShellVerifyCommand parses every shell script with the shell that would +// run it. `-n` reads the file and exits without executing any of it. +// +// bash, not sh: skills ship `#!/bin/bash` almost exclusively, and +// SkillInterpreterCommand runs them with bash for that reason. /bin/sh is dash +// on Debian, where an array literal, `function f()`, a C-style for loop and +// process substitution are all outright syntax errors — so checking with sh +// would refuse installs over scripts that run perfectly. func skillShellVerifyCommand(skillDir string, scripts []string) string { - return forEachScript(skillDir, scripts, "sh -n \"$f\"") + return "if command -v bash >/dev/null 2>&1; then parser=bash; else parser=sh; fi; " + + forEachScript(skillDir, scripts, `"$parser" -n "$f"`) +} + +// skillVerifyOptionalFlag separates the files whose failure refuses the install +// from the ones whose failure is only reported. It is the checker's own argv +// contract; both sides must keep using this spelling. +const skillVerifyOptionalFlag = "--optional" + +// auxiliaryScriptDirs are the directory names Python and JavaScript projects +// universally use for files that ship with a project without being part of +// what it offers. A skill's tests import pytest; its examples import whatever +// they illustrate. Nothing the skill exposes loads either, so neither may +// decide whether the skill installs. +var auxiliaryScriptDirs = map[string]struct{}{ + "test": {}, "tests": {}, "testing": {}, "__tests__": {}, + "example": {}, "examples": {}, "sample": {}, "samples": {}, + "benchmark": {}, "benchmarks": {}, "fixtures": {}, + "doc": {}, "docs": {}, +} + +// skillAuxiliaryScript reports whether a bundled path is one of those files. +// +// The rule is the naming convention the language communities already share, +// not a list grown one skill at a time: a directory the ecosystem reserves for +// tests, examples or docs, or a file name pytest and setuptools already treat +// as theirs. +func skillAuxiliaryScript(rel string) bool { + segments := strings.Split(path.Clean(rel), "/") + for _, segment := range segments[:len(segments)-1] { + if _, ok := auxiliaryScriptDirs[strings.ToLower(segment)]; ok { + return true + } + } + base := strings.ToLower(segments[len(segments)-1]) + stem := strings.TrimSuffix(base, path.Ext(base)) + return stem == "conftest" || stem == "setup" || + strings.HasPrefix(stem, "test_") || strings.HasSuffix(stem, "_test") +} + +// splitAuxiliaryScripts separates the files whose failure is a failed install +// from the ones whose failure is a note, preserving the caller's order. +func splitAuxiliaryScripts(scripts []string) (entry, auxiliary []string) { + for _, rel := range scripts { + if skillAuxiliaryScript(rel) { + auxiliary = append(auxiliary, rel) + continue + } + entry = append(entry, rel) + } + return entry, auxiliary } func forEachScript(skillDir string, scripts []string, check string) string { diff --git a/internal/application/service/tenant_skill_verify.py b/internal/application/service/tenant_skill_verify.py index 0ae0ae7b..78d6264d 100644 --- a/internal/application/service/tenant_skill_verify.py +++ b/internal/application/service/tenant_skill_verify.py @@ -3,14 +3,36 @@ Run inside the sandbox, by the interpreter a real skill call would use, as the ordinary execution user, after the tree has been given its final permissions: - python3 - [ ...] + python3 -