refactor(skills): enhance skill upload size management and error handling

- Updated .env.example to clarify skill zip upload limits and their implications for Nginx configuration.
- Refactored frontend logic to differentiate between skill zip uploads and other uploads, ensuring accurate error messages for oversized requests.
- Improved utility functions for managing upload limits and added tests to validate behavior for oversized JSON bodies.
- Enhanced error handling in various handlers to provide clearer feedback on upload size constraints, particularly for sandbox and catalog installations.
main
wizardchen 2026-09-02 12:22:48 +08:00 committed by lyingbug
parent 17324c46e4
commit bbe95a6c1b
11 changed files with 231 additions and 59 deletions

View File

@ -187,7 +187,7 @@ LOCAL_STORAGE_BASE_DIR=/data/files
# LOCAL_STORAGE_PATH_PREFIX=
# 统一文件大小限制(MB,默认 50)。影响知识库/附件单文件上传、docreader gRPC 消息、浏览器知识库校验。属部署期配置:Go/Nginx/docreader/浏览器四层启动时读一次,运行中改不生效,改后须同步重启四层。
# MAX_FILE_SIZE_MB=50
# 技能 zip 上传与 GitHub/ClawHub 来源下载上限(MB,默认 256)。与知识库上传分开:ppt-master 一类技能包会超过 50MB。GitHub zipball 按整仓压缩包计,不是 SKILL.md 子树;仓过大仍会在下载阶段被拒。未设置时不低于 MAX_FILE_SIZE_MB,且不超过 512。Nginx 只对技能上传的两条路由放宽到这个值,其余路由仍是 MAX_FILE_SIZE_MB。改后须重启 app 与 frontend。
# 技能 zip 上传与 GitHub/ClawHub 来源下载上限(MB,默认 256)。与知识库上传分开:ppt-master 一类技能包会超过 50MB。GitHub zipball 按整仓压缩包计,不是 SKILL.md 子树;仓过大仍会在下载阶段被拒。未设置时不低于 MAX_FILE_SIZE_MB,且不超过 512。Nginx 只对 POST /api/v1/skills/catalog 与 POST /api/v1/sandbox-configs/{id}/skills 放宽到这个值,/install、PATCH 等子路径仍是 MAX_FILE_SIZE_MB。改后须重启 app 与 frontend。
# MAX_SKILL_BUNDLE_SIZE_MB=256
# ========== B4. 对象存储 provider(按 STORAGE_TYPE 选其一)==========

View File

@ -25,9 +25,9 @@ window.__RUNTIME_CONFIG__ = {
EOF
# 处理 nginx 配置。
# 两个上限分开注入:全站保持知识库的 MAX_FILE_SIZE,只有技能包上传的两条路由
# 放宽到 MAX_SKILL_BUNDLE_SIZE。合成一个全站上限会让每个上传端点都能收到
# 技能包那么大的 body。
# 两个上限分开注入:全站保持知识库的 MAX_FILE_SIZE,只有技能 zip 上传的两条
# 集合路由放宽到 MAX_SKILL_BUNDLE_SIZE(不含 /install、PATCH 等子路径)。
# 合成一个全站上限会让每个上传端点都能收到技能包那么大的 body。
export MAX_FILE_SIZE=${FILE_MB}M
export MAX_SKILL_BUNDLE_SIZE=${SKILL_MB}M
export APP_HOST=${APP_HOST:-app}

View File

@ -112,12 +112,13 @@ server {
proxy_set_header X-Forwarded-Proto $scheme;
}
# 技能包上传的两条路由,body 上限放宽到 MAX_SKILL_BUNDLE_SIZE。
# 技能 zip 上传的两条集合路由,body 上限放宽到 MAX_SKILL_BUNDLE_SIZE。
# 单独成一个 location 而不是抬高全站上限:client_max_body_size 是按
# location 生效的,写在 server 级会让知识库上传等端点也能收到技能包大小的
# body。这里放在 /api/ 之前,正则 location 优先于前缀 location 命中。
# 匹配上的 GET(列表、文件内容)不受影响——body 上限对无 body 的请求无意义。
location ~ ^/api/v1/(?:skills/catalog|sandbox-configs/[^/]+/skills)(?:/|$) {
# 只匹配集合本身(POST 上传 / GET 列表),不带 /install、PATCH 等子路径:
# 那些是小 JSON,必须继续走下面 /api/ 的 MAX_FILE_SIZE。
location ~ ^/api/v1/(?:skills/catalog|sandbox-configs/[^/]+/skills)/?$ {
client_max_body_size ${MAX_SKILL_BUNDLE_SIZE};
# 正则 location 的 proxy_pass 不能带 URI,原始路径原样透传,

View File

@ -5,9 +5,19 @@ import { isSkillBundleUploadUrl } from './uploadLimit.ts'
test('skill catalog and sandbox skill uploads use the skill-bundle 413 path', () => {
assert.equal(isSkillBundleUploadUrl('/api/v1/skills/catalog'), true)
assert.equal(isSkillBundleUploadUrl('/api/v1/skills/catalog/'), true)
assert.equal(isSkillBundleUploadUrl('/api/v1/skills/catalog?foo=1'), true)
assert.equal(isSkillBundleUploadUrl('/api/v1/sandbox-configs/cfg-a/skills'), true)
assert.equal(isSkillBundleUploadUrl('https://host/api/v1/sandbox-configs/cfg-a/skills'), true)
assert.equal(isSkillBundleUploadUrl('skills/catalog'), true)
assert.equal(isSkillBundleUploadUrl('sandbox-configs/cfg-a/skills'), true)
})
test('skill JSON subpaths stay on the knowledge-size 413 path', () => {
assert.equal(isSkillBundleUploadUrl('/api/v1/skills/catalog/cat-1/install'), false)
assert.equal(isSkillBundleUploadUrl('/api/v1/skills/catalog/cat-1'), false)
assert.equal(isSkillBundleUploadUrl('/api/v1/sandbox-configs/cfg-a/skills/sk-1'), false)
assert.equal(isSkillBundleUploadUrl('/api/v1/sandbox-configs/cfg-a/skills/sk-1/reinstall'), false)
})
test('knowledge and other API uploads stay on the knowledge-size 413 path', () => {

View File

@ -1,9 +1,9 @@
/** True when nginx's skill-upload location (256MB) applies, not the knowledge 50MB cap. */
export function isSkillBundleUploadUrl(url: string | undefined): boolean {
if (!url) return false
const path = url.split('?')[0]
return /(?:^|\/)api\/v1\/skills\/catalog(?:\/|$)/.test(path)
|| /(?:^|\/)api\/v1\/sandbox-configs\/[^/]+\/skills(?:\/|$)/.test(path)
|| /(?:^|\/)skills\/catalog(?:\/|$)/.test(path)
|| /(?:^|\/)sandbox-configs\/[^/]+\/skills(?:\/|$)/.test(path)
const path = url.split('?')[0].replace(/\/+$/, '')
return /(?:^|\/)api\/v1\/skills\/catalog$/.test(path)
|| /(?:^|\/)api\/v1\/sandbox-configs\/[^/]+\/skills$/.test(path)
|| /(?:^|\/)skills\/catalog$/.test(path)
|| /(?:^|\/)sandbox-configs\/[^/]+\/skills$/.test(path)
}

View File

@ -284,13 +284,17 @@ func (s *TenantSkillService) upsertCatalogFromBundle(
}
now := s.now()
if existing != nil {
stored, releaseReplaced, err := s.storeCatalogBundle(ctx, tenantID, existing, archive, requireStore)
stored, replaced, err := s.storeCatalogBundle(ctx, tenantID, existing, archive, requireStore)
if err != nil {
return nil, err
}
if !stored {
return existing, nil
}
keepOld, err := s.pinReplacedCatalogBundle(ctx, tenantID, existing, replaced)
if err != nil {
return nil, err
}
existing.Version = bundle.Version
existing.Description = bundle.Description
existing.Instructions = bundle.Instructions
@ -300,9 +304,13 @@ func (s *TenantSkillService) upsertCatalogFromBundle(
// The stored definition still names the old archive, so leave that
// archive alone. The object just written is the leak, and it is the
// harmless one: the next re-register overwrites that same key.
// Installs that were pinned above still name the old archive, which
// is also what the uncommitted definition names.
return nil, err
}
releaseReplaced()
if replaced.oldRef != "" && !replaced.sameDigest && !keepOld {
s.deleteBundleBestEffort(ctx, tenantID, replaced.oldRef)
}
return existing, nil
}
@ -312,7 +320,7 @@ func (s *TenantSkillService) upsertCatalogFromBundle(
Instructions: bundle.Instructions,
CreatedAt: now, UpdatedAt: now,
}
stored, releaseReplaced, err := s.storeCatalogBundle(ctx, tenantID, row, archive, requireStore)
stored, _, err := s.storeCatalogBundle(ctx, tenantID, row, archive, requireStore)
if err != nil {
return nil, err
}
@ -338,64 +346,94 @@ func (s *TenantSkillService) upsertCatalogFromBundle(
}
return s.upsertCatalogFromBundle(ctx, tenantID, bundle, archive, requireStore)
}
releaseReplaced()
return row, nil
}
// storeCatalogBundle writes the archive and reports whether the definition now
// names it, along with the work that settles the archive it replaced.
//
// That second return value has to be called by whoever commits the row, and
// only once the commit succeeds. Retiring the old object here instead would
// delete bytes the stored definition still points at the moment the write below
// fails: the row would name a ref that no longer resolves, and the object just
// saved would be unreferenced, which takes the definition's files down for good
// rather than leaving the failed re-register as a no-op.
// catalogBundleReplacement is the archive a definition just stopped naming.
// Pinning has to happen before the catalog row is committed: once the row
// names the new object, an unpinned install with an empty BundleRef would
// follow it and lose the tree its image was built from.
type catalogBundleReplacement struct {
oldRef string
oldSHA string
sameDigest bool
}
// storeCatalogBundle writes the archive and reports whether the in-memory
// definition now names it. The caller must pin any replacement and only then
// commit the row; retiring the old object before that commit would delete
// bytes the stored definition still points at.
func (s *TenantSkillService) storeCatalogBundle(
ctx context.Context, tenantID uint64, catalog *types.TenantSkillCatalogEntity, archive []byte, requireStore bool,
) (bool, func(), error) {
noop := func() {}
) (bool, catalogBundleReplacement, error) {
var none catalogBundleReplacement
if catalog == nil || len(archive) == 0 {
return false, noop, nil
return false, none, nil
}
digest := skillArchiveSHA256(archive)
if strings.TrimSpace(catalog.BundleRef) != "" &&
strings.TrimSpace(catalog.BundleSHA256) != "" &&
catalog.BundleSHA256 == digest {
oldRef := strings.TrimSpace(catalog.BundleRef)
oldSHA := strings.TrimSpace(catalog.BundleSHA256)
if oldRef != "" && oldSHA != "" && oldSHA == digest &&
s.catalogBundleStillHeld(ctx, tenantID, catalog) {
// Same bytes already on the definition. Installing onto another
// sandbox must not mint a second object.
return true, noop, nil
return true, none, nil
}
fs, err := s.fileServiceForTenant(ctx, tenantID)
if err != nil {
if requireStore {
return false, noop, err
return false, none, err
}
logger.Warnf(ctx, "[skill] catalog bundle storage unavailable: %v", err)
return false, noop, nil
return false, none, nil
}
oldRef := strings.TrimSpace(catalog.BundleRef)
oldSHA := strings.TrimSpace(catalog.BundleSHA256)
ref, err := fs.SaveBytes(ctx, archive, tenantID,
fmt.Sprintf("tenant-skills/catalog/%s.zip", catalog.ID), false)
if err != nil {
if requireStore {
return false, noop, fmt.Errorf("store catalog bundle: %w", err)
return false, none, fmt.Errorf("store catalog bundle: %w", err)
}
logger.Warnf(ctx, "[skill] catalog bundle store failed: %v", err)
return false, noop, nil
return false, none, nil
}
catalog.BundleRef = ref
if oldRef == "" || oldRef == ref {
return true, noop, nil
return true, none, nil
}
return true, func() {
if !s.pinInstallsToReplacedBundle(ctx, tenantID, catalog, oldRef, oldSHA) {
s.deleteBundleBestEffort(ctx, tenantID, oldRef)
}
return true, catalogBundleReplacement{
oldRef: oldRef, oldSHA: oldSHA, sameDigest: oldSHA == digest,
}, nil
}
func (s *TenantSkillService) catalogBundleStillHeld(
ctx context.Context, tenantID uint64, catalog *types.TenantSkillCatalogEntity,
) bool {
if catalog == nil || strings.TrimSpace(catalog.BundleRef) == "" {
return false
}
want := strings.TrimSpace(catalog.BundleSHA256)
archive, ok := s.trySkillBundle(ctx, tenantID, &types.TenantSkillEntity{
Name: catalog.Name, BundleRef: catalog.BundleRef, BundleSHA256: catalog.BundleSHA256,
})
return ok && len(archive) > 0 && (want == "" || archiveMatchesSHA(archive, want))
}
// pinReplacedCatalogBundle stamps the previous archive onto installs that are
// still serving it. It must run before the catalog row is committed: a stamp
// that fails after that write leaves those installs following the new object
// while their SHA still names the old tree.
func (s *TenantSkillService) pinReplacedCatalogBundle(
ctx context.Context, tenantID uint64, catalog *types.TenantSkillCatalogEntity, replaced catalogBundleReplacement,
) (bool, error) {
if replaced.oldRef == "" || replaced.sameDigest {
return false, nil
}
if catalog != nil && replaced.oldRef == strings.TrimSpace(catalog.BundleRef) {
return false, nil
}
return s.pinInstallsToReplacedBundle(ctx, tenantID, catalog, replaced.oldRef, replaced.oldSHA)
}
// pinInstallsToReplacedBundle hands the archive a definition just replaced to
// the installs that are still serving it, and reports whether the object has to
// be kept.
@ -407,14 +445,14 @@ func (s *TenantSkillService) storeCatalogBundle(
// once it names it, nothing else has to remember which version it was.
func (s *TenantSkillService) pinInstallsToReplacedBundle(
ctx context.Context, tenantID uint64, catalog *types.TenantSkillCatalogEntity, oldRef, oldSHA string,
) bool {
) (bool, error) {
if catalog == nil || strings.TrimSpace(oldRef) == "" {
return false, nil
}
installs, err := s.skills.ListSkillsByCatalog(ctx, tenantID, catalog.ID)
if err != nil {
// Keeping an archive nothing reads costs storage; dropping one an
// install still needs costs that install its files.
logger.Warnf(ctx, "[skill] list installs of catalog %s before replacing its bundle failed: %v",
return true, fmt.Errorf("list installs of catalog %s before replacing its bundle: %w",
catalog.ID, err)
return true
}
keep := false
for _, row := range installs {
@ -430,17 +468,14 @@ func (s *TenantSkillService) pinInstallsToReplacedBundle(
if strings.TrimSpace(row.BundleSHA256) != oldSHA {
continue
}
// Best-effort: a row that could not be stamped still reads the object
// through the catalog until the next write, and keep is set either way
// so the bytes it needs stay where they are.
if err := s.updateSkillFields(ctx, row.TenantID, row.SandboxConfigID, row.ID,
func(e *types.TenantSkillEntity) { e.BundleRef = oldRef }); err != nil {
logger.Warnf(ctx, "[skill] pin install %s to the replaced archive of catalog %s failed: %v",
return true, fmt.Errorf("pin install %s to the replaced archive of catalog %s: %w",
row.ID, catalog.ID, err)
}
keep = true
}
return keep
return keep, nil
}
func skillUserErrorMessage(err error) string {

View File

@ -347,6 +347,81 @@ func TestRegisterCatalogKeepsTheReplacedZipForInstallsStillOnIt(t *testing.T) {
require.Contains(t, content.Content, "v1")
}
// A stamp that fails must not move the definition: the install still has an
// empty BundleRef, so committing the new catalog object would make read_skill
// and the file browser follow v2 while this sandbox is still running v1.
func TestRegisterCatalogDoesNotMoveTheDefinitionWhenPinFails(t *testing.T) {
fx := newInstallFixture(t)
ctx := context.Background()
first := zipBundle(t, map[string]string{
"SKILL.md": validSkillMD,
"scripts/extract.py": "print('v1')\n",
})
skillID, err := fx.svc.InstallSkill(ctx, 7, "cfg-1", first)
require.NoError(t, err)
installed, err := fx.skillRepo.GetSkill(ctx, 7, "cfg-1", skillID)
require.NoError(t, err)
catalogID := installed.CatalogID
firstRef := fx.catalogRefFor(t, catalogID)
fx.skillRepo.updateFailsWhen = func(e *types.TenantSkillEntity) bool {
return strings.TrimSpace(e.BundleRef) == firstRef
}
second := zipBundle(t, map[string]string{
"SKILL.md": validSkillMD,
"scripts/extract.py": "print('v2')\n",
})
_, err = fx.svc.RegisterCatalogFromArchive(ctx, 7, second)
require.Error(t, err)
stored, err := fx.skillRepo.GetCatalog(ctx, 7, catalogID)
require.NoError(t, err)
require.Equal(t, firstRef, stored.BundleRef,
"the definition must keep naming v1 when the install could not be pinned")
installed, err = fx.skillRepo.GetSkill(ctx, 7, "cfg-1", skillID)
require.NoError(t, err)
require.Empty(t, installed.BundleRef)
content, err := fx.svc.ReadSkillFile(ctx, 7, "cfg-1", skillID, "scripts/extract.py")
require.NoError(t, err)
require.Contains(t, content.Content, "v1")
}
func TestRegisterCatalogSkipsSaveWhenTheSameZipIsStillStored(t *testing.T) {
fx := newInstallFixture(t)
ctx := context.Background()
archive := zipBundle(t, map[string]string{"SKILL.md": validSkillMD})
cat, err := fx.svc.RegisterCatalogFromArchive(ctx, 7, archive)
require.NoError(t, err)
require.Equal(t, 1, fx.savedBundles)
firstRef := cat.BundleRef
cat, err = fx.svc.RegisterCatalogFromArchive(ctx, 7, archive)
require.NoError(t, err)
require.Equal(t, 1, fx.savedBundles, "the same bytes must not mint a second object")
require.Equal(t, firstRef, cat.BundleRef)
}
func TestRegisterCatalogRewritesAMissingObjectWithTheSameSHA(t *testing.T) {
fx := newInstallFixture(t)
ctx := context.Background()
archive := zipBundle(t, map[string]string{"SKILL.md": validSkillMD})
cat, err := fx.svc.RegisterCatalogFromArchive(ctx, 7, archive)
require.NoError(t, err)
firstRef := cat.BundleRef
delete(fx.storedBundles, firstRef)
cat, err = fx.svc.RegisterCatalogFromArchive(ctx, 7, archive)
require.NoError(t, err)
require.Equal(t, 2, fx.savedBundles)
require.NotEqual(t, firstRef, cat.BundleRef)
files, err := fx.svc.ListCatalogFiles(ctx, 7, cat.ID)
require.NoError(t, err)
require.NotEmpty(t, files)
}
// The mirror of the test above: once the last install of a pinned archive is
// gone, nothing can reach those bytes again, so removal is what reclaims them.
func TestRemovingThePinnedInstallReclaimsTheReplacedZip(t *testing.T) {

View File

@ -407,6 +407,10 @@ func skillSourceRequestTooLargeError() error {
return apperrors.NewBadRequestError("skill source request is too large")
}
func skillJSONRequestTooLargeError() error {
return apperrors.NewBadRequestError("skill request is too large")
}
type skillPatchRequest struct {
// Enabled is a pointer because its absence is not a request to disable the
// skill; a body may carry envs instead.
@ -434,8 +438,13 @@ type skillPatchRequest struct {
// @Security ApiKeyAuth
// @Router /sandbox-configs/{id}/skills/{skillId} [patch]
func (h *SandboxSkillHandler) Patch(c *gin.Context) {
limitJSONBody(c, skillSourceJSONMaxBytes)
var req skillPatchRequest
if err := c.ShouldBindJSON(&req); err != nil {
if isRequestBodyTooLarge(err) {
_ = c.Error(skillJSONRequestTooLargeError())
return
}
_ = c.Error(apperrors.NewBadRequestError(err.Error()))
return
}

View File

@ -577,6 +577,26 @@ func TestSandboxSkillPatchWithoutEnabledFieldReturns400(t *testing.T) {
require.True(t, svc.skills["skill-1"].Enabled)
}
func TestSandboxSkillPatchRejectsAnOversizedJSONBody(t *testing.T) {
svc := &fakeSandboxSkillService{skills: map[string]*types.TenantSkillEntity{
"skill-1": {
ID: "skill-1", TenantID: testSkillTenantID, SandboxConfigID: "cfg-a",
Name: "pdf", Status: types.SkillStatusReady, Enabled: true,
},
}}
router := newSkillTestRouter(NewSandboxSkillHandler(svc, nil))
req := httptest.NewRequest(http.MethodPatch, "/sandbox-configs/cfg-a/skills/skill-1",
bytes.NewReader(oversizedSkillSourceJSON(1)))
req.Header.Set("Content-Type", "application/json")
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
require.Equal(t, http.StatusBadRequest, w.Code)
require.Contains(t, w.Body.String(), "skill request is too large")
require.Equal(t, 0, svc.patchCalls)
}
// Removal rebuilds the image, so it is accepted and followed, never awaited.
func TestSandboxSkillDeleteIsAccepted(t *testing.T) {
svc := &fakeSandboxSkillService{}

View File

@ -136,8 +136,13 @@ func (h *SkillHandler) InstallCatalog(c *gin.Context) {
_ = c.Error(apperrors.NewInternalServerError("skill catalog is not configured"))
return
}
limitJSONBody(c, skillSourceJSONMaxBytes)
var req catalogInstallRequest
if err := c.ShouldBindJSON(&req); err != nil {
if isRequestBodyTooLarge(err) {
_ = c.Error(skillJSONRequestTooLargeError())
return
}
_ = c.Error(apperrors.NewBadRequestError("invalid install request"))
return
}

View File

@ -19,12 +19,13 @@ import (
)
type fakeSkillCatalog struct {
list []service.SkillCatalogView
registerID string
installs map[string]string
installErrs map[string]string
deleteErr error
source string
list []service.SkillCatalogView
registerID string
installs map[string]string
installErrs map[string]string
deleteErr error
source string
installCalls int
}
func (f *fakeSkillCatalog) ListCatalog(context.Context, uint64) ([]service.SkillCatalogView, error) {
@ -47,6 +48,7 @@ func (f *fakeSkillCatalog) RegisterCatalogFromSource(
func (f *fakeSkillCatalog) InstallCatalogToConfigs(
context.Context, uint64, string, []string,
) (*service.CatalogInstallResult, error) {
f.installCalls++
return &service.CatalogInstallResult{Installs: f.installs, Errors: f.installErrs}, nil
}
@ -134,6 +136,21 @@ func TestRegisterCatalogFromSourceRejectsAnOversizedJSONBody(t *testing.T) {
require.Empty(t, catalog.source)
}
func TestInstallCatalogRejectsAnOversizedJSONBody(t *testing.T) {
catalog := &fakeSkillCatalog{installs: map[string]string{"cfg-1": "sk-1"}}
router := newCatalogRouter(NewSkillHandler(&fakeUsableSkillLister{}, catalog))
req := httptest.NewRequest(http.MethodPost, "/skills/catalog/cat-1/install",
bytes.NewReader(oversizedSkillSourceJSON(1)))
req.Header.Set("Content-Type", "application/json")
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
require.Equal(t, http.StatusBadRequest, w.Code)
require.Contains(t, w.Body.String(), "skill request is too large")
require.Equal(t, 0, catalog.installCalls)
}
func TestInstallCatalogAcceptsPerConfigIDs(t *testing.T) {
catalog := &fakeSkillCatalog{installs: map[string]string{"cfg-1": "sk-1"}}
router := newCatalogRouter(NewSkillHandler(&fakeUsableSkillLister{}, catalog))