From 3d81b977f12f7c19bb560d2118a7651f7fa00ba8 Mon Sep 17 00:00:00 2001 From: lyingbug Date: Tue, 8 Sep 2026 15:36:33 +0800 Subject: [PATCH] refactor(resource-access): unify scoped authorization and transfer recovery (#3097) * refactor(resource-access): unify scoped authorization and transfer recovery * fix(resource-access): close reparse and transfer review gaps * fix: enforce explicit file bindings and recover move cleanup * fix: align mutation admission and cleanup scope checks --- internal/application/access/README.md | 294 +++++ internal/application/access/context.go | 94 ++ internal/application/access/context_test.go | 140 +++ internal/application/access/files.go | 256 ++++ internal/application/access/files_test.go | 162 +++ internal/application/access/kb_shares.go | 62 + internal/application/access/kb_shares_test.go | 84 ++ internal/application/access/kb_transfer.go | 227 ++++ .../application/access/kb_transfer_test.go | 96 ++ internal/application/access/kb_write.go | 45 + internal/application/access/kb_write_test.go | 85 ++ .../application/access/knowledge_state.go | 38 + internal/application/access/knowledgebase.go | 156 +++ .../application/access/knowledgebase_test.go | 282 +++++ internal/application/access/message_files.go | 249 ++++ internal/application/access/ownership.go | 69 ++ internal/application/access/ownership_test.go | 103 ++ internal/application/repository/chunk.go | 54 +- .../repository/chunk_fields_test.go | 60 + internal/application/repository/knowledge.go | 11 +- .../repository/knowledge_finalize_test.go | 61 +- .../repository/knowledge_list_filter_test.go | 2 +- .../repository/knowledge_transfer.go | 74 ++ .../repository/knowledge_transfer_test.go | 30 + .../repository/resource_references.go | 57 + .../repository/retriever/doris/move.go | 41 + .../repository/retriever/doris/move_test.go | 36 + .../retriever/elasticsearch/v7/move.go | 59 + .../retriever/elasticsearch/v7/move_test.go | 59 + .../retriever/elasticsearch/v8/move.go | 48 + .../retriever/elasticsearch/v8/move_test.go | 59 + .../repository/retriever/milvus/move.go | 61 + .../repository/retriever/milvus/move_test.go | 82 ++ .../retriever/opensearch/byquery.go | 16 +- .../repository/retriever/opensearch/move.go | 49 + .../retriever/opensearch/move_test.go | 61 + .../repository/retriever/postgres/move.go | 16 + .../repository/retriever/qdrant/move.go | 26 + .../repository/retriever/sqlite/move.go | 17 + .../repository/retriever/sqlite/move_test.go | 72 ++ .../retriever/tencentvectordb/move.go | 33 + .../repository/retriever/weaviate/move.go | 77 ++ .../retriever/weaviate/move_test.go | 130 +++ internal/application/service/agent_share.go | 20 +- internal/application/service/chunk.go | 37 +- .../service/chunk_edit_parent_test.go | 17 +- internal/application/service/chunk_write.go | 143 +++ internal/application/service/custom_agent.go | 21 +- .../custom_agent_suggestion_scope_test.go | 10 +- .../application/service/datasource_service.go | 13 +- .../service/datasource_service_test.go | 8 +- .../service/document_write_access_test.go | 660 +++++++++++ internal/application/service/evaluation.go | 5 +- internal/application/service/extract.go | 40 +- .../service/file/resolve_tenant.go | 69 ++ .../application/service/image_multimodal.go | 6 +- internal/application/service/kbshare.go | 10 +- internal/application/service/knowledge.go | 122 +- .../application/service/knowledge_auto_tag.go | 10 +- .../service/knowledge_auto_tag_test.go | 8 +- .../service/knowledge_batch_reparse_test.go | 14 +- .../service/knowledge_caller_scope_test.go | 186 +++ .../knowledge_cleanup_regression_test.go | 133 +++ .../service/knowledge_clone_move.go | 1029 +++++++++-------- .../application/service/knowledge_create.go | 13 +- .../application/service/knowledge_delete.go | 387 +++---- .../service/knowledge_delete_plan.go | 169 +++ internal/application/service/knowledge_faq.go | 275 ++--- .../service/knowledge_faq_batch.go | 149 +++ .../service/knowledge_faq_import.go | 68 +- .../service/knowledge_folder_move_test.go | 34 +- .../service/knowledge_index_content_test.go | 36 +- .../service/knowledge_move_wiki_test.go | 241 +++- .../service/knowledge_post_process.go | 53 +- ...nowledge_post_process_wiki_enqueue_test.go | 27 +- .../application/service/knowledge_process.go | 194 +++- .../knowledge_process_parent_child_test.go | 6 +- .../service/knowledge_reparse_scope.go | 60 + .../service/knowledge_shared_access_test.go | 77 +- .../knowledge_shared_storage_failure_test.go | 46 + .../application/service/knowledge_transfer.go | 525 +++++++++ .../service/knowledge_transfer_test.go | 457 ++++++++ .../service/knowledge_update_test.go | 45 + .../application/service/knowledge_write.go | 141 +++ .../service/knowledge_write_access_test.go | 442 +++++++ internal/application/service/knowledgebase.go | 100 +- .../service/knowledgebase_access.go | 74 ++ .../service/knowledgebase_pr3_test.go | 45 +- .../service/knowledgebase_search.go | 11 +- .../knowledgebase_search_fanout_test.go | 37 +- .../service/knowledgebase_search_shared.go | 126 +- .../knowledgebase_search_storegroup.go | 25 +- internal/application/service/message.go | 4 +- internal/application/service/resource.go | 41 + .../resource_file_authorization_test.go | 169 +++ .../service/resource_references_test.go | 71 ++ .../resource_review_regression_test.go | 284 +++++ .../service/retriever/composite.go | 39 + .../keywords_vector_hybrid_indexer.go | 35 + .../service/retriever/move_test.go | 38 + internal/application/service/session.go | 13 +- .../service/session_knowledge_qa.go | 25 +- internal/application/service/tag.go | 65 +- internal/application/service/tag_access.go | 60 + .../application/service/web_search_state.go | 2 +- internal/application/service/wiki_ingest.go | 18 +- internal/filetransport/response.go | 70 ++ internal/filetransport/response_test.go | 77 ++ internal/handler/chunk.go | 14 +- internal/handler/document_write_scope_test.go | 99 ++ internal/handler/kb_access.go | 81 ++ internal/handler/kb_access_test.go | 229 ++++ internal/handler/knowledge.go | 581 ++++------ .../knowledge_mutation_admission_test.go | 253 ++++ internal/handler/knowledge_ownership_test.go | 157 +++ internal/handler/knowledge_transfer_test.go | 176 +++ internal/handler/knowledgebase.go | 341 ++---- internal/handler/rbac_lookups.go | 69 +- internal/handler/session/artifact_download.go | 78 +- .../handler/session/artifact_download_test.go | 125 +- internal/handler/session/handler.go | 3 + internal/handler/session/qa.go | 15 +- .../handler/session/temporary_document.go | 25 +- internal/handler/session/wiki_fixer_scope.go | 10 +- internal/handler/shared_agent_access.go | 82 ++ internal/handler/shared_agent_access_test.go | 245 ++++ internal/handler/tag.go | 40 +- internal/handler/tag_delete_test.go | 109 ++ internal/middleware/auth_context.go | 5 + internal/middleware/auth_context_test.go | 4 + internal/middleware/kb_access.go | 241 +--- internal/middleware/kb_access_test.go | 34 +- internal/middleware/rbac.go | 182 ++- .../rbac_api_key_shortcircuit_test.go | 9 +- internal/middleware/rbac_audit_test.go | 57 + internal/router/files.go | 628 ++-------- internal/router/rbac.go | 2 +- internal/router/router_files_test.go | 234 +++- internal/router/routes_knowledge.go | 4 +- internal/router/task.go | 79 +- internal/router/task_delete_scope_test.go | 55 + internal/router/task_processing_scope_test.go | 72 ++ internal/storageurl/storageurl.go | 8 +- internal/types/caller.go | 43 + internal/types/const.go | 10 +- internal/types/context_clone.go | 16 +- internal/types/context_helpers.go | 3 +- internal/types/file_reference.go | 56 + internal/types/file_reference_test.go | 34 + internal/types/interfaces/chunk.go | 7 + internal/types/interfaces/knowledge.go | 16 +- internal/types/interfaces/resource.go | 26 + internal/types/interfaces/retriever.go | 15 + internal/types/knowledge.go | 20 + internal/types/resource.go | 7 + internal/types/shared_agent_access.go | 60 + internal/types/shared_agent_access_test.go | 54 + internal/types/task.go | 20 +- 158 files changed, 12746 insertions(+), 3155 deletions(-) create mode 100644 internal/application/access/README.md create mode 100644 internal/application/access/context.go create mode 100644 internal/application/access/context_test.go create mode 100644 internal/application/access/files.go create mode 100644 internal/application/access/files_test.go create mode 100644 internal/application/access/kb_shares.go create mode 100644 internal/application/access/kb_shares_test.go create mode 100644 internal/application/access/kb_transfer.go create mode 100644 internal/application/access/kb_transfer_test.go create mode 100644 internal/application/access/kb_write.go create mode 100644 internal/application/access/kb_write_test.go create mode 100644 internal/application/access/knowledge_state.go create mode 100644 internal/application/access/knowledgebase.go create mode 100644 internal/application/access/knowledgebase_test.go create mode 100644 internal/application/access/message_files.go create mode 100644 internal/application/access/ownership.go create mode 100644 internal/application/access/ownership_test.go create mode 100644 internal/application/repository/chunk_fields_test.go create mode 100644 internal/application/repository/knowledge_transfer.go create mode 100644 internal/application/repository/knowledge_transfer_test.go create mode 100644 internal/application/repository/resource_references.go create mode 100644 internal/application/repository/retriever/doris/move.go create mode 100644 internal/application/repository/retriever/doris/move_test.go create mode 100644 internal/application/repository/retriever/elasticsearch/v7/move.go create mode 100644 internal/application/repository/retriever/elasticsearch/v7/move_test.go create mode 100644 internal/application/repository/retriever/elasticsearch/v8/move.go create mode 100644 internal/application/repository/retriever/elasticsearch/v8/move_test.go create mode 100644 internal/application/repository/retriever/milvus/move.go create mode 100644 internal/application/repository/retriever/milvus/move_test.go create mode 100644 internal/application/repository/retriever/opensearch/move.go create mode 100644 internal/application/repository/retriever/opensearch/move_test.go create mode 100644 internal/application/repository/retriever/postgres/move.go create mode 100644 internal/application/repository/retriever/qdrant/move.go create mode 100644 internal/application/repository/retriever/sqlite/move.go create mode 100644 internal/application/repository/retriever/sqlite/move_test.go create mode 100644 internal/application/repository/retriever/tencentvectordb/move.go create mode 100644 internal/application/repository/retriever/weaviate/move.go create mode 100644 internal/application/repository/retriever/weaviate/move_test.go create mode 100644 internal/application/service/chunk_write.go create mode 100644 internal/application/service/document_write_access_test.go create mode 100644 internal/application/service/file/resolve_tenant.go create mode 100644 internal/application/service/knowledge_caller_scope_test.go create mode 100644 internal/application/service/knowledge_cleanup_regression_test.go create mode 100644 internal/application/service/knowledge_delete_plan.go create mode 100644 internal/application/service/knowledge_faq_batch.go create mode 100644 internal/application/service/knowledge_reparse_scope.go create mode 100644 internal/application/service/knowledge_shared_storage_failure_test.go create mode 100644 internal/application/service/knowledge_transfer.go create mode 100644 internal/application/service/knowledge_transfer_test.go create mode 100644 internal/application/service/knowledge_write.go create mode 100644 internal/application/service/knowledge_write_access_test.go create mode 100644 internal/application/service/knowledgebase_access.go create mode 100644 internal/application/service/resource_file_authorization_test.go create mode 100644 internal/application/service/resource_references_test.go create mode 100644 internal/application/service/resource_review_regression_test.go create mode 100644 internal/application/service/retriever/move_test.go create mode 100644 internal/application/service/tag_access.go create mode 100644 internal/filetransport/response.go create mode 100644 internal/filetransport/response_test.go create mode 100644 internal/handler/document_write_scope_test.go create mode 100644 internal/handler/kb_access.go create mode 100644 internal/handler/kb_access_test.go create mode 100644 internal/handler/knowledge_mutation_admission_test.go create mode 100644 internal/handler/knowledge_ownership_test.go create mode 100644 internal/handler/knowledge_transfer_test.go create mode 100644 internal/handler/shared_agent_access.go create mode 100644 internal/handler/shared_agent_access_test.go create mode 100644 internal/handler/tag_delete_test.go create mode 100644 internal/router/task_delete_scope_test.go create mode 100644 internal/router/task_processing_scope_test.go create mode 100644 internal/types/caller.go create mode 100644 internal/types/file_reference.go create mode 100644 internal/types/file_reference_test.go create mode 100644 internal/types/shared_agent_access.go create mode 100644 internal/types/shared_agent_access_test.go diff --git a/internal/application/access/README.md b/internal/application/access/README.md new file mode 100644 index 00000000..2e199178 --- /dev/null +++ b/internal/application/access/README.md @@ -0,0 +1,294 @@ +# Resource access boundaries + +`access` contains shared permission resolution without depending on Gin or the +application service implementations. HTTP adapters keep parameter parsing, status +codes, RBAC rollout behavior, and response projection at their existing boundaries. + +| Entry point | Shared rule | Additional boundary | +| --- | --- | --- | +| Resource mutations with URL/body KB selectors | `CheckOwnershipOrRole`: creator or sufficient tenant role | API-key capabilities, KB scope and tenant visibility remain separate checks | +| KB/document detail and content routes | `ResolveKB`: own tenant, organization share, then read-only shared agent | Tenant role/ownership guards and API-key capabilities | +| Shared documents, chunks, search targets and suggestions | `KBSharePermissions`: organization grants only | Existing user requirements and caller-specific filtering/error behavior | +| FAQ and tag service reads | `resolveKBReadTenant`: execution tenant or organization share | Direct cross-tenant calls require a user; no implicit agent fallback | +| KB list, document search and batch restoration with `agent_id` | Shared-agent lookup, then `SharedAgentKBScope` | API-key intersection; dynamic list/search selections also apply capability filtering | +| Wiki fixer source tenant | Organization Editor permission | Built-in fixer only; exactly one KB | +| Message file shared-KB fallback | Organization Viewer permission | Persisted message reference, exact resource handle, KB/resource owner match | +| KB file proxy | `ResolveKBFile`: exact Viewer grant and current explicit KB binding | Owner tenant and exports namespace; original uploads retain their separate download permission | +| Message file proxy | `ResolveMessageFile`: session ownership and exact persisted output reference | Current shared-agent or organization-KB permission for source-owned resources | +| Message artifact download | `ResolveMessageArtifact`: session ownership and persisted artifact index | Session-owned output remains accessible; source-owned output rechecks current sharing | +| FAQ and tag mutations | `RequireKBWrite`: exact Editor operation grant | API-key ingest capability and KB scope; tenant-role/creator checks remain at the admission boundary | + +## Ownership and write roles + +URL middleware and body-based handlers use the same lazy ownership decision. +The middleware adapter resolves identity, RBAC rollout configuration and the +configuration-gated superuser flag. API-key principals, sufficient roles, +cross-tenant superusers and disabled enforcement skip the creator lookup. +API keys still require their route capabilities and allowed KB scope. + +When a lookup is needed, it must check tenant visibility before returning the +creator. An empty creator never matches an empty user. Middleware passes missing +resources to the handler, reports lookup failures as 503 and audits only actual +policy denials. Body-based handlers retain their existing 404/403/500 mapping. +KB, initialization, Wiki and body-based KB lookups share the same tenant check. + +## Caller and resource tenant + +`types.Caller` holds the authenticated tenant, user and tenant role. Auth records +it once; `KBRequest.Caller` and `KBAccess.Caller` use that same identity. The +legacy `TenantIDContextKey` continues to scope repository/model execution, so +existing storage APIs do not need a coordinated signature migration. + +Use `types.WithExecutionTenant` to switch execution tenants. It captures the +caller first, even when identity is missing, and grants no access by itself. +Use `KBAccess.Context` after authorization to carry both the exact KB grant and +its execution tenant, or `WithGrant` to carry the grant without switching yet. +Grants bind the KB ID, owner tenant, caller and resolved permission; changing +execution tenant cannot turn a Viewer grant into Editor or expose another KB. +HTTP adapters retain Gin's caller keys for existing handlers. + +Service reads use `KBPermissions` to check caller ownership, exact upstream +grants, then organization shares for the original caller. This check also applies +to documents/chunks returned by the initial tenant-scoped batch query, not just +rows found during cross-tenant expansion. Search preflight, result enrichment, +FAQ/tag reads, search targets and suggestion scopes follow the same boundary. +General document search enumerates the caller's own/shared KBs; its execution +tenant is not a new identity. + +After authorizing a shared agent, `WithSharedAgent` carries its configured, +source-tenant-bound read scope into the pipeline. An arbitrary execution switch +cannot substitute for that grant. `logger.CloneContext` preserves the caller and +resource grants for detached work within the operation. Context grants contain +immutable snapshots; deriving a context does not mutate a sibling search branch. +They are not persisted as authority for subsequent requests. API-key scope is +reapplied when consuming any grant. + +## Organization shares + +`KBSharePermissions` fixes the caller tenant and role for one operation and caches +the granular decision per KB. Reading many documents/chunks from one KB therefore +queries its membership once. Both denied decisions and errors are cached within +that operation. The next request creates a new resolver and observes revocations. +The resolver is not a process-wide cache and is not shared between goroutines. + +The share service remains responsible for the role cap across share permission, +organization membership and caller tenant role. The resolver returns lookup errors +so strict search preflight can report an infrastructure failure while best-effort +result expansion can omit inaccessible rows. `ResolveKB` retains its independent +read-only agent fallback when organization resolution cannot grant access. + +## Shared-agent KB scope + +A scope is created only after looking up an accessible shared agent. `all` means +all KBs in that agent's tenant; `selected` means its explicit nonempty ID set. +`none`, unknown modes and empty selections authorize no KBs. A nil/empty ID slice +never implicitly grants an entire tenant. Specifying an agent does not permit +falling back to a different shared agent. + +These rules do not replace message/session ownership, persisted file-reference +validation, resource binding, Agent tool search-target restrictions, or embed/IM +capability checks. Raw tenant files and message files have different authorization +contracts and must not be admitted by a generic KB grant alone. + +## FAQ and tag writes + +FAQ/tag service mutations consume an explicit Editor grant; setting an execution +tenant or supplying a KB ID is insufficient. `ResolveKB` keeps the effective +permission used for response projection, but caps its context grant to the +requested operation. Thus an owner read does not mint a reusable write grant. +HTTP routes retain their existing role/creator admission checks. API-key ingest +capability and KB scope are reapplied when consuming a write grant. + +Trusted FAQ-import, data-source sync and automatic-tagging workers use +`WithKBTaskWrite` after loading and validating their target KB and child-resource +bindings. This represents continuation of an admitted job, not renewed user +authorization on every retry. It carries exactly one KB, preserves the caller +or its absence, survives in-operation context cloning, and grants no tenant-wide +role. Services use the KB owner's TenantInfo for model/index configuration. + +Batch FAQ updates resolve all source/destination tags, entries and exclusions +before mutation. Batch deletion also resolves all parent documents first. +Tag-only batch updates use the same plan. Tag groups execute in ascending seq_id +order, then explicit entry updates override only their specified fields. +Import retains per-entry content-validation results, but out-of-scope resource +references are rejected before admission and checked again in the worker. +This preflight prevents partial writes caused by invalid resource selections; +it does not add a transaction across the database, task queue and search index. + +## File authorization and transport + +File resolvers return request-local `FileAccess` locators before any storage +reader is opened. The KB catalog accepts only explicit resource bindings to live +documents in the exact live KB. Registered physical aliases resolve to the same +resource ID; chunk/wiki text is never authorization evidence. Historical files +without explicit bindings fail closed and need a trusted migration. A same-tenant +path or handle prefix cannot replace a KB binding. Message evidence comes from +persisted content, references, images, artifacts and tool results; tool arguments +do not prove that the message returned a file. The organization-shared KB +fallback also checks the catalog for an independent live KB/file binding; +retrieval text and a currently shared KB ID alone are insufficient. + +Source-owned shared-agent files also require the current live agent share. +KB sources need an explicit live KB binding allowed by the agent's current +`all`/`selected` selection and the caller's API-key KB scope; `none`, unknown +modes, removed KB selections and deleted sources fail closed. Independently +generated artifacts remain accessible when explicitly bound to the exact +authorized message as an artifact. Mentioning a file in message text or the +artifact list does not create that binding. Caller-owned artifacts retain their +session ownership contract. + +The storage adapter uses the authorized owner and backend. The shared +`filetransport` package owns response headers, safe inline/download disposition, +streaming and reader closure. Seekable readers use standard HTTP Range and HEAD +handling; streaming-only backends return full content with `Accept-Ranges: none` +without buffering the object (RFC 9110 sections 14.2 and 14.3). Permission-controlled responses use `private, no-store` so a later +request rechecks current access. Tenant-wide and signed-token file routes keep +their own authorization and caching contracts. + +## Document and chunk writes + +Document metadata, manual content, reparse, document tags/folders, image edits, chunk mutations +and generated-question edits consume explicit Editor grants after loading the +persisted document/KB binding. Full chunk objects cannot reparent existing rows. +Batch inputs are deduplicated and fully validated, including missing IDs and +each KB's grant, before writes. Text edits also check the parent/image-child +relationships that they will modify. Body-based HTTP routes explicitly resolve +an Editor operation; a previously cached owner read is not a write admission. + +Document deletion shares a preflight plan and executor for single and batch +calls. The plan validates every selection and KB, loads owner TenantInfo and +routing metadata, and captures image references before marking rows as deleting. +Index/graph failures leave the document and original file available for retry. +Wiki deletion collects all chunk types before deleting chunks, so parent, image +and summary citations are removed along with text citations. +Physical file cleanup remains best effort after database deletion; this does not +introduce a transaction across database, indexes and storage or durable blob GC. + +New queued deletion payloads pin `knowledge_base_id` as well as tenant and exact +document IDs. Workers consume a private cleanup scope bound to those references, +not general KB write permission; already deleted IDs are successful no-ops. +A moved document rejects the batch before side effects. Pre-upgrade payloads that +omit the KB are accepted only when remaining rows resolve to one same-tenant KB; +they cannot reconstruct the KB that existed at enqueue time. Failure callbacks +skip rejected scopes and condition status updates on tenant, KB and deleting +state. Scope/identity errors are not retried; infrastructure failures are. + +Single/batch delete, clear-content and batch-reparse admission reject already +moving documents with HTTP 409 before enqueue. Workers recheck current state; +a deletion conflict is terminal for that admitted task, so retries cannot +follow a document into a different KB. This does not serialize admission with +a concurrent move. Synchronous metadata/image/chunk/summary handlers preserve +application 403/409 errors, including wrapped errors. + +Batch reparse and batch document tags resolve an Editor operation and apply the +same creator-or-Admin admission policy as single-document writes, respecting +RBAC rollout/API-key behavior. Batch reparse payloads likewise carry the admitted +KB ID. Workers validate the whole batch before any submission and use exactly one `WithKBTaskWrite` grant; +each reparse reload consumes that grant and rejects a document still moving. +Missing/mixed/moved selections fail without submitting earlier batch members. +Legacy reparse payloads can reconstruct only a present, single-KB binding. + +Session-history, temporary-KB and clone cleanup use exact server-derived +references from their already admitted operations. These cleanup contexts cannot +be used for document/chunk updates or additional IDs. Parsing and enrichment +workers use repositories for their internal persistence and rollback; they must +keep their existing task ownership checks, abort checks and resource bindings. +Repository access is not an incoming-request authorization API. + +## Clone and move operations + +`WithKBTransfer` admits the exact source/target pair after HTTP role/creator +checks. Clone consumes source Viewer, target Editor and the API-key +`manage_kbs` capability; move consumes Editor on both sides and `ingest`. +Both operations remain within one tenant. Creating a clone reserves its target +ID before enqueue and carries its creator into the worker. The ID of a new +resource is not checked against an existing-resource API-key allow-list. + +Workers reload both KBs, validate their owners and consume a private pair grant. +Document/FAQ plans validate all selected resources, chunks, parents and tags +before destructive work. Lifecycle reads include every chunk type and indexing +state. A document's `_knowledge_transfer` metadata records task identity and +progress; retries reuse the reserved target, replace incomplete clones, skip +completed vector moves and retry outstanding wiki cleanup/parser enqueue. +This field is internal operation state, not caller-supplied custom metadata. +For `reparse`, transfer `done` means target binding and parser admission are +complete, not that parsing has finished. `ParseStatus` remains the parser's +state. `reparse_pending` already admits the target parser; extending the +`moving` write guard to that phase would also block legitimate worker writes. +Locking all writes until parsing finishes is a separate processing/concurrency +contract, including ordinary reparses, and is not provided by this transfer marker. + +Failed clone chunk/index writes attempt rollback after conditionally claiming +the original destination. Rollback failures are returned, references retained +when cleanup is uncertain, and later retries replace the failed copy. Accounting +is committed only after the clone succeeds. FAQ containers do not inherit the +source document's transfer marker. + +Source wiki cleanup starts only after the document ownership CAS succeeds. +The transfer checkpoint preserves source chunk IDs and summary before reparse +removes them. Retract page lists are persisted before page source refs are removed, so queue +failures retain retry evidence. Cleanup errors propagate; redelivery resumes cleanup after a +committed move without repeating vector/chunk relocation. These retries do not +provide a distributed transaction or recovery after retries are exhausted. + +Vector reuse relocates KB metadata while preserving physical vector/chunk IDs. +Weaviate and Milvus drain the first source-filtered page until a follow-up query +is empty, avoiding offset limits/skipped rows. Repeated IDs or partial failures +return errors and leave the document checkpoint pending for retry. +OpenSearch selects the entire source KB/document pair, even without DB chunks, +and rejects version conflicts, timeouts and partial failures before DB relocation. +OpenSearch and Elasticsearch 7/8 require present non-negative `total`/`updated` +counts with `updated == total`; their move scripts do not skip/delete documents. +A zero/zero result allows a retry after metadata already reached the target. +Reparse cleanup loads and validates the source KB before touching any backend, +reusing its storage binding throughout; a failed/missing lookup cannot fall +back to the tenant default store. +Vector reuse never copies vectors and then deletes by the same document ID. Source and +target must use the same vector store for reuse, and the same concrete file +storage instance for either mode. Doris ANN tables cannot safely replace rows +without a delete/insert gap, so their moves require `reparse`. Clone still +requires compatible embedding models and vector stores; a move also retains +the existing same-model requirement. This is not a cross-storage migration API. + +Conditional document checkpoints bind tenant, KB, status and the stored update +time. They keep storage accounting in the same database transaction and read +back timestamp precision. Delete stops when its checkpoint no longer matches. +Ordinary document/chunk writes reject a document in the `moving` phase. Parser +admission and failure callbacks verify the task's persisted tenant/KB binding; +table-summary failure cleanup also stops if its document changed. + +## Follow-up work outside this refactor + +These are separate follow-ups, not capabilities provided by this package: + +1. **Complete worker contracts and concurrency control.** Audit enrichment, + table-summary, multimodal, wiki and indexing workers end to end. Standardize + payload resource bindings, processing-attempt IDs and conditional writes, + including operations admitted before a concurrent move. Do not replace + these checks with an execution-tenant switch or a blanket task write grant. +2. **Durable recovery across systems.** Add a transactional outbox and explicit + recovery/cancellation for exhausted transfer tasks. Current retries use the + same admitted task identity; a new move request is not a resume API. Database, + queue, vector, graph and blob operations are not one atomic transaction. + Physical file cleanup remains best effort and needs durable garbage collection. +3. **External-backend integration coverage.** Exercise clone/move, partial + backend failures, retries, pagination and search/edit behavior against live + PostgreSQL, Elasticsearch 7/8, OpenSearch, Qdrant, Milvus, Weaviate, + TencentVectorDB and Doris. Local SQLite vector tests and backend compilation + do not establish this coverage. Doris ANN vector reuse remains unsupported. +4. **Finish service-layer migration.** Apply explicit operation grants to the + remaining KB configuration/duplicate/delete, wiki and resource-administration + service methods, preserving their existing route admission policies. Keep + user/organization membership, embed, IM and signed-resource contracts separate. +5. **Trusted legacy file migration.** Backfill historical file bindings from + verified provenance, not arbitrary chunk/wiki mentions. Unbound historical + files now fail closed; request-path text fallback has been removed. Migrate + old queued payloads and transfer records without captured source wiki refs. + A legacy delete task without a KB ID can reconstruct only its present + unambiguous binding, not its original enqueue-time binding. + +No public request schema or database column migration is required by this +refactor. Internal service interfaces require operation contexts, queued payloads +add optional scope fields, and malformed/out-of-scope batch selections now fail +before mutation instead of being silently skipped. Callers must also handle a +409 for document edits/deletes while a move is in progress. diff --git a/internal/application/access/context.go b/internal/application/access/context.go new file mode 100644 index 00000000..fbfe0796 --- /dev/null +++ b/internal/application/access/context.go @@ -0,0 +1,94 @@ +package access + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/types" +) + +// Context grants contain values, not mutable KB objects. Deriving a context +// copies the grant slice so parallel search branches cannot widen each other. +type kbGrant struct { + caller types.Caller + kbID string + tenantID uint64 + permission types.OrgMemberRole + task bool +} + +type agentGrant struct { + caller types.Caller + scope types.SharedAgentKBScope +} + +// WithSharedAgent scopes execution after the caller has authorized this exact +// agent. Only its configured KBs receive read access; other source-tenant KBs +// and the source tenant's organization memberships are not inherited. +func WithSharedAgent(ctx context.Context, agent *types.CustomAgent) context.Context { + grant := agentGrant{caller: types.CallerFromContext(ctx), scope: types.NewSharedAgentKBScope(agent)} + ctx = context.WithValue(ctx, types.SharedAgentGrantContextKey, grant) + if agent == nil { + return ctx + } + return types.WithExecutionTenant(ctx, agent.TenantID) +} + +// HasKBGrant checks only previously resolved resource access. It never infers +// ownership from the execution tenant, and always reapplies API-key scope. +func HasKBGrant(ctx context.Context, kbID string, tenantID uint64, required types.OrgMemberRole) bool { + if !required.IsValid() || types.AuthorizeTenantAPIKeyKnowledgeBases(ctx, kbID) != nil { + return false + } + caller := types.CallerFromContext(ctx) + if kbID == "" || tenantID == 0 { + return false + } + grants, _ := ctx.Value(types.KBGrantsContextKey).([]kbGrant) + for _, grant := range grants { + if (caller.TenantID != 0 || grant.task) && grant.caller == caller && grant.kbID == kbID && + grant.tenantID == tenantID && + grant.permission.HasPermission(required) { + return true + } + } + grant, ok := ctx.Value(types.SharedAgentGrantContextKey).(agentGrant) + return ok && caller.TenantID != 0 && required == types.OrgRoleViewer && grant.caller == caller && + grant.scope.Allows(kbID, tenantID) +} + +// KBPermissions combines caller ownership, exact context grants and organization +// shares for one service operation. Pass nil shares when that entry point does +// not permit organization expansion (for example, a caller without a user). +type KBPermissions struct { + ctx context.Context + caller types.Caller + shares *KBSharePermissions +} + +// NewKBPermissions resolves reads for one caller and operation. +func NewKBPermissions(ctx context.Context, shares KBShareLookup) *KBPermissions { + caller := types.CallerFromContext(ctx) + return &KBPermissions{ + ctx: ctx, + caller: caller, + shares: NewKBSharePermissions(ctx, shares, caller.TenantID, caller.Role), + } +} + +// Check combines caller ownership, exact grants and organization sharing. +func (p *KBPermissions) Check(kbID string, ownerTenantID uint64, required types.OrgMemberRole) (bool, error) { + if kbID == "" || ownerTenantID == 0 || !required.IsValid() { + return false, nil + } + if err := types.AuthorizeTenantAPIKeyKnowledgeBases(p.ctx, kbID); err != nil { + return false, err + } + if (p.caller.TenantID == ownerTenantID && required == types.OrgRoleViewer) || + HasKBGrant(p.ctx, kbID, ownerTenantID, required) { + return true, nil + } + if p.caller.TenantID == 0 || p.caller.TenantID == ownerTenantID { + return false, nil + } + return p.shares.Check(kbID, required) +} diff --git a/internal/application/access/context_test.go b/internal/application/access/context_test.go new file mode 100644 index 00000000..bef519c8 --- /dev/null +++ b/internal/application/access/context_test.go @@ -0,0 +1,140 @@ +package access + +import ( + "context" + "testing" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/stretchr/testify/require" +) + +func callerContext() context.Context { + ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(1)) + ctx = context.WithValue(ctx, types.UserIDContextKey, "user") + return context.WithValue(ctx, types.TenantRoleContextKey, types.TenantRoleContributor) +} + +func TestKBGrantSurvivesExecutionSwitchAndDetachWithoutWidening(t *testing.T) { + ctx := callerContext() + kb := &types.KnowledgeBase{ID: "shared", TenantID: 2} + grant, err := ResolveKB( + ctx, + KBRequest{Caller: types.CallerFromContext(ctx)}, + kb, + types.OrgRoleViewer, + &shareLookup{permission: types.OrgRoleViewer}, + nil, + ) + require.NoError(t, err) + ctx = grant.Context(ctx) + require.Equal(t, uint64(2), types.MustTenantIDFromContext(ctx)) + ctx = logger.CloneContext(types.WithExecutionTenant(ctx, 3)) + require.Equal( + t, + types.Caller{TenantID: 1, UserID: "user", Role: types.TenantRoleContributor}, + types.CallerFromContext(ctx), + ) + require.True(t, HasKBGrant(ctx, "shared", 2, types.OrgRoleViewer)) + // A later mutation of the service model cannot mutate the captured grant. + kb.ID = "private" + require.False(t, HasKBGrant(ctx, "private", 2, types.OrgRoleViewer)) + require.False(t, HasKBGrant(ctx, "shared", 3, types.OrgRoleViewer)) + require.False(t, HasKBGrant(ctx, "shared", 2, types.OrgRoleEditor)) + for _, caller := range []types.Caller{ + {TenantID: 9, UserID: "user", Role: types.TenantRoleContributor}, + {TenantID: 1, UserID: "other", Role: types.TenantRoleContributor}, + {TenantID: 1, UserID: "user", Role: types.TenantRoleViewer}, + } { + otherCtx := types.WithCaller(ctx, caller) + require.False(t, HasKBGrant(otherCtx, "shared", 2, types.OrgRoleViewer)) + require.Equal( + t, + caller, + types.CallerFromContext(grant.Context(otherCtx)), + "reusing a grant cannot impersonate its caller", + ) + require.Equal( + t, + uint64(3), + types.MustTenantIDFromContext(grant.Context(otherCtx)), + "a caller mismatch must not change execution scope", + ) + } + narrowed := types.WithTenantAPIKeyScope( + ctx, + types.TenantAPIKeyScope{KnowledgeBaseIDs: types.StringArray{"another"}}, + ) + require.False(t, HasKBGrant(narrowed, "shared", 2, types.OrgRoleViewer)) +} + +func TestExecutionTenantDoesNotGrantOwnershipOrChangeShareIdentity(t *testing.T) { + ctx := types.WithExecutionTenant(callerContext(), 2) + shares := &shareLookup{} + allowed, err := NewKBPermissions(ctx, shares).Check("private", 2, types.OrgRoleViewer) + require.NoError(t, err) + require.False(t, allowed) + require.Equal(t, uint64(1), shares.caller) + require.Equal(t, types.TenantRoleContributor, shares.role) + ctx = types.WithExecutionTenant(context.Background(), 2) + allowed, err = NewKBPermissions(ctx, nil).Check("private", 2, types.OrgRoleViewer) + require.NoError(t, err) + require.False(t, allowed, "an absent caller must not become the execution tenant") +} + +func TestSharedAgentGrantPreservesConfiguredScopeAcrossDetach(t *testing.T) { + for _, mode := range []string{"all", "selected", "none", "unknown"} { + t.Run(mode, func(t *testing.T) { + agent := &types.CustomAgent{ + TenantID: 2, + Config: types.CustomAgentConfig{KBSelectionMode: mode, KnowledgeBases: []string{"selected"}}, + } + ctx := logger.CloneContext(WithSharedAgent(callerContext(), agent)) + require.Equal(t, uint64(1), types.CallerFromContext(ctx).TenantID) + require.Equal(t, uint64(2), types.MustTenantIDFromContext(ctx)) + require.Equal(t, mode == "all" || mode == "selected", HasKBGrant(ctx, "selected", 2, types.OrgRoleViewer)) + require.Equal(t, mode == "all", HasKBGrant(ctx, "other", 2, types.OrgRoleViewer)) + require.False(t, HasKBGrant(ctx, "selected", 3, types.OrgRoleViewer)) + require.False(t, HasKBGrant(ctx, "selected", 2, types.OrgRoleEditor)) + }) + } +} + +func TestKBGrantBranchesRemainIndependent(t *testing.T) { + base := callerContext() + grant := &KBAccess{Caller: types.CallerFromContext(base), EffectiveTenantID: 2, Permission: types.OrgRoleViewer} + grant.KnowledgeBase = &types.KnowledgeBase{ID: "first", TenantID: 2} + first := grant.WithGrant(base) + grant.KnowledgeBase = &types.KnowledgeBase{ID: "second", TenantID: 2} + second := grant.WithGrant(base) + require.True(t, HasKBGrant(first, "first", 2, types.OrgRoleViewer)) + require.False(t, HasKBGrant(first, "second", 2, types.OrgRoleViewer)) + require.True(t, HasKBGrant(second, "second", 2, types.OrgRoleViewer)) + require.False(t, HasKBGrant(second, "first", 2, types.OrgRoleViewer)) +} + +func TestKBPermissionsOwnerShortcutOnlyGrantsRead(t *testing.T) { + roles := []types.TenantRole{types.TenantRoleViewer, types.TenantRoleContributor, types.TenantRoleAdmin} + for _, role := range roles { + ctx := context.WithValue(callerContext(), types.TenantRoleContextKey, role) + permissions := NewKBPermissions(ctx, nil) + for _, required := range []types.OrgMemberRole{types.OrgRoleViewer, types.OrgRoleEditor, types.OrgRoleAdmin} { + allowed, err := permissions.Check("kb", 1, required) + require.NoError(t, err) + require.Equal(t, required == types.OrgRoleViewer, allowed) + } + grant := &KBAccess{ + Caller: types.CallerFromContext(ctx), + KnowledgeBase: &types.KnowledgeBase{ID: "kb", TenantID: 1}, + EffectiveTenantID: 1, + Permission: types.OrgRoleEditor, + } + permissions = NewKBPermissions(grant.Context(ctx), nil) + allowed, err := permissions.Check("kb", 1, types.OrgRoleEditor) + require.NoError(t, err) + require.True(t, allowed) + allowed, err = permissions.Check("kb", 1, types.OrgRoleAdmin) + require.NoError(t, err) + require.False(t, allowed) + } +} diff --git a/internal/application/access/files.go b/internal/application/access/files.go new file mode 100644 index 00000000..f18e6bcb --- /dev/null +++ b/internal/application/access/files.go @@ -0,0 +1,256 @@ +package access + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + secutils "github.com/Tencent/WeKnora/internal/utils" +) + +// FileAccess is a request-local result, produced before storage is opened. +// Physical locators are not a substitute for KB/message authorization. +type FileAccess struct { + OwnerTenantID uint64 + Path string + Filename string + StorageBackendID string +} + +// FileCatalog resolves logical references to their stored owner and locator. +type FileCatalog interface { + ResolvePath(context.Context, string) (string, *types.StoredResource, error) +} + +func resolveFile( + ctx context.Context, + catalog FileCatalog, + reference string, +) (FileAccess, *types.StoredResource, error) { + file := FileAccess{Path: reference, Filename: reference} + if catalog == nil { + if _, resource := types.ParseResourcePath(reference); resource { + return file, nil, ErrNotFound + } + return file, nil, nil + } + path, resource, err := catalog.ResolvePath(ctx, reference) + if err != nil { + return file, nil, ErrNotFound + } + file.Path = path + file.Filename = path + if resource != nil { + file.OwnerTenantID = resource.TenantID + file.StorageBackendID = resource.StorageBackendID + if strings.TrimSpace(resource.OriginalName) != "" { + file.Filename = resource.OriginalName + } + } + return file, resource, nil +} + +// ResolveKBFile requires the route's exact KB grant and an independent live +// explicit binding check. Rewriting the execution tenant is insufficient. +func ResolveKBFile( + ctx context.Context, + grant *KBAccess, + kbID, reference string, + catalog FileCatalog, + bindings interfaces.KBResourceLookup, +) (FileAccess, error) { + var empty FileAccess + if grant == nil || grant.KnowledgeBase == nil || grant.KnowledgeBase.ID != kbID || + grant.Caller != types.CallerFromContext(ctx) || !grant.Permission.HasPermission(types.OrgRoleViewer) { + return empty, ErrForbidden + } + if err := types.AuthorizeTenantAPIKeyKnowledgeBases(ctx, kbID); err != nil { + return empty, err + } + file, resource, err := resolveFile(ctx, catalog, reference) + if err != nil { + return empty, err + } + owner := grant.EffectiveTenantID + if owner == 0 || (resource != nil && resource.TenantID != owner) { + return empty, ErrForbidden + } + if err := secutils.ValidateKBScopedStoragePath(file.Path, owner); err != nil { + return empty, ErrForbidden + } + if bindings == nil { + return empty, ErrForbidden + } + bound, err := bindings.IsReferencedByKnowledgeBase(ctx, owner, kbID, reference) + if err != nil { + return empty, fmt.Errorf("check KB file binding: %w", err) + } + if !bound { + return empty, ErrForbidden + } + file.OwnerTenantID = owner + return file, nil +} + +// MessageReferencesFile examines only persisted rendering/output fields. Tool +// arguments and request metadata are not evidence of a returned file. +func MessageReferencesFile(message *types.Message, reference string) bool { + if message == nil { + return false + } + if types.ContainsStorageReference(message.Content, reference) { + return true + } + for _, artifact := range message.Artifacts { + if artifact.URL == reference { + return true + } + } + for _, value := range []interface{}{message.KnowledgeReferences, message.Images} { + data, _ := json.Marshal(value) + if types.ContainsStorageReference(string(data), reference) { + return true + } + } + for _, step := range message.AgentSteps { + for _, call := range step.ToolCalls { + if call.Result == nil { + continue + } + data, _ := json.Marshal(call.Result) + if types.ContainsStorageReference(string(data), reference) { + return true + } + } + } + return false +} + +// ResolveMessageFile loads the session-authorized message using the original +// caller and rechecks its current sharing relationships on every request. +func ResolveMessageFile(ctx context.Context, sessionID, messageID, reference string, messages MessageFileLookup, + agents SharedAgentFileLookup, catalog FileCatalog, kbShares MessageKBShareAuthorizer, +) (FileAccess, error) { + caller := types.CallerFromContext(ctx) + if caller.TenantID == 0 { + return FileAccess{}, ErrUnauthorized + } + if messages == nil { + return FileAccess{}, ErrNotFound + } + ctx = types.WithExecutionTenant(ctx, caller.TenantID) + message, err := messages.GetMessage(ctx, sessionID, messageID) + if err != nil || message == nil { + return FileAccess{}, ErrNotFound + } + return AuthorizeMessageFile(ctx, message, reference, agents, catalog, kbShares) +} + +// AuthorizeMessageFile also serves index-based artifact downloads after their +// session/message lookup. It never accepts an unchecked client message. +func AuthorizeMessageFile(ctx context.Context, message *types.Message, reference string, agents SharedAgentFileLookup, + catalog FileCatalog, kbShares MessageKBShareAuthorizer, +) (FileAccess, error) { + caller := types.CallerFromContext(ctx) + if caller.TenantID == 0 { + return FileAccess{}, ErrUnauthorized + } + if !MessageReferencesFile(message, reference) { + return FileAccess{}, ErrForbidden + } + file, resource, err := resolveFile(ctx, catalog, reference) + if err != nil { + return FileAccess{}, err + } + owner := message.AgentTenantID + if resource != nil { + owner = resource.TenantID + } + if owner == 0 { + return FileAccess{}, ErrForbidden + } + if message.Role == "user" && owner != caller.TenantID { + return FileAccess{}, ErrForbidden + } + if kbShares.Bindings == nil { + kbShares.Bindings, _ = catalog.(interfaces.KBResourceLookup) + } + kbAuthorized := false + if resource != nil && message.AgentTenantID != 0 && message.AgentTenantID != owner { + kbAuthorized = kbShares.resourceAccessibleViaSharedKB(ctx, message, resource, caller.TenantID, caller.Role) + if !kbAuthorized { + return FileAccess{}, ErrForbidden + } + } + if owner != caller.TenantID && !kbAuthorized { + if message.AgentTenantID == 0 { + kbAuthorized = kbShares.resourceAccessibleViaSharedKB(ctx, message, resource, caller.TenantID, caller.Role) + } + if !kbAuthorized { + if message.AgentID == "" || agents == nil { + return FileAccess{}, ErrForbidden + } + agent, err := agents.GetSharedAgentForTenant(ctx, caller.TenantID, caller.Role, message.AgentID, owner) + if err != nil || agent == nil || agent.TenantID != owner { + return FileAccess{}, ErrForbidden + } + bindings, ok := catalog.(interfaces.MessageFileBindingLookup) + if !ok { + return FileAccess{}, ErrForbidden + } + origins, err := bindings.GetMessageFileBindings(ctx, owner, reference, message.ID) + if err != nil || origins == nil { + return FileAccess{}, ErrForbidden + } + allowed := origins.MessageArtifact + scope := types.NewSharedAgentKBScope(agent) + for _, kbID := range origins.KnowledgeBaseIDs { + if scope.Allows(kbID, owner) && types.AuthorizeTenantAPIKeyKnowledgeBases(ctx, kbID) == nil { + allowed = true + } + } + if !allowed { + return FileAccess{}, ErrForbidden + } + } + } + if resource == nil { + if err := secutils.ValidateStoragePathTenant(file.Path, owner); err != nil { + return FileAccess{}, ErrForbidden + } + } + file.OwnerTenantID = owner + return file, nil +} + +// ResolveMessageArtifact selects a persisted artifact after session ownership +// was checked. Session-owned output stays downloadable independently of the +// agent; source-owned output still requires its current sharing permission. +func ResolveMessageArtifact(ctx context.Context, message *types.Message, index int, agents SharedAgentFileLookup, + catalog FileCatalog, kbShares MessageKBShareAuthorizer, +) (FileAccess, error) { + caller := types.CallerFromContext(ctx) + if caller.TenantID == 0 { + return FileAccess{}, ErrUnauthorized + } + if message == nil || index < 0 || index >= len(message.Artifacts) { + return FileAccess{}, ErrNotFound + } + artifact := message.Artifacts[index] + file, resource, err := resolveFile(ctx, catalog, artifact.URL) + if err != nil { + return FileAccess{}, err + } + if (resource != nil && resource.TenantID == caller.TenantID) || + (resource == nil && secutils.ValidateStoragePathTenant(file.Path, caller.TenantID) == nil) { + file.OwnerTenantID = caller.TenantID + file.Filename = artifact.FileName + return file, nil + } + file, err = AuthorizeMessageFile(ctx, message, artifact.URL, agents, catalog, kbShares) + file.Filename = artifact.FileName + return file, err +} diff --git a/internal/application/access/files_test.go b/internal/application/access/files_test.go new file mode 100644 index 00000000..e6693527 --- /dev/null +++ b/internal/application/access/files_test.go @@ -0,0 +1,162 @@ +package access + +import ( + "context" + "errors" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/stretchr/testify/require" +) + +type fileCatalog struct{ resource *types.StoredResource } + +func (c fileCatalog) ResolvePath(_ context.Context, ref string) (string, *types.StoredResource, error) { + if c.resource == nil { + return ref, nil, nil + } + return c.resource.PhysicalPath, c.resource, nil +} + +type fileBinding struct { + allowed bool + err error + kb string + tenant uint64 +} + +func (b *fileBinding) IsReferencedByKnowledgeBase(_ context.Context, tenant uint64, kb, _ string) (bool, error) { + b.kb, b.tenant = kb, tenant + return b.allowed, b.err +} + +type fileMessages struct { + message *types.Message + tenant uint64 +} + +func (m *fileMessages) GetMessage(ctx context.Context, _, _ string) (*types.Message, error) { + m.tenant = types.MustTenantIDFromContext(ctx) + return m.message, nil +} + +func TestKBFilesRequireExactGrantAndBinding(t *testing.T) { + base := callerContext() + grant := &KBAccess{ + Caller: types.CallerFromContext(base), + KnowledgeBase: &types.KnowledgeBase{ID: "kb", TenantID: 2}, + EffectiveTenantID: 2, + Permission: types.OrgRoleViewer, + } + ctx := grant.Context(base) + bindings := &fileBinding{allowed: true} + const ref = "resource://AbCdEfGhIjKlMnOpQrStUv" + catalog := fileCatalog{ + &types.StoredResource{TenantID: 2, PhysicalPath: "local://2/exports/a.png", OriginalName: "a.png"}, + } + file, err := ResolveKBFile(ctx, grant, "kb", ref, catalog, bindings) + require.NoError(t, err) + require.Equal(t, uint64(2), file.OwnerTenantID) + require.Equal(t, "kb", bindings.kb) + require.Equal(t, uint64(2), bindings.tenant) + bindings.allowed = false + _, err = ResolveKBFile(ctx, grant, "kb", ref, catalog, bindings) + require.ErrorIs(t, err, ErrForbidden, "a same-tenant file still needs an exact KB binding") + bindings.allowed = true + _, err = ResolveKBFile(ctx, grant, "another", ref, catalog, bindings) + require.ErrorIs(t, err, ErrForbidden) + _, err = ResolveKBFile(ctx, nil, "kb", ref, catalog, bindings) + require.ErrorIs(t, err, ErrForbidden, "execution tenant alone is not a grant") + bindings.err = errors.New("database unavailable") + _, err = ResolveKBFile(ctx, grant, "kb", ref, catalog, bindings) + require.ErrorIs(t, err, bindings.err) +} + +func TestMessageFilesRequireReferenceAndRecheckRevocation(t *testing.T) { + const ref = "resource://AbCdEfGhIjKlMnOpQrStUv" + messages := &fileMessages{ + message: &types.Message{ID: "message", AgentID: "agent", AgentTenantID: 2, Role: "assistant"}, + } + agents := &agentLookup{agent: &types.CustomAgent{ID: "agent", TenantID: 2}} + catalog := artifactFileCatalog{ + fileCatalog{&types.StoredResource{TenantID: 2, PhysicalPath: "local://2/exports/a.png"}}, + } + ctx := types.WithExecutionTenant(callerContext(), 2) + _, err := ResolveMessageFile(ctx, "session", "message", ref, messages, agents, catalog, MessageKBShareAuthorizer{}) + require.ErrorIs(t, err, ErrForbidden) + messages.message.Content = "![image](" + ref + "x)" + _, err = ResolveMessageFile(ctx, "session", "message", ref, messages, agents, catalog, MessageKBShareAuthorizer{}) + require.ErrorIs(t, err, ErrForbidden, "a longer handle is not the requested handle") + messages.message.Content = "![image](" + ref + ")" + _, err = ResolveMessageFile(ctx, "session", "message", ref, messages, agents, catalog, MessageKBShareAuthorizer{}) + require.NoError(t, err) + require.Equal(t, uint64(1), messages.tenant, "session lookup must use the caller") + agents.agent = nil + _, err = ResolveMessageFile(ctx, "session", "message", ref, messages, agents, catalog, MessageKBShareAuthorizer{}) + require.ErrorIs(t, err, ErrForbidden, "historical references cannot bypass share revocation") +} + +func TestMessageArtifactsKeepSessionOwnershipSeparateFromAgentOutput(t *testing.T) { + const ref = "resource://AbCdEfGhIjKlMnOpQrStUv" + message := &types.Message{ + AgentID: "agent", + AgentTenantID: 2, + Artifacts: types.MessageArtifacts{{URL: ref, FileName: "report.pdf"}}, + } + catalog := fileCatalog{&types.StoredResource{TenantID: 1, PhysicalPath: "local://1/exports/report.pdf"}} + file, err := ResolveMessageArtifact(callerContext(), message, 0, nil, catalog, MessageKBShareAuthorizer{}) + require.NoError(t, err) + require.Equal(t, uint64(1), file.OwnerTenantID) + catalog.resource.TenantID = 2 + catalog.resource.PhysicalPath = "local://2/exports/report.pdf" + _, err = ResolveMessageArtifact(callerContext(), message, 0, nil, catalog, MessageKBShareAuthorizer{}) + require.ErrorIs(t, err, ErrForbidden) +} + +type messageFileKBs struct{ kb *types.KnowledgeBase } + +func (k messageFileKBs) GetKnowledgeBasesByIDsOnly(context.Context, []string) ([]*types.KnowledgeBase, error) { + return []*types.KnowledgeBase{k.kb}, nil +} + +func TestMessageSharedKBFilesRequireLiveBinding(t *testing.T) { + const ref = "resource://AbCdEfGhIjKlMnOpQrStUv" + message := &types.Message{ + AgentTenantID: 1, Role: "assistant", + KnowledgeReferences: types.References{{KnowledgeBaseID: "shared", Content: ref}}, + } + catalog := fileCatalog{&types.StoredResource{ + Handle: "AbCdEfGhIjKlMnOpQrStUv", TenantID: 2, PhysicalPath: "local://2/exports/a.png", + }} + binding := &fileBinding{allowed: true} + shares := &shareLookup{permission: types.OrgRoleViewer} + authorizer := MessageKBShareAuthorizer{ + ShareGuard: shares, KBs: messageFileKBs{kb: &types.KnowledgeBase{ID: "shared", TenantID: 2}}, Bindings: binding, + } + _, err := AuthorizeMessageFile(callerContext(), message, ref, nil, catalog, authorizer) + require.NoError(t, err) + require.Equal(t, "shared", binding.kb) + require.Equal(t, uint64(2), binding.tenant) + binding.allowed = false + _, err = AuthorizeMessageFile(callerContext(), message, ref, nil, catalog, authorizer) + require.ErrorIs(t, err, ErrForbidden, "retrieval text alone must not authorize an unrelated same-tenant resource") + binding.allowed = true + binding.err = errors.New("binding lookup unavailable") + _, err = AuthorizeMessageFile(callerContext(), message, ref, nil, catalog, authorizer) + require.ErrorIs(t, err, ErrForbidden) + binding.err = nil + authorizer.Bindings = nil + _, err = AuthorizeMessageFile(callerContext(), message, ref, nil, catalog, authorizer) + require.ErrorIs(t, err, ErrForbidden, "missing live lookup must fail closed") +} + +// This fixture represents a catalog claim made by the artifact collector. +type artifactFileCatalog struct{ fileCatalog } + +func (artifactFileCatalog) GetMessageFileBindings( + _ context.Context, + _ uint64, + _, messageID string, +) (*types.MessageFileBindings, error) { + return &types.MessageFileBindings{MessageArtifact: messageID == "message"}, nil +} diff --git a/internal/application/access/kb_shares.go b/internal/application/access/kb_shares.go new file mode 100644 index 00000000..b73957db --- /dev/null +++ b/internal/application/access/kb_shares.go @@ -0,0 +1,62 @@ +package access + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/types" +) + +// KBSharePermissions resolves organization grants for one caller during one +// operation. It deliberately does not grant tenant ownership or shared-agent +// access. Callers retain their own policy for lookup failures (abort or filter). +// Create a new instance for each operation; it is not a cross-request cache and +// must not be shared between goroutines. +type KBSharePermissions struct { + ctx context.Context + lookup KBShareLookup + tenantID uint64 + role types.TenantRole + results map[string]kbSharePermission +} + +type kbSharePermission struct { + role types.OrgMemberRole + shared bool + err error +} + +// NewKBSharePermissions creates a request-local permission cache. +func NewKBSharePermissions( + ctx context.Context, + lookup KBShareLookup, + tenantID uint64, + role types.TenantRole, +) *KBSharePermissions { + return &KBSharePermissions{ + ctx: ctx, + lookup: lookup, + tenantID: tenantID, + role: role, + results: make(map[string]kbSharePermission), + } +} + +// Permission caches both grants and failures so multiple documents/chunks in a +// KB observe the same decision, without repeating the membership queries. +func (p *KBSharePermissions) Permission(kbID string) (types.OrgMemberRole, bool, error) { + if kbID == "" || p.tenantID == 0 || p.lookup == nil { + return "", false, nil + } + result, ok := p.results[kbID] + if !ok { + result.role, result.shared, result.err = p.lookup.CheckTenantKBPermission(p.ctx, kbID, p.tenantID, p.role) + p.results[kbID] = result + } + return result.role, result.shared, result.err +} + +// Check tests the required role against the cached organization permission. +func (p *KBSharePermissions) Check(kbID string, required types.OrgMemberRole) (bool, error) { + role, shared, err := p.Permission(kbID) + return err == nil && shared && role.IsValid() && required.IsValid() && role.HasPermission(required), err +} diff --git a/internal/application/access/kb_shares_test.go b/internal/application/access/kb_shares_test.go new file mode 100644 index 00000000..2fc5cc11 --- /dev/null +++ b/internal/application/access/kb_shares_test.go @@ -0,0 +1,84 @@ +package access + +import ( + "context" + "errors" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/stretchr/testify/require" +) + +type permissionLookupFunc func(context.Context, string, uint64, types.TenantRole) (types.OrgMemberRole, bool, error) + +func (f permissionLookupFunc) CheckTenantKBPermission( + ctx context.Context, + id string, + tenant uint64, + role types.TenantRole, +) (types.OrgMemberRole, bool, error) { + return f(ctx, id, tenant, role) +} + +func TestKBSharePermissionsCachePerOperation(t *testing.T) { + calls := map[string]int{} + granted := true + lookup := permissionLookupFunc( + func(_ context.Context, kb string, tenant uint64, role types.TenantRole) (types.OrgMemberRole, bool, error) { + require.Equal(t, uint64(1), tenant) + require.Equal(t, types.TenantRoleContributor, role) + calls[kb]++ + return types.OrgRoleEditor, granted, nil + }, + ) + permissions := NewKBSharePermissions(context.Background(), lookup, 1, types.TenantRoleContributor) + for _, required := range []types.OrgMemberRole{types.OrgRoleViewer, types.OrgRoleEditor, types.OrgRoleAdmin} { + allowed, err := permissions.Check("kb", required) + require.NoError(t, err) + require.Equal(t, required != types.OrgRoleAdmin, allowed) + } + require.Equal(t, 1, calls["kb"], "different minimum roles share the same membership lookup") + _, err := permissions.Check("other", types.OrgRoleViewer) + require.NoError(t, err) + require.Equal(t, 1, calls["other"]) + granted = false + fresh := NewKBSharePermissions(context.Background(), lookup, 1, types.TenantRoleContributor) + allowed, err := fresh.Check("kb", types.OrgRoleViewer) + require.NoError(t, err) + require.False(t, allowed, "a new operation must observe share revocation") + require.Equal(t, 2, calls["kb"]) +} + +func TestKBSharePermissionsCachesFailuresWithoutGranting(t *testing.T) { + failure := errors.New("database unavailable") + calls := 0 + lookup := permissionLookupFunc( + func(context.Context, string, uint64, types.TenantRole) (types.OrgMemberRole, bool, error) { + calls++ + return types.OrgRoleAdmin, true, failure + }, + ) + permissions := NewKBSharePermissions(context.Background(), lookup, 1, types.TenantRoleAdmin) + for i := 0; i < 3; i++ { + allowed, err := permissions.Check("kb", types.OrgRoleViewer) + require.False(t, allowed) + require.ErrorIs(t, err, failure, "callers must retain the infrastructure error") + } + require.Equal(t, 1, calls) +} + +func TestKBSharePermissionsMissingLookupAndInvalidRolesDeny(t *testing.T) { + permissions := NewKBSharePermissions(context.Background(), nil, 1, types.TenantRoleAdmin) + allowed, err := permissions.Check("kb", types.OrgRoleViewer) + require.NoError(t, err) + require.False(t, allowed) + lookup := permissionLookupFunc( + func(context.Context, string, uint64, types.TenantRole) (types.OrgMemberRole, bool, error) { + return types.OrgRoleEditor, true, nil + }, + ) + permissions = NewKBSharePermissions(context.Background(), lookup, 1, types.TenantRoleAdmin) + allowed, err = permissions.Check("kb", "unknown") + require.NoError(t, err) + require.False(t, allowed) +} diff --git a/internal/application/access/kb_transfer.go b/internal/application/access/kb_transfer.go new file mode 100644 index 00000000..0cac5b40 --- /dev/null +++ b/internal/application/access/kb_transfer.go @@ -0,0 +1,227 @@ +package access + +import ( + "context" + "fmt" + + "github.com/Tencent/WeKnora/internal/types" +) + +// KBTransferOperation selects the source and destination permission contract. +type KBTransferOperation string + +// Supported knowledge-base transfer operations. +const ( + KBTransferClone KBTransferOperation = "clone" + KBTransferMove KBTransferOperation = "move" +) + +type kbTransferGrant struct { + caller types.Caller + tenant uint64 + source, target, taskID, creatorID string + operation KBTransferOperation + create bool +} + +// WithKBTransfer admits a pair of resources after the entry point has resolved +// the source and destination grants and applied its creator/role policy. A new +// clone destination must be a server-reserved ID; it is not an existing KB. +func WithKBTransfer( + ctx context.Context, + source, target *types.KnowledgeBase, + operation KBTransferOperation, + taskID string, + create bool, +) (context.Context, error) { + caller := types.CallerFromContext(ctx) + if taskID == "" { + return ctx, ErrForbidden + } + if err := validateTransferPair(source, target, caller.TenantID, operation, create); err != nil { + return ctx, err + } + if err := transferAPIScope(ctx, source.ID, target.ID, operation, create); err != nil { + return ctx, err + } + required := types.OrgRoleViewer + if operation == KBTransferMove { + required = types.OrgRoleEditor + } + if !HasKBGrant(ctx, source.ID, source.TenantID, required) || + (!create && !HasKBGrant(ctx, target.ID, target.TenantID, types.OrgRoleEditor)) { + return ctx, ErrForbidden + } + return withTransferGrant(ctx, source, target, operation, taskID, create, false), nil +} + +// WithKBTransferTask continues an admitted task. Both KBs must first be loaded +// with their persisted owners (or a reserved new clone destination). This scope +// cannot authorize a different pair or upgrade the worker into a tenant user. +func WithKBTransferTask( + ctx context.Context, + source, target *types.KnowledgeBase, + expectedTenant uint64, + operation KBTransferOperation, + taskID string, + create bool, +) (context.Context, error) { + if taskID == "" { + return ctx, ErrForbidden + } + if err := validateTransferPair(source, target, expectedTenant, operation, create); err != nil { + return ctx, err + } + ctx = types.WithExecutionTenant(ctx, expectedTenant) + return withTransferGrant(ctx, source, target, operation, taskID, create, true), nil +} + +func withTransferGrant( + ctx context.Context, + source, target *types.KnowledgeBase, + operation KBTransferOperation, + taskID string, + create, task bool, +) context.Context { + caller := types.CallerFromContext(ctx) + grant := kbTransferGrant{ + caller: caller, + tenant: source.TenantID, + source: source.ID, + target: target.ID, + taskID: taskID, + creatorID: target.CreatorID, + operation: operation, + create: create, + } + required := types.OrgRoleViewer + if operation == KBTransferMove { + required = types.OrgRoleEditor + } + ctx = context.WithValue(ctx, types.KBGrantsContextKey, []kbGrant{ + {caller: caller, kbID: source.ID, tenantID: source.TenantID, permission: required, task: task}, + {caller: caller, kbID: target.ID, tenantID: target.TenantID, permission: types.OrgRoleEditor, task: task}, + }) + return context.WithValue(ctx, types.KBTransferContextKey, grant) +} + +// RequireKBTransfer consumes the exact caller, operation and resource-pair grant. +func RequireKBTransfer(ctx context.Context, source, target *types.KnowledgeBase, operation KBTransferOperation) error { + grant, ok := ctx.Value(types.KBTransferContextKey).(kbTransferGrant) + if !ok || grant.caller != types.CallerFromContext(ctx) || grant.operation != operation { + return ErrForbidden + } + if err := validateTransferPair(source, target, grant.tenant, operation, grant.create); err != nil { + return err + } + if grant.source != source.ID || grant.target != target.ID { + return ErrForbidden + } + return transferAPIScope(ctx, source.ID, target.ID, operation, grant.create) +} + +// CloneDestination exposes only the destination already admitted by the +// request/worker; CopyKnowledgeBase cannot turn an empty ID into ambient write. +func CloneDestination(ctx context.Context, sourceID string) (id string, create bool, creatorID string, err error) { + g, ok := ctx.Value(types.KBTransferContextKey).(kbTransferGrant) + if !ok || g.caller != types.CallerFromContext(ctx) || g.operation != KBTransferClone || g.source != sourceID { + return "", false, "", ErrForbidden + } + return g.target, g.create, g.creatorID, nil +} + +// TransferTaskID returns the admitted operation identity for retry checkpoints. +func TransferTaskID(ctx context.Context) string { + g, ok := ctx.Value(types.KBTransferContextKey).(kbTransferGrant) + if !ok || g.caller != types.CallerFromContext(ctx) { + return "" + } + return g.taskID +} + +func validateTransferPair( + source, target *types.KnowledgeBase, + tenant uint64, + operation KBTransferOperation, + create bool, +) error { + if source == nil || target == nil || tenant == 0 || source.ID == "" || target.ID == "" || + source.TenantID != tenant || + target.TenantID != tenant { + return ErrForbidden + } + if source.ID == target.ID { + return fmt.Errorf("source and target knowledge bases must differ") + } + if operation != KBTransferClone && operation != KBTransferMove || create && operation != KBTransferClone { + return ErrForbidden + } + return nil +} + +func transferAPIScope( + ctx context.Context, + sourceID, targetID string, + operation KBTransferOperation, + create bool, +) error { + ids := []string{sourceID} + if !create { + ids = append(ids, targetID) + } + if err := types.AuthorizeTenantAPIKeyKnowledgeBases(ctx, ids...); err != nil { + return err + } + capability := types.APIKeyCapabilityManageKnowledgeBases + if operation == KBTransferMove { + capability = types.APIKeyCapabilityIngest + } + if scope, ok := types.TenantAPIKeyScopeFromContext(ctx); ok && !scope.FullAccess && + !scope.HasCapability(capability) { + return ErrForbidden + } + return nil +} + +// ValidateKBTransferCompatibility is shared by synchronous admission and task +// execution, so invalid modes/configurations fail before resource mutations. +func ValidateKBTransferCompatibility( + source, target *types.KnowledgeBase, + operation KBTransferOperation, + mode string, + tenant *types.Tenant, +) error { + if source == nil || target == nil { + return ErrNotFound + } + if source.ID == target.ID { + return fmt.Errorf("source and target knowledge bases must differ") + } + if source.Type != target.Type { + return fmt.Errorf("source and target knowledge bases must have the same type") + } + if source.EmbeddingModelID != target.EmbeddingModelID { + return fmt.Errorf("source and target knowledge bases use different embedding models") + } + if operation == KBTransferMove && mode != "reuse_vectors" && mode != "reparse" { + return fmt.Errorf("unknown move mode: %s", mode) + } + if (operation == KBTransferClone || mode == "reuse_vectors") && !source.SharesStoreWith(target) { + return fmt.Errorf("source and target knowledge bases use different vector stores; use reparse mode for moves") + } + // A move retains file paths too, so resolving them through another storage + // instance would make the document unreadable, even when vectors are reparsed. + defaultID, defaultProvider := "", "" + if tenant != nil { + if tenant.DefaultStorageBackendID != nil { + defaultID = *tenant.DefaultStorageBackendID + } + if tenant.StorageEngineConfig != nil { + defaultProvider = tenant.StorageEngineConfig.DefaultProvider + } + } + if !source.SharesStorageBackendWith(target, defaultID, defaultProvider) { + return fmt.Errorf("source and target knowledge bases use different storage instances") + } + return nil +} diff --git a/internal/application/access/kb_transfer_test.go b/internal/application/access/kb_transfer_test.go new file mode 100644 index 00000000..f3fa0ee0 --- /dev/null +++ b/internal/application/access/kb_transfer_test.go @@ -0,0 +1,96 @@ +package access + +import ( + "context" + "fmt" + "testing" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/stretchr/testify/require" +) + +func TestTransferAdmissionRequiresIndependentGrants(t *testing.T) { + source, target := &types.KnowledgeBase{ID: "source", TenantID: 1}, &types.KnowledgeBase{ID: "target", TenantID: 1} + ctx := types.WithCaller( + context.Background(), + types.Caller{TenantID: 1, UserID: "user", Role: types.TenantRoleAdmin}, + ) + _, err := WithKBTransfer(ctx, source, target, KBTransferMove, "task", false) + require.ErrorIs(t, err, ErrForbidden) + request := KBRequest{Caller: types.CallerFromContext(ctx)} + sourceRead, err := ResolveKB(ctx, request, source, types.OrgRoleViewer, nil, nil) + require.NoError(t, err) + targetRead, err := ResolveKB(ctx, request, target, types.OrgRoleViewer, nil, nil) + require.NoError(t, err) + reads := targetRead.WithGrant(sourceRead.WithGrant(ctx)) + _, err = WithKBTransfer(reads, source, target, KBTransferClone, "task", false) + require.ErrorIs(t, err, ErrForbidden, "owner read projection must not authorize target replacement") + targetWrite, err := ResolveKB(ctx, request, target, types.OrgRoleEditor, nil, nil) + require.NoError(t, err) + cloneCtx, err := WithKBTransfer(targetWrite.WithGrant(reads), source, target, KBTransferClone, "task", false) + require.NoError(t, err) + require.NoError(t, RequireKBTransfer(logger.CloneContext(cloneCtx), source, target, KBTransferClone)) + require.False(t, HasKBGrant(cloneCtx, source.ID, 1, types.OrgRoleEditor)) + _, err = WithKBTransfer(targetWrite.WithGrant(reads), source, target, KBTransferMove, "task", false) + require.ErrorIs(t, err, ErrForbidden) +} + +func TestTransferTaskScopeIsExactAndDoesNotCreateCaller(t *testing.T) { + source, target := &types.KnowledgeBase{ID: "source", TenantID: 7}, &types.KnowledgeBase{ID: "target", TenantID: 7} + ctx, err := WithKBTransferTask(context.Background(), source, target, 7, KBTransferMove, "task", false) + require.NoError(t, err) + ctx = logger.CloneContext(ctx) + require.Zero(t, types.CallerFromContext(ctx).TenantID) + require.NoError(t, RequireKBTransfer(ctx, source, target, KBTransferMove)) + require.ErrorIs(t, RequireKBTransfer(ctx, source, target, KBTransferClone), ErrForbidden) + require.ErrorIs( + t, + RequireKBTransfer(ctx, source, &types.KnowledgeBase{ID: "third", TenantID: 7}, KBTransferMove), + ErrForbidden, + ) + require.False(t, HasKBGrant(ctx, "third", 7, types.OrgRoleViewer)) + _, err = WithKBTransferTask( + ctx, + source, + &types.KnowledgeBase{ID: "target", TenantID: 8}, + 7, + KBTransferMove, + "task", + false, + ) + require.ErrorIs(t, err, ErrForbidden) + changed := types.WithCaller(ctx, types.Caller{TenantID: 7, UserID: "different"}) + require.ErrorIs(t, RequireKBTransfer(changed, source, target, KBTransferMove), ErrForbidden) +} + +func TestTransferAPICapabilitiesRemainSeparate(t *testing.T) { + source, target := &types.KnowledgeBase{ID: "source", TenantID: 1}, &types.KnowledgeBase{ID: "target", TenantID: 1} + for _, tc := range []struct { + operation KBTransferOperation + capability string + create bool + ids []string + allowed bool + }{ + {KBTransferClone, "manage_kbs", false, []string{"source", "target"}, true}, + {KBTransferClone, "ingest", false, []string{"source", "target"}, false}, + {KBTransferMove, "ingest", false, []string{"source", "target"}, true}, + {KBTransferMove, "manage_kbs", false, []string{"source", "target"}, false}, + {KBTransferMove, "ingest", false, []string{"source"}, false}, + {KBTransferClone, "manage_kbs", true, []string{"source"}, true}, + } { + t.Run( + fmt.Sprintf("%s/%s/create=%v/ids=%d", tc.operation, tc.capability, tc.create, len(tc.ids)), + func(t *testing.T) { + ctx, err := WithKBTransferTask(context.Background(), source, target, 1, tc.operation, "task", tc.create) + require.NoError(t, err) + ctx = types.WithTenantAPIKeyScope( + ctx, + types.TenantAPIKeyScope{Capabilities: types.StringArray{tc.capability}, KnowledgeBaseIDs: tc.ids}, + ) + require.Equal(t, tc.allowed, RequireKBTransfer(ctx, source, target, tc.operation) == nil) + }, + ) + } +} diff --git a/internal/application/access/kb_write.go b/internal/application/access/kb_write.go new file mode 100644 index 00000000..282f663c --- /dev/null +++ b/internal/application/access/kb_write.go @@ -0,0 +1,45 @@ +package access + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/types" +) + +// RequireKBWrite consumes an explicit operation grant. Tenant ownership and +// role checks belong to the entry point that resolved the grant; an execution +// tenant alone, or a shared-agent/read grant, never authorizes a mutation. +func RequireKBWrite(ctx context.Context, kb *types.KnowledgeBase) error { + if kb == nil || kb.ID == "" || kb.TenantID == 0 { + return ErrNotFound + } + if scope, ok := types.TenantAPIKeyScopeFromContext(ctx); ok && !scope.FullAccess && + !scope.HasCapability(types.APIKeyCapabilityIngest) { + return ErrForbidden + } + if !HasKBGrant(ctx, kb.ID, kb.TenantID, types.OrgRoleEditor) { + return ErrForbidden + } + return nil +} + +// WithKBTaskWrite is for trusted workers continuing an admitted operation. +// The worker supplies a server-loaded KB and verifies its child-resource +// bindings before any side effects. It grants only this KB, preserves the +// original caller (including an absent caller), and never creates an Admin. +// This is a task execution grant, not a way to authorize an incoming request. +func WithKBTaskWrite(ctx context.Context, kb *types.KnowledgeBase, expectedTenant uint64) (context.Context, error) { + if kb == nil || kb.ID == "" || expectedTenant == 0 || kb.TenantID != expectedTenant { + return ctx, ErrForbidden + } + ctx = types.WithExecutionTenant(ctx, expectedTenant) + grant := kbGrant{ + caller: types.CallerFromContext(ctx), + kbID: kb.ID, + tenantID: expectedTenant, + permission: types.OrgRoleEditor, + task: true, + } + // A worker gets exactly one scope, without inheriting another task's KBs. + return context.WithValue(ctx, types.KBGrantsContextKey, []kbGrant{grant}), nil +} diff --git a/internal/application/access/kb_write_test.go b/internal/application/access/kb_write_test.go new file mode 100644 index 00000000..ee6106ed --- /dev/null +++ b/internal/application/access/kb_write_test.go @@ -0,0 +1,85 @@ +package access + +import ( + "context" + "testing" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/stretchr/testify/require" +) + +func TestKBWriteConsumesOperationGrant(t *testing.T) { + ctx := callerContext() + kb := &types.KnowledgeBase{ID: "kb", TenantID: types.CallerFromContext(ctx).TenantID} + read, err := ResolveKB(ctx, KBRequest{Caller: types.CallerFromContext(ctx)}, kb, types.OrgRoleViewer, nil, nil) + require.NoError(t, err) + require.Equal(t, types.OrgRoleAdmin, read.Permission, "owner permission projection remains unchanged") + require.ErrorIs(t, RequireKBWrite(read.Context(ctx), kb), ErrForbidden, "an owner read does not grant mutation") + require.ErrorIs(t, RequireKBWrite(types.WithExecutionTenant(ctx, kb.TenantID), kb), ErrForbidden) + write, err := ResolveKB(ctx, KBRequest{Caller: types.CallerFromContext(ctx)}, kb, types.OrgRoleEditor, nil, nil) + require.NoError(t, err) + ctx = logger.CloneContext(write.Context(ctx)) + require.NoError(t, RequireKBWrite(ctx, kb)) + require.ErrorIs(t, RequireKBWrite(ctx, &types.KnowledgeBase{ID: "other", TenantID: kb.TenantID}), ErrForbidden) + require.ErrorIs(t, RequireKBWrite(ctx, &types.KnowledgeBase{ID: kb.ID, TenantID: kb.TenantID + 1}), ErrForbidden) + for _, tt := range []struct { + name string + scope types.TenantAPIKeyScope + allowed bool + }{ + {"retrieve only", types.TenantAPIKeyScope{Capabilities: types.StringArray{"retrieve"}}, false}, + { + "ingest", + types.TenantAPIKeyScope{ + Capabilities: types.StringArray{"ingest"}, + KnowledgeBaseIDs: types.StringArray{"kb"}, + }, + true, + }, + + { + "wrong KB", + types.TenantAPIKeyScope{ + Capabilities: types.StringArray{"ingest"}, + KnowledgeBaseIDs: types.StringArray{"other"}, + }, + false, + }, + + {"full access", types.TenantAPIKeyScope{FullAccess: true}, true}, + } { + t.Run(tt.name, func(t *testing.T) { + err := RequireKBWrite(types.WithTenantAPIKeyScope(ctx, tt.scope), kb) + require.Equal(t, tt.allowed, err == nil) + }) + } +} + +func TestKBTaskWriteIsExplicitAndBounded(t *testing.T) { + kb := &types.KnowledgeBase{ID: "kb", TenantID: 7} + base := context.Background() + ctx, err := WithKBTaskWrite(base, kb, 7) + require.NoError(t, err) + ctx = logger.CloneContext(ctx) + require.Zero(t, types.CallerFromContext(ctx).TenantID, "a worker must not impersonate the tenant owner") + require.NoError(t, RequireKBWrite(ctx, kb)) + allowed, err := NewKBPermissions(ctx, nil).Check("kb", 7, types.OrgRoleViewer) + require.NoError(t, err) + require.True(t, allowed, "a scoped write task can read its KB") + allowed, err = NewKBPermissions(ctx, nil).Check("other", 7, types.OrgRoleViewer) + require.NoError(t, err) + require.False(t, allowed) + require.ErrorIs(t, RequireKBWrite(ctx, &types.KnowledgeBase{ID: "other", TenantID: 7}), ErrForbidden) + _, err = WithKBTaskWrite(base, kb, 8) + require.ErrorIs(t, err, ErrForbidden) + _, err = WithKBTaskWrite(base, kb, 0) + require.ErrorIs(t, err, ErrForbidden) + require.False(t, HasKBGrant(base, "kb", 7, types.OrgRoleEditor), "the parent context stays unmodified") + request := callerContext() + task, err := WithKBTaskWrite(request, kb, 7) + require.NoError(t, err) + require.Equal(t, types.CallerFromContext(request), types.CallerFromContext(task)) + changed := types.WithCaller(task, types.Caller{TenantID: 99, UserID: "another"}) + require.ErrorIs(t, RequireKBWrite(changed, kb), ErrForbidden) +} diff --git a/internal/application/access/knowledge_state.go b/internal/application/access/knowledge_state.go new file mode 100644 index 00000000..96bdcefa --- /dev/null +++ b/internal/application/access/knowledge_state.go @@ -0,0 +1,38 @@ +package access + +import ( + "encoding/json" + + apperrors "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/types" +) + +// RejectMovingKnowledge is shared by admission and persisted service checks. +// Admission prevents misleading task acceptance; workers must recheck after +// enqueue because a move can claim the document in the intervening window. +func RejectMovingKnowledge(knowledge *types.Knowledge) error { + if knowledge == nil { + return apperrors.NewNotFoundError("knowledge not found") + } + fields := map[string]json.RawMessage{} + if len(knowledge.Metadata) != 0 { + if err := json.Unmarshal(knowledge.Metadata, &fields); err != nil { + return err + } + } + raw := fields[types.KnowledgeTransferMetadataKey] + if len(raw) == 0 { + return nil + } + var state struct { + Operation KBTransferOperation `json:"operation"` + Phase string `json:"phase"` + } + if err := json.Unmarshal(raw, &state); err != nil { + return err + } + if state.Operation == KBTransferMove && state.Phase == "moving" { + return apperrors.NewConflictError("knowledge has an unfinished move; retry the move first") + } + return nil +} diff --git a/internal/application/access/knowledgebase.go b/internal/application/access/knowledgebase.go new file mode 100644 index 00000000..4a875aab --- /dev/null +++ b/internal/application/access/knowledgebase.go @@ -0,0 +1,156 @@ +// Package access resolves resource permissions independently of HTTP routing. +// Tenant roles, API-key capabilities, and resource ownership guards remain +// separate checks; a KB grant does not replace them. +package access + +import ( + "context" + "errors" + "fmt" + + "github.com/Tencent/WeKnora/internal/types" +) + +// Resource access errors retain distinct adapter response mappings. +var ( + ErrUnauthorized = errors.New("kb_access: unauthorized") + ErrNotFound = errors.New("kb_access: not found") + ErrForbidden = errors.New("kb_access: forbidden") + ErrInvalidAgentSource = errors.New("kb_access: invalid agent source") +) + +// KBRequest keeps the authenticated caller separate from the resource tenant. +// AgentSourceTenantID is parsed only when the explicit-agent fallback is used. +type KBRequest struct { + Caller types.Caller + AgentID string + AgentSourceTenantID string +} + +// KBAccess binds a resolved resource permission to its original caller. +type KBAccess struct { + KnowledgeBase *types.KnowledgeBase + Caller types.Caller + EffectiveTenantID uint64 + Permission types.OrgMemberRole + // A read resolution must not mint a write grant even for the KB owner. + operationPermission types.OrgMemberRole +} + +// Context scopes resource operations without changing the authenticated caller. +func (a *KBAccess) Context(ctx context.Context) context.Context { + if a == nil || a.KnowledgeBase == nil { + return ctx + } + scoped := a.WithGrant(ctx) + if types.CallerFromContext(scoped) != a.Caller.Normalize() { + return ctx + } + return types.WithExecutionTenant(scoped, a.EffectiveTenantID) +} + +// WithGrant carries this authorization without changing execution scope. +func (a *KBAccess) WithGrant(ctx context.Context) context.Context { + if a == nil || a.KnowledgeBase == nil { + return ctx + } + caller := a.Caller.Normalize() + if existing, ok := ctx.Value(types.CallerContextKey).(types.Caller); ok && existing != caller { + return ctx // A grant cannot replace an already authenticated caller. + } + grants, _ := ctx.Value(types.KBGrantsContextKey).([]kbGrant) + permission := a.Permission + if a.operationPermission.IsValid() && permission.HasPermission(a.operationPermission) { + permission = a.operationPermission + } + grants = append( + append([]kbGrant(nil), grants...), + kbGrant{caller: caller, kbID: a.KnowledgeBase.ID, tenantID: a.EffectiveTenantID, permission: permission}, + ) + return context.WithValue(types.WithCaller(ctx, caller), types.KBGrantsContextKey, grants) +} + +// KBShareLookup resolves organization sharing with the caller role cap. +type KBShareLookup interface { + CheckTenantKBPermission(context.Context, string, uint64, types.TenantRole) (types.OrgMemberRole, bool, error) +} + +// SharedAgentLookup verifies access to a particular shared agent. +type SharedAgentLookup interface { + GetSharedAgentForTenant(context.Context, uint64, types.TenantRole, string, ...uint64) (*types.CustomAgent, error) +} + +// AgentShareLookup also supports read access through any reachable agent. +type AgentShareLookup interface { + SharedAgentLookup + TenantCanAccessKBViaSomeSharedAgent(context.Context, uint64, types.TenantRole, *types.KnowledgeBase) (bool, error) +} + +// ResolveKB authorizes a server-loaded KB (or a document's persisted KB/tenant +// reference). Its tenant is authoritative; resolving the owner from a share +// record again would add a lookup and a second source of truth. It never treats +// an effective resource tenant as the caller. +// Share lookup errors do not grant access; an independent agent grant may still +// allow reading. An explicit agent selector never falls back to another agent. +func ResolveKB(ctx context.Context, request KBRequest, kb *types.KnowledgeBase, required types.OrgMemberRole, + shares KBShareLookup, agents AgentShareLookup, +) (*KBAccess, error) { + if request.Caller.TenantID == 0 { + return nil, ErrUnauthorized + } + if kb == nil || kb.ID == "" { + return nil, ErrNotFound + } + if err := types.AuthorizeTenantAPIKeyKnowledgeBases(ctx, kb.ID); err != nil { + return nil, err + } + request.Caller = request.Caller.Normalize() + if caller, ok := ctx.Value(types.CallerContextKey).(types.Caller); ok && caller != request.Caller { + return nil, ErrForbidden + } + grant := func(permission types.OrgMemberRole) (*KBAccess, error) { + return &KBAccess{ + KnowledgeBase: kb, Caller: request.Caller, + EffectiveTenantID: kb.TenantID, Permission: permission, operationPermission: required, + }, nil + } + if kb.TenantID == request.Caller.TenantID { + return grant(types.OrgRoleAdmin) + } + if shares != nil { + permissions := NewKBSharePermissions(ctx, shares, request.Caller.TenantID, request.Caller.Role) + permission, shared, err := permissions.Permission(kb.ID) + if err == nil && shared && permission.HasPermission(required) { + return grant(permission) + } + } + if required != types.OrgRoleViewer || agents == nil { + return nil, ErrForbidden + } + if request.AgentID != "" { + source, err := types.ParseAgentSourceTenantID(request.AgentSourceTenantID) + if err != nil { + return nil, fmt.Errorf("%w: %s", ErrInvalidAgentSource, err) + } + agent, err := agents.GetSharedAgentForTenant( + ctx, + request.Caller.TenantID, + request.Caller.Role, + request.AgentID, + source, + ) + if err == nil && types.SharedAgentIncludesKB(agent, kb) { + return grant(types.OrgRoleViewer) + } + } else { + allowed, + err := agents.TenantCanAccessKBViaSomeSharedAgent(ctx, + request.Caller.TenantID, + request.Caller.Role, + kb) + if err == nil && allowed { + return grant(types.OrgRoleViewer) + } + } + return nil, ErrForbidden +} diff --git a/internal/application/access/knowledgebase_test.go b/internal/application/access/knowledgebase_test.go new file mode 100644 index 00000000..a29f81c5 --- /dev/null +++ b/internal/application/access/knowledgebase_test.go @@ -0,0 +1,282 @@ +package access + +import ( + "context" + "errors" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/stretchr/testify/require" +) + +type shareLookup struct { + permission types.OrgMemberRole + err error + caller uint64 + role types.TenantRole +} + +func (s *shareLookup) CheckTenantKBPermission( + _ context.Context, + _ string, + caller uint64, + role types.TenantRole, +) (types.OrgMemberRole, bool, error) { + s.caller, s.role = caller, role + return s.permission, s.permission != "", s.err +} + +type agentLookup struct { + agent *types.CustomAgent + any bool + err error + anyCalls int + source uint64 +} + +func (s *agentLookup) GetSharedAgentForTenant( + _ context.Context, + _ uint64, + _ types.TenantRole, + _ string, + source ...uint64, +) (*types.CustomAgent, error) { + s.source = source[0] + return s.agent, s.err +} + +func (s *agentLookup) TenantCanAccessKBViaSomeSharedAgent( + context.Context, + uint64, + types.TenantRole, + *types.KnowledgeBase, +) (bool, error) { + s.anyCalls++ + return s.any, s.err +} + +func TestResolveKBPermissionMatrix(t *testing.T) { + for _, tt := range []struct { + name string + own bool + permission types.OrgMemberRole + required types.OrgMemberRole + shareError bool + agentID, source, mode string + selected []string + wrongTenant, missingAgent, agentError, anyAgent bool + want types.OrgMemberRole + wantErr error + }{ + {name: "own KB", own: true, required: types.OrgRoleEditor, want: types.OrgRoleAdmin}, + { + name: "organization reader", + permission: types.OrgRoleViewer, + required: types.OrgRoleViewer, + want: types.OrgRoleViewer, + }, + + { + name: "organization editor", + permission: types.OrgRoleEditor, + required: types.OrgRoleEditor, + want: types.OrgRoleEditor, + }, + + { + name: "reader cannot write", + permission: types.OrgRoleViewer, + required: types.OrgRoleEditor, + wantErr: ErrForbidden, + }, + + { + name: "share lookup fails closed", + permission: types.OrgRoleEditor, + shareError: true, + required: types.OrgRoleViewer, + wantErr: ErrForbidden, + }, + + { + name: "independent agent grant after share error", + shareError: true, + anyAgent: true, + required: types.OrgRoleViewer, + want: types.OrgRoleViewer, + }, + + {name: "any shared agent read", anyAgent: true, required: types.OrgRoleViewer, want: types.OrgRoleViewer}, + {name: "any agent cannot write", anyAgent: true, required: types.OrgRoleEditor, wantErr: ErrForbidden}, + { + name: "agent lookup error", + anyAgent: true, + agentError: true, + required: types.OrgRoleViewer, + wantErr: ErrForbidden, + }, + + { + name: "explicit all", + agentID: "agent", + source: "2", + mode: "all", + required: types.OrgRoleViewer, + want: types.OrgRoleViewer, + }, + + { + name: "explicit selected", + agentID: "agent", + mode: "selected", + selected: []string{"kb"}, + required: types.OrgRoleViewer, + want: types.OrgRoleViewer, + }, + + { + name: "selection miss does not fall back", + agentID: "agent", + mode: "selected", + selected: []string{"other"}, + anyAgent: true, + required: types.OrgRoleViewer, + wantErr: ErrForbidden, + }, + + { + name: "none does not fall back", + agentID: "agent", + mode: "none", + anyAgent: true, + required: types.OrgRoleViewer, + wantErr: ErrForbidden, + }, + + { + name: "unknown mode denied", + agentID: "agent", + mode: "invalid", + required: types.OrgRoleViewer, + wantErr: ErrForbidden, + }, + + { + name: "tenant mismatch denied", + agentID: "agent", + mode: "all", + wrongTenant: true, + required: types.OrgRoleViewer, + wantErr: ErrForbidden, + }, + + { + name: "missing explicit agent does not fall back", + agentID: "agent", + missingAgent: true, + anyAgent: true, + required: types.OrgRoleViewer, + wantErr: ErrForbidden, + }, + + { + name: "invalid source rejected", + agentID: "agent", + source: "bad", + mode: "all", + required: types.OrgRoleViewer, + wantErr: ErrInvalidAgentSource, + }, + + { + name: "own grant precedes agent parsing", + own: true, + agentID: "agent", + source: "bad", + required: types.OrgRoleViewer, + want: types.OrgRoleAdmin, + }, + + { + name: "share grant precedes agent parsing", + permission: types.OrgRoleViewer, + agentID: "agent", + source: "bad", + required: types.OrgRoleViewer, + want: types.OrgRoleViewer, + }, + } { + t.Run(tt.name, func(t *testing.T) { + kb := &types.KnowledgeBase{ID: "kb", TenantID: 2} + if tt.own { + kb.TenantID = 1 + } + shares := &shareLookup{permission: tt.permission} + if tt.shareError { + shares.err = errors.New("unavailable") + } + agents := &agentLookup{ + any: tt.anyAgent, + agent: &types.CustomAgent{ + TenantID: 2, + Config: types.CustomAgentConfig{KBSelectionMode: tt.mode, KnowledgeBases: tt.selected}, + }, + } + if tt.wrongTenant { + agents.agent.TenantID = 3 + } + if tt.missingAgent { + agents.agent = nil + } + if tt.agentError { + agents.err = errors.New("unavailable") + } + request := KBRequest{ + Caller: types.Caller{TenantID: 1, Role: types.TenantRoleViewer}, + AgentID: tt.agentID, + AgentSourceTenantID: tt.source, + } + grant, err := ResolveKB(context.Background(), request, kb, tt.required, shares, agents) + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + require.Nil(t, grant) + } else { + require.NoError(t, err) + require.Equal(t, tt.want, grant.Permission) + require.Equal(t, uint64(1), grant.Caller.TenantID) + require.Equal(t, kb.TenantID, grant.EffectiveTenantID) + } + if !tt.own { + require.Equal(t, uint64(1), shares.caller) + require.Equal(t, types.TenantRoleViewer, shares.role) + } + if tt.agentID != "" { + require.Zero(t, agents.anyCalls) + } + if tt.source == "2" { + require.Equal(t, uint64(2), agents.source) + } + }) + } +} + +func TestResolveKBMissingIdentityResourceAndAPIKeyScope(t *testing.T) { + kb := &types.KnowledgeBase{ID: "kb", TenantID: 1} + _, err := ResolveKB(context.Background(), KBRequest{}, kb, types.OrgRoleViewer, nil, nil) + require.ErrorIs(t, err, ErrUnauthorized) + _, err = ResolveKB( + context.Background(), + KBRequest{Caller: types.Caller{TenantID: 1}}, + nil, + types.OrgRoleViewer, + nil, + nil, + ) + require.ErrorIs(t, err, ErrNotFound) + ctx := types.WithTenantAPIKeyScope( + context.Background(), + types.TenantAPIKeyScope{KnowledgeBaseIDs: types.StringArray{"other"}}, + ) + _, err = ResolveKB(ctx, KBRequest{Caller: types.Caller{TenantID: 1}}, kb, types.OrgRoleViewer, nil, nil) + require.Error(t, err, "API key scope must apply even to owned KBs") +} diff --git a/internal/application/access/message_files.go b/internal/application/access/message_files.go new file mode 100644 index 00000000..7f7d6bd1 --- /dev/null +++ b/internal/application/access/message_files.go @@ -0,0 +1,249 @@ +package access + +import ( + "context" + "strings" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// MessageFileLookup is the narrow message-service surface needed by the +// message-scoped file proxy. Keeping it small makes the authorization boundary +// independently testable. +type MessageFileLookup interface { + GetMessage(ctx context.Context, sessionID, messageID string) (*types.Message, error) +} + +// SharedAgentFileLookup verifies that a source workspace's agent is still +// shared to the caller. Revoking the share therefore also revokes historical +// message-file access. +type SharedAgentFileLookup interface { + GetSharedAgentForTenant( + ctx context.Context, + tenantID uint64, + callerTenantRole types.TenantRole, + agentID string, + sourceTenantID ...uint64, + ) (*types.CustomAgent, error) +} + +// KBSharePermissionGuard is the org-share surface the message-scoped proxy +// needs for its shared-KB fallback. +type KBSharePermissionGuard = KBShareLookup + +// KBTenantLookup resolves knowledge bases without a tenant filter, so a +// message's persisted retrieval evidence can be mapped back to the knowledge +// bases the chunks were retrieved from. +type KBTenantLookup interface { + GetKnowledgeBasesByIDsOnly(ctx context.Context, ids []string) ([]*types.KnowledgeBase, error) +} + +// KnowledgeOwnerLookup resolves a knowledge entry to its knowledge base for +// message references that predate the denormalized knowledge_base_id field. +type KnowledgeOwnerLookup interface { + GetKnowledgeByIDOnly(ctx context.Context, id string) (*types.Knowledge, error) +} + +// MessageKBShareAuthorizer bundles the read-only lookups behind the +// message proxy's org-shared-KB fallback. A nil ShareGuard or KBs lookup +// disables the fallback. A live Bindings lookup is also required and can +// be supplied by the resource catalog. A nil Knowledges lookup only disables the +// pre-denormalization KnowledgeID path. +type MessageKBShareAuthorizer struct { + ShareGuard KBSharePermissionGuard + KBs KBTenantLookup + Knowledges KnowledgeOwnerLookup + Bindings interfaces.KBResourceLookup +} + +// resourceAccessibleViaSharedKB reports whether the message's persisted +// retrieval evidence proves the resource came from an org-shared KB the +// caller may read. This is the fallback for replies whose agent belongs to +// the caller's own workspace while the retrieved resources belong to the +// workspace that shared the knowledge base (#3022). +// +// Evidence required from one retrieval record: a canonical resource:// +// handle (not a prefix of a longer token) appears in the chunk text or +// image_info, its knowledge base belongs to the resource's tenant, and +// that KB is org-shared to the caller with at least viewer permission, and +// an independent live resource binding confirms the file still belongs to it. +// Smart-reasoning turns persist that evidence on AgentSteps when +// KnowledgeReferences was never filled. Any lookup failure fails closed. +func (a MessageKBShareAuthorizer) resourceAccessibleViaSharedKB( + ctx context.Context, + message *types.Message, + resource *types.StoredResource, + callerTenantID uint64, + callerTenantRole types.TenantRole, +) bool { + if a.ShareGuard == nil || a.KBs == nil || a.Bindings == nil || message == nil || resource == nil { + return false + } + handle, ok := types.ParseResourcePath(types.BuildResourcePath(resource.Handle)) + if !ok { + return false + } + + kbIDs := a.collectSharedKBEvidenceIDs(ctx, message, handle) + if len(kbIDs) == 0 { + return false + } + + kbs, err := a.KBs.GetKnowledgeBasesByIDsOnly(ctx, kbIDs) + if err != nil { + return false + } + permissions := NewKBSharePermissions(ctx, a.ShareGuard, callerTenantID, callerTenantRole) + for _, kb := range kbs { + if kb == nil || kb.TenantID != resource.TenantID { + continue + } + shared, err := permissions.Check(kb.ID, types.OrgRoleViewer) + if err == nil && shared { + bound, bindingErr := a.Bindings.IsReferencedByKnowledgeBase( + ctx, resource.TenantID, kb.ID, types.BuildResourcePath(handle)) + if bindingErr == nil && bound { + return true + } + } + } + return false +} + +func (a MessageKBShareAuthorizer) collectSharedKBEvidenceIDs( + ctx context.Context, + message *types.Message, + handle string, +) []string { + seenKB := make(map[string]bool) + seenKnowledge := make(map[string]bool) + var kbIDs, knowledgeIDs []string + addKB := func(id string) { + if id == "" || seenKB[id] { + return + } + seenKB[id] = true + kbIDs = append(kbIDs, id) + } + addKnowledge := func(id string) { + if id == "" || seenKnowledge[id] { + return + } + seenKnowledge[id] = true + knowledgeIDs = append(knowledgeIDs, id) + } + + for _, ref := range message.KnowledgeReferences { + if !searchResultHasResourceHandle(ref, handle) { + continue + } + if ref.KnowledgeBaseID != "" { + addKB(ref.KnowledgeBaseID) + continue + } + addKnowledge(ref.KnowledgeID) + } + collectKBEvidenceFromValue(message.AgentSteps, handle, "", "", addKB, addKnowledge) + + if a.Knowledges != nil { + for _, knowledgeID := range knowledgeIDs { + knowledge, err := a.Knowledges.GetKnowledgeByIDOnly(ctx, knowledgeID) + if err != nil || knowledge == nil { + continue + } + addKB(knowledge.KnowledgeBaseID) + } + } + return kbIDs +} + +func searchResultHasResourceHandle(ref *types.SearchResult, handle string) bool { + if ref == nil { + return false + } + return textHasResourceHandle(ref.Content, handle) || + textHasResourceHandle(ref.MatchedContent, handle) || + textHasResourceHandle(ref.ImageInfo, handle) +} + +func textHasResourceHandle(text, handle string) bool { + if text == "" || handle == "" { + return false + } + want := types.BuildResourcePath(handle) + for _, ref := range types.ScanResourceReferences(text) { + if ref == want { + return true + } + } + return false +} + +func collectKBEvidenceFromValue( + v interface{}, + handle, kbID, knowledgeID string, + addKB, addKnowledge func(string), +) { + switch val := v.(type) { + case string: + if !textHasResourceHandle(val, handle) { + return + } + if kbID != "" { + addKB(kbID) + return + } + addKnowledge(knowledgeID) + case types.AgentSteps: + for _, step := range val { + collectKBEvidenceFromValue(step, handle, kbID, knowledgeID, addKB, addKnowledge) + } + case types.AgentStep: + collectKBEvidenceFromValue(val.ToolCalls, handle, kbID, knowledgeID, addKB, addKnowledge) + case []types.ToolCall: + for _, call := range val { + collectKBEvidenceFromValue(call, handle, kbID, knowledgeID, addKB, addKnowledge) + } + case types.ToolCall: + if val.Result != nil { + collectKBEvidenceFromValue(val.Result.Output, handle, kbID, knowledgeID, addKB, addKnowledge) + collectKBEvidenceFromValue(val.Result.Data, handle, kbID, knowledgeID, addKB, addKnowledge) + } + case map[string]interface{}: + nextKB := firstNonEmptyString( + stringFromAnyMap(val, "knowledge_base_id"), + stringFromAnyMap(val, "knowledge_base"), + kbID, + ) + nextKnowledge := firstNonEmptyString(stringFromAnyMap(val, "knowledge_id"), knowledgeID) + for _, nested := range val { + collectKBEvidenceFromValue(nested, handle, nextKB, nextKnowledge, addKB, addKnowledge) + } + case []interface{}: + for _, item := range val { + collectKBEvidenceFromValue(item, handle, kbID, knowledgeID, addKB, addKnowledge) + } + case []map[string]interface{}: + for _, item := range val { + collectKBEvidenceFromValue(item, handle, kbID, knowledgeID, addKB, addKnowledge) + } + } +} + +func stringFromAnyMap(m map[string]interface{}, key string) string { + if m == nil { + return "" + } + s, _ := m[key].(string) + return strings.TrimSpace(s) +} + +func firstNonEmptyString(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return strings.TrimSpace(value) + } + } + return "" +} diff --git a/internal/application/access/ownership.go b/internal/application/access/ownership.go new file mode 100644 index 00000000..29efc876 --- /dev/null +++ b/internal/application/access/ownership.go @@ -0,0 +1,69 @@ +package access + +import ( + "errors" + + "github.com/Tencent/WeKnora/internal/types" +) + +var ( + // ErrResourceNotFound means the ownership lookup found no resource visible + // in the caller's tenant. HTTP adapters decide how to produce the 404. + ErrResourceNotFound = errors.New("rbac: resource not found") + // ErrOwnershipForbidden indicates neither ownership nor the required role. + ErrOwnershipForbidden = errors.New("rbac: ownership or role insufficient") +) + +// OwnershipRequest carries server-resolved identity and rollout policy. The +// cross-tenant bypass must already account for the cluster configuration. +// This gate does not replace API-key capabilities or resource-scope checks. +type OwnershipRequest struct { + UserID string + Role types.TenantRole + APIKey bool + CrossTenantSuperuser bool + Enforce bool +} + +// OwnershipDecision contains the details adapters need for logs and audits. +// The error returned with it determines whether access was allowed. +type OwnershipDecision struct { + CreatorID string + EnforcementSkipped bool + LookupFailed bool +} + +// CheckOwnershipOrRole performs no resource lookup when API-key authorization, +// role, superuser status, or the RBAC rollout policy already settles the gate. +// The lookup must enforce tenant visibility before returning a creator ID. +func CheckOwnershipOrRole( + request OwnershipRequest, + required types.TenantRole, + lookup func() (string, error), +) (OwnershipDecision, error) { + var decision OwnershipDecision + if request.APIKey || request.Role.HasPermission(required) || request.CrossTenantSuperuser { + return decision, nil + } + if !request.Enforce { + decision.EnforcementSkipped = true + return decision, nil + } + if lookup == nil { + decision.LookupFailed = true + return decision, errors.New("rbac: creator lookup unavailable") + } + creator, err := lookup() + decision.CreatorID = creator + if errors.Is(err, ErrResourceNotFound) { + return decision, ErrResourceNotFound + } + if err != nil { + decision.LookupFailed = true + return decision, err + } + if creator != "" && creator == request.UserID { + return decision, nil + } + return decision, ErrOwnershipForbidden +} diff --git a/internal/application/access/ownership_test.go b/internal/application/access/ownership_test.go new file mode 100644 index 00000000..77806b64 --- /dev/null +++ b/internal/application/access/ownership_test.go @@ -0,0 +1,103 @@ +package access + +import ( + "errors" + "fmt" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/stretchr/testify/require" +) + +func TestCheckOwnershipOrRoleSkipsLookupWhenGateAlreadyResolved(t *testing.T) { + for _, tt := range []struct { + name string + request OwnershipRequest + skipped bool + }{ + {name: "API key", request: OwnershipRequest{APIKey: true, Role: types.TenantRoleViewer, Enforce: true}}, + {name: "admin", request: OwnershipRequest{Role: types.TenantRoleAdmin, Enforce: true}}, + {name: "owner role", request: OwnershipRequest{Role: types.TenantRoleOwner, Enforce: true}}, + { + name: "cross tenant superuser", + request: OwnershipRequest{ + Role: types.TenantRoleViewer, + CrossTenantSuperuser: true, + Enforce: true, + }, + }, + + {name: "enforcement disabled", request: OwnershipRequest{Role: types.TenantRoleViewer}, skipped: true}, + {name: "role before rollout", request: OwnershipRequest{Role: types.TenantRoleAdmin}}, + } { + t.Run(tt.name, func(t *testing.T) { + decision, err := CheckOwnershipOrRole(tt.request, types.TenantRoleAdmin, func() (string, error) { + t.Fatal("ownership lookup must not run on a fast path") + return "", errors.New("unavailable") + }) + require.NoError(t, err) + require.Equal(t, tt.skipped, decision.EnforcementSkipped) + require.False(t, decision.LookupFailed) + }) + } +} + +func TestCheckOwnershipOrRoleResolvesCreator(t *testing.T) { + failure := errors.New("database unavailable") + for _, tt := range []struct { + name, user, creator string + lookupErr, wantErr error + lookupFailed bool + }{ + {name: "creator", user: "user", creator: "user"}, + {name: "other creator", user: "user", creator: "other", wantErr: ErrOwnershipForbidden}, + {name: "tenant owned", user: "user", wantErr: ErrOwnershipForbidden}, + {name: "missing identity cannot match empty creator", wantErr: ErrOwnershipForbidden}, + {name: "missing resource", user: "user", lookupErr: ErrResourceNotFound, wantErr: ErrResourceNotFound}, + { + name: "wrapped missing resource", + user: "user", + lookupErr: fmt.Errorf("load: %w", + ErrResourceNotFound), + wantErr: ErrResourceNotFound, + }, + + { + name: "lookup failure wins over creator match", + user: "user", + creator: "user", + lookupErr: failure, + wantErr: failure, + lookupFailed: true, + }, + + { + name: "lookup denial remains a lookup failure", + lookupErr: ErrOwnershipForbidden, + wantErr: ErrOwnershipForbidden, + lookupFailed: true, + }, + } { + t.Run(tt.name, func(t *testing.T) { + calls := 0 + decision, err := CheckOwnershipOrRole( + OwnershipRequest{UserID: tt.user, Role: types.TenantRoleContributor, Enforce: true}, + types.TenantRoleAdmin, + func() (string, error) { + calls++ + return tt.creator, tt.lookupErr + }, + ) + require.Equal(t, 1, calls) + require.ErrorIs(t, err, tt.wantErr) + require.Equal(t, tt.creator, decision.CreatorID) + require.Equal(t, tt.lookupFailed, decision.LookupFailed) + }) + } +} + +func TestCheckOwnershipOrRoleMissingLookupFailsClosed(t *testing.T) { + decision, err := CheckOwnershipOrRole(OwnershipRequest{Enforce: true}, types.TenantRoleAdmin, nil) + require.Error(t, err) + require.True(t, decision.LookupFailed) +} diff --git a/internal/application/repository/chunk.go b/internal/application/repository/chunk.go index ccb9fe87..d06c83cf 100644 --- a/internal/application/repository/chunk.go +++ b/internal/application/repository/chunk.go @@ -575,7 +575,7 @@ func (r *chunkRepository) DeleteByKnowledgeList(ctx context.Context, tenantID ui func (r *chunkRepository) MoveChunksByKnowledgeID(ctx context.Context, tenantID uint64, knowledgeID string, targetKBID string) error { return r.db.WithContext(ctx).Model(&types.Chunk{}). Where("tenant_id = ? AND knowledge_id = ?", tenantID, knowledgeID). - Update("knowledge_base_id", targetKBID).Error + Updates(map[string]any{"knowledge_base_id": targetKBID, "tag_id": ""}).Error } // DeleteChunksByTagID deletes all chunks with the specified tag ID @@ -953,30 +953,22 @@ func (r *chunkRepository) UpdateChunkFieldsByTagID( newTagID *string, excludeIDs []string, ) ([]string, error) { - // First, get the IDs of chunks that will be affected (for is_enabled sync) + if isEnabled == nil && setFlags == 0 && clearFlags == 0 && newTagID == nil { + return nil, nil + } + // Return every affected entry, including tag-only and flag-only changes. + // Callers use these IDs for index synchronization and subsequent patches. var affectedIDs []string - if isEnabled != nil { - var chunks []*types.Chunk - query := r.db.WithContext(ctx). - Select("id"). - Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", - tenantID, kbID, types.ChunkTypeFAQ) - if tagID != "" { - query = query.Where("tag_id = ?", tagID) - } - - if len(excludeIDs) > 0 { - query = query.Where("id NOT IN ?", excludeIDs) - } - - // Only get chunks that need to change - query = query.Where("is_enabled != ?", *isEnabled) - if err := query.Find(&chunks).Error; err != nil { - return nil, err - } - for _, c := range chunks { - affectedIDs = append(affectedIDs, c.ID) - } + selection := r.db.WithContext(ctx).Model(&types.Chunk{}). + Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", tenantID, kbID, types.ChunkTypeFAQ) + if tagID != "" { + selection = selection.Where("tag_id = ?", tagID) + } + if len(excludeIDs) > 0 { + selection = selection.Where("id NOT IN ?", excludeIDs) + } + if err := selection.Pluck("id", &affectedIDs).Error; err != nil { + return nil, err } // Build update query @@ -1309,3 +1301,17 @@ func (r *chunkRepository) ListRecentDocumentChunksWithQuestions( return chunks, nil } + +func (r *chunkRepository) ListAllChunksByKnowledgeID( + ctx context.Context, + tenantID uint64, + knowledgeID string, +) ([]*types.Chunk, error) { + var chunks []*types.Chunk + err := r.db.WithContext(ctx). + Where("tenant_id = ? AND knowledge_id = ?", tenantID, knowledgeID). + Order("id ASC"). + Find(&chunks). + Error + return chunks, err +} diff --git a/internal/application/repository/chunk_fields_test.go b/internal/application/repository/chunk_fields_test.go new file mode 100644 index 00000000..f8a9be27 --- /dev/null +++ b/internal/application/repository/chunk_fields_test.go @@ -0,0 +1,60 @@ +package repository + +import ( + "context" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/stretchr/testify/require" +) + +func TestTagFieldUpdatesReturnAllAffectedChunks(t *testing.T) { + for _, flagsOnly := range []bool{false, true} { + name := "tag only" + if flagsOnly { + name = "flags only" + } + t.Run(name, func(t *testing.T) { + db := setupChunkTestDB(t) + repo := NewChunkRepository(db) + for _, chunk := range []*types.Chunk{ + {ID: "selected", TenantID: 7, KnowledgeBaseID: "kb", ChunkType: types.ChunkTypeFAQ, TagID: "tag"}, + {ID: "excluded", TenantID: 7, KnowledgeBaseID: "kb", ChunkType: types.ChunkTypeFAQ, TagID: "tag"}, + {ID: "other-kb", TenantID: 7, KnowledgeBaseID: "other", ChunkType: types.ChunkTypeFAQ, TagID: "tag"}, + {ID: "other-tenant", TenantID: 8, KnowledgeBaseID: "kb", ChunkType: types.ChunkTypeFAQ, TagID: "tag"}, + {ID: "deleted", TenantID: 7, KnowledgeBaseID: "kb", ChunkType: types.ChunkTypeFAQ, TagID: "tag"}, + } { + require.NoError(t, db.Create(chunk).Error) + } + require.NoError(t, db.Delete(&types.Chunk{ID: "deleted"}).Error) + next := "next" + var tag *string + var flags types.ChunkFlags + if flagsOnly { + flags = types.ChunkFlagRecommended + } else { + tag = &next + } + ids, err := repo.UpdateChunkFieldsByTagID( + context.Background(), + 7, + "kb", + "tag", + nil, + flags, + 0, + tag, + []string{"excluded"}, + ) + require.NoError(t, err) + require.Equal(t, []string{"selected"}, ids) + selected, err := repo.GetChunkByID(context.Background(), 7, "selected") + require.NoError(t, err) + if flagsOnly { + require.True(t, selected.Flags.HasFlag(types.ChunkFlagRecommended)) + } else { + require.Equal(t, "next", selected.TagID) + } + }) + } +} diff --git a/internal/application/repository/knowledge.go b/internal/application/repository/knowledge.go index 065912c6..db94133e 100644 --- a/internal/application/repository/knowledge.go +++ b/internal/application/repository/knowledge.go @@ -560,15 +560,20 @@ func (r *knowledgeRepository) UpdateKnowledgeColumns( // to normal queries and have not moved out of the transient deleting state. func (r *knowledgeRepository) UpdateActiveDeletingKnowledgeColumns( ctx context.Context, - id string, + tenantID uint64, + kbID, id string, values map[string]interface{}, ) (bool, error) { - if len(values) == 0 { + if tenantID == 0 || kbID == "" || len(values) == 0 { return false, nil } result := r.db.WithContext(ctx). Model(&types.Knowledge{}). - Where("id = ? AND parse_status = ?", id, types.ParseStatusDeleting). + Where("tenant_id = ? AND knowledge_base_id = ? AND id = ? AND parse_status = ?", + tenantID, + kbID, + id, + types.ParseStatusDeleting). Updates(values) if result.Error != nil { return false, result.Error diff --git a/internal/application/repository/knowledge_finalize_test.go b/internal/application/repository/knowledge_finalize_test.go index abcb68a9..501ea8a6 100644 --- a/internal/application/repository/knowledge_finalize_test.go +++ b/internal/application/repository/knowledge_finalize_test.go @@ -353,22 +353,63 @@ func TestUpdateActiveDeletingKnowledgeColumns_GuardsStateAndSoftDelete(t *testin activeCompletedID := insertKnowledgeWithStatus(t, db, types.ParseStatusCompleted, false) deletedDeletingID := insertKnowledgeWithStatus(t, db, types.ParseStatusDeleting, true) - updated, err := repo.UpdateActiveDeletingKnowledgeColumns(ctx, activeDeletingID, map[string]interface{}{ - "parse_status": types.ParseStatusFailed, - "error_message": "delete task exhausted retries", - }) + require.NoError( + t, + db.Exec( + "UPDATE knowledges SET knowledge_base_id = ? WHERE id IN ?", + "delete-kb", + []string{activeDeletingID, activeCompletedID, deletedDeletingID}, + ).Error, + ) + for _, scope := range []struct { + tenant uint64 + kb string + }{{2, "delete-kb"}, {1, "other-kb"}, {0, "delete-kb"}, {1, ""}} { + updated, err := repo.UpdateActiveDeletingKnowledgeColumns( + ctx, + scope.tenant, + scope.kb, + activeDeletingID, + map[string]interface{}{"parse_status": types.ParseStatusFailed}, + ) + require.NoError(t, err) + require.False(t, updated) + } + + updated, err := repo.UpdateActiveDeletingKnowledgeColumns( + ctx, + 1, + "delete-kb", + activeDeletingID, + map[string]interface{}{ + "parse_status": types.ParseStatusFailed, + "error_message": "delete task exhausted retries", + }, + ) require.NoError(t, err) assert.True(t, updated) - updated, err = repo.UpdateActiveDeletingKnowledgeColumns(ctx, activeCompletedID, map[string]interface{}{ - "parse_status": types.ParseStatusFailed, - }) + updated, err = repo.UpdateActiveDeletingKnowledgeColumns( + ctx, + 1, + "delete-kb", + activeCompletedID, + map[string]interface{}{ + "parse_status": types.ParseStatusFailed, + }, + ) require.NoError(t, err) assert.False(t, updated) - updated, err = repo.UpdateActiveDeletingKnowledgeColumns(ctx, deletedDeletingID, map[string]interface{}{ - "parse_status": types.ParseStatusFailed, - }) + updated, err = repo.UpdateActiveDeletingKnowledgeColumns( + ctx, + 1, + "delete-kb", + deletedDeletingID, + map[string]interface{}{ + "parse_status": types.ParseStatusFailed, + }, + ) require.NoError(t, err) assert.False(t, updated) diff --git a/internal/application/repository/knowledge_list_filter_test.go b/internal/application/repository/knowledge_list_filter_test.go index a3283622..364e7b6e 100644 --- a/internal/application/repository/knowledge_list_filter_test.go +++ b/internal/application/repository/knowledge_list_filter_test.go @@ -84,7 +84,7 @@ func TestGetKnowledgeBatch_TracksDeletionUntilSoftDelete(t *testing.T) { assert.Equal(t, id, rows[0].ID) assert.Equal(t, types.ParseStatusDeleting, rows[0].ParseStatus) - updated, err := repo.UpdateActiveDeletingKnowledgeColumns(ctx, id, map[string]interface{}{ + updated, err := repo.UpdateActiveDeletingKnowledgeColumns(ctx, 1, kbID, id, map[string]interface{}{ "parse_status": types.ParseStatusFailed, }) require.NoError(t, err) diff --git a/internal/application/repository/knowledge_transfer.go b/internal/application/repository/knowledge_transfer.go new file mode 100644 index 00000000..99b12078 --- /dev/null +++ b/internal/application/repository/knowledge_transfer.go @@ -0,0 +1,74 @@ +package repository + +import ( + "context" + "fmt" + "time" + + "github.com/Tencent/WeKnora/internal/types" + "gorm.io/gorm" +) + +// UpdateKnowledgeForTransfer is a compare-and-swap checkpoint. Full-row Save +// could resurrect a deleted document or overwrite a concurrently moved row. +// Accounting changes are committed with the checkpoint, making retry cleanup +// safe even if the worker crashes between the database and queue operations. +func (r *knowledgeRepository) UpdateKnowledgeForTransfer(ctx context.Context, before, after *types.Knowledge) error { + if before == nil || after == nil || before.ID == "" || before.ID != after.ID || before.TenantID == 0 || + before.TenantID != after.TenantID { + return fmt.Errorf("invalid knowledge transfer checkpoint") + } + // PostgreSQL persists timestamps at microsecond precision. Keep the returned + // checkpoint identical to storage so the next CAS in this task can match it. + after.UpdatedAt = time.Now().UTC().Truncate(time.Microsecond) + var storedUpdatedAt time.Time + err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + result := tx.Model(&types.Knowledge{}). + Where("id = ? AND tenant_id = ? AND knowledge_base_id = ? AND parse_status = ? AND "+ + "updated_at = ?", + before.ID, + before.TenantID, + before.KnowledgeBaseID, + before.ParseStatus, + before.UpdatedAt). + Updates(map[string]any{ + "knowledge_base_id": after.KnowledgeBaseID, "parse_status": after.ParseStatus, + "metadata": after.Metadata, "storage_size": after.StorageSize, "updated_at": after.UpdatedAt, + "enable_status": after.EnableStatus, "embedding_model_id": after.EmbeddingModelID, + "description": after.Description, + "processed_at": after.ProcessedAt, + "error_message": after.ErrorMessage, + }) + if result.Error != nil { + return result.Error + } + if result.RowsAffected != 1 { + return fmt.Errorf("knowledge changed during transfer") + } + delta := after.StorageSize - before.StorageSize + if delta != 0 { + result := tx.Model(&types.Tenant{}).Where("id = ?", before.TenantID). + UpdateColumn("storage_used", + gorm.Expr("CASE WHEN storage_used + ? < 0 THEN 0 ELSE storage_used + ? END", + delta, + delta)) + if result.Error != nil { + return result.Error + } + if result.RowsAffected != 1 { + return fmt.Errorf("transfer tenant not found") + } + } + var stored struct{ UpdatedAt time.Time } + if err := tx.Model(&types.Knowledge{}).Select("updated_at"). + Where("id = ? AND tenant_id = ?", after.ID, after.TenantID).Take(&stored).Error; err != nil { + return err + } + storedUpdatedAt = stored.UpdatedAt + return nil + }) + if err == nil { + after.UpdatedAt = storedUpdatedAt + } + return err +} diff --git a/internal/application/repository/knowledge_transfer_test.go b/internal/application/repository/knowledge_transfer_test.go new file mode 100644 index 00000000..d5664f09 --- /dev/null +++ b/internal/application/repository/knowledge_transfer_test.go @@ -0,0 +1,30 @@ +package repository + +import ( + "context" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/stretchr/testify/require" +) + +func TestTransferCheckpointReturnsStoredTimestampPrecision(t *testing.T) { + db := setupKnowledgeTestDB(t) + // Emulate a database that stores fewer fractional digits than time.Now. + require.NoError(t, db.Exec(`CREATE TRIGGER truncate_checkpoint_time AFTER UPDATE ON knowledges + BEGIN UPDATE knowledges SET updated_at = strftime('%Y-%m-%d %H:%M:%S', NEW.updated_at) || '+00:00' + WHERE id = NEW.id; END`).Error) + row := &types.Knowledge{ID: "doc", TenantID: 7, KnowledgeBaseID: "source", ParseStatus: types.ParseStatusCompleted} + require.NoError(t, db.Omit("custom_metadata").Create(row).Error) + repo := NewKnowledgeRepository(db) + before, err := repo.GetKnowledgeByID(context.Background(), 7, "doc") + require.NoError(t, err) + after := *before + after.ParseStatus = types.ParseStatusProcessing + require.NoError(t, repo.UpdateKnowledgeForTransfer(context.Background(), before, &after)) + require.Zero(t, after.UpdatedAt.Nanosecond()) + final := after + final.KnowledgeBaseID = "target" + final.ParseStatus = types.ParseStatusCompleted + require.NoError(t, repo.UpdateKnowledgeForTransfer(context.Background(), &after, &final)) +} diff --git a/internal/application/repository/resource_references.go b/internal/application/repository/resource_references.go new file mode 100644 index 00000000..baf32ee1 --- /dev/null +++ b/internal/application/repository/resource_references.go @@ -0,0 +1,57 @@ +package repository + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/types" +) + +// IsReferencedByKnowledgeBase accepts only explicit bindings to live documents. +// Text mentioning a handle or physical path is never ownership evidence. +func (r *resourceRepository) IsReferencedByKnowledgeBase( + ctx context.Context, tenantID uint64, kbID, resourceID string, +) (bool, error) { + if tenantID == 0 || kbID == "" || resourceID == "" { + return false, nil + } + var count int64 + err := r.db.WithContext(ctx).Table("resource_bindings AS b"). + Joins("JOIN knowledges AS k ON k.id = b.owner_id AND k.tenant_id = b.tenant_id"). + Joins("JOIN knowledge_bases AS kb ON kb.id = k.knowledge_base_id "+ + "AND kb.tenant_id = k.tenant_id AND kb.deleted_at IS NULL"). + Where("b.resource_id = ? AND b.owner_type = ? AND b.tenant_id = ? "+ + "AND k.knowledge_base_id = ? AND k.deleted_at IS NULL", + resourceID, types.ResourceOwnerKnowledge, tenantID, kbID).Count(&count).Error + return count > 0, err +} + +func (r *resourceRepository) GetMessageFileBindings( + ctx context.Context, tenantID uint64, resourceID, messageID string, +) (*types.MessageFileBindings, error) { + result := &types.MessageFileBindings{} + if tenantID == 0 || resourceID == "" { + return result, nil + } + err := r.db.WithContext(ctx).Table("resource_bindings AS b"). + Joins("JOIN knowledges AS k ON k.id = b.owner_id AND k.tenant_id = b.tenant_id"). + Joins("JOIN knowledge_bases AS kb ON kb.id = k.knowledge_base_id "+ + "AND kb.tenant_id = k.tenant_id AND kb.deleted_at IS NULL"). + Where("b.resource_id = ? AND b.owner_type = ? AND b.tenant_id = ? AND k.deleted_at IS NULL", + resourceID, types.ResourceOwnerKnowledge, tenantID). + Distinct("k.knowledge_base_id").Pluck("k.knowledge_base_id", &result.KnowledgeBaseIDs).Error + if err != nil { + return nil, err + } + if messageID != "" { + var count int64 + err = r.db.WithContext(ctx).Model(&types.ResourceBinding{}). + Where("resource_id = ? AND tenant_id = ? AND owner_type = ? AND owner_id = ? AND relation = ?", + resourceID, tenantID, types.ResourceOwnerMessage, messageID, types.ResourceRelationArtifact). + Count(&count).Error + if err != nil { + return nil, err + } + result.MessageArtifact = count > 0 + } + return result, nil +} diff --git a/internal/application/repository/retriever/doris/move.go b/internal/application/repository/retriever/doris/move.go new file mode 100644 index 00000000..1b745d55 --- /dev/null +++ b/internal/application/repository/retriever/doris/move.go @@ -0,0 +1,41 @@ +package doris + +import ( + "context" + "fmt" +) + +func (r *dorisRepository) ValidateKnowledgeIndexMove(ctx context.Context) error { + mode, err := r.resolveCompatMode(ctx) + if err != nil { + return err + } + if mode.usesRewriteChunkUpdates() { + // ANN DUPLICATE KEY tables implement replacement as DELETE + INSERT. + // A failed insert would lose the only vector copy; changing physical IDs + // would break the stable source-ID identity used by subsequent edits. + return fmt.Errorf("reuse_vectors move is not supported by Doris ANN tables; use reparse mode") + } + return nil +} + +func (r *dorisRepository) MoveKnowledgeIndices( + ctx context.Context, + sourceKB, targetKB, knowledgeID string, + _ []string, + dimension int, + _ string, +) error { + if err := r.ValidateKnowledgeIndexMove(ctx); err != nil { + return err + } + if dimension <= 0 { + return fmt.Errorf("invalid embedding dimension") + } + query := fmt.Sprintf( + "UPDATE `%s` SET knowledge_base_id = ?, tag_id = '' WHERE knowledge_base_id = ? AND knowledge_id = ?", + r.getTableName(dimension), + ) + _, err := r.db.ExecContext(ctx, query, targetKB, sourceKB, knowledgeID) + return err +} diff --git a/internal/application/repository/retriever/doris/move_test.go b/internal/application/repository/retriever/doris/move_test.go new file mode 100644 index 00000000..a22c521f --- /dev/null +++ b/internal/application/repository/retriever/doris/move_test.go @@ -0,0 +1,36 @@ +package doris + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/stretchr/testify/require" +) + +func TestMoveIndicesRejectsNonAtomicANNReplacement(t *testing.T) { + repo, mock, _, cleanup := newTestRepo(t) + defer cleanup() + primeCompatMode(repo, dorisCompatModeInnerProductDuplicate, nil) + require.ErrorContains( + t, + repo.MoveKnowledgeIndices(context.Background(), "source", "target", "doc", []string{"chunk"}, 2, "document"), + "use reparse mode", + ) + require.NoError(t, mock.ExpectationsWereMet(), "rejection must issue no vector delete or insert") +} + +func TestMoveIndicesLegacyUpdatesOnlyExactSourceDocument(t *testing.T) { + repo, mock, _, cleanup := newTestRepo(t) + defer cleanup() + primeCompatMode(repo, dorisCompatModeLegacy, nil) + mock.ExpectExec("UPDATE `weknora_embeddings_2` SET knowledge_base_id = \\?, tag_id = '' WHERE "+ + "knowledge_base_id = \\? AND knowledge_id = \\?"). + WithArgs("target", "source", "doc"). + WillReturnResult(sqlmock.NewResult(0, 3)) + require.NoError( + t, + repo.MoveKnowledgeIndices(context.Background(), "source", "target", "doc", []string{"chunk"}, 2, "document"), + ) + require.NoError(t, mock.ExpectationsWereMet()) +} diff --git a/internal/application/repository/retriever/elasticsearch/v7/move.go b/internal/application/repository/retriever/elasticsearch/v7/move.go new file mode 100644 index 00000000..3e85a08e --- /dev/null +++ b/internal/application/repository/retriever/elasticsearch/v7/move.go @@ -0,0 +1,59 @@ +package v7 + +import ( + "bytes" + "context" + "encoding/json" + "fmt" +) + +func (e *elasticsearchRepository) MoveKnowledgeIndices( + ctx context.Context, + sourceKB, targetKB, knowledgeID string, + _ []string, + _ int, + _ string, +) error { + body, err := json.Marshal(map[string]any{ + "query": map[string]any{"bool": map[string]any{"filter": []any{ + map[string]any{"term": map[string]any{e.idField("knowledge_base_id"): sourceKB}}, + map[string]any{"term": map[string]any{e.idField("knowledge_id"): knowledgeID}}, + }}}, + "script": map[string]any{ + "lang": "painless", + "source": "ctx._source.knowledge_base_id = params.target; ctx._source.tag_id = ''", + "params": map[string]any{"target": targetKB}, + }, + }) + if err != nil { + return err + } + resp, err := e.client.UpdateByQuery( + []string{e.index}, + e.client.UpdateByQuery.WithContext(ctx), + e.client.UpdateByQuery.WithBody(bytes.NewReader(body)), + e.client.UpdateByQuery.WithRefresh(true), + ) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + if resp.IsError() { + return fmt.Errorf("move indices: %s", resp.String()) + } + var result struct { + Total *int64 `json:"total"` + Updated *int64 `json:"updated"` + TimedOut bool `json:"timed_out"` + Conflicts int `json:"version_conflicts"` + Failures []json.RawMessage `json:"failures"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return err + } + if result.Total == nil || result.Updated == nil || *result.Total < 0 || *result.Total != *result.Updated || + result.TimedOut || result.Conflicts != 0 || len(result.Failures) != 0 { + return fmt.Errorf("move indices was incomplete") + } + return nil +} diff --git a/internal/application/repository/retriever/elasticsearch/v7/move_test.go b/internal/application/repository/retriever/elasticsearch/v7/move_test.go new file mode 100644 index 00000000..6f1d63a2 --- /dev/null +++ b/internal/application/repository/retriever/elasticsearch/v7/move_test.go @@ -0,0 +1,59 @@ +package v7 + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/elastic/go-elasticsearch/v7" + "github.com/stretchr/testify/require" +) + +func TestMoveIndicesRequiresCompleteCounts(t *testing.T) { + for _, tt := range []struct { + name, body string + fails bool + }{ + {"complete", `{"total":100,"updated":100}`, false}, + {"empty retry", `{"total":0,"updated":0}`, false}, + {"partial", `{"total":100,"updated":40}`, true}, + {"missing total", `{"updated":40}`, true}, + {"missing updated", `{"total":40}`, true}, + {"missing counts", `{}`, true}, + {"negative", `{"total":-1,"updated":-1}`, true}, + {"conflict", `{"total":1,"updated":1,"version_conflicts":1}`, true}, + {"timeout", `{"total":1,"updated":1,"timed_out":true}`, true}, + } { + t.Run(tt.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Elastic-Product", "Elasticsearch") + if r.URL.Path == "/" { + _, _ = w.Write( + []byte( + `{"version":{"number":"7.17.0","build_flavor":"default"},"tagline":"You Know, for Search"}`, + ), + ) + return + } + if r.URL.Path != "/vectors/_update_by_query" { + t.Errorf("unexpected path: %s", r.URL.Path) + } + _, _ = w.Write([]byte(tt.body)) + })) + defer server.Close() + client, err := elasticsearch.NewClient( + elasticsearch.Config{Addresses: []string{server.URL}, DisableRetry: true}, + ) + require.NoError(t, err) + repo := &elasticsearchRepository{client: client, index: "vectors"} + err = repo.MoveKnowledgeIndices(context.Background(), "source", "target", "doc", nil, 3, "file") + if tt.fails { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/internal/application/repository/retriever/elasticsearch/v8/move.go b/internal/application/repository/retriever/elasticsearch/v8/move.go new file mode 100644 index 00000000..547b87bb --- /dev/null +++ b/internal/application/repository/retriever/elasticsearch/v8/move.go @@ -0,0 +1,48 @@ +// Package v8 implements retrieval against its configured vector backend. +package v8 + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/elastic/go-elasticsearch/v8/typedapi/types" +) + +func (e *elasticsearchRepository) MoveKnowledgeIndices( + ctx context.Context, + sourceKB, targetKB, knowledgeID string, + _ []string, + _ int, + _ string, +) error { + query := &types.Query{Bool: &types.BoolQuery{Filter: []types.Query{ + { + Terms: &types.TermsQuery{ + TermsQuery: map[string]types.TermsQueryField{e.idField("knowledge_base_id"): []string{sourceKB}}, + }, + }, + { + Terms: &types.TermsQuery{ + TermsQuery: map[string]types.TermsQueryField{e.idField("knowledge_id"): []string{knowledgeID}}, + }, + }, + }}} + source := "ctx._source.knowledge_base_id = params.target; ctx._source.tag_id = ''" + target, err := json.Marshal(targetKB) + if err != nil { + return err + } + script := &types.Script{Source: &source, Params: map[string]json.RawMessage{"target": target}} + result, err := e.client.UpdateByQuery(e.index).Query(query).Script(script).Refresh(true).Do(ctx) + if err != nil { + return err + } + if result == nil || result.Total == nil || result.Updated == nil || + *result.Total < 0 || *result.Total != *result.Updated || (result.TimedOut != nil && *result.TimedOut) || + (result.VersionConflicts != nil && *result.VersionConflicts != 0) || + len(result.Failures) != 0 { + return fmt.Errorf("move indices was incomplete") + } + return nil +} diff --git a/internal/application/repository/retriever/elasticsearch/v8/move_test.go b/internal/application/repository/retriever/elasticsearch/v8/move_test.go new file mode 100644 index 00000000..c2763006 --- /dev/null +++ b/internal/application/repository/retriever/elasticsearch/v8/move_test.go @@ -0,0 +1,59 @@ +package v8 + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/elastic/go-elasticsearch/v8" + "github.com/stretchr/testify/require" +) + +func TestMoveIndicesRequiresCompleteCounts(t *testing.T) { + for _, tt := range []struct { + name, body string + fails bool + }{ + {"complete", `{"total":100,"updated":100}`, false}, + {"empty retry", `{"total":0,"updated":0}`, false}, + {"partial", `{"total":100,"updated":40}`, true}, + {"missing total", `{"updated":40}`, true}, + {"missing updated", `{"total":40}`, true}, + {"missing counts", `{}`, true}, + {"negative", `{"total":-1,"updated":-1}`, true}, + {"conflict", `{"total":1,"updated":1,"version_conflicts":1}`, true}, + {"timeout", `{"total":1,"updated":1,"timed_out":true}`, true}, + } { + t.Run(tt.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Elastic-Product", "Elasticsearch") + if r.URL.Path == "/" { + _, _ = w.Write( + []byte( + `{"version":{"number":"7.17.0","build_flavor":"default"},"tagline":"You Know, for Search"}`, + ), + ) + return + } + if r.URL.Path != "/vectors/_update_by_query" { + t.Errorf("unexpected path: %s", r.URL.Path) + } + _, _ = w.Write([]byte(tt.body)) + })) + defer server.Close() + client, err := elasticsearch.NewTypedClient( + elasticsearch.Config{Addresses: []string{server.URL}, DisableRetry: true}, + ) + require.NoError(t, err) + repo := &elasticsearchRepository{client: client, index: "vectors"} + err = repo.MoveKnowledgeIndices(context.Background(), "source", "target", "doc", nil, 3, "file") + if tt.fails { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/internal/application/repository/retriever/milvus/move.go b/internal/application/repository/retriever/milvus/move.go new file mode 100644 index 00000000..cd981b79 --- /dev/null +++ b/internal/application/repository/retriever/milvus/move.go @@ -0,0 +1,61 @@ +package milvus + +import ( + "context" + "fmt" +) + +func (m *milvusRepository) MoveKnowledgeIndices( + ctx context.Context, + sourceKB, targetKB, knowledgeID string, + _ []string, + dimension int, + _ string, +) error { + collection := m.getCollectionName(dimension) + filter := &universalFilterCondition{Operator: operatorAnd, Value: []*universalFilterCondition{ + {Field: fieldKnowledgeBaseID, Operator: operatorEqual, Value: sourceKB}, + {Field: fieldKnowledgeID, Operator: operatorEqual, Value: knowledgeID}, + }} + size, offset := 100, 0 + return drainMoveRows(ctx, targetKB, func(ctx context.Context) ([]*MilvusVectorEmbeddingWithScore, error) { + rows, _, err := m.searchByFilter(ctx, collection, filter, &size, &offset) + return rows, err + }, func(ctx context.Context, rows []*MilvusVectorEmbedding) error { + _, err := m.client.Upsert(ctx, createUpsert(collection, rows)) + return err + }) +} + +// Drain the source predicate without offset windows. A repeated ID means the +// backend has not made the acknowledged update visible; fail for retry instead +// of advancing the document checkpoint or silently omitting remaining vectors. +func drainMoveRows(ctx context.Context, targetKB string, + read func(context.Context) ([]*MilvusVectorEmbeddingWithScore, error), + write func(context.Context, []*MilvusVectorEmbedding) error, +) error { + seen := map[string]bool{} + for { + rows, err := read(ctx) + if err != nil { + return err + } + if len(rows) == 0 { + return nil + } + batch := make([]*MilvusVectorEmbedding, 0, len(rows)) + for _, row := range rows { + if row == nil || row.ID == "" || seen[row.ID] { + return fmt.Errorf("invalid or repeated move index") + } + seen[row.ID] = true + copyOfRow := row.MilvusVectorEmbedding + copyOfRow.KnowledgeBaseID = targetKB + copyOfRow.TagID = "" + batch = append(batch, ©OfRow) + } + if err := write(ctx, batch); err != nil { + return err + } + } +} diff --git a/internal/application/repository/retriever/milvus/move_test.go b/internal/application/repository/retriever/milvus/move_test.go new file mode 100644 index 00000000..93b3afb1 --- /dev/null +++ b/internal/application/repository/retriever/milvus/move_test.go @@ -0,0 +1,82 @@ +package milvus + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMoveRowsDrainsPagesAndRetriesPartialUpsert(t *testing.T) { + remaining := map[string]*MilvusVectorEmbeddingWithScore{} + moved := map[string]*MilvusVectorEmbedding{} + for i := range 205 { + id := fmt.Sprint(i) + remaining[id] = &MilvusVectorEmbeddingWithScore{MilvusVectorEmbedding: MilvusVectorEmbedding{ + ID: id, + KnowledgeID: "doc", + KnowledgeBaseID: "source", + TagID: "old-tag", + ChunkID: "chunk-" + id, + Content: "text", + Embedding: []float32{1, 2, 3}, + IsEnabled: i%2 == 0, + }} + } + reads, writes := 0, 0 + failure := errors.New("partial upsert") + read := func(context.Context) ([]*MilvusVectorEmbeddingWithScore, error) { + reads++ + var rows []*MilvusVectorEmbeddingWithScore + for _, row := range remaining { + rows = append(rows, row) + if len(rows) == 100 { + break + } + } + return rows, nil + } + write := func(_ context.Context, rows []*MilvusVectorEmbedding) error { + writes++ + for i, row := range rows { + original := remaining[row.ID] + require.NotNil(t, original) + require.Equal(t, "source", original.KnowledgeBaseID, "read rows must not be mutated before acknowledgement") + require.Equal(t, "target", row.KnowledgeBaseID) + require.Empty(t, row.TagID) + expected := original.MilvusVectorEmbedding + expected.KnowledgeBaseID = "target" + expected.TagID = "" + require.Equal(t, expected, *row) + moved[row.ID] = row + delete(remaining, row.ID) + if writes == 2 && i == 16 { + return failure + } + } + return nil + } + require.ErrorIs(t, drainMoveRows(context.Background(), "target", read, write), failure) + require.Len(t, remaining, 88) + require.NoError(t, drainMoveRows(context.Background(), "target", read, write)) + require.Empty(t, remaining) + require.Len(t, moved, 205) + require.Equal(t, 4, reads, "retry must verify an empty source predicate") +} + +func TestMoveRowsRejectsUnacknowledgedProgress(t *testing.T) { + row := &MilvusVectorEmbeddingWithScore{ + MilvusVectorEmbedding: MilvusVectorEmbedding{ID: "id", KnowledgeBaseID: "source"}, + } + err := drainMoveRows( + context.Background(), + "target", + func(context.Context) ([]*MilvusVectorEmbeddingWithScore, error) { + return []*MilvusVectorEmbeddingWithScore{row}, nil + }, + func(context.Context, []*MilvusVectorEmbedding) error { return nil }, + ) + require.ErrorContains(t, err, "repeated") +} diff --git a/internal/application/repository/retriever/opensearch/byquery.go b/internal/application/repository/retriever/opensearch/byquery.go index 912fcaac..1c0f06e6 100644 --- a/internal/application/repository/retriever/opensearch/byquery.go +++ b/internal/application/repository/retriever/opensearch/byquery.go @@ -17,8 +17,15 @@ import ( // bounded id + type. A non-zero version_conflicts count with no hard failures // is logged as a warning but not treated as an error. func inspectByQueryResponse(body io.Reader) error { + return inspectByQueryResult(body, false) +} + +func inspectByQueryResult(body io.Reader, requireComplete bool) error { var r struct { - VersionConflicts int `json:"version_conflicts"` + Total *int64 `json:"total"` + Updated *int64 `json:"updated"` + TimedOut bool `json:"timed_out"` + VersionConflicts int `json:"version_conflicts"` Failures []struct { ID string `json:"id"` Cause struct { @@ -30,6 +37,13 @@ func inspectByQueryResponse(body io.Reader) error { if err := json.NewDecoder(body).Decode(&r); err != nil { return fmt.Errorf("opensearch: parse by-query response: %w", ErrTransport) } + if requireComplete && (r.TimedOut || r.VersionConflicts != 0) { + return fmt.Errorf("opensearch: incomplete move (timed_out=%t, version_conflicts=%d): %w", + r.TimedOut, r.VersionConflicts, ErrTransport) + } + if requireComplete && (r.Total == nil || r.Updated == nil || *r.Total < 0 || *r.Updated != *r.Total) { + return fmt.Errorf("opensearch: incomplete move document counts: %w", ErrTransport) + } log := logger.GetLogger(context.Background()) if len(r.Failures) == 0 { if r.VersionConflicts > 0 { diff --git a/internal/application/repository/retriever/opensearch/move.go b/internal/application/repository/retriever/opensearch/move.go new file mode 100644 index 00000000..5e12238a --- /dev/null +++ b/internal/application/repository/retriever/opensearch/move.go @@ -0,0 +1,49 @@ +package opensearch + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + + osapi "github.com/opensearch-project/opensearch-go/v4/opensearchapi" +) + +// MoveKnowledgeIndices changes every vector for the source document, including +// historical vectors whose chunks no longer exist, while preserving vector IDs. +func (r *Repository) MoveKnowledgeIndices( + ctx context.Context, + sourceKB, targetKB, knowledgeID string, + _ []string, + _ int, + _ string, +) error { + body, err := json.Marshal(map[string]any{ + "query": map[string]any{"bool": map[string]any{"filter": []any{ + map[string]any{"term": map[string]any{"knowledge_base_id": sourceKB}}, + map[string]any{"term": map[string]any{"knowledge_id": knowledgeID}}, + }}}, + "script": map[string]any{ + "lang": "painless", + "source": "ctx._source.knowledge_base_id = params.target; ctx._source.tag_id = '';", + "params": map[string]any{"target": targetKB}, + }, + }) + if err != nil { + return fmt.Errorf("opensearch: marshal move: %w", err) + } + refresh := true + resp, err := r.client.UpdateByQuery(ctx, osapi.UpdateByQueryReq{ + Indices: []string{r.baseIndex + "_*"}, Body: bytes.NewReader(body), + Params: osapi.UpdateByQueryParams{Refresh: &refresh}, + }) + if err != nil { + return wrapTransport(err) + } + if resp == nil { + return fmt.Errorf("opensearch: missing move response: %w", ErrTransport) + } + defer drainAndClose(resp.Inspect().Response.Body) + return inspectByQueryResult(io.LimitReader(resp.Inspect().Response.Body, 16<<20), true) +} diff --git a/internal/application/repository/retriever/opensearch/move_test.go b/internal/application/repository/retriever/opensearch/move_test.go new file mode 100644 index 00000000..1570e9ad --- /dev/null +++ b/internal/application/repository/retriever/opensearch/move_test.go @@ -0,0 +1,61 @@ +package opensearch + +import ( + "context" + "encoding/json" + "net/http" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMoveIndicesRequiresCompleteResult(t *testing.T) { + for _, tt := range []struct { + name, response string + fails bool + }{ + {"complete", `{"total":2,"updated":2,"version_conflicts":0,"failures":[]}`, false}, + {"partial", `{"total":100,"updated":40,"failures":[]}`, true}, + {"empty", `{"total":0,"updated":0,"failures":[]}`, false}, + {"missing counts", `{"failures":[]}`, true}, + {"missing total", `{"updated":2,"failures":[]}`, true}, + {"missing updated", `{"total":2,"failures":[]}`, true}, + {"negative", `{"total":-1,"updated":-1,"failures":[]}`, true}, + {"conflict", `{"updated":1,"version_conflicts":1,"failures":[]}`, true}, + {"timeout", `{"timed_out":true,"version_conflicts":0,"failures":[]}`, true}, + {"failure", `{"failures":[{"id":"doc","cause":{"type":"unavailable"}}]}`, true}, + } { + t.Run(tt.name, func(t *testing.T) { + repo, server := newTestRepo(t, func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(tt.response)) + }) + defer server.Close() + err := repo.MoveKnowledgeIndices( + context.Background(), "source", "target", "doc", []string{"chunk"}, 1, "file", + ) + if tt.fails { + require.ErrorIs(t, err, ErrTransport) + } else { + require.NoError(t, err) + } + }) + } +} + +func TestMoveIndicesIncludesOrphansEvenWithoutDatabaseChunks(t *testing.T) { + var body map[string]any + repo, server := newTestRepo(t, func(w http.ResponseWriter, r *http.Request) { + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Errorf("decode move request: %v", err) + } + _, _ = w.Write([]byte(`{"total":1,"updated":1,"failures":[]}`)) + }) + defer server.Close() + require.NoError(t, repo.MoveKnowledgeIndices(context.Background(), "source", "target", "doc", nil, 1, "file")) + want := `{"bool":{"filter":[{"term":{"knowledge_base_id":"source"}},{"term":{"knowledge_id":"doc"}}]}}` + query, err := json.Marshal(body["query"]) + require.NoError(t, err) + require.JSONEq(t, want, string(query), "all vectors for this exact source document must be selected") + script := body["script"].(map[string]any) + require.Equal(t, map[string]any{"target": "target"}, script["params"]) +} diff --git a/internal/application/repository/retriever/postgres/move.go b/internal/application/repository/retriever/postgres/move.go new file mode 100644 index 00000000..dd480091 --- /dev/null +++ b/internal/application/repository/retriever/postgres/move.go @@ -0,0 +1,16 @@ +// Package postgres implements retrieval against its configured vector backend. +package postgres + +import "context" + +func (g *pgRepository) MoveKnowledgeIndices( + ctx context.Context, + sourceKB, targetKB, knowledgeID string, + _ []string, + _ int, + _ string, +) error { + return g.db.WithContext(ctx).Model(&pgVector{}). + Where("knowledge_base_id = ? AND knowledge_id = ?", sourceKB, knowledgeID). + Updates(map[string]any{"knowledge_base_id": targetKB, "tag_id": ""}).Error +} diff --git a/internal/application/repository/retriever/qdrant/move.go b/internal/application/repository/retriever/qdrant/move.go new file mode 100644 index 00000000..731a5374 --- /dev/null +++ b/internal/application/repository/retriever/qdrant/move.go @@ -0,0 +1,26 @@ +// Package qdrant implements vector retrieval and metadata updates in Qdrant. +package qdrant + +import ( + "context" + + "github.com/qdrant/go-client/qdrant" +) + +func (q *qdrantRepository) MoveKnowledgeIndices( + ctx context.Context, + sourceKB, targetKB, knowledgeID string, + _ []string, + dimension int, + _ string, +) error { + wait := true + _, err := q.client.SetPayload(ctx, &qdrant.SetPayloadPoints{ + CollectionName: q.getCollectionName(dimension), Wait: &wait, + Payload: newQdrantValueMap(map[string]any{fieldKnowledgeBaseID: targetKB, fieldTagID: ""}), + PointsSelector: qdrant.NewPointsSelectorFilter(&qdrant.Filter{Must: []*qdrant.Condition{ + qdrant.NewMatch(fieldKnowledgeBaseID, sourceKB), qdrant.NewMatch(fieldKnowledgeID, knowledgeID), + }}), + }) + return err +} diff --git a/internal/application/repository/retriever/sqlite/move.go b/internal/application/repository/retriever/sqlite/move.go new file mode 100644 index 00000000..a91c483c --- /dev/null +++ b/internal/application/repository/retriever/sqlite/move.go @@ -0,0 +1,17 @@ +// Package sqlite implements local retrieval with SQLite FTS and vectors. +package sqlite + +import "context" + +func (r *sqliteRepository) MoveKnowledgeIndices( + ctx context.Context, + sourceKB, targetKB, knowledgeID string, + _ []string, + _ int, + _ string, +) error { + // FTS and vector rows keep their IDs. Queries join them to this metadata row. + return r.db.WithContext(ctx).Model(&sqliteEmbedding{}). + Where("knowledge_base_id = ? AND knowledge_id = ?", sourceKB, knowledgeID). + Updates(map[string]any{"knowledge_base_id": targetKB, "tag_id": ""}).Error +} diff --git a/internal/application/repository/retriever/sqlite/move_test.go b/internal/application/repository/retriever/sqlite/move_test.go new file mode 100644 index 00000000..45e72f8d --- /dev/null +++ b/internal/application/repository/retriever/sqlite/move_test.go @@ -0,0 +1,72 @@ +package sqlite + +import ( + "context" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/stretchr/testify/require" +) + +func TestMoveIndicesPreservesVectorsAndExactScopeAcrossRetry(t *testing.T) { + repo := newSQLiteRetrieverTestRepository(t) + ctx := context.Background() + for _, row := range []struct{ chunk, kb, doc string }{ + { + "text", + "source", + "doc", + }, + { + "image", + "source", + "doc", + }, + { + "other", + "source", + "other-doc", + }, + { + "foreign", + "third", + "doc", + }, + } { + saveSQLiteTestVector(t, repo, sqliteTestIndex(row.chunk, row.kb, row.doc, "old-tag", true), []float32{1, 0}) + } + var before []sqliteEmbedding + require.NoError(t, repo.db.Order("id").Find(&before).Error) + for range 2 { + require.NoError( + t, + repo.MoveKnowledgeIndices(ctx, "source", "target", "doc", []string{"text", "image"}, 2, "document"), + ) + } + var after []sqliteEmbedding + require.NoError(t, repo.db.Order("id").Find(&after).Error) + require.Len(t, after, len(before)) + for i, row := range after { + require.Equal(t, before[i].ID, row.ID) + require.Equal(t, before[i].SourceID, row.SourceID) + if row.KnowledgeID == "doc" && before[i].KnowledgeBaseID == "source" { + require.Equal(t, "target", row.KnowledgeBaseID) + require.Empty(t, row.TagID) + } else { + require.Equal(t, before[i].KnowledgeBaseID, row.KnowledgeBaseID) + require.Equal(t, "old-tag", row.TagID) + } + } + results, err := repo.vectorRetrieve( + ctx, + types.RetrieveParams{ + Embedding: []float32{1, 0}, + TopK: 10, + KnowledgeBaseIDs: []string{"target"}, + RetrieverType: types.VectorRetrieverType, + }, + ) + require.NoError(t, err) + require.Len(t, results, 1) + require.Len(t, results[0].Results, 2, "target vectors remain searchable after an identity-preserving move") +} diff --git a/internal/application/repository/retriever/tencentvectordb/move.go b/internal/application/repository/retriever/tencentvectordb/move.go new file mode 100644 index 00000000..f05579d7 --- /dev/null +++ b/internal/application/repository/retriever/tencentvectordb/move.go @@ -0,0 +1,33 @@ +// Package tencentvectordb implements retrieval against its configured vector backend. +package tencentvectordb + +import ( + "context" + + "github.com/tencent/vectordatabase-sdk-go/tcvectordb" +) + +func (r *repository) MoveKnowledgeIndices( + ctx context.Context, + sourceKB, targetKB, knowledgeID string, + _ []string, + dimension int, + _ string, +) error { + filter := tcvectordb.NewFilter( + tcvectordb.In( + fieldKnowledgeBaseID, + []string{sourceKB}, + ) + " and " + tcvectordb.In( + fieldKnowledgeID, + []string{knowledgeID}, + ), + ) + _, err := r.client.Database(r.databaseName). + Collection(r.collectionName(dimension)). + Update(ctx, tcvectordb.UpdateDocumentParams{ + QueryFilter: filter, + UpdateFields: map[string]tcvectordb.Field{fieldKnowledgeBaseID: {Val: targetKB}, fieldTagID: {Val: ""}}, + }) + return err +} diff --git a/internal/application/repository/retriever/weaviate/move.go b/internal/application/repository/retriever/weaviate/move.go new file mode 100644 index 00000000..0410c409 --- /dev/null +++ b/internal/application/repository/retriever/weaviate/move.go @@ -0,0 +1,77 @@ +// Package weaviate implements retrieval and metadata updates in Weaviate. +package weaviate + +import ( + "context" + "fmt" + + "github.com/weaviate/weaviate-go-client/v5/weaviate/filters" + "github.com/weaviate/weaviate-go-client/v5/weaviate/graphql" +) + +func (w *weaviateRepository) MoveKnowledgeIndices( + ctx context.Context, + sourceKB, targetKB, knowledgeID string, + _ []string, + dimension int, + _ string, +) error { + collection := w.getCollectionName(dimension) + where := filters.Where().WithOperator(filters.And).WithOperands([]*filters.WhereBuilder{ + filters.Where().WithPath([]string{fieldKnowledgeBaseID}).WithOperator(filters.Equal).WithValueString(sourceKB), + filters.Where().WithPath([]string{fieldKnowledgeID}).WithOperator(filters.Equal).WithValueString(knowledgeID), + }) + seen := make(map[string]bool) + for { + result, err := w.client.GraphQL().Get().WithClassName(collection).WithWhere(where).WithLimit(100). + WithFields(graphql.Field{Name: "_additional", Fields: []graphql.Field{{Name: "id"}}}). + Do(ctx) + if err != nil { + return err + } + if result == nil || len(result.Errors) != 0 { + return fmt.Errorf("failed to list move indices") + } + get, ok := result.Data["Get"].(map[string]interface{}) + if !ok { + return fmt.Errorf("invalid move index response") + } + rows, ok := get[collection].([]interface{}) + if !ok { + return fmt.Errorf("invalid move index collection") + } + if len(rows) == 0 { + return nil + } + var ids []string + for _, row := range rows { + data, ok := row.(map[string]interface{}) + if !ok { + return fmt.Errorf("invalid move index row") + } + additional, ok := data["_additional"].(map[string]interface{}) + if !ok { + return fmt.Errorf("missing move index ID") + } + id, ok := additional["id"].(string) + if !ok || id == "" { + return fmt.Errorf("invalid move index ID") + } + if seen[id] { + return fmt.Errorf("move indices made no progress for %s", id) + } + seen[id] = true + ids = append(ids, id) + } + // Drain the first filtered page; after+where is unsupported by Weaviate. + // Successful updates remove these IDs from the predicate. Only an empty + // follow-up query proves completion, including after partial-page updates. + for _, id := range ids { + if err := w.client.Data().Updater().WithClassName(collection).WithID(id).WithMerge(). + WithProperties(map[string]interface{}{fieldKnowledgeBaseID: targetKB, fieldTagID: ""}). + Do(ctx); err != nil { + return err + } + } + } +} diff --git a/internal/application/repository/retriever/weaviate/move_test.go b/internal/application/repository/retriever/weaviate/move_test.go new file mode 100644 index 00000000..eabc2ad3 --- /dev/null +++ b/internal/application/repository/retriever/weaviate/move_test.go @@ -0,0 +1,130 @@ +package weaviate + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "path" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + sdk "github.com/weaviate/weaviate-go-client/v5/weaviate" +) + +func TestMoveKnowledgeIndicesDrainsBeyondOffsetLimitAndRetries(t *testing.T) { + const count = 10005 + var mu sync.Mutex + remaining := make(map[string]bool, count) + for i := range count { + remaining[fmt.Sprintf("00000000-0000-0000-0000-%012d", i)] = true + } + failed := false + patches, queries := 0, 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + w.Header().Set("Content-Type", "application/json") + switch { + case r.URL.Path == "/v1/graphql": + queries++ + var body struct { + Query string `json:"query"` + } + if !assert.NoError(t, json.NewDecoder(r.Body).Decode(&body)) { + w.WriteHeader(400) + return + } + assert.Contains(t, body.Query, "knowledge_base_id") + assert.Contains(t, body.Query, "knowledge_id") + assert.Contains(t, body.Query, "source") + assert.Contains(t, body.Query, "doc") + if strings.Contains(body.Query, "offset") || strings.Contains(body.Query, "after") { + _ = json.NewEncoder(w). + Encode(map[string]any{"errors": []any{map[string]any{"message": "pagination unsupported"}}}) + return + } + rows := make([]any, 0, 100) + for id := range remaining { + rows = append(rows, map[string]any{"_additional": map[string]any{"id": id}}) + if len(rows) == 100 { + break + } + } + _ = json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"Get": map[string]any{"Move_3": rows}}}) + case r.Method == http.MethodPatch: + id := path.Base(r.URL.Path) + assert.True(t, remaining[id], "only source-filtered IDs should be moved") + var body struct { + Properties map[string]any `json:"properties"` + Vector []float32 `json:"vector"` + } + if !assert.NoError(t, json.NewDecoder(r.Body).Decode(&body)) { + w.WriteHeader(400) + return + } + assert.Equal(t, map[string]any{fieldKnowledgeBaseID: "target", fieldTagID: ""}, body.Properties) + assert.Empty(t, body.Vector, "metadata merge must preserve existing vectors") + if patches == 137 && !failed { + failed = true + w.WriteHeader(http.StatusServiceUnavailable) + return + } + patches++ + delete(remaining, id) + w.WriteHeader(http.StatusNoContent) + default: + // SDK probes the server version before object writes. + if r.URL.Path == "/v1/meta" { + _, _ = w.Write([]byte(`{"version":"1.30.0"}`)) + return + } + t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + client, err := sdk.NewClient(sdk.Config{Scheme: "http", Host: strings.TrimPrefix(server.URL, "http://")}) + require.NoError(t, err) + repo := &weaviateRepository{client: client, collectionBaseName: "Move"} + require.Error(t, repo.MoveKnowledgeIndices(context.Background(), "source", "target", "doc", nil, 3, "")) + mu.Lock() + left := len(remaining) + mu.Unlock() + require.Equal(t, count-137, left) + require.NoError(t, repo.MoveKnowledgeIndices(context.Background(), "source", "target", "doc", nil, 3, "")) + mu.Lock() + defer mu.Unlock() + require.Empty(t, remaining) + require.Equal(t, count, patches) + require.Greater(t, queries, 100) +} + +func TestMoveKnowledgeIndicesRejectsNoProgress(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/v1/graphql": + _, _ = w.Write( + []byte(`{"data":{"Get":{"Move_3":[{"_additional":{"id":"00000000-0000-0000-0000-000000000001"}}]}}}`), + ) + case "/v1/meta": + _, _ = w.Write([]byte(`{"version":"1.30.0"}`)) + default: + w.WriteHeader(http.StatusNoContent) + } + })) + defer server.Close() + client, err := sdk.NewClient(sdk.Config{Scheme: "http", Host: strings.TrimPrefix(server.URL, "http://")}) + require.NoError(t, err) + repo := &weaviateRepository{client: client, collectionBaseName: "Move"} + require.ErrorContains( + t, + repo.MoveKnowledgeIndices(context.Background(), "source", "target", "doc", nil, 3, ""), + "no progress", + ) +} diff --git a/internal/application/service/agent_share.go b/internal/application/service/agent_share.go index eec993d6..9e30ed29 100644 --- a/internal/application/service/agent_share.go +++ b/internal/application/service/agent_share.go @@ -512,27 +512,9 @@ func (s *agentShareService) TenantCanAccessKBViaSomeSharedAgent(ctx context.Cont return false, err } for _, info := range list { - if info.Agent == nil { - continue - } - agent := info.Agent - if agent.TenantID != kb.TenantID { - continue - } - mode := agent.Config.KBSelectionMode - if mode == "none" { - continue - } - if mode == "all" { + if types.SharedAgentIncludesKB(info.Agent, kb) { return true, nil } - if mode == "selected" { - for _, id := range agent.Config.KnowledgeBases { - if id == kb.ID { - return true, nil - } - } - } } return false, nil } diff --git a/internal/application/service/chunk.go b/internal/application/service/chunk.go index de8ac951..8727ca04 100644 --- a/internal/application/service/chunk.go +++ b/internal/application/service/chunk.go @@ -86,6 +86,9 @@ func (s *chunkService) GetRepository() interfaces.ChunkRepository { // Returns: // - error: Any error encountered during chunk creation func (s *chunkService) CreateChunks(ctx context.Context, chunks []*types.Chunk) error { + if err := s.validateChunkWrites(ctx, chunks, true); err != nil { + return err + } err := s.chunkRepository.CreateChunks(ctx, chunks) if err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ @@ -217,6 +220,9 @@ func (s *chunkService) ListPagedChunksByKnowledgeID(ctx context.Context, // // This method handles the actual update logic for a chunk, including updating the vector database representation func (s *chunkService) UpdateChunk(ctx context.Context, chunk *types.Chunk) error { + if err := s.validateChunkWrites(ctx, []*types.Chunk{chunk}, false); err != nil { + return err + } logger.Infof(ctx, "Updating chunk, ID: %s, knowledge ID: %s", chunk.ID, chunk.KnowledgeID) // Update the chunk in the repository @@ -235,6 +241,9 @@ func (s *chunkService) UpdateChunk(ctx context.Context, chunk *types.Chunk) erro // UpdateChunks updates chunks in batch func (s *chunkService) UpdateChunks(ctx context.Context, chunks []*types.Chunk) error { + if err := s.validateChunkWrites(ctx, chunks, false); err != nil { + return err + } if len(chunks) == 0 { return nil } @@ -262,6 +271,9 @@ func (s *chunkService) UpdateChunks(ctx context.Context, chunks []*types.Chunk) // Returns: // - error: Any error encountered during deletion func (s *chunkService) DeleteChunk(ctx context.Context, id string) error { + if _, err := s.writableChunk(ctx, id); err != nil { + return err + } tenantID := types.MustTenantIDFromContext(ctx) err := s.chunkRepository.DeleteChunk(ctx, tenantID, id) if err != nil { @@ -289,10 +301,15 @@ func (s *chunkService) DeleteChunks(ctx context.Context, ids []string) error { logger.Info(ctx, "Start deleting chunks in batch") logger.Infof(ctx, "Deleting %d chunks", len(ids)) + checkedIDs, err := s.writableChunkIDs(ctx, ids) + if err != nil { + return err + } + ids = checkedIDs tenantID := types.MustTenantIDFromContext(ctx) logger.Infof(ctx, "Tenant ID: %d", tenantID) - err := s.chunkRepository.DeleteChunks(ctx, tenantID, ids) + err = s.chunkRepository.DeleteChunks(ctx, tenantID, ids) if err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ "chunk_ids": ids, @@ -314,6 +331,9 @@ func (s *chunkService) DeleteChunks(ctx context.Context, ids []string) error { // Returns: // - error: Any error encountered during bulk deletion func (s *chunkService) DeleteChunksByKnowledgeID(ctx context.Context, knowledgeID string) error { + if _, err := loadKnowledgeWriteBatch(ctx, s.knowledgeRepo, s.kbRepository, []string{knowledgeID}); err != nil { + return err + } logger.Info(ctx, "Start deleting all chunks by knowledge ID") logger.Infof(ctx, "Knowledge ID: %s", knowledgeID) @@ -334,6 +354,9 @@ func (s *chunkService) DeleteChunksByKnowledgeID(ctx context.Context, knowledgeI } func (s *chunkService) DeleteByKnowledgeList(ctx context.Context, ids []string) error { + if _, err := loadKnowledgeWriteBatch(ctx, s.knowledgeRepo, s.kbRepository, ids); err != nil { + return err + } logger.Info(ctx, "Start deleting all chunks by knowledge IDs") logger.Infof(ctx, "Knowledge IDs: %v", ids) @@ -382,14 +405,16 @@ func (s *chunkService) ListChunkByParentID( func (s *chunkService) UpdateDocumentChunk( ctx context.Context, chunkID string, content *string, isEnabled *bool, expectedRevision *int, ) (*types.Chunk, error) { - tenantID := types.MustTenantIDFromContext(ctx) - chunk, err := s.chunkRepository.GetChunkByID(ctx, tenantID, chunkID) + chunk, err := s.writableChunk(ctx, chunkID) if err != nil { return nil, err } if chunk.ChunkType != types.ChunkTypeText { return nil, fmt.Errorf("only text chunks can be edited") } + if err := s.validateDocumentChunkRelations(ctx, chunk); err != nil { + return nil, err + } if expectedRevision != nil && *expectedRevision != chunk.ContentRevision { return nil, ErrChunkRevisionConflict } @@ -481,7 +506,7 @@ func (s *chunkService) UpdateDocumentChunk( } } if bodyChanged || newEnabled != revision.IsEnabled { - knowledge, getErr := s.knowledgeRepo.GetKnowledgeByID(ctx, tenantID, chunk.KnowledgeID) + knowledge, getErr := s.knowledgeRepo.GetKnowledgeByID(ctx, chunk.TenantID, chunk.KnowledgeID) if getErr == nil { if err := enqueueSummaryRefresh( ctx, s.knowledgeRepo, s.task, s.kbRepository, s.spanTracker, knowledge, @@ -700,7 +725,7 @@ func (s *chunkService) UpsertGeneratedQuestion( if question == "" { return nil, fmt.Errorf("question cannot be empty") } - chunk, err := s.chunkRepository.GetChunkByID(ctx, types.MustTenantIDFromContext(ctx), chunkID) + chunk, err := s.writableChunk(ctx, chunkID) if err != nil { return nil, err } @@ -756,7 +781,7 @@ func (s *chunkService) DeleteGeneratedQuestion(ctx context.Context, chunkID stri tenantID := types.MustTenantIDFromContext(ctx) // 1. Get the chunk - chunk, err := s.chunkRepository.GetChunkByID(ctx, tenantID, chunkID) + chunk, err := s.writableChunk(ctx, chunkID) if err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ "chunk_id": chunkID, diff --git a/internal/application/service/chunk_edit_parent_test.go b/internal/application/service/chunk_edit_parent_test.go index 846dfdc2..1d06f7d6 100644 --- a/internal/application/service/chunk_edit_parent_test.go +++ b/internal/application/service/chunk_edit_parent_test.go @@ -3,11 +3,11 @@ package service import ( "context" "encoding/json" - "errors" "strings" "testing" "time" + "github.com/Tencent/WeKnora/internal/application/access" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" ) @@ -49,7 +49,7 @@ type editableChunkKBRepo struct { } func (editableChunkKBRepo) GetKnowledgeBaseByID(context.Context, string) (*types.KnowledgeBase, error) { - return &types.KnowledgeBase{}, nil + return &types.KnowledgeBase{ID: "kb", TenantID: 1}, nil } type editableChunkKnowledgeRepo struct { @@ -57,7 +57,7 @@ type editableChunkKnowledgeRepo struct { } func (editableChunkKnowledgeRepo) GetKnowledgeByID(context.Context, uint64, string) (*types.Knowledge, error) { - return nil, errors.New("summary refresh not configured in unit test") + return &types.Knowledge{ID: "knowledge", TenantID: 1, KnowledgeBaseID: "kb"}, nil } type imageSyncChunkRepo struct { @@ -165,6 +165,17 @@ func TestUpdateDocumentChunkPreservesGeneratedQuestionsAcrossRevision(t *testing kbRepository: editableChunkKBRepo{}, } ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(1)) + ctx = (&access.KBAccess{ + KnowledgeBase: &types.KnowledgeBase{ + ID: "kb", + TenantID: 1, + }, + Caller: types.CallerFromContext(ctx), + EffectiveTenantID: 1, + Permission: types.OrgRoleEditor, + }).Context( + ctx, + ) newContent := "new body" updated, err := service.UpdateDocumentChunk(ctx, "chunk", &newContent, nil, nil) diff --git a/internal/application/service/chunk_write.go b/internal/application/service/chunk_write.go new file mode 100644 index 00000000..5006e8d0 --- /dev/null +++ b/internal/application/service/chunk_write.go @@ -0,0 +1,143 @@ +package service + +import ( + "context" + + apperrors "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/types" +) + +func (s *chunkService) writableChunk(ctx context.Context, id string) (*types.Chunk, error) { + tenant, err := writeExecutionTenant(ctx) + if err != nil { + return nil, err + } + chunk, err := s.chunkRepository.GetChunkByID(ctx, tenant, id) + if err != nil { + return nil, err + } + if chunk == nil || chunk.ID != id || chunk.TenantID != tenant { + return nil, apperrors.NewNotFoundError("chunk not found") + } + knowledge, _, err := loadKnowledgeWrite(ctx, s.knowledgeRepo, s.kbRepository, chunk.KnowledgeID) + if err != nil { + return nil, err + } + if chunk.KnowledgeBaseID != knowledge.KnowledgeBaseID { + return nil, apperrors.NewForbiddenError("chunk does not belong to its knowledge base") + } + copyOfChunk := *chunk + return ©OfChunk, nil +} + +func sameChunkDocument(a, b *types.Chunk) bool { + return a != nil && b != nil && a.TenantID == b.TenantID && + a.KnowledgeBaseID == b.KnowledgeBaseID && a.KnowledgeID == b.KnowledgeID +} + +// Batch validation is completed before calling the repository. Supplied full +// rows cannot reparent an existing chunk to a different authorized document. +func (s *chunkService) validateChunkWrites(ctx context.Context, chunks []*types.Chunk, create bool) error { + ids := make([]string, 0, len(chunks)) + for _, chunk := range chunks { + if chunk == nil || chunk.ID == "" { + return apperrors.NewBadRequestError("chunk ID cannot be empty") + } + ids = append(ids, chunk.KnowledgeID) + } + knowledge, err := loadKnowledgeWriteBatch(ctx, s.knowledgeRepo, s.kbRepository, ids) + if err != nil { + return err + } + parents := make(map[string]*types.Knowledge, len(knowledge)) + for _, row := range knowledge { + parents[row.ID] = row + } + storedByID := make(map[string]*types.Chunk) + if !create && len(chunks) > 0 { + chunkIDs := make([]string, 0, len(chunks)) + for _, chunk := range chunks { + chunkIDs = append(chunkIDs, chunk.ID) + } + stored, err := s.chunkRepository.ListChunksByID(ctx, knowledge[0].TenantID, chunkIDs) + if err != nil { + return err + } + for _, chunk := range stored { + if chunk != nil { + storedByID[chunk.ID] = chunk + } + } + } + for _, chunk := range chunks { + parent := parents[chunk.KnowledgeID] + if parent == nil || parent.TenantID != chunk.TenantID || parent.KnowledgeBaseID != chunk.KnowledgeBaseID { + return apperrors.NewForbiddenError("chunk does not belong to its knowledge document") + } + if !create { + stored := storedByID[chunk.ID] + if stored == nil || stored.ID != chunk.ID || !sameChunkDocument(stored, chunk) { + return apperrors.NewForbiddenError("chunk ownership cannot be changed") + } + } + } + return nil +} + +func (s *chunkService) writableChunkIDs(ctx context.Context, ids []string) ([]string, error) { + ids, err := writeResourceIDs(ids) + if err != nil || len(ids) == 0 { + return ids, err + } + tenant, err := writeExecutionTenant(ctx) + if err != nil { + return nil, err + } + chunks, err := s.chunkRepository.ListChunksByID(ctx, tenant, ids) + if err != nil { + return nil, err + } + byID := make(map[string]*types.Chunk, len(chunks)) + for _, chunk := range chunks { + if chunk != nil { + byID[chunk.ID] = chunk + } + } + ordered := make([]*types.Chunk, 0, len(ids)) + for _, id := range ids { + if byID[id] == nil { + return nil, apperrors.NewNotFoundError("chunk not found") + } + ordered = append(ordered, byID[id]) + } + // Rows were loaded from storage already; only their parent grants remain. + return ids, s.validateChunkWrites(ctx, ordered, true) +} + +// Editing a text chunk may also rewrite its parent and image children. Validate +// those persisted relationships before saving the first revision. +func (s *chunkService) validateDocumentChunkRelations(ctx context.Context, chunk *types.Chunk) error { + parents := []*types.Chunk{chunk} + if chunk.ParentChunkID != "" { + parent, err := s.chunkRepository.GetChunkByID(ctx, chunk.TenantID, chunk.ParentChunkID) + if err != nil { + return err + } + if parent == nil || parent.ID != chunk.ParentChunkID || !sameChunkDocument(chunk, parent) { + return apperrors.NewForbiddenError("parent chunk does not belong to its document") + } + parents = append(parents, parent) + } + for _, parent := range parents { + children, err := s.chunkRepository.ListChunkByParentID(ctx, chunk.TenantID, parent.ID) + if err != nil { + return err + } + for _, child := range children { + if !sameChunkDocument(chunk, child) || child.ParentChunkID != parent.ID { + return apperrors.NewForbiddenError("child chunk does not belong to its document") + } + } + } + return nil +} diff --git a/internal/application/service/custom_agent.go b/internal/application/service/custom_agent.go index 02be0a10..714b91ab 100644 --- a/internal/application/service/custom_agent.go +++ b/internal/application/service/custom_agent.go @@ -8,6 +8,7 @@ import ( "time" "github.com/Tencent/WeKnora/internal/agent/tools" + "github.com/Tencent/WeKnora/internal/application/access" "github.com/Tencent/WeKnora/internal/application/repository" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" @@ -579,7 +580,7 @@ func (s *customAgentService) getSuggestedQuestions( resolvedTags := resolvedSuggestionTagScopes{} if len(scopeTagIDs) > 0 { var err error - resolvedTags, err = s.resolveSuggestionTagScopes(ctx, tenantID, tagScopes) + resolvedTags, err = s.resolveSuggestionTagScopes(ctx, tagScopes) if err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ "agent_id": agentID, @@ -673,7 +674,7 @@ func (s *customAgentService) getSuggestedQuestions( // querying a KB shared from tenant B would hit `tenant_id = A` and get zero // rows back — the symptom is "suggested questions never appear for shared KBs". scopeKBIDs := mergeUniqueStrings(queryKBIDs, resolvedTags.KnowledgeBaseIDs) - kbGroups := s.groupKBIDsByEffectiveTenant(ctx, tenantID, scopeKBIDs) + kbGroups := s.groupKBIDsByEffectiveTenant(ctx, scopeKBIDs) // Always keep the caller's tenant in the iteration so knowledge_ids-only // requests (no kbIDs) still execute one query under the caller's tenant. if len(scopeKBIDs) == 0 { @@ -834,7 +835,6 @@ type resolvedSuggestionTagScopes struct { // source tenant that owns the tag and chunk rows. func (s *customAgentService) resolveSuggestionTagScopes( ctx context.Context, - callerTenantID uint64, tagScopes []types.TagScope, ) (resolvedSuggestionTagScopes, error) { result := resolvedSuggestionTagScopes{TagIDsByTenant: make(map[uint64][]string)} @@ -862,7 +862,7 @@ func (s *customAgentService) resolveSuggestionTagScopes( for kbID := range byKB { kbIDs = append(kbIDs, kbID) } - kbGroups := s.groupKBIDsByEffectiveTenant(ctx, callerTenantID, kbIDs) + kbGroups := s.groupKBIDsByEffectiveTenant(ctx, kbIDs) for tenantID, groupKBIDs := range kbGroups { for _, kbID := range groupKBIDs { requested := mergeUniqueStrings(nil, byKB[kbID]) @@ -1083,9 +1083,9 @@ func wikiSuggestionFromPage(page *types.WikiPage, locale string) string { // kbIDs is empty. func (s *customAgentService) groupKBIDsByEffectiveTenant( ctx context.Context, - callerTenantID uint64, kbIDs []string, ) map[uint64][]string { + callerTenantID := types.CallerFromContext(ctx).TenantID out := make(map[uint64][]string) if len(kbIDs) == 0 { return out @@ -1106,20 +1106,13 @@ func (s *customAgentService) groupKBIDsByEffectiveTenant( kbByID[kb.ID] = kb } } - callerRole := types.TenantRoleFromContext(ctx) + permissions := access.NewKBPermissions(ctx, s.kbShareService) for _, kbID := range kbIDs { kb := kbByID[kbID] if kb == nil { continue } - if kb.TenantID == callerTenantID { - out[callerTenantID] = append(out[callerTenantID], kbID) - continue - } - if s.kbShareService == nil { - continue - } - ok, err := s.kbShareService.HasTenantKBPermission(ctx, kbID, callerTenantID, callerRole, types.OrgRoleViewer) + ok, err := permissions.Check(kbID, kb.TenantID, types.OrgRoleViewer) if err != nil || !ok { continue } diff --git a/internal/application/service/custom_agent_suggestion_scope_test.go b/internal/application/service/custom_agent_suggestion_scope_test.go index a87a51d8..215325ee 100644 --- a/internal/application/service/custom_agent_suggestion_scope_test.go +++ b/internal/application/service/custom_agent_suggestion_scope_test.go @@ -66,14 +66,13 @@ type suggestionKBShareService struct { allowed map[string]bool } -func (s *suggestionKBShareService) HasTenantKBPermission( +func (s *suggestionKBShareService) CheckTenantKBPermission( _ context.Context, kbID string, _ uint64, _ types.TenantRole, - _ types.OrgMemberRole, -) (bool, error) { - return s.allowed[kbID], nil +) (types.OrgMemberRole, bool, error) { + return types.OrgRoleViewer, s.allowed[kbID], nil } func TestResolveSuggestionTagScopes_UsesSourceTenantForSharedKB(t *testing.T) { @@ -97,8 +96,7 @@ func TestResolveSuggestionTagScopes_UsesSourceTenantForSharedKB(t *testing.T) { } resolved, err := svc.resolveSuggestionTagScopes( - context.Background(), - callerTenant, + context.WithValue(context.Background(), types.TenantIDContextKey, callerTenant), []types.TagScope{{KnowledgeBaseID: kbID, TagIDs: []string{tagID}}}, ) require.NoError(t, err) diff --git a/internal/application/service/datasource_service.go b/internal/application/service/datasource_service.go index 61ec2d27..950c0aa3 100644 --- a/internal/application/service/datasource_service.go +++ b/internal/application/service/datasource_service.go @@ -13,6 +13,7 @@ import ( "strings" "time" + "github.com/Tencent/WeKnora/internal/application/access" "github.com/Tencent/WeKnora/internal/datasource" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/tracing/langfuse" @@ -513,8 +514,10 @@ func (s *DataSourceService) ManualSync(ctx context.Context, dsID string) (*types logger.Infof(ctx, "sync task enqueued: ds=%s syncLog=%s", dsID, syncLog.ID) recordKBActivity(ctx, s.audit, ds.TenantID, ds.KnowledgeBaseID, types.AuditActionDataSourceSyncStarted, "data_source", ds.ID, types.AuditOutcomeAccepted, - map[string]any{"name": ds.Name, "type": ds.Type, "sync_log_id": syncLog.ID, - "task_id": info.ID, "trigger": "manual", "processing_status": "pending"}) + map[string]any{ + "name": ds.Name, "type": ds.Type, "sync_log_id": syncLog.ID, + "task_id": info.ID, "trigger": "manual", "processing_status": "pending", + }) return syncLog, nil } @@ -627,6 +630,10 @@ func (s *DataSourceService) ProcessSync(ctx context.Context, task *asynq.Task) e return nil } + ctx, err = access.WithKBTaskWrite(ctx, kb, ds.TenantID) + if err != nil { + return fmt.Errorf("%w: data source KB does not belong to its tenant", asynq.SkipRetry) + } wasPaused := ds.Status == types.DataSourceStatusPaused // Get connector @@ -719,7 +726,7 @@ func (s *DataSourceService) ProcessSync(ctx context.Context, task *asynq.Task) e } // Process fetched items and write to knowledge base - var result = &types.SyncResult{ + result := &types.SyncResult{ Total: len(items), } diff --git a/internal/application/service/datasource_service_test.go b/internal/application/service/datasource_service_test.go index 4a64723a..bc90b995 100644 --- a/internal/application/service/datasource_service_test.go +++ b/internal/application/service/datasource_service_test.go @@ -254,9 +254,15 @@ func (r *processSyncTenantRepo) GetTenantByID(context.Context, uint64) (*types.T type processSyncTagService struct { interfaces.KnowledgeTagService + ctx context.Context } -func (*processSyncTagService) FindOrCreateTagByName(context.Context, string, string) (*types.KnowledgeTag, error) { +func (s *processSyncTagService) FindOrCreateTagByName( + ctx context.Context, + _ string, + _ string, +) (*types.KnowledgeTag, error) { + s.ctx = ctx return nil, nil } diff --git a/internal/application/service/document_write_access_test.go b/internal/application/service/document_write_access_test.go new file mode 100644 index 00000000..f2fdf880 --- /dev/null +++ b/internal/application/service/document_write_access_test.go @@ -0,0 +1,660 @@ +package service + +import ( + "context" + "encoding/json" + "errors" + "path/filepath" + "testing" + + "github.com/Tencent/WeKnora/internal/application/access" + "github.com/Tencent/WeKnora/internal/application/repository" + apperrors "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/hibiken/asynq" + "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +type documentKBLookup struct { + interfaces.KnowledgeBaseService + values map[string]*types.KnowledgeBase + err error +} + +func (r *documentKBLookup) GetKnowledgeBaseByID(_ context.Context, id string) (*types.KnowledgeBase, error) { + if r.err != nil { + return nil, r.err + } + if r.values[id] == nil { + return nil, repository.ErrKnowledgeBaseNotFound + } + return r.values[id], nil +} + +type documentKBRepo struct { + interfaces.KnowledgeBaseRepository + lookup *documentKBLookup +} + +func (r documentKBRepo) GetKnowledgeBaseByID(ctx context.Context, id string) (*types.KnowledgeBase, error) { + return r.lookup.GetKnowledgeBaseByID(ctx, id) +} + +type documentKnowledgeSpy struct { + interfaces.KnowledgeRepository + writes int +} + +func (r *documentKnowledgeSpy) UpdateKnowledge(ctx context.Context, k *types.Knowledge) error { + r.writes++ + return r.KnowledgeRepository.UpdateKnowledge(ctx, k) +} + +func (r *documentKnowledgeSpy) SetKnowledgeTags(ctx context.Context, id string, tags []string) error { + r.writes++ + return r.KnowledgeRepository.SetKnowledgeTags(ctx, id, tags) +} + +func (r *documentKnowledgeSpy) DeleteKnowledgeList(ctx context.Context, tenant uint64, ids []string) error { + r.writes++ + return r.KnowledgeRepository.DeleteKnowledgeList(ctx, tenant, ids) +} + +type documentChunkSpy struct { + interfaces.ChunkRepository + writes int + imageErr error +} + +func (r *documentChunkSpy) CreateChunks(ctx context.Context, chunks []*types.Chunk) error { + r.writes++ + return r.ChunkRepository.CreateChunks(ctx, chunks) +} + +func (r *documentChunkSpy) UpdateChunks(ctx context.Context, chunks []*types.Chunk) error { + r.writes++ + return r.ChunkRepository.UpdateChunks(ctx, chunks) +} + +func (r *documentChunkSpy) UpdateChunk(ctx context.Context, chunk *types.Chunk) error { + r.writes++ + return r.ChunkRepository.UpdateChunk(ctx, chunk) +} + +func (r *documentChunkSpy) DeleteChunks(ctx context.Context, tenant uint64, ids []string) error { + r.writes++ + return r.ChunkRepository.DeleteChunks(ctx, tenant, ids) +} + +func (r *documentChunkSpy) DeleteChunk(ctx context.Context, tenant uint64, id string) error { + r.writes++ + return r.ChunkRepository.DeleteChunk(ctx, tenant, id) +} + +func (r *documentChunkSpy) DeleteByKnowledgeList(ctx context.Context, tenant uint64, ids []string) error { + r.writes++ + return r.ChunkRepository.DeleteByKnowledgeList(ctx, tenant, ids) +} + +func (r *documentChunkSpy) DeleteChunksByKnowledgeID(ctx context.Context, tenant uint64, id string) error { + r.writes++ + return r.ChunkRepository.DeleteChunksByKnowledgeID(ctx, tenant, id) +} + +func (r *documentChunkSpy) SaveChunkRevision( + ctx context.Context, + chunk *types.Chunk, + rev *types.ChunkRevision, + expected int, +) error { + r.writes++ + return r.ChunkRepository.SaveChunkRevision(ctx, chunk, rev, expected) +} + +func (r *documentChunkSpy) ListImageInfoByKnowledgeIDs( + ctx context.Context, + tenant uint64, + ids []string, +) ([]interfaces.ChunkImageInfo, error) { + if r.imageErr != nil { + return nil, r.imageErr + } + return r.ChunkRepository.ListImageInfoByKnowledgeIDs(ctx, tenant, ids) +} + +type documentGraphSpy struct { + interfaces.RetrieveGraphRepository + err error + calls int +} + +func (r *documentGraphSpy) DelGraph(context.Context, []types.NameSpace) error { + r.calls++ + return r.err +} + +type documentTenantSpy struct { + interfaces.TenantRepository + adjustments []int64 +} + +func (r *documentTenantSpy) GetTenantByID(_ context.Context, id uint64) (*types.Tenant, error) { + return &types.Tenant{ID: id}, nil +} + +func (r *documentTenantSpy) AdjustStorageUsed(_ context.Context, _ uint64, delta int64) error { + r.adjustments = append(r.adjustments, delta) + return nil +} + +type documentFileSpy struct { + interfaces.FileService + deleted []string + beforeDelete func() +} + +func (r *documentFileSpy) DeleteFile(_ context.Context, path string) error { + if r.beforeDelete != nil { + r.beforeDelete() + } + r.deleted = append(r.deleted, path) + return nil +} + +type documentWriteFixture struct { + svc *knowledgeService + chunks *chunkService + repo *documentKnowledgeSpy + chunkRepo *documentChunkSpy + kbs *documentKBLookup + graph *documentGraphSpy + files *documentFileSpy + tenants *documentTenantSpy + db *gorm.DB + ctx context.Context +} + +func newDocumentWriteFixture(t *testing.T) *documentWriteFixture { + t.Helper() + db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "document.db")), &gorm.Config{}) + require.NoError(t, err) + sqlDB, err := db.DB() + require.NoError(t, err) + sqlDB.SetMaxOpenConns(1) + t.Cleanup(func() { require.NoError(t, sqlDB.Close()) }) + require.NoError( + t, + db.AutoMigrate( + &types.Knowledge{}, + &types.Chunk{}, + &types.KnowledgeTag{}, + &types.KnowledgeTagRelation{}, + &types.ChunkRevision{}, + ), + ) + repo := &documentKnowledgeSpy{KnowledgeRepository: repository.NewKnowledgeRepository(db)} + chunkRepo := &documentChunkSpy{ChunkRepository: repository.NewChunkRepository(db)} + kbs := &documentKBLookup{values: map[string]*types.KnowledgeBase{ + "kb": {ID: "kb", TenantID: 7}, "other": {ID: "other", TenantID: 7}, + }} + for _, row := range []*types.Knowledge{ + { + ID: "doc", + TenantID: 7, + KnowledgeBaseID: "kb", + Title: "original", + Type: types.KnowledgeTypeManual, + ParseStatus: types.ManualKnowledgeStatusDraft, + StorageSize: 5, + }, + + { + ID: "other-doc", + TenantID: 7, + KnowledgeBaseID: "other", + Title: "other", + ParseStatus: types.ParseStatusCompleted, + }, + } { + require.NoError(t, db.Create(row).Error) + } + for _, row := range []*types.Chunk{ + { + ID: "chunk", + TenantID: 7, + KnowledgeID: "doc", + KnowledgeBaseID: "kb", + ChunkType: types.ChunkTypeText, + Content: "original", + IndexStatus: "ready", + }, + + { + ID: "other-chunk", + TenantID: 7, + KnowledgeID: "other-doc", + KnowledgeBaseID: "other", + ChunkType: types.ChunkTypeText, + }, + } { + require.NoError(t, db.Create(row).Error) + } + require.NoError(t, db.Create(&types.KnowledgeTag{ID: "tag", TenantID: 7, KnowledgeBaseID: "kb", Name: "tag"}).Error) + graph, files, tenants := &documentGraphSpy{}, &documentFileSpy{}, &documentTenantSpy{} + chunks := &chunkService{chunkRepository: chunkRepo, knowledgeRepo: repo, kbRepository: documentKBRepo{lookup: kbs}} + svc := &knowledgeService{ + repo: repo, + chunkRepo: chunkRepo, + chunkService: chunks, + kbService: kbs, + graphEngine: graph, + fileSvc: files, + tenantRepo: tenants, + tagRepo: repository.NewKnowledgeTagRepository(db), + } + ctx := types.WithCaller( + context.Background(), + types.Caller{TenantID: 1, UserID: "editor", Role: types.TenantRoleAdmin}, + ) + grant := &access.KBAccess{ + KnowledgeBase: kbs.values["kb"], + Caller: types.CallerFromContext(ctx), + EffectiveTenantID: 7, + Permission: types.OrgRoleEditor, + } + ctx = grant.Context(ctx) + ctx = context.WithValue(ctx, types.TenantInfoContextKey, &types.Tenant{ID: 7}) + return &documentWriteFixture{ + svc: svc, + chunks: chunks, + repo: repo, + chunkRepo: chunkRepo, + kbs: kbs, + graph: graph, + files: files, + tenants: tenants, + db: db, + ctx: ctx, + } +} + +func requireForbiddenWrite(t *testing.T, err error) { + t.Helper() + var appErr *apperrors.AppError + require.ErrorAs(t, err, &appErr) + require.Equal(t, 403, appErr.HTTPCode) +} + +func (f *documentWriteFixture) requireNoWrites(t *testing.T) { + t.Helper() + require.Zero(t, f.repo.writes) + require.Zero(t, f.chunkRepo.writes) + require.Zero(t, f.graph.calls) + require.Empty(t, f.files.deleted) +} + +func TestDocumentAndChunkWritesRequireOperationGrant(t *testing.T) { + f := newDocumentWriteFixture(t) + ctx := types.WithExecutionTenant( + types.WithCaller(context.Background(), types.Caller{TenantID: 7, Role: types.TenantRoleAdmin}), + 7, + ) + // Even an owner read resolution projects Admin without authorizing writes. + grant, err := access.ResolveKB( + ctx, + access.KBRequest{Caller: types.CallerFromContext(ctx)}, + f.kbs.values["kb"], + types.OrgRoleViewer, + nil, + nil, + ) + require.NoError(t, err) + ctx = grant.Context(ctx) + row, _ := f.chunkRepo.GetChunkByID(f.ctx, 7, "chunk") + ops := map[string]func() error{ + "set tags": func() error { return f.svc.SetKnowledgeTags(ctx, "doc", nil) }, + "move folder": func() error { + _, + err := f.svc.MoveKnowledgeToFolder(ctx, + "kb", + []string{"doc"}, + "folder") + return err + }, + + "rename folder": func() error { + _, + err := f.svc.RenameKnowledgeFolder(ctx, + "kb", + "folder", + "next") + return err + }, + + "metadata": func() error { return f.svc.UpdateKnowledge(ctx, &types.Knowledge{ID: "doc", Title: "new"}) }, + "manual": func() error { + _, err := f.svc.UpdateManualKnowledge( + ctx, + "doc", + &types.ManualKnowledgePayload{Content: "body", Status: "draft"}, + ) + return err + }, + "tag": func() error { return f.svc.UpdateKnowledgeTag(ctx, "doc", []string{"tag"}) }, + "batch tags": func() error { + return f.svc.UpdateKnowledgeTagBatch(ctx, + "", + map[string][]string{"doc": {"tag"}}) + }, + + "delete document": func() error { return f.svc.DeleteKnowledge(ctx, "doc") }, + "delete documents": func() error { return f.svc.DeleteKnowledgeList(ctx, []string{"doc"}) }, + "create chunks": func() error { return f.chunks.CreateChunks(ctx, []*types.Chunk{row}) }, + "update chunk": func() error { return f.chunks.UpdateChunk(ctx, row) }, + "update chunks": func() error { return f.chunks.UpdateChunks(ctx, []*types.Chunk{row}) }, + "edit chunk": func() error { + body := "new" + _, err := f.chunks.UpdateDocumentChunk(ctx, "chunk", &body, nil, nil) + return err + }, + "upsert question": func() error { + _, + err := f.chunks.UpsertGeneratedQuestion(ctx, + "chunk", + "", + "new question") + return err + }, + + "delete question": func() error { return f.chunks.DeleteGeneratedQuestion(ctx, "chunk", "q") }, + "regenerate questions": func() error { + _, + err := f.svc.RegenerateChunkQuestions(ctx, + "chunk") + return err + }, + + "update image": func() error { return f.svc.UpdateImageInfo(ctx, "doc", "chunk", "[]") }, + "delete chunk": func() error { return f.chunks.DeleteChunk(ctx, "chunk") }, + "delete chunks": func() error { return f.chunks.DeleteChunks(ctx, []string{"chunk"}) }, + "delete document chunks": func() error { return f.chunks.DeleteChunksByKnowledgeID(ctx, "doc") }, + "delete batch document chunks": func() error { return f.chunks.DeleteByKnowledgeList(ctx, []string{"doc"}) }, + } + for name, op := range ops { + t.Run(name, func(t *testing.T) { requireForbiddenWrite(t, op()); f.requireNoWrites(t) }) + } + require.NoError(t, f.svc.UpdateKnowledge(f.ctx, &types.Knowledge{ID: "doc", Title: "shared edit"})) + stored, err := f.repo.GetKnowledgeByID(f.ctx, 7, "doc") + require.NoError(t, err) + require.Equal(t, "shared edit", stored.Title) +} + +func TestDocumentBatchPreflightDoesNotPartiallyMutate(t *testing.T) { + for _, invalid := range []string{"other-doc", "missing", ""} { + t.Run(invalid, func(t *testing.T) { + f := newDocumentWriteFixture(t) + require.Error(t, f.svc.DeleteKnowledgeList(f.ctx, []string{"doc", invalid})) + f.requireNoWrites(t) + require.Error( + t, + f.svc.UpdateKnowledgeTagBatch(f.ctx, "", map[string][]string{"doc": {"tag"}, invalid: nil}), + ) + f.requireNoWrites(t) + _, err := f.svc.MoveKnowledgeToFolder(f.ctx, "kb", []string{"doc", invalid}, "next") + require.Error(t, err) + f.requireNoWrites(t) + require.Error(t, f.chunks.DeleteByKnowledgeList(f.ctx, []string{"doc", invalid})) + f.requireNoWrites(t) + }) + } + for _, invalid := range []string{"other-chunk", "missing", ""} { + t.Run("chunks/"+invalid, func(t *testing.T) { + f := newDocumentWriteFixture(t) + require.Error(t, f.chunks.DeleteChunks(f.ctx, []string{"chunk", invalid})) + f.requireNoWrites(t) + good, err := f.chunkRepo.GetChunkByID(f.ctx, 7, "chunk") + require.NoError(t, err) + forged := *good + forged.ID = invalid + require.Error(t, f.chunks.UpdateChunks(f.ctx, []*types.Chunk{good, &forged})) + f.requireNoWrites(t) + }) + } +} + +func TestDeletePreflightResolvesKBAndImagesBeforeChangingStatus(t *testing.T) { + for _, stage := range []string{"KB lookup", "KB owner", "image lookup"} { + t.Run(stage, func(t *testing.T) { + f := newDocumentWriteFixture(t) + switch stage { + case "KB lookup": + f.kbs.err = errors.New("database unavailable") + case "KB owner": + f.kbs.values["kb"].TenantID = 9 + case "image lookup": + f.chunkRepo.imageErr = errors.New("database unavailable") + } + require.Error(t, f.svc.DeleteKnowledge(f.ctx, "doc")) + f.requireNoWrites(t) + row, err := f.repo.GetKnowledgeByID(f.ctx, 7, "doc") + require.NoError(t, err) + require.Equal(t, types.ManualKnowledgeStatusDraft, row.ParseStatus) + }) + } +} + +func deleteDocumentTask(t *testing.T, payload types.KnowledgeListDeletePayload) *asynq.Task { + t.Helper() + data, err := json.Marshal(payload) + require.NoError(t, err) + return asynq.NewTask(types.TypeKnowledgeListDelete, data) +} + +func TestDeleteTaskScopeRejectsMovedDocumentsBeforeAnyMutation(t *testing.T) { + f := newDocumentWriteFixture(t) + task := deleteDocumentTask( + t, + types.KnowledgeListDeletePayload{ + TenantID: 7, + KnowledgeBaseID: "kb", + KnowledgeIDs: []string{"doc", "other-doc"}, + }, + ) + require.ErrorIs(t, f.svc.ProcessKnowledgeListDelete(context.Background(), task), asynq.SkipRetry) + f.requireNoWrites(t) + task = deleteDocumentTask( + t, + types.KnowledgeListDeletePayload{TenantID: 7, KnowledgeIDs: []string{"doc", "other-doc"}}, + ) + require.ErrorIs(t, f.svc.ProcessKnowledgeListDelete(context.Background(), task), asynq.SkipRetry) + f.requireNoWrites(t) +} + +func TestDeleteTaskRetriesAfterPartialCleanupAndAlreadyDeletedRows(t *testing.T) { + f := newDocumentWriteFixture(t) + require.NoError(t, f.db.Model(&types.Knowledge{}).Where("id = ?", "doc").Update("file_path", "local://doc").Error) + task := deleteDocumentTask( + t, + types.KnowledgeListDeletePayload{ + TenantID: 7, + KnowledgeBaseID: "kb", + KnowledgeIDs: []string{"doc", "doc", "already-gone"}, + }, + ) + f.graph.err = errors.New("graph unavailable") + err := f.svc.ProcessKnowledgeListDelete(context.Background(), task) + require.ErrorContains(t, err, "graph unavailable") + require.NotErrorIs(t, err, asynq.SkipRetry) + row, err := f.repo.GetKnowledgeByID(f.ctx, 7, "doc") + require.NoError(t, err) + require.Equal(t, types.ParseStatusDeleting, row.ParseStatus) + require.Empty(t, f.files.deleted) + require.Empty(t, f.tenants.adjustments) + f.graph.err = nil + f.files.beforeDelete = func() { _, err := f.repo.GetKnowledgeByID(f.ctx, 7, "doc"); require.Error(t, err) } + require.NoError(t, f.svc.ProcessKnowledgeListDelete(context.Background(), task)) + require.Equal(t, []string{"local://doc"}, f.files.deleted) + require.Equal(t, []int64{-5}, f.tenants.adjustments) + // A completed task does not need live KB metadata or current user grants. + delete(f.kbs.values, "kb") + writes, calls := f.repo.writes, f.graph.calls + require.NoError(t, f.svc.ProcessKnowledgeListDelete(context.Background(), task)) + require.Equal(t, writes, f.repo.writes) + require.Equal(t, calls, f.graph.calls) +} + +func TestKnowledgeCleanupScopeCannotAuthorizeGeneralWritesOrExtraIDs(t *testing.T) { + f := newDocumentWriteFixture(t) + ctx := withKnowledgeCleanup(context.Background(), 7, map[string]string{"doc": "kb"}) + require.Zero(t, types.CallerFromContext(ctx).TenantID) + requireForbiddenWrite(t, f.svc.UpdateKnowledge(ctx, &types.Knowledge{ID: "doc", Title: "no"})) + requireForbiddenWrite(t, f.svc.DeleteKnowledgeList(ctx, []string{"doc", "other-doc"})) + requireForbiddenWrite( + t, + f.svc.DeleteKnowledgeList(types.WithCaller(ctx, types.Caller{TenantID: 7}), []string{"doc"}), + ) + f.requireNoWrites(t) + require.NoError(t, f.svc.DeleteKnowledge(ctx, "doc")) +} + +func TestReferencedKnowledgeCleanupPinsItsIndependentKB(t *testing.T) { + f := newDocumentWriteFixture(t) + requireForbiddenWrite(t, deleteReferencedKnowledge(f.ctx, f.svc, "kb", []string{"doc", "other-doc"})) + f.requireNoWrites(t) + // Session cleanup authority comes from exact persisted references and can + // continue even when the caller has no permission to edit the history KB. + ctx := types.WithExecutionTenant(types.WithCaller(context.Background(), types.Caller{TenantID: 7}), 7) + require.NoError(t, deleteReferencedKnowledge(ctx, f.svc, "kb", []string{"doc", "already-gone"})) +} + +func TestChunkEditRejectsCrossDocumentParentBeforeSavingRevision(t *testing.T) { + f := newDocumentWriteFixture(t) + require.NoError( + t, + f.db.Model(&types.Chunk{}).Where("id = ?", "chunk").Update("parent_chunk_id", "other-chunk").Error, + ) + body := "edit" + _, err := f.chunks.UpdateDocumentChunk(f.ctx, "chunk", &body, nil, nil) + requireForbiddenWrite(t, err) + f.requireNoWrites(t) +} + +func TestChunkEditRejectsCrossDocumentChildBeforeSavingRevision(t *testing.T) { + f := newDocumentWriteFixture(t) + require.NoError( + t, + f.db.Model(&types.Chunk{}).Where("id = ?", "other-chunk").Update("parent_chunk_id", "chunk").Error, + ) + body := "edit" + _, err := f.chunks.UpdateDocumentChunk(f.ctx, "chunk", &body, nil, nil) + requireForbiddenWrite(t, err) + f.requireNoWrites(t) +} + +func TestChunkBatchWriteAllowsOnlyStoredOwnership(t *testing.T) { + f := newDocumentWriteFixture(t) + row, err := f.chunkRepo.GetChunkByID(f.ctx, 7, "chunk") + require.NoError(t, err) + row.Content = "updated" + require.NoError(t, f.chunks.UpdateChunks(f.ctx, []*types.Chunk{row})) + stored, err := f.chunkRepo.GetChunkByID(f.ctx, 7, "chunk") + require.NoError(t, err) + require.Equal(t, "updated", stored.Content) + require.NoError(t, f.chunks.DeleteChunks(f.ctx, []string{"chunk", "chunk"})) + _, err = f.chunkRepo.GetChunkByID(f.ctx, 7, "chunk") + require.ErrorIs(t, err, repository.ErrChunkNotFound) +} + +func TestDocumentWriteTaskAndAPIKeyScopesStayNarrow(t *testing.T) { + f := newDocumentWriteFixture(t) + ctx, err := access.WithKBTaskWrite(context.Background(), f.kbs.values["kb"], 7) + require.NoError(t, err) + require.NoError(t, f.svc.UpdateKnowledge(ctx, &types.Knowledge{ID: "doc", Title: "worker edit"})) + requireForbiddenWrite(t, f.svc.UpdateKnowledge(ctx, &types.Knowledge{ID: "other-doc", Title: "no"})) + scope := types.TenantAPIKeyScope{ + KnowledgeBaseIDs: types.StringArray{"kb"}, + Capabilities: types.StringArray{string(types.APIKeyCapabilityRetrieve)}, + } + requireForbiddenWrite( + t, + f.svc.UpdateKnowledge(types.WithTenantAPIKeyScope(f.ctx, scope), &types.Knowledge{ID: "doc", Title: "no"}), + ) + require.Zero(t, types.CallerFromContext(ctx).TenantID) +} + +func TestLegacySingleKBDeleteTaskRemainsExecutable(t *testing.T) { + f := newDocumentWriteFixture(t) + task := deleteDocumentTask( + t, + types.KnowledgeListDeletePayload{TenantID: 7, KnowledgeIDs: []string{"doc", "already-gone"}}, + ) + require.NoError(t, f.svc.ProcessKnowledgeListDelete(context.Background(), task)) + _, err := f.repo.GetKnowledgeByID(f.ctx, 7, "doc") + require.Error(t, err) +} + +func TestMultiKBDeleteRequiresAndAcceptsEveryKBGrant(t *testing.T) { + f := newDocumentWriteFixture(t) + grant := &access.KBAccess{ + KnowledgeBase: f.kbs.values["other"], + Caller: types.CallerFromContext(f.ctx), + EffectiveTenantID: 7, + Permission: types.OrgRoleEditor, + } + require.NoError(t, f.svc.DeleteKnowledgeList(grant.Context(f.ctx), []string{"doc", "other-doc"})) + rows, err := f.repo.GetKnowledgeBatch(f.ctx, 7, []string{"doc", "other-doc"}) + require.NoError(t, err) + require.Empty(t, rows) +} + +func TestDeleteCheckpointRejectsChangedOrDeletedDocument(t *testing.T) { + for _, action := range []string{"moved", "deleted"} { + t.Run(action, func(t *testing.T) { + f := newDocumentWriteFixture(t) + plan, err := f.svc.planKnowledgeDelete(f.ctx, []string{"doc"}) + require.NoError(t, err) + if action == "moved" { + require.NoError( + t, + f.db.Model(&types.Knowledge{}).Where("id = ?", "doc").Update("knowledge_base_id", "other").Error, + ) + } else { + require.NoError(t, f.db.Where("id = ?", "doc").Delete(&types.Knowledge{}).Error) + } + require.Error(t, f.svc.executeKnowledgeDelete(plan, true)) + require.Zero(t, f.graph.calls) + require.Zero(t, f.chunkRepo.writes) + require.Empty(t, f.files.deleted) + row, err := f.repo.GetKnowledgeByID(f.ctx, 7, "doc") + if action == "moved" { + require.NoError(t, err) + require.Equal(t, "other", row.KnowledgeBaseID) + require.Equal(t, types.ManualKnowledgeStatusDraft, row.ParseStatus) + } else { + require.Error(t, err) + } + }) + } +} + +func TestDocumentWritesRejectMoveInProgress(t *testing.T) { + f := newDocumentWriteFixture(t) + row, err := f.repo.GetKnowledgeByID(f.ctx, 7, "doc") + require.NoError(t, err) + require.NoError(t, setTransferState(row, knowledgeTransferState{Operation: access.KBTransferMove, Phase: "moving"})) + require.NoError(t, f.db.Model(&types.Knowledge{}).Where("id = ?", "doc").Update("metadata", row.Metadata).Error) + for _, op := range []func() error{ + func() error { return f.svc.DeleteKnowledge(f.ctx, "doc") }, + func() error { return f.svc.UpdateKnowledge(f.ctx, &types.Knowledge{ID: "doc", Title: "changed"}) }, + func() error { return f.chunks.DeleteChunk(f.ctx, "chunk") }, + func() error { return f.svc.DeleteKnowledgeList(f.ctx, []string{"doc"}) }, + } { + var appErr *apperrors.AppError + require.ErrorAs(t, op(), &appErr) + require.Equal(t, 409, appErr.HTTPCode) + } + f.requireNoWrites(t) +} diff --git a/internal/application/service/evaluation.go b/internal/application/service/evaluation.go index 184613f2..4a3844cb 100644 --- a/internal/application/service/evaluation.go +++ b/internal/application/service/evaluation.go @@ -363,7 +363,10 @@ func (e *EvaluationService) EvalDataset(ctx context.Context, detail *types.Evalu // Setup cleanup of temporary resources defer func() { logger.Infof(ctx, "Cleaning up resources - deleting knowledge: %s", knowledge.ID) - if err := e.knowledgeService.DeleteKnowledge(ctx, knowledge.ID); err != nil { + if err := deleteReferencedKnowledge(ctx, + e.knowledgeService, + knowledgeBaseID, + []string{knowledge.ID}); err != nil { logger.Errorf(ctx, "Failed to delete knowledge: %v, knowledge ID: %s", err, knowledge.ID) } diff --git a/internal/application/service/extract.go b/internal/application/service/extract.go index 3d789d6d..d43afe01 100644 --- a/internal/application/service/extract.go +++ b/internal/application/service/extract.go @@ -10,6 +10,7 @@ import ( "time" "github.com/Tencent/WeKnora/internal/agent/tools" + "github.com/Tencent/WeKnora/internal/application/access" chatpipeline "github.com/Tencent/WeKnora/internal/application/service/chat_pipeline" "github.com/Tencent/WeKnora/internal/application/service/retriever" "github.com/Tencent/WeKnora/internal/config" @@ -468,7 +469,7 @@ func (s *DataTableSummaryService) Handle(ctx context.Context, t *asynq.Task) err ctx = logger.WithRequestID(ctx, uuid.New().String()) ctx = logger.WithField(ctx, "knowledge", payload.KnowledgeID) - ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID) + ctx = types.WithExecutionTenant(ctx, payload.TenantID) logger.Infof(ctx, "Processing table extraction for knowledge: %s", payload.KnowledgeID) @@ -478,6 +479,12 @@ func (s *DataTableSummaryService) Handle(ctx context.Context, t *asynq.Task) err return err } + ctx, err = access.WithKBTaskWrite(ctx, resources.knowledgeBase, payload.TenantID) + if err != nil { + return err + } + ctx = context.WithValue(ctx, types.TenantInfoContextKey, resources.tenant) + // 3. 加载表格数据并生成摘要 chunks, err := s.processTableData(ctx, resources) if err != nil { @@ -508,12 +515,20 @@ type extractionResources struct { // 思路:集中加载所有依赖,统一错误处理,避免分散的资源获取逻辑 func (s *DataTableSummaryService) prepareResources(ctx context.Context, payload DataTableSummaryPayload) (*extractionResources, error) { // 获取并验证知识文件 - knowledge, err := s.knowledgeService.GetKnowledgeByID(ctx, payload.KnowledgeID) + knowledge, err := s.knowledgeService.GetRepository().GetKnowledgeByID(ctx, payload.TenantID, payload.KnowledgeID) if err != nil { logger.Errorf(ctx, "failed to get knowledge: %v", err) return nil, err } + if knowledge == nil || knowledge.ID != payload.KnowledgeID || knowledge.TenantID != payload.TenantID { + return nil, fmt.Errorf("invalid table summary knowledge scope") + } + kb, err := knowledgeWriteKB(ctx, s.knowledgeBaseService, knowledge) + if err != nil { + return nil, err + } + // 验证文件类型 fileType := strings.ToLower(knowledge.FileType) if fileType != "csv" && fileType != "xlsx" && fileType != "xls" { @@ -542,13 +557,6 @@ func (s *DataTableSummaryService) prepareResources(ctx context.Context, payload return nil, err } - // Load the KB to discover its VectorStoreID binding so the factory can - // route to the bound store (or fall back to tenant engines if unbound). - kb, err := s.knowledgeBaseService.GetKnowledgeBaseByID(ctx, knowledge.KnowledgeBaseID) - if err != nil { - logger.Errorf(ctx, "failed to get knowledge base for vector store lookup: %v", err) - return nil, err - } var vectorStoreID *string if kb != nil { vectorStoreID = kb.VectorStoreID @@ -774,12 +782,12 @@ func (s *DataTableSummaryService) cleanupOnFailure(ctx context.Context, resource logger.Warnf(ctx, "Starting cleanup due to failure: %v", indexErr) // 1. 更新知识状态为失败 - resources.knowledge.ParseStatus = types.ParseStatusFailed - resources.knowledge.ErrorMessage = indexErr.Error() - if err := s.knowledgeService.UpdateKnowledge(ctx, resources.knowledge); err != nil { - logger.Errorf(ctx, "Failed to update knowledge status: %v", err) - } else { - logger.Infof(ctx, "Updated knowledge %s status to failed", resources.knowledge.ID) + before, after := *resources.knowledge, *resources.knowledge + after.ParseStatus = types.ParseStatusFailed + after.ErrorMessage = indexErr.Error() + if err := s.knowledgeService.GetRepository().UpdateKnowledgeForTransfer(ctx, &before, &after); err != nil { + logger.Warnf(ctx, "Table summary cleanup skipped after knowledge changed: %v", err) + return } // 提取chunk IDs @@ -790,7 +798,7 @@ func (s *DataTableSummaryService) cleanupOnFailure(ctx context.Context, resource // 删除已创建的chunks if len(chunkIDs) > 0 { - if err := s.chunkService.DeleteChunks(ctx, chunkIDs); err != nil { + if err := s.chunkService.GetRepository().DeleteChunks(ctx, resources.knowledge.TenantID, chunkIDs); err != nil { logger.Errorf(ctx, "Failed to delete chunks: %v", err) } else { logger.Infof(ctx, "Deleted %d chunks", len(chunkIDs)) diff --git a/internal/application/service/file/resolve_tenant.go b/internal/application/service/file/resolve_tenant.go new file mode 100644 index 00000000..fe67ec9f --- /dev/null +++ b/internal/application/service/file/resolve_tenant.go @@ -0,0 +1,69 @@ +package file + +import ( + "context" + "net/http" + "os" + "strings" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// ResolveTenantFileServiceWithFallback applies the storage resolution and +// fallback rule shared by file proxies and artifact downloads: when tenant-level +// resolution fails and the requested provider matches the process-global +// STORAGE_TYPE, serve through the global file service instead of failing. +// Returns ok=false (already logged) when no service can be resolved; the +// caller decides the HTTP status. +func ResolveTenantFileServiceWithFallback( + ctx context.Context, + logTag string, + tenant *types.Tenant, + backendID, provider, absDir string, + storageResolver interfaces.StorageBackendResolver, + globalFileService interfaces.FileService, +) (fileSvc interfaces.FileService, resolvedProvider string, ok bool) { + var err error + if storageResolver != nil { + fileSvc, resolvedProvider, err = storageResolver.ResolveFileService(ctx, tenant, backendID, provider, absDir) + } else if tenant.StorageEngineConfig != nil { + fileSvc, resolvedProvider, err = NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, absDir) + } else { + err = http.ErrMissingFile + } + if err == nil { + return fileSvc, resolvedProvider, true + } + + globalStorageType := strings.ToLower(strings.TrimSpace(os.Getenv("STORAGE_TYPE"))) + if globalStorageType == "" { + globalStorageType = "local" + } + if provider == globalStorageType && globalFileService != nil { + logger.Warnf( + ctx, + "[Router] %s tenant storage config missing or invalid, fallback to global file "+ + "service: tenant_id=%d provider=%s err=%v", + + logTag, + tenant.ID, + provider, + err, + ) + return globalFileService, globalStorageType, true + } + logger.Warnf( + ctx, + "[Router] %s resolve file service failed without fallback: tenant_id=%d "+ + "provider=%s global_storage_type=%s err=%v", + + logTag, + tenant.ID, + provider, + globalStorageType, + err, + ) + return nil, "", false +} diff --git a/internal/application/service/image_multimodal.go b/internal/application/service/image_multimodal.go index 46b4d892..aedd671c 100644 --- a/internal/application/service/image_multimodal.go +++ b/internal/application/service/image_multimodal.go @@ -342,7 +342,7 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e } // Persist chunks - if err := s.chunkService.CreateChunks(ctx, newChunks); err != nil { + if err := s.chunkService.GetRepository().CreateChunks(ctx, newChunks); err != nil { handleErr = fmt.Errorf("create multimodal chunks: %w", err) return handleErr } @@ -446,7 +446,7 @@ func (s *ImageMultimodalService) indexChunks(ctx context.Context, payload types. continue } dbChunk.Status = int(types.ChunkStatusIndexed) - if uerr := s.chunkService.UpdateChunk(ctx, dbChunk); uerr != nil { + if uerr := s.chunkService.GetRepository().UpdateChunk(ctx, dbChunk); uerr != nil { logger.Warnf(ctx, "[ImageMultimodal] Failed to update chunk %s status to indexed: %v", chunk.ID, uerr) } } @@ -503,7 +503,7 @@ func (s *ImageMultimodalService) indexChunks(ctx context.Context, payload types. continue } dbChunk.Status = int(types.ChunkStatusIndexed) - if err := s.chunkService.UpdateChunk(ctx, dbChunk); err != nil { + if err := s.chunkService.GetRepository().UpdateChunk(ctx, dbChunk); err != nil { logger.Warnf(ctx, "[ImageMultimodal] Failed to update chunk %s status to indexed: %v", chunk.ID, err) } } diff --git a/internal/application/service/kbshare.go b/internal/application/service/kbshare.go index 1b5033fe..a17676cb 100644 --- a/internal/application/service/kbshare.go +++ b/internal/application/service/kbshare.go @@ -5,6 +5,7 @@ import ( "errors" "time" + "github.com/Tencent/WeKnora/internal/application/access" "github.com/Tencent/WeKnora/internal/application/repository" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" @@ -489,14 +490,7 @@ func (s *kbShareService) CheckTenantKBPermission(ctx context.Context, kbID strin // HasTenantKBPermission is a thin "do I have at least N" wrapper over // CheckTenantKBPermission for callers that don't need the granular role. func (s *kbShareService) HasTenantKBPermission(ctx context.Context, kbID string, callerTenantID uint64, callerTenantRole types.TenantRole, requiredRole types.OrgMemberRole) (bool, error) { - role, isShared, err := s.CheckTenantKBPermission(ctx, kbID, callerTenantID, callerTenantRole) - if err != nil { - return false, err - } - if !isShared { - return false, nil - } - return role.HasPermission(requiredRole), nil + return access.NewKBSharePermissions(ctx, s, callerTenantID, callerTenantRole).Check(kbID, requiredRole) } // GetKBSourceTenant gets the source tenant ID for a shared knowledge base diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index 7a513913..16f148b7 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "reflect" + "sort" "strings" "sync" "time" @@ -592,7 +593,18 @@ func (s *knowledgeService) MoveKnowledgeToFolder(ctx context.Context, if err != nil { return 0, err } - tenantID := ctx.Value(types.TenantIDContextKey).(uint64) + rows, err := loadKnowledgeWriteBatch(ctx, s.repo, s.kbService, ids) + if err != nil { + return 0, err + } + ids = make([]string, 0, len(rows)) + for _, row := range rows { + if row.KnowledgeBaseID != kbID { + return 0, werrors.NewForbiddenError("knowledge outside target KB") + } + ids = append(ids, row.ID) + } + tenantID := rows[0].TenantID affected, err := s.repo.UpdateKnowledgeFolderPath(ctx, tenantID, kbID, ids, normalized) if err != nil { logger.Errorf(ctx, "Failed to move knowledge to folder %q: %v", normalized, err) @@ -627,7 +639,18 @@ func (s *knowledgeService) RenameKnowledgeFolder(ctx context.Context, return 0, werrors.NewBadRequestError("不能将文件夹移动到它自己的子目录下") } - tenantID := ctx.Value(types.TenantIDContextKey).(uint64) + kb, err := s.kbService.GetKnowledgeBaseByID(ctx, kbID) + if err != nil { + return 0, err + } + if kb == nil || kb.ID != kbID { + return 0, werrors.NewNotFoundError("knowledge base not found") + } + ctx, err = requireKBWrite(ctx, kb) + if err != nil { + return 0, err + } + tenantID := kb.TenantID affected, err := s.repo.RenameKnowledgeFolderPath(ctx, tenantID, kbID, source, target) if err != nil { logger.Errorf(ctx, "Failed to rename folder %q to %q: %v", source, target, err) @@ -689,7 +712,10 @@ func (s *knowledgeService) GetKnowledgeFile(ctx context.Context, id string) (io. } func (s *knowledgeService) UpdateKnowledge(ctx context.Context, knowledge *types.Knowledge) error { - record, err := s.repo.GetKnowledgeByID(ctx, ctx.Value(types.TenantIDContextKey).(uint64), knowledge.ID) + if knowledge == nil { + return werrors.NewBadRequestError("knowledge cannot be nil") + } + record, _, err := loadKnowledgeWrite(ctx, s.repo, s.kbService, knowledge.ID) if err != nil { logger.Errorf(ctx, "Failed to get knowledge record: %v", err) return err @@ -769,48 +795,48 @@ func (s *knowledgeService) GetKnowledgeBatchWithSharedAccess(ctx context.Context if len(ids) == 0 { return nil, nil } - ownList, err := s.repo.GetKnowledgeBatch(ctx, tenantID, ids) + permissions := kbReadPermissions(ctx, s.kbShareService) + rows, err := s.repo.GetKnowledgeBatch(ctx, tenantID, ids) if err != nil { return nil, err } + ownList := make([]*types.Knowledge, 0, len(rows)) foundSet := make(map[string]bool) - for _, k := range ownList { + appendAllowed := func(k *types.Knowledge) { + if k == nil || foundSet[k.ID] { + return + } + allowed, err := permissions.Check(k.KnowledgeBaseID, k.TenantID, types.OrgRoleViewer) + if err == nil && allowed { + ownList = append(ownList, k) + foundSet[k.ID] = true + } + } + for _, k := range rows { + appendAllowed(k) if k != nil { foundSet[k.ID] = true } } - userIDVal := ctx.Value(types.UserIDContextKey) - if userIDVal == nil { - return ownList, nil - } - userID, ok := userIDVal.(string) - if !ok || userID == "" { - return ownList, nil - } - // Plan 3: shared-KB permission is keyed on (tenant, tenant_role) - // rather than user. callerTenantRole drives the 3-D cap. - callerTenantRole := types.TenantRoleFromContext(ctx) for _, id := range ids { if foundSet[id] { continue } k, err := s.repo.GetKnowledgeByIDOnly(ctx, id) - if err != nil || k == nil || k.KnowledgeBaseID == "" { - continue + if err != nil && !errors.Is(err, repository.ErrKnowledgeNotFound) { + return nil, err } - hasPermission, err := s.kbShareService.HasTenantKBPermission(ctx, k.KnowledgeBaseID, tenantID, callerTenantRole, types.OrgRoleViewer) - if err != nil || !hasPermission { - continue + if err == nil { + appendAllowed(k) } - foundSet[k.ID] = true - ownList = append(ownList, k) + foundSet[id] = true } return ownList, nil } // SetKnowledgeTags replaces all tags for a single knowledge entry. func (s *knowledgeService) SetKnowledgeTags(ctx context.Context, knowledgeID string, tagIDs []string) error { - return s.repo.SetKnowledgeTags(ctx, knowledgeID, tagIDs) + return s.UpdateKnowledgeTag(ctx, knowledgeID, tagIDs) } // ListKnowledgeIDsByTagIDs returns document knowledge IDs carrying any of the @@ -909,11 +935,11 @@ func (s *knowledgeService) GetKnowledgeTags(ctx context.Context, knowledgeIDs [] // UpdateKnowledgeTag updates the tags assigned to a knowledge document. func (s *knowledgeService) UpdateKnowledgeTag(ctx context.Context, knowledgeID string, tagIDs []string) error { - tenantID := ctx.Value(types.TenantIDContextKey).(uint64) - knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID) + knowledge, _, err := loadKnowledgeWrite(ctx, s.repo, s.kbService, knowledgeID) if err != nil { return err } + tenantID := knowledge.TenantID // Validate all tag IDs if err := s.validateKnowledgeTagIDs(ctx, tenantID, knowledge.KnowledgeBaseID, tagIDs); err != nil { @@ -925,29 +951,25 @@ func (s *knowledgeService) UpdateKnowledgeTag(ctx context.Context, knowledgeID s // UpdateKnowledgeTagBatch updates tags for document knowledge items in batch. // authorizedKBID restricts all updates to knowledge items belonging to this KB; -// pass empty string to skip the check (caller must ensure authorization by other means). -func (s *knowledgeService) UpdateKnowledgeTagBatch(ctx context.Context, authorizedKBID string, updates map[string][]string) error { +// an empty value allows multiple KBs only when every KB has an explicit write grant. +func (s *knowledgeService) UpdateKnowledgeTagBatch( + ctx context.Context, + authorizedKBID string, + updates map[string][]string, +) error { if len(updates) == 0 { return nil } - tenantIDVal := ctx.Value(types.TenantIDContextKey) - if tenantIDVal == nil { - return werrors.NewUnauthorizedError("workspace ID not found in context") - } - tenantID, ok := tenantIDVal.(uint64) - if !ok { - return werrors.NewUnauthorizedError("invalid workspace ID in context") - } - - // Get all knowledge items in batch knowledgeIDs := make([]string, 0, len(updates)) - for knowledgeID := range updates { - knowledgeIDs = append(knowledgeIDs, knowledgeID) + for id := range updates { + knowledgeIDs = append(knowledgeIDs, id) } - knowledgeList, err := s.repo.GetKnowledgeBatch(ctx, tenantID, knowledgeIDs) + sort.Strings(knowledgeIDs) + knowledgeList, err := loadKnowledgeWriteBatch(ctx, s.repo, s.kbService, knowledgeIDs) if err != nil { return err } + tenantID := knowledgeList[0].TenantID // Validate all requested IDs were found and belong to the authorized KB if authorizedKBID != "" { @@ -1002,15 +1024,15 @@ func (s *knowledgeService) UpdateKnowledgeTagBatch(ctx context.Context, authoriz if !ok { return werrors.NewBadRequestError(fmt.Sprintf("标签 %s 不存在", tagID)) } - if tag.KnowledgeBaseID != knowledge.KnowledgeBaseID { + if tag.TenantID != tenantID || tag.KnowledgeBaseID != knowledge.KnowledgeBaseID { return werrors.NewBadRequestError(fmt.Sprintf("标签 %s 不属于知识库 %s", tagID, knowledge.KnowledgeBaseID)) } } } // Set tags for each knowledge - for knowledgeID, tagIDs := range updates { - if err := s.repo.SetKnowledgeTags(ctx, knowledgeID, tagIDs); err != nil { + for _, knowledgeID := range knowledgeIDs { + if err := s.repo.SetKnowledgeTags(ctx, knowledgeID, updates[knowledgeID]); err != nil { return err } } @@ -1021,10 +1043,12 @@ func (s *knowledgeService) UpdateKnowledgeTagBatch(ctx context.Context, authoriz // SearchKnowledge searches knowledge items by keyword across the tenant and shared knowledge bases. // fileTypes: optional list of file extensions to filter by (e.g., ["csv", "xlsx"]) func (s *knowledgeService) SearchKnowledge(ctx context.Context, keyword string, offset, limit int, fileTypes []string) ([]*types.Knowledge, bool, int64, error) { - tenantID, ok := ctx.Value(types.TenantIDContextKey).(uint64) - if !ok { + caller := types.CallerFromContext(ctx) + tenantID := caller.TenantID + if tenantID == 0 { return nil, false, 0, werrors.NewUnauthorizedError("Workspace ID not found in context") } + ctx = types.WithExecutionTenant(ctx, tenantID) scopes := make([]types.KnowledgeSearchScope, 0) @@ -1041,9 +1065,9 @@ func (s *knowledgeService) SearchKnowledge(ctx context.Context, keyword string, // Shared knowledge bases (document type only). Plan 3 of #1303 keys // the share lookup on (tenantID, callerTenantRole); userID is no // longer load-bearing for org-share access. - if userIDVal := ctx.Value(types.UserIDContextKey); userIDVal != nil { - if userID, ok := userIDVal.(string); ok && userID != "" { - callerTenantRole := types.TenantRoleFromContext(ctx) + if s.kbShareService != nil { + if caller.UserID != "" { + callerTenantRole := caller.Role sharedList, err := s.kbShareService.ListSharedKnowledgeBases(ctx, tenantID, callerTenantRole) if err == nil { for _, info := range sharedList { diff --git a/internal/application/service/knowledge_auto_tag.go b/internal/application/service/knowledge_auto_tag.go index a2459fe8..87ab729e 100644 --- a/internal/application/service/knowledge_auto_tag.go +++ b/internal/application/service/knowledge_auto_tag.go @@ -7,6 +7,7 @@ import ( "sort" "strings" + "github.com/Tencent/WeKnora/internal/application/access" "github.com/Tencent/WeKnora/internal/common" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/models/chat" @@ -113,7 +114,7 @@ func (s *KnowledgeAutoTagService) Handle(ctx context.Context, task *asynq.Task) if err := json.Unmarshal(task.Payload(), &payload); err != nil { return fmt.Errorf("unmarshal knowledge auto tag payload: %w", err) } - ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID) + ctx = types.WithExecutionTenant(ctx, payload.TenantID) if payload.Language != "" { ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language) } @@ -153,6 +154,10 @@ func (s *KnowledgeAutoTagService) Handle(ctx context.Context, task *asynq.Task) if kb == nil || kb.TenantID != payload.TenantID || kb.Type != types.KnowledgeBaseTypeDocument { return skip("knowledge_base_not_eligible", nil) } + ctx, err = access.WithKBTaskWrite(ctx, kb, payload.TenantID) + if err != nil { + return err + } config := kb.AutoTagConfig if config == nil || !config.Enabled { return skip("auto_tag_disabled", nil) @@ -251,6 +256,9 @@ func (s *KnowledgeAutoTagService) Handle(ctx context.Context, task *asynq.Task) if len(added) == 0 { return skip("all_matches_already_attached", types.JSONMap{"matched_tag_count": len(validIDs)}) } + if err := access.RequireKBWrite(ctx, kb); err != nil { + return err + } if err := s.knowledgeRepo.AddKnowledgeTagRelations( ctx, payload.TenantID, diff --git a/internal/application/service/knowledge_auto_tag_test.go b/internal/application/service/knowledge_auto_tag_test.go index 4a214529..2e662edc 100644 --- a/internal/application/service/knowledge_auto_tag_test.go +++ b/internal/application/service/knowledge_auto_tag_test.go @@ -7,6 +7,7 @@ import ( "strings" "testing" + "github.com/Tencent/WeKnora/internal/application/access" "github.com/Tencent/WeKnora/internal/models/chat" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" @@ -36,6 +37,7 @@ type autoTagKnowledgeRepo struct { existing map[string][]*types.KnowledgeTag added []string addErr error + writeCtx context.Context } func (r *autoTagKnowledgeRepo) GetKnowledgeByIDOnly(context.Context, string) (*types.Knowledge, error) { @@ -49,8 +51,9 @@ func (r *autoTagKnowledgeRepo) GetKnowledgeTags( } func (r *autoTagKnowledgeRepo) AddKnowledgeTagRelations( - _ context.Context, _ uint64, _, _ string, tagIDs []string, + ctx context.Context, _ uint64, _, _ string, tagIDs []string, ) error { + r.writeCtx = ctx if r.addErr != nil { return r.addErr } @@ -173,6 +176,9 @@ func TestAutoTagHandleAttachesMatchedTags(t *testing.T) { fixture := newAutoTagFixture(t, `{"matches":[{"index":1,"confidence":0.9}]}`) require.NoError(t, fixture.handle(t)) assert.Equal(t, []string{"tag-a"}, fixture.repo.added) + require.Zero(t, types.CallerFromContext(fixture.repo.writeCtx).TenantID) + require.NoError(t, access.RequireKBWrite(fixture.repo.writeCtx, &types.KnowledgeBase{ID: "kb", TenantID: 7})) + require.Error(t, access.RequireKBWrite(fixture.repo.writeCtx, &types.KnowledgeBase{ID: "other", TenantID: 7})) } // A missing confidence must not be read as zero: models frequently omit the diff --git a/internal/application/service/knowledge_batch_reparse_test.go b/internal/application/service/knowledge_batch_reparse_test.go index 5babdab6..c4efc008 100644 --- a/internal/application/service/knowledge_batch_reparse_test.go +++ b/internal/application/service/knowledge_batch_reparse_test.go @@ -5,6 +5,7 @@ import ( "errors" "testing" + "github.com/Tencent/WeKnora/internal/application/access" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" "github.com/hibiken/asynq" @@ -81,18 +82,21 @@ func TestReparseKnowledgeManualEnqueueFailureIsVisible(t *testing.T) { repo := &reparseFailureKnowledgeRepo{knowledge: knowledge} svc := &knowledgeService{ repo: repo, - kbService: &reparseFailureKBService{kb: &types.KnowledgeBase{ID: "kb-1"}}, + kbService: &reparseFailureKBService{kb: &types.KnowledgeBase{ID: "kb-1", TenantID: 7}}, task: failingReparseTaskEnqueuer{err: enqueueErr}, } ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(7)) + ctx, grantErr := access.WithKBTaskWrite(ctx, &types.KnowledgeBase{ID: "kb-1", TenantID: 7}, 7) + require.NoError(t, grantErr) + got, err := svc.ReparseKnowledge(ctx, knowledge.ID, nil) require.Error(t, err) - require.Same(t, knowledge, got) - require.Equal(t, types.ParseStatusFailed, knowledge.ParseStatus) - require.Equal(t, "disabled", knowledge.EnableStatus) - require.Equal(t, "Failed to enqueue processing task", knowledge.ErrorMessage) + require.NotNil(t, got) + require.Equal(t, types.ParseStatusFailed, got.ParseStatus) + require.Equal(t, "disabled", got.EnableStatus) + require.Equal(t, "Failed to enqueue processing task", got.ErrorMessage) require.GreaterOrEqual(t, repo.updateCalls, 2, "pending and failed states must both be persisted") } diff --git a/internal/application/service/knowledge_caller_scope_test.go b/internal/application/service/knowledge_caller_scope_test.go new file mode 100644 index 00000000..af680ad8 --- /dev/null +++ b/internal/application/service/knowledge_caller_scope_test.go @@ -0,0 +1,186 @@ +package service + +import ( + "context" + "testing" + + "github.com/Tencent/WeKnora/internal/application/access" + "github.com/Tencent/WeKnora/internal/application/repository" + apperrors "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/stretchr/testify/require" +) + +type callerScopeShares struct { + interfaces.KBShareService + callers []uint64 +} + +func (s *callerScopeShares) CheckTenantKBPermission( + _ context.Context, + kbID string, + caller uint64, + _ types.TenantRole, +) (types.OrgMemberRole, bool, error) { + s.callers = append(s.callers, caller) + // Tenant 2 has access to a third tenant's KB; caller 1 does not. + return types.OrgRoleViewer, kbID == "onward" && caller == 2, nil +} + +func TestSharedExecutionFiltersDocumentsChunksAndSearchScope(t *testing.T) { + shares := &callerScopeShares{} + svc, db := newKnowledgeSharedAccessService(t, shares) + require.NoError(t, db.AutoMigrate(&types.Chunk{})) + kbs := []*types.KnowledgeBase{ + {ID: "own", TenantID: 1}, + {ID: "granted", TenantID: 2}, + {ID: "private", TenantID: 2}, + {ID: "onward", TenantID: 3}, + } + var ids []string + for _, kb := range kbs { + ids = append(ids, kb.ID) + seedKnowledge(t, db, &types.Knowledge{ID: kb.ID, TenantID: kb.TenantID, KnowledgeBaseID: kb.ID, Type: "file"}) + require.NoError( + t, + db.Create( + &types.Chunk{ + ID: kb.ID, + TenantID: kb.TenantID, + KnowledgeBaseID: kb.ID, + KnowledgeID: kb.ID, + Content: kb.ID, + }, + ).Error, + ) + } + search := &knowledgeBaseService{ + kgRepo: svc.repo, + chunkRepo: repository.NewChunkRepository(db), + kbShareService: shares, + } + kbService := &suggestionKBService{kbs: make(map[string]*types.KnowledgeBase)} + for _, kb := range kbs { + kbService.kbs[kb.ID] = kb + } + session := &sessionService{knowledgeBaseService: kbService, knowledgeService: svc, kbShareService: shares} + base := newSharedAccessContext() + grant := &access.KBAccess{ + KnowledgeBase: kbs[1], + Caller: types.CallerFromContext(base), + EffectiveTenantID: 2, + Permission: types.OrgRoleViewer, + } + for _, execution := range []uint64{1, 2, 3} { + ctx := logger.CloneContext(types.WithExecutionTenant(grant.Context(base), execution)) + rows, err := svc.GetKnowledgeBatchWithSharedAccess(ctx, execution, ids) + require.NoError(t, err) + got := make([]string, 0, len(rows)) + for _, row := range rows { + got = append(got, row.ID) + } + require.ElementsMatch(t, []string{"own", "granted"}, got) + knowledgeMap, err := search.fetchKnowledgeDataWithShared(ctx, execution, ids) + require.NoError(t, err) + require.Len(t, knowledgeMap, 2) + require.NotNil(t, knowledgeMap["own"]) + require.NotNil(t, knowledgeMap["granted"]) + chunks, err := search.listChunksByIDWithShared(ctx, execution, ids) + require.NoError(t, err) + got = got[:0] + for _, chunk := range chunks { + got = append(got, chunk.ID) + } + require.ElementsMatch(t, []string{"own", "granted"}, got) + require.NoError(t, search.authorizeKBAccess(ctx, kbs[:2])) + require.Error(t, search.authorizeKBAccess(ctx, kbs[2:3]), "execution tenant is not ownership") + require.Error(t, search.authorizeKBAccess(ctx, kbs[3:]), "source tenant shares are not inherited") + targets, err := session.buildSearchTargets( + ctx, + execution, + ids, + nil, + []types.TagScope{{KnowledgeBaseID: "private", TagIDs: []string{"tag"}}}, + ) + require.NoError(t, err) + require.Len(t, targets, 2, "denied KBs must not become search or tag targets") + got = got[:0] + for _, target := range targets { + got = append(got, target.KnowledgeBaseID) + } + require.ElementsMatch(t, []string{"own", "granted"}, got) + _, err = resolveKBReadTenant(ctx, kbs[1], shares) + require.NoError(t, err, "FAQ/tag reads accept the exact upstream grant") + _, err = resolveKBReadTenant(ctx, kbs[2], shares) + require.Error(t, err, "FAQ/tag reads cannot access another KB in the execution tenant") + } + for _, caller := range shares.callers { + require.Equal(t, uint64(1), caller) + } +} + +type callerScopeKBList struct { + interfaces.KnowledgeBaseService + tenant uint64 +} + +func (s *callerScopeKBList) ListKnowledgeBases(ctx context.Context) ([]*types.KnowledgeBase, error) { + s.tenant = types.MustTenantIDFromContext(ctx) + return []*types.KnowledgeBase{{ID: "own", TenantID: s.tenant, Type: types.KnowledgeBaseTypeDocument}}, nil +} + +type callerScopeSearchRepo struct { + interfaces.KnowledgeRepository + scopes []types.KnowledgeSearchScope +} + +func (r *callerScopeSearchRepo) SearchKnowledgeInScopes( + _ context.Context, + scopes []types.KnowledgeSearchScope, + _ string, + _, _ int, + _ []string, +) ([]*types.Knowledge, bool, int64, error) { + r.scopes = scopes + return nil, false, 0, nil +} + +func TestDocumentSearchListsCallerTenantAfterExecutionSwitch(t *testing.T) { + kbs := &callerScopeKBList{} + repo := &callerScopeSearchRepo{} + svc := &knowledgeService{kbService: kbs, repo: repo} + ctx := types.WithExecutionTenant(newSharedAccessContext(), 2) + _, _, _, err := svc.SearchKnowledge(ctx, "query", 0, 10, nil) + require.NoError(t, err) + require.Equal(t, uint64(1), kbs.tenant) + require.Equal(t, []types.KnowledgeSearchScope{{TenantID: 1, KBID: "own"}}, repo.scopes) + require.Equal(t, uint64(2), types.MustTenantIDFromContext(ctx), "search must not mutate the parent context") +} + +func TestSharedAgentBatchUsesGrantInsteadOfSourceTenantOwnership(t *testing.T) { + svc, db := newKnowledgeSharedAccessService(t, &callerScopeShares{}) + for _, kbID := range []string{"selected", "private"} { + seedKnowledge(t, db, &types.Knowledge{ID: kbID, TenantID: 2, KnowledgeBaseID: kbID, Type: "file"}) + } + agent := &types.CustomAgent{ + TenantID: 2, + Config: types.CustomAgentConfig{KBSelectionMode: "selected", KnowledgeBases: []string{"selected"}}, + } + ctx := logger.CloneContext(access.WithSharedAgent(newSharedAccessContext(), agent)) + rows, err := svc.GetKnowledgeBatchWithSharedAccess(ctx, 2, []string{"selected", "private"}) + require.NoError(t, err) + require.Len(t, rows, 1) + require.Equal(t, "selected", rows[0].ID) + ctx = types.WithTenantAPIKeyScope(ctx, types.TenantAPIKeyScope{KnowledgeBaseIDs: types.StringArray{"private"}}) + search := &knowledgeBaseService{} + app, ok := apperrors.IsAppError( + search.authorizeKBAccess(ctx, []*types.KnowledgeBase{{ID: "selected", TenantID: 2}}), + ) + require.True(t, ok) + require.Equal(t, apperrors.ErrForbidden, app.Code, "API-key scope denial must not become a lookup failure") + rows, err = svc.GetKnowledgeBatchWithSharedAccess(ctx, 2, []string{"selected", "private"}) + require.NoError(t, err) + require.Empty(t, rows) +} diff --git a/internal/application/service/knowledge_cleanup_regression_test.go b/internal/application/service/knowledge_cleanup_regression_test.go new file mode 100644 index 00000000..035e7534 --- /dev/null +++ b/internal/application/service/knowledge_cleanup_regression_test.go @@ -0,0 +1,133 @@ +package service + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "github.com/Tencent/WeKnora/internal/application/access" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/hibiken/asynq" + "github.com/stretchr/testify/require" +) + +func TestDeleteTaskMovingConflictStopsRetries(t *testing.T) { + f := transferFixture(t, access.KBTransferMove) + row, err := f.repo.GetKnowledgeByID(f.ctx, 7, "doc") + require.NoError(t, err) + require.NoError(t, setTransferState(row, knowledgeTransferState{Operation: access.KBTransferMove, Phase: "moving"})) + require.NoError(t, f.repo.UpdateKnowledge(f.ctx, row)) + beforeWrites := f.repo.writes + raw, err := json.Marshal( + types.KnowledgeListDeletePayload{TenantID: 7, KnowledgeBaseID: "kb", KnowledgeIDs: []string{"doc"}}, + ) + require.NoError(t, err) + err = f.svc.ProcessKnowledgeListDelete(context.Background(), asynq.NewTask(types.TypeKnowledgeListDelete, raw)) + require.ErrorIs(t, err, asynq.SkipRetry) + require.ErrorContains(t, err, "unfinished move") + require.Equal(t, beforeWrites, f.repo.writes) + require.Zero(t, f.chunkRepo.writes) + require.Zero(t, f.graph.calls) +} + +func TestDeleteWikiRemovesEveryChunkType(t *testing.T) { + f := newDocumentWriteFixture(t) + f.kbs.values["kb"].IndexingStrategy.WikiEnabled = true + refs := types.StringArray{"chunk"} + chunkTypes := []string{ + types.ChunkTypeParentText, types.ChunkTypeImageOCR, types.ChunkTypeImageCaption, types.ChunkTypeSummary, + } + for _, typ := range chunkTypes { + require.NoError( + t, + f.db.Create( + &types.Chunk{ID: typ, TenantID: 7, KnowledgeID: "doc", KnowledgeBaseID: "kb", ChunkType: typ}, + ).Error, + ) + refs = append(refs, typ) + } + refs = append(refs, "other-chunk") + page := &types.WikiPage{ + ID: "page", + Slug: "concept/shared", + SourceRefs: types.StringArray{"doc", "other-doc"}, + ChunkRefs: refs, + } + wiki := &moveWikiPageRepo{pages: []*types.WikiPage{page}} + meta := &moveWikiMetaService{} + f.svc.wikiRepo = wiki + f.svc.wikiService = meta + f.svc.taskPendingRepo = &moveWikiPendingRepo{} + f.svc.task = &transferQueue{} + require.NoError(t, f.svc.DeleteKnowledge(f.ctx, "doc")) + require.Len(t, meta.updated, 1) + require.Equal(t, types.StringArray{"other-chunk"}, meta.updated[0].ChunkRefs) + require.Equal(t, types.StringArray{"other-doc"}, meta.updated[0].SourceRefs) +} + +type cleanupKBFailure struct { + interfaces.KnowledgeBaseService + value *types.KnowledgeBase + err error +} + +func (s *cleanupKBFailure) GetKnowledgeBaseByID(context.Context, string) (*types.KnowledgeBase, error) { + return s.value, s.err +} + +func TestCleanupStopsBeforeSideEffectsWhenKBUnavailable(t *testing.T) { + for _, scenario := range []string{"storage error", "nil KB", "wrong KB", "wrong tenant"} { + t.Run(scenario, func(t *testing.T) { + f := newDocumentWriteFixture(t) + lookup := &cleanupKBFailure{} + switch scenario { + case "storage error": + lookup.err = errors.New("database unavailable") + case "wrong KB": + lookup.value = &types.KnowledgeBase{ID: "other", TenantID: 7} + case "wrong tenant": + lookup.value = &types.KnowledgeBase{ID: "kb", TenantID: 8} + } + f.svc.kbService = lookup + row, err := f.repo.GetKnowledgeByID(f.ctx, 7, "doc") + require.NoError(t, err) + row.EmbeddingModelID = "bound-model" + // Engine/model dependencies remain absent: touching a fallback backend + // would panic instead of preserving the lookup error and original data. + err = f.svc.cleanupKnowledgeResources(f.ctx, row) + require.Error(t, err) + if lookup.err != nil { + require.ErrorIs(t, err, lookup.err) + } + require.Zero(t, f.chunkRepo.writes) + require.Zero(t, f.graph.calls) + require.Empty(t, f.files.deleted) + require.Empty(t, f.tenants.adjustments) + }) + } +} + +func TestMoveReparseKBLookupFailurePreservesSourceCheckpoint(t *testing.T) { + f := transferFixture(t, access.KBTransferMove) + row, err := f.repo.GetKnowledgeByID(f.ctx, 7, "doc") + require.NoError(t, err) + require.NoError( + t, + row.SetManualMetadata(types.NewManualKnowledgeMetadata("content", types.ManualKnowledgeStatusPublish, 1)), + ) + require.NoError(t, f.repo.UpdateKnowledge(f.ctx, row)) + failure := errors.New("source KB load failed") + f.svc.kbService = &cleanupKBFailure{err: failure} + err = f.svc.moveOneKnowledge(f.ctx, "doc", f.kbs.values["kb"], f.kbs.values["other"], "reparse") + require.ErrorIs(t, err, failure) + row, err = f.repo.GetKnowledgeByID(f.ctx, 7, "doc") + require.NoError(t, err) + require.Equal(t, "kb", row.KnowledgeBaseID) + state, err := transferState(row) + require.NoError(t, err) + require.Equal(t, "moving", state.Phase) + require.Zero(t, f.chunkRepo.writes) + require.Zero(t, f.graph.calls) +} diff --git a/internal/application/service/knowledge_clone_move.go b/internal/application/service/knowledge_clone_move.go index 1742c534..3add2513 100644 --- a/internal/application/service/knowledge_clone_move.go +++ b/internal/application/service/knowledge_clone_move.go @@ -12,6 +12,7 @@ import ( "strings" "time" + "github.com/Tencent/WeKnora/internal/application/access" "github.com/Tencent/WeKnora/internal/application/service/retriever" werrors "github.com/Tencent/WeKnora/internal/errors" "github.com/Tencent/WeKnora/internal/logger" @@ -21,7 +22,6 @@ import ( "github.com/google/uuid" "github.com/hibiken/asynq" "github.com/redis/go-redis/v9" - "golang.org/x/sync/errgroup" ) // copyOwnedObject copies srcPath into a NEW object owned by the destination @@ -202,67 +202,15 @@ func cleanupCopiedObjects(ctx context.Context, svc interfaces.FileService, paths } func (s *knowledgeService) CloneKnowledgeBase(ctx context.Context, srcID, dstID string) error { - srcKB, dstKB, err := s.kbService.CopyKnowledgeBase(ctx, srcID, dstID) + source, target, err := s.kbService.CopyKnowledgeBase(ctx, srcID, dstID) if err != nil { - logger.Errorf(ctx, "Failed to copy knowledge base: %v", err) return err } - - addKnowledge, err := s.repo.AminusB(ctx, srcKB.TenantID, srcKB.ID, dstKB.TenantID, dstKB.ID) - if err != nil { - logger.Errorf(ctx, "Failed to get knowledge: %v", err) - return err + if source.Type == types.KnowledgeBaseTypeFAQ { + p := &types.KBCloneProgress{TaskID: access.TransferTaskID(ctx), SourceID: source.ID, TargetID: target.ID} + return s.cloneFAQKnowledgeBase(ctx, source, target, p, func(*types.KBCloneProgress, error, string) {}) } - - delKnowledge, err := s.repo.AminusB(ctx, dstKB.TenantID, dstKB.ID, srcKB.TenantID, srcKB.ID) - if err != nil { - logger.Errorf(ctx, "Failed to get knowledge: %v", err) - return err - } - logger.Infof(ctx, "Knowledge after update to add: %d, delete: %d", len(addKnowledge), len(delKnowledge)) - - batch := 10 - g, gctx := errgroup.WithContext(ctx) - for ids := range slices.Chunk(delKnowledge, batch) { - g.Go(func() error { - err := s.DeleteKnowledgeList(gctx, ids) - if err != nil { - logger.Errorf(gctx, "delete partial knowledge %v: %v", ids, err) - return err - } - return nil - }) - } - err = g.Wait() - if err != nil { - logger.Errorf(ctx, "delete total knowledge %d: %v", len(delKnowledge), err) - return err - } - - // Copy context out of auto-stop task - g, gctx = errgroup.WithContext(ctx) - g.SetLimit(batch) - for _, knowledge := range addKnowledge { - g.Go(func() error { - srcKn, err := s.repo.GetKnowledgeByID(gctx, srcKB.TenantID, knowledge) - if err != nil { - logger.Errorf(gctx, "get knowledge %s: %v", knowledge, err) - return err - } - err = s.cloneKnowledge(gctx, srcKn, dstKB) - if err != nil { - logger.Errorf(gctx, "clone knowledge %s: %v", knowledge, err) - return err - } - return nil - }) - } - err = g.Wait() - if err != nil { - logger.Errorf(ctx, "add total knowledge %d: %v", len(addKnowledge), err) - return err - } - return nil + return s.executeKnowledgeClone(ctx, source, target, nil) } // CloneChunk clone chunks from one knowledge to another @@ -281,15 +229,33 @@ func (s *knowledgeService) CloneKnowledgeBase(ctx context.Context, srcID, dstID // It also ensures that the chunk's relationships (like pre and next chunk IDs) are maintained // by mapping the source chunk IDs to the new target chunk IDs. func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowledge) (err error) { - chunkPage := 1 + sourceKB, err := knowledgeWriteKB(ctx, s.kbService, src) + if err != nil { + return err + } + targetKB, err := knowledgeWriteKB(ctx, s.kbService, dst) + if err != nil { + return err + } + if err := access.RequireKBTransfer(ctx, sourceKB, targetKB, access.KBTransferClone); err != nil { + return err + } + stored, err := s.repo.GetKnowledgeByID(ctx, dst.TenantID, dst.ID) + if err != nil { + return err + } + if stored == nil || stored.ID != dst.ID || stored.TenantID != dst.TenantID || + stored.KnowledgeBaseID != targetKB.ID { + return access.ErrForbidden + } + sourceChunks, err := s.transferChunks(ctx, src, sourceKB.ID) + if err != nil { + return err + } chunkPageSize := 100 srcTodst := map[string]string{} tagIDMapping := map[string]string{} // srcTagID -> dstTagID targetChunks := make([]*types.Chunk, 0, 10) - chunkType := []types.ChunkType{ - types.ChunkTypeText, types.ChunkTypeParentText, types.ChunkTypeSummary, - types.ChunkTypeImageCaption, types.ChunkTypeImageOCR, - } // Resolve the destination FileService so extracted images can be copied // into objects owned by the destination knowledge. urlCache dedups identical @@ -302,89 +268,96 @@ func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowl dstSvc := s.resolveFileService(ctx, dstKB) urlCache := map[string]string{} var copiedURLs []string + chunksAttempted := false + var rollbackIndices func() error defer func() { - if err != nil { - cleanupCopiedObjects(ctx, dstSvc, copiedURLs) + if err == nil { + return } - }() - - for { - sourceChunks, _, err := s.chunkRepo.ListPagedChunksByKnowledgeID(ctx, - src.TenantID, - src.ID, - &types.Pagination{ - Page: chunkPage, - PageSize: chunkPageSize, - }, - chunkType, - nil, - "", - "", - "", - "", - nil, - ) - chunkPage++ - if err != nil { - return err - } - if len(sourceChunks) == 0 { - break - } - now := time.Now() - for _, sourceChunk := range sourceChunks { - // Map TagID to target knowledge base - targetTagID := "" - if sourceChunk.TagID != "" { - if mappedTagID, ok := tagIDMapping[sourceChunk.TagID]; ok { - targetTagID = mappedTagID - } else { - // Try to find or create the tag in target knowledge base - targetTagID = s.getOrCreateTagInTarget(ctx, src.TenantID, dst.TenantID, dst.KnowledgeBaseID, sourceChunk.TagID, tagIDMapping) + if chunksAttempted { + // A failed create may have committed before losing its acknowledgement. + // Claim the original destination before removing any persisted data. + before, after := *stored, *stored + after.ParseStatus = types.ParseStatusFailed + after.ErrorMessage = err.Error() + if checkpointErr := s.repo.UpdateKnowledgeForTransfer(ctx, &before, &after); checkpointErr != nil { + err = errors.Join(err, fmt.Errorf("claim failed clone cleanup: %w", checkpointErr)) + return + } + if rollbackIndices != nil { + if cleanupErr := rollbackIndices(); cleanupErr != nil { + err = errors.Join(err, fmt.Errorf("clean failed clone indices: %w", cleanupErr)) + return } } - - // Deep-copy extracted images into objects owned by the destination - // knowledge so deleting the source never breaks this clone. Content - // URL rewriting happens in a final pass below, once urlCache holds - // the complete old->new mapping (image objects live in independent - // child chunks, so a parent text chunk's ![](url) reference cannot be - // rewritten until its child image chunk has been processed). - newImageInfo, copied, copyErr := cloneChunkImageInfo( - ctx, dstSvc, sourceChunk.ImageInfo, dst.TenantID, dst.ID, urlCache) - if copyErr != nil { - err = fmt.Errorf("clone chunk image copy failed: %w", copyErr) - return err + if cleanupErr := s.chunkRepo.DeleteChunksByKnowledgeID(ctx, dst.TenantID, dst.ID); cleanupErr != nil { + err = errors.Join(err, fmt.Errorf("clean failed clone chunks: %w", cleanupErr)) + return } - copiedURLs = append(copiedURLs, copied...) - - targetChunk := &types.Chunk{ - ID: uuid.New().String(), - TenantID: dst.TenantID, - KnowledgeID: dst.ID, - KnowledgeBaseID: dst.KnowledgeBaseID, - TagID: targetTagID, - Content: sourceChunk.Content, - ChunkIndex: sourceChunk.ChunkIndex, - IsEnabled: sourceChunk.IsEnabled, - Flags: sourceChunk.Flags, - Status: sourceChunk.Status, - StartAt: sourceChunk.StartAt, - EndAt: sourceChunk.EndAt, - PreChunkID: sourceChunk.PreChunkID, - NextChunkID: sourceChunk.NextChunkID, - ChunkType: sourceChunk.ChunkType, - ParentChunkID: sourceChunk.ParentChunkID, - Metadata: sourceChunk.Metadata, - ContentHash: sourceChunk.ContentHash, - ImageInfo: newImageInfo, - CreatedAt: now, - UpdatedAt: now, - } - targetChunks = append(targetChunks, targetChunk) - srcTodst[sourceChunk.ID] = targetChunk.ID } + cleanupCopiedObjects(ctx, dstSvc, copiedURLs) + }() + + now := time.Now() + for _, sourceChunk := range sourceChunks { + // Map TagID to target knowledge base + targetTagID := "" + if sourceChunk.TagID != "" { + if mappedTagID, ok := tagIDMapping[sourceChunk.TagID]; ok { + targetTagID = mappedTagID + } else { + // Try to find or create the tag in target knowledge base + targetTagID = s.getOrCreateTagInTarget(ctx, + src.TenantID, + dst.TenantID, + dst.KnowledgeBaseID, + sourceChunk.TagID, + tagIDMapping) + } + } + + // Deep-copy extracted images into objects owned by the destination + // knowledge so deleting the source never breaks this clone. Content + // URL rewriting happens in a final pass below, once urlCache holds + // the complete old->new mapping (image objects live in independent + // child chunks, so a parent text chunk's ![](url) reference cannot be + // rewritten until its child image chunk has been processed). + newImageInfo, copied, copyErr := cloneChunkImageInfo( + ctx, dstSvc, sourceChunk.ImageInfo, dst.TenantID, dst.ID, urlCache) + copiedURLs = append(copiedURLs, copied...) + if copyErr != nil { + err = fmt.Errorf("clone chunk image copy failed: %w", copyErr) + return err + } + + targetChunk := &types.Chunk{ + ID: uuid.New().String(), + TenantID: dst.TenantID, + KnowledgeID: dst.ID, + KnowledgeBaseID: dst.KnowledgeBaseID, + TagID: targetTagID, + Content: sourceChunk.Content, + ChunkIndex: sourceChunk.ChunkIndex, + IsEnabled: sourceChunk.IsEnabled, + Flags: sourceChunk.Flags, + Status: sourceChunk.Status, + IndexStatus: sourceChunk.IndexStatus, + StartAt: sourceChunk.StartAt, + EndAt: sourceChunk.EndAt, + PreChunkID: sourceChunk.PreChunkID, + NextChunkID: sourceChunk.NextChunkID, + ChunkType: sourceChunk.ChunkType, + ParentChunkID: sourceChunk.ParentChunkID, + Metadata: sourceChunk.Metadata, + ContentHash: sourceChunk.ContentHash, + ImageInfo: newImageInfo, + CreatedAt: now, + UpdatedAt: now, + } + targetChunks = append(targetChunks, targetChunk) + srcTodst[sourceChunk.ID] = targetChunk.ID } + for _, targetChunk := range targetChunks { // Rewrite in-content Markdown image URLs now that urlCache holds the // complete old->new mapping across all chunks. This fixes parent text @@ -408,6 +381,7 @@ func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowl } } for chunks := range slices.Chunk(targetChunks, chunkPageSize) { + chunksAttempted = true err := s.chunkRepo.CreateChunks(ctx, chunks) if err != nil { return err @@ -420,10 +394,10 @@ func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowl // VectorStore backends are not bit-compatible, so callers that allow // source/target KBs to bind to different stores must perform their own // cross-store migration before invoking this. - var sourceStoreID *string - if srcKB, loadErr := s.kbService.GetKnowledgeBaseByID(ctx, src.KnowledgeBaseID); loadErr == nil && srcKB != nil { - sourceStoreID = srcKB.VectorStoreID + if len(srcTodst) == 0 || dst.EmbeddingModelID == "" { + return nil } + sourceStoreID := sourceKB.VectorStoreID retrieveEngine, err := retriever.CreateRetrieveEngineForKB( ctx, s.retrieveEngine, s.ownership, tenantID, sourceStoreID) if err != nil { @@ -433,6 +407,9 @@ func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowl if err != nil { return err } + rollbackIndices = func() error { + return retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{dst.ID}, embeddingModel.GetDimensions(), dst.Type) + } if err := retrieveEngine.CopyIndices(ctx, src.KnowledgeBaseID, dst.KnowledgeBaseID, map[string]string{src.ID: dst.ID}, srcTodst, @@ -463,8 +440,47 @@ func (s *knowledgeService) ProcessKBClone(ctx context.Context, t *asynq.Task) er ctx = payload.Initiator.Apply(ctx) ctx = withKBActivityTask(ctx, payload.TaskID, kbActivityTrigger(ctx)) - // Add tenant ID to context - ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID) + if payload.TenantID == 0 || payload.SourceID == "" || payload.TaskID == "" { + return fmt.Errorf("invalid clone task: %w", asynq.SkipRetry) + } + ctx = types.WithExecutionTenant(ctx, payload.TenantID) + source, err := s.kbService.GetKnowledgeBaseByID(ctx, payload.SourceID) + if err != nil { + return err + } + create := payload.CreateTarget || payload.TargetID == "" + if payload.TargetID == "" { + // Compatibility for tasks admitted before destination IDs were reserved at + // enqueue time. The same legacy task always resolves the same target. + payload.TargetID = uuid.NewSHA1(uuid.NameSpaceOID, + []byte(fmt.Sprintf("kb-clone:%d:%s:%s", + payload.TenantID, + payload.SourceID, + payload.TaskID))). + String() + } + target := &types.KnowledgeBase{ID: payload.TargetID, TenantID: payload.TenantID, CreatorID: payload.CreatorID} + if !create { + target, err = s.kbService.GetKnowledgeBaseByID(ctx, payload.TargetID) + if err != nil { + return err + } + } + if source == nil || source.ID != payload.SourceID || target == nil || target.ID != payload.TargetID { + return fmt.Errorf("invalid clone binding: %w", asynq.SkipRetry) + } + ctx, err = access.WithKBTransferTask( + ctx, + source, + target, + payload.TenantID, + access.KBTransferClone, + payload.TaskID, + create, + ) + if err != nil { + return fmt.Errorf("invalid clone scope: %v: %w", err, asynq.SkipRetry) + } // Get tenant info and add to context tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID) @@ -472,8 +488,36 @@ func (s *knowledgeService) ProcessKBClone(ctx context.Context, t *asynq.Task) er logger.Errorf(ctx, "Failed to get tenant info: %v", err) return fmt.Errorf("failed to get tenant info: %w", err) } + if tenantInfo == nil || tenantInfo.ID != payload.TenantID { + return fmt.Errorf("invalid clone tenant: %w", asynq.SkipRetry) + } ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo) + // Get source and target knowledge bases + srcKB, dstKB, err := s.kbService.CopyKnowledgeBase(ctx, payload.SourceID, payload.TargetID) + if err != nil { + retry, maxRetry := 0, 0 + retry, _ = asynq.GetRetryCount(ctx) + maxRetry, _ = asynq.GetMaxRetry(ctx) + status := types.KBCloneStatusProcessing + if retry >= maxRetry { + status = types.KBCloneStatusFailed + } + _ = s.saveKBCloneProgress( + ctx, + &types.KBCloneProgress{ + TaskID: payload.TaskID, + SourceID: payload.SourceID, + TargetID: payload.TargetID, + Status: status, + Error: err.Error(), + Message: "Clone preflight failed", + UpdatedAt: time.Now().Unix(), + }, + ) + return err + } + // Check if this is the last retry retryCount, _ := asynq.GetRetryCount(ctx) maxRetry, _ := asynq.GetMaxRetry(ctx) @@ -510,13 +554,6 @@ func (s *knowledgeService) ProcessKBClone(ctx context.Context, t *asynq.Task) er logger.Errorf(ctx, "Failed to update KB clone progress: %v", err) } - // Get source and target knowledge bases - srcKB, dstKB, err := s.kbService.CopyKnowledgeBase(ctx, payload.SourceID, payload.TargetID) - if err != nil { - logger.Errorf(ctx, "Failed to copy knowledge base: %v", err) - handleError(progress, err, "Failed to copy knowledge base configuration") - return err - } if retryCount == 0 { recordKBActivity(ctx, s.audit, payload.TenantID, payload.TargetID, types.AuditActionKBCloneStarted, "knowledge_base", payload.TargetID, types.AuditOutcomeAccepted, @@ -534,93 +571,21 @@ func (s *knowledgeService) ProcessKBClone(ctx context.Context, t *asynq.Task) er return nil } - // Document type: use Knowledge-level diff based on file_hash - addKnowledge, err := s.repo.AminusB(ctx, srcKB.TenantID, srcKB.ID, dstKB.TenantID, dstKB.ID) + err = s.executeKnowledgeClone(ctx, srcKB, dstKB, func(done, total int) { + progress.Total = total + progress.Processed = done + if total > 0 { + progress.Progress = done * 100 / total + } + progress.Message = fmt.Sprintf("Processed %d/%d clone operations", done, total) + progress.UpdatedAt = time.Now().Unix() + _ = s.saveKBCloneProgress(ctx, progress) + }) if err != nil { - logger.Errorf(ctx, "Failed to get knowledge to add: %v", err) - handleError(progress, err, "Failed to calculate knowledge difference") - return err - } - - delKnowledge, err := s.repo.AminusB(ctx, dstKB.TenantID, dstKB.ID, srcKB.TenantID, srcKB.ID) - if err != nil { - logger.Errorf(ctx, "Failed to get knowledge to delete: %v", err) - handleError(progress, err, "Failed to calculate knowledge difference") - return err - } - - totalOperations := len(addKnowledge) + len(delKnowledge) - progress.Total = totalOperations - progress.Message = fmt.Sprintf("Found %d knowledge to add, %d to delete", len(addKnowledge), len(delKnowledge)) - progress.UpdatedAt = time.Now().Unix() - _ = s.saveKBCloneProgress(ctx, progress) - - logger.Infof(ctx, "Knowledge after update to add: %d, delete: %d", len(addKnowledge), len(delKnowledge)) - - processedCount := 0 - batch := 10 - - // Delete knowledge in target that doesn't exist in source - g, gctx := errgroup.WithContext(ctx) - for ids := range slices.Chunk(delKnowledge, batch) { - g.Go(func() error { - err := s.DeleteKnowledgeList(gctx, ids) - if err != nil { - logger.Errorf(gctx, "delete partial knowledge %v: %v", ids, err) - return err - } - return nil - }) - } - if err := g.Wait(); err != nil { - logger.Errorf(ctx, "delete total knowledge %d: %v", len(delKnowledge), err) - handleError(progress, err, "Failed to delete knowledge") - return err - } - - processedCount += len(delKnowledge) - if totalOperations > 0 { - progress.Progress = processedCount * 100 / totalOperations - } - progress.Processed = processedCount - progress.Message = fmt.Sprintf("Deleted %d knowledge, cloning %d...", len(delKnowledge), len(addKnowledge)) - progress.UpdatedAt = time.Now().Unix() - _ = s.saveKBCloneProgress(ctx, progress) - - // Clone knowledge from source to target - g, gctx = errgroup.WithContext(ctx) - g.SetLimit(batch) - for _, knowledge := range addKnowledge { - g.Go(func() error { - srcKn, err := s.repo.GetKnowledgeByID(gctx, srcKB.TenantID, knowledge) - if err != nil { - logger.Errorf(gctx, "get knowledge %s: %v", knowledge, err) - return err - } - err = s.cloneKnowledge(gctx, srcKn, dstKB) - if err != nil { - logger.Errorf(gctx, "clone knowledge %s: %v", knowledge, err) - return err - } - - // Update progress - processedCount++ - if totalOperations > 0 { - progress.Progress = processedCount * 100 / totalOperations - } - progress.Processed = processedCount - progress.Message = fmt.Sprintf("Cloned %d/%d knowledge", processedCount-len(delKnowledge), len(addKnowledge)) - progress.UpdatedAt = time.Now().Unix() - _ = s.saveKBCloneProgress(ctx, progress) - - return nil - }) - } - if err := g.Wait(); err != nil { - logger.Errorf(ctx, "add total knowledge %d: %v", len(addKnowledge), err) handleError(progress, err, "Failed to clone knowledge") return err } + totalOperations := progress.Total // Mark as completed progress.Status = types.KBCloneStatusCompleted @@ -646,6 +611,11 @@ func (s *knowledgeService) cloneFAQKnowledgeBase( progress *types.KBCloneProgress, handleError func(*types.KBCloneProgress, error, string), ) (retErr error) { + srcByID, dstByID, err := s.preflightFAQClone(ctx, srcKB, dstKB) + if err != nil { + return err + } + // Deep-copy extracted FAQ images into objects owned by the destination KB. // urlCache dedups identical source images across chunks; copiedURLs tracks // new objects for best-effort cleanup if the clone fails partway through. @@ -683,6 +653,33 @@ func (s *knowledgeService) cloneFAQKnowledgeBase( handleError(progress, err, "Failed to calculate FAQ chunk difference") return err } + // Validate the complete diff, including matched status/tag updates, before + // any tag creation, index deletion or chunk write. A stored-but-unindexed + // destination from a failed delivery must be replaced, not counted as done. + for _, id := range diff.ChunksToAdd { + if srcByID[id] == nil { + return fmt.Errorf("FAQ source diff escaped transfer scope") + } + } + for _, id := range diff.ChunksToDelete { + if dstByID[id] == nil { + return fmt.Errorf("FAQ target diff escaped transfer scope") + } + } + matched := make([]types.FAQChunkSyncPair, 0, len(diff.MatchedPairs)) + for _, pair := range diff.MatchedPairs { + src, dst := srcByID[pair.SrcChunkID], dstByID[pair.DstChunkID] + if src == nil || dst == nil { + return fmt.Errorf("FAQ matched diff escaped transfer scope") + } + if dst.Status != int(types.ChunkStatusIndexed) { + diff.ChunksToAdd = append(diff.ChunksToAdd, src.ID) + diff.ChunksToDelete = append(diff.ChunksToDelete, dst.ID) + } else { + matched = append(matched, pair) + } + } + diff.MatchedPairs = matched chunksToAdd := diff.ChunksToAdd chunksToDelete := diff.ChunksToDelete @@ -792,12 +789,9 @@ func (s *knowledgeService) cloneFAQKnowledgeBase( } batchIDs := chunksToAdd[i:end] - // Get source chunks - srcChunks, err := s.chunkRepo.ListChunksByID(ctx, srcKB.TenantID, batchIDs) - if err != nil { - logger.Errorf(ctx, "Failed to get source FAQ chunks: %v", err) - handleError(progress, err, "Failed to get source FAQ entries") - return err + srcChunks := make([]*types.Chunk, 0, len(batchIDs)) + for _, id := range batchIDs { + srcChunks = append(srcChunks, srcByID[id]) } // Create new chunks for destination @@ -849,6 +843,10 @@ func (s *knowledgeService) cloneFAQKnowledgeBase( return err } + // Saved rows now own these images, including when indexing later fails. + // A later batch failure must not delete files from an earlier saved batch. + copiedImageURLs = nil + // Index in vector store using existing method // This will index standard question + similar questions based on FAQConfig if err := s.indexFAQChunks(ctx, dstKB, dstKnowledge, newChunks, embeddingModel, false, false); err != nil { @@ -861,8 +859,8 @@ func (s *knowledgeService) cloneFAQKnowledgeBase( for _, chunk := range newChunks { chunk.Status = int(types.ChunkStatusIndexed) } - if err := s.chunkService.UpdateChunks(ctx, newChunks); err != nil { - logger.Warnf(ctx, "Failed to update FAQ chunks status: %v", err) + if err := s.chunkRepo.UpdateChunks(ctx, newChunks); err != nil { + return fmt.Errorf("failed to update FAQ chunks status: %w", err) // Don't fail the whole operation for status update failure } @@ -942,7 +940,17 @@ func (s *knowledgeService) getOrCreateFAQKnowledge(ctx context.Context, kb *type knowledge.Description = srcKnowledge.Description knowledge.Source = srcKnowledge.Source knowledge.Channel = srcKnowledge.Channel - knowledge.Metadata = srcKnowledge.Metadata + fields := map[string]json.RawMessage{} + if len(srcKnowledge.Metadata) > 0 { + if err := json.Unmarshal(srcKnowledge.Metadata, &fields); err != nil { + return nil, fmt.Errorf("decode source FAQ metadata: %w", err) + } + delete(fields, types.KnowledgeTransferMetadataKey) + knowledge.Metadata, err = json.Marshal(fields) + if err != nil { + return nil, err + } + } } if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil { @@ -963,7 +971,11 @@ func (s *knowledgeService) saveKBCloneProgress(ctx context.Context, progress *ty // SaveKBCloneProgress saves the KB clone progress to Redis (public method for handler use) func (s *knowledgeService) SaveKBCloneProgress(ctx context.Context, progress *types.KBCloneProgress) error { - return s.saveKBCloneProgress(ctx, progress) + data, err := json.Marshal(progress) + if err != nil { + return err + } + return s.redisClient.SetNX(ctx, getKBCloneProgressKey(progress.TaskID), data, kbCloneProgressTTL).Err() } // GetKBCloneProgress retrieves the progress of a knowledge base clone task @@ -1004,7 +1016,11 @@ func (s *knowledgeService) saveKnowledgeMoveProgress(ctx context.Context, progre // SaveKnowledgeMoveProgress saves the knowledge move progress to Redis (public method for handler use) func (s *knowledgeService) SaveKnowledgeMoveProgress(ctx context.Context, progress *types.KnowledgeMoveProgress) error { - return s.saveKnowledgeMoveProgress(ctx, progress) + data, err := json.Marshal(progress) + if err != nil { + return err + } + return s.redisClient.SetNX(ctx, getKnowledgeMoveProgressKey(progress.TaskID), data, knowledgeMoveProgressTTL).Err() } // GetKnowledgeMoveProgress retrieves the progress of a knowledge move task @@ -1029,135 +1045,140 @@ func (s *knowledgeService) GetKnowledgeMoveProgress(ctx context.Context, taskID func (s *knowledgeService) ProcessKnowledgeMove(ctx context.Context, t *asynq.Task) error { var payload types.KnowledgeMovePayload if err := json.Unmarshal(t.Payload(), &payload); err != nil { - return fmt.Errorf("failed to unmarshal knowledge move payload: %w", err) + return fmt.Errorf("invalid move payload: %v: %w", err, asynq.SkipRetry) + } + if payload.TenantID == 0 || payload.TaskID == "" || payload.SourceKBID == "" || payload.TargetKBID == "" { + return fmt.Errorf("invalid move scope: %w", asynq.SkipRetry) } ctx = payload.Initiator.Apply(ctx) ctx = withKBActivityTask(ctx, payload.TaskID, kbActivityTrigger(ctx)) - - // Add tenant ID to context - ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID) - - // Get tenant info and add to context - tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID) - if err != nil { - logger.Errorf(ctx, "ProcessKnowledgeMove: failed to get tenant info: %v", err) - return fmt.Errorf("failed to get tenant info: %w", err) - } - ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo) - - // Check if this is the last retry - retryCount, _ := asynq.GetRetryCount(ctx) - maxRetry, _ := asynq.GetMaxRetry(ctx) - isLastRetry := retryCount >= maxRetry - - logger.Infof(ctx, "ProcessKnowledgeMove: task=%s, source=%s, target=%s, mode=%s, count=%d, retry=%d/%d", - payload.TaskID, payload.SourceKBID, payload.TargetKBID, payload.Mode, len(payload.KnowledgeIDs), retryCount, maxRetry) - - // Helper function to handle errors - only mark as failed on last retry - handleError := func(progress *types.KnowledgeMoveProgress, err error, message string) { - if isLastRetry { - progress.Status = types.KBCloneStatusFailed - progress.Error = err.Error() - progress.Message = message - progress.UpdatedAt = time.Now().Unix() - _ = s.saveKnowledgeMoveProgress(ctx, progress) - for _, kbID := range []string{payload.SourceKBID, payload.TargetKBID} { - recordKBActivity(ctx, s.audit, payload.TenantID, kbID, types.AuditActionKnowledgeMoveFailed, - "knowledge_move", payload.TaskID, types.AuditOutcomeFailed, - map[string]any{"source_kb_id": payload.SourceKBID, "target_kb_id": payload.TargetKBID, - "task_id": payload.TaskID, "count": len(payload.KnowledgeIDs), "mode": payload.Mode}) - } - } - } - if retryCount == 0 { - for _, kbID := range []string{payload.SourceKBID, payload.TargetKBID} { - recordKBActivity(ctx, s.audit, payload.TenantID, kbID, types.AuditActionKnowledgeMoveStarted, - "knowledge_move", payload.TaskID, types.AuditOutcomeAccepted, - map[string]any{"source_kb_id": payload.SourceKBID, "target_kb_id": payload.TargetKBID, - "task_id": payload.TaskID, "count": len(payload.KnowledgeIDs), "mode": payload.Mode}) - } - } - - // Update progress to processing - progress := &types.KnowledgeMoveProgress{ - TaskID: payload.TaskID, - SourceKBID: payload.SourceKBID, - TargetKBID: payload.TargetKBID, - Status: types.KBCloneStatusProcessing, - Total: len(payload.KnowledgeIDs), - Progress: 0, - Message: "Starting knowledge move...", - UpdatedAt: time.Now().Unix(), - } - _ = s.saveKnowledgeMoveProgress(ctx, progress) - - // Get source and target knowledge bases + ctx = types.WithExecutionTenant(ctx, payload.TenantID) sourceKB, err := s.kbService.GetKnowledgeBaseByID(ctx, payload.SourceKBID) if err != nil { - handleError(progress, err, "Failed to get source knowledge base") return err } targetKB, err := s.kbService.GetKnowledgeBaseByID(ctx, payload.TargetKBID) if err != nil { - handleError(progress, err, "Failed to get target knowledge base") return err } - - // Validate compatibility - if sourceKB.Type != targetKB.Type { - err := fmt.Errorf("type mismatch: source=%s, target=%s", sourceKB.Type, targetKB.Type) - handleError(progress, err, "Source and target knowledge bases must be the same type") + if sourceKB == nil || sourceKB.ID != payload.SourceKBID || targetKB == nil || targetKB.ID != payload.TargetKBID { + return fmt.Errorf("invalid move binding: %w", asynq.SkipRetry) + } + ctx, err = access.WithKBTransferTask( + ctx, + sourceKB, + targetKB, + payload.TenantID, + access.KBTransferMove, + payload.TaskID, + false, + ) + if err != nil { + return fmt.Errorf("invalid move scope: %v: %w", err, asynq.SkipRetry) + } + tenant, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID) + if err != nil { return err } - if sourceKB.EmbeddingModelID != targetKB.EmbeddingModelID { - err := fmt.Errorf("embedding model mismatch: source=%s, target=%s", sourceKB.EmbeddingModelID, targetKB.EmbeddingModelID) - handleError(progress, err, "Source and target must use the same embedding model") + if tenant == nil || tenant.ID != payload.TenantID { + return fmt.Errorf("invalid move tenant: %w", asynq.SkipRetry) + } + ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant) + // No item is claimed and no failure callback can alter document state until + // every requested document and its children passed the same pair preflight. + retry, _ := asynq.GetRetryCount(ctx) + maxRetry, _ := asynq.GetMaxRetry(ctx) + items, err := s.planKnowledgeMove(ctx, sourceKB, targetKB, payload.KnowledgeIDs, payload.Mode) + if err != nil { + status := types.KBCloneStatusProcessing + if retry >= maxRetry { + status = types.KBCloneStatusFailed + } + _ = s.saveKnowledgeMoveProgress( + ctx, + &types.KnowledgeMoveProgress{ + TaskID: payload.TaskID, + SourceKBID: sourceKB.ID, + TargetKBID: targetKB.ID, + Status: status, + Error: err.Error(), + Message: "Move preflight failed", + UpdatedAt: time.Now().Unix(), + }, + ) return err } - - // Process each knowledge item - for i, knowledgeID := range payload.KnowledgeIDs { - err := s.moveOneKnowledge(ctx, knowledgeID, sourceKB, targetKB, payload.Mode) - if err != nil { - logger.Errorf(ctx, "ProcessKnowledgeMove: failed to move knowledge %s: %v", knowledgeID, err) + progress := &types.KnowledgeMoveProgress{ + TaskID: payload.TaskID, + SourceKBID: sourceKB.ID, + TargetKBID: targetKB.ID, + Status: types.KBCloneStatusProcessing, + Total: len(items), + UpdatedAt: time.Now().Unix(), + } + record := func(action types.AuditAction, outcome types.AuditOutcome) { + for _, kbID := range []string{sourceKB.ID, targetKB.ID} { + recordKBActivity( + ctx, + s.audit, + payload.TenantID, + kbID, + action, + "knowledge_move", + payload.TaskID, + outcome, + map[string]any{ + "source_kb_id": sourceKB.ID, + "target_kb_id": targetKB.ID, + "task_id": payload.TaskID, + "count": len(items), + "mode": payload.Mode, + }, + ) + } + } + if retry == 0 { + record(types.AuditActionKnowledgeMoveStarted, types.AuditOutcomeAccepted) + } + var failures error + for _, item := range items { + if err := s.moveOneKnowledge(ctx, item.ID, sourceKB, targetKB, payload.Mode); err != nil { + failures = errors.Join(failures, fmt.Errorf("knowledge %s: %w", item.ID, err)) + if retry >= maxRetry { + s.markMoveItemFailed(ctx, item.ID, sourceKB, targetKB, payload.Mode, err) + } progress.Failed++ } - progress.Processed = i + 1 - if progress.Total > 0 { - progress.Progress = progress.Processed * 100 / progress.Total - } - progress.Message = fmt.Sprintf("Moved %d/%d knowledge items", progress.Processed, progress.Total) + progress.Processed++ + progress.Progress = progress.Processed * 100 / progress.Total + progress.Message = fmt.Sprintf( + "Moved %d/%d knowledge items", + progress.Processed-progress.Failed, + progress.Total, + ) progress.UpdatedAt = time.Now().Unix() _ = s.saveKnowledgeMoveProgress(ctx, progress) } - - // Mark as completed - if progress.Failed > 0 && progress.Failed == progress.Total { - progress.Status = types.KBCloneStatusFailed - progress.Message = fmt.Sprintf("Knowledge move failed: all %d items failed", progress.Total) - } else { - progress.Status = types.KBCloneStatusCompleted - progress.Message = fmt.Sprintf("Knowledge move completed: %d/%d succeeded", progress.Processed-progress.Failed, progress.Total) + if failures != nil { + progress.Error = failures.Error() + if retry >= maxRetry { + progress.Status = types.KBCloneStatusFailed + outcome := types.AuditOutcomeFailed + if progress.Failed < progress.Total { + outcome = types.AuditOutcomePartial + } + record(types.AuditActionKnowledgeMoveFailed, outcome) + } + _ = s.saveKnowledgeMoveProgress(ctx, progress) + // Return an error for partial failures too. Completed items are recognized + // from their persisted task marker and skipped on the next delivery. + return failures } + progress.Status = types.KBCloneStatusCompleted progress.Progress = 100 - progress.UpdatedAt = time.Now().Unix() + progress.Error = "" _ = s.saveKnowledgeMoveProgress(ctx, progress) - - logger.Infof(ctx, "ProcessKnowledgeMove: task=%s completed, processed=%d, failed=%d", payload.TaskID, progress.Processed, progress.Failed) - outcome := types.AuditOutcomeSuccess - action := types.AuditActionKnowledgeMoveCompleted - if progress.Failed == progress.Total && progress.Total > 0 { - outcome = types.AuditOutcomeFailed - action = types.AuditActionKnowledgeMoveFailed - } else if progress.Failed > 0 { - outcome = types.AuditOutcomePartial - } - for _, kbID := range []string{payload.SourceKBID, payload.TargetKBID} { - recordKBActivity(ctx, s.audit, payload.TenantID, kbID, action, - "knowledge_move", payload.TaskID, outcome, - map[string]any{"source_kb_id": payload.SourceKBID, "target_kb_id": payload.TargetKBID, - "task_id": payload.TaskID, "count": progress.Total, "failed": progress.Failed, "mode": payload.Mode}) - } + record(types.AuditActionKnowledgeMoveCompleted, types.AuditOutcomeSuccess) return nil } @@ -1168,45 +1189,76 @@ func (s *knowledgeService) moveOneKnowledge( sourceKB, targetKB *types.KnowledgeBase, mode string, ) error { - tenantID := ctx.Value(types.TenantIDContextKey).(uint64) - - // Get the knowledge item + if err := access.RequireKBTransfer(ctx, sourceKB, targetKB, access.KBTransferMove); err != nil { + return err + } + tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) + if err := access.ValidateKBTransferCompatibility(sourceKB, + targetKB, + access.KBTransferMove, + mode, + tenant); err != nil { + return err + } + tenantID := sourceKB.TenantID knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID) if err != nil { - return fmt.Errorf("failed to get knowledge %s: %w", knowledgeID, err) + return err } - - // Only move completed items - if knowledge.ParseStatus != types.ParseStatusCompleted { - return fmt.Errorf("knowledge %s is not in completed status (current: %s)", knowledgeID, knowledge.ParseStatus) + if knowledge == nil || knowledge.ID != knowledgeID { + return access.ErrNotFound } - - // Reject a cross-store reuse_vectors move BEFORE mutating status, so a - // rejected move leaves the knowledge untouched (Completed) rather than - // stranded in Processing. reuse_vectors copies indices through the source - // store only; a cross-store copy would corrupt vector data. The handler - // rejects this synchronously — this is defense-in-depth for directly - // enqueued tasks. Cross-store moves must use reparse mode. - if mode == "reuse_vectors" && !sourceKB.SharesStoreWith(targetKB) { - return fmt.Errorf( - "reuse_vectors move across different vector stores is not supported "+ - "(source KB %s, target KB %s); use reparse mode", sourceKB.ID, targetKB.ID) + if err := validateMoveItem(ctx, knowledge, sourceKB, targetKB, mode); err != nil { + return err } - - // Mark as processing during move - knowledge.ParseStatus = types.ParseStatusProcessing - if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil { - return fmt.Errorf("failed to mark knowledge as processing: %w", err) + copyOfKnowledge := *knowledge + knowledge = ©OfKnowledge + state, err := transferState(knowledge) + if err != nil { + return err } - - // From the source KB's point of view the document is leaving for good, so it - // needs the same wiki reconciliation a delete performs: wiki_pages carry - // source_refs back to this knowledge and are what the folder tree and the - // wiki graph are built from, and nothing below touches them. This must run - // while KnowledgeBaseID still points at the source and before any chunk is - // removed, since the cleanup matches pages by chunk_refs. - if sourceKB.IsWikiEnabled() { - s.cleanupWikiOnKnowledgeDelete(ctx, knowledge) + if matchesTransfer(ctx, state, sourceKB, targetKB, access.KBTransferMove, knowledgeID, mode) { + if state.Phase == "done" || state.Phase == "reparse_pending" { + if err := s.cleanupMovedSourceWiki(ctx, knowledge, sourceKB, targetKB); err != nil { + return err + } + if state.Phase == "reparse_pending" { + return s.enqueueMovedKnowledge(ctx, knowledge, sourceKB, targetKB) + } + if mode == "reuse_vectors" && targetKB.IsWikiEnabled() { + _, err := EnqueueWikiIngest(ctx, s.task, s.taskPendingRepo, tenantID, targetKB.ID, knowledge.ID) + return err + } + return nil + } + } else { + state = &knowledgeTransferState{ + TaskID: access.TransferTaskID(ctx), + Operation: access.KBTransferMove, + SourceKB: sourceKB.ID, + TargetKB: targetKB.ID, + SourceID: knowledge.ID, + Mode: mode, + Phase: "moving", + } + if sourceKB.IsWikiEnabled() { + chunks, err := s.transferChunks(ctx, knowledge, sourceKB.ID) + if err != nil { + return err + } + for _, chunk := range chunks { + state.WikiChunkIDs = append(state.WikiChunkIDs, chunk.ID) + } + state.WikiSummary = knowledge.Description + } + before := *knowledge + knowledge.ParseStatus = types.ParseStatusProcessing + if err := setTransferState(knowledge, *state); err != nil { + return err + } + if err := s.repo.UpdateKnowledgeForTransfer(ctx, &before, knowledge); err != nil { + return err + } } switch mode { @@ -1214,11 +1266,15 @@ func (s *knowledgeService) moveOneKnowledge( if err := s.moveKnowledgeReuseVectors(ctx, knowledge, sourceKB, targetKB); err != nil { return err } + if err := s.cleanupMovedSourceWiki(ctx, knowledge, sourceKB, targetKB); err != nil { + return err + } // reparse re-ingests through KnowledgePostProcess once the new chunks // land; reuse_vectors keeps the existing chunks and never re-enters that // pipeline, so the target KB has to be told about the document here. if targetKB.IsWikiEnabled() { - EnqueueWikiIngest(ctx, s.task, s.taskPendingRepo, tenantID, targetKB.ID, knowledge.ID) + _, err := EnqueueWikiIngest(ctx, s.task, s.taskPendingRepo, tenantID, targetKB.ID, knowledge.ID) + return err } return nil case "reparse": @@ -1228,7 +1284,7 @@ func (s *knowledgeService) moveOneKnowledge( } } -// moveKnowledgeReuseVectors moves knowledge by copying vector indices and updating DB references. +// moveKnowledgeReuseVectors relocates vector metadata while retaining physical IDs. func (s *knowledgeService) moveKnowledgeReuseVectors( ctx context.Context, knowledge *types.Knowledge, @@ -1236,65 +1292,62 @@ func (s *knowledgeService) moveKnowledgeReuseVectors( ) error { tenantID := ctx.Value(types.TenantIDContextKey).(uint64) - // reuse_vectors copies index entries directly between KBs, which only works - // inside the same VectorStore backend (CopyIndices is routed through the - // source store). A cross-store reuse_vectors move would write target-KB rows - // into the source store and then delete the source indices, corrupting data. - // The MoveKnowledge handler rejects this up front; this is defense-in-depth - // for any path that enqueues a move task directly. Cross-store moves must use - // reparse mode (moveKnowledgeReparse), which re-indexes into the target store. + // Metadata relocation is supported only within the same vector store. + // Cross-store moves must reparse into the target store. if !sourceKB.SharesStoreWith(targetKB) { return fmt.Errorf( "reuse_vectors move across different vector stores is not supported "+ "(source KB %s, target KB %s); use reparse mode", sourceKB.ID, targetKB.ID) } - // 1. Get old chunk IDs for vector index copy mapping - oldChunks, err := s.chunkRepo.ListChunksByKnowledgeID(ctx, tenantID, knowledge.ID) + if err := access.RequireKBTransfer(ctx, sourceKB, targetKB, access.KBTransferMove); err != nil { + return err + } + oldChunks, err := s.transferChunks(ctx, knowledge, sourceKB.ID, targetKB.ID) if err != nil { - return fmt.Errorf("failed to list chunks: %w", err) + return err + } + chunkIDs := make([]string, 0, len(oldChunks)) + for _, chunk := range oldChunks { + chunkIDs = append(chunkIDs, chunk.ID) + } + if knowledge.EmbeddingModelID != "" { + engine, err := retriever.CreateRetrieveEngineForKB( + ctx, + s.retrieveEngine, + s.ownership, + tenantID, + sourceKB.VectorStoreID, + ) + if err != nil { + return err + } + model, err := s.modelService.GetEmbeddingModel(ctx, knowledge.EmbeddingModelID) + if err != nil { + return err + } + // Move index metadata in place. Copy + delete by the unchanged document ID + // would also delete the just-created target indices. + if err := engine.MoveKnowledgeIndices(ctx, + sourceKB.ID, + targetKB.ID, + knowledge.ID, + chunkIDs, + model.GetDimensions(), + sourceKB.Type); err != nil { + return err + } } - // Build identity mapping (same chunk IDs, just moving between KBs) - chunkIDMapping := make(map[string]string, len(oldChunks)) - for _, c := range oldChunks { - chunkIDMapping[c.ID] = c.ID - } - - // 2. Copy vector indices from source KB to target KB - if len(chunkIDMapping) > 0 && knowledge.EmbeddingModelID != "" { - // Same VectorStore backend is guaranteed by the SharesStoreWith guard at - // the top of this function, so routing CopyIndices through the source - // KB's binding also resolves the target's store. - var sourceStoreID *string - if sourceKB != nil { - sourceStoreID = sourceKB.VectorStoreID - } - retrieveEngine, err := retriever.CreateRetrieveEngineForKB( - ctx, s.retrieveEngine, s.ownership, tenantID, sourceStoreID) - if err != nil { - return fmt.Errorf("failed to init retrieve engine: %w", err) - } - embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, knowledge.EmbeddingModelID) - if err != nil { - return fmt.Errorf("failed to get embedding model: %w", err) - } - - // Copy indices from source KB to target KB - knowledgeIDMapping := map[string]string{knowledge.ID: knowledge.ID} - if err := retrieveEngine.CopyIndices(ctx, sourceKB.ID, targetKB.ID, - knowledgeIDMapping, chunkIDMapping, - embeddingModel.GetDimensions(), sourceKB.Type, - ); err != nil { - return fmt.Errorf("failed to copy indices: %w", err) - } - - // Delete indices from source KB - if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, - embeddingModel.GetDimensions(), sourceKB.Type, - ); err != nil { - logger.Warnf(ctx, "moveKnowledgeReuseVectors: failed to delete old indices for knowledge %s: %v", knowledge.ID, err) - // Non-fatal: indices will be orphaned but won't affect correctness + // Source graph namespaces must not keep exposing a document after it + // leaves the KB. The exact namespace deletion is repeatable on retry. + if s.graphEngine != nil { + if err := s.graphEngine.DelGraph(ctx, + []types.NameSpace{{ + KnowledgeBase: sourceKB.ID, + Knowledge: knowledge.ID, + }}); err != nil { + return err } } @@ -1307,11 +1360,20 @@ func (s *knowledgeService) moveKnowledgeReuseVectors( if err := s.repo.DeleteKnowledgeTagRelations(ctx, knowledge.ID); err != nil { return fmt.Errorf("failed to clear knowledge tag relations: %w", err) } + before := *knowledge knowledge.KnowledgeBaseID = targetKB.ID knowledge.ParseStatus = types.ParseStatusCompleted - knowledge.UpdatedAt = time.Now() - if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil { - return fmt.Errorf("failed to update knowledge: %w", err) + knowledge.ErrorMessage = "" + state, err := transferState(knowledge) + if err != nil || state == nil { + return fmt.Errorf("move state unavailable") + } + state.Phase = "done" + if err := setTransferState(knowledge, *state); err != nil { + return err + } + if err := s.repo.UpdateKnowledgeForTransfer(ctx, &before, knowledge); err != nil { + return err } return nil @@ -1321,39 +1383,81 @@ func (s *knowledgeService) moveKnowledgeReuseVectors( func (s *knowledgeService) moveKnowledgeReparse( ctx context.Context, knowledge *types.Knowledge, - _, targetKB *types.KnowledgeBase, + sourceKB, targetKB *types.KnowledgeBase, ) error { - tenantID := ctx.Value(types.TenantIDContextKey).(uint64) - - // 1. Clean up existing chunks and vector indices - if err := s.cleanupKnowledgeResources(ctx, knowledge); err != nil { - logger.Warnf(ctx, "moveKnowledgeReparse: cleanup partial error for knowledge %s: %v", knowledge.ID, err) - // Continue - partial cleanup is acceptable + if err := access.RequireKBTransfer(ctx, sourceKB, targetKB, access.KBTransferMove); err != nil { + return err + } + before := *knowledge + cleanup := *knowledge + cleanup.StorageSize = 0 + if err := s.cleanupKnowledgeResources(ctx, &cleanup); err != nil { + return fmt.Errorf("failed to clean up source: %w", err) } - - // 2. Update knowledge to belong to target KB (tags are KB-scoped; clear relations) if err := s.repo.DeleteKnowledgeTagRelations(ctx, knowledge.ID); err != nil { - return fmt.Errorf("failed to clear knowledge tag relations: %w", err) + return err } knowledge.KnowledgeBaseID = targetKB.ID knowledge.EmbeddingModelID = targetKB.EmbeddingModelID knowledge.ParseStatus = types.ParseStatusPending + knowledge.ErrorMessage = "" knowledge.EnableStatus = "disabled" knowledge.Description = "" knowledge.ProcessedAt = nil - knowledge.UpdatedAt = time.Now() - if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil { - return fmt.Errorf("failed to update knowledge: %w", err) + knowledge.StorageSize = 0 + state, err := transferState(knowledge) + if err != nil || state == nil { + return fmt.Errorf("move state unavailable") } + state.Phase = "reparse_pending" + if err := setTransferState(knowledge, *state); err != nil { + return err + } + if err := s.repo.UpdateKnowledgeForTransfer(ctx, &before, knowledge); err != nil { + return err + } + if err := s.cleanupMovedSourceWiki(ctx, knowledge, sourceKB, targetKB); err != nil { + return err + } + return s.enqueueMovedKnowledge(ctx, knowledge, sourceKB, targetKB) +} +func (s *knowledgeService) enqueueMovedKnowledge( + ctx context.Context, + knowledge *types.Knowledge, + sourceKB, targetKB *types.KnowledgeBase, +) error { + if err := access.RequireKBTransfer(ctx, sourceKB, targetKB, access.KBTransferMove); err != nil { + return err + } + if knowledge.ParseStatus == types.ParseStatusFailed { + before := *knowledge + knowledge.ParseStatus = types.ParseStatusPending + knowledge.ErrorMessage = "" + if err := s.repo.UpdateKnowledgeForTransfer(ctx, &before, knowledge); err != nil { + return err + } + } + tenantID := knowledge.TenantID + taskID := "move-reparse:" + access.TransferTaskID(ctx) + ":" + knowledge.ID // 3. Enqueue document processing task with target KB's configuration if knowledge.IsManual() { meta, err := knowledge.ManualMetadata() if err != nil || meta == nil { return fmt.Errorf("failed to get manual metadata for reparse: %w", err) } - s.triggerManualProcessing(ctx, targetKB, knowledge, meta.Content, false) - return nil + _, err = s.enqueueManualProcessing( + ctx, + knowledge, + meta.Content, + false, + asynq.TaskID(taskID), + asynq.Retention(7*24*time.Hour), + ) + if err != nil && !errors.Is(err, asynq.ErrTaskIDConflict) && !errors.Is(err, asynq.ErrDuplicateTask) { + return err + } + return s.acknowledgeMovedReparse(ctx, knowledge, sourceKB, targetKB) } if knowledge.FilePath != "" { @@ -1387,16 +1491,23 @@ func (s *knowledgeService) moveKnowledgeReparse( return fmt.Errorf("failed to marshal document process payload: %w", err) } - task := asynq.NewTask(types.TypeDocumentProcess, payloadBytes, - documentProcessTaskOptions(s.config, asynq.MaxRetry(3))...) + task := asynq.NewTask( + types.TypeDocumentProcess, + payloadBytes, + documentProcessTaskOptions( + s.config, + asynq.MaxRetry(3), + asynq.TaskID(taskID), + asynq.Retention(7*24*time.Hour), + )...) info, err := s.task.Enqueue(task) - if err != nil { + if err != nil && !errors.Is(err, asynq.ErrTaskIDConflict) && !errors.Is(err, asynq.ErrDuplicateTask) { return fmt.Errorf("failed to enqueue document process task: %w", err) } - logger.Infof(ctx, "moveKnowledgeReparse: enqueued reparse task id=%s for knowledge=%s", info.ID, knowledge.ID) + _ = info } - return nil + return s.acknowledgeMovedReparse(ctx, knowledge, sourceKB, targetKB) } // getOrCreateTagInTarget finds or creates a tag in the target knowledge base based on the source tag. diff --git a/internal/application/service/knowledge_create.go b/internal/application/service/knowledge_create.go index 410a2be8..d294f2c1 100644 --- a/internal/application/service/knowledge_create.go +++ b/internal/application/service/knowledge_create.go @@ -1017,21 +1017,18 @@ func (s *knowledgeService) UpdateManualKnowledge(ctx context.Context, return nil, werrors.NewValidationError("状态仅支持 draft 或 publish") } - tenantID := ctx.Value(types.TenantIDContextKey).(uint64) - existing, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID) + existing, kb, err := loadKnowledgeWrite(ctx, s.repo, s.kbService, knowledgeID) if err != nil { - logger.Errorf(ctx, "Failed to load knowledge: %v", err) return nil, err } if !existing.IsManual() { return nil, werrors.NewBadRequestError("仅支持手工知识的在线编辑") } - - kb, err := s.kbService.GetKnowledgeBaseByID(ctx, existing.KnowledgeBaseID) + ctx, err = withKBWriteTenantInfo(ctx, kb, s.tenantRepo) if err != nil { - logger.Errorf(ctx, "Failed to get knowledge base for manual update: %v", err) return nil, err } + tenantID := existing.TenantID var version int if meta, err := existing.ManualMetadata(); err == nil && meta != nil { @@ -1115,7 +1112,7 @@ func (s *knowledgeService) UpdateManualKnowledge(ctx context.Context, // enqueueManualProcessing enqueues a manual:process Asynq task for async cleanup + re-indexing. func (s *knowledgeService) enqueueManualProcessing(ctx context.Context, - knowledge *types.Knowledge, content string, needCleanup bool, + knowledge *types.Knowledge, content string, needCleanup bool, options ...asynq.Option, ) (string, error) { requestID, _ := types.RequestIDFromContext(ctx) payload := types.ManualProcessPayload{ @@ -1134,7 +1131,7 @@ func (s *knowledgeService) enqueueManualProcessing(ctx context.Context, task := asynq.NewTask(types.TypeManualProcess, payloadBytes, asynq.Queue(types.QueueDefault), asynq.MaxRetry(3), asynq.Timeout(30*time.Minute)) - info, err := s.task.Enqueue(task) + info, err := s.task.Enqueue(task, options...) if err != nil { return "", fmt.Errorf("failed to enqueue manual process task: %w", err) } diff --git a/internal/application/service/knowledge_delete.go b/internal/application/service/knowledge_delete.go index f550e09a..c24c67d5 100644 --- a/internal/application/service/knowledge_delete.go +++ b/internal/application/service/knowledge_delete.go @@ -4,10 +4,11 @@ import ( "context" "encoding/json" "errors" + "fmt" "strings" - "time" "github.com/Tencent/WeKnora/internal/application/service/retriever" + apperrors "github.com/Tencent/WeKnora/internal/errors" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" @@ -120,150 +121,11 @@ func deleteExtractedImages( // DeleteKnowledge deletes a knowledge entry and all related resources func (s *knowledgeService) DeleteKnowledge(ctx context.Context, id string) error { - // Get the knowledge entry - knowledge, err := s.repo.GetKnowledgeByID(ctx, ctx.Value(types.TenantIDContextKey).(uint64), id) + plan, err := s.planKnowledgeDelete(ctx, []string{id}) if err != nil { return err } - - // Mark as deleting first to prevent async task conflicts - // This ensures that any running async tasks will detect the deletion and abort - originalStatus := knowledge.ParseStatus - knowledge.ParseStatus = types.ParseStatusDeleting - knowledge.UpdatedAt = time.Now() - if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil { - logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge failed to mark as deleting") - // Continue with deletion even if marking fails - } else { - logger.Infof(ctx, "Marked knowledge %s as deleting (previous status: %s)", id, originalStatus) - } - - // Best-effort: purge any queued downstream tasks for this knowledge - // (multimodal / post-process / question / summary / graph extract). - // Worker checkpoints already drop them on the floor, but dequeuing - // here avoids waking workers just to no-op when the parse was still - // in flight at delete time. No-op in Lite mode and on completed rows - // (no queued descendants anyway). - if originalStatus == types.ParseStatusPending || - originalStatus == types.ParseStatusProcessing { - s.dequeueKnowledgeTasks(ctx, id) - } - - // Resolve file service for this KB before spawning goroutines - kb, _ := s.kbService.GetKnowledgeBaseByID(ctx, knowledge.KnowledgeBaseID) - kbFileSvc := s.resolveFileService(ctx, kb) - - // Collect image URLs before chunks are deleted (ImageInfo references are lost after deletion) - tenantID := ctx.Value(types.TenantIDContextKey).(uint64) - chunkImageInfos, err := s.chunkService.GetRepository().ListImageInfoByKnowledgeIDs(ctx, tenantID, []string{id}) - if err != nil { - logger.Errorf(ctx, "Failed to collect image URLs for cleanup: %v", err) - } - var imageInfoStrs []string - for _, ci := range chunkImageInfos { - imageInfoStrs = append(imageInfoStrs, ci.ImageInfo) - } - imageURLs := collectImageURLs(ctx, imageInfoStrs) - - wg := errgroup.Group{} - // Delete knowledge embeddings from vector store. - // Skip entirely when the knowledge has no embedding model (e.g. Wiki-only KB): - // nothing was ever written to the vector store, so there is nothing to delete, - // and GetEmbeddingModel would fail with "model ID cannot be empty". - if strings.TrimSpace(knowledge.EmbeddingModelID) != "" { - wg.Go(func() error { - // kb was already loaded above for resolveFileService — reuse its - // VectorStoreID for engine routing. - var boundStoreID *string - if kb != nil { - boundStoreID = kb.VectorStoreID - } - retrieveEngine, err := retriever.CreateRetrieveEngineForKB( - ctx, - s.retrieveEngine, - s.ownership, - tenantID, - boundStoreID, - ) - if err != nil { - logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed") - return err - } - embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, knowledge.EmbeddingModelID) - if err != nil { - logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed") - return err - } - if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions(), knowledge.Type); err != nil { - logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed") - return err - } - return nil - }) - } else { - logger.Infof(ctx, "Knowledge %s has no embedding model, skipping vector store cleanup", knowledge.ID) - } - - // Clean wiki pages before deleting chunks so cleanup can still identify - // which chunk_refs belonged to this source document. - if kb != nil && kb.IsWikiEnabled() { - s.cleanupWikiOnKnowledgeDelete(ctx, knowledge) - } - - // Delete all chunks associated with this knowledge - wg.Go(func() error { - if err := s.chunkService.DeleteChunksByKnowledgeID(ctx, knowledge.ID); err != nil { - logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete chunks failed") - return err - } - return nil - }) - - // Delete the knowledge graph - wg.Go(func() error { - namespace := types.NameSpace{KnowledgeBase: knowledge.KnowledgeBaseID, Knowledge: knowledge.ID} - if err := s.graphEngine.DelGraph(ctx, []types.NameSpace{namespace}); err != nil { - logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge graph failed") - return err - } - return nil - }) - - if err = wg.Wait(); err != nil { - return err - } - if err := s.repo.DeleteKnowledgeTagRelations(ctx, id); err != nil { - logger.Warnf(ctx, "Failed to delete tag relations for knowledge %s: %v", id, err) - } - // Delete the knowledge row FIRST, then drop its physical file. Physical - // cleanup is deliberately deferred until the row is gone: if any of the - // index/chunk/graph cleanups above failed we already returned early with the - // row (and its file) intact, so the queued retry — or a user-triggered - // reparse — can still read the original file. Deleting the file before the - // row could leave a "file missing but row present" zombie that can neither be - // reparsed nor cleanly re-deleted (issue #2192). Orphaning a file after the - // row is gone is the tolerable failure mode instead. - if err := s.repo.DeleteKnowledge(ctx, tenantID, id); err != nil { - return err - } - - // Best-effort physical cleanup. Errors here only leak storage; they must not - // fail the delete now that the row is already gone. - if knowledge.FilePath != "" { - if err := kbFileSvc.DeleteFile(ctx, knowledge.FilePath); err != nil { - logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete file failed") - } - } - deleteExtractedImages(ctx, kbFileSvc, knowledgeResourceOwners(s.resourceCatalog, knowledge.ID), imageURLs) - tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) - tenantInfo.StorageUsed -= knowledge.StorageSize - if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, -knowledge.StorageSize); err != nil { - logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge update tenant storage used failed") - } - recordKBActivity(ctx, s.audit, tenantID, knowledge.KnowledgeBaseID, types.AuditActionKnowledgeDeleted, - "knowledge", knowledge.ID, types.AuditOutcomeSuccess, - map[string]any{"title": knowledge.Title, "type": knowledge.Type}) - return nil + return s.executeKnowledgeDelete(plan, true) } // cleanupWikiOnKnowledgeDelete handles wiki pages when a source document is deleted. @@ -277,9 +139,9 @@ func (s *knowledgeService) DeleteKnowledge(ctx context.Context, id string) error // - It writes a tombstone + scrubs pending ingest ops so new pages cannot be // born with a stale source_ref (guards (a) queued ingest and (b) ingest // tasks mid-LLM call — both consult the tombstone before writing). -// - It immediately reconciles any pages already present (delete-if-only-ref -// or strip-ref-if-multi). -// - It *unconditionally* enqueues a retract task. Crucially we DO NOT gate +// - It persists a retract task before reconciling existing pages (delete +// if only source, or strip the source if shared), preserving retry evidence. +// Crucially we DO NOT gate // enqueue on "pages currently exist": in the ingest/delete race the // knowledge may have pages that exist only after this function returns // (the ingest task fires later and, absent the tombstone, would have @@ -287,20 +149,42 @@ func (s *knowledgeService) DeleteKnowledge(ctx context.Context, id string) error // run time, so even with an empty PageSlugs it will do the right thing — // and at worst it's a cheap no-op. func (s *knowledgeService) cleanupWikiOnKnowledgeDelete(ctx context.Context, knowledge *types.Knowledge) { + if err := s.cleanupWikiReferences(ctx, knowledge, s.wikiChunkRefsForKnowledge(ctx, knowledge)); err != nil { + logger.Warnf(ctx, "wiki cleanup failed: %v", err) + } +} + +func (s *knowledgeService) cleanupWikiReferences( + ctx context.Context, + knowledge *types.Knowledge, + sourceChunkRefs map[string]bool, +) error { if knowledge == nil { - return + return nil } kbID := knowledge.KnowledgeBaseID knowledgeID := knowledge.ID if kbID == "" || knowledgeID == "" { - return + return nil } + var cleanupErr error + // (1) Tombstone + scrub pending ingest — must happen first so any // wiki_ingest task that wakes up between here and the retract enqueue // below sees "knowledge gone" and bails out. - s.markKnowledgeDeletedForWiki(ctx, kbID, knowledgeID) - s.scrubWikiPendingIngest(ctx, kbID, knowledgeID, "cleanup") + if s.redisClient != nil { + key := WikiDeletedTombstoneKey(kbID, knowledgeID) + if err := s.redisClient.Set(ctx, key, "1", wikiDeletedTTL).Err(); err != nil { + cleanupErr = errors.Join(cleanupErr, err) + } + } + if s.taskPendingRepo != nil { + err := s.taskPendingRepo.DeleteByDedupKey(ctx, wikiTaskType, wikiTaskScope, kbID, knowledgeID, WikiOpIngest) + if err != nil { + cleanupErr = errors.Join(cleanupErr, err) + } + } // Pull title/summary from the knowledge itself — do NOT read them from // existing wiki pages. In the race window wiki pages may not exist yet, @@ -322,9 +206,8 @@ func (s *knowledgeService) cleanupWikiOnKnowledgeDelete(ctx context.Context, kno pages, err := s.wikiRepo.ListBySourceRef(ctx, kbID, knowledgeID) if err != nil { logger.Warnf(ctx, "wiki cleanup: failed to list pages by source ref %s: %v", knowledgeID, err) - pages = nil + return errors.Join(cleanupErr, err) } - sourceChunkRefs := s.wikiChunkRefsForKnowledge(ctx, knowledge) // Prefer the on-disk summary if the summary page already exists (it's // richer than the raw user-provided description). Leave docSummary @@ -336,22 +219,37 @@ func (s *knowledgeService) cleanupWikiOnKnowledgeDelete(ctx context.Context, kno } } + // Persist the retract page set before removing source refs. Otherwise an + // enqueue failure would leave the retry unable to rediscover those pages. + var pageSlugs, folderIDs []string + for _, page := range pages { + if page.PageType != types.WikiPageTypeIndex { + pageSlugs = append(pageSlugs, page.Slug) + folderIDs = append(folderIDs, page.FolderID) + } + } + lang := types.LanguageFromContextOrDefault(ctx) + tenantID, _ := types.TenantIDFromContext(ctx) + if err := enqueueWikiRetract(ctx, s.task, s.taskPendingRepo, WikiRetractPayload{ + TenantID: tenantID, KnowledgeBaseID: kbID, KnowledgeID: knowledgeID, + DocTitle: docTitle, DocSummary: docSummary, Language: lang, + PageSlugs: pageSlugs, FolderIDs: uniqueWikiFolderIDs(folderIDs), + }); err != nil { + return errors.Join(cleanupErr, err) + } + var deletedSlugs []string - var retractSlugs []string - var affectedFolderIDs []string for _, page := range pages { if page.PageType == types.WikiPageTypeIndex { continue } - if page.FolderID != "" { - affectedFolderIDs = append(affectedFolderIDs, page.FolderID) - } remaining := removeSourceRef(page.SourceRefs, knowledgeID) if len(remaining) == 0 { if err := s.wikiService.DeletePage(ctx, kbID, page.Slug); err != nil { logger.Warnf(ctx, "wiki cleanup: failed to delete page %s: %v", page.Slug, err) + cleanupErr = errors.Join(cleanupErr, err) } else { deletedSlugs = append(deletedSlugs, page.Slug) } @@ -360,8 +258,7 @@ func (s *knowledgeService) cleanupWikiOnKnowledgeDelete(ctx context.Context, kno page.ChunkRefs = removeChunkRefs(page.ChunkRefs, sourceChunkRefs) if err := s.wikiService.UpdatePageMeta(ctx, page); err != nil { logger.Warnf(ctx, "wiki cleanup: failed to update source refs for page %s: %v", page.Slug, err) - } else { - retractSlugs = append(retractSlugs, page.Slug) + cleanupErr = errors.Join(cleanupErr, err) } } } @@ -371,34 +268,14 @@ func (s *knowledgeService) cleanupWikiOnKnowledgeDelete(ctx context.Context, kno len(deletedSlugs), knowledgeID, deletedSlugs) } - allAffectedSlugs := append(retractSlugs, deletedSlugs...) - - // (3) Unconditionally enqueue the retract task. See function comment — - // an empty PageSlugs is not a bug, it's the signal "re-query at run - // time". The handler will ListPagesBySourceRef again, pick up any - // pages that materialised after we looked, and also rebuild the index - // so the knowledge's disappearance is reflected in the UI. - lang := types.LanguageFromContextOrDefault(ctx) - tenantID, _ := types.TenantIDFromContext(ctx) - EnqueueWikiRetract(ctx, s.task, s.taskPendingRepo, WikiRetractPayload{ - TenantID: tenantID, - KnowledgeBaseID: kbID, - KnowledgeID: knowledgeID, - DocTitle: docTitle, - DocSummary: docSummary, - Language: lang, - PageSlugs: allAffectedSlugs, - FolderIDs: uniqueWikiFolderIDs(affectedFolderIDs), - }) - logger.Infof(ctx, "wiki cleanup: enqueued retract task for knowledge %s (%d known slugs: %v)", - knowledgeID, len(allAffectedSlugs), allAffectedSlugs) + return cleanupErr } func (s *knowledgeService) wikiChunkRefsForKnowledge(ctx context.Context, knowledge *types.Knowledge) map[string]bool { if knowledge == nil || s.chunkRepo == nil { return nil } - chunks, err := s.chunkRepo.ListChunksByKnowledgeID(ctx, knowledge.TenantID, knowledge.ID) + chunks, err := s.chunkRepo.ListAllChunksByKnowledgeID(ctx, knowledge.TenantID, knowledge.ID) if err != nil { logger.Warnf(ctx, "wiki cleanup: failed to list chunks for knowledge %s: %v", knowledge.ID, err) return nil @@ -413,19 +290,6 @@ func (s *knowledgeService) wikiChunkRefsForKnowledge(ctx context.Context, knowle return refs } -// markKnowledgeDeletedForWiki writes a short-TTL tombstone so any wiki_ingest -// task still running or queued for this knowledge can short-circuit before -// resurrecting a page with a stale source_ref. No-op when Redis is absent. -func (s *knowledgeService) markKnowledgeDeletedForWiki(ctx context.Context, kbID, knowledgeID string) { - if s.redisClient == nil || kbID == "" || knowledgeID == "" { - return - } - key := WikiDeletedTombstoneKey(kbID, knowledgeID) - if err := s.redisClient.Set(ctx, key, "1", wikiDeletedTTL).Err(); err != nil { - logger.Warnf(ctx, "wiki cleanup: failed to write tombstone %s: %v", key, err) - } -} - // scrubWikiPendingIngest removes queued WikiOpIngest entries for a knowledge // from task_pending_ops. Used by both the delete path (we're about to // soft-delete the doc, no point ingesting it) and the reparse path (the @@ -441,7 +305,8 @@ func (s *knowledgeService) scrubWikiPendingIngest(ctx context.Context, kbID, kno if s.taskPendingRepo == nil || kbID == "" || knowledgeID == "" { return } - if err := s.taskPendingRepo.DeleteByDedupKey(ctx, wikiTaskType, wikiTaskScope, kbID, knowledgeID, WikiOpIngest); err != nil { + err := s.taskPendingRepo.DeleteByDedupKey(ctx, wikiTaskType, wikiTaskScope, kbID, knowledgeID, WikiOpIngest) + if err != nil { logger.Warnf(ctx, "wiki %s: failed to scrub pending ingest ops for knowledge %s: %v", reason, knowledgeID, err) return } @@ -552,15 +417,20 @@ func buildKnowledgeVectorDeleteGroups( // DeleteKnowledgeList deletes a knowledge entry and all related resources func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string) error { - if len(ids) == 0 { - return nil - } - // 1. Get the knowledge entry - tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) - knowledgeList, err := s.repo.GetKnowledgeBatch(ctx, tenantInfo.ID, ids) + plan, err := s.planKnowledgeDelete(ctx, ids) if err != nil { return err } + return s.executeKnowledgeDelete(plan, false) +} + +func (s *knowledgeService) executeKnowledgeDelete(plan *knowledgeDeletePlan, single bool) error { + if len(plan.ids) == 0 { + return nil + } + ctx, ids, knowledgeList := plan.ctx, plan.ids, plan.knowledge + knowledgeBases, kbFileServices := plan.kbs, plan.files + tenantInfo, _ := types.TenantInfoFromContext(ctx) // Mark all as deleting first to prevent async task conflicts. // Remember which entries still had queued / in-flight downstream tasks @@ -568,12 +438,12 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string var inFlightIDs []string for _, knowledge := range knowledgeList { prev := knowledge.ParseStatus + before := *knowledge knowledge.ParseStatus = types.ParseStatusDeleting - knowledge.UpdatedAt = time.Now() - if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil { + if err := s.repo.UpdateKnowledgeForTransfer(ctx, &before, knowledge); err != nil { logger.GetLogger(ctx).WithField("error", err).WithField("knowledge_id", knowledge.ID). Errorf("DeleteKnowledgeList failed to mark as deleting") - // Continue with deletion even if marking fails + return err } if prev == types.ParseStatusPending || prev == types.ParseStatusProcessing { inFlightIDs = append(inFlightIDs, knowledge.ID) @@ -582,28 +452,14 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string logger.Infof(ctx, "Marked %d knowledge entries as deleting", len(knowledgeList)) // Best-effort dequeue of downstream tasks for in-flight entries. - // See DeleteKnowledge for the rationale; loop is per-knowledge because + // Workers also check deletion state; this avoids waking them unnecessarily. + // The loop is per-knowledge because // the inspector only filters by knowledge_id, not by ID set. for _, kid := range inFlightIDs { s.dequeueKnowledgeTasks(ctx, kid) } - // Pre-resolve KB metadata and file services so goroutines don't need DB access. - knowledgeBases := make(map[string]*types.KnowledgeBase) - kbFileServices := make(map[string]interfaces.FileService) - for _, knowledge := range knowledgeList { - if _, ok := kbFileServices[knowledge.KnowledgeBaseID]; !ok { - kb, _ := s.kbService.GetKnowledgeBaseByID(ctx, knowledge.KnowledgeBaseID) - knowledgeBases[knowledge.KnowledgeBaseID] = kb - kbFileServices[knowledge.KnowledgeBaseID] = s.resolveFileService(ctx, kb) - } - } - - // Collect image URLs before chunks are deleted - chunkImageInfos, err := s.chunkService.GetRepository().ListImageInfoByKnowledgeIDs(ctx, tenantInfo.ID, ids) - if err != nil { - logger.Errorf(ctx, "Failed to collect image URLs for batch cleanup: %v", err) - } + chunkImageInfos := plan.imageInfo knowledgeToKB := make(map[string]string) for _, k := range knowledgeList { knowledgeToKB[k.ID] = k.KnowledgeBaseID @@ -664,7 +520,7 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string // 3. Clean wiki pages before deleting chunks so cleanup can still identify // which chunk_refs belonged to each source document. for _, knowledge := range knowledgeList { - kb, _ := s.kbService.GetKnowledgeBaseByID(ctx, knowledge.KnowledgeBaseID) + kb := knowledgeBases[knowledge.KnowledgeBaseID] if kb != nil && kb.IsWikiEnabled() { s.cleanupWikiOnKnowledgeDelete(ctx, knowledge) } @@ -672,7 +528,7 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string // 4. Delete all chunks associated with this knowledge wg.Go(func() error { - if err := s.chunkService.DeleteByKnowledgeList(ctx, ids); err != nil { + if err := s.chunkRepo.DeleteByKnowledgeList(ctx, tenantInfo.ID, ids); err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete chunks failed") return err } @@ -695,7 +551,7 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string return nil }) - if err = wg.Wait(); err != nil { + if err := wg.Wait(); err != nil { return err } for _, knowledgeID := range ids { @@ -704,7 +560,7 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string } } // 6. Delete the knowledge rows FIRST, then drop their physical files. See - // DeleteKnowledge for the rationale: deferring file removal until the rows are + // Deferring file removal until the rows are // gone avoids "file missing but row present" zombies that break reparse / // re-delete when an earlier cleanup step failed (issue #2192). A failure below // only orphans storage. @@ -731,7 +587,7 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string } deleteExtractedImages(ctx, fSvc, knowledgeResourceOwners(s.resourceCatalog, kbKnowledgeIDs[kbID]...), urls) } - tenantInfo.StorageUsed += storageAdjust + // TenantInfo can be shared by concurrent cleanup branches; update only storage accounting in the repository. if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, storageAdjust); err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge update tenant storage used failed") } @@ -752,6 +608,13 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string details["knowledge_ids"] = knowledgeIDs } kbActivityAppendSampleTitles(details, titles...) + if single { + knowledge := knowledges[0] + recordKBActivity(ctx, s.audit, tenantInfo.ID, kbID, types.AuditActionKnowledgeDeleted, + "knowledge", knowledge.ID, types.AuditOutcomeSuccess, + map[string]any{"title": knowledge.Title, "type": knowledge.Type}) + continue + } recordKBActivity(ctx, s.audit, tenantInfo.ID, kbID, types.AuditActionKnowledgeBatchDeleted, "knowledge", "", types.AuditOutcomeSuccess, details) } @@ -768,23 +631,14 @@ func (s *knowledgeService) cleanupKnowledgeResources(ctx context.Context, knowle return nil } + kb, err := knowledgeWriteKB(ctx, s.kbService, knowledge) + if err != nil { + return fmt.Errorf("resolve cleanup knowledge base: %w", err) + } tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) if knowledge.EmbeddingModelID != "" { - // Load KB to discover its VectorStoreID binding. Falls back to tenant - // effective engines if the KB has no binding or the load fails. - // - // Silent fallback risk: if a bound KB fails to load here due to a - // transient DB error, the cleanup will delete from env engines and - // leave orphan vectors in the bound store. Warn so operators can spot it. - var boundStoreID *string - if kb, loadErr := s.kbService.GetKnowledgeBaseByID(ctx, knowledge.KnowledgeBaseID); loadErr == nil && kb != nil { - boundStoreID = kb.VectorStoreID - } else if loadErr != nil { - logger.GetLogger(ctx).WithField("error", loadErr).WithField("knowledge_base_id", knowledge.KnowledgeBaseID). - Warnf("cleanupKnowledgeResources: failed to load KB for vector store resolution; falling back to tenant effective engines") - } retrieveEngine, err := retriever.CreateRetrieveEngineForKB( - ctx, s.retrieveEngine, s.ownership, tenantInfo.ID, boundStoreID) + ctx, s.retrieveEngine, s.ownership, tenantInfo.ID, kb.VectorStoreID) if err != nil { logger.GetLogger(ctx).WithField("error", err).Error("Failed to init retrieve engine during cleanup") cleanupErr = errors.Join(cleanupErr, err) @@ -803,7 +657,6 @@ func (s *knowledgeService) cleanupKnowledgeResources(ctx context.Context, knowle } // Collect image URLs before chunks are deleted - kb, _ := s.kbService.GetKnowledgeBaseByID(ctx, knowledge.KnowledgeBaseID) fileSvc := s.resolveFileService(ctx, kb) chunkImageInfos, imgErr := s.chunkService.GetRepository().ListImageInfoByKnowledgeIDs(ctx, tenantInfo.ID, []string{knowledge.ID}) if imgErr != nil { @@ -816,7 +669,7 @@ func (s *knowledgeService) cleanupKnowledgeResources(ctx context.Context, knowle } imageURLs := collectImageURLs(ctx, imageInfoStrs) - if err := s.chunkService.DeleteChunksByKnowledgeID(ctx, knowledge.ID); err != nil { + if err := s.chunkRepo.DeleteChunksByKnowledgeID(ctx, knowledge.TenantID, knowledge.ID); err != nil { logger.GetLogger(ctx).WithField("error", err).Error("Failed to delete manual knowledge chunks") cleanupErr = errors.Join(cleanupErr, err) } @@ -860,20 +713,46 @@ func (s *knowledgeService) ProcessKnowledgeListDelete(ctx context.Context, t *as logger.Infof(ctx, "Processing knowledge list delete task for %d knowledge items", len(payload.KnowledgeIDs)) - // Get tenant info - tenant, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID) - if err != nil { - logger.Errorf(ctx, "Failed to get tenant %d: %v", payload.TenantID, err) - return err + if payload.TenantID == 0 { + return fmt.Errorf("invalid delete task tenant: %w", asynq.SkipRetry) } - - // Set context values - ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID) - ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant) - - // Delete knowledge list - if err := s.DeleteKnowledgeList(ctx, payload.KnowledgeIDs); err != nil { - logger.Errorf(ctx, "Failed to delete knowledge list: %v", err) + ids, err := writeResourceIDs(payload.KnowledgeIDs) + if err != nil { + return fmt.Errorf("invalid delete task IDs: %w", asynq.SkipRetry) + } + if len(ids) == 0 { + return nil + } + ctx = types.WithExecutionTenant(ctx, payload.TenantID) + kbID := payload.KnowledgeBaseID + if kbID == "" { + // Pre-upgrade tasks were admitted by single-KB endpoints but omitted + // the KB ID. Reconstruct only an unambiguous, same-tenant scope. + rows, err := s.repo.GetKnowledgeBatch(ctx, payload.TenantID, ids) + if err != nil { + return err + } + if len(rows) == 0 { + return nil + } + for _, row := range rows { + if row == nil || row.TenantID != payload.TenantID || row.KnowledgeBaseID == "" || + (kbID != "" && row.KnowledgeBaseID != kbID) { + return fmt.Errorf("ambiguous legacy delete scope: %w", asynq.SkipRetry) + } + kbID = row.KnowledgeBaseID + } + } + bindings := make(map[string]string, len(ids)) + for _, id := range ids { + bindings[id] = kbID + } + ctx = withKnowledgeCleanup(ctx, payload.TenantID, bindings) + if err := s.DeleteKnowledgeList(ctx, ids); err != nil { + var appErr *apperrors.AppError + if errors.As(err, &appErr) && (appErr.HTTPCode == 403 || appErr.HTTPCode == 400 || appErr.HTTPCode == 409) { + return fmt.Errorf("invalid delete task scope: %v: %w", err, asynq.SkipRetry) + } return err } diff --git a/internal/application/service/knowledge_delete_plan.go b/internal/application/service/knowledge_delete_plan.go new file mode 100644 index 00000000..f08f17ae --- /dev/null +++ b/internal/application/service/knowledge_delete_plan.go @@ -0,0 +1,169 @@ +package service + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/application/access" + apperrors "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +type knowledgeCleanupKey struct{} + +// Cleanup continues a previously admitted operation on exact persisted +// references. It never grants general KB write access to a session owner. +type knowledgeCleanupScope struct { + caller types.Caller + tenant uint64 + bindings map[string]string // knowledge ID -> KB ID, copied when admitted +} + +func withKnowledgeCleanup(ctx context.Context, tenant uint64, bindings map[string]string) context.Context { + ctx = types.WithExecutionTenant(ctx, tenant) + copyOfBindings := make(map[string]string, len(bindings)) + for id, kb := range bindings { + copyOfBindings[id] = kb + } + return context.WithValue(ctx, knowledgeCleanupKey{}, knowledgeCleanupScope{ + caller: types.CallerFromContext(ctx), tenant: tenant, bindings: copyOfBindings, + }) +} + +// Only call with IDs obtained from an already authorized persisted relationship +// (session messages, a temporary KB, or a clone diff), never a request body. +// expectedKB additionally pins workflows with an independently known KB. +func deleteReferencedKnowledge( + ctx context.Context, + svc interfaces.KnowledgeService, + expectedKB string, + ids []string, +) error { + tenant, err := writeExecutionTenant(ctx) + if err != nil { + return err + } + ids, err = writeResourceIDs(ids) + if err != nil || len(ids) == 0 { + return err + } + rows, err := svc.GetRepository().GetKnowledgeBatch(ctx, tenant, ids) + if err != nil { + return err + } + wanted := make(map[string]bool, len(ids)) + for _, id := range ids { + wanted[id] = true + } + bindings := make(map[string]string, len(rows)) + remaining := make([]string, 0, len(rows)) + for _, row := range rows { + if row == nil || !wanted[row.ID] || row.TenantID != tenant || row.KnowledgeBaseID == "" || + (expectedKB != "" && row.KnowledgeBaseID != expectedKB) { + return apperrors.NewForbiddenError("cleanup resource binding changed") + } + bindings[row.ID] = row.KnowledgeBaseID + remaining = append(remaining, row.ID) + } + if len(remaining) == 0 { + return nil + } + return svc.DeleteKnowledgeList(withKnowledgeCleanup(ctx, tenant, bindings), remaining) +} + +type knowledgeDeletePlan struct { + ctx context.Context + ids []string + knowledge []*types.Knowledge + kbs map[string]*types.KnowledgeBase + files map[string]interfaces.FileService + imageInfo []interfaces.ChunkImageInfo +} + +func (s *knowledgeService) planKnowledgeDelete(ctx context.Context, ids []string) (*knowledgeDeletePlan, error) { + ids, err := writeResourceIDs(ids) + if err != nil { + return nil, err + } + plan := &knowledgeDeletePlan{ + ctx: ctx, + kbs: make(map[string]*types.KnowledgeBase), + files: make(map[string]interfaces.FileService), + } + if len(ids) == 0 { + return plan, nil + } + tenant, err := writeExecutionTenant(ctx) + if err != nil { + return nil, err + } + cleanup, isCleanup := ctx.Value(knowledgeCleanupKey{}).(knowledgeCleanupScope) + if isCleanup { + if cleanup.tenant != tenant || cleanup.caller != types.CallerFromContext(ctx) { + return nil, apperrors.NewForbiddenError("invalid cleanup scope") + } + for _, id := range ids { + if cleanup.bindings[id] == "" { + return nil, apperrors.NewForbiddenError("knowledge outside cleanup scope") + } + } + } + rows, err := s.repo.GetKnowledgeBatch(ctx, tenant, ids) + if err != nil { + return nil, err + } + byID := make(map[string]*types.Knowledge, len(rows)) + for _, row := range rows { + if row != nil && row.TenantID == tenant { + byID[row.ID] = row + } + } + for _, id := range ids { + row := byID[id] + if row == nil { + if isCleanup { + // An earlier attempt already removed this exact resource. + continue + } + return nil, apperrors.NewNotFoundError("knowledge not found") + } + if isCleanup && cleanup.bindings[id] != row.KnowledgeBaseID { + return nil, apperrors.NewForbiddenError("cleanup resource binding changed") + } + if err := access.RejectMovingKnowledge(row); err != nil { + return nil, err + } + if _, ok := plan.kbs[row.KnowledgeBaseID]; !ok { + kb, err := knowledgeWriteKB(ctx, s.kbService, row) + if err != nil { + return nil, err + } + if !isCleanup { + if _, err := requireKBWrite(ctx, kb); err != nil { + return nil, err + } + } + plan.kbs[kb.ID] = kb + } + copyOfRow := *row + plan.knowledge = append(plan.knowledge, ©OfRow) + plan.ids = append(plan.ids, id) + } + if len(plan.ids) == 0 { + return plan, nil + } + // Resolve owner storage/index context and all KB routing before mutations. + ctx, err = withKBWriteTenantInfo(ctx, plan.kbs[plan.knowledge[0].KnowledgeBaseID], s.tenantRepo) + if err != nil { + return nil, err + } + plan.ctx = ctx + for id, kb := range plan.kbs { + plan.files[id] = s.resolveFileService(ctx, kb) + } + plan.imageInfo, err = s.chunkRepo.ListImageInfoByKnowledgeIDs(ctx, tenant, plan.ids) + if err != nil { + return nil, err + } + return plan, nil +} diff --git a/internal/application/service/knowledge_faq.go b/internal/application/service/knowledge_faq.go index b0afb1e8..b935342a 100644 --- a/internal/application/service/knowledge_faq.go +++ b/internal/application/service/knowledge_faq.go @@ -32,32 +32,9 @@ func (s *knowledgeService) ListFAQEntries(ctx context.Context, return nil, err } - // Check if this is a shared knowledge base access - tenantID := ctx.Value(types.TenantIDContextKey).(uint64) - effectiveTenantID := tenantID - - // If the kb belongs to a different tenant, check for shared access - if kb.TenantID != tenantID { - // Get user ID from context - userIDVal := ctx.Value(types.UserIDContextKey) - if userIDVal == nil { - return nil, werrors.NewForbiddenError("无权访问该知识库") - } - _ = userIDVal.(string) // userID retained only for legacy log fields - callerTenantRole := types.TenantRoleFromContext(ctx) - - // Check if the caller's tenant has at least viewer permission via org sharing. - hasPermission, err := s.kbShareService.HasTenantKBPermission(ctx, kbID, tenantID, callerTenantRole, types.OrgRoleViewer) - if err != nil || !hasPermission { - return nil, werrors.NewForbiddenError("无权访问该知识库") - } - - // Use the source tenant ID for data access - sourceTenantID, err := s.kbShareService.GetKBSourceTenant(ctx, kbID) - if err != nil { - return nil, werrors.NewForbiddenError("无权访问该知识库") - } - effectiveTenantID = sourceTenantID + effectiveTenantID, err := resolveKBReadTenant(ctx, kb, s.kbShareService) + if err != nil { + return nil, err } faqKnowledge, err := s.findFAQKnowledge(ctx, effectiveTenantID, kb.ID) @@ -141,7 +118,7 @@ func (s *knowledgeService) CreateFAQEntry(ctx context.Context, return nil, werrors.NewBadRequestError("请求体不能为空") } - kb, err := s.validateFAQKnowledgeBase(ctx, kbID) + kb, ctx, err := s.writableFAQKnowledgeBase(ctx, kbID) if err != nil { return nil, err } @@ -342,7 +319,7 @@ func (s *knowledgeService) UpdateFAQEntry(ctx context.Context, if payload == nil { return nil, werrors.NewBadRequestError("请求体不能为空") } - kb, err := s.validateFAQKnowledgeBase(ctx, kbID) + kb, ctx, err := s.writableFAQKnowledgeBase(ctx, kbID) if err != nil { return nil, err } @@ -510,7 +487,7 @@ func (s *knowledgeService) AddSimilarQuestions(ctx context.Context, return nil, werrors.NewBadRequestError("相似问列表不能为空") } - kb, err := s.validateFAQKnowledgeBase(ctx, kbID) + kb, ctx, err := s.writableFAQKnowledgeBase(ctx, kbID) if err != nil { return nil, err } @@ -655,7 +632,7 @@ func (s *knowledgeService) AddSimilarQuestions(ctx context.Context, func (s *knowledgeService) UpdateFAQEntryStatus(ctx context.Context, kbID string, entryID string, isEnabled bool, ) error { - kb, err := s.validateFAQKnowledgeBase(ctx, kbID) + kb, ctx, err := s.writableFAQKnowledgeBase(ctx, kbID) if err != nil { return err } @@ -704,7 +681,7 @@ func (s *knowledgeService) UpdateFAQEntryFieldsBatch(ctx context.Context, if req == nil || (len(req.ByID) == 0 && len(req.ByTag) == 0) { return nil } - kb, err := s.validateFAQKnowledgeBase(ctx, kbID) + kb, ctx, err := s.writableFAQKnowledgeBase(ctx, kbID) if err != nil { return err } @@ -713,25 +690,17 @@ func (s *knowledgeService) UpdateFAQEntryFieldsBatch(ctx context.Context, enabledUpdates := make(map[string]bool) tagUpdates := make(map[string]string) - // Convert exclude seq_ids to UUIDs - excludeUUIDs := make([]string, 0, len(req.ExcludeIDs)) - if len(req.ExcludeIDs) > 0 { - excludeChunks, err := s.chunkRepo.ListChunksBySeqID(ctx, tenantID, req.ExcludeIDs) - if err == nil { - for _, c := range excludeChunks { - excludeUUIDs = append(excludeUUIDs, c.ID) - } - } + plan, err := s.planFAQFields(ctx, kb, req) + if err != nil { + return err } + excludeUUIDs := plan.excludeIDs // Handle ByTag updates first (by tag seq_id) if len(req.ByTag) > 0 { - for tagSeqID, update := range req.ByTag { - // Convert tag seq_id to UUID - tag, err := s.tagRepo.GetBySeqID(ctx, tenantID, tagSeqID) - if err != nil { - return werrors.NewNotFoundError(fmt.Sprintf("标签 %d 不存在", tagSeqID)) - } + for _, tagSeqID := range sortedFAQIDs(req.ByTag) { + update := req.ByTag[tagSeqID] + tag := plan.tags[tagSeqID] var setFlags, clearFlags types.ChunkFlags @@ -748,10 +717,7 @@ func (s *knowledgeService) UpdateFAQEntryFieldsBatch(ctx context.Context, var newTagUUID *string if update.TagID != nil { if *update.TagID > 0 { - newTag, err := s.tagRepo.GetBySeqID(ctx, tenantID, *update.TagID) - if err != nil { - return werrors.NewNotFoundError(fmt.Sprintf("标签 %d 不存在", *update.TagID)) - } + newTag := plan.tags[*update.TagID] newTagUUID = &newTag.ID } else { emptyStr := "" @@ -768,6 +734,19 @@ func (s *knowledgeService) UpdateFAQEntryFieldsBatch(ctx context.Context, return err } + // Preserve group changes when an explicit ByID patch follows it. + for _, id := range affectedIDs { + if chunk := plan.chunksByID[id]; chunk != nil { + if update.IsEnabled != nil { + chunk.IsEnabled = *update.IsEnabled + } + chunk.Flags = (chunk.Flags | setFlags) &^ clearFlags + if newTagUUID != nil { + chunk.TagID = *newTagUUID + } + } + } + // Collect affected IDs for retriever sync if len(affectedIDs) > 0 { if update.IsEnabled != nil { @@ -786,33 +765,15 @@ func (s *knowledgeService) UpdateFAQEntryFieldsBatch(ctx context.Context, // Handle ByID updates (by entry seq_id) if len(req.ByID) > 0 { - entrySeqIDs := make([]int64, 0, len(req.ByID)) - for entrySeqID := range req.ByID { - entrySeqIDs = append(entrySeqIDs, entrySeqID) - } - chunks, err := s.chunkRepo.ListChunksBySeqID(ctx, tenantID, entrySeqIDs) - if err != nil { - return err - } - - // Build chunk seq_id to chunk map - chunkBySeqID := make(map[int64]*types.Chunk) - for _, chunk := range chunks { - chunkBySeqID[chunk.SeqID] = chunk - } + chunkBySeqID := plan.chunks setFlags := make(map[string]types.ChunkFlags) clearFlags := make(map[string]types.ChunkFlags) chunksToUpdate := make([]*types.Chunk, 0) - for entrySeqID, update := range req.ByID { - chunk, exists := chunkBySeqID[entrySeqID] - if !exists { - continue - } - if chunk.KnowledgeBaseID != kb.ID || chunk.ChunkType != types.ChunkTypeFAQ { - continue - } + for _, entrySeqID := range sortedFAQIDs(req.ByID) { + update := req.ByID[entrySeqID] + chunk := chunkBySeqID[entrySeqID] needUpdate := false @@ -839,10 +800,7 @@ func (s *knowledgeService) UpdateFAQEntryFieldsBatch(ctx context.Context, if update.TagID != nil { var newTagID string if *update.TagID > 0 { - newTag, err := s.tagRepo.GetBySeqID(ctx, tenantID, *update.TagID) - if err != nil { - return werrors.NewNotFoundError(fmt.Sprintf("标签 %d 不存在", *update.TagID)) - } + newTag := plan.tags[*update.TagID] newTagID = newTag.ID } if chunk.TagID != newTagID { @@ -900,7 +858,7 @@ func (s *knowledgeService) UpdateFAQEntryFieldsBatch(ctx context.Context, // UpdateFAQEntryTag updates the tag assigned to an FAQ entry. func (s *knowledgeService) UpdateFAQEntryTag(ctx context.Context, kbID string, entryID string, tagID *string) error { - kb, err := s.validateFAQKnowledgeBase(ctx, kbID) + kb, ctx, err := s.writableFAQKnowledgeBase(ctx, kbID) if err != nil { return err } @@ -948,104 +906,15 @@ func (s *knowledgeService) UpdateFAQEntryTag(ctx context.Context, kbID string, e // UpdateFAQEntryTagBatch updates tags for FAQ entries in batch. // Key: entry seq_id, Value: tag seq_id (nil to remove tag) func (s *knowledgeService) UpdateFAQEntryTagBatch(ctx context.Context, kbID string, updates map[int64]*int64) error { - if len(updates) == 0 { - return nil + req := &types.FAQEntryFieldsBatchUpdate{ByID: make(map[int64]types.FAQEntryFieldsUpdate, len(updates))} + for id, tag := range updates { + value := int64(0) // nil in the tag API means remove the tag. + if tag != nil { + value = *tag + } + req.ByID[id] = types.FAQEntryFieldsUpdate{TagID: &value} } - kb, err := s.validateFAQKnowledgeBase(ctx, kbID) - if err != nil { - return err - } - tenantID := ctx.Value(types.TenantIDContextKey).(uint64) - - // Get all chunks in batch by seq_id - entrySeqIDs := make([]int64, 0, len(updates)) - for entrySeqID := range updates { - entrySeqIDs = append(entrySeqIDs, entrySeqID) - } - chunks, err := s.chunkRepo.ListChunksBySeqID(ctx, tenantID, entrySeqIDs) - if err != nil { - return err - } - - // Build chunk seq_id to chunk map - chunkBySeqID := make(map[int64]*types.Chunk) - for _, chunk := range chunks { - chunkBySeqID[chunk.SeqID] = chunk - } - - // Build tag seq_id set for validation - tagSeqIDSet := make(map[int64]bool) - for _, tagSeqID := range updates { - if tagSeqID != nil && *tagSeqID > 0 { - tagSeqIDSet[*tagSeqID] = true - } - } - - // Validate all tags in batch by seq_id - tagMap := make(map[int64]*types.KnowledgeTag) - if len(tagSeqIDSet) > 0 { - tagSeqIDs := make([]int64, 0, len(tagSeqIDSet)) - for tagSeqID := range tagSeqIDSet { - tagSeqIDs = append(tagSeqIDs, tagSeqID) - } - tags, err := s.tagRepo.GetBySeqIDs(ctx, tenantID, tagSeqIDs) - if err != nil { - return err - } - for _, tag := range tags { - if tag.KnowledgeBaseID != kb.ID { - return werrors.NewBadRequestError(fmt.Sprintf("标签 %d 不属于当前知识库", tag.SeqID)) - } - tagMap[tag.SeqID] = tag - } - } - - // Update chunks - chunksToUpdate := make([]*types.Chunk, 0) - for entrySeqID, tagSeqID := range updates { - chunk, exists := chunkBySeqID[entrySeqID] - if !exists { - continue - } - if chunk.KnowledgeBaseID != kb.ID || chunk.ChunkType != types.ChunkTypeFAQ { - continue - } - - var resolvedTagID string - if tagSeqID != nil && *tagSeqID > 0 { - tag, ok := tagMap[*tagSeqID] - if !ok { - return werrors.NewBadRequestError(fmt.Sprintf("标签 %d 不存在", *tagSeqID)) - } - resolvedTagID = tag.ID - } - - chunk.TagID = resolvedTagID - chunk.UpdatedAt = time.Now() - chunksToUpdate = append(chunksToUpdate, chunk) - } - - if len(chunksToUpdate) > 0 { - if err := s.chunkRepo.UpdateChunks(ctx, chunksToUpdate); err != nil { - return err - } - - // Sync tag updates to retriever engines - tagUpdates := make(map[string]string) - for _, chunk := range chunksToUpdate { - tagUpdates[chunk.ID] = chunk.TagID - } - retrieveEngine, err := retriever.CreateRetrieveEngineForKB( - ctx, s.retrieveEngine, s.ownership, tenantID, kb.VectorStoreID) - if err != nil { - return err - } - if err := retrieveEngine.BatchUpdateChunkTagID(ctx, tagUpdates); err != nil { - return err - } - } - - return nil + return s.UpdateFAQEntryFieldsBatch(ctx, kbID, req) } // SearchFAQEntries searches FAQ entries using hybrid search. @@ -1389,41 +1258,47 @@ func (s *knowledgeService) DeleteFAQEntries(ctx context.Context, if len(entrySeqIDs) == 0 { return werrors.NewBadRequestError("请选择需要删除的 FAQ 条目") } - kb, err := s.validateFAQKnowledgeBase(ctx, kbID) + kb, ctx, err := s.writableFAQKnowledgeBase(ctx, kbID) if err != nil { return err } tenantID := ctx.Value(types.TenantIDContextKey).(uint64) - var faqKnowledge *types.Knowledge - chunksToRemove := make([]*types.Chunk, 0, len(entrySeqIDs)) - for _, seqID := range entrySeqIDs { - if seqID <= 0 { - continue - } - chunk, err := s.chunkRepo.GetChunkBySeqID(ctx, tenantID, seqID) - if err != nil { - return werrors.NewNotFoundError("FAQ条目不存在") - } - if chunk.KnowledgeBaseID != kb.ID || chunk.ChunkType != types.ChunkTypeFAQ { - return werrors.NewBadRequestError("包含无效的 FAQ 条目") - } - if err := s.chunkService.DeleteChunk(ctx, chunk.ID); err != nil { - return err - } - if faqKnowledge == nil { - faqKnowledge, err = s.repo.GetKnowledgeByID(ctx, tenantID, chunk.KnowledgeID) + selected, err := s.loadFAQWriteChunks(ctx, kb, entrySeqIDs) + if err != nil { + return err + } + chunksToRemove := make([]*types.Chunk, 0, len(selected)) + knowledges := make(map[string]*types.Knowledge) + groups := make(map[string][]*types.Chunk) + for _, id := range sortedFAQIDs(selected) { + chunk := selected[id] + if knowledges[chunk.KnowledgeID] == nil { + knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, chunk.KnowledgeID) if err != nil { return err } + if knowledge == nil || knowledge.TenantID != tenantID || knowledge.KnowledgeBaseID != kb.ID || + knowledge.Type != types.KnowledgeTypeFAQ { + return werrors.NewForbiddenError("FAQ 文档不属于当前知识库") + } + knowledges[chunk.KnowledgeID] = knowledge } + groups[chunk.KnowledgeID] = append(groups[chunk.KnowledgeID], chunk) chunksToRemove = append(chunksToRemove, chunk) } - if len(chunksToRemove) > 0 && faqKnowledge != nil { - if err := s.deleteFAQChunkVectors(ctx, kb, faqKnowledge, chunksToRemove); err != nil { + // All entries and parent documents are authorized before any deletion. + for _, chunk := range chunksToRemove { + if err := s.chunkService.DeleteChunk(ctx, chunk.ID); err != nil { return err } } + for id, chunks := range groups { + if err := s.deleteFAQChunkVectors(ctx, kb, knowledges[id], chunks); err != nil { + return err + } + } + details := map[string]any{"count": len(chunksToRemove), "source_type": "faq"} titles := make([]string, 0, len(chunksToRemove)) for _, chunk := range chunksToRemove { @@ -1630,6 +1505,9 @@ func (s *knowledgeService) validateFAQKnowledgeBase(ctx context.Context, kbID st if err != nil { return nil, err } + if kb == nil || kb.ID != kbID { + return nil, werrors.NewNotFoundError("知识库不存在") + } kb.EnsureDefaults() if kb.Type != types.KnowledgeBaseTypeFAQ { return nil, werrors.NewBadRequestError("仅 FAQ 知识库支持该操作") @@ -1906,7 +1784,9 @@ func (s *knowledgeService) buildFAQTagResolver( if len(seqIDs) > 0 { if tags, err := s.tagRepo.GetBySeqIDs(ctx, tenantID, seqIDs); err == nil { for _, t := range tags { - seqIDToUUID[t.SeqID] = t.ID + if validateFAQTagScope(t, tenantID, kbID) == nil { + seqIDToUUID[t.SeqID] = t.ID + } } } else { logger.Warnf(ctx, "buildFAQTagResolver: batch GetBySeqIDs failed (%d ids), fallback per-entry: %v", @@ -1953,6 +1833,9 @@ func (s *knowledgeService) resolveTagID(ctx context.Context, kbID string, payloa if err != nil { return "", fmt.Errorf("failed to find tag by seq_id %d: %w", payload.TagID, err) } + if err := validateFAQTagScope(tag, tenantID, kbID); err != nil { + return "", err + } return tag.ID, nil } diff --git a/internal/application/service/knowledge_faq_batch.go b/internal/application/service/knowledge_faq_batch.go new file mode 100644 index 00000000..4087020c --- /dev/null +++ b/internal/application/service/knowledge_faq_batch.go @@ -0,0 +1,149 @@ +package service + +import ( + "context" + "fmt" + "sort" + + apperrors "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/types" +) + +func sortedFAQIDs[V any](values map[int64]V) []int64 { + ids := make([]int64, 0, len(values)) + for id := range values { + ids = append(ids, id) + } + sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] }) + return ids +} + +// loadFAQWriteChunks validates the entire selection before the first mutation. +// Copies keep planning from mutating repository-owned snapshots on rejection. +func (s *knowledgeService) loadFAQWriteChunks( + ctx context.Context, + kb *types.KnowledgeBase, + ids []int64, +) (map[int64]*types.Chunk, error) { + wanted := make(map[int64]bool, len(ids)) + for _, id := range ids { + if id <= 0 { + return nil, apperrors.NewBadRequestError("FAQ 条目 ID 必须为正整数") + } + wanted[id] = true + } + result := make(map[int64]*types.Chunk, len(wanted)) + if len(wanted) == 0 { + return result, nil + } + chunks, err := s.chunkRepo.ListChunksBySeqID(ctx, kb.TenantID, sortedFAQIDs(wanted)) + if err != nil { + return nil, err + } + for _, chunk := range chunks { + if chunk == nil || !wanted[chunk.SeqID] { + continue + } + if chunk.TenantID != kb.TenantID || chunk.KnowledgeBaseID != kb.ID || chunk.ChunkType != types.ChunkTypeFAQ { + return nil, apperrors.NewForbiddenError("FAQ 条目不属于当前知识库") + } + snapshot := *chunk + result[chunk.SeqID] = &snapshot + } + if len(result) != len(wanted) { + return nil, apperrors.NewNotFoundError("FAQ 条目不存在") + } + return result, nil +} + +type faqFieldPlan struct { + chunks map[int64]*types.Chunk + chunksByID map[string]*types.Chunk + tags map[int64]*types.KnowledgeTag + excludeIDs []string +} + +func (s *knowledgeService) planFAQFields( + ctx context.Context, + kb *types.KnowledgeBase, + req *types.FAQEntryFieldsBatchUpdate, +) (*faqFieldPlan, error) { + ids := append(sortedFAQIDs(req.ByID), req.ExcludeIDs...) + chunks, err := s.loadFAQWriteChunks(ctx, kb, ids) + if err != nil { + return nil, err + } + plan := &faqFieldPlan{ + chunks: chunks, + chunksByID: make(map[string]*types.Chunk), + tags: make(map[int64]*types.KnowledgeTag), + } + for _, chunk := range chunks { + plan.chunksByID[chunk.ID] = chunk + } + for _, id := range req.ExcludeIDs { + plan.excludeIDs = append(plan.excludeIDs, chunks[id].ID) + } + tagIDs := make(map[int64]bool) + for id := range req.ByTag { + if id <= 0 { + return nil, apperrors.NewBadRequestError("标签 ID 必须为正整数") + } + tagIDs[id] = true + } + for _, updates := range []map[int64]types.FAQEntryFieldsUpdate{req.ByTag, req.ByID} { + for _, update := range updates { + if update.TagID != nil && *update.TagID > 0 { + tagIDs[*update.TagID] = true + } + } + } + if len(tagIDs) > 0 { + tags, err := s.tagRepo.GetBySeqIDs(ctx, kb.TenantID, sortedFAQIDs(tagIDs)) + if err != nil { + return nil, err + } + for _, tag := range tags { + if tag == nil || !tagIDs[tag.SeqID] { + continue + } + if err := validateFAQTagScope(tag, kb.TenantID, kb.ID); err != nil { + return nil, err + } + plan.tags[tag.SeqID] = tag + } + for _, id := range sortedFAQIDs(tagIDs) { + if plan.tags[id] == nil { + return nil, apperrors.NewNotFoundError(fmt.Sprintf("标签 %d 不存在", id)) + } + } + } + return plan, nil +} + +func validateFAQTagScope(tag *types.KnowledgeTag, tenantID uint64, kbID string) error { + if tag == nil { + return apperrors.NewNotFoundError("标签不存在") + } + if tag.TenantID != tenantID || tag.KnowledgeBaseID != kbID { + return apperrors.NewForbiddenError("标签不属于当前知识库") + } + return nil +} + +func (s *knowledgeService) validateFAQImportTags( + ctx context.Context, + kb *types.KnowledgeBase, + entries []types.FAQEntryPayload, +) error { + // Import still reports per-entry content validation failures. Referenced + // resources must all be in scope before admission or execution, however. + req := &types.FAQEntryFieldsBatchUpdate{ByTag: make(map[int64]types.FAQEntryFieldsUpdate)} + for _, entry := range entries { + if entry.TagID != 0 { + req.ByTag[entry.TagID] = types.FAQEntryFieldsUpdate{} + } + } + _, err := s.planFAQFields(ctx, kb, req) + return err +} diff --git a/internal/application/service/knowledge_faq_import.go b/internal/application/service/knowledge_faq_import.go index 7bd6770f..c944b846 100644 --- a/internal/application/service/knowledge_faq_import.go +++ b/internal/application/service/knowledge_faq_import.go @@ -11,6 +11,8 @@ import ( "strings" "time" + "github.com/Tencent/WeKnora/internal/application/access" + "github.com/Tencent/WeKnora/internal/application/repository" "github.com/Tencent/WeKnora/internal/application/service/retriever" werrors "github.com/Tencent/WeKnora/internal/errors" "github.com/Tencent/WeKnora/internal/logger" @@ -39,10 +41,13 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context, } // 验证知识库是否存在且有效 - kb, err := s.validateFAQKnowledgeBase(ctx, kbID) + kb, ctx, err := s.writableFAQKnowledgeBase(ctx, kbID) if err != nil { return "", err } + if err := s.validateFAQImportTags(ctx, kb, payload.Entries); err != nil { + return "", err + } tenantID := ctx.Value(types.TenantIDContextKey).(uint64) @@ -1386,7 +1391,7 @@ func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string, } }() - kb, err = s.validateFAQKnowledgeBase(ctx, kbID) + kb, ctx, err = s.writableFAQKnowledgeBase(ctx, kbID) if err != nil { return err } @@ -2245,7 +2250,29 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task) ctx = logger.WithRequestID(ctx, uuid.New().String()) ctx = logger.WithField(ctx, "faq_import", payload.TaskID) - ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID) + ctx = types.WithExecutionTenant(ctx, payload.TenantID) + kb, err := s.validateFAQKnowledgeBase(ctx, payload.KBID) + if err != nil { + if errors.Is(err, repository.ErrKnowledgeBaseNotFound) { + return fmt.Errorf("%w: FAQ task KB no longer exists", asynq.SkipRetry) + } + return err + } + ctx, err = access.WithKBTaskWrite(ctx, kb, payload.TenantID) + if err != nil { + return fmt.Errorf("%w: FAQ task KB does not belong to its tenant", asynq.SkipRetry) + } + knowledge, err := s.repo.GetKnowledgeByID(ctx, payload.TenantID, payload.KnowledgeID) + if err != nil { + if errors.Is(err, repository.ErrKnowledgeNotFound) { + return fmt.Errorf("%w: FAQ task document no longer exists", asynq.SkipRetry) + } + return err + } + if knowledge == nil || knowledge.TenantID != payload.TenantID || knowledge.KnowledgeBaseID != payload.KBID || + knowledge.Type != types.KnowledgeTypeFAQ { + return fmt.Errorf("%w: FAQ task document does not belong to its KB", asynq.SkipRetry) + } // 获取任务重试信息,用于判断是否是最后一次重试 retryCount, _ := asynq.GetRetryCount(ctx) @@ -2287,6 +2314,9 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task) logger.Infof(ctx, "Processing FAQ import task: task_id=%s, kb_id=%s, total_entries=%d, dry_run=%v, retry=%d/%d", payload.TaskID, payload.KBID, len(payload.Entries), payload.DryRun, retryCount, maxRetry) + if err := s.validateFAQImportTags(ctx, kb, payload.Entries); err != nil { + return err + } // 保存原始总数量 originalTotalEntries := len(payload.Entries) @@ -2361,31 +2391,6 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task) logger.Warnf(ctx, "Failed to update FAQ import progress: %v", err) } - // 幂等性检查:获取knowledge记录(FAQ任务使用knowledge ID作为taskID) - knowledge, err := s.repo.GetKnowledgeByID(ctx, payload.TenantID, payload.KnowledgeID) - if err != nil { - logger.Errorf(ctx, "failed to get FAQ knowledge: %v", err) - return nil - } - - if knowledge == nil { - return nil - } - - kb, err := s.kbService.GetKnowledgeBaseByID(ctx, payload.KBID) - if err != nil { - logger.Errorf(ctx, "Failed to get knowledge base: %v", err) - // 如果是最后一次重试,更新状态为失败 - if isLastRetry { - if updateErr := s.updateFAQImportProgressStatus(ctx, payload.TaskID, payload.InstanceID, payload.EnqueuedAt, types.FAQImportStatusFailed, 0, originalTotalEntries, 0, "获取知识库失败", err.Error()); updateErr != nil { - logger.Errorf(ctx, "Failed to update task status to failed: %v", updateErr) - } - s.recordFAQImportKBActivity(ctx, &payload, progress, originalTotalEntries, types.AuditActionFAQImportFailed, types.AuditOutcomeFailed) - } - s.cleanupFAQEntriesFileOnFinalFailure(ctx, payload.EntriesURL, retryCount, maxRetry) - return fmt.Errorf("failed to get knowledge base: %w", err) - } - // 检查任务状态 - 幂等性处理(复用之前获取的 existingProgress) var processedCount int if existingProgress != nil { @@ -2825,8 +2830,11 @@ func (s *knowledgeService) UpdateLastFAQImportResultDisplayStatus(ctx context.Co return werrors.NewBadRequestError("invalid display status, must be 'open' or 'close'") } - // 获取当前空间ID - tenantID := ctx.Value(types.TenantIDContextKey).(uint64) + kb, ctx, err := s.writableFAQKnowledgeBase(ctx, kbID) + if err != nil { + return err + } + tenantID := kb.TenantID // 查找FAQ类型的knowledge knowledgeList, err := s.repo.ListKnowledgeByKnowledgeBaseID(ctx, tenantID, kbID) diff --git a/internal/application/service/knowledge_folder_move_test.go b/internal/application/service/knowledge_folder_move_test.go index e235ad8f..18bad69f 100644 --- a/internal/application/service/knowledge_folder_move_test.go +++ b/internal/application/service/knowledge_folder_move_test.go @@ -4,6 +4,7 @@ import ( "context" "testing" + "github.com/Tencent/WeKnora/internal/application/access" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" "github.com/stretchr/testify/assert" @@ -40,12 +41,23 @@ func (r *folderMoveRepoStub) RenameKnowledgeFolderPath( } func folderMoveContext() context.Context { - return context.WithValue(context.Background(), types.TenantIDContextKey, uint64(1)) + ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(1)) + return (&access.KBAccess{ + KnowledgeBase: &types.KnowledgeBase{ + ID: "kb-1", + TenantID: 1, + }, + Caller: types.CallerFromContext(ctx), + EffectiveTenantID: 1, + Permission: types.OrgRoleEditor, + }).Context( + ctx, + ) } func TestMoveKnowledgeToFolderNormalizesDestination(t *testing.T) { repo := &folderMoveRepoStub{} - svc := &knowledgeService{repo: repo} + svc := &knowledgeService{repo: repo, kbService: &writeKBLookup{kb: &types.KnowledgeBase{ID: "kb-1", TenantID: 1}}} affected, err := svc.MoveKnowledgeToFolder(folderMoveContext(), "kb-1", []string{"k1", "k2"}, "/docs//spec/") require.NoError(t, err) @@ -56,7 +68,7 @@ func TestMoveKnowledgeToFolderNormalizesDestination(t *testing.T) { func TestMoveKnowledgeToFolderAcceptsRootAndRejectsEmptyBatch(t *testing.T) { repo := &folderMoveRepoStub{} - svc := &knowledgeService{repo: repo} + svc := &knowledgeService{repo: repo, kbService: &writeKBLookup{kb: &types.KnowledgeBase{ID: "kb-1", TenantID: 1}}} // The empty destination is meaningful: it files documents back at the top level. _, err := svc.MoveKnowledgeToFolder(folderMoveContext(), "kb-1", []string{"k1"}, "") @@ -70,7 +82,7 @@ func TestMoveKnowledgeToFolderAcceptsRootAndRejectsEmptyBatch(t *testing.T) { func TestMoveKnowledgeToFolderRejectsTraversalAndUnsafeInput(t *testing.T) { repo := &folderMoveRepoStub{} - svc := &knowledgeService{repo: repo} + svc := &knowledgeService{repo: repo, kbService: &writeKBLookup{kb: &types.KnowledgeBase{ID: "kb-1", TenantID: 1}}} _, err := svc.MoveKnowledgeToFolder(folderMoveContext(), "kb-1", []string{"k1"}, "../../etc/docs") require.NoError(t, err) @@ -82,7 +94,7 @@ func TestMoveKnowledgeToFolderRejectsTraversalAndUnsafeInput(t *testing.T) { func TestRenameKnowledgeFolderValidatesPaths(t *testing.T) { repo := &folderMoveRepoStub{} - svc := &knowledgeService{repo: repo} + svc := &knowledgeService{repo: repo, kbService: &writeKBLookup{kb: &types.KnowledgeBase{ID: "kb-1", TenantID: 1}}} ctx := folderMoveContext() affected, err := svc.RenameKnowledgeFolder(ctx, "kb-1", "docs", "handbook") @@ -112,3 +124,15 @@ func TestRenameKnowledgeFolderValidatesPaths(t *testing.T) { assert.Equal(t, int64(0), affected) assert.Equal(t, before, repo.renameCalls) } + +func (r *folderMoveRepoStub) GetKnowledgeBatch( + _ context.Context, + tenant uint64, + ids []string, +) ([]*types.Knowledge, error) { + rows := make([]*types.Knowledge, 0, len(ids)) + for _, id := range ids { + rows = append(rows, &types.Knowledge{ID: id, TenantID: tenant, KnowledgeBaseID: "kb-1"}) + } + return rows, nil +} diff --git a/internal/application/service/knowledge_index_content_test.go b/internal/application/service/knowledge_index_content_test.go index 80ee18f6..55e6045d 100644 --- a/internal/application/service/knowledge_index_content_test.go +++ b/internal/application/service/knowledge_index_content_test.go @@ -6,6 +6,7 @@ import ( "errors" "testing" + "github.com/Tencent/WeKnora/internal/application/access" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" "github.com/hibiken/asynq" @@ -74,7 +75,7 @@ type metadataUpdateKBService struct { func (metadataUpdateKBService) GetKnowledgeBaseByID( _ context.Context, id string, ) (*types.KnowledgeBase, error) { - return &types.KnowledgeBase{ID: id, SummaryModelID: "summary-model"}, nil + return &types.KnowledgeBase{ID: id, TenantID: 7, SummaryModelID: "summary-model"}, nil } type metadataUpdateTaskEnqueuer struct { @@ -114,6 +115,17 @@ func TestUpdateKnowledgeMetadataDoesNotReindexChunks(t *testing.T) { repo: repo, chunkRepo: chunkRepo, kbService: metadataUpdateKBService{}, task: taskEnqueuer, } ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(7)) + ctx = (&access.KBAccess{ + KnowledgeBase: &types.KnowledgeBase{ + ID: "kb-1", + TenantID: 7, + }, + Caller: types.CallerFromContext(ctx), + EffectiveTenantID: 7, + Permission: types.OrgRoleEditor, + }).Context( + ctx, + ) err := service.UpdateKnowledge(ctx, &types.Knowledge{ ID: "knowledge-1", @@ -143,6 +155,17 @@ func TestUpdateKnowledgeMetadataDoesNotLeaveTasklessPendingStatus(t *testing.T) task: &metadataUpdateTaskEnqueuer{err: errors.New("queue unavailable")}, } ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(7)) + ctx = (&access.KBAccess{ + KnowledgeBase: &types.KnowledgeBase{ + ID: "kb-1", + TenantID: 7, + }, + Caller: types.CallerFromContext(ctx), + EffectiveTenantID: 7, + Permission: types.OrgRoleEditor, + }).Context( + ctx, + ) require.NoError(t, service.UpdateKnowledge(ctx, &types.Knowledge{ ID: "knowledge-1", CustomMetadata: types.JSON(`{"region":"Shanghai"}`), @@ -162,6 +185,17 @@ func TestUpdateKnowledgeUnchangedMetadataDoesNotRefreshSummary(t *testing.T) { repo: repo, chunkRepo: &metadataUpdateChunkRepo{}, kbService: metadataUpdateKBService{}, task: taskEnqueuer, } ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(7)) + ctx = (&access.KBAccess{ + KnowledgeBase: &types.KnowledgeBase{ + ID: "kb-1", + TenantID: 7, + }, + Caller: types.CallerFromContext(ctx), + EffectiveTenantID: 7, + Permission: types.OrgRoleEditor, + }).Context( + ctx, + ) require.NoError(t, service.UpdateKnowledge(ctx, &types.Knowledge{ ID: "knowledge-1", diff --git a/internal/application/service/knowledge_move_wiki_test.go b/internal/application/service/knowledge_move_wiki_test.go index bf18bf01..188fe3b7 100644 --- a/internal/application/service/knowledge_move_wiki_test.go +++ b/internal/application/service/knowledge_move_wiki_test.go @@ -2,8 +2,11 @@ package service import ( "context" + "encoding/json" + "errors" "testing" + "github.com/Tencent/WeKnora/internal/application/access" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" "github.com/stretchr/testify/assert" @@ -19,6 +22,7 @@ import ( type moveWikiKnowledgeRepo struct { interfaces.KnowledgeRepository knowledge *types.Knowledge + commitErr error } func (r *moveWikiKnowledgeRepo) GetKnowledgeByID( @@ -40,22 +44,33 @@ func (r *moveWikiKnowledgeRepo) DeleteKnowledgeTagRelations(_ context.Context, _ type moveWikiPageRepo struct { interfaces.WikiPageRepository - listedKBs []string + listedKBs []string + listErr error + beforeList func() + pages []*types.WikiPage } func (r *moveWikiPageRepo) ListBySourceRef( _ context.Context, kbID, _ string, ) ([]*types.WikiPage, error) { r.listedKBs = append(r.listedKBs, kbID) - return nil, nil + if r.beforeList != nil { + r.beforeList() + } + return r.pages, r.listErr } type moveWikiPendingRepo struct { interfaces.TaskPendingOpsRepository - ops []*types.TaskPendingOp + ops []*types.TaskPendingOp + err error + failKB string } func (r *moveWikiPendingRepo) Enqueue(_ context.Context, op *types.TaskPendingOp) error { + if r.err != nil && (r.failKB == "" || r.failKB == op.ScopeID) { + return r.err + } r.ops = append(r.ops, op) return nil } @@ -67,6 +82,8 @@ func (r *moveWikiPendingRepo) DeleteByDedupKey(_ context.Context, _, _, _, _, _ type moveWikiChunkRepo struct { interfaces.ChunkRepository movedToKB string + moves int + moveErr error } func (r *moveWikiChunkRepo) ListChunksByKnowledgeID( @@ -78,6 +95,10 @@ func (r *moveWikiChunkRepo) ListChunksByKnowledgeID( func (r *moveWikiChunkRepo) MoveChunksByKnowledgeID( _ context.Context, _ uint64, _ string, targetKBID string, ) error { + r.moves++ + if r.moveErr != nil { + return r.moveErr + } r.movedToKB = targetKBID return nil } @@ -85,6 +106,7 @@ func (r *moveWikiChunkRepo) MoveChunksByKnowledgeID( func wikiEnabledKB(id string) *types.KnowledgeBase { return &types.KnowledgeBase{ ID: id, + TenantID: 1, IndexingStrategy: types.IndexingStrategy{WikiEnabled: true}, } } @@ -124,25 +146,25 @@ func newMoveWikiService(t *testing.T) ( } func moveWikiCtx() context.Context { - return context.WithValue(context.Background(), types.TenantIDContextKey, uint64(1)) + ctx, _ := access.WithKBTransferTask( + context.Background(), + wikiEnabledKB("kb-src"), + wikiEnabledKB("kb-dst"), + 1, + access.KBTransferMove, + "move-task", + false, + ) + return ctx } -func TestMoveOneKnowledgeRetractsWikiFromSourceKB(t *testing.T) { - // An unknown mode makes the move itself a no-op, which pins the cleanup as - // unconditional: it runs before the mode dispatch, while the knowledge still - // belongs to the source KB. +func TestMoveOneKnowledgeRejectsUnknownModeBeforeWikiCleanup(t *testing.T) { svc, wikiRepo, pendingRepo, _ := newMoveWikiService(t) - - err := svc.moveOneKnowledge(moveWikiCtx(), "kn-1", - wikiEnabledKB("kb-src"), wikiEnabledKB("kb-dst"), "bogus") - - require.Error(t, err) - assert.Equal(t, []string{"kb-src"}, wikiRepo.listedKBs) - - srcOps := opsFor(pendingRepo.ops, "kb-src") - require.Len(t, srcOps, 1) - assert.Equal(t, WikiOpRetract, srcOps[0].Op) - assert.Equal(t, "kn-1", srcOps[0].DedupKey) + err := svc.moveOneKnowledge(moveWikiCtx(), "kn-1", wikiEnabledKB("kb-src"), wikiEnabledKB("kb-dst"), "bogus") + require.ErrorContains(t, err, "unknown move mode") + require.Empty(t, wikiRepo.listedKBs) + require.Empty(t, pendingRepo.ops) + require.Equal(t, types.ParseStatusCompleted, svc.repo.(*moveWikiKnowledgeRepo).knowledge.ParseStatus) } func TestMoveOneKnowledgeReuseVectorsIngestsIntoTargetKB(t *testing.T) { @@ -165,10 +187,187 @@ func TestMoveOneKnowledgeReuseVectorsIngestsIntoTargetKB(t *testing.T) { func TestMoveOneKnowledgeSkipsWikiWorkForNonWikiKBs(t *testing.T) { svc, wikiRepo, pendingRepo, _ := newMoveWikiService(t) - err := svc.moveOneKnowledge(moveWikiCtx(), "kn-1", - &types.KnowledgeBase{ID: "kb-src"}, &types.KnowledgeBase{ID: "kb-dst"}, "reuse_vectors") + err := svc.moveOneKnowledge( + moveWikiCtx(), + "kn-1", + &types.KnowledgeBase{ + ID: "kb-src", + TenantID: 1, + }, + &types.KnowledgeBase{ID: "kb-dst", TenantID: 1}, + "reuse_vectors", + ) require.NoError(t, err) assert.Empty(t, wikiRepo.listedKBs) assert.Empty(t, pendingRepo.ops) } + +func (r *moveWikiKnowledgeRepo) UpdateKnowledgeForTransfer( + ctx context.Context, + _ *types.Knowledge, + after *types.Knowledge, +) error { + if after.KnowledgeBaseID == "kb-dst" && r.commitErr != nil { + return r.commitErr + } + return r.UpdateKnowledge(ctx, after) +} + +func (r *moveWikiChunkRepo) ListAllChunksByKnowledgeID(context.Context, uint64, string) ([]*types.Chunk, error) { + return nil, nil +} + +type moveWikiMetaService struct { + interfaces.WikiPageService + updated []*types.WikiPage +} + +func (s *moveWikiMetaService) UpdatePageMeta(_ context.Context, page *types.WikiPage) error { + s.updated = append(s.updated, page) + return nil +} + +func TestMoveWikiCleanupRunsOnlyAfterOwnershipCommit(t *testing.T) { + for _, stage := range []string{"chunks", "CAS"} { + t.Run(stage, func(t *testing.T) { + svc, wiki, pending, chunks := newMoveWikiService(t) + repo := svc.repo.(*moveWikiKnowledgeRepo) + failure := errors.New("transfer failed") + if stage == "chunks" { + chunks.moveErr = failure + } else { + repo.commitErr = failure + } + err := svc.moveOneKnowledge( + moveWikiCtx(), + "kn-1", + wikiEnabledKB("kb-src"), + wikiEnabledKB("kb-dst"), + "reuse_vectors", + ) + require.ErrorIs(t, err, failure) + require.Equal(t, "kb-src", repo.knowledge.KnowledgeBaseID) + require.Empty(t, wiki.listedKBs) + require.Empty(t, pending.ops) + }) + } +} + +func TestMoveWikiCleanupFailureRetriesWithoutMovingAgain(t *testing.T) { + for _, stage := range []string{"lookup", "enqueue", "target enqueue"} { + t.Run(stage, func(t *testing.T) { + svc, wiki, pending, chunks := newMoveWikiService(t) + repo := svc.repo.(*moveWikiKnowledgeRepo) + failure := errors.New("wiki unavailable") + if stage == "lookup" { + wiki.listErr = failure + } else { + pending.err = failure + if stage == "target enqueue" { + pending.failKB = "kb-dst" + } + } + wiki.beforeList = func() { require.Equal(t, "kb-dst", repo.knowledge.KnowledgeBaseID) } + err := svc.moveOneKnowledge( + moveWikiCtx(), + "kn-1", + wikiEnabledKB("kb-src"), + wikiEnabledKB("kb-dst"), + "reuse_vectors", + ) + require.ErrorIs(t, err, failure) + require.Equal(t, "kb-dst", repo.knowledge.KnowledgeBaseID) + require.Equal(t, []string{"kb-src"}, wiki.listedKBs) + require.Empty(t, opsFor(pending.ops, "kb-dst")) + wiki.listErr = nil + pending.err = nil + require.NoError( + t, + svc.moveOneKnowledge( + moveWikiCtx(), + "kn-1", + wikiEnabledKB("kb-src"), + wikiEnabledKB("kb-dst"), + "reuse_vectors", + ), + ) + require.Equal(t, 1, chunks.moves) + require.Equal(t, []string{"kb-src", "kb-src"}, wiki.listedKBs) + require.NotEmpty(t, opsFor(pending.ops, "kb-dst")) + }) + } +} + +func TestMoveReparseWikiUsesDurableSourceRefsAfterChunksDeleted(t *testing.T) { + f := transferFixture(t, access.KBTransferMove) + f.kbs.values["kb"].IndexingStrategy.WikiEnabled = true + row, err := f.repo.GetKnowledgeByID(f.ctx, 7, "doc") + require.NoError(t, err) + require.NoError( + t, + row.SetManualMetadata(types.NewManualKnowledgeMetadata("# content", types.ManualKnowledgeStatusPublish, 1)), + ) + row.Description = "source summary" + require.NoError(t, f.repo.UpdateKnowledge(f.ctx, row)) + page := &types.WikiPage{ + ID: "page", + KnowledgeBaseID: "kb", + SourceRefs: types.StringArray{"doc", "other-doc"}, + ChunkRefs: types.StringArray{"chunk", "other-chunk"}, + } + wiki := &moveWikiPageRepo{pages: []*types.WikiPage{page}} + meta := &moveWikiMetaService{} + pending := &moveWikiPendingRepo{} + wiki.beforeList = func() { + saved, err := f.repo.GetKnowledgeByID(f.ctx, 7, "doc") + require.NoError(t, err) + require.Equal(t, "other", saved.KnowledgeBaseID) + chunks, err := f.chunkRepo.ListAllChunksByKnowledgeID(f.ctx, 7, "doc") + require.NoError(t, err) + require.Empty(t, chunks) + } + f.svc.wikiRepo = wiki + f.svc.wikiService = meta + f.svc.taskPendingRepo = pending + f.svc.task = &transferQueue{} + require.NoError(t, f.svc.ProcessKnowledgeMove(context.Background(), moveTask(t, []string{"doc"}, "reparse"))) + require.Equal(t, []string{"kb"}, wiki.listedKBs) + require.Len(t, meta.updated, 1) + require.Equal(t, types.StringArray{"other-doc"}, meta.updated[0].SourceRefs) + require.Equal(t, types.StringArray{"other-chunk"}, meta.updated[0].ChunkRefs) + ops := opsFor(pending.ops, "kb") + require.Len(t, ops, 1) + var payload WikiPendingOp + require.NoError(t, json.Unmarshal(ops[0].Payload, &payload)) + require.Equal(t, "source summary", payload.DocSummary) +} + +func TestMoveWikiRetractionIsDurableBeforeRemovingPageSources(t *testing.T) { + svc, wiki, pending, chunks := newMoveWikiService(t) + page := &types.WikiPage{ID: "page", Slug: "concept/source", SourceRefs: types.StringArray{"kn-1", "other"}} + wiki.pages = []*types.WikiPage{page} + meta := &moveWikiMetaService{} + svc.wikiService = meta + pending.err = errors.New("enqueue unavailable") + err := svc.moveOneKnowledge( + moveWikiCtx(), + "kn-1", + wikiEnabledKB("kb-src"), + wikiEnabledKB("kb-dst"), + "reuse_vectors", + ) + require.ErrorIs(t, err, pending.err) + require.Empty(t, meta.updated) + require.Contains(t, page.SourceRefs, "kn-1") + pending.err = nil + require.NoError( + t, + svc.moveOneKnowledge(moveWikiCtx(), "kn-1", wikiEnabledKB("kb-src"), wikiEnabledKB("kb-dst"), "reuse_vectors"), + ) + require.Equal(t, 1, chunks.moves) + require.Len(t, meta.updated, 1) + var op WikiPendingOp + require.NoError(t, json.Unmarshal(opsFor(pending.ops, "kb-src")[0].Payload, &op)) + require.Equal(t, []string{"concept/source"}, op.PageSlugs) +} diff --git a/internal/application/service/knowledge_post_process.go b/internal/application/service/knowledge_post_process.go index f3f28a6b..35146c50 100644 --- a/internal/application/service/knowledge_post_process.go +++ b/internal/application/service/knowledge_post_process.go @@ -8,6 +8,7 @@ import ( "sort" "time" + "github.com/Tencent/WeKnora/internal/application/access" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/tracing/langfuse" "github.com/Tencent/WeKnora/internal/types" @@ -83,11 +84,39 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta logger.Infof(ctx, "[KnowledgePostProcess] Orchestrating post processing for knowledge: %s", payload.KnowledgeID) - ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID) + ctx = types.WithExecutionTenant(ctx, payload.TenantID) if payload.Language != "" { ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language) } + // 1. Fetch Knowledge and KB + knowledge, err := s.knowledgeRepo.GetKnowledgeByIDOnly(ctx, payload.KnowledgeID) + if err != nil { + return fmt.Errorf("get knowledge %s: %w", payload.KnowledgeID, err) + } + if knowledge == nil { + logger.Warnf(ctx, "[KnowledgePostProcess] Knowledge %s not found, aborting.", payload.KnowledgeID) + return nil + } + + if err := validateProcessingKnowledge(knowledge, + payload.TenantID, + payload.KnowledgeBaseID, + payload.KnowledgeID); err != nil { + return err + } + kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID) + if err != nil || kb == nil { + return fmt.Errorf("get knowledge base %s: %w", payload.KnowledgeBaseID, err) + } + + if kb.ID != payload.KnowledgeBaseID || kb.TenantID != payload.TenantID { + return fmt.Errorf("postprocess task KB owner changed: %w", asynq.SkipRetry) + } + ctx, err = access.WithKBTaskWrite(ctx, kb, payload.TenantID) + if err != nil { + return fmt.Errorf("invalid postprocess scope: %v: %w", err, asynq.SkipRetry) + } // Resolve attempt: payload carries it from the upstream stage, but // fall back to the latest known attempt for compatibility with // in-flight tasks queued before this code shipped. @@ -108,16 +137,6 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta postSpan := s.tracker().BeginStage(ctx, payload.KnowledgeID, attempt, types.StagePostProcess, nil) - // 1. Fetch Knowledge and KB - knowledge, err := s.knowledgeRepo.GetKnowledgeByIDOnly(ctx, payload.KnowledgeID) - if err != nil { - return fmt.Errorf("get knowledge %s: %w", payload.KnowledgeID, err) - } - if knowledge == nil { - logger.Warnf(ctx, "[KnowledgePostProcess] Knowledge %s not found, aborting.", payload.KnowledgeID) - return nil - } - // Skip post-processing entirely when the knowledge has been cancelled // by the user or marked for deletion. We must NOT enqueue summary / // question / graph / wiki child tasks for an aborted knowledge. We @@ -136,11 +155,6 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta return nil } - kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID) - if err != nil || kb == nil { - return fmt.Errorf("get knowledge base %s: %w", payload.KnowledgeBaseID, err) - } - processOverrides, _ := knowledge.ProcessOverrides() eff := ResolveProcessConfig(kb, processOverrides) @@ -150,6 +164,13 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta return fmt.Errorf("list chunks for knowledge %s: %w", payload.KnowledgeID, err) } + for _, chunk := range chunks { + if chunk == nil || chunk.TenantID != payload.TenantID || chunk.KnowledgeID != payload.KnowledgeID || + chunk.KnowledgeBaseID != payload.KnowledgeBaseID { + return fmt.Errorf("postprocess chunk binding changed: %w", asynq.SkipRetry) + } + } + // Gather all text-like chunks (including newly added OCR and Caption from multimodal tasks) var textChunks []*types.Chunk for _, c := range chunks { diff --git a/internal/application/service/knowledge_post_process_wiki_enqueue_test.go b/internal/application/service/knowledge_post_process_wiki_enqueue_test.go index a3596678..88c37a9d 100644 --- a/internal/application/service/knowledge_post_process_wiki_enqueue_test.go +++ b/internal/application/service/knowledge_post_process_wiki_enqueue_test.go @@ -117,21 +117,30 @@ func newWikiEnqueueTestService( ) (*KnowledgePostProcessService, *wikiEnqueueFailureKnowledgeRepo) { repo := &wikiEnqueueFailureKnowledgeRepo{ knowledge: &types.Knowledge{ - ID: knowledgeID, - ParseStatus: types.ParseStatusProcessing, + ID: knowledgeID, + TenantID: 7, + KnowledgeBaseID: "kb-wiki", + ParseStatus: types.ParseStatusProcessing, }, } pendingRepo.knowledgeRepo = repo return &KnowledgePostProcessService{ knowledgeRepo: repo, kbService: &wikiEnqueueFailureKBService{kb: &types.KnowledgeBase{ - ID: "kb-wiki", + ID: "kb-wiki", + TenantID: 7, IndexingStrategy: types.IndexingStrategy{ WikiEnabled: true, }, }}, chunkService: &wikiEnqueueFailureChunkService{chunks: []*types.Chunk{ - {ID: "chunk-1", ChunkType: types.ChunkTypeText}, + { + ID: "chunk-1", + TenantID: 7, + KnowledgeID: knowledgeID, + KnowledgeBaseID: "kb-wiki", + ChunkType: types.ChunkTypeText, + }, }}, taskEnqueuer: queue, pendingRepo: pendingRepo, @@ -219,3 +228,13 @@ func TestKnowledgePostProcessRetriesWikiTriggerWithoutDoubleAccounting(t *testin queue.taskTypes, ) } + +func TestPostProcessRejectsMovedKnowledgeBeforeWikiOrSpanWrites(t *testing.T) { + pending := &wikiEnqueueFailurePendingRepo{} + queue := &wikiEnqueueFailureTaskQueue{} + svc, repo := newWikiEnqueueTestService("moved-doc", pending, queue) + repo.knowledge.KnowledgeBaseID = "other-kb" + require.ErrorIs(t, svc.Handle(context.Background(), newWikiEnqueuePostProcessTask(t, "moved-doc")), asynq.SkipRetry) + require.Zero(t, repo.expectedSubtasks) + require.Equal(t, types.ParseStatusProcessing, repo.knowledge.ParseStatus) +} diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go index ce9365fc..04e2210f 100644 --- a/internal/application/service/knowledge_process.go +++ b/internal/application/service/knowledge_process.go @@ -10,6 +10,8 @@ import ( "strings" "time" + "github.com/Tencent/WeKnora/internal/application/access" + "github.com/Tencent/WeKnora/internal/application/repository" "github.com/Tencent/WeKnora/internal/application/service/retriever" "github.com/Tencent/WeKnora/internal/common" werrors "github.com/Tencent/WeKnora/internal/errors" @@ -32,13 +34,21 @@ func (s *knowledgeService) cloneKnowledge( src *types.Knowledge, targetKB *types.KnowledgeBase, ) (err error) { - if src.ParseStatus != "completed" { - logger.GetLogger(ctx).WithField("knowledge_id", src.ID).Errorf("MoveKnowledge parse status is not completed") - return nil + sourceKB, err := knowledgeWriteKB(ctx, s.kbService, src) + if err != nil { + return err + } + if err := access.RequireKBTransfer(ctx, sourceKB, targetKB, access.KBTransferClone); err != nil { + return err + } + if src.ParseStatus != types.ParseStatusCompleted { + return fmt.Errorf("source knowledge %s is not completed", src.ID) + } + if _, err := s.transferChunks(ctx, src, sourceKB.ID); err != nil { + return err } - tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) dst := &types.Knowledge{ - ID: uuid.New().String(), + ID: uuid.NewString(), TenantID: targetKB.TenantID, KnowledgeBaseID: targetKB.ID, Type: src.Type, @@ -55,8 +65,21 @@ func (s *knowledgeService) cloneKnowledge( FileSize: src.FileSize, FileHash: src.FileHash, FilePath: src.FilePath, - StorageSize: src.StorageSize, + StorageSize: 0, Metadata: src.Metadata, + CustomMetadata: src.CustomMetadata, + } + + state := knowledgeTransferState{ + TaskID: access.TransferTaskID(ctx), + Operation: access.KBTransferClone, + SourceKB: sourceKB.ID, + TargetKB: targetKB.ID, + SourceID: src.ID, + Phase: "cloning", + } + if err := setTransferState(dst, state); err != nil { + return err } // Deep-copy the source document file into an object owned by the destination @@ -80,37 +103,52 @@ func (s *knowledgeService) cloneKnowledge( } defer func() { - if err != nil { - if len(copiedFilePaths) > 0 { - cleanupCopiedObjects(ctx, s.resolveFileService(ctx, targetKB), copiedFilePaths) - } - dst.ParseStatus = "failed" - dst.ErrorMessage = err.Error() - _ = s.repo.UpdateKnowledge(ctx, dst) - logger.GetLogger(ctx).WithField("error", err).Errorf("MoveKnowledge failed to move knowledge") - } else { - dst.ParseStatus = "completed" - dst.EnableStatus = "enabled" - _ = s.repo.UpdateKnowledge(ctx, dst) - logger.GetLogger(ctx).WithField("knowledge_id", dst.ID).Infof("MoveKnowledge move knowledge successfully") + if err == nil { + return } + stored, loadErr := s.repo.GetKnowledgeByID(ctx, dst.TenantID, dst.ID) + if errors.Is(loadErr, repository.ErrKnowledgeNotFound) { + cleanupCopiedObjects(ctx, s.resolveFileService(ctx, targetKB), copiedFilePaths) + return + } + // An uncertain save/read must not delete objects that may be referenced. + if loadErr != nil || stored == nil || validateTransferKnowledge(stored, targetKB) != nil { + return + } + marker, markerErr := transferState(stored) + if markerErr != nil || !matchesTransfer(ctx, marker, sourceKB, targetKB, access.KBTransferClone, src.ID, "") || + stored.ParseStatus == types.ParseStatusCompleted { + return + } + before, after := *stored, *stored + after.ParseStatus = types.ParseStatusFailed + after.ErrorMessage = err.Error() + _ = s.repo.UpdateKnowledgeForTransfer(ctx, &before, &after) }() - if err = s.repo.CreateKnowledge(ctx, dst); err != nil { - logger.GetLogger(ctx).WithField("error", err).Errorf("MoveKnowledge create knowledge failed") - return + return err } - tenantInfo.StorageUsed += dst.StorageSize - if err = s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, dst.StorageSize); err != nil { - logger.GetLogger(ctx).WithField("error", err).Errorf("MoveKnowledge update tenant storage used failed") - return + // Create timestamps can be rounded by the database before the first CAS. + persisted, err := s.repo.GetKnowledgeByID(ctx, dst.TenantID, dst.ID) + if err != nil { + return err } + if err := validateTransferKnowledge(persisted, targetKB); err != nil { + return err + } + dst.UpdatedAt = persisted.UpdatedAt if err = s.CloneChunk(ctx, src, dst); err != nil { - logger.GetLogger(ctx).WithField("knowledge_id", dst.ID). - WithField("error", err).Errorf("MoveKnowledge move chunks failed") - return + return err } - return + before := *dst + state.Phase = "done" + if err = setTransferState(dst, state); err != nil { + return err + } + dst.ParseStatus = types.ParseStatusCompleted + dst.EnableStatus = "enabled" + dst.StorageSize = src.StorageSize + return s.repo.UpdateKnowledgeForTransfer(ctx, &before, dst) } // processDocumentFromPassage handles asynchronous processing of text passages @@ -294,7 +332,7 @@ func (s *knowledgeService) processChunks(ctx context.Context, logger.Infof(ctx, "Cleaning up existing chunks and index data for knowledge: %s", knowledge.ID) // 删除旧的chunks - if err := s.chunkService.DeleteChunksByKnowledgeID(ctx, knowledge.ID); err != nil { + if err := s.chunkRepo.DeleteChunksByKnowledgeID(ctx, knowledge.TenantID, knowledge.ID); err != nil { logger.Warnf(ctx, "Failed to delete existing chunks (may not exist): %v", err) // 不返回错误,继续处理(可能没有旧数据) } @@ -494,7 +532,7 @@ func (s *knowledgeService) processChunks(ctx context.Context, s.beginStage(ctx, knowledge.ID, types.StageChunking, types.JSONMap{ "chunks_planned": len(insertChunks), }) - if err := s.chunkService.CreateChunks(ctx, insertChunks); err != nil { + if err := s.chunkRepo.CreateChunks(ctx, insertChunks); err != nil { knowledge.ParseStatus = types.ParseStatusFailed knowledge.ErrorMessage = err.Error() knowledge.UpdatedAt = time.Now() @@ -574,7 +612,7 @@ func (s *knowledgeService) processChunks(ctx context.Context, if aborted, status := s.isKnowledgeAborted(ctx, knowledge.TenantID, knowledge.ID); aborted { logger.Infof(ctx, "Knowledge aborted (%s) before indexing: %s", status, knowledge.ID) if status == types.ParseStatusDeleting { - if err := s.chunkService.DeleteChunksByKnowledgeID(ctx, knowledge.ID); err != nil { + if err := s.chunkRepo.DeleteChunksByKnowledgeID(ctx, knowledge.TenantID, knowledge.ID); err != nil { logger.Warnf(ctx, "Failed to cleanup chunks after deletion detected: %v", err) } } @@ -589,7 +627,7 @@ func (s *knowledgeService) processChunks(ctx context.Context, s.repo.UpdateKnowledge(ctx, knowledge) // delete failed chunks - if err := s.chunkService.DeleteChunksByKnowledgeID(ctx, knowledge.ID); err != nil { + if err := s.chunkRepo.DeleteChunksByKnowledgeID(ctx, knowledge.TenantID, knowledge.ID); err != nil { logger.Errorf(ctx, "Delete chunks failed: %v", err) } @@ -622,7 +660,7 @@ func (s *knowledgeService) processChunks(ctx context.Context, if aborted, status := s.isKnowledgeAborted(ctx, knowledge.TenantID, knowledge.ID); aborted { logger.Infof(ctx, "Knowledge aborted (%s) after indexing: %s", status, knowledge.ID) if status == types.ParseStatusDeleting { - if err := s.chunkService.DeleteChunksByKnowledgeID(ctx, knowledge.ID); err != nil { + if err := s.chunkRepo.DeleteChunksByKnowledgeID(ctx, knowledge.TenantID, knowledge.ID); err != nil { logger.Warnf(ctx, "Failed to cleanup chunks after deletion detected: %v", err) } if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions(), kb.Type); err != nil { @@ -1319,7 +1357,7 @@ func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asyn } // Save summary chunk - if err := s.chunkService.CreateChunks(ctx, []*types.Chunk{summaryChunk}); err != nil { + if err := s.chunkRepo.CreateChunks(ctx, []*types.Chunk{summaryChunk}); err != nil { logger.Errorf(ctx, "Failed to create summary chunk: %v", err) summaryErr = err return fmt.Errorf("failed to create summary chunk: %w", err) @@ -1726,7 +1764,7 @@ func (s *knowledgeService) processQuestionGenerationForKnowledge(ctx context.Con } // Update chunk in database - if err := s.chunkService.UpdateChunk(ctx, chunk); err != nil { + if err := s.chunkRepo.UpdateChunk(ctx, chunk); err != nil { chunkUpdateFailed++ logger.Warnf(ctx, "Failed to update chunk %s: %v", chunk.ID, err) continue @@ -2059,7 +2097,7 @@ func (s *knowledgeService) processQuestionGenerationForChunks(ctx context.Contex logger.Warnf(ctx, "Failed to set document metadata for chunk %s: %v", chunk.ID, err) continue } - if err := s.chunkService.UpdateChunk(ctx, chunk); err != nil { + if err := s.chunkRepo.UpdateChunk(ctx, chunk); err != nil { logger.Warnf(ctx, "Failed to update chunk %s: %v", chunk.ID, err) continue } @@ -2180,13 +2218,12 @@ func (s *knowledgeService) RegenerateChunkQuestions( return nil, fmt.Errorf("questions can only be generated for text chunks") } generationRevision := chunk.ContentRevision - knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, chunk.KnowledgeID) + knowledge, kb, err := loadKnowledgeWrite(ctx, s.repo, s.kbService, chunk.KnowledgeID) if err != nil { return nil, err } - kb, err := s.kbService.GetKnowledgeBaseByID(ctx, chunk.KnowledgeBaseID) - if err != nil { - return nil, err + if knowledge.KnowledgeBaseID != chunk.KnowledgeBaseID || chunk.TenantID != knowledge.TenantID { + return nil, werrors.NewForbiddenError("chunk does not belong to its knowledge document") } if kb.SummaryModelID == "" { return nil, fmt.Errorf("summary model is required for question generation") @@ -2200,7 +2237,7 @@ func (s *knowledgeService) RegenerateChunkQuestions( return "" } neighbor, getErr := s.chunkRepo.GetChunkByID(ctx, tenantID, id) - if getErr != nil { + if getErr != nil || !sameChunkDocument(chunk, neighbor) { return "" } return neighbor.Content @@ -2414,13 +2451,14 @@ func (s *knowledgeService) ReparseKnowledge( ) (*types.Knowledge, error) { logger.Info(ctx, "Start re-parsing knowledge") - tenantID := ctx.Value(types.TenantIDContextKey).(uint64) - existing, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID) + existing, kb, err := loadKnowledgeWrite(ctx, s.repo, s.kbService, knowledgeID) if err != nil { logger.Errorf(ctx, "Failed to load knowledge: %v", err) return nil, err } + tenantID := existing.TenantID + // Allocate a fresh span tree attempt up front. Doing this BEFORE // the cleanup + enqueue means: (a) the UI immediately sees a new // attempt with all five stages back to "pending" instead of the @@ -2434,13 +2472,6 @@ func (s *knowledgeService) ReparseKnowledge( logger.Warnf(ctx, "[Reparse] OpenAttempt failed for %s: %v (will fall back in worker)", existing.ID, err) } - // Get knowledge base configuration - kb, err := s.kbService.GetKnowledgeBaseByID(ctx, existing.KnowledgeBaseID) - if err != nil { - logger.Errorf(ctx, "Failed to get knowledge base for reparse: %v", err) - return nil, err - } - // When the caller supplies new overrides (e.g. via the reparse confirm // dialog), validate them against this knowledge's file type, then persist // to metadata so both this call's enqueue and the worker re-read the same @@ -2914,6 +2945,10 @@ func (s *knowledgeService) UpdateImageInfo( chunkID string, imageInfo string, ) error { + knowledge, _, err := loadKnowledgeWrite(ctx, s.repo, s.kbService, knowledgeID) + if err != nil { + return err + } imageInfo = common.CleanInvalidUTF8(imageInfo) var images []*types.ImageInfo if err := json.Unmarshal([]byte(imageInfo), &images); err != nil { @@ -2932,6 +2967,11 @@ func (s *knowledgeService) UpdateImageInfo( logger.Errorf(ctx, "Failed to get chunk: %v", err) return err } + if chunk == nil || chunk.ID != chunkID || chunk.KnowledgeID != knowledge.ID || + chunk.TenantID != knowledge.TenantID || + chunk.KnowledgeBaseID != knowledge.KnowledgeBaseID { + return werrors.NewForbiddenError("chunk does not belong to its knowledge document") + } chunk.ImageInfo = imageInfo tenantID := ctx.Value(types.TenantIDContextKey).(uint64) chunkChildren, err := s.chunkService.ListChunkByParentID(ctx, tenantID, chunkID) @@ -2942,6 +2982,11 @@ func (s *knowledgeService) UpdateImageInfo( }) return err } + for _, child := range chunkChildren { + if !sameChunkDocument(chunk, child) { + return werrors.NewForbiddenError("image child does not belong to its document") + } + } logger.Infof(ctx, "Found %d chunks with parent chunk ID: %s", len(chunkChildren), chunkID) // Iterate through each chunk and update its content based on the image information @@ -3029,7 +3074,7 @@ func (s *knowledgeService) UpdateImageInfo( logger.Infof(ctx, "Updated %d chunks out of %d total chunks", len(updateChunk), len(chunkChildren)+1) if len(addChunk) > 0 { - err := s.chunkService.CreateChunks(ctx, addChunk) + err := s.chunkRepo.CreateChunks(ctx, addChunk) if err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ "add_chunk_size": len(addChunk), @@ -3040,7 +3085,7 @@ func (s *knowledgeService) UpdateImageInfo( // Update the chunks for _, c := range updateChunk { - err := s.chunkService.UpdateChunk(ctx, c) + err := s.chunkRepo.UpdateChunk(ctx, c) if err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ "chunk_id": c.ID, @@ -3061,7 +3106,7 @@ func (s *knowledgeService) UpdateImageInfo( } // Update the knowledge file hash - knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID) + knowledge, err = s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID) if err != nil { logger.Errorf(ctx, "Failed to get knowledge: %v", err) return err @@ -3088,7 +3133,7 @@ func (s *knowledgeService) ProcessManualUpdate(ctx context.Context, t *asynq.Tas ctx = logger.WithRequestID(ctx, payload.RequestId) ctx = logger.WithField(ctx, "manual_process", payload.KnowledgeID) - ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID) + ctx = types.WithExecutionTenant(ctx, payload.TenantID) tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID) if err != nil { @@ -3107,6 +3152,13 @@ func (s *knowledgeService) ProcessManualUpdate(ctx context.Context, t *asynq.Tas return nil } + if err := validateProcessingKnowledge(knowledge, + payload.TenantID, + payload.KnowledgeBaseID, + payload.KnowledgeID); err != nil { + return err + } + // Skip if already completed or being deleted if knowledge.ParseStatus == types.ParseStatusCompleted { logger.Infof(ctx, "ProcessManualUpdate: already completed, skipping: %s", payload.KnowledgeID) @@ -3130,6 +3182,13 @@ func (s *knowledgeService) ProcessManualUpdate(ctx context.Context, t *asynq.Tas s.repo.UpdateKnowledge(ctx, knowledge) return nil } + if kb == nil || kb.ID != payload.KnowledgeBaseID || kb.TenantID != payload.TenantID { + return fmt.Errorf("processing task KB owner changed: %w", asynq.SkipRetry) + } + ctx, err = access.WithKBTaskWrite(ctx, kb, payload.TenantID) + if err != nil { + return fmt.Errorf("invalid processing scope: %v: %w", err, asynq.SkipRetry) + } // Re-check abort status right before marking processing — see the same // note in ProcessDocument for the cancel race this guards. @@ -3185,7 +3244,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e ctx = logger.WithRequestID(ctx, payload.RequestId) ctx = logger.WithField(ctx, "document_process", payload.KnowledgeID) - ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID) + ctx = types.WithExecutionTenant(ctx, payload.TenantID) if payload.Language != "" { ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language) } @@ -3216,6 +3275,13 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e return nil } + if err := validateProcessingKnowledge(knowledge, + payload.TenantID, + payload.KnowledgeBaseID, + payload.KnowledgeID); err != nil { + return err + } + // 检查是否正在删除 / 已被用户取消 - 如果是则直接退出 if knowledge.ParseStatus == types.ParseStatusDeleting { logger.Infof(ctx, "Knowledge is being deleted, aborting processing: %s", payload.KnowledgeID) @@ -3261,6 +3327,13 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e s.repo.UpdateKnowledge(ctx, knowledge) return nil } + if kb == nil || kb.ID != payload.KnowledgeBaseID || kb.TenantID != payload.TenantID { + return fmt.Errorf("processing task KB owner changed: %w", asynq.SkipRetry) + } + ctx, err = access.WithKBTaskWrite(ctx, kb, payload.TenantID) + if err != nil { + return fmt.Errorf("invalid processing scope: %v: %w", err, asynq.SkipRetry) + } processOverrides, _ := knowledge.ProcessOverrides() eff := ResolveProcessConfig(kb, processOverrides) @@ -3926,6 +3999,11 @@ func (s *knowledgeService) ProcessKnowledgeListReparse(ctx context.Context, t *a logger.Infof(ctx, "Processing knowledge list reparse task for %d knowledge items", len(payload.KnowledgeIDs)) + ctx, ids, err := s.reparseTaskScope(ctx, payload) + if err != nil || len(ids) == 0 { + return err + } + tenant, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID) if err != nil { logger.Errorf(ctx, "Failed to get tenant %d: %v", payload.TenantID, err) @@ -3935,7 +4013,7 @@ func (s *knowledgeService) ProcessKnowledgeListReparse(ctx context.Context, t *a ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID) ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant) - outcome, err := runKnowledgeListReparseSubmissions(payload.KnowledgeIDs, func(id string) error { + outcome, err := runKnowledgeListReparseSubmissions(ids, func(id string) error { _, err := s.ReparseKnowledge(ctx, id, payload.ProcessConfig) return err }) diff --git a/internal/application/service/knowledge_process_parent_child_test.go b/internal/application/service/knowledge_process_parent_child_test.go index a217a86a..e4df0190 100644 --- a/internal/application/service/knowledge_process_parent_child_test.go +++ b/internal/application/service/knowledge_process_parent_child_test.go @@ -29,11 +29,11 @@ func (r *parentChildKnowledgeRepo) UpdateKnowledge( } type parentChildChunkService struct { - interfaces.ChunkService + interfaces.ChunkRepository created []*types.Chunk } -func (s *parentChildChunkService) DeleteChunksByKnowledgeID(context.Context, string) error { +func (s *parentChildChunkService) DeleteChunksByKnowledgeID(context.Context, uint64, string) error { return nil } @@ -160,7 +160,7 @@ func TestProcessChunksIndexesEveryTextChild(t *testing.T) { ctx := context.WithValue(context.Background(), types.TenantInfoContextKey, tenant) svc := &knowledgeService{ repo: &parentChildKnowledgeRepo{knowledge: knowledge}, - chunkService: chunkService, + chunkRepo: chunkService, modelService: parentChildModelService{embedder: parentChildEmbedder{}}, retrieveEngine: parentChildRetrieveRegistry{engine: retrieveEngine}, graphEngine: parentChildGraphRepo{}, diff --git a/internal/application/service/knowledge_reparse_scope.go b/internal/application/service/knowledge_reparse_scope.go new file mode 100644 index 00000000..44afcc3f --- /dev/null +++ b/internal/application/service/knowledge_reparse_scope.go @@ -0,0 +1,60 @@ +package service + +import ( + "context" + "fmt" + + "github.com/Tencent/WeKnora/internal/application/access" + "github.com/Tencent/WeKnora/internal/types" + "github.com/hibiken/asynq" +) + +// Validate the whole batch before any destructive submission. The exact KB +// grant is consumed again when ReparseKnowledge reloads each document. +func (s *knowledgeService) reparseTaskScope( + ctx context.Context, payload types.KnowledgeListReparsePayload, +) (context.Context, []string, error) { + if payload.TenantID == 0 { + return ctx, nil, fmt.Errorf("invalid reparse task tenant: %w", asynq.SkipRetry) + } + ids, err := writeResourceIDs(payload.KnowledgeIDs) + if err != nil { + return ctx, nil, fmt.Errorf("invalid reparse task IDs: %w", asynq.SkipRetry) + } + if len(ids) == 0 { + return ctx, ids, nil + } + ctx = types.WithExecutionTenant(ctx, payload.TenantID) + rows, err := s.repo.GetKnowledgeBatch(ctx, payload.TenantID, ids) + if err != nil { + return ctx, nil, err + } + kbID := payload.KnowledgeBaseID + seen := make(map[string]bool, len(rows)) + for _, row := range rows { + if row == nil || row.TenantID != payload.TenantID || row.KnowledgeBaseID == "" || + (kbID != "" && row.KnowledgeBaseID != kbID) { + return ctx, nil, fmt.Errorf("reparse task binding changed: %w", asynq.SkipRetry) + } + if err := access.RejectMovingKnowledge(row); err != nil { + return ctx, nil, fmt.Errorf("reparse task document unavailable: %v: %w", err, asynq.SkipRetry) + } + // Legacy payloads can reconstruct only their current unambiguous KB. + kbID = row.KnowledgeBaseID + seen[row.ID] = true + } + for _, id := range ids { + if !seen[id] { + return ctx, nil, fmt.Errorf("reparse task document missing: %w", asynq.SkipRetry) + } + } + kb, err := s.kbService.GetKnowledgeBaseByID(ctx, kbID) + if err != nil { + return ctx, nil, err + } + if kb == nil || kb.ID != kbID || kb.TenantID != payload.TenantID { + return ctx, nil, fmt.Errorf("reparse task KB binding changed: %w", asynq.SkipRetry) + } + ctx, err = access.WithKBTaskWrite(ctx, kb, payload.TenantID) + return ctx, ids, err +} diff --git a/internal/application/service/knowledge_shared_access_test.go b/internal/application/service/knowledge_shared_access_test.go index be0e2121..a405ec0d 100644 --- a/internal/application/service/knowledge_shared_access_test.go +++ b/internal/application/service/knowledge_shared_access_test.go @@ -21,45 +21,64 @@ type fakeKBShareService struct { func (f *fakeKBShareService) ShareKnowledgeBase(context.Context, string, string, string, uint64, types.OrgMemberRole) (*types.KnowledgeBaseShare, error) { return nil, errors.New("not implemented") } + func (f *fakeKBShareService) UpdateSharePermission(context.Context, string, types.OrgMemberRole, string, uint64) error { return errors.New("not implemented") } + func (f *fakeKBShareService) RemoveShare(context.Context, string, string, uint64) error { return errors.New("not implemented") } + func (f *fakeKBShareService) ListSharesByKnowledgeBase(context.Context, string, uint64) ([]*types.KnowledgeBaseShare, error) { return nil, errors.New("not implemented") } + func (f *fakeKBShareService) ListSharesByOrganization(context.Context, string) ([]*types.KnowledgeBaseShare, error) { return nil, errors.New("not implemented") } + func (f *fakeKBShareService) ListSharedKnowledgeBases(context.Context, uint64, types.TenantRole) ([]*types.SharedKnowledgeBaseInfo, error) { return nil, errors.New("not implemented") } + func (f *fakeKBShareService) ListSharedKnowledgeBasesInOrganization(context.Context, string, uint64, types.TenantRole) ([]*types.OrganizationSharedKnowledgeBaseItem, error) { return nil, errors.New("not implemented") } + func (f *fakeKBShareService) ListSharedKnowledgeBaseIDsByOrganizations(context.Context, []string, uint64) (map[string][]string, error) { return nil, errors.New("not implemented") } + func (f *fakeKBShareService) GetShare(context.Context, string) (*types.KnowledgeBaseShare, error) { return nil, errors.New("not implemented") } + func (f *fakeKBShareService) GetShareByKBAndOrg(context.Context, string, string) (*types.KnowledgeBaseShare, error) { return nil, errors.New("not implemented") } -func (f *fakeKBShareService) CheckTenantKBPermission(context.Context, string, uint64, types.TenantRole) (types.OrgMemberRole, bool, error) { - return "", false, errors.New("not implemented") + +func (f *fakeKBShareService) CheckTenantKBPermission( + _ context.Context, + kbID string, + _ uint64, + _ types.TenantRole, +) (types.OrgMemberRole, bool, error) { + return types.OrgRoleViewer, f.allowedKBs[kbID], nil } + func (f *fakeKBShareService) HasTenantKBPermission(ctx context.Context, kbID string, callerTenantID uint64, callerTenantRole types.TenantRole, requiredRole types.OrgMemberRole) (bool, error) { return f.allowedKBs[kbID], nil } + func (f *fakeKBShareService) GetKBSourceTenant(context.Context, string) (uint64, error) { return 0, errors.New("not implemented") } + func (f *fakeKBShareService) CountSharesByKnowledgeBaseIDs(context.Context, []string) (map[string]int64, error) { return nil, errors.New("not implemented") } + func (f *fakeKBShareService) CountByOrganizations(context.Context, []string) (map[string]int64, error) { return nil, errors.New("not implemented") } @@ -153,3 +172,57 @@ func TestGetKnowledgeBatchWithSharedAccess_ExcludesSharedKnowledgeWithoutPermiss } var _ interfaces.KBShareService = (*fakeKBShareService)(nil) + +type countingKBShareService struct { + fakeKBShareService + calls int +} + +func (s *countingKBShareService) CheckTenantKBPermission( + ctx context.Context, + id string, + tenantID uint64, + role types.TenantRole, +) (types.OrgMemberRole, bool, error) { + s.calls++ + return s.fakeKBShareService.CheckTenantKBPermission(ctx, id, tenantID, role) +} + +func TestGetKnowledgeBatchWithSharedAccessChecksEachKBOnceAndRechecksNextRequest(t *testing.T) { + shares := &countingKBShareService{ + fakeKBShareService: fakeKBShareService{allowedKBs: map[string]bool{"kb-shared": true}}, + } + svc, db := newKnowledgeSharedAccessService(t, shares) + for _, id := range []string{"first", "second"} { + seedKnowledge(t, db, &types.Knowledge{ID: id, TenantID: 2, KnowledgeBaseID: "kb-shared", Type: "file"}) + } + got, err := svc.GetKnowledgeBatchWithSharedAccess(newSharedAccessContext(), 1, []string{"first", "second"}) + require.NoError(t, err) + require.Len(t, got, 2) + require.Equal(t, 1, shares.calls) + shares.allowedKBs["kb-shared"] = false + got, err = svc.GetKnowledgeBatchWithSharedAccess(newSharedAccessContext(), 1, []string{"first", "second"}) + require.NoError(t, err) + require.Empty(t, got) + require.Equal(t, 2, shares.calls) +} + +func TestResolveKBReadTenantPreservesServiceBoundary(t *testing.T) { + kb := &types.KnowledgeBase{ID: "kb", TenantID: 2} + shares := &fakeKBShareService{allowedKBs: map[string]bool{"kb": true}} + tenant, err := resolveKBReadTenant(newSharedAccessContext(), kb, shares) + require.NoError(t, err) + require.Equal(t, uint64(2), tenant) + _, err = resolveKBReadTenant( + context.WithValue(context.Background(), types.TenantIDContextKey, uint64(1)), + kb, + shares, + ) + require.Error(t, err, "direct cross-tenant calls still require a user identity") + _, err = resolveKBReadTenant(newSharedAccessContext(), kb, nil) + require.Error(t, err, "a missing share service cannot grant access") + ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(2)) + tenant, err = resolveKBReadTenant(ctx, kb, nil) + require.NoError(t, err, "a service execution context already scoped to the KB remains usable") + require.Equal(t, uint64(2), tenant) +} diff --git a/internal/application/service/knowledge_shared_storage_failure_test.go b/internal/application/service/knowledge_shared_storage_failure_test.go new file mode 100644 index 00000000..cac55cc1 --- /dev/null +++ b/internal/application/service/knowledge_shared_storage_failure_test.go @@ -0,0 +1,46 @@ +package service + +import ( + "context" + "errors" + "testing" + + "github.com/Tencent/WeKnora/internal/application/repository" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/stretchr/testify/require" +) + +type failingSharedKnowledgeRepo struct { + interfaces.KnowledgeRepository + err error +} + +func (r *failingSharedKnowledgeRepo) GetKnowledgeBatch(context.Context, uint64, []string) ([]*types.Knowledge, error) { + return []*types.Knowledge{{ID: "own", TenantID: 1, KnowledgeBaseID: "own-kb"}}, nil +} + +func (r *failingSharedKnowledgeRepo) GetKnowledgeByIDOnly(context.Context, string) (*types.Knowledge, error) { + return nil, r.err +} + +func TestSharedDocumentReadsPropagateStorageFailure(t *testing.T) { + for _, failure := range []error{errors.New("database unavailable"), repository.ErrKnowledgeNotFound} { + repo := &failingSharedKnowledgeRepo{err: failure} + svc := &knowledgeService{repo: repo} + search := &knowledgeBaseService{kgRepo: repo} + rows, err := svc.GetKnowledgeBatchWithSharedAccess(newSharedAccessContext(), 1, []string{"own", "shared"}) + found, searchErr := search.fetchKnowledgeDataWithShared(newSharedAccessContext(), 1, []string{"own", "shared"}) + if errors.Is(failure, repository.ErrKnowledgeNotFound) { + require.NoError(t, err) + require.NoError(t, searchErr) + require.Len(t, rows, 1) + require.Len(t, found, 1) + } else { + require.ErrorIs(t, err, failure) + require.ErrorIs(t, searchErr, failure) + require.Nil(t, rows) + require.Nil(t, found) + } + } +} diff --git a/internal/application/service/knowledge_transfer.go b/internal/application/service/knowledge_transfer.go new file mode 100644 index 00000000..415d3b43 --- /dev/null +++ b/internal/application/service/knowledge_transfer.go @@ -0,0 +1,525 @@ +package service + +import ( + "context" + "encoding/json" + "fmt" + "sort" + + "github.com/Tencent/WeKnora/internal/application/access" + "github.com/Tencent/WeKnora/internal/application/service/retriever" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/hibiken/asynq" +) + +// Transfer state lives with the document, so queue retries do not depend on +// an expiring progress cache. It is server metadata, never custom metadata. +type knowledgeTransferState struct { + TaskID string `json:"task_id"` + Operation access.KBTransferOperation `json:"operation"` + SourceKB string `json:"source_kb"` + TargetKB string `json:"target_kb"` + SourceID string `json:"source_id"` + Mode string `json:"mode,omitempty"` + WikiChunkIDs []string `json:"wiki_chunk_ids,omitempty"` + WikiSummary string `json:"wiki_summary,omitempty"` + Phase string `json:"phase"` +} + +func transferState(k *types.Knowledge) (*knowledgeTransferState, error) { + fields := map[string]json.RawMessage{} + if len(k.Metadata) != 0 { + if err := json.Unmarshal(k.Metadata, &fields); err != nil { + return nil, err + } + } + raw := fields[types.KnowledgeTransferMetadataKey] + if len(raw) == 0 { + return nil, nil + } + var state knowledgeTransferState + if err := json.Unmarshal(raw, &state); err != nil { + return nil, err + } + return &state, nil +} + +func setTransferState(k *types.Knowledge, state knowledgeTransferState) error { + fields := map[string]json.RawMessage{} + if len(k.Metadata) != 0 { + if err := json.Unmarshal(k.Metadata, &fields); err != nil { + return err + } + } + if fields == nil { + fields = map[string]json.RawMessage{} + } + raw, err := json.Marshal(state) + if err != nil { + return err + } + fields[types.KnowledgeTransferMetadataKey] = raw + k.Metadata, err = json.Marshal(fields) + return err +} + +func matchesTransfer( + ctx context.Context, + state *knowledgeTransferState, + source, target *types.KnowledgeBase, + operation access.KBTransferOperation, + sourceID, mode string, +) bool { + return state != nil && state.TaskID != "" && state.TaskID == access.TransferTaskID(ctx) && + state.Operation == operation && + state.SourceKB == source.ID && + state.TargetKB == target.ID && + state.SourceID == sourceID && + state.Mode == mode +} + +func validateTransferKnowledge(k *types.Knowledge, kb *types.KnowledgeBase) error { + if k == nil || k.ID == "" || k.TenantID != kb.TenantID || k.KnowledgeBaseID != kb.ID { + return fmt.Errorf("knowledge binding does not match the transfer scope") + } + return nil +} + +// Read every chunk type and validate persisted document/KB/tag relationships +// before a clone or move can delete, index, create tags, or copy files. +func (s *knowledgeService) transferChunks( + ctx context.Context, + k *types.Knowledge, + allowedKBs ...string, +) ([]*types.Chunk, error) { + allowed := map[string]bool{} + for _, id := range allowedKBs { + allowed[id] = true + } + var all []*types.Chunk + seen := map[string]bool{} + tags := map[string]string{} + chunks, err := s.chunkRepo.ListAllChunksByKnowledgeID(ctx, k.TenantID, k.ID) + if err != nil { + return nil, err + } + for _, c := range chunks { + if c == nil || c.ID == "" || seen[c.ID] || c.TenantID != k.TenantID || c.KnowledgeID != k.ID || + !allowed[c.KnowledgeBaseID] { + return nil, fmt.Errorf("chunk binding does not match the transfer scope") + } + seen[c.ID] = true + if c.TagID != "" && tags[c.TagID] != c.KnowledgeBaseID { + tag, err := s.tagRepo.GetByID(ctx, k.TenantID, c.TagID) + if err != nil { + return nil, err + } + if tag == nil || tag.ID != c.TagID || tag.TenantID != k.TenantID || + tag.KnowledgeBaseID != c.KnowledgeBaseID { + return nil, fmt.Errorf("chunk tag belongs to another knowledge base") + } + tags[c.TagID] = c.KnowledgeBaseID + } + all = append(all, c) + } + + for _, c := range all { + if c.ParentChunkID != "" && !seen[c.ParentChunkID] { + return nil, fmt.Errorf("chunk parent belongs to another document") + } + } + return all, nil +} + +func (s *knowledgeService) planKnowledgeMove( + ctx context.Context, + source, target *types.KnowledgeBase, + ids []string, + mode string, +) ([]*types.Knowledge, error) { + if err := access.RequireKBTransfer(ctx, source, target, access.KBTransferMove); err != nil { + return nil, err + } + tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) + if err := access.ValidateKBTransferCompatibility(source, target, access.KBTransferMove, mode, tenant); err != nil { + return nil, err + } + ids, err := writeResourceIDs(ids) + if err != nil { + return nil, err + } + if len(ids) == 0 { + return nil, fmt.Errorf("knowledge IDs cannot be empty") + } + rows, err := s.repo.GetKnowledgeBatch(ctx, source.TenantID, ids) + if err != nil { + return nil, err + } + byID := map[string]*types.Knowledge{} + for _, row := range rows { + if row != nil { + byID[row.ID] = row + } + } + engineChecked := false + result := make([]*types.Knowledge, 0, len(ids)) + for _, id := range ids { + row := byID[id] + if row == nil || row.TenantID != source.TenantID { + return nil, fmt.Errorf("knowledge %s not found in transfer tenant", id) + } + if err := validateMoveItem(ctx, row, source, target, mode); err != nil { + return nil, err + } + allowed := []string{row.KnowledgeBaseID} + state, _ := transferState(row) + if matchesTransfer(ctx, state, source, target, access.KBTransferMove, row.ID, mode) && state.Phase == "moving" { + allowed = []string{source.ID, target.ID} + } + _, err = s.transferChunks(ctx, row, allowed...) + if err != nil { + return nil, err + } + if mode == "reuse_vectors" && row.EmbeddingModelID != "" { + if row.EmbeddingModelID != source.EmbeddingModelID { + return nil, fmt.Errorf("knowledge %s uses a different embedding model", row.ID) + } + if !engineChecked { + engine, err := retriever.CreateRetrieveEngineForKB( + ctx, + s.retrieveEngine, + s.ownership, + source.TenantID, + source.VectorStoreID, + ) + if err != nil { + return nil, err + } + if err := engine.ValidateKnowledgeIndexMove(ctx); err != nil { + return nil, err + } + engineChecked = true + } + } + snapshot := *row + result = append(result, &snapshot) + } + return result, nil +} + +func validateMoveItem(ctx context.Context, k *types.Knowledge, source, target *types.KnowledgeBase, mode string) error { + if k == nil || k.TenantID != source.TenantID { + return access.ErrForbidden + } + state, err := transferState(k) + if err != nil { + return err + } + if matchesTransfer(ctx, state, source, target, access.KBTransferMove, k.ID, mode) { + switch state.Phase { + case "moving": + if k.KnowledgeBaseID == source.ID && + (k.ParseStatus == types.ParseStatusProcessing || k.ParseStatus == types.ParseStatusFailed) { + return nil + } + case "reparse_pending", "done": + if k.KnowledgeBaseID == target.ID { + return nil + } + } + return fmt.Errorf("knowledge %s changed during move", k.ID) + } + if k.KnowledgeBaseID != source.ID || k.ParseStatus != types.ParseStatusCompleted { + return fmt.Errorf("knowledge %s must be completed in the source knowledge base", k.ID) + } + if state != nil && state.Phase != "done" { + return fmt.Errorf("knowledge %s has an unfinished transfer", k.ID) + } + if mode == "reparse" { + if k.IsManual() { + meta, err := k.ManualMetadata() + if err != nil || meta == nil || meta.Content == "" { + return fmt.Errorf("knowledge %s has no manual content to reparse", k.ID) + } + } else if k.FilePath == "" { + return fmt.Errorf("knowledge %s has no stored file to reparse", k.ID) + } + } + return nil +} + +type knowledgeClonePlan struct { + add []*types.Knowledge + remove []string +} + +func (s *knowledgeService) planKnowledgeClone( + ctx context.Context, + source, target *types.KnowledgeBase, +) (*knowledgeClonePlan, error) { + if err := access.RequireKBTransfer(ctx, source, target, access.KBTransferClone); err != nil { + return nil, err + } + src, err := s.repo.ListKnowledgeByKnowledgeBaseID(ctx, source.TenantID, source.ID) + if err != nil { + return nil, err + } + dst, err := s.repo.ListKnowledgeByKnowledgeBaseID(ctx, target.TenantID, target.ID) + if err != nil { + return nil, err + } + for _, pair := range []struct { + rows []*types.Knowledge + kb *types.KnowledgeBase + }{{src, source}, {dst, target}} { + for _, k := range pair.rows { + if err := validateTransferKnowledge(k, pair.kb); err != nil { + return nil, err + } + if _, err := s.transferChunks(ctx, k, pair.kb.ID); err != nil { + return nil, err + } + } + } + sort.Slice(src, func(i, j int) bool { return src[i].ID < src[j].ID }) + sort.Slice(dst, func(i, j int) bool { return dst[i].ID < dst[j].ID }) + plan := &knowledgeClonePlan{} + used := map[string]bool{} + for _, k := range src { + if k.ParseStatus != types.ParseStatusCompleted { + return nil, fmt.Errorf("source knowledge %s is not completed", k.ID) + } + matched := false + // Prefer exact source bindings created by this task; unlike file hashes, + // these also identify manual/empty-hash documents across retries. + for _, other := range dst { + state, err := transferState(other) + if err != nil { + return nil, err + } + if !used[other.ID] && matchesTransfer(ctx, state, source, target, access.KBTransferClone, k.ID, "") { + used[other.ID] = true + if state.Phase == "done" && other.ParseStatus == types.ParseStatusCompleted { + matched = true + } else { + plan.remove = append(plan.remove, other.ID) + } + break + } + } + if !matched && k.FileHash != "" { + for _, other := range dst { + if !used[other.ID] && other.FileHash == k.FileHash && other.ParseStatus == types.ParseStatusCompleted { + used[other.ID] = true + matched = true + break + } + } + } + if !matched { + plan.add = append(plan.add, k) + } + } + for _, k := range dst { + if used[k.ID] { + continue + } + if k.ParseStatus == types.ParseStatusProcessing || k.ParseStatus == types.ParseStatusPending || + k.ParseStatus == types.ParseStatusDeleting { + return nil, fmt.Errorf("target knowledge %s is busy", k.ID) + } + plan.remove = append(plan.remove, k.ID) + } + return plan, nil +} + +func (s *knowledgeService) executeKnowledgeClone( + ctx context.Context, + source, target *types.KnowledgeBase, + progress func(int, int), +) error { + plan, err := s.planKnowledgeClone(ctx, source, target) + if err != nil { + return err + } + total, done := len(plan.add)+len(plan.remove), 0 + if progress != nil { + progress(done, total) + } + if len(plan.remove) > 0 { + // The complete binding/preflight plan above precedes the first deletion. + if err := deleteReferencedKnowledge(ctx, s, target.ID, plan.remove); err != nil { + return err + } + done += len(plan.remove) + if progress != nil { + progress(done, total) + } + } + // Serial execution keeps storage accounting, tag creation and progress + // coherent; each completed document is independently resumable. + for _, k := range plan.add { + if err := s.cloneKnowledge(ctx, k, target); err != nil { + return err + } + done++ + if progress != nil { + progress(done, total) + } + } + return nil +} + +// acknowledgeMovedReparse completes transfer admission, not document parsing. +// The target processing pipeline owns ParseStatus after its task is enqueued. +func (s *knowledgeService) acknowledgeMovedReparse( + ctx context.Context, + knowledge *types.Knowledge, + source, target *types.KnowledgeBase, +) error { + // The processing worker can already have updated the row after enqueue. + // Reload before checkpointing; never overwrite its parse status or metadata. + current, err := s.repo.GetKnowledgeByID(ctx, knowledge.TenantID, knowledge.ID) + if err != nil { + return err + } + if err := validateTransferKnowledge(current, target); err != nil { + return err + } + state, err := transferState(current) + if err != nil { + return err + } + if !matchesTransfer(ctx, state, source, target, access.KBTransferMove, knowledge.ID, "reparse") { + return access.ErrForbidden + } + if state.Phase == "done" { + return nil + } + before := *current + after := *current + state.Phase = "done" + if err := setTransferState(&after, *state); err != nil { + return err + } + return s.repo.UpdateKnowledgeForTransfer(ctx, &before, &after) +} + +func (s *knowledgeService) preflightFAQClone( + ctx context.Context, + source, target *types.KnowledgeBase, +) (map[string]*types.Chunk, map[string]*types.Chunk, error) { + if err := access.RequireKBTransfer(ctx, source, target, access.KBTransferClone); err != nil { + return nil, nil, err + } + maps := []map[string]*types.Chunk{{}, {}} + for i, kb := range []*types.KnowledgeBase{source, target} { + rows, err := s.repo.ListKnowledgeByKnowledgeBaseID(ctx, kb.TenantID, kb.ID) + if err != nil { + return nil, nil, err + } + if len(rows) > 1 { + return nil, nil, fmt.Errorf("FAQ knowledge base has multiple document containers") + } + for _, row := range rows { + if err := validateTransferKnowledge(row, kb); err != nil { + return nil, nil, err + } + if row.Type != types.KnowledgeTypeFAQ { + return nil, nil, fmt.Errorf("invalid FAQ container") + } + chunks, err := s.transferChunks(ctx, row, kb.ID) + if err != nil { + return nil, nil, err + } + for _, chunk := range chunks { + if chunk.ChunkType != types.ChunkTypeFAQ { + return nil, nil, fmt.Errorf("invalid FAQ chunk type") + } + maps[i][chunk.ID] = chunk + } + } + } + return maps[0], maps[1], nil +} + +func (s *knowledgeService) markMoveItemFailed( + ctx context.Context, + id string, + source, target *types.KnowledgeBase, + mode string, + taskErr error, +) { + if access.RequireKBTransfer(ctx, source, target, access.KBTransferMove) != nil { + return + } + row, err := s.repo.GetKnowledgeByID(ctx, source.TenantID, id) + if err != nil || row == nil || row.ID != id || row.TenantID != source.TenantID { + return + } + state, err := transferState(row) + if err != nil || !matchesTransfer(ctx, state, source, target, access.KBTransferMove, id, mode) { + return + } + sourceFailed := row.KnowledgeBaseID == source.ID && state.Phase == "moving" && + row.ParseStatus == types.ParseStatusProcessing + enqueueFailed := row.KnowledgeBaseID == target.ID && state.Phase == "reparse_pending" && + row.ParseStatus == types.ParseStatusPending + if !sourceFailed && !enqueueFailed { + return + } + before, after := *row, *row + after.ParseStatus = types.ParseStatusFailed + after.ErrorMessage = taskErr.Error() + if err := s.repo.UpdateKnowledgeForTransfer(ctx, &before, &after); err != nil { + logger.Errorf(ctx, "Failed to record move failure for knowledge %s, task %s: %v (move error: %v)", + row.ID, state.TaskID, err, taskErr) + } +} + +// A stale parser must not start after a move has claimed a completed document, +// including the interval before its knowledge_base_id changes. Reparse tasks +// emitted by the move are admitted only once the destination checkpoint exists. +func validateProcessingKnowledge(knowledge *types.Knowledge, tenant uint64, kbID, knowledgeID string) error { + if knowledge == nil || tenant == 0 || kbID == "" || knowledge.ID != knowledgeID || knowledge.TenantID != tenant || + knowledge.KnowledgeBaseID != kbID { + return fmt.Errorf("processing task binding changed: %w", asynq.SkipRetry) + } + state, err := transferState(knowledge) + if err != nil { + return fmt.Errorf("invalid processing metadata: %v: %w", err, asynq.SkipRetry) + } + if state != nil && state.Operation == access.KBTransferMove && state.Phase == "moving" { + return fmt.Errorf("knowledge is being moved: %w", asynq.SkipRetry) + } + return nil +} + +// Cleanup consumes the durable source references only after target ownership +// is committed. A done transfer is retried here when wiki reconciliation fails. +func (s *knowledgeService) cleanupMovedSourceWiki(ctx context.Context, knowledge *types.Knowledge, + source, target *types.KnowledgeBase, +) error { + if !source.IsWikiEnabled() { + return nil + } + if knowledge.KnowledgeBaseID != target.ID { + return access.ErrForbidden + } + state, err := transferState(knowledge) + if err != nil { + return err + } + if state == nil || state.SourceKB != source.ID || state.TargetKB != target.ID { + return access.ErrForbidden + } + old := *knowledge + old.KnowledgeBaseID = source.ID + old.Description = state.WikiSummary + refs := make(map[string]bool, len(state.WikiChunkIDs)) + for _, id := range state.WikiChunkIDs { + refs[id] = true + } + return s.cleanupWikiReferences(ctx, &old, refs) +} diff --git a/internal/application/service/knowledge_transfer_test.go b/internal/application/service/knowledge_transfer_test.go new file mode 100644 index 00000000..28487f41 --- /dev/null +++ b/internal/application/service/knowledge_transfer_test.go @@ -0,0 +1,457 @@ +package service + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "github.com/Tencent/WeKnora/internal/application/access" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/alicebob/miniredis/v2" + "github.com/hibiken/asynq" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/require" +) + +func transferFixture(t *testing.T, operation access.KBTransferOperation) *documentWriteFixture { + t.Helper() + f := newDocumentWriteFixture(t) + require.NoError(t, f.db.AutoMigrate(&types.Tenant{})) + require.NoError(t, f.db.Create(&types.Tenant{ID: 7, Name: "tenant", StorageUsed: 5}).Error) + require.NoError( + t, + f.db.Model(&types.Knowledge{}).Where("id = ?", "doc").Update("parse_status", types.ParseStatusCompleted).Error, + ) + ctx, err := access.WithKBTransferTask( + context.Background(), + f.kbs.values["kb"], + f.kbs.values["other"], + 7, + operation, + "transfer-task", + false, + ) + require.NoError(t, err) + f.ctx = context.WithValue(ctx, types.TenantInfoContextKey, &types.Tenant{ID: 7, StorageUsed: 5}) + mr := miniredis.RunT(t) + f.svc.redisClient = redis.NewClient(&redis.Options{Addr: mr.Addr()}) + t.Cleanup(func() { require.NoError(t, f.svc.redisClient.Close()) }) + return f +} + +func moveTask(t *testing.T, ids []string, mode string) *asynq.Task { + t.Helper() + raw, err := json.Marshal( + types.KnowledgeMovePayload{ + TenantID: 7, + TaskID: "transfer-task", + SourceKBID: "kb", + TargetKBID: "other", + KnowledgeIDs: ids, + Mode: mode, + }, + ) + require.NoError(t, err) + return asynq.NewTask(types.TypeKnowledgeMove, raw) +} + +func TestMoveBatchPreflightHasNoPartialMutations(t *testing.T) { + for _, ids := range [][]string{{"doc", "missing"}, {"doc", "other-doc"}, {"doc", ""}} { + t.Run(ids[1], func(t *testing.T) { + f := transferFixture(t, access.KBTransferMove) + err := f.svc.ProcessKnowledgeMove(context.Background(), moveTask(t, ids, "reuse_vectors")) + require.Error(t, err) + row, err := f.repo.GetKnowledgeByID(f.ctx, 7, "doc") + require.NoError(t, err) + require.Equal(t, types.ParseStatusCompleted, row.ParseStatus) + require.Equal(t, "kb", row.KnowledgeBaseID) + require.Empty(t, row.Metadata) + require.Zero(t, f.chunkRepo.writes) + require.Zero(t, f.graph.calls) + }) + } +} + +func TestMoveRejectsInvalidPairAndModeBeforeWrites(t *testing.T) { + for _, change := range []string{"tenant", "same KB", "mode", "chunk", "tag"} { + t.Run(change, func(t *testing.T) { + f := transferFixture(t, access.KBTransferMove) + mode := "reuse_vectors" + switch change { + case "tenant": + f.kbs.values["other"].TenantID = 8 + case "same KB": + f.kbs.values["other"].ID = "kb" + case "mode": + mode = "bogus" + case "chunk": + require.NoError( + t, + f.db.Model(&types.Chunk{}).Where("id = ?", "chunk").Update("knowledge_base_id", "third").Error, + ) + case "tag": + require.NoError(t, f.db.Model(&types.Chunk{}).Where("id = ?", "chunk").Update("tag_id", "tag").Error) + require.NoError( + t, + f.db.Model(&types.KnowledgeTag{}).Where("id = ?", "tag").Update("knowledge_base_id", "other").Error, + ) + } + require.Error(t, f.svc.ProcessKnowledgeMove(context.Background(), moveTask(t, []string{"doc"}, mode))) + row, err := f.repo.GetKnowledgeByID(f.ctx, 7, "doc") + require.NoError(t, err) + require.Equal(t, types.ParseStatusCompleted, row.ParseStatus) + require.Empty(t, row.Metadata) + }) + } +} + +type transferFaultRepo struct { + interfaces.KnowledgeRepository + failID string + claims int +} + +func (r *transferFaultRepo) DeleteKnowledgeTagRelations(ctx context.Context, id string) error { + if id == r.failID { + return errors.New("temporary tag cleanup failure") + } + return r.KnowledgeRepository.DeleteKnowledgeTagRelations(ctx, id) +} + +func (r *transferFaultRepo) UpdateKnowledgeForTransfer(ctx context.Context, before, after *types.Knowledge) error { + r.claims++ + return r.KnowledgeRepository.UpdateKnowledgeForTransfer(ctx, before, after) +} + +func TestMovePartialFailureResumesAndSkipsCompletedDocuments(t *testing.T) { + f := transferFixture(t, access.KBTransferMove) + require.NoError( + t, + f.db.Create( + &types.Knowledge{ID: "second", TenantID: 7, KnowledgeBaseID: "kb", ParseStatus: types.ParseStatusCompleted}, + ).Error, + ) + require.NoError( + t, + f.db.Create( + &types.Chunk{ + ID: "second-chunk", + TenantID: 7, + KnowledgeID: "second", + KnowledgeBaseID: "kb", + ChunkType: types.ChunkTypeImageOCR, + }, + ).Error, + ) + fault := &transferFaultRepo{KnowledgeRepository: f.repo, failID: "second"} + f.svc.repo = fault + task := moveTask(t, []string{"doc", "doc", "second"}, "reuse_vectors") + require.ErrorContains(t, f.svc.ProcessKnowledgeMove(context.Background(), task), "temporary tag cleanup failure") + first, err := f.repo.GetKnowledgeByID(f.ctx, 7, "doc") + require.NoError(t, err) + require.Equal(t, "other", first.KnowledgeBaseID) + firstUpdated := first.UpdatedAt + second, err := f.repo.GetKnowledgeByID(f.ctx, 7, "second") + require.NoError(t, err) + require.Equal(t, "kb", second.KnowledgeBaseID) + state, err := transferState(second) + require.NoError(t, err) + require.Equal(t, "moving", state.Phase) + fault.failID = "" + require.NoError(t, f.svc.ProcessKnowledgeMove(context.Background(), task)) + first, err = f.repo.GetKnowledgeByID(f.ctx, 7, "doc") + require.NoError(t, err) + require.Equal(t, firstUpdated, first.UpdatedAt, "completed item must not be moved again") + second, err = f.repo.GetKnowledgeByID(f.ctx, 7, "second") + require.NoError(t, err) + require.Equal(t, "other", second.KnowledgeBaseID) + require.Equal(t, types.ParseStatusCompleted, second.ParseStatus) + progress, err := f.svc.GetKnowledgeMoveProgress(f.ctx, "transfer-task") + require.NoError(t, err) + require.Equal(t, 2, progress.Total) + require.Zero(t, progress.Failed) + require.Equal(t, types.KBCloneStatusCompleted, progress.Status) +} + +type transferQueue struct { + fail bool + ids []string +} + +func (q *transferQueue) Enqueue(_ *asynq.Task, options ...asynq.Option) (*asynq.TaskInfo, error) { + for _, option := range options { + if option.Type() == asynq.TaskIDOpt { + q.ids = append(q.ids, option.Value().(string)) + } + } + if q.fail { + return nil, errors.New("queue unavailable") + } + return &asynq.TaskInfo{ID: "processing-task"}, nil +} + +func TestMoveReparseRetryOnlyReenqueuesAndAccountsOnce(t *testing.T) { + f := transferFixture(t, access.KBTransferMove) + row, err := f.repo.GetKnowledgeByID(f.ctx, 7, "doc") + require.NoError(t, err) + require.NoError( + t, + row.SetManualMetadata(types.NewManualKnowledgeMetadata("# content", types.ManualKnowledgeStatusPublish, 1)), + ) + require.NoError(t, f.repo.UpdateKnowledge(f.ctx, row)) + queue := &transferQueue{fail: true} + f.svc.task = queue + task := moveTask(t, []string{"doc"}, "reparse") + require.ErrorContains(t, f.svc.ProcessKnowledgeMove(context.Background(), task), "queue unavailable") + row, err = f.repo.GetKnowledgeByID(f.ctx, 7, "doc") + require.NoError(t, err) + require.Equal(t, "other", row.KnowledgeBaseID) + require.Zero(t, row.StorageSize) + cleanupCalls := f.graph.calls + var tenant types.Tenant + require.NoError(t, f.db.First(&tenant, 7).Error) + require.Zero(t, tenant.StorageUsed) + // A parser may already have written manual metadata before the move's ack. + meta, err := row.ManualMetadata() + require.NoError(t, err) + require.NoError(t, row.SetManualMetadata(meta)) + require.NoError(t, f.repo.UpdateKnowledge(f.ctx, row)) + queue.fail = false + require.NoError(t, f.svc.ProcessKnowledgeMove(context.Background(), task)) + require.Equal(t, cleanupCalls, f.graph.calls) + require.Len(t, queue.ids, 2) + require.Equal(t, queue.ids[0], queue.ids[1]) + require.NoError(t, f.db.First(&tenant, 7).Error) + require.Zero(t, tenant.StorageUsed) + row, err = f.repo.GetKnowledgeByID(f.ctx, 7, "doc") + require.NoError(t, err) + state, err := transferState(row) + require.NoError(t, err) + require.Equal(t, "done", state.Phase) +} + +type cloneFaultChunks struct { + interfaces.ChunkRepository + fail bool +} + +func (r *cloneFaultChunks) CreateChunks(ctx context.Context, chunks []*types.Chunk) error { + if err := r.ChunkRepository.CreateChunks(ctx, chunks); err != nil { + return err + } + if r.fail { + return errors.New("lost clone acknowledgement") + } + return nil +} + +func TestCloneRetryReplacesIncompleteCopyWithoutDuplicatingEmptyHashes(t *testing.T) { + f := transferFixture(t, access.KBTransferClone) + fault := &cloneFaultChunks{ChunkRepository: f.chunkRepo, fail: true} + f.svc.chunkRepo = fault + require.ErrorContains( + t, + f.svc.executeKnowledgeClone(f.ctx, f.kbs.values["kb"], f.kbs.values["other"], nil), + "lost clone acknowledgement", + ) + rows, err := f.repo.ListKnowledgeByKnowledgeBaseID(f.ctx, 7, "other") + require.NoError(t, err) + require.Len(t, rows, 1) + require.Equal(t, types.ParseStatusFailed, rows[0].ParseStatus) + failedChunks, chunkErr := f.chunkRepo.ListAllChunksByKnowledgeID(f.ctx, 7, rows[0].ID) + require.NoError(t, chunkErr) + require.Empty(t, failedChunks, "a lost create acknowledgement must also be rolled back") + fault.fail = false + require.NoError(t, f.svc.executeKnowledgeClone(f.ctx, f.kbs.values["kb"], f.kbs.values["other"], nil)) + rows, err = f.repo.ListKnowledgeByKnowledgeBaseID(f.ctx, 7, "other") + require.NoError(t, err) + require.Len(t, rows, 1) + require.Equal(t, types.ParseStatusCompleted, rows[0].ParseStatus) + id := rows[0].ID + require.NoError(t, f.svc.executeKnowledgeClone(f.ctx, f.kbs.values["kb"], f.kbs.values["other"], nil)) + rows, err = f.repo.ListKnowledgeByKnowledgeBaseID(f.ctx, 7, "other") + require.NoError(t, err) + require.Len(t, rows, 1) + require.Equal(t, id, rows[0].ID) + var tenant types.Tenant + require.NoError(t, f.db.First(&tenant, 7).Error) + require.EqualValues(t, 10, tenant.StorageUsed) +} + +func TestClonePreflightDoesNotDeleteTargetForUnreadySource(t *testing.T) { + f := transferFixture(t, access.KBTransferClone) + require.NoError( + t, + f.db.Model(&types.Knowledge{}).Where("id = ?", "doc").Update("parse_status", types.ParseStatusProcessing).Error, + ) + require.Error(t, f.svc.executeKnowledgeClone(f.ctx, f.kbs.values["kb"], f.kbs.values["other"], nil)) + _, err := f.repo.GetKnowledgeByID(f.ctx, 7, "other-doc") + require.NoError(t, err) + require.Zero(t, f.repo.writes) + require.Zero(t, f.chunkRepo.writes) +} + +func TestTransferCheckpointCannotResurrectOrOverwriteChangedDocument(t *testing.T) { + f := transferFixture(t, access.KBTransferMove) + before, err := f.repo.GetKnowledgeByID(f.ctx, 7, "doc") + require.NoError(t, err) + after := *before + after.StorageSize = 0 + after.KnowledgeBaseID = "other" + require.NoError(t, f.db.Model(&types.Knowledge{}).Where("id = ?", "doc").Update("knowledge_base_id", "third").Error) + require.Error(t, f.repo.UpdateKnowledgeForTransfer(f.ctx, before, &after)) + var tenant types.Tenant + require.NoError(t, f.db.First(&tenant, 7).Error) + require.EqualValues(t, 5, tenant.StorageUsed) + require.NoError(t, f.db.Where("id = ?", "doc").Delete(&types.Knowledge{}).Error) + require.Error(t, f.repo.UpdateKnowledgeForTransfer(f.ctx, before, &after)) + _, err = f.repo.GetKnowledgeByID(f.ctx, 7, "doc") + require.Error(t, err) +} + +func TestProcessingTasksRejectOldKnowledgeBaseAfterMove(t *testing.T) { + for _, taskType := range []string{types.TypeDocumentProcess, types.TypeManualProcess} { + t.Run(taskType, func(t *testing.T) { + f := transferFixture(t, access.KBTransferMove) + require.NoError( + t, + f.db.Model(&types.Knowledge{}). + Where("id = ?", "doc"). + Updates(map[string]any{"knowledge_base_id": "other", "parse_status": types.ParseStatusPending}). + Error, + ) + payload, err := json.Marshal( + types.DocumentProcessPayload{TenantID: 7, KnowledgeID: "doc", KnowledgeBaseID: "kb"}, + ) + require.NoError(t, err) + task := asynq.NewTask(taskType, payload) + if taskType == types.TypeDocumentProcess { + err = f.svc.ProcessDocument(context.Background(), task) + } else { + err = f.svc.ProcessManualUpdate(context.Background(), task) + } + require.ErrorIs(t, err, asynq.SkipRetry) + require.Zero(t, f.repo.writes) + require.Zero(t, f.graph.calls) + }) + } +} + +func TestCopyServiceResumesReservedTargetAndRequiresTransferGrant(t *testing.T) { + repo := newFakeKBRepo() + repo.rows["src"] = &types.KnowledgeBase{ID: "src", TenantID: 1} + svc := newPR3KBService(repo, &fakeRegistry{}, &fakeOwnership{}) + _, _, err := svc.CopyKnowledgeBase(ctxWithTenant(1), "src", "") + require.ErrorIs(t, err, access.ErrForbidden) + require.Len(t, repo.rows, 1) + ctx, err := access.WithKBTransferTask( + context.Background(), + repo.rows["src"], + &types.KnowledgeBase{ID: "reserved", TenantID: 1, CreatorID: "creator"}, + 1, + access.KBTransferClone, + "task", + true, + ) + require.NoError(t, err) + for range 2 { + _, target, err := svc.CopyKnowledgeBase(ctx, "src", "reserved") + require.NoError(t, err) + require.Equal(t, "reserved", target.ID) + require.Equal(t, "creator", target.CreatorID) + } + require.Len(t, repo.rows, 2) + _, _, err = svc.CopyKnowledgeBase(ctx, "src", "other") + require.ErrorIs(t, err, access.ErrForbidden) +} + +func TestProcessingCannotEnterClaimedMoveBeforeKBBindingChanges(t *testing.T) { + f := transferFixture(t, access.KBTransferMove) + row, err := f.repo.GetKnowledgeByID(f.ctx, 7, "doc") + require.NoError(t, err) + require.NoError( + t, + setTransferState( + row, + knowledgeTransferState{ + TaskID: "transfer-task", + Operation: access.KBTransferMove, + SourceKB: "kb", + TargetKB: "other", + SourceID: "doc", + Mode: "reuse_vectors", + Phase: "moving", + }, + ), + ) + row.ParseStatus = types.ParseStatusProcessing + require.NoError(t, f.repo.UpdateKnowledge(f.ctx, row)) + f.repo.writes = 0 + payload, err := json.Marshal(types.DocumentProcessPayload{TenantID: 7, KnowledgeID: "doc", KnowledgeBaseID: "kb"}) + require.NoError(t, err) + require.ErrorIs( + t, + f.svc.ProcessDocument(context.Background(), asynq.NewTask(types.TypeDocumentProcess, payload)), + asynq.SkipRetry, + ) + require.Zero(t, f.repo.writes) + require.Zero(t, f.graph.calls) +} + +func TestAdmissionProgressCannotOverwriteFinishedWorker(t *testing.T) { + f := transferFixture(t, access.KBTransferMove) + require.NoError( + t, + f.svc.saveKnowledgeMoveProgress( + f.ctx, + &types.KnowledgeMoveProgress{TaskID: "task", Status: types.KBCloneStatusCompleted, Progress: 100}, + ), + ) + require.NoError( + t, + f.svc.SaveKnowledgeMoveProgress( + f.ctx, + &types.KnowledgeMoveProgress{TaskID: "task", Status: types.KBCloneStatusPending}, + ), + ) + progress, err := f.svc.GetKnowledgeMoveProgress(f.ctx, "task") + require.NoError(t, err) + require.Equal(t, types.KBCloneStatusCompleted, progress.Status) + require.Equal(t, 100, progress.Progress) +} + +func TestClonePreservesFailedChunkIndexStatus(t *testing.T) { + f := transferFixture(t, access.KBTransferClone) + require.NoError(t, f.db.Exec(`CREATE TRIGGER truncate_clone_created_time AFTER INSERT ON knowledges + BEGIN UPDATE knowledges SET updated_at = strftime('%Y-%m-%d %H:%M:%S', NEW.updated_at) || '+00:00' + WHERE id = NEW.id; END`).Error) + require.NoError(t, f.db.Model(&types.Chunk{}).Where("id = ?", "chunk").Update("index_status", "failed").Error) + require.NoError(t, f.svc.executeKnowledgeClone(f.ctx, f.kbs.values["kb"], f.kbs.values["other"], nil)) + rows, err := f.repo.ListKnowledgeByKnowledgeBaseID(f.ctx, 7, "other") + require.NoError(t, err) + require.Len(t, rows, 1) + chunks, err := f.chunkRepo.ListAllChunksByKnowledgeID(f.ctx, 7, rows[0].ID) + require.NoError(t, err) + require.Len(t, chunks, 1) + require.Equal(t, "failed", chunks[0].IndexStatus) +} + +func TestTableSummaryCleanupDoesNotFollowMovedDocument(t *testing.T) { + f := newDocumentWriteFixture(t) + row, err := f.repo.GetKnowledgeByID(f.ctx, 7, "doc") + require.NoError(t, err) + chunk, err := f.chunkRepo.GetChunkByID(f.ctx, 7, "chunk") + require.NoError(t, err) + require.NoError(t, f.db.Model(&types.Knowledge{}).Where("id = ?", "doc").Update("knowledge_base_id", "other").Error) + worker := &DataTableSummaryService{knowledgeService: f.svc, chunkService: f.chunks} + worker.cleanupOnFailure( + f.ctx, &extractionResources{knowledge: row}, []*types.Chunk{chunk}, errors.New("index failed"), + ) + f.requireNoWrites(t) + current, err := f.repo.GetKnowledgeByID(f.ctx, 7, "doc") + require.NoError(t, err) + require.Equal(t, "other", current.KnowledgeBaseID) + require.Equal(t, row.ParseStatus, current.ParseStatus) +} diff --git a/internal/application/service/knowledge_update_test.go b/internal/application/service/knowledge_update_test.go index b5662a35..cfec0384 100644 --- a/internal/application/service/knowledge_update_test.go +++ b/internal/application/service/knowledge_update_test.go @@ -4,6 +4,7 @@ import ( "context" "testing" + "github.com/Tencent/WeKnora/internal/application/access" "github.com/Tencent/WeKnora/internal/types" "github.com/stretchr/testify/require" ) @@ -21,6 +22,17 @@ func TestUpdateKnowledgeExplicitEmptyDescriptionClearsSummary(t *testing.T) { repo: repo, kbService: metadataUpdateKBService{}, } ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(7)) + ctx = (&access.KBAccess{ + KnowledgeBase: &types.KnowledgeBase{ + ID: "kb-1", + TenantID: 7, + }, + Caller: types.CallerFromContext(ctx), + EffectiveTenantID: 7, + Permission: types.OrgRoleEditor, + }).Context( + ctx, + ) err := service.UpdateKnowledge(ctx, &types.Knowledge{ ID: "knowledge-1", @@ -45,6 +57,17 @@ func TestUpdateKnowledgeOmittedDescriptionPreservesExisting(t *testing.T) { repo: repo, kbService: metadataUpdateKBService{}, } ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(7)) + ctx = (&access.KBAccess{ + KnowledgeBase: &types.KnowledgeBase{ + ID: "kb-1", + TenantID: 7, + }, + Caller: types.CallerFromContext(ctx), + EffectiveTenantID: 7, + Permission: types.OrgRoleEditor, + }).Context( + ctx, + ) err := service.UpdateKnowledge(ctx, &types.Knowledge{ ID: "knowledge-1", @@ -68,6 +91,17 @@ func TestUpdateKnowledgeManualDescriptionSetsCompletedStatus(t *testing.T) { repo: repo, kbService: metadataUpdateKBService{}, } ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(7)) + ctx = (&access.KBAccess{ + KnowledgeBase: &types.KnowledgeBase{ + ID: "kb-1", + TenantID: 7, + }, + Caller: types.CallerFromContext(ctx), + EffectiveTenantID: 7, + Permission: types.OrgRoleEditor, + }).Context( + ctx, + ) err := service.UpdateKnowledge(ctx, &types.Knowledge{ ID: "knowledge-1", @@ -93,6 +127,17 @@ func TestUpdateKnowledgeDescriptionOnlyDoesNotTouchMetadata(t *testing.T) { repo: repo, kbService: metadataUpdateKBService{}, } ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(7)) + ctx = (&access.KBAccess{ + KnowledgeBase: &types.KnowledgeBase{ + ID: "kb-1", + TenantID: 7, + }, + Caller: types.CallerFromContext(ctx), + EffectiveTenantID: 7, + Permission: types.OrgRoleEditor, + }).Context( + ctx, + ) err := service.UpdateKnowledge(ctx, &types.Knowledge{ ID: "knowledge-1", diff --git a/internal/application/service/knowledge_write.go b/internal/application/service/knowledge_write.go new file mode 100644 index 00000000..5fe8bce7 --- /dev/null +++ b/internal/application/service/knowledge_write.go @@ -0,0 +1,141 @@ +package service + +import ( + "context" + "strings" + + "github.com/Tencent/WeKnora/internal/application/access" + apperrors "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +type knowledgeBaseWriteLookup interface { + GetKnowledgeBaseByID(context.Context, string) (*types.KnowledgeBase, error) +} + +func writeResourceIDs(ids []string) ([]string, error) { + result := make([]string, 0, len(ids)) + seen := make(map[string]bool, len(ids)) + for _, id := range ids { + if strings.TrimSpace(id) == "" { + return nil, apperrors.NewBadRequestError("resource ID cannot be empty") + } + if !seen[id] { + seen[id] = true + result = append(result, id) + } + } + return result, nil +} + +func writeExecutionTenant(ctx context.Context) (uint64, error) { + tenant, ok := types.TenantIDFromContext(ctx) + if !ok || tenant == 0 { + return 0, apperrors.NewUnauthorizedError("workspace context unavailable") + } + return tenant, nil +} + +// Resolve persisted bindings before consuming grants. An input object's KB or +// tenant must never select the scope used to authorize an existing resource. +func knowledgeWriteKB( + ctx context.Context, + lookup knowledgeBaseWriteLookup, + knowledge *types.Knowledge, +) (*types.KnowledgeBase, error) { + if knowledge == nil || knowledge.ID == "" || knowledge.KnowledgeBaseID == "" || knowledge.TenantID == 0 { + return nil, apperrors.NewNotFoundError("knowledge not found") + } + kb, err := lookup.GetKnowledgeBaseByID(ctx, knowledge.KnowledgeBaseID) + if err != nil { + return nil, err + } + if kb == nil || kb.ID != knowledge.KnowledgeBaseID || kb.TenantID != knowledge.TenantID { + return nil, apperrors.NewForbiddenError("knowledge does not belong to its knowledge base") + } + return kb, nil +} + +func loadKnowledgeWrite( + ctx context.Context, + repo interfaces.KnowledgeRepository, + lookup knowledgeBaseWriteLookup, + id string, +) (*types.Knowledge, *types.KnowledgeBase, error) { + tenant, err := writeExecutionTenant(ctx) + if err != nil { + return nil, nil, err + } + knowledge, err := repo.GetKnowledgeByID(ctx, tenant, id) + if err != nil { + return nil, nil, err + } + if knowledge == nil || knowledge.ID != id || knowledge.TenantID != tenant { + return nil, nil, apperrors.NewNotFoundError("knowledge not found") + } + if err := access.RejectMovingKnowledge(knowledge); err != nil { + return nil, nil, err + } + kb, err := knowledgeWriteKB(ctx, lookup, knowledge) + if err != nil { + return nil, nil, err + } + if _, err := requireKBWrite(ctx, kb); err != nil { + return nil, nil, err + } + copyOfKnowledge := *knowledge + return ©OfKnowledge, kb, nil +} + +// loadKnowledgeWriteBatch validates every requested ID, including missing +// entries, before any caller can apply changes. Each KB needs its own grant. +func loadKnowledgeWriteBatch( + ctx context.Context, + repo interfaces.KnowledgeRepository, + lookup knowledgeBaseWriteLookup, + ids []string, +) ([]*types.Knowledge, error) { + ids, err := writeResourceIDs(ids) + if err != nil || len(ids) == 0 { + return nil, err + } + tenant, err := writeExecutionTenant(ctx) + if err != nil { + return nil, err + } + rows, err := repo.GetKnowledgeBatch(ctx, tenant, ids) + if err != nil { + return nil, err + } + byID := make(map[string]*types.Knowledge, len(rows)) + for _, row := range rows { + if row != nil && row.TenantID == tenant { + byID[row.ID] = row + } + } + checked := make(map[string]bool) + result := make([]*types.Knowledge, 0, len(ids)) + for _, id := range ids { + row := byID[id] + if row == nil { + return nil, apperrors.NewNotFoundError("knowledge not found") + } + if err := access.RejectMovingKnowledge(row); err != nil { + return nil, err + } + if !checked[row.KnowledgeBaseID] { + kb, err := knowledgeWriteKB(ctx, lookup, row) + if err != nil { + return nil, err + } + if _, err := requireKBWrite(ctx, kb); err != nil { + return nil, err + } + checked[row.KnowledgeBaseID] = true + } + copyOfRow := *row + result = append(result, ©OfRow) + } + return result, nil +} diff --git a/internal/application/service/knowledge_write_access_test.go b/internal/application/service/knowledge_write_access_test.go new file mode 100644 index 00000000..15a56239 --- /dev/null +++ b/internal/application/service/knowledge_write_access_test.go @@ -0,0 +1,442 @@ +package service + +import ( + "context" + "encoding/json" + "testing" + + "github.com/Tencent/WeKnora/internal/application/access" + "github.com/Tencent/WeKnora/internal/application/repository" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/hibiken/asynq" + "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +type writeKBLookup struct { + interfaces.KnowledgeBaseService + kb *types.KnowledgeBase +} + +func (s *writeKBLookup) GetKnowledgeBaseByID(context.Context, string) (*types.KnowledgeBase, error) { + return s.kb, nil +} + +type writeChunkSpy struct { + interfaces.ChunkRepository + writes int +} + +func (s *writeChunkSpy) UpdateChunkFieldsByTagID( + ctx context.Context, + tenant uint64, + kb, tag string, + enabled *bool, + set, clearFlags types.ChunkFlags, + next *string, + exclude []string, +) ([]string, error) { + s.writes++ + return s.ChunkRepository.UpdateChunkFieldsByTagID(ctx, tenant, kb, tag, enabled, set, clearFlags, next, exclude) +} + +func (s *writeChunkSpy) UpdateChunks(ctx context.Context, chunks []*types.Chunk) error { + s.writes++ + return s.ChunkRepository.UpdateChunks(ctx, chunks) +} + +func (s *writeChunkSpy) UpdateChunkFlagsBatch( + ctx context.Context, + tenant uint64, + kb string, + set, clearFlags map[string]types.ChunkFlags, +) error { + s.writes++ + return s.ChunkRepository.UpdateChunkFlagsBatch(ctx, tenant, kb, set, clearFlags) +} + +type writeChunkDeleter struct { + interfaces.ChunkService + deleted []string +} + +func (s *writeChunkDeleter) DeleteChunk(_ context.Context, id string) error { + s.deleted = append(s.deleted, id) + return nil +} + +type writeIndexSpy struct { + interfaces.RetrieveEngineService + enabled map[string]bool + tags map[string]string +} + +func (*writeIndexSpy) EngineType() types.RetrieverEngineType { + return types.PostgresRetrieverEngineType +} + +func (*writeIndexSpy) Support() []types.RetrieverType { + return []types.RetrieverType{types.VectorRetrieverType} +} + +func (s *writeIndexSpy) BatchUpdateChunkEnabledStatus(_ context.Context, values map[string]bool) error { + s.enabled = values + return nil +} + +func (s *writeIndexSpy) BatchUpdateChunkTagID(_ context.Context, values map[string]string) error { + s.tags = values + return nil +} + +type faqWriteFixture struct { + svc *knowledgeService + tags *knowledgeTagService + db *gorm.DB + ctx context.Context + chunks *writeChunkSpy + deletes *writeChunkDeleter + index *writeIndexSpy + kb *types.KnowledgeBase +} + +func newFAQWriteFixture(t *testing.T) *faqWriteFixture { + t.Helper() + db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{}) + require.NoError(t, err) + require.NoError(t, db.AutoMigrate(&types.Knowledge{}, &types.Chunk{}, &types.KnowledgeTag{})) + store := "write-store" + kb := &types.KnowledgeBase{ID: "kb", TenantID: 7, Type: types.KnowledgeBaseTypeFAQ, VectorStoreID: &store} + lookup := &writeKBLookup{kb: kb} + chunks := &writeChunkSpy{ChunkRepository: repository.NewChunkRepository(db)} + deletes := &writeChunkDeleter{} + index := &writeIndexSpy{} + svc := &knowledgeService{ + kbService: lookup, chunkRepo: chunks, chunkService: deletes, + tagRepo: repository.NewKnowledgeTagRepository(db), repo: repository.NewKnowledgeRepository(db), + retrieveEngine: &fakeFanoutRegistry{byStore: map[string]interfaces.RetrieveEngineService{store: index}}, + ownership: &fakeOwnership{owned: map[string]uint64{store: 7}}, + } + base := types.WithCaller( + context.Background(), + types.Caller{TenantID: 1, UserID: "caller", Role: types.TenantRoleAdmin}, + ) + grant := &access.KBAccess{ + KnowledgeBase: kb, + Caller: types.CallerFromContext(base), + EffectiveTenantID: 7, + Permission: types.OrgRoleEditor, + } + ctx := context.WithValue(grant.Context(base), types.TenantInfoContextKey, &types.Tenant{ID: 7}) + for _, tag := range []*types.KnowledgeTag{ + {ID: "tag", SeqID: 11, TenantID: 7, KnowledgeBaseID: "kb", Name: "tag"}, + {ID: "next", SeqID: 12, TenantID: 7, KnowledgeBaseID: "kb", Name: "next"}, + {ID: "foreign", SeqID: 22, TenantID: 7, KnowledgeBaseID: "other", Name: "foreign"}, + } { + require.NoError(t, db.Create(tag).Error) + } + for _, chunk := range []*types.Chunk{ + { + ID: "one", + SeqID: 1, + TenantID: 7, + KnowledgeBaseID: "kb", + KnowledgeID: "faq", + ChunkType: types.ChunkTypeFAQ, + TagID: "tag", + IsEnabled: true, + }, + + { + ID: "two", + SeqID: 2, + TenantID: 7, + KnowledgeBaseID: "other", + KnowledgeID: "foreign-faq", + ChunkType: types.ChunkTypeFAQ, + TagID: "foreign", + IsEnabled: true, + }, + } { + require.NoError(t, db.Create(chunk).Error) + } + require.NoError( + t, + db.Create(&types.Knowledge{ID: "faq", TenantID: 7, KnowledgeBaseID: "kb", Type: types.KnowledgeTypeFAQ}).Error, + ) + tags := &knowledgeTagService{kbService: lookup, repo: svc.tagRepo} + tags.chunkRepo = chunks + svc.tagService = tags + return &faqWriteFixture{ + svc: svc, + tags: tags, + db: db, + ctx: ctx, + chunks: chunks, + deletes: deletes, + index: index, + kb: kb, + } +} + +func TestFAQAndTagWritesRejectUnscopedServiceCalls(t *testing.T) { + f := newFAQWriteFixture(t) + ctx := types.WithExecutionTenant(types.WithCaller(context.Background(), types.Caller{TenantID: 1}), 7) + operations := map[string]func(context.Context) error{ + "create FAQ": func(ctx context.Context) error { + _, err := f.svc.CreateFAQEntry(ctx, "kb", &types.FAQEntryPayload{}) + return err + }, + "update FAQ": func(ctx context.Context) error { + _, err := f.svc.UpdateFAQEntry(ctx, "kb", 1, &types.FAQEntryPayload{}) + return err + }, + "similar questions": func(ctx context.Context) error { + _, err := f.svc.AddSimilarQuestions(ctx, "kb", 1, []string{"question"}) + return err + }, + "status": func(ctx context.Context) error { return f.svc.UpdateFAQEntryStatus(ctx, "kb", "one", false) }, + "fields": func(ctx context.Context) error { + return f.svc.UpdateFAQEntryFieldsBatch( + ctx, + "kb", + &types.FAQEntryFieldsBatchUpdate{ByID: map[int64]types.FAQEntryFieldsUpdate{1: {}}}, + ) + }, + "single tag": func(ctx context.Context) error { return f.svc.UpdateFAQEntryTag(ctx, "kb", "one", nil) }, + "batch tags": func(ctx context.Context) error { + return f.svc.UpdateFAQEntryTagBatch(ctx, "kb", map[int64]*int64{1: nil}) + }, + "delete": func(ctx context.Context) error { return f.svc.DeleteFAQEntries(ctx, "kb", []int64{1}) }, + "import": func(ctx context.Context) error { + _, err := f.svc.UpsertFAQEntries( + ctx, + "kb", + &types.FAQBatchUpsertPayload{Entries: []types.FAQEntryPayload{{}}}, + ) + return err + }, + "import display": func(ctx context.Context) error { + return f.svc.UpdateLastFAQImportResultDisplayStatus(ctx, "kb", "close") + }, + "create tag": func(ctx context.Context) error { + _, + err := f.tags.CreateTag(ctx, + "kb", + "new", + "", + 0) + return err + }, + + "find/create tag": func(ctx context.Context) error { + _, + err := f.tags.FindOrCreateTagByName(ctx, + "kb", + "tag") + return err + }, + + "update tag": func(ctx context.Context) error { + _, + err := f.tags.UpdateTag(ctx, + "tag", + nil, + nil, + nil) + return err + }, + + "delete tag": func(ctx context.Context) error { return f.tags.DeleteTag(ctx, "tag", true, false, nil) }, + } + for name, operation := range operations { + t.Run(name, func(t *testing.T) { require.Error(t, operation(ctx)) }) + } + require.Zero(t, f.chunks.writes) + require.Empty(t, f.deletes.deleted) + // The same service accepts a correctly scoped shared Editor operation. + tag, err := f.tags.CreateTag(f.ctx, "kb", "created", "", 0) + require.NoError(t, err) + require.Equal(t, uint64(7), tag.TenantID) +} + +func TestFAQBatchPreflightRejectsBeforeAnyWrite(t *testing.T) { + no := false + foreign := int64(22) + for name, req := range map[string]*types.FAQEntryFieldsBatchUpdate{ + "foreign entry after group": { + ByTag: map[int64]types.FAQEntryFieldsUpdate{11: {IsEnabled: &no}}, + ByID: map[int64]types.FAQEntryFieldsUpdate{2: {IsEnabled: &no}}, + }, + + "missing entry after group": { + ByTag: map[int64]types.FAQEntryFieldsUpdate{11: {IsEnabled: &no}}, + ByID: map[int64]types.FAQEntryFieldsUpdate{999: {}}, + }, + + "foreign source tag": {ByTag: map[int64]types.FAQEntryFieldsUpdate{ + 11: {IsEnabled: &no}, + 22: {IsEnabled: &no}, + }}, + + "foreign destination tag": {ByID: map[int64]types.FAQEntryFieldsUpdate{1: {TagID: &foreign}}}, + "foreign exclusion": { + ByTag: map[int64]types.FAQEntryFieldsUpdate{11: {IsEnabled: &no}}, + ExcludeIDs: []int64{2}, + }, + + "missing exclusion": { + ByTag: map[int64]types.FAQEntryFieldsUpdate{11: {IsEnabled: &no}}, + ExcludeIDs: []int64{999}, + }, + } { + t.Run(name, func(t *testing.T) { + f := newFAQWriteFixture(t) + require.Error(t, f.svc.UpdateFAQEntryFieldsBatch(f.ctx, "kb", req)) + require.Zero(t, f.chunks.writes) + chunk, err := f.chunks.GetChunkByID(f.ctx, 7, "one") + require.NoError(t, err) + require.True(t, chunk.IsEnabled) + require.Equal(t, "tag", chunk.TagID) + }) + } +} + +func TestFAQDeleteValidatesEntireSelectionAndParents(t *testing.T) { + for _, ids := range [][]int64{{1, 2}, {1, 999}, {1, 0}} { + t.Run(string(mustWriteJSON(t, ids)), func(t *testing.T) { + f := newFAQWriteFixture(t) + require.Error(t, f.svc.DeleteFAQEntries(f.ctx, "kb", ids)) + require.Empty(t, f.deletes.deleted) + }) + } + t.Run("foreign parent", func(t *testing.T) { + f := newFAQWriteFixture(t) + require.NoError( + t, + f.db.Model(&types.Knowledge{}).Where("id = ?", "faq").Update("knowledge_base_id", "other").Error, + ) + require.Error(t, f.svc.DeleteFAQEntries(f.ctx, "kb", []int64{1})) + require.Empty(t, f.deletes.deleted) + }) +} + +func TestFAQFieldGroupsAndExplicitUpdatesPreservePrecedence(t *testing.T) { + f := newFAQWriteFixture(t) + yes, no := true, false + next := int64(12) + req := &types.FAQEntryFieldsBatchUpdate{ + ByTag: map[int64]types.FAQEntryFieldsUpdate{11: {TagID: &next, IsRecommended: &yes}}, + ByID: map[int64]types.FAQEntryFieldsUpdate{1: {IsEnabled: &no, IsRecommended: &no}}, + } + require.NoError(t, f.svc.UpdateFAQEntryFieldsBatch(f.ctx, "kb", req)) + chunk, err := f.chunks.GetChunkByID(f.ctx, 7, "one") + require.NoError(t, err) + require.False(t, chunk.IsEnabled) + require.Equal(t, "next", chunk.TagID, "an explicit enabled patch must preserve the group's tag change") + require.False(t, chunk.Flags.HasFlag(types.ChunkFlagRecommended)) + require.Equal(t, map[string]string{"one": "next"}, f.index.tags) + require.Equal(t, map[string]bool{"one": false}, f.index.enabled) +} + +func TestFAQTagOnlyBatchUsesValidatedPlanAndSynchronizesIndex(t *testing.T) { + f := newFAQWriteFixture(t) + foreign := int64(22) + require.Error(t, f.svc.UpdateFAQEntryTagBatch(f.ctx, "kb", map[int64]*int64{1: &foreign})) + require.Zero(t, f.chunks.writes) + require.NoError(t, f.svc.UpdateFAQEntryTagBatch(f.ctx, "kb", map[int64]*int64{1: nil})) + require.Equal(t, map[string]string{"one": ""}, f.index.tags) +} + +func TestFAQImportRejectsForeignTagsBeforeEnqueue(t *testing.T) { + f := newFAQWriteFixture(t) + _, err := f.svc.UpsertFAQEntries( + f.ctx, + "kb", + &types.FAQBatchUpsertPayload{Entries: []types.FAQEntryPayload{{TagID: 22}}}, + ) + require.Error(t, err) // Redis, file storage and the queue are intentionally unwired. + require.Zero(t, f.chunks.writes) + _, err = f.svc.resolveTagID(f.ctx, "kb", &types.FAQEntryPayload{TagID: 22}) + require.Error(t, err) + resolver := f.svc.buildFAQTagResolver(f.ctx, "kb", []types.FAQEntryPayload{{TagID: 22}}) + _, err = resolver(&types.FAQEntryPayload{TagID: 22}) + require.Error(t, err) +} + +func TestFAQImportWorkerRejectsMismatchedScopeBeforeSideEffects(t *testing.T) { + for _, tenant := range []uint64{7, 8} { + t.Run(string(mustWriteJSON(t, tenant)), func(t *testing.T) { + f := newFAQWriteFixture(t) + require.NoError( + t, + f.db.Create( + &types.Knowledge{ + ID: "foreign-faq", + TenantID: 7, + KnowledgeBaseID: "other", + Type: types.KnowledgeTypeFAQ, + }, + ).Error, + ) + payload := types.FAQImportPayload{TenantID: tenant, KBID: "kb", KnowledgeID: "foreign-faq", TaskID: "task"} + err := f.svc.ProcessFAQImport( + context.Background(), + asynq.NewTask(types.TypeFAQImport, mustWriteJSON(t, payload)), + ) + require.ErrorIs(t, err, asynq.SkipRetry) + require.Zero(t, f.chunks.writes) + }) + } +} + +func TestTagDeleteRejectsInvalidExclusionsBeforeDeletion(t *testing.T) { + for _, id := range []string{"two", "missing"} { + t.Run(id, func(t *testing.T) { + f := newFAQWriteFixture(t) + require.Error(t, f.tags.DeleteTag(f.ctx, "tag", true, false, []string{id})) + tag, err := f.svc.tagRepo.GetByID(f.ctx, 7, "tag") + require.NoError(t, err) + require.NotNil(t, tag) + chunk, err := f.chunks.GetChunkByID(f.ctx, 7, "one") + require.NoError(t, err) + require.NotNil(t, chunk) + }) + } +} + +func mustWriteJSON(t *testing.T, value interface{}) []byte { + t.Helper() + data, err := json.Marshal(value) + require.NoError(t, err) + return data +} + +func TestSharedFAQWriteLoadsOwnerTenantInfoWithoutReplacingCaller(t *testing.T) { + f := newFAQWriteFixture(t) + f.svc.tenantRepo = &processSyncTenantRepo{tenant: &types.Tenant{ID: 7}} + ctx := context.WithValue(f.ctx, types.TenantInfoContextKey, &types.Tenant{ID: 1}) + _, scoped, err := f.svc.writableFAQKnowledgeBase(ctx, "kb") + require.NoError(t, err) + owner, _ := types.TenantInfoFromContext(scoped) + require.Equal(t, uint64(7), owner.ID) + require.Equal(t, uint64(1), types.CallerFromContext(scoped).TenantID) + original, _ := types.TenantInfoFromContext(ctx) + require.Equal(t, uint64(1), original.ID) +} + +func TestDataSourceTagCreationReceivesOnlyItsTaskKBGrant(t *testing.T) { + h := newSyncDeletionHarness(t, false, "ds-tag-scope", "log-tag-scope", nil, nil) + tags := h.svc.tagService.(*processSyncTagService) + _, err := h.run(t) + require.NoError(t, err) + require.NotNil(t, tags.ctx) + require.Zero(t, types.CallerFromContext(tags.ctx).TenantID) + require.NoError( + t, + access.RequireKBWrite(tags.ctx, &types.KnowledgeBase{ID: h.ds.KnowledgeBaseID, TenantID: h.ds.TenantID}), + ) + require.Error(t, access.RequireKBWrite(tags.ctx, &types.KnowledgeBase{ID: "other", TenantID: h.ds.TenantID})) +} diff --git a/internal/application/service/knowledgebase.go b/internal/application/service/knowledgebase.go index 42a52dfd..76189da6 100644 --- a/internal/application/service/knowledgebase.go +++ b/internal/application/service/knowledgebase.go @@ -9,6 +9,8 @@ import ( "strings" "time" + "github.com/Tencent/WeKnora/internal/application/access" + "github.com/Tencent/WeKnora/internal/application/repository" "github.com/Tencent/WeKnora/internal/application/service/retriever" "github.com/Tencent/WeKnora/internal/datasource" apperrors "github.com/Tencent/WeKnora/internal/errors" @@ -1099,72 +1101,57 @@ func (s *knowledgeBaseService) SetEmbeddingModel(ctx context.Context, id string, // CopyKnowledgeBase copies a knowledge base to a new knowledge base (shallow copy). // Source and target must belong to the tenant in context; cross-tenant access is rejected. // -// Defensive checks: -// -// - When dstKB != "" (clone into an existing target), the source's -// EmbeddingModelID and VectorStoreID must match the target's. Mismatched -// embedding models would silently mix incompatible vector spaces; -// mismatched vector stores would require copying physical vector data -// between stores, which is not yet supported. -// - When dstKB == "" (create a new target), VectorStoreID is copied from -// the source so the new KB shares the same physical vector index. GORM -// `<-:create` allows INSERT, so the new row is well-formed. -// -// The handler's CopyKnowledgeBase endpoint runs the same checks synchronously -// before enqueueing the async clone task, so the 400 errors here are -// defense-in-depth for the worker entry point. +// The transfer grant fixes both resource IDs and whether a new destination +// may be created. Existing targets must be compatible with the source. A +// worker retry reloads the reserved target instead of creating another KB. func (s *knowledgeBaseService) CopyKnowledgeBase(ctx context.Context, srcKB string, dstKB string, ) (*types.KnowledgeBase, *types.KnowledgeBase, error) { - tenantID := types.MustTenantIDFromContext(ctx) - // Load source KB with tenant scope to prevent cross-tenant cloning - sourceKB, err := s.repo.GetKnowledgeBaseByIDAndTenant(ctx, srcKB, tenantID) + destinationID, create, creatorID, err := access.CloneDestination(ctx, srcKB) if err != nil { - logger.Errorf(ctx, "Get source knowledge base failed: %v", err) return nil, nil, err } + if dstKB != "" && dstKB != destinationID { + return nil, nil, access.ErrForbidden + } + tenantID := types.MustTenantIDFromContext(ctx) + sourceKB, err := s.repo.GetKnowledgeBaseByIDAndTenant(ctx, srcKB, tenantID) + if err != nil { + return nil, nil, err + } + if sourceKB == nil || sourceKB.ID != srcKB || sourceKB.TenantID != tenantID { + return nil, nil, access.ErrForbidden + } + sourceCopy := *sourceKB + sourceKB = &sourceCopy sourceKB.EnsureDefaults() - var targetKB *types.KnowledgeBase - if dstKB != "" { - // Load target KB with tenant scope so we only clone into the caller's tenant - targetKB, err = s.repo.GetKnowledgeBaseByIDAndTenant(ctx, dstKB, tenantID) - if err != nil { + targetKB, err := s.repo.GetKnowledgeBaseByIDAndTenant(ctx, destinationID, tenantID) + if err != nil && (!create || !errors.Is(err, repository.ErrKnowledgeBaseNotFound)) { + return nil, nil, err + } + if targetKB != nil { + targetCopy := *targetKB + targetKB = &targetCopy + targetKB.EnsureDefaults() + if err := access.RequireKBTransfer(ctx, sourceKB, targetKB, access.KBTransferClone); err != nil { return nil, nil, err } - - // Defense 1: embedding model must match. Mixing incompatible - // vector spaces would produce semantically broken search results. - if sourceKB.EmbeddingModelID != targetKB.EmbeddingModelID { - return nil, nil, apperrors.NewBadRequestError( - "source and target knowledge bases use different embedding models; " + - "clone into a target with the same embedding model") - } - - // Defense 2: vector store binding must match. Cross-store cloning - // would require copying physical vector data between stores. - // (both nil → equal; both same UUID → equal; otherwise → rejected) - if !sourceKB.SharesStoreWith(targetKB) { - return nil, nil, apperrors.NewBadRequestError( - "source and target knowledge bases are bound to different vector stores; " + - "cross-store cloning is not yet supported") - } - - // Defense 3: the concrete storage instance must match. Comparing only - // provider names would incorrectly allow COS-A -> COS-B clones. - if tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant); tenant != nil { - defaultID, defaultProvider := "", "" - if tenant.DefaultStorageBackendID != nil { - defaultID = *tenant.DefaultStorageBackendID - } - if tenant.StorageEngineConfig != nil { - defaultProvider = tenant.StorageEngineConfig.DefaultProvider - } - if !sourceKB.SharesStorageBackendWith(targetKB, defaultID, defaultProvider) { - return nil, nil, apperrors.NewBadRequestError( - "source and target knowledge bases use different storage instances; cross-storage-backend cloning is not supported") - } + tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) + if err := access.ValidateKBTransferCompatibility(sourceKB, + targetKB, + access.KBTransferClone, + "", + tenant); err != nil { + return nil, nil, apperrors.NewBadRequestError(err.Error()) } } else { + reserved := &types.KnowledgeBase{ID: destinationID, TenantID: tenantID} + if !create { + return nil, nil, access.ErrNotFound + } + if err := access.RequireKBTransfer(ctx, sourceKB, reserved, access.KBTransferClone); err != nil { + return nil, nil, err + } var faqConfig *types.FAQConfig if sourceKB.FAQConfig != nil { cfg := *sourceKB.FAQConfig @@ -1173,7 +1160,8 @@ func (s *knowledgeBaseService) CopyKnowledgeBase(ctx context.Context, // Preserve VectorStoreID so the cloned KB lands on the same // physical index. GORM `<-:create` permits the value at INSERT. targetKB = &types.KnowledgeBase{ - ID: uuid.New().String(), + ID: destinationID, + CreatorID: creatorID, Name: sourceKB.Name, Type: sourceKB.Type, Description: sourceKB.Description, diff --git a/internal/application/service/knowledgebase_access.go b/internal/application/service/knowledgebase_access.go new file mode 100644 index 00000000..b7d4a8a7 --- /dev/null +++ b/internal/application/service/knowledgebase_access.go @@ -0,0 +1,74 @@ +package service + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/application/access" + apperrors "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// Service reads accept exact upstream grants. Otherwise cross-tenant reads +// require a user and organization permission resolved for the original caller. +func kbReadPermissions(ctx context.Context, shares access.KBShareLookup) *access.KBPermissions { + if types.CallerFromContext(ctx).UserID == "" { + shares = nil + } + return access.NewKBPermissions(ctx, shares) +} + +func resolveKBReadTenant(ctx context.Context, kb *types.KnowledgeBase, shares access.KBShareLookup) (uint64, error) { + if kb != nil { + allowed, err := kbReadPermissions(ctx, shares).Check(kb.ID, kb.TenantID, types.OrgRoleViewer) + if err == nil && allowed { + return kb.TenantID, nil + } + } + return 0, apperrors.NewForbiddenError("无权访问该知识库") +} + +func requireKBWrite(ctx context.Context, kb *types.KnowledgeBase) (context.Context, error) { + if err := access.RequireKBWrite(ctx, kb); err != nil { + return ctx, apperrors.NewForbiddenError("无权修改该知识库") + } + return types.WithExecutionTenant(ctx, kb.TenantID), nil +} + +func (s *knowledgeService) writableFAQKnowledgeBase( + ctx context.Context, + kbID string, +) (*types.KnowledgeBase, context.Context, error) { + kb, err := s.validateFAQKnowledgeBase(ctx, kbID) + if err != nil { + return nil, ctx, err + } + ctx, err = requireKBWrite(ctx, kb) + if err == nil { + ctx, err = withKBWriteTenantInfo(ctx, kb, s.tenantRepo) + } + return kb, ctx, err +} + +// A shared KB mutation must also resolve models/index backends against its +// owner. The authenticated caller remains unchanged when TenantInfo changes. +func withKBWriteTenantInfo( + ctx context.Context, + kb *types.KnowledgeBase, + tenants interfaces.TenantRepository, +) (context.Context, error) { + if tenant, ok := types.TenantInfoFromContext(ctx); ok && tenant != nil && tenant.ID == kb.TenantID { + return ctx, nil + } + if tenants == nil { + return ctx, apperrors.NewServiceUnavailableError("无法获取知识库所属空间") + } + tenant, err := tenants.GetTenantByID(ctx, kb.TenantID) + if err != nil { + return ctx, err + } + if tenant == nil || tenant.ID != kb.TenantID { + return ctx, apperrors.NewNotFoundError("知识库所属空间不存在") + } + return context.WithValue(ctx, types.TenantInfoContextKey, tenant), nil +} diff --git a/internal/application/service/knowledgebase_pr3_test.go b/internal/application/service/knowledgebase_pr3_test.go index d24616f7..d5bd3a61 100644 --- a/internal/application/service/knowledgebase_pr3_test.go +++ b/internal/application/service/knowledgebase_pr3_test.go @@ -6,6 +6,8 @@ import ( "testing" "time" + "github.com/Tencent/WeKnora/internal/application/access" + "github.com/Tencent/WeKnora/internal/application/repository" "github.com/Tencent/WeKnora/internal/application/service/retriever" apperrors "github.com/Tencent/WeKnora/internal/errors" "github.com/Tencent/WeKnora/internal/storageallowlist" @@ -87,22 +89,27 @@ func (r *fakeKBRepo) CreateKnowledgeBase(_ context.Context, kb *types.KnowledgeB r.rows[kb.ID] = kb return nil } + func (r *fakeKBRepo) GetKnowledgeBaseByID(_ context.Context, id string) (*types.KnowledgeBase, error) { return r.rows[id], nil } + func (r *fakeKBRepo) GetKnowledgeBaseByIDAndTenant(_ context.Context, id string, tenantID uint64) (*types.KnowledgeBase, error) { kb := r.rows[id] if kb == nil || kb.TenantID != tenantID { - return nil, stderrors.New("not found") + return nil, repository.ErrKnowledgeBaseNotFound } return kb, nil } + func (r *fakeKBRepo) GetKnowledgeBaseByIDs(_ context.Context, _ []string) ([]*types.KnowledgeBase, error) { return nil, nil } + func (r *fakeKBRepo) ListKnowledgeBases(_ context.Context) ([]*types.KnowledgeBase, error) { return nil, nil } + func (r *fakeKBRepo) ListKnowledgeBasesByTenantID(_ context.Context, tenantID uint64) ([]*types.KnowledgeBase, error) { rows := make([]*types.KnowledgeBase, 0, len(r.rows)) for _, kb := range r.rows { @@ -112,6 +119,7 @@ func (r *fakeKBRepo) ListKnowledgeBasesByTenantID(_ context.Context, tenantID ui } return rows, nil } + func (r *fakeKBRepo) UpdateKnowledgeBase(_ context.Context, _ *types.KnowledgeBase) error { return nil } @@ -119,9 +127,11 @@ func (r *fakeKBRepo) DeleteKnowledgeBase(_ context.Context, _ string) error { re func (r *fakeKBRepo) TogglePinKnowledgeBase(_ context.Context, _ string, _ uint64) (*types.KnowledgeBase, error) { return nil, nil } + func (r *fakeKBRepo) CountByVectorStoreID(_ context.Context, _ *gorm.DB, _ uint64, _ string) (int64, error) { return 0, nil } + func (r *fakeKBRepo) CountByModelID(_ context.Context, _ uint64, _ string) (int64, error) { return 0, nil } @@ -322,7 +332,7 @@ func TestCopyKnowledgeBase_Defenses(t *testing.T) { repo.rows["src"] = mkKB("src", 1, "embed-1", &storeA) svc := newPR3KBService(repo, &fakeRegistry{}, &fakeOwnership{}) - src, tgt, err := svc.CopyKnowledgeBase(ctxWithTenant(1), "src", "") + src, tgt, err := svc.CopyKnowledgeBase(copyTaskTestContext(t, repo, ""), "src", "") require.NoError(t, err) require.NotNil(t, src.VectorStoreID) require.NotNil(t, tgt.VectorStoreID) @@ -340,7 +350,7 @@ func TestCopyKnowledgeBase_Defenses(t *testing.T) { repo.rows["src"] = mkKB("src", 1, "embed-1", nil) svc := newPR3KBService(repo, &fakeRegistry{}, &fakeOwnership{}) - _, tgt, err := svc.CopyKnowledgeBase(ctxWithTenant(1), "src", "") + _, tgt, err := svc.CopyKnowledgeBase(copyTaskTestContext(t, repo, ""), "src", "") require.NoError(t, err) assert.Nil(t, tgt.VectorStoreID) }) @@ -351,7 +361,7 @@ func TestCopyKnowledgeBase_Defenses(t *testing.T) { repo.rows["dst"] = mkKB("dst", 1, "embed-1", nil) svc := newPR3KBService(repo, &fakeRegistry{}, &fakeOwnership{}) - _, _, err := svc.CopyKnowledgeBase(ctxWithTenant(1), "src", "dst") + _, _, err := svc.CopyKnowledgeBase(copyTaskTestContext(t, repo, "dst"), "src", "dst") require.NoError(t, err) }) @@ -361,7 +371,7 @@ func TestCopyKnowledgeBase_Defenses(t *testing.T) { repo.rows["dst"] = mkKB("dst", 1, "embed-1", &storeA) svc := newPR3KBService(repo, &fakeRegistry{}, &fakeOwnership{}) - _, _, err := svc.CopyKnowledgeBase(ctxWithTenant(1), "src", "dst") + _, _, err := svc.CopyKnowledgeBase(copyTaskTestContext(t, repo, "dst"), "src", "dst") require.NoError(t, err) }) @@ -371,7 +381,7 @@ func TestCopyKnowledgeBase_Defenses(t *testing.T) { repo.rows["dst"] = mkKB("dst", 1, "embed-2", &storeA) svc := newPR3KBService(repo, &fakeRegistry{}, &fakeOwnership{}) - _, _, err := svc.CopyKnowledgeBase(ctxWithTenant(1), "src", "dst") + _, _, err := svc.CopyKnowledgeBase(copyTaskTestContext(t, repo, "dst"), "src", "dst") require.Error(t, err) assert.Contains(t, err.Error(), "different embedding models") }) @@ -382,7 +392,7 @@ func TestCopyKnowledgeBase_Defenses(t *testing.T) { repo.rows["dst"] = mkKB("dst", 1, "embed-1", &storeB) svc := newPR3KBService(repo, &fakeRegistry{}, &fakeOwnership{}) - _, _, err := svc.CopyKnowledgeBase(ctxWithTenant(1), "src", "dst") + _, _, err := svc.CopyKnowledgeBase(copyTaskTestContext(t, repo, "dst"), "src", "dst") require.Error(t, err) assert.Contains(t, err.Error(), "different vector stores") assert.NotContains(t, err.Error(), "Phase 4", "internal roadmap labels must not leak to end-user error messages") @@ -394,7 +404,7 @@ func TestCopyKnowledgeBase_Defenses(t *testing.T) { repo.rows["dst"] = mkKB("dst", 1, "embed-1", &storeA) svc := newPR3KBService(repo, &fakeRegistry{}, &fakeOwnership{}) - _, _, err := svc.CopyKnowledgeBase(ctxWithTenant(1), "src", "dst") + _, _, err := svc.CopyKnowledgeBase(copyTaskTestContext(t, repo, "dst"), "src", "dst") require.Error(t, err) assert.Contains(t, err.Error(), "different vector stores") }) @@ -525,3 +535,22 @@ func TestDuplicateKnowledgeBase_UsesLocalizedEnglishSuffix(t *testing.T) { assert.Equal(t, "Source KB Copy", target.Name) } + +func copyTaskTestContext(t *testing.T, repo *fakeKBRepo, dst string) context.Context { + t.Helper() + target := repo.rows[dst] + if target == nil { + target = &types.KnowledgeBase{ID: "reserved-copy", TenantID: 1} + } + ctx, err := access.WithKBTransferTask( + context.Background(), + repo.rows["src"], + target, + 1, + access.KBTransferClone, + "clone-task", + dst == "", + ) + require.NoError(t, err) + return ctx +} diff --git a/internal/application/service/knowledgebase_search.go b/internal/application/service/knowledgebase_search.go index 2be71595..cca50c9a 100644 --- a/internal/application/service/knowledgebase_search.go +++ b/internal/application/service/knowledgebase_search.go @@ -97,7 +97,7 @@ func (s *knowledgeBaseService) ResolveEmbeddingModelKeys(ctx context.Context, kb // Resolve each unique (modelID, tenantID) to a model identity key resolvedKeys := make(map[modelRef]string, len(uniqueRefs)) for ref := range uniqueRefs { - tenantCtx := context.WithValue(ctx, types.TenantIDContextKey, ref.TenantID) + tenantCtx := types.WithExecutionTenant(ctx, ref.TenantID) model, err := s.modelService.GetModelByID(tenantCtx, ref.ModelID) if err != nil || model == nil { logger.Warnf(ctx, "ResolveEmbeddingModelKeys: cannot resolve model %s for tenant %d: %v", ref.ModelID, ref.TenantID, err) @@ -143,7 +143,6 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context, searchKBIDs, secutils.SanitizeForLog(params.QueryText)) tenantInfo, _ := types.TenantInfoFromContext(ctx) - requestTenantID := types.MustTenantIDFromContext(ctx) // Batch-load every KB in scope. Required for store grouping, // embedding-model consistency validation, and FAQ type detection. @@ -162,13 +161,13 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context, return nil, apperrors.NewNotFoundError("knowledge base not found") } - // Authorize every KB the caller asked for. Same-tenant KBs are - // always accessible; foreign-tenant KBs (Organization-shared) must - // pass an explicit per-KB permission check. Without this guard, a + // Authorize every KB for the original caller, using exact upstream + // grants or organization permissions. Execution in a shared tenant + // does not grant access to its other KBs. Without this guard, a // caller could pass arbitrary KB UUIDs in params.KnowledgeBaseIDs // and reach foreign tenants' bound vector stores via the per-group // engine resolution downstream. - if err := s.authorizeKBAccess(ctx, kbs, requestTenantID); err != nil { + if err := s.authorizeKBAccess(ctx, kbs); err != nil { return nil, err } diff --git a/internal/application/service/knowledgebase_search_fanout_test.go b/internal/application/service/knowledgebase_search_fanout_test.go index 12043a9d..79bbd3ee 100644 --- a/internal/application/service/knowledgebase_search_fanout_test.go +++ b/internal/application/service/knowledgebase_search_fanout_test.go @@ -415,27 +415,35 @@ func (f *fakeRetrieveEngineService) Retrieve(ctx context.Context, p types.Retrie func (f *fakeRetrieveEngineService) Index(context.Context, embedding.Embedder, *types.IndexInfo, []types.RetrieverType) error { panic("unused") } + func (f *fakeRetrieveEngineService) BatchIndex(context.Context, embedding.Embedder, []*types.IndexInfo, []types.RetrieverType) error { panic("unused") } + func (f *fakeRetrieveEngineService) EstimateStorageSize(context.Context, embedding.Embedder, []*types.IndexInfo, []types.RetrieverType) int64 { panic("unused") } + func (f *fakeRetrieveEngineService) CopyIndices(context.Context, string, map[string]string, map[string]string, string, int, string) error { panic("unused") } + func (f *fakeRetrieveEngineService) DeleteByChunkIDList(context.Context, []string, int, string) error { panic("unused") } + func (f *fakeRetrieveEngineService) DeleteBySourceIDList(context.Context, []string, int, string) error { panic("unused") } + func (f *fakeRetrieveEngineService) DeleteByKnowledgeIDList(context.Context, []string, int, string) error { panic("unused") } + func (f *fakeRetrieveEngineService) BatchUpdateChunkEnabledStatus(context.Context, map[string]bool) error { panic("unused") } + func (f *fakeRetrieveEngineService) BatchUpdateChunkTagID(context.Context, map[string]string) error { panic("unused") } @@ -455,9 +463,11 @@ func (r *fakeFanoutRegistry) Register(interfaces.RetrieveEngineService) error { func (r *fakeFanoutRegistry) GetRetrieveEngineService(types.RetrieverEngineType) (interfaces.RetrieveEngineService, error) { return nil, stderrors.New("not used in fan-out tests") } + func (r *fakeFanoutRegistry) GetAllRetrieveEngineServices() []interfaces.RetrieveEngineService { return nil } + func (r *fakeFanoutRegistry) GetByStoreID(id string) (interfaces.RetrieveEngineService, error) { if svc, ok := r.byStore[id]; ok { return svc, nil @@ -775,7 +785,7 @@ func TestRetrieveFromStores_PerGroupTimeout(t *testing.T) { // --------------------------------------------------------------------------- // fakeKBShareForAuth implements just enough of KBShareService for the -// authorizeKBAccess test matrix. Only HasTenantKBPermission is exercised; +// authorizeKBAccess test matrix. Only CheckTenantKBPermission is exercised; // the embedded interface keeps the type assignable. type fakeKBShareForAuth struct { // allowed maps kbID → tenantID → allowed. Mirrors the Plan 3 (#1303) @@ -785,17 +795,10 @@ type fakeKBShareForAuth struct { interfaces.KBShareService } -func (f *fakeKBShareForAuth) HasTenantKBPermission( - _ context.Context, kbID string, callerTenantID uint64, - _ types.TenantRole, _ types.OrgMemberRole, -) (bool, error) { - if f.err != nil { - return false, f.err - } - if perTenant, ok := f.allowed[kbID]; ok { - return perTenant[callerTenantID], nil - } - return false, nil +func (f *fakeKBShareForAuth) CheckTenantKBPermission( + _ context.Context, kbID string, callerTenantID uint64, _ types.TenantRole, +) (types.OrgMemberRole, bool, error) { + return types.OrgRoleViewer, f.allowed[kbID][callerTenantID], f.err } func ctxWithTenantForAuth(tenantID uint64) context.Context { @@ -809,7 +812,7 @@ func TestAuthorizeKBAccess_SameTenantAllPass(t *testing.T) { {ID: "kb-1", TenantID: 7}, {ID: "kb-2", TenantID: 7}, } - err := s.authorizeKBAccess(ctxWithTenantForAuth(7), kbs, 7) + err := s.authorizeKBAccess(ctxWithTenantForAuth(7), kbs) require.NoError(t, err) } @@ -825,7 +828,7 @@ func TestAuthorizeKBAccess_ForeignTenantWithShare_OK(t *testing.T) { {ID: "kb-own", TenantID: 7}, {ID: "kb-foreign", TenantID: 99}, } - err := s.authorizeKBAccess(ctxWithTenantForAuth(7), kbs, 7) + err := s.authorizeKBAccess(ctxWithTenantForAuth(7), kbs) require.NoError(t, err) } @@ -840,7 +843,7 @@ func TestAuthorizeKBAccess_ForeignTenantNoShare_NotFound(t *testing.T) { kbs := []*types.KnowledgeBase{ {ID: "kb-foreign", TenantID: 99}, } - err := s.authorizeKBAccess(ctxWithTenantForAuth(7), kbs, 7) + err := s.authorizeKBAccess(ctxWithTenantForAuth(7), kbs) require.Error(t, err) app, ok := apperrors.IsAppError(err) require.True(t, ok, "expected typed AppError, got %T", err) @@ -853,7 +856,7 @@ func TestAuthorizeKBAccess_PermissionLookupError_500(t *testing.T) { share := &fakeKBShareForAuth{err: stderrors.New("share infra down")} s := &knowledgeBaseService{kbShareService: share} kbs := []*types.KnowledgeBase{{ID: "kb-foreign", TenantID: 99}} - err := s.authorizeKBAccess(ctxWithTenantForAuth(7), kbs, 7) + err := s.authorizeKBAccess(ctxWithTenantForAuth(7), kbs) require.Error(t, err) app, ok := apperrors.IsAppError(err) require.True(t, ok) @@ -863,7 +866,7 @@ func TestAuthorizeKBAccess_PermissionLookupError_500(t *testing.T) { func TestAuthorizeKBAccess_EmptyKBs_OK(t *testing.T) { t.Parallel() s := &knowledgeBaseService{kbShareService: &fakeKBShareForAuth{}} - err := s.authorizeKBAccess(ctxWithTenantForAuth(7), nil, 7) + err := s.authorizeKBAccess(ctxWithTenantForAuth(7), nil) require.NoError(t, err) } diff --git a/internal/application/service/knowledgebase_search_shared.go b/internal/application/service/knowledgebase_search_shared.go index c7a0aec5..a47bbb12 100644 --- a/internal/application/service/knowledgebase_search_shared.go +++ b/internal/application/service/knowledgebase_search_shared.go @@ -2,7 +2,9 @@ package service import ( "context" + "errors" + "github.com/Tencent/WeKnora/internal/application/repository" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" ) @@ -30,52 +32,42 @@ func (s *knowledgeBaseService) fetchKnowledgeData(ctx context.Context, } // fetchKnowledgeDataWithShared gets knowledge data in batch, including knowledge -// from shared KBs the user has access to. +// from KBs authorized for the original caller. Initial tenant-scoped rows +// require the same permission check as cross-tenant expansion. func (s *knowledgeBaseService) fetchKnowledgeDataWithShared(ctx context.Context, tenantID uint64, knowledgeIDs []string, ) (map[string]*types.Knowledge, error) { - knowledgeMap, err := s.fetchKnowledgeData(ctx, tenantID, knowledgeIDs) + rows, err := s.fetchKnowledgeData(ctx, tenantID, knowledgeIDs) if err != nil { return nil, err } - - missingIDs := s.findMissingIDs(knowledgeIDs, func(id string) bool { - return knowledgeMap[id] != nil - }) - if len(missingIDs) == 0 { - return knowledgeMap, nil + permissions := kbReadPermissions(ctx, s.kbShareService) + knowledgeMap := make(map[string]*types.Knowledge, len(rows)) + appendAllowed := func(k *types.Knowledge) { + if k == nil { + return + } + allowed, err := permissions.Check(k.KnowledgeBaseID, k.TenantID, types.OrgRoleViewer) + if err == nil && allowed { + knowledgeMap[k.ID] = k + } } - logger.Infof(ctx, "[fetchKnowledgeDataWithShared] %d knowledge IDs not found in current tenant, attempting shared KB lookup", len(missingIDs)) - - userID, ok := s.extractUserID(ctx) - if !ok { - logger.Warnf(ctx, "[fetchKnowledgeDataWithShared] userID not found or empty in context, skipping shared KB lookup") - return knowledgeMap, nil + for _, k := range rows { + appendAllowed(k) } - - logger.Infof(ctx, "[fetchKnowledgeDataWithShared] Looking up %d missing knowledge IDs with userID=%s", len(missingIDs), userID) - callerTenantRole := types.TenantRoleFromContext(ctx) - for _, id := range missingIDs { + for _, id := range knowledgeIDs { + if rows[id] != nil { + continue + } k, err := s.kgRepo.GetKnowledgeByIDOnly(ctx, id) - if err != nil || k == nil || k.KnowledgeBaseID == "" { - logger.Debugf(ctx, "[fetchKnowledgeDataWithShared] Knowledge %s not found or has no KB", id) - continue + if err != nil && !errors.Is(err, repository.ErrKnowledgeNotFound) { + return nil, err } - hasPermission, err := s.kbShareService.HasTenantKBPermission(ctx, k.KnowledgeBaseID, tenantID, callerTenantRole, types.OrgRoleViewer) - if err != nil { - logger.Debugf(ctx, "[fetchKnowledgeDataWithShared] Permission check error for KB %s: %v", k.KnowledgeBaseID, err) - continue + if err == nil { + appendAllowed(k) } - if !hasPermission { - logger.Debugf(ctx, "[fetchKnowledgeDataWithShared] No permission for KB %s", k.KnowledgeBaseID) - continue - } - logger.Debugf(ctx, "[fetchKnowledgeDataWithShared] Found shared knowledge %s in KB %s", id, k.KnowledgeBaseID) - knowledgeMap[k.ID] = k } - - logger.Infof(ctx, "[fetchKnowledgeDataWithShared] After shared lookup, total knowledge found: %d", len(knowledgeMap)) return knowledgeMap, nil } @@ -84,58 +76,41 @@ func (s *knowledgeBaseService) listChunksByIDWithShared(ctx context.Context, tenantID uint64, chunkIDs []string, ) ([]*types.Chunk, error) { - chunks, err := s.chunkRepo.ListChunksByID(ctx, tenantID, chunkIDs) + rows, err := s.chunkRepo.ListChunksByID(ctx, tenantID, chunkIDs) if err != nil { return nil, err } - - foundSet := make(map[string]bool, len(chunks)) - for _, c := range chunks { + permissions := kbReadPermissions(ctx, s.kbShareService) + chunks := make([]*types.Chunk, 0, len(rows)) + foundSet := make(map[string]bool) + appendAllowed := func(c *types.Chunk) { + if c == nil || foundSet[c.ID] { + return + } + allowed, err := permissions.Check(c.KnowledgeBaseID, c.TenantID, types.OrgRoleViewer) + if err == nil && allowed { + chunks = append(chunks, c) + foundSet[c.ID] = true + } + } + for _, c := range rows { + appendAllowed(c) if c != nil { foundSet[c.ID] = true } } - - missing := s.findMissingIDs(chunkIDs, func(id string) bool { - return foundSet[id] - }) + missing := s.findMissingIDs(chunkIDs, func(id string) bool { return foundSet[id] }) if len(missing) == 0 { return chunks, nil } - logger.Infof(ctx, "[listChunksByIDWithShared] %d chunks not found in current tenant, attempting shared KB lookup", len(missing)) - - userID, ok := s.extractUserID(ctx) - if !ok { - logger.Warnf(ctx, "[listChunksByIDWithShared] userID not found or empty in context, skipping shared KB lookup") - return chunks, nil - } - - logger.Infof(ctx, "[listChunksByIDWithShared] Looking up %d missing chunks with userID=%s", len(missing), userID) - callerTenantRole := types.TenantRoleFromContext(ctx) crossChunks, err := s.chunkRepo.ListChunksByIDOnly(ctx, missing) if err != nil { logger.Warnf(ctx, "[listChunksByIDWithShared] Failed to fetch chunks by ID only: %v", err) - return chunks, nil + return nil, err } - logger.Infof(ctx, "[listChunksByIDWithShared] Found %d chunks without tenant filter", len(crossChunks)) - for _, c := range crossChunks { - if c == nil || c.KnowledgeBaseID == "" { - continue - } - hasPermission, err := s.kbShareService.HasTenantKBPermission(ctx, c.KnowledgeBaseID, tenantID, callerTenantRole, types.OrgRoleViewer) - if err != nil { - logger.Debugf(ctx, "[listChunksByIDWithShared] Permission check error for KB %s: %v", c.KnowledgeBaseID, err) - continue - } - if !hasPermission { - logger.Debugf(ctx, "[listChunksByIDWithShared] No permission for KB %s", c.KnowledgeBaseID) - continue - } - chunks = append(chunks, c) + appendAllowed(c) } - - logger.Infof(ctx, "[listChunksByIDWithShared] After shared lookup, total chunks: %d", len(chunks)) return chunks, nil } @@ -149,16 +124,3 @@ func (s *knowledgeBaseService) findMissingIDs(ids []string, exists func(string) } return missing } - -// extractUserID extracts the user ID from context, returning ("", false) if not found. -func (s *knowledgeBaseService) extractUserID(ctx context.Context) (string, bool) { - userIDVal := ctx.Value(types.UserIDContextKey) - if userIDVal == nil { - return "", false - } - userID, ok := userIDVal.(string) - if !ok || userID == "" { - return "", false - } - return userID, true -} diff --git a/internal/application/service/knowledgebase_search_storegroup.go b/internal/application/service/knowledgebase_search_storegroup.go index 8e194c94..7f8c724b 100644 --- a/internal/application/service/knowledgebase_search_storegroup.go +++ b/internal/application/service/knowledgebase_search_storegroup.go @@ -6,6 +6,7 @@ import ( "fmt" "time" + "github.com/Tencent/WeKnora/internal/application/access" "github.com/Tencent/WeKnora/internal/application/service/retriever" apperrors "github.com/Tencent/WeKnora/internal/errors" "github.com/Tencent/WeKnora/internal/logger" @@ -184,9 +185,9 @@ func classifyFactoryError( } // authorizeKBAccess rejects multi-KB searches whose scope includes a KB -// that the caller is not entitled to read. Same-tenant KBs always pass. -// Foreign-tenant KBs (Organization-shared) must pass an explicit -// tenant-scoped permission check via kbShareService.HasTenantKBPermission, +// that the original caller is not entitled to read. Exact upstream grants +// support shared KB/agent execution; other foreign KBs need an organization +// permission check for the caller via access.KBPermissions, // applying the 3-D cap (share role + caller's tenant-org role + tenant // Viewer cap) introduced in Plan 3 of #1303. // @@ -198,20 +199,24 @@ func classifyFactoryError( func (s *knowledgeBaseService) authorizeKBAccess( ctx context.Context, kbs []*types.KnowledgeBase, - requestTenantID uint64, ) error { if len(kbs) == 0 { return nil } - callerTenantRole := types.TenantRoleFromContext(ctx) + kbIDs := make([]string, 0, len(kbs)) + for _, kb := range kbs { + kbIDs = append(kbIDs, kb.ID) + } + if err := types.AuthorizeTenantAPIKeyKnowledgeBases(ctx, kbIDs...); err != nil { + return err + } + + requestTenantID := types.CallerFromContext(ctx).TenantID + permissions := access.NewKBPermissions(ctx, s.kbShareService) for _, kb := range kbs { - if kb.TenantID == requestTenantID { - continue - } - hasPermission, permErr := s.kbShareService.HasTenantKBPermission( - ctx, kb.ID, requestTenantID, callerTenantRole, types.OrgRoleViewer) + hasPermission, permErr := permissions.Check(kb.ID, kb.TenantID, types.OrgRoleViewer) if permErr != nil { logger.ErrorWithFields(ctx, permErr, map[string]interface{}{ "caller_tenant_id": requestTenantID, diff --git a/internal/application/service/message.go b/internal/application/service/message.go index 26074725..b0b71926 100644 --- a/internal/application/service/message.go +++ b/internal/application/service/message.go @@ -446,7 +446,7 @@ func (s *messageService) DeleteMessageKnowledge(ctx context.Context, knowledgeID return } logger.Infof(ctx, "Deleting chat history knowledge entry: %s", knowledgeID) - if err := s.knowService.DeleteKnowledge(ctx, knowledgeID); err != nil { + if err := deleteReferencedKnowledge(ctx, s.knowService, "", []string{knowledgeID}); err != nil { logger.Warnf(ctx, "Failed to delete chat history knowledge %s: %v", knowledgeID, err) } } @@ -466,7 +466,7 @@ func (s *messageService) DeleteSessionKnowledge(ctx context.Context, sessionID s } logger.Infof(ctx, "Deleting %d chat history knowledge entries for session %s", len(knowledgeIDs), sessionID) - if err := s.knowService.DeleteKnowledgeList(ctx, knowledgeIDs); err != nil { + if err := deleteReferencedKnowledge(ctx, s.knowService, "", knowledgeIDs); err != nil { logger.Warnf(ctx, "Failed to batch delete chat history knowledge for session %s: %v", sessionID, err) } } diff --git a/internal/application/service/resource.go b/internal/application/service/resource.go index f6b25772..c46443bc 100644 --- a/internal/application/service/resource.go +++ b/internal/application/service/resource.go @@ -155,6 +155,47 @@ func (s *resourceCatalog) Bind(ctx context.Context, reference, ownerType, ownerI }) } +func (s *resourceCatalog) IsReferencedByKnowledgeBase( + ctx context.Context, + tenantID uint64, + kbID, reference string, +) (bool, error) { + physical, resource, err := s.ResolvePath(ctx, reference) + if err != nil { + return false, err + } + if resource == nil { + resource, err = s.repo.GetByTenantLocation(ctx, tenantID, resourceLocationHash(physical)) + if err != nil { + return false, err + } + } + if resource == nil || resource.TenantID != tenantID { + return false, nil + } + return s.repo.IsReferencedByKnowledgeBase(ctx, tenantID, kbID, resource.ID) +} + +// GetMessageFileBindings resolves registered aliases before reading authoritative origins. +func (s *resourceCatalog) GetMessageFileBindings( + ctx context.Context, tenantID uint64, reference, messageID string, +) (*types.MessageFileBindings, error) { + physical, resource, err := s.ResolvePath(ctx, reference) + if err != nil { + return nil, err + } + if resource == nil { + resource, err = s.repo.GetByTenantLocation(ctx, tenantID, resourceLocationHash(physical)) + if err != nil { + return nil, err + } + } + if resource == nil || resource.TenantID != tenantID { + return &types.MessageFileBindings{}, nil + } + return s.repo.GetMessageFileBindings(ctx, tenantID, resource.ID, messageID) +} + // Release implements interfaces.ResourceCatalog. // // Unbinding and counting are deliberately not a single transaction. A racing diff --git a/internal/application/service/resource_file_authorization_test.go b/internal/application/service/resource_file_authorization_test.go new file mode 100644 index 00000000..6475d518 --- /dev/null +++ b/internal/application/service/resource_file_authorization_test.go @@ -0,0 +1,169 @@ +package service + +import ( + "context" + "testing" + + "github.com/Tencent/WeKnora/internal/application/access" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/stretchr/testify/require" +) + +type catalogFileShares struct{} + +func (catalogFileShares) CheckTenantKBPermission( + _ context.Context, + kb string, + _ uint64, + _ types.TenantRole, +) (types.OrgMemberRole, bool, error) { + return types.OrgRoleViewer, kb == "shared", nil +} + +func (catalogFileShares) GetKnowledgeBasesByIDsOnly(context.Context, []string) ([]*types.KnowledgeBase, error) { + return []*types.KnowledgeBase{{ID: "shared", TenantID: 7}}, nil +} + +type catalogFileAgent struct{ agent *types.CustomAgent } + +func (a *catalogFileAgent) GetSharedAgentForTenant( + context.Context, + uint64, + types.TenantRole, + string, + ...uint64, +) (*types.CustomAgent, error) { + return a.agent, nil +} + +func TestCatalogFileAuthorizationRejectsPrivateHandlesInSharedText(t *testing.T) { + catalog, db := newResourceCatalogForTest(t) + require.NoError(t, db.AutoMigrate(&types.KnowledgeBase{}, &types.Knowledge{}, &types.Chunk{}, &types.WikiPage{})) + ctx := newSharedAccessContext() + const physical = "local://7/exports/private.png" + ref, err := catalog.Register(ctx, 7, physical, interfaces.ResourceRegistration{}) + require.NoError(t, err) + for _, kb := range []string{"private", "shared"} { + require.NoError(t, db.Create(&types.KnowledgeBase{ID: kb, TenantID: 7}).Error) + require.NoError( + t, + db.Create(&types.Knowledge{ID: kb + "-doc", TenantID: 7, KnowledgeBaseID: kb, Type: "file"}).Error, + ) + } + require.NoError( + t, + catalog.Bind(ctx, ref, types.ResourceOwnerKnowledge, "private-doc", types.ResourceRelationSourceFile), + ) + text := "![private](" + ref + ") ![legacy](" + physical + ")" + require.NoError( + t, + db.Create( + &types.Chunk{ID: "chunk", TenantID: 7, KnowledgeBaseID: "shared", KnowledgeID: "shared-doc", Content: text}, + ).Error, + ) + require.NoError( + t, + db.Create(&types.WikiPage{ID: "wiki", TenantID: 7, KnowledgeBaseID: "shared", Content: text}).Error, + ) + grant := &access.KBAccess{ + KnowledgeBase: &types.KnowledgeBase{ID: "shared", TenantID: 7}, + Caller: types.CallerFromContext(ctx), + EffectiveTenantID: 7, + Permission: types.OrgRoleViewer, + } + lookup := catalog.(interfaces.KBResourceLookup) + message := &types.Message{ + ID: "message", + AgentTenantID: 1, + Content: text, + KnowledgeReferences: []*types.SearchResult{{KnowledgeBaseID: "shared", Content: text}}, + } + shares := access.MessageKBShareAuthorizer{ShareGuard: catalogFileShares{}, KBs: catalogFileShares{}} + check := func(allowed bool) { + t.Helper() + for _, path := range []string{ref, physical} { + _, err := access.ResolveKBFile(ctx, grant, "shared", path, catalog, lookup) + if allowed { + require.NoError(t, err) + } else { + require.ErrorIs(t, err, access.ErrForbidden) + } + _, err = access.AuthorizeMessageFile(ctx, message, path, nil, catalog, shares) + if allowed && path == ref { + require.NoError(t, err) + } else { + require.ErrorIs(t, err, access.ErrForbidden) + } + } + } + check(false) + require.NoError( + t, + catalog.Bind(ctx, ref, types.ResourceOwnerKnowledge, "shared-doc", types.ResourceRelationSourceFile), + ) + check(true) + require.NoError(t, db.Where("id = ?", "shared").Delete(&types.KnowledgeBase{}).Error) + check(false) +} + +func TestCatalogSharedAgentFileRequiresCurrentSelectionOrExactArtifactBinding(t *testing.T) { + catalog, db := newResourceCatalogForTest(t) + require.NoError(t, db.AutoMigrate(&types.KnowledgeBase{}, &types.Knowledge{})) + ctx := newSharedAccessContext() + const physical = "local://7/exports/source.pdf" + ref, err := catalog.Register(ctx, 7, physical, interfaces.ResourceRegistration{}) + require.NoError(t, err) + require.NoError(t, db.Create(&types.KnowledgeBase{ID: "private", TenantID: 7}).Error) + doc := &types.Knowledge{ID: "doc", TenantID: 7, KnowledgeBaseID: "private", Type: "file"} + require.NoError(t, db.Create(doc).Error) + require.NoError(t, catalog.Bind(ctx, ref, types.ResourceOwnerKnowledge, "doc", types.ResourceRelationSourceFile)) + agents := &catalogFileAgent{agent: &types.CustomAgent{ID: "agent", TenantID: 7}} + message := &types.Message{ + ID: "message", + AgentID: "agent", + AgentTenantID: 7, + Content: ref + " " + physical, + Artifacts: types.MessageArtifacts{{URL: ref}}, + } + check := func(allowed bool) { + t.Helper() + for _, path := range []string{ref, physical} { + _, err := access.AuthorizeMessageFile( + ctx, + message, + path, + agents, + catalog, + access.MessageKBShareAuthorizer{}, + ) + if allowed { + require.NoError(t, err) + } else { + require.ErrorIs(t, err, access.ErrForbidden) + } + } + } + for _, mode := range []string{"none", "", "unknown", "selected", "all"} { + agents.agent.Config = types.CustomAgentConfig{KBSelectionMode: mode, KnowledgeBases: []string{"private"}} + check(mode == "selected" || mode == "all") + } + agents.agent.Config = types.CustomAgentConfig{KBSelectionMode: "selected", KnowledgeBases: []string{"other"}} + check(false) + agents.agent.Config.KnowledgeBases = []string{"private"} + scoped := types.WithTenantAPIKeyScope(ctx, types.TenantAPIKeyScope{KnowledgeBaseIDs: types.StringArray{"other"}}) + _, err = access.AuthorizeMessageFile(scoped, message, ref, agents, catalog, access.MessageKBShareAuthorizer{}) + require.ErrorIs(t, err, access.ErrForbidden) + require.NoError(t, db.Delete(doc).Error) + check(false) + agents.agent.Config.KBSelectionMode = "none" + require.NoError( + t, + catalog.Bind(ctx, ref, types.ResourceOwnerMessage, "another-message", types.ResourceRelationArtifact), + ) + check(false) + require.NoError(t, catalog.Bind(ctx, ref, types.ResourceOwnerMessage, "message", types.ResourceRelationArtifact)) + check(true) + agents.agent = nil + check(false) +} diff --git a/internal/application/service/resource_references_test.go b/internal/application/service/resource_references_test.go new file mode 100644 index 00000000..fefffa4c --- /dev/null +++ b/internal/application/service/resource_references_test.go @@ -0,0 +1,71 @@ +package service + +import ( + "context" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/stretchr/testify/require" +) + +func TestKBFileBindingRequiresLiveKnowledgeInExactKB(t *testing.T) { + catalog, db := newResourceCatalogForTest(t) + require.NoError(t, db.AutoMigrate(&types.KnowledgeBase{}, &types.Knowledge{}, &types.Chunk{}, &types.WikiPage{})) + ctx := context.Background() + ref, err := catalog.Register(ctx, 7, "local://7/exports/image.png", interfaces.ResourceRegistration{}) + require.NoError(t, err) + require.NoError(t, db.Create(&types.KnowledgeBase{ID: "kb", TenantID: 7}).Error) + knowledge := &types.Knowledge{ID: "doc", TenantID: 7, KnowledgeBaseID: "kb", Type: "file"} + require.NoError(t, db.Create(knowledge).Error) + require.NoError( + t, + catalog.Bind(ctx, ref, types.ResourceOwnerKnowledge, "doc", types.ResourceRelationExtractedImage), + ) + lookup := catalog.(interfaces.KBResourceLookup) + for _, reference := range []string{ref, "local://7/exports/image.png"} { + ok, err := lookup.IsReferencedByKnowledgeBase(ctx, 7, "kb", reference) + require.NoError(t, err) + require.True(t, ok) + ok, err = lookup.IsReferencedByKnowledgeBase(ctx, 7, "other-kb", reference) + require.NoError(t, err) + require.False(t, ok) + } + require.NoError(t, db.Delete(knowledge).Error) + ok, err := lookup.IsReferencedByKnowledgeBase(ctx, 7, "kb", ref) + require.NoError(t, err) + require.False(t, ok, "a stale binding cannot authorize a deleted document") +} + +func TestKBFileLegacyTextReferencesDoNotAuthorizeFiles(t *testing.T) { + catalog, db := newResourceCatalogForTest(t) + require.NoError(t, db.AutoMigrate(&types.KnowledgeBase{}, &types.Knowledge{}, &types.Chunk{}, &types.WikiPage{})) + lookup := catalog.(interfaces.KBResourceLookup) + const reference = "local://7/exports/image.png" + chunk := &types.Chunk{ + ID: "chunk", + TenantID: 7, + KnowledgeBaseID: "kb", + Content: "![image](" + reference + ".private)", + } + require.NoError(t, db.Create(chunk).Error) + ok, err := lookup.IsReferencedByKnowledgeBase(context.Background(), 7, "kb", reference) + require.NoError(t, err) + require.False(t, ok) + require.NoError(t, db.Model(chunk).Update("image_info", `[{"url":"`+reference+`"}]`).Error) + ok, err = lookup.IsReferencedByKnowledgeBase(context.Background(), 7, "kb", reference) + require.NoError(t, err) + require.False(t, ok, "text is not an ownership binding") + require.NoError(t, db.Delete(chunk).Error) + ok, err = lookup.IsReferencedByKnowledgeBase(context.Background(), 7, "kb", reference) + require.NoError(t, err) + require.False(t, ok) + page := &types.WikiPage{ID: "page", TenantID: 7, KnowledgeBaseID: "kb", Content: "![image](" + reference + ")"} + require.NoError(t, db.Create(page).Error) + ok, err = lookup.IsReferencedByKnowledgeBase(context.Background(), 7, "kb", reference) + require.NoError(t, err) + require.False(t, ok, "text is not an ownership binding") + ok, err = lookup.IsReferencedByKnowledgeBase(context.Background(), 8, "kb", reference) + require.NoError(t, err) + require.False(t, ok) +} diff --git a/internal/application/service/resource_review_regression_test.go b/internal/application/service/resource_review_regression_test.go new file mode 100644 index 00000000..8bf5c2ab --- /dev/null +++ b/internal/application/service/resource_review_regression_test.go @@ -0,0 +1,284 @@ +package service + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "github.com/Tencent/WeKnora/internal/application/access" + apperrors "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/hibiken/asynq" + "github.com/stretchr/testify/require" +) + +func TestReparseRejectsMovingDocumentBeforeSideEffects(t *testing.T) { + f := newDocumentWriteFixture(t) + row, err := f.repo.GetKnowledgeByID(f.ctx, 7, "doc") + require.NoError(t, err) + require.NoError(t, setTransferState(row, knowledgeTransferState{Operation: access.KBTransferMove, Phase: "moving"})) + require.NoError(t, f.repo.UpdateKnowledge(f.ctx, row)) + f.repo.writes = 0 + _, err = f.svc.ReparseKnowledge(f.ctx, "doc", nil) + var appErr *apperrors.AppError + require.ErrorAs(t, err, &appErr) + require.Equal(t, 409, appErr.HTTPCode) + require.Zero(t, f.repo.writes) + require.Zero(t, f.chunkRepo.writes) + require.Zero(t, f.graph.calls) +} + +func TestReparseRequiresExplicitWriteGrant(t *testing.T) { + f := newDocumentWriteFixture(t) + ctx := types.WithExecutionTenant(context.Background(), 7) + _, err := f.svc.ReparseKnowledge(ctx, "doc", nil) + require.Error(t, err) + require.Zero(t, f.repo.writes) + require.Zero(t, f.chunkRepo.writes) +} + +func TestReparseTaskPinsOriginalKBBeforeAnySubmission(t *testing.T) { + for _, scenario := range []string{"moved", "missing", "mixed legacy", "blank ID", "moving"} { + t.Run(scenario, func(t *testing.T) { + f := newDocumentWriteFixture(t) + payload := types.KnowledgeListReparsePayload{ + TenantID: 7, KnowledgeBaseID: "kb", KnowledgeIDs: []string{"doc"}, + } + switch scenario { + case "moved": + require.NoError( + t, + f.db.Model(&types.Knowledge{}).Where("id = ?", "doc").Update("knowledge_base_id", "other").Error, + ) + case "missing": + payload.KnowledgeIDs = append(payload.KnowledgeIDs, "missing") + case "mixed legacy": + payload.KnowledgeBaseID = "" + payload.KnowledgeIDs = append(payload.KnowledgeIDs, "other-doc") + case "blank ID": + payload.KnowledgeIDs = append(payload.KnowledgeIDs, " ") + case "moving": + row, err := f.repo.GetKnowledgeByID(f.ctx, 7, "doc") + require.NoError(t, err) + require.NoError( + t, + setTransferState(row, knowledgeTransferState{Operation: access.KBTransferMove, Phase: "moving"}), + ) + require.NoError(t, f.repo.UpdateKnowledge(f.ctx, row)) + } + f.repo.writes = 0 + raw, err := json.Marshal(payload) + require.NoError(t, err) + err = f.svc.ProcessKnowledgeListReparse( + context.Background(), asynq.NewTask(types.TypeKnowledgeListReparse, raw), + ) + require.ErrorIs(t, err, asynq.SkipRetry) + require.Zero(t, f.repo.writes) + require.Zero(t, f.chunkRepo.writes) + }) + } +} + +func TestReparseTaskGrantCannotFollowMoveAfterPreflight(t *testing.T) { + f := newDocumentWriteFixture(t) + ctx, ids, err := f.svc.reparseTaskScope(context.Background(), types.KnowledgeListReparsePayload{ + TenantID: 7, KnowledgeBaseID: "kb", KnowledgeIDs: []string{"doc", "doc"}, + }) + require.NoError(t, err) + require.Equal(t, []string{"doc"}, ids) + require.NoError(t, access.RequireKBWrite(ctx, f.kbs.values["kb"])) + require.Error(t, access.RequireKBWrite(ctx, f.kbs.values["other"])) + require.NoError(t, f.db.Model(&types.Knowledge{}).Where("id = ?", "doc").Update("knowledge_base_id", "other").Error) + _, err = f.svc.ReparseKnowledge(ctx, "doc", nil) + require.Error(t, err) + require.Zero(t, f.repo.writes) + require.Zero(t, f.chunkRepo.writes) +} + +type sharedChunkFailureRepo struct { + interfaces.ChunkRepository + failure error +} + +func (r sharedChunkFailureRepo) ListChunksByID(context.Context, uint64, []string) ([]*types.Chunk, error) { + return []*types.Chunk{{ID: "own", TenantID: 1, KnowledgeBaseID: "own"}}, nil +} + +func (r sharedChunkFailureRepo) ListChunksByIDOnly(context.Context, []string) ([]*types.Chunk, error) { + return nil, r.failure +} + +func TestSharedChunkReadPropagatesStorageFailure(t *testing.T) { + failure := errors.New("shared chunk storage unavailable") + svc := &knowledgeBaseService{chunkRepo: sharedChunkFailureRepo{failure: failure}} + rows, err := svc.listChunksByIDWithShared(newSharedAccessContext(), 1, []string{"own", "shared"}) + require.ErrorIs(t, err, failure) + require.Nil(t, rows) +} + +func TestFAQCloneDoesNotInheritTransferState(t *testing.T) { + f := transferFixture(t, access.KBTransferClone) + kb := &types.KnowledgeBase{ID: "new-faq", TenantID: 7} + src := &types.Knowledge{ + Metadata: types.JSON(`{"custom":{"value":3},"_knowledge_transfer":{"operation":"move","phase":"moving"}}`), + } + before := string(src.Metadata) + dst, err := f.svc.getOrCreateFAQKnowledge(f.ctx, kb, src) + require.NoError(t, err) + require.JSONEq(t, `{"custom":{"value":3}}`, string(dst.Metadata)) + require.Equal(t, before, string(src.Metadata)) + require.NoError(t, access.RejectMovingKnowledge(dst)) +} + +type partialCloneEngine struct { + parentChildRetrieveEngine + vectors map[string]int + cleanupFailure bool +} + +func (e *partialCloneEngine) Support() []types.RetrieverType { + return []types.RetrieverType{types.VectorRetrieverType, types.KeywordsRetrieverType} +} + +func (e *partialCloneEngine) CopyIndices( + _ context.Context, _ string, documents, _ map[string]string, _ string, _ int, _ string, +) error { + for _, id := range documents { + e.vectors[id] = 1 + } + return errors.New("partial vector copy") +} + +func (e *partialCloneEngine) DeleteByKnowledgeIDList(_ context.Context, ids []string, _ int, _ string) error { + if e.cleanupFailure { + return errors.New("vector cleanup unavailable") + } + for _, id := range ids { + delete(e.vectors, id) + } + return nil +} + +func TestClonePartialIndexFailureCleansOnlyItsDestination(t *testing.T) { + for _, cleanupFailure := range []bool{false, true} { + t.Run(map[bool]string{false: "cleaned", true: "cleanup fails visibly"}[cleanupFailure], func(t *testing.T) { + t.Setenv("RETRIEVE_DRIVER", "postgres") + f := transferFixture(t, access.KBTransferClone) + engine := &partialCloneEngine{vectors: map[string]int{"doc": 1}, cleanupFailure: cleanupFailure} + f.svc.retrieveEngine = parentChildRetrieveRegistry{engine: engine} + f.svc.modelService = parentChildModelService{embedder: parentChildEmbedder{}} + f.kbs.values["other"].EmbeddingModelID = "model" + src, err := f.repo.GetKnowledgeByID(f.ctx, 7, "doc") + require.NoError(t, err) + err = f.svc.cloneKnowledge(f.ctx, src, f.kbs.values["other"]) + require.ErrorContains(t, err, "partial vector copy") + rows, err := f.repo.ListKnowledgeByKnowledgeBaseID(f.ctx, 7, "other") + require.NoError(t, err) + var dst *types.Knowledge + for _, row := range rows { + if row.ID != "other-doc" { + dst = row + } + } + require.NotNil(t, dst) + require.Equal(t, types.ParseStatusFailed, dst.ParseStatus) + chunks, err := f.chunkRepo.ListAllChunksByKnowledgeID(f.ctx, 7, dst.ID) + require.NoError(t, err) + if cleanupFailure { + require.Len(t, chunks, 1, "retain image references when index cleanup is uncertain") + require.Equal(t, 1, engine.vectors[dst.ID]) + require.Contains(t, dst.ErrorMessage, "vector cleanup unavailable") + } else { + require.Empty(t, chunks) + require.Zero(t, engine.vectors[dst.ID]) + } + require.Equal(t, 1, engine.vectors["doc"]) + var tenant types.Tenant + require.NoError(t, f.db.First(&tenant, 7).Error) + require.EqualValues(t, 5, tenant.StorageUsed, "failed clones must not change storage accounting") + }) + } +} + +func TestReparseWorkerSubmitsPinnedAndLegacySingleKBTasks(t *testing.T) { + for _, kbID := range []string{"kb", ""} { + t.Run("scope="+kbID, func(t *testing.T) { + f := newDocumentWriteFixture(t) + row, err := f.repo.GetKnowledgeByID(f.ctx, 7, "doc") + require.NoError(t, err) + require.NoError( + t, + row.SetManualMetadata( + types.NewManualKnowledgeMetadata("# content", types.ManualKnowledgeStatusPublish, 1), + ), + ) + require.NoError(t, f.repo.UpdateKnowledge(f.ctx, row)) + f.svc.task = &transferQueue{} + raw, err := json.Marshal(types.KnowledgeListReparsePayload{ + TenantID: 7, KnowledgeBaseID: kbID, KnowledgeIDs: []string{"doc"}, + }) + require.NoError(t, err) + require.NoError( + t, + f.svc.ProcessKnowledgeListReparse( + context.Background(), asynq.NewTask(types.TypeKnowledgeListReparse, raw), + ), + ) + row, err = f.repo.GetKnowledgeByID(f.ctx, 7, "doc") + require.NoError(t, err) + require.Equal(t, types.ParseStatusPending, row.ParseStatus) + }) + } +} + +func TestMovedReparseTransferPhaseDoesNotBlockItsParser(t *testing.T) { + row := &types.Knowledge{ID: "doc", TenantID: 7, KnowledgeBaseID: "target", ParseStatus: types.ParseStatusPending} + for _, phase := range []string{"reparse_pending", "done"} { + require.NoError( + t, + setTransferState(row, knowledgeTransferState{ + Operation: access.KBTransferMove, Mode: "reparse", Phase: phase, + }), + ) + require.NoError(t, validateProcessingKnowledge(row, 7, "target", "doc")) + require.ErrorIs(t, validateProcessingKnowledge(row, 7, "source", "doc"), asynq.SkipRetry) + require.Equal(t, types.ParseStatusPending, row.ParseStatus, "transfer state is independent of parse completion") + } +} + +type orphanMoveEngine struct { + partialCloneEngine + calls int + chunks []string +} + +func (e *orphanMoveEngine) MoveKnowledgeIndices( + _ context.Context, _, _, _ string, chunks []string, _ int, _ string, +) error { + e.calls++ + e.chunks = append([]string(nil), chunks...) + return nil +} + +func TestMoveStillVisitsVectorBackendWithoutDatabaseChunks(t *testing.T) { + t.Setenv("RETRIEVE_DRIVER", "postgres") + f := transferFixture(t, access.KBTransferMove) + engine := &orphanMoveEngine{} + f.svc.retrieveEngine = parentChildRetrieveRegistry{engine: engine} + f.svc.modelService = parentChildModelService{embedder: parentChildEmbedder{}} + f.kbs.values["kb"].EmbeddingModelID = "model" + f.kbs.values["other"].EmbeddingModelID = "model" + require.NoError( + t, + f.db.Model(&types.Knowledge{}).Where("id = ?", "doc").Update("embedding_model_id", "model").Error, + ) + require.NoError(t, f.db.Where("knowledge_id = ?", "doc").Delete(&types.Chunk{}).Error) + require.NoError(t, f.svc.ProcessKnowledgeMove(context.Background(), moveTask(t, []string{"doc"}, "reuse_vectors"))) + require.Equal(t, 1, engine.calls) + require.Empty(t, engine.chunks) + row, err := f.repo.GetKnowledgeByID(f.ctx, 7, "doc") + require.NoError(t, err) + require.Equal(t, "other", row.KnowledgeBaseID) +} diff --git a/internal/application/service/retriever/composite.go b/internal/application/service/retriever/composite.go index 977dd049..cef6171a 100644 --- a/internal/application/service/retriever/composite.go +++ b/internal/application/service/retriever/composite.go @@ -312,3 +312,42 @@ func (c *CompositeRetrieveEngine) EstimateStorageSize(ctx context.Context, } return sum.Load() } + +// ValidateKnowledgeIndexMove checks all stores before the first mutation. +func (c *CompositeRetrieveEngine) ValidateKnowledgeIndexMove(ctx context.Context) error { + for _, info := range c.engineInfos { + if _, ok := info.retrieveEngine.(interfaces.KnowledgeIndexMover); !ok { + return fmt.Errorf("retriever %s does not support moving indices", info.retrieveEngine.EngineType()) + } + if validator, ok := info.retrieveEngine.(interface{ ValidateKnowledgeIndexMove(context.Context) error }); ok { + if err := validator.ValidateKnowledgeIndexMove(ctx); err != nil { + return err + } + } + } + return nil +} + +// MoveKnowledgeIndices updates each validated store with retry-safe operations. +func (c *CompositeRetrieveEngine) MoveKnowledgeIndices( + ctx context.Context, + sourceKB, targetKB, knowledgeID string, + chunkIDs []string, + dimension int, + knowledgeType string, +) error { + if err := c.ValidateKnowledgeIndexMove(ctx); err != nil { + return err + } + return c.concurrentExecWithError(ctx, func(ctx context.Context, info *engineInfo) error { + return info.retrieveEngine.(interfaces.KnowledgeIndexMover).MoveKnowledgeIndices( + ctx, + sourceKB, + targetKB, + knowledgeID, + chunkIDs, + dimension, + knowledgeType, + ) + }) +} diff --git a/internal/application/service/retriever/keywords_vector_hybrid_indexer.go b/internal/application/service/retriever/keywords_vector_hybrid_indexer.go index 3e62ceb8..efb6d582 100644 --- a/internal/application/service/retriever/keywords_vector_hybrid_indexer.go +++ b/internal/application/service/retriever/keywords_vector_hybrid_indexer.go @@ -2,6 +2,7 @@ package retriever import ( "context" + "fmt" "regexp" "slices" "strings" @@ -346,3 +347,37 @@ func (v *KeywordsVectorHybridRetrieveEngineService) BatchUpdateChunkTagID( ) error { return v.indexRepository.BatchUpdateChunkTagID(ctx, chunkTagMap) } + +// ValidateKnowledgeIndexMove checks backend support before any mutation. +func (v *KeywordsVectorHybridRetrieveEngineService) ValidateKnowledgeIndexMove(ctx context.Context) error { + if _, ok := v.indexRepository.(interfaces.KnowledgeIndexMover); !ok { + return fmt.Errorf("retriever %s does not support moving indices", v.EngineType()) + } + if validator, ok := v.indexRepository.(interface{ ValidateKnowledgeIndexMove(context.Context) error }); ok { + return validator.ValidateKnowledgeIndexMove(ctx) + } + + return nil +} + +// MoveKnowledgeIndices delegates metadata relocation to the supported backend. +func (v *KeywordsVectorHybridRetrieveEngineService) MoveKnowledgeIndices( + ctx context.Context, + sourceKB, targetKB, knowledgeID string, + chunkIDs []string, + dimension int, + knowledgeType string, +) error { + if err := v.ValidateKnowledgeIndexMove(ctx); err != nil { + return err + } + return v.indexRepository.(interfaces.KnowledgeIndexMover).MoveKnowledgeIndices( + ctx, + sourceKB, + targetKB, + knowledgeID, + chunkIDs, + dimension, + knowledgeType, + ) +} diff --git a/internal/application/service/retriever/move_test.go b/internal/application/service/retriever/move_test.go new file mode 100644 index 00000000..5050bf25 --- /dev/null +++ b/internal/application/service/retriever/move_test.go @@ -0,0 +1,38 @@ +package retriever + +import ( + "context" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/stretchr/testify/require" +) + +type moveTestEngine struct { + fakeEngine + calls int +} + +func (e *moveTestEngine) MoveKnowledgeIndices(context.Context, string, string, string, []string, int, string) error { + e.calls++ + return nil +} + +func TestCompositeMoveChecksAllEnginesBeforeMutation(t *testing.T) { + supported := &moveTestEngine{fakeEngine: fakeEngine{engineType: types.PostgresRetrieverEngineType}} + unsupported := &fakeEngine{engineType: types.ElasticsearchRetrieverEngineType} + c := &CompositeRetrieveEngine{ + engineInfos: []*engineInfo{{retrieveEngine: supported}, {retrieveEngine: unsupported}}, + } + require.Error( + t, + c.MoveKnowledgeIndices(context.Background(), "source", "target", "doc", []string{"chunk"}, 2, "document"), + ) + require.Zero(t, supported.calls) + c.engineInfos = c.engineInfos[:1] + require.NoError( + t, + c.MoveKnowledgeIndices(context.Background(), "source", "target", "doc", []string{"chunk"}, 2, "document"), + ) + require.Equal(t, 1, supported.calls) +} diff --git a/internal/application/service/session.go b/internal/application/service/session.go index d7d07fce..a1e026c2 100644 --- a/internal/application/service/session.go +++ b/internal/application/service/session.go @@ -6,18 +6,17 @@ import ( "fmt" "strings" + "github.com/Tencent/WeKnora/internal/application/repository" + chatpipeline "github.com/Tencent/WeKnora/internal/application/service/chat_pipeline" "github.com/Tencent/WeKnora/internal/config" apperrors "github.com/Tencent/WeKnora/internal/errors" "github.com/Tencent/WeKnora/internal/event" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/models/chat" + "github.com/Tencent/WeKnora/internal/sandbox" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" "github.com/google/uuid" - - "github.com/Tencent/WeKnora/internal/application/repository" - chatpipeline "github.com/Tencent/WeKnora/internal/application/service/chat_pipeline" - "github.com/Tencent/WeKnora/internal/sandbox" ) func sessionUserIDFromContext(ctx context.Context) string { @@ -495,7 +494,7 @@ func (s *sessionService) DeleteSession(ctx context.Context, id string) error { return } if len(knowledgeIDs) > 0 { - if err := s.knowledgeService.DeleteKnowledgeList(bgCtx, knowledgeIDs); err != nil { + if err := deleteReferencedKnowledge(bgCtx, s.knowledgeService, "", knowledgeIDs); err != nil { logger.Warnf(bgCtx, "Failed to delete chat history knowledge for session %s: %v", id, err) } } @@ -572,7 +571,7 @@ func (s *sessionService) BatchDeleteSessions(ctx context.Context, ids []string) return } if len(knowledgeIDs) > 0 { - if err := s.knowledgeService.DeleteKnowledgeList(bgCtx, knowledgeIDs); err != nil { + if err := deleteReferencedKnowledge(bgCtx, s.knowledgeService, "", knowledgeIDs); err != nil { logger.Warnf(bgCtx, "Failed to delete chat history knowledge for session %s: %v", sessionID, err) } } @@ -629,7 +628,7 @@ func (s *sessionService) DeleteAllSessions(ctx context.Context) error { return } if len(knowledgeIDs) > 0 { - if err := s.knowledgeService.DeleteKnowledgeList(bgCtx, knowledgeIDs); err != nil { + if err := deleteReferencedKnowledge(bgCtx, s.knowledgeService, "", knowledgeIDs); err != nil { logger.Warnf(bgCtx, "Failed to delete chat history knowledge for session %s: %v", sessionID, err) } } diff --git a/internal/application/service/session_knowledge_qa.go b/internal/application/service/session_knowledge_qa.go index bf0233b2..7742531d 100644 --- a/internal/application/service/session_knowledge_qa.go +++ b/internal/application/service/session_knowledge_qa.go @@ -445,6 +445,7 @@ func (s *sessionService) buildSearchTargets( knowledgeIDs []string, tagScopes []types.TagScope, ) (types.SearchTargets, error) { + caller := types.CallerFromContext(ctx) var targets types.SearchTargets tagIDsByKB := mergeTagScopesByKB(tagScopes) @@ -455,7 +456,7 @@ func (s *sessionService) buildSearchTargets( fullKBSet := make(map[string]bool) // First pass: batch-fetch KBs, then resolve tenant per ID (tenant scope already set by caller) - callerTenantRole := types.TenantRoleFromContext(ctx) + permissions := kbReadPermissions(ctx, s.kbShareService) kbIDsToFetch := append([]string(nil), knowledgeBaseIDs...) for kbID := range tagIDsByKB { kbIDsToFetch = append(kbIDsToFetch, kbID) @@ -474,25 +475,17 @@ func (s *sessionService) buildSearchTargets( } } } - userID, _ := types.UserIDFromContext(ctx) resolveKBTenant := func(kbID string) uint64 { if kbTenantMap[kbID] != 0 { return kbTenantMap[kbID] } kb := kbByID[kbID] - if kb == nil { - kbTenantMap[kbID] = tenantID - } else if kb.TenantID == tenantID { - kbTenantMap[kbID] = tenantID - } else if s.kbShareService != nil && userID != "" { - hasAccess, _ := s.kbShareService.HasTenantKBPermission(ctx, kbID, tenantID, callerTenantRole, types.OrgRoleViewer) - if hasAccess { + kbTenantMap[kbID] = caller.TenantID + if kb != nil { + kbTenantMap[kbID] = 0 + if allowed, err := permissions.Check(kbID, kb.TenantID, types.OrgRoleViewer); err == nil && allowed { kbTenantMap[kbID] = kb.TenantID - } else { - kbTenantMap[kbID] = tenantID } - } else { - kbTenantMap[kbID] = tenantID } return kbTenantMap[kbID] } @@ -501,6 +494,9 @@ func (s *sessionService) buildSearchTargets( for _, kbID := range knowledgeBaseIDs { fullKBSet[kbID] = true kbTenant := resolveKBTenant(kbID) + if kbTenant == 0 { + continue + } if len(tagIDsByKB[kbID]) > 0 { continue } @@ -563,6 +559,9 @@ func (s *sessionService) buildSearchTargets( continue } kbTenant := resolveKBTenant(kbID) + if kbTenant == 0 { + continue + } kb := kbByID[kbID] explicitKnowledgeIDs := uniqueNonEmptyStrings(kbToKnowledgeIDs[kbID]) diff --git a/internal/application/service/tag.go b/internal/application/service/tag.go index e0f4f6b8..50d8010e 100644 --- a/internal/application/service/tag.go +++ b/internal/application/service/tag.go @@ -29,6 +29,7 @@ type knowledgeTagService struct { modelService interfaces.ModelService task interfaces.TaskEnqueuer kbShareService interfaces.KBShareService + tenantRepo interfaces.TenantRepository audit interfaces.AuditLogService } @@ -43,6 +44,7 @@ func NewKnowledgeTagService( modelService interfaces.ModelService, task interfaces.TaskEnqueuer, kbShareService interfaces.KBShareService, + tenantRepo interfaces.TenantRepository, audit interfaces.AuditLogService, ) (interfaces.KnowledgeTagService, error) { return &knowledgeTagService{ @@ -55,6 +57,7 @@ func NewKnowledgeTagService( modelService: modelService, task: task, kbShareService: kbShareService, + tenantRepo: tenantRepo, audit: audit, }, nil } @@ -79,27 +82,11 @@ func (s *knowledgeTagService) ListTags( return nil, err } - // Check access permission - tenantID := types.MustTenantIDFromContext(ctx) - if kb.TenantID != tenantID { - // Get user ID from context - userIDVal := ctx.Value(types.UserIDContextKey) - if userIDVal == nil { - return nil, werrors.NewForbiddenError("无权访问该知识库") - } - _ = userIDVal.(string) - callerTenantRole := types.TenantRoleFromContext(ctx) - - // Check whether the caller's tenant has at least viewer permission via org sharing. - hasPermission, err := s.kbShareService.HasTenantKBPermission(ctx, kbID, tenantID, callerTenantRole, types.OrgRoleViewer) - if err != nil || !hasPermission { - return nil, werrors.NewForbiddenError("无权访问该知识库") - } + effectiveTenantID, err := resolveKBReadTenant(ctx, kb, s.kbShareService) + if err != nil { + return nil, err } - // Use kb's tenant ID for data access - effectiveTenantID := kb.TenantID - tags, total, err := s.repo.ListByKB(ctx, effectiveTenantID, kbID, page, keyword) if err != nil { return nil, err @@ -158,6 +145,10 @@ func (s *knowledgeTagService) CreateTag( if err != nil { return nil, err } + ctx, err = requireKBWrite(ctx, kb) + if err != nil { + return nil, err + } // Check if tag with same name already exists existingTag, err := s.repo.GetByName(ctx, kb.TenantID, kbID, name) @@ -202,11 +193,18 @@ func (s *knowledgeTagService) UpdateTag( if id == "" { return nil, werrors.NewBadRequestError("标签ID不能为空") } - tenantID := types.MustTenantIDFromContext(ctx) + tenantID, ok := types.TenantIDFromContext(ctx) + if !ok || tenantID == 0 { + return nil, werrors.NewForbiddenError("无权修改标签") + } tag, err := s.repo.GetByID(ctx, tenantID, id) if err != nil { return nil, err } + _, ctx, err = s.requireTagWrite(ctx, tag) + if err != nil { + return nil, err + } if name != nil { newName := strings.TrimSpace(*name) @@ -237,14 +235,22 @@ func (s *knowledgeTagService) DeleteTag(ctx context.Context, id string, force bo if id == "" { return werrors.NewBadRequestError("标签ID不能为空") } - tenantID := types.MustTenantIDFromContext(ctx) + tenantID, ok := types.TenantIDFromContext(ctx) + if !ok || tenantID == 0 { + return werrors.NewForbiddenError("无权修改标签") + } tag, err := s.repo.GetByID(ctx, tenantID, id) if err != nil { return err } - - // Get KB info for embedding model - kb, err := s.kbService.GetKnowledgeBaseByID(ctx, tag.KnowledgeBaseID) + kb, ctx, err := s.requireTagWrite(ctx, tag) + if err != nil { + return err + } + if err := s.validateTagDeleteExclusions(ctx, kb, excludeIDs); err != nil { + return err + } + ctx, err = withKBWriteTenantInfo(ctx, kb, s.tenantRepo) if err != nil { return err } @@ -291,9 +297,10 @@ func (s *knowledgeTagService) DeleteTag(ctx context.Context, id string, force bo } // Enqueue async task to delete knowledge files payload := types.KnowledgeListDeletePayload{ - TenantID: tenantID, - KnowledgeIDs: knowledgeIDs, - Initiator: types.TaskInitiatorFromContext(ctx), + KnowledgeBaseID: kb.ID, + TenantID: tenantID, + KnowledgeIDs: knowledgeIDs, + Initiator: types.TaskInitiatorFromContext(ctx), } langfuse.InjectTracing(ctx, &payload) payloadBytes, err := json.Marshal(payload) @@ -476,6 +483,10 @@ func (s *knowledgeTagService) FindOrCreateTagByName(ctx context.Context, kbID st if err != nil { return nil, err } + ctx, err = requireKBWrite(ctx, kb) + if err != nil { + return nil, err + } tenantID := kb.TenantID diff --git a/internal/application/service/tag_access.go b/internal/application/service/tag_access.go new file mode 100644 index 00000000..edf7e4c7 --- /dev/null +++ b/internal/application/service/tag_access.go @@ -0,0 +1,60 @@ +package service + +import ( + "context" + + apperrors "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/types" +) + +func (s *knowledgeTagService) requireTagWrite( + ctx context.Context, + tag *types.KnowledgeTag, +) (*types.KnowledgeBase, context.Context, error) { + if tag == nil { + return nil, ctx, apperrors.NewNotFoundError("标签不存在") + } + kb, err := s.kbService.GetKnowledgeBaseByID(ctx, tag.KnowledgeBaseID) + if err != nil { + return nil, ctx, err + } + if kb == nil || kb.ID != tag.KnowledgeBaseID || kb.TenantID != tag.TenantID { + return nil, ctx, apperrors.NewForbiddenError("标签不属于当前知识库") + } + ctx, err = requireKBWrite(ctx, kb) + return kb, ctx, err +} + +func (s *knowledgeTagService) validateTagDeleteExclusions( + ctx context.Context, + kb *types.KnowledgeBase, + ids []string, +) error { + if len(ids) == 0 { + return nil + } + if kb.Type != types.KnowledgeBaseTypeFAQ { + return apperrors.NewBadRequestError("仅 FAQ 条目删除支持排除条目") + } + wanted := make(map[string]bool, len(ids)) + for _, id := range ids { + wanted[id] = true + } + chunks, err := s.chunkRepo.ListChunksByID(ctx, kb.TenantID, ids) + if err != nil { + return err + } + for _, chunk := range chunks { + if chunk == nil || !wanted[chunk.ID] { + continue + } + if chunk.TenantID != kb.TenantID || chunk.KnowledgeBaseID != kb.ID || chunk.ChunkType != types.ChunkTypeFAQ { + return apperrors.NewForbiddenError("排除条目不属于当前知识库") + } + delete(wanted, chunk.ID) + } + if len(wanted) != 0 { + return apperrors.NewNotFoundError("排除条目不存在") + } + return nil +} diff --git a/internal/application/service/web_search_state.go b/internal/application/service/web_search_state.go index 811458cb..00221cfc 100644 --- a/internal/application/service/web_search_state.go +++ b/internal/application/service/web_search_state.go @@ -121,7 +121,7 @@ func (s *webSearchStateService) DeleteWebSearchTempKBState(ctx context.Context, // Delete all knowledge items for _, kid := range state.KnowledgeIDs { - if delErr := s.knowledgeService.DeleteKnowledge(ctx, kid); delErr != nil { + if delErr := deleteReferencedKnowledge(ctx, s.knowledgeService, state.KBID, []string{kid}); delErr != nil { logger.Warnf(ctx, "Failed to delete temp knowledge %s: %v", kid, delErr) } } diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index eb03de4a..685221fd 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -600,12 +600,18 @@ func enqueueWikiIngestTrigger( // because there is no "user upload arriving in waves" pattern to // debounce against — a deletion fires once and we want the cleanup // to land promptly. -func EnqueueWikiRetract( +func EnqueueWikiRetract(ctx context.Context, task interfaces.TaskEnqueuer, + pendingRepo interfaces.TaskPendingOpsRepository, payload WikiRetractPayload, +) { + _ = enqueueWikiRetract(ctx, task, pendingRepo, payload) +} + +func enqueueWikiRetract( ctx context.Context, task interfaces.TaskEnqueuer, pendingRepo interfaces.TaskPendingOpsRepository, payload WikiRetractPayload, -) { +) error { op := WikiPendingOp{ Op: WikiOpRetract, KnowledgeID: payload.KnowledgeID, @@ -618,7 +624,7 @@ func EnqueueWikiRetract( payloadBytes, err := json.Marshal(op) if err != nil { logger.Warnf(ctx, "wiki retract: failed to marshal pending op: %v", err) - return + return err } accepted, err := enqueueWikiPendingOp(ctx, pendingRepo, &types.TaskPendingOp{ TenantID: payload.TenantID, @@ -631,11 +637,11 @@ func EnqueueWikiRetract( }) if err != nil { logger.Warnf(ctx, "wiki retract: failed to enqueue pending op: %v", err) - return + return err } if !accepted { logger.Infof(ctx, "wiki retract: skip enqueue for deleted KB %s", payload.KnowledgeBaseID) - return + return nil } trigger := WikiIngestPayload{ @@ -653,7 +659,9 @@ func EnqueueWikiRetract( ) if _, err := task.Enqueue(t); err != nil { logger.Warnf(ctx, "wiki retract: failed to enqueue trigger task: %v", err) + return err } + return nil } // Handle implements interfaces.TaskHandler for asynq task processing. The diff --git a/internal/filetransport/response.go b/internal/filetransport/response.go new file mode 100644 index 00000000..4327fdea --- /dev/null +++ b/internal/filetransport/response.go @@ -0,0 +1,70 @@ +// Package filetransport streams already-authorized files. It performs no +// resource lookup or permission inference; callers own those boundaries. +package filetransport + +import ( + "io" + "mime" + "net/http" + "path/filepath" + "strconv" + "time" + + secutils "github.com/Tencent/WeKnora/internal/utils" +) + +// Options defines response metadata for an already-authorized reader. +type Options struct { + Filename string + Download bool + ContentType string + Disposition string + CacheControl string + Size int64 +} + +// Serve closes reader on every path. Seekable backends support standard Range +// and HEAD via ServeContent; streaming-only backends return the full response +// without buffering the object just to implement seeking (RFC 9110 section 14.2). +func Serve(w http.ResponseWriter, r *http.Request, reader io.ReadCloser, options Options) error { + defer func() { _ = reader.Close() }() + contentType, inline := secutils.SafeContentTypeByFilename(options.Filename) + if options.Download { + inline = false + } + if options.ContentType != "" { + contentType = options.ContentType + } + w.Header().Set("Content-Type", contentType) + w.Header().Set("X-Content-Type-Options", "nosniff") + disposition := "attachment" + if inline { + disposition = "inline" + } + if options.Disposition != "" { + w.Header().Set("Content-Disposition", options.Disposition) + } else if options.Filename != "" { + w.Header().Set("Content-Disposition", + mime.FormatMediaType(disposition, + map[string]string{"filename": filepath.Base(options.Filename)})) + } else { + w.Header().Set("Content-Disposition", disposition) + } + if options.CacheControl != "" { + w.Header().Set("Cache-Control", options.CacheControl) + } + if seeker, ok := reader.(io.ReadSeeker); ok { + http.ServeContent(w, r, options.Filename, time.Time{}, seeker) + return nil + } + w.Header().Set("Accept-Ranges", "none") + if options.Size > 0 { + w.Header().Set("Content-Length", strconv.FormatInt(options.Size, 10)) + } + w.WriteHeader(http.StatusOK) + if r.Method == http.MethodHead { + return nil + } + _, err := io.Copy(w, reader) + return err +} diff --git a/internal/filetransport/response_test.go b/internal/filetransport/response_test.go new file mode 100644 index 00000000..77642559 --- /dev/null +++ b/internal/filetransport/response_test.go @@ -0,0 +1,77 @@ +package filetransport + +import ( + "bytes" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +type seekFile struct { + *bytes.Reader + closed bool +} + +func (f *seekFile) Close() error { f.closed = true; return nil } + +func TestServeSeekableFile(t *testing.T) { + for _, tt := range []struct { + name, method, rangeHeader, body string + status int + }{ + {"whole", "GET", "", "0123456789", 200}, + {"range", "GET", "bytes=2-5", "2345", 206}, + {"suffix", "GET", "bytes=-3", "789", 206}, + {"head", "HEAD", "", "", 200}, + {"unsatisfiable", "GET", "bytes=20-30", "", 416}, + } { + t.Run(tt.name, func(t *testing.T) { + reader := &seekFile{Reader: bytes.NewReader([]byte("0123456789"))} + req := httptest.NewRequest(tt.method, "/file", nil) + req.Header.Set("Range", tt.rangeHeader) + w := httptest.NewRecorder() + require.NoError(t, Serve(w, req, reader, Options{Filename: "image.png", CacheControl: "private, no-store"})) + require.Equal(t, tt.status, w.Code) + if tt.status != 416 { + require.Equal(t, tt.body, w.Body.String()) + } + require.True(t, reader.closed) + require.Equal(t, "nosniff", w.Header().Get("X-Content-Type-Options")) + if tt.status == 206 { + require.Contains(t, w.Header().Get("Content-Range"), "/10") + } + }) + } +} + +type streamFile struct{ read, closed bool } + +func (f *streamFile) Read([]byte) (int, error) { f.read = true; return 0, io.EOF } +func (f *streamFile) Close() error { f.closed = true; return nil } + +func TestServeStreamingHEADClosesWithoutReading(t *testing.T) { + reader := &streamFile{} + w := httptest.NewRecorder() + require.NoError( + t, + Serve(w, httptest.NewRequest("HEAD", "/file", nil), reader, Options{Filename: "a.pdf", Size: 123}), + ) + require.False(t, reader.read) + require.True(t, reader.closed) + require.Equal(t, "123", w.Header().Get("Content-Length")) +} + +func TestServeActiveContentAndStreamingRange(t *testing.T) { + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/file", nil) + req.Header.Set("Range", "bytes=0-1") + require.NoError(t, Serve(w, req, io.NopCloser(strings.NewReader("")), Options{Filename: "../payload.svg"})) + require.Equal(t, http.StatusOK, w.Code, "non-seekable readers return the complete body") + require.Equal(t, "", w.Body.String()) + require.Equal(t, "none", w.Header().Get("Accept-Ranges")) + require.Equal(t, "attachment; filename=payload.svg", w.Header().Get("Content-Disposition")) +} diff --git a/internal/handler/chunk.go b/internal/handler/chunk.go index dd832146..162bc952 100644 --- a/internal/handler/chunk.go +++ b/internal/handler/chunk.go @@ -231,7 +231,12 @@ func (h *ChunkHandler) UpdateChunk(c *gin.Context) { c.Error(errors.NewConflictError("Chunk was modified by another user; refresh and retry")) return } - c.Error(errors.NewInternalServerError(err.Error())) + var appErr *errors.AppError + if stderrors.As(err, &appErr) { + _ = c.Error(appErr) + } else { + _ = c.Error(errors.NewInternalServerError(err.Error())) + } return } @@ -289,7 +294,12 @@ func (h *ChunkHandler) RevertChunk(c *gin.Context) { return } if err != nil { - c.Error(errors.NewBadRequestError(err.Error())) + var appErr *errors.AppError + if stderrors.As(err, &appErr) { + _ = c.Error(appErr) + } else { + _ = c.Error(errors.NewBadRequestError(err.Error())) + } return } knowledge, getErr := h.kgService.GetKnowledgeByID(c.Request.Context(), knowledgeID) diff --git a/internal/handler/document_write_scope_test.go b/internal/handler/document_write_scope_test.go new file mode 100644 index 00000000..370ce48a --- /dev/null +++ b/internal/handler/document_write_scope_test.go @@ -0,0 +1,99 @@ +package handler + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/application/access" + "github.com/Tencent/WeKnora/internal/middleware" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/gin-gonic/gin" + "github.com/hibiken/asynq" + "github.com/stretchr/testify/require" +) + +type documentWriteHandlerKnowledge struct { + interfaces.KnowledgeService + called bool +} + +func (*documentWriteHandlerKnowledge) GetKnowledgeByIDOnly(context.Context, string) (*types.Knowledge, error) { + return &types.Knowledge{ID: "doc", TenantID: 7, KnowledgeBaseID: "kb"}, nil +} + +func (s *documentWriteHandlerKnowledge) UpdateKnowledgeTagBatch( + ctx context.Context, + kb string, + _ map[string][]string, +) error { + if err := access.RequireKBWrite(ctx, &types.KnowledgeBase{ID: kb, TenantID: 7}); err != nil { + return err + } + s.called = true + return nil +} + +type documentDeleteEnqueuer struct{ task *asynq.Task } + +func (q *documentDeleteEnqueuer) Enqueue(task *asynq.Task, _ ...asynq.Option) (*asynq.TaskInfo, error) { + q.task = task + return &asynq.TaskInfo{ID: "delete-task"}, nil +} + +func documentHandlerRouter() *gin.Engine { + r := gin.New() + r.Use(middleware.ErrorHandler(), func(c *gin.Context) { + ctx := types.WithCaller( + c.Request.Context(), + types.Caller{TenantID: 7, UserID: "user", Role: types.TenantRoleAdmin}, + ) + ctx = types.WithExecutionTenant(ctx, 7) + c.Set(types.TenantIDContextKey.String(), uint64(7)) + c.Request = c.Request.WithContext(ctx) + c.Next() + }) + return r +} + +func TestDocumentTagBodyRouteIssuesWriteGrant(t *testing.T) { + for _, body := range []string{`{"kb_id":"kb","updates":{"doc":["tag"]}}`, `{"updates":{"doc":["tag"]}}`} { + t.Run(body, func(t *testing.T) { + kg := &documentWriteHandlerKnowledge{} + h := &KnowledgeHandler{ + kgService: kg, + kbService: &stubKBService{get: func(context.Context, string) (*types.KnowledgeBase, error) { + return &types.KnowledgeBase{ID: "kb", TenantID: 7}, nil + }}, + } + r := documentHandlerRouter() + r.PUT("/tags", h.UpdateKnowledgeTagBatch) + req := httptest.NewRequest(http.MethodPut, "/tags", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + require.Equal(t, 200, w.Code, w.Body.String()) + require.True(t, kg.called) + }) + } +} + +func TestSingleDocumentDeletePersistsAuthorizedKBInTask(t *testing.T) { + q := &documentDeleteEnqueuer{} + h := &KnowledgeHandler{kgService: &documentWriteHandlerKnowledge{}, asynqClient: q} + r := documentHandlerRouter() + r.DELETE("/:id", h.DeleteKnowledge) + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodDelete, "/doc", nil)) + require.Equal(t, 200, w.Code, w.Body.String()) + require.NotNil(t, q.task) + var payload types.KnowledgeListDeletePayload + require.NoError(t, json.Unmarshal(q.task.Payload(), &payload)) + require.Equal(t, uint64(7), payload.TenantID) + require.Equal(t, "kb", payload.KnowledgeBaseID) + require.Equal(t, []string{"doc"}, payload.KnowledgeIDs) +} diff --git a/internal/handler/kb_access.go b/internal/handler/kb_access.go new file mode 100644 index 00000000..6cc1a4e1 --- /dev/null +++ b/internal/handler/kb_access.go @@ -0,0 +1,81 @@ +package handler + +import ( + stderrors "errors" + + "github.com/Tencent/WeKnora/internal/application/access" + "github.com/Tencent/WeKnora/internal/application/repository" + apperrors "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/middleware" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/gin-gonic/gin" +) + +// resolvedKBAccess reuses only a grant for this caller and this resource. A +// Viewer grant cannot satisfy an Editor check after the tenant context switches. +func resolvedKBAccess(c *gin.Context, kbID string, required types.OrgMemberRole) (*access.KBAccess, bool) { + grant, ok := middleware.KBAccessFromContext(c) + return grant, ok && grant.KnowledgeBase != nil && grant.KnowledgeBase.ID == kbID && + grant.Caller == middleware.KBAccessRequest(c).Caller && + access.HasKBGrant(grant.WithGrant(c.Request.Context()), kbID, grant.EffectiveTenantID, required) +} + +// resolveHandlerKBAccess is also used by body/query-based endpoints that cannot +// resolve a single KB in route middleware. It always enforces access, preserving +// the handler checks even when the middleware's RBAC rollout flag is disabled. +func resolveHandlerKBAccess(c *gin.Context, kbID string, kbService middleware.KBLookup, + shares interfaces.KBShareService, agents interfaces.AgentShareService, +) (*access.KBAccess, error) { + return resolveHandlerKBAccessFor(c, kbID, kbService, shares, agents, types.OrgRoleViewer) +} + +func resolveHandlerKBAccessFor(c *gin.Context, kbID string, kbService middleware.KBLookup, + shares interfaces.KBShareService, agents interfaces.AgentShareService, required types.OrgMemberRole, +) (*access.KBAccess, error) { + ctx := c.Request.Context() + request := middleware.KBAccessRequest(c) + if request.Caller.TenantID == 0 { + return nil, apperrors.NewUnauthorizedError("Unauthorized") + } + if kbID == "" { + return nil, apperrors.NewBadRequestError("Knowledge base ID cannot be empty") + } + if err := requireTenantAPIKeyKnowledgeBase(ctx, kbID); err != nil { + return nil, err + } + if grant, ok := resolvedKBAccess(c, kbID, required); ok { + return grant, nil + } + kb, err := kbService.GetKnowledgeBaseByID(ctx, kbID) + if err != nil { + if stderrors.Is(err, repository.ErrKnowledgeBaseNotFound) { + return nil, apperrors.NewNotFoundError("knowledge base not found") + } + logger.ErrorWithFields(ctx, err, nil) + return nil, apperrors.NewInternalServerError(err.Error()) + } + grant, err := access.ResolveKB(ctx, request, kb, required, shares, agents) + if err == nil { + // Preserve authorization for body-based routes without changing the + // execution tenant until the handler performs its resource operation. + c.Request = c.Request.WithContext(grant.WithGrant(ctx)) + } + return grant, kbAccessHTTPError(err) +} + +func kbAccessHTTPError(err error) error { + switch { + case stderrors.Is(err, access.ErrUnauthorized): + return apperrors.NewUnauthorizedError("Unauthorized") + case stderrors.Is(err, access.ErrNotFound): + return apperrors.NewNotFoundError("knowledge base not found") + case stderrors.Is(err, access.ErrForbidden): + return apperrors.NewForbiddenError("Permission denied to access this knowledge base") + case stderrors.Is(err, access.ErrInvalidAgentSource): + return apperrors.NewBadRequestError("invalid agent_source_tenant_id") + default: + return err + } +} diff --git a/internal/handler/kb_access_test.go b/internal/handler/kb_access_test.go new file mode 100644 index 00000000..2ad4ae39 --- /dev/null +++ b/internal/handler/kb_access_test.go @@ -0,0 +1,229 @@ +package handler + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/middleware" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +type handlerKBShareStub struct { + interfaces.KBShareService + calls int + caller uint64 +} + +func (s *handlerKBShareStub) CheckTenantKBPermission( + _ context.Context, + _ string, + caller uint64, + _ types.TenantRole, +) (types.OrgMemberRole, bool, error) { + s.calls++ + s.caller = caller + return types.OrgRoleViewer, true, nil +} + +type handlerKnowledgeAccessStub struct { + interfaces.KnowledgeService + knowledge *types.Knowledge +} + +func (s *handlerKnowledgeAccessStub) GetKnowledgeByIDOnly(context.Context, string) (*types.Knowledge, error) { + return s.knowledge, nil +} + +func TestKBGuardAndHandlersShareResolution(t *testing.T) { + gin.SetMode(gin.TestMode) + for _, ginCallerKey := range []bool{true, false} { + name := "request context only" + if ginCallerKey { + name = "both context surfaces" + } + t.Run(name, func(t *testing.T) { + kb := &types.KnowledgeBase{ID: "kb", TenantID: 2} + lookups := 0 + svc := &stubKBOnlyService{getByID: func(context.Context, string) (*types.KnowledgeBase, error) { + lookups++ + return kb, nil + }} + shares := &handlerKBShareStub{} + kg := &handlerKnowledgeAccessStub{ + knowledge: &types.Knowledge{ID: "doc", KnowledgeBaseID: "kb", TenantID: 2}, + } + h := &KnowledgeHandler{kbService: svc, kgService: kg, kbShareService: shares} + kbHandler := &KnowledgeBaseHandler{service: svc, kbShareService: shares} + r := gin.New() + r.Use(middleware.ErrorHandler(), func(c *gin.Context) { + ctx := context.WithValue(c.Request.Context(), types.TenantIDContextKey, uint64(1)) + ctx = context.WithValue(ctx, types.TenantRoleContextKey, types.TenantRoleViewer) + c.Request = c.Request.WithContext(ctx) + if ginCallerKey { + c.Set(types.TenantIDContextKey.String(), uint64(1)) + } + c.Next() + }) + enabled := true + r.GET("/:id", middleware.RequireKBAccess( + middleware.KBIDFromParam("id"), + types.OrgRoleViewer, + svc, + shares, + nil, + &config.Config{Tenant: &config.TenantConfig{EnableRBAC: &enabled}}, + ), func(c *gin.Context) { + _, _, effective, permission, err := kbHandler.validateAndGetKnowledgeBase(c) + require.NoError(t, err) + require.Equal(t, uint64(2), effective) + require.Equal(t, types.OrgRoleViewer, permission) + _, _, _, permission, err = h.validateKnowledgeBaseAccess(c) + require.NoError(t, err) + require.Equal(t, types.OrgRoleViewer, permission) + _, ctx, err := h.resolveKnowledgeAndValidateKBAccess(c, "doc", types.OrgRoleViewer) + require.NoError(t, err) + require.Equal(t, uint64(2), types.MustTenantIDFromContext(ctx)) + require.Equal(t, uint64(1), types.CallerFromContext(ctx).TenantID) + require.Equal(t, 1, lookups, "handlers must reuse the guarded KB") + require.Equal(t, 1, shares.calls, "handlers must reuse the permission decision") + // The effective tenant is 2, but the write must still be checked + // as caller 1 and rejected, not upgraded to resource ownership. + _, _, err = h.resolveKnowledgeAndValidateKBAccess(c, "doc", types.OrgRoleEditor) + require.Error(t, err) + require.Equal(t, uint64(1), shares.caller) + c.Status(http.StatusNoContent) + }) + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/kb", nil)) + require.Equal(t, http.StatusNoContent, w.Code, w.Body.String()) + }) + } +} + +func TestKBHandlerDoesNotReuseGrantForAnotherResourceOrCaller(t *testing.T) { + gin.SetMode(gin.TestMode) + for _, tt := range []struct { + name, requestedID string + caller uint64 + }{ + {name: "another KB", requestedID: "other", caller: 1}, + {name: "another caller", requestedID: "kb", caller: 3}, + } { + t.Run(tt.name, func(t *testing.T) { + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = httptest.NewRequest(http.MethodGet, "/", nil) + c.Set(types.TenantIDContextKey.String(), tt.caller) + c.Set(middleware.KBAccessContextKey, &middleware.KBAccess{ + KnowledgeBase: &types.KnowledgeBase{ID: "kb", TenantID: 2}, + Caller: types.Caller{ + TenantID: 1, + Role: types.TenantRoleViewer, + }, EffectiveTenantID: 2, Permission: types.OrgRoleAdmin, + }) + svc := &stubKBOnlyService{getByID: func(context.Context, string) (*types.KnowledgeBase, error) { + return &types.KnowledgeBase{ID: tt.requestedID, TenantID: 2}, nil + }} + _, err := resolveHandlerKBAccess(c, tt.requestedID, svc, nil, nil) + require.Error(t, err) + }) + } +} + +func TestKBHandlerAPIKeyScopeStillAppliesToCachedGrant(t *testing.T) { + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + ctx := types.WithTenantAPIKeyScope( + context.Background(), + types.TenantAPIKeyScope{KnowledgeBaseIDs: types.StringArray{"other"}}, + ) + c.Request = httptest.NewRequest(http.MethodGet, "/", nil).WithContext(ctx) + c.Set(types.TenantIDContextKey.String(), uint64(1)) + c.Set(middleware.KBAccessContextKey, &middleware.KBAccess{ + KnowledgeBase: &types.KnowledgeBase{ID: "kb", TenantID: 1}, + Caller: types.Caller{ + TenantID: 1, + Role: types.TenantRoleViewer, + }, EffectiveTenantID: 1, Permission: types.OrgRoleAdmin, + }) + _, err := resolveHandlerKBAccess(c, "kb", nil, nil, nil) + require.Error(t, err) +} + +type handlerAgentAccessStub struct { + interfaces.AgentShareService + anyCalls int +} + +func (s *handlerAgentAccessStub) GetSharedAgentForTenant( + context.Context, + uint64, + types.TenantRole, + string, + ...uint64, +) (*types.CustomAgent, error) { + return &types.CustomAgent{TenantID: 2, Config: types.CustomAgentConfig{KBSelectionMode: "none"}}, nil +} + +func (s *handlerAgentAccessStub) TenantCanAccessKBViaSomeSharedAgent( + context.Context, + uint64, + types.TenantRole, + *types.KnowledgeBase, +) (bool, error) { + s.anyCalls++ + return true, nil +} + +func TestKBHandlersHonorExplicitAgentWithoutRouteGuard(t *testing.T) { + for _, query := range []string{"", "?agent_id=restricted"} { + t.Run(query, func(t *testing.T) { + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = httptest.NewRequest(http.MethodGet, "/kb"+query, nil) + c.Set(types.TenantIDContextKey.String(), uint64(1)) + c.Params = gin.Params{{Key: "id", Value: "kb"}} + svc := &stubKBOnlyService{getByID: func(context.Context, string) (*types.KnowledgeBase, error) { + return &types.KnowledgeBase{ID: "kb", TenantID: 2}, nil + }} + agents := &handlerAgentAccessStub{} + h := &KnowledgeHandler{kbService: svc, agentShareService: agents} + kbHandler := &KnowledgeBaseHandler{service: svc, agentShareService: agents} + _, _, _, _, kbErr := kbHandler.validateAndGetKnowledgeBase(c) + _, _, _, _, knowledgeErr := h.validateKnowledgeBaseAccessWithKBID(c, "kb") + if query == "" { + require.NoError(t, kbErr) + require.NoError(t, knowledgeErr) + require.Equal(t, 2, agents.anyCalls) + } else { + require.Error(t, kbErr) + require.Error(t, knowledgeErr) + require.Zero(t, agents.anyCalls) + } + }) + } +} + +func TestKBHandlerEnforcesAccessWhenRouteRBACIsDisabled(t *testing.T) { + svc := &stubKBOnlyService{getByID: func(context.Context, string) (*types.KnowledgeBase, error) { + return &types.KnowledgeBase{ID: "kb", TenantID: 2}, nil + }} + r := gin.New() + r.Use(middleware.ErrorHandler(), func(c *gin.Context) { + c.Set(types.TenantIDContextKey.String(), uint64(1)) + c.Next() + }) + disabled := false + r.GET("/:id", middleware.RequireKBAccess(middleware.KBIDFromParam("id"), types.OrgRoleViewer, + svc, nil, nil, &config.Config{Tenant: &config.TenantConfig{EnableRBAC: &disabled}}), func(c *gin.Context) { + _, err := resolveHandlerKBAccess(c, c.Param("id"), svc, nil, nil) + require.Error(t, err) + _ = c.Error(err) + }) + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/kb", nil)) + require.Equal(t, http.StatusForbidden, w.Code) +} diff --git a/internal/handler/knowledge.go b/internal/handler/knowledge.go index 68f98f60..05820223 100644 --- a/internal/handler/knowledge.go +++ b/internal/handler/knowledge.go @@ -3,21 +3,18 @@ package handler import ( "context" "encoding/json" + goerrors "errors" "fmt" - "io" - "mime" "net/http" "strconv" "strings" "time" - goerrors "errors" - - "github.com/Tencent/WeKnora/internal/agent/tools" + "github.com/Tencent/WeKnora/internal/application/access" "github.com/Tencent/WeKnora/internal/application/repository" - "github.com/Tencent/WeKnora/internal/application/service" "github.com/Tencent/WeKnora/internal/config" "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/filetransport" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/middleware" "github.com/Tencent/WeKnora/internal/tracing/langfuse" @@ -64,13 +61,11 @@ func NewKnowledgeHandler( // requireKBOwnershipOrAdmin enforces the same "KB creator OR Admin+" matrix // used by OwnedKBOrAdmin for routes whose KB id comes from the request body. func (h *KnowledgeHandler) requireKBOwnershipOrAdmin(c *gin.Context, kbID string) error { - creator, err := resolveKBCreatorByKBID(c, h.kbService, kbID) evalErr := middleware.EvaluateOwnershipOrRole( c.Request.Context(), h.cfg, types.TenantRoleAdmin, - creator, - err, + func() (string, error) { return resolveKBCreatorByKBID(c, h.kbService, kbID) }, ) if evalErr == nil { return nil @@ -97,133 +92,74 @@ func (h *KnowledgeHandler) validateKnowledgeBaseAccess(c *gin.Context) (*types.K // validateKnowledgeBaseAccessWithKBID validates access to the given knowledge base ID (e.g. from query or body). // Enforces per-API-key KB scope before tenant/share/agent resolution. // Returns the knowledge base, kbID, effective tenant ID, permission, and error. -func (h *KnowledgeHandler) validateKnowledgeBaseAccessWithKBID(c *gin.Context, kbID string) (*types.KnowledgeBase, string, uint64, types.OrgMemberRole, error) { - ctx := c.Request.Context() - tenantID := c.GetUint64(types.TenantIDContextKey.String()) - if tenantID == 0 { - logger.Error(ctx, "Failed to get tenant ID") - return nil, "", 0, "", errors.NewUnauthorizedError("Unauthorized") - } - userID, userExists := c.Get(types.UserIDContextKey.String()) - callerTenantRole := types.TenantRoleFromContext(ctx) +func (h *KnowledgeHandler) validateKnowledgeBaseAccessWithKBID( + c *gin.Context, + kbID string, +) (*types.KnowledgeBase, string, uint64, types.OrgMemberRole, error) { kbID = secutils.SanitizeForLog(kbID) - if kbID == "" { - return nil, "", 0, "", errors.NewBadRequestError("Knowledge base ID cannot be empty") - } - if err := requireTenantAPIKeyKnowledgeBase(ctx, kbID); err != nil { + grant, err := resolveHandlerKBAccess(c, kbID, h.kbService, h.kbShareService, h.agentShareService) + if err != nil { return nil, kbID, 0, "", err } - kb, err := h.kbService.GetKnowledgeBaseByID(ctx, kbID) - if err != nil { - // Same not-found-vs-real-error split as knowledgebase.go's - // validateAndGetKnowledgeBase: ErrKnowledgeBaseNotFound is the - // expected outcome for a probed/stale kb id and must surface as - // 404, not the generic 500 the original code produced. - if goerrors.Is(err, repository.ErrKnowledgeBaseNotFound) { - return nil, kbID, 0, "", errors.NewNotFoundError("knowledge base not found") - } - logger.ErrorWithFields(ctx, err, nil) - return nil, kbID, 0, "", errors.NewInternalServerError(err.Error()) - } - if kb.TenantID == tenantID { - return kb, kbID, tenantID, types.OrgRoleAdmin, nil - } - if h.kbShareService != nil { - permission, isShared, permErr := h.kbShareService.CheckTenantKBPermission(ctx, kbID, tenantID, callerTenantRole) - if permErr == nil && isShared { - sourceTenantID, srcErr := h.kbShareService.GetKBSourceTenant(ctx, kbID) - if srcErr == nil { - logger.Infof(ctx, "Tenant %d accessing shared KB %s with permission %s, source tenant: %d", - tenantID, kbID, permission, sourceTenantID) - return kb, kbID, sourceTenantID, permission, nil - } - } - } - if h.agentShareService != nil { - can, err := h.agentShareService.TenantCanAccessKBViaSomeSharedAgent(ctx, tenantID, callerTenantRole, kb) - if err == nil && can { - logger.Infof(ctx, "Tenant %d accessing KB %s via some shared agent", tenantID, kbID) - return kb, kbID, kb.TenantID, types.OrgRoleViewer, nil - } - } - _ = userID - _ = userExists - logger.Warnf(ctx, "Permission denied to access KB %s, tenant ID: %d, KB tenant: %d", kbID, tenantID, kb.TenantID) - return nil, kbID, 0, "", errors.NewForbiddenError("Permission denied to access this knowledge base") + return grant.KnowledgeBase, kbID, grant.EffectiveTenantID, grant.Permission, nil } -// resolveKnowledgeAndValidateKBAccess resolves knowledge by ID and validates KB access (owner or shared with required permission). -// Returns the knowledge, context with effectiveTenantID set for downstream service calls, and error. -func (h *KnowledgeHandler) resolveKnowledgeAndValidateKBAccess(c *gin.Context, knowledgeID string, requiredPermission types.OrgMemberRole) (*types.Knowledge, context.Context, error) { +func (h *KnowledgeHandler) validateKnowledgeBaseWriteAccessWithKBID( + c *gin.Context, + kbID string, +) (*types.KnowledgeBase, string, uint64, types.OrgMemberRole, error) { + grant, err := resolveHandlerKBAccessFor( + c, + kbID, + h.kbService, + h.kbShareService, + h.agentShareService, + types.OrgRoleEditor, + ) + if err != nil { + return nil, kbID, 0, "", err + } + if err := access.RequireKBWrite(grant.Context(c.Request.Context()), grant.KnowledgeBase); err != nil { + return nil, kbID, 0, "", kbAccessHTTPError(err) + } + return grant.KnowledgeBase, kbID, grant.EffectiveTenantID, grant.Permission, nil +} + +// resolveKnowledgeAndValidateKBAccess resolves a document and checks its parent +// KB using the same policy as KB routes. The persisted document supplies the KB +// and owner reference, so unguarded callers do not need an additional KB lookup. +func (h *KnowledgeHandler) resolveKnowledgeAndValidateKBAccess( + c *gin.Context, + knowledgeID string, + requiredPermission types.OrgMemberRole, +) (*types.Knowledge, context.Context, error) { ctx := c.Request.Context() - tenantID := c.GetUint64(types.TenantIDContextKey.String()) - if tenantID == 0 { + request := middleware.KBAccessRequest(c) + if request.Caller.TenantID == 0 { return nil, ctx, errors.NewUnauthorizedError("Unauthorized") } - userID, userExists := c.Get(types.UserIDContextKey.String()) - callerTenantRole := types.TenantRoleFromContext(ctx) - knowledge, err := h.kgService.GetKnowledgeByIDOnly(ctx, knowledgeID) - if err != nil { + if err != nil || knowledge == nil { return nil, ctx, errors.NewNotFoundError("Knowledge not found") } if err := requireTenantAPIKeyKnowledgeBase(ctx, knowledge.KnowledgeBaseID); err != nil { return nil, ctx, err } - - // Owner: knowledge belongs to caller's tenant - if knowledge.TenantID == tenantID { - return knowledge, context.WithValue(ctx, types.TenantIDContextKey, tenantID), nil - } - - // Shared KB: check organization permission - if h.kbShareService != nil { - permission, isShared, permErr := h.kbShareService.CheckTenantKBPermission(ctx, knowledge.KnowledgeBaseID, tenantID, callerTenantRole) - if permErr == nil && isShared && permission.HasPermission(requiredPermission) { - effectiveTenantID := knowledge.TenantID - return knowledge, context.WithValue(ctx, types.TenantIDContextKey, effectiveTenantID), nil + if grant, ok := resolvedKBAccess(c, knowledge.KnowledgeBaseID, requiredPermission); ok { + if grant.EffectiveTenantID != knowledge.TenantID { + return nil, ctx, errors.NewForbiddenError("Permission denied to access this knowledge") } + return knowledge, grant.Context(ctx), nil } - // Shared agent: request passes agent_id, or user has any shared agent that can access this KB - if h.agentShareService != nil && requiredPermission == types.OrgRoleViewer { - agentID := c.Query("agent_id") - if agentID != "" { - sourceTenantID, parseErr := types.ParseAgentSourceTenantID(c.Query(types.AgentSourceTenantIDParam)) - if parseErr != nil { - return nil, ctx, errors.NewBadRequestError(parseErr.Error()) - } - agent, err := h.agentShareService.GetSharedAgentForTenant(ctx, tenantID, callerTenantRole, agentID, sourceTenantID) - if err == nil && agent != nil { - if knowledge.TenantID != agent.TenantID { - return nil, ctx, errors.NewForbiddenError("Permission denied to access this knowledge") - } - mode := agent.Config.KBSelectionMode - if mode == "none" { - return nil, ctx, errors.NewForbiddenError("Permission denied to access this knowledge") - } - if mode == "all" { - return knowledge, context.WithValue(ctx, types.TenantIDContextKey, knowledge.TenantID), nil - } - if mode == "selected" { - for _, kbID := range agent.Config.KnowledgeBases { - if kbID == knowledge.KnowledgeBaseID { - return knowledge, context.WithValue(ctx, types.TenantIDContextKey, knowledge.TenantID), nil - } - } - return nil, ctx, errors.NewForbiddenError("Permission denied to access this knowledge") - } - } - } else { - kbRef := &types.KnowledgeBase{ID: knowledge.KnowledgeBaseID, TenantID: knowledge.TenantID} - can, err := h.agentShareService.TenantCanAccessKBViaSomeSharedAgent(ctx, tenantID, callerTenantRole, kbRef) - if err == nil && can { - return knowledge, context.WithValue(ctx, types.TenantIDContextKey, knowledge.TenantID), nil - } - } + kb := &types.KnowledgeBase{ID: knowledge.KnowledgeBaseID, TenantID: knowledge.TenantID} + grant, err := access.ResolveKB(ctx, request, kb, requiredPermission, h.kbShareService, h.agentShareService) + if goerrors.Is(err, access.ErrForbidden) { + return nil, ctx, errors.NewForbiddenError("Permission denied to access this knowledge") } - _ = userID - _ = userExists - return nil, ctx, errors.NewForbiddenError("Permission denied to access this knowledge") + if err != nil { + return nil, ctx, kbAccessHTTPError(err) + } + return knowledge, grant.Context(ctx), nil } // handleDuplicateKnowledgeError handles cases where duplicate knowledge is detected @@ -248,12 +184,13 @@ func (h *KnowledgeHandler) handleDuplicateKnowledgeError(c *gin.Context, // enqueueKnowledgeListDelete enqueues an async batch-delete task for the // given knowledge IDs and returns the asynq task ID. func (h *KnowledgeHandler) enqueueKnowledgeListDelete( - ctx context.Context, tenantID uint64, ids []string, + ctx context.Context, tenantID uint64, kbID string, ids []string, ) (string, error) { payload := types.KnowledgeListDeletePayload{ - TenantID: tenantID, - KnowledgeIDs: ids, - Initiator: types.TaskInitiatorFromContext(ctx), + KnowledgeBaseID: kbID, + TenantID: tenantID, + KnowledgeIDs: ids, + Initiator: types.TaskInitiatorFromContext(ctx), } langfuse.InjectTracing(ctx, &payload) payloadBytes, err := json.Marshal(payload) @@ -272,13 +209,14 @@ func (h *KnowledgeHandler) enqueueKnowledgeListDelete( // enqueueKnowledgeListReparse enqueues an async batch-reparse task for the // given knowledge IDs and returns the asynq task ID. func (h *KnowledgeHandler) enqueueKnowledgeListReparse( - ctx context.Context, tenantID uint64, ids []string, processConfig *types.KnowledgeProcessOverrides, + ctx context.Context, tenantID uint64, kbID string, ids []string, processConfig *types.KnowledgeProcessOverrides, ) (string, error) { payload := types.KnowledgeListReparsePayload{ - TenantID: tenantID, - KnowledgeIDs: ids, - ProcessConfig: processConfig, - Initiator: types.TaskInitiatorFromContext(ctx), + KnowledgeBaseID: kbID, + TenantID: tenantID, + KnowledgeIDs: ids, + ProcessConfig: processConfig, + Initiator: types.TaskInitiatorFromContext(ctx), } langfuse.InjectTracing(ctx, &payload) payloadBytes, err := json.Marshal(payload) @@ -323,7 +261,7 @@ func (h *KnowledgeHandler) CreateKnowledgeFromFile(c *gin.Context) { c.Error(err) return } - ctx = context.WithValue(ctx, types.TenantIDContextKey, effectiveTenantID) + ctx = types.WithExecutionTenant(c.Request.Context(), effectiveTenantID) // Check write permission if permission != types.OrgRoleAdmin && permission != types.OrgRoleEditor { @@ -472,7 +410,7 @@ func (h *KnowledgeHandler) CreateKnowledgeFromURL(c *gin.Context) { c.Error(err) return } - ctx = context.WithValue(ctx, types.TenantIDContextKey, effectiveTenantID) + ctx = types.WithExecutionTenant(c.Request.Context(), effectiveTenantID) // Check write permission if permission != types.OrgRoleAdmin && permission != types.OrgRoleEditor { @@ -569,7 +507,7 @@ func (h *KnowledgeHandler) CreateManualKnowledge(c *gin.Context) { c.Error(err) return } - ctx = context.WithValue(ctx, types.TenantIDContextKey, effectiveTenantID) + ctx = types.WithExecutionTenant(c.Request.Context(), effectiveTenantID) // Check write permission if permission != types.OrgRoleAdmin && permission != types.OrgRoleEditor { @@ -955,7 +893,7 @@ func (h *KnowledgeHandler) ListKnowledge(c *gin.Context) { } // Update context with effective tenant ID for shared KB access - ctx = context.WithValue(ctx, types.TenantIDContextKey, effectiveTenantID) + ctx = types.WithExecutionTenant(c.Request.Context(), effectiveTenantID) // Parse pagination parameters from query string var pagination types.Pagination @@ -1062,7 +1000,7 @@ func (h *KnowledgeHandler) ListKnowledgeFolders(c *gin.Context) { c.Error(err) return } - ctx = context.WithValue(ctx, types.TenantIDContextKey, effectiveTenantID) + ctx = types.WithExecutionTenant(c.Request.Context(), effectiveTenantID) tree, err := h.kgService.ListKnowledgeFolderTree(ctx, kbID) if err != nil { @@ -1127,7 +1065,7 @@ func (h *KnowledgeHandler) MoveKnowledgeToFolder(c *gin.Context) { c.Error(err) return } - ctx = context.WithValue(ctx, types.TenantIDContextKey, effectiveTenantID) + ctx = types.WithExecutionTenant(c.Request.Context(), effectiveTenantID) // Guard against cross-KB moves: the service layer scopes by tenant, so the // handler must confirm every entry belongs to the requested knowledge base. @@ -1186,7 +1124,7 @@ func (h *KnowledgeHandler) RenameKnowledgeFolder(c *gin.Context) { return } - _, kbID, effectiveTenantID, permission, err := h.validateKnowledgeBaseAccess(c) + _, kbID, effectiveTenantID, permission, err := h.validateKnowledgeBaseWriteAccessWithKBID(c, c.Param("id")) if err != nil { c.Error(err) return @@ -1199,7 +1137,7 @@ func (h *KnowledgeHandler) RenameKnowledgeFolder(c *gin.Context) { c.Error(err) return } - ctx = context.WithValue(ctx, types.TenantIDContextKey, effectiveTenantID) + ctx = types.WithExecutionTenant(c.Request.Context(), effectiveTenantID) affected, err := h.kgService.RenameKnowledgeFolder(ctx, kbID, req.From, req.To) if err != nil { @@ -1246,7 +1184,7 @@ func (h *KnowledgeHandler) requireKnowledgeWriteAccess( c *gin.Context, requestedKBID string, ) (string, uint64, error) { - _, kbID, effectiveTenantID, permission, err := h.validateKnowledgeBaseAccessWithKBID(c, requestedKBID) + _, kbID, effectiveTenantID, permission, err := h.validateKnowledgeBaseWriteAccessWithKBID(c, requestedKBID) if err != nil { return "", 0, err } @@ -1310,12 +1248,17 @@ func (h *KnowledgeHandler) DeleteKnowledge(c *gin.Context) { return } - _, effCtx, err := h.resolveKnowledgeAndValidateKBAccess(c, id, types.OrgRoleEditor) + knowledge, effCtx, err := h.resolveKnowledgeAndValidateKBAccess(c, id, types.OrgRoleEditor) if err != nil { c.Error(err) return } + if err := access.RejectMovingKnowledge(knowledge); err != nil { + _ = c.Error(err) + return + } + // Reuse the batch async pipeline so single-item delete shares the same // hardening (asynq retries, business-aware queue routing, marking-as-deleting // inside the worker) as BatchDeleteKnowledge / ClearKnowledgeBaseContents. @@ -1327,7 +1270,7 @@ func (h *KnowledgeHandler) DeleteKnowledge(c *gin.Context) { } logger.Infof(ctx, "Enqueuing knowledge delete, ID: %s", secutils.SanitizeForLog(id)) - taskID, err := h.enqueueKnowledgeListDelete(effCtx, effectiveTenantID, []string{id}) + taskID, err := h.enqueueKnowledgeListDelete(effCtx, effectiveTenantID, knowledge.KnowledgeBaseID, []string{id}) if err != nil { logger.Errorf(ctx, "Failed to enqueue knowledge delete task: %v", err) c.Error(errors.NewInternalServerError("Failed to enqueue delete task")) @@ -1384,7 +1327,7 @@ func (h *KnowledgeHandler) BatchDeleteKnowledge(c *gin.Context) { } // Validate KB access (editor or admin) using the kb_id from body. - _, kbID, effectiveTenantID, permission, err := h.validateKnowledgeBaseAccessWithKBID(c, req.KBID) + _, kbID, effectiveTenantID, permission, err := h.validateKnowledgeBaseWriteAccessWithKBID(c, req.KBID) if err != nil { c.Error(err) return @@ -1397,11 +1340,10 @@ func (h *KnowledgeHandler) BatchDeleteKnowledge(c *gin.Context) { c.Error(err) return } - ctx = context.WithValue(ctx, types.TenantIDContextKey, effectiveTenantID) + ctx = types.WithExecutionTenant(c.Request.Context(), effectiveTenantID) // Single batch fetch to validate that every id exists and belongs to the - // requested KB. The service-layer DeleteKnowledgeList only enforces tenant - // scope, not KB scope, so the handler must guard against cross-KB deletion. + // requested KB before admitting a task with that exact KB scope. knowledgeList, err := h.kgService.GetKnowledgeBatch(ctx, effectiveTenantID, ids) if err != nil { logger.ErrorWithFields(ctx, err, nil) @@ -1413,6 +1355,10 @@ func (h *KnowledgeHandler) BatchDeleteKnowledge(c *gin.Context) { return } for _, k := range knowledgeList { + if err := access.RejectMovingKnowledge(k); err != nil { + _ = c.Error(err) + return + } if k.KnowledgeBaseID != kbID { c.Error(errors.NewBadRequestError( fmt.Sprintf("Knowledge %s does not belong to knowledge base %s", @@ -1421,7 +1367,7 @@ func (h *KnowledgeHandler) BatchDeleteKnowledge(c *gin.Context) { } } - taskID, err := h.enqueueKnowledgeListDelete(ctx, effectiveTenantID, ids) + taskID, err := h.enqueueKnowledgeListDelete(ctx, effectiveTenantID, kbID, ids) if err != nil { logger.Errorf(ctx, "Failed to enqueue batch knowledge delete task: %v", err) c.Error(errors.NewInternalServerError("Failed to enqueue batch delete task")) @@ -1458,7 +1404,7 @@ func (h *KnowledgeHandler) ClearKnowledgeBaseContents(c *gin.Context) { ctx := c.Request.Context() logger.Info(ctx, "Start clearing knowledge base contents") - kb, kbID, effectiveTenantID, permission, err := h.validateKnowledgeBaseAccess(c) + kb, kbID, effectiveTenantID, permission, err := h.validateKnowledgeBaseWriteAccessWithKBID(c, c.Param("id")) if err != nil { c.Error(err) return @@ -1471,7 +1417,7 @@ func (h *KnowledgeHandler) ClearKnowledgeBaseContents(c *gin.Context) { return } - ctx = context.WithValue(ctx, types.TenantIDContextKey, effectiveTenantID) + ctx = types.WithExecutionTenant(c.Request.Context(), effectiveTenantID) knowledgeList, err := h.kgService.ListKnowledgeByKnowledgeBaseID(ctx, kbID) if err != nil { @@ -1492,10 +1438,14 @@ func (h *KnowledgeHandler) ClearKnowledgeBaseContents(c *gin.Context) { knowledgeIDs := make([]string, 0, len(knowledgeList)) for _, knowledge := range knowledgeList { + if err := access.RejectMovingKnowledge(knowledge); err != nil { + _ = c.Error(err) + return + } knowledgeIDs = append(knowledgeIDs, knowledge.ID) } - taskID, err := h.enqueueKnowledgeListDelete(ctx, effectiveTenantID, knowledgeIDs) + taskID, err := h.enqueueKnowledgeListDelete(ctx, effectiveTenantID, kbID, knowledgeIDs) if err != nil { logger.Errorf(ctx, "Failed to enqueue knowledge list delete task: %v", err) c.Error(errors.NewInternalServerError("Failed to enqueue cleanup task")) @@ -1552,34 +1502,14 @@ func (h *KnowledgeHandler) DownloadKnowledgeFile(c *gin.Context) { c.Error(errors.NewInternalServerError("Failed to retrieve file").WithDetails(err.Error())) return } - defer file.Close() - - logger.Infof( - ctx, - "Knowledge file retrieved successfully, ID: %s, filename: %s", - secutils.SanitizeForLog(id), - secutils.SanitizeForLog(filename), - ) - - // Set response headers for file download c.Header("Content-Description", "File Transfer") c.Header("Content-Transfer-Encoding", "binary") - cd := mime.FormatMediaType("attachment", map[string]string{"filename": filename}) - c.Header("Content-Disposition", cd) - c.Header("Content-Type", "application/octet-stream") c.Header("Expires", "0") - c.Header("Cache-Control", "must-revalidate") - c.Header("Pragma", "public") - - // Stream file content to response - c.Stream(func(w io.Writer) bool { - if _, err := io.Copy(w, file); err != nil { - logger.Errorf(ctx, "Failed to send file: %v", err) - return false - } - logger.Debug(ctx, "File sending completed") - return false - }) + if err := filetransport.Serve(c.Writer, c.Request, file, filetransport.Options{ + Filename: filename, Download: true, ContentType: "application/octet-stream", CacheControl: "private, no-store", + }); err != nil { + logger.Errorf(ctx, "Failed to send file: %v", err) + } } // mimeTypeByExt returns the MIME type for a given file extension. @@ -1621,25 +1551,11 @@ func (h *KnowledgeHandler) PreviewKnowledgeFile(c *gin.Context) { c.Error(errors.NewInternalServerError("Failed to retrieve file").WithDetails(err.Error())) return } - defer file.Close() - - contentType, inline := secutils.SafeContentTypeByFilename(filename) - c.Header("Content-Type", contentType) - c.Header("X-Content-Type-Options", "nosniff") - disposition := "inline" - if !inline { - disposition = "attachment" + if err := filetransport.Serve(c.Writer, c.Request, file, filetransport.Options{ + Filename: filename, CacheControl: "private, no-store", + }); err != nil { + logger.Errorf(ctx, "Failed to stream preview: %v", err) } - c.Header("Content-Disposition", mime.FormatMediaType(disposition, map[string]string{"filename": filename})) - c.Header("Cache-Control", "private, max-age=3600") - - c.Stream(func(w io.Writer) bool { - if _, err := io.Copy(w, file); err != nil { - logger.Errorf(ctx, "Failed to stream preview: %v", err) - return false - } - return false - }) } // GetKnowledgeBatchRequest defines parameters for batch knowledge retrieval @@ -1667,13 +1583,11 @@ type GetKnowledgeBatchRequest struct { func (h *KnowledgeHandler) GetKnowledgeBatch(c *gin.Context) { ctx := c.Request.Context() - tenantID, ok := c.Get(types.TenantIDContextKey.String()) - if !ok { - logger.Error(ctx, "Failed to get tenant ID") + effectiveTenantID := middleware.KBAccessRequest(c).Caller.TenantID + if effectiveTenantID == 0 { c.Error(errors.NewUnauthorizedError("Unauthorized")) return } - effectiveTenantID := tenantID.(uint64) var req GetKnowledgeBatchRequest if err := c.ShouldBindQuery(&req); err != nil { @@ -1686,40 +1600,23 @@ func (h *KnowledgeHandler) GetKnowledgeBatch(c *gin.Context) { return } - // agentAllowedKBIDs restricts results to the agent's configured KB scope. - // nil = no agent restriction; empty slice = agent has no KB access (none mode). - var agentAllowedKBIDs []string - - // Optional agent_id: when using shared agent, resolve agent and use its tenant for batch retrieval (so shared KB files can be loaded after refresh) - if agentID := secutils.SanitizeForLog(req.AgentID); agentID != "" && h.agentShareService != nil { - userIDVal, ok := c.Get(types.UserIDContextKey.String()) - if !ok { - c.Error(errors.NewUnauthorizedError("Unauthorized")) + // A nil scope means no agent selector. Every resolved scope, including + // mode=all, binds results to the authorized agent's source tenant. + var agentScope *types.SharedAgentKBScope + if agentID := secutils.SanitizeForLog(req.AgentID); agentID != "" { + agent, err := resolveSharedAgentForRequest(c, agentID, h.agentShareService) + if err != nil { + _ = c.Error(err) return } - userID, _ := userIDVal.(string) - currentTenantID := c.GetUint64(types.TenantIDContextKey.String()) - if currentTenantID == 0 { - c.Error(errors.NewUnauthorizedError("Unauthorized")) - return - } - callerTenantRole := types.TenantRoleFromContext(ctx) - agent, err := h.agentShareService.GetSharedAgentForTenant(ctx, currentTenantID, callerTenantRole, agentID, req.AgentSourceTenantID) - if err != nil || agent == nil { - logger.Warnf(ctx, "GetKnowledgeBatch: invalid or inaccessible shared agent %s: %v", agentID, err) - c.Error(errors.NewForbiddenError("Invalid or inaccessible shared agent").WithDetails(err.Error())) - return - } - _ = userID + ctx = access.WithSharedAgent(ctx, agent) + scope := types.NewSharedAgentKBScope(agent) + agentScope = &scope effectiveTenantID = agent.TenantID - agentAllowedKBIDs = resolveAgentAllowedKBIDs(agent) - - if agentAllowedKBIDs != nil && len(agentAllowedKBIDs) == 0 { + if scope.IsEmpty() { c.JSON(http.StatusOK, gin.H{"success": true, "data": []*types.Knowledge{}}) return } - logger.Infof(ctx, "Batch retrieving knowledge with agent_id, effective tenant ID: %d, IDs count: %d, allowed KBs: %v", - effectiveTenantID, len(req.IDs), agentAllowedKBIDs) } var knowledges []*types.Knowledge @@ -1730,42 +1627,46 @@ func (h *KnowledgeHandler) GetKnowledgeBatch(c *gin.Context) { // Optional kb_id: validate KB access and use effective tenant for shared KB if kbID := secutils.SanitizeForLog(req.KBID); kbID != "" { - _, _, effID, _, err := h.validateKnowledgeBaseAccessWithKBID(c, kbID) - if err != nil { - c.Error(err) + _, _, effID, _, accessErr := h.validateKnowledgeBaseAccessWithKBID(c, kbID) + if accessErr != nil { + _ = c.Error(accessErr) return } - if agentAllowedKBIDs != nil && !sliceContains(agentAllowedKBIDs, kbID) { + if agentScope != nil && !agentScope.Allows(kbID, effID) { c.Error(errors.NewForbiddenError("Knowledge base not accessible through this agent")) return } scopeKBID = kbID effectiveTenantID = effID - ctx = context.WithValue(ctx, types.TenantIDContextKey, effectiveTenantID) + ctx = types.WithExecutionTenant(c.Request.Context(), effectiveTenantID) logger.Infof(ctx, "Batch retrieving knowledge with kb_id, effective tenant ID: %d, IDs count: %d", effectiveTenantID, len(req.IDs)) + knowledges, err = h.kgService.GetKnowledgeBatch(ctx, effectiveTenantID, req.IDs) + } else if agentScope != nil { + // Keep the authorized agent's read scope and the original caller. + ctx = types.WithExecutionTenant(ctx, effectiveTenantID) knowledges, err = h.kgService.GetKnowledgeBatch(ctx, effectiveTenantID, req.IDs) } else { - // No kb_id: use GetKnowledgeBatchWithSharedAccess (or effectiveTenantID may already be set by agent_id for shared agent) - logger.Infof(ctx, "Batch retrieving knowledge without kb_id, effective tenant ID: %d, IDs count: %d", - effectiveTenantID, len(req.IDs)) - knowledges, err = h.kgService.GetKnowledgeBatchWithSharedAccess(ctx, effectiveTenantID, req.IDs) } + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + _ = c.Error(errors.NewInternalServerError("Failed to retrieve knowledge list").WithDetails(err.Error())) + return + } // Build the effective allowed-KB set from explicit kb_id, shared agent // scope, and per-API-key KB restrictions. var allowedKBSet map[string]bool if scopeKBID != "" { allowedKBSet = map[string]bool{scopeKBID: true} - } else if agentAllowedKBIDs != nil { - allowedKBSet = make(map[string]bool, len(agentAllowedKBIDs)) - for _, id := range agentAllowedKBIDs { - allowedKBSet[id] = true - } } + if agentScope != nil { + knowledges = filterKnowledgeByAgentScope(knowledges, *agentScope) + } + if apiKeySet := tenantAPIKeyAllowedKBSet(ctx); apiKeySet != nil { allowedKBSet = intersectKBAllowSet(allowedKBSet, apiKeySet) } @@ -1773,12 +1674,6 @@ func (h *KnowledgeHandler) GetKnowledgeBatch(c *gin.Context) { knowledges = filterKnowledgesByKBAllowSet(knowledges, allowedKBSet) } - if err != nil { - logger.ErrorWithFields(ctx, err, nil) - c.Error(errors.NewInternalServerError("Failed to retrieve knowledge list").WithDetails(err.Error())) - return - } - logger.Infof(ctx, "Batch knowledge retrieval successful, requested count: %d, returned count: %d", len(req.IDs), len(knowledges)) @@ -1842,7 +1737,12 @@ func (h *KnowledgeHandler) UpdateKnowledge(c *gin.Context) { if err := h.kgService.UpdateKnowledge(effCtx, &knowledge); err != nil { logger.ErrorWithFields(ctx, err, nil) - c.Error(errors.NewInternalServerError(err.Error())) + var appErr *errors.AppError + if goerrors.As(err, &appErr) { + _ = c.Error(appErr) + } else { + _ = c.Error(errors.NewInternalServerError(err.Error())) + } return } updated, getErr := h.kgService.GetKnowledgeByID(effCtx, id) @@ -1874,7 +1774,12 @@ func (h *KnowledgeHandler) RegenerateKnowledgeSummary(c *gin.Context) { knowledge, err := h.kgService.GetKnowledgeByID(effCtx, id) if err != nil { logger.ErrorWithFields(ctx, err, nil) - c.Error(errors.NewInternalServerError(err.Error())) + var appErr *errors.AppError + if goerrors.As(err, &appErr) { + _ = c.Error(appErr) + } else { + _ = c.Error(errors.NewInternalServerError(err.Error())) + } return } if knowledge.SummaryStatus == "" || knowledge.SummaryStatus == types.SummaryStatusNone { @@ -1887,7 +1792,12 @@ func (h *KnowledgeHandler) RegenerateKnowledgeSummary(c *gin.Context) { } if err != nil { logger.ErrorWithFields(ctx, err, nil) - c.Error(errors.NewBadRequestError(err.Error())) + var appErr *errors.AppError + if goerrors.As(err, &appErr) { + _ = c.Error(appErr) + } else { + _ = c.Error(errors.NewBadRequestError(err.Error())) + } return } c.JSON(http.StatusOK, gin.H{"success": true, "data": knowledge}) @@ -2098,7 +2008,7 @@ func (h *KnowledgeHandler) UpdateKnowledgeTagBatch(c *gin.Context) { c.Error(errors.NewUnauthorizedError("Unauthorized")) return } - ctx = context.WithValue(ctx, types.TenantIDContextKey, tenantID) + ctx = types.WithExecutionTenant(ctx, tenantID) var req knowledgeTagBatchRequest if err := c.ShouldBindJSON(&req); err != nil { @@ -2109,7 +2019,7 @@ func (h *KnowledgeHandler) UpdateKnowledgeTagBatch(c *gin.Context) { // Resolve effective tenant and the authorized KB scope. var authorizedKBID string if kbID := secutils.SanitizeForLog(req.KBID); kbID != "" { - _, _, effID, permission, err := h.validateKnowledgeBaseAccessWithKBID(c, kbID) + _, _, effID, permission, err := h.validateKnowledgeBaseWriteAccessWithKBID(c, kbID) if err != nil { c.Error(err) return @@ -2119,7 +2029,7 @@ func (h *KnowledgeHandler) UpdateKnowledgeTagBatch(c *gin.Context) { return } authorizedKBID = kbID - ctx = context.WithValue(ctx, types.TenantIDContextKey, effID) + ctx = types.WithExecutionTenant(c.Request.Context(), effID) } else if len(req.Updates) > 0 { // No kb_id: infer from first knowledge ID so shared-KB updates work without client sending kb_id var firstKnowledgeID string @@ -2137,6 +2047,11 @@ func (h *KnowledgeHandler) UpdateKnowledgeTagBatch(c *gin.Context) { ctx = effCtx } } + if err := h.requireKBOwnershipOrAdmin(c, authorizedKBID); err != nil { + _ = c.Error(err) + return + } + if err := h.kgService.UpdateKnowledgeTagBatch(ctx, authorizedKBID, req.Updates); err != nil { logger.ErrorWithFields(ctx, err, nil) c.Error(err) @@ -2198,7 +2113,12 @@ func (h *KnowledgeHandler) UpdateImageInfo(c *gin.Context) { err = h.kgService.UpdateImageInfo(effCtx, id, chunkID, secutils.SanitizeForLog(request.ImageInfo)) if err != nil { logger.ErrorWithFields(ctx, err, nil) - c.Error(errors.NewInternalServerError(err.Error())) + var appErr *errors.AppError + if goerrors.As(err, &appErr) { + _ = c.Error(appErr) + } else { + _ = c.Error(errors.NewInternalServerError(err.Error())) + } return } @@ -2262,36 +2182,15 @@ func (h *KnowledgeHandler) SearchKnowledge(c *gin.Context) { agentID := c.Query("agent_id") if agentID != "" { - userIDVal, ok := c.Get(types.UserIDContextKey.String()) - if !ok { - c.Error(errors.NewUnauthorizedError("user ID not found")) - return - } - _ = userIDVal - currentTenantID := c.GetUint64(types.TenantIDContextKey.String()) - if currentTenantID == 0 { - c.Error(errors.NewUnauthorizedError("workspace ID not found")) - return - } - callerTenantRole := types.TenantRoleFromContext(ctx) - requestedSourceTenantID, parseErr := types.ParseAgentSourceTenantID(c.Query(types.AgentSourceTenantIDParam)) - if parseErr != nil { - c.Error(errors.NewBadRequestError(parseErr.Error())) - return - } - agent, err := h.agentShareService.GetSharedAgentForTenant(ctx, currentTenantID, callerTenantRole, agentID, requestedSourceTenantID) + agent, err := resolveSharedAgentForRequest(c, agentID, h.agentShareService) if err != nil { - if goerrors.Is(err, service.ErrAgentShareNotFound) || goerrors.Is(err, service.ErrAgentSharePermission) || goerrors.Is(err, service.ErrAgentNotFoundForShare) { - c.Error(errors.NewForbiddenError("no permission for this shared agent")) - return - } - logger.ErrorWithFields(ctx, err, nil) - c.Error(errors.NewInternalServerError("Failed to verify shared agent access").WithDetails(err.Error())) + _ = c.Error(err) return } sourceTenantID := agent.TenantID - mode := agent.Config.KBSelectionMode - if mode == "none" { + ctx = access.WithSharedAgent(ctx, agent) + scope := types.NewSharedAgentKBScope(agent) + if scope.IsEmpty() { c.JSON(http.StatusOK, gin.H{ "success": true, "data": []interface{}{}, @@ -2301,41 +2200,24 @@ func (h *KnowledgeHandler) SearchKnowledge(c *gin.Context) { return } var scopes []types.KnowledgeSearchScope - if mode == "selected" && len(agent.Config.KnowledgeBases) > 0 { - for _, kbID := range agent.Config.KnowledgeBases { + if !scope.IsAll() { + for _, kbID := range scope.IDs() { if kbID != "" { scopes = append(scopes, types.KnowledgeSearchScope{TenantID: sourceTenantID, KBID: kbID}) } } } - if len(scopes) == 0 { + if scope.IsAll() { kbs, err := h.kbService.ListKnowledgeBasesByTenantID(ctx, sourceTenantID) if err != nil { logger.ErrorWithFields(ctx, err, nil) c.Error(errors.NewInternalServerError("Failed to list knowledge bases").WithDetails(err.Error())) return } - // `all` mode: authoritative server-side capability filter. Mirrors the - // logic in ListKnowledgeBases so @file search, KB listing, and runtime - // all agree on what "mode=all" actually means for this agent. The - // filter is agent-mode aware so quick-answer (RAG-only) skips - // wiki-only KBs even though it has no `allowed_tools`. - filter := tools.DeriveKBFilterForAgent(agent.Config.AgentMode, agent.Config.AllowedTools) - removed := 0 - for _, kb := range kbs { - if kb == nil || kb.Type != types.KnowledgeBaseTypeDocument { - continue + for _, kb := range filterKnowledgeBasesForSharedAgent(kbs, agent) { + if kb.Type == types.KnowledgeBaseTypeDocument { + scopes = append(scopes, types.KnowledgeSearchScope{TenantID: kb.TenantID, KBID: kb.ID}) } - if !filter.IsEmpty() && !tools.KBSatisfiesAgentRequirements(kb.Capabilities(), agent.Config.AgentMode, agent.Config.AllowedTools) { - removed++ - continue - } - scopes = append(scopes, types.KnowledgeSearchScope{TenantID: sourceTenantID, KBID: kb.ID}) - } - if removed > 0 { - logger.Infof(ctx, - "SearchKnowledge(agent=%s, mode=all): capability filter removed %d KBs", - agentID, removed) } } scopes = filterKnowledgeSearchScopesForAPIKey(ctx, scopes) @@ -2499,29 +2381,43 @@ func (h *KnowledgeHandler) MoveKnowledge(c *gin.Context) { return } - // Validate type match - if sourceKB.Type != targetKB.Type { - c.Error(errors.NewBadRequestError("Source and target knowledge bases must be the same type")) + // Resolve explicit write grants independently for both body IDs. + if _, err := resolveHandlerKBAccessFor(c, req.SourceKBID, h.kbService, nil, nil, types.OrgRoleEditor); err != nil { + _ = c.Error(err) return } - - // Validate embedding model match - if sourceKB.EmbeddingModelID != targetKB.EmbeddingModelID { - c.Error(errors.NewBadRequestError("Source and target must use the same embedding model")) + if _, err := resolveHandlerKBAccessFor(c, req.TargetKBID, h.kbService, nil, nil, types.OrgRoleEditor); err != nil { + _ = c.Error(err) return } - - // reuse_vectors copies index entries directly between KBs, which only works - // inside the same VectorStore backend. A cross-store reuse_vectors move would - // route CopyIndices through the SOURCE store and then delete the source - // indices, corrupting the vector data. Reject it and point the caller at - // reparse mode, which re-indexes into the target store safely. - if req.Mode == "reuse_vectors" && !sourceKB.SharesStoreWith(targetKB) { - c.Error(errors.NewBadRequestError( - "reuse_vectors move across different vector stores is not supported; " + - "use reparse mode to move into a different store")) + taskID := utils.GenerateTaskID("kg_move", tenantID.(uint64), req.SourceKBID) + ctx, err = access.WithKBTransfer(c.Request.Context(), sourceKB, targetKB, access.KBTransferMove, taskID, false) + if err != nil { + _ = c.Error(kbAccessHTTPError(err)) return } + tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) + if err := access.ValidateKBTransferCompatibility(sourceKB, + targetKB, + access.KBTransferMove, + req.Mode, + tenant); err != nil { + _ = c.Error(errors.NewBadRequestError(err.Error())) + return + } + uniqueIDs := make([]string, 0, len(req.KnowledgeIDs)) + seen := make(map[string]bool, len(req.KnowledgeIDs)) + for _, id := range req.KnowledgeIDs { + if strings.TrimSpace(id) == "" { + _ = c.Error(errors.NewBadRequestError("Knowledge ID cannot be empty")) + return + } + if !seen[id] { + seen[id] = true + uniqueIDs = append(uniqueIDs, id) + } + } + req.KnowledgeIDs = uniqueIDs // Validate all knowledge IDs belong to source KB and are in completed status for _, kID := range req.KnowledgeIDs { @@ -2530,8 +2426,13 @@ func (h *KnowledgeHandler) MoveKnowledge(c *gin.Context) { c.Error(errors.NewBadRequestError(fmt.Sprintf("Knowledge item %s not found", kID))) return } - if knowledge.KnowledgeBaseID != req.SourceKBID { - c.Error(errors.NewBadRequestError(fmt.Sprintf("Knowledge item %s does not belong to the source knowledge base", kID))) + if knowledge == nil || knowledge.ID != kID || knowledge.TenantID != tenantID.(uint64) || + knowledge.KnowledgeBaseID != req.SourceKBID { + _ = c.Error( + errors.NewBadRequestError( + fmt.Sprintf("Knowledge item %s does not belong to the source knowledge base", kID), + ), + ) return } if knowledge.ParseStatus != types.ParseStatusCompleted { @@ -2540,9 +2441,6 @@ func (h *KnowledgeHandler) MoveKnowledge(c *gin.Context) { } } - // Generate task ID - taskID := utils.GenerateTaskID("kg_move", tenantID.(uint64), req.SourceKBID) - // Create move payload payload := types.KnowledgeMovePayload{ TenantID: tenantID.(uint64), @@ -2643,26 +2541,6 @@ func (h *KnowledgeHandler) GetKnowledgeMoveProgress(c *gin.Context) { }) } -// resolveAgentAllowedKBIDs returns the set of knowledge base IDs that the -// shared agent is allowed to access based on its KBSelectionMode config. -// Returns nil when no restriction applies ("all" mode), or a concrete slice -// (possibly empty for "none" mode) when the results must be filtered. -func resolveAgentAllowedKBIDs(agent *types.CustomAgent) []string { - switch agent.Config.KBSelectionMode { - case "all": - return nil - case "none": - return []string{} - case "selected": - return agent.Config.KnowledgeBases - default: - if len(agent.Config.KnowledgeBases) > 0 { - return agent.Config.KnowledgeBases - } - return nil - } -} - // parseCommaSeparatedTagIDs splits a comma-separated string of tag IDs and // filters out empty strings and the "__untagged__" sentinel value. func parseCommaSeparatedTagIDs(raw string) []string { @@ -2701,15 +2579,6 @@ func parseFilterTime(raw string) (time.Time, error) { return time.Time{}, lastErr } -func sliceContains(ss []string, target string) bool { - for _, s := range ss { - if s == target { - return true - } - } - return false -} - type batchReparseKnowledgeRequest struct { KBID string `json:"kb_id" binding:"required"` IDs []string `json:"ids" binding:"required"` @@ -2763,7 +2632,7 @@ func (h *KnowledgeHandler) BatchReparseKnowledge(c *gin.Context) { return } - _, kbID, effectiveTenantID, permission, err := h.validateKnowledgeBaseAccessWithKBID(c, req.KBID) + _, kbID, effectiveTenantID, permission, err := h.validateKnowledgeBaseWriteAccessWithKBID(c, req.KBID) if err != nil { c.Error(err) return @@ -2772,7 +2641,11 @@ func (h *KnowledgeHandler) BatchReparseKnowledge(c *gin.Context) { c.Error(errors.NewForbiddenError("no permission to reparse knowledge in this kb")) return } - ctx = context.WithValue(ctx, types.TenantIDContextKey, effectiveTenantID) + if err := h.requireKBOwnershipOrAdmin(c, kbID); err != nil { + _ = c.Error(err) + return + } + ctx = types.WithExecutionTenant(c.Request.Context(), effectiveTenantID) knowledgeList, err := h.kgService.GetKnowledgeBatch(ctx, effectiveTenantID, ids) if err != nil { @@ -2785,6 +2658,10 @@ func (h *KnowledgeHandler) BatchReparseKnowledge(c *gin.Context) { return } for _, k := range knowledgeList { + if err := access.RejectMovingKnowledge(k); err != nil { + _ = c.Error(err) + return + } if k.KnowledgeBaseID != kbID { c.Error(errors.NewBadRequestError( fmt.Sprintf("Knowledge %s does not belong to knowledge base %s", @@ -2793,7 +2670,7 @@ func (h *KnowledgeHandler) BatchReparseKnowledge(c *gin.Context) { } } - taskID, err := h.enqueueKnowledgeListReparse(ctx, effectiveTenantID, ids, req.ProcessConfig) + taskID, err := h.enqueueKnowledgeListReparse(ctx, effectiveTenantID, kbID, ids, req.ProcessConfig) if err != nil { logger.Errorf(ctx, "Failed to enqueue batch knowledge reparse task: %v", err) c.Error(errors.NewInternalServerError("Failed to enqueue batch reparse task")) diff --git a/internal/handler/knowledge_mutation_admission_test.go b/internal/handler/knowledge_mutation_admission_test.go new file mode 100644 index 00000000..dcf70951 --- /dev/null +++ b/internal/handler/knowledge_mutation_admission_test.go @@ -0,0 +1,253 @@ +package handler + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/application/access" + apperrors "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +type mutationAdmissionKnowledge struct { + interfaces.KnowledgeService + rows []*types.Knowledge + err error + tagsCalled bool + writeGrant bool +} + +func (s *mutationAdmissionKnowledge) GetKnowledgeByIDOnly(context.Context, string) (*types.Knowledge, error) { + return s.rows[0], nil +} + +func (s *mutationAdmissionKnowledge) GetKnowledgeByID(context.Context, string) (*types.Knowledge, error) { + return s.rows[0], nil +} + +func (s *mutationAdmissionKnowledge) GetKnowledgeBatch( + ctx context.Context, + _ uint64, + _ []string, +) ([]*types.Knowledge, error) { + s.writeGrant = access.RequireKBWrite(ctx, &types.KnowledgeBase{ID: "kb", TenantID: 7}) == nil + return s.rows, nil +} + +func (s *mutationAdmissionKnowledge) ListKnowledgeByKnowledgeBaseID( + context.Context, + string, +) ([]*types.Knowledge, error) { + return s.rows, nil +} + +func (s *mutationAdmissionKnowledge) UpdateKnowledge(context.Context, *types.Knowledge) error { + return s.err +} + +func (s *mutationAdmissionKnowledge) UpdateImageInfo(context.Context, string, string, string) error { + return s.err +} + +func (s *mutationAdmissionKnowledge) RegenerateKnowledgeSummary(context.Context, string) (*types.Knowledge, error) { + return nil, s.err +} + +func (s *mutationAdmissionKnowledge) UpdateKnowledgeTagBatch(context.Context, string, map[string][]string) error { + s.tagsCalled = true + return s.err +} + +func mutationRequest(r *gin.Engine, method, path, body string) *httptest.ResponseRecorder { + req := httptest.NewRequest(method, path, strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + return w +} + +func admissionKnowledge(moving bool) *types.Knowledge { + k := &types.Knowledge{ID: "doc", TenantID: 7, KnowledgeBaseID: "kb"} + if moving { + k.Metadata = []byte(`{"_knowledge_transfer":{"operation":"move","phase":"moving"}}`) + } + return k +} + +func admissionKB(creator string) *stubKBService { + return &stubKBService{get: func(context.Context, string) (*types.KnowledgeBase, error) { + return &types.KnowledgeBase{ID: "kb", TenantID: 7, CreatorID: creator}, nil + }} +} + +func TestMovingDocumentAdmissionRejectsBeforeEnqueue(t *testing.T) { + for _, route := range []string{"single", "batch", "clear", "reparse"} { + t.Run(route, func(t *testing.T) { + kg := &mutationAdmissionKnowledge{rows: []*types.Knowledge{admissionKnowledge(true)}} + if route == "batch" || route == "reparse" { + kg.rows = append([]*types.Knowledge{{ID: "other", TenantID: 7, KnowledgeBaseID: "kb"}}, kg.rows...) + } + queue := &documentDeleteEnqueuer{} + h := &KnowledgeHandler{ + cfg: transferHandlerConfig(), + kbService: admissionKB("user"), + kgService: kg, + asynqClient: queue, + } + r := documentHandlerRouter() + method, path, body := http.MethodDelete, "/doc", "" + switch route { + case "single": + r.DELETE("/:id", h.DeleteKnowledge) + case "batch": + r.POST("/batch", h.BatchDeleteKnowledge) + method, path, body = http.MethodPost, "/batch", `{"kb_id":"kb","ids":["other","doc"]}` + case "clear": + r.DELETE("/:id", h.ClearKnowledgeBaseContents) + path = "/kb" + case "reparse": + r.POST("/reparse", h.BatchReparseKnowledge) + method, path, body = http.MethodPost, "/reparse", `{"kb_id":"kb","ids":["other","doc"]}` + } + w := mutationRequest(r, method, path, body) + require.Equal(t, 409, w.Code, w.Body.String()) + require.Nil(t, queue.task) + }) + } +} + +func TestBatchMutationRequiresCreatorOrAdminAndWriteGrant(t *testing.T) { + for _, route := range []string{"reparse", "tags explicit", "tags inferred"} { + for _, creator := range []string{"user", "colleague"} { + t.Run(route+creator, func(t *testing.T) { + kg := &mutationAdmissionKnowledge{rows: []*types.Knowledge{admissionKnowledge(false)}} + queue := &documentDeleteEnqueuer{} + h := &KnowledgeHandler{ + cfg: transferHandlerConfig(), + kbService: admissionKB(creator), + kgService: kg, + asynqClient: queue, + } + r := transferHandlerRouter(nil) + method, body := http.MethodPost, `{"kb_id":"kb","ids":["doc"]}` + if route == "reparse" { + r.POST("/batch", h.BatchReparseKnowledge) + } else { + r.PUT("/batch", h.UpdateKnowledgeTagBatch) + method = http.MethodPut + body = `{"kb_id":"kb","updates":{"doc":["tag"]}}` + if route == "tags inferred" { + body = `{"updates":{"doc":["tag"]}}` + } + } + w := mutationRequest(r, method, "/batch", body) + if creator == "user" { + require.Equal(t, 200, w.Code, w.Body.String()) + if route == "reparse" { + require.True(t, kg.writeGrant) + require.NotNil(t, queue.task) + } else { + require.True(t, kg.tagsCalled) + } + } else { + require.Equal(t, 403, w.Code, w.Body.String()) + require.Nil(t, queue.task) + require.False(t, kg.tagsCalled) + } + }) + } + } +} + +type mutationChunkService struct { + interfaces.ChunkService + err error +} + +func (s *mutationChunkService) GetChunkByID(context.Context, string) (*types.Chunk, error) { + return &types.Chunk{ID: "chunk", KnowledgeID: "doc"}, nil +} + +func (s *mutationChunkService) UpdateDocumentChunk( + context.Context, + string, + *string, + *bool, + *int, +) (*types.Chunk, error) { + return nil, s.err +} + +func (s *mutationChunkService) RevertDocumentChunk(context.Context, string, int, *int) (*types.Chunk, error) { + return nil, s.err +} + +func TestMutationHandlersPreserveApplicationStatus(t *testing.T) { + for _, status := range []int{409, 403} { + for _, route := range []string{"metadata", "image", "summary", "chunk", "revert"} { + t.Run(fmt.Sprint(status)+route, func(t *testing.T) { + app := apperrors.NewConflictError("moving") + if status == 403 { + app = apperrors.NewForbiddenError("binding changed") + } + failure := fmt.Errorf("operation: %w", app) + kg := &mutationAdmissionKnowledge{rows: []*types.Knowledge{admissionKnowledge(false)}, err: failure} + h := &KnowledgeHandler{kgService: kg} + chunks := &ChunkHandler{service: &mutationChunkService{err: failure}} + r := documentHandlerRouter() + method, path, body := http.MethodPut, "/doc", `{"title":"new"}` + switch route { + case "metadata": + r.PUT("/:id", h.UpdateKnowledge) + case "image": + r.PUT("/:id/:chunk_id", h.UpdateImageInfo) + path = "/doc/chunk" + body = `{"image_info":"[]"}` + case "summary": + r.POST("/:id", h.RegenerateKnowledgeSummary) + method = http.MethodPost + body = "" + case "chunk": + r.PUT("/:knowledge_id/:id", chunks.UpdateChunk) + path = "/doc/chunk" + body = `{"content":"new"}` + case "revert": + r.POST("/:knowledge_id/:id", chunks.RevertChunk) + method = http.MethodPost + path = "/doc/chunk" + body = `{"revision":0}` + } + w := mutationRequest(r, method, path, body) + require.Equal(t, status, w.Code, w.Body.String()) + }) + } + } +} + +func TestBatchReparsePreservesAdminAndRBACRolloutPolicy(t *testing.T) { + for _, enforced := range []bool{true, false} { + t.Run(fmt.Sprint(enforced), func(t *testing.T) { + cfg := transferHandlerConfig() + cfg.Tenant.EnableRBAC = &enforced + kg := &mutationAdmissionKnowledge{rows: []*types.Knowledge{admissionKnowledge(false)}} + queue := &documentDeleteEnqueuer{} + h := &KnowledgeHandler{cfg: cfg, kbService: admissionKB("colleague"), kgService: kg, asynqClient: queue} + r := transferHandlerRouter(nil) + if enforced { + r = documentHandlerRouter() + } // Admin may edit another creator's KB. + r.POST("/batch", h.BatchReparseKnowledge) + w := mutationRequest(r, http.MethodPost, "/batch", `{"kb_id":"kb","ids":["doc"]}`) + require.Equal(t, 200, w.Code, w.Body.String()) + require.True(t, kg.writeGrant) + require.NotNil(t, queue.task) + }) + } +} diff --git a/internal/handler/knowledge_ownership_test.go b/internal/handler/knowledge_ownership_test.go new file mode 100644 index 00000000..6a368b32 --- /dev/null +++ b/internal/handler/knowledge_ownership_test.go @@ -0,0 +1,157 @@ +package handler + +import ( + "context" + stderrors "errors" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/Tencent/WeKnora/internal/application/repository" + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/middleware" + "github.com/Tencent/WeKnora/internal/types" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +func TestBodyKBOwnershipSkipsUnnecessaryLookup(t *testing.T) { + for _, tt := range []struct { + name string + role types.TenantRole + apiKey, superuser, disabled bool + }{ + {name: "admin", role: types.TenantRoleAdmin}, + {name: "owner", role: types.TenantRoleOwner}, + {name: "API key", role: types.TenantRoleViewer, apiKey: true}, + {name: "cross tenant superuser", role: types.TenantRoleViewer, superuser: true}, + {name: "RBAC disabled", role: types.TenantRoleContributor, disabled: true}, + } { + t.Run(tt.name, func(t *testing.T) { + c := newKBLookupCtx(t, 1, "kb") + ctx := context.WithValue(c.Request.Context(), types.TenantRoleContextKey, tt.role) + ctx = context.WithValue( + ctx, + types.UserContextKey, + &types.User{ID: "user", CanAccessAllTenants: tt.superuser}, + ) + if tt.apiKey { + ctx = types.WithTenantAPIKeyScope( + ctx, + types.TenantAPIKeyScope{KnowledgeBaseIDs: types.StringArray{"kb"}}, + ) + } + c.Request = c.Request.WithContext(ctx) + enabled := !tt.disabled + h := &KnowledgeHandler{ + cfg: &config.Config{ + Tenant: &config.TenantConfig{EnableRBAC: &enabled, EnableCrossTenantAccess: tt.superuser}, + }, + kbService: &stubKBService{get: func(context.Context, string) (*types.KnowledgeBase, error) { + t.Fatal("body-carried KB ownership must be looked up lazily") + return nil, stderrors.New("unavailable") + }}, + } + require.NoError(t, h.requireKBOwnershipOrAdmin(c, "kb")) + }) + } +} + +func TestBodyKBOwnershipPreservesStatusAndTenantBoundary(t *testing.T) { + for _, tt := range []struct { + name string + kb *types.KnowledgeBase + lookupErr error + status int + }{ + { + name: "creator", + kb: &types.KnowledgeBase{ + ID: "kb", + TenantID: 1, + CreatorID: "user", + }, + status: http.StatusNoContent, + }, + + { + name: "noncreator", + kb: &types.KnowledgeBase{ + ID: "kb", + TenantID: 1, + CreatorID: "other", + }, + status: http.StatusForbidden, + }, + + {name: "tenant owned", kb: &types.KnowledgeBase{ID: "kb", TenantID: 1}, status: http.StatusForbidden}, + { + name: "same creator in another tenant", + kb: &types.KnowledgeBase{ + ID: "kb", + TenantID: 2, + CreatorID: "user", + }, + status: http.StatusNotFound, + }, + + {name: "nil resource", status: http.StatusNotFound}, + { + name: "wrapped not found", + lookupErr: fmt.Errorf("lookup: %w", + repository.ErrKnowledgeBaseNotFound), + status: http.StatusNotFound, + }, + + { + name: "database failure", + lookupErr: stderrors.New("database unavailable"), + status: http.StatusInternalServerError, + }, + } { + t.Run(tt.name, func(t *testing.T) { + enabled := true + calls := 0 + h := &KnowledgeHandler{ + cfg: &config.Config{Tenant: &config.TenantConfig{EnableRBAC: &enabled}}, + kbService: &stubKBService{get: func(context.Context, string) (*types.KnowledgeBase, error) { + calls++ + return tt.kb, tt.lookupErr + }}, + } + r := gin.New() + r.Use(middleware.ErrorHandler(), func(c *gin.Context) { + ctx := context.WithValue(c.Request.Context(), types.TenantIDContextKey, uint64(1)) + ctx = context.WithValue(ctx, types.TenantRoleContextKey, types.TenantRoleContributor) + ctx = context.WithValue(ctx, types.UserIDContextKey, "user") + c.Request = c.Request.WithContext(ctx) + c.Next() + }) + r.POST("/write", func(c *gin.Context) { + if err := h.requireKBOwnershipOrAdmin(c, "kb"); err != nil { + _ = c.Error(err) + return + } + c.Status(http.StatusNoContent) + }) + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/write", nil)) + require.Equal(t, tt.status, w.Code, w.Body.String()) + require.Equal(t, 1, calls) + }) + } +} + +func TestBodyWriteStillEnforcesAPIKeyKBScope(t *testing.T) { + c := newKBLookupCtx(t, 1, "kb") + c.Set(types.TenantIDContextKey.String(), uint64(1)) + ctx := types.WithTenantAPIKeyScope( + c.Request.Context(), + types.TenantAPIKeyScope{KnowledgeBaseIDs: types.StringArray{"allowed"}}, + ) + c.Request = c.Request.WithContext(ctx) + h := &KnowledgeHandler{} + _, _, err := h.requireKnowledgeWriteAccess(c, "outside-scope") + require.Error(t, err, "skipping human ownership checks must not skip KB scope") +} diff --git a/internal/handler/knowledge_transfer_test.go b/internal/handler/knowledge_transfer_test.go new file mode 100644 index 00000000..8b56a92a --- /dev/null +++ b/internal/handler/knowledge_transfer_test.go @@ -0,0 +1,176 @@ +package handler + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/middleware" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +type transferHandlerKnowledge struct{ interfaces.KnowledgeService } + +func (*transferHandlerKnowledge) SaveKBCloneProgress(context.Context, *types.KBCloneProgress) error { + return nil +} + +func (*transferHandlerKnowledge) SaveKnowledgeMoveProgress(context.Context, *types.KnowledgeMoveProgress) error { + return nil +} + +func (*transferHandlerKnowledge) GetKnowledgeByID(_ context.Context, id string) (*types.Knowledge, error) { + return &types.Knowledge{ + ID: id, + TenantID: 7, + KnowledgeBaseID: "source", + ParseStatus: types.ParseStatusCompleted, + }, nil +} + +func transferHandlerRouter(scope *types.TenantAPIKeyScope) *gin.Engine { + r := gin.New() + r.Use(middleware.ErrorHandler(), func(c *gin.Context) { + ctx := types.WithCaller( + c.Request.Context(), + types.Caller{TenantID: 7, UserID: "user", Role: types.TenantRoleContributor}, + ) + ctx = types.WithExecutionTenant(ctx, 7) + if scope != nil { + ctx = types.WithTenantAPIKeyScope(ctx, *scope) + } + c.Set(types.TenantIDContextKey.String(), uint64(7)) + c.Request = c.Request.WithContext(ctx) + c.Next() + }) + return r +} + +func transferHandlerConfig() *config.Config { + enabled := true + return &config.Config{Tenant: &config.TenantConfig{EnableRBAC: &enabled}} +} + +func transferHandlerKB() *stubKBService { + return &stubKBService{get: func(_ context.Context, id string) (*types.KnowledgeBase, error) { + return &types.KnowledgeBase{ID: id, TenantID: 7, CreatorID: "user"}, nil + }} +} + +func transferRequest(r *gin.Engine, body string) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodPost, "/transfer", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + return w +} + +func TestCopyAdmissionReservesDestinationAndPreservesCreator(t *testing.T) { + q := &documentDeleteEnqueuer{} + h := &KnowledgeBaseHandler{ + cfg: transferHandlerConfig(), + service: transferHandlerKB(), + knowledgeService: &transferHandlerKnowledge{}, + asynqClient: q, + } + r := transferHandlerRouter(nil) + r.POST("/transfer", h.CopyKnowledgeBase) + w := transferRequest(r, `{"source_id":"source"}`) + require.Equal(t, 200, w.Code, w.Body.String()) + require.NotNil(t, q.task) + var payload types.KBClonePayload + require.NoError(t, json.Unmarshal(q.task.Payload(), &payload)) + require.NotEmpty(t, payload.TargetID) + require.NotEqual(t, "source", payload.TargetID) + require.True(t, payload.CreateTarget) + require.Equal(t, "user", payload.CreatorID) + require.Equal(t, uint64(7), payload.TenantID) + require.Contains(t, w.Body.String(), payload.TargetID) +} + +func TestCopyAdmissionRequiresExistingDestinationOwnership(t *testing.T) { + q := &documentDeleteEnqueuer{} + kb := &stubKBService{get: func(_ context.Context, id string) (*types.KnowledgeBase, error) { + return &types.KnowledgeBase{ID: id, TenantID: 7, CreatorID: "someone-else"}, nil + }} + h := &KnowledgeBaseHandler{ + cfg: transferHandlerConfig(), + service: kb, + knowledgeService: &transferHandlerKnowledge{}, + asynqClient: q, + } + r := transferHandlerRouter(nil) + r.POST("/transfer", h.CopyKnowledgeBase) + w := transferRequest(r, `{"source_id":"source","target_id":"target"}`) + require.Equal(t, 403, w.Code, w.Body.String()) + require.Nil(t, q.task) +} + +func TestCopyAdmissionKeepsManageCapabilityForNewDestination(t *testing.T) { + for _, capability := range []string{"manage_kbs", "ingest"} { + t.Run(capability, func(t *testing.T) { + q := &documentDeleteEnqueuer{} + h := &KnowledgeBaseHandler{ + cfg: transferHandlerConfig(), + service: transferHandlerKB(), + knowledgeService: &transferHandlerKnowledge{}, + asynqClient: q, + } + r := transferHandlerRouter( + &types.TenantAPIKeyScope{ + Capabilities: types.StringArray{capability}, + KnowledgeBaseIDs: types.StringArray{"source"}, + }, + ) + r.POST("/transfer", h.CopyKnowledgeBase) + w := transferRequest(r, `{"source_id":"source"}`) + if capability == "manage_kbs" { + require.Equal(t, 200, w.Code, w.Body.String()) + require.NotNil(t, q.task) + } else { + require.Equal(t, 403, w.Code, w.Body.String()) + require.Nil(t, q.task) + } + }) + } +} + +func TestMoveAdmissionDeduplicatesIDsAndRequiresBothKBs(t *testing.T) { + for _, both := range []bool{true, false} { + ids := types.StringArray{"source"} + if both { + ids = append(ids, "target") + } + q := &documentDeleteEnqueuer{} + h := &KnowledgeHandler{ + cfg: transferHandlerConfig(), + kbService: transferHandlerKB(), + kgService: &transferHandlerKnowledge{}, + asynqClient: q, + } + r := transferHandlerRouter( + &types.TenantAPIKeyScope{Capabilities: types.StringArray{"ingest"}, KnowledgeBaseIDs: ids}, + ) + r.POST("/transfer", h.MoveKnowledge) + w := transferRequest( + r, + `{"source_kb_id":"source","target_kb_id":"target","knowledge_ids":["doc","doc"],"mode":"reuse_vectors"}`, + ) + if !both { + require.Equal(t, 403, w.Code, w.Body.String()) + require.Nil(t, q.task) + continue + } + require.Equal(t, 200, w.Code, w.Body.String()) + var payload types.KnowledgeMovePayload + require.NoError(t, json.Unmarshal(q.task.Payload(), &payload)) + require.Equal(t, []string{"doc"}, payload.KnowledgeIDs) + } +} diff --git a/internal/handler/knowledgebase.go b/internal/handler/knowledgebase.go index f531b8f3..30c12107 100644 --- a/internal/handler/knowledgebase.go +++ b/internal/handler/knowledgebase.go @@ -9,12 +9,13 @@ import ( "strings" "time" - "github.com/Tencent/WeKnora/internal/agent/tools" + "github.com/Tencent/WeKnora/internal/application/access" "github.com/Tencent/WeKnora/internal/application/repository" - "github.com/Tencent/WeKnora/internal/application/service" + "github.com/Tencent/WeKnora/internal/config" "github.com/Tencent/WeKnora/internal/errors" apperrors "github.com/Tencent/WeKnora/internal/errors" "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/middleware" "github.com/Tencent/WeKnora/internal/storageurl" "github.com/Tencent/WeKnora/internal/tracing/langfuse" "github.com/Tencent/WeKnora/internal/types" @@ -22,11 +23,13 @@ import ( "github.com/Tencent/WeKnora/internal/utils" secutils "github.com/Tencent/WeKnora/internal/utils" "github.com/gin-gonic/gin" + "github.com/google/uuid" "github.com/hibiken/asynq" ) // KnowledgeBaseHandler defines the HTTP handler for knowledge base operations type KnowledgeBaseHandler struct { + cfg *config.Config service interfaces.KnowledgeBaseService knowledgeService interfaces.KnowledgeService kbShareService interfaces.KBShareService @@ -45,6 +48,7 @@ type KnowledgeBaseHandler struct { // NewKnowledgeBaseHandler creates a new knowledge base handler instance func NewKnowledgeBaseHandler( + cfg *config.Config, service interfaces.KnowledgeBaseService, knowledgeService interfaces.KnowledgeService, kbShareService interfaces.KBShareService, @@ -56,6 +60,7 @@ func NewKnowledgeBaseHandler( storageResolver interfaces.StorageBackendResolver, ) *KnowledgeBaseHandler { return &KnowledgeBaseHandler{ + cfg: cfg, service: service, knowledgeService: knowledgeService, kbShareService: kbShareService, @@ -348,7 +353,7 @@ func (h *KnowledgeBaseHandler) HybridSearch(c *gin.Context) { // Execute hybrid search with default search parameters // Note: For shared KBs, the service uses effectiveTenantID internally via context - results, err := h.service.HybridSearch(ctx, id, req) + results, err := h.service.HybridSearch(c.Request.Context(), id, req) if err != nil { // Service-layer typed AppErrors (e.g. ErrVectorStoreBindingInvalid, // ErrVectorStoreUnavailable, BadRequest from multi-store fan-out) @@ -445,115 +450,15 @@ func (h *KnowledgeBaseHandler) CreateKnowledgeBase(c *gin.Context) { // Returns the knowledge base, knowledge base ID, effective tenant ID for embedding, permission level, and any errors encountered // For owned KBs, effectiveTenantID is the caller's tenant ID // For shared KBs, effectiveTenantID is the source tenant ID (owner's tenant) -func (h *KnowledgeBaseHandler) validateAndGetKnowledgeBase(c *gin.Context) (*types.KnowledgeBase, string, uint64, types.OrgMemberRole, error) { - ctx := c.Request.Context() - - // Get tenant ID from context - tenantID, exists := c.Get(types.TenantIDContextKey.String()) - if !exists { - logger.Error(ctx, "Failed to get tenant ID") - return nil, "", 0, "", apperrors.NewUnauthorizedError("Unauthorized") - } - - // Get user ID from context (needed for shared KB permission check) - userID, userExists := c.Get(types.UserIDContextKey.String()) - callerTenantRole := types.TenantRoleFromContext(ctx) - - // Get knowledge base ID from URL parameter +func (h *KnowledgeBaseHandler) validateAndGetKnowledgeBase( + c *gin.Context, +) (*types.KnowledgeBase, string, uint64, types.OrgMemberRole, error) { id := secutils.SanitizeForLog(c.Param("id")) - if id == "" { - logger.Error(ctx, "Knowledge base ID is empty") - return nil, "", 0, "", apperrors.NewBadRequestError("Knowledge base ID cannot be empty") - } - if err := requireTenantAPIKeyKnowledgeBase(ctx, id); err != nil { + grant, err := resolveHandlerKBAccess(c, id, h.service, h.kbShareService, h.agentShareService) + if err != nil { return nil, id, 0, "", err } - - // Verify tenant has permission to access this knowledge base - kb, err := h.service.GetKnowledgeBaseByID(ctx, id) - if err != nil { - // repo.GetKnowledgeBaseByID surfaces ErrKnowledgeBaseNotFound for - // missing or cross-tenant rows. Map it to 404 here so the four - // callers (Get / Update / Delete / TogglePin / Copy / Hybrid-search - // path) don't have to wrap NewInternalServerError into a 500 for - // every probe of a non-existent id. - if stderrors.Is(err, repository.ErrKnowledgeBaseNotFound) { - return nil, id, 0, "", apperrors.NewNotFoundError("knowledge base not found") - } - logger.ErrorWithFields(ctx, err, nil) - return nil, id, 0, "", apperrors.NewInternalServerError(err.Error()) - } - - // Check 1: Verify tenant ownership (owner has full access) - if kb.TenantID == tenantID.(uint64) { - return kb, id, tenantID.(uint64), types.OrgRoleAdmin, nil - } - - // Check 2: If not owner, check organization shared access - if h.kbShareService != nil { - // Check if caller's tenant has shared access through organization - permission, isShared, permErr := h.kbShareService.CheckTenantKBPermission(ctx, id, tenantID.(uint64), callerTenantRole) - if permErr == nil && isShared { - // Tenant has shared access, get the source tenant ID for embedding queries - sourceTenantID, srcErr := h.kbShareService.GetKBSourceTenant(ctx, id) - if srcErr == nil { - logger.Infof(ctx, "Tenant %d accessing shared KB %s with permission %s, source tenant: %d", - tenantID.(uint64), id, permission, sourceTenantID) - return kb, id, sourceTenantID, permission, nil - } - } - } - - // Check 3: Shared agent — allow if request has agent_id (and agent can access this KB) OR caller's tenant has any shared agent that can access this KB (e.g. opened from "通过智能体可见" list without agent_id) - if h.agentShareService != nil { - currentTenantID := tenantID.(uint64) - agentID := c.Query("agent_id") - if agentID != "" { - sourceTenantID, parseErr := types.ParseAgentSourceTenantID(c.Query(types.AgentSourceTenantIDParam)) - if parseErr != nil { - return kb, id, 0, types.OrgMemberRole(""), apperrors.NewBadRequestError(parseErr.Error()) - } - agent, err := h.agentShareService.GetSharedAgentForTenant(ctx, currentTenantID, callerTenantRole, agentID, sourceTenantID) - if err == nil && agent != nil { - if kb.TenantID != agent.TenantID { - logger.Warnf(ctx, "Shared agent workspace mismatch, KB %s tenant: %d, agent tenant: %d", id, kb.TenantID, agent.TenantID) - } else { - mode := agent.Config.KBSelectionMode - if mode == "none" { - // no-op, fall through - } else if mode == "all" { - logger.Infof(ctx, "Tenant %d accessing KB %s via shared agent %s (mode=all)", currentTenantID, id, agentID) - return kb, id, kb.TenantID, types.OrgRoleViewer, nil - } else if mode == "selected" { - for _, allowedID := range agent.Config.KnowledgeBases { - if allowedID == id { - logger.Infof(ctx, "Tenant %d accessing KB %s via shared agent %s (mode=selected)", currentTenantID, id, agentID) - return kb, id, kb.TenantID, types.OrgRoleViewer, nil - } - } - } - } - } - } else { - // No agent_id in query: allow if caller's tenant has any shared agent that can access this KB (e.g. from space list "通过智能体可见") - can, err := h.agentShareService.TenantCanAccessKBViaSomeSharedAgent(ctx, currentTenantID, callerTenantRole, kb) - if err == nil && can { - logger.Infof(ctx, "Tenant %d accessing KB %s via some shared agent (no agent_id in query)", currentTenantID, id) - return kb, id, kb.TenantID, types.OrgRoleViewer, nil - } - } - } - _ = userID - _ = userExists - - // No permission: not owner and no shared access - logger.Warnf( - ctx, - "Tenant has no permission to access this knowledge base, knowledge base ID: %s, "+ - "request tenant ID: %d, knowledge base tenant ID: %d", - id, tenantID.(uint64), kb.TenantID, - ) - return nil, id, 0, "", apperrors.NewForbiddenError("No permission to operate") + return grant.KnowledgeBase, id, grant.EffectiveTenantID, grant.Permission, nil } // GetKnowledgeBase godoc @@ -608,86 +513,26 @@ func (h *KnowledgeBaseHandler) ListKnowledgeBases(c *gin.Context) { agentID := c.Query("agent_id") if agentID != "" { - userIDVal, ok := c.Get(types.UserIDContextKey.String()) - if !ok { - c.Error(apperrors.NewUnauthorizedError("user ID not found")) - return - } - _ = userIDVal - currentTenantID := c.GetUint64(types.TenantIDContextKey.String()) - if currentTenantID == 0 { - c.Error(apperrors.NewUnauthorizedError("workspace ID not found")) - return - } - callerTenantRole := types.TenantRoleFromContext(ctx) - requestedSourceTenantID, parseErr := types.ParseAgentSourceTenantID(c.Query(types.AgentSourceTenantIDParam)) - if parseErr != nil { - c.Error(apperrors.NewBadRequestError(parseErr.Error())) - return - } - agent, err := h.agentShareService.GetSharedAgentForTenant(ctx, currentTenantID, callerTenantRole, agentID, requestedSourceTenantID) + agent, err := resolveSharedAgentForRequest(c, agentID, h.agentShareService) if err != nil { - if stderrors.Is(err, service.ErrAgentShareNotFound) || stderrors.Is(err, service.ErrAgentSharePermission) || stderrors.Is(err, service.ErrAgentNotFoundForShare) { - c.Error(apperrors.NewForbiddenError("no permission for this shared agent")) - return - } - logger.ErrorWithFields(ctx, err, nil) - c.Error(apperrors.NewInternalServerError(err.Error())) + _ = c.Error(err) return } - mode := agent.Config.KBSelectionMode - if mode == "none" { + currentTenantID := middleware.KBAccessRequest(c).Caller.TenantID + scope := types.NewSharedAgentKBScope(agent) + if scope.IsEmpty() { c.JSON(http.StatusOK, gin.H{"success": true, "data": []interface{}{}}) return } - sourceTenantID := agent.TenantID - kbs, err := h.service.ListKnowledgeBasesByTenantID(ctx, sourceTenantID) + kbs, err := h.service.ListKnowledgeBasesByTenantID(ctx, agent.TenantID) if err != nil { logger.ErrorWithFields(ctx, err, nil) c.Error(apperrors.NewInternalServerError(err.Error())) return } - if mode == "selected" && len(agent.Config.KnowledgeBases) > 0 { - allowed := make(map[string]bool) - for _, id := range agent.Config.KnowledgeBases { - allowed[id] = true - } - filtered := make([]*types.KnowledgeBase, 0, len(kbs)) - for _, kb := range kbs { - if allowed[kb.ID] { - filtered = append(filtered, kb) - } - } - kbs = filtered - } + kbs = filterKnowledgeBasesForSharedAgent(kbs, agent) kbs = filterKnowledgeBasesForAPIKeyScope(ctx, kbs) - // `all` mode: authoritative server-side capability filter so a client - // that bypassed the frontend (old tab, curl, rogue plugin) can't @ a - // KB whose capabilities don't match this agent. The filter combines - // tool-derived requirements (smart-reasoning) with the implicit - // RAG-only requirement of quick-answer mode (which has no - // `allowed_tools` but still needs vector/keyword chunks to work). - // Non-`all` modes already constrain the scope explicitly. - if mode == "all" { - filter := tools.DeriveKBFilterForAgent(agent.Config.AgentMode, agent.Config.AllowedTools) - if !filter.IsEmpty() { - before := len(kbs) - kept := make([]*types.KnowledgeBase, 0, before) - for _, kb := range kbs { - if tools.KBSatisfiesAgentRequirements(kb.Capabilities(), agent.Config.AgentMode, agent.Config.AllowedTools) { - kept = append(kept, kb) - } - } - if removed := before - len(kept); removed > 0 { - logger.Infof(ctx, - "ListKnowledgeBases(agent=%s, mode=all): capability filter removed %d of %d KBs", - agentID, removed, before) - } - kbs = kept - } - } - c.JSON(http.StatusOK, gin.H{ "success": true, "data": h.buildKBListResponse(ctx, kbs, currentTenantID), @@ -1029,112 +874,86 @@ func (h *KnowledgeBaseHandler) CopyKnowledgeBase(c *gin.Context) { return } - // Get tenant ID from context - tenantID, exists := c.Get(types.TenantIDContextKey.String()) - if !exists { - logger.Error(ctx, "Failed to get tenant ID") - c.Error(apperrors.NewUnauthorizedError("Unauthorized")) + caller := types.CallerFromContext(ctx) + if caller.TenantID == 0 { + _ = c.Error(errors.NewUnauthorizedError("Unauthorized")) return } - kbIDs := []string{req.SourceID} - if req.TargetID != "" { - kbIDs = append(kbIDs, req.TargetID) - } - if err := requireTenantAPIKeyKnowledgeBases(ctx, kbIDs...); err != nil { + tenantID := caller.TenantID + sourceGrant, err := resolveHandlerKBAccessFor(c, req.SourceID, h.service, nil, nil, types.OrgRoleViewer) + if err != nil { c.Error(err) return } - - // Validate source knowledge base exists and belongs to caller's tenant (prevent cross-tenant clone) - sourceKB, err := h.service.GetKnowledgeBaseByID(ctx, req.SourceID) - if err != nil { - if stderrors.Is(err, repository.ErrKnowledgeBaseNotFound) { - c.Error(errors.NewNotFoundError("Source knowledge base not found")) - return + sourceKB := sourceGrant.KnowledgeBase + if sourceKB.TenantID != caller.TenantID { + _ = c.Error(errors.NewForbiddenError("No permission to copy this knowledge base")) + return + } + taskID := req.TaskID + if taskID == "" { + taskID = utils.GenerateTaskID("kb_clone", caller.TenantID, req.SourceID) + } else if err := requireTaskProgressTenant(ctx, taskID); err != nil { + _ = c.Error(err) + return + } + create := req.TargetID == "" + targetKB := &types.KnowledgeBase{ID: uuid.NewString(), TenantID: caller.TenantID} + if create { + if !types.IsSyntheticUserID(caller.UserID) { + targetKB.CreatorID = caller.UserID } - logger.ErrorWithFields(ctx, err, nil) - c.Error(errors.NewInternalServerError(err.Error())) - return - } - if sourceKB.TenantID != tenantID.(uint64) { - logger.Warnf(ctx, - "Copy rejected: source knowledge base belongs to another tenant, source_id: %s, caller_tenant: %d, kb_tenant: %d", - secutils.SanitizeForLog(req.SourceID), tenantID.(uint64), sourceKB.TenantID) - c.Error(errors.NewForbiddenError("No permission to copy this knowledge base")) - return - } - - // If target_id provided, validate target belongs to caller's tenant - // and run the pre-flight defenses synchronously so a mismatched - // clone is rejected with 400 before the task is enqueued. The same - // checks are re-applied inside the async worker (service.CopyKnowledgeBase) - // as defense in depth. - if req.TargetID != "" { - targetKB, err := h.service.GetKnowledgeBaseByID(ctx, req.TargetID) + } else { + targetGrant, err := resolveHandlerKBAccessFor(c, req.TargetID, h.service, nil, nil, types.OrgRoleEditor) if err != nil { - if stderrors.Is(err, repository.ErrKnowledgeBaseNotFound) { - c.Error(errors.NewNotFoundError("Target knowledge base not found")) - return - } - logger.ErrorWithFields(ctx, err, nil) - c.Error(errors.NewInternalServerError(err.Error())) + _ = c.Error(err) return } - if targetKB.TenantID != tenantID.(uint64) { - logger.Warnf(ctx, "Copy rejected: target knowledge base belongs to another tenant, target_id: %s", - secutils.SanitizeForLog(req.TargetID)) + targetKB = targetGrant.KnowledgeBase + if targetKB.TenantID != caller.TenantID { c.Error(errors.NewForbiddenError("No permission to copy to this knowledge base")) return } - // Pre-flight defense 1: embedding model must match. - // Without this check the async clone would run with incompatible - // vector spaces and produce semantically broken results. - if sourceKB.EmbeddingModelID != targetKB.EmbeddingModelID { - c.Error(apperrors.NewBadRequestError( - "source and target knowledge bases use different embedding models; " + - "clone into a target with the same embedding model")) + if err := middleware.EvaluateOwnershipOrRole(c.Request.Context(), + h.cfg, + types.TenantRoleAdmin, + func() (string, + error, + ) { + return targetKB.CreatorID, + nil + }); err != nil { + _ = c.Error(errors.NewForbiddenError("No permission to replace this knowledge base's contents")) return } - // Pre-flight defense 2: vector store binding must match. - // Cross-store cloning would require copying physical vector data - // between stores, which is not yet supported. - if !sourceKB.SharesStoreWith(targetKB) { - c.Error(apperrors.NewBadRequestError( - "source and target knowledge bases are bound to different vector stores; " + - "cross-store cloning is not yet supported")) + tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) + if err := access.ValidateKBTransferCompatibility(sourceKB, + targetKB, + access.KBTransferClone, + "", + tenant); err != nil { + _ = c.Error(errors.NewBadRequestError(err.Error())) return } - // Pre-flight defense 3: compare concrete instance IDs, not just the - // provider type (COS-A and COS-B are different physical stores). - if tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant); tenant != nil { - defaultID, defaultProvider := "", "" - if tenant.DefaultStorageBackendID != nil { - defaultID = *tenant.DefaultStorageBackendID - } - if tenant.StorageEngineConfig != nil { - defaultProvider = tenant.StorageEngineConfig.DefaultProvider - } - if !sourceKB.SharesStorageBackendWith(targetKB, defaultID, defaultProvider) { - c.Error(apperrors.NewBadRequestError( - "source and target knowledge bases use different storage instances; cross-storage-backend cloning is not supported")) - return - } - } } - - // Generate task ID if not provided - taskID := req.TaskID - if taskID == "" { - taskID = utils.GenerateTaskID("kb_clone", tenantID.(uint64), req.SourceID) + ctx, err = access.WithKBTransfer(c.Request.Context(), sourceKB, targetKB, access.KBTransferClone, taskID, create) + if err != nil { + _ = c.Error(kbAccessHTTPError(err)) + return } + // Reserve the destination in the payload; enqueue failures do not leave + // empty KBs and every worker delivery creates/resumes the same destination. + req.TargetID = targetKB.ID // Create KB clone payload payload := types.KBClonePayload{ - TenantID: tenantID.(uint64), - TaskID: taskID, - SourceID: req.SourceID, - TargetID: req.TargetID, - Initiator: types.TaskInitiatorFromContext(ctx), + TenantID: tenantID, + TaskID: taskID, + SourceID: req.SourceID, + TargetID: req.TargetID, + CreateTarget: create, + CreatorID: targetKB.CreatorID, + Initiator: types.TaskInitiatorFromContext(ctx), } langfuse.InjectTracing(ctx, &payload) diff --git a/internal/handler/rbac_lookups.go b/internal/handler/rbac_lookups.go index fbc181a9..cd048643 100644 --- a/internal/handler/rbac_lookups.go +++ b/internal/handler/rbac_lookups.go @@ -36,30 +36,7 @@ func (h *KnowledgeBaseHandler) KBCreatorLookup(c *gin.Context) (string, error) { if id == "" { return "", errors.New("missing :id param for KB creator lookup") } - ctx := c.Request.Context() - tenantID, ok := types.TenantIDFromContext(ctx) - if !ok { - // 没有空间上下文意味着 auth 中间件未完成;当作 lookup 失败让上层 503, - // 而不是 silently 走 fail-open 给一个不该有的访问。 - return "", errors.New("workspace context missing") - } - kb, err := h.service.GetKnowledgeBaseByID(ctx, id) - if err != nil { - if errors.Is(err, apprepo.ErrKnowledgeBaseNotFound) { - return "", middleware.ErrResourceNotFound - } - return "", err - } - if kb == nil { - return "", middleware.ErrResourceNotFound - } - // 显式重校验空间:repo.GetKnowledgeBaseByID 不带 tenant 过滤, - // 万一未来 :id 被攻击者从他人空间 UUID 试探到,也不会借由 - // "ownership match" 通过中间件。 - if kb.TenantID != tenantID { - return "", middleware.ErrResourceNotFound - } - return kb.CreatorID, nil + return resolveKBCreatorByKBID(c, h.service, id) } // KBCreatorLookupFromKbIDParam is the same lookup as KBCreatorLookup @@ -74,25 +51,7 @@ func (h *KnowledgeBaseHandler) KBCreatorLookupFromKbIDParam(c *gin.Context) (str if id == "" { return "", errors.New("missing :kbId param for KB creator lookup") } - ctx := c.Request.Context() - tenantID, ok := types.TenantIDFromContext(ctx) - if !ok { - return "", errors.New("workspace context missing") - } - kb, err := h.service.GetKnowledgeBaseByID(ctx, id) - if err != nil { - if errors.Is(err, apprepo.ErrKnowledgeBaseNotFound) { - return "", middleware.ErrResourceNotFound - } - return "", err - } - if kb == nil { - return "", middleware.ErrResourceNotFound - } - if kb.TenantID != tenantID { - return "", middleware.ErrResourceNotFound - } - return kb.CreatorID, nil + return resolveKBCreatorByKBID(c, h.service, id) } // AgentCreatorLookup resolves :id -> CustomAgent.CreatedBy. Built-in @@ -226,30 +185,12 @@ func (h *WikiPageHandler) KBCreatorLookupFromKBPath(c *gin.Context) (string, err if kbID == "" { return "", errors.New("missing :kb_id param for wiki owner lookup") } - ctx := c.Request.Context() - tenantID, ok := types.TenantIDFromContext(ctx) - if !ok { - return "", errors.New("workspace context missing") - } - kb, err := h.kbService.GetKnowledgeBaseByID(ctx, kbID) - if err != nil { - if errors.Is(err, apprepo.ErrKnowledgeBaseNotFound) { - return "", middleware.ErrResourceNotFound - } - return "", err - } - if kb == nil { - return "", middleware.ErrResourceNotFound - } - if kb.TenantID != tenantID { - return "", middleware.ErrResourceNotFound - } - return kb.CreatorID, nil + return resolveKBCreatorByKBID(c, h.kbService, kbID) } // resolveKBCreatorByKBID resolves a KB id to CreatorID, scoped to the -// caller's tenant. Used by cross-KB handlers whose body carries kb_id -// instead of a URL param (batch-delete, move, etc.). +// caller's tenant. Shared by KB, initialization and Wiki route lookups and +// cross-KB handlers whose body carries kb_id (batch-delete, move, etc.). func resolveKBCreatorByKBID( c *gin.Context, kbService interfaces.KnowledgeBaseService, diff --git a/internal/handler/session/artifact_download.go b/internal/handler/session/artifact_download.go index 6186bed8..945060b0 100644 --- a/internal/handler/session/artifact_download.go +++ b/internal/handler/session/artifact_download.go @@ -2,15 +2,18 @@ package session import ( stderrors "errors" - "io" "mime" "net/http" "path/filepath" "strconv" "strings" + "github.com/Tencent/WeKnora/internal/application/access" + filesvc "github.com/Tencent/WeKnora/internal/application/service/file" "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/filetransport" "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/storageurl" "github.com/Tencent/WeKnora/internal/types" secutils "github.com/Tencent/WeKnora/internal/utils" "github.com/gin-gonic/gin" @@ -180,12 +183,12 @@ func (h *Handler) DownloadMessageArtifact(c *gin.Context) { return } if index >= len(msg.Artifacts) { - c.Error(errors.NewNotFoundError("artifact index out of range")) + _ = c.Error(errors.NewNotFoundError("artifact index out of range")) return } artifact := msg.Artifacts[index] if artifact.URL == "" { - c.Error(errors.NewNotFoundError("artifact storage path missing")) + _ = c.Error(errors.NewNotFoundError("artifact storage path missing")) return } @@ -193,27 +196,64 @@ func (h *Handler) DownloadMessageArtifact(c *gin.Context) { c.Error(errors.NewInternalServerError("file service unavailable")) return } - reader, err := h.fileService.GetFile(ctx, artifact.URL) + file, err := access.ResolveMessageArtifact(ctx, msg, index, h.agentShareService, h.resourceCatalog, + access.MessageKBShareAuthorizer{ShareGuard: h.kbShareService, KBs: h.knowledgebaseService}) + if err != nil { + _ = c.Error(errors.NewNotFoundError("artifact not accessible")) + return + } + ctx = types.WithExecutionTenant(ctx, file.OwnerTenantID) + fileService := h.fileService + if h.tenantService != nil { + tenant, lookupErr := h.tenantService.GetTenantByID(ctx, file.OwnerTenantID) + if lookupErr != nil || tenant == nil { + _ = c.Error(errors.NewNotFoundError("artifact workspace unavailable")) + return + } + backendID, providerPath, scoped := types.ParseStorageBackendPath(file.Path) + if !scoped { + providerPath = file.Path + } + if file.StorageBackendID != "" { + backendID = file.StorageBackendID + } + var ok bool + fileService, _, ok = filesvc.ResolveTenantFileServiceWithFallback( + ctx, + "artifact download", + tenant, + backendID, + types.ParseProviderScheme(providerPath), + storageurl.LocalStorageBaseDir(), + h.storageResolver, + h.fileService, + ) + if !ok { + _ = c.Error(errors.NewNotFoundError("artifact storage unavailable")) + return + } + } + reader, err := fileService.GetFile(ctx, file.Path) if err != nil { logger.Warnf(ctx, "artifact download read failed: session=%s message=%s idx=%d err=%v", sessionID, messageID, index, err) - c.Error(errors.NewNotFoundError("artifact blob missing")) + _ = c.Error(errors.NewNotFoundError("artifact blob missing")) return } - defer reader.Close() - - // Force download semantics — artifacts are never rendered inline, matching - // the /files endpoint's active-content protection. - c.Header("Content-Type", mimeTypeFor(artifact.FileName)) - c.Header("X-Content-Type-Options", "nosniff") - c.Header("Content-Disposition", buildAttachmentHeader(artifact.FileName)) - if artifact.FileSize > 0 { - c.Header("Content-Length", strconv.FormatInt(artifact.FileSize, 10)) - } - c.Status(http.StatusOK) - if _, err := io.Copy(c.Writer, reader); err != nil { - logger.Warnf(ctx, "artifact download stream failed: session=%s message=%s idx=%d err=%v", - sessionID, messageID, index, err) + if err := filetransport.Serve(c.Writer, c.Request, reader, filetransport.Options{ + Filename: artifact.FileName, Download: true, ContentType: mimeTypeFor(artifact.FileName), + Disposition: buildAttachmentHeader(artifact.FileName), + Size: artifact.FileSize, + CacheControl: "private, no-store", + }); err != nil { + logger.Warnf( + ctx, + "artifact download stream failed: session=%s message=%s idx=%d err=%v", + sessionID, + messageID, + index, + err, + ) } } diff --git a/internal/handler/session/artifact_download_test.go b/internal/handler/session/artifact_download_test.go index e58a1d20..6d6921d3 100644 --- a/internal/handler/session/artifact_download_test.go +++ b/internal/handler/session/artifact_download_test.go @@ -21,9 +21,8 @@ import ( // ----------------------------------------------------------------------------- // Test doubles // -// The download handler only touches three collaborators: SessionService (for -// ownership check), MessageService (for the message + artifact list), and -// FileService (for the blob stream). We embed the full interfaces so the +// The download handler checks session ownership, persisted artifacts and +// source sharing before resolving the owner's file storage. We embed interfaces so the // zero-value struct compiles, then override just the methods each test // exercises. A stray call to an un-stubbed method deliberately nil-panics // so the failure surfaces immediately. @@ -58,18 +57,63 @@ func (s *stubMessageServiceForArtifacts) GetSessionArtifacts(ctx context.Context // fakeArtifactFileService serves canned bytes for a single URL. type fakeArtifactFileService struct { interfaces.FileService - url string - data []byte + url string + data []byte + calls int } func (f *fakeArtifactFileService) GetFile(_ context.Context, url string) (io.ReadCloser, error) { + f.calls++ if url != f.url { return nil, stderrors.New("not found") } return io.NopCloser(strings.NewReader(string(f.data))), nil } -func (f *fakeArtifactFileService) SaveFile(_ context.Context, _ *multipart.FileHeader, _ uint64, _ string) (string, error) { +type artifactCatalogStub struct { + interfaces.ResourceCatalog + resource *types.StoredResource +} + +func (s *artifactCatalogStub) ResolvePath(context.Context, string) (string, *types.StoredResource, error) { + return s.resource.PhysicalPath, s.resource, nil +} + +type artifactTenantStub struct { + interfaces.TenantService + t *testing.T +} + +func (s *artifactTenantStub) GetTenantByID(ctx context.Context, id uint64) (*types.Tenant, error) { + if id != 7 || types.MustTenantIDFromContext(ctx) != 7 || types.CallerFromContext(ctx).TenantID != 42 { + s.t.Fatal("artifact storage must use its owner while preserving the caller") + } + return &types.Tenant{ID: id}, nil +} + +type artifactStorageStub struct { + interfaces.StorageBackendResolver + t *testing.T + file interfaces.FileService +} + +func (s *artifactStorageStub) ResolveFileService( + _ context.Context, + tenant *types.Tenant, + backendID, provider, _ string, +) (interfaces.FileService, string, error) { + if tenant.ID != 7 || backendID != "backend-7" || provider != "local" { + s.t.Fatalf("unexpected storage target: tenant=%d backend=%s provider=%s", tenant.ID, backendID, provider) + } + return s.file, provider, nil +} + +func (f *fakeArtifactFileService) SaveFile( + _ context.Context, + _ *multipart.FileHeader, + _ uint64, + _ string, +) (string, error) { return "", nil } @@ -92,7 +136,10 @@ func (f *fakeArtifactFileService) DeleteFile(_ context.Context, _ string) error func newArtifactTestRouter(h *Handler) *gin.Engine { gin.SetMode(gin.TestMode) r := gin.New() - r.Use(middleware.ErrorHandler()) + r.Use(middleware.ErrorHandler(), func(c *gin.Context) { + c.Request = c.Request.WithContext(context.WithValue(c.Request.Context(), types.TenantIDContextKey, uint64(42))) + c.Next() + }) // Match the production route's wildcard names (see router.go — GET tree // binds :id to align with /sessions/:id) so paramSessionID resolves via // the same code path exercised in prod. @@ -109,7 +156,7 @@ func newArtifactTestRouter(h *Handler) *gin.Engine { func TestDownloadMessageArtifact_HappyPath(t *testing.T) { sessionID := "sess-1" messageID := "msg-1" - url := "fake://tenant-42/report.pptx" + url := "local://42/exports/report.pptx" body := []byte("PPTX-BYTES") h := &Handler{ @@ -180,6 +227,60 @@ func TestDownloadMessageArtifact_SessionNotOwnedReturns404(t *testing.T) { } } +func TestDownloadMessageArtifact_SourceStorageAndShareRevocation(t *testing.T) { + const ref = "resource://AbCdEfGhIjKlMnOpQrStUv" + const physical = "local://7/exports/report.pdf" + shares := &resolveAgentShareStub{agent: &types.CustomAgent{ID: "agent", TenantID: 7}} + ownerFiles := &fakeArtifactFileService{url: physical, data: []byte("PDF-BYTES")} + globalFiles := &fakeArtifactFileService{} + h := &Handler{ + sessionService: &stubSessionServiceForArtifacts{ + getSession: func(context.Context, string) (*types.Session, error) { + return &types.Session{ID: "sess-1", TenantID: 42}, nil + }, + }, + messageService: &stubMessageServiceForArtifacts{ + getMessage: func(context.Context, string, string) (*types.Message, error) { + return &types.Message{ + ID: "msg-1", AgentID: "agent", AgentTenantID: 7, + Artifacts: types.MessageArtifacts{{URL: ref, FileName: "report.pdf"}}, + }, nil + }, + }, + agentShareService: shares, + resourceCatalog: &artifactCatalogStub{resource: &types.StoredResource{ + TenantID: 7, PhysicalPath: physical, StorageBackendID: "backend-7", + }}, + tenantService: &artifactTenantStub{t: t}, + storageResolver: &artifactStorageStub{t: t, file: ownerFiles}, + fileService: globalFiles, + } + router := newArtifactTestRouter(h) + request := func() *httptest.ResponseRecorder { + w := httptest.NewRecorder() + router.ServeHTTP( + w, + httptest.NewRequest(http.MethodGet, "/sessions/sess-1/messages/msg-1/artifacts/0/download", nil), + ) + return w + } + w := request() + if w.Code != http.StatusOK || w.Body.String() != "PDF-BYTES" || + w.Header().Get("Cache-Control") != "private, no-store" { + t.Fatalf("status=%d body=%q cache=%q", w.Code, w.Body.String(), w.Header().Get("Cache-Control")) + } + shares.agent = nil + w = request() + if w.Code != http.StatusNotFound || ownerFiles.calls != 1 || globalFiles.calls != 0 { + t.Fatalf( + "revoked artifact: status=%d owner reads=%d global reads=%d", + w.Code, + ownerFiles.calls, + globalFiles.calls, + ) + } +} + func TestDownloadMessageArtifact_IndexOutOfRange(t *testing.T) { sessionID := "sess-1" messageID := "msg-1" @@ -269,3 +370,11 @@ func TestBuildAttachmentHeader_CJK(t *testing.T) { t.Fatalf("filename* contains raw CJK: %q", got) } } + +func (s *artifactCatalogStub) GetMessageFileBindings( + _ context.Context, + _ uint64, + _, messageID string, +) (*types.MessageFileBindings, error) { + return &types.MessageFileBindings{MessageArtifact: messageID == "msg-1"}, nil +} diff --git a/internal/handler/session/handler.go b/internal/handler/session/handler.go index 7498998f..779437b3 100644 --- a/internal/handler/session/handler.go +++ b/internal/handler/session/handler.go @@ -28,6 +28,7 @@ type Handler struct { agentShareService interfaces.AgentShareService // Service for resolving shared agents (KB scope in retrieval) kbShareService interfaces.KBShareService // Service for resolving shared KB permissions fileService interfaces.FileService // Service for file storage (image uploads) + resourceCatalog interfaces.ResourceCatalog storageResolver interfaces.StorageBackendResolver modelService interfaces.ModelService // Service for model management (VLM access) attachmentProcessor *AttachmentProcessor // Processor for file attachments @@ -52,6 +53,7 @@ func NewHandler( agentShareService interfaces.AgentShareService, kbShareService interfaces.KBShareService, fileService interfaces.FileService, + resourceCatalog interfaces.ResourceCatalog, storageResolver interfaces.StorageBackendResolver, modelService interfaces.ModelService, documentReader interfaces.DocumentReader, @@ -72,6 +74,7 @@ func NewHandler( agentShareService: agentShareService, kbShareService: kbShareService, fileService: fileService, + resourceCatalog: resourceCatalog, storageResolver: storageResolver, modelService: modelService, temporaryDocuments: temporaryDocuments, diff --git a/internal/handler/session/qa.go b/internal/handler/session/qa.go index 795491c3..1e37fe58 100644 --- a/internal/handler/session/qa.go +++ b/internal/handler/session/qa.go @@ -14,6 +14,7 @@ import ( "sync" "time" + "github.com/Tencent/WeKnora/internal/application/access" "github.com/Tencent/WeKnora/internal/errors" "github.com/Tencent/WeKnora/internal/event" "github.com/Tencent/WeKnora/internal/logger" @@ -236,7 +237,7 @@ func (h *Handler) parseQARequest(c *gin.Context, logPrefix string) (*qaRequestCo tenantID := c.GetUint64(types.TenantIDContextKey.String()) attachmentRuntimeCtx := ctx if effectiveTenantID != 0 { - attachmentRuntimeCtx = context.WithValue(ctx, types.TenantIDContextKey, effectiveTenantID) + attachmentRuntimeCtx = types.WithExecutionTenant(ctx, effectiveTenantID) } // Use ASR only when the agent has audio upload enabled. @@ -631,8 +632,16 @@ func (h *Handler) setupSSEStream(reqCtx *qaRequestContext, generateTitle bool) * baseCtx := reqCtx.ctx if reqCtx.effectiveTenantID != 0 && h.tenantService != nil { if tenant, err := h.tenantService.GetTenantByID(reqCtx.ctx, reqCtx.effectiveTenantID); err == nil && tenant != nil { - baseCtx = context.WithValue(context.WithValue(reqCtx.ctx, types.TenantIDContextKey, reqCtx.effectiveTenantID), types.TenantInfoContextKey, tenant) - logger.Infof(reqCtx.ctx, "Using effective tenant %d for shared agent (model/KB/MCP)", reqCtx.effectiveTenantID) + baseCtx = types.WithExecutionTenant(reqCtx.ctx, reqCtx.effectiveTenantID) + if reqCtx.customAgent != nil { + baseCtx = access.WithSharedAgent(baseCtx, reqCtx.customAgent) + } + baseCtx = context.WithValue(baseCtx, types.TenantInfoContextKey, tenant) + logger.Infof( + reqCtx.ctx, + "Using effective tenant %d for shared agent (model/KB/MCP)", + reqCtx.effectiveTenantID, + ) } } // The session's sandbox stays bound to the session owner even when the diff --git a/internal/handler/session/temporary_document.go b/internal/handler/session/temporary_document.go index 22eaef45..32918417 100644 --- a/internal/handler/session/temporary_document.go +++ b/internal/handler/session/temporary_document.go @@ -2,13 +2,12 @@ package session import ( "fmt" - "io" - "mime" "net/http" "path/filepath" "strings" apperrors "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/filetransport" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" secutils "github.com/Tencent/WeKnora/internal/utils" @@ -151,25 +150,11 @@ func (h *Handler) PreviewTemporaryDocument(c *gin.Context) { c.Error(apperrors.NewInternalServerError("Failed to retrieve attachment").WithDetails(err.Error())) return } - defer file.Close() - - contentType, inline := secutils.SafeContentTypeByFilename(filename) - c.Header("Content-Type", contentType) - c.Header("X-Content-Type-Options", "nosniff") - disposition := "inline" - if !inline { - disposition = "attachment" + if err := filetransport.Serve(c.Writer, c.Request, file, filetransport.Options{ + Filename: filename, CacheControl: "private, no-store", + }); err != nil { + logger.Errorf(ctx, "Failed to stream attachment preview: %v", err) } - c.Header("Content-Disposition", mime.FormatMediaType(disposition, map[string]string{"filename": filename})) - c.Header("Cache-Control", "private, max-age=3600") - - c.Stream(func(w io.Writer) bool { - if _, err := io.Copy(w, file); err != nil { - logger.Errorf(ctx, "Failed to stream attachment preview: %v", err) - return false - } - return false - }) } func (h *Handler) DeleteTemporaryDocument(c *gin.Context) { diff --git a/internal/handler/session/wiki_fixer_scope.go b/internal/handler/session/wiki_fixer_scope.go index bc65131f..0f4871bc 100644 --- a/internal/handler/session/wiki_fixer_scope.go +++ b/internal/handler/session/wiki_fixer_scope.go @@ -3,6 +3,7 @@ package session import ( "context" + "github.com/Tencent/WeKnora/internal/application/access" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" secutils "github.com/Tencent/WeKnora/internal/utils" @@ -12,9 +13,7 @@ type wikiFixerKBLookup interface { GetKnowledgeBaseByIDOnly(ctx context.Context, id string) (*types.KnowledgeBase, error) } -type wikiFixerKBSharePermission interface { - CheckTenantKBPermission(ctx context.Context, kbID string, callerTenantID uint64, callerTenantRole types.TenantRole) (types.OrgMemberRole, bool, error) -} +type wikiFixerKBSharePermission = access.KBShareLookup func (h *Handler) resolveWikiFixerTenantScope( ctx context.Context, @@ -64,12 +63,13 @@ func resolveBuiltinWikiFixerTenantScope( return agent, 0 } - permission, isShared, err := kbShare.CheckTenantKBPermission(ctx, kb.ID, currentTenantID, callerTenantRole) + permissions := access.NewKBSharePermissions(ctx, kbShare, currentTenantID, callerTenantRole) + allowed, err := permissions.Check(kb.ID, types.OrgRoleEditor) if err != nil { logger.Warnf(ctx, "wiki fixer: failed to check shared KB %s permission: %v", secutils.SanitizeForLog(kb.ID), err) return agent, 0 } - if !isShared || !permission.HasPermission(types.OrgRoleEditor) { + if !allowed { return agent, 0 } diff --git a/internal/handler/shared_agent_access.go b/internal/handler/shared_agent_access.go new file mode 100644 index 00000000..dfcd47f0 --- /dev/null +++ b/internal/handler/shared_agent_access.go @@ -0,0 +1,82 @@ +package handler + +import ( + stderrors "errors" + + "github.com/Tencent/WeKnora/internal/agent/tools" + "github.com/Tencent/WeKnora/internal/application/access" + "github.com/Tencent/WeKnora/internal/application/service" + apperrors "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/middleware" + "github.com/Tencent/WeKnora/internal/types" + "github.com/gin-gonic/gin" +) + +// resolveSharedAgentForRequest is the identity/share boundary for KB listing, +// document search and batch restoration. KB-scope enforcement follows lookup. +func resolveSharedAgentForRequest( + c *gin.Context, + agentID string, + agents access.SharedAgentLookup, +) (*types.CustomAgent, error) { + ctx := c.Request.Context() + request := middleware.KBAccessRequest(c) + userID := request.Caller.UserID + if request.Caller.TenantID == 0 || userID == "" { + return nil, apperrors.NewUnauthorizedError("Unauthorized") + } + source, err := types.ParseAgentSourceTenantID(request.AgentSourceTenantID) + if err != nil { + return nil, apperrors.NewBadRequestError(err.Error()) + } + if agents == nil { + return nil, apperrors.NewForbiddenError("no permission for this shared agent") + } + agent, err := agents.GetSharedAgentForTenant(ctx, request.Caller.TenantID, request.Caller.Role, agentID, source) + if err != nil { + if stderrors.Is(err, service.ErrAgentShareNotFound) || stderrors.Is(err, service.ErrAgentSharePermission) || + stderrors.Is(err, service.ErrAgentNotFoundForShare) { + return nil, apperrors.NewForbiddenError("no permission for this shared agent") + } + logger.ErrorWithFields(ctx, err, nil) + return nil, apperrors.NewInternalServerError("Failed to verify shared agent access") + } + if agent == nil || agent.TenantID == 0 || (source != 0 && agent.TenantID != source) { + return nil, apperrors.NewForbiddenError("no permission for this shared agent") + } + return agent, nil +} + +func filterKnowledgeByAgentScope(knowledges []*types.Knowledge, scope types.SharedAgentKBScope) []*types.Knowledge { + filtered := make([]*types.Knowledge, 0, len(knowledges)) + for _, knowledge := range knowledges { + if knowledge != nil && scope.Allows(knowledge.KnowledgeBaseID, knowledge.TenantID) { + filtered = append(filtered, knowledge) + } + } + return filtered +} + +// Scope filtering is shared by @KB listing and @file search. Capability checks +// constrain dynamic "all" selections; explicit selections keep their existing +// behavior. This filter does not replace API-key scope checks at the endpoint. +func filterKnowledgeBasesForSharedAgent(kbs []*types.KnowledgeBase, agent *types.CustomAgent) []*types.KnowledgeBase { + scope := types.NewSharedAgentKBScope(agent) + filtered := make([]*types.KnowledgeBase, 0, len(kbs)) + if scope.IsEmpty() { + return filtered + } + filter := tools.DeriveKBFilterForAgent(agent.Config.AgentMode, agent.Config.AllowedTools) + for _, kb := range kbs { + if kb == nil || !scope.Allows(kb.ID, kb.TenantID) { + continue + } + if scope.IsAll() && !filter.IsEmpty() && + !tools.KBSatisfiesAgentRequirements(kb.Capabilities(), agent.Config.AgentMode, agent.Config.AllowedTools) { + continue + } + filtered = append(filtered, kb) + } + return filtered +} diff --git a/internal/handler/shared_agent_access_test.go b/internal/handler/shared_agent_access_test.go new file mode 100644 index 00000000..13632e90 --- /dev/null +++ b/internal/handler/shared_agent_access_test.go @@ -0,0 +1,245 @@ +package handler + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "github.com/Tencent/WeKnora/internal/middleware" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +type scopedAgentStub struct { + interfaces.AgentShareService + agent *types.CustomAgent + err error + caller, source uint64 +} + +func (s *scopedAgentStub) GetSharedAgentForTenant( + _ context.Context, + caller uint64, + _ types.TenantRole, + _ string, + source ...uint64, +) (*types.CustomAgent, error) { + s.caller, s.source = caller, source[0] + return s.agent, s.err +} + +type scopedKBStub struct { + interfaces.KnowledgeBaseService + calls int + kbs []*types.KnowledgeBase +} + +func (s *scopedKBStub) ListKnowledgeBasesByTenantID(context.Context, uint64) ([]*types.KnowledgeBase, error) { + s.calls++ + return s.kbs, nil +} + +type scopedKnowledgeStub struct { + interfaces.KnowledgeService + batchCalls, searchCalls int + knowledges []*types.Knowledge + batchErr error +} + +func (s *scopedKnowledgeStub) GetKnowledgeBatch(context.Context, uint64, []string) ([]*types.Knowledge, error) { + s.batchCalls++ + return s.knowledges, s.batchErr +} + +func (s *scopedKnowledgeStub) SearchKnowledgeForScopes( + _ context.Context, + scopes []types.KnowledgeSearchScope, + _ string, + _, _ int, + _ []string, +) ([]*types.Knowledge, bool, int64, error) { + s.searchCalls++ + var out []*types.Knowledge + for _, k := range s.knowledges { + if k == nil { + continue + } + for _, scope := range scopes { + if k.TenantID == scope.TenantID && k.KnowledgeBaseID == scope.KBID { + out = append(out, k) + break + } + } + } + return out, false, int64(len(out)), nil +} + +func TestSharedAgentScopesAgreeAcrossListSearchAndBatch(t *testing.T) { + gin.SetMode(gin.TestMode) + for _, tt := range []struct { + name, mode string + ids []string + expected []string + restricted bool + }{ + {name: "all", mode: "all", expected: []string{"a", "b"}}, + {name: "selected", mode: "selected", ids: []string{"a"}, expected: []string{"a"}}, + {name: "selected empty", mode: "selected"}, + {name: "selected blank", mode: "selected", ids: []string{""}}, + {name: "none", mode: "none", ids: []string{"a"}}, + {name: "unknown", mode: "invalid", ids: []string{"a"}}, + {name: "API key intersects all", mode: "all", restricted: true, expected: []string{"b"}}, + {name: "API key intersects selection", mode: "selected", ids: []string{"a"}, restricted: true}, + } { + t.Run(tt.name, func(t *testing.T) { + agents := &scopedAgentStub{ + agent: &types.CustomAgent{ + ID: "agent", + TenantID: 2, + Config: types.CustomAgentConfig{KBSelectionMode: tt.mode, KnowledgeBases: tt.ids}, + }, + } + kbs := &scopedKBStub{kbs: []*types.KnowledgeBase{ + {ID: "a", TenantID: 2, Type: types.KnowledgeBaseTypeDocument}, + {ID: "b", TenantID: 2, Type: types.KnowledgeBaseTypeDocument}, + {ID: "foreign", TenantID: 3, Type: types.KnowledgeBaseTypeDocument}, + nil, + }} + knowledges := &scopedKnowledgeStub{knowledges: []*types.Knowledge{ + {ID: "a", TenantID: 2, KnowledgeBaseID: "a"}, + {ID: "b", TenantID: 2, KnowledgeBaseID: "b"}, + {ID: "foreign", TenantID: 3, KnowledgeBaseID: "a"}, + nil, + }} + kh := &KnowledgeHandler{kgService: knowledges, kbService: kbs, agentShareService: agents} + bh := &KnowledgeBaseHandler{service: kbs, agentShareService: agents} + r := gin.New() + r.Use(middleware.ErrorHandler(), func(c *gin.Context) { + c.Set(types.TenantIDContextKey.String(), uint64(1)) + c.Set(types.UserIDContextKey.String(), "user") + ctx := context.WithValue(c.Request.Context(), types.TenantIDContextKey, uint64(1)) + if tt.restricted { + ctx = types.WithTenantAPIKeyScope( + ctx, + types.TenantAPIKeyScope{KnowledgeBaseIDs: types.StringArray{"b"}}, + ) + } + c.Request = c.Request.WithContext(ctx) + c.Next() + }) + r.GET("/kbs", bh.ListKnowledgeBases) + r.GET("/search", kh.SearchKnowledge) + r.GET("/batch", kh.GetKnowledgeBatch) + for _, path := range []string{"/kbs", "/search", "/batch"} { + w := httptest.NewRecorder() + r.ServeHTTP( + w, + httptest.NewRequest( + http.MethodGet, + path+"?agent_id=agent&agent_source_tenant_id=2&keyword=test&ids=a&ids=b", + nil, + ), + ) + require.Equal(t, http.StatusOK, w.Code, "%s: %s", path, w.Body.String()) + var body struct { + Data []struct { + ID string `json:"id"` + } `json:"data"` + } + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &body)) + var ids []string + for _, row := range body.Data { + ids = append(ids, row.ID) + } + require.ElementsMatch(t, tt.expected, ids, path) + } + require.Equal(t, uint64(1), agents.caller) + require.Equal(t, uint64(2), agents.source) + if types.NewSharedAgentKBScope(agents.agent).IsEmpty() { + require.Zero(t, kbs.calls) + require.Zero(t, knowledges.batchCalls) + require.Zero(t, knowledges.searchCalls) + } + }) + } +} + +func TestSharedAgentLookupFailuresDoNotReachBatchRetrieval(t *testing.T) { + for _, tt := range []struct { + name string + agent *types.CustomAgent + err error + source string + status int + }{ + {name: "revoked or missing", status: http.StatusForbidden}, + {name: "source mismatch", agent: &types.CustomAgent{TenantID: 3}, source: "2", status: http.StatusForbidden}, + {name: "invalid source", source: "invalid", status: http.StatusBadRequest}, + {name: "lookup unavailable", err: errors.New("offline"), status: http.StatusInternalServerError}, + } { + t.Run(tt.name, func(t *testing.T) { + r := gin.New() + r.Use(middleware.ErrorHandler(), func(c *gin.Context) { + c.Set(types.TenantIDContextKey.String(), uint64(1)) + c.Set(types.UserIDContextKey.String(), "user") + c.Next() + }) + h := &KnowledgeHandler{agentShareService: &scopedAgentStub{agent: tt.agent, err: tt.err}} + r.GET("/batch", h.GetKnowledgeBatch) + w := httptest.NewRecorder() + r.ServeHTTP( + w, + httptest.NewRequest( + http.MethodGet, + "/batch?ids=a&agent_id=agent&agent_source_tenant_id="+tt.source, + nil, + ), + ) + require.Equal(t, tt.status, w.Code, w.Body.String()) + }) + } +} + +func TestSharedAgentCapabilityFilterRetainsExplicitSelection(t *testing.T) { + kb := &types.KnowledgeBase{ID: "wiki", TenantID: 2, IndexingStrategy: types.IndexingStrategy{WikiEnabled: true}} + agent := &types.CustomAgent{ + TenantID: 2, + Config: types.CustomAgentConfig{AgentMode: "quick-answer", KBSelectionMode: "all"}, + } + require.Empty(t, filterKnowledgeBasesForSharedAgent([]*types.KnowledgeBase{kb}, agent)) + agent.Config.KBSelectionMode = "selected" + agent.Config.KnowledgeBases = []string{"wiki"} + require.Len(t, filterKnowledgeBasesForSharedAgent([]*types.KnowledgeBase{kb}, agent), 1) +} + +func TestExplicitKBBatchReadPropagatesRepositoryFailure(t *testing.T) { + gin.SetMode(gin.TestMode) + kbs := &scopedBatchKBStub{} + knowledge := &scopedKnowledgeStub{batchErr: errors.New("storage unavailable")} + h := &KnowledgeHandler{kgService: knowledge, kbService: kbs} + r := gin.New() + r.Use(middleware.ErrorHandler(), func(c *gin.Context) { + ctx := types.WithExecutionTenant(c.Request.Context(), 1) + ctx = types.WithCaller(ctx, types.Caller{TenantID: 1, UserID: "user", Role: types.TenantRoleAdmin}) + c.Request = c.Request.WithContext(ctx) + c.Next() + }) + r.GET("/batch", h.GetKnowledgeBatch) + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/batch?kb_id=kb&ids=doc", nil)) + require.Equal(t, http.StatusInternalServerError, w.Code, w.Body.String()) + require.Equal(t, 1, knowledge.batchCalls) +} + +type scopedBatchKBStub struct { + interfaces.KnowledgeBaseService +} + +func (*scopedBatchKBStub) GetKnowledgeBaseByID(context.Context, string) (*types.KnowledgeBase, error) { + return &types.KnowledgeBase{ID: "kb", TenantID: 1}, nil +} diff --git a/internal/handler/tag.go b/internal/handler/tag.go index 3145d179..59b2a2b5 100644 --- a/internal/handler/tag.go +++ b/internal/handler/tag.go @@ -2,16 +2,17 @@ package handler import ( "context" + stderrors "errors" + "io" "net/http" "strconv" - "github.com/gin-gonic/gin" - "github.com/Tencent/WeKnora/internal/errors" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" secutils "github.com/Tencent/WeKnora/internal/utils" + "github.com/gin-gonic/gin" ) // TagHandler handles knowledge base tag operations. @@ -235,19 +236,42 @@ func (h *TagHandler) DeleteTag(c *gin.Context) { contentOnly := c.Query("content_only") == "true" var req DeleteTagRequest - _ = c.ShouldBindJSON(&req) + if err := c.ShouldBindJSON(&req); err != nil && !stderrors.Is(err, io.EOF) { + _ = c.Error(errors.NewBadRequestError("删除选项不合法")) + return + } var excludeUUIDs []string if len(req.ExcludeIDs) > 0 { tenantID := types.MustTenantIDFromContext(ctx) + wanted := make(map[int64]bool, len(req.ExcludeIDs)) + for _, id := range req.ExcludeIDs { + if id <= 0 { + _ = c.Error(errors.NewBadRequestError("排除条目 ID 必须为正整数")) + return + } + wanted[id] = true + } chunks, err := h.getChunksBySeqIDs(ctx, tenantID, req.ExcludeIDs) if err != nil { - logger.Warnf(ctx, "Failed to resolve exclude_ids: %v", err) - } else { - excludeUUIDs = make([]string, len(chunks)) - for i, chunk := range chunks { - excludeUUIDs[i] = chunk.ID + _ = c.Error(err) + return + } + for _, chunk := range chunks { + if chunk == nil || !wanted[chunk.SeqID] { + continue } + if chunk.TenantID != tenantID || chunk.KnowledgeBaseID != c.Param("id") || + chunk.ChunkType != types.ChunkTypeFAQ { + _ = c.Error(errors.NewForbiddenError("排除条目不属于当前知识库")) + return + } + excludeUUIDs = append(excludeUUIDs, chunk.ID) + delete(wanted, chunk.SeqID) + } + if len(wanted) != 0 { + _ = c.Error(errors.NewNotFoundError("排除条目不存在")) + return } } diff --git a/internal/handler/tag_delete_test.go b/internal/handler/tag_delete_test.go new file mode 100644 index 00000000..44693763 --- /dev/null +++ b/internal/handler/tag_delete_test.go @@ -0,0 +1,109 @@ +package handler + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/middleware" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +type tagDeleteServiceSpy struct { + interfaces.KnowledgeTagService + calls int + excluded []string +} + +func (s *tagDeleteServiceSpy) DeleteTag(_ context.Context, _ string, _, _ bool, excluded []string) error { + s.calls++ + s.excluded = excluded + return nil +} + +type tagDeleteChunks struct { + interfaces.ChunkRepository + chunks []*types.Chunk + err error +} + +func (s *tagDeleteChunks) ListChunksBySeqID(context.Context, uint64, []int64) ([]*types.Chunk, error) { + return s.chunks, s.err +} + +func TestTagDeletionNeverIgnoresInvalidExclusions(t *testing.T) { + gin.SetMode(gin.TestMode) + for _, tt := range []struct { + name, body string + chunks []*types.Chunk + err error + status int + }{ + {name: "invalid JSON", body: `{`, status: 400}, + {name: "missing exclusion", body: `{"exclude_ids":[1]}`, status: 404}, + {name: "lookup failure", body: `{"exclude_ids":[1]}`, err: errors.New("database unavailable"), status: 500}, + { + name: "foreign exclusion", + body: `{"exclude_ids":[1]}`, + chunks: []*types.Chunk{{ + ID: "one", + SeqID: 1, + TenantID: 7, + KnowledgeBaseID: "other", + ChunkType: types.ChunkTypeFAQ, + }}, + status: 403, + }, + + { + name: "valid exclusion", + body: `{"exclude_ids":[1]}`, + chunks: []*types.Chunk{{ + ID: "one", + SeqID: 1, + TenantID: 7, + KnowledgeBaseID: "kb", + ChunkType: types.ChunkTypeFAQ, + }}, + status: 200, + }, + + {name: "optional empty body", status: 200}, + } { + t.Run(tt.name, func(t *testing.T) { + svc := &tagDeleteServiceSpy{} + h := &TagHandler{tagService: svc, chunkRepo: &tagDeleteChunks{chunks: tt.chunks, err: tt.err}} + router := gin.New() + router.Use(middleware.ErrorHandler(), func(c *gin.Context) { + c.Request = c.Request.WithContext( + context.WithValue(c.Request.Context(), types.TenantIDContextKey, uint64(7)), + ) + c.Next() + }) + router.DELETE("/knowledge-bases/:id/tags/:tag_id", h.DeleteTag) + w := httptest.NewRecorder() + req := httptest.NewRequest( + http.MethodDelete, + "/knowledge-bases/kb/tags/tag?force=true", + strings.NewReader(tt.body), + ) + req.Header.Set("Content-Type", "application/json") + router.ServeHTTP(w, req) + require.Equal(t, tt.status, w.Code, w.Body.String()) + if tt.status == 200 { + require.Equal(t, 1, svc.calls) + if tt.body != "" { + require.Equal(t, []string{"one"}, svc.excluded) + } + } else { + require.Zero(t, svc.calls) + } + }) + } +} diff --git a/internal/middleware/auth_context.go b/internal/middleware/auth_context.go index 47121306..f3e2a341 100644 --- a/internal/middleware/auth_context.go +++ b/internal/middleware/auth_context.go @@ -77,5 +77,10 @@ func applyAuthSession(c *gin.Context, s authSession) { set(key, v) } + userID := "" + if s.User != nil { + userID = s.User.ID + } + ctx = types.WithCaller(ctx, types.Caller{TenantID: s.TenantID, UserID: userID, Role: s.Role}) c.Request = c.Request.WithContext(ctx) } diff --git a/internal/middleware/auth_context_test.go b/internal/middleware/auth_context_test.go index d756f16d..7986ec77 100644 --- a/internal/middleware/auth_context_test.go +++ b/internal/middleware/auth_context_test.go @@ -34,6 +34,10 @@ func TestApplyAuthSessionSetsBothSurfaces(t *testing.T) { }) ctx := c.Request.Context() + caller := types.Caller{TenantID: 7, UserID: "u1", Role: types.TenantRoleAdmin} + if got := types.CallerFromContext(types.WithExecutionTenant(ctx, 9)); got != caller { + t.Fatalf("execution switch changed authenticated caller: %+v", got) + } if got, ok := types.TenantIDFromContext(ctx); !ok || got != 7 { t.Fatalf("ctx tenant id = %d, ok=%v", got, ok) } diff --git a/internal/middleware/kb_access.go b/internal/middleware/kb_access.go index 66bbfc1c..456d5531 100644 --- a/internal/middleware/kb_access.go +++ b/internal/middleware/kb_access.go @@ -4,6 +4,7 @@ import ( "context" stderrors "errors" + "github.com/Tencent/WeKnora/internal/application/access" apprepo "github.com/Tencent/WeKnora/internal/application/repository" "github.com/Tencent/WeKnora/internal/config" apperrors "github.com/Tencent/WeKnora/internal/errors" @@ -13,54 +14,10 @@ import ( "github.com/gin-gonic/gin" ) -// kb_access.go centralises the share-fallback that previously lived as -// near-identical 30-line helpers in five handler files (chunk.go, -// faq.go, tag.go, knowledge.go, knowledgebase.go). Each was a copy of -// the same three checks: -// -// 1. KB belongs to caller's tenant -> grant own access -// 2. Org-shared KB -> grant min(share, role) cap -// 3. Shared agent carries the KB -> grant Viewer (read-only) -// -// Putting the resolution in a route-level gin.HandlerFunc makes the -// route declaration the single source of truth for "what permission -// is required" and "where does the kb_id come from". Handlers no -// longer carry an effectiveCtxForKB / validateAndGetKnowledgeBase -// helper — the guard runs first, stashes the resolution under -// KBAccessContextKey and rewrites c.Request to carry the -// effective-tenant-ID context, then handlers just read TenantIDFromContext -// the way they always did. -// -// Plan 3's 3-D cap (tenant Viewer pinned to OrgRoleViewer) is enforced -// inside CheckTenantKBPermission itself, so the guard here just -// propagates the result. -// -// ⚠️ Tenant id has TWO surfaces on a gin.Context: -// -// - c.Request.Context() (read via types.TenantIDFromContext) -// - c.Keys[TenantIDContextKey] (read via c.Get / c.GetUint64) -// -// The guard rewrites ONLY the request context — c.Keys is intentionally -// left at the caller's own tenant so handlers that still run their own -// share resolution (currently knowledge.go / knowledgebase.go, which -// branch on the ?agent_id query) keep working unchanged. New handlers -// MUST read tenant from c.Request.Context() (the "effective" tenant -// for shared KBs); reading from c.Keys after this guard runs gives -// the caller's own tenant, which is almost always the wrong answer -// for KB-scoped routes. The FAQ / Tag / Chunk handlers in this PR -// were converted to the ctx-based read; the migration of knowledge.go -// / knowledgebase.go is a follow-up. - -// KBAccess captures the result of a successful KB access resolution. -// Stashed on gin.Context under KBAccessContextKey so handlers that -// need the resolved KB / permission (e.g. to render -// my_permission in the response) can pull it without re-running the -// resolution. -type KBAccess struct { - KnowledgeBase *types.KnowledgeBase - EffectiveTenantID uint64 - Permission types.OrgMemberRole -} +// KBAccess aliases the shared policy result. This adapter resolves route +// parameters, applies the RBAC rollout switch, and scopes downstream services +// to the resource tenant. Gin's tenant key retains the authenticated caller. +type KBAccess = access.KBAccess // KBAccessContextKey is the gin.Context key under which a successful // KB access resolution is stored. @@ -75,10 +32,10 @@ func KBAccessFromContext(c *gin.Context) (*KBAccess, bool) { return nil, false } a, ok := v.(*KBAccess) - return a, ok + return a, ok && a != nil } -// KBLookup is the minimum surface ResolveKBAccess needs from the +// KBLookup is the minimum surface the guard needs from the // knowledge-base service: a single method that turns an ID into a // KnowledgeBase pointer (or repo.ErrKnowledgeBaseNotFound). Defining // it as a tiny dedicated interface keeps the guard testable without @@ -254,9 +211,9 @@ func RequireKBAccess( // regardless of whether RBAC enforcement is active. enforcing := rbacEnforcementEnabled(cfg) - access, err := resolveKBAccessOnce(ctx, c, kbID, requiredPermission, kbService, kbShareService, agentShareService) + grant, err := resolveKBAccess(ctx, c, kbID, requiredPermission, kbService, kbShareService, agentShareService) switch { - case stderrors.Is(err, errKBAccessUnauthorized): + case stderrors.Is(err, access.ErrUnauthorized): if !enforcing { logger.Warnf(ctx, "[rbac] kb-access would 401 (enforcement off): kb=%s", kbID) c.Next() @@ -265,14 +222,14 @@ func RequireKBAccess( _ = c.Error(apperrors.NewUnauthorizedError("Unauthorized")) c.Abort() return - case stderrors.Is(err, errKBAccessNotFound): + case stderrors.Is(err, access.ErrNotFound): // 404 still fires when enforcement is off — a missing KB is // not an authorisation event, the client genuinely asked // for nothing. _ = c.Error(apperrors.NewNotFoundError("knowledge base not found")) c.Abort() return - case stderrors.Is(err, errKBAccessForbidden): + case stderrors.Is(err, access.ErrForbidden): if !enforcing { logger.Warnf(ctx, "[rbac] kb-access would 403 (enforcement off): kb=%s required=%s", kbID, requiredPermission) @@ -282,7 +239,7 @@ func RequireKBAccess( _ = c.Error(apperrors.NewForbiddenError("Permission denied to access this knowledge base")) c.Abort() return - case stderrors.Is(err, errKBAccessBadRequest): + case stderrors.Is(err, access.ErrInvalidAgentSource): _ = c.Error(apperrors.NewBadRequestError("invalid agent_source_tenant_id")) c.Abort() return @@ -299,25 +256,37 @@ func RequireKBAccess( // effective tenant id. Handlers reading tenant from context now // see the source-tenant for shared KBs (so retrieval queries // hit the right embedding store) without having to know. - c.Set(KBAccessContextKey, access) - newCtx := context.WithValue(ctx, types.TenantIDContextKey, access.EffectiveTenantID) + c.Set(KBAccessContextKey, grant) + newCtx := grant.Context(ctx) c.Request = c.Request.WithContext(newCtx) c.Next() } } -// resolveKBAccessOnce performs the actual three-step resolution. Kept -// unexported and using package-private sentinel errors so the guard's -// error mapping is the only public surface. -// -// The shared-agent step honours the ?agent_id query parameter when -// present, mirroring the in-handler resolution in -// knowledgebase.go's validateAndGetKnowledgeBase: a request with a -// specific agent_id is validated against THAT agent's KB scope (mode = -// all / selected / none), not against "any shared agent". Falling -// back to "any shared agent" when agent_id is empty preserves the -// "通过智能体可见" KB list entry point. -func resolveKBAccessOnce( +// KBAccessRequest captures caller identity before a guard scopes the request +// context to a shared resource. The stored grant also supports nested guards. +func KBAccessRequest(c *gin.Context) access.KBRequest { + ctx := c.Request.Context() + caller := types.CallerFromContext(ctx) + if _, captured := ctx.Value(types.CallerContextKey).(types.Caller); !captured { + // Compatibility for direct handler invocations that only seed Gin. + if tenantID, ok := c.Get(types.TenantIDContextKey.String()); ok { + caller.TenantID, _ = tenantID.(uint64) + } else if grant, found := KBAccessFromContext(c); found { + caller = grant.Caller + } + if userID, ok := c.Get(types.UserIDContextKey.String()); ok { + caller.UserID, _ = userID.(string) + } + } + return access.KBRequest{ + Caller: caller, + AgentID: c.Query("agent_id"), + AgentSourceTenantID: c.Query(types.AgentSourceTenantIDParam), + } +} + +func resolveKBAccess( ctx context.Context, c *gin.Context, kbID string, @@ -326,140 +295,16 @@ func resolveKBAccessOnce( kbShareService interfaces.KBShareService, agentShareService interfaces.AgentShareService, ) (*KBAccess, error) { - tenantID, ok := types.TenantIDFromContext(ctx) - if !ok || tenantID == 0 { - return nil, errKBAccessUnauthorized + request := KBAccessRequest(c) + if request.Caller.TenantID == 0 { + return nil, access.ErrUnauthorized } - callerTenantRole := types.TenantRoleFromContext(ctx) - kb, err := kbService.GetKnowledgeBaseByID(ctx, kbID) if err != nil { if stderrors.Is(err, apprepo.ErrKnowledgeBaseNotFound) { - return nil, errKBAccessNotFound + return nil, access.ErrNotFound } return nil, err } - if kb == nil { - return nil, errKBAccessNotFound - } - - // 1. Own KB. - if kb.TenantID == tenantID { - return &KBAccess{ - KnowledgeBase: kb, - EffectiveTenantID: tenantID, - Permission: types.OrgRoleAdmin, - }, nil - } - - // 2. Org-shared KB. Plan 3's 3-D cap is applied inside - // CheckTenantKBPermission; we just check the result satisfies - // the minimum requirement. - if kbShareService != nil { - permission, isShared, permErr := kbShareService.CheckTenantKBPermission(ctx, kbID, tenantID, callerTenantRole) - if permErr == nil && isShared && permission.HasPermission(requiredPermission) { - source, srcErr := kbShareService.GetKBSourceTenant(ctx, kbID) - if srcErr == nil { - logger.Infof(ctx, "[kb_access] tenant %d -> shared KB %s perm=%s source=%d", - tenantID, kbID, permission, source) - return &KBAccess{ - KnowledgeBase: kb, - EffectiveTenantID: source, - Permission: permission, - }, nil - } - } - } - - // 3. Shared agent that carries this KB — only ever grants read. - if requiredPermission == types.OrgRoleViewer && agentShareService != nil { - access, agentErr := resolveSharedAgentAccess(ctx, c, tenantID, callerTenantRole, kb, agentShareService) - if agentErr != nil { - return nil, agentErr - } - if access != nil { - return access, nil - } - } - - logger.Warnf(ctx, "[kb_access] tenant %d -> KB %s denied (required=%s)", tenantID, kbID, requiredPermission) - return nil, errKBAccessForbidden + return access.ResolveKB(ctx, request, kb, requiredPermission, kbShareService, agentShareService) } - -// resolveSharedAgentAccess implements the agent-share fallback, -// mirroring knowledgebase.go's in-handler logic so the guard and the -// (still-present) handler resolver agree on which requests pass. -// -// - If ?agent_id=X is provided, validate against THAT agent's -// KBSelectionMode (all / selected / none). A mismatch means deny; -// we do NOT fall back to "any shared agent" because the client -// explicitly named one (typically from a @-mention or a deep link -// scoped to that agent). -// - If ?agent_id is empty, allow when any shared agent reachable by -// the caller can access this KB ("通过智能体可见" KB list entry). -func resolveSharedAgentAccess( - ctx context.Context, - c *gin.Context, - tenantID uint64, - callerTenantRole types.TenantRole, - kb *types.KnowledgeBase, - agentShareService interfaces.AgentShareService, -) (*KBAccess, error) { - agentID := c.Query("agent_id") - if agentID != "" { - sourceTenantID, err := types.ParseAgentSourceTenantID(c.Query(types.AgentSourceTenantIDParam)) - if err != nil { - return nil, errKBAccessBadRequest - } - agent, err := agentShareService.GetSharedAgentForTenant(ctx, tenantID, callerTenantRole, agentID, sourceTenantID) - if err != nil || agent == nil { - return nil, nil - } - if kb.TenantID != agent.TenantID { - logger.Warnf(ctx, "[kb_access] shared agent workspace mismatch: kb=%s kb.tenant=%d agent.tenant=%d", - kb.ID, kb.TenantID, agent.TenantID) - return nil, nil - } - switch agent.Config.KBSelectionMode { - case "all": - logger.Infof(ctx, "[kb_access] tenant %d -> KB %s via shared agent %s (mode=all)", - tenantID, kb.ID, agentID) - return &KBAccess{ - KnowledgeBase: kb, - EffectiveTenantID: kb.TenantID, - Permission: types.OrgRoleViewer, - }, nil - case "selected": - for _, allowedID := range agent.Config.KnowledgeBases { - if allowedID == kb.ID { - logger.Infof(ctx, "[kb_access] tenant %d -> KB %s via shared agent %s (mode=selected)", - tenantID, kb.ID, agentID) - return &KBAccess{ - KnowledgeBase: kb, - EffectiveTenantID: kb.TenantID, - Permission: types.OrgRoleViewer, - }, nil - } - } - } - return nil, nil - } - - can, err := agentShareService.TenantCanAccessKBViaSomeSharedAgent(ctx, tenantID, callerTenantRole, kb) - if err == nil && can { - logger.Infof(ctx, "[kb_access] tenant %d -> KB %s via some shared agent", tenantID, kb.ID) - return &KBAccess{ - KnowledgeBase: kb, - EffectiveTenantID: kb.TenantID, - Permission: types.OrgRoleViewer, - }, nil - } - return nil, nil -} - -var ( - errKBAccessUnauthorized = stderrors.New("kb_access: unauthorized") - errKBAccessNotFound = stderrors.New("kb_access: not found") - errKBAccessForbidden = stderrors.New("kb_access: forbidden") - errKBAccessBadRequest = stderrors.New("kb_access: bad request") -) diff --git a/internal/middleware/kb_access_test.go b/internal/middleware/kb_access_test.go index d552ab8f..57e72203 100644 --- a/internal/middleware/kb_access_test.go +++ b/internal/middleware/kb_access_test.go @@ -33,13 +33,12 @@ func (s *stubKBLookup) GetKnowledgeBaseByID(_ context.Context, id string) (*type } // stubKBShareForGuard implements just the methods the guard touches — -// CheckTenantKBPermission and GetKBSourceTenant. The other methods on +// CheckTenantKBPermission. The other methods on // the interface panic so any unintended new dependency surfaces // immediately. type stubKBShareForGuard struct { permission map[string]types.OrgMemberRole shared map[string]bool - source map[string]uint64 } func (s *stubKBShareForGuard) CheckTenantKBPermission(_ context.Context, kbID string, _ uint64, _ types.TenantRole) (types.OrgMemberRole, bool, error) { @@ -50,48 +49,57 @@ func (s *stubKBShareForGuard) CheckTenantKBPermission(_ context.Context, kbID st } func (s *stubKBShareForGuard) GetKBSourceTenant(_ context.Context, kbID string) (uint64, error) { - if v, ok := s.source[kbID]; ok { - return v, nil - } - return 0, errors.New("not found") + panic("the loaded KB already carries its authoritative tenant") } func (s *stubKBShareForGuard) ShareKnowledgeBase(context.Context, string, string, string, uint64, types.OrgMemberRole) (*types.KnowledgeBaseShare, error) { panic("not implemented") } + func (s *stubKBShareForGuard) UpdateSharePermission(context.Context, string, types.OrgMemberRole, string, uint64) error { panic("not implemented") } + func (s *stubKBShareForGuard) RemoveShare(context.Context, string, string, uint64) error { panic("not implemented") } + func (s *stubKBShareForGuard) ListSharesByKnowledgeBase(context.Context, string, uint64) ([]*types.KnowledgeBaseShare, error) { panic("not implemented") } + func (s *stubKBShareForGuard) ListSharesByOrganization(context.Context, string) ([]*types.KnowledgeBaseShare, error) { panic("not implemented") } + func (s *stubKBShareForGuard) ListSharedKnowledgeBases(context.Context, uint64, types.TenantRole) ([]*types.SharedKnowledgeBaseInfo, error) { panic("not implemented") } + func (s *stubKBShareForGuard) ListSharedKnowledgeBasesInOrganization(context.Context, string, uint64, types.TenantRole) ([]*types.OrganizationSharedKnowledgeBaseItem, error) { panic("not implemented") } + func (s *stubKBShareForGuard) ListSharedKnowledgeBaseIDsByOrganizations(context.Context, []string, uint64) (map[string][]string, error) { panic("not implemented") } + func (s *stubKBShareForGuard) GetShare(context.Context, string) (*types.KnowledgeBaseShare, error) { panic("not implemented") } + func (s *stubKBShareForGuard) GetShareByKBAndOrg(context.Context, string, string) (*types.KnowledgeBaseShare, error) { panic("not implemented") } + func (s *stubKBShareForGuard) HasTenantKBPermission(context.Context, string, uint64, types.TenantRole, types.OrgMemberRole) (bool, error) { panic("not implemented") } + func (s *stubKBShareForGuard) CountSharesByKnowledgeBaseIDs(context.Context, []string) (map[string]int64, error) { panic("not implemented") } + func (s *stubKBShareForGuard) CountByOrganizations(context.Context, []string) (map[string]int64, error) { panic("not implemented") } @@ -121,36 +129,47 @@ func (s *stubAgentShareForGuard) TenantCanAccessKBViaSomeSharedAgent(_ context.C func (s *stubAgentShareForGuard) ShareAgent(context.Context, string, string, string, uint64, types.OrgMemberRole) (*types.AgentShare, error) { panic("not implemented") } + func (s *stubAgentShareForGuard) RemoveShare(context.Context, string, string, uint64) error { panic("not implemented") } + func (s *stubAgentShareForGuard) ListSharesByAgent(context.Context, string, uint64) ([]*types.AgentShare, error) { panic("not implemented") } + func (s *stubAgentShareForGuard) ListSharesByOrganization(context.Context, string) ([]*types.AgentShare, error) { panic("not implemented") } + func (s *stubAgentShareForGuard) ListSharedAgents(context.Context, uint64, types.TenantRole) ([]*types.SharedAgentInfo, error) { panic("not implemented") } + func (s *stubAgentShareForGuard) ListSharedAgentsInOrganization(context.Context, string, uint64, types.TenantRole) ([]*types.OrganizationSharedAgentItem, error) { panic("not implemented") } + func (s *stubAgentShareForGuard) ListSharedAgentsInOrganizations(context.Context, []string, uint64, types.TenantRole) (map[string][]*types.OrganizationSharedAgentItem, error) { panic("not implemented") } + func (s *stubAgentShareForGuard) SetSharedAgentDisabledByMe(context.Context, uint64, string, uint64, bool) error { panic("not implemented") } + func (s *stubAgentShareForGuard) GetShare(context.Context, string) (*types.AgentShare, error) { panic("not implemented") } + func (s *stubAgentShareForGuard) GetShareByAgentAndOrg(context.Context, string, string) (*types.AgentShare, error) { panic("not implemented") } + func (s *stubAgentShareForGuard) GetShareByAgentIDForTenant(context.Context, uint64, string, uint64) (*types.AgentShare, error) { panic("not implemented") } + func (s *stubAgentShareForGuard) CountByOrganizations(context.Context, []string) (map[string]int64, error) { panic("not implemented") } @@ -277,7 +296,6 @@ func TestRequireKBAccess_SharedKB_RewritesTenantContext(t *testing.T) { share := &stubKBShareForGuard{ permission: map[string]types.OrgMemberRole{"kb-shared": types.OrgRoleEditor}, shared: map[string]bool{"kb-shared": true}, - source: map[string]uint64{"kb-shared": 200}, } _, c := runGuard(t, 100, "kb-shared", types.OrgRoleEditor, @@ -297,7 +315,6 @@ func TestRequireKBAccess_SharedKB_PermissionBelowMin_Aborts(t *testing.T) { share := &stubKBShareForGuard{ permission: map[string]types.OrgMemberRole{"kb-shared": types.OrgRoleViewer}, shared: map[string]bool{"kb-shared": true}, - source: map[string]uint64{"kb-shared": 200}, } _, c := runGuard(t, 100, "kb-shared", types.OrgRoleEditor, // require Editor @@ -500,7 +517,6 @@ func TestRequireKBAccess_Forbidden_FailOpenWhenRBACDisabled(t *testing.T) { share := &stubKBShareForGuard{ permission: map[string]types.OrgMemberRole{"kb-shared": types.OrgRoleViewer}, shared: map[string]bool{"kb-shared": true}, - source: map[string]uint64{"kb-shared": 200}, } kbsvc := &stubKBLookup{kbs: map[string]*types.KnowledgeBase{ "kb-shared": {ID: "kb-shared", TenantID: 200}, diff --git a/internal/middleware/rbac.go b/internal/middleware/rbac.go index 2dc314ac..746783bc 100644 --- a/internal/middleware/rbac.go +++ b/internal/middleware/rbac.go @@ -6,6 +6,7 @@ import ( "net/http" "sync" + "github.com/Tencent/WeKnora/internal/application/access" "github.com/Tencent/WeKnora/internal/config" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" @@ -20,7 +21,7 @@ import ( // own 404 — middleware-level 403 would hide real "URL is wrong" failures // behind a permissions error, which breaks client diagnostics and // operator dashboards. -var ErrResourceNotFound = errors.New("rbac: resource not found") +var ErrResourceNotFound = access.ErrResourceNotFound // CreatorLookup resolves the creator user ID for the resource targeted // by the current request, based on whatever is on the gin.Context (URL @@ -75,7 +76,7 @@ func RequireRole(min types.TenantRole, cfg *config.Config) gin.HandlerFunc { c.Next() return } - role := types.TenantRoleFromContext(ctx) + role := types.CallerFromContext(ctx).Role if role.HasPermission(min) { c.Next() return @@ -100,7 +101,7 @@ func RequireRole(min types.TenantRole, cfg *config.Config) gin.HandlerFunc { // dedup inside the service so probing clients can't fill the // table. if svc := AuditServiceFromContext(c); svc != nil { - tenantID, _ := types.TenantIDFromContext(ctx) + tenantID := types.CallerFromContext(ctx).TenantID _ = svc.LogDenied(ctx, c, tenantID, uid, string(role), min) } c.JSON(http.StatusForbidden, gin.H{ @@ -175,7 +176,7 @@ func RequireSystemAdmin(cfg *config.Config) gin.HandlerFunc { uid, c.Request.URL.Path) // Durable audit row for the reject — same dedup as RequireRole. if svc := AuditServiceFromContext(c); svc != nil { - tenantID, _ := types.TenantIDFromContext(ctx) + tenantID := types.CallerFromContext(ctx).TenantID _ = svc.LogDenied(ctx, c, tenantID, uid, "user", "system_admin") } c.JSON(http.StatusForbidden, gin.H{ @@ -194,7 +195,8 @@ func RequireSystemAdmin(cfg *config.Config) gin.HandlerFunc { // is responsible for translating the URL into the resource's creator // user ID (see CreatorLookup for the return-value contract). // -// Decision order: +// API-key principals skip this human ownership gate; route capabilities and +// allowed KB scope remain mandatory. For human callers, decision order is: // 1. role >= min -> allow without running lookup. // 2. cross-tenant superuser -> allow without running lookup. // 3. enforcement off -> log, allow without running lookup. This is the @@ -213,81 +215,35 @@ func RequireOwnershipOrRole(min types.TenantRole, lookup CreatorLookup, cfg *con warnOnNilConfig(cfg) return func(c *gin.Context) { ctx := c.Request.Context() - // API-key principals are authorized solely by the APIKeyGate. - // Ownership ("creator OR Admin+") is a human concept that cannot - // apply to a machine principal (its synthetic system-user never - // matches creator_id), so short-circuit here. KB-scope for API - // keys is still enforced by the KBAccess guards + handler checks. - if _, ok := types.TenantAPIKeyScopeFromContext(ctx); ok { - c.Next() - return + request := ownershipRequestFromContext(ctx, cfg) + var resolveCreator func() (string, error) + if lookup != nil { + resolveCreator = func() (string, error) { return lookup(c) } } - role := types.TenantRoleFromContext(ctx) - - // 1. Fast path: role meets the bar. - if role.HasPermission(min) { - c.Next() - return - } - - // 2. Cross-tenant superuser bypass — same reasoning as RequireRole. - if IsCrossTenantSuperuser(ctx, cfg) { - c.Next() - return - } - - uid, _ := types.UserIDFromContext(ctx) - - // 3. Fail-open shortcut: when enforcement is off, do NOT run the - // lookup. Running it would add a hidden DB roundtrip on every - // mutating request during the rollout window — see #1318 review. - if !rbacEnforcementEnabled(cfg) { - logger.Warnf(ctx, - "[rbac] ownership/role would be checked (enforcement off, lookup skipped): "+ - "user=%s have=%s need=%s path=%s", - uid, role, min, c.Request.URL.Path) - c.Next() - return - } - - creator, err := lookup(c) + decision, err := access.CheckOwnershipOrRole(request, min, resolveCreator) + logOwnershipDecision(ctx, request, min, decision, err, c.Request.URL.Path) switch { - case errors.Is(err, ErrResourceNotFound): - // 4. Hand off to the handler so the client sees a real 404 - // rather than a fake "no permission" 403. + case err == nil, errors.Is(err, ErrResourceNotFound): + // A missing resource remains the handler's responsibility so it + // can produce its existing 404 response. c.Next() return - case err != nil: - // 5. Genuine failure — surface it as 5xx so monitoring catches it. + case errors.Is(err, ErrOwnershipForbidden) && !decision.LookupFailed: + if svc := AuditServiceFromContext(c); svc != nil { + tenantID := types.CallerFromContext(ctx).TenantID + _ = svc.LogDenied(ctx, c, tenantID, request.UserID, string(request.Role), min) + } + c.JSON(http.StatusForbidden, gin.H{ + "error": "Forbidden: must own the resource or have the required role", + }) + default: logger.Errorf(ctx, "[rbac] creator lookup failed: user=%s path=%s err=%v", - uid, c.Request.URL.Path, err) + request.UserID, c.Request.URL.Path, err) c.JSON(http.StatusServiceUnavailable, gin.H{ "error": "Service Unavailable: cannot verify resource ownership", }) - c.Abort() - return } - - // 6. Ownership match wins even when role is below min — that's the - // whole point: Contributors can edit their own resources. - if creator != "" && creator == uid { - c.Next() - return - } - - // 7-8. Tenant-owned (creator=="") or non-creator with insufficient role. - logger.Warnf(ctx, - "[rbac] ownership/role insufficient: user=%s have=%s need=%s creator=%q path=%s", - uid, role, min, creator, c.Request.URL.Path) - // Same durable audit hook as RequireRole — subject to dedup. - if svc := AuditServiceFromContext(c); svc != nil { - tenantID, _ := types.TenantIDFromContext(ctx) - _ = svc.LogDenied(ctx, c, tenantID, uid, string(role), min) - } - c.JSON(http.StatusForbidden, gin.H{ - "error": "Forbidden: must own the resource or have the required role", - }) c.Abort() } } @@ -303,61 +259,55 @@ func rbacEnforcementEnabled(cfg *config.Config) bool { // ErrOwnershipForbidden is returned by EvaluateOwnershipOrRole when the // caller is neither the resource creator nor meets the minimum role. -var ErrOwnershipForbidden = errors.New("rbac: ownership or role insufficient") +var ErrOwnershipForbidden = access.ErrOwnershipForbidden -// EvaluateOwnershipOrRole applies the same decision matrix as -// RequireOwnershipOrRole for handlers that resolve creator_id out-of-band -// (e.g. KB id carried in a JSON body rather than a URL param). -// -// Returns nil when access is allowed. ErrResourceNotFound means the -// handler should issue its own 404. ErrOwnershipForbidden maps to 403. -// Any other error is a transient lookup failure (503). +// EvaluateOwnershipOrRole is the handler adapter for the same lazy decision +// used by RequireOwnershipOrRole. Body/query-based callers supply a closure so +// ownership lookup is skipped on exactly the same fast paths as URL routes. +// Not-found, forbidden and lookup failures remain distinct; each handler maps +// them to its existing response format and status. func EvaluateOwnershipOrRole( ctx context.Context, cfg *config.Config, min types.TenantRole, - creatorID string, - lookupErr error, + lookup func() (string, error), ) error { - // API-key principals are authorized solely by the APIKeyGate (route - // policy) plus the KB allow-list handlers enforce separately - // (requireTenantAPIKeyKnowledgeBase(s)). Ownership ("creator OR Admin+") - // is a human concept that never applies to a machine principal, so - // short-circuit here — exactly as RequireOwnershipOrRole does for the - // middleware form. Without this, a scoped key (synthesized as Viewer for - // legacy-guard compatibility) would be 403'd by the body-carried-KB - // ownership checks even though the gate + allow-list already admitted it. - if _, ok := types.TenantAPIKeyScopeFromContext(ctx); ok { - return nil + request := ownershipRequestFromContext(ctx, cfg) + decision, err := access.CheckOwnershipOrRole(request, min, lookup) + logOwnershipDecision(ctx, request, min, decision, err, "") + return err +} + +func ownershipRequestFromContext(ctx context.Context, cfg *config.Config) access.OwnershipRequest { + caller := types.CallerFromContext(ctx) + _, apiKey := types.TenantAPIKeyScopeFromContext(ctx) + return access.OwnershipRequest{ + UserID: caller.UserID, + Role: caller.Role, + APIKey: apiKey, + CrossTenantSuperuser: IsCrossTenantSuperuser(ctx, cfg), + Enforce: rbacEnforcementEnabled(cfg), } - role := types.TenantRoleFromContext(ctx) - if role.HasPermission(min) { - return nil - } - if IsCrossTenantSuperuser(ctx, cfg) { - return nil - } - if !rbacEnforcementEnabled(cfg) { - uid, _ := types.UserIDFromContext(ctx) +} + +func logOwnershipDecision( + ctx context.Context, + request access.OwnershipRequest, + required types.TenantRole, + decision access.OwnershipDecision, + err error, + path string, +) { + switch { + case decision.EnforcementSkipped: logger.Warnf(ctx, - "[rbac] ownership/role would be checked (enforcement off, lookup skipped): user=%s have=%s need=%s", - uid, role, min) - return nil + "[rbac] ownership/role would be checked (enforcement off, lookup skipped): user=%s have=%s need=%s path=%s", + request.UserID, request.Role, required, path) + case errors.Is(err, ErrOwnershipForbidden) && !decision.LookupFailed: + logger.Warnf(ctx, + "[rbac] ownership/role insufficient: user=%s have=%s need=%s creator=%q path=%s", + request.UserID, request.Role, required, decision.CreatorID, path) } - if errors.Is(lookupErr, ErrResourceNotFound) { - return ErrResourceNotFound - } - if lookupErr != nil { - return lookupErr - } - uid, _ := types.UserIDFromContext(ctx) - if creatorID != "" && creatorID == uid { - return nil - } - logger.Warnf(ctx, - "[rbac] ownership/role insufficient: user=%s have=%s need=%s creator=%q", - uid, role, min, creatorID) - return ErrOwnershipForbidden } // isCrossTenantSuperuser was moved to access.go (renamed to diff --git a/internal/middleware/rbac_api_key_shortcircuit_test.go b/internal/middleware/rbac_api_key_shortcircuit_test.go index f767ced3..573a5489 100644 --- a/internal/middleware/rbac_api_key_shortcircuit_test.go +++ b/internal/middleware/rbac_api_key_shortcircuit_test.go @@ -108,7 +108,10 @@ func TestEvaluateOwnershipOrRole_ShortCircuitsAPIKey(t *testing.T) { ctx = context.WithValue(ctx, types.TenantRoleContextKey, types.TenantRoleViewer) if err := EvaluateOwnershipOrRole(ctx, cfgRBAC(true), - types.TenantRoleAdmin, "some-other-human-user", nil); err != nil { + types.TenantRoleAdmin, func() (string, error) { + t.Fatal("API-key principals must not query resource ownership") + return "some-other-human-user", nil + }); err != nil { t.Fatalf("API-key principal should short-circuit EvaluateOwnershipOrRole, got %v", err) } } @@ -118,7 +121,9 @@ func TestEvaluateOwnershipOrRole_ShortCircuitsAPIKey(t *testing.T) { func TestEvaluateOwnershipOrRole_JWTViewerStillDenied(t *testing.T) { ctx := context.WithValue(context.Background(), types.TenantRoleContextKey, types.TenantRoleViewer) err := EvaluateOwnershipOrRole(ctx, cfgRBAC(true), - types.TenantRoleAdmin, "some-other-human-user", nil) + types.TenantRoleAdmin, func() (string, error) { + return "some-other-human-user", nil + }) if err == nil { t.Fatal("JWT Viewer with foreign creator must be denied by EvaluateOwnershipOrRole") } diff --git a/internal/middleware/rbac_audit_test.go b/internal/middleware/rbac_audit_test.go index a4e295bd..37d0f321 100644 --- a/internal/middleware/rbac_audit_test.go +++ b/internal/middleware/rbac_audit_test.go @@ -2,6 +2,8 @@ package middleware import ( "context" + "errors" + "fmt" "net/http" "net/http/httptest" "sync" @@ -135,3 +137,58 @@ func TestRequireRole_NilAuditServiceDoesNotPanic(t *testing.T) { t.Fatalf("expected 403 even with nil audit service, got %d", w.Code) } } + +func TestOwnershipAuditOnlyRecordsPolicyDenials(t *testing.T) { + for _, tt := range []struct { + name string + creator string + lookupErr error + disabled bool + status int + auditCalls int + }{ + {name: "creator allowed", creator: "u1", status: http.StatusOK}, + {name: "policy denied", creator: "other", status: http.StatusForbidden, auditCalls: 1}, + { + name: "not found passes to handler", + lookupErr: fmt.Errorf("load: %w", + ErrResourceNotFound), + status: http.StatusOK, + }, + + { + name: "database failure", + lookupErr: errors.New("database unavailable"), + status: http.StatusServiceUnavailable, + }, + + { + name: "lookup returned denial sentinel", + lookupErr: ErrOwnershipForbidden, + status: http.StatusServiceUnavailable, + }, + + {name: "enforcement disabled", disabled: true, creator: "other", status: http.StatusOK}, + } { + t.Run(tt.name, func(t *testing.T) { + audit := &stubDenyAudit{} + lookup := func(*gin.Context) (string, error) { + if tt.disabled { + t.Fatal("disabled enforcement must skip lookup") + } + return tt.creator, tt.lookupErr + } + w := auditableHarness(t, types.TenantRoleContributor, "u1", 7, audit, + RequireOwnershipOrRole(types.TenantRoleAdmin, lookup, cfgRBAC(!tt.disabled))) + if w.Code != tt.status || len(audit.calls) != tt.auditCalls { + t.Fatalf( + "status=%d audit calls=%d; want status=%d audit calls=%d", + w.Code, + len(audit.calls), + tt.status, + tt.auditCalls, + ) + } + }) + } +} diff --git a/internal/router/files.go b/internal/router/files.go index 5ad7c876..a00b58c8 100644 --- a/internal/router/files.go +++ b/internal/router/files.go @@ -2,23 +2,25 @@ package router import ( "context" + "errors" "io" - "mime" "net/http" "os" "path/filepath" "strconv" "strings" + "github.com/Tencent/WeKnora/internal/application/access" filesvc "github.com/Tencent/WeKnora/internal/application/service/file" - "github.com/gin-gonic/gin" - "github.com/Tencent/WeKnora/internal/config" + apperrors "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/filetransport" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/middleware" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" secutils "github.com/Tencent/WeKnora/internal/utils" + "github.com/gin-gonic/gin" ) // files.go hosts every file-proxy surface the router exposes: @@ -42,247 +44,11 @@ type getRouteRegistrar interface { GET(string, ...gin.HandlerFunc) gin.IRoutes } -// messageFileLookup is the narrow message-service surface needed by the -// message-scoped file proxy. Keeping it small makes the authorization boundary -// independently testable. -type messageFileLookup interface { - GetMessage(ctx context.Context, sessionID, messageID string) (*types.Message, error) -} - -// sharedAgentFileLookup verifies that a source workspace's agent is still -// shared to the caller. Revoking the share therefore also revokes historical -// message-file access. -type sharedAgentFileLookup interface { - GetSharedAgentForTenant( - ctx context.Context, - tenantID uint64, - callerTenantRole types.TenantRole, - agentID string, - sourceTenantID ...uint64, - ) (*types.CustomAgent, error) -} - -// kbSharePermissionGuard is the org-share surface the message-scoped proxy -// needs for its shared-KB fallback. -type kbSharePermissionGuard interface { - HasTenantKBPermission( - ctx context.Context, - kbID string, - callerTenantID uint64, - callerTenantRole types.TenantRole, - requiredRole types.OrgMemberRole, - ) (bool, error) -} - -// kbTenantLookup resolves knowledge bases without a tenant filter, so a -// message's persisted retrieval evidence can be mapped back to the knowledge -// bases the chunks were retrieved from. -type kbTenantLookup interface { - GetKnowledgeBasesByIDsOnly(ctx context.Context, ids []string) ([]*types.KnowledgeBase, error) -} - -// knowledgeOwnerLookup resolves a knowledge entry to its knowledge base for -// message references that predate the denormalized knowledge_base_id field. -type knowledgeOwnerLookup interface { - GetKnowledgeByIDOnly(ctx context.Context, id string) (*types.Knowledge, error) -} - -// messageKBShareAuthorizer bundles the read-only lookups behind the -// message proxy's org-shared-KB fallback. A nil ShareGuard or KBs lookup -// disables the fallback, which keeps older call sites and tests on the -// shared-agent-only behavior. A nil Knowledges lookup only disables the -// pre-denormalization KnowledgeID path. -type messageKBShareAuthorizer struct { - ShareGuard kbSharePermissionGuard - KBs kbTenantLookup - Knowledges knowledgeOwnerLookup -} - -// resourceAccessibleViaSharedKB reports whether the message's persisted -// retrieval evidence proves the resource came from an org-shared KB the -// caller may read. This is the fallback for replies whose agent belongs to -// the caller's own workspace while the retrieved resources belong to the -// workspace that shared the knowledge base (#3022). -// -// Evidence required from one retrieval record: a canonical resource:// -// handle (not a prefix of a longer token) appears in the chunk text or -// image_info, its knowledge base belongs to the resource's tenant, and -// that KB is org-shared to the caller with at least viewer permission. -// Smart-reasoning turns persist that evidence on AgentSteps when -// KnowledgeReferences was never filled. Any lookup failure fails closed. -func (a messageKBShareAuthorizer) resourceAccessibleViaSharedKB( - ctx context.Context, - message *types.Message, - resource *types.StoredResource, - callerTenantID uint64, - callerTenantRole types.TenantRole, -) bool { - if a.ShareGuard == nil || a.KBs == nil || message == nil || resource == nil { - return false - } - handle, ok := types.ParseResourcePath(types.BuildResourcePath(resource.Handle)) - if !ok { - return false - } - - kbIDs := a.collectSharedKBEvidenceIDs(ctx, message, handle) - if len(kbIDs) == 0 { - return false - } - - kbs, err := a.KBs.GetKnowledgeBasesByIDsOnly(ctx, kbIDs) - if err != nil { - return false - } - for _, kb := range kbs { - if kb == nil || kb.TenantID != resource.TenantID { - continue - } - shared, err := a.ShareGuard.HasTenantKBPermission( - ctx, kb.ID, callerTenantID, callerTenantRole, types.OrgRoleViewer) - if err == nil && shared { - return true - } - } - return false -} - -func (a messageKBShareAuthorizer) collectSharedKBEvidenceIDs( - ctx context.Context, - message *types.Message, - handle string, -) []string { - seenKB := make(map[string]bool) - seenKnowledge := make(map[string]bool) - var kbIDs, knowledgeIDs []string - addKB := func(id string) { - if id == "" || seenKB[id] { - return - } - seenKB[id] = true - kbIDs = append(kbIDs, id) - } - addKnowledge := func(id string) { - if id == "" || seenKnowledge[id] { - return - } - seenKnowledge[id] = true - knowledgeIDs = append(knowledgeIDs, id) - } - - for _, ref := range message.KnowledgeReferences { - if !searchResultHasResourceHandle(ref, handle) { - continue - } - if ref.KnowledgeBaseID != "" { - addKB(ref.KnowledgeBaseID) - continue - } - addKnowledge(ref.KnowledgeID) - } - collectKBEvidenceFromValue(message.AgentSteps, handle, "", "", addKB, addKnowledge) - - if a.Knowledges != nil { - for _, knowledgeID := range knowledgeIDs { - knowledge, err := a.Knowledges.GetKnowledgeByIDOnly(ctx, knowledgeID) - if err != nil || knowledge == nil { - continue - } - addKB(knowledge.KnowledgeBaseID) - } - } - return kbIDs -} - -func searchResultHasResourceHandle(ref *types.SearchResult, handle string) bool { - if ref == nil { - return false - } - return textHasResourceHandle(ref.Content, handle) || - textHasResourceHandle(ref.MatchedContent, handle) || - textHasResourceHandle(ref.ImageInfo, handle) -} - -func textHasResourceHandle(text, handle string) bool { - if text == "" || handle == "" { - return false - } - want := types.BuildResourcePath(handle) - for _, ref := range types.ScanResourceReferences(text) { - if ref == want { - return true - } - } - return false -} - -func collectKBEvidenceFromValue( - v interface{}, - handle, kbID, knowledgeID string, - addKB, addKnowledge func(string), -) { - switch val := v.(type) { - case string: - if !textHasResourceHandle(val, handle) { - return - } - if kbID != "" { - addKB(kbID) - return - } - addKnowledge(knowledgeID) - case types.AgentSteps: - for _, step := range val { - collectKBEvidenceFromValue(step, handle, kbID, knowledgeID, addKB, addKnowledge) - } - case types.AgentStep: - collectKBEvidenceFromValue(val.ToolCalls, handle, kbID, knowledgeID, addKB, addKnowledge) - case []types.ToolCall: - for _, call := range val { - collectKBEvidenceFromValue(call, handle, kbID, knowledgeID, addKB, addKnowledge) - } - case types.ToolCall: - if val.Result != nil { - collectKBEvidenceFromValue(val.Result.Output, handle, kbID, knowledgeID, addKB, addKnowledge) - collectKBEvidenceFromValue(val.Result.Data, handle, kbID, knowledgeID, addKB, addKnowledge) - } - case map[string]interface{}: - nextKB := firstNonEmptyString( - stringFromAnyMap(val, "knowledge_base_id"), - stringFromAnyMap(val, "knowledge_base"), - kbID, - ) - nextKnowledge := firstNonEmptyString(stringFromAnyMap(val, "knowledge_id"), knowledgeID) - for _, nested := range val { - collectKBEvidenceFromValue(nested, handle, nextKB, nextKnowledge, addKB, addKnowledge) - } - case []interface{}: - for _, item := range val { - collectKBEvidenceFromValue(item, handle, kbID, knowledgeID, addKB, addKnowledge) - } - case []map[string]interface{}: - for _, item := range val { - collectKBEvidenceFromValue(item, handle, kbID, knowledgeID, addKB, addKnowledge) - } - } -} - -func stringFromAnyMap(m map[string]interface{}, key string) string { - if m == nil { - return "" - } - s, _ := m[key].(string) - return strings.TrimSpace(s) -} - -func firstNonEmptyString(values ...string) string { - for _, value := range values { - if strings.TrimSpace(value) != "" { - return strings.TrimSpace(value) - } - } - return "" -} +type ( + messageFileLookup = access.MessageFileLookup + sharedAgentFileLookup = access.SharedAgentFileLookup + messageKBShareAuthorizer = access.MessageKBShareAuthorizer +) // localStorageBaseDir resolves LOCAL_STORAGE_BASE_DIR with the container // default. @@ -372,68 +138,17 @@ func resolveFileService( return filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, absDir) } -// resolveTenantFileServiceWithFallback is resolveFileService plus the -// fallback rule shared by /files and the KB-scoped proxy: when tenant-level -// resolution fails and the requested provider matches the process-global -// STORAGE_TYPE, serve through the global file service instead of failing. -// Returns ok=false (already logged) when no service can be resolved; the -// caller decides the HTTP status. -func resolveTenantFileServiceWithFallback( - ctx context.Context, - logTag string, - tenant *types.Tenant, - backendID, provider, absDir string, - storageResolver interfaces.StorageBackendResolver, - globalFileService interfaces.FileService, -) (fileSvc interfaces.FileService, resolvedProvider string, ok bool) { - var err error - if storageResolver != nil { - fileSvc, resolvedProvider, err = storageResolver.ResolveFileService(ctx, tenant, backendID, provider, absDir) - } else if tenant.StorageEngineConfig != nil { - fileSvc, resolvedProvider, err = filesvc.NewFileServiceFromStorageConfig(provider, tenant.StorageEngineConfig, absDir) - } else { - err = http.ErrMissingFile - } - if err == nil { - return fileSvc, resolvedProvider, true - } - - globalStorageType := strings.ToLower(strings.TrimSpace(os.Getenv("STORAGE_TYPE"))) - if globalStorageType == "" { - globalStorageType = "local" - } - if provider == globalStorageType && globalFileService != nil { - logger.Warnf(ctx, "[Router] %s tenant storage config missing or invalid, fallback to global file service: tenant_id=%d provider=%s err=%v", - logTag, tenant.ID, provider, err) - return globalFileService, globalStorageType, true - } - logger.Warnf(ctx, "[Router] %s resolve file service failed without fallback: tenant_id=%d provider=%s global_storage_type=%s err=%v", - logTag, tenant.ID, provider, globalStorageType, err) - return nil, "", false -} - // streamStoredFile writes the shared success response of every file proxy: // safe content type, nosniff, disposition for non-inline types, the route's // cache policy, then the body (skipped for HEAD). Closes reader. func streamStoredFile(c *gin.Context, reader io.ReadCloser, contentType string, inline bool, cacheControl, logTag string, fileNames ...string) { - defer reader.Close() - c.Header("Content-Type", contentType) - c.Header("X-Content-Type-Options", "nosniff") - disposition := "inline" - if !inline { - disposition = "attachment" - } + name := "" if len(fileNames) > 0 { - c.Header("Content-Disposition", mime.FormatMediaType(disposition, map[string]string{"filename": filepath.Base(fileNames[0])})) - } else if !inline { - c.Header("Content-Disposition", disposition) + name = fileNames[0] } - c.Header("Cache-Control", cacheControl) - c.Status(http.StatusOK) - if c.Request.Method == http.MethodHead { - return - } - if _, err := io.Copy(c.Writer, reader); err != nil { + if err := filetransport.Serve(c.Writer, c.Request, reader, filetransport.Options{ + Filename: name, Download: !inline, ContentType: contentType, CacheControl: cacheControl, + }); err != nil { logger.Warnf(c.Request.Context(), "[Router] %s write response failed: %v", logTag, err) } } @@ -486,7 +201,7 @@ func newFileServeHandler( } backendID, provider := parseStorageTarget(filePath) - fileSvc, resolvedProvider, ok := resolveTenantFileServiceWithFallback( + fileSvc, resolvedProvider, ok := filesvc.ResolveTenantFileServiceWithFallback( c.Request.Context(), "/files", tenant, backendID, provider, absDir, storageResolver, globalFileService) if !ok { c.Status(http.StatusBadRequest) @@ -591,9 +306,6 @@ func serveResourceGrants( fileName = resource.PhysicalPath } contentType, inline := secutils.SafeContentTypeByFilename(fileName) - if resource.MimeType != "" && inline { - contentType = resource.MimeType - } streamStoredFile(c, reader, contentType, inline, "private, max-age=300", "resource grant (resource_id="+resource.ID+")", fileName) } @@ -603,13 +315,10 @@ func serveResourceGrants( // serveKBScopedFiles registers the KB-scoped file proxy used to render images // embedded in a knowledge base's content (chunks / wiki pages). Unlike the -// tenant-scoped /files route — which enforces file_path.tenant == caller.tenant -// and therefore cannot serve objects owned by another tenant — this route is -// gated by RequireKBAccess. That guard resolves org-shared / agent-visible KBs -// and rewrites the request context's tenant ID to the KB's *owner* (source) -// tenant, so images stored under the owner tenant (local:///exports/...) -// become reachable by tenants that legitimately share the KB, while still -// enforcing that the requested path belongs to that owner tenant. +// tenant-scoped /files route, this route consumes RequireKBAccess's exact KB +// grant. The file must be owned by that KB's tenant and have a live document +// binding or an exact reference in the KB's chunks/wiki pages. Only then does +// storage execute under the owner tenant. // // Route: // - GET /api/v1/knowledge-bases/:id/files?file_path= @@ -622,14 +331,8 @@ func serveKBScopedFiles( resourceCatalogs ...interfaces.ResourceCatalog, ) { logger.Infof(context.Background(), "[Router] Serving KB-scoped files from /knowledge-bases/:id/files") - // API-key access mirrors /files: KB-restricted keys are denied (an - // arbitrary file_path under the KB owner tenant cannot be bounded to a - // key's allow-list), while full-access and tenant-wide retrieve keys pass - // — KBAccessRead still confines them to KBs they may read (own / - // org-shared / agent-visible), exactly as it does for a JWT Viewer. The - // route is declared to the gate with the retrieve policy so it is reachable - // at all; AllowFileServeAPIKey then applies the stricter not-KB-restricted - // constraint. + // Preserve the existing file-route API-key policy: KB-restricted keys are + // denied; full-access and tenant-wide retrieve keys still need KBAccessRead. g.apiKeyRoute(r, http.MethodGet, "/knowledge-bases/:id/files", apiKeyRetrieve(apiKeyFullAccess()), middleware.AllowFileServeAPIKey(), @@ -644,24 +347,6 @@ func serveKBScopedFiles( ) } -// newKBScopedFileServeHandler builds the handler backing serveKBScopedFiles. -// The effective (owner) tenant is taken from the request context, which -// RequireKBAccess has already rewritten to the KB's source tenant. The owner -// tenant's storage config is loaded via TenantService so the file is fetched -// from the backend that actually holds it — the caller's own storage config is -// irrelevant here. -func newKBScopedFileServeHandler( - tenantService interfaces.TenantService, - globalFileService interfaces.FileService, - resolvers ...interfaces.StorageBackendResolver, -) gin.HandlerFunc { - var storageResolver interfaces.StorageBackendResolver - if len(resolvers) > 0 { - storageResolver = resolvers[0] - } - return newKBScopedFileServeHandlerWithResources(tenantService, globalFileService, storageResolver, nil) -} - func firstResourceCatalog(catalogs []interfaces.ResourceCatalog) interfaces.ResourceCatalog { if len(catalogs) == 0 { return nil @@ -675,71 +360,32 @@ func newKBScopedFileServeHandlerWithResources( storageResolver interfaces.StorageBackendResolver, resourceCatalog interfaces.ResourceCatalog, ) gin.HandlerFunc { - absDir := localStorageAbsDir() - return func(c *gin.Context) { - ctx := c.Request.Context() - - filePath, ok := requireFilePathQuery(c) + reference, ok := requireFilePathQuery(c) if !ok { return } - - // RequireKBAccess rewrote the request context tenant ID to the KB's - // owner (source) tenant for shared KBs; for own KBs it equals the - // caller's tenant. Either way it is the tenant that owns this KB's - // storage objects, so the requested path must belong to it. - ownerTenantID, ok := types.TenantIDFromContext(ctx) - if !ok || ownerTenantID == 0 { - c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized: workspace context missing"}) + grant, _ := middleware.KBAccessFromContext(c) + bindings, _ := resourceCatalog.(interfaces.KBResourceLookup) + file, err := access.ResolveKBFile( + c.Request.Context(), + grant, + c.Param("id"), + reference, + resourceCatalog, + bindings, + ) + if fileAccessError(c, err) { return } - filePath, _, ok = resolveCatalogResource(c, resourceCatalog, filePath, ownerTenantID) - if !ok { - return - } - - if err := secutils.ValidateKBScopedStoragePath(filePath, ownerTenantID); err != nil { - logger.Warnf(ctx, "[Router] /knowledge-bases/:id/files denied path not allowed for KB proxy: owner_tenant_id=%d file_path=%q err=%v", - ownerTenantID, filePath, err) - c.JSON(http.StatusForbidden, gin.H{"error": "forbidden: file path not accessible"}) - return - } - - tenant, err := tenantService.GetTenantByID(ctx, ownerTenantID) - if err != nil || tenant == nil { - logger.Warnf(ctx, "[Router] /knowledge-bases/:id/files owner tenant lookup failed: owner_tenant_id=%d err=%v", - ownerTenantID, err) - c.Status(http.StatusNotFound) - return - } - - backendID, provider := parseStorageTarget(filePath) - fileSvc, resolvedProvider, ok := resolveTenantFileServiceWithFallback( - ctx, "/knowledge-bases/:id/files", tenant, backendID, provider, absDir, storageResolver, globalFileService) - if !ok { - c.Status(http.StatusBadRequest) - return - } - - reader, err := fileSvc.GetFile(ctx, filePath) - if err != nil { - logger.Warnf(ctx, "[Router] /knowledge-bases/:id/files get file failed: owner_tenant_id=%d provider=%s path=%q err=%v", - ownerTenantID, resolvedProvider, filePath, err) - c.Status(http.StatusNotFound) - return - } - - contentType, inline := secutils.SafeContentTypeByFilename(filePath) - // Cross-tenant shared content — keep it private so shared proxies / - // CDNs do not cache one tenant's view for another. - streamStoredFile(c, reader, contentType, inline, "private, max-age=86400", "/knowledge-bases/:id/files", filePath) + serveAuthorizedFile(c, file, tenantService, globalFileService, storageResolver, "KB files") } } // newMessageScopedFileServeHandler serves resources rendered inside one // assistant message. The message service first proves that the caller owns the -// containing session. For cross-workspace resources we then require either the +// containing session, and the persisted message must reference the exact file. +// For cross-workspace resources we then require either the // message's agent to still be shared from the resource-owning workspace, or — // when the reply was produced by the caller's own agent over an org-shared // knowledge base — that the resource's KB is still shared to the caller. @@ -756,147 +402,83 @@ func newMessageScopedFileServeHandler( resourceCatalog interfaces.ResourceCatalog, kbShareAuth messageKBShareAuthorizer, ) gin.HandlerFunc { - absDir := localStorageAbsDir() - return func(c *gin.Context) { - ctx := c.Request.Context() - filePath, ok := requireFilePathQuery(c) + reference, ok := requireFilePathQuery(c) if !ok { return } - - callerTenantID, ok := types.TenantIDFromContext(ctx) - if !ok || callerTenantID == 0 { - c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized: workspace context missing"}) + file, err := access.ResolveMessageFile(c.Request.Context(), c.Param("id"), c.Param("message_id"), reference, + messageService, agentShareService, resourceCatalog, kbShareAuth) + if fileAccessError(c, err) { return } - if messageService == nil { - c.Status(http.StatusNotFound) - return - } - - message, err := messageService.GetMessage(ctx, c.Param("id"), c.Param("message_id")) - if err != nil || message == nil { - // Do not reveal whether a message exists outside the caller's session. - c.Status(http.StatusNotFound) - return - } - - resolvedPath := filePath - var resource *types.StoredResource - if resourceCatalog != nil { - resolvedPath, resource, err = resourceCatalog.ResolvePath(ctx, filePath) - if err != nil { - c.Status(http.StatusNotFound) - return - } - } - - ownerTenantID := message.AgentTenantID - kbShareAuthorized := false - if resource != nil { - ownerTenantID = resource.TenantID - // A modern message records its source tenant. A resource from any - // other tenant cannot be smuggled through that message even if the - // caller happens to have another shared agent with the same ID — - // unless the reply's own retrieval references prove the resource - // came from an org-shared KB the caller may still read (#3022). - if message.AgentTenantID != 0 && message.AgentTenantID != ownerTenantID { - kbShareAuthorized = kbShareAuth.resourceAccessibleViaSharedKB( - ctx, message, resource, callerTenantID, types.TenantRoleFromContext(ctx)) - if !kbShareAuthorized { - c.JSON(http.StatusForbidden, gin.H{"error": "forbidden: resource not accessible from this message"}) - return - } - } - } - if ownerTenantID == 0 { - c.JSON(http.StatusForbidden, gin.H{"error": "forbidden: message resource workspace missing"}) - return - } - - if ownerTenantID != callerTenantID && !kbShareAuthorized { - // Messages written before agent_tenant_id was populated have no - // recorded source tenant; offer them the same org-shared KB - // fallback before the shared-agent relation is demanded. - if message.AgentTenantID == 0 { - kbShareAuthorized = kbShareAuth.resourceAccessibleViaSharedKB( - ctx, message, resource, callerTenantID, types.TenantRoleFromContext(ctx)) - } - if !kbShareAuthorized { - if message.AgentID == "" || agentShareService == nil { - c.JSON(http.StatusForbidden, gin.H{"error": "forbidden: shared agent access required"}) - return - } - agent, shareErr := agentShareService.GetSharedAgentForTenant( - ctx, - callerTenantID, - types.TenantRoleFromContext(ctx), - message.AgentID, - ownerTenantID, - ) - if shareErr != nil || agent == nil || agent.TenantID != ownerTenantID { - c.JSON(http.StatusForbidden, gin.H{"error": "forbidden: shared agent access revoked"}) - return - } - } - } - - // Registered resources carry authoritative tenant ownership. Legacy - // provider paths must still encode the authorized owner tenant. - if resource == nil { - if err := secutils.ValidateStoragePathTenant(resolvedPath, ownerTenantID); err != nil { - logger.Warnf(ctx, - "[Router] message files denied cross-tenant or invalid path: owner_tenant_id=%d file_path=%q err=%v", - ownerTenantID, resolvedPath, err) - c.JSON(http.StatusForbidden, gin.H{"error": "forbidden: file path not accessible"}) - return - } - } - - ownerTenant, err := tenantService.GetTenantByID(ctx, ownerTenantID) - if err != nil || ownerTenant == nil { - c.Status(http.StatusNotFound) - return - } - - backendID, provider := parseStorageTarget(resolvedPath) - fileSvc, resolvedProvider, ok := resolveTenantFileServiceWithFallback( - ctx, - "/sessions/:id/messages/:message_id/files", - ownerTenant, - backendID, - provider, - absDir, - storageResolver, - globalFileService, - ) - if !ok { - c.Status(http.StatusBadRequest) - return - } - - reader, err := fileSvc.GetFile(ctx, resolvedPath) - if err != nil { - logger.Warnf(ctx, - "[Router] message files get file failed: owner_tenant_id=%d provider=%s path=%q err=%v", - ownerTenantID, resolvedProvider, resolvedPath, err) - c.Status(http.StatusNotFound) - return - } - - contentType, inline := secutils.SafeContentTypeByFilename(resolvedPath) - if resource != nil && resource.MimeType != "" && inline { - contentType = resource.MimeType - } - fileName := resolvedPath - if resource != nil && strings.TrimSpace(resource.OriginalName) != "" { - fileName = resource.OriginalName - } - streamStoredFile(c, reader, contentType, inline, "private, max-age=86400", "message files", fileName) + serveAuthorizedFile(c, file, tenantService, globalFileService, storageResolver, "message files") } } +// Storage consumes the authorized locator; it never infers permissions from +// a context tenant or attempts a different resource after an authorization error. +func serveAuthorizedFile(c *gin.Context, file access.FileAccess, tenants interfaces.TenantService, + global interfaces.FileService, resolver interfaces.StorageBackendResolver, tag string, +) { + ctx := types.WithExecutionTenant(c.Request.Context(), file.OwnerTenantID) + tenant, err := tenants.GetTenantByID(ctx, file.OwnerTenantID) + if err != nil || tenant == nil { + c.Status(http.StatusNotFound) + return + } + backendID, provider := parseStorageTarget(file.Path) + if file.StorageBackendID != "" { + backendID = file.StorageBackendID + } + svc, _, ok := filesvc.ResolveTenantFileServiceWithFallback( + ctx, + tag, + tenant, + backendID, + provider, + localStorageAbsDir(), + resolver, + global, + ) + if !ok { + c.Status(http.StatusBadRequest) + return + } + reader, err := svc.GetFile(ctx, file.Path) + if err != nil { + c.Status(http.StatusNotFound) + return + } + if err := filetransport.Serve(c.Writer, c.Request, reader, filetransport.Options{ + Filename: file.Filename, CacheControl: "private, no-store", + }); err != nil { + logger.Warnf(ctx, "%s stream failed: %v", tag, err) + } +} + +func fileAccessError(c *gin.Context, err error) bool { + if err == nil { + return false + } + switch { + case errors.Is(err, access.ErrNotFound): + c.Status(http.StatusNotFound) + case errors.Is(err, access.ErrUnauthorized): + c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized: workspace context missing"}) + case errors.Is(err, access.ErrForbidden): + c.JSON(http.StatusForbidden, gin.H{"error": "forbidden: file not accessible from this resource"}) + default: + if app, ok := apperrors.IsAppError(err); ok { + c.JSON(app.HTTPCode, gin.H{"error": app.Message}) + } else { + logger.Warnf(c.Request.Context(), "file authorization failed: %v", err) + c.Status(http.StatusServiceUnavailable) + } + } + return true +} + // serveMessageScopedFiles registers the authenticated proxy used by the chat // renderer for assistant-message resources. The chat API-key capability is // sufficient because GetMessage enforces ownership of the API key's session. diff --git a/internal/router/rbac.go b/internal/router/rbac.go index ce872ead..f7f370ea 100644 --- a/internal/router/rbac.go +++ b/internal/router/rbac.go @@ -546,7 +546,7 @@ func (g *rbacGuards) PathTenantMatch() gin.HandlerFunc { // validateAndGetKnowledgeBase helpers that used to be re-implemented // in chunk.go, faq.go, tag.go, knowledge.go and knowledgebase.go; // the share-fallback logic now lives in exactly one place -// (middleware/kb_access.go). +// (application/access/knowledgebase.go). // KBAccessRead gates a KB-scoped read route on the caller having at // least Viewer-level access. The agent-share fallback only activates diff --git a/internal/router/router_files_test.go b/internal/router/router_files_test.go index afe089a3..f0341824 100644 --- a/internal/router/router_files_test.go +++ b/internal/router/router_files_test.go @@ -11,10 +11,10 @@ import ( "testing" "time" - "github.com/gin-gonic/gin" - + "github.com/Tencent/WeKnora/internal/middleware" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/gin-gonic/gin" ) var _ interfaces.FileService = (*stubFileService)(nil) @@ -24,7 +24,9 @@ type stubFileService struct { } type stubResourceCatalog struct { - resource *types.StoredResource + resource *types.StoredResource + fileBindings *types.MessageFileBindings + bound func(context.Context, uint64, string, string) (bool, error) } type stubMessageFileLookup struct { @@ -58,6 +60,17 @@ func (s *stubSharedAgentFileLookup) GetSharedAgentForTenant( return s.get(ctx, tenantID, callerTenantRole, agentID, sourceTenantID...) } +func (s *stubResourceCatalog) IsReferencedByKnowledgeBase( + ctx context.Context, + tenant uint64, + kb, ref string, +) (bool, error) { + if s.bound == nil { + return false, nil + } + return s.bound(ctx, tenant, kb, ref) +} + func (s *stubResourceCatalog) Register( context.Context, uint64, @@ -382,9 +395,8 @@ func TestServeFilesAPIKeyScopeMatrix(t *testing.T) { } // newKBScopedFilesTestEngine wires newKBScopedFileServeHandler behind a -// middleware that injects effectiveTenantID into the request context, mirroring -// what RequireKBAccess does after resolving an org-shared KB to its source -// tenant. This lets the handler be exercised without the full RBAC stack. +// middleware that carries the exact KB grant and owner execution tenant, +// mirroring RequireKBAccess without the full RBAC stack. func newKBScopedFilesTestEngine( effectiveTenantID uint64, tenantSvc interfaces.TenantService, @@ -394,10 +406,24 @@ func newKBScopedFilesTestEngine( engine.GET("/knowledge-bases/:id/files", func(c *gin.Context) { ctx := context.WithValue(c.Request.Context(), types.TenantIDContextKey, effectiveTenantID) - c.Request = c.Request.WithContext(ctx) + grant := &middleware.KBAccess{ + KnowledgeBase: &types.KnowledgeBase{ID: "kb-1", TenantID: effectiveTenantID}, + Caller: types.CallerFromContext(ctx), + EffectiveTenantID: effectiveTenantID, + Permission: types.OrgRoleViewer, + } + c.Set(middleware.KBAccessContextKey, grant) + c.Request = c.Request.WithContext(grant.Context(ctx)) c.Next() }, - newKBScopedFileServeHandler(tenantSvc, global), + newKBScopedFileServeHandlerWithResources( + tenantSvc, + global, + nil, + &stubResourceCatalog{bound: func(_ context.Context, _ uint64, kb, ref string) (bool, error) { + return kb == "kb-1" && ref == "local://10008/exports/img.jpg", nil + }}, + ), ) return engine } @@ -515,6 +541,26 @@ func TestKBScopedFilesRequiresFilePath(t *testing.T) { } } +func TestKBScopedFilesRejectsUnboundFileBeforeStorage(t *testing.T) { + gin.SetMode(gin.TestMode) + engine := newKBScopedFilesTestEngine(10008, + &stubTenantService{get: func(context.Context, uint64) (*types.Tenant, error) { + t.Fatal("an unbound file must be rejected before resolving storage") + return nil, nil + }}, + &stubFileService{getFile: func(context.Context, string) (io.ReadCloser, error) { + t.Fatal("an unbound file must never be opened") + return nil, nil + }}, + ) + w := httptest.NewRecorder() + engine.ServeHTTP(w, httptest.NewRequest(http.MethodGet, + "/knowledge-bases/kb-1/files?file_path="+url.QueryEscape("local://10008/exports/another-kb.png"), nil)) + if w.Code != http.StatusForbidden { + t.Fatalf("status=%d, want 403", w.Code) + } +} + func newMessageScopedFilesTestEngine( callerTenantID uint64, messageService messageFileLookup, @@ -555,13 +601,15 @@ func TestMessageScopedFilesServesSharedAgentResource(t *testing.T) { physical = "local://7/exports/chart.png" ) var requestedPath string + revoked := false + storageCalls := 0 engine := newMessageScopedFilesTestEngine( callerTenantID, &stubMessageFileLookup{get: func(_ context.Context, sessionID, messageID string) (*types.Message, error) { if sessionID != "session-1" || messageID != "message-1" { t.Fatalf("unexpected message scope %s/%s", sessionID, messageID) } - return &types.Message{AgentID: "agent-1", AgentTenantID: ownerTenantID}, nil + return &types.Message{ID: "message-1", AgentID: "agent-1", AgentTenantID: ownerTenantID, Content: ref}, nil }}, &stubSharedAgentFileLookup{get: func( _ context.Context, @@ -573,21 +621,28 @@ func TestMessageScopedFilesServesSharedAgentResource(t *testing.T) { if tenantID != callerTenantID || agentID != "agent-1" || len(sourceTenantID) != 1 || sourceTenantID[0] != ownerTenantID { t.Fatalf("unexpected shared-agent lookup tenant=%d agent=%s source=%v", tenantID, agentID, sourceTenantID) } + if revoked { + return nil, nil + } return &types.CustomAgent{ID: agentID, TenantID: ownerTenantID}, nil }}, &stubTenantService{get: func(_ context.Context, id uint64) (*types.Tenant, error) { return &types.Tenant{ID: id}, nil }}, &stubFileService{getFile: func(_ context.Context, filePath string) (io.ReadCloser, error) { + storageCalls++ requestedPath = filePath return io.NopCloser(strings.NewReader("shared-agent-image")), nil }}, - &stubResourceCatalog{resource: &types.StoredResource{ - TenantID: ownerTenantID, - PhysicalPath: physical, - OriginalName: "chart.png", - MimeType: "image/png", - }}, + &stubResourceCatalog{ + fileBindings: &types.MessageFileBindings{MessageArtifact: true}, + resource: &types.StoredResource{ + TenantID: ownerTenantID, + PhysicalPath: physical, + OriginalName: "chart.png", + MimeType: "image/png", + }, + }, messageKBShareAuthorizer{}, ) @@ -605,6 +660,15 @@ func TestMessageScopedFilesServesSharedAgentResource(t *testing.T) { if got := recorder.Header().Get("Content-Disposition"); !strings.Contains(got, "chart.png") { t.Fatalf("Content-Disposition = %q, want original filename chart.png", got) } + if got := recorder.Header().Get("Cache-Control"); got != "private, no-store" { + t.Fatalf("Cache-Control = %q, want private, no-store", got) + } + revoked = true + recorder = httptest.NewRecorder() + engine.ServeHTTP(recorder, req.Clone(context.Background())) + if recorder.Code != http.StatusForbidden || storageCalls != 1 { + t.Fatalf("revoked request: status=%d storage calls=%d", recorder.Code, storageCalls) + } } func TestMessageScopedFilesServesSameTenantResource(t *testing.T) { @@ -623,7 +687,7 @@ func TestMessageScopedFilesServesSameTenantResource(t *testing.T) { if sessionID != "session-1" || messageID != "message-1" { t.Fatalf("unexpected message scope %s/%s", sessionID, messageID) } - return &types.Message{AgentTenantID: tenantID}, nil + return &types.Message{AgentTenantID: tenantID, Content: ref}, nil }}, &stubSharedAgentFileLookup{get: func( context.Context, uint64, types.TenantRole, string, ...uint64, @@ -690,7 +754,7 @@ func TestMessageScopedFilesRejectsRevokedSharedAgent(t *testing.T) { engine := newMessageScopedFilesTestEngine( 42, &stubMessageFileLookup{get: func(context.Context, string, string) (*types.Message, error) { - return &types.Message{AgentID: "agent-1", AgentTenantID: 7}, nil + return &types.Message{AgentID: "agent-1", AgentTenantID: 7, Content: ref}, nil }}, &stubSharedAgentFileLookup{get: func( context.Context, uint64, types.TenantRole, string, ...uint64, @@ -719,6 +783,56 @@ func TestMessageScopedFilesRejectsRevokedSharedAgent(t *testing.T) { } } +func TestMessageScopedFilesRejectsUnreferencedFilesBeforeStorage(t *testing.T) { + gin.SetMode(gin.TestMode) + const ref = "resource://AbCdEfGhIjKlMnOpQrStUv" + for _, content := range []string{"", ref + "suffix"} { + t.Run(content, func(t *testing.T) { + engine := newMessageScopedFilesTestEngine( + 42, + &stubMessageFileLookup{get: func(context.Context, string, string) (*types.Message, error) { + return &types.Message{ + AgentID: "agent-1", AgentTenantID: 7, Content: content, + AgentSteps: types.AgentSteps{ + {ToolCalls: []types.ToolCall{{Args: map[string]interface{}{"file": ref}}}}, + }, + }, nil + }}, + &stubSharedAgentFileLookup{ + get: func(context.Context, + uint64, + types.TenantRole, + string, + ...uint64) (*types.CustomAgent, + error, + ) { + t.Fatal("tool arguments and handle prefixes must not authorize a message file") + return nil, nil + }, + }, + &stubTenantService{get: func(context.Context, uint64) (*types.Tenant, error) { + t.Fatal("unreferenced files must be denied before resolving storage") + return nil, nil + }}, + &stubFileService{getFile: func(context.Context, string) (io.ReadCloser, error) { + t.Fatal("unreferenced files must never be opened") + return nil, nil + }}, + &stubResourceCatalog{ + resource: &types.StoredResource{TenantID: 7, PhysicalPath: "local://7/exports/chart.png"}, + }, + messageKBShareAuthorizer{}, + ) + w := httptest.NewRecorder() + engine.ServeHTTP(w, httptest.NewRequest(http.MethodGet, + "/sessions/session-1/messages/message-1/files?file_path="+url.QueryEscape(ref), nil)) + if w.Code != http.StatusForbidden { + t.Fatalf("status=%d, want 403", w.Code) + } + }) + } +} + func TestMessageScopedFilesRejectsResourceOutsideMessageTenant(t *testing.T) { gin.SetMode(gin.TestMode) const ref = "resource://AbCdEfGhIjKlMnOpQrStUv" @@ -726,7 +840,7 @@ func TestMessageScopedFilesRejectsResourceOutsideMessageTenant(t *testing.T) { engine := newMessageScopedFilesTestEngine( 42, &stubMessageFileLookup{get: func(context.Context, string, string) (*types.Message, error) { - return &types.Message{AgentID: "agent-1", AgentTenantID: 8}, nil + return &types.Message{AgentID: "agent-1", AgentTenantID: 8, Content: ref}, nil }}, &stubSharedAgentFileLookup{get: func( context.Context, uint64, types.TenantRole, string, ...uint64, @@ -760,14 +874,11 @@ type stubKBShareGuard struct { ) (bool, error) } -func (s *stubKBShareGuard) HasTenantKBPermission( - ctx context.Context, - kbID string, - callerTenantID uint64, - callerTenantRole types.TenantRole, - requiredRole types.OrgMemberRole, -) (bool, error) { - return s.hasPermission(ctx, kbID, callerTenantID, callerTenantRole, requiredRole) +func (s *stubKBShareGuard) CheckTenantKBPermission( + ctx context.Context, kbID string, callerTenantID uint64, callerTenantRole types.TenantRole, +) (types.OrgMemberRole, bool, error) { + allowed, err := s.hasPermission(ctx, kbID, callerTenantID, callerTenantRole, types.OrgRoleViewer) + return types.OrgRoleViewer, allowed, err } type stubKBTenantLookup struct { @@ -861,13 +972,16 @@ func newOrgSharedKBTestEngineFromMessage( requestedPath = path return io.NopCloser(strings.NewReader("shared-kb-image")), nil }}, - &stubResourceCatalog{resource: &types.StoredResource{ - Handle: "ShArEdKbHaNdLe00000000", - TenantID: ownerTenantID, - PhysicalPath: physical, - OriginalName: "quadrant.jpg", - MimeType: "image/jpeg", - }}, + &stubResourceCatalog{ + bound: func(context.Context, uint64, string, string) (bool, error) { return true, nil }, + resource: &types.StoredResource{ + Handle: "ShArEdKbHaNdLe00000000", + TenantID: ownerTenantID, + PhysicalPath: physical, + OriginalName: "quadrant.jpg", + MimeType: "image/jpeg", + }, + }, messageKBShareAuthorizer{ ShareGuard: shareGuard, KBs: &stubKBTenantLookup{kbs: kbs}, @@ -999,12 +1113,15 @@ func TestMessageScopedFilesServesLegacyMessageViaOrgSharedKB(t *testing.T) { &stubFileService{getFile: func(_ context.Context, _ string) (io.ReadCloser, error) { return io.NopCloser(strings.NewReader("legacy-shared-kb-image")), nil }}, - &stubResourceCatalog{resource: &types.StoredResource{ - Handle: "ShArEdKbHaNdLe00000000", - TenantID: ownerTenantID, - PhysicalPath: "local://10005/images/quadrant.jpg", - MimeType: "image/jpeg", - }}, + &stubResourceCatalog{ + bound: func(context.Context, uint64, string, string) (bool, error) { return true, nil }, + resource: &types.StoredResource{ + Handle: "ShArEdKbHaNdLe00000000", + TenantID: ownerTenantID, + PhysicalPath: "local://10005/images/quadrant.jpg", + MimeType: "image/jpeg", + }, + }, messageKBShareAuthorizer{ ShareGuard: &stubKBShareGuard{hasPermission: func( context.Context, string, uint64, types.TenantRole, types.OrgMemberRole, @@ -1055,12 +1172,15 @@ func TestMessageScopedFilesOrgSharedKBResolvesKnowledgeOwner(t *testing.T) { &stubFileService{getFile: func(_ context.Context, _ string) (io.ReadCloser, error) { return io.NopCloser(strings.NewReader("resolved-kb-image")), nil }}, - &stubResourceCatalog{resource: &types.StoredResource{ - Handle: "ShArEdKbHaNdLe00000000", - TenantID: 10005, - PhysicalPath: "local://10005/images/quadrant.jpg", - MimeType: "image/jpeg", - }}, + &stubResourceCatalog{ + bound: func(context.Context, uint64, string, string) (bool, error) { return true, nil }, + resource: &types.StoredResource{ + Handle: "ShArEdKbHaNdLe00000000", + TenantID: 10005, + PhysicalPath: "local://10005/images/quadrant.jpg", + MimeType: "image/jpeg", + }, + }, messageKBShareAuthorizer{ ShareGuard: &stubKBShareGuard{hasPermission: func( context.Context, string, uint64, types.TenantRole, types.OrgMemberRole, @@ -1194,12 +1314,15 @@ func TestMessageScopedFilesOrgSharedKBFailsClosedOnShareError(t *testing.T) { t.Fatal("GetFile should not run after share lookup error") return nil, nil }}, - &stubResourceCatalog{resource: &types.StoredResource{ - Handle: "ShArEdKbHaNdLe00000000", - TenantID: 10005, - PhysicalPath: "local://10005/images/quadrant.jpg", - MimeType: "image/jpeg", - }}, + &stubResourceCatalog{ + bound: func(context.Context, uint64, string, string) (bool, error) { return true, nil }, + resource: &types.StoredResource{ + Handle: "ShArEdKbHaNdLe00000000", + TenantID: 10005, + PhysicalPath: "local://10005/images/quadrant.jpg", + MimeType: "image/jpeg", + }, + }, messageKBShareAuthorizer{ ShareGuard: &stubKBShareGuard{hasPermission: func( context.Context, string, uint64, types.TenantRole, types.OrgMemberRole, @@ -1250,3 +1373,12 @@ func TestServeFilesForcesActiveContentDownload(t *testing.T) { t.Fatalf("X-Content-Type-Options = %q, want nosniff", got) } } + +func (s *stubResourceCatalog) GetMessageFileBindings( + context.Context, + uint64, + string, + string, +) (*types.MessageFileBindings, error) { + return s.fileBindings, nil +} diff --git a/internal/router/routes_knowledge.go b/internal/router/routes_knowledge.go index dc11aa4d..e5a30516 100644 --- a/internal/router/routes_knowledge.go +++ b/internal/router/routes_knowledge.go @@ -62,8 +62,8 @@ func RegisterChunkRoutes(r *gin.RouterGroup, handler *handler.ChunkHandler, g *r // edit/delete any of its documents while a non-owner Contributor gets // 403. KB-scoped upload routes (`/knowledge-bases/:id/knowledge/...`) // reuse OwnedKBOrAdmin because the URL :id is the KB id directly. -// Cross-:id batch operations stay Contributor-gated — they don't have -// a single owning KB to check against. +// Body-scoped batch operations have a Contributor route gate and resolve +// their KB ownership plus Editor operation grant inside the handler. func RegisterKnowledgeRoutes(r *gin.RouterGroup, handler *handler.KnowledgeHandler, g *rbacGuards) { // 知识库下的知识路由组(URL :id is the KB id)。Scoped API key 需要 // ingest 能力才能写内容,且仍受 KB 范围限制;清空 KB 只允许 full-access key。 diff --git a/internal/router/task.go b/internal/router/task.go index 7dcd6d3f..a3ebad50 100644 --- a/internal/router/task.go +++ b/internal/router/task.go @@ -338,7 +338,9 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux { // document-related asynq payload. Kept narrow so we don't accidentally // depend on the full payload schema and survive future field churn. type deadLetterKnowledgePayload struct { - KnowledgeID string `json:"knowledge_id,omitempty"` + TenantID uint64 `json:"tenant_id"` + KnowledgeBaseID string `json:"knowledge_base_id"` + KnowledgeID string `json:"knowledge_id,omitempty"` // Attempt threads through DocumentProcess / ManualProcess / // KnowledgePostProcess payloads (added when span tracking shipped) // — extracted here so the dead-letter callback can also close the @@ -369,10 +371,6 @@ var taskTypesAffectingKnowledgeStatus = map[string]struct{}{ types.TypeManualProcess: {}, } -type deadLetterKnowledgeListDeletePayload struct { - KnowledgeIDs []string `json:"knowledge_ids,omitempty"` -} - // newDeadLetterKnowledgeFailer returns the callback wired into the asynq // dead-letter middleware. When a document-related task exhausts its retry // budget, this callback marks the corresponding Knowledge row as failed so @@ -391,7 +389,7 @@ func newDeadLetterKnowledgeFailer(ks interfaces.KnowledgeService, tracker servic return nil } return func(ctx context.Context, t *asynq.Task, taskErr error) { - if t == nil { + if t == nil || taskErr == nil || errors.Is(taskErr, asynq.SkipRetry) { return } if t.Type() == types.TypeKnowledgeListDelete { @@ -402,7 +400,19 @@ func newDeadLetterKnowledgeFailer(ks interfaces.KnowledgeService, tracker servic return } var probe deadLetterKnowledgePayload - if err := json.Unmarshal(t.Payload(), &probe); err != nil || probe.KnowledgeID == "" { + if err := json.Unmarshal(t.Payload(), &probe); err != nil || probe.KnowledgeID == "" || + probe.KnowledgeBaseID == "" || + probe.TenantID == 0 { + return + } + row, err := repo.GetKnowledgeByID(ctx, probe.TenantID, probe.KnowledgeID) + if err != nil || row == nil || row.ID != probe.KnowledgeID || row.TenantID != probe.TenantID || + row.KnowledgeBaseID != probe.KnowledgeBaseID { + return + } + switch row.ParseStatus { + case types.ParseStatusPending, types.ParseStatusProcessing, types.ParseStatusFinalizing: + default: return } errMsg := "task " + t.Type() + " exhausted retries: " + taskErr.Error() @@ -410,14 +420,16 @@ func newDeadLetterKnowledgeFailer(ks interfaces.KnowledgeService, tracker servic if len(errMsg) > 8192 { errMsg = errMsg[:8192] } - // Single UPDATE so we never end up with parse_status=failed but - // stale error_message (or vice versa) when the second write - // fails. - if err := repo.UpdateKnowledgeColumns(ctx, probe.KnowledgeID, map[string]interface{}{ - "parse_status": types.ParseStatusFailed, - "error_message": errMsg, - }); err != nil { - logger.Warnf(ctx, "dead-letter callback: failed to mark knowledge %s as failed: %v", probe.KnowledgeID, err) + before, after := *row, *row + after.ParseStatus = types.ParseStatusFailed + after.ErrorMessage = errMsg + if err := repo.UpdateKnowledgeForTransfer(ctx, &before, &after); err != nil { + logger.Warnf( + ctx, + "dead-letter callback: knowledge %s changed or failed to update: %v", + probe.KnowledgeID, + err, + ) return } // Close the matching root span so the timeline stops showing @@ -437,10 +449,29 @@ func markKnowledgeListDeleteFailed( t *asynq.Task, taskErr error, ) { - var payload deadLetterKnowledgeListDeletePayload - if err := json.Unmarshal(t.Payload(), &payload); err != nil || len(payload.KnowledgeIDs) == 0 { + // A rejected scope must not gain a side effect through the failure path. + if errors.Is(taskErr, asynq.SkipRetry) { return } + var payload types.KnowledgeListDeletePayload + if err := json.Unmarshal(t.Payload(), &payload); err != nil || payload.TenantID == 0 || + len(payload.KnowledgeIDs) == 0 { + return + } + if payload.KnowledgeBaseID == "" { + rows, err := repo.GetKnowledgeBatch(ctx, payload.TenantID, payload.KnowledgeIDs) + if err != nil || len(rows) == 0 { + return + } + for _, row := range rows { + if row == nil || row.TenantID != payload.TenantID || row.KnowledgeBaseID == "" || + (payload.KnowledgeBaseID != "" && payload.KnowledgeBaseID != row.KnowledgeBaseID) { + return + } + payload.KnowledgeBaseID = row.KnowledgeBaseID + } + } + errMsg := "delete task exhausted retries: " + taskErr.Error() if len(errMsg) > 8192 { errMsg = errMsg[:8192] @@ -449,10 +480,16 @@ func markKnowledgeListDeleteFailed( if knowledgeID == "" { continue } - updated, err := repo.UpdateActiveDeletingKnowledgeColumns(ctx, knowledgeID, map[string]interface{}{ - "parse_status": types.ParseStatusFailed, - "error_message": errMsg, - }) + updated, err := repo.UpdateActiveDeletingKnowledgeColumns( + ctx, + payload.TenantID, + payload.KnowledgeBaseID, + knowledgeID, + map[string]interface{}{ + "parse_status": types.ParseStatusFailed, + "error_message": errMsg, + }, + ) if err != nil { logger.Warnf(ctx, "dead-letter callback: failed to mark delete failure for knowledge %s: %v", knowledgeID, err) continue diff --git a/internal/router/task_delete_scope_test.go b/internal/router/task_delete_scope_test.go new file mode 100644 index 00000000..b4649676 --- /dev/null +++ b/internal/router/task_delete_scope_test.go @@ -0,0 +1,55 @@ +package router + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "testing" + + "github.com/Tencent/WeKnora/internal/application/repository" + "github.com/Tencent/WeKnora/internal/types" + "github.com/hibiken/asynq" + "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +func TestDeleteFailureCallbackPreservesTaskScope(t *testing.T) { + db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{}) + require.NoError(t, err) + require.NoError(t, db.AutoMigrate(&types.Knowledge{})) + for _, row := range []*types.Knowledge{ + {ID: "doc", TenantID: 7, KnowledgeBaseID: "kb", ParseStatus: types.ParseStatusDeleting}, + {ID: "other-kb", TenantID: 7, KnowledgeBaseID: "other", ParseStatus: types.ParseStatusDeleting}, + {ID: "other-tenant", TenantID: 8, KnowledgeBaseID: "kb", ParseStatus: types.ParseStatusDeleting}, + } { + require.NoError(t, db.Create(row).Error) + } + repo := repository.NewKnowledgeRepository(db) + payload := types.KnowledgeListDeletePayload{ + TenantID: 7, + KnowledgeBaseID: "kb", + KnowledgeIDs: []string{"doc", "other-kb", "other-tenant"}, + } + data, err := json.Marshal(payload) + require.NoError(t, err) + task := asynq.NewTask(types.TypeKnowledgeListDelete, data) + markKnowledgeListDeleteFailed(context.Background(), repo, task, fmt.Errorf("rejected scope: %w", asynq.SkipRetry)) + var count int64 + require.NoError( + t, + db.Model(&types.Knowledge{}).Where("parse_status = ?", types.ParseStatusFailed).Count(&count).Error, + ) + require.Zero(t, count) + markKnowledgeListDeleteFailed(context.Background(), repo, task, errors.New("graph unavailable")) + row, err := repo.GetKnowledgeByID(context.Background(), 7, "doc") + require.NoError(t, err) + require.Equal(t, types.ParseStatusFailed, row.ParseStatus) + row, err = repo.GetKnowledgeByID(context.Background(), 7, "other-kb") + require.NoError(t, err) + require.Equal(t, types.ParseStatusDeleting, row.ParseStatus) + row, err = repo.GetKnowledgeByID(context.Background(), 8, "other-tenant") + require.NoError(t, err) + require.Equal(t, types.ParseStatusDeleting, row.ParseStatus) +} diff --git a/internal/router/task_processing_scope_test.go b/internal/router/task_processing_scope_test.go new file mode 100644 index 00000000..d44c1b2e --- /dev/null +++ b/internal/router/task_processing_scope_test.go @@ -0,0 +1,72 @@ +package router + +import ( + "context" + "encoding/json" + "errors" + "path/filepath" + "testing" + + "github.com/Tencent/WeKnora/internal/application/repository" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/hibiken/asynq" + "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +type transferFailureKnowledge struct { + interfaces.KnowledgeService + repo interfaces.KnowledgeRepository +} + +func (s transferFailureKnowledge) GetRepository() interfaces.KnowledgeRepository { return s.repo } + +func TestProcessingFailureCallbackDoesNotFollowMovedDocuments(t *testing.T) { + db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "callback.db")), &gorm.Config{}) + require.NoError(t, err) + sqlDB, err := db.DB() + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, sqlDB.Close()) }) + require.NoError(t, db.AutoMigrate(&types.Knowledge{})) + repo := repository.NewKnowledgeRepository(db) + callback := newDeadLetterKnowledgeFailer(transferFailureKnowledge{repo: repo}, nil) + for _, tc := range []struct { + id, kb, status string + tenant uint64 + skip, changed bool + }{ + {"active", "source", types.ParseStatusProcessing, 7, false, true}, + {"moved", "target", types.ParseStatusProcessing, 7, false, false}, + {"foreign", "source", types.ParseStatusProcessing, 8, false, false}, + {"done", "source", types.ParseStatusCompleted, 7, false, false}, + {"rejected", "source", types.ParseStatusProcessing, 7, true, false}, + } { + t.Run(tc.id, func(t *testing.T) { + require.NoError( + t, + db.Create( + &types.Knowledge{ID: tc.id, TenantID: tc.tenant, KnowledgeBaseID: tc.kb, ParseStatus: tc.status}, + ).Error, + ) + payload, err := json.Marshal( + types.DocumentProcessPayload{TenantID: 7, KnowledgeBaseID: "source", KnowledgeID: tc.id}, + ) + require.NoError(t, err) + failure := errors.New("timeout") + if tc.skip { + failure = asynq.SkipRetry + } + callback(context.Background(), asynq.NewTask(types.TypeDocumentProcess, payload), failure) + row, err := repo.GetKnowledgeByID(context.Background(), tc.tenant, tc.id) + require.NoError(t, err) + if tc.changed { + require.Equal(t, types.ParseStatusFailed, row.ParseStatus) + } else { + require.Equal(t, tc.status, row.ParseStatus) + require.Empty(t, row.ErrorMessage) + } + }) + } +} diff --git a/internal/storageurl/storageurl.go b/internal/storageurl/storageurl.go index 86926876..66d7001d 100644 --- a/internal/storageurl/storageurl.go +++ b/internal/storageurl/storageurl.go @@ -15,11 +15,11 @@ package storageurl import ( "context" - "regexp" "strings" "sync" "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" ) @@ -27,11 +27,7 @@ import ( // legacy `provider://` paths, and canonical `storage:///provider://` // paths. The trailing character class stops at Markdown/HTML delimiters so a // reference inside `![alt](…)` or `src="…"` is matched without them. -var Pattern = regexp.MustCompile( - `\b(?:resource://[0-9A-Za-z_-]+|` + - `(?:storage://[0-9A-Za-z_-]+/)?` + - `(?:local|minio|s3|cos|tos|oss|obs|ks3)://[^\s)\]>"]+)`, -) +var Pattern = types.StorageReferencePattern // IsHTTPURL reports whether s is an http(s) URL — the only form an external // client can fetch; any provider scheme (oss://, local://, …) is not. Scheme diff --git a/internal/types/caller.go b/internal/types/caller.go new file mode 100644 index 00000000..901ebe4b --- /dev/null +++ b/internal/types/caller.go @@ -0,0 +1,43 @@ +package types + +import "context" + +// Caller is the authenticated identity used for resource authorization. Its +// tenant and role do not change when queries execute in a resource's tenant. +// Machine capabilities remain in TenantAPIKeyScope; Principal is separate. +type Caller struct { + TenantID uint64 + UserID string + Role TenantRole +} + +// Normalize defaults an invalid tenant role to Viewer. +func (caller Caller) Normalize() Caller { + if !caller.Role.IsValid() { + caller.Role = TenantRoleViewer + } + return caller +} + +// WithCaller captures the authenticated identity independently of execution scope. +func WithCaller(ctx context.Context, caller Caller) context.Context { + return context.WithValue(ctx, CallerContextKey, caller.Normalize()) +} + +// CallerFromContext supports legacy, unscoped service contexts. Always capture +// the caller before changing execution tenants, including a missing identity. +func CallerFromContext(ctx context.Context) Caller { + if caller, ok := ctx.Value(CallerContextKey).(Caller); ok { + return caller + } + tenantID, _ := TenantIDFromContext(ctx) + userID, _ := UserIDFromContext(ctx) + return Caller{TenantID: tenantID, UserID: userID, Role: TenantRoleFromContext(ctx)} +} + +// WithExecutionTenant changes repository/model scope, never authorization. +// It does not grant access to any resources in the destination tenant. +func WithExecutionTenant(ctx context.Context, tenantID uint64) context.Context { + ctx = WithCaller(ctx, CallerFromContext(ctx)) + return context.WithValue(ctx, TenantIDContextKey, tenantID) +} diff --git a/internal/types/const.go b/internal/types/const.go index 32234409..3b74d6d7 100644 --- a/internal/types/const.go +++ b/internal/types/const.go @@ -4,8 +4,16 @@ package types type ContextKey string const ( - // TenantIDContextKey is the context key for tenant ID + // TenantIDContextKey scopes execution (repository/model queries). TenantIDContextKey ContextKey = "TenantID" + // CallerContextKey captures the authenticated resource caller. + CallerContextKey ContextKey = "ResourceCaller" + // KBGrantsContextKey carries immutable operation grants. + KBGrantsContextKey ContextKey = "ResourceKBGrants" + // KBTransferContextKey carries one admitted resource pair. + KBTransferContextKey ContextKey = "ResourceKBTransfer" + // SharedAgentGrantContextKey carries an authorized agent read scope. + SharedAgentGrantContextKey ContextKey = "ResourceSharedAgentGrant" // TenantInfoContextKey is the context key for tenant information TenantInfoContextKey ContextKey = "TenantInfo" // RequestIDContextKey is the context key for request ID diff --git a/internal/types/context_clone.go b/internal/types/context_clone.go index ba49c6cf..258d547b 100644 --- a/internal/types/context_clone.go +++ b/internal/types/context_clone.go @@ -25,12 +25,16 @@ var contextCloneAcrossDetach = map[ContextKey]bool{ // principal in the same workspace, so all of this has to survive; a // detached goroutine that loses its tenant reads another tenant's rows or // none at all. - TenantIDContextKey: true, - TenantInfoContextKey: true, - UserContextKey: true, - UserIDContextKey: true, - PrincipalContextKey: true, - SystemAdminContextKey: true, + TenantIDContextKey: true, + CallerContextKey: true, + KBGrantsContextKey: true, + KBTransferContextKey: true, + SharedAgentGrantContextKey: true, + TenantInfoContextKey: true, + UserContextKey: true, + UserIDContextKey: true, + PrincipalContextKey: true, + SystemAdminContextKey: true, // TenantRoleContextKey: the caller's resolved role in the active tenant // (PR 2 #1303). Must survive for the same reason as TenantIDContextKey — // any handler that does `ctx := logger.CloneContext(c.Request.Context())` diff --git a/internal/types/context_helpers.go b/internal/types/context_helpers.go index 01aaeac7..628d084e 100644 --- a/internal/types/context_helpers.go +++ b/internal/types/context_helpers.go @@ -20,7 +20,8 @@ func DefaultLanguage() string { return "zh-CN" } -// TenantIDFromContext extracts the tenant ID from ctx. +// TenantIDFromContext extracts the execution tenant ID from ctx. +// Authorization should use CallerFromContext instead. // Returns (0, false) when the key is absent or the value is not uint64. func TenantIDFromContext(ctx context.Context) (uint64, bool) { v, ok := ctx.Value(TenantIDContextKey).(uint64) diff --git a/internal/types/file_reference.go b/internal/types/file_reference.go new file mode 100644 index 00000000..6275fd2d --- /dev/null +++ b/internal/types/file_reference.go @@ -0,0 +1,56 @@ +package types + +import ( + "encoding/json" + "regexp" + "strings" +) + +// StorageReferencePattern is shared by response rewriting and persisted +// reference checks. Matching an entire token prevents path/handle prefix grants. +var StorageReferencePattern = regexp.MustCompile( + `\b(?:resource://[0-9A-Za-z_-]+|(?:storage://[0-9A-Za-z_-]+/)?` + + `(?:local|minio|s3|cos|tos|oss|obs|ks3)://[^\s)\]>"]+)`, +) + +// ContainsStorageReference matches an exact token in text or nested JSON. +func ContainsStorageReference(text, reference string) bool { + if reference == "" { + return false + } + // Image metadata and tool outputs may contain nested JSON strings. Decode + // before tokenizing so escaping neither hides a path nor grants its prefix. + trimmed := strings.TrimSpace(text) + if len(trimmed) > 0 && strings.ContainsRune("[{\"", rune(trimmed[0])) { + var value interface{} + if json.Unmarshal([]byte(trimmed), &value) == nil { + return containsStorageReferenceValue(value, reference) + } + } + for _, candidate := range StorageReferencePattern.FindAllString(text, -1) { + if candidate == reference { + return true + } + } + return false +} + +func containsStorageReferenceValue(value interface{}, reference string) bool { + switch value := value.(type) { + case string: + return ContainsStorageReference(value, reference) + case []interface{}: + for _, item := range value { + if containsStorageReferenceValue(item, reference) { + return true + } + } + case map[string]interface{}: + for _, item := range value { + if containsStorageReferenceValue(item, reference) { + return true + } + } + } + return false +} diff --git a/internal/types/file_reference_test.go b/internal/types/file_reference_test.go new file mode 100644 index 00000000..265534cd --- /dev/null +++ b/internal/types/file_reference_test.go @@ -0,0 +1,34 @@ +package types + +import "testing" + +func TestContainsStorageReference(t *testing.T) { + for _, tt := range []struct { + name, text, reference string + want bool + }{ + {"markdown", "![image](local://7/exports/chart.png)", "local://7/exports/chart.png", true}, + {"path prefix", "![image](local://7/exports/chart.png.bak)", "local://7/exports/chart.png", false}, + { + "nested image metadata", + `{"image_info":"[{\"url\":\"local://7/exports/chart.png\"}]"}`, + "local://7/exports/chart.png", + true, + }, + + { + "escaped handle suffix", + `{"url":"resource://AbCdEfGhIjKlMnOpQrStUv\u0058"}`, + "resource://AbCdEfGhIjKlMnOpQrStUv", + false, + }, + + {"empty reference", `{"url":"local://7/exports/chart.png"}`, "", false}, + } { + t.Run(tt.name, func(t *testing.T) { + if got := ContainsStorageReference(tt.text, tt.reference); got != tt.want { + t.Fatalf("ContainsStorageReference = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/types/interfaces/chunk.go b/internal/types/interfaces/chunk.go index 28cf8a4c..c8f7db2b 100644 --- a/internal/types/interfaces/chunk.go +++ b/internal/types/interfaces/chunk.go @@ -14,6 +14,10 @@ type ChunkImageInfo struct { // ChunkRepository defines the interface for chunk repository operations type ChunkRepository interface { + // ListAllChunksByKnowledgeID includes every type and storage status for + // scoped lifecycle operations; UI pagination hides unindexed chunks. + ListAllChunksByKnowledgeID(ctx context.Context, tenantID uint64, knowledgeID string) ([]*types.Chunk, error) + // CreateChunks creates chunks CreateChunks(ctx context.Context, chunks []*types.Chunk) error // GetChunkByID gets a chunk by id @@ -131,6 +135,9 @@ type ChunkRepository interface { ListRecentDocumentChunksWithQuestions(ctx context.Context, tenantID uint64, kbIDs []string, knowledgeIDs []string, limit int) ([]*types.Chunk, error) } +// ChunkService mutations require explicit KB write grants and validate persisted +// document bindings. Trusted processing/rollback code uses ChunkRepository after +// its own task admission; the execution tenant alone is not a write grant. // ChunkService defines the interface for chunk service operations type ChunkService interface { // CreateChunks creates chunks diff --git a/internal/types/interfaces/knowledge.go b/internal/types/interfaces/knowledge.go index b8179351..7dcf240d 100644 --- a/internal/types/interfaces/knowledge.go +++ b/internal/types/interfaces/knowledge.go @@ -101,7 +101,8 @@ type KnowledgeService interface { RenameKnowledgeFolder(ctx context.Context, kbID string, from string, to string) (int64, error) // DeleteKnowledge deletes knowledge by ID. DeleteKnowledge(ctx context.Context, id string) error - // DeleteKnowledgeList deletes multiple knowledge entries by IDs. + // DeleteKnowledgeList requires an explicit write grant for every affected KB. + // It validates the complete selection before changing any deletion state. DeleteKnowledgeList(ctx context.Context, ids []string) error // GetKnowledgeFile retrieves the file associated with the knowledge. GetKnowledgeFile(ctx context.Context, id string) (io.ReadCloser, string, error) @@ -178,7 +179,7 @@ type KnowledgeService interface { ExportFAQEntriesJSON(ctx context.Context, kbID string) ([]byte, error) // UpdateKnowledgeTagBatch updates tag for document knowledge items in batch. // authorizedKBID restricts all updates to knowledge items belonging to this KB; - // pass empty string to skip (caller must ensure authorization by other means). + // an empty value still requires an explicit write grant for every affected KB. UpdateKnowledgeTagBatch(ctx context.Context, authorizedKBID string, updates map[string][]string) error // SetKnowledgeTags replaces all tags for a single knowledge entry. SetKnowledgeTags(ctx context.Context, knowledgeID string, tagIDs []string) error @@ -241,6 +242,10 @@ type KnowledgeRepository interface { tenantID uint64, kbID string, page *types.Pagination, filter types.KnowledgeListFilter, ) ([]*types.Knowledge, int64, error) UpdateKnowledge(ctx context.Context, knowledge *types.Knowledge) error + // UpdateKnowledgeForTransfer conditionally persists a transfer checkpoint + // and its storage delta in one transaction, without inserting missing rows. + UpdateKnowledgeForTransfer(ctx context.Context, before, after *types.Knowledge) error + // UpdateKnowledgeBatch updates knowledge items in batch UpdateKnowledgeBatch(ctx context.Context, knowledgeList []*types.Knowledge) error DeleteKnowledge(ctx context.Context, tenantID uint64, id string) error @@ -291,7 +296,12 @@ type KnowledgeRepository interface { UpdateKnowledgeColumns(ctx context.Context, id string, values map[string]interface{}) error // UpdateActiveDeletingKnowledgeColumns updates an active, non-deleted knowledge row // only when it is still in the transient deleting state. - UpdateActiveDeletingKnowledgeColumns(ctx context.Context, id string, values map[string]interface{}) (bool, error) + UpdateActiveDeletingKnowledgeColumns( + ctx context.Context, + tenantID uint64, + kbID, id string, + values map[string]interface{}, + ) (bool, error) // FinalizeSubtask atomically decrements pending_subtasks_count for the // given knowledge and promotes parse_status from "finalizing" to // "completed" when the count reaches zero. Returns the post-decrement diff --git a/internal/types/interfaces/resource.go b/internal/types/interfaces/resource.go index 12dd33f6..c7ceb24f 100644 --- a/internal/types/interfaces/resource.go +++ b/internal/types/interfaces/resource.go @@ -26,6 +26,16 @@ type ResourceRepository interface { CreateBinding(ctx context.Context, binding *types.ResourceBinding) error DeleteBinding(ctx context.Context, resourceID, ownerType, ownerID string) error CountBindings(ctx context.Context, resourceID string) (int64, error) + IsReferencedByKnowledgeBase( + ctx context.Context, + tenantID uint64, + kbID, resourceID string, + ) (bool, error) + GetMessageFileBindings( + ctx context.Context, + tenantID uint64, + resourceID, messageID string, + ) (*types.MessageFileBindings, error) CreateGrant(ctx context.Context, grant *types.ResourceAccessGrant) error GetValidGrant(ctx context.Context, tokenHash string, now time.Time) (*types.ResourceAccessGrant, error) DeleteExpiredGrants(ctx context.Context, before time.Time) error @@ -62,3 +72,19 @@ type ResourceCatalog interface { CreateAccessGrant(ctx context.Context, reference string, ttl time.Duration) (string, error) ResolveAccessGrant(ctx context.Context, token string) (*types.StoredResource, error) } + +// KBResourceLookup verifies an explicit binding to a live knowledge document. +// Text references and common storage tenancy never establish ownership. +type KBResourceLookup interface { + IsReferencedByKnowledgeBase(ctx context.Context, tenantID uint64, kbID, reference string) (bool, error) +} + +// MessageFileBindingLookup resolves explicit KB and message-artifact bindings. +// Only the catalog can turn a raw locator into a registered resource identity. +type MessageFileBindingLookup interface { + GetMessageFileBindings( + ctx context.Context, + tenantID uint64, + reference, messageID string, + ) (*types.MessageFileBindings, error) +} diff --git a/internal/types/interfaces/retriever.go b/internal/types/interfaces/retriever.go index 58181055..f19d4d16 100644 --- a/internal/types/interfaces/retriever.go +++ b/internal/types/interfaces/retriever.go @@ -154,3 +154,18 @@ type RetrieveEngineService interface { // RetrieveEngine retrieves the engine RetrieveEngine } + +// KnowledgeIndexMover changes the KB binding of existing indices, preserving +// their chunk IDs and vectors. It must match both source KB and document, +// clear KB-scoped tags, and be safe to repeat after a partial failure. +// CopyIndices followed by DeleteByKnowledgeIDList cannot implement this: the +// unchanged knowledge ID also selects the destination rows for deletion. +type KnowledgeIndexMover interface { + MoveKnowledgeIndices( + ctx context.Context, + sourceKB, targetKB, knowledgeID string, + chunkIDs []string, + dimension int, + knowledgeType string, + ) error +} diff --git a/internal/types/knowledge.go b/internal/types/knowledge.go index aeac79bc..6253423a 100644 --- a/internal/types/knowledge.go +++ b/internal/types/knowledge.go @@ -335,6 +335,9 @@ func (k *Knowledge) ManualMetadata() (*ManualKnowledgeMetadata, error) { return &metadata, nil } +// KnowledgeTransferMetadataKey is reserved for server-owned transfer recovery state. +const KnowledgeTransferMetadataKey = "_knowledge_transfer" + // SetManualMetadata sets manual knowledge metadata onto the knowledge instance. func (k *Knowledge) SetManualMetadata(meta *ManualKnowledgeMetadata) error { if meta == nil { @@ -345,6 +348,23 @@ func (k *Knowledge) SetManualMetadata(meta *ManualKnowledgeMetadata) error { if err != nil { return err } + // Manual processing may finish before the move worker acknowledges its + // task. Preserve the recovery marker when updating manual content/status. + old, err := k.Metadata.Map() + if err != nil { + return err + } + if state, ok := old[KnowledgeTransferMetadataKey]; ok { + fields, err := jsonValue.Map() + if err != nil { + return err + } + fields[KnowledgeTransferMetadataKey] = state + jsonValue, err = json.Marshal(fields) + if err != nil { + return err + } + } k.Metadata = jsonValue return nil } diff --git a/internal/types/resource.go b/internal/types/resource.go index 1acacd6b..350dcab7 100644 --- a/internal/types/resource.go +++ b/internal/types/resource.go @@ -82,6 +82,13 @@ func (r *StoredResource) BeforeCreate(_ *gorm.DB) error { return nil } +// MessageFileBindings contains authoritative file origins for an authorized message. +// MessageArtifact requires an explicit artifact binding to that exact message. +type MessageFileBindings struct { + KnowledgeBaseIDs []string + MessageArtifact bool +} + // ResourceBinding connects a resource to a domain object that owns or uses it. type ResourceBinding struct { ID string `json:"id" gorm:"type:varchar(36);primaryKey"` diff --git a/internal/types/shared_agent_access.go b/internal/types/shared_agent_access.go new file mode 100644 index 00000000..8af93e79 --- /dev/null +++ b/internal/types/shared_agent_access.go @@ -0,0 +1,60 @@ +package types + +// SharedAgentKBScope describes the KBs exposed by an already-authorized shared +// agent. It is not proof that a caller can use that agent. The zero value, an +// unknown selection mode, and an empty selection all deny access. +type SharedAgentKBScope struct { + tenantID uint64 + all bool + ids []string + selected map[string]struct{} +} + +// NewSharedAgentKBScope snapshots an authorized agent configuration. +func NewSharedAgentKBScope(agent *CustomAgent) SharedAgentKBScope { + if agent == nil || agent.TenantID == 0 { + return SharedAgentKBScope{} + } + scope := SharedAgentKBScope{tenantID: agent.TenantID} + switch agent.Config.KBSelectionMode { + case "all": + scope.all = true + case "selected": + scope.selected = make(map[string]struct{}, len(agent.Config.KnowledgeBases)) + for _, id := range agent.Config.KnowledgeBases { + if id == "" { + continue + } + if _, exists := scope.selected[id]; exists { + continue + } + scope.selected[id] = struct{}{} + scope.ids = append(scope.ids, id) + } + } + return scope +} + +// IsAll reports whether every KB in the source tenant is selected. +func (s SharedAgentKBScope) IsAll() bool { return s.all } + +// IsEmpty reports whether the scope authorizes no KBs. +func (s SharedAgentKBScope) IsEmpty() bool { return !s.all && len(s.ids) == 0 } + +// IDs returns a copy of the explicit selection. Use IsAll to distinguish a +// whole-tenant scope from an empty selection; nil never means unrestricted. +func (s SharedAgentKBScope) IDs() []string { return append([]string{}, s.ids...) } + +// Allows checks both the KB selection and the authoritative owner tenant. +func (s SharedAgentKBScope) Allows(kbID string, tenantID uint64) bool { + if kbID == "" || s.tenantID == 0 || s.tenantID != tenantID { + return false + } + _, selected := s.selected[kbID] + return s.all || selected +} + +// SharedAgentIncludesKB also binds an "all" selection to the agent's tenant. +func SharedAgentIncludesKB(agent *CustomAgent, kb *KnowledgeBase) bool { + return kb != nil && NewSharedAgentKBScope(agent).Allows(kb.ID, kb.TenantID) +} diff --git a/internal/types/shared_agent_access_test.go b/internal/types/shared_agent_access_test.go new file mode 100644 index 00000000..dd3a11f7 --- /dev/null +++ b/internal/types/shared_agent_access_test.go @@ -0,0 +1,54 @@ +package types + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSharedAgentKBScope(t *testing.T) { + for _, tt := range []struct { + name, mode string + ids []string + all, empty, allows bool + }{ + {name: "all", mode: "all", all: true, allows: true}, + {name: "selected", mode: "selected", ids: []string{"kb"}, allows: true}, + {name: "other selection", mode: "selected", ids: []string{"other"}}, + {name: "empty selection", mode: "selected", empty: true}, + {name: "blank selection", mode: "selected", ids: []string{""}, empty: true}, + {name: "none", mode: "none", ids: []string{"kb"}, empty: true}, + {name: "unknown mode", mode: "unknown", ids: []string{"kb"}, empty: true}, + {name: "missing mode", empty: true}, + } { + t.Run(tt.name, func(t *testing.T) { + agent := &CustomAgent{ + TenantID: 2, + Config: CustomAgentConfig{KBSelectionMode: tt.mode, KnowledgeBases: tt.ids}, + } + scope := NewSharedAgentKBScope(agent) + require.Equal(t, tt.all, scope.IsAll()) + require.Equal(t, tt.empty, scope.IsEmpty()) + require.Equal(t, tt.allows, scope.Allows("kb", 2)) + require.False(t, scope.Allows("kb", 3), "even all is tenant-bound") + require.False(t, scope.Allows("", 2)) + require.Equal(t, tt.allows, SharedAgentIncludesKB(agent, &KnowledgeBase{ID: "kb", TenantID: 2})) + }) + } + require.True(t, NewSharedAgentKBScope(nil).IsEmpty()) + require.False(t, SharedAgentIncludesKB(nil, &KnowledgeBase{ID: "kb", TenantID: 2})) +} + +func TestSharedAgentKBScopeCopiesSelection(t *testing.T) { + agent := &CustomAgent{ + TenantID: 2, + Config: CustomAgentConfig{KBSelectionMode: "selected", KnowledgeBases: []string{"kb", "", "kb"}}, + } + scope := NewSharedAgentKBScope(agent) + require.Equal(t, []string{"kb"}, scope.IDs()) + agent.Config.KnowledgeBases[0] = "other" + ids := scope.IDs() + ids[0] = "other" + require.True(t, scope.Allows("kb", 2)) + require.False(t, scope.Allows("other", 2)) +} diff --git a/internal/types/task.go b/internal/types/task.go index 6b8495ac..fcbeb8b1 100644 --- a/internal/types/task.go +++ b/internal/types/task.go @@ -407,6 +407,10 @@ type KBClonePayload struct { SourceID string `json:"source_id"` TargetID string `json:"target_id"` Initiator TaskInitiator `json:"initiator,omitempty"` + // New clone destinations are reserved by the server at admission and + // created by the worker once. Retries retain both ID and creator. + CreateTarget bool `json:"create_target,omitempty"` + CreatorID string `json:"creator_id,omitempty"` } // IndexDeletePayload represents the index delete task payload @@ -440,18 +444,20 @@ type KBDeletePayload struct { // KnowledgeListDeletePayload represents the batch knowledge delete task payload type KnowledgeListDeletePayload struct { TracingContext - TenantID uint64 `json:"tenant_id"` - KnowledgeIDs []string `json:"knowledge_ids"` - Initiator TaskInitiator `json:"initiator,omitempty"` + TenantID uint64 `json:"tenant_id"` + KnowledgeIDs []string `json:"knowledge_ids"` + Initiator TaskInitiator `json:"initiator,omitempty"` + KnowledgeBaseID string `json:"knowledge_base_id,omitempty"` } // KnowledgeListReparsePayload represents the batch knowledge reparse task payload type KnowledgeListReparsePayload struct { TracingContext - TenantID uint64 `json:"tenant_id"` - KnowledgeIDs []string `json:"knowledge_ids"` - ProcessConfig *KnowledgeProcessOverrides `json:"process_config,omitempty"` - Initiator TaskInitiator `json:"initiator,omitempty"` + KnowledgeBaseID string `json:"knowledge_base_id,omitempty"` + TenantID uint64 `json:"tenant_id"` + KnowledgeIDs []string `json:"knowledge_ids"` + ProcessConfig *KnowledgeProcessOverrides `json:"process_config,omitempty"` + Initiator TaskInitiator `json:"initiator,omitempty"` } // KnowledgeMovePayload represents the knowledge move task payload