From b720267c8bd90c6cce94230cb92f276e5a60304b Mon Sep 17 00:00:00 2001 From: hanxiantao <601803023@qq.com> Date: Sat, 8 Aug 2026 14:23:10 +0800 Subject: [PATCH 1/7] feat(knowledge): add create-or-update file workflow --- docs/api/knowledge.md | 94 +- docs/docs.go | 240 ++++ docs/swagger.json | 240 ++++ docs/swagger.yaml | 156 +++ frontend/src/api/knowledge-base/index.ts | 8 + frontend/src/i18n/locales/en-US.ts | 14 +- frontend/src/i18n/locales/ko-KR.ts | 14 +- frontend/src/i18n/locales/ru-RU.ts | 14 +- frontend/src/i18n/locales/zh-CN.ts | 14 +- .../src/views/knowledge/KnowledgeBase.vue | 38 +- .../components/DocumentActionMenu.vue | 52 +- .../knowledge/components/DocumentCardView.vue | 34 +- .../knowledge/components/DocumentListView.vue | 30 +- .../views/knowledge/wikiStatusRefresh.test.ts | 31 +- .../src/views/knowledge/wikiStatusRefresh.ts | 4 + internal/application/repository/knowledge.go | 479 ++++++- .../knowledge_file_update_apply_test.go | 215 ++++ .../knowledge_file_update_slot_test.go | 300 +++++ .../retriever/opensearch/bulk_update.go | 3 + .../retriever/opensearch/mapping.go | 3 + .../retriever/opensearch/repository.go | 71 +- .../retriever/opensearch/repository_test.go | 54 + .../retriever/opensearch/retrieve.go | 5 +- internal/application/service/extract.go | 2 +- .../application/service/image_multimodal.go | 2 +- internal/application/service/knowledge.go | 60 +- .../application/service/knowledge_create.go | 5 +- .../service/knowledge_create_test.go | 3 +- .../application/service/knowledge_delete.go | 48 +- .../service/knowledge_file_update.go | 1117 +++++++++++++++++ .../knowledge_file_update_coordinator_test.go | 158 +++ .../service/knowledge_file_update_test.go | 549 ++++++++ .../service/knowledge_housekeeping.go | 95 +- .../service/knowledge_housekeeping_test.go | 70 +- .../service/knowledge_post_process.go | 2 +- .../application/service/knowledge_process.go | 9 +- internal/application/service/wiki_ingest.go | 2 +- internal/container/container.go | 29 +- .../engine_factory_opensearch_test.go | 39 +- .../container/recover_pending_file_updates.go | 58 + .../recover_pending_file_updates_test.go | 60 + internal/container/reset_pending_tasks.go | 1 + .../knowledge_file_update_migration_test.go | 57 + internal/handler/knowledge.go | 240 ++++ .../knowledge_create_or_update_test.go | 254 ++++ .../router/retrieve_driver_readiness_test.go | 16 + internal/router/router.go | 67 +- internal/router/routes_knowledge.go | 3 + internal/router/sync_task.go | 1 + internal/router/task.go | 137 +- .../task_file_update_deadletter_test.go | 214 ++++ .../router/task_inspector_kb_cancel_test.go | 12 + internal/types/interfaces/knowledge.go | 90 ++ internal/types/knowledge.go | 101 ++ internal/types/task.go | 42 + internal/types/tenant.go | 22 +- .../types/tenant_retriever_drivers_test.go | 14 + internal/types/vectorstore.go | 8 +- migrations/mysql/00-init-db.sql | 25 + migrations/sqlite/000000_init.down.sql | 1 + migrations/sqlite/000000_init.up.sql | 26 + ...00080_knowledge_file_update_slots.down.sql | 1 + .../000080_knowledge_file_update_slots.up.sql | 24 + ...00080_knowledge_file_update_slots.down.sql | 1 + .../000080_knowledge_file_update_slots.up.sql | 33 + 65 files changed, 5706 insertions(+), 105 deletions(-) create mode 100644 internal/application/repository/knowledge_file_update_apply_test.go create mode 100644 internal/application/repository/knowledge_file_update_slot_test.go create mode 100644 internal/application/service/knowledge_file_update.go create mode 100644 internal/application/service/knowledge_file_update_coordinator_test.go create mode 100644 internal/application/service/knowledge_file_update_test.go create mode 100644 internal/container/recover_pending_file_updates.go create mode 100644 internal/container/recover_pending_file_updates_test.go create mode 100644 internal/database/knowledge_file_update_migration_test.go create mode 100644 internal/handler/knowledge_create_or_update_test.go create mode 100644 internal/router/retrieve_driver_readiness_test.go create mode 100644 internal/router/task_file_update_deadletter_test.go create mode 100644 internal/types/tenant_retriever_drivers_test.go create mode 100644 migrations/sqlite/000080_knowledge_file_update_slots.down.sql create mode 100644 migrations/sqlite/000080_knowledge_file_update_slots.up.sql create mode 100644 migrations/versioned/000080_knowledge_file_update_slots.down.sql create mode 100644 migrations/versioned/000080_knowledge_file_update_slots.up.sql diff --git a/docs/api/knowledge.md b/docs/api/knowledge.md index 94dc51c737..6a9561074f 100644 --- a/docs/api/knowledge.md +++ b/docs/api/knowledge.md @@ -7,6 +7,7 @@ | 方法 | 路径 | 描述 | | ------ | ------------------------------------------ | ------------------------------------------ | | POST | `/knowledge-bases/:id/knowledge/file` | 上传文件创建知识（multipart） | +| POST | `/knowledge-bases/:id/knowledge/file/create-or-update` | 新增或原位修改文件知识（multipart） | | POST | `/knowledge-bases/:id/knowledge/url` | 从 URL 创建知识（网页抓取或文件下载） | | POST | `/knowledge-bases/:id/knowledge/manual` | 创建手工 Markdown 知识 | | GET | `/knowledge-bases/:id/knowledge` | 列出知识库下的知识（支持分页/筛选） | @@ -20,6 +21,8 @@ | PUT | `/knowledge/manual/:id` | 更新手工 Markdown 知识 | | POST | `/knowledge/:id/reparse` | 重新解析知识（异步） | | POST | `/knowledge/:id/cancel-parse` | 取消正在进行的解析任务 | +| POST | `/knowledge/:id/file-update/retry` | 重试失败的文件更新版本 | +| DELETE | `/knowledge/:id/file-update` | 丢弃失败及待处理的文件更新版本 | | GET | `/knowledge/:id/download` | 下载原始文件（attachment） | | GET | `/knowledge/:id/preview` | 内联预览文件（按扩展名设置 Content-Type） | | PUT | `/knowledge/image/:id/:chunk_id` | 更新分块图像信息 | @@ -34,7 +37,7 @@ > **公共说明**： > - 路径中的 `:id`（知识库路径下）为**知识库 ID**，`/knowledge/:id` 中的 `:id` 为**知识 ID**。 > - 所有写操作（创建、更新、删除、迁移、重新解析、取消解析）需要当前用户在知识库所属组织内具有 `editor` 或 `admin` 权限；清空知识库内容仅 KB **所有者**（admin 且空间匹配）可操作。 -> - 关键状态字段：`parse_status` 取值 `pending` / `processing` / `finalizing` / `completed` / `failed` / `cancelled`；`enable_status` 取值 `enabled` / `disabled`。 +> - 关键状态字段：`parse_status` 取值 `pending` / `processing` / `finalizing` / `replacing` / `completed` / `failed` / `cancelled`；`enable_status` 取值 `enabled` / `disabled`。 > - `processing` 指 DocReader / 分块 / 向量化阶段；`finalizing` 指主解析已完成、仍在执行摘要 / 问题生成 / 图谱抽取等索引优化任务；只有当全部子任务到达终态后才进入 `completed`。 > - `cancelled` 表示解析被用户主动取消，可通过 `reparse` 重新触发。`pending` / `processing` / `finalizing` 这三种状态都可通过 `cancel-parse` 终止。 @@ -112,6 +115,95 @@ curl --location 'http://localhost:8080/api/v1/knowledge-bases/kb-00000001/knowle 文件重复时返回 409 与已存在知识的引用；超过大小限制返回 400 `文件大小不能超过 N MB`。 +## POST `/knowledge-bases/:id/knowledge/file/create-or-update` - 新增或修改文件知识 + +通过同一个 `multipart/form-data` 接口新增或原位修改**文件知识**。服务端优先使用显式 `knowledge_id`，未提供时按本次生效文件名在同知识库内查找唯一文件知识： + +- 不传 `knowledge_id` 且同知识库内没有同名文件知识：进入新增分支，复用文件创建链路。 +- 不传 `knowledge_id` 且唯一同名文件知识命中：进入修改分支，保留原 knowledge ID。 +- 不传 `knowledge_id` 但存在多条同名文件知识：返回 409，调用方必须显式传 `knowledge_id`。 +- 传入已有 `knowledge_id`：原位异步替换源文件并重建索引，既有引用继续有效。 + +修改分支采用 latest-wins：每条 knowledge 最多保存一个 active 版本和一个最新 pending 版本。active 处理期间再次上传时，新请求覆盖原 pending；active 完成后自动处理当时最新的 pending。 + +**路径参数**: + +| 字段 | 类型 | 说明 | +| ---- | ------ | --------- | +| id | string | 知识库 ID | + +**表单字段**: + +| 字段 | 类型 | 必填 | 新增分支 | 修改分支 | +| ------------------------- | ------- | ---- | ------------------------------------- | ------------------------------------------------------ | +| `file` | file | 是 | 新文件；唯一同名命中时转为修改 | 替换后的新文件 | +| `knowledge_id` | string | 否 | 省略；按文件名唯一命中时更新，否则新增 | 已有 knowledge ID | +| `expected_file_hash` | string | 否 | 忽略 | 可选乐观锁；与当前 `file_hash` 不一致时返回 409 | +| `expected_update_version` | integer | 否 | 忽略 | 可选乐观锁；与最后接受的更新版本不一致时返回 409 | +| `fileName` | string | 否 | 自定义文件名 | 新文件名；相对路径的目录部分写入 `folder_path` | +| `metadata` | string | 否 | 沿用现有创建语义 | 省略时保留；提供时合并用户 metadata | +| `enable_multimodel` | string | 否 | `"true"` / `"false"` | 省略时保留，提供时更新 | +| `process_config` | string | 否 | 解析配置覆盖 JSON | 省略时保留，提供时替换 process overrides | +| `tag_ids` | string | 否 | 标签 ID 列表，逗号分隔 | 省略时保留；提供时整体替换，显式空值清空标签 | +| `channel` | string | 否 | 省略时默认为 `api` | 省略时保留原 channel，提供时更新 | + +**请求**（修改分支）: + +```curl +curl --location 'http://localhost:8080/api/v1/knowledge-bases/kb-00000001/knowledge/file/create-or-update' \ +--header 'X-API-Key: sk-xxxxx' \ +--form 'file=@"/Users/xxxx/tests/report-v2.pdf"' \ +--form 'knowledge_id="4c4e7c1a-09cf-485b-a7b5-24b8cdc5acf5"' \ +--form 'expected_update_version="12"' +``` + +**响应**（修改任务已接受，HTTP 202）: + +```json +{ + "success": true, + "data": { + "action": "updated", + "knowledge": { + "id": "4c4e7c1a-09cf-485b-a7b5-24b8cdc5acf5", + "knowledge_base_id": "kb-00000001", + "parse_status": "processing", + "file_update_version": 13, + "file_update_state": "pending" + }, + "update_version": 13, + "update_state": "pending", + "accepted_file_hash": "..." + } +} +``` + +`action` 取值： + +- `created`：新 knowledge 已创建（HTTP 202），文件进入异步解析。 +- `updated`：更新版本已持久化为 active 或最新 pending（HTTP 202）。 +- `unchanged`：内容和显式配置没有变化（HTTP 200），不增加版本。 + +调用方通过 `GET /knowledge/:id` 同时轮询 `file_update_state` 和 `parse_status`。前者取值 `idle` / `active` / `pending` / `failed`，后者表示当前已生效文件的解析状态。`file_update_version` 可作为下一次请求的 `expected_update_version`。 + +失败槽会保留 active payload 以便恢复： + +- `POST /knowledge/:id/file-update/retry`：仅当最新 active 为 `failed` 时重新入队。 +- `DELETE /knowledge/:id/file-update`：丢弃精确匹配的 failed active 和最新 pending，并回收不再引用的暂存文件。 + +删除 knowledge 会先设置 `deleting` 并撤销更新槽；已排队的陈旧更新任务不会恢复该 knowledge。 + +| 状态码 | 说明 | +| ------ | ---- | +| 200 | `action=unchanged`，内容未变化 | +| 202 | `action=created` 或 `updated`，任务已接受 | +| 400 | 文件、JSON 字段、类型或解析配置无效 | +| 403 | 缺少知识库写权限或 API Key `ingest` 能力 | +| 404 | 路径知识库或修改目标 knowledge 不存在 | +| 409 | 正在删除、跨知识库、同名多条、乐观锁失败或并发修改 | +| 413 | 文件超过大小限制 | +| 503 | 检索引擎未注册，或更新已持久化但协调任务暂时无法入队 | + ## POST `/knowledge-bases/:id/knowledge/url` - 从 URL 创建知识 可创建**网页知识**或**远程文件知识**。后端根据下列规则自动判定： diff --git a/docs/docs.go b/docs/docs.go index 9f87c24c8f..ff02fcdd7f 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -5242,6 +5242,140 @@ const docTemplate = `{ } } }, + "/knowledge-bases/{id}/knowledge/file/create-or-update": { + "post": { + "security": [ + { + "Bearer": [] + }, + { + "ApiKeyAuth": [] + } + ], + "description": "knowledge_id 为空时按文件名匹配同知识库唯一文件知识，命中则更新，否则新增；提供时保留原 knowledge ID，按 active + 最新 pending 的 latest-wins 规则异步更新", + "consumes": [ + "multipart/form-data" + ], + "produces": [ + "application/json" + ], + "tags": [ + "知识管理" + ], + "summary": "新增或修改文件知识", + "operationId": "createOrUpdateKnowledgeFromFile", + "parameters": [ + { + "type": "string", + "description": "知识库ID", + "name": "id", + "in": "path", + "required": true + }, + { + "type": "file", + "description": "上传的文件", + "name": "file", + "in": "formData", + "required": true + }, + { + "type": "string", + "description": "要修改的知识ID；省略时按文件名匹配唯一文件知识，未命中则新增", + "name": "knowledge_id", + "in": "formData" + }, + { + "type": "string", + "description": "修改时可选的当前文件hash", + "name": "expected_file_hash", + "in": "formData" + }, + { + "type": "integer", + "description": "修改时可选的最后接受更新版本", + "name": "expected_update_version", + "in": "formData" + }, + { + "type": "string", + "description": "自定义文件名", + "name": "fileName", + "in": "formData" + }, + { + "type": "string", + "description": "元数据JSON", + "name": "metadata", + "in": "formData" + }, + { + "type": "boolean", + "description": "启用多模态处理", + "name": "enable_multimodel", + "in": "formData" + }, + { + "type": "string", + "description": "分类ID列表，逗号分隔", + "name": "tag_ids", + "in": "formData" + }, + { + "type": "string", + "description": "来源渠道", + "name": "channel", + "in": "formData" + }, + { + "type": "string", + "description": "处理配置JSON（KnowledgeProcessOverrides）", + "name": "process_config", + "in": "formData" + } + ], + "responses": { + "200": { + "description": "请求幂等且内容未变化", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "202": { + "description": "新增或修改任务已接受", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "400": { + "description": "请求参数错误", + "schema": { + "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" + } + }, + "404": { + "description": "修改目标不存在", + "schema": { + "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" + } + }, + "409": { + "description": "状态、版本或重复冲突", + "schema": { + "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" + } + }, + "413": { + "description": "文件超限", + "schema": { + "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" + } + } + } + } + }, "/knowledge-bases/{id}/knowledge/folders": { "get": { "security": [ @@ -6933,6 +7067,100 @@ const docTemplate = `{ } } }, + "/knowledge/{id}/file-update": { + "delete": { + "security": [ + { + "Bearer": [] + }, + { + "ApiKeyAuth": [] + } + ], + "description": "删除精确匹配的失败 active 和最新 pending 暂存版本，不影响并发提交的新版本", + "produces": [ + "application/json" + ], + "tags": [ + "知识管理" + ], + "summary": "丢弃失败的文件更新", + "parameters": [ + { + "type": "string", + "description": "知识ID", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "待更新版本已丢弃", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "409": { + "description": "没有失败更新或状态已变化", + "schema": { + "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" + } + } + } + } + }, + "/knowledge/{id}/file-update/retry": { + "post": { + "security": [ + { + "Bearer": [] + }, + { + "ApiKeyAuth": [] + } + ], + "description": "重新唤醒 update slot 中保留的失败 active 版本；并发上传的新版本不会被覆盖", + "produces": [ + "application/json" + ], + "tags": [ + "知识管理" + ], + "summary": "重试失败的文件更新", + "parameters": [ + { + "type": "string", + "description": "知识ID", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "重试已提交", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "409": { + "description": "没有失败更新或状态已变化", + "schema": { + "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" + } + }, + "503": { + "description": "任务系统暂时不可用", + "schema": { + "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" + } + } + } + } + }, "/knowledge/{id}/preview": { "get": { "security": [ @@ -16649,6 +16877,18 @@ const docTemplate = `{ "description": "File type of the knowledge", "type": "string" }, + "file_update_error": { + "description": "FileUpdateError contains a sanitized update failure summary.", + "type": "string" + }, + "file_update_state": { + "description": "FileUpdateState projects the durable update slot state.", + "type": "string" + }, + "file_update_version": { + "description": "FileUpdateVersion is the latest accepted file update version.", + "type": "integer" + }, "folder_path": { "description": "FolderPath is the canonical relative directory this entry belongs to\ninside the knowledge base, e.g. \"docs/spec\" for a folder upload of\n\"docs/spec/design.md\". Empty means the knowledge base root. It is a\ndisplay/navigation concern only: it never affects where the file is\nphysically stored (see FilePath).", "type": "string" diff --git a/docs/swagger.json b/docs/swagger.json index e1eb820d5f..348ed050bb 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -5235,6 +5235,140 @@ } } }, + "/knowledge-bases/{id}/knowledge/file/create-or-update": { + "post": { + "security": [ + { + "Bearer": [] + }, + { + "ApiKeyAuth": [] + } + ], + "description": "knowledge_id 为空时按文件名匹配同知识库唯一文件知识，命中则更新，否则新增；提供时保留原 knowledge ID，按 active + 最新 pending 的 latest-wins 规则异步更新", + "consumes": [ + "multipart/form-data" + ], + "produces": [ + "application/json" + ], + "tags": [ + "知识管理" + ], + "summary": "新增或修改文件知识", + "operationId": "createOrUpdateKnowledgeFromFile", + "parameters": [ + { + "type": "string", + "description": "知识库ID", + "name": "id", + "in": "path", + "required": true + }, + { + "type": "file", + "description": "上传的文件", + "name": "file", + "in": "formData", + "required": true + }, + { + "type": "string", + "description": "要修改的知识ID；省略时按文件名匹配唯一文件知识，未命中则新增", + "name": "knowledge_id", + "in": "formData" + }, + { + "type": "string", + "description": "修改时可选的当前文件hash", + "name": "expected_file_hash", + "in": "formData" + }, + { + "type": "integer", + "description": "修改时可选的最后接受更新版本", + "name": "expected_update_version", + "in": "formData" + }, + { + "type": "string", + "description": "自定义文件名", + "name": "fileName", + "in": "formData" + }, + { + "type": "string", + "description": "元数据JSON", + "name": "metadata", + "in": "formData" + }, + { + "type": "boolean", + "description": "启用多模态处理", + "name": "enable_multimodel", + "in": "formData" + }, + { + "type": "string", + "description": "分类ID列表，逗号分隔", + "name": "tag_ids", + "in": "formData" + }, + { + "type": "string", + "description": "来源渠道", + "name": "channel", + "in": "formData" + }, + { + "type": "string", + "description": "处理配置JSON（KnowledgeProcessOverrides）", + "name": "process_config", + "in": "formData" + } + ], + "responses": { + "200": { + "description": "请求幂等且内容未变化", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "202": { + "description": "新增或修改任务已接受", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "400": { + "description": "请求参数错误", + "schema": { + "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" + } + }, + "404": { + "description": "修改目标不存在", + "schema": { + "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" + } + }, + "409": { + "description": "状态、版本或重复冲突", + "schema": { + "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" + } + }, + "413": { + "description": "文件超限", + "schema": { + "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" + } + } + } + } + }, "/knowledge-bases/{id}/knowledge/folders": { "get": { "security": [ @@ -6926,6 +7060,100 @@ } } }, + "/knowledge/{id}/file-update": { + "delete": { + "security": [ + { + "Bearer": [] + }, + { + "ApiKeyAuth": [] + } + ], + "description": "删除精确匹配的失败 active 和最新 pending 暂存版本，不影响并发提交的新版本", + "produces": [ + "application/json" + ], + "tags": [ + "知识管理" + ], + "summary": "丢弃失败的文件更新", + "parameters": [ + { + "type": "string", + "description": "知识ID", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "待更新版本已丢弃", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "409": { + "description": "没有失败更新或状态已变化", + "schema": { + "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" + } + } + } + } + }, + "/knowledge/{id}/file-update/retry": { + "post": { + "security": [ + { + "Bearer": [] + }, + { + "ApiKeyAuth": [] + } + ], + "description": "重新唤醒 update slot 中保留的失败 active 版本；并发上传的新版本不会被覆盖", + "produces": [ + "application/json" + ], + "tags": [ + "知识管理" + ], + "summary": "重试失败的文件更新", + "parameters": [ + { + "type": "string", + "description": "知识ID", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "重试已提交", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "409": { + "description": "没有失败更新或状态已变化", + "schema": { + "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" + } + }, + "503": { + "description": "任务系统暂时不可用", + "schema": { + "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" + } + } + } + } + }, "/knowledge/{id}/preview": { "get": { "security": [ @@ -16642,6 +16870,18 @@ "description": "File type of the knowledge", "type": "string" }, + "file_update_error": { + "description": "FileUpdateError contains a sanitized update failure summary.", + "type": "string" + }, + "file_update_state": { + "description": "FileUpdateState projects the durable update slot state.", + "type": "string" + }, + "file_update_version": { + "description": "FileUpdateVersion is the latest accepted file update version.", + "type": "integer" + }, "folder_path": { "description": "FolderPath is the canonical relative directory this entry belongs to\ninside the knowledge base, e.g. \"docs/spec\" for a folder upload of\n\"docs/spec/design.md\". Empty means the knowledge base root. It is a\ndisplay/navigation concern only: it never affects where the file is\nphysically stored (see FilePath).", "type": "string" diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 72a2a86fd6..c3bae9a2da 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1429,6 +1429,15 @@ definitions: file_type: description: File type of the knowledge type: string + file_update_error: + description: FileUpdateError contains a sanitized update failure summary. + type: string + file_update_state: + description: FileUpdateState projects the durable update slot state. + type: string + file_update_version: + description: FileUpdateVersion is the latest accepted file update version. + type: integer folder_path: description: |- FolderPath is the canonical relative directory this entry belongs to @@ -9111,6 +9120,95 @@ paths: summary: 从文件创建知识 tags: - 知识管理 + /knowledge-bases/{id}/knowledge/file/create-or-update: + post: + consumes: + - multipart/form-data + description: knowledge_id 为空时按文件名匹配同知识库唯一文件知识，命中则更新，否则新增；提供时保留原 knowledge ID，按 + active + 最新 pending 的 latest-wins 规则异步更新 + operationId: createOrUpdateKnowledgeFromFile + parameters: + - description: 知识库ID + in: path + name: id + required: true + type: string + - description: 上传的文件 + in: formData + name: file + required: true + type: file + - description: 要修改的知识ID；省略时按文件名匹配唯一文件知识，未命中则新增 + in: formData + name: knowledge_id + type: string + - description: 修改时可选的当前文件hash + in: formData + name: expected_file_hash + type: string + - description: 修改时可选的最后接受更新版本 + in: formData + name: expected_update_version + type: integer + - description: 自定义文件名 + in: formData + name: fileName + type: string + - description: 元数据JSON + in: formData + name: metadata + type: string + - description: 启用多模态处理 + in: formData + name: enable_multimodel + type: boolean + - description: 分类ID列表，逗号分隔 + in: formData + name: tag_ids + type: string + - description: 来源渠道 + in: formData + name: channel + type: string + - description: 处理配置JSON（KnowledgeProcessOverrides） + in: formData + name: process_config + type: string + produces: + - application/json + responses: + "200": + description: 请求幂等且内容未变化 + schema: + additionalProperties: true + type: object + "202": + description: 新增或修改任务已接受 + schema: + additionalProperties: true + type: object + "400": + description: 请求参数错误 + schema: + $ref: '#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError' + "404": + description: 修改目标不存在 + schema: + $ref: '#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError' + "409": + description: 状态、版本或重复冲突 + schema: + $ref: '#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError' + "413": + description: 文件超限 + schema: + $ref: '#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError' + security: + - Bearer: [] + - ApiKeyAuth: [] + summary: 新增或修改文件知识 + tags: + - 知识管理 /knowledge-bases/{id}/knowledge/folders: get: consumes: @@ -9882,6 +9980,64 @@ paths: summary: 下载知识文件 tags: - 知识管理 + /knowledge/{id}/file-update: + delete: + description: 删除精确匹配的失败 active 和最新 pending 暂存版本，不影响并发提交的新版本 + parameters: + - description: 知识ID + in: path + name: id + required: true + type: string + produces: + - application/json + responses: + "200": + description: 待更新版本已丢弃 + schema: + additionalProperties: true + type: object + "409": + description: 没有失败更新或状态已变化 + schema: + $ref: '#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError' + security: + - Bearer: [] + - ApiKeyAuth: [] + summary: 丢弃失败的文件更新 + tags: + - 知识管理 + /knowledge/{id}/file-update/retry: + post: + description: 重新唤醒 update slot 中保留的失败 active 版本；并发上传的新版本不会被覆盖 + parameters: + - description: 知识ID + in: path + name: id + required: true + type: string + produces: + - application/json + responses: + "200": + description: 重试已提交 + schema: + additionalProperties: true + type: object + "409": + description: 没有失败更新或状态已变化 + schema: + $ref: '#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError' + "503": + description: 任务系统暂时不可用 + schema: + $ref: '#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError' + security: + - Bearer: [] + - ApiKeyAuth: [] + summary: 重试失败的文件更新 + tags: + - 知识管理 /knowledge/{id}/preview: get: consumes: diff --git a/frontend/src/api/knowledge-base/index.ts b/frontend/src/api/knowledge-base/index.ts index 7eecec5199..d463ad6f78 100644 --- a/frontend/src/api/knowledge-base/index.ts +++ b/frontend/src/api/knowledge-base/index.ts @@ -358,6 +358,14 @@ export function cancelKnowledgeParse(id: string) { return post(`/api/v1/knowledge/${id}/cancel-parse`); } +export function retryKnowledgeFileUpdate(id: string) { + return post(`/api/v1/knowledge/${id}/file-update/retry`); +} + +export function discardKnowledgeFileUpdate(id: string) { + return del(`/api/v1/knowledge/${id}/file-update`); +} + export function getKnowledgeSpans(id: string, attempt?: number) { const qs = attempt ? `?attempt=${attempt}` : ''; return get(`/api/v1/knowledge/${id}/spans${qs}`); diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index d28822f68e..545e49ceda 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -573,7 +573,7 @@ export default { generatingSummary: 'Generating summary...', documentSummary: 'Summary', detailSectionMeta: 'Basic info', - confirmDeleteDocument: 'Confirm deletion of document "{fileName}", recovery will be impossible after deletion', + confirmDeleteDocument: 'Delete document "{fileName}"? Pending file updates will also be cancelled. This cannot be undone.', confirmDelete: 'Confirm Delete', viewModeGrid: 'Grid view', viewModeList: 'List view', @@ -608,6 +608,17 @@ export default { statusCompleted: 'Completed', statusProcessing: 'Processing', statusFinalizing: 'Optimizing', + statusReplacing: 'Replacing file', + fileUpdateActive: 'Updating file', + fileUpdatePending: 'Update pending v{version}', + fileUpdateFailed: 'File update failed', + retryFileUpdate: 'Retry latest version', + retryFileUpdateSubmitted: 'File update resubmitted', + retryFileUpdateFailed: 'Failed to retry file update', + discardFileUpdate: 'Discard pending update', + discardFileUpdateConfirm: 'Discard the failed version and its latest pending file?', + discardFileUpdateSubmitted: 'Pending update discarded', + discardFileUpdateFailed: 'Failed to discard pending update', statusFailed: 'Failed', statusCancelled: 'Cancelled', statusDraft: 'Draft', @@ -627,6 +638,7 @@ export default { parseStatusProcessing: 'Processing', parseStatusCompleted: 'Completed', parseStatusFinalizing: 'Optimizing', + parseStatusReplacing: 'Replacing file', parseStatusFailed: 'Failed', parseStatusCancelled: 'Cancelled', parseStatusDraft: 'Draft', diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index 4f7b4cfefa..e9be1e1a34 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -5454,7 +5454,7 @@ export default { generatingSummary: '요약 생성 중...', documentSummary: '요약', detailSectionMeta: '기본 정보', - confirmDeleteDocument: '"{fileName}" 문서를 삭제하시겠습니까? 삭제 후 복구할 수 없습니다', + confirmDeleteDocument: '"{fileName}" 문서를 삭제하시겠습니까? 대기 중인 파일 업데이트도 취소되며 복구할 수 없습니다.', confirmDelete: '삭제 확인', viewModeGrid: '카드 보기', viewModeList: '목록 보기', @@ -5489,6 +5489,17 @@ export default { statusCompleted: '완료', statusProcessing: '처리 중', statusFinalizing: '최적화 중', + statusReplacing: '파일 교체 중', + fileUpdateActive: '파일 업데이트 중', + fileUpdatePending: '업데이트 대기 v{version}', + fileUpdateFailed: '파일 업데이트 실패', + retryFileUpdate: '최신 버전 다시 시도', + retryFileUpdateSubmitted: '파일 업데이트를 다시 제출했습니다', + retryFileUpdateFailed: '파일 업데이트 재시도 실패', + discardFileUpdate: '대기 업데이트 삭제', + discardFileUpdateConfirm: '실패한 버전과 최신 대기 파일을 삭제하시겠습니까?', + discardFileUpdateSubmitted: '대기 업데이트를 삭제했습니다', + discardFileUpdateFailed: '대기 업데이트 삭제 실패', statusFailed: '실패', statusCancelled: '취소됨', statusDraft: '초안', @@ -5508,6 +5519,7 @@ export default { parseStatusProcessing: '처리 중', parseStatusCompleted: '완료됨', parseStatusFinalizing: '최적화 중', + parseStatusReplacing: '파일 교체 중', parseStatusFailed: '실패', parseStatusCancelled: '취소됨', parseStatusDraft: '초안', diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index dbb4af1ac3..79af818055 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -5454,7 +5454,7 @@ export default { generatingSummary: 'Генерация резюме...', documentSummary: 'Резюме', detailSectionMeta: 'Основная информация', - confirmDeleteDocument: 'Подтвердить удаление документа "{fileName}", после удаления восстановление невозможно', + confirmDeleteDocument: 'Удалить документ "{fileName}"? Ожидающие обновления файла также будут отменены. Восстановление невозможно.', confirmDelete: 'Подтвердить удаление', viewModeGrid: 'Сетка', viewModeList: 'Список', @@ -5489,6 +5489,17 @@ export default { statusCompleted: 'Завершено', statusProcessing: 'Обработка', statusFinalizing: 'Оптимизация', + statusReplacing: 'Замена файла', + fileUpdateActive: 'Обновление файла', + fileUpdatePending: 'Ожидает обновления v{version}', + fileUpdateFailed: 'Ошибка обновления файла', + retryFileUpdate: 'Повторить последнюю версию', + retryFileUpdateSubmitted: 'Обновление файла отправлено повторно', + retryFileUpdateFailed: 'Не удалось повторить обновление файла', + discardFileUpdate: 'Отменить ожидающее обновление', + discardFileUpdateConfirm: 'Отменить ошибочную версию и последний ожидающий файл?', + discardFileUpdateSubmitted: 'Ожидающее обновление отменено', + discardFileUpdateFailed: 'Не удалось отменить ожидающее обновление', statusFailed: 'Ошибка', statusCancelled: 'Отменено', statusDraft: 'Черновик', @@ -5508,6 +5519,7 @@ export default { parseStatusProcessing: 'Обработка', parseStatusCompleted: 'Завершено', parseStatusFinalizing: 'Оптимизация', + parseStatusReplacing: 'Замена файла', parseStatusFailed: 'Ошибка', parseStatusCancelled: 'Отменено', parseStatusDraft: 'Черновик', diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index 06eb7a75cb..0bae4980f6 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -5454,7 +5454,7 @@ export default { generatingSummary: '生成摘要中...', documentSummary: '摘要', detailSectionMeta: '基本信息', - confirmDeleteDocument: '确认删除文档"{fileName}"，删除后将无法恢复', + confirmDeleteDocument: '确认删除文档"{fileName}"？待更新版本会同时取消，删除后无法恢复。', confirmDelete: '确认删除', viewModeGrid: '卡片视图', viewModeList: '列表视图', @@ -5489,6 +5489,17 @@ export default { statusCompleted: '已完成', statusProcessing: '解析中', statusFinalizing: '优化中', + statusReplacing: '正在替换文件', + fileUpdateActive: '正在更新文件', + fileUpdatePending: '有待更新 v{version}', + fileUpdateFailed: '文件更新失败', + retryFileUpdate: '重试最新版本', + retryFileUpdateSubmitted: '文件更新已重新提交', + retryFileUpdateFailed: '文件更新重试失败', + discardFileUpdate: '丢弃待更新', + discardFileUpdateConfirm: '确认丢弃失败版本及其最新待更新文件？', + discardFileUpdateSubmitted: '已丢弃待更新版本', + discardFileUpdateFailed: '丢弃待更新版本失败', statusFailed: '失败', statusCancelled: '已取消', statusDraft: '草稿', @@ -5508,6 +5519,7 @@ export default { parseStatusProcessing: '处理中', parseStatusCompleted: '已完成', parseStatusFinalizing: '整理优化中', + parseStatusReplacing: '正在替换文件', parseStatusFailed: '失败', parseStatusCancelled: '已取消', parseStatusDraft: '草稿', diff --git a/frontend/src/views/knowledge/KnowledgeBase.vue b/frontend/src/views/knowledge/KnowledgeBase.vue index 1b1963bfce..c8870a3ef4 100644 --- a/frontend/src/views/knowledge/KnowledgeBase.vue +++ b/frontend/src/views/knowledge/KnowledgeBase.vue @@ -31,6 +31,8 @@ import { createKnowledgeFromURL, reparseKnowledge, cancelKnowledgeParse, + retryKnowledgeFileUpdate, + discardKnowledgeFileUpdate, batchDeleteKnowledge, batchReparseKnowledge, getKnowledgeSpans, @@ -581,6 +583,7 @@ const parseStatusOptions = computed(() => [ { label: t('knowledgeBase.parseStatusFailed'), value: 'failed' }, { label: t('knowledgeBase.parseStatusCancelled'), value: 'cancelled' }, { label: t('knowledgeBase.parseStatusFinalizing'), value: 'finalizing' }, + { label: t('knowledgeBase.parseStatusReplacing'), value: 'replacing' }, { label: t('knowledgeBase.parseStatusDraft'), value: 'draft' }, ]); const selectedSource = ref(''); @@ -1316,6 +1319,9 @@ type KnowledgeCard = { metadata?: any; error_message?: string; tags?: Array<{ id: string; name: string; color?: string }>; + file_update_version?: number; + file_update_state?: string; + file_update_error?: string; }; // needsStatusPolling decides whether a card row is still "in flight" // enough that the doc list should keep refreshing it. Keep in sync with @@ -1324,7 +1330,7 @@ type KnowledgeCard = { // graph extract still running), and a `completed` row whose summary // hasn't landed yet keeps polling so the description fills in. const needsStatusPolling = (item: KnowledgeCard) => { - return knowledgeNeedsStatusPolling(item); + return knowledgeNeedsStatusPolling(item) || item.file_update_state === 'active' || item.file_update_state === 'pending'; }; const updateStatus = (analyzeList: KnowledgeCard[]) => { @@ -2145,9 +2151,31 @@ const confirmCancelParseKnowledge = async (item: KnowledgeCard) => { } }; +const confirmRetryFileUpdate = async (item: KnowledgeCard) => { + if (!item?.id) return; + try { + await retryKnowledgeFileUpdate(item.id); + MessagePlugin.success(t('knowledgeBase.retryFileUpdateSubmitted')); + loadKnowledgeFiles(kbId.value); + } catch (error: any) { + MessagePlugin.error(error?.message || t('knowledgeBase.retryFileUpdateFailed')); + } +}; + +const confirmDiscardFileUpdate = async (item: KnowledgeCard) => { + if (!item?.id) return; + try { + await discardKnowledgeFileUpdate(item.id); + MessagePlugin.success(t('knowledgeBase.discardFileUpdateSubmitted')); + loadKnowledgeFiles(kbId.value); + } catch (error: any) { + MessagePlugin.error(error?.message || t('knowledgeBase.discardFileUpdateFailed')); + } +}; + // Bridge card-view actions back to existing per-card handlers. const handleCardAction = ( - action: 'edit' | 'reparse' | 'cancel-parse' | 'move' | 'move-folder' | 'delete' | 'view-trace' | 'batch-manage', + action: 'edit' | 'reparse' | 'cancel-parse' | 'retry-file-update' | 'discard-file-update' | 'move' | 'move-folder' | 'delete' | 'view-trace' | 'batch-manage', item: KnowledgeCard, ) => { const idx = (cardList.value || []).findIndex((i: KnowledgeCard) => i.id === item.id); @@ -2157,6 +2185,8 @@ const handleCardAction = ( return confirmRebuildKnowledge(idx, item); } if (action === 'cancel-parse') return confirmCancelParseKnowledge(item); + if (action === 'retry-file-update') return confirmRetryFileUpdate(item); + if (action === 'discard-file-update') return confirmDiscardFileUpdate(item); if (action === 'move') return handleMoveKnowledge(item); if (action === 'delete') return confirmDeleteKnowledge(idx, item); if (action === 'view-trace') return handleViewTrace(idx, item); @@ -2165,13 +2195,15 @@ const handleCardAction = ( // Bridge list-view actions back to existing per-card handlers. const handleListAction = ( - action: 'edit' | 'reparse' | 'cancel-parse' | 'move' | 'move-folder' | 'delete' | 'view-trace' | 'batch-manage', + action: 'edit' | 'reparse' | 'cancel-parse' | 'retry-file-update' | 'discard-file-update' | 'move' | 'move-folder' | 'delete' | 'view-trace' | 'batch-manage', item: KnowledgeCard, ) => { const idx = (cardList.value || []).findIndex((i: KnowledgeCard) => i.id === item.id); if (action === 'edit') return handleManualEdit(idx, item); if (action === 'reparse') return confirmRebuildKnowledge(idx, item); if (action === 'cancel-parse') return confirmCancelParseKnowledge(item); + if (action === 'retry-file-update') return confirmRetryFileUpdate(item); + if (action === 'discard-file-update') return confirmDiscardFileUpdate(item); if (action === 'move') return handleMoveKnowledge(item); if (action === 'delete') return confirmDeleteKnowledge(idx, item); if (action === 'view-trace') return handleViewTrace(idx, item); diff --git a/frontend/src/views/knowledge/components/DocumentActionMenu.vue b/frontend/src/views/knowledge/components/DocumentActionMenu.vue index 0d85aa2ab1..76ef9a0912 100644 --- a/frontend/src/views/knowledge/components/DocumentActionMenu.vue +++ b/frontend/src/views/knowledge/components/DocumentActionMenu.vue @@ -8,6 +8,7 @@ interface KnowledgeItem { title?: string; type?: string; parse_status?: string; + file_update_state?: string; } const props = defineProps<{ @@ -23,6 +24,8 @@ const emit = defineEmits<{ (e: 'view-trace'): void; (e: 'reparse'): void; (e: 'cancel-parse'): void; + (e: 'retry-file-update'): void; + (e: 'discard-file-update'): void; (e: 'move'): void; (e: 'move-folder'): void; (e: 'batch-manage'): void; @@ -36,13 +39,36 @@ const CANCELABLE_PARSE_STATUSES = new Set(['pending', 'processing', 'finalizing' const isParseInFlight = computed(() => CANCELABLE_PARSE_STATUSES.has(String(props.item.parse_status ?? '')) ); +const isReplacing = computed(() => props.item.parse_status === 'replacing'); +const hasFailedFileUpdate = computed(() => props.item.file_update_state === 'failed'); const fileName = computed(() => props.item.file_name || props.item.title || props.item.id); + + {{ item.file_update_state === 'active' + ? t('knowledgeBase.fileUpdateActive') + : item.file_update_state === 'pending' + ? t('knowledgeBase.fileUpdatePending', { version: item.file_update_version }) + : t('knowledgeBase.fileUpdateFailed') }} +
@@ -404,6 +421,8 @@ const handleAction = (action: 'edit' | 'reparse' | 'cancel-parse' | 'move' | 'mo @view-trace="handleAction('view-trace', item)" @reparse="handleAction('reparse', item)" @cancel-parse="handleAction('cancel-parse', item)" + @retry-file-update="handleAction('retry-file-update', item)" + @discard-file-update="handleAction('discard-file-update', item)" @move="handleAction('move', item)" @move-folder="handleAction('move-folder', item)" @batch-manage="handleAction('batch-manage', item)" @@ -830,6 +849,13 @@ const handleAction = (action: 'edit' | 'reparse' | 'cancel-parse' | 'move' | 'mo margin-right: 2px; } +.doc-list-row .cell-status { + flex-direction: column; + align-items: flex-start; + justify-content: center; + gap: 3px; +} + .icon-spin { animation: doc-list-spin 0.9s linear infinite; } diff --git a/frontend/src/views/knowledge/wikiStatusRefresh.test.ts b/frontend/src/views/knowledge/wikiStatusRefresh.test.ts index 17fb602d4d..be992debef 100644 --- a/frontend/src/views/knowledge/wikiStatusRefresh.test.ts +++ b/frontend/src/views/knowledge/wikiStatusRefresh.test.ts @@ -1,7 +1,10 @@ import assert from 'node:assert/strict' import test from 'node:test' -import { shouldRefreshWikiStatusAfterKnowledgePoll } from './wikiStatusRefresh.ts' +import { + knowledgeNeedsStatusPolling, + shouldRefreshWikiStatusAfterKnowledgePoll, +} from './wikiStatusRefresh.ts' test('refreshes wiki status when a polled document leaves an in-flight state', () => { assert.equal( @@ -22,3 +25,29 @@ test('does not refresh wiki status for ordinary in-flight polling updates', () = false, ) }) + +test('keeps polling while a document is being replaced', () => { + assert.equal(knowledgeNeedsStatusPolling({ parse_status: 'replacing' }), true) +}) + +test('keeps polling (no wiki refresh) while a replacement moves to pending', () => { + // replacing -> pending is still in-flight; polling continues, so the wiki + // status must not be refreshed yet. + assert.equal( + shouldRefreshWikiStatusAfterKnowledgePoll( + { parse_status: 'replacing' }, + { parse_status: 'pending' }, + ), + false, + ) +}) + +test('refreshes wiki status when a replacement settles onto completed', () => { + assert.equal( + shouldRefreshWikiStatusAfterKnowledgePoll( + { parse_status: 'replacing' }, + { parse_status: 'completed', summary_status: 'completed' }, + ), + true, + ) +}) diff --git a/frontend/src/views/knowledge/wikiStatusRefresh.ts b/frontend/src/views/knowledge/wikiStatusRefresh.ts index 1cbef7ac3c..82b82c9b0b 100644 --- a/frontend/src/views/knowledge/wikiStatusRefresh.ts +++ b/frontend/src/views/knowledge/wikiStatusRefresh.ts @@ -8,6 +8,10 @@ export function isKnowledgeParseInFlight(status?: string): boolean { } export function knowledgeNeedsStatusPolling(item: KnowledgePollStatus): boolean { + // `replacing` is driven by an external API, not the local parse pipeline, + // so it is not "parse in flight", but the list must keep polling until the + // async replacement lands the row back on pending/completed/failed. + if (item.parse_status === 'replacing') return true if (isKnowledgeParseInFlight(item.parse_status)) return true return item.parse_status === 'completed' && (item.summary_status === 'pending' || item.summary_status === 'processing') diff --git a/internal/application/repository/knowledge.go b/internal/application/repository/knowledge.go index c2989e8f2a..acf80ebe52 100644 --- a/internal/application/repository/knowledge.go +++ b/internal/application/repository/knowledge.go @@ -9,9 +9,15 @@ import ( "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" "gorm.io/gorm" + "gorm.io/gorm/clause" ) -var ErrKnowledgeNotFound = errors.New("knowledge not found") +var ( + ErrKnowledgeNotFound = errors.New("knowledge not found") + ErrKnowledgeFileUpdateVersionConflict = errors.New("knowledge file update version conflict") + ErrKnowledgeFileUpdateDeleting = errors.New("knowledge is being deleted") + ErrKnowledgeFileUpdateStateConflict = errors.New("knowledge file update state conflict") +) // likeEscapeChar is the SQL ESCAPE character paired with escapeLikeKeyword. const likeEscapeChar = `\` @@ -70,6 +76,7 @@ func (r *knowledgeRepository) GetKnowledgeByID( } return nil, err } + r.projectKnowledgeFileUpdateSlots(ctx, tenantID, []*types.Knowledge{&knowledge}) return &knowledge, nil } @@ -94,6 +101,7 @@ func (r *knowledgeRepository) ListKnowledgeByKnowledgeBaseID( Order("created_at DESC").Find(&knowledges).Error; err != nil { return nil, err } + r.projectKnowledgeFileUpdateSlots(ctx, tenantID, knowledges) return knowledges, nil } @@ -200,6 +208,7 @@ func (r *knowledgeRepository) ListPagedKnowledgeByKnowledgeBaseID( Find(&knowledges).Error; err != nil { return nil, 0, err } + r.projectKnowledgeFileUpdateSlots(ctx, tenantID, knowledges) return knowledges, total, nil } @@ -344,9 +353,55 @@ func (r *knowledgeRepository) GetKnowledgeBatch( Find(&knowledge).Error; err != nil { return nil, err } + r.projectKnowledgeFileUpdateSlots(ctx, tenantID, knowledge) return knowledge, nil } +// projectKnowledgeFileUpdateSlots adds read-only update state without making +// existing knowledge queries depend on the new table. This is intentionally +// best-effort for deployments that disable automatic migrations. +func (r *knowledgeRepository) projectKnowledgeFileUpdateSlots( + ctx context.Context, tenantID uint64, knowledges []*types.Knowledge, +) { + if len(knowledges) == 0 { + return + } + ids := make([]string, 0, len(knowledges)) + byID := make(map[string]*types.Knowledge, len(knowledges)) + for _, knowledge := range knowledges { + if knowledge == nil { + continue + } + knowledge.FileUpdateState = types.KnowledgeFileUpdateStateIdle + ids = append(ids, knowledge.ID) + byID[knowledge.ID] = knowledge + } + var slots []*types.KnowledgeFileUpdateSlot + if err := r.db.WithContext(ctx). + Where("tenant_id = ? AND knowledge_id IN ?", tenantID, ids). + Find(&slots).Error; err != nil { + return + } + for _, slot := range slots { + knowledge := byID[slot.KnowledgeID] + if knowledge == nil { + continue + } + knowledge.FileUpdateVersion = slot.LatestVersion + switch { + case slot.ActiveState == types.KnowledgeFileUpdateStateFailed: + knowledge.FileUpdateState = types.KnowledgeFileUpdateStateFailed + knowledge.FileUpdateError = "文件更新失败" + case slot.PendingVersion != nil: + knowledge.FileUpdateState = types.KnowledgeFileUpdateResultPending + case slot.ActiveVersion != nil: + knowledge.FileUpdateState = types.KnowledgeFileUpdateResultActive + default: + knowledge.FileUpdateState = types.KnowledgeFileUpdateStateIdle + } + } +} + // CheckKnowledgeExists checks if knowledge already exists func (r *knowledgeRepository) CheckKnowledgeExists( ctx context.Context, @@ -356,6 +411,31 @@ func (r *knowledgeRepository) CheckKnowledgeExists( ) (bool, *types.Knowledge, error) { query := r.db.WithContext(ctx).Model(&types.Knowledge{}). Where("tenant_id = ? AND knowledge_base_id = ? AND parse_status <> ?", tenantID, kbID, "failed") + return checkKnowledgeExistsQuery(query, params) +} + +// CheckKnowledgeExistsExcluding checks duplicate identity while excluding the +// row being replaced. It prevents a replacement from conflicting with itself +// without weakening duplicate protection against other knowledge rows. +func (r *knowledgeRepository) CheckKnowledgeExistsExcluding( + ctx context.Context, + tenantID uint64, + kbID string, + excludeKnowledgeID string, + params *types.KnowledgeCheckParams, +) (bool, *types.Knowledge, error) { + query := r.db.WithContext(ctx).Model(&types.Knowledge{}). + Where("tenant_id = ? AND knowledge_base_id = ? AND parse_status <> ?", tenantID, kbID, types.ParseStatusFailed) + if excludeKnowledgeID != "" { + query = query.Where("id <> ?", excludeKnowledgeID) + } + return checkKnowledgeExistsQuery(query, params) +} + +func checkKnowledgeExistsQuery(query *gorm.DB, params *types.KnowledgeCheckParams) (bool, *types.Knowledge, error) { + if params == nil { + return false, nil, nil + } switch params.Type { case "file": @@ -532,6 +612,403 @@ func (r *knowledgeRepository) UpdateKnowledgeColumns( return r.db.WithContext(ctx).Model(&types.Knowledge{}).Where("id = ?", id).Updates(values).Error } +// ClaimKnowledgeFileUpdate serializes in-place replacements using the exact +// source version observed by the request. GORM's normal model scope also +// excludes soft-deleted rows. +func (r *knowledgeRepository) ClaimKnowledgeFileUpdate( + ctx context.Context, + tenantID uint64, + knowledgeID string, + kbID string, + expectedStatus string, + expectedFilePath string, + expectedFileHash string, +) (bool, error) { + result := r.db.WithContext(ctx). + Model(&types.Knowledge{}). + Where( + "tenant_id = ? AND id = ? AND knowledge_base_id = ? AND parse_status = ? AND file_path = ? AND file_hash = ?", + tenantID, knowledgeID, kbID, expectedStatus, expectedFilePath, expectedFileHash, + ). + Updates(map[string]interface{}{ + "parse_status": types.ParseStatusReplacing, + "error_message": "", + "updated_at": time.Now(), + }) + if result.Error != nil { + return false, result.Error + } + return result.RowsAffected == 1, nil +} + +// UpdateApplyingKnowledgeFileColumns applies compensation or the final file +// switch only if the task still owns the source version it originally claimed. +func (r *knowledgeRepository) UpdateApplyingKnowledgeFileColumns( + ctx context.Context, + tenantID uint64, + knowledgeID string, + kbID string, + expectedFilePath string, + expectedFileHash string, + values map[string]interface{}, +) (bool, error) { + if len(values) == 0 { + return false, nil + } + result := r.db.WithContext(ctx). + Model(&types.Knowledge{}). + Where( + "tenant_id = ? AND id = ? AND knowledge_base_id = ? AND parse_status = ? AND file_path = ? AND file_hash = ?", + tenantID, knowledgeID, kbID, types.ParseStatusReplacing, expectedFilePath, expectedFileHash, + ). + Updates(values) + if result.Error != nil { + return false, result.Error + } + return result.RowsAffected == 1, nil +} + +// StageKnowledgeFileUpdate accepts a new latest-wins replacement under a row +// lock. When no active version exists, the new version is promoted directly. +func (r *knowledgeRepository) StageKnowledgeFileUpdate( + ctx context.Context, + tenantID uint64, + knowledgeID string, + kbID string, + payload types.JSON, + expectedVersion *uint64, +) (*types.KnowledgeFileUpdateStageResult, error) { + var staged *types.KnowledgeFileUpdateStageResult + err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + // Deletion locks the knowledge before its slot as well. Keeping the same + // order prevents an upload that passed the HTTP precheck from recreating + // a slot after deletion has claimed the resource. + knowledgeQuery := tx.Model(&types.Knowledge{}). + Select("id", "parse_status"). + Where("id = ? AND tenant_id = ? AND knowledge_base_id = ?", knowledgeID, tenantID, kbID) + if tx.Dialector.Name() != "sqlite" { + knowledgeQuery = knowledgeQuery.Clauses(clause.Locking{Strength: "UPDATE"}) + } + var knowledge types.Knowledge + if err := knowledgeQuery.Take(&knowledge).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return ErrKnowledgeNotFound + } + return err + } + if knowledge.ParseStatus == types.ParseStatusDeleting { + return ErrKnowledgeFileUpdateDeleting + } + + now := time.Now() + seed := &types.KnowledgeFileUpdateSlot{ + KnowledgeID: knowledgeID, + TenantID: tenantID, + KnowledgeBaseID: kbID, + ActiveState: types.KnowledgeFileUpdateStateIdle, + CreatedAt: now, + UpdatedAt: now, + } + if err := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(seed).Error; err != nil { + return err + } + + query := tx.Where("knowledge_id = ?", knowledgeID) + if tx.Dialector.Name() != "sqlite" { + query = query.Clauses(clause.Locking{Strength: "UPDATE"}) + } + var slot types.KnowledgeFileUpdateSlot + if err := query.First(&slot).Error; err != nil { + return err + } + if slot.TenantID != tenantID || slot.KnowledgeBaseID != kbID { + return ErrKnowledgeFileUpdateVersionConflict + } + if expectedVersion != nil && slot.LatestVersion != *expectedVersion { + return ErrKnowledgeFileUpdateVersionConflict + } + + version := slot.LatestVersion + 1 + updates := map[string]interface{}{ + "latest_version": version, + "last_error": "", + "updated_at": now, + } + state := types.KnowledgeFileUpdateResultPending + activeVersion := uint64(0) + if slot.ActiveVersion == nil || slot.ActiveState == types.KnowledgeFileUpdateStateFailed { + updates["active_version"] = version + updates["active_state"] = types.KnowledgeFileUpdateStateWaiting + updates["active_payload"] = payload + updates["pending_version"] = nil + updates["pending_payload"] = nil + state = types.KnowledgeFileUpdateResultActive + activeVersion = version + } else { + updates["pending_version"] = version + updates["pending_payload"] = payload + activeVersion = *slot.ActiveVersion + } + + result := tx.Model(&types.KnowledgeFileUpdateSlot{}). + Where("knowledge_id = ? AND latest_version = ?", knowledgeID, slot.LatestVersion). + Updates(updates) + if result.Error != nil { + return result.Error + } + if result.RowsAffected != 1 { + return ErrKnowledgeFileUpdateVersionConflict + } + staged = &types.KnowledgeFileUpdateStageResult{ + Version: version, + State: state, + ActiveVersion: activeVersion, + ReplacedPendingPayload: append(types.JSON(nil), slot.PendingPayload...), + } + if slot.ActiveState == types.KnowledgeFileUpdateStateFailed { + staged.ReplacedActivePayload = append(types.JSON(nil), slot.ActivePayload...) + } + return nil + }) + return staged, err +} + +func (r *knowledgeRepository) GetKnowledgeFileUpdateSlot( + ctx context.Context, tenantID uint64, knowledgeID string, +) (*types.KnowledgeFileUpdateSlot, error) { + var slot types.KnowledgeFileUpdateSlot + err := r.db.WithContext(ctx). + Where("tenant_id = ? AND knowledge_id = ?", tenantID, knowledgeID). + First(&slot).Error + if err != nil { + return nil, err + } + return &slot, nil +} + +func (r *knowledgeRepository) PrepareKnowledgeFileUpdate( + ctx context.Context, tenantID uint64, knowledgeID string, version uint64, payload types.JSON, +) (bool, error) { + result := r.db.WithContext(ctx).Model(&types.KnowledgeFileUpdateSlot{}). + Where("tenant_id = ? AND knowledge_id = ? AND active_version = ? AND active_state = ?", + tenantID, knowledgeID, version, types.KnowledgeFileUpdateStateWaiting). + Updates(map[string]interface{}{"active_payload": payload, "updated_at": time.Now()}) + return result.RowsAffected == 1, result.Error +} + +func (r *knowledgeRepository) TransitionKnowledgeFileUpdateState( + ctx context.Context, + tenantID uint64, + knowledgeID string, + version uint64, + fromState string, + toState string, + lastError string, +) (bool, error) { + updates := map[string]interface{}{ + "active_state": toState, + "updated_at": time.Now(), + } + if lastError != "" || toState != types.KnowledgeFileUpdateStateFailed { + updates["last_error"] = lastError + } + result := r.db.WithContext(ctx).Model(&types.KnowledgeFileUpdateSlot{}). + Where("tenant_id = ? AND knowledge_id = ? AND active_version = ? AND active_state = ?", + tenantID, knowledgeID, version, fromState). + Updates(updates) + return result.RowsAffected == 1, result.Error +} + +// CompleteKnowledgeFileUpdate clears the finished active version and promotes +// the latest pending version in the same transaction. +func (r *knowledgeRepository) CompleteKnowledgeFileUpdate( + ctx context.Context, tenantID uint64, knowledgeID string, version uint64, +) (*types.KnowledgeFileUpdateSlot, error) { + var completed *types.KnowledgeFileUpdateSlot + err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + query := tx.Where("tenant_id = ? AND knowledge_id = ?", tenantID, knowledgeID) + if tx.Dialector.Name() != "sqlite" { + query = query.Clauses(clause.Locking{Strength: "UPDATE"}) + } + var slot types.KnowledgeFileUpdateSlot + if err := query.First(&slot).Error; err != nil { + return err + } + if slot.ActiveVersion == nil || *slot.ActiveVersion != version { + completed = &slot + return nil + } + + updates := map[string]interface{}{ + "active_version": nil, + "active_state": types.KnowledgeFileUpdateStateIdle, + "active_payload": nil, + "last_error": "", + "updated_at": time.Now(), + } + if slot.PendingVersion != nil { + updates["active_version"] = *slot.PendingVersion + updates["active_state"] = types.KnowledgeFileUpdateStateWaiting + updates["active_payload"] = slot.PendingPayload + updates["pending_version"] = nil + updates["pending_payload"] = nil + } + if err := tx.Model(&types.KnowledgeFileUpdateSlot{}). + Where("knowledge_id = ? AND active_version = ?", knowledgeID, version). + Updates(updates).Error; err != nil { + return err + } + if err := tx.Where("knowledge_id = ?", knowledgeID).First(&slot).Error; err != nil { + return err + } + completed = &slot + return nil + }) + return completed, err +} + +// CancelKnowledgeFileUpdates deletes the coordination row and returns its +// payloads so the caller can clean staged files after the transaction commits. +func (r *knowledgeRepository) CancelKnowledgeFileUpdates( + ctx context.Context, tenantID uint64, knowledgeID string, +) (*types.KnowledgeFileUpdateSlot, error) { + var cancelled *types.KnowledgeFileUpdateSlot + err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + query := tx.Where("tenant_id = ? AND knowledge_id = ?", tenantID, knowledgeID) + if tx.Dialector.Name() != "sqlite" { + query = query.Clauses(clause.Locking{Strength: "UPDATE"}) + } + var slot types.KnowledgeFileUpdateSlot + if err := query.First(&slot).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil + } + return err + } + if err := tx.Delete(&types.KnowledgeFileUpdateSlot{}, "knowledge_id = ?", knowledgeID).Error; err != nil { + return err + } + cancelled = &slot + return nil + }) + return cancelled, err +} + +// CancelFailedKnowledgeFileUpdate removes only the exact failed active +// version observed by the caller. A concurrent upload cannot be discarded. +func (r *knowledgeRepository) CancelFailedKnowledgeFileUpdate( + ctx context.Context, tenantID uint64, knowledgeID string, activeVersion uint64, +) (*types.KnowledgeFileUpdateSlot, error) { + var cancelled *types.KnowledgeFileUpdateSlot + err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + query := tx.Where("tenant_id = ? AND knowledge_id = ?", tenantID, knowledgeID) + if tx.Dialector.Name() != "sqlite" { + query = query.Clauses(clause.Locking{Strength: "UPDATE"}) + } + var slot types.KnowledgeFileUpdateSlot + if err := query.First(&slot).Error; err != nil { + return err + } + if slot.ActiveVersion == nil || *slot.ActiveVersion != activeVersion || + slot.ActiveState != types.KnowledgeFileUpdateStateFailed { + return ErrKnowledgeFileUpdateStateConflict + } + result := tx.Where( + "tenant_id = ? AND knowledge_id = ? AND active_version = ? AND active_state = ?", + tenantID, knowledgeID, activeVersion, types.KnowledgeFileUpdateStateFailed, + ).Delete(&types.KnowledgeFileUpdateSlot{}) + if result.Error != nil { + return result.Error + } + if result.RowsAffected != 1 { + return ErrKnowledgeFileUpdateStateConflict + } + cancelled = &slot + return nil + }) + return cancelled, err +} + +// BeginKnowledgeDeletion serializes deletion against file-update staging and +// returns removed slots so staged files can be cleaned after the commit. +func (r *knowledgeRepository) BeginKnowledgeDeletion( + ctx context.Context, tenantID uint64, knowledgeIDs []string, +) ([]*types.KnowledgeFileUpdateSlot, error) { + if len(knowledgeIDs) == 0 { + return nil, nil + } + uniqueIDs := make([]string, 0, len(knowledgeIDs)) + seen := make(map[string]struct{}, len(knowledgeIDs)) + for _, id := range knowledgeIDs { + if id == "" { + continue + } + if _, ok := seen[id]; ok { + continue + } + seen[id] = struct{}{} + uniqueIDs = append(uniqueIDs, id) + } + if len(uniqueIDs) == 0 { + return nil, ErrKnowledgeNotFound + } + + var cancelled []*types.KnowledgeFileUpdateSlot + err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + knowledgeQuery := tx.Model(&types.Knowledge{}). + Select("id"). + Where("tenant_id = ? AND id IN ?", tenantID, uniqueIDs). + Order("id ASC") + if tx.Dialector.Name() != "sqlite" { + knowledgeQuery = knowledgeQuery.Clauses(clause.Locking{Strength: "UPDATE"}) + } + var rows []types.Knowledge + if err := knowledgeQuery.Find(&rows).Error; err != nil { + return err + } + if len(rows) != len(uniqueIDs) { + return ErrKnowledgeNotFound + } + + if err := tx.Model(&types.Knowledge{}). + Where("tenant_id = ? AND id IN ?", tenantID, uniqueIDs). + Updates(map[string]interface{}{ + "parse_status": types.ParseStatusDeleting, + "updated_at": time.Now(), + }).Error; err != nil { + return err + } + + slotQuery := tx.Where("tenant_id = ? AND knowledge_id IN ?", tenantID, uniqueIDs). + Order("knowledge_id ASC") + if tx.Dialector.Name() != "sqlite" { + slotQuery = slotQuery.Clauses(clause.Locking{Strength: "UPDATE"}) + } + if err := slotQuery.Find(&cancelled).Error; err != nil { + return err + } + if len(cancelled) == 0 { + return nil + } + return tx.Where("tenant_id = ? AND knowledge_id IN ?", tenantID, uniqueIDs). + Delete(&types.KnowledgeFileUpdateSlot{}).Error + }) + return cancelled, err +} + +func (r *knowledgeRepository) ListRecoverableKnowledgeFileUpdates( + ctx context.Context, limit int, +) ([]*types.KnowledgeFileUpdateSlot, error) { + if limit <= 0 { + limit = 1000 + } + var slots []*types.KnowledgeFileUpdateSlot + err := r.db.WithContext(ctx). + Where("active_version IS NOT NULL OR pending_version IS NOT NULL"). + Order("updated_at ASC").Limit(limit).Find(&slots).Error + return slots, err +} + // UpdateActiveDeletingKnowledgeColumns only touches rows that are still visible // to normal queries and have not moved out of the transient deleting state. func (r *knowledgeRepository) UpdateActiveDeletingKnowledgeColumns( diff --git a/internal/application/repository/knowledge_file_update_apply_test.go b/internal/application/repository/knowledge_file_update_apply_test.go new file mode 100644 index 0000000000..3cee69dd5a --- /dev/null +++ b/internal/application/repository/knowledge_file_update_apply_test.go @@ -0,0 +1,215 @@ +package repository + +import ( + "context" + "sync" + "sync/atomic" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm" +) + +// insertFileKnowledge seeds a file knowledge row with an explicit source +// version (path + hash) so the in-place replacement helpers can be exercised +// against their exact optimistic-lock predicates. +func insertFileKnowledge( + t *testing.T, db *gorm.DB, tenantID uint64, kbID, status, filePath, fileHash string, +) string { + t.Helper() + id := uuid.New().String() + require.NoError(t, db.Exec(` + INSERT INTO knowledges + (id, tenant_id, knowledge_base_id, type, title, source, parse_status, file_name, file_size, file_path, file_hash) + VALUES (?, ?, ?, 'file', 'replace-test', 'file', ?, 'report.pdf', 1024, ?, ?) + `, id, tenantID, kbID, status, filePath, fileHash).Error) + return id +} + +func reloadFileVersion(t *testing.T, db *gorm.DB, id string) (status, path, hash string) { + t.Helper() + row := db.Raw(`SELECT parse_status, file_path, file_hash FROM knowledges WHERE id = ?`, id).Row() + require.NoError(t, row.Scan(&status, &path, &hash)) + return status, path, hash +} + +func reloadFileDisplay(t *testing.T, db *gorm.DB, id string) (title, fileName, folderPath string) { + t.Helper() + row := db.Raw(`SELECT title, file_name, folder_path FROM knowledges WHERE id = ?`, id).Row() + require.NoError(t, row.Scan(&title, &fileName, &folderPath)) + return title, fileName, folderPath +} + +// TestClaimKnowledgeFileUpdate_Success verifies the happy path: a terminal +// row matching the observed version transitions to replacing exactly once. +func TestClaimKnowledgeFileUpdate_Success(t *testing.T) { + db := setupKnowledgeTestDB(t) + repo := NewKnowledgeRepository(db).(*knowledgeRepository) + ctx := context.Background() + + kbID := uuid.New().String() + id := insertFileKnowledge(t, db, 1, kbID, types.ParseStatusCompleted, "old/path.pdf", "hash-old") + + claimed, err := repo.ClaimKnowledgeFileUpdate(ctx, 1, id, kbID, + types.ParseStatusCompleted, "old/path.pdf", "hash-old") + require.NoError(t, err) + assert.True(t, claimed) + + status, _, _ := reloadFileVersion(t, db, id) + assert.Equal(t, types.ParseStatusReplacing, status) +} + +// TestClaimKnowledgeFileUpdate_GuardsMismatch verifies that a claim fails +// without mutating the row when any part of the observed identity is stale: +// wrong status, wrong hash, wrong path, wrong tenant, or wrong KB. +func TestClaimKnowledgeFileUpdate_GuardsMismatch(t *testing.T) { + db := setupKnowledgeTestDB(t) + repo := NewKnowledgeRepository(db).(*knowledgeRepository) + ctx := context.Background() + + kbID := uuid.New().String() + id := insertFileKnowledge(t, db, 1, kbID, types.ParseStatusCompleted, "old/path.pdf", "hash-old") + + cases := []struct { + name string + tenant uint64 + kb string + status string + path string + hash string + }{ + {"wrong status", 1, kbID, types.ParseStatusPending, "old/path.pdf", "hash-old"}, + {"wrong hash", 1, kbID, types.ParseStatusCompleted, "old/path.pdf", "hash-different"}, + {"wrong path", 1, kbID, types.ParseStatusCompleted, "other/path.pdf", "hash-old"}, + {"wrong tenant", 2, kbID, types.ParseStatusCompleted, "old/path.pdf", "hash-old"}, + {"wrong kb", 1, uuid.New().String(), types.ParseStatusCompleted, "old/path.pdf", "hash-old"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + claimed, err := repo.ClaimKnowledgeFileUpdate(ctx, tc.tenant, id, tc.kb, tc.status, tc.path, tc.hash) + require.NoError(t, err) + assert.False(t, claimed) + status, _, _ := reloadFileVersion(t, db, id) + assert.Equal(t, types.ParseStatusCompleted, status, "row must stay in its terminal state") + }) + } +} + +// TestClaimKnowledgeFileUpdate_ConcurrentExactlyOne is the core concurrency +// guarantee: two requests reading the same version must produce exactly one +// winner, so only one replacement task is ever enqueued for a knowledge. +func TestClaimKnowledgeFileUpdate_ConcurrentExactlyOne(t *testing.T) { + db := setupKnowledgeTestDB(t) + repo := NewKnowledgeRepository(db).(*knowledgeRepository) + ctx := context.Background() + + kbID := uuid.New().String() + id := insertFileKnowledge(t, db, 1, kbID, types.ParseStatusFailed, "old/path.pdf", "hash-old") + + const n = 16 + var wins atomic.Int32 + var wg sync.WaitGroup + for i := 0; i < n; i++ { + wg.Add(1) + go func() { + defer wg.Done() + claimed, err := repo.ClaimKnowledgeFileUpdate(ctx, 1, id, kbID, + types.ParseStatusFailed, "old/path.pdf", "hash-old") + if err != nil { + t.Errorf("claim: %v", err) + return + } + if claimed { + wins.Add(1) + } + }() + } + wg.Wait() + + assert.Equal(t, int32(1), wins.Load(), "exactly one concurrent claim must win") + status, _, _ := reloadFileVersion(t, db, id) + assert.Equal(t, types.ParseStatusReplacing, status) +} + +// TestUpdateApplyingKnowledgeFileColumns_GuardsVersion verifies the file switch / +// compensation write only lands while the row is still the replacing version +// the task originally claimed. +func TestUpdateApplyingKnowledgeFileColumns_GuardsVersion(t *testing.T) { + db := setupKnowledgeTestDB(t) + repo := NewKnowledgeRepository(db).(*knowledgeRepository) + ctx := context.Background() + + kbID := uuid.New().String() + id := insertFileKnowledge(t, db, 1, kbID, types.ParseStatusReplacing, "old/path.pdf", "hash-old") + + // Stale version (wrong hash) must not update. + updated, err := repo.UpdateApplyingKnowledgeFileColumns(ctx, 1, id, kbID, "old/path.pdf", "stale-hash", + map[string]interface{}{"parse_status": types.ParseStatusFailed}) + require.NoError(t, err) + assert.False(t, updated) + + // Empty values map is a no-op. + updated, err = repo.UpdateApplyingKnowledgeFileColumns(ctx, 1, id, kbID, "old/path.pdf", "hash-old", nil) + require.NoError(t, err) + assert.False(t, updated) + + // Correct version applies the switch. + updated, err = repo.UpdateApplyingKnowledgeFileColumns(ctx, 1, id, kbID, "old/path.pdf", "hash-old", + map[string]interface{}{ + "file_path": "new/path.pdf", + "file_name": "new-name.pdf", + "title": "new-name.pdf", + "folder_path": "docs/spec", + "file_hash": "hash-new", + "parse_status": types.ParseStatusPending, + }) + require.NoError(t, err) + assert.True(t, updated) + + status, path, hash := reloadFileVersion(t, db, id) + assert.Equal(t, types.ParseStatusPending, status) + assert.Equal(t, "new/path.pdf", path) + assert.Equal(t, "hash-new", hash) + title, fileName, folderPath := reloadFileDisplay(t, db, id) + assert.Equal(t, "new-name.pdf", title) + assert.Equal(t, "new-name.pdf", fileName) + assert.Equal(t, "docs/spec", folderPath) + + // A second write with the old version is now stale and must not land. + updated, err = repo.UpdateApplyingKnowledgeFileColumns(ctx, 1, id, kbID, "old/path.pdf", "hash-old", + map[string]interface{}{"parse_status": types.ParseStatusFailed}) + require.NoError(t, err) + assert.False(t, updated) +} + +// TestCheckKnowledgeExistsExcluding verifies duplicate detection skips the row +// being replaced but still catches a collision with a different knowledge. +func TestCheckKnowledgeExistsExcluding(t *testing.T) { + db := setupKnowledgeTestDB(t) + repo := NewKnowledgeRepository(db).(*knowledgeRepository) + ctx := context.Background() + + kbID := uuid.New().String() + selfID := insertFileKnowledge(t, db, 1, kbID, types.ParseStatusCompleted, "self/path.pdf", "hash-shared") + + // Same hash but excluding self: no duplicate. + exists, dup, err := repo.CheckKnowledgeExistsExcluding(ctx, 1, kbID, selfID, &types.KnowledgeCheckParams{ + Type: "file", FileHash: "hash-shared", + }) + require.NoError(t, err) + assert.False(t, exists) + assert.Nil(t, dup) + + // A different knowledge with the same hash is a real duplicate. + otherID := insertFileKnowledge(t, db, 1, kbID, types.ParseStatusCompleted, "other/path.pdf", "hash-shared") + exists, dup, err = repo.CheckKnowledgeExistsExcluding(ctx, 1, kbID, selfID, &types.KnowledgeCheckParams{ + Type: "file", FileHash: "hash-shared", + }) + require.NoError(t, err) + assert.True(t, exists) + require.NotNil(t, dup) + assert.Equal(t, otherID, dup.ID) +} diff --git a/internal/application/repository/knowledge_file_update_slot_test.go b/internal/application/repository/knowledge_file_update_slot_test.go new file mode 100644 index 0000000000..cc186b4c9b --- /dev/null +++ b/internal/application/repository/knowledge_file_update_slot_test.go @@ -0,0 +1,300 @@ +package repository + +import ( + "context" + "encoding/json" + "sync" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm" +) + +const knowledgeFileUpdateSlotsTestDDL = ` +CREATE TABLE knowledge_file_update_slots ( + knowledge_id VARCHAR(36) PRIMARY KEY, + tenant_id INTEGER NOT NULL, + knowledge_base_id VARCHAR(36) NOT NULL, + latest_version INTEGER NOT NULL DEFAULT 0, + active_version INTEGER, + active_state VARCHAR(16) NOT NULL DEFAULT 'idle', + active_payload TEXT, + pending_version INTEGER, + pending_payload TEXT, + last_error TEXT NOT NULL DEFAULT '', + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + CHECK (active_state IN ('idle', 'waiting', 'applying', 'retry_wait', 'failed')), + CHECK ((active_state = 'idle') = (active_version IS NULL)), + CHECK ((active_version IS NULL) = (active_payload IS NULL)), + CHECK ((pending_version IS NULL) = (pending_payload IS NULL)) +);` + +func setupKnowledgeFileUpdateSlotTestDB(t *testing.T) (*gorm.DB, *knowledgeRepository) { + t.Helper() + db := setupKnowledgeTestDB(t) + require.NoError(t, db.Exec(knowledgeFileUpdateSlotsTestDDL).Error) + return db, NewKnowledgeRepository(db).(*knowledgeRepository) +} + +func updatePayload(t *testing.T, path, hash string) types.JSON { + t.Helper() + payload, err := json.Marshal(types.KnowledgeFileUpdatePayload{ + NewFilePath: path, + NewFileHash: hash, + NewFileName: "document.md", + NewFileType: "md", + }) + require.NoError(t, err) + return types.JSON(payload) +} + +func decodeUpdatePayload(t *testing.T, payload types.JSON) types.KnowledgeFileUpdatePayload { + t.Helper() + var decoded types.KnowledgeFileUpdatePayload + require.NoError(t, json.Unmarshal(payload, &decoded)) + return decoded +} + +func TestStageKnowledgeFileUpdateLatestWinsAndPromotes(t *testing.T) { + db, repo := setupKnowledgeFileUpdateSlotTestDB(t) + ctx := context.Background() + kbID := uuid.New().String() + knowledgeID := insertFileKnowledge( + t, db, 1, kbID, types.ParseStatusCompleted, "old/path.md", "old-hash", + ) + + first, err := repo.StageKnowledgeFileUpdate(ctx, 1, knowledgeID, kbID, updatePayload(t, "a", "ha"), nil) + require.NoError(t, err) + assert.Equal(t, uint64(1), first.Version) + assert.Equal(t, types.KnowledgeFileUpdateResultActive, first.State) + + second, err := repo.StageKnowledgeFileUpdate(ctx, 1, knowledgeID, kbID, updatePayload(t, "b", "hb"), nil) + require.NoError(t, err) + assert.Equal(t, uint64(2), second.Version) + assert.Equal(t, types.KnowledgeFileUpdateResultPending, second.State) + assert.Empty(t, second.ReplacedPendingPayload) + + third, err := repo.StageKnowledgeFileUpdate(ctx, 1, knowledgeID, kbID, updatePayload(t, "c", "hc"), nil) + require.NoError(t, err) + assert.Equal(t, uint64(3), third.Version) + assert.Equal(t, "b", decodeUpdatePayload(t, third.ReplacedPendingPayload).NewFilePath) + + slot, err := repo.GetKnowledgeFileUpdateSlot(ctx, 1, knowledgeID) + require.NoError(t, err) + require.NotNil(t, slot.ActiveVersion) + require.NotNil(t, slot.PendingVersion) + assert.Equal(t, uint64(1), *slot.ActiveVersion) + assert.Equal(t, uint64(3), *slot.PendingVersion) + assert.Equal(t, "a", decodeUpdatePayload(t, slot.ActivePayload).NewFilePath) + assert.Equal(t, "c", decodeUpdatePayload(t, slot.PendingPayload).NewFilePath) + + slot, err = repo.CompleteKnowledgeFileUpdate(ctx, 1, knowledgeID, 1) + require.NoError(t, err) + require.NotNil(t, slot.ActiveVersion) + assert.Equal(t, uint64(3), *slot.ActiveVersion) + assert.Nil(t, slot.PendingVersion) + assert.Equal(t, types.KnowledgeFileUpdateStateWaiting, slot.ActiveState) + assert.Equal(t, "c", decodeUpdatePayload(t, slot.ActivePayload).NewFilePath) +} + +func TestStageKnowledgeFileUpdateExpectedVersion(t *testing.T) { + db, repo := setupKnowledgeFileUpdateSlotTestDB(t) + ctx := context.Background() + kbID := uuid.New().String() + knowledgeID := insertFileKnowledge( + t, db, 1, kbID, types.ParseStatusCompleted, "old/path.md", "old-hash", + ) + + _, err := repo.StageKnowledgeFileUpdate(ctx, 1, knowledgeID, kbID, updatePayload(t, "a", "ha"), nil) + require.NoError(t, err) + stale := uint64(0) + _, err = repo.StageKnowledgeFileUpdate(ctx, 1, knowledgeID, kbID, updatePayload(t, "b", "hb"), &stale) + require.ErrorIs(t, err, ErrKnowledgeFileUpdateVersionConflict) + + slot, err := repo.GetKnowledgeFileUpdateSlot(ctx, 1, knowledgeID) + require.NoError(t, err) + assert.Equal(t, uint64(1), slot.LatestVersion) +} + +func TestStageKnowledgeFileUpdateConcurrentQueueIsBounded(t *testing.T) { + db, repo := setupKnowledgeFileUpdateSlotTestDB(t) + ctx := context.Background() + kbID := uuid.New().String() + knowledgeID := insertFileKnowledge( + t, db, 1, kbID, types.ParseStatusCompleted, "old/path.md", "old-hash", + ) + + const requests = 100 + var wg sync.WaitGroup + for i := 0; i < requests; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + _, err := repo.StageKnowledgeFileUpdate( + ctx, 1, knowledgeID, kbID, + updatePayload(t, "path-"+string(rune('a'+i)), "hash"), nil, + ) + assert.NoError(t, err) + }(i) + } + wg.Wait() + + slot, err := repo.GetKnowledgeFileUpdateSlot(ctx, 1, knowledgeID) + require.NoError(t, err) + assert.Equal(t, uint64(requests), slot.LatestVersion) + require.NotNil(t, slot.ActiveVersion) + require.NotNil(t, slot.PendingVersion) + assert.NotEmpty(t, slot.ActivePayload) + assert.NotEmpty(t, slot.PendingPayload) +} + +func TestStageKnowledgeFileUpdateReplacesFailedActiveAndPending(t *testing.T) { + db, repo := setupKnowledgeFileUpdateSlotTestDB(t) + ctx := context.Background() + kbID := uuid.New().String() + knowledgeID := insertFileKnowledge( + t, db, 1, kbID, types.ParseStatusCompleted, "old/path.md", "old-hash", + ) + + first, err := repo.StageKnowledgeFileUpdate(ctx, 1, knowledgeID, kbID, updatePayload(t, "a", "ha"), nil) + require.NoError(t, err) + _, err = repo.StageKnowledgeFileUpdate(ctx, 1, knowledgeID, kbID, updatePayload(t, "b", "hb"), nil) + require.NoError(t, err) + failed, err := repo.TransitionKnowledgeFileUpdateState( + ctx, 1, knowledgeID, first.ActiveVersion, + types.KnowledgeFileUpdateStateWaiting, types.KnowledgeFileUpdateStateFailed, "apply failed", + ) + require.NoError(t, err) + require.True(t, failed) + + latest, err := repo.StageKnowledgeFileUpdate(ctx, 1, knowledgeID, kbID, updatePayload(t, "c", "hc"), nil) + require.NoError(t, err) + assert.Equal(t, uint64(3), latest.Version) + assert.Equal(t, types.KnowledgeFileUpdateResultActive, latest.State) + assert.Equal(t, "a", decodeUpdatePayload(t, latest.ReplacedActivePayload).NewFilePath) + assert.Equal(t, "b", decodeUpdatePayload(t, latest.ReplacedPendingPayload).NewFilePath) + + slot, err := repo.GetKnowledgeFileUpdateSlot(ctx, 1, knowledgeID) + require.NoError(t, err) + require.NotNil(t, slot.ActiveVersion) + assert.Equal(t, uint64(3), *slot.ActiveVersion) + assert.Nil(t, slot.PendingVersion) + assert.Equal(t, types.KnowledgeFileUpdateStateWaiting, slot.ActiveState) + assert.Equal(t, "c", decodeUpdatePayload(t, slot.ActivePayload).NewFilePath) +} + +func TestKnowledgeQueryProjectsLatestFileUpdateState(t *testing.T) { + db, repo := setupKnowledgeFileUpdateSlotTestDB(t) + ctx := context.Background() + kbID := uuid.New().String() + knowledgeID := insertFileKnowledge( + t, db, 1, kbID, types.ParseStatusCompleted, "old/path.md", "old-hash", + ) + + _, err := repo.StageKnowledgeFileUpdate(ctx, 1, knowledgeID, kbID, updatePayload(t, "a", "ha"), nil) + require.NoError(t, err) + _, err = repo.StageKnowledgeFileUpdate(ctx, 1, knowledgeID, kbID, updatePayload(t, "b", "hb"), nil) + require.NoError(t, err) + + knowledge, err := repo.GetKnowledgeByID(ctx, 1, knowledgeID) + require.NoError(t, err) + assert.Equal(t, uint64(2), knowledge.FileUpdateVersion) + assert.Equal(t, types.KnowledgeFileUpdateResultPending, knowledge.FileUpdateState) +} + +func TestBeginKnowledgeDeletionMarksDeletingAndRevokesSlot(t *testing.T) { + db, repo := setupKnowledgeFileUpdateSlotTestDB(t) + ctx := context.Background() + kbID := uuid.NewString() + knowledgeID := insertFileKnowledge( + t, db, 1, kbID, types.ParseStatusCompleted, "old/path.md", "old-hash", + ) + _, err := repo.StageKnowledgeFileUpdate( + ctx, 1, knowledgeID, kbID, updatePayload(t, "staged/a.md", "ha"), nil, + ) + require.NoError(t, err) + _, err = repo.StageKnowledgeFileUpdate( + ctx, 1, knowledgeID, kbID, updatePayload(t, "staged/b.md", "hb"), nil, + ) + require.NoError(t, err) + + cancelled, err := repo.BeginKnowledgeDeletion(ctx, 1, []string{knowledgeID}) + require.NoError(t, err) + require.Len(t, cancelled, 1) + assert.Equal(t, "staged/a.md", decodeUpdatePayload(t, cancelled[0].ActivePayload).NewFilePath) + assert.Equal(t, "staged/b.md", decodeUpdatePayload(t, cancelled[0].PendingPayload).NewFilePath) + + status, _, _ := reloadFileVersion(t, db, knowledgeID) + assert.Equal(t, types.ParseStatusDeleting, status) + _, err = repo.GetKnowledgeFileUpdateSlot(ctx, 1, knowledgeID) + require.ErrorIs(t, err, gorm.ErrRecordNotFound) + + _, err = repo.StageKnowledgeFileUpdate( + ctx, 1, knowledgeID, kbID, updatePayload(t, "staged/late.md", "late"), nil, + ) + require.ErrorIs(t, err, ErrKnowledgeFileUpdateDeleting) + _, err = repo.GetKnowledgeFileUpdateSlot(ctx, 1, knowledgeID) + require.ErrorIs(t, err, gorm.ErrRecordNotFound) +} + +func TestBeginKnowledgeDeletionRollsBackStatusWhenSlotDeleteFails(t *testing.T) { + db, repo := setupKnowledgeFileUpdateSlotTestDB(t) + ctx := context.Background() + kbID := uuid.NewString() + knowledgeID := insertFileKnowledge( + t, db, 1, kbID, types.ParseStatusCompleted, "old/path.md", "old-hash", + ) + _, err := repo.StageKnowledgeFileUpdate( + ctx, 1, knowledgeID, kbID, updatePayload(t, "staged/a.md", "ha"), nil, + ) + require.NoError(t, err) + require.NoError(t, db.Exec(` + CREATE TRIGGER reject_file_update_slot_delete + BEFORE DELETE ON knowledge_file_update_slots + BEGIN + SELECT RAISE(ABORT, 'slot delete rejected'); + END; + `).Error) + + _, err = repo.BeginKnowledgeDeletion(ctx, 1, []string{knowledgeID}) + require.Error(t, err) + status, _, _ := reloadFileVersion(t, db, knowledgeID) + assert.Equal(t, types.ParseStatusCompleted, status) + _, err = repo.GetKnowledgeFileUpdateSlot(ctx, 1, knowledgeID) + require.NoError(t, err) +} + +func TestCancelFailedKnowledgeFileUpdateDoesNotRemoveNewerActive(t *testing.T) { + db, repo := setupKnowledgeFileUpdateSlotTestDB(t) + ctx := context.Background() + kbID := uuid.NewString() + knowledgeID := insertFileKnowledge( + t, db, 1, kbID, types.ParseStatusCompleted, "old/path.md", "old-hash", + ) + first, err := repo.StageKnowledgeFileUpdate( + ctx, 1, knowledgeID, kbID, updatePayload(t, "staged/a.md", "ha"), nil, + ) + require.NoError(t, err) + failed, err := repo.TransitionKnowledgeFileUpdateState( + ctx, 1, knowledgeID, first.ActiveVersion, + types.KnowledgeFileUpdateStateWaiting, types.KnowledgeFileUpdateStateFailed, "failed", + ) + require.NoError(t, err) + require.True(t, failed) + latest, err := repo.StageKnowledgeFileUpdate( + ctx, 1, knowledgeID, kbID, updatePayload(t, "staged/b.md", "hb"), nil, + ) + require.NoError(t, err) + + _, err = repo.CancelFailedKnowledgeFileUpdate(ctx, 1, knowledgeID, first.ActiveVersion) + require.ErrorIs(t, err, ErrKnowledgeFileUpdateStateConflict) + slot, err := repo.GetKnowledgeFileUpdateSlot(ctx, 1, knowledgeID) + require.NoError(t, err) + require.NotNil(t, slot.ActiveVersion) + assert.Equal(t, latest.ActiveVersion, *slot.ActiveVersion) + assert.Equal(t, "staged/b.md", decodeUpdatePayload(t, slot.ActivePayload).NewFilePath) +} diff --git a/internal/application/repository/retriever/opensearch/bulk_update.go b/internal/application/repository/retriever/opensearch/bulk_update.go index 97e64caf79..2e3927ec4a 100644 --- a/internal/application/repository/retriever/opensearch/bulk_update.go +++ b/internal/application/repository/retriever/opensearch/bulk_update.go @@ -75,6 +75,9 @@ func (r *Repository) BatchUpdateChunkTagID(ctx context.Context, chunkTagMap map[ func (r *Repository) updateByQueryScript( ctx context.Context, chunkIDs []string, source string, params map[string]any, ) error { + if err := r.ensureClusterValidated(ctx); err != nil { + return err + } body, err := json.Marshal(map[string]any{ "query": map[string]any{ "terms": map[string]any{"chunk_id": chunkIDs}, diff --git a/internal/application/repository/retriever/opensearch/mapping.go b/internal/application/repository/retriever/opensearch/mapping.go index ca3b2ad906..34e85a5ffe 100644 --- a/internal/application/repository/retriever/opensearch/mapping.go +++ b/internal/application/repository/retriever/opensearch/mapping.go @@ -195,6 +195,9 @@ func buildKeywordsMapping(cfg internalCfg) ([]byte, error) { // failures can be retried by the next caller (sync.Once cannot be // reset). func (r *Repository) ensureKeywordsIndex(ctx context.Context) error { + if err := r.ensureClusterValidated(ctx); err != nil { + return err + } r.keywordsMu.Lock() defer r.keywordsMu.Unlock() diff --git a/internal/application/repository/retriever/opensearch/repository.go b/internal/application/repository/retriever/opensearch/repository.go index 8ffe7a7330..1e8b9a9a62 100644 --- a/internal/application/repository/retriever/opensearch/repository.go +++ b/internal/application/repository/retriever/opensearch/repository.go @@ -26,10 +26,11 @@ import ( // except the once / initErr maps are read-only after NewRepository returns; // per-dimension index initialization is guarded by sync.Once. // -// Lifecycle: NewRepository validates connectivity + cluster version + -// k-NN plugin, but does NOT create any index. Index creation happens -// lazily on first Save / BatchSave / Retrieve once the embedding -// dimension is known (see ensureReady — per-dimension index naming). +// Lifecycle: NewRepository validates cluster version + k-NN plugin when the +// cluster is reachable. A transient transport failure defers that probe until +// first use so a short OpenSearch outage does not prevent the API process from +// starting. It does NOT create any index. Index creation happens lazily on +// first Save / BatchSave / Retrieve once the embedding dimension is known. // // Concurrency: A single Repository instance is shared across N goroutines // concurrently retrieving from the same store (the multi-store fan-out @@ -65,6 +66,13 @@ type Repository struct { keywordsReady bool keywordsErr error + // A startup transport failure defers the version/plugin probe. The first + // real operation retries it under this mutex; permanent probe failures are + // cached until restart, while transport failures remain retryable. + clusterProbeMu sync.Mutex + clusterProbePending bool + clusterProbeErr error + // sink receives audit events (index created / reindex executed). nil // means no auditing; use r.auditSink() to get a non-nil sink. Set via // WithAuditSink at construction. @@ -75,11 +83,10 @@ type Repository struct { // red if the interface drifts and our implementation lags). var _ interfaces.RetrieveEngineRepository = (*Repository)(nil) -// NewRepository builds a new OpenSearch k-NN repository and verifies the -// backing cluster is reachable + version-compatible + has the k-NN -// plugin installed on every cluster node. It does NOT create any index — -// callers (Save / Retrieve) trigger lazy per-dimension creation via -// ensureReady on first use. +// NewRepository builds a new OpenSearch k-NN repository. Permanent validation +// errors (authentication, unsupported version, missing k-NN plugin) fail the +// constructor. Transient transport errors defer validation until first use. +// It does NOT create any index; callers trigger lazy per-dimension creation. // // storeID is the VectorStore.ID owning this repository instance. It is // folded into the base index name so multiple OpenSearch VectorStores @@ -131,13 +138,6 @@ func NewRepository( return nil, fmt.Errorf("opensearch: invalid index config: %w", err) } - if err := probeVersion(ctx, client); err != nil { - return nil, err // already wraps ErrVersionUnsupported / ErrTransport - } - if err := probeKNNPlugin(ctx, client); err != nil { - return nil, err // already wraps ErrConfigInvalid / ErrTransport - } - r := &Repository{ client: client, baseIndex: base, @@ -148,11 +148,47 @@ func NewRepository( for _, opt := range opts { opt(r) } + if err := probeCluster(ctx, client); err != nil { + if !isTransientErr(err) { + return nil, err + } + r.clusterProbePending = true + log.Warnf("[OpenSearch] startup probe deferred after transient failure: %v", err) + return r, nil + } log.Infof("[OpenSearch] repository ready (baseIndex=%s, knn_engine=%s, hnsw_m=%d)", base, icfg.knnEngine, icfg.hnswM) return r, nil } +func probeCluster(ctx context.Context, client *osapi.Client) error { + if err := probeVersion(ctx, client); err != nil { + return err + } + return probeKNNPlugin(ctx, client) +} + +func (r *Repository) ensureClusterValidated(ctx context.Context) error { + r.clusterProbeMu.Lock() + defer r.clusterProbeMu.Unlock() + if r.clusterProbeErr != nil { + return r.clusterProbeErr + } + if !r.clusterProbePending { + return nil + } + if err := probeCluster(ctx, r.client); err != nil { + if !isTransientErr(err) { + r.clusterProbeErr = err + r.clusterProbePending = false + } + return err + } + r.clusterProbePending = false + logger.GetLogger(ctx).Infof("[OpenSearch] deferred startup probe recovered (baseIndex=%s)", r.baseIndex) + return nil +} + // ensureReady creates the per-dimension index (alias-backed) the first // time a given embedding dimension is seen. Concurrent callers for the // same dim block on the same sync.Once. @@ -180,6 +216,9 @@ func (r *Repository) ensureReady(ctx context.Context, dim int) error { return fmt.Errorf("opensearch: dim %d out of range (1..16000): %w", dim, ErrDimensionMismatch) } + if err := r.ensureClusterValidated(ctx); err != nil { + return err + } r.onceMu.Lock() once, ok := r.once[dim] diff --git a/internal/application/repository/retriever/opensearch/repository_test.go b/internal/application/repository/retriever/opensearch/repository_test.go index 687f0cd440..8d444fa2be 100644 --- a/internal/application/repository/retriever/opensearch/repository_test.go +++ b/internal/application/repository/retriever/opensearch/repository_test.go @@ -1122,6 +1122,60 @@ func TestNewRepository_AcceptsLongStoreID(t *testing.T) { } } +func TestNewRepositoryDefersTransientStartupProbeAndRecovers(t *testing.T) { + const transientResponses = int32(8) + var infoCalls atomic.Int32 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/": + if infoCalls.Add(1) <= transientResponses { + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = w.Write([]byte(`{"error":{"type":"unavailable","reason":"starting"},"status":503}`)) + return + } + _, _ = w.Write([]byte(`{"version":{"distribution":"opensearch","number":"3.3.2"}}`)) + case "/_cat/plugins": + _, _ = w.Write([]byte(`[{"name":"node-1","component":"opensearch-knn"}]`)) + default: + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{}`)) + } + })) + defer ts.Close() + + repoIface, err := NewRepository(context.Background(), newTestClient(t, ts.URL), "", nil) + if err != nil { + t.Fatalf("transient startup probe must not fail construction: %v", err) + } + repo, ok := repoIface.(*Repository) + if !ok { + t.Fatalf("repository type: %T", repoIface) + } + if !repo.clusterProbePending { + t.Fatal("transient startup failure must leave a deferred probe") + } + var probeErr error + for attempt := 0; attempt < 10; attempt++ { + probeErr = repo.ensureClusterValidated(context.Background()) + if probeErr == nil { + break + } + if !errors.Is(probeErr, ErrTransport) { + t.Fatalf("deferred transient probe classification: %v", probeErr) + } + } + if probeErr != nil { + t.Fatalf("deferred probe should eventually recover: %v", probeErr) + } + if repo.clusterProbePending { + t.Fatal("successful deferred probe must clear pending state") + } + if got := infoCalls.Load(); got <= transientResponses { + t.Fatalf("cluster info calls: want recovery after %d transient responses, got %d", + transientResponses, got) + } +} + // ============================================================================ // Stub coverage — remaining stubs return the not-enabled sentinel // diff --git a/internal/application/repository/retriever/opensearch/retrieve.go b/internal/application/repository/retriever/opensearch/retrieve.go index 61101d7811..23d4d1f1a1 100644 --- a/internal/application/repository/retriever/opensearch/retrieve.go +++ b/internal/application/repository/retriever/opensearch/retrieve.go @@ -31,6 +31,9 @@ func (r *Repository) Retrieve( ctx context.Context, params types.RetrieveParams, ) ([]*types.RetrieveResult, error) { + if err := r.ensureClusterValidated(ctx); err != nil { + return nil, err + } dim, multiIndex := resolveDim(params) switch params.RetrieverType { @@ -140,7 +143,7 @@ func (r *Repository) search(ctx context.Context, indexPattern string, body []byt // Field-by-field decode (vs map[string]any) keeps the JSON shape // pinned at compile time. type hit struct { - ID string `json:"_id"` // equals chunk_id per the indexing invariant + ID string `json:"_id"` // equals chunk_id per the indexing invariant Score float64 `json:"_score"` Source struct { Content string `json:"content"` diff --git a/internal/application/service/extract.go b/internal/application/service/extract.go index f9a9d59c84..2cd8880ace 100644 --- a/internal/application/service/extract.go +++ b/internal/application/service/extract.go @@ -285,7 +285,7 @@ func (s *ChunkExtractService) Handle(ctx context.Context, t *asynq.Task) error { if p.KnowledgeID != "" && s.knowledgeRepo != nil { if k, kerr := s.knowledgeRepo.GetKnowledgeByIDOnly(ctx, p.KnowledgeID); kerr == nil && k != nil { switch k.ParseStatus { - case types.ParseStatusCancelled, types.ParseStatusDeleting: + case types.ParseStatusCancelled, types.ParseStatusDeleting, types.ParseStatusReplacing: logger.Infof(ctx, "graph extract: knowledge %s aborted (%s), skipping chunk %s", p.KnowledgeID, k.ParseStatus, p.ChunkID) graphOut["skipped"] = "knowledge_" + k.ParseStatus diff --git a/internal/application/service/image_multimodal.go b/internal/application/service/image_multimodal.go index 46b4d892c0..0c05be8a61 100644 --- a/internal/application/service/image_multimodal.go +++ b/internal/application/service/image_multimodal.go @@ -381,7 +381,7 @@ func (s *ImageMultimodalService) shouldDropOrphanedMultimodal( return false, err } switch k.ParseStatus { - case types.ParseStatusCancelled, types.ParseStatusDeleting: + case types.ParseStatusCancelled, types.ParseStatusDeleting, types.ParseStatusReplacing: return true, nil } } diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index 590a6b38ca..f1c70922d8 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -397,7 +397,7 @@ func (s *knowledgeService) isKnowledgeDeleting(ctx context.Context, tenantID uin } // isKnowledgeAborted returns (true, status) when the knowledge has been -// marked as deleting OR cancelled so async pipeline workers should bail +// marked as deleting, cancelled, or replacing so async pipeline workers should bail // out. Status is returned so callers can branch on cleanup behavior: // deleting → existing cleanup of partial chunks/index applies; // cancelled → keep partially written data per user expectation. @@ -417,7 +417,7 @@ func (s *knowledgeService) isKnowledgeAborted( return true, types.ParseStatusDeleting } switch knowledge.ParseStatus { - case types.ParseStatusDeleting, types.ParseStatusCancelled: + case types.ParseStatusDeleting, types.ParseStatusCancelled, types.ParseStatusReplacing: return true, knowledge.ParseStatus } return false, knowledge.ParseStatus @@ -451,6 +451,40 @@ func (s *knowledgeService) checkStorageEngineConfigured(ctx context.Context, kb return werrors.NewBadRequestError("请先为知识库选择存储引擎，再上传内容。请前往知识库设置页面进行配置。") } +// checkRetrieveEngineConfigured resolves the KB's effective vector store before +// file bytes are persisted. This prevents accepting work that is guaranteed to +// fail later with a missing repository registration. +func (s *knowledgeService) checkRetrieveEngineConfigured( + ctx context.Context, kb *types.KnowledgeBase, tenantID uint64, +) error { + // Some focused unit tests construct a partial service. Production always + // injects the registry through the container. + if s == nil || s.retrieveEngine == nil || kb == nil { + return nil + } + if kb.VectorStoreID != nil && *kb.VectorStoreID != "" && s.ownership == nil { + return werrors.NewServiceUnavailableError("知识库检索引擎暂时不可用") + } + if _, ok := types.TenantInfoFromContext(ctx); !ok && s.tenantRepo != nil { + tenant, err := s.tenantRepo.GetTenantByID(ctx, tenantID) + if err != nil { + return werrors.NewServiceUnavailableError("知识库检索引擎暂时不可用") + } + ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant) + } + if _, err := retriever.CreateRetrieveEngineForKB( + ctx, s.retrieveEngine, s.ownership, tenantID, kb.VectorStoreID, + ); err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "tenant_id": tenantID, + "kb_id": kb.ID, + "reason": "retrieve engine preflight failed", + }) + return werrors.NewServiceUnavailableError("知识库检索引擎暂时不可用") + } + return nil +} + func kbIDOrEmpty(kb *types.KnowledgeBase) string { if kb == nil { return "" @@ -694,6 +728,9 @@ func (s *knowledgeService) UpdateKnowledge(ctx context.Context, knowledge *types logger.Errorf(ctx, "Failed to get knowledge record: %v", err) return err } + if record.ParseStatus == types.ParseStatusReplacing { + return werrors.NewConflictError("knowledge file is being replaced") + } // if need other fields update, please add here if knowledge.Title != "" { record.Title = knowledge.Title @@ -803,6 +840,17 @@ func (s *knowledgeService) GetKnowledgeBatchWithSharedAccess(ctx context.Context // SetKnowledgeTags replaces all tags for a single knowledge entry. func (s *knowledgeService) SetKnowledgeTags(ctx context.Context, knowledgeID string, tagIDs []string) error { + tenantID, _ := ctx.Value(types.TenantIDContextKey).(uint64) + knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID) + if err != nil { + return err + } + if knowledge.ParseStatus == types.ParseStatusReplacing { + return werrors.NewConflictError("knowledge file is being replaced") + } + if err := s.validateKnowledgeTagIDs(ctx, tenantID, knowledge.KnowledgeBaseID, tagIDs); err != nil { + return err + } return s.repo.SetKnowledgeTags(ctx, knowledgeID, tagIDs) } @@ -907,6 +955,9 @@ func (s *knowledgeService) UpdateKnowledgeTag(ctx context.Context, knowledgeID s if err != nil { return err } + if knowledge.ParseStatus == types.ParseStatusReplacing { + return werrors.NewConflictError("knowledge file is being replaced") + } // Validate all tag IDs if err := s.validateKnowledgeTagIDs(ctx, tenantID, knowledge.KnowledgeBaseID, tagIDs); err != nil { @@ -941,6 +992,11 @@ func (s *knowledgeService) UpdateKnowledgeTagBatch(ctx context.Context, authoriz if err != nil { return err } + for _, knowledge := range knowledgeList { + if knowledge.ParseStatus == types.ParseStatusReplacing { + return werrors.NewConflictError(fmt.Sprintf("knowledge %s file is being replaced", knowledge.ID)) + } + } // Validate all requested IDs were found and belong to the authorized KB if authorizedKBID != "" { diff --git a/internal/application/service/knowledge_create.go b/internal/application/service/knowledge_create.go index b1d26ec430..13d2c4e6a6 100644 --- a/internal/application/service/knowledge_create.go +++ b/internal/application/service/knowledge_create.go @@ -66,6 +66,10 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context, if err := s.checkStorageEngineConfigured(ctx, kb); err != nil { return nil, err } + tenantID := ctx.Value(types.TenantIDContextKey).(uint64) + if err := s.checkRetrieveEngineConfigured(ctx, kb, tenantID); err != nil { + return nil, err + } // Early reject before the whole-file hash below. resolveFileImportProcessConfig // gates the same extension set, but this path must keep returning @@ -85,7 +89,6 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context, } // Check if file already exists - tenantID := ctx.Value(types.TenantIDContextKey).(uint64) logger.Infof(ctx, "Checking if file exists, tenant ID: %d", tenantID) exists, existingKnowledge, err := s.repo.CheckKnowledgeExists(ctx, tenantID, kbID, &types.KnowledgeCheckParams{ Type: "file", diff --git a/internal/application/service/knowledge_create_test.go b/internal/application/service/knowledge_create_test.go index 856b6cec33..c4811d8268 100644 --- a/internal/application/service/knowledge_create_test.go +++ b/internal/application/service/knowledge_create_test.go @@ -67,6 +67,7 @@ type createKnowledgeFileServiceStub struct { savedWithKnowledgeID string deleteCalls int deletedPath string + deleteErr error } func (s *createKnowledgeFileServiceStub) CheckConnectivity(ctx context.Context) error { @@ -108,7 +109,7 @@ func (s *createKnowledgeFileServiceStub) GetFileURL(ctx context.Context, filePat func (s *createKnowledgeFileServiceStub) DeleteFile(ctx context.Context, filePath string) error { s.deleteCalls++ s.deletedPath = filePath - return nil + return s.deleteErr } func (s *createKnowledgeFileServiceStub) CopyFile(ctx context.Context, srcPath string, tenantID uint64, knowledgeID string) (string, error) { diff --git a/internal/application/service/knowledge_delete.go b/internal/application/service/knowledge_delete.go index d3aa09a8c1..a933a586a1 100644 --- a/internal/application/service/knowledge_delete.go +++ b/internal/application/service/knowledge_delete.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "strings" - "time" "github.com/Tencent/WeKnora/internal/application/service/retriever" "github.com/Tencent/WeKnora/internal/logger" @@ -58,22 +57,20 @@ func deleteExtractedImages(ctx context.Context, fileSvc interfaces.FileService, // DeleteKnowledge deletes a knowledge entry and all related resources func (s *knowledgeService) DeleteKnowledge(ctx context.Context, id string) error { // Get the knowledge entry - knowledge, err := s.repo.GetKnowledgeByID(ctx, ctx.Value(types.TenantIDContextKey).(uint64), id) + tenantID := ctx.Value(types.TenantIDContextKey).(uint64) + knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, id) if err != nil { return err } - // Mark as deleting first to prevent async task conflicts // This ensures that any running async tasks will detect the deletion and abort originalStatus := knowledge.ParseStatus - knowledge.ParseStatus = types.ParseStatusDeleting - knowledge.UpdatedAt = time.Now() - if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil { - logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge failed to mark as deleting") - // Continue with deletion even if marking fails - } else { - logger.Infof(ctx, "Marked knowledge %s as deleting (previous status: %s)", id, originalStatus) + cancelledSlots, err := s.repo.BeginKnowledgeDeletion(ctx, tenantID, []string{id}) + if err != nil { + return err } + knowledge.ParseStatus = types.ParseStatusDeleting + logger.Infof(ctx, "Marked knowledge %s as deleting (previous status: %s)", id, originalStatus) // Best-effort: purge any queued downstream tasks for this knowledge // (multimodal / post-process / question / summary / graph extract). @@ -89,9 +86,11 @@ func (s *knowledgeService) DeleteKnowledge(ctx context.Context, id string) error // Resolve file service for this KB before spawning goroutines kb, _ := s.kbService.GetKnowledgeBaseByID(ctx, knowledge.KnowledgeBaseID) kbFileSvc := s.resolveFileService(ctx, kb) + for _, slot := range cancelledSlots { + s.cleanupCancelledKnowledgeFileUpdate(ctx, kb, knowledge.FilePath, slot) + } // Collect image URLs before chunks are deleted (ImageInfo references are lost after deletion) - tenantID := ctx.Value(types.TenantIDContextKey).(uint64) chunkImageInfos, err := s.chunkService.GetRepository().ListImageInfoByKnowledgeIDs(ctx, tenantID, []string{id}) if err != nil { logger.Errorf(ctx, "Failed to collect image URLs for cleanup: %v", err) @@ -498,23 +497,21 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string if err != nil { return err } - - // Mark all as deleting first to prevent async task conflicts. + // Atomically mark all as deleting and revoke their update slots before any + // asynchronous cleanup can race with a late file-update coordinator. // Remember which entries still had queued / in-flight downstream tasks // so we can dequeue them in one pass after marking. var inFlightIDs []string for _, knowledge := range knowledgeList { prev := knowledge.ParseStatus - knowledge.ParseStatus = types.ParseStatusDeleting - knowledge.UpdatedAt = time.Now() - if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil { - logger.GetLogger(ctx).WithField("error", err).WithField("knowledge_id", knowledge.ID). - Errorf("DeleteKnowledgeList failed to mark as deleting") - // Continue with deletion even if marking fails - } if prev == types.ParseStatusPending || prev == types.ParseStatusProcessing { inFlightIDs = append(inFlightIDs, knowledge.ID) } + knowledge.ParseStatus = types.ParseStatusDeleting + } + cancelledSlots, err := s.repo.BeginKnowledgeDeletion(ctx, tenantInfo.ID, ids) + if err != nil { + return err } logger.Infof(ctx, "Marked %d knowledge entries as deleting", len(knowledgeList)) @@ -528,13 +525,24 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string // Pre-resolve KB metadata and file services so goroutines don't need DB access. knowledgeBases := make(map[string]*types.KnowledgeBase) kbFileServices := make(map[string]interfaces.FileService) + knowledgeByID := make(map[string]*types.Knowledge, len(knowledgeList)) for _, knowledge := range knowledgeList { + knowledgeByID[knowledge.ID] = knowledge if _, ok := kbFileServices[knowledge.KnowledgeBaseID]; !ok { kb, _ := s.kbService.GetKnowledgeBaseByID(ctx, knowledge.KnowledgeBaseID) knowledgeBases[knowledge.KnowledgeBaseID] = kb kbFileServices[knowledge.KnowledgeBaseID] = s.resolveFileService(ctx, kb) } } + for _, slot := range cancelledSlots { + knowledge := knowledgeByID[slot.KnowledgeID] + if knowledge == nil { + continue + } + s.cleanupCancelledKnowledgeFileUpdate( + ctx, knowledgeBases[knowledge.KnowledgeBaseID], knowledge.FilePath, slot, + ) + } // Collect image URLs before chunks are deleted chunkImageInfos, err := s.chunkService.GetRepository().ListImageInfoByKnowledgeIDs(ctx, tenantInfo.ID, ids) diff --git a/internal/application/service/knowledge_file_update.go b/internal/application/service/knowledge_file_update.go new file mode 100644 index 0000000000..f0b0aebe50 --- /dev/null +++ b/internal/application/service/knowledge_file_update.go @@ -0,0 +1,1117 @@ +package service + +import ( + "context" + "encoding/json" + stderrors "errors" + "fmt" + "reflect" + "slices" + "strings" + "time" + + "github.com/Tencent/WeKnora/internal/application/repository" + werrors "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/tracing/langfuse" + "github.com/Tencent/WeKnora/internal/types" + secutils "github.com/Tencent/WeKnora/internal/utils" + "github.com/hibiken/asynq" + "gorm.io/gorm" +) + +var replaceableKnowledgeStatuses = map[string]struct{}{ + types.ParseStatusCompleted: {}, + types.ParseStatusFailed: {}, + types.ParseStatusCancelled: {}, +} + +// CreateOrUpdateKnowledgeFromFile implements the unified HTTP upsert contract. +// An explicit KnowledgeID always wins. Without it, a unique filename match in +// the same KB is treated as an in-place update; otherwise the request creates +// a new knowledge. +func (s *knowledgeService) CreateOrUpdateKnowledgeFromFile( + ctx context.Context, + req *types.KnowledgeFileCreateOrUpdateRequest, +) (*types.KnowledgeFileUpsertResult, error) { + if req == nil || req.File == nil { + return nil, werrors.NewBadRequestError("File upload failed") + } + if req.KnowledgeID != "" { + return s.UpdateKnowledgeFile(ctx, knowledgeFileUpdateRequestFromUpsert(req)) + } + + _, safeFilename, err := resolveKnowledgeFileUpdateName(req.File.Filename, req.CustomFileName) + if err != nil { + return nil, err + } + existing, err := s.findUniqueFileKnowledgeByName(ctx, req.KnowledgeBaseID, safeFilename) + if err != nil { + return nil, err + } + if existing != nil { + updateReq := knowledgeFileUpdateRequestFromUpsert(req) + updateReq.KnowledgeID = existing.ID + return s.UpdateKnowledgeFile(ctx, updateReq) + } + + channel := req.Channel + if channel == "" { + channel = types.ChannelAPI + } + knowledge, err := s.CreateKnowledgeFromFile( + ctx, + req.KnowledgeBaseID, + req.File, + req.Metadata, + req.EnableMultimodel, + req.CustomFileName, + req.TagIDs, + channel, + req.ProcessOverrides, + ) + if err != nil { + if duplicate, ok := err.(*types.DuplicateKnowledgeError); ok { + existing := duplicate.Knowledge + if existing == nil { + existing = knowledge + } + return &types.KnowledgeFileUpsertResult{ + Action: "unchanged", Knowledge: existing, + }, nil + } + return nil, err + } + return &types.KnowledgeFileUpsertResult{Action: "created", Knowledge: knowledge}, nil +} + +func knowledgeFileUpdateRequestFromUpsert( + req *types.KnowledgeFileCreateOrUpdateRequest, +) *types.KnowledgeFileUpdateRequest { + return &types.KnowledgeFileUpdateRequest{ + KnowledgeBaseID: req.KnowledgeBaseID, + KnowledgeID: req.KnowledgeID, + File: req.File, + CustomFileName: req.CustomFileName, + ExpectedFileHash: req.ExpectedFileHash, + ExpectedUpdateVersion: req.ExpectedUpdateVersion, + Metadata: req.Metadata, + MetadataProvided: req.MetadataProvided, + TagIDs: req.TagIDs, + TagIDsProvided: req.TagIDsProvided, + Channel: req.Channel, + ChannelProvided: req.ChannelProvided, + ProcessOverrides: req.ProcessOverrides, + } +} + +func resolveKnowledgeFileUpdateName( + uploadFileName string, + customFileName string, +) (folderPath string, safeFilename string, err error) { + fileName := uploadFileName + if customFileName != "" { + folderPath, fileName = types.SplitKnowledgeRelativePath(customFileName) + if fileName == "" { + fileName = uploadFileName + } + } + safeFilename, valid := secutils.ValidateInput(fileName) + if !valid { + return "", "", werrors.NewValidationError("文件名包含非法字符") + } + if folderPath != "" { + safeFolderPath, folderValid := secutils.ValidateInput(folderPath) + if !folderValid { + return "", "", werrors.NewValidationError("文件夹路径包含非法字符") + } + folderPath = types.NormalizeKnowledgeFolderPath(safeFolderPath) + } + return folderPath, safeFilename, nil +} + +func (s *knowledgeService) findUniqueFileKnowledgeByName( + ctx context.Context, kbID string, fileName string, +) (*types.Knowledge, error) { + if fileName == "" { + return nil, nil + } + tenantID, ok := ctx.Value(types.TenantIDContextKey).(uint64) + if !ok || tenantID == 0 { + return nil, werrors.NewUnauthorizedError("tenant context is required") + } + knowledges, err := s.repo.ListKnowledgeByKnowledgeBaseID(ctx, tenantID, kbID) + if err != nil { + return nil, err + } + var match *types.Knowledge + for _, knowledge := range knowledges { + if knowledge == nil || + knowledge.Type != "file" || + knowledge.FileName != fileName || + knowledge.FilePath == "" || + knowledge.ParseStatus == types.ParseStatusDeleting { + continue + } + if match != nil { + return nil, werrors.NewConflictError( + "multiple file knowledge entries match this filename; pass knowledge_id explicitly", + ) + } + match = knowledge + } + return match, nil +} + +// UpdateKnowledgeFile validates and durably stages the latest requested file. +// The coordinator claims the current knowledge only when it is safe to apply. +func (s *knowledgeService) UpdateKnowledgeFile( + ctx context.Context, + req *types.KnowledgeFileUpdateRequest, +) (*types.KnowledgeFileUpsertResult, error) { + if req == nil || req.File == nil { + return nil, werrors.NewBadRequestError("File upload failed") + } + tenantID, ok := ctx.Value(types.TenantIDContextKey).(uint64) + if !ok || tenantID == 0 { + return nil, werrors.NewUnauthorizedError("tenant context is required") + } + + existing, err := s.repo.GetKnowledgeByID(ctx, tenantID, req.KnowledgeID) + if err != nil { + if stderrors.Is(err, repository.ErrKnowledgeNotFound) { + return nil, werrors.NewNotFoundError("knowledge not found") + } + return nil, err + } + if existing.KnowledgeBaseID != req.KnowledgeBaseID { + return nil, werrors.NewConflictError("knowledge does not belong to the requested knowledge base") + } + if existing.Type != "file" || existing.FilePath == "" { + return nil, werrors.NewBadRequestError("only file knowledge can be replaced") + } + if existing.ParseStatus == types.ParseStatusDeleting { + return nil, werrors.NewConflictError("knowledge is being deleted") + } + if req.ExpectedFileHash != "" && req.ExpectedFileHash != existing.FileHash { + return nil, werrors.NewConflictError("expected_file_hash does not match the current file") + } + + maxSizeMB := secutils.GetMaxFileSizeMB() + if req.File.Size > maxSizeMB*1024*1024 { + return nil, werrors.NewBadRequestError(fmt.Sprintf("文件大小不能超过%dMB", maxSizeMB)) + } + + folderPath, safeFilename, err := resolveKnowledgeFileUpdateName(req.File.Filename, req.CustomFileName) + if err != nil { + return nil, err + } + fileType := getFileType(safeFilename) + if IsVideoType(fileType) { + return nil, werrors.NewBadRequestError("暂不支持上传视频文件") + } + if !isValidFileType(safeFilename) { + return nil, ErrInvalidFileType + } + + kb, err := s.kbService.GetKnowledgeBaseByID(ctx, req.KnowledgeBaseID) + if err != nil { + return nil, err + } + if kb.Type == types.KnowledgeBaseTypeFAQ { + return nil, werrors.NewBadRequestError("FAQ 知识库不支持文件上传，请使用 FAQ 导入功能") + } + if err := s.checkStorageEngineConfigured(ctx, kb); err != nil { + return nil, err + } + if err := s.checkRetrieveEngineConfigured(ctx, kb, tenantID); err != nil { + return nil, err + } + if req.TagIDsProvided { + if err := s.validateKnowledgeTagIDs(ctx, tenantID, req.KnowledgeBaseID, req.TagIDs); err != nil { + return nil, err + } + } + + storedOverrides, err := existing.ProcessOverrides() + if err != nil { + return nil, fmt.Errorf("parse stored process overrides: %w", err) + } + effectiveOverrides := storedOverrides + if req.ProcessOverrides != nil { + effectiveOverrides = req.ProcessOverrides + } + validationOverrides := effectiveOverrides + if validationOverrides == nil { + validationOverrides = &types.KnowledgeProcessOverrides{} + } + if err := ValidateProcessOverrides(ctx, kb, validationOverrides, []string{fileType}); err != nil { + return nil, err + } + + newHash, err := calculateFileHash(req.File) + if err != nil { + return nil, err + } + if (existing.FileUpdateState == "" || existing.FileUpdateState == types.KnowledgeFileUpdateStateIdle) && + newHash == existing.FileHash && safeFilename == existing.FileName && folderPath == existing.FolderPath && + !req.MetadataProvided && !req.TagIDsProvided && !req.ChannelProvided && req.ProcessOverrides == nil { + s.attachTagsToKnowledge(ctx, existing) + return &types.KnowledgeFileUpsertResult{Action: "unchanged", Knowledge: existing}, nil + } + + exists, duplicate, err := s.repo.CheckKnowledgeExistsExcluding( + ctx, + tenantID, + req.KnowledgeBaseID, + existing.ID, + &types.KnowledgeCheckParams{ + Type: "file", + FileName: safeFilename, + FileSize: req.File.Size, + FileHash: newHash, + }, + ) + if err != nil { + return nil, err + } + if exists { + duplicateID := "" + if duplicate != nil { + duplicateID = duplicate.ID + } + return nil, werrors.NewConflictError(fmt.Sprintf("replacement file already exists as knowledge %s", duplicateID)) + } + + payload := types.KnowledgeFileUpdatePayload{ + TenantID: tenantID, + KnowledgeBaseID: req.KnowledgeBaseID, + KnowledgeID: existing.ID, + NewFileName: safeFilename, + NewFolderPath: folderPath, + NewFileType: fileType, + NewFileSize: req.File.Size, + NewFileHash: newHash, + Metadata: req.Metadata, + MetadataProvided: req.MetadataProvided, + TagIDs: req.TagIDs, + TagIDsProvided: req.TagIDsProvided, + Channel: req.Channel, + ChannelProvided: req.ChannelProvided, + ProcessConfig: req.ProcessOverrides, + ProcessProvided: req.ProcessOverrides != nil, + Initiator: types.TaskInitiatorFromContext(ctx), + } + if existing.FileUpdateState == types.KnowledgeFileUpdateResultActive || + existing.FileUpdateState == types.KnowledgeFileUpdateResultPending { + slot, slotErr := s.repo.GetKnowledgeFileUpdateSlot(ctx, tenantID, existing.ID) + if slotErr != nil { + return nil, slotErr + } + latest := slot.ActivePayload + if slot.PendingVersion != nil { + latest = slot.PendingPayload + } + var accepted types.KnowledgeFileUpdatePayload + if json.Unmarshal(latest, &accepted) == nil && sameKnowledgeFileUpdate(&accepted, &payload) { + s.attachTagsToKnowledge(ctx, existing) + return &types.KnowledgeFileUpsertResult{ + Action: "unchanged", + Knowledge: existing, + UpdateVersion: slot.LatestVersion, + UpdateState: existing.FileUpdateState, + AcceptedFileHash: newHash, + }, nil + } + } + + fileSvc := s.resolveFileService(ctx, kb) + if fileSvc == nil { + return nil, fmt.Errorf("file service is not configured") + } + newFilePath, err := fileSvc.SaveFile(ctx, req.File, tenantID, existing.ID) + if err != nil { + return nil, err + } + cleanupStaged := func() { + if deleteErr := fileSvc.DeleteFile(ctx, newFilePath); deleteErr != nil { + logger.Errorf(ctx, "Failed to delete staged replacement file %s: %v", newFilePath, deleteErr) + } + } + payload.NewFilePath = newFilePath + langfuse.InjectTracing(ctx, &payload) + payloadBytes, err := json.Marshal(payload) + if err != nil { + cleanupStaged() + return nil, fmt.Errorf("encode knowledge file update: %w", err) + } + + staged, err := s.repo.StageKnowledgeFileUpdate( + ctx, tenantID, existing.ID, req.KnowledgeBaseID, types.JSON(payloadBytes), req.ExpectedUpdateVersion, + ) + if err != nil { + cleanupStaged() + if stderrors.Is(err, repository.ErrKnowledgeFileUpdateVersionConflict) { + return nil, werrors.NewConflictError("expected_update_version does not match the latest accepted version") + } + if stderrors.Is(err, repository.ErrKnowledgeFileUpdateDeleting) || + stderrors.Is(err, repository.ErrKnowledgeNotFound) { + return nil, werrors.NewConflictError("knowledge is being deleted") + } + return nil, err + } + s.deleteStagedPayloadBestEffort(ctx, kb, staged.ReplacedPendingPayload) + s.deleteSupersededActivePayloadBestEffort(ctx, kb, existing.FilePath, staged.ReplacedActivePayload) + + taskID, err := s.enqueueKnowledgeFileUpdate(ctx, types.KnowledgeFileUpdateTaskPayload{ + TenantID: tenantID, + KnowledgeBaseID: req.KnowledgeBaseID, + KnowledgeID: existing.ID, + ActiveVersion: staged.ActiveVersion, + }, 0) + if err != nil { + return nil, werrors.NewServiceUnavailableError( + "file update was saved but the worker is temporarily unavailable; retry is safe") + } + + existing.FileUpdateVersion = staged.Version + existing.FileUpdateState = staged.State + s.attachTagsToKnowledge(ctx, existing) + return &types.KnowledgeFileUpsertResult{ + Action: "updated", + Knowledge: existing, + TaskID: taskID, + UpdateVersion: staged.Version, + UpdateState: staged.State, + AcceptedFileHash: newHash, + }, nil +} + +// applyKnowledgeFileUpdatePayload cleans old derived resources, switches the +// source file in place, and enqueues the normal document parser. +func (s *knowledgeService) applyKnowledgeFileUpdatePayload( + ctx context.Context, payload *types.KnowledgeFileUpdatePayload, +) error { + ctx = payload.Initiator.Apply(ctx) + tenant, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID) + if err != nil { + return fmt.Errorf("load replacement tenant: %w", err) + } + ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID) + ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant) + ctx = withAttempt(ctx, payload.Attempt) + + kb, err := s.kbService.GetKnowledgeBaseByID(ctx, payload.KnowledgeBaseID) + if err != nil { + return fmt.Errorf("load replacement knowledge base: %w", err) + } + current, err := s.repo.GetKnowledgeByID(ctx, payload.TenantID, payload.KnowledgeID) + if err != nil { + if stderrors.Is(err, repository.ErrKnowledgeNotFound) { + s.deleteReplacementFileBestEffort(ctx, kb, payload.NewFilePath) + return nil + } + return err + } + + // A retry after the file switch only has to finish tag persistence and + // deterministic parser enqueue. A parse failure is not retried here. + if current.FilePath == payload.NewFilePath && current.FileHash == payload.NewFileHash { + switch current.ParseStatus { + case types.ParseStatusPending: + return s.finishKnowledgeFileUpdate(ctx, kb, current, payload) + case types.ParseStatusProcessing, types.ParseStatusFinalizing, types.ParseStatusCompleted, + types.ParseStatusFailed, types.ParseStatusCancelled: + return nil + } + } + + if current.KnowledgeBaseID != payload.KnowledgeBaseID || + current.ParseStatus != types.ParseStatusReplacing || + current.FilePath != payload.OldFilePath || + current.FileHash != payload.OldFileHash { + s.deleteReplacementFileBestEffort(ctx, kb, payload.NewFilePath) + return nil + } + + s.dequeueKnowledgeTasks(ctx, current.ID) + if payload.Attempt == 0 { + if _, attempt, openErr := s.tracker().OpenAttempt(ctx, current.ID, payload.LangfuseTraceID); openErr == nil { + payload.Attempt = attempt + ctx = withAttempt(ctx, attempt) + } else { + logger.Warnf(ctx, "Open replacement attempt failed for %s: %v", current.ID, openErr) + } + } + if kb.IsWikiEnabled() { + s.prepareWikiForReparse(ctx, current) + } + if err := s.cleanupKnowledgeResources(ctx, current); err != nil { + // cleanupKnowledgeResources adjusts tenant quota before returning a + // combined error. Persisting zero prevents a retry from decrementing it + // a second time while the remaining cleanup is retried. + if current.StorageSize == 0 { + _, persistErr := s.repo.UpdateApplyingKnowledgeFileColumns( + ctx, payload.TenantID, current.ID, payload.KnowledgeBaseID, + payload.OldFilePath, payload.OldFileHash, + map[string]interface{}{"storage_size": 0}, + ) + if persistErr != nil { + logger.Errorf(ctx, "Failed to persist replacement cleanup storage size: %v", persistErr) + } + } + return err + } + + metadata, err := replacementMetadata(current, payload) + if err != nil { + return err + } + updated, err := s.repo.UpdateApplyingKnowledgeFileColumns( + ctx, + payload.TenantID, + current.ID, + payload.KnowledgeBaseID, + payload.OldFilePath, + payload.OldFileHash, + map[string]interface{}{ + "file_path": payload.NewFilePath, + "file_name": payload.NewFileName, + "title": payload.NewFileName, + "folder_path": payload.NewFolderPath, + "file_type": payload.NewFileType, + "file_size": payload.NewFileSize, + "file_hash": payload.NewFileHash, + "metadata": metadata, + "channel": replacementChannel(current.Channel, payload), + "embedding_model_id": kb.EmbeddingModelID, + "storage_size": 0, + "description": "", + "processed_at": nil, + "pending_subtasks_count": 0, + "summary_status": types.SummaryStatusNone, + "enable_status": "disabled", + "error_message": "", + "parse_status": types.ParseStatusPending, + "updated_at": time.Now(), + }, + ) + if err != nil { + return err + } + if !updated { + s.deleteReplacementFileBestEffort(ctx, kb, payload.NewFilePath) + return nil + } + + current.FilePath = payload.NewFilePath + current.FileName = payload.NewFileName + current.Title = payload.NewFileName + current.FolderPath = payload.NewFolderPath + current.FileType = payload.NewFileType + current.FileSize = payload.NewFileSize + current.FileHash = payload.NewFileHash + current.Metadata = metadata + current.Channel = replacementChannel(current.Channel, payload) + current.EmbeddingModelID = kb.EmbeddingModelID + current.StorageSize = 0 + current.Description = "" + current.ProcessedAt = nil + current.PendingSubtasksCount = 0 + current.SummaryStatus = types.SummaryStatusNone + current.EnableStatus = "disabled" + current.ErrorMessage = "" + current.ParseStatus = types.ParseStatusPending + + return s.finishKnowledgeFileUpdate(ctx, kb, current, payload) +} + +const knowledgeFileUpdateWakeDelay = 5 * time.Second + +// ProcessKnowledgeFileUpdate coordinates the durable active/pending slots. It +// never trusts a file payload from Redis; the active version is loaded from the +// database on every execution. +func (s *knowledgeService) ProcessKnowledgeFileUpdate(ctx context.Context, task *asynq.Task) error { + var wake types.KnowledgeFileUpdateTaskPayload + if err := json.Unmarshal(task.Payload(), &wake); err != nil { + return fmt.Errorf("decode knowledge file update task: %w", err) + } + tenant, err := s.tenantRepo.GetTenantByID(ctx, wake.TenantID) + if err != nil { + return fmt.Errorf("load file update tenant: %w", err) + } + ctx = context.WithValue(ctx, types.TenantIDContextKey, wake.TenantID) + ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant) + + slot, err := s.repo.GetKnowledgeFileUpdateSlot(ctx, wake.TenantID, wake.KnowledgeID) + if err != nil { + if stderrors.Is(err, gorm.ErrRecordNotFound) { + return nil + } + return err + } + if slot.ActiveVersion == nil { + return nil + } + if *slot.ActiveVersion != wake.ActiveVersion { + if slot.ActiveState == types.KnowledgeFileUpdateStateFailed { + return nil + } + _, err := s.enqueueKnowledgeFileUpdate(ctx, types.KnowledgeFileUpdateTaskPayload{ + TenantID: slot.TenantID, + KnowledgeBaseID: slot.KnowledgeBaseID, + KnowledgeID: slot.KnowledgeID, + ActiveVersion: *slot.ActiveVersion, + }, 0) + return err + } + if slot.ActiveState == types.KnowledgeFileUpdateStateFailed { + return nil + } + if slot.ActiveState == types.KnowledgeFileUpdateStateRetryWait { + claimed, err := s.repo.TransitionKnowledgeFileUpdateState( + ctx, wake.TenantID, wake.KnowledgeID, wake.ActiveVersion, + types.KnowledgeFileUpdateStateRetryWait, types.KnowledgeFileUpdateStateWaiting, "", + ) + if err != nil || !claimed { + return err + } + slot.ActiveState = types.KnowledgeFileUpdateStateWaiting + } + + kb, err := s.kbService.GetKnowledgeBaseByID(ctx, wake.KnowledgeBaseID) + if err != nil { + return err + } + current, err := s.repo.GetKnowledgeByID(ctx, wake.TenantID, wake.KnowledgeID) + if err != nil { + if stderrors.Is(err, repository.ErrKnowledgeNotFound) { + return s.cancelKnowledgeFileUpdate(ctx, kb, wake.TenantID, wake.KnowledgeID) + } + return err + } + if current.ParseStatus == types.ParseStatusDeleting { + return s.cancelKnowledgeFileUpdate(ctx, kb, wake.TenantID, wake.KnowledgeID) + } + + var active types.KnowledgeFileUpdatePayload + if err := json.Unmarshal(slot.ActivePayload, &active); err != nil { + return s.failKnowledgeFileUpdate(ctx, wake, slot.ActiveState, fmt.Errorf("decode active update payload: %w", err)) + } + + if slot.ActiveState == types.KnowledgeFileUpdateStateWaiting { + if _, terminal := replaceableKnowledgeStatuses[current.ParseStatus]; !terminal { + if current.ParseStatus != types.ParseStatusReplacing || active.OldFilePath == "" || + current.FilePath != active.OldFilePath || current.FileHash != active.OldFileHash { + return s.deferKnowledgeFileUpdate(ctx, wake) + } + } else { + if active.OldFilePath == "" { + active.OldParseStatus = current.ParseStatus + active.OldFilePath = current.FilePath + active.OldFileHash = current.FileHash + active.DocumentTaskID = secutils.GenerateTaskID( + "knowledge_file_parse", wake.TenantID, wake.KnowledgeID, + ) + prepared, err := json.Marshal(active) + if err != nil { + return err + } + updated, err := s.repo.PrepareKnowledgeFileUpdate( + ctx, wake.TenantID, wake.KnowledgeID, wake.ActiveVersion, types.JSON(prepared), + ) + if err != nil || !updated { + return err + } + } + claimed, err := s.repo.ClaimKnowledgeFileUpdate( + ctx, wake.TenantID, wake.KnowledgeID, wake.KnowledgeBaseID, + current.ParseStatus, active.OldFilePath, active.OldFileHash, + ) + if err != nil { + return err + } + if !claimed { + return s.deferKnowledgeFileUpdate(ctx, wake) + } + } + + moved, err := s.repo.TransitionKnowledgeFileUpdateState( + ctx, wake.TenantID, wake.KnowledgeID, wake.ActiveVersion, + types.KnowledgeFileUpdateStateWaiting, types.KnowledgeFileUpdateStateApplying, "", + ) + if err != nil || !moved { + return err + } + } + + if err := s.applyKnowledgeFileUpdatePayload(ctx, &active); err != nil { + return s.failKnowledgeFileUpdate(ctx, wake, types.KnowledgeFileUpdateStateApplying, err) + } + + completed, err := s.repo.CompleteKnowledgeFileUpdate( + ctx, wake.TenantID, wake.KnowledgeID, wake.ActiveVersion, + ) + if err != nil { + return err + } + if completed.ActiveVersion != nil { + _, err = s.enqueueKnowledgeFileUpdate(ctx, types.KnowledgeFileUpdateTaskPayload{ + TenantID: completed.TenantID, + KnowledgeBaseID: completed.KnowledgeBaseID, + KnowledgeID: completed.KnowledgeID, + ActiveVersion: *completed.ActiveVersion, + }, 0) + } + return err +} + +// RetryKnowledgeFileUpdate re-arms the retained failed active payload. The +// exact active version is guarded so a concurrent upload wins safely. +func (s *knowledgeService) RetryKnowledgeFileUpdate( + ctx context.Context, knowledgeID string, +) (*types.Knowledge, error) { + tenantID, ok := ctx.Value(types.TenantIDContextKey).(uint64) + if !ok || tenantID == 0 { + return nil, werrors.NewUnauthorizedError("tenant context is required") + } + knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID) + if err != nil { + return nil, err + } + if knowledge.ParseStatus == types.ParseStatusDeleting { + return nil, werrors.NewConflictError("knowledge is being deleted") + } + slot, err := s.repo.GetKnowledgeFileUpdateSlot(ctx, tenantID, knowledgeID) + if err != nil || slot.ActiveVersion == nil { + if stderrors.Is(err, gorm.ErrRecordNotFound) || (err == nil && slot.ActiveVersion == nil) { + return nil, werrors.NewConflictError("no failed file update is available") + } + return nil, err + } + if slot.ActiveState != types.KnowledgeFileUpdateStateFailed { + return nil, werrors.NewConflictError("file update is not failed") + } + version := *slot.ActiveVersion + moved, err := s.repo.TransitionKnowledgeFileUpdateState( + ctx, tenantID, knowledgeID, version, + types.KnowledgeFileUpdateStateFailed, types.KnowledgeFileUpdateStateWaiting, "", + ) + if err != nil { + return nil, err + } + if !moved { + return nil, werrors.NewConflictError("file update changed; refresh and retry") + } + if _, err := s.enqueueKnowledgeFileUpdate(ctx, types.KnowledgeFileUpdateTaskPayload{ + TenantID: tenantID, + KnowledgeBaseID: slot.KnowledgeBaseID, + KnowledgeID: knowledgeID, + ActiveVersion: version, + }, 0); err != nil { + _, _ = s.repo.TransitionKnowledgeFileUpdateState( + ctx, tenantID, knowledgeID, version, + types.KnowledgeFileUpdateStateWaiting, types.KnowledgeFileUpdateStateFailed, + "retry enqueue failed", + ) + return nil, werrors.NewServiceUnavailableError("file update retry is temporarily unavailable") + } + return s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID) +} + +// DiscardKnowledgeFileUpdate removes the exact failed active version and its +// pending successor, then restores a claimed old source to its prior terminal +// state when the file switch had not happened yet. +func (s *knowledgeService) DiscardKnowledgeFileUpdate( + ctx context.Context, knowledgeID string, +) (*types.Knowledge, error) { + tenantID, ok := ctx.Value(types.TenantIDContextKey).(uint64) + if !ok || tenantID == 0 { + return nil, werrors.NewUnauthorizedError("tenant context is required") + } + knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID) + if err != nil { + return nil, err + } + if knowledge.ParseStatus == types.ParseStatusDeleting { + return nil, werrors.NewConflictError("knowledge is being deleted") + } + slot, err := s.repo.GetKnowledgeFileUpdateSlot(ctx, tenantID, knowledgeID) + if err != nil || slot.ActiveVersion == nil { + if stderrors.Is(err, gorm.ErrRecordNotFound) || (err == nil && slot.ActiveVersion == nil) { + return nil, werrors.NewConflictError("no failed file update is available") + } + return nil, err + } + if slot.ActiveState != types.KnowledgeFileUpdateStateFailed { + return nil, werrors.NewConflictError("only a failed file update can be discarded") + } + cancelled, err := s.repo.CancelFailedKnowledgeFileUpdate( + ctx, tenantID, knowledgeID, *slot.ActiveVersion, + ) + if err != nil { + if stderrors.Is(err, repository.ErrKnowledgeFileUpdateStateConflict) { + return nil, werrors.NewConflictError("file update changed; refresh and retry") + } + return nil, err + } + + var active types.KnowledgeFileUpdatePayload + if json.Unmarshal(cancelled.ActivePayload, &active) == nil && + knowledge.ParseStatus == types.ParseStatusReplacing && active.OldFilePath != "" { + restoreStatus := active.OldParseStatus + if _, ok := replaceableKnowledgeStatuses[restoreStatus]; !ok { + restoreStatus = types.ParseStatusFailed + } + updated, updateErr := s.repo.UpdateApplyingKnowledgeFileColumns( + ctx, tenantID, knowledgeID, knowledge.KnowledgeBaseID, + active.OldFilePath, active.OldFileHash, + map[string]interface{}{ + "parse_status": restoreStatus, + "error_message": "", + "updated_at": time.Now(), + }, + ) + if updateErr != nil { + return nil, updateErr + } + if updated { + knowledge.ParseStatus = restoreStatus + } + } + kb, _ := s.kbService.GetKnowledgeBaseByID(ctx, knowledge.KnowledgeBaseID) + s.cleanupCancelledKnowledgeFileUpdate(ctx, kb, knowledge.FilePath, cancelled) + return s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID) +} + +func (s *knowledgeService) enqueueKnowledgeFileUpdate( + ctx context.Context, wake types.KnowledgeFileUpdateTaskPayload, delay time.Duration, +) (string, error) { + payload, err := json.Marshal(wake) + if err != nil { + return "", err + } + opts := []asynq.Option{ + asynq.Queue(types.QueueMaintenance), + asynq.MaxRetry(3), + asynq.Timeout(2 * time.Hour), + asynq.Unique(2 * time.Hour), + } + if delay > 0 { + opts = append(opts, asynq.ProcessIn(delay)) + } + info, err := s.task.Enqueue(asynq.NewTask(types.TypeKnowledgeFileUpdate, payload), opts...) + if stderrors.Is(err, asynq.ErrTaskIDConflict) || stderrors.Is(err, asynq.ErrDuplicateTask) { + return "", nil + } + if err != nil { + logger.Errorf(ctx, "Enqueue knowledge file update failed: knowledge_id=%s version=%d err=%v", + wake.KnowledgeID, wake.ActiveVersion, err) + return "", err + } + if info == nil { + return "", nil + } + return info.ID, nil +} + +func (s *knowledgeService) deferKnowledgeFileUpdate( + ctx context.Context, wake types.KnowledgeFileUpdateTaskPayload, +) error { + claimed, err := s.repo.TransitionKnowledgeFileUpdateState( + ctx, wake.TenantID, wake.KnowledgeID, wake.ActiveVersion, + types.KnowledgeFileUpdateStateWaiting, types.KnowledgeFileUpdateStateRetryWait, "", + ) + if err != nil || !claimed { + return err + } + wake.WakeSequence++ + if _, err := s.enqueueKnowledgeFileUpdate(ctx, wake, knowledgeFileUpdateWakeDelay); err != nil { + _, restoreErr := s.repo.TransitionKnowledgeFileUpdateState( + ctx, wake.TenantID, wake.KnowledgeID, wake.ActiveVersion, + types.KnowledgeFileUpdateStateRetryWait, types.KnowledgeFileUpdateStateWaiting, "", + ) + if restoreErr != nil { + logger.Errorf(ctx, "Restore file update wait state failed: %v", restoreErr) + } + return err + } + return nil +} + +func (s *knowledgeService) failKnowledgeFileUpdate( + ctx context.Context, + wake types.KnowledgeFileUpdateTaskPayload, + fromState string, + cause error, +) error { + retried, retriedOK := asynq.GetRetryCount(ctx) + maxRetry, maxRetryOK := asynq.GetMaxRetry(ctx) + if !retriedOK || !maxRetryOK || retried < maxRetry { + return cause + } + message := cause.Error() + if len(message) > 512 { + message = message[:512] + } + markedFailed, err := s.repo.TransitionKnowledgeFileUpdateState( + ctx, wake.TenantID, wake.KnowledgeID, wake.ActiveVersion, + fromState, types.KnowledgeFileUpdateStateFailed, message, + ) + if err != nil { + logger.Errorf(ctx, "Mark knowledge file update failed: %v", err) + } + if markedFailed { + s.restoreFailedKnowledgeFileUpdateClaim(ctx, wake) + } + return cause +} + +func (s *knowledgeService) restoreFailedKnowledgeFileUpdateClaim( + ctx context.Context, + wake types.KnowledgeFileUpdateTaskPayload, +) { + slot, err := s.repo.GetKnowledgeFileUpdateSlot(ctx, wake.TenantID, wake.KnowledgeID) + if err != nil { + logger.Errorf(ctx, "Load failed knowledge file update slot: %v", err) + return + } + if slot.ActiveVersion == nil || *slot.ActiveVersion != wake.ActiveVersion { + return + } + var active types.KnowledgeFileUpdatePayload + if err := json.Unmarshal(slot.ActivePayload, &active); err != nil { + logger.Errorf(ctx, "Decode failed knowledge file update payload: %v", err) + return + } + if active.OldFilePath == "" { + return + } + restoreStatus := active.OldParseStatus + if _, ok := replaceableKnowledgeStatuses[restoreStatus]; !ok { + restoreStatus = types.ParseStatusFailed + } + kbID := wake.KnowledgeBaseID + if kbID == "" { + kbID = active.KnowledgeBaseID + } + updated, err := s.repo.UpdateApplyingKnowledgeFileColumns( + ctx, + wake.TenantID, + wake.KnowledgeID, + kbID, + active.OldFilePath, + active.OldFileHash, + map[string]interface{}{ + "parse_status": restoreStatus, + "error_message": "", + "updated_at": time.Now(), + }, + ) + if err != nil { + logger.Errorf(ctx, "Restore failed knowledge file update status: %v", err) + return + } + if updated { + logger.Infof(ctx, "Restored failed knowledge file update status: knowledge_id=%s status=%s", + wake.KnowledgeID, restoreStatus) + } +} + +func (s *knowledgeService) cancelKnowledgeFileUpdate( + ctx context.Context, kb *types.KnowledgeBase, tenantID uint64, knowledgeID string, +) error { + slot, err := s.repo.CancelKnowledgeFileUpdates(ctx, tenantID, knowledgeID) + if err != nil || slot == nil { + return err + } + s.deleteStagedPayloadBestEffort(ctx, kb, slot.ActivePayload) + s.deleteStagedPayloadBestEffort(ctx, kb, slot.PendingPayload) + return nil +} + +func (s *knowledgeService) cleanupCancelledKnowledgeFileUpdate( + ctx context.Context, + kb *types.KnowledgeBase, + currentPath string, + slot *types.KnowledgeFileUpdateSlot, +) { + if slot == nil { + return + } + // An applying retry may have switched the active payload into the current + // source already. Leave that path for the normal delete flow to remove. + s.deleteSupersededActivePayloadBestEffort(ctx, kb, currentPath, slot.ActivePayload) + s.deleteStagedPayloadBestEffort(ctx, kb, slot.PendingPayload) +} + +func (s *knowledgeService) deleteStagedPayloadBestEffort( + ctx context.Context, kb *types.KnowledgeBase, raw types.JSON, +) { + if len(raw) == 0 { + return + } + var payload types.KnowledgeFileUpdatePayload + if err := json.Unmarshal(raw, &payload); err != nil { + logger.Errorf(ctx, "Decode staged file update payload for cleanup failed: %v", err) + return + } + s.deleteReplacementFileBestEffort(ctx, kb, payload.NewFilePath) +} + +func (s *knowledgeService) deleteSupersededActivePayloadBestEffort( + ctx context.Context, kb *types.KnowledgeBase, currentPath string, raw types.JSON, +) { + if len(raw) == 0 { + return + } + var payload types.KnowledgeFileUpdatePayload + if err := json.Unmarshal(raw, &payload); err != nil { + logger.Errorf(ctx, "Decode superseded active file update payload failed: %v", err) + return + } + if payload.NewFilePath == currentPath { + return + } + s.deleteReplacementFileBestEffort(ctx, kb, payload.NewFilePath) +} + +func sameKnowledgeFileUpdate(a, b *types.KnowledgeFileUpdatePayload) bool { + if a == nil || b == nil { + return false + } + return a.NewFileName == b.NewFileName && + a.NewFolderPath == b.NewFolderPath && + a.NewFileType == b.NewFileType && + a.NewFileSize == b.NewFileSize && + a.NewFileHash == b.NewFileHash && + a.MetadataProvided == b.MetadataProvided && + reflect.DeepEqual(a.Metadata, b.Metadata) && + a.TagIDsProvided == b.TagIDsProvided && + slices.Equal(a.TagIDs, b.TagIDs) && + a.ChannelProvided == b.ChannelProvided && + a.Channel == b.Channel && + a.ProcessProvided == b.ProcessProvided && + reflect.DeepEqual(a.ProcessConfig, b.ProcessConfig) +} + +func replacementMetadata( + knowledge *types.Knowledge, + payload *types.KnowledgeFileUpdatePayload, +) (types.JSON, error) { + metadata, err := knowledge.Metadata.Map() + if err != nil { + return nil, fmt.Errorf("parse knowledge metadata: %w", err) + } + if metadata == nil { + metadata = make(map[string]interface{}) + } + if payload.MetadataProvided { + for key, value := range payload.Metadata { + if key == "process_overrides" { + continue + } + metadata[key] = value + } + } + bytes, err := json.Marshal(metadata) + if err != nil { + return nil, err + } + knowledge.Metadata = types.JSON(bytes) + if payload.ProcessProvided { + if err := knowledge.SetProcessOverrides(payload.ProcessConfig); err != nil { + return nil, err + } + } + return knowledge.Metadata, nil +} + +func replacementChannel(current string, payload *types.KnowledgeFileUpdatePayload) string { + if payload.ChannelProvided { + return payload.Channel + } + return current +} + +func (s *knowledgeService) finishKnowledgeFileUpdate( + ctx context.Context, + kb *types.KnowledgeBase, + knowledge *types.Knowledge, + payload *types.KnowledgeFileUpdatePayload, +) error { + if payload.TagIDsProvided { + if err := s.repo.SetKnowledgeTags(ctx, knowledge.ID, payload.TagIDs); err != nil { + return fmt.Errorf("set replacement knowledge tags: %w", err) + } + } + overrides, err := knowledge.ProcessOverrides() + if err != nil { + return err + } + eff := ResolveProcessConfig(kb, overrides) + questionCount := eff.QuestionGenerationConfig.QuestionCount + if questionCount <= 0 { + questionCount = 3 + } + lang, _ := types.LanguageFromContext(ctx) + documentPayload := types.DocumentProcessPayload{ + TenantID: payload.TenantID, + KnowledgeID: knowledge.ID, + KnowledgeBaseID: knowledge.KnowledgeBaseID, + FilePath: knowledge.FilePath, + FileName: knowledge.FileName, + FileType: knowledge.FileType, + EnableMultimodel: eff.EnableMultimodel, + EnableQuestionGeneration: eff.QuestionGenerationConfig.Enabled, + QuestionCount: questionCount, + Language: lang, + Attempt: payload.Attempt, + } + langfuse.InjectTracing(ctx, &documentPayload) + payloadBytes, err := json.Marshal(documentPayload) + if err != nil { + return err + } + documentTask := asynq.NewTask(types.TypeDocumentProcess, payloadBytes) + _, err = s.task.Enqueue( + documentTask, + documentProcessTaskOptions(s.config, asynq.TaskID(payload.DocumentTaskID), asynq.MaxRetry(3))..., + ) + if err != nil && !stderrors.Is(err, asynq.ErrTaskIDConflict) && !stderrors.Is(err, asynq.ErrDuplicateTask) { + return fmt.Errorf("enqueue replacement document process: %w", err) + } + + s.deleteReplacementFileBestEffort(ctx, kb, payload.OldFilePath) + if slices.Contains([]string{"csv", "xlsx", "xls"}, knowledge.FileType) { + NewDataTableSummaryTask(ctx, s.task, payload.TenantID, knowledge.ID, kb.SummaryModelID, kb.EmbeddingModelID) + } + recordKBActivity(ctx, s.audit, payload.TenantID, payload.KnowledgeBaseID, + types.AuditActionKnowledgeUpdated, "knowledge", knowledge.ID, types.AuditOutcomeAccepted, + map[string]any{ + "title": knowledge.Title, + "source_type": "file", + "file_type": knowledge.FileType, + "processing_status": types.ParseStatusPending, + "trigger": kbActivityTrigger(ctx), + }) + return nil +} + +func (s *knowledgeService) deleteReplacementFileBestEffort( + ctx context.Context, + kb *types.KnowledgeBase, + filePath string, +) { + if strings.TrimSpace(filePath) == "" { + return + } + fileSvc := s.resolveFileServiceForPath(ctx, kb, filePath) + if fileSvc == nil { + logger.Errorf(ctx, "Cannot delete replacement file %s: file service is not configured", filePath) + return + } + if err := fileSvc.DeleteFile(ctx, filePath); err != nil { + logger.Errorf(ctx, "Failed to delete replacement file %s: %v", filePath, err) + } +} diff --git a/internal/application/service/knowledge_file_update_coordinator_test.go b/internal/application/service/knowledge_file_update_coordinator_test.go new file mode 100644 index 0000000000..79c2df69b3 --- /dev/null +++ b/internal/application/service/knowledge_file_update_coordinator_test.go @@ -0,0 +1,158 @@ +package service + +import ( + "context" + "encoding/json" + "testing" + + "github.com/Tencent/WeKnora/internal/application/repository" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/google/uuid" + "github.com/hibiken/asynq" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +type fileUpdateTenantRepoStub struct { + interfaces.TenantRepository +} + +func (fileUpdateTenantRepoStub) GetTenantByID(context.Context, uint64) (*types.Tenant, error) { + return &types.Tenant{ID: 1}, nil +} + +func setupFileUpdateCoordinatorRepo(t *testing.T) (*gorm.DB, interfaces.KnowledgeRepository) { + t.Helper() + db, err := gorm.Open(sqlite.Open("file:"+uuid.NewString()+"?mode=memory&cache=shared"), &gorm.Config{}) + require.NoError(t, err) + sqlDB, err := db.DB() + require.NoError(t, err) + sqlDB.SetMaxOpenConns(1) + t.Cleanup(func() { _ = sqlDB.Close() }) + require.NoError(t, db.Exec(` + CREATE TABLE knowledges ( + id VARCHAR(36) PRIMARY KEY, + tenant_id INTEGER NOT NULL, + knowledge_base_id VARCHAR(36) NOT NULL, + type VARCHAR(50) NOT NULL, + parse_status VARCHAR(50) NOT NULL, + file_path TEXT, + file_hash VARCHAR(64), + deleted_at DATETIME + ); + CREATE TABLE knowledge_file_update_slots ( + knowledge_id VARCHAR(36) PRIMARY KEY, + tenant_id INTEGER NOT NULL, + knowledge_base_id VARCHAR(36) NOT NULL, + latest_version INTEGER NOT NULL DEFAULT 0, + active_version INTEGER, + active_state VARCHAR(16) NOT NULL DEFAULT 'idle', + active_payload TEXT, + pending_version INTEGER, + pending_payload TEXT, + last_error TEXT NOT NULL DEFAULT '', + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP + ); + `).Error) + return db, repository.NewKnowledgeRepository(db) +} + +func TestProcessKnowledgeFileUpdateDefersWhileCurrentVersionIsProcessing(t *testing.T) { + db, repo := setupFileUpdateCoordinatorRepo(t) + knowledgeID := uuid.NewString() + kbID := uuid.NewString() + require.NoError(t, db.Exec(` + INSERT INTO knowledges(id, tenant_id, knowledge_base_id, type, parse_status, file_path, file_hash) + VALUES (?, 1, ?, 'file', ?, 'current/path.md', 'current-hash') + `, knowledgeID, kbID, types.ParseStatusProcessing).Error) + active, err := json.Marshal(types.KnowledgeFileUpdatePayload{ + TenantID: 1, KnowledgeBaseID: kbID, KnowledgeID: knowledgeID, + NewFilePath: "staged/latest.md", NewFileName: "latest.md", NewFileHash: "latest-hash", + }) + require.NoError(t, err) + staged, err := repo.StageKnowledgeFileUpdate( + context.Background(), 1, knowledgeID, kbID, types.JSON(active), nil, + ) + require.NoError(t, err) + + taskQueue := &fileUpdateTaskStub{} + svc := &knowledgeService{ + repo: repo, + tenantRepo: fileUpdateTenantRepoStub{}, + kbService: &createKnowledgeFileKBServiceStub{kb: &types.KnowledgeBase{ID: kbID}}, + task: taskQueue, + } + wake, err := json.Marshal(types.KnowledgeFileUpdateTaskPayload{ + TenantID: 1, KnowledgeBaseID: kbID, KnowledgeID: knowledgeID, ActiveVersion: staged.ActiveVersion, + }) + require.NoError(t, err) + + err = svc.ProcessKnowledgeFileUpdate( + context.Background(), asynq.NewTask(types.TypeKnowledgeFileUpdate, wake), + ) + require.NoError(t, err) + + slot, err := repo.GetKnowledgeFileUpdateSlot(context.Background(), 1, knowledgeID) + require.NoError(t, err) + assert.Equal(t, types.KnowledgeFileUpdateStateRetryWait, slot.ActiveState) + require.Len(t, taskQueue.tasks, 1) + assert.Equal(t, types.TypeKnowledgeFileUpdate, taskQueue.tasks[0].Type()) + var delayed types.KnowledgeFileUpdateTaskPayload + require.NoError(t, json.Unmarshal(taskQueue.tasks[0].Payload(), &delayed)) + assert.Equal(t, uint64(1), delayed.WakeSequence, + "the delayed wake must not share the currently executing task's unique fingerprint") + + var status string + require.NoError(t, db.Raw(`SELECT parse_status FROM knowledges WHERE id = ?`, knowledgeID).Scan(&status).Error) + assert.Equal(t, types.ParseStatusProcessing, status) +} + +func TestProcessKnowledgeFileUpdateStaleWakeRearmsCurrentActive(t *testing.T) { + db, repo := setupFileUpdateCoordinatorRepo(t) + knowledgeID := uuid.NewString() + kbID := uuid.NewString() + require.NoError(t, db.Exec(` + INSERT INTO knowledges(id, tenant_id, knowledge_base_id, type, parse_status, file_path, file_hash) + VALUES (?, 1, ?, 'file', ?, 'current/path.md', 'current-hash') + `, knowledgeID, kbID, types.ParseStatusCompleted).Error) + firstPayload, err := json.Marshal(types.KnowledgeFileUpdatePayload{ + TenantID: 1, KnowledgeBaseID: kbID, KnowledgeID: knowledgeID, + NewFilePath: "staged/first.md", NewFileHash: "first-hash", + }) + require.NoError(t, err) + first, err := repo.StageKnowledgeFileUpdate( + context.Background(), 1, knowledgeID, kbID, types.JSON(firstPayload), nil, + ) + require.NoError(t, err) + latestPayload, err := json.Marshal(types.KnowledgeFileUpdatePayload{ + TenantID: 1, KnowledgeBaseID: kbID, KnowledgeID: knowledgeID, + NewFilePath: "staged/latest.md", NewFileHash: "latest-hash", + }) + require.NoError(t, err) + latest, err := repo.StageKnowledgeFileUpdate( + context.Background(), 1, knowledgeID, kbID, types.JSON(latestPayload), nil, + ) + require.NoError(t, err) + _, err = repo.CompleteKnowledgeFileUpdate(context.Background(), 1, knowledgeID, first.ActiveVersion) + require.NoError(t, err) + + taskQueue := &fileUpdateTaskStub{} + svc := &knowledgeService{repo: repo, tenantRepo: fileUpdateTenantRepoStub{}, task: taskQueue} + staleWake, err := json.Marshal(types.KnowledgeFileUpdateTaskPayload{ + TenantID: 1, KnowledgeBaseID: kbID, KnowledgeID: knowledgeID, ActiveVersion: first.ActiveVersion, + }) + require.NoError(t, err) + + err = svc.ProcessKnowledgeFileUpdate( + context.Background(), asynq.NewTask(types.TypeKnowledgeFileUpdate, staleWake), + ) + require.NoError(t, err) + require.Len(t, taskQueue.tasks, 1) + var currentWake types.KnowledgeFileUpdateTaskPayload + require.NoError(t, json.Unmarshal(taskQueue.tasks[0].Payload(), ¤tWake)) + assert.Equal(t, latest.Version, currentWake.ActiveVersion) +} diff --git a/internal/application/service/knowledge_file_update_test.go b/internal/application/service/knowledge_file_update_test.go new file mode 100644 index 0000000000..af9271caaf --- /dev/null +++ b/internal/application/service/knowledge_file_update_test.go @@ -0,0 +1,549 @@ +package service + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "github.com/Tencent/WeKnora/internal/application/repository" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/hibiken/asynq" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fileUpdateRepoStub struct { + interfaces.KnowledgeRepository + + knowledge *types.Knowledge + listKnowledges []*types.Knowledge + stageResult *types.KnowledgeFileUpdateStageResult + slot *types.KnowledgeFileUpdateSlot + stagedPayload types.JSON + stagedExpected *uint64 + stageCalls int + applyCalls int + applyValues map[string]interface{} +} + +func (r *fileUpdateRepoStub) TransitionKnowledgeFileUpdateState( + _ context.Context, _ uint64, _ string, version uint64, fromState, toState, lastError string, +) (bool, error) { + if r.slot == nil || r.slot.ActiveVersion == nil || *r.slot.ActiveVersion != version || + r.slot.ActiveState != fromState { + return false, nil + } + r.slot.ActiveState = toState + r.slot.LastError = lastError + return true, nil +} + +func (r *fileUpdateRepoStub) CancelFailedKnowledgeFileUpdate( + _ context.Context, _ uint64, _ string, version uint64, +) (*types.KnowledgeFileUpdateSlot, error) { + if r.slot == nil || r.slot.ActiveVersion == nil || *r.slot.ActiveVersion != version || + r.slot.ActiveState != types.KnowledgeFileUpdateStateFailed { + return nil, repository.ErrKnowledgeFileUpdateStateConflict + } + cancelled := r.slot + r.slot = nil + return cancelled, nil +} + +func (r *fileUpdateRepoStub) GetKnowledgeFileUpdateSlot( + context.Context, uint64, string, +) (*types.KnowledgeFileUpdateSlot, error) { + return r.slot, nil +} + +func (r *fileUpdateRepoStub) GetKnowledgeByID( + context.Context, uint64, string, +) (*types.Knowledge, error) { + copy := *r.knowledge + return ©, nil +} + +func (r *fileUpdateRepoStub) ListKnowledgeByKnowledgeBaseID( + context.Context, uint64, string, +) ([]*types.Knowledge, error) { + return r.listKnowledges, nil +} + +func (r *fileUpdateRepoStub) CheckKnowledgeExistsExcluding( + context.Context, uint64, string, string, *types.KnowledgeCheckParams, +) (bool, *types.Knowledge, error) { + return false, nil, nil +} + +func (r *fileUpdateRepoStub) StageKnowledgeFileUpdate( + _ context.Context, + _ uint64, + _ string, + _ string, + payload types.JSON, + expectedVersion *uint64, +) (*types.KnowledgeFileUpdateStageResult, error) { + r.stageCalls++ + r.stagedPayload = append(types.JSON(nil), payload...) + r.stagedExpected = expectedVersion + return r.stageResult, nil +} + +func (r *fileUpdateRepoStub) UpdateApplyingKnowledgeFileColumns( + _ context.Context, + _ uint64, + knowledgeID string, + kbID string, + expectedFilePath string, + expectedFileHash string, + values map[string]interface{}, +) (bool, error) { + r.applyCalls++ + r.applyValues = values + if r.knowledge == nil || + r.knowledge.ID != knowledgeID || + r.knowledge.KnowledgeBaseID != kbID || + r.knowledge.ParseStatus != types.ParseStatusReplacing || + r.knowledge.FilePath != expectedFilePath || + r.knowledge.FileHash != expectedFileHash { + return false, nil + } + if status, ok := values["parse_status"].(string); ok { + r.knowledge.ParseStatus = status + } + if message, ok := values["error_message"].(string); ok { + r.knowledge.ErrorMessage = message + } + return true, nil +} + +func (r *fileUpdateRepoStub) GetKnowledgeTags( + context.Context, []string, +) (map[string][]*types.KnowledgeTag, error) { + return map[string][]*types.KnowledgeTag{}, nil +} + +type fileUpdateTaskStub struct { + tasks []*asynq.Task +} + +func (s *fileUpdateTaskStub) Enqueue(task *asynq.Task, _ ...asynq.Option) (*asynq.TaskInfo, error) { + s.tasks = append(s.tasks, task) + return &asynq.TaskInfo{ID: "update-task", Queue: types.QueueMaintenance}, nil +} + +func newFileUpdateService( + repo *fileUpdateRepoStub, fileSvc interfaces.FileService, task interfaces.TaskEnqueuer, +) *knowledgeService { + return &knowledgeService{ + repo: repo, + kbService: &createKnowledgeFileKBServiceStub{kb: &types.KnowledgeBase{ + ID: repo.knowledge.KnowledgeBaseID, + }}, + fileSvc: fileSvc, + task: task, + } +} + +func TestUpdateKnowledgeFileFirstVersionBecomesActive(t *testing.T) { + repo := &fileUpdateRepoStub{ + knowledge: &types.Knowledge{ + ID: "knowledge-1", KnowledgeBaseID: "kb-1", Type: "file", + FilePath: "old/path.md", FileName: "old.md", FileHash: "old-hash", + ParseStatus: types.ParseStatusCompleted, + }, + stageResult: &types.KnowledgeFileUpdateStageResult{ + Version: 1, State: types.KnowledgeFileUpdateResultActive, ActiveVersion: 1, + }, + } + fileSvc := &createKnowledgeFileServiceStub{} + task := &fileUpdateTaskStub{} + expected := uint64(0) + + result, err := newFileUpdateService(repo, fileSvc, task).UpdateKnowledgeFile( + newCreateKnowledgeFileContext(), + &types.KnowledgeFileUpdateRequest{ + KnowledgeBaseID: "kb-1", KnowledgeID: "knowledge-1", + File: newMultipartFileHeader(t, "new.md", "new content"), + ExpectedUpdateVersion: &expected, + }, + ) + + require.NoError(t, err) + require.NotNil(t, result) + assert.Equal(t, uint64(1), result.UpdateVersion) + assert.Equal(t, types.KnowledgeFileUpdateResultActive, result.UpdateState) + assert.Equal(t, types.ParseStatusCompleted, result.Knowledge.ParseStatus, + "accepting an update must not interrupt the current parsed version") + assert.Equal(t, &expected, repo.stagedExpected) + require.Len(t, task.tasks, 1) + assert.Equal(t, types.TypeKnowledgeFileUpdate, task.tasks[0].Type()) + + var staged types.KnowledgeFileUpdatePayload + require.NoError(t, json.Unmarshal(repo.stagedPayload, &staged)) + assert.Equal(t, "new.md", staged.NewFileName) + assert.Empty(t, staged.NewFolderPath) + assert.NotEmpty(t, staged.NewFileHash) + assert.Empty(t, staged.OldFilePath, "the coordinator captures the current source only when it is safe to apply") +} + +func TestCreateOrUpdateKnowledgeFromFileMatchesUniqueFilename(t *testing.T) { + existing := &types.Knowledge{ + ID: "knowledge-1", KnowledgeBaseID: "kb-1", Type: "file", + FilePath: "old/path.md", FileName: "doc.md", FileHash: "old-hash", + ParseStatus: types.ParseStatusCompleted, + } + repo := &fileUpdateRepoStub{ + knowledge: existing, + listKnowledges: []*types.Knowledge{existing}, + stageResult: &types.KnowledgeFileUpdateStageResult{ + Version: 1, State: types.KnowledgeFileUpdateResultActive, ActiveVersion: 1, + }, + } + fileSvc := &createKnowledgeFileServiceStub{} + task := &fileUpdateTaskStub{} + + result, err := newFileUpdateService(repo, fileSvc, task).CreateOrUpdateKnowledgeFromFile( + newCreateKnowledgeFileContext(), + &types.KnowledgeFileCreateOrUpdateRequest{ + KnowledgeBaseID: "kb-1", + File: newMultipartFileHeader(t, "doc.md", "new content"), + }, + ) + + require.NoError(t, err) + require.NotNil(t, result) + assert.Equal(t, "updated", result.Action) + assert.Equal(t, uint64(1), result.UpdateVersion) + assert.Equal(t, 1, repo.stageCalls) + assert.Equal(t, "knowledge-1", fileSvc.savedWithKnowledgeID) +} + +func TestCreateOrUpdateKnowledgeFromFileMatchesPathQualifiedUniqueFilename(t *testing.T) { + existing := &types.Knowledge{ + ID: "knowledge-1", KnowledgeBaseID: "kb-1", Type: "file", + FilePath: "old/path.md", FileName: "doc.md", FileHash: "old-hash", + ParseStatus: types.ParseStatusCompleted, + } + repo := &fileUpdateRepoStub{ + knowledge: existing, + listKnowledges: []*types.Knowledge{existing}, + stageResult: &types.KnowledgeFileUpdateStageResult{ + Version: 1, State: types.KnowledgeFileUpdateResultActive, ActiveVersion: 1, + }, + } + fileSvc := &createKnowledgeFileServiceStub{} + task := &fileUpdateTaskStub{} + + result, err := newFileUpdateService(repo, fileSvc, task).CreateOrUpdateKnowledgeFromFile( + newCreateKnowledgeFileContext(), + &types.KnowledgeFileCreateOrUpdateRequest{ + KnowledgeBaseID: "kb-1", + File: newMultipartFileHeader(t, "doc.md", "new content"), + CustomFileName: "docs/spec/doc.md", + }, + ) + + require.NoError(t, err) + require.NotNil(t, result) + assert.Equal(t, "updated", result.Action) + assert.Equal(t, 1, repo.stageCalls) + var staged types.KnowledgeFileUpdatePayload + require.NoError(t, json.Unmarshal(repo.stagedPayload, &staged)) + assert.Equal(t, "doc.md", staged.NewFileName) + assert.Equal(t, "docs/spec", staged.NewFolderPath) +} + +func TestCreateOrUpdateKnowledgeFromFileRejectsAmbiguousFilename(t *testing.T) { + first := &types.Knowledge{ + ID: "knowledge-1", KnowledgeBaseID: "kb-1", Type: "file", + FilePath: "old/one.md", FileName: "doc.md", FileHash: "one", + ParseStatus: types.ParseStatusCompleted, + } + second := &types.Knowledge{ + ID: "knowledge-2", KnowledgeBaseID: "kb-1", Type: "file", + FilePath: "old/two.md", FileName: "doc.md", FileHash: "two", + ParseStatus: types.ParseStatusCompleted, + } + repo := &fileUpdateRepoStub{ + knowledge: first, + listKnowledges: []*types.Knowledge{first, second}, + } + fileSvc := &createKnowledgeFileServiceStub{} + task := &fileUpdateTaskStub{} + + result, err := newFileUpdateService(repo, fileSvc, task).CreateOrUpdateKnowledgeFromFile( + newCreateKnowledgeFileContext(), + &types.KnowledgeFileCreateOrUpdateRequest{ + KnowledgeBaseID: "kb-1", + File: newMultipartFileHeader(t, "doc.md", "new content"), + }, + ) + + require.Error(t, err) + require.Nil(t, result) + assert.Zero(t, repo.stageCalls) + assert.Zero(t, fileSvc.saveCalls) + assert.Empty(t, task.tasks) +} + +func TestUpdateKnowledgeFilePathQualifiedNameStagesFolderMove(t *testing.T) { + repo := &fileUpdateRepoStub{ + knowledge: &types.Knowledge{ + ID: "knowledge-1", KnowledgeBaseID: "kb-1", Type: "file", + FilePath: "old/path.md", FileName: "old.md", FileHash: "old-hash", + ParseStatus: types.ParseStatusCompleted, + }, + stageResult: &types.KnowledgeFileUpdateStageResult{ + Version: 1, State: types.KnowledgeFileUpdateResultActive, ActiveVersion: 1, + }, + } + task := &fileUpdateTaskStub{} + + result, err := newFileUpdateService(repo, &createKnowledgeFileServiceStub{}, task).UpdateKnowledgeFile( + newCreateKnowledgeFileContext(), + &types.KnowledgeFileUpdateRequest{ + KnowledgeBaseID: "kb-1", + KnowledgeID: "knowledge-1", + File: newMultipartFileHeader(t, "upload.bin", "latest content"), + CustomFileName: "docs/spec/latest.md", + }, + ) + + require.NoError(t, err) + require.NotNil(t, result) + var staged types.KnowledgeFileUpdatePayload + require.NoError(t, json.Unmarshal(repo.stagedPayload, &staged)) + assert.Equal(t, "latest.md", staged.NewFileName) + assert.Equal(t, "docs/spec", staged.NewFolderPath) + assert.Equal(t, "md", staged.NewFileType) +} + +func TestUpdateKnowledgeFileWhileProcessingOnlyReplacesPending(t *testing.T) { + superseded, err := json.Marshal(types.KnowledgeFileUpdatePayload{NewFilePath: "staged/b.md"}) + require.NoError(t, err) + repo := &fileUpdateRepoStub{ + knowledge: &types.Knowledge{ + ID: "knowledge-1", KnowledgeBaseID: "kb-1", Type: "file", + FilePath: "old/path.md", FileName: "old.md", FileHash: "old-hash", + ParseStatus: types.ParseStatusProcessing, + }, + stageResult: &types.KnowledgeFileUpdateStageResult{ + Version: 3, State: types.KnowledgeFileUpdateResultPending, ActiveVersion: 1, + ReplacedPendingPayload: types.JSON(superseded), + }, + } + fileSvc := &createKnowledgeFileServiceStub{} + task := &fileUpdateTaskStub{} + + result, err := newFileUpdateService(repo, fileSvc, task).UpdateKnowledgeFile( + newCreateKnowledgeFileContext(), + &types.KnowledgeFileUpdateRequest{ + KnowledgeBaseID: "kb-1", KnowledgeID: "knowledge-1", + File: newMultipartFileHeader(t, "latest.md", "latest content"), + }, + ) + + require.NoError(t, err) + assert.Equal(t, uint64(3), result.UpdateVersion) + assert.Equal(t, types.KnowledgeFileUpdateResultPending, result.UpdateState) + require.Len(t, task.tasks, 1, "pending submissions opportunistically wake the active coordinator") + assert.Equal(t, 1, fileSvc.deleteCalls) + assert.Equal(t, "staged/b.md", fileSvc.deletedPath) +} + +func TestUpdateKnowledgeFileDoesNotRollbackWhenSupersededCleanupFails(t *testing.T) { + superseded, err := json.Marshal(types.KnowledgeFileUpdatePayload{NewFilePath: "staged/b.md"}) + require.NoError(t, err) + repo := &fileUpdateRepoStub{ + knowledge: &types.Knowledge{ + ID: "knowledge-1", KnowledgeBaseID: "kb-1", Type: "file", + FilePath: "old/path.md", FileName: "old.md", FileHash: "old-hash", + ParseStatus: types.ParseStatusProcessing, + }, + stageResult: &types.KnowledgeFileUpdateStageResult{ + Version: 3, State: types.KnowledgeFileUpdateResultPending, ActiveVersion: 1, + ReplacedPendingPayload: types.JSON(superseded), + }, + } + fileSvc := &createKnowledgeFileServiceStub{deleteErr: errors.New("storage unavailable")} + task := &fileUpdateTaskStub{} + + result, err := newFileUpdateService(repo, fileSvc, task).UpdateKnowledgeFile( + newCreateKnowledgeFileContext(), + &types.KnowledgeFileUpdateRequest{ + KnowledgeBaseID: "kb-1", KnowledgeID: "knowledge-1", + File: newMultipartFileHeader(t, "latest.md", "latest content"), + }, + ) + + require.NoError(t, err) + assert.Equal(t, uint64(3), result.UpdateVersion) + assert.Equal(t, 1, fileSvc.deleteCalls) + require.Len(t, task.tasks, 1) +} + +func TestUpdateKnowledgeFileSameLatestPendingIsUnchanged(t *testing.T) { + file := newMultipartFileHeader(t, "latest.md", "latest content") + hash, err := calculateFileHash(file) + require.NoError(t, err) + pending, err := json.Marshal(types.KnowledgeFileUpdatePayload{ + NewFileName: "latest.md", NewFileType: "md", NewFileSize: file.Size, NewFileHash: hash, + }) + require.NoError(t, err) + activeVersion, pendingVersion := uint64(1), uint64(2) + repo := &fileUpdateRepoStub{ + knowledge: &types.Knowledge{ + ID: "knowledge-1", KnowledgeBaseID: "kb-1", Type: "file", + FilePath: "old/path.md", FileName: "old.md", FileHash: "old-hash", + ParseStatus: types.ParseStatusProcessing, + FileUpdateVersion: 2, FileUpdateState: types.KnowledgeFileUpdateResultPending, + }, + slot: &types.KnowledgeFileUpdateSlot{ + LatestVersion: 2, ActiveVersion: &activeVersion, PendingVersion: &pendingVersion, + PendingPayload: types.JSON(pending), + }, + } + fileSvc := &createKnowledgeFileServiceStub{} + task := &fileUpdateTaskStub{} + + result, err := newFileUpdateService(repo, fileSvc, task).UpdateKnowledgeFile( + newCreateKnowledgeFileContext(), + &types.KnowledgeFileUpdateRequest{ + KnowledgeBaseID: "kb-1", KnowledgeID: "knowledge-1", File: file, + }, + ) + + require.NoError(t, err) + assert.Equal(t, "unchanged", result.Action) + assert.Equal(t, uint64(2), result.UpdateVersion) + assert.Equal(t, types.KnowledgeFileUpdateResultPending, result.UpdateState) + assert.Zero(t, repo.stageCalls) + assert.Empty(t, task.tasks) + assert.Zero(t, fileSvc.saveCalls) + assert.Zero(t, fileSvc.deleteCalls) +} + +func TestRetryKnowledgeFileUpdateRearmsExactFailedVersion(t *testing.T) { + version := uint64(7) + repo := &fileUpdateRepoStub{ + knowledge: &types.Knowledge{ + ID: "knowledge-1", TenantID: 1, KnowledgeBaseID: "kb-1", Type: "file", + FilePath: "old/path.md", ParseStatus: types.ParseStatusCompleted, + }, + slot: &types.KnowledgeFileUpdateSlot{ + KnowledgeID: "knowledge-1", TenantID: 1, KnowledgeBaseID: "kb-1", + ActiveVersion: &version, ActiveState: types.KnowledgeFileUpdateStateFailed, + }, + } + task := &fileUpdateTaskStub{} + svc := newFileUpdateService(repo, &createKnowledgeFileServiceStub{}, task) + + _, err := svc.RetryKnowledgeFileUpdate(newCreateKnowledgeFileContext(), "knowledge-1") + require.NoError(t, err) + assert.Equal(t, types.KnowledgeFileUpdateStateWaiting, repo.slot.ActiveState) + require.Len(t, task.tasks, 1) + var wake types.KnowledgeFileUpdateTaskPayload + require.NoError(t, json.Unmarshal(task.tasks[0].Payload(), &wake)) + assert.Equal(t, version, wake.ActiveVersion) +} + +func TestDiscardKnowledgeFileUpdateCleansFailedActiveAndPending(t *testing.T) { + version, pendingVersion := uint64(7), uint64(8) + active, err := json.Marshal(types.KnowledgeFileUpdatePayload{NewFilePath: "staged/a.md"}) + require.NoError(t, err) + pending, err := json.Marshal(types.KnowledgeFileUpdatePayload{NewFilePath: "staged/b.md"}) + require.NoError(t, err) + repo := &fileUpdateRepoStub{ + knowledge: &types.Knowledge{ + ID: "knowledge-1", TenantID: 1, KnowledgeBaseID: "kb-1", Type: "file", + FilePath: "old/path.md", ParseStatus: types.ParseStatusCompleted, + }, + slot: &types.KnowledgeFileUpdateSlot{ + KnowledgeID: "knowledge-1", TenantID: 1, KnowledgeBaseID: "kb-1", + ActiveVersion: &version, ActiveState: types.KnowledgeFileUpdateStateFailed, + ActivePayload: types.JSON(active), PendingVersion: &pendingVersion, PendingPayload: types.JSON(pending), + }, + } + fileSvc := &createKnowledgeFileServiceStub{} + svc := newFileUpdateService(repo, fileSvc, &fileUpdateTaskStub{}) + + _, err = svc.DiscardKnowledgeFileUpdate(newCreateKnowledgeFileContext(), "knowledge-1") + require.NoError(t, err) + assert.Nil(t, repo.slot) + assert.Equal(t, 2, fileSvc.deleteCalls) +} + +func TestRestoreFailedKnowledgeFileUpdateClaimRestoresReplacingStatus(t *testing.T) { + version := uint64(7) + active, err := json.Marshal(types.KnowledgeFileUpdatePayload{ + KnowledgeBaseID: "kb-1", + OldParseStatus: types.ParseStatusCompleted, + OldFilePath: "old/path.md", + OldFileHash: "old-hash", + NewFilePath: "staged/latest.md", + NewFileName: "latest.md", + NewFileHash: "latest-hash", + DocumentTaskID: "doc-task", + }) + require.NoError(t, err) + repo := &fileUpdateRepoStub{ + knowledge: &types.Knowledge{ + ID: "knowledge-1", TenantID: 1, KnowledgeBaseID: "kb-1", Type: "file", + FilePath: "old/path.md", FileHash: "old-hash", + ParseStatus: types.ParseStatusReplacing, + ErrorMessage: "cleanup failed", + }, + slot: &types.KnowledgeFileUpdateSlot{ + KnowledgeID: "knowledge-1", TenantID: 1, KnowledgeBaseID: "kb-1", + ActiveVersion: &version, + ActiveState: types.KnowledgeFileUpdateStateFailed, + ActivePayload: types.JSON(active), + }, + } + svc := newFileUpdateService(repo, &createKnowledgeFileServiceStub{}, &fileUpdateTaskStub{}) + + svc.restoreFailedKnowledgeFileUpdateClaim(context.Background(), types.KnowledgeFileUpdateTaskPayload{ + TenantID: 1, KnowledgeBaseID: "kb-1", KnowledgeID: "knowledge-1", ActiveVersion: version, + }) + + assert.Equal(t, types.ParseStatusCompleted, repo.knowledge.ParseStatus) + assert.Empty(t, repo.knowledge.ErrorMessage) + assert.Equal(t, 1, repo.applyCalls) +} + +func TestRestoreFailedKnowledgeFileUpdateClaimDoesNotOverwriteSwitchedVersion(t *testing.T) { + version := uint64(7) + active, err := json.Marshal(types.KnowledgeFileUpdatePayload{ + KnowledgeBaseID: "kb-1", + OldParseStatus: types.ParseStatusCompleted, + OldFilePath: "old/path.md", + OldFileHash: "old-hash", + NewFilePath: "staged/latest.md", + NewFileHash: "latest-hash", + }) + require.NoError(t, err) + repo := &fileUpdateRepoStub{ + knowledge: &types.Knowledge{ + ID: "knowledge-1", TenantID: 1, KnowledgeBaseID: "kb-1", Type: "file", + FilePath: "staged/latest.md", FileHash: "latest-hash", + ParseStatus: types.ParseStatusPending, + }, + slot: &types.KnowledgeFileUpdateSlot{ + KnowledgeID: "knowledge-1", TenantID: 1, KnowledgeBaseID: "kb-1", + ActiveVersion: &version, + ActiveState: types.KnowledgeFileUpdateStateFailed, + ActivePayload: types.JSON(active), + }, + } + svc := newFileUpdateService(repo, &createKnowledgeFileServiceStub{}, &fileUpdateTaskStub{}) + + svc.restoreFailedKnowledgeFileUpdateClaim(context.Background(), types.KnowledgeFileUpdateTaskPayload{ + TenantID: 1, KnowledgeBaseID: "kb-1", KnowledgeID: "knowledge-1", ActiveVersion: version, + }) + + assert.Equal(t, types.ParseStatusPending, repo.knowledge.ParseStatus) + assert.Equal(t, 1, repo.applyCalls) +} diff --git a/internal/application/service/knowledge_housekeeping.go b/internal/application/service/knowledge_housekeeping.go index 95abd863bb..5c18371775 100644 --- a/internal/application/service/knowledge_housekeeping.go +++ b/internal/application/service/knowledge_housekeeping.go @@ -20,6 +20,8 @@ package service import ( "context" + "encoding/json" + "errors" "os" "strings" "sync" @@ -29,6 +31,7 @@ import ( "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/hibiken/asynq" "github.com/robfig/cron/v3" "gorm.io/gorm" ) @@ -45,6 +48,7 @@ type HousekeepingService struct { // nil-safe — a nil inspector disables the queue check and the sweep // falls back to the span/updated_at heuristics alone. inspector interfaces.TaskInspector + task interfaces.TaskEnqueuer mu sync.Mutex started bool @@ -54,12 +58,16 @@ type HousekeepingService struct { // the cron — call Start in the application bootstrap so a misconfigured // cron schedule cannot prevent the rest of the service from coming up. func NewHousekeepingService( - db *gorm.DB, cfg *config.Config, inspector interfaces.TaskInspector, + db *gorm.DB, + cfg *config.Config, + inspector interfaces.TaskInspector, + task interfaces.TaskEnqueuer, ) *HousekeepingService { return &HousekeepingService{ db: db, cfg: cfg, inspector: inspector, + task: task, cron: cron.New(cron.WithSeconds(), cron.WithChain( cron.Recover(cron.DefaultLogger), )), @@ -112,7 +120,7 @@ func (h *HousekeepingService) runSweep(ctx context.Context) { threshold := h.staleThreshold() cutoff := time.Now().Add(-threshold) - // Sweep A: knowledge stuck in "pending", "processing", or "finalizing". + // Sweep A: knowledge stuck in a parsing or file-replacement state. // // Two-stage check is critical here: knowledge.updated_at advances // only at parse_status transitions, but a long stage (DocReader on @@ -138,7 +146,7 @@ func (h *HousekeepingService) runSweep(ctx context.Context) { var candidates []types.Knowledge if err := h.db.WithContext(ctx). Where("parse_status IN ? AND updated_at < ?", - []string{types.ParseStatusPending, types.ParseStatusProcessing, types.ParseStatusFinalizing}, cutoff). + []string{types.ParseStatusPending, types.ParseStatusProcessing, types.ParseStatusFinalizing, types.ParseStatusReplacing}, cutoff). Find(&candidates).Error; err != nil { logger.Warnf(ctx, "[Housekeeping] knowledge candidate query failed: %v", err) return @@ -164,7 +172,7 @@ func (h *HousekeepingService) runSweep(ctx context.Context) { } res := h.db.WithContext(ctx).Model(&types.Knowledge{}). Where("id IN ? AND parse_status IN ?", stuckIDs, - []string{types.ParseStatusPending, types.ParseStatusProcessing, types.ParseStatusFinalizing}). + []string{types.ParseStatusPending, types.ParseStatusProcessing, types.ParseStatusFinalizing, types.ParseStatusReplacing}). Updates(map[string]interface{}{ "parse_status": types.ParseStatusFailed, "error_message": "task stuck in processing > " + threshold.String() + ", recovered by housekeeping", @@ -196,7 +204,13 @@ func (h *HousekeepingService) runSweep(ctx context.Context) { queueSkipped) } - // Sweep B: knowledge summary stuck. Summary is post-parse; threshold + // Sweep B: a durable file-update slot can outlive its Redis wake after a + // restart or enqueue outage. Re-arm only stale slots with no queued/active + // task; failed slots intentionally retain their payload for user retry or + // discard and are counted for operational visibility. + h.recoverStaleKnowledgeFileUpdates(ctx, cutoff) + + // Sweep C: knowledge summary stuck. Summary is post-parse; threshold // is shorter because summary tasks are bounded by a single LLM call. // No span heartbeat exists for the summary stage (it lives in a // downstream asynq task), so we accept the original simple check. @@ -211,6 +225,77 @@ func (h *HousekeepingService) runSweep(ctx context.Context) { } } +func (h *HousekeepingService) recoverStaleKnowledgeFileUpdates(ctx context.Context, cutoff time.Time) { + var slots []*types.KnowledgeFileUpdateSlot + if err := h.db.WithContext(ctx). + Where("active_version IS NOT NULL AND active_state IN ? AND updated_at < ?", []string{ + types.KnowledgeFileUpdateStateWaiting, + types.KnowledgeFileUpdateStateApplying, + types.KnowledgeFileUpdateStateRetryWait, + types.KnowledgeFileUpdateStateFailed, + }, cutoff). + Order("updated_at ASC").Limit(1000).Find(&slots).Error; err != nil { + logger.Warnf(ctx, "[Housekeeping] file update slot query failed: %v", err) + return + } + + failed := 0 + rearmed := 0 + for _, slot := range slots { + if slot == nil || slot.ActiveVersion == nil { + continue + } + if slot.ActiveState == types.KnowledgeFileUpdateStateFailed { + failed++ + continue + } + if h.inspector != nil { + queued, err := h.inspector.HasQueuedTasksForKnowledge(ctx, slot.KnowledgeID) + if err != nil { + logger.Warnf(ctx, "[Housekeeping] file update queue probe failed for %s: %v", slot.KnowledgeID, err) + continue + } + if queued { + continue + } + } + if h.task == nil { + logger.Warnf(ctx, "[Housekeeping] cannot re-arm file update %s: task enqueuer unavailable", slot.KnowledgeID) + continue + } + wake := types.KnowledgeFileUpdateTaskPayload{ + TenantID: slot.TenantID, + KnowledgeBaseID: slot.KnowledgeBaseID, + KnowledgeID: slot.KnowledgeID, + ActiveVersion: *slot.ActiveVersion, + WakeSequence: uint64(time.Now().UnixNano()), + } + payload, err := json.Marshal(wake) + if err != nil { + logger.Warnf(ctx, "[Housekeeping] encode file update wake failed for %s: %v", slot.KnowledgeID, err) + continue + } + _, err = h.task.Enqueue(asynq.NewTask(types.TypeKnowledgeFileUpdate, payload), + asynq.Queue(types.QueueMaintenance), + asynq.MaxRetry(3), + asynq.Timeout(2*time.Hour), + asynq.Unique(2*time.Hour), + ) + if err != nil && !errors.Is(err, asynq.ErrTaskIDConflict) && !errors.Is(err, asynq.ErrDuplicateTask) { + logger.Warnf(ctx, "[Housekeeping] re-arm file update %s failed: %v", slot.KnowledgeID, err) + continue + } + rearmed++ + _ = h.db.WithContext(ctx).Model(&types.KnowledgeFileUpdateSlot{}). + Where("knowledge_id = ? AND active_version = ? AND active_state = ?", + slot.KnowledgeID, *slot.ActiveVersion, slot.ActiveState). + Update("updated_at", time.Now()).Error + } + if rearmed > 0 || failed > 0 { + logger.Infof(ctx, "[Housekeeping] file update slots: rearmed=%d retained_failed=%d", rearmed, failed) + } +} + // filterByLastSpanActivity returns the subset of candidates whose most // recent span row predates `cutoff` — i.e. genuinely stuck. Candidates // with no span rows at all also pass through (they're lite-mode or diff --git a/internal/application/service/knowledge_housekeeping_test.go b/internal/application/service/knowledge_housekeeping_test.go index 9c290df2b0..73f38412d9 100644 --- a/internal/application/service/knowledge_housekeeping_test.go +++ b/internal/application/service/knowledge_housekeeping_test.go @@ -2,6 +2,7 @@ package service import ( "context" + "encoding/json" "errors" "testing" "time" @@ -9,6 +10,7 @@ import ( "github.com/Tencent/WeKnora/internal/config" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/hibiken/asynq" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gorm.io/driver/sqlite" @@ -68,12 +70,29 @@ CREATE TABLE IF NOT EXISTS knowledge_processing_spans ( ); ` +const housekeepingFileUpdateSlotsDDL = ` +CREATE TABLE IF NOT EXISTS knowledge_file_update_slots ( + knowledge_id VARCHAR(64) PRIMARY KEY, + tenant_id INTEGER NOT NULL, + knowledge_base_id VARCHAR(64) NOT NULL, + latest_version INTEGER NOT NULL DEFAULT 0, + active_version INTEGER, + active_state VARCHAR(16) NOT NULL DEFAULT 'idle', + active_payload TEXT, + pending_version INTEGER, + pending_payload TEXT, + last_error TEXT NOT NULL DEFAULT '', + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP +);` + func setupHousekeepingDB(t *testing.T) *gorm.DB { t.Helper() db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) require.NoError(t, err) require.NoError(t, db.Exec(knowledgeTestDDL).Error) require.NoError(t, db.Exec(housekeepingSpansDDL).Error) + require.NoError(t, db.Exec(housekeepingFileUpdateSlotsDDL).Error) return db } @@ -105,6 +124,15 @@ type fakeTaskInspector struct { err error } +type housekeepingTaskEnqueuer struct { + tasks []*asynq.Task +} + +func (e *housekeepingTaskEnqueuer) Enqueue(task *asynq.Task, _ ...asynq.Option) (*asynq.TaskInfo, error) { + e.tasks = append(e.tasks, task) + return &asynq.TaskInfo{ID: "housekeeping-wake"}, nil +} + func (f fakeTaskInspector) CancelTasksForKnowledge( _ context.Context, _ string, ) (int, int, error) { @@ -143,7 +171,7 @@ func newHousekeepingSvcWithInspector(db *gorm.DB, inspector interfaces.TaskInspe // default of 2h+10min is just a constant scale factor. DocumentProcessTimeout: 1 * time.Hour, }} - return NewHousekeepingService(db, cfg, inspector) + return NewHousekeepingService(db, cfg, inspector, nil) } // TestHousekeeping_RecoversAbandoned exercises the happy path: a @@ -308,3 +336,43 @@ func TestHousekeeping_PreservesRecentlyTouched(t *testing.T) { assert.Equal(t, types.ParseStatusProcessing, status, "knowledge updated within the cutoff must be left alone") } + +func TestHousekeepingRearmsStaleFileUpdateAndRetainsFailedSlot(t *testing.T) { + db := setupHousekeepingDB(t) + stale := time.Now().Add(-3 * time.Hour) + for _, fixture := range []struct { + id string + state string + }{ + {id: "update-waiting", state: types.KnowledgeFileUpdateStateWaiting}, + {id: "update-applying-queued", state: types.KnowledgeFileUpdateStateApplying}, + {id: "update-failed", state: types.KnowledgeFileUpdateStateFailed}, + } { + require.NoError(t, db.Exec(` + INSERT INTO knowledge_file_update_slots + (knowledge_id, tenant_id, knowledge_base_id, latest_version, active_version, active_state, active_payload, updated_at) + VALUES (?, 1, 'kb-1', 1, 1, ?, '{}', ?) + `, fixture.id, fixture.state, stale).Error) + } + task := &housekeepingTaskEnqueuer{} + cfg := &config.Config{KnowledgeBase: &config.KnowledgeBaseConfig{ + DocumentProcessTimeout: time.Hour, + }} + svc := NewHousekeepingService(db, cfg, fakeTaskInspector{ + queued: map[string]bool{"update-applying-queued": true}, + }, task) + + svc.runSweep(context.Background()) + + require.Len(t, task.tasks, 1) + assert.Equal(t, types.TypeKnowledgeFileUpdate, task.tasks[0].Type()) + var wake types.KnowledgeFileUpdateTaskPayload + require.NoError(t, json.Unmarshal(task.tasks[0].Payload(), &wake)) + assert.Equal(t, "update-waiting", wake.KnowledgeID) + + var failedState string + require.NoError(t, db.Raw( + `SELECT active_state FROM knowledge_file_update_slots WHERE knowledge_id = 'update-failed'`, + ).Row().Scan(&failedState)) + assert.Equal(t, types.KnowledgeFileUpdateStateFailed, failedState) +} diff --git a/internal/application/service/knowledge_post_process.go b/internal/application/service/knowledge_post_process.go index e08a7514f4..52f7950916 100644 --- a/internal/application/service/knowledge_post_process.go +++ b/internal/application/service/knowledge_post_process.go @@ -126,7 +126,7 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta // long after the user cancelled (the AbortAttempt sweep ran before // we opened postSpan, so the sweep didn't catch this row). switch knowledge.ParseStatus { - case types.ParseStatusCancelled, types.ParseStatusDeleting: + case types.ParseStatusCancelled, types.ParseStatusDeleting, types.ParseStatusReplacing: logger.Infof(ctx, "[KnowledgePostProcess] Knowledge %s aborted (%s), skipping post-processing.", payload.KnowledgeID, knowledge.ParseStatus, diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go index 64b9eb6847..3d2a0abc9a 100644 --- a/internal/application/service/knowledge_process.go +++ b/internal/application/service/knowledge_process.go @@ -1094,7 +1094,7 @@ func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asyn // Short-circuit when the user cancelled parsing or the row is being deleted. if knowledge != nil { switch knowledge.ParseStatus { - case types.ParseStatusCancelled, types.ParseStatusDeleting: + case types.ParseStatusCancelled, types.ParseStatusDeleting, types.ParseStatusReplacing: logger.Infof(ctx, "Summary generation: knowledge aborted (%s), skipping: %s", knowledge.ParseStatus, payload.KnowledgeID) summaryOut["skipped"] = "knowledge_" + knowledge.ParseStatus @@ -1550,7 +1550,7 @@ func (s *knowledgeService) processQuestionGenerationForKnowledge(ctx context.Con // Short-circuit when the user cancelled parsing or the row is being deleted. if knowledge != nil { switch knowledge.ParseStatus { - case types.ParseStatusCancelled, types.ParseStatusDeleting: + case types.ParseStatusCancelled, types.ParseStatusDeleting, types.ParseStatusReplacing: exitStatus = "knowledge_" + knowledge.ParseStatus logger.Infof(ctx, "Question generation: knowledge aborted (%s), skipping: %s", knowledge.ParseStatus, payload.KnowledgeID) @@ -1891,7 +1891,7 @@ func (s *knowledgeService) processQuestionGenerationForChunks(ctx context.Contex // batch, so a cancel stops burning LLM quota on the remaining batches. if knowledge != nil { switch knowledge.ParseStatus { - case types.ParseStatusCancelled, types.ParseStatusDeleting: + case types.ParseStatusCancelled, types.ParseStatusDeleting, types.ParseStatusReplacing: exitStatus = "knowledge_" + knowledge.ParseStatus logger.Infof(ctx, "Question generation: knowledge aborted (%s), skipping batch %d", knowledge.ParseStatus, payload.BatchIndex) @@ -2396,6 +2396,9 @@ func (s *knowledgeService) ReparseKnowledge( logger.Errorf(ctx, "Failed to load knowledge: %v", err) return nil, err } + if existing.ParseStatus == types.ParseStatusReplacing { + return nil, werrors.NewConflictError("knowledge file is being replaced") + } // Allocate a fresh span tree attempt up front. Doing this BEFORE // the cleanup + enqueue means: (a) the UI immediately sees a new diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index 4b5d7caf8c..ad4c7a06f0 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -2688,7 +2688,7 @@ func (s *wikiIngestService) isKnowledgeGone(ctx context.Context, kbID, knowledge return true } switch kn.ParseStatus { - case types.ParseStatusDeleting, types.ParseStatusCancelled: + case types.ParseStatusDeleting, types.ParseStatusCancelled, types.ParseStatusReplacing: return true } return false diff --git a/internal/container/container.go b/internal/container/container.go index a5fe47f9f5..a4fa3247d9 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -402,6 +402,7 @@ func BuildContainer(container *dig.Container) *dig.Container { // persistence succeeded immediately before trigger enqueue failed). Re-arm // them only after the matching handlers are ready. must(container.Invoke(recoverPendingWikiTasks)) + must(container.Invoke(recoverPendingFileUpdates)) logger.Infof(ctx, "[Container] Container initialization completed successfully") return container @@ -594,7 +595,7 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) { encodedPassword := url.QueryEscape(dbPassword) // Check if postgres is in RETRIEVE_DRIVER to determine skip_embedding - retrieveDriver := strings.Split(os.Getenv("RETRIEVE_DRIVER"), ",") + retrieveDriver := types.ParseRetrieverDrivers(os.Getenv("RETRIEVE_DRIVER")) skipEmbedding := "true" if slices.Contains(retrieveDriver, "postgres") { skipEmbedding = "false" @@ -1055,7 +1056,7 @@ func initRetrieveEngineRegistry( // is absent from this process, which happens when startup skipped it after // a construction failure or when another instance registered it. registry := retriever.NewRetrieveEngineRegistry(storeRepo, engineFactory) - retrieveDriver := strings.Split(os.Getenv("RETRIEVE_DRIVER"), ",") + retrieveDriver := types.ParseRetrieverDrivers(os.Getenv("RETRIEVE_DRIVER")) log := logger.GetLogger(context.Background()) // Audit sink for OpenSearch driver events (index created / reindex). Driver // events fire under a tenant-scoped ctx at indexing time; the env-path @@ -1135,15 +1136,15 @@ func initRetrieveEngineRegistry( } client, err := openSearchRepo.NewOpenSearchClient(cc) if err != nil { - log.Errorf("Create opensearch client failed: %v", err) + return nil, fmt.Errorf("create opensearch client: %w", err) } else if repo, err := openSearchRepo.NewRepository( context.Background(), client, "", nil, openSearchRepo.WithAuditSink(auditSink), ); err != nil { - log.Errorf("Create opensearch repository failed: %v", err) + return nil, fmt.Errorf("create opensearch repository: %w", err) } else if err := registry.Register( retriever.NewKVHybridRetrieveEngine(repo, types.OpenSearchRetrieverEngineType), ); err != nil { - log.Errorf("Register opensearch retrieve engine failed: %v", err) + return nil, fmt.Errorf("register opensearch retrieve engine: %w", err) } else { log.Infof("Register opensearch retrieve engine success") } @@ -1354,6 +1355,24 @@ func initRetrieveEngineRegistry( } } } + + engineMapping := types.GetRetrieverEngineMapping() + for _, driver := range retrieveDriver { + params, ok := engineMapping[driver] + if !ok { + return nil, fmt.Errorf("unsupported retrieve driver %q", driver) + } + checked := make(map[types.RetrieverEngineType]struct{}) + for _, param := range params { + if _, ok := checked[param.RetrieverEngineType]; ok { + continue + } + checked[param.RetrieverEngineType] = struct{}{} + if _, err := registry.GetRetrieveEngineService(param.RetrieverEngineType); err != nil { + return nil, fmt.Errorf("configured retrieve driver %q is unavailable: %w", driver, err) + } + } + } // ─── DB store registration (byStoreID) ─── if storeReg, ok := registry.(*retriever.RetrieveEngineRegistry); ok { loadDBStoresIntoRegistry(storeReg, db, cfg, auditSink) diff --git a/internal/container/engine_factory_opensearch_test.go b/internal/container/engine_factory_opensearch_test.go index 8599d312c9..df414fd5f2 100644 --- a/internal/container/engine_factory_opensearch_test.go +++ b/internal/container/engine_factory_opensearch_test.go @@ -91,7 +91,7 @@ func TestInitRetrieveEngineRegistry_OpenSearchEnvPath(t *testing.T) { t.Fatalf("open in-mem db: %v", err) } - t.Setenv("RETRIEVE_DRIVER", "opensearch") + t.Setenv("RETRIEVE_DRIVER", " postgres, opensearch ") t.Setenv("OPENSEARCH_ADDR", ts.URL) // nil store repository and engine factory: this exercises the env-driver @@ -104,3 +104,40 @@ func TestInitRetrieveEngineRegistry_OpenSearchEnvPath(t *testing.T) { t.Errorf("opensearch engine not registered via env path: %v", err) } } + +func TestInitRetrieveEngineRegistry_OpenSearchTransientStartupRegistersDeferred(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = w.Write([]byte(`{"error":{"type":"unavailable","reason":"starting"},"status":503}`)) + })) + defer ts.Close() + + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + if err != nil { + t.Fatalf("open in-mem db: %v", err) + } + t.Setenv("RETRIEVE_DRIVER", "opensearch") + t.Setenv("OPENSEARCH_ADDR", ts.URL) + + registry, err := initRetrieveEngineRegistry(db, &config.Config{}, &fakeAuditSvc{}, nil, nil) + if err != nil { + t.Fatalf("transient OpenSearch startup failure must not fail registry init: %v", err) + } + if _, err := registry.GetRetrieveEngineService(types.OpenSearchRetrieverEngineType); err != nil { + t.Fatalf("deferred OpenSearch repository must remain registered: %v", err) + } +} + +func TestInitRetrieveEngineRegistryRejectsUnknownConfiguredDriver(t *testing.T) { + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + if err != nil { + t.Fatalf("open in-mem db: %v", err) + } + t.Setenv("RETRIEVE_DRIVER", " postgres, unknown-driver ") + + _, err = initRetrieveEngineRegistry(db, &config.Config{}, &fakeAuditSvc{}, nil, nil) + if err == nil { + t.Fatal("configured unknown retrieve driver must fail startup") + } +} diff --git a/internal/container/recover_pending_file_updates.go b/internal/container/recover_pending_file_updates.go new file mode 100644 index 0000000000..9be959300b --- /dev/null +++ b/internal/container/recover_pending_file_updates.go @@ -0,0 +1,58 @@ +package container + +import ( + "context" + "encoding/json" + "errors" + "time" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/hibiken/asynq" +) + +// recoverPendingFileUpdates recreates ephemeral coordinator triggers from the +// durable update slots after all Redis/Lite handlers have been registered. +func recoverPendingFileUpdates(repo interfaces.KnowledgeRepository, task interfaces.TaskEnqueuer) { + if repo == nil || task == nil { + return + } + ctx := context.Background() + slots, err := repo.ListRecoverableKnowledgeFileUpdates(ctx, 1000) + if err != nil { + logger.Warnf(ctx, "[FileUpdateRecovery] list update slots failed: %v", err) + return + } + recovered := 0 + for _, slot := range slots { + if slot == nil || slot.ActiveVersion == nil || slot.ActiveState == types.KnowledgeFileUpdateStateFailed { + continue + } + payload, err := json.Marshal(types.KnowledgeFileUpdateTaskPayload{ + TenantID: slot.TenantID, + KnowledgeBaseID: slot.KnowledgeBaseID, + KnowledgeID: slot.KnowledgeID, + ActiveVersion: *slot.ActiveVersion, + }) + if err != nil { + logger.Warnf(ctx, "[FileUpdateRecovery] encode trigger failed: knowledge_id=%s err=%v", slot.KnowledgeID, err) + continue + } + _, err = task.Enqueue( + asynq.NewTask(types.TypeKnowledgeFileUpdate, payload), + asynq.Queue(types.QueueMaintenance), + asynq.MaxRetry(3), + asynq.Timeout(2*time.Hour), + asynq.Unique(2*time.Hour), + ) + if err != nil && !errors.Is(err, asynq.ErrTaskIDConflict) && !errors.Is(err, asynq.ErrDuplicateTask) { + logger.Warnf(ctx, "[FileUpdateRecovery] enqueue trigger failed: knowledge_id=%s err=%v", slot.KnowledgeID, err) + continue + } + recovered++ + } + if recovered > 0 { + logger.Infof(ctx, "[FileUpdateRecovery] recreated %d coordinator trigger(s)", recovered) + } +} diff --git a/internal/container/recover_pending_file_updates_test.go b/internal/container/recover_pending_file_updates_test.go new file mode 100644 index 0000000000..2ea9c1c09a --- /dev/null +++ b/internal/container/recover_pending_file_updates_test.go @@ -0,0 +1,60 @@ +package container + +import ( + "context" + "encoding/json" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/hibiken/asynq" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type recoverFileUpdateRepoStub struct { + interfaces.KnowledgeRepository + slots []*types.KnowledgeFileUpdateSlot +} + +func (r *recoverFileUpdateRepoStub) ListRecoverableKnowledgeFileUpdates( + context.Context, int, +) ([]*types.KnowledgeFileUpdateSlot, error) { + return r.slots, nil +} + +type recoverFileUpdateTaskStub struct { + tasks []*asynq.Task +} + +func (s *recoverFileUpdateTaskStub) Enqueue( + task *asynq.Task, _ ...asynq.Option, +) (*asynq.TaskInfo, error) { + s.tasks = append(s.tasks, task) + return &asynq.TaskInfo{ID: "recovered"}, nil +} + +func TestRecoverPendingFileUpdatesSkipsFailedSlots(t *testing.T) { + activeVersion := uint64(4) + failedVersion := uint64(8) + repo := &recoverFileUpdateRepoStub{slots: []*types.KnowledgeFileUpdateSlot{ + { + KnowledgeID: "knowledge-active", TenantID: 7, KnowledgeBaseID: "kb-1", + ActiveVersion: &activeVersion, ActiveState: types.KnowledgeFileUpdateStateRetryWait, + }, + { + KnowledgeID: "knowledge-failed", TenantID: 7, KnowledgeBaseID: "kb-1", + ActiveVersion: &failedVersion, ActiveState: types.KnowledgeFileUpdateStateFailed, + }, + }} + task := &recoverFileUpdateTaskStub{} + + recoverPendingFileUpdates(repo, task) + + require.Len(t, task.tasks, 1) + assert.Equal(t, types.TypeKnowledgeFileUpdate, task.tasks[0].Type()) + var payload types.KnowledgeFileUpdateTaskPayload + require.NoError(t, json.Unmarshal(task.tasks[0].Payload(), &payload)) + assert.Equal(t, "knowledge-active", payload.KnowledgeID) + assert.Equal(t, activeVersion, payload.ActiveVersion) +} diff --git a/internal/container/reset_pending_tasks.go b/internal/container/reset_pending_tasks.go index 349f9e4944..bc3eea8bd5 100644 --- a/internal/container/reset_pending_tasks.go +++ b/internal/container/reset_pending_tasks.go @@ -173,6 +173,7 @@ func resettableParseStatuses() []string { types.ParseStatusPending, types.ParseStatusProcessing, types.ParseStatusFinalizing, + types.ParseStatusReplacing, types.ParseStatusDeleting, } } diff --git a/internal/database/knowledge_file_update_migration_test.go b/internal/database/knowledge_file_update_migration_test.go new file mode 100644 index 0000000000..d458825d75 --- /dev/null +++ b/internal/database/knowledge_file_update_migration_test.go @@ -0,0 +1,57 @@ +package database + +import ( + "database/sql" + "path/filepath" + "runtime" + "testing" + + "github.com/golang-migrate/migrate/v4" + sqlite3migrate "github.com/golang-migrate/migrate/v4/database/sqlite3" + _ "github.com/golang-migrate/migrate/v4/source/file" + _ "github.com/mattn/go-sqlite3" + "github.com/stretchr/testify/require" +) + +// TestKnowledgeFileUpdateSlotMigratesExistingSQLite verifies the production +// auto-migration path for databases that already applied the upstream SQLite +// migrations before the update-slot table existed. +func TestKnowledgeFileUpdateSlotMigratesExistingSQLite(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "legacy.db") + db, err := sql.Open("sqlite3", dbPath) + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + + _, err = db.Exec(` + CREATE TABLE schema_migrations (version INTEGER NOT NULL PRIMARY KEY, dirty BOOLEAN NOT NULL); + INSERT INTO schema_migrations(version, dirty) VALUES (2, 0); + `) + require.NoError(t, err) + + driver, err := sqlite3migrate.WithInstance(db, &sqlite3migrate.Config{}) + require.NoError(t, err) + _, currentFile, _, ok := runtime.Caller(0) + require.True(t, ok) + repoRoot := filepath.Clean(filepath.Join(filepath.Dir(currentFile), "..", "..")) + migrator, err := migrate.NewWithDatabaseInstance( + "file://"+filepath.Join(repoRoot, "migrations", "sqlite"), "sqlite3", driver, + ) + require.NoError(t, err) + t.Cleanup(func() { + _, _ = migrator.Close() + }) + + require.NoError(t, migrator.Up()) + version, dirty, err := migrator.Version() + require.NoError(t, err) + require.Equal(t, uint(80), version) + require.False(t, dirty) + + var tableName string + err = db.QueryRow(` + SELECT name FROM sqlite_master + WHERE type = 'table' AND name = 'knowledge_file_update_slots' + `).Scan(&tableName) + require.NoError(t, err) + require.Equal(t, "knowledge_file_update_slots", tableName) +} diff --git a/internal/handler/knowledge.go b/internal/handler/knowledge.go index 8bf2fe5aa2..6cbdd5a610 100644 --- a/internal/handler/knowledge.go +++ b/internal/handler/knowledge.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "mime" + "mime/multipart" "net/http" "strconv" "strings" @@ -438,6 +439,174 @@ func (h *KnowledgeHandler) CreateKnowledgeFromFile(c *gin.Context) { }) } +type createOrUpdateFileForm struct { + File *multipart.FileHeader + KnowledgeID string + ExpectedFileHash string + ExpectedUpdateVersion *uint64 + CustomFileName string + Metadata map[string]string + MetadataProvided bool + EnableMultimodel *bool + ProcessOverrides *types.KnowledgeProcessOverrides + TagIDs []string + TagIDsProvided bool + Channel string + ChannelProvided bool +} + +func parseCreateOrUpdateFileForm(c *gin.Context) (*createOrUpdateFileForm, error) { + file, err := c.FormFile("file") + if err != nil { + return nil, errors.NewBadRequestError("File upload failed").WithDetails(err.Error()) + } + maxSizeMB := utils.GetMaxFileSizeMB() + if file.Size > maxSizeMB*1024*1024 { + return nil, &errors.AppError{ + Code: errors.ErrBadRequest, + Message: fmt.Sprintf("文件大小不能超过%dMB", maxSizeMB), + HTTPCode: http.StatusRequestEntityTooLarge, + } + } + + form := &createOrUpdateFileForm{ + File: file, + KnowledgeID: strings.TrimSpace(c.PostForm("knowledge_id")), + ExpectedFileHash: strings.TrimSpace(c.PostForm("expected_file_hash")), + CustomFileName: c.PostForm("fileName"), + } + if raw, provided := c.GetPostForm("expected_update_version"); provided { + version, err := strconv.ParseUint(strings.TrimSpace(raw), 10, 64) + if err != nil { + return nil, errors.NewBadRequestError("Invalid expected_update_version format").WithDetails(err.Error()) + } + form.ExpectedUpdateVersion = &version + } + if raw, provided := c.GetPostForm("metadata"); provided { + form.MetadataProvided = true + form.Metadata = make(map[string]string) + if strings.TrimSpace(raw) != "" { + if err := json.Unmarshal([]byte(raw), &form.Metadata); err != nil { + return nil, errors.NewBadRequestError("Invalid metadata format").WithDetails(err.Error()) + } + } + } + if raw, provided := c.GetPostForm("enable_multimodel"); provided && strings.TrimSpace(raw) != "" { + value, err := strconv.ParseBool(raw) + if err != nil { + return nil, errors.NewBadRequestError("Invalid enable_multimodel format").WithDetails(err.Error()) + } + form.EnableMultimodel = &value + } + if raw, provided := c.GetPostForm("process_config"); provided { + form.ProcessOverrides = &types.KnowledgeProcessOverrides{} + if strings.TrimSpace(raw) != "" { + if err := json.Unmarshal([]byte(raw), form.ProcessOverrides); err != nil { + return nil, errors.NewBadRequestError("Invalid process_config format").WithDetails(err.Error()) + } + } + } + if form.EnableMultimodel != nil && + (form.ProcessOverrides == nil || form.ProcessOverrides.EnableMultimodel == nil) { + if form.ProcessOverrides == nil { + form.ProcessOverrides = &types.KnowledgeProcessOverrides{} + } + form.ProcessOverrides.EnableMultimodel = form.EnableMultimodel + } + if raw, provided := c.GetPostForm("tag_ids"); provided { + form.TagIDsProvided = true + form.TagIDs = parseCommaSeparatedTagIDs(raw) + } + if raw, provided := c.GetPostForm("channel"); provided { + form.ChannelProvided = true + form.Channel = raw + } + return form, nil +} + +// CreateOrUpdateKnowledgeFromFile godoc +// @ID createOrUpdateKnowledgeFromFile +// @Summary 新增或修改文件知识 +// @Description knowledge_id 为空时按文件名匹配同知识库唯一文件知识，命中则更新，否则新增；提供时保留原 knowledge ID，按 active + 最新 pending 的 latest-wins 规则异步更新 +// @Tags 知识管理 +// @Accept multipart/form-data +// @Produce json +// @Param id path string true "知识库ID" +// @Param file formData file true "上传的文件" +// @Param knowledge_id formData string false "要修改的知识ID；省略时按文件名匹配唯一文件知识，未命中则新增" +// @Param expected_file_hash formData string false "修改时可选的当前文件hash" +// @Param expected_update_version formData integer false "修改时可选的最后接受更新版本" +// @Param fileName formData string false "自定义文件名" +// @Param metadata formData string false "元数据JSON" +// @Param enable_multimodel formData bool false "启用多模态处理" +// @Param tag_ids formData string false "分类ID列表，逗号分隔" +// @Param channel formData string false "来源渠道" +// @Param process_config formData string false "处理配置JSON（KnowledgeProcessOverrides）" +// @Success 200 {object} map[string]interface{} "请求幂等且内容未变化" +// @Success 202 {object} map[string]interface{} "新增或修改任务已接受" +// @Failure 400 {object} errors.AppError "请求参数错误" +// @Failure 404 {object} errors.AppError "修改目标不存在" +// @Failure 409 {object} errors.AppError "状态、版本或重复冲突" +// @Failure 413 {object} errors.AppError "文件超限" +// @Security Bearer +// @Security ApiKeyAuth +// @Router /knowledge-bases/{id}/knowledge/file/create-or-update [post] +func (h *KnowledgeHandler) CreateOrUpdateKnowledgeFromFile(c *gin.Context) { + ctx := c.Request.Context() + _, kbID, effectiveTenantID, permission, err := h.validateKnowledgeBaseAccess(c) + if err != nil { + c.Error(err) + return + } + ctx = context.WithValue(ctx, types.TenantIDContextKey, effectiveTenantID) + if permission != types.OrgRoleAdmin && permission != types.OrgRoleEditor { + c.Error(errors.NewForbiddenError("No permission to create or update knowledge")) + return + } + + form, err := parseCreateOrUpdateFileForm(c) + if err != nil { + c.Error(err) + return + } + result, err := h.kgService.CreateOrUpdateKnowledgeFromFile(ctx, &types.KnowledgeFileCreateOrUpdateRequest{ + KnowledgeBaseID: kbID, + KnowledgeID: form.KnowledgeID, + File: form.File, + CustomFileName: form.CustomFileName, + ExpectedFileHash: form.ExpectedFileHash, + ExpectedUpdateVersion: form.ExpectedUpdateVersion, + EnableMultimodel: form.EnableMultimodel, + Metadata: form.Metadata, + MetadataProvided: form.MetadataProvided, + TagIDs: form.TagIDs, + TagIDsProvided: form.TagIDsProvided, + Channel: form.Channel, + ChannelProvided: form.ChannelProvided, + ProcessOverrides: form.ProcessOverrides, + }) + if err != nil { + if appErr, ok := errors.IsAppError(err); ok { + c.Error(appErr) + return + } + if goerrors.Is(err, service.ErrInvalidFileType) { + c.Error(errors.NewBadRequestError(err.Error())) + return + } + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "knowledge_id": secutils.SanitizeForLog(form.KnowledgeID), + }) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + status := http.StatusAccepted + if result.Action == "unchanged" { + status = http.StatusOK + } + c.JSON(status, gin.H{"success": true, "data": result}) +} + // CreateKnowledgeFromURL godoc // @Summary 从URL创建知识 // @Description 从指定URL抓取内容并创建知识条目。当提供 file_name/file_type 或 URL 路径含已知文件扩展名时，自动切换为文件下载模式 @@ -2047,6 +2216,77 @@ func (h *KnowledgeHandler) CancelKnowledgeParse(c *gin.Context) { }) } +// RetryKnowledgeFileUpdate godoc +// @Summary 重试失败的文件更新 +// @Description 重新唤醒 update slot 中保留的失败 active 版本；并发上传的新版本不会被覆盖 +// @Tags 知识管理 +// @Produce json +// @Param id path string true "知识ID" +// @Success 200 {object} map[string]interface{} "重试已提交" +// @Failure 409 {object} errors.AppError "没有失败更新或状态已变化" +// @Failure 503 {object} errors.AppError "任务系统暂时不可用" +// @Security Bearer +// @Security ApiKeyAuth +// @Router /knowledge/{id}/file-update/retry [post] +func (h *KnowledgeHandler) RetryKnowledgeFileUpdate(c *gin.Context) { + id := secutils.SanitizeForLog(c.Param("id")) + if id == "" { + c.Error(errors.NewBadRequestError("Knowledge ID cannot be empty")) + return + } + _, effCtx, err := h.resolveKnowledgeAndValidateKBAccess(c, id, types.OrgRoleEditor) + if err != nil { + c.Error(err) + return + } + knowledge, err := h.kgService.RetryKnowledgeFileUpdate(effCtx, id) + if err != nil { + if appErr, ok := errors.IsAppError(err); ok { + c.Error(appErr) + return + } + logger.ErrorWithFields(c.Request.Context(), err, map[string]interface{}{"knowledge_id": id}) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + c.JSON(http.StatusOK, gin.H{"success": true, "data": knowledge}) +} + +// DiscardKnowledgeFileUpdate godoc +// @Summary 丢弃失败的文件更新 +// @Description 删除精确匹配的失败 active 和最新 pending 暂存版本，不影响并发提交的新版本 +// @Tags 知识管理 +// @Produce json +// @Param id path string true "知识ID" +// @Success 200 {object} map[string]interface{} "待更新版本已丢弃" +// @Failure 409 {object} errors.AppError "没有失败更新或状态已变化" +// @Security Bearer +// @Security ApiKeyAuth +// @Router /knowledge/{id}/file-update [delete] +func (h *KnowledgeHandler) DiscardKnowledgeFileUpdate(c *gin.Context) { + id := secutils.SanitizeForLog(c.Param("id")) + if id == "" { + c.Error(errors.NewBadRequestError("Knowledge ID cannot be empty")) + return + } + _, effCtx, err := h.resolveKnowledgeAndValidateKBAccess(c, id, types.OrgRoleEditor) + if err != nil { + c.Error(err) + return + } + knowledge, err := h.kgService.DiscardKnowledgeFileUpdate(effCtx, id) + if err != nil { + if appErr, ok := errors.IsAppError(err); ok { + c.Error(appErr) + return + } + logger.ErrorWithFields(c.Request.Context(), err, map[string]interface{}{"knowledge_id": id}) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + c.JSON(http.StatusOK, gin.H{"success": true, "data": knowledge}) +} + type knowledgeTagBatchRequest struct { Updates map[string][]string `json:"updates" binding:"required,min=1"` KBID string `json:"kb_id"` // Optional: scope to this KB (validates editor access and uses effective tenant for shared KB) diff --git a/internal/handler/knowledge_create_or_update_test.go b/internal/handler/knowledge_create_or_update_test.go new file mode 100644 index 0000000000..df659e7238 --- /dev/null +++ b/internal/handler/knowledge_create_or_update_test.go @@ -0,0 +1,254 @@ +package handler + +import ( + "bytes" + "context" + "encoding/json" + "mime/multipart" + "net/http" + "net/http/httptest" + "testing" + + "github.com/Tencent/WeKnora/internal/middleware" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/gin-gonic/gin" +) + +// upsertKnowledgeServiceStub records which branch the create-or-update handler +// dispatched to and returns canned results. +type upsertKnowledgeServiceStub struct { + interfaces.KnowledgeService + createCalls int + updateCalls int + upsertCalls int + lastUpsert *types.KnowledgeFileCreateOrUpdateRequest + lastUpdate *types.KnowledgeFileUpdateRequest + createResult *types.Knowledge + createErr error + updateResult *types.KnowledgeFileUpsertResult + updateErr error + upsertResult *types.KnowledgeFileUpsertResult + upsertErr error +} + +type upsertKnowledgeBaseServiceStub struct { + interfaces.KnowledgeBaseService + tenantID uint64 +} + +func (s *upsertKnowledgeBaseServiceStub) GetKnowledgeBaseByID( + _ context.Context, + id string, +) (*types.KnowledgeBase, error) { + return &types.KnowledgeBase{ID: id, TenantID: s.tenantID}, nil +} + +func (s *upsertKnowledgeServiceStub) CreateKnowledgeFromFile( + _ context.Context, + _ string, + _ *multipart.FileHeader, + _ map[string]string, + _ *bool, + _ string, + _ []string, + _ string, + _ *types.KnowledgeProcessOverrides, +) (*types.Knowledge, error) { + s.createCalls++ + return s.createResult, s.createErr +} + +func (s *upsertKnowledgeServiceStub) UpdateKnowledgeFile( + _ context.Context, + req *types.KnowledgeFileUpdateRequest, +) (*types.KnowledgeFileUpsertResult, error) { + s.updateCalls++ + s.lastUpdate = req + return s.updateResult, s.updateErr +} + +func (s *upsertKnowledgeServiceStub) CreateOrUpdateKnowledgeFromFile( + _ context.Context, + req *types.KnowledgeFileCreateOrUpdateRequest, +) (*types.KnowledgeFileUpsertResult, error) { + s.upsertCalls++ + s.lastUpsert = req + return s.upsertResult, s.upsertErr +} + +func newUpsertKnowledgeRouter(svc interfaces.KnowledgeService) *gin.Engine { + gin.SetMode(gin.TestMode) + router := gin.New() + router.Use(middleware.ErrorHandler()) + router.Use(func(c *gin.Context) { + c.Set(types.TenantIDContextKey.String(), uint64(1)) + c.Set(types.UserIDContextKey.String(), "u-test") + c.Next() + }) + handler := &KnowledgeHandler{ + kgService: svc, + kbService: &upsertKnowledgeBaseServiceStub{tenantID: 1}, + } + router.POST("/knowledge-bases/:id/knowledge/file/create-or-update", handler.CreateOrUpdateKnowledgeFromFile) + return router +} + +// performUpsertRequest builds a multipart create-or-update request with the +// given form fields plus a small file part. +func performUpsertRequest(t *testing.T, router http.Handler, fields map[string]string) *httptest.ResponseRecorder { + t.Helper() + var body bytes.Buffer + writer := multipart.NewWriter(&body) + part, err := writer.CreateFormFile("file", "report.pdf") + if err != nil { + t.Fatalf("create form file: %v", err) + } + if _, err := part.Write([]byte("%PDF-1.4 test")); err != nil { + t.Fatalf("write file part: %v", err) + } + for k, v := range fields { + if err := writer.WriteField(k, v); err != nil { + t.Fatalf("write field %s: %v", k, err) + } + } + if err := writer.Close(); err != nil { + t.Fatalf("close writer: %v", err) + } + req := httptest.NewRequest(http.MethodPost, "/knowledge-bases/kb-1/knowledge/file/create-or-update", &body) + req.Header.Set("Content-Type", writer.FormDataContentType()) + response := httptest.NewRecorder() + router.ServeHTTP(response, req) + return response +} + +func decodeUpsertResult(t *testing.T, response *httptest.ResponseRecorder) *types.KnowledgeFileUpsertResult { + t.Helper() + var body struct { + Success bool `json:"success"` + Data *types.KnowledgeFileUpsertResult `json:"data"` + } + if err := json.Unmarshal(response.Body.Bytes(), &body); err != nil { + t.Fatalf("decode response: %v; body=%s", err, response.Body.String()) + } + return body.Data +} + +// TestCreateOrUpdate_Created verifies an accepted create-or-update request +// returns 202 and the service result. +func TestCreateOrUpdate_Created(t *testing.T) { + svc := &upsertKnowledgeServiceStub{upsertResult: &types.KnowledgeFileUpsertResult{ + Action: "created", Knowledge: &types.Knowledge{ID: "k-new"}, + }} + response := performUpsertRequest(t, newUpsertKnowledgeRouter(svc), nil) + + if response.Code != http.StatusAccepted { + t.Fatalf("status = %d, want 202; body=%s", response.Code, response.Body.String()) + } + if svc.upsertCalls != 1 { + t.Fatalf("upsert calls = %d, want 1", svc.upsertCalls) + } + result := decodeUpsertResult(t, response) + if result.Action != "created" || result.Knowledge.ID != "k-new" { + t.Fatalf("result = %+v, want action=created id=k-new", result) + } +} + +// TestCreateOrUpdate_Unchanged verifies an idempotent service result maps to +// HTTP 200. +func TestCreateOrUpdate_DuplicateIsUnchanged(t *testing.T) { + existing := &types.Knowledge{ID: "k-existing"} + svc := &upsertKnowledgeServiceStub{upsertResult: &types.KnowledgeFileUpsertResult{ + Action: "unchanged", Knowledge: existing, + }} + response := performUpsertRequest(t, newUpsertKnowledgeRouter(svc), nil) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", response.Code, response.Body.String()) + } + result := decodeUpsertResult(t, response) + if result.Action != "unchanged" || result.Knowledge.ID != "k-existing" { + t.Fatalf("result = %+v, want action=unchanged id=k-existing", result) + } +} + +// TestCreateOrUpdate_ForwardsFields verifies a supplied knowledge_id and other +// multipart fields are forwarded to the service. +func TestCreateOrUpdate_ForwardsFields(t *testing.T) { + svc := &upsertKnowledgeServiceStub{ + upsertResult: &types.KnowledgeFileUpsertResult{ + Action: "updated", + Knowledge: &types.Knowledge{ID: "k-existing"}, + TaskID: "task-1", + }, + } + response := performUpsertRequest(t, newUpsertKnowledgeRouter(svc), map[string]string{ + "knowledge_id": "k-existing", + "tag_ids": "t1,t2", + "channel": "api", + "expected_update_version": "7", + }) + + if response.Code != http.StatusAccepted { + t.Fatalf("status = %d, want 202; body=%s", response.Code, response.Body.String()) + } + if svc.upsertCalls != 1 { + t.Fatalf("upsert calls = %d, want 1", svc.upsertCalls) + } + if svc.lastUpsert.KnowledgeID != "k-existing" { + t.Fatalf("KnowledgeID = %q, want k-existing", svc.lastUpsert.KnowledgeID) + } + if !svc.lastUpsert.TagIDsProvided || len(svc.lastUpsert.TagIDs) != 2 { + t.Fatalf("TagIDs = %+v provided=%v, want 2 provided", svc.lastUpsert.TagIDs, svc.lastUpsert.TagIDsProvided) + } + if !svc.lastUpsert.ChannelProvided || svc.lastUpsert.Channel != "api" { + t.Fatalf("Channel = %q provided=%v, want api provided", svc.lastUpsert.Channel, svc.lastUpsert.ChannelProvided) + } + if svc.lastUpsert.ExpectedUpdateVersion == nil || *svc.lastUpsert.ExpectedUpdateVersion != 7 { + t.Fatalf("ExpectedUpdateVersion = %v, want 7", svc.lastUpsert.ExpectedUpdateVersion) + } + result := decodeUpsertResult(t, response) + if result.Action != "updated" || result.TaskID != "task-1" { + t.Fatalf("result = %+v, want action=updated task-1", result) + } +} + +// TestCreateOrUpdate_UpdateUnchangedReturns200 verifies an unchanged update +// result maps to HTTP 200 rather than 202. +func TestCreateOrUpdate_UpdateUnchangedReturns200(t *testing.T) { + svc := &upsertKnowledgeServiceStub{ + upsertResult: &types.KnowledgeFileUpsertResult{ + Action: "unchanged", + Knowledge: &types.Knowledge{ID: "k-existing"}, + }, + } + response := performUpsertRequest(t, newUpsertKnowledgeRouter(svc), map[string]string{ + "knowledge_id": "k-existing", + }) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", response.Code, response.Body.String()) + } + result := decodeUpsertResult(t, response) + if result.Action != "unchanged" { + t.Fatalf("result = %+v, want action=unchanged", result) + } +} + +func TestCreateOrUpdateRejectsInvalidExpectedUpdateVersion(t *testing.T) { + svc := &upsertKnowledgeServiceStub{} + response := performUpsertRequest(t, newUpsertKnowledgeRouter(svc), map[string]string{ + "knowledge_id": "k-existing", + "expected_update_version": "not-a-number", + }) + + if response.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400; body=%s", response.Code, response.Body.String()) + } + if svc.updateCalls != 0 { + t.Fatalf("update calls = %d, want 0", svc.updateCalls) + } + if svc.upsertCalls != 0 { + t.Fatalf("upsert calls = %d, want 0", svc.upsertCalls) + } +} diff --git a/internal/router/retrieve_driver_readiness_test.go b/internal/router/retrieve_driver_readiness_test.go new file mode 100644 index 0000000000..074f040240 --- /dev/null +++ b/internal/router/retrieve_driver_readiness_test.go @@ -0,0 +1,16 @@ +package router + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCompareRetrieveDriversReportsSanitizedDifference(t *testing.T) { + missing, unexpected := compareRetrieveDrivers( + []string{"opensearch", "postgres"}, + []string{"postgres", "qdrant"}, + ) + assert.Equal(t, []string{"opensearch"}, missing) + assert.Equal(t, []string{"qdrant"}, unexpected) +} diff --git a/internal/router/router.go b/internal/router/router.go index 90e950687f..455e302c89 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -2,7 +2,9 @@ package router import ( "context" + "net/http" "os" + "sort" "strings" "time" @@ -19,6 +21,7 @@ import ( "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/middleware" "github.com/Tencent/WeKnora/internal/tracing/langfuse" + "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" _ "github.com/Tencent/WeKnora/docs" // swagger docs @@ -84,6 +87,7 @@ type RouterParams struct { DataSourceCredentialsHandler *handler.DataSourceCredentialsHandler WeKnoraCloudHandler *handler.WeKnoraCloudHandler WikiPageHandler *handler.WikiPageHandler + RetrieveEngineRegistry interfaces.RetrieveEngineRegistry } // NewRouter 创建新的路由 @@ -125,7 +129,25 @@ func NewRouter(params RouterParams) *gin.Engine { // 健康检查（不需要认证） r.GET("/health", func(c *gin.Context) { - c.JSON(200, gin.H{"status": "ok"}) + c.JSON(http.StatusOK, gin.H{"status": "ok"}) + }) + r.GET("/ready", func(c *gin.Context) { + configured := types.ParseRetrieverDrivers(os.Getenv("RETRIEVE_DRIVER")) + registered := registeredRetrieveDrivers(params.RetrieveEngineRegistry) + missing, unexpected := compareRetrieveDrivers(configured, registered) + status := http.StatusOK + state := "ok" + if len(missing) > 0 { + status = http.StatusServiceUnavailable + state = "unavailable" + } + c.JSON(status, gin.H{ + "status": state, + "configured_retrieve_drivers": configured, + "registered_retrieve_drivers": registered, + "missing_retrieve_drivers": missing, + "unexpected_retrieve_drivers": unexpected, + }) }) // Swagger API 文档（仅在非生产环境下启用） @@ -278,6 +300,49 @@ func NewRouter(params RouterParams) *gin.Engine { return r } +func registeredRetrieveDrivers(registry interfaces.RetrieveEngineRegistry) []string { + if registry == nil { + return nil + } + seen := make(map[string]struct{}) + for _, engine := range registry.GetAllRetrieveEngineServices() { + if engine == nil { + continue + } + seen[string(engine.EngineType())] = struct{}{} + } + drivers := make([]string, 0, len(seen)) + for driver := range seen { + drivers = append(drivers, driver) + } + sort.Strings(drivers) + return drivers +} + +func compareRetrieveDrivers(configured, registered []string) (missing, unexpected []string) { + configuredSet := make(map[string]struct{}, len(configured)) + registeredSet := make(map[string]struct{}, len(registered)) + for _, driver := range configured { + configuredSet[driver] = struct{}{} + } + for _, driver := range registered { + registeredSet[driver] = struct{}{} + } + for _, driver := range configured { + if _, ok := registeredSet[driver]; !ok { + missing = append(missing, driver) + } + } + for _, driver := range registered { + if _, ok := configuredSet[driver]; !ok { + unexpected = append(unexpected, driver) + } + } + sort.Strings(missing) + sort.Strings(unexpected) + return missing, unexpected +} + // trustedProxies returns the proxy CIDRs/IPs whose X-Forwarded-For headers // gin should trust when resolving the client IP. Defaults to loopback and // private ranges (covers the bundled nginx in a container network); override diff --git a/internal/router/routes_knowledge.go b/internal/router/routes_knowledge.go index dc11aa4d98..6c7016f42f 100644 --- a/internal/router/routes_knowledge.go +++ b/internal/router/routes_knowledge.go @@ -71,6 +71,7 @@ func RegisterKnowledgeRoutes(r *gin.RouterGroup, handler *handler.KnowledgeHandl kbRead := kb.With(apiKeyRetrieve(apiKeyFullAccess())) { kb.POST("/file", g.OwnedKBOrAdmin(), g.KBAccessWrite("id"), handler.CreateKnowledgeFromFile) + kb.POST("/file/create-or-update", g.OwnedKBOrAdmin(), g.KBAccessWrite("id"), handler.CreateOrUpdateKnowledgeFromFile) kb.POST("/url", g.OwnedKBOrAdmin(), g.KBAccessWrite("id"), handler.CreateKnowledgeFromURL) kb.POST("/manual", g.OwnedKBOrAdmin(), g.KBAccessWrite("id"), handler.CreateManualKnowledge) kbRead.GET("", g.Viewer(), g.KBAccessRead("id"), handler.ListKnowledge) @@ -109,6 +110,8 @@ func RegisterKnowledgeRoutes(r *gin.RouterGroup, handler *handler.KnowledgeHandl k.PUT("/manual/:id", g.OwnedKnowledgeKBOrAdmin(), g.KBAccessWriteFromKnowledgeIDParam("id"), handler.UpdateManualKnowledge) k.POST("/:id/reparse", g.OwnedKnowledgeKBOrAdmin(), g.KBAccessWriteFromKnowledgeIDParam("id"), handler.ReparseKnowledge) k.POST("/:id/cancel-parse", g.OwnedKnowledgeKBOrAdmin(), g.KBAccessWriteFromKnowledgeIDParam("id"), handler.CancelKnowledgeParse) + k.POST("/:id/file-update/retry", g.OwnedKnowledgeKBOrAdmin(), g.KBAccessWriteFromKnowledgeIDParam("id"), handler.RetryKnowledgeFileUpdate) + k.DELETE("/:id/file-update", g.OwnedKnowledgeKBOrAdmin(), g.KBAccessWriteFromKnowledgeIDParam("id"), handler.DiscardKnowledgeFileUpdate) // Downloading exposes the original source file, so it has a stricter // boundary than viewing parsed content or previewing it: tenant Viewers // cannot download from their own workspace, and org-shared Viewer access diff --git a/internal/router/sync_task.go b/internal/router/sync_task.go index c316500326..b762497273 100644 --- a/internal/router/sync_task.go +++ b/internal/router/sync_task.go @@ -137,6 +137,7 @@ func RegisterSyncHandlers(params SyncTaskParams) { params.Executor.RegisterHandler(types.TypeChunkExtract, params.ChunkExtractor.Handle) params.Executor.RegisterHandler(types.TypeDataTableSummary, params.DataTableSummary.Handle) params.Executor.RegisterHandler(types.TypeDocumentProcess, params.KnowledgeService.ProcessDocument) + params.Executor.RegisterHandler(types.TypeKnowledgeFileUpdate, params.KnowledgeService.ProcessKnowledgeFileUpdate) params.Executor.RegisterHandler(types.TypeTemporaryDocumentProcess, params.TemporaryDocument.Process) params.Executor.RegisterHandler(types.TypeManualProcess, params.KnowledgeService.ProcessManualUpdate) params.Executor.RegisterHandler(types.TypeFAQImport, params.KnowledgeService.ProcessFAQImport) diff --git a/internal/router/task.go b/internal/router/task.go index 3097fa2abb..b099dc0028 100644 --- a/internal/router/task.go +++ b/internal/router/task.go @@ -224,6 +224,27 @@ func NewWikiAsynqServer(svc interfaces.SystemSettingService) *asynq.Server { } func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux { + mux := newAsynqServeMux(params) + + // Run the same mux on every pool. Shared and dedicated servers intentionally + // overlap, but Redis dequeue is atomic, so each task still executes once. + runPool := func(name string, srv *asynq.Server) { + go func() { + if err := srv.Run(mux); err != nil { + log.Fatalf("could not run %s asynq server: %v", name, err) + } + }() + } + runPool("core-pool", params.CoreServer) + runPool("postprocess-pool", params.PostProcessServer) + runPool("enrichment-pool", params.EnrichmentServer) + runPool("maintenance-pool", params.MaintenanceServer) + runPool("shared-pool", params.SharedServer) + runPool("wiki-pool", params.WikiServer) + return mux +} + +func newAsynqServeMux(params AsynqTaskParams) *asynq.ServeMux { // Create a new mux and register all handlers mux := asynq.NewServeMux() @@ -263,6 +284,7 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux { // Register document processing handler mux.HandleFunc(types.TypeDocumentProcess, params.KnowledgeService.ProcessDocument) + mux.HandleFunc(types.TypeKnowledgeFileUpdate, params.KnowledgeService.ProcessKnowledgeFileUpdate) mux.HandleFunc(types.TypeTemporaryDocumentProcess, params.TemporaryDocument.Process) // Register manual knowledge processing handler (cleanup + re-indexing) @@ -310,21 +332,6 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux { mux.HandleFunc(types.TypeWikiIngest, params.WikiIngest.Handle) mux.HandleFunc(types.TypeWikiFinalize, params.WikiIngest.Handle) - // Run the same mux on every pool. Shared and dedicated servers intentionally - // overlap, but Redis dequeue is atomic, so each task still executes once. - runPool := func(name string, srv *asynq.Server) { - go func() { - if err := srv.Run(mux); err != nil { - log.Fatalf("could not run %s asynq server: %v", name, err) - } - }() - } - runPool("core-pool", params.CoreServer) - runPool("postprocess-pool", params.PostProcessServer) - runPool("enrichment-pool", params.EnrichmentServer) - runPool("maintenance-pool", params.MaintenanceServer) - runPool("shared-pool", params.SharedServer) - runPool("wiki-pool", params.WikiServer) return mux } @@ -392,6 +399,10 @@ func newDeadLetterKnowledgeFailer(ks interfaces.KnowledgeService, tracker servic markKnowledgeListDeleteFailed(ctx, repo, t, taskErr) return } + if t.Type() == types.TypeKnowledgeFileUpdate { + markKnowledgeFileUpdateFailed(ctx, repo, t, taskErr) + return + } if _, ok := taskTypesAffectingKnowledgeStatus[t.Type()]; !ok { return } @@ -425,6 +436,102 @@ func newDeadLetterKnowledgeFailer(ks interfaces.KnowledgeService, tracker servic } } +func markKnowledgeFileUpdateFailed( + ctx context.Context, + repo interfaces.KnowledgeRepository, + t *asynq.Task, + taskErr error, +) { + var wake types.KnowledgeFileUpdateTaskPayload + if err := json.Unmarshal(t.Payload(), &wake); err != nil || + wake.TenantID == 0 || wake.KnowledgeID == "" || wake.ActiveVersion == 0 { + return + } + slot, err := repo.GetKnowledgeFileUpdateSlot(ctx, wake.TenantID, wake.KnowledgeID) + if err != nil { + logger.Warnf(ctx, "dead-letter callback: failed to load file update slot for %s: %v", + wake.KnowledgeID, err) + return + } + if slot.ActiveVersion == nil || *slot.ActiveVersion != wake.ActiveVersion || + slot.ActiveState == types.KnowledgeFileUpdateStateFailed || + slot.ActiveState == types.KnowledgeFileUpdateStateIdle { + return + } + errMsg := "task " + t.Type() + " exhausted retries: " + taskErr.Error() + if len(errMsg) > 512 { + errMsg = errMsg[:512] + } + marked, err := repo.TransitionKnowledgeFileUpdateState( + ctx, + wake.TenantID, + wake.KnowledgeID, + wake.ActiveVersion, + slot.ActiveState, + types.KnowledgeFileUpdateStateFailed, + errMsg, + ) + if err != nil { + logger.Warnf(ctx, "dead-letter callback: failed to mark file update %s as failed: %v", + wake.KnowledgeID, err) + return + } + if !marked { + return + } + restoreFailedKnowledgeFileUpdateClaim(ctx, repo, wake, slot) + logger.Infof(ctx, "dead-letter callback: marked file update %s as failed", wake.KnowledgeID) +} + +func restoreFailedKnowledgeFileUpdateClaim( + ctx context.Context, + repo interfaces.KnowledgeRepository, + wake types.KnowledgeFileUpdateTaskPayload, + slot *types.KnowledgeFileUpdateSlot, +) { + var active types.KnowledgeFileUpdatePayload + if err := json.Unmarshal(slot.ActivePayload, &active); err != nil || active.OldFilePath == "" { + return + } + restoreStatus := active.OldParseStatus + if !isRestorableFileUpdateParseStatus(restoreStatus) { + restoreStatus = types.ParseStatusFailed + } + kbID := wake.KnowledgeBaseID + if kbID == "" { + kbID = slot.KnowledgeBaseID + } + if kbID == "" { + kbID = active.KnowledgeBaseID + } + _, err := repo.UpdateApplyingKnowledgeFileColumns( + ctx, + wake.TenantID, + wake.KnowledgeID, + kbID, + active.OldFilePath, + active.OldFileHash, + map[string]interface{}{ + "parse_status": restoreStatus, + "error_message": "", + "updated_at": time.Now(), + }, + ) + if err != nil { + logger.Warnf(ctx, "dead-letter callback: failed to restore file update %s status: %v", + wake.KnowledgeID, err) + } +} + +func isRestorableFileUpdateParseStatus(status string) bool { + switch status { + case types.ParseStatusCompleted, types.ParseStatusFailed, types.ParseStatusCancelled: + return true + default: + return false + } +} + func markKnowledgeListDeleteFailed( ctx context.Context, repo interfaces.KnowledgeRepository, diff --git a/internal/router/task_file_update_deadletter_test.go b/internal/router/task_file_update_deadletter_test.go new file mode 100644 index 0000000000..d56a77afbc --- /dev/null +++ b/internal/router/task_file_update_deadletter_test.go @@ -0,0 +1,214 @@ +package router + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/hibiken/asynq" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestKnowledgeFileUpdateDeadLetterDoesNotOverwriteKnowledgeStatus(t *testing.T) { + _, included := taskTypesAffectingKnowledgeStatus[types.TypeKnowledgeFileUpdate] + assert.False(t, included, + "the file-update coordinator owns failure state through its versioned slot") +} + +type taskRouteHandlerStub struct { + interfaces.TaskHandler +} + +func (taskRouteHandlerStub) Handle(context.Context, *asynq.Task) error { return nil } + +type taskRouteKnowledgeServiceStub struct { + interfaces.KnowledgeService + repo interfaces.KnowledgeRepository + fileUpdateCalls int +} + +func (s *taskRouteKnowledgeServiceStub) GetRepository() interfaces.KnowledgeRepository { + return s.repo +} + +func (s *taskRouteKnowledgeServiceStub) ProcessDocument(context.Context, *asynq.Task) error { + return nil +} + +func (s *taskRouteKnowledgeServiceStub) ProcessKnowledgeFileUpdate(context.Context, *asynq.Task) error { + s.fileUpdateCalls++ + return nil +} + +func (s *taskRouteKnowledgeServiceStub) ProcessManualUpdate(context.Context, *asynq.Task) error { + return nil +} + +func (s *taskRouteKnowledgeServiceStub) ProcessFAQImport(context.Context, *asynq.Task) error { + return nil +} + +func (s *taskRouteKnowledgeServiceStub) ProcessQuestionGeneration(context.Context, *asynq.Task) error { + return nil +} + +func (s *taskRouteKnowledgeServiceStub) ProcessSummaryGeneration(context.Context, *asynq.Task) error { + return nil +} + +func (s *taskRouteKnowledgeServiceStub) ProcessKBClone(context.Context, *asynq.Task) error { + return nil +} + +func (s *taskRouteKnowledgeServiceStub) ProcessKnowledgeMove(context.Context, *asynq.Task) error { + return nil +} + +func (s *taskRouteKnowledgeServiceStub) ProcessKnowledgeListDelete(context.Context, *asynq.Task) error { + return nil +} + +func (s *taskRouteKnowledgeServiceStub) ProcessKnowledgeListReparse(context.Context, *asynq.Task) error { + return nil +} + +type taskRouteKnowledgeBaseServiceStub struct { + interfaces.KnowledgeBaseService +} + +func (taskRouteKnowledgeBaseServiceStub) ProcessKBDelete(context.Context, *asynq.Task) error { + return nil +} + +type taskRouteTagServiceStub struct { + interfaces.KnowledgeTagService +} + +func (taskRouteTagServiceStub) ProcessIndexDelete(context.Context, *asynq.Task) error { + return nil +} + +type taskRouteDataSourceServiceStub struct { + interfaces.DataSourceService +} + +func (taskRouteDataSourceServiceStub) ProcessSync(context.Context, *asynq.Task) error { + return nil +} + +type taskRouteTemporaryDocumentServiceStub struct { + interfaces.TemporaryDocumentService +} + +func (taskRouteTemporaryDocumentServiceStub) Process(context.Context, *asynq.Task) error { + return nil +} + +func TestRunAsynqServerRegistersKnowledgeFileUpdateHandler(t *testing.T) { + knowledgeSvc := &taskRouteKnowledgeServiceStub{} + handler := taskRouteHandlerStub{} + mux := newAsynqServeMux(AsynqTaskParams{ + KnowledgeService: knowledgeSvc, + KnowledgeBaseService: taskRouteKnowledgeBaseServiceStub{}, + TagService: taskRouteTagServiceStub{}, + DataSourceService: taskRouteDataSourceServiceStub{}, + ChunkExtractor: handler, + DataTableSummary: handler, + ImageMultimodal: handler, + KnowledgePostProcess: handler, + WikiIngest: handler, + TemporaryDocument: taskRouteTemporaryDocumentServiceStub{}, + }) + + payload, err := json.Marshal(types.KnowledgeFileUpdateTaskPayload{TenantID: 1}) + require.NoError(t, err) + task := asynq.NewTask(types.TypeKnowledgeFileUpdate, payload) + _, pattern := mux.Handler(task) + require.Equal(t, types.TypeKnowledgeFileUpdate, pattern) + require.NoError(t, mux.ProcessTask(context.Background(), task)) + assert.Equal(t, 1, knowledgeSvc.fileUpdateCalls) +} + +type taskRouteKnowledgeRepoStub struct { + interfaces.KnowledgeRepository + slot *types.KnowledgeFileUpdateSlot + applyCalls int + applyValues map[string]interface{} +} + +func (r *taskRouteKnowledgeRepoStub) GetKnowledgeFileUpdateSlot( + context.Context, uint64, string, +) (*types.KnowledgeFileUpdateSlot, error) { + return r.slot, nil +} + +func (r *taskRouteKnowledgeRepoStub) TransitionKnowledgeFileUpdateState( + _ context.Context, + _ uint64, + _ string, + version uint64, + fromState string, + toState string, + lastError string, +) (bool, error) { + if r.slot == nil || r.slot.ActiveVersion == nil || + *r.slot.ActiveVersion != version || r.slot.ActiveState != fromState { + return false, nil + } + r.slot.ActiveState = toState + r.slot.LastError = lastError + return true, nil +} + +func (r *taskRouteKnowledgeRepoStub) UpdateApplyingKnowledgeFileColumns( + _ context.Context, + _ uint64, + _ string, + _ string, + _ string, + _ string, + values map[string]interface{}, +) (bool, error) { + r.applyCalls++ + r.applyValues = values + return true, nil +} + +func TestKnowledgeFileUpdateDeadLetterMarksSlotFailedAndRestoresReplacingStatus(t *testing.T) { + version := uint64(6) + active, err := json.Marshal(types.KnowledgeFileUpdatePayload{ + KnowledgeBaseID: "kb-1", + OldParseStatus: types.ParseStatusCompleted, + OldFilePath: "old/path.md", + OldFileHash: "old-hash", + NewFilePath: "staged/latest.md", + NewFileHash: "new-hash", + }) + require.NoError(t, err) + repo := &taskRouteKnowledgeRepoStub{slot: &types.KnowledgeFileUpdateSlot{ + KnowledgeID: "knowledge-1", + TenantID: 1, + KnowledgeBaseID: "kb-1", + ActiveVersion: &version, + ActiveState: types.KnowledgeFileUpdateStateApplying, + ActivePayload: types.JSON(active), + }} + callback := newDeadLetterKnowledgeFailer(&taskRouteKnowledgeServiceStub{repo: repo}, nil) + require.NotNil(t, callback) + payload, err := json.Marshal(types.KnowledgeFileUpdateTaskPayload{ + TenantID: 1, KnowledgeBaseID: "kb-1", KnowledgeID: "knowledge-1", ActiveVersion: version, + }) + require.NoError(t, err) + + callback(context.Background(), asynq.NewTask(types.TypeKnowledgeFileUpdate, payload), errors.New("context canceled")) + + assert.Equal(t, types.KnowledgeFileUpdateStateFailed, repo.slot.ActiveState) + assert.Contains(t, repo.slot.LastError, "context canceled") + require.Equal(t, 1, repo.applyCalls) + assert.Equal(t, types.ParseStatusCompleted, repo.applyValues["parse_status"]) + assert.Equal(t, "", repo.applyValues["error_message"]) +} diff --git a/internal/router/task_inspector_kb_cancel_test.go b/internal/router/task_inspector_kb_cancel_test.go index 58d92381bf..f105a7a770 100644 --- a/internal/router/task_inspector_kb_cancel_test.go +++ b/internal/router/task_inspector_kb_cancel_test.go @@ -82,6 +82,18 @@ func TestMatchesKnowledgePreservesPerKnowledgeAllowList(t *testing.T) { if matchesKnowledge(types.TypeKBClone, []byte(`{"knowledge_id":"knowledge-1"}`), "knowledge-1") { t.Fatal("KB clone must not become cancellable through the per-knowledge API") } + if matchesKnowledge(types.TypeKnowledgeFileUpdate, []byte(`{"knowledge_id":"knowledge-1"}`), "knowledge-1") { + t.Fatal("file update coordinator must not cancel itself while applying a replacement") + } + if !matchesKnowledgeBase( + types.TypeKnowledgeFileUpdate, + []byte(`{"knowledge_base_id":"kb-1","knowledge_id":"knowledge-1"}`), + "kb-1", + map[string]struct{}{"knowledge-1": {}}, + nil, + ) { + t.Fatal("KB deletion still needs to cancel queued file update coordinators") + } } func TestCancelTasksForKnowledgeBaseRescansMutatedPages(t *testing.T) { diff --git a/internal/types/interfaces/knowledge.go b/internal/types/interfaces/knowledge.go index d615c8bb37..a0672a539b 100644 --- a/internal/types/interfaces/knowledge.go +++ b/internal/types/interfaces/knowledge.go @@ -24,6 +24,23 @@ type KnowledgeService interface { channel string, processOverrides *types.KnowledgeProcessOverrides, ) (*types.Knowledge, error) + // CreateOrUpdateKnowledgeFromFile creates a new file knowledge, or updates + // an existing file knowledge selected by explicit knowledge_id or unique + // filename match. + CreateOrUpdateKnowledgeFromFile( + ctx context.Context, + req *types.KnowledgeFileCreateOrUpdateRequest, + ) (*types.KnowledgeFileUpsertResult, error) + // UpdateKnowledgeFile stages and asynchronously replaces the source file + // of an existing file knowledge while preserving its ID. + UpdateKnowledgeFile( + ctx context.Context, + req *types.KnowledgeFileUpdateRequest, + ) (*types.KnowledgeFileUpsertResult, error) + // RetryKnowledgeFileUpdate re-arms the retained failed active version. + RetryKnowledgeFileUpdate(ctx context.Context, knowledgeID string) (*types.Knowledge, error) + // DiscardKnowledgeFileUpdate removes a failed active version and its latest pending version. + DiscardKnowledgeFileUpdate(ctx context.Context, knowledgeID string) (*types.Knowledge, error) // CreateKnowledgeFromURL creates knowledge from a URL. // When fileName or fileType is provided (or the URL path has a known file extension), // the URL is treated as a direct file download instead of a web page crawl. @@ -193,6 +210,8 @@ type KnowledgeService interface { ProcessManualUpdate(ctx context.Context, t *asynq.Task) error // ProcessDocument handles Asynq document processing tasks ProcessDocument(ctx context.Context, t *asynq.Task) error + // ProcessKnowledgeFileUpdate coordinates durable active/pending file updates. + ProcessKnowledgeFileUpdate(ctx context.Context, t *asynq.Task) error // ProcessFAQImport handles Asynq FAQ import tasks ProcessFAQImport(ctx context.Context, t *asynq.Task) error // ProcessQuestionGeneration handles Asynq question generation tasks @@ -256,6 +275,15 @@ type KnowledgeRepository interface { kbID string, params *types.KnowledgeCheckParams, ) (bool, *types.Knowledge, error) + // CheckKnowledgeExistsExcluding is CheckKnowledgeExists with one knowledge + // ID excluded, used when validating an in-place replacement. + CheckKnowledgeExistsExcluding( + ctx context.Context, + tenantID uint64, + kbID string, + excludeKnowledgeID string, + params *types.KnowledgeCheckParams, + ) (bool, *types.Knowledge, error) // ListKnowledgeFolderCounts aggregates the number of knowledge entries // stored directly in each folder_path of a knowledge base. ListKnowledgeFolderCounts( @@ -289,6 +317,68 @@ type KnowledgeRepository interface { // statement so callers that flip several related fields (e.g. parse_status + // error_message) cannot leave the row in a half-updated state. UpdateKnowledgeColumns(ctx context.Context, id string, values map[string]interface{}) error + // ClaimKnowledgeFileUpdate atomically moves one exact file version from a + // terminal parse status to replacing. + ClaimKnowledgeFileUpdate( + ctx context.Context, + tenantID uint64, + knowledgeID string, + kbID string, + expectedStatus string, + expectedFilePath string, + expectedFileHash string, + ) (bool, error) + // UpdateApplyingKnowledgeFileColumns updates a row only while it is still the + // file version claimed by a replacement task. + UpdateApplyingKnowledgeFileColumns( + ctx context.Context, + tenantID uint64, + knowledgeID string, + kbID string, + expectedFilePath string, + expectedFileHash string, + values map[string]interface{}, + ) (bool, error) + StageKnowledgeFileUpdate( + ctx context.Context, + tenantID uint64, + knowledgeID string, + kbID string, + payload types.JSON, + expectedVersion *uint64, + ) (*types.KnowledgeFileUpdateStageResult, error) + GetKnowledgeFileUpdateSlot( + ctx context.Context, tenantID uint64, knowledgeID string, + ) (*types.KnowledgeFileUpdateSlot, error) + PrepareKnowledgeFileUpdate( + ctx context.Context, tenantID uint64, knowledgeID string, version uint64, payload types.JSON, + ) (bool, error) + TransitionKnowledgeFileUpdateState( + ctx context.Context, + tenantID uint64, + knowledgeID string, + version uint64, + fromState string, + toState string, + lastError string, + ) (bool, error) + CompleteKnowledgeFileUpdate( + ctx context.Context, tenantID uint64, knowledgeID string, version uint64, + ) (*types.KnowledgeFileUpdateSlot, error) + CancelKnowledgeFileUpdates( + ctx context.Context, tenantID uint64, knowledgeID string, + ) (*types.KnowledgeFileUpdateSlot, error) + CancelFailedKnowledgeFileUpdate( + ctx context.Context, tenantID uint64, knowledgeID string, activeVersion uint64, + ) (*types.KnowledgeFileUpdateSlot, error) + // BeginKnowledgeDeletion atomically marks active knowledge rows as deleting + // and removes their file-update coordination slots. + BeginKnowledgeDeletion( + ctx context.Context, tenantID uint64, knowledgeIDs []string, + ) ([]*types.KnowledgeFileUpdateSlot, error) + ListRecoverableKnowledgeFileUpdates( + ctx context.Context, limit int, + ) ([]*types.KnowledgeFileUpdateSlot, error) // UpdateActiveDeletingKnowledgeColumns updates an active, non-deleted knowledge row // only when it is still in the transient deleting state. UpdateActiveDeletingKnowledgeColumns(ctx context.Context, id string, values map[string]interface{}) (bool, error) diff --git a/internal/types/knowledge.go b/internal/types/knowledge.go index 2133997ac4..26be4d469a 100644 --- a/internal/types/knowledge.go +++ b/internal/types/knowledge.go @@ -3,6 +3,7 @@ package types import ( "encoding/json" "fmt" + "mime/multipart" "sort" "strings" "time" @@ -66,6 +67,21 @@ const ( // queued downstream tasks, but the knowledge row and any already-written // chunks/index are kept so the user can re-trigger parsing via reparse. ParseStatusCancelled = "cancelled" + // ParseStatusReplacing indicates that a validated replacement file has + // been staged and the asynchronous worker is rebuilding this knowledge in + // place. Other write operations must not mutate the row in this state. + ParseStatusReplacing = "replacing" +) + +// File knowledge update slot states. +const ( + KnowledgeFileUpdateStateIdle = "idle" + KnowledgeFileUpdateStateWaiting = "waiting" + KnowledgeFileUpdateStateApplying = "applying" + KnowledgeFileUpdateStateRetryWait = "retry_wait" + KnowledgeFileUpdateStateFailed = "failed" + KnowledgeFileUpdateResultActive = "active" + KnowledgeFileUpdateResultPending = "pending" ) // Summary status constants for async summary generation @@ -188,6 +204,41 @@ type Knowledge struct { DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"` // Knowledge base name (not stored in database, populated on query) KnowledgeBaseName string `json:"knowledge_base_name" gorm:"-"` + // FileUpdateVersion is the latest accepted file update version. + FileUpdateVersion uint64 `json:"file_update_version" gorm:"-"` + // FileUpdateState projects the durable update slot state. + FileUpdateState string `json:"file_update_state" gorm:"-"` + // FileUpdateError contains a sanitized update failure summary. + FileUpdateError string `json:"file_update_error,omitempty" gorm:"-"` +} + +// KnowledgeFileUpdateSlot stores at most one active and one latest pending +// replacement for a file knowledge. +type KnowledgeFileUpdateSlot struct { + KnowledgeID string `gorm:"column:knowledge_id;primaryKey"` + TenantID uint64 `gorm:"column:tenant_id"` + KnowledgeBaseID string `gorm:"column:knowledge_base_id"` + LatestVersion uint64 `gorm:"column:latest_version"` + ActiveVersion *uint64 `gorm:"column:active_version"` + ActiveState string `gorm:"column:active_state"` + ActivePayload JSON `gorm:"column:active_payload;type:json"` + PendingVersion *uint64 `gorm:"column:pending_version"` + PendingPayload JSON `gorm:"column:pending_payload;type:json"` + LastError string `gorm:"column:last_error"` + CreatedAt time.Time `gorm:"column:created_at"` + UpdatedAt time.Time `gorm:"column:updated_at"` +} + +func (KnowledgeFileUpdateSlot) TableName() string { return "knowledge_file_update_slots" } + +// KnowledgeFileUpdateStageResult describes the durable state after accepting +// an update. Superseded payloads are returned for staged-file cleanup. +type KnowledgeFileUpdateStageResult struct { + Version uint64 + State string + ActiveVersion uint64 + ReplacedActivePayload JSON + ReplacedPendingPayload JSON } // CustomMetadataText returns stable human-readable metadata for summaries and @@ -219,6 +270,56 @@ func (k *Knowledge) CustomMetadataText() string { return strings.Join(lines, "\n") } +// KnowledgeFileUpdateRequest carries an in-place file replacement request. +// The Provided flags distinguish an omitted multipart field from an explicit +// empty value, which is required for preserving or clearing optional fields. +type KnowledgeFileUpdateRequest struct { + KnowledgeBaseID string + KnowledgeID string + File *multipart.FileHeader + CustomFileName string + ExpectedFileHash string + ExpectedUpdateVersion *uint64 + Metadata map[string]string + MetadataProvided bool + TagIDs []string + TagIDsProvided bool + Channel string + ChannelProvided bool + ProcessOverrides *KnowledgeProcessOverrides +} + +// KnowledgeFileCreateOrUpdateRequest carries the unified multipart file +// upsert request. When KnowledgeID is empty, the service first tries to find a +// unique existing file knowledge by the effective filename; if no match exists, +// it creates a new knowledge. +type KnowledgeFileCreateOrUpdateRequest struct { + KnowledgeBaseID string + KnowledgeID string + File *multipart.FileHeader + CustomFileName string + ExpectedFileHash string + ExpectedUpdateVersion *uint64 + EnableMultimodel *bool + Metadata map[string]string + MetadataProvided bool + TagIDs []string + TagIDsProvided bool + Channel string + ChannelProvided bool + ProcessOverrides *KnowledgeProcessOverrides +} + +// KnowledgeFileUpsertResult is returned by the create-or-update HTTP flow. +type KnowledgeFileUpsertResult struct { + Action string `json:"action"` + Knowledge *Knowledge `json:"knowledge"` + TaskID string `json:"task_id,omitempty"` + UpdateVersion uint64 `json:"update_version,omitempty"` + UpdateState string `json:"update_state,omitempty"` + AcceptedFileHash string `json:"accepted_file_hash,omitempty"` +} + // GetMetadata returns the metadata as a map[string]string. func (k *Knowledge) GetMetadata() map[string]string { metadata := make(map[string]string) diff --git a/internal/types/task.go b/internal/types/task.go index 9f16b2e727..ae430b6246 100644 --- a/internal/types/task.go +++ b/internal/types/task.go @@ -78,6 +78,7 @@ var queueDefinitions = []QueueDefinition{ {Name: QueueMaintenance, Pool: WorkerPoolMaintenance, Weight: 1, TaskTypes: []string{ TypeFAQImport, TypeKBClone, TypeIndexDelete, TypeKBDelete, TypeKnowledgeListDelete, TypeKnowledgeListReparse, TypeKnowledgeMove, + TypeKnowledgeFileUpdate, }}, {Name: QueueWiki, Pool: WorkerPoolWiki, Weight: 1, TaskTypes: []string{TypeWikiIngest, TypeWikiFinalize}}, } @@ -229,6 +230,7 @@ type WorkerServerStat struct { const ( TypeChunkExtract = "chunk:extract" TypeDocumentProcess = "document:process" // 文档处理任务 + TypeKnowledgeFileUpdate = "knowledge:file_update" // 文件知识更新协调任务 TypeFAQImport = "faq:import" // FAQ导入任务（包含dry run模式） TypeQuestionGeneration = "question:generation" // 问题生成任务 TypeSummaryGeneration = "summary:generation" // 摘要生成任务 @@ -425,6 +427,46 @@ type KnowledgeListReparsePayload struct { Initiator TaskInitiator `json:"initiator,omitempty"` } +// KnowledgeFileUpdatePayload describes a staged, in-place file replacement. +// It contains storage references and metadata only; file bytes never enter the +// task queue. Old and new identities make retries and stale deliveries safe. +type KnowledgeFileUpdatePayload struct { + TracingContext + TenantID uint64 `json:"tenant_id"` + KnowledgeBaseID string `json:"knowledge_base_id"` + KnowledgeID string `json:"knowledge_id"` + OldParseStatus string `json:"old_parse_status,omitempty"` + OldFilePath string `json:"old_file_path"` + OldFileHash string `json:"old_file_hash"` + NewFilePath string `json:"new_file_path"` + NewFileName string `json:"new_file_name"` + NewFolderPath string `json:"new_folder_path,omitempty"` + NewFileType string `json:"new_file_type"` + NewFileSize int64 `json:"new_file_size"` + NewFileHash string `json:"new_file_hash"` + Metadata map[string]string `json:"metadata,omitempty"` + MetadataProvided bool `json:"metadata_provided,omitempty"` + TagIDs []string `json:"tag_ids,omitempty"` + TagIDsProvided bool `json:"tag_ids_provided,omitempty"` + Channel string `json:"channel,omitempty"` + ChannelProvided bool `json:"channel_provided,omitempty"` + ProcessConfig *KnowledgeProcessOverrides `json:"process_config,omitempty"` + ProcessProvided bool `json:"process_provided,omitempty"` + DocumentTaskID string `json:"document_task_id"` + Initiator TaskInitiator `json:"initiator,omitempty"` + Attempt int `json:"attempt,omitempty"` +} + +// KnowledgeFileUpdateTaskPayload wakes the durable active/pending coordinator. +// The actual file payload is always loaded from knowledge_file_update_slots. +type KnowledgeFileUpdateTaskPayload struct { + TenantID uint64 `json:"tenant_id"` + KnowledgeBaseID string `json:"knowledge_base_id"` + KnowledgeID string `json:"knowledge_id"` + ActiveVersion uint64 `json:"active_version"` + WakeSequence uint64 `json:"wake_sequence,omitempty"` +} + // KnowledgeMovePayload represents the knowledge move task payload type KnowledgeMovePayload struct { TracingContext diff --git a/internal/types/tenant.go b/internal/types/tenant.go index ce24d61dc3..582f3ed1e7 100644 --- a/internal/types/tenant.go +++ b/internal/types/tenant.go @@ -62,13 +62,31 @@ func GetRetrieverEngineMapping() map[string][]RetrieverEngineParams { return retrieverEngineMapping } +// ParseRetrieverDrivers normalizes RETRIEVE_DRIVER consistently for registry, +// tenant defaults, migrations, and env-backed vector-store projections. +func ParseRetrieverDrivers(raw string) []string { + result := make([]string, 0) + seen := make(map[string]struct{}) + for _, driver := range strings.Split(raw, ",") { + driver = strings.ToLower(strings.TrimSpace(driver)) + if driver == "" { + continue + } + if _, ok := seen[driver]; ok { + continue + } + seen[driver] = struct{}{} + result = append(result, driver) + } + return result +} + // GetDefaultRetrieverEngines returns the default retriever engines based on RETRIEVE_DRIVER env func GetDefaultRetrieverEngines() []RetrieverEngineParams { result := []RetrieverEngineParams{} seen := make(map[string]bool) - for _, driver := range strings.Split(os.Getenv("RETRIEVE_DRIVER"), ",") { - driver = strings.TrimSpace(driver) + for _, driver := range ParseRetrieverDrivers(os.Getenv("RETRIEVE_DRIVER")) { if params, ok := retrieverEngineMapping[driver]; ok { for _, p := range params { key := string(p.RetrieverType) + ":" + string(p.RetrieverEngineType) diff --git a/internal/types/tenant_retriever_drivers_test.go b/internal/types/tenant_retriever_drivers_test.go new file mode 100644 index 0000000000..5d02d7c247 --- /dev/null +++ b/internal/types/tenant_retriever_drivers_test.go @@ -0,0 +1,14 @@ +package types + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseRetrieverDriversNormalizesAndDeduplicates(t *testing.T) { + assert.Equal(t, + []string{"postgres", "opensearch"}, + ParseRetrieverDrivers(" POSTGRES, opensearch, postgres, "), + ) +} diff --git a/internal/types/vectorstore.go b/internal/types/vectorstore.go index 48b3d621a6..8a7b97d055 100644 --- a/internal/types/vectorstore.go +++ b/internal/types/vectorstore.go @@ -805,15 +805,9 @@ func BuildEnvVectorStores(retrieveDriver string, envLookup EnvLookupFunc) []Vect return nil } - drivers := strings.Split(retrieveDriver, ",") var stores []VectorStore - for _, driver := range drivers { - driver = strings.TrimSpace(driver) - if driver == "" { - continue - } - + for _, driver := range ParseRetrieverDrivers(retrieveDriver) { store := buildEnvStoreForDriver(driver, envLookup) if store != nil { stores = append(stores, *store) diff --git a/migrations/mysql/00-init-db.sql b/migrations/mysql/00-init-db.sql index a4c7aec608..d78d154b11 100644 --- a/migrations/mysql/00-init-db.sql +++ b/migrations/mysql/00-init-db.sql @@ -227,3 +227,28 @@ CREATE TABLE chunk_revisions ( UNIQUE KEY idx_chunk_revisions_chunk_revision (chunk_id, revision), KEY idx_chunk_revisions_tenant_chunk (tenant_id, chunk_id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE TABLE knowledge_file_update_slots ( + knowledge_id VARCHAR(36) PRIMARY KEY, + tenant_id BIGINT NOT NULL, + knowledge_base_id VARCHAR(36) NOT NULL, + latest_version BIGINT NOT NULL DEFAULT 0, + active_version BIGINT NULL, + active_state VARCHAR(16) NOT NULL DEFAULT 'idle', + active_payload JSON NULL, + pending_version BIGINT NULL, + pending_payload JSON NULL, + last_error TEXT NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + CONSTRAINT chk_knowledge_file_update_active_state + CHECK (active_state IN ('idle', 'waiting', 'applying', 'retry_wait', 'failed')), + CONSTRAINT chk_knowledge_file_update_active_idle + CHECK ((active_state = 'idle') = (active_version IS NULL)), + CONSTRAINT chk_knowledge_file_update_active_payload + CHECK ((active_version IS NULL) = (active_payload IS NULL)), + CONSTRAINT chk_knowledge_file_update_pending_payload + CHECK ((pending_version IS NULL) = (pending_payload IS NULL)), + KEY idx_knowledge_file_update_slots_tenant_kb (tenant_id, knowledge_base_id), + KEY idx_knowledge_file_update_slots_state (active_state, updated_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='文件知识更新协调槽'; diff --git a/migrations/sqlite/000000_init.down.sql b/migrations/sqlite/000000_init.down.sql index 2c81c06eca..4e01d1b5d9 100644 --- a/migrations/sqlite/000000_init.down.sql +++ b/migrations/sqlite/000000_init.down.sql @@ -24,6 +24,7 @@ DROP TABLE IF EXISTS chunks; DROP TABLE IF EXISTS messages; DROP TABLE IF EXISTS temporary_documents; DROP TABLE IF EXISTS sessions; +DROP TABLE IF EXISTS knowledge_file_update_slots; DROP TABLE IF EXISTS knowledges; DROP TABLE IF EXISTS knowledge_bases; DROP TABLE IF EXISTS storage_backends; diff --git a/migrations/sqlite/000000_init.up.sql b/migrations/sqlite/000000_init.up.sql index d1baea4af0..8e1e8d97c9 100644 --- a/migrations/sqlite/000000_init.up.sql +++ b/migrations/sqlite/000000_init.up.sql @@ -1101,3 +1101,29 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_wiki_page_revisions_page_version CREATE INDEX IF NOT EXISTS idx_wiki_page_revisions_kb_slug ON wiki_page_revisions (knowledge_base_id, slug); + +-- File knowledge update coordination slot. Migration 000080 repeats this +-- idempotently so databases already initialized at version 0 are upgraded. +CREATE TABLE IF NOT EXISTS knowledge_file_update_slots ( + knowledge_id VARCHAR(36) PRIMARY KEY, + tenant_id INTEGER NOT NULL, + knowledge_base_id VARCHAR(36) NOT NULL, + latest_version INTEGER NOT NULL DEFAULT 0, + active_version INTEGER, + active_state VARCHAR(16) NOT NULL DEFAULT 'idle', + active_payload TEXT, + pending_version INTEGER, + pending_payload TEXT, + last_error TEXT NOT NULL DEFAULT '', + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + CHECK (active_state IN ('idle', 'waiting', 'applying', 'retry_wait', 'failed')), + CHECK ((active_state = 'idle') = (active_version IS NULL)), + CHECK ((active_version IS NULL) = (active_payload IS NULL)), + CHECK ((pending_version IS NULL) = (pending_payload IS NULL)) +); + +CREATE INDEX IF NOT EXISTS idx_knowledge_file_update_slots_tenant_kb + ON knowledge_file_update_slots(tenant_id, knowledge_base_id); +CREATE INDEX IF NOT EXISTS idx_knowledge_file_update_slots_state + ON knowledge_file_update_slots(active_state, updated_at); diff --git a/migrations/sqlite/000080_knowledge_file_update_slots.down.sql b/migrations/sqlite/000080_knowledge_file_update_slots.down.sql new file mode 100644 index 0000000000..d1a06bc0b6 --- /dev/null +++ b/migrations/sqlite/000080_knowledge_file_update_slots.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS knowledge_file_update_slots; diff --git a/migrations/sqlite/000080_knowledge_file_update_slots.up.sql b/migrations/sqlite/000080_knowledge_file_update_slots.up.sql new file mode 100644 index 0000000000..20885cfc46 --- /dev/null +++ b/migrations/sqlite/000080_knowledge_file_update_slots.up.sql @@ -0,0 +1,24 @@ +-- File knowledge update coordination slot. +CREATE TABLE IF NOT EXISTS knowledge_file_update_slots ( + knowledge_id VARCHAR(36) PRIMARY KEY, + tenant_id INTEGER NOT NULL, + knowledge_base_id VARCHAR(36) NOT NULL, + latest_version INTEGER NOT NULL DEFAULT 0, + active_version INTEGER, + active_state VARCHAR(16) NOT NULL DEFAULT 'idle', + active_payload TEXT, + pending_version INTEGER, + pending_payload TEXT, + last_error TEXT NOT NULL DEFAULT '', + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + CHECK (active_state IN ('idle', 'waiting', 'applying', 'retry_wait', 'failed')), + CHECK ((active_state = 'idle') = (active_version IS NULL)), + CHECK ((active_version IS NULL) = (active_payload IS NULL)), + CHECK ((pending_version IS NULL) = (pending_payload IS NULL)) +); + +CREATE INDEX IF NOT EXISTS idx_knowledge_file_update_slots_tenant_kb + ON knowledge_file_update_slots(tenant_id, knowledge_base_id); +CREATE INDEX IF NOT EXISTS idx_knowledge_file_update_slots_state + ON knowledge_file_update_slots(active_state, updated_at); diff --git a/migrations/versioned/000080_knowledge_file_update_slots.down.sql b/migrations/versioned/000080_knowledge_file_update_slots.down.sql new file mode 100644 index 0000000000..d1a06bc0b6 --- /dev/null +++ b/migrations/versioned/000080_knowledge_file_update_slots.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS knowledge_file_update_slots; diff --git a/migrations/versioned/000080_knowledge_file_update_slots.up.sql b/migrations/versioned/000080_knowledge_file_update_slots.up.sql new file mode 100644 index 0000000000..04be895905 --- /dev/null +++ b/migrations/versioned/000080_knowledge_file_update_slots.up.sql @@ -0,0 +1,33 @@ +DO $$ BEGIN RAISE NOTICE '[Migration 000080] Creating knowledge file update slots...'; END $$; + +CREATE TABLE IF NOT EXISTS knowledge_file_update_slots ( + knowledge_id VARCHAR(36) PRIMARY KEY, + tenant_id BIGINT NOT NULL, + knowledge_base_id VARCHAR(36) NOT NULL, + latest_version BIGINT NOT NULL DEFAULT 0, + active_version BIGINT, + active_state VARCHAR(16) NOT NULL DEFAULT 'idle', + active_payload JSONB, + pending_version BIGINT, + pending_payload JSONB, + last_error TEXT NOT NULL DEFAULT '', + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT chk_knowledge_file_update_active_state + CHECK (active_state IN ('idle', 'waiting', 'applying', 'retry_wait', 'failed')), + CONSTRAINT chk_knowledge_file_update_active_idle + CHECK ((active_state = 'idle') = (active_version IS NULL)), + CONSTRAINT chk_knowledge_file_update_active_payload + CHECK ((active_version IS NULL) = (active_payload IS NULL)), + CONSTRAINT chk_knowledge_file_update_pending_payload + CHECK ((pending_version IS NULL) = (pending_payload IS NULL)) +); + +COMMENT ON TABLE knowledge_file_update_slots IS '文件知识更新协调槽'; + +CREATE INDEX IF NOT EXISTS idx_knowledge_file_update_slots_tenant_kb + ON knowledge_file_update_slots(tenant_id, knowledge_base_id); +CREATE INDEX IF NOT EXISTS idx_knowledge_file_update_slots_state + ON knowledge_file_update_slots(active_state, updated_at); + +DO $$ BEGIN RAISE NOTICE '[Migration 000080] Knowledge file update slots ready'; END $$; From 6ca50316dbe53ee7b578b639ee58b32693191a98 Mon Sep 17 00:00:00 2001 From: hanxiantao <601803023@qq.com> Date: Sun, 9 Aug 2026 15:59:55 +0800 Subject: [PATCH 2/7] chore(knowledge): scope create-or-update changes --- .../retriever/opensearch/bulk_update.go | 3 - .../retriever/opensearch/mapping.go | 3 - .../retriever/opensearch/repository.go | 71 +++++-------------- .../retriever/opensearch/repository_test.go | 54 -------------- .../retriever/opensearch/retrieve.go | 5 +- internal/application/service/knowledge.go | 37 ---------- .../application/service/knowledge_create.go | 5 +- .../service/knowledge_file_update.go | 3 - internal/container/container.go | 28 ++------ .../engine_factory_opensearch_test.go | 39 +--------- .../knowledge_file_update_migration_test.go | 1 + .../router/retrieve_driver_readiness_test.go | 16 ----- internal/router/router.go | 67 +---------------- .../task_file_update_deadletter_test.go | 1 + internal/types/tenant.go | 22 +----- .../types/tenant_retriever_drivers_test.go | 14 ---- internal/types/vectorstore.go | 8 ++- 17 files changed, 36 insertions(+), 341 deletions(-) delete mode 100644 internal/router/retrieve_driver_readiness_test.go delete mode 100644 internal/types/tenant_retriever_drivers_test.go diff --git a/internal/application/repository/retriever/opensearch/bulk_update.go b/internal/application/repository/retriever/opensearch/bulk_update.go index 2e3927ec4a..97e64caf79 100644 --- a/internal/application/repository/retriever/opensearch/bulk_update.go +++ b/internal/application/repository/retriever/opensearch/bulk_update.go @@ -75,9 +75,6 @@ func (r *Repository) BatchUpdateChunkTagID(ctx context.Context, chunkTagMap map[ func (r *Repository) updateByQueryScript( ctx context.Context, chunkIDs []string, source string, params map[string]any, ) error { - if err := r.ensureClusterValidated(ctx); err != nil { - return err - } body, err := json.Marshal(map[string]any{ "query": map[string]any{ "terms": map[string]any{"chunk_id": chunkIDs}, diff --git a/internal/application/repository/retriever/opensearch/mapping.go b/internal/application/repository/retriever/opensearch/mapping.go index 34e85a5ffe..ca3b2ad906 100644 --- a/internal/application/repository/retriever/opensearch/mapping.go +++ b/internal/application/repository/retriever/opensearch/mapping.go @@ -195,9 +195,6 @@ func buildKeywordsMapping(cfg internalCfg) ([]byte, error) { // failures can be retried by the next caller (sync.Once cannot be // reset). func (r *Repository) ensureKeywordsIndex(ctx context.Context) error { - if err := r.ensureClusterValidated(ctx); err != nil { - return err - } r.keywordsMu.Lock() defer r.keywordsMu.Unlock() diff --git a/internal/application/repository/retriever/opensearch/repository.go b/internal/application/repository/retriever/opensearch/repository.go index 1e8b9a9a62..8ffe7a7330 100644 --- a/internal/application/repository/retriever/opensearch/repository.go +++ b/internal/application/repository/retriever/opensearch/repository.go @@ -26,11 +26,10 @@ import ( // except the once / initErr maps are read-only after NewRepository returns; // per-dimension index initialization is guarded by sync.Once. // -// Lifecycle: NewRepository validates cluster version + k-NN plugin when the -// cluster is reachable. A transient transport failure defers that probe until -// first use so a short OpenSearch outage does not prevent the API process from -// starting. It does NOT create any index. Index creation happens lazily on -// first Save / BatchSave / Retrieve once the embedding dimension is known. +// Lifecycle: NewRepository validates connectivity + cluster version + +// k-NN plugin, but does NOT create any index. Index creation happens +// lazily on first Save / BatchSave / Retrieve once the embedding +// dimension is known (see ensureReady — per-dimension index naming). // // Concurrency: A single Repository instance is shared across N goroutines // concurrently retrieving from the same store (the multi-store fan-out @@ -66,13 +65,6 @@ type Repository struct { keywordsReady bool keywordsErr error - // A startup transport failure defers the version/plugin probe. The first - // real operation retries it under this mutex; permanent probe failures are - // cached until restart, while transport failures remain retryable. - clusterProbeMu sync.Mutex - clusterProbePending bool - clusterProbeErr error - // sink receives audit events (index created / reindex executed). nil // means no auditing; use r.auditSink() to get a non-nil sink. Set via // WithAuditSink at construction. @@ -83,10 +75,11 @@ type Repository struct { // red if the interface drifts and our implementation lags). var _ interfaces.RetrieveEngineRepository = (*Repository)(nil) -// NewRepository builds a new OpenSearch k-NN repository. Permanent validation -// errors (authentication, unsupported version, missing k-NN plugin) fail the -// constructor. Transient transport errors defer validation until first use. -// It does NOT create any index; callers trigger lazy per-dimension creation. +// NewRepository builds a new OpenSearch k-NN repository and verifies the +// backing cluster is reachable + version-compatible + has the k-NN +// plugin installed on every cluster node. It does NOT create any index — +// callers (Save / Retrieve) trigger lazy per-dimension creation via +// ensureReady on first use. // // storeID is the VectorStore.ID owning this repository instance. It is // folded into the base index name so multiple OpenSearch VectorStores @@ -138,6 +131,13 @@ func NewRepository( return nil, fmt.Errorf("opensearch: invalid index config: %w", err) } + if err := probeVersion(ctx, client); err != nil { + return nil, err // already wraps ErrVersionUnsupported / ErrTransport + } + if err := probeKNNPlugin(ctx, client); err != nil { + return nil, err // already wraps ErrConfigInvalid / ErrTransport + } + r := &Repository{ client: client, baseIndex: base, @@ -148,47 +148,11 @@ func NewRepository( for _, opt := range opts { opt(r) } - if err := probeCluster(ctx, client); err != nil { - if !isTransientErr(err) { - return nil, err - } - r.clusterProbePending = true - log.Warnf("[OpenSearch] startup probe deferred after transient failure: %v", err) - return r, nil - } log.Infof("[OpenSearch] repository ready (baseIndex=%s, knn_engine=%s, hnsw_m=%d)", base, icfg.knnEngine, icfg.hnswM) return r, nil } -func probeCluster(ctx context.Context, client *osapi.Client) error { - if err := probeVersion(ctx, client); err != nil { - return err - } - return probeKNNPlugin(ctx, client) -} - -func (r *Repository) ensureClusterValidated(ctx context.Context) error { - r.clusterProbeMu.Lock() - defer r.clusterProbeMu.Unlock() - if r.clusterProbeErr != nil { - return r.clusterProbeErr - } - if !r.clusterProbePending { - return nil - } - if err := probeCluster(ctx, r.client); err != nil { - if !isTransientErr(err) { - r.clusterProbeErr = err - r.clusterProbePending = false - } - return err - } - r.clusterProbePending = false - logger.GetLogger(ctx).Infof("[OpenSearch] deferred startup probe recovered (baseIndex=%s)", r.baseIndex) - return nil -} - // ensureReady creates the per-dimension index (alias-backed) the first // time a given embedding dimension is seen. Concurrent callers for the // same dim block on the same sync.Once. @@ -216,9 +180,6 @@ func (r *Repository) ensureReady(ctx context.Context, dim int) error { return fmt.Errorf("opensearch: dim %d out of range (1..16000): %w", dim, ErrDimensionMismatch) } - if err := r.ensureClusterValidated(ctx); err != nil { - return err - } r.onceMu.Lock() once, ok := r.once[dim] diff --git a/internal/application/repository/retriever/opensearch/repository_test.go b/internal/application/repository/retriever/opensearch/repository_test.go index 8d444fa2be..687f0cd440 100644 --- a/internal/application/repository/retriever/opensearch/repository_test.go +++ b/internal/application/repository/retriever/opensearch/repository_test.go @@ -1122,60 +1122,6 @@ func TestNewRepository_AcceptsLongStoreID(t *testing.T) { } } -func TestNewRepositoryDefersTransientStartupProbeAndRecovers(t *testing.T) { - const transientResponses = int32(8) - var infoCalls atomic.Int32 - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/": - if infoCalls.Add(1) <= transientResponses { - w.WriteHeader(http.StatusServiceUnavailable) - _, _ = w.Write([]byte(`{"error":{"type":"unavailable","reason":"starting"},"status":503}`)) - return - } - _, _ = w.Write([]byte(`{"version":{"distribution":"opensearch","number":"3.3.2"}}`)) - case "/_cat/plugins": - _, _ = w.Write([]byte(`[{"name":"node-1","component":"opensearch-knn"}]`)) - default: - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte(`{}`)) - } - })) - defer ts.Close() - - repoIface, err := NewRepository(context.Background(), newTestClient(t, ts.URL), "", nil) - if err != nil { - t.Fatalf("transient startup probe must not fail construction: %v", err) - } - repo, ok := repoIface.(*Repository) - if !ok { - t.Fatalf("repository type: %T", repoIface) - } - if !repo.clusterProbePending { - t.Fatal("transient startup failure must leave a deferred probe") - } - var probeErr error - for attempt := 0; attempt < 10; attempt++ { - probeErr = repo.ensureClusterValidated(context.Background()) - if probeErr == nil { - break - } - if !errors.Is(probeErr, ErrTransport) { - t.Fatalf("deferred transient probe classification: %v", probeErr) - } - } - if probeErr != nil { - t.Fatalf("deferred probe should eventually recover: %v", probeErr) - } - if repo.clusterProbePending { - t.Fatal("successful deferred probe must clear pending state") - } - if got := infoCalls.Load(); got <= transientResponses { - t.Fatalf("cluster info calls: want recovery after %d transient responses, got %d", - transientResponses, got) - } -} - // ============================================================================ // Stub coverage — remaining stubs return the not-enabled sentinel // diff --git a/internal/application/repository/retriever/opensearch/retrieve.go b/internal/application/repository/retriever/opensearch/retrieve.go index 23d4d1f1a1..61101d7811 100644 --- a/internal/application/repository/retriever/opensearch/retrieve.go +++ b/internal/application/repository/retriever/opensearch/retrieve.go @@ -31,9 +31,6 @@ func (r *Repository) Retrieve( ctx context.Context, params types.RetrieveParams, ) ([]*types.RetrieveResult, error) { - if err := r.ensureClusterValidated(ctx); err != nil { - return nil, err - } dim, multiIndex := resolveDim(params) switch params.RetrieverType { @@ -143,7 +140,7 @@ func (r *Repository) search(ctx context.Context, indexPattern string, body []byt // Field-by-field decode (vs map[string]any) keeps the JSON shape // pinned at compile time. type hit struct { - ID string `json:"_id"` // equals chunk_id per the indexing invariant + ID string `json:"_id"` // equals chunk_id per the indexing invariant Score float64 `json:"_score"` Source struct { Content string `json:"content"` diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index f1c70922d8..7d74b75e02 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -451,40 +451,6 @@ func (s *knowledgeService) checkStorageEngineConfigured(ctx context.Context, kb return werrors.NewBadRequestError("请先为知识库选择存储引擎，再上传内容。请前往知识库设置页面进行配置。") } -// checkRetrieveEngineConfigured resolves the KB's effective vector store before -// file bytes are persisted. This prevents accepting work that is guaranteed to -// fail later with a missing repository registration. -func (s *knowledgeService) checkRetrieveEngineConfigured( - ctx context.Context, kb *types.KnowledgeBase, tenantID uint64, -) error { - // Some focused unit tests construct a partial service. Production always - // injects the registry through the container. - if s == nil || s.retrieveEngine == nil || kb == nil { - return nil - } - if kb.VectorStoreID != nil && *kb.VectorStoreID != "" && s.ownership == nil { - return werrors.NewServiceUnavailableError("知识库检索引擎暂时不可用") - } - if _, ok := types.TenantInfoFromContext(ctx); !ok && s.tenantRepo != nil { - tenant, err := s.tenantRepo.GetTenantByID(ctx, tenantID) - if err != nil { - return werrors.NewServiceUnavailableError("知识库检索引擎暂时不可用") - } - ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant) - } - if _, err := retriever.CreateRetrieveEngineForKB( - ctx, s.retrieveEngine, s.ownership, tenantID, kb.VectorStoreID, - ); err != nil { - logger.ErrorWithFields(ctx, err, map[string]interface{}{ - "tenant_id": tenantID, - "kb_id": kb.ID, - "reason": "retrieve engine preflight failed", - }) - return werrors.NewServiceUnavailableError("知识库检索引擎暂时不可用") - } - return nil -} - func kbIDOrEmpty(kb *types.KnowledgeBase) string { if kb == nil { return "" @@ -848,9 +814,6 @@ func (s *knowledgeService) SetKnowledgeTags(ctx context.Context, knowledgeID str if knowledge.ParseStatus == types.ParseStatusReplacing { return werrors.NewConflictError("knowledge file is being replaced") } - if err := s.validateKnowledgeTagIDs(ctx, tenantID, knowledge.KnowledgeBaseID, tagIDs); err != nil { - return err - } return s.repo.SetKnowledgeTags(ctx, knowledgeID, tagIDs) } diff --git a/internal/application/service/knowledge_create.go b/internal/application/service/knowledge_create.go index 13d2c4e6a6..b1d26ec430 100644 --- a/internal/application/service/knowledge_create.go +++ b/internal/application/service/knowledge_create.go @@ -66,10 +66,6 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context, if err := s.checkStorageEngineConfigured(ctx, kb); err != nil { return nil, err } - tenantID := ctx.Value(types.TenantIDContextKey).(uint64) - if err := s.checkRetrieveEngineConfigured(ctx, kb, tenantID); err != nil { - return nil, err - } // Early reject before the whole-file hash below. resolveFileImportProcessConfig // gates the same extension set, but this path must keep returning @@ -89,6 +85,7 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context, } // Check if file already exists + tenantID := ctx.Value(types.TenantIDContextKey).(uint64) logger.Infof(ctx, "Checking if file exists, tenant ID: %d", tenantID) exists, existingKnowledge, err := s.repo.CheckKnowledgeExists(ctx, tenantID, kbID, &types.KnowledgeCheckParams{ Type: "file", diff --git a/internal/application/service/knowledge_file_update.go b/internal/application/service/knowledge_file_update.go index f0b0aebe50..8d353f44c7 100644 --- a/internal/application/service/knowledge_file_update.go +++ b/internal/application/service/knowledge_file_update.go @@ -224,9 +224,6 @@ func (s *knowledgeService) UpdateKnowledgeFile( if err := s.checkStorageEngineConfigured(ctx, kb); err != nil { return nil, err } - if err := s.checkRetrieveEngineConfigured(ctx, kb, tenantID); err != nil { - return nil, err - } if req.TagIDsProvided { if err := s.validateKnowledgeTagIDs(ctx, tenantID, req.KnowledgeBaseID, req.TagIDs); err != nil { return nil, err diff --git a/internal/container/container.go b/internal/container/container.go index 5cf8fe006a..7069702c31 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -596,7 +596,7 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) { encodedPassword := url.QueryEscape(dbPassword) // Check if postgres is in RETRIEVE_DRIVER to determine skip_embedding - retrieveDriver := types.ParseRetrieverDrivers(os.Getenv("RETRIEVE_DRIVER")) + retrieveDriver := strings.Split(os.Getenv("RETRIEVE_DRIVER"), ",") skipEmbedding := "true" if slices.Contains(retrieveDriver, "postgres") { skipEmbedding = "false" @@ -1057,7 +1057,7 @@ func initRetrieveEngineRegistry( // is absent from this process, which happens when startup skipped it after // a construction failure or when another instance registered it. registry := retriever.NewRetrieveEngineRegistry(storeRepo, engineFactory) - retrieveDriver := types.ParseRetrieverDrivers(os.Getenv("RETRIEVE_DRIVER")) + retrieveDriver := strings.Split(os.Getenv("RETRIEVE_DRIVER"), ",") log := logger.GetLogger(context.Background()) // Audit sink for OpenSearch driver events (index created / reindex). Driver // events fire under a tenant-scoped ctx at indexing time; the env-path @@ -1137,15 +1137,15 @@ func initRetrieveEngineRegistry( } client, err := openSearchRepo.NewOpenSearchClient(cc) if err != nil { - return nil, fmt.Errorf("create opensearch client: %w", err) + log.Errorf("Create opensearch client failed: %v", err) } else if repo, err := openSearchRepo.NewRepository( context.Background(), client, "", nil, openSearchRepo.WithAuditSink(auditSink), ); err != nil { - return nil, fmt.Errorf("create opensearch repository: %w", err) + log.Errorf("Create opensearch repository failed: %v", err) } else if err := registry.Register( retriever.NewKVHybridRetrieveEngine(repo, types.OpenSearchRetrieverEngineType), ); err != nil { - return nil, fmt.Errorf("register opensearch retrieve engine: %w", err) + log.Errorf("Register opensearch retrieve engine failed: %v", err) } else { log.Infof("Register opensearch retrieve engine success") } @@ -1356,24 +1356,6 @@ func initRetrieveEngineRegistry( } } } - - engineMapping := types.GetRetrieverEngineMapping() - for _, driver := range retrieveDriver { - params, ok := engineMapping[driver] - if !ok { - return nil, fmt.Errorf("unsupported retrieve driver %q", driver) - } - checked := make(map[types.RetrieverEngineType]struct{}) - for _, param := range params { - if _, ok := checked[param.RetrieverEngineType]; ok { - continue - } - checked[param.RetrieverEngineType] = struct{}{} - if _, err := registry.GetRetrieveEngineService(param.RetrieverEngineType); err != nil { - return nil, fmt.Errorf("configured retrieve driver %q is unavailable: %w", driver, err) - } - } - } // ─── DB store registration (byStoreID) ─── if storeReg, ok := registry.(*retriever.RetrieveEngineRegistry); ok { loadDBStoresIntoRegistry(storeReg, db, cfg, auditSink) diff --git a/internal/container/engine_factory_opensearch_test.go b/internal/container/engine_factory_opensearch_test.go index df414fd5f2..8599d312c9 100644 --- a/internal/container/engine_factory_opensearch_test.go +++ b/internal/container/engine_factory_opensearch_test.go @@ -91,7 +91,7 @@ func TestInitRetrieveEngineRegistry_OpenSearchEnvPath(t *testing.T) { t.Fatalf("open in-mem db: %v", err) } - t.Setenv("RETRIEVE_DRIVER", " postgres, opensearch ") + t.Setenv("RETRIEVE_DRIVER", "opensearch") t.Setenv("OPENSEARCH_ADDR", ts.URL) // nil store repository and engine factory: this exercises the env-driver @@ -104,40 +104,3 @@ func TestInitRetrieveEngineRegistry_OpenSearchEnvPath(t *testing.T) { t.Errorf("opensearch engine not registered via env path: %v", err) } } - -func TestInitRetrieveEngineRegistry_OpenSearchTransientStartupRegistersDeferred(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusServiceUnavailable) - _, _ = w.Write([]byte(`{"error":{"type":"unavailable","reason":"starting"},"status":503}`)) - })) - defer ts.Close() - - db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) - if err != nil { - t.Fatalf("open in-mem db: %v", err) - } - t.Setenv("RETRIEVE_DRIVER", "opensearch") - t.Setenv("OPENSEARCH_ADDR", ts.URL) - - registry, err := initRetrieveEngineRegistry(db, &config.Config{}, &fakeAuditSvc{}, nil, nil) - if err != nil { - t.Fatalf("transient OpenSearch startup failure must not fail registry init: %v", err) - } - if _, err := registry.GetRetrieveEngineService(types.OpenSearchRetrieverEngineType); err != nil { - t.Fatalf("deferred OpenSearch repository must remain registered: %v", err) - } -} - -func TestInitRetrieveEngineRegistryRejectsUnknownConfiguredDriver(t *testing.T) { - db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) - if err != nil { - t.Fatalf("open in-mem db: %v", err) - } - t.Setenv("RETRIEVE_DRIVER", " postgres, unknown-driver ") - - _, err = initRetrieveEngineRegistry(db, &config.Config{}, &fakeAuditSvc{}, nil, nil) - if err == nil { - t.Fatal("configured unknown retrieve driver must fail startup") - } -} diff --git a/internal/database/knowledge_file_update_migration_test.go b/internal/database/knowledge_file_update_migration_test.go index d458825d75..326e3045a6 100644 --- a/internal/database/knowledge_file_update_migration_test.go +++ b/internal/database/knowledge_file_update_migration_test.go @@ -23,6 +23,7 @@ func TestKnowledgeFileUpdateSlotMigratesExistingSQLite(t *testing.T) { t.Cleanup(func() { _ = db.Close() }) _, err = db.Exec(` + CREATE TABLE knowledge_bases (id TEXT PRIMARY KEY); CREATE TABLE schema_migrations (version INTEGER NOT NULL PRIMARY KEY, dirty BOOLEAN NOT NULL); INSERT INTO schema_migrations(version, dirty) VALUES (2, 0); `) diff --git a/internal/router/retrieve_driver_readiness_test.go b/internal/router/retrieve_driver_readiness_test.go deleted file mode 100644 index 074f040240..0000000000 --- a/internal/router/retrieve_driver_readiness_test.go +++ /dev/null @@ -1,16 +0,0 @@ -package router - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestCompareRetrieveDriversReportsSanitizedDifference(t *testing.T) { - missing, unexpected := compareRetrieveDrivers( - []string{"opensearch", "postgres"}, - []string{"postgres", "qdrant"}, - ) - assert.Equal(t, []string{"opensearch"}, missing) - assert.Equal(t, []string{"qdrant"}, unexpected) -} diff --git a/internal/router/router.go b/internal/router/router.go index 14647d7fe3..cc9b113685 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -2,9 +2,7 @@ package router import ( "context" - "net/http" "os" - "sort" "strings" "time" @@ -21,7 +19,6 @@ import ( "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/middleware" "github.com/Tencent/WeKnora/internal/tracing/langfuse" - "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" _ "github.com/Tencent/WeKnora/docs" // swagger docs @@ -87,7 +84,6 @@ type RouterParams struct { DataSourceCredentialsHandler *handler.DataSourceCredentialsHandler WeKnoraCloudHandler *handler.WeKnoraCloudHandler WikiPageHandler *handler.WikiPageHandler - RetrieveEngineRegistry interfaces.RetrieveEngineRegistry } // NewRouter 创建新的路由 @@ -129,25 +125,7 @@ func NewRouter(params RouterParams) *gin.Engine { // 健康检查（不需要认证） r.GET("/health", func(c *gin.Context) { - c.JSON(http.StatusOK, gin.H{"status": "ok"}) - }) - r.GET("/ready", func(c *gin.Context) { - configured := types.ParseRetrieverDrivers(os.Getenv("RETRIEVE_DRIVER")) - registered := registeredRetrieveDrivers(params.RetrieveEngineRegistry) - missing, unexpected := compareRetrieveDrivers(configured, registered) - status := http.StatusOK - state := "ok" - if len(missing) > 0 { - status = http.StatusServiceUnavailable - state = "unavailable" - } - c.JSON(status, gin.H{ - "status": state, - "configured_retrieve_drivers": configured, - "registered_retrieve_drivers": registered, - "missing_retrieve_drivers": missing, - "unexpected_retrieve_drivers": unexpected, - }) + c.JSON(200, gin.H{"status": "ok"}) }) // Swagger API 文档（仅在非生产环境下启用） @@ -314,49 +292,6 @@ func NewRouter(params RouterParams) *gin.Engine { return r } -func registeredRetrieveDrivers(registry interfaces.RetrieveEngineRegistry) []string { - if registry == nil { - return nil - } - seen := make(map[string]struct{}) - for _, engine := range registry.GetAllRetrieveEngineServices() { - if engine == nil { - continue - } - seen[string(engine.EngineType())] = struct{}{} - } - drivers := make([]string, 0, len(seen)) - for driver := range seen { - drivers = append(drivers, driver) - } - sort.Strings(drivers) - return drivers -} - -func compareRetrieveDrivers(configured, registered []string) (missing, unexpected []string) { - configuredSet := make(map[string]struct{}, len(configured)) - registeredSet := make(map[string]struct{}, len(registered)) - for _, driver := range configured { - configuredSet[driver] = struct{}{} - } - for _, driver := range registered { - registeredSet[driver] = struct{}{} - } - for _, driver := range configured { - if _, ok := registeredSet[driver]; !ok { - missing = append(missing, driver) - } - } - for _, driver := range registered { - if _, ok := configuredSet[driver]; !ok { - unexpected = append(unexpected, driver) - } - } - sort.Strings(missing) - sort.Strings(unexpected) - return missing, unexpected -} - // trustedProxies returns the proxy CIDRs/IPs whose X-Forwarded-For headers // gin should trust when resolving the client IP. Defaults to loopback and // private ranges (covers the bundled nginx in a container network); override diff --git a/internal/router/task_file_update_deadletter_test.go b/internal/router/task_file_update_deadletter_test.go index d56a77afbc..8ce3a4d434 100644 --- a/internal/router/task_file_update_deadletter_test.go +++ b/internal/router/task_file_update_deadletter_test.go @@ -120,6 +120,7 @@ func TestRunAsynqServerRegistersKnowledgeFileUpdateHandler(t *testing.T) { DataTableSummary: handler, ImageMultimodal: handler, KnowledgePostProcess: handler, + KnowledgeAutoTag: handler, WikiIngest: handler, TemporaryDocument: taskRouteTemporaryDocumentServiceStub{}, }) diff --git a/internal/types/tenant.go b/internal/types/tenant.go index 582f3ed1e7..ce24d61dc3 100644 --- a/internal/types/tenant.go +++ b/internal/types/tenant.go @@ -62,31 +62,13 @@ func GetRetrieverEngineMapping() map[string][]RetrieverEngineParams { return retrieverEngineMapping } -// ParseRetrieverDrivers normalizes RETRIEVE_DRIVER consistently for registry, -// tenant defaults, migrations, and env-backed vector-store projections. -func ParseRetrieverDrivers(raw string) []string { - result := make([]string, 0) - seen := make(map[string]struct{}) - for _, driver := range strings.Split(raw, ",") { - driver = strings.ToLower(strings.TrimSpace(driver)) - if driver == "" { - continue - } - if _, ok := seen[driver]; ok { - continue - } - seen[driver] = struct{}{} - result = append(result, driver) - } - return result -} - // GetDefaultRetrieverEngines returns the default retriever engines based on RETRIEVE_DRIVER env func GetDefaultRetrieverEngines() []RetrieverEngineParams { result := []RetrieverEngineParams{} seen := make(map[string]bool) - for _, driver := range ParseRetrieverDrivers(os.Getenv("RETRIEVE_DRIVER")) { + for _, driver := range strings.Split(os.Getenv("RETRIEVE_DRIVER"), ",") { + driver = strings.TrimSpace(driver) if params, ok := retrieverEngineMapping[driver]; ok { for _, p := range params { key := string(p.RetrieverType) + ":" + string(p.RetrieverEngineType) diff --git a/internal/types/tenant_retriever_drivers_test.go b/internal/types/tenant_retriever_drivers_test.go deleted file mode 100644 index 5d02d7c247..0000000000 --- a/internal/types/tenant_retriever_drivers_test.go +++ /dev/null @@ -1,14 +0,0 @@ -package types - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestParseRetrieverDriversNormalizesAndDeduplicates(t *testing.T) { - assert.Equal(t, - []string{"postgres", "opensearch"}, - ParseRetrieverDrivers(" POSTGRES, opensearch, postgres, "), - ) -} diff --git a/internal/types/vectorstore.go b/internal/types/vectorstore.go index 8a7b97d055..48b3d621a6 100644 --- a/internal/types/vectorstore.go +++ b/internal/types/vectorstore.go @@ -805,9 +805,15 @@ func BuildEnvVectorStores(retrieveDriver string, envLookup EnvLookupFunc) []Vect return nil } + drivers := strings.Split(retrieveDriver, ",") var stores []VectorStore - for _, driver := range ParseRetrieverDrivers(retrieveDriver) { + for _, driver := range drivers { + driver = strings.TrimSpace(driver) + if driver == "" { + continue + } + store := buildEnvStoreForDriver(driver, envLookup) if store != nil { stores = append(stores, *store) From a81a8a130a26761501c4738d3a0b1d4b4e7d7537 Mon Sep 17 00:00:00 2001 From: hanxiantao <601803023@qq.com> Date: Mon, 10 Aug 2026 10:05:08 +0800 Subject: [PATCH 3/7] fix(knowledge): preserve upsert idempotency with explicit config --- .../service/knowledge_file_update.go | 76 +++++++++- .../service/knowledge_file_update_test.go | 132 ++++++++++++++++++ 2 files changed, 205 insertions(+), 3 deletions(-) diff --git a/internal/application/service/knowledge_file_update.go b/internal/application/service/knowledge_file_update.go index 8d353f44c7..e8e1c0145f 100644 --- a/internal/application/service/knowledge_file_update.go +++ b/internal/application/service/knowledge_file_update.go @@ -250,9 +250,13 @@ func (s *knowledgeService) UpdateKnowledgeFile( if err != nil { return nil, err } - if (existing.FileUpdateState == "" || existing.FileUpdateState == types.KnowledgeFileUpdateStateIdle) && - newHash == existing.FileHash && safeFilename == existing.FileName && folderPath == existing.FolderPath && - !req.MetadataProvided && !req.TagIDsProvided && !req.ChannelProvided && req.ProcessOverrides == nil { + unchanged, err := s.sameAsCurrentKnowledgeFile( + ctx, existing, req, storedOverrides, folderPath, safeFilename, newHash, + ) + if err != nil { + return nil, err + } + if unchanged { s.attachTagsToKnowledge(ctx, existing) return &types.KnowledgeFileUpsertResult{Action: "unchanged", Knowledge: existing}, nil } @@ -992,6 +996,72 @@ func sameKnowledgeFileUpdate(a, b *types.KnowledgeFileUpdatePayload) bool { reflect.DeepEqual(a.ProcessConfig, b.ProcessConfig) } +func (s *knowledgeService) sameAsCurrentKnowledgeFile( + ctx context.Context, + existing *types.Knowledge, + req *types.KnowledgeFileUpdateRequest, + storedOverrides *types.KnowledgeProcessOverrides, + folderPath string, + fileName string, + fileHash string, +) (bool, error) { + if existing == nil || + (existing.FileUpdateState != "" && existing.FileUpdateState != types.KnowledgeFileUpdateStateIdle) || + fileHash != existing.FileHash || fileName != existing.FileName || folderPath != existing.FolderPath { + return false, nil + } + if req.ChannelProvided && req.Channel != existing.Channel { + return false, nil + } + if req.ProcessOverrides != nil && !reflect.DeepEqual(req.ProcessOverrides, storedOverrides) { + return false, nil + } + if req.MetadataProvided { + metadata, err := existing.Metadata.Map() + if err != nil { + return false, fmt.Errorf("parse stored metadata: %w", err) + } + for key, value := range req.Metadata { + if key == "process_overrides" { + continue + } + if current, ok := metadata[key]; !ok || !reflect.DeepEqual(current, value) { + return false, nil + } + } + } + if req.TagIDsProvided { + tagMap, err := s.repo.GetKnowledgeTags(ctx, []string{existing.ID}) + if err != nil { + return false, err + } + existing.Tags = tagMap[existing.ID] + currentIDs := make([]string, 0, len(existing.Tags)) + for _, tag := range existing.Tags { + if tag != nil { + currentIDs = append(currentIDs, tag.ID) + } + } + if !sameKnowledgeTagIDs(req.TagIDs, currentIDs) { + return false, nil + } + } + return true, nil +} + +func sameKnowledgeTagIDs(a, b []string) bool { + set := func(values []string) map[string]struct{} { + result := make(map[string]struct{}, len(values)) + for _, value := range values { + if value != "" { + result[value] = struct{}{} + } + } + return result + } + return reflect.DeepEqual(set(a), set(b)) +} + func replacementMetadata( knowledge *types.Knowledge, payload *types.KnowledgeFileUpdatePayload, diff --git a/internal/application/service/knowledge_file_update_test.go b/internal/application/service/knowledge_file_update_test.go index af9271caaf..1e8446187b 100644 --- a/internal/application/service/knowledge_file_update_test.go +++ b/internal/application/service/knowledge_file_update_test.go @@ -26,6 +26,8 @@ type fileUpdateRepoStub struct { stageCalls int applyCalls int applyValues map[string]interface{} + tags map[string][]*types.KnowledgeTag + tagsErr error } func (r *fileUpdateRepoStub) TransitionKnowledgeFileUpdateState( @@ -122,6 +124,12 @@ func (r *fileUpdateRepoStub) UpdateApplyingKnowledgeFileColumns( func (r *fileUpdateRepoStub) GetKnowledgeTags( context.Context, []string, ) (map[string][]*types.KnowledgeTag, error) { + if r.tagsErr != nil { + return nil, r.tagsErr + } + if r.tags != nil { + return r.tags, nil + } return map[string][]*types.KnowledgeTag{}, nil } @@ -426,6 +434,130 @@ func TestUpdateKnowledgeFileSameLatestPendingIsUnchanged(t *testing.T) { assert.Zero(t, fileSvc.deleteCalls) } +func TestUpdateKnowledgeFileSameCurrentFileAndExplicitChannelIsUnchanged(t *testing.T) { + file := newMultipartFileHeader(t, "latest.md", "latest content") + hash, err := calculateFileHash(file) + require.NoError(t, err) + repo := &fileUpdateRepoStub{ + knowledge: &types.Knowledge{ + ID: "knowledge-1", KnowledgeBaseID: "kb-1", Type: "file", + FilePath: "current/path.md", FileName: "latest.md", FileHash: hash, + Channel: "api-e2e", ParseStatus: types.ParseStatusCompleted, + FileUpdateVersion: 4, FileUpdateState: types.KnowledgeFileUpdateStateIdle, + }, + } + fileSvc := &createKnowledgeFileServiceStub{} + task := &fileUpdateTaskStub{} + + result, err := newFileUpdateService(repo, fileSvc, task).UpdateKnowledgeFile( + newCreateKnowledgeFileContext(), + &types.KnowledgeFileUpdateRequest{ + KnowledgeBaseID: "kb-1", KnowledgeID: "knowledge-1", File: file, + Channel: "api-e2e", ChannelProvided: true, + }, + ) + + require.NoError(t, err) + require.NotNil(t, result) + assert.Equal(t, "unchanged", result.Action) + assert.Equal(t, uint64(4), result.Knowledge.FileUpdateVersion) + assert.Zero(t, repo.stageCalls) + assert.Zero(t, fileSvc.saveCalls) + assert.Empty(t, task.tasks) +} + +func TestUpdateKnowledgeFileSameCurrentFileAndDifferentChannelStagesUpdate(t *testing.T) { + file := newMultipartFileHeader(t, "latest.md", "latest content") + hash, err := calculateFileHash(file) + require.NoError(t, err) + repo := &fileUpdateRepoStub{ + knowledge: &types.Knowledge{ + ID: "knowledge-1", KnowledgeBaseID: "kb-1", Type: "file", + FilePath: "current/path.md", FileName: "latest.md", FileHash: hash, + Channel: "api-e2e", ParseStatus: types.ParseStatusCompleted, + FileUpdateVersion: 4, FileUpdateState: types.KnowledgeFileUpdateStateIdle, + }, + stageResult: &types.KnowledgeFileUpdateStageResult{ + Version: 5, State: types.KnowledgeFileUpdateResultActive, ActiveVersion: 5, + }, + } + fileSvc := &createKnowledgeFileServiceStub{} + task := &fileUpdateTaskStub{} + + result, err := newFileUpdateService(repo, fileSvc, task).UpdateKnowledgeFile( + newCreateKnowledgeFileContext(), + &types.KnowledgeFileUpdateRequest{ + KnowledgeBaseID: "kb-1", KnowledgeID: "knowledge-1", File: file, + Channel: "another-channel", ChannelProvided: true, + }, + ) + + require.NoError(t, err) + require.NotNil(t, result) + assert.Equal(t, "updated", result.Action) + assert.Equal(t, uint64(5), result.UpdateVersion) + assert.Equal(t, 1, repo.stageCalls) + assert.Equal(t, 1, fileSvc.saveCalls) + require.Len(t, task.tasks, 1) +} + +func TestSameAsCurrentKnowledgeFileComparesExplicitConfig(t *testing.T) { + storedOverrides := &types.KnowledgeProcessOverrides{ + ParserEngineOverrides: map[string]string{"mode": "accurate"}, + } + existing := &types.Knowledge{ + ID: "knowledge-1", FileName: "latest.md", FileHash: "same-hash", + Channel: "api-e2e", FileUpdateState: types.KnowledgeFileUpdateStateIdle, + Metadata: types.JSON(`{"source":"sync","owner":"docs"}`), + } + repo := &fileUpdateRepoStub{ + knowledge: existing, + tags: map[string][]*types.KnowledgeTag{ + existing.ID: {{ID: "tag-1"}, {ID: "tag-2"}}, + }, + } + svc := &knowledgeService{repo: repo} + + request := func() *types.KnowledgeFileUpdateRequest { + return &types.KnowledgeFileUpdateRequest{ + Channel: "api-e2e", ChannelProvided: true, + Metadata: map[string]string{"source": "sync"}, MetadataProvided: true, + TagIDs: []string{"tag-2", "tag-1", "tag-1"}, TagIDsProvided: true, + ProcessOverrides: &types.KnowledgeProcessOverrides{ + ParserEngineOverrides: map[string]string{"mode": "accurate"}, + }, + } + } + + unchanged, err := svc.sameAsCurrentKnowledgeFile( + context.Background(), existing, request(), storedOverrides, "", "latest.md", "same-hash", + ) + require.NoError(t, err) + assert.True(t, unchanged) + + tests := map[string]func(*types.KnowledgeFileUpdateRequest){ + "channel": func(req *types.KnowledgeFileUpdateRequest) { req.Channel = "web" }, + "metadata": func(req *types.KnowledgeFileUpdateRequest) { + req.Metadata["source"] = "manual" + }, + "tags": func(req *types.KnowledgeFileUpdateRequest) { req.TagIDs = []string{"tag-1"} }, + "process config": func(req *types.KnowledgeFileUpdateRequest) { + req.ProcessOverrides.ParserEngineOverrides["mode"] = "fast" + }, + } + for name, mutate := range tests { + t.Run(name, func(t *testing.T) { + req := request() + mutate(req) + unchanged, err := svc.sameAsCurrentKnowledgeFile( + context.Background(), existing, req, storedOverrides, "", "latest.md", "same-hash", + ) + require.NoError(t, err) + assert.False(t, unchanged) + }) + } +} + func TestRetryKnowledgeFileUpdateRearmsExactFailedVersion(t *testing.T) { version := uint64(7) repo := &fileUpdateRepoStub{ From 00946980f07de3a1b3881863fa286051fec37495 Mon Sep 17 00:00:00 2001 From: hanxiantao <601803023@qq.com> Date: Tue, 11 Aug 2026 15:14:28 +0800 Subject: [PATCH 4/7] fix(migrations): move knowledge update slots to latest versions --- internal/database/knowledge_file_update_migration_test.go | 2 +- migrations/sqlite/000000_init.up.sql | 2 +- ...s.down.sql => 000004_knowledge_file_update_slots.down.sql} | 0 ...slots.up.sql => 000004_knowledge_file_update_slots.up.sql} | 0 ...s.down.sql => 000084_knowledge_file_update_slots.down.sql} | 0 ...slots.up.sql => 000084_knowledge_file_update_slots.up.sql} | 4 ++-- 6 files changed, 4 insertions(+), 4 deletions(-) rename migrations/sqlite/{000080_knowledge_file_update_slots.down.sql => 000004_knowledge_file_update_slots.down.sql} (100%) rename migrations/sqlite/{000080_knowledge_file_update_slots.up.sql => 000004_knowledge_file_update_slots.up.sql} (100%) rename migrations/versioned/{000080_knowledge_file_update_slots.down.sql => 000084_knowledge_file_update_slots.down.sql} (100%) rename migrations/versioned/{000080_knowledge_file_update_slots.up.sql => 000084_knowledge_file_update_slots.up.sql} (92%) diff --git a/internal/database/knowledge_file_update_migration_test.go b/internal/database/knowledge_file_update_migration_test.go index 326e3045a6..2b15fdade9 100644 --- a/internal/database/knowledge_file_update_migration_test.go +++ b/internal/database/knowledge_file_update_migration_test.go @@ -45,7 +45,7 @@ func TestKnowledgeFileUpdateSlotMigratesExistingSQLite(t *testing.T) { require.NoError(t, migrator.Up()) version, dirty, err := migrator.Version() require.NoError(t, err) - require.Equal(t, uint(80), version) + require.Equal(t, uint(4), version) require.False(t, dirty) var tableName string diff --git a/migrations/sqlite/000000_init.up.sql b/migrations/sqlite/000000_init.up.sql index 0732c3f3ff..dea3f10394 100644 --- a/migrations/sqlite/000000_init.up.sql +++ b/migrations/sqlite/000000_init.up.sql @@ -1121,7 +1121,7 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_wiki_page_revisions_page_version CREATE INDEX IF NOT EXISTS idx_wiki_page_revisions_kb_slug ON wiki_page_revisions (knowledge_base_id, slug); --- File knowledge update coordination slot. Migration 000080 repeats this +-- File knowledge update coordination slot. Migration 000004 repeats this -- idempotently so databases already initialized at version 0 are upgraded. CREATE TABLE IF NOT EXISTS knowledge_file_update_slots ( knowledge_id VARCHAR(36) PRIMARY KEY, diff --git a/migrations/sqlite/000080_knowledge_file_update_slots.down.sql b/migrations/sqlite/000004_knowledge_file_update_slots.down.sql similarity index 100% rename from migrations/sqlite/000080_knowledge_file_update_slots.down.sql rename to migrations/sqlite/000004_knowledge_file_update_slots.down.sql diff --git a/migrations/sqlite/000080_knowledge_file_update_slots.up.sql b/migrations/sqlite/000004_knowledge_file_update_slots.up.sql similarity index 100% rename from migrations/sqlite/000080_knowledge_file_update_slots.up.sql rename to migrations/sqlite/000004_knowledge_file_update_slots.up.sql diff --git a/migrations/versioned/000080_knowledge_file_update_slots.down.sql b/migrations/versioned/000084_knowledge_file_update_slots.down.sql similarity index 100% rename from migrations/versioned/000080_knowledge_file_update_slots.down.sql rename to migrations/versioned/000084_knowledge_file_update_slots.down.sql diff --git a/migrations/versioned/000080_knowledge_file_update_slots.up.sql b/migrations/versioned/000084_knowledge_file_update_slots.up.sql similarity index 92% rename from migrations/versioned/000080_knowledge_file_update_slots.up.sql rename to migrations/versioned/000084_knowledge_file_update_slots.up.sql index 04be895905..4efd6b70f4 100644 --- a/migrations/versioned/000080_knowledge_file_update_slots.up.sql +++ b/migrations/versioned/000084_knowledge_file_update_slots.up.sql @@ -1,4 +1,4 @@ -DO $$ BEGIN RAISE NOTICE '[Migration 000080] Creating knowledge file update slots...'; END $$; +DO $$ BEGIN RAISE NOTICE '[Migration 000084] Creating knowledge file update slots...'; END $$; CREATE TABLE IF NOT EXISTS knowledge_file_update_slots ( knowledge_id VARCHAR(36) PRIMARY KEY, @@ -30,4 +30,4 @@ CREATE INDEX IF NOT EXISTS idx_knowledge_file_update_slots_tenant_kb CREATE INDEX IF NOT EXISTS idx_knowledge_file_update_slots_state ON knowledge_file_update_slots(active_state, updated_at); -DO $$ BEGIN RAISE NOTICE '[Migration 000080] Knowledge file update slots ready'; END $$; +DO $$ BEGIN RAISE NOTICE '[Migration 000084] Knowledge file update slots ready'; END $$; From 79a3d1edec56d2df47cfcccf9d1fd47120b03af3 Mon Sep 17 00:00:00 2001 From: hanxiantao <601803023@qq.com> Date: Thu, 13 Aug 2026 10:24:06 +0800 Subject: [PATCH 5/7] refactor(knowledge): keep only create-or-update API --- docs/api/knowledge.md | 7 -- docs/docs.go | 94 -------------- docs/swagger.json | 94 -------------- docs/swagger.yaml | 58 --------- frontend/src/api/knowledge-base/index.ts | 8 -- frontend/src/i18n/locales/en-US.ts | 7 -- frontend/src/i18n/locales/ko-KR.ts | 7 -- frontend/src/i18n/locales/ru-RU.ts | 7 -- frontend/src/i18n/locales/zh-CN.ts | 7 -- .../src/views/knowledge/KnowledgeBase.vue | 32 +---- .../components/DocumentActionMenu.vue | 19 --- .../knowledge/components/DocumentCardView.vue | 6 +- .../knowledge/components/DocumentListView.vue | 6 +- internal/application/repository/knowledge.go | 35 ------ .../knowledge_file_update_slot_test.go | 31 ----- .../service/knowledge_file_update.go | 118 ------------------ .../service/knowledge_file_update_test.go | 63 ---------- internal/handler/knowledge.go | 71 ----------- internal/router/routes_knowledge.go | 2 - internal/types/interfaces/knowledge.go | 7 -- 20 files changed, 6 insertions(+), 673 deletions(-) diff --git a/docs/api/knowledge.md b/docs/api/knowledge.md index 6a9561074f..df226246df 100644 --- a/docs/api/knowledge.md +++ b/docs/api/knowledge.md @@ -21,8 +21,6 @@ | PUT | `/knowledge/manual/:id` | 更新手工 Markdown 知识 | | POST | `/knowledge/:id/reparse` | 重新解析知识（异步） | | POST | `/knowledge/:id/cancel-parse` | 取消正在进行的解析任务 | -| POST | `/knowledge/:id/file-update/retry` | 重试失败的文件更新版本 | -| DELETE | `/knowledge/:id/file-update` | 丢弃失败及待处理的文件更新版本 | | GET | `/knowledge/:id/download` | 下载原始文件（attachment） | | GET | `/knowledge/:id/preview` | 内联预览文件（按扩展名设置 Content-Type） | | PUT | `/knowledge/image/:id/:chunk_id` | 更新分块图像信息 | @@ -186,11 +184,6 @@ curl --location 'http://localhost:8080/api/v1/knowledge-bases/kb-00000001/knowle 调用方通过 `GET /knowledge/:id` 同时轮询 `file_update_state` 和 `parse_status`。前者取值 `idle` / `active` / `pending` / `failed`，后者表示当前已生效文件的解析状态。`file_update_version` 可作为下一次请求的 `expected_update_version`。 -失败槽会保留 active payload 以便恢复： - -- `POST /knowledge/:id/file-update/retry`：仅当最新 active 为 `failed` 时重新入队。 -- `DELETE /knowledge/:id/file-update`：丢弃精确匹配的 failed active 和最新 pending，并回收不再引用的暂存文件。 - 删除 knowledge 会先设置 `deleting` 并撤销更新槽；已排队的陈旧更新任务不会恢复该 knowledge。 | 状态码 | 说明 | diff --git a/docs/docs.go b/docs/docs.go index 41ace13b75..83396498db 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -7067,100 +7067,6 @@ const docTemplate = `{ } } }, - "/knowledge/{id}/file-update": { - "delete": { - "security": [ - { - "Bearer": [] - }, - { - "ApiKeyAuth": [] - } - ], - "description": "删除精确匹配的失败 active 和最新 pending 暂存版本，不影响并发提交的新版本", - "produces": [ - "application/json" - ], - "tags": [ - "知识管理" - ], - "summary": "丢弃失败的文件更新", - "parameters": [ - { - "type": "string", - "description": "知识ID", - "name": "id", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "待更新版本已丢弃", - "schema": { - "type": "object", - "additionalProperties": true - } - }, - "409": { - "description": "没有失败更新或状态已变化", - "schema": { - "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" - } - } - } - } - }, - "/knowledge/{id}/file-update/retry": { - "post": { - "security": [ - { - "Bearer": [] - }, - { - "ApiKeyAuth": [] - } - ], - "description": "重新唤醒 update slot 中保留的失败 active 版本；并发上传的新版本不会被覆盖", - "produces": [ - "application/json" - ], - "tags": [ - "知识管理" - ], - "summary": "重试失败的文件更新", - "parameters": [ - { - "type": "string", - "description": "知识ID", - "name": "id", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "重试已提交", - "schema": { - "type": "object", - "additionalProperties": true - } - }, - "409": { - "description": "没有失败更新或状态已变化", - "schema": { - "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" - } - }, - "503": { - "description": "任务系统暂时不可用", - "schema": { - "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" - } - } - } - } - }, "/knowledge/{id}/preview": { "get": { "security": [ diff --git a/docs/swagger.json b/docs/swagger.json index 6c56615923..b553c3c58a 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -7060,100 +7060,6 @@ } } }, - "/knowledge/{id}/file-update": { - "delete": { - "security": [ - { - "Bearer": [] - }, - { - "ApiKeyAuth": [] - } - ], - "description": "删除精确匹配的失败 active 和最新 pending 暂存版本，不影响并发提交的新版本", - "produces": [ - "application/json" - ], - "tags": [ - "知识管理" - ], - "summary": "丢弃失败的文件更新", - "parameters": [ - { - "type": "string", - "description": "知识ID", - "name": "id", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "待更新版本已丢弃", - "schema": { - "type": "object", - "additionalProperties": true - } - }, - "409": { - "description": "没有失败更新或状态已变化", - "schema": { - "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" - } - } - } - } - }, - "/knowledge/{id}/file-update/retry": { - "post": { - "security": [ - { - "Bearer": [] - }, - { - "ApiKeyAuth": [] - } - ], - "description": "重新唤醒 update slot 中保留的失败 active 版本；并发上传的新版本不会被覆盖", - "produces": [ - "application/json" - ], - "tags": [ - "知识管理" - ], - "summary": "重试失败的文件更新", - "parameters": [ - { - "type": "string", - "description": "知识ID", - "name": "id", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "重试已提交", - "schema": { - "type": "object", - "additionalProperties": true - } - }, - "409": { - "description": "没有失败更新或状态已变化", - "schema": { - "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" - } - }, - "503": { - "description": "任务系统暂时不可用", - "schema": { - "$ref": "#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError" - } - } - } - } - }, "/knowledge/{id}/preview": { "get": { "security": [ diff --git a/docs/swagger.yaml b/docs/swagger.yaml index dccf86f5bb..015ad0d93c 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -10022,64 +10022,6 @@ paths: summary: 下载知识文件 tags: - 知识管理 - /knowledge/{id}/file-update: - delete: - description: 删除精确匹配的失败 active 和最新 pending 暂存版本，不影响并发提交的新版本 - parameters: - - description: 知识ID - in: path - name: id - required: true - type: string - produces: - - application/json - responses: - "200": - description: 待更新版本已丢弃 - schema: - additionalProperties: true - type: object - "409": - description: 没有失败更新或状态已变化 - schema: - $ref: '#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError' - security: - - Bearer: [] - - ApiKeyAuth: [] - summary: 丢弃失败的文件更新 - tags: - - 知识管理 - /knowledge/{id}/file-update/retry: - post: - description: 重新唤醒 update slot 中保留的失败 active 版本；并发上传的新版本不会被覆盖 - parameters: - - description: 知识ID - in: path - name: id - required: true - type: string - produces: - - application/json - responses: - "200": - description: 重试已提交 - schema: - additionalProperties: true - type: object - "409": - description: 没有失败更新或状态已变化 - schema: - $ref: '#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError' - "503": - description: 任务系统暂时不可用 - schema: - $ref: '#/definitions/github_com_Tencent_WeKnora_internal_errors.AppError' - security: - - Bearer: [] - - ApiKeyAuth: [] - summary: 重试失败的文件更新 - tags: - - 知识管理 /knowledge/{id}/preview: get: consumes: diff --git a/frontend/src/api/knowledge-base/index.ts b/frontend/src/api/knowledge-base/index.ts index 968a6e798a..323d87c9a8 100644 --- a/frontend/src/api/knowledge-base/index.ts +++ b/frontend/src/api/knowledge-base/index.ts @@ -360,14 +360,6 @@ export function cancelKnowledgeParse(id: string) { return post(`/api/v1/knowledge/${id}/cancel-parse`); } -export function retryKnowledgeFileUpdate(id: string) { - return post(`/api/v1/knowledge/${id}/file-update/retry`); -} - -export function discardKnowledgeFileUpdate(id: string) { - return del(`/api/v1/knowledge/${id}/file-update`); -} - export function getKnowledgeSpans(id: string, attempt?: number) { const qs = attempt ? `?attempt=${attempt}` : ''; return get(`/api/v1/knowledge/${id}/spans${qs}`); diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index 7a234e1f70..8e0b2f6dc9 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -614,13 +614,6 @@ export default { fileUpdateActive: 'Updating file', fileUpdatePending: 'Update pending v{version}', fileUpdateFailed: 'File update failed', - retryFileUpdate: 'Retry latest version', - retryFileUpdateSubmitted: 'File update resubmitted', - retryFileUpdateFailed: 'Failed to retry file update', - discardFileUpdate: 'Discard pending update', - discardFileUpdateConfirm: 'Discard the failed version and its latest pending file?', - discardFileUpdateSubmitted: 'Pending update discarded', - discardFileUpdateFailed: 'Failed to discard pending update', statusFailed: 'Failed', statusCancelled: 'Cancelled', statusDraft: 'Draft', diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index 46006ff718..631c6cdae0 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -5765,13 +5765,6 @@ export default { fileUpdateActive: '파일 업데이트 중', fileUpdatePending: '업데이트 대기 v{version}', fileUpdateFailed: '파일 업데이트 실패', - retryFileUpdate: '최신 버전 다시 시도', - retryFileUpdateSubmitted: '파일 업데이트를 다시 제출했습니다', - retryFileUpdateFailed: '파일 업데이트 재시도 실패', - discardFileUpdate: '대기 업데이트 삭제', - discardFileUpdateConfirm: '실패한 버전과 최신 대기 파일을 삭제하시겠습니까?', - discardFileUpdateSubmitted: '대기 업데이트를 삭제했습니다', - discardFileUpdateFailed: '대기 업데이트 삭제 실패', statusFailed: '실패', statusCancelled: '취소됨', statusDraft: '초안', diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index 2cb890c489..8282a036a6 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -5765,13 +5765,6 @@ export default { fileUpdateActive: 'Обновление файла', fileUpdatePending: 'Ожидает обновления v{version}', fileUpdateFailed: 'Ошибка обновления файла', - retryFileUpdate: 'Повторить последнюю версию', - retryFileUpdateSubmitted: 'Обновление файла отправлено повторно', - retryFileUpdateFailed: 'Не удалось повторить обновление файла', - discardFileUpdate: 'Отменить ожидающее обновление', - discardFileUpdateConfirm: 'Отменить ошибочную версию и последний ожидающий файл?', - discardFileUpdateSubmitted: 'Ожидающее обновление отменено', - discardFileUpdateFailed: 'Не удалось отменить ожидающее обновление', statusFailed: 'Ошибка', statusCancelled: 'Отменено', statusDraft: 'Черновик', diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index 9383a8ad8e..64d1c17640 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -5767,13 +5767,6 @@ export default { fileUpdateActive: '正在更新文件', fileUpdatePending: '有待更新 v{version}', fileUpdateFailed: '文件更新失败', - retryFileUpdate: '重试最新版本', - retryFileUpdateSubmitted: '文件更新已重新提交', - retryFileUpdateFailed: '文件更新重试失败', - discardFileUpdate: '丢弃待更新', - discardFileUpdateConfirm: '确认丢弃失败版本及其最新待更新文件？', - discardFileUpdateSubmitted: '已丢弃待更新版本', - discardFileUpdateFailed: '丢弃待更新版本失败', statusFailed: '失败', statusCancelled: '已取消', statusDraft: '草稿', diff --git a/frontend/src/views/knowledge/KnowledgeBase.vue b/frontend/src/views/knowledge/KnowledgeBase.vue index fdafa3a630..7f603f6191 100644 --- a/frontend/src/views/knowledge/KnowledgeBase.vue +++ b/frontend/src/views/knowledge/KnowledgeBase.vue @@ -31,8 +31,6 @@ import { createKnowledgeFromURL, reparseKnowledge, cancelKnowledgeParse, - retryKnowledgeFileUpdate, - discardKnowledgeFileUpdate, batchDeleteKnowledge, batchReparseKnowledge, getKnowledgeSpans, @@ -2155,28 +2153,6 @@ const confirmCancelParseKnowledge = async (item: KnowledgeCard) => { } }; -const confirmRetryFileUpdate = async (item: KnowledgeCard) => { - if (!item?.id) return; - try { - await retryKnowledgeFileUpdate(item.id); - MessagePlugin.success(t('knowledgeBase.retryFileUpdateSubmitted')); - loadKnowledgeFiles(kbId.value); - } catch (error: any) { - MessagePlugin.error(error?.message || t('knowledgeBase.retryFileUpdateFailed')); - } -}; - -const confirmDiscardFileUpdate = async (item: KnowledgeCard) => { - if (!item?.id) return; - try { - await discardKnowledgeFileUpdate(item.id); - MessagePlugin.success(t('knowledgeBase.discardFileUpdateSubmitted')); - loadKnowledgeFiles(kbId.value); - } catch (error: any) { - MessagePlugin.error(error?.message || t('knowledgeBase.discardFileUpdateFailed')); - } -}; - const downloadKnowledge = async (item: KnowledgeCard) => { if (!item?.id) return; try { @@ -2200,7 +2176,7 @@ const downloadKnowledge = async (item: KnowledgeCard) => { // Bridge card-view actions back to existing per-card handlers. const handleCardAction = ( - action: 'download' | 'edit' | 'reparse' | 'cancel-parse' | 'retry-file-update' | 'discard-file-update' | 'move' | 'move-folder' | 'delete' | 'view-trace' | 'batch-manage', + action: 'download' | 'edit' | 'reparse' | 'cancel-parse' | 'move' | 'move-folder' | 'delete' | 'view-trace' | 'batch-manage', item: KnowledgeCard, ) => { const idx = (cardList.value || []).findIndex((i: KnowledgeCard) => i.id === item.id); @@ -2211,8 +2187,6 @@ const handleCardAction = ( return confirmRebuildKnowledge(idx, item); } if (action === 'cancel-parse') return confirmCancelParseKnowledge(item); - if (action === 'retry-file-update') return confirmRetryFileUpdate(item); - if (action === 'discard-file-update') return confirmDiscardFileUpdate(item); if (action === 'move') return handleMoveKnowledge(item); if (action === 'delete') return confirmDeleteKnowledge(idx, item); if (action === 'view-trace') return handleViewTrace(idx, item); @@ -2221,7 +2195,7 @@ const handleCardAction = ( // Bridge list-view actions back to existing per-card handlers. const handleListAction = ( - action: 'download' | 'edit' | 'reparse' | 'cancel-parse' | 'retry-file-update' | 'discard-file-update' | 'move' | 'move-folder' | 'delete' | 'view-trace' | 'batch-manage', + action: 'download' | 'edit' | 'reparse' | 'cancel-parse' | 'move' | 'move-folder' | 'delete' | 'view-trace' | 'batch-manage', item: KnowledgeCard, ) => { const idx = (cardList.value || []).findIndex((i: KnowledgeCard) => i.id === item.id); @@ -2229,8 +2203,6 @@ const handleListAction = ( if (action === 'edit') return handleManualEdit(idx, item); if (action === 'reparse') return confirmRebuildKnowledge(idx, item); if (action === 'cancel-parse') return confirmCancelParseKnowledge(item); - if (action === 'retry-file-update') return confirmRetryFileUpdate(item); - if (action === 'discard-file-update') return confirmDiscardFileUpdate(item); if (action === 'move') return handleMoveKnowledge(item); if (action === 'delete') return confirmDeleteKnowledge(idx, item); if (action === 'view-trace') return handleViewTrace(idx, item); diff --git a/frontend/src/views/knowledge/components/DocumentActionMenu.vue b/frontend/src/views/knowledge/components/DocumentActionMenu.vue index 9732d36821..19389558a1 100644 --- a/frontend/src/views/knowledge/components/DocumentActionMenu.vue +++ b/frontend/src/views/knowledge/components/DocumentActionMenu.vue @@ -26,8 +26,6 @@ const emit = defineEmits<{ (e: 'view-trace'): void; (e: 'reparse'): void; (e: 'cancel-parse'): void; - (e: 'retry-file-update'): void; - (e: 'discard-file-update'): void; (e: 'move'): void; (e: 'move-folder'): void; (e: 'batch-manage'): void; @@ -42,7 +40,6 @@ const isParseInFlight = computed(() => CANCELABLE_PARSE_STATUSES.has(String(props.item.parse_status ?? '')) ); const isReplacing = computed(() => props.item.parse_status === 'replacing'); -const hasFailedFileUpdate = computed(() => props.item.file_update_state === 'failed'); const fileName = computed(() => props.item.file_name || props.item.title || props.item.id); @@ -53,22 +50,6 @@ const fileName = computed(() => props.item.file_name || props.item.title || prop {{ $t('knowledgeBase.statusReplacing') }}
-
- - {{ $t('knowledgeBase.retryFileUpdate') }} -
- - -
- - {{ $t('knowledgeBase.discardFileUpdate') }} -
- -
{ }; // --- Action handlers --- -const handleAction = (action: 'download' | 'edit' | 'view-trace' | 'reparse' | 'cancel-parse' | 'retry-file-update' | 'discard-file-update' | 'move' | 'move-folder' | 'batch-manage' | 'delete', item: KnowledgeCard) => { +const handleAction = (action: 'download' | 'edit' | 'view-trace' | 'reparse' | 'cancel-parse' | 'move' | 'move-folder' | 'batch-manage' | 'delete', item: KnowledgeCard) => { // The folder picker opens inside this same popup, so keep the menu open. if (action === 'move-folder') { folderPickerItemId.value = item.id; @@ -410,8 +410,6 @@ const handleAction = (action: 'download' | 'edit' | 'view-trace' | 'reparse' | ' @view-trace="handleAction('view-trace', item)" @reparse="handleAction('reparse', item)" @cancel-parse="handleAction('cancel-parse', item)" - @retry-file-update="handleAction('retry-file-update', item)" - @discard-file-update="handleAction('discard-file-update', item)" @move="handleAction('move', item)" @move-folder="handleAction('move-folder', item)" @batch-manage="handleAction('batch-manage', item)" diff --git a/frontend/src/views/knowledge/components/DocumentListView.vue b/frontend/src/views/knowledge/components/DocumentListView.vue index 9a251c19ae..5437a1bb80 100644 --- a/frontend/src/views/knowledge/components/DocumentListView.vue +++ b/frontend/src/views/knowledge/components/DocumentListView.vue @@ -64,7 +64,7 @@ const emit = defineEmits<{ (e: 'open', item: KnowledgeItem): void; (e: 'toggle-row', id: string, checked: boolean, shiftKey: boolean): void; (e: 'toggle-all', checked: boolean): void; - (e: 'action', action: 'download' | 'edit' | 'reparse' | 'cancel-parse' | 'retry-file-update' | 'discard-file-update' | 'move' | 'move-folder' | 'delete' | 'view-trace' | 'batch-manage', item: KnowledgeItem): void; + (e: 'action', action: 'download' | 'edit' | 'reparse' | 'cancel-parse' | 'move' | 'move-folder' | 'delete' | 'view-trace' | 'batch-manage', item: KnowledgeItem): void; (e: 'probe-trace', item: KnowledgeItem): void; (e: 'tag-edit', item: KnowledgeItem): void; (e: 'open-folder', path: string): void; @@ -246,7 +246,7 @@ const onFolderPicked = (item: KnowledgeItem, path: string) => { emit('move-to-folder', item, path); }; -const handleAction = (action: 'download' | 'edit' | 'reparse' | 'cancel-parse' | 'retry-file-update' | 'discard-file-update' | 'move' | 'move-folder' | 'delete' | 'view-trace' | 'batch-manage', item: KnowledgeItem) => { +const handleAction = (action: 'download' | 'edit' | 'reparse' | 'cancel-parse' | 'move' | 'move-folder' | 'delete' | 'view-trace' | 'batch-manage', item: KnowledgeItem) => { // The folder picker opens inside this same popup, so keep the menu open. if (action === 'move-folder') { folderPickerItemId.value = item.id; @@ -426,8 +426,6 @@ const handleAction = (action: 'download' | 'edit' | 'reparse' | 'cancel-parse' | @view-trace="handleAction('view-trace', item)" @reparse="handleAction('reparse', item)" @cancel-parse="handleAction('cancel-parse', item)" - @retry-file-update="handleAction('retry-file-update', item)" - @discard-file-update="handleAction('discard-file-update', item)" @move="handleAction('move', item)" @move-folder="handleAction('move-folder', item)" @batch-manage="handleAction('batch-manage', item)" diff --git a/internal/application/repository/knowledge.go b/internal/application/repository/knowledge.go index acf80ebe52..846494eeef 100644 --- a/internal/application/repository/knowledge.go +++ b/internal/application/repository/knowledge.go @@ -894,41 +894,6 @@ func (r *knowledgeRepository) CancelKnowledgeFileUpdates( return cancelled, err } -// CancelFailedKnowledgeFileUpdate removes only the exact failed active -// version observed by the caller. A concurrent upload cannot be discarded. -func (r *knowledgeRepository) CancelFailedKnowledgeFileUpdate( - ctx context.Context, tenantID uint64, knowledgeID string, activeVersion uint64, -) (*types.KnowledgeFileUpdateSlot, error) { - var cancelled *types.KnowledgeFileUpdateSlot - err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { - query := tx.Where("tenant_id = ? AND knowledge_id = ?", tenantID, knowledgeID) - if tx.Dialector.Name() != "sqlite" { - query = query.Clauses(clause.Locking{Strength: "UPDATE"}) - } - var slot types.KnowledgeFileUpdateSlot - if err := query.First(&slot).Error; err != nil { - return err - } - if slot.ActiveVersion == nil || *slot.ActiveVersion != activeVersion || - slot.ActiveState != types.KnowledgeFileUpdateStateFailed { - return ErrKnowledgeFileUpdateStateConflict - } - result := tx.Where( - "tenant_id = ? AND knowledge_id = ? AND active_version = ? AND active_state = ?", - tenantID, knowledgeID, activeVersion, types.KnowledgeFileUpdateStateFailed, - ).Delete(&types.KnowledgeFileUpdateSlot{}) - if result.Error != nil { - return result.Error - } - if result.RowsAffected != 1 { - return ErrKnowledgeFileUpdateStateConflict - } - cancelled = &slot - return nil - }) - return cancelled, err -} - // BeginKnowledgeDeletion serializes deletion against file-update staging and // returns removed slots so staged files can be cleaned after the commit. func (r *knowledgeRepository) BeginKnowledgeDeletion( diff --git a/internal/application/repository/knowledge_file_update_slot_test.go b/internal/application/repository/knowledge_file_update_slot_test.go index cc186b4c9b..6ddb3d3807 100644 --- a/internal/application/repository/knowledge_file_update_slot_test.go +++ b/internal/application/repository/knowledge_file_update_slot_test.go @@ -267,34 +267,3 @@ func TestBeginKnowledgeDeletionRollsBackStatusWhenSlotDeleteFails(t *testing.T) _, err = repo.GetKnowledgeFileUpdateSlot(ctx, 1, knowledgeID) require.NoError(t, err) } - -func TestCancelFailedKnowledgeFileUpdateDoesNotRemoveNewerActive(t *testing.T) { - db, repo := setupKnowledgeFileUpdateSlotTestDB(t) - ctx := context.Background() - kbID := uuid.NewString() - knowledgeID := insertFileKnowledge( - t, db, 1, kbID, types.ParseStatusCompleted, "old/path.md", "old-hash", - ) - first, err := repo.StageKnowledgeFileUpdate( - ctx, 1, knowledgeID, kbID, updatePayload(t, "staged/a.md", "ha"), nil, - ) - require.NoError(t, err) - failed, err := repo.TransitionKnowledgeFileUpdateState( - ctx, 1, knowledgeID, first.ActiveVersion, - types.KnowledgeFileUpdateStateWaiting, types.KnowledgeFileUpdateStateFailed, "failed", - ) - require.NoError(t, err) - require.True(t, failed) - latest, err := repo.StageKnowledgeFileUpdate( - ctx, 1, knowledgeID, kbID, updatePayload(t, "staged/b.md", "hb"), nil, - ) - require.NoError(t, err) - - _, err = repo.CancelFailedKnowledgeFileUpdate(ctx, 1, knowledgeID, first.ActiveVersion) - require.ErrorIs(t, err, ErrKnowledgeFileUpdateStateConflict) - slot, err := repo.GetKnowledgeFileUpdateSlot(ctx, 1, knowledgeID) - require.NoError(t, err) - require.NotNil(t, slot.ActiveVersion) - assert.Equal(t, latest.ActiveVersion, *slot.ActiveVersion) - assert.Equal(t, "staged/b.md", decodeUpdatePayload(t, slot.ActivePayload).NewFilePath) -} diff --git a/internal/application/service/knowledge_file_update.go b/internal/application/service/knowledge_file_update.go index e8e1c0145f..f3c51438a3 100644 --- a/internal/application/service/knowledge_file_update.go +++ b/internal/application/service/knowledge_file_update.go @@ -667,124 +667,6 @@ func (s *knowledgeService) ProcessKnowledgeFileUpdate(ctx context.Context, task return err } -// RetryKnowledgeFileUpdate re-arms the retained failed active payload. The -// exact active version is guarded so a concurrent upload wins safely. -func (s *knowledgeService) RetryKnowledgeFileUpdate( - ctx context.Context, knowledgeID string, -) (*types.Knowledge, error) { - tenantID, ok := ctx.Value(types.TenantIDContextKey).(uint64) - if !ok || tenantID == 0 { - return nil, werrors.NewUnauthorizedError("tenant context is required") - } - knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID) - if err != nil { - return nil, err - } - if knowledge.ParseStatus == types.ParseStatusDeleting { - return nil, werrors.NewConflictError("knowledge is being deleted") - } - slot, err := s.repo.GetKnowledgeFileUpdateSlot(ctx, tenantID, knowledgeID) - if err != nil || slot.ActiveVersion == nil { - if stderrors.Is(err, gorm.ErrRecordNotFound) || (err == nil && slot.ActiveVersion == nil) { - return nil, werrors.NewConflictError("no failed file update is available") - } - return nil, err - } - if slot.ActiveState != types.KnowledgeFileUpdateStateFailed { - return nil, werrors.NewConflictError("file update is not failed") - } - version := *slot.ActiveVersion - moved, err := s.repo.TransitionKnowledgeFileUpdateState( - ctx, tenantID, knowledgeID, version, - types.KnowledgeFileUpdateStateFailed, types.KnowledgeFileUpdateStateWaiting, "", - ) - if err != nil { - return nil, err - } - if !moved { - return nil, werrors.NewConflictError("file update changed; refresh and retry") - } - if _, err := s.enqueueKnowledgeFileUpdate(ctx, types.KnowledgeFileUpdateTaskPayload{ - TenantID: tenantID, - KnowledgeBaseID: slot.KnowledgeBaseID, - KnowledgeID: knowledgeID, - ActiveVersion: version, - }, 0); err != nil { - _, _ = s.repo.TransitionKnowledgeFileUpdateState( - ctx, tenantID, knowledgeID, version, - types.KnowledgeFileUpdateStateWaiting, types.KnowledgeFileUpdateStateFailed, - "retry enqueue failed", - ) - return nil, werrors.NewServiceUnavailableError("file update retry is temporarily unavailable") - } - return s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID) -} - -// DiscardKnowledgeFileUpdate removes the exact failed active version and its -// pending successor, then restores a claimed old source to its prior terminal -// state when the file switch had not happened yet. -func (s *knowledgeService) DiscardKnowledgeFileUpdate( - ctx context.Context, knowledgeID string, -) (*types.Knowledge, error) { - tenantID, ok := ctx.Value(types.TenantIDContextKey).(uint64) - if !ok || tenantID == 0 { - return nil, werrors.NewUnauthorizedError("tenant context is required") - } - knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID) - if err != nil { - return nil, err - } - if knowledge.ParseStatus == types.ParseStatusDeleting { - return nil, werrors.NewConflictError("knowledge is being deleted") - } - slot, err := s.repo.GetKnowledgeFileUpdateSlot(ctx, tenantID, knowledgeID) - if err != nil || slot.ActiveVersion == nil { - if stderrors.Is(err, gorm.ErrRecordNotFound) || (err == nil && slot.ActiveVersion == nil) { - return nil, werrors.NewConflictError("no failed file update is available") - } - return nil, err - } - if slot.ActiveState != types.KnowledgeFileUpdateStateFailed { - return nil, werrors.NewConflictError("only a failed file update can be discarded") - } - cancelled, err := s.repo.CancelFailedKnowledgeFileUpdate( - ctx, tenantID, knowledgeID, *slot.ActiveVersion, - ) - if err != nil { - if stderrors.Is(err, repository.ErrKnowledgeFileUpdateStateConflict) { - return nil, werrors.NewConflictError("file update changed; refresh and retry") - } - return nil, err - } - - var active types.KnowledgeFileUpdatePayload - if json.Unmarshal(cancelled.ActivePayload, &active) == nil && - knowledge.ParseStatus == types.ParseStatusReplacing && active.OldFilePath != "" { - restoreStatus := active.OldParseStatus - if _, ok := replaceableKnowledgeStatuses[restoreStatus]; !ok { - restoreStatus = types.ParseStatusFailed - } - updated, updateErr := s.repo.UpdateApplyingKnowledgeFileColumns( - ctx, tenantID, knowledgeID, knowledge.KnowledgeBaseID, - active.OldFilePath, active.OldFileHash, - map[string]interface{}{ - "parse_status": restoreStatus, - "error_message": "", - "updated_at": time.Now(), - }, - ) - if updateErr != nil { - return nil, updateErr - } - if updated { - knowledge.ParseStatus = restoreStatus - } - } - kb, _ := s.kbService.GetKnowledgeBaseByID(ctx, knowledge.KnowledgeBaseID) - s.cleanupCancelledKnowledgeFileUpdate(ctx, kb, knowledge.FilePath, cancelled) - return s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID) -} - func (s *knowledgeService) enqueueKnowledgeFileUpdate( ctx context.Context, wake types.KnowledgeFileUpdateTaskPayload, delay time.Duration, ) (string, error) { diff --git a/internal/application/service/knowledge_file_update_test.go b/internal/application/service/knowledge_file_update_test.go index 1e8446187b..5e1898d909 100644 --- a/internal/application/service/knowledge_file_update_test.go +++ b/internal/application/service/knowledge_file_update_test.go @@ -6,7 +6,6 @@ import ( "errors" "testing" - "github.com/Tencent/WeKnora/internal/application/repository" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" "github.com/hibiken/asynq" @@ -42,18 +41,6 @@ func (r *fileUpdateRepoStub) TransitionKnowledgeFileUpdateState( return true, nil } -func (r *fileUpdateRepoStub) CancelFailedKnowledgeFileUpdate( - _ context.Context, _ uint64, _ string, version uint64, -) (*types.KnowledgeFileUpdateSlot, error) { - if r.slot == nil || r.slot.ActiveVersion == nil || *r.slot.ActiveVersion != version || - r.slot.ActiveState != types.KnowledgeFileUpdateStateFailed { - return nil, repository.ErrKnowledgeFileUpdateStateConflict - } - cancelled := r.slot - r.slot = nil - return cancelled, nil -} - func (r *fileUpdateRepoStub) GetKnowledgeFileUpdateSlot( context.Context, uint64, string, ) (*types.KnowledgeFileUpdateSlot, error) { @@ -558,56 +545,6 @@ func TestSameAsCurrentKnowledgeFileComparesExplicitConfig(t *testing.T) { } } -func TestRetryKnowledgeFileUpdateRearmsExactFailedVersion(t *testing.T) { - version := uint64(7) - repo := &fileUpdateRepoStub{ - knowledge: &types.Knowledge{ - ID: "knowledge-1", TenantID: 1, KnowledgeBaseID: "kb-1", Type: "file", - FilePath: "old/path.md", ParseStatus: types.ParseStatusCompleted, - }, - slot: &types.KnowledgeFileUpdateSlot{ - KnowledgeID: "knowledge-1", TenantID: 1, KnowledgeBaseID: "kb-1", - ActiveVersion: &version, ActiveState: types.KnowledgeFileUpdateStateFailed, - }, - } - task := &fileUpdateTaskStub{} - svc := newFileUpdateService(repo, &createKnowledgeFileServiceStub{}, task) - - _, err := svc.RetryKnowledgeFileUpdate(newCreateKnowledgeFileContext(), "knowledge-1") - require.NoError(t, err) - assert.Equal(t, types.KnowledgeFileUpdateStateWaiting, repo.slot.ActiveState) - require.Len(t, task.tasks, 1) - var wake types.KnowledgeFileUpdateTaskPayload - require.NoError(t, json.Unmarshal(task.tasks[0].Payload(), &wake)) - assert.Equal(t, version, wake.ActiveVersion) -} - -func TestDiscardKnowledgeFileUpdateCleansFailedActiveAndPending(t *testing.T) { - version, pendingVersion := uint64(7), uint64(8) - active, err := json.Marshal(types.KnowledgeFileUpdatePayload{NewFilePath: "staged/a.md"}) - require.NoError(t, err) - pending, err := json.Marshal(types.KnowledgeFileUpdatePayload{NewFilePath: "staged/b.md"}) - require.NoError(t, err) - repo := &fileUpdateRepoStub{ - knowledge: &types.Knowledge{ - ID: "knowledge-1", TenantID: 1, KnowledgeBaseID: "kb-1", Type: "file", - FilePath: "old/path.md", ParseStatus: types.ParseStatusCompleted, - }, - slot: &types.KnowledgeFileUpdateSlot{ - KnowledgeID: "knowledge-1", TenantID: 1, KnowledgeBaseID: "kb-1", - ActiveVersion: &version, ActiveState: types.KnowledgeFileUpdateStateFailed, - ActivePayload: types.JSON(active), PendingVersion: &pendingVersion, PendingPayload: types.JSON(pending), - }, - } - fileSvc := &createKnowledgeFileServiceStub{} - svc := newFileUpdateService(repo, fileSvc, &fileUpdateTaskStub{}) - - _, err = svc.DiscardKnowledgeFileUpdate(newCreateKnowledgeFileContext(), "knowledge-1") - require.NoError(t, err) - assert.Nil(t, repo.slot) - assert.Equal(t, 2, fileSvc.deleteCalls) -} - func TestRestoreFailedKnowledgeFileUpdateClaimRestoresReplacingStatus(t *testing.T) { version := uint64(7) active, err := json.Marshal(types.KnowledgeFileUpdatePayload{ diff --git a/internal/handler/knowledge.go b/internal/handler/knowledge.go index 6cbdd5a610..3e4861bf47 100644 --- a/internal/handler/knowledge.go +++ b/internal/handler/knowledge.go @@ -2216,77 +2216,6 @@ func (h *KnowledgeHandler) CancelKnowledgeParse(c *gin.Context) { }) } -// RetryKnowledgeFileUpdate godoc -// @Summary 重试失败的文件更新 -// @Description 重新唤醒 update slot 中保留的失败 active 版本；并发上传的新版本不会被覆盖 -// @Tags 知识管理 -// @Produce json -// @Param id path string true "知识ID" -// @Success 200 {object} map[string]interface{} "重试已提交" -// @Failure 409 {object} errors.AppError "没有失败更新或状态已变化" -// @Failure 503 {object} errors.AppError "任务系统暂时不可用" -// @Security Bearer -// @Security ApiKeyAuth -// @Router /knowledge/{id}/file-update/retry [post] -func (h *KnowledgeHandler) RetryKnowledgeFileUpdate(c *gin.Context) { - id := secutils.SanitizeForLog(c.Param("id")) - if id == "" { - c.Error(errors.NewBadRequestError("Knowledge ID cannot be empty")) - return - } - _, effCtx, err := h.resolveKnowledgeAndValidateKBAccess(c, id, types.OrgRoleEditor) - if err != nil { - c.Error(err) - return - } - knowledge, err := h.kgService.RetryKnowledgeFileUpdate(effCtx, id) - if err != nil { - if appErr, ok := errors.IsAppError(err); ok { - c.Error(appErr) - return - } - logger.ErrorWithFields(c.Request.Context(), err, map[string]interface{}{"knowledge_id": id}) - c.Error(errors.NewInternalServerError(err.Error())) - return - } - c.JSON(http.StatusOK, gin.H{"success": true, "data": knowledge}) -} - -// DiscardKnowledgeFileUpdate godoc -// @Summary 丢弃失败的文件更新 -// @Description 删除精确匹配的失败 active 和最新 pending 暂存版本，不影响并发提交的新版本 -// @Tags 知识管理 -// @Produce json -// @Param id path string true "知识ID" -// @Success 200 {object} map[string]interface{} "待更新版本已丢弃" -// @Failure 409 {object} errors.AppError "没有失败更新或状态已变化" -// @Security Bearer -// @Security ApiKeyAuth -// @Router /knowledge/{id}/file-update [delete] -func (h *KnowledgeHandler) DiscardKnowledgeFileUpdate(c *gin.Context) { - id := secutils.SanitizeForLog(c.Param("id")) - if id == "" { - c.Error(errors.NewBadRequestError("Knowledge ID cannot be empty")) - return - } - _, effCtx, err := h.resolveKnowledgeAndValidateKBAccess(c, id, types.OrgRoleEditor) - if err != nil { - c.Error(err) - return - } - knowledge, err := h.kgService.DiscardKnowledgeFileUpdate(effCtx, id) - if err != nil { - if appErr, ok := errors.IsAppError(err); ok { - c.Error(appErr) - return - } - logger.ErrorWithFields(c.Request.Context(), err, map[string]interface{}{"knowledge_id": id}) - c.Error(errors.NewInternalServerError(err.Error())) - return - } - c.JSON(http.StatusOK, gin.H{"success": true, "data": knowledge}) -} - type knowledgeTagBatchRequest struct { Updates map[string][]string `json:"updates" binding:"required,min=1"` KBID string `json:"kb_id"` // Optional: scope to this KB (validates editor access and uses effective tenant for shared KB) diff --git a/internal/router/routes_knowledge.go b/internal/router/routes_knowledge.go index 6c7016f42f..2a8b95cced 100644 --- a/internal/router/routes_knowledge.go +++ b/internal/router/routes_knowledge.go @@ -110,8 +110,6 @@ func RegisterKnowledgeRoutes(r *gin.RouterGroup, handler *handler.KnowledgeHandl k.PUT("/manual/:id", g.OwnedKnowledgeKBOrAdmin(), g.KBAccessWriteFromKnowledgeIDParam("id"), handler.UpdateManualKnowledge) k.POST("/:id/reparse", g.OwnedKnowledgeKBOrAdmin(), g.KBAccessWriteFromKnowledgeIDParam("id"), handler.ReparseKnowledge) k.POST("/:id/cancel-parse", g.OwnedKnowledgeKBOrAdmin(), g.KBAccessWriteFromKnowledgeIDParam("id"), handler.CancelKnowledgeParse) - k.POST("/:id/file-update/retry", g.OwnedKnowledgeKBOrAdmin(), g.KBAccessWriteFromKnowledgeIDParam("id"), handler.RetryKnowledgeFileUpdate) - k.DELETE("/:id/file-update", g.OwnedKnowledgeKBOrAdmin(), g.KBAccessWriteFromKnowledgeIDParam("id"), handler.DiscardKnowledgeFileUpdate) // Downloading exposes the original source file, so it has a stricter // boundary than viewing parsed content or previewing it: tenant Viewers // cannot download from their own workspace, and org-shared Viewer access diff --git a/internal/types/interfaces/knowledge.go b/internal/types/interfaces/knowledge.go index ca72d2f883..824a81808e 100644 --- a/internal/types/interfaces/knowledge.go +++ b/internal/types/interfaces/knowledge.go @@ -37,10 +37,6 @@ type KnowledgeService interface { ctx context.Context, req *types.KnowledgeFileUpdateRequest, ) (*types.KnowledgeFileUpsertResult, error) - // RetryKnowledgeFileUpdate re-arms the retained failed active version. - RetryKnowledgeFileUpdate(ctx context.Context, knowledgeID string) (*types.Knowledge, error) - // DiscardKnowledgeFileUpdate removes a failed active version and its latest pending version. - DiscardKnowledgeFileUpdate(ctx context.Context, knowledgeID string) (*types.Knowledge, error) // CreateKnowledgeFromURL creates knowledge from a URL. // When fileName or fileType is provided (or the URL path has a known file extension), // the URL is treated as a direct file download instead of a web page crawl. @@ -368,9 +364,6 @@ type KnowledgeRepository interface { CancelKnowledgeFileUpdates( ctx context.Context, tenantID uint64, knowledgeID string, ) (*types.KnowledgeFileUpdateSlot, error) - CancelFailedKnowledgeFileUpdate( - ctx context.Context, tenantID uint64, knowledgeID string, activeVersion uint64, - ) (*types.KnowledgeFileUpdateSlot, error) // BeginKnowledgeDeletion atomically marks active knowledge rows as deleting // and removes their file-update coordination slots. BeginKnowledgeDeletion( From 61a373b5691b00bdf8778f471a78245976eaec18 Mon Sep 17 00:00:00 2001 From: hanxiantao <601803023@qq.com> Date: Sat, 22 Aug 2026 08:57:44 +0800 Subject: [PATCH 6/7] fix(migrations): renumber knowledge file update slots --- internal/database/knowledge_file_update_migration_test.go | 4 +++- migrations/sqlite/000000_init.up.sql | 2 +- ...s.down.sql => 000012_knowledge_file_update_slots.down.sql} | 0 ...slots.up.sql => 000012_knowledge_file_update_slots.up.sql} | 0 ...s.down.sql => 000085_knowledge_file_update_slots.down.sql} | 0 ...slots.up.sql => 000085_knowledge_file_update_slots.up.sql} | 4 ++-- 6 files changed, 6 insertions(+), 4 deletions(-) rename migrations/sqlite/{000004_knowledge_file_update_slots.down.sql => 000012_knowledge_file_update_slots.down.sql} (100%) rename migrations/sqlite/{000004_knowledge_file_update_slots.up.sql => 000012_knowledge_file_update_slots.up.sql} (100%) rename migrations/versioned/{000084_knowledge_file_update_slots.down.sql => 000085_knowledge_file_update_slots.down.sql} (100%) rename migrations/versioned/{000084_knowledge_file_update_slots.up.sql => 000085_knowledge_file_update_slots.up.sql} (92%) diff --git a/internal/database/knowledge_file_update_migration_test.go b/internal/database/knowledge_file_update_migration_test.go index 2b15fdade9..1cb9cd2a4b 100644 --- a/internal/database/knowledge_file_update_migration_test.go +++ b/internal/database/knowledge_file_update_migration_test.go @@ -24,6 +24,8 @@ func TestKnowledgeFileUpdateSlotMigratesExistingSQLite(t *testing.T) { _, err = db.Exec(` CREATE TABLE knowledge_bases (id TEXT PRIMARY KEY); + CREATE TABLE tenants (id INTEGER PRIMARY KEY); + CREATE TABLE messages (id TEXT PRIMARY KEY); CREATE TABLE schema_migrations (version INTEGER NOT NULL PRIMARY KEY, dirty BOOLEAN NOT NULL); INSERT INTO schema_migrations(version, dirty) VALUES (2, 0); `) @@ -45,7 +47,7 @@ func TestKnowledgeFileUpdateSlotMigratesExistingSQLite(t *testing.T) { require.NoError(t, migrator.Up()) version, dirty, err := migrator.Version() require.NoError(t, err) - require.Equal(t, uint(4), version) + require.Equal(t, uint(12), version) require.False(t, dirty) var tableName string diff --git a/migrations/sqlite/000000_init.up.sql b/migrations/sqlite/000000_init.up.sql index dea3f10394..0936c36ac8 100644 --- a/migrations/sqlite/000000_init.up.sql +++ b/migrations/sqlite/000000_init.up.sql @@ -1121,7 +1121,7 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_wiki_page_revisions_page_version CREATE INDEX IF NOT EXISTS idx_wiki_page_revisions_kb_slug ON wiki_page_revisions (knowledge_base_id, slug); --- File knowledge update coordination slot. Migration 000004 repeats this +-- File knowledge update coordination slot. Migration 000012 repeats this -- idempotently so databases already initialized at version 0 are upgraded. CREATE TABLE IF NOT EXISTS knowledge_file_update_slots ( knowledge_id VARCHAR(36) PRIMARY KEY, diff --git a/migrations/sqlite/000004_knowledge_file_update_slots.down.sql b/migrations/sqlite/000012_knowledge_file_update_slots.down.sql similarity index 100% rename from migrations/sqlite/000004_knowledge_file_update_slots.down.sql rename to migrations/sqlite/000012_knowledge_file_update_slots.down.sql diff --git a/migrations/sqlite/000004_knowledge_file_update_slots.up.sql b/migrations/sqlite/000012_knowledge_file_update_slots.up.sql similarity index 100% rename from migrations/sqlite/000004_knowledge_file_update_slots.up.sql rename to migrations/sqlite/000012_knowledge_file_update_slots.up.sql diff --git a/migrations/versioned/000084_knowledge_file_update_slots.down.sql b/migrations/versioned/000085_knowledge_file_update_slots.down.sql similarity index 100% rename from migrations/versioned/000084_knowledge_file_update_slots.down.sql rename to migrations/versioned/000085_knowledge_file_update_slots.down.sql diff --git a/migrations/versioned/000084_knowledge_file_update_slots.up.sql b/migrations/versioned/000085_knowledge_file_update_slots.up.sql similarity index 92% rename from migrations/versioned/000084_knowledge_file_update_slots.up.sql rename to migrations/versioned/000085_knowledge_file_update_slots.up.sql index 4efd6b70f4..7181a1eb62 100644 --- a/migrations/versioned/000084_knowledge_file_update_slots.up.sql +++ b/migrations/versioned/000085_knowledge_file_update_slots.up.sql @@ -1,4 +1,4 @@ -DO $$ BEGIN RAISE NOTICE '[Migration 000084] Creating knowledge file update slots...'; END $$; +DO $$ BEGIN RAISE NOTICE '[Migration 000085] Creating knowledge file update slots...'; END $$; CREATE TABLE IF NOT EXISTS knowledge_file_update_slots ( knowledge_id VARCHAR(36) PRIMARY KEY, @@ -30,4 +30,4 @@ CREATE INDEX IF NOT EXISTS idx_knowledge_file_update_slots_tenant_kb CREATE INDEX IF NOT EXISTS idx_knowledge_file_update_slots_state ON knowledge_file_update_slots(active_state, updated_at); -DO $$ BEGIN RAISE NOTICE '[Migration 000084] Knowledge file update slots ready'; END $$; +DO $$ BEGIN RAISE NOTICE '[Migration 000085] Knowledge file update slots ready'; END $$; From fc8e52bd4d26071ee31ae6728cfd623a4a32a7d3 Mon Sep 17 00:00:00 2001 From: hanxiantao <601803023@qq.com> Date: Sat, 22 Aug 2026 09:05:52 +0800 Subject: [PATCH 7/7] fix(sqlite): remove update slots from init migration --- migrations/sqlite/000000_init.down.sql | 1 - migrations/sqlite/000000_init.up.sql | 26 -------------------------- 2 files changed, 27 deletions(-) diff --git a/migrations/sqlite/000000_init.down.sql b/migrations/sqlite/000000_init.down.sql index 4e01d1b5d9..2c81c06eca 100644 --- a/migrations/sqlite/000000_init.down.sql +++ b/migrations/sqlite/000000_init.down.sql @@ -24,7 +24,6 @@ DROP TABLE IF EXISTS chunks; DROP TABLE IF EXISTS messages; DROP TABLE IF EXISTS temporary_documents; DROP TABLE IF EXISTS sessions; -DROP TABLE IF EXISTS knowledge_file_update_slots; DROP TABLE IF EXISTS knowledges; DROP TABLE IF EXISTS knowledge_bases; DROP TABLE IF EXISTS storage_backends; diff --git a/migrations/sqlite/000000_init.up.sql b/migrations/sqlite/000000_init.up.sql index 0936c36ac8..635cce402b 100644 --- a/migrations/sqlite/000000_init.up.sql +++ b/migrations/sqlite/000000_init.up.sql @@ -1120,29 +1120,3 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_wiki_page_revisions_page_version CREATE INDEX IF NOT EXISTS idx_wiki_page_revisions_kb_slug ON wiki_page_revisions (knowledge_base_id, slug); - --- File knowledge update coordination slot. Migration 000012 repeats this --- idempotently so databases already initialized at version 0 are upgraded. -CREATE TABLE IF NOT EXISTS knowledge_file_update_slots ( - knowledge_id VARCHAR(36) PRIMARY KEY, - tenant_id INTEGER NOT NULL, - knowledge_base_id VARCHAR(36) NOT NULL, - latest_version INTEGER NOT NULL DEFAULT 0, - active_version INTEGER, - active_state VARCHAR(16) NOT NULL DEFAULT 'idle', - active_payload TEXT, - pending_version INTEGER, - pending_payload TEXT, - last_error TEXT NOT NULL DEFAULT '', - created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - CHECK (active_state IN ('idle', 'waiting', 'applying', 'retry_wait', 'failed')), - CHECK ((active_state = 'idle') = (active_version IS NULL)), - CHECK ((active_version IS NULL) = (active_payload IS NULL)), - CHECK ((pending_version IS NULL) = (pending_payload IS NULL)) -); - -CREATE INDEX IF NOT EXISTS idx_knowledge_file_update_slots_tenant_kb - ON knowledge_file_update_slots(tenant_id, knowledge_base_id); -CREATE INDEX IF NOT EXISTS idx_knowledge_file_update_slots_state - ON knowledge_file_update_slots(active_state, updated_at);