From 1b1c60269309821e17d45a4d0519244aa7f85524 Mon Sep 17 00:00:00 2001 From: Jones <3049453512@qq.com> Date: Fri, 21 Aug 2026 03:28:03 +0800 Subject: [PATCH] fix(knowledge): use current parser rules for batch reparse --- .../service/knowledge_batch_reparse_test.go | 135 ++++++++++++++++++ .../application/service/knowledge_process.go | 43 ++++++ 2 files changed, 178 insertions(+) diff --git a/internal/application/service/knowledge_batch_reparse_test.go b/internal/application/service/knowledge_batch_reparse_test.go index 5babdab620..d3d8490b35 100644 --- a/internal/application/service/knowledge_batch_reparse_test.go +++ b/internal/application/service/knowledge_batch_reparse_test.go @@ -47,6 +47,40 @@ type reparseFailureKBService struct { kb *types.KnowledgeBase } +type parserRulesKnowledgeRepo struct { + interfaces.KnowledgeRepository + knowledge *types.Knowledge + getErr error + updateErr error + requestedTenant uint64 + updateCalls int + updatedID string + updatedColumn string + updatedValue interface{} +} + +func (r *parserRulesKnowledgeRepo) GetKnowledgeByID( + _ context.Context, + tenantID uint64, + _ string, +) (*types.Knowledge, error) { + r.requestedTenant = tenantID + return r.knowledge, r.getErr +} + +func (r *parserRulesKnowledgeRepo) UpdateKnowledgeColumn( + _ context.Context, + id string, + column string, + value interface{}, +) error { + r.updateCalls++ + r.updatedID = id + r.updatedColumn = column + r.updatedValue = value + return r.updateErr +} + func (s *reparseFailureKBService) GetKnowledgeBaseByID( _ context.Context, _ string, @@ -134,3 +168,104 @@ func TestRunKnowledgeListReparseSubmissionsSucceeds(t *testing.T) { require.NoError(t, err) require.Equal(t, knowledgeListReparseOutcome{Submitted: 2}, outcome) } + +func TestClearStoredParserEngineRulesUsesCurrentKnowledgeBaseRules(t *testing.T) { + enableMultimodel := true + knowledge := &types.Knowledge{ + ID: "knowledge-1", + TenantID: 7, + Metadata: types.JSON(`{ + "source_id":"keep-me", + "process_overrides":{ + "parser_engine_rules":[{"file_types":["pdf"],"engine":"old-top-level"}], + "chunking_config":{ + "chunk_size":1024, + "chunk_overlap":128, + "enable_parent_child":true, + "parser_engine_rules":[{"file_types":["docx"],"engine":"old-nested"}] + }, + "enable_multimodel":true, + "parser_engine_overrides":{"pdf_force_scanned":"true"} + } + }`), + } + repo := &parserRulesKnowledgeRepo{knowledge: knowledge} + svc := &knowledgeService{repo: repo} + ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(7)) + + err := svc.clearStoredParserEngineRules(ctx, knowledge.ID) + + require.NoError(t, err) + require.Equal(t, uint64(7), repo.requestedTenant) + require.Equal(t, 1, repo.updateCalls) + require.Equal(t, knowledge.ID, repo.updatedID) + require.Equal(t, "metadata", repo.updatedColumn) + require.Equal(t, knowledge.Metadata, repo.updatedValue) + + overrides, err := knowledge.ProcessOverrides() + require.NoError(t, err) + require.NotNil(t, overrides) + require.Empty(t, overrides.ParserEngineRules) + require.NotNil(t, overrides.ChunkingConfig) + require.Empty(t, overrides.ChunkingConfig.ParserEngineRules) + require.Equal(t, 1024, overrides.ChunkingConfig.ChunkSize) + require.Equal(t, 128, overrides.ChunkingConfig.ChunkOverlap) + require.True(t, overrides.ChunkingConfig.EnableParentChild) + require.Equal(t, &enableMultimodel, overrides.EnableMultimodel) + require.Equal(t, map[string]string{"pdf_force_scanned": "true"}, overrides.ParserEngineOverrides) + + metadata, err := knowledge.Metadata.Map() + require.NoError(t, err) + require.Equal(t, "keep-me", metadata["source_id"]) + + currentRules := []types.ParserEngineRule{{FileTypes: []string{"pdf", "docx"}, Engine: "current"}} + effective := ResolveProcessConfig(&types.KnowledgeBase{ + ChunkingConfig: types.ChunkingConfig{ParserEngineRules: currentRules}, + }, overrides) + require.Equal(t, currentRules, effective.ChunkingConfig.ParserEngineRules) +} + +func TestClearStoredParserEngineRulesWithoutSnapshotsIsNoOp(t *testing.T) { + knowledge := &types.Knowledge{ + ID: "knowledge-2", + TenantID: 7, + Metadata: types.JSON(`{ + "source_id":"keep-me", + "process_overrides":{ + "chunking_config":{"chunk_size":2048}, + "parser_engine_overrides":{"xlsx_first_row_as_header":"true"} + } + }`), + } + originalMetadata := append(types.JSON(nil), knowledge.Metadata...) + repo := &parserRulesKnowledgeRepo{knowledge: knowledge} + svc := &knowledgeService{repo: repo} + ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(7)) + + err := svc.clearStoredParserEngineRules(ctx, knowledge.ID) + + require.NoError(t, err) + require.Zero(t, repo.updateCalls) + require.Equal(t, originalMetadata, knowledge.Metadata) +} + +func TestClearStoredParserEngineRulesPropagatesUpdateFailure(t *testing.T) { + updateErr := errors.New("metadata update failed") + knowledge := &types.Knowledge{ + ID: "knowledge-3", + TenantID: 7, + Metadata: types.JSON(`{ + "process_overrides":{ + "parser_engine_rules":[{"file_types":["pdf"],"engine":"old"}] + } + }`), + } + repo := &parserRulesKnowledgeRepo{knowledge: knowledge, updateErr: updateErr} + svc := &knowledgeService{repo: repo} + ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(7)) + + err := svc.clearStoredParserEngineRules(ctx, knowledge.ID) + + require.ErrorIs(t, err, updateErr) + require.Equal(t, 1, repo.updateCalls) +} diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go index 6c2f7350b5..ee6b2290fb 100644 --- a/internal/application/service/knowledge_process.go +++ b/internal/application/service/knowledge_process.go @@ -3870,6 +3870,44 @@ func (s *knowledgeService) enqueueImageMultimodalTasks( } } +// clearStoredParserEngineRules drops only the upload-time parser selection from +// a knowledge's process overrides. Batch reparse can then resolve parser rules +// from the current knowledge-base config while retaining document-specific +// chunking and parser options. +func (s *knowledgeService) clearStoredParserEngineRules(ctx context.Context, knowledgeID string) error { + tenantID := ctx.Value(types.TenantIDContextKey).(uint64) + knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID) + if err != nil { + return err + } + if knowledge == nil { + return fmt.Errorf("knowledge %s not found", secutils.SanitizeForLog(knowledgeID)) + } + + overrides, err := knowledge.ProcessOverrides() + if err != nil { + return err + } + if overrides == nil { + return nil + } + + changed := len(overrides.ParserEngineRules) > 0 + overrides.ParserEngineRules = nil + if overrides.ChunkingConfig != nil && len(overrides.ChunkingConfig.ParserEngineRules) > 0 { + overrides.ChunkingConfig.ParserEngineRules = nil + changed = true + } + if !changed { + return nil + } + + if err := knowledge.SetProcessOverrides(overrides); err != nil { + return err + } + return s.repo.UpdateKnowledgeColumn(ctx, knowledge.ID, "metadata", knowledge.Metadata) +} + // ProcessKnowledgeListReparse handles Asynq knowledge list reparse tasks. func (s *knowledgeService) ProcessKnowledgeListReparse(ctx context.Context, t *asynq.Task) error { var payload types.KnowledgeListReparsePayload @@ -3893,6 +3931,11 @@ func (s *knowledgeService) ProcessKnowledgeListReparse(ctx context.Context, t *a ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant) outcome, err := runKnowledgeListReparseSubmissions(payload.KnowledgeIDs, func(id string) error { + if payload.ProcessConfig == nil { + if err := s.clearStoredParserEngineRules(ctx, id); err != nil { + return err + } + } _, err := s.ReparseKnowledge(ctx, id, payload.ProcessConfig) return err })