From e59dc9c61749168ec0e2ec5028fc582fd428ac9c Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Tue, 31 Mar 2026 11:19:06 +0200 Subject: [PATCH 01/46] form flow for building blocks --- .../review-subsidy-calculation.form-flow.json | 16 ++ ...bsidy-calculator-process.process-link.json | 4 +- backend/building-block/build.gradle | 1 + .../BuildingBlockAutoConfiguration.kt | 54 +++- .../BuildingBlockHttpSecurityConfigurer.kt | 11 + ...BuildingBlockFormFlowDefinitionExporter.kt | 69 ++++++ ...BuildingBlockFormFlowDefinitionImporter.kt | 125 ++++++++++ .../BuildingBlockFormFlowDefinitionService.kt | 71 ++++++ ...BuildingBlockFormFlowManagementResource.kt | 123 +++++++++ .../service/BuildingBlockFormFlowIT.kt | 212 ++++++++++++++++ ...dingBlockFormFlowDefinitionExporterTest.kt | 158 ++++++++++++ ...dingBlockFormFlowDefinitionImporterTest.kt | 92 +++++++ ...ldingBlockFormFlowDefinitionServiceTest.kt | 104 ++++++++ ...ildingBlockFormFlowManagementResourceIT.kt | 234 ++++++++++++++++++ .../building-block-form-flow-process.bpmn | 39 +++ .../bezwaar/1-0-0/form/bb-form.form.json | 11 + .../bpmn/building-block-form-flow-main.bpmn | 42 ++++ .../liquibase/13-22-0/13-22-0-master.xml | 2 + .../20260330-form-flow-blueprint-support.xml | 145 +++++++++++ ...ingBlockFormFlowDefinitionExportRequest.kt | 25 ++ .../FormFlowProcessLinkActivityHandler.kt | 22 +- .../FormFlowAutoConfiguration.kt | 5 +- .../formflow/common/ValtimoFormFlow.kt | 4 +- .../FormFlowDefinitionBlueprintId.kt | 72 ++++++ .../domain/definition/FormFlowDefinitionId.kt | 24 +- .../domain/definition/FormFlowStepId.kt | 5 +- .../domain/instance/FormFlowInstance.kt | 5 +- .../handler/FormFlowStepTypeFormHandler.kt | 6 +- .../importer/FormFlowDefinitionImporter.kt | 12 +- .../mapper/FormFlowProcessLinkMapper.kt | 25 +- .../FormFlowDefinitionRepository.kt | 21 +- .../formflow/service/FormFlowService.kt | 74 +++++- .../service/FormFlowValtimoService.kt | 8 +- .../web/rest/FormFlowManagementResource.kt | 2 +- .../web/rest/result/FormFlowDefinitionDto.kt | 14 +- .../formflow/domain/FormFlowInstanceIT.kt | 12 +- .../formflow/domain/FormFlowInstanceTest.kt | 6 +- .../FormFlowDefinitionBlueprintIdTest.kt | 132 ++++++++++ .../formflow/service/FormFlowServiceTest.kt | 6 +- .../rest/FormFlowManagementResourceIntTest.kt | 4 +- .../ritense/importer/ValtimoImportTypes.kt | 1 + backend/process-link/build.gradle | 1 + .../service/ProcessLinkServiceIntTest.kt | 2 + 43 files changed, 1928 insertions(+), 73 deletions(-) create mode 100644 backend/app/gzac/src/dev/resources/config/building-block/subsidy-calculator/1-0-0/form-flow/review-subsidy-calculation.form-flow.json create mode 100644 backend/building-block/src/main/kotlin/com/ritense/buildingblock/service/BuildingBlockFormFlowDefinitionExporter.kt create mode 100644 backend/building-block/src/main/kotlin/com/ritense/buildingblock/service/BuildingBlockFormFlowDefinitionImporter.kt create mode 100644 backend/building-block/src/main/kotlin/com/ritense/buildingblock/service/BuildingBlockFormFlowDefinitionService.kt create mode 100644 backend/building-block/src/main/kotlin/com/ritense/buildingblock/web/rest/BuildingBlockFormFlowManagementResource.kt create mode 100644 backend/building-block/src/test/kotlin/com/ritense/buildingblock/processlink/service/BuildingBlockFormFlowIT.kt create mode 100644 backend/building-block/src/test/kotlin/com/ritense/buildingblock/service/BuildingBlockFormFlowDefinitionExporterTest.kt create mode 100644 backend/building-block/src/test/kotlin/com/ritense/buildingblock/service/BuildingBlockFormFlowDefinitionImporterTest.kt create mode 100644 backend/building-block/src/test/kotlin/com/ritense/buildingblock/service/BuildingBlockFormFlowDefinitionServiceTest.kt create mode 100644 backend/building-block/src/test/kotlin/com/ritense/buildingblock/web/rest/BuildingBlockFormFlowManagementResourceIT.kt create mode 100644 backend/building-block/src/test/resources/config/building-block/bezwaar/1-0-0/bpmn/building-block-form-flow-process.bpmn create mode 100644 backend/building-block/src/test/resources/config/building-block/bezwaar/1-0-0/form/bb-form.form.json create mode 100644 backend/building-block/src/test/resources/config/case/bb-case/1-0-0/bpmn/building-block-form-flow-main.bpmn create mode 100644 backend/core/src/main/resources/config/liquibase/13-22-0/20260330-form-flow-blueprint-support.xml create mode 100644 backend/exporter/src/main/kotlin/com/ritense/exporter/request/BuildingBlockFormFlowDefinitionExportRequest.kt create mode 100644 backend/form-flow/src/main/kotlin/com/ritense/formflow/domain/definition/FormFlowDefinitionBlueprintId.kt create mode 100644 backend/form-flow/src/test/kotlin/com/ritense/formflow/domain/definition/FormFlowDefinitionBlueprintIdTest.kt diff --git a/backend/app/gzac/src/dev/resources/config/building-block/subsidy-calculator/1-0-0/form-flow/review-subsidy-calculation.form-flow.json b/backend/app/gzac/src/dev/resources/config/building-block/subsidy-calculator/1-0-0/form-flow/review-subsidy-calculation.form-flow.json new file mode 100644 index 0000000000..d0d34c241e --- /dev/null +++ b/backend/app/gzac/src/dev/resources/config/building-block/subsidy-calculator/1-0-0/form-flow/review-subsidy-calculation.form-flow.json @@ -0,0 +1,16 @@ +{ + "startStep": "step1", + "steps": [ + { + "key": "step1", + "nextSteps": [], + "onComplete": ["${valtimoFormFlow.completeTask(additionalProperties, step.submissionData)}"], + "type": { + "name": "form", + "properties": { + "definition": "review-subsidy-calculation" + } + } + } + ] +} \ No newline at end of file diff --git a/backend/app/gzac/src/dev/resources/config/building-block/subsidy-calculator/1-0-0/process-link/subsidy-calculator-process.process-link.json b/backend/app/gzac/src/dev/resources/config/building-block/subsidy-calculator/1-0-0/process-link/subsidy-calculator-process.process-link.json index 8edfec4aeb..981917b625 100644 --- a/backend/app/gzac/src/dev/resources/config/building-block/subsidy-calculator/1-0-0/process-link/subsidy-calculator-process.process-link.json +++ b/backend/app/gzac/src/dev/resources/config/building-block/subsidy-calculator/1-0-0/process-link/subsidy-calculator-process.process-link.json @@ -2,8 +2,8 @@ { "activityId": "ReviewCalculationTask", "activityType": "bpmn:UserTask:create", - "processLinkType": "form", - "formDefinitionName": "review-subsidy-calculation" + "processLinkType": "form-flow", + "formFlowDefinitionKey": "review-subsidy-calculation" }, { "activityId": "PatchZaakCalculationStartTask", diff --git a/backend/building-block/build.gradle b/backend/building-block/build.gradle index d69bb818fa..4ed57503e8 100644 --- a/backend/building-block/build.gradle +++ b/backend/building-block/build.gradle @@ -53,6 +53,7 @@ dependencies { implementation project(":backend:process-document") implementation project(":backend:case") implementation project(":backend:form") + implementation project(":backend:form-flow") implementation project(":backend:process-link") implementation project(":backend:plugin") implementation project(":backend:plugin-valtimo") diff --git a/backend/building-block/src/main/kotlin/com/ritense/buildingblock/configuration/BuildingBlockAutoConfiguration.kt b/backend/building-block/src/main/kotlin/com/ritense/buildingblock/configuration/BuildingBlockAutoConfiguration.kt index 4007e2ebb0..bad8d2e7e4 100644 --- a/backend/building-block/src/main/kotlin/com/ritense/buildingblock/configuration/BuildingBlockAutoConfiguration.kt +++ b/backend/building-block/src/main/kotlin/com/ritense/buildingblock/configuration/BuildingBlockAutoConfiguration.kt @@ -47,6 +47,9 @@ import com.ritense.buildingblock.service.BuildingBlockFieldService import com.ritense.buildingblock.service.BuildingBlockFormDefinitionExporter import com.ritense.buildingblock.service.BuildingBlockFormDefinitionImporter import com.ritense.buildingblock.service.BuildingBlockFormDefinitionService +import com.ritense.buildingblock.service.BuildingBlockFormFlowDefinitionExporter +import com.ritense.buildingblock.service.BuildingBlockFormFlowDefinitionImporter +import com.ritense.buildingblock.service.BuildingBlockFormFlowDefinitionService import com.ritense.buildingblock.service.BuildingBlockInstanceService import com.ritense.buildingblock.service.BuildingBlockJsonSchemaDocumentDefinitionExporter import com.ritense.buildingblock.service.BuildingBlockJsonSchemaDocumentDefinitionImporter @@ -60,12 +63,14 @@ import com.ritense.buildingblock.service.ProcessDefinitionBuildingBlockDefinitio import com.ritense.buildingblock.web.rest.BuildingBlockDefinitionArtworkResource import com.ritense.buildingblock.web.rest.BuildingBlockDocumentDefinitionResource import com.ritense.buildingblock.web.rest.BuildingBlockFieldResource +import com.ritense.buildingblock.web.rest.BuildingBlockFormFlowManagementResource import com.ritense.buildingblock.web.rest.BuildingBlockFormManagementResource import com.ritense.buildingblock.web.rest.BuildingBlockManagementResource import com.ritense.buildingblock.web.rest.BuildingBlockProcessResource import com.ritense.buildingblock.web.rest.BuildingBlockValueResolverResource import com.ritense.case.service.CaseDefinitionService import com.ritense.case.service.finalization.CaseDefinitionFinalizationChecker +import com.ritense.document.autoconfiguration.DocumentAuthorizationAutoConfiguration import com.ritense.document.repository.impl.JsonSchemaDocumentDefinitionRepository import com.ritense.document.service.DocumentDefinitionService import com.ritense.document.service.DocumentService @@ -73,6 +78,7 @@ import com.ritense.document.service.impl.JsonSchemaDocumentDefinitionService import com.ritense.document.service.impl.JsonSchemaDocumentService import com.ritense.exporter.ExportService import com.ritense.form.repository.FormDefinitionRepository +import com.ritense.formflow.service.FormFlowService import com.ritense.importer.ImportService import com.ritense.importer.ValtimoImportService import com.ritense.plugin.service.BuildingBlockPluginConfigurationResolver @@ -92,7 +98,6 @@ import com.ritense.valtimo.service.OperatonTaskService import com.ritense.valueresolver.ValueResolverService import org.operaton.bpm.engine.RepositoryService import org.springframework.beans.factory.annotation.Value -import com.ritense.document.autoconfiguration.DocumentAuthorizationAutoConfiguration import org.springframework.boot.autoconfigure.AutoConfiguration import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.boot.autoconfigure.domain.EntityScan @@ -498,7 +503,12 @@ class BuildingBlockAutoConfiguration { buildingBlockDefinitionRepository: BuildingBlockDefinitionRepository, documentDefinitionService: DocumentDefinitionService, formDefinitionRepository: FormDefinitionRepository, - ) = BuildingBlockDefinitionExporter(objectMapper, buildingBlockDefinitionRepository, documentDefinitionService, formDefinitionRepository) + ) = BuildingBlockDefinitionExporter( + objectMapper, + buildingBlockDefinitionRepository, + documentDefinitionService, + formDefinitionRepository + ) @Bean @ConditionalOnMissingBean(BuildingBlockDefinitionArtworkExporter::class) @@ -596,4 +606,44 @@ class BuildingBlockAutoConfiguration { ): BuildingBlockFormDefinitionImporter { return BuildingBlockFormDefinitionImporter(buildingBlockFormDefinitionService) } + + @Bean + @ConditionalOnMissingBean(BuildingBlockFormFlowDefinitionService::class) + fun buildingBlockFormFlowDefinitionService( + formFlowService: FormFlowService, + definitionChecker: BuildingBlockDefinitionChecker + ): BuildingBlockFormFlowDefinitionService { + return BuildingBlockFormFlowDefinitionService(formFlowService, definitionChecker) + } + + @Bean + @ConditionalOnMissingBean(BuildingBlockFormFlowManagementResource::class) + fun buildingBlockFormFlowManagementResource( + buildingBlockFormFlowDefinitionService: BuildingBlockFormFlowDefinitionService, + buildingBlockFormFlowDefinitionImporter: BuildingBlockFormFlowDefinitionImporter, + ): BuildingBlockFormFlowManagementResource { + return BuildingBlockFormFlowManagementResource( + buildingBlockFormFlowDefinitionService, + buildingBlockFormFlowDefinitionImporter + ) + } + + @Bean + @ConditionalOnMissingBean(BuildingBlockFormFlowDefinitionExporter::class) + fun buildingBlockFormFlowDefinitionExporter( + objectMapper: ObjectMapper, + formFlowService: FormFlowService + ): BuildingBlockFormFlowDefinitionExporter { + return BuildingBlockFormFlowDefinitionExporter(objectMapper, formFlowService) + } + + @Bean + @ConditionalOnMissingBean(BuildingBlockFormFlowDefinitionImporter::class) + fun buildingBlockFormFlowDefinitionImporter( + formFlowService: FormFlowService, + objectMapper: ObjectMapper, + resourceLoader: ResourceLoader, + ): BuildingBlockFormFlowDefinitionImporter { + return BuildingBlockFormFlowDefinitionImporter(formFlowService, objectMapper, resourceLoader) + } } diff --git a/backend/building-block/src/main/kotlin/com/ritense/buildingblock/security/config/BuildingBlockHttpSecurityConfigurer.kt b/backend/building-block/src/main/kotlin/com/ritense/buildingblock/security/config/BuildingBlockHttpSecurityConfigurer.kt index 6ddece3360..68f3961a1e 100644 --- a/backend/building-block/src/main/kotlin/com/ritense/buildingblock/security/config/BuildingBlockHttpSecurityConfigurer.kt +++ b/backend/building-block/src/main/kotlin/com/ritense/buildingblock/security/config/BuildingBlockHttpSecurityConfigurer.kt @@ -106,6 +106,17 @@ class BuildingBlockHttpSecurityConfigurer : HttpSecurityConfigurer { .hasAuthority(ADMIN) .requestMatchers(antMatcher(GET, "$MANAGEMENT_BASE_PATH/{key}/version/{versionTag}/form/{name}/exists")) .hasAuthority(ADMIN) + // Form flow management endpoints + .requestMatchers(antMatcher(GET, "$MANAGEMENT_BASE_PATH/{key}/version/{versionTag}/form-flow-definition")) + .hasAuthority(ADMIN) + .requestMatchers(antMatcher(POST, "$MANAGEMENT_BASE_PATH/{key}/version/{versionTag}/form-flow-definition")) + .hasAuthority(ADMIN) + .requestMatchers(antMatcher(GET, "$MANAGEMENT_BASE_PATH/{key}/version/{versionTag}/form-flow-definition/{definitionKey}")) + .hasAuthority(ADMIN) + .requestMatchers(antMatcher(PUT, "$MANAGEMENT_BASE_PATH/{key}/version/{versionTag}/form-flow-definition/{definitionKey}")) + .hasAuthority(ADMIN) + .requestMatchers(antMatcher(DELETE, "$MANAGEMENT_BASE_PATH/{key}/version/{versionTag}/form-flow-definition/{definitionKey}")) + .hasAuthority(ADMIN) } } catch (e: Exception) { throw HttpConfigurerConfigurationException(e) diff --git a/backend/building-block/src/main/kotlin/com/ritense/buildingblock/service/BuildingBlockFormFlowDefinitionExporter.kt b/backend/building-block/src/main/kotlin/com/ritense/buildingblock/service/BuildingBlockFormFlowDefinitionExporter.kt new file mode 100644 index 0000000000..349ca084c6 --- /dev/null +++ b/backend/building-block/src/main/kotlin/com/ritense/buildingblock/service/BuildingBlockFormFlowDefinitionExporter.kt @@ -0,0 +1,69 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.buildingblock.service + +import com.fasterxml.jackson.databind.ObjectMapper +import com.ritense.exporter.ExportFile +import com.ritense.exporter.ExportPrettyPrinter +import com.ritense.exporter.ExportResult +import com.ritense.exporter.Exporter +import com.ritense.exporter.request.BuildingBlockFormDefinitionExportRequest +import com.ritense.exporter.request.BuildingBlockFormFlowDefinitionExportRequest +import com.ritense.formflow.domain.definition.configuration.FormFlowDefinition +import com.ritense.formflow.domain.definition.configuration.step.FormStepTypeProperties +import com.ritense.formflow.handler.FormFlowStepTypeFormHandler +import com.ritense.formflow.service.FormFlowService +import org.springframework.transaction.annotation.Transactional + +@Transactional(readOnly = true) +class BuildingBlockFormFlowDefinitionExporter( + private val objectMapper: ObjectMapper, + private val formFlowService: FormFlowService, +) : Exporter { + + override fun supports() = BuildingBlockFormFlowDefinitionExportRequest::class.java + + override fun export(request: BuildingBlockFormFlowDefinitionExportRequest): ExportResult { + val buildingBlockDefinitionId = request.buildingBlockDefinitionId + val definition = formFlowService.findDefinition(request.formFlowDefinitionKey, buildingBlockDefinitionId) + + val relatedFormRequests = definition.steps + .map { it.type } + .filter { it.name == FormFlowStepTypeFormHandler.TYPE } + .map { type -> + val formDefinitionName = (type.properties as FormStepTypeProperties).definition + BuildingBlockFormDefinitionExportRequest(formDefinitionName, buildingBlockDefinitionId) + } + .toSet() + + val formattedVersionTag = buildingBlockDefinitionId.versionTag.let { + "${it.major}-${it.minor}-${it.patch}" + } + + return ExportResult( + ExportFile( + PATH.format(buildingBlockDefinitionId.key, formattedVersionTag, definition.id.key), + objectMapper.writer(ExportPrettyPrinter()).writeValueAsBytes(FormFlowDefinition.fromEntity(definition)) + ), + relatedFormRequests + ) + } + + companion object { + private const val PATH = "config/building-block/%s/%s/form-flow/%s.form-flow.json" + } +} diff --git a/backend/building-block/src/main/kotlin/com/ritense/buildingblock/service/BuildingBlockFormFlowDefinitionImporter.kt b/backend/building-block/src/main/kotlin/com/ritense/buildingblock/service/BuildingBlockFormFlowDefinitionImporter.kt new file mode 100644 index 0000000000..05d958bb74 --- /dev/null +++ b/backend/building-block/src/main/kotlin/com/ritense/buildingblock/service/BuildingBlockFormFlowDefinitionImporter.kt @@ -0,0 +1,125 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.buildingblock.service + +import com.fasterxml.jackson.databind.ObjectMapper +import com.ritense.formflow.domain.definition.FormFlowDefinitionId +import com.ritense.formflow.domain.definition.configuration.FormFlowDefinition +import com.ritense.formflow.expression.ExpressionProcessorFactoryHolder +import com.ritense.formflow.service.FormFlowService +import com.ritense.importer.ImportRequest +import com.ritense.importer.Importer +import com.ritense.importer.ValtimoImportTypes.Companion.BUILDING_BLOCK_DEFINITION +import com.ritense.importer.ValtimoImportTypes.Companion.BUILDING_BLOCK_FORM_DEFINITION +import com.ritense.importer.ValtimoImportTypes.Companion.BUILDING_BLOCK_FORM_FLOW_DEFINITION +import com.ritense.logging.withLoggingContext +import com.ritense.valtimo.contract.buildingblock.BuildingBlockDefinitionId +import io.github.oshai.kotlinlogging.KotlinLogging +import org.everit.json.schema.loader.SchemaLoader +import org.json.JSONObject +import org.json.JSONTokener +import org.springframework.core.io.Resource +import org.springframework.core.io.ResourceLoader +import org.springframework.core.io.support.ResourcePatternUtils +import org.springframework.transaction.annotation.Transactional + +@Transactional +class BuildingBlockFormFlowDefinitionImporter( + private val formFlowService: FormFlowService, + private val objectMapper: ObjectMapper, + private val resourceLoader: ResourceLoader, +) : Importer { + + override fun type() = BUILDING_BLOCK_FORM_FLOW_DEFINITION + + override fun dependsOn() = setOf(BUILDING_BLOCK_DEFINITION, BUILDING_BLOCK_FORM_DEFINITION) + + override fun supports(fileName: String) = fileName.matches(PATH_REGEX) + + override fun import(request: ImportRequest) { + val buildingBlockDefinitionId = request.buildingBlockDefinitionId + ?: throw IllegalArgumentException("Building block definition ID is required for form flow import") + + val formFlowKey = PATH_REGEX.matchEntire(request.fileName)!!.groupValues[1] + + deploy(formFlowKey, request.content.toString(Charsets.UTF_8), buildingBlockDefinitionId) + } + + override fun partOfCaseDefinition() = false + + override fun partOfBuildingBlockDefinition() = true + + fun isAutoDeployed(formFlowDefinitionKey: String): Boolean { + withLoggingContext("formFlowDefinitionKey" to formFlowDefinitionKey) { + return ResourcePatternUtils.getResourcePatternResolver(resourceLoader) + .getResources(FORM_FLOW_DEFINITIONS_PATH.replace("{formFlowKey}", formFlowDefinitionKey)) + .size > 0 + } + } + + private fun deploy(formFlowKey: String, formFlowJson: String, buildingBlockDefinitionId: BuildingBlockDefinitionId) { + withLoggingContext("formFlowDefinitionKey" to formFlowKey) { + validate(formFlowJson) + + val formFlowDefinitionConfig = objectMapper.readValue(formFlowJson, FormFlowDefinition::class.java) + + validate(formFlowDefinitionConfig) + + try { + val existingDefinition = formFlowService.findDefinitionOrNull(formFlowKey, buildingBlockDefinitionId) + val definitionId = FormFlowDefinitionId.newId(formFlowKey, buildingBlockDefinitionId) + + if (existingDefinition != null && formFlowDefinitionConfig.contentEquals(existingDefinition)) { + logger.info { "Form Flow already deployed - $definitionId" } + return + } + + formFlowService.save(formFlowDefinitionConfig.toDefinition(definitionId)) + logger.info { "Deployed Form Flow - $definitionId" } + } catch (e: Exception) { + throw RuntimeException("Failed to deploy Form Flow $formFlowKey", e) + } + } + } + + private fun validate(formFlowJson: String) { + val definitionJsonObject = JSONObject(JSONTokener(formFlowJson)) + val schema = SchemaLoader.load(JSONObject(JSONTokener(loadFormFlowSchemaResource().inputStream))) + schema.validate(definitionJsonObject) + } + + private fun validate(formFlowDefinitionConfig: FormFlowDefinition) { + val expressionProcessor = ExpressionProcessorFactoryHolder.getInstance().create() + formFlowDefinitionConfig.steps.forEach { step -> + step.onBack.forEach { expression -> expressionProcessor.validate(expression) } + step.onOpen.forEach { expression -> expressionProcessor.validate(expression) } + step.onComplete.forEach { expression -> expressionProcessor.validate(expression) } + } + } + + private fun loadFormFlowSchemaResource(): Resource { + return ResourcePatternUtils.getResourcePatternResolver(resourceLoader).getResource(FORM_FLOW_SCHEMA_PATH) + } + + private companion object { + private const val FORM_FLOW_SCHEMA_PATH = "classpath:config/form-flow/schema/formflow.schema.json" + private const val FORM_FLOW_DEFINITIONS_PATH = + "classpath:config/building-block/*/*/form-flow/{formFlowKey}.form-flow.json" + val PATH_REGEX = """/form-flow/([^/]+)\.form-flow\.json""".toRegex() + val logger = KotlinLogging.logger {} + } +} diff --git a/backend/building-block/src/main/kotlin/com/ritense/buildingblock/service/BuildingBlockFormFlowDefinitionService.kt b/backend/building-block/src/main/kotlin/com/ritense/buildingblock/service/BuildingBlockFormFlowDefinitionService.kt new file mode 100644 index 0000000000..53259d9f8a --- /dev/null +++ b/backend/building-block/src/main/kotlin/com/ritense/buildingblock/service/BuildingBlockFormFlowDefinitionService.kt @@ -0,0 +1,71 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.buildingblock.service + +import com.ritense.formflow.domain.definition.FormFlowDefinition +import com.ritense.formflow.service.FormFlowService +import com.ritense.formflow.web.rest.result.FormFlowDefinitionDto +import com.ritense.valtimo.contract.buildingblock.BuildingBlockDefinitionChecker +import com.ritense.valtimo.contract.buildingblock.BuildingBlockDefinitionId +import org.springframework.data.domain.Page +import org.springframework.data.domain.Pageable +import org.springframework.transaction.annotation.Transactional + +@Transactional +class BuildingBlockFormFlowDefinitionService( + private val formFlowService: FormFlowService, + private val definitionChecker: BuildingBlockDefinitionChecker, +) { + @Transactional(readOnly = true) + fun getFormFlowDefinitions( + buildingBlockDefinitionId: BuildingBlockDefinitionId, + pageable: Pageable + ): Page { + definitionChecker.assertBuildingBlockDefinitionExists(buildingBlockDefinitionId) + return formFlowService.getFormFlowDefinitions(buildingBlockDefinitionId, pageable) + } + + @Transactional(readOnly = true) + fun getFormFlowDefinition( + buildingBlockDefinitionId: BuildingBlockDefinitionId, + definitionKey: String + ): FormFlowDefinition? { + definitionChecker.assertBuildingBlockDefinitionExists(buildingBlockDefinitionId) + return formFlowService.findDefinitionOrNull(definitionKey, buildingBlockDefinitionId) + } + + fun save( + buildingBlockDefinitionId: BuildingBlockDefinitionId, + dto: FormFlowDefinitionDto + ): FormFlowDefinition { + definitionChecker.assertCanUpdateBuildingBlockDefinition(buildingBlockDefinitionId) + return formFlowService.save(dto.toEntity(buildingBlockDefinitionId)) + } + + fun delete( + buildingBlockDefinitionId: BuildingBlockDefinitionId, + definitionKey: String + ) { + definitionChecker.assertCanUpdateBuildingBlockDefinition(buildingBlockDefinitionId) + formFlowService.deleteByKeyAndBuildingBlockDefinition(definitionKey, buildingBlockDefinitionId) + } + + fun isAutoDeployed(definitionKey: String): Boolean { + // Building block form flows deployed via classpath are considered read-only + return false + } +} diff --git a/backend/building-block/src/main/kotlin/com/ritense/buildingblock/web/rest/BuildingBlockFormFlowManagementResource.kt b/backend/building-block/src/main/kotlin/com/ritense/buildingblock/web/rest/BuildingBlockFormFlowManagementResource.kt new file mode 100644 index 0000000000..bb0f90f236 --- /dev/null +++ b/backend/building-block/src/main/kotlin/com/ritense/buildingblock/web/rest/BuildingBlockFormFlowManagementResource.kt @@ -0,0 +1,123 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.buildingblock.web.rest + +import com.ritense.buildingblock.service.BuildingBlockFormFlowDefinitionImporter +import com.ritense.buildingblock.service.BuildingBlockFormFlowDefinitionService +import com.ritense.formflow.web.rest.result.FormFlowDefinitionDto +import com.ritense.valtimo.contract.annotation.SkipComponentScan +import com.ritense.valtimo.contract.buildingblock.BuildingBlockDefinitionId +import com.ritense.valtimo.contract.domain.ValtimoMediaType.APPLICATION_JSON_UTF8_VALUE +import org.springframework.data.domain.Page +import org.springframework.data.domain.Pageable +import org.springframework.http.HttpStatus +import org.springframework.http.ResponseEntity +import org.springframework.transaction.annotation.Transactional +import org.springframework.web.bind.annotation.DeleteMapping +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.PutMapping +import org.springframework.web.bind.annotation.RequestBody +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + +@RestController +@SkipComponentScan +@RequestMapping("/api/management/v1/building-block", produces = [APPLICATION_JSON_UTF8_VALUE]) +class BuildingBlockFormFlowManagementResource( + private val buildingBlockFormFlowDefinitionService: BuildingBlockFormFlowDefinitionService, + private val buildingBlockFormFlowDefinitionImporter: BuildingBlockFormFlowDefinitionImporter, +) { + + @GetMapping("/{key}/version/{versionTag}/form-flow-definition") + @Transactional + fun getAllFormFlowDefinitions( + @PathVariable key: String, + @PathVariable versionTag: String, + pageable: Pageable + ): ResponseEntity> { + val buildingBlockId = BuildingBlockDefinitionId.of(key, versionTag) + val definitions = buildingBlockFormFlowDefinitionService.getFormFlowDefinitions(buildingBlockId, pageable) + .map { FormFlowDefinitionDto.of(it, buildingBlockFormFlowDefinitionImporter.isAutoDeployed(it.id.key)) } + return ResponseEntity.ok(definitions) + } + + @GetMapping("/{key}/version/{versionTag}/form-flow-definition/{definitionKey}") + @Transactional + fun getFormFlowDefinition( + @PathVariable key: String, + @PathVariable versionTag: String, + @PathVariable definitionKey: String + ): ResponseEntity { + val buildingBlockId = BuildingBlockDefinitionId.of(key, versionTag) + val definition = buildingBlockFormFlowDefinitionService.getFormFlowDefinition(buildingBlockId, definitionKey) + ?: return ResponseEntity.notFound().build() + return ResponseEntity.ok( + FormFlowDefinitionDto.of( + definition, + buildingBlockFormFlowDefinitionImporter.isAutoDeployed(definitionKey) + ) + ) + } + + @PostMapping("/{key}/version/{versionTag}/form-flow-definition") + @Transactional + fun createFormFlowDefinition( + @PathVariable key: String, + @PathVariable versionTag: String, + @RequestBody definitionDto: FormFlowDefinitionDto + ): ResponseEntity { + val buildingBlockId = BuildingBlockDefinitionId.of(key, versionTag) + if (buildingBlockFormFlowDefinitionService.getFormFlowDefinition(buildingBlockId, definitionDto.key) != null) { + return ResponseEntity.badRequest().build() + } + val saved = buildingBlockFormFlowDefinitionService.save(buildingBlockId, definitionDto) + return ResponseEntity.ok(FormFlowDefinitionDto.of(saved, false)) + } + + @PutMapping("/{key}/version/{versionTag}/form-flow-definition/{definitionKey}") + @Transactional + fun updateFormFlowDefinition( + @PathVariable key: String, + @PathVariable versionTag: String, + @PathVariable definitionKey: String, + @RequestBody definitionDto: FormFlowDefinitionDto + ): ResponseEntity { + val buildingBlockId = BuildingBlockDefinitionId.of(key, versionTag) + if (buildingBlockFormFlowDefinitionImporter.isAutoDeployed(definitionKey)) { + return ResponseEntity.status(HttpStatus.FORBIDDEN).build() + } + val saved = buildingBlockFormFlowDefinitionService.save(buildingBlockId, definitionDto) + return ResponseEntity.ok(FormFlowDefinitionDto.of(saved, false)) + } + + @DeleteMapping("/{key}/version/{versionTag}/form-flow-definition/{definitionKey}") + @Transactional + fun deleteFormFlowDefinition( + @PathVariable key: String, + @PathVariable versionTag: String, + @PathVariable definitionKey: String + ): ResponseEntity { + val buildingBlockId = BuildingBlockDefinitionId.of(key, versionTag) + if (buildingBlockFormFlowDefinitionImporter.isAutoDeployed(definitionKey)) { + return ResponseEntity.status(HttpStatus.FORBIDDEN).build() + } + buildingBlockFormFlowDefinitionService.delete(buildingBlockId, definitionKey) + return ResponseEntity.ok().build() + } +} diff --git a/backend/building-block/src/test/kotlin/com/ritense/buildingblock/processlink/service/BuildingBlockFormFlowIT.kt b/backend/building-block/src/test/kotlin/com/ritense/buildingblock/processlink/service/BuildingBlockFormFlowIT.kt new file mode 100644 index 0000000000..d652e2f7fe --- /dev/null +++ b/backend/building-block/src/test/kotlin/com/ritense/buildingblock/processlink/service/BuildingBlockFormFlowIT.kt @@ -0,0 +1,212 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.buildingblock.processlink.service + +import com.fasterxml.jackson.databind.ObjectMapper +import com.ritense.authorization.AuthorizationContext.Companion.runWithoutAuthorization +import com.ritense.buildingblock.BaseIntegrationTest +import com.ritense.buildingblock.processlink.domain.BuildingBlockProcessLink +import com.ritense.buildingblock.repository.BuildingBlockInstanceRepository +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.domain.impl.request.NewDocumentRequest +import com.ritense.document.service.DocumentService +import com.ritense.formflow.FormFlowTaskOpenResultProperties +import com.ritense.formflow.domain.definition.FormFlowDefinition +import com.ritense.formflow.domain.definition.FormFlowDefinitionId +import com.ritense.formflow.domain.definition.FormFlowStep +import com.ritense.formflow.domain.definition.FormFlowStepId +import com.ritense.formflow.domain.definition.configuration.FormFlowStepType +import com.ritense.formflow.domain.definition.configuration.step.FormStepTypeProperties +import com.ritense.formflow.domain.instance.FormFlowInstanceId +import com.ritense.formflow.repository.FormFlowDefinitionRepository +import com.ritense.formflow.service.FormFlowService +import com.ritense.formflow.web.rest.FormFlowResource +import com.ritense.formflow.web.rest.dto.FormFlowProcessLinkCreateRequestDto +import com.ritense.processdocument.domain.impl.request.NewDocumentAndStartProcessRequest +import com.ritense.processdocument.service.ProcessDocumentService +import com.ritense.processlink.domain.ActivityTypeWithEventName +import com.ritense.processlink.repository.ProcessLinkRepository +import com.ritense.processlink.service.ProcessLinkActivityService +import com.ritense.valtimo.contract.buildingblock.BuildingBlockDefinitionId +import com.ritense.valtimo.operaton.repository.OperatonTaskSpecificationHelper.Companion.byProcessInstanceId +import com.ritense.valtimo.service.OperatonTaskService +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.operaton.bpm.engine.RepositoryService +import org.operaton.bpm.engine.RuntimeService +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.transaction.annotation.Transactional +import java.util.UUID + +/** + * Integration test verifying that a building block sub-process with a form-flow-protected user task + * correctly writes the form submission to the building block's document. + * + * Flow: + * 1. A case is created and the main process starts. + * 2. The main process calls the building block sub-process (building-block-form-flow-process). + * 3. The sub-process has a user task linked to a building-block-owned form flow. + * 4. Completing the form flow step calls valtimoFormFlow.completeTask, which resolves + * additionalProperties["documentId"] from the sub-process business key (= BB document ID) + * and writes the submission to the BB document. + */ +@Transactional +class BuildingBlockFormFlowIT @Autowired constructor( + private val buildingBlockInstanceRepository: BuildingBlockInstanceRepository, + private val documentService: DocumentService, + private val processDocumentService: ProcessDocumentService, + private val formFlowDefinitionRepository: FormFlowDefinitionRepository, + private val processLinkActivityService: ProcessLinkActivityService, + private val processLinkRepository: ProcessLinkRepository, + private val formFlowResource: FormFlowResource, + private val operatonTaskService: OperatonTaskService, + private val formFlowService: FormFlowService, + private val repositoryService: RepositoryService, + private val runtimeService: RuntimeService, + private val objectMapper: ObjectMapper, +) : BaseIntegrationTest() { + + @Test + fun `completing building block form flow writes submission to building block document`() { + val bbId = BuildingBlockDefinitionId(BUILDING_BLOCK_KEY, BUILDING_BLOCK_VERSION) + val formFlowDefinitionId = FormFlowDefinitionId.existingId(FORM_FLOW_KEY, bbId) + + // Save the form flow definition directly via repository — bypasses the "final building block" + // write check that would block formFlowDefinitionImporter.deploy() for bezwaar + val step = FormFlowStep( + id = FormFlowStepId(FORM_FLOW_STEP_KEY), + onComplete = listOf("\${valtimoFormFlow.completeTask(additionalProperties, step.submissionData)}"), + type = FormFlowStepType("form", FormStepTypeProperties("bb-form")) + ) + formFlowDefinitionRepository.save( + FormFlowDefinition(formFlowDefinitionId, FORM_FLOW_STEP_KEY, setOf(step)) + ) + + // Link the form flow to the user task in the building block sub-process. + // The process link must reference the sub-process definition (which contains the user task). + val subProcessDefinitionId = repositoryService.createProcessDefinitionQuery() + .processDefinitionKey(SUB_PROCESS_KEY) + .latestVersion() + .singleResult() + ?.id ?: error("Process definition '$SUB_PROCESS_KEY' not deployed") + + processLinkService.createProcessLink( + FormFlowProcessLinkCreateRequestDto( + subProcessDefinitionId, + USER_TASK_ID, + ActivityTypeWithEventName.USER_TASK_CREATE, + FORM_FLOW_KEY + ), + bbId + ) + + // Register the building block process link on the call activity in the main process. + // Without this, BuildingBlockCallActivityListener won't create the BB document or set + // the buildingBlockDocumentId variable that the call activity's businessKey expression depends on. + val mainProcessDefinitionId = repositoryService.createProcessDefinitionQuery() + .processDefinitionKey(MAIN_PROCESS_KEY) + .latestVersion() + .singleResult() + ?.id ?: error("Process definition '$MAIN_PROCESS_KEY' not deployed") + + processLinkRepository.save( + BuildingBlockProcessLink( + id = UUID.randomUUID(), + processDefinitionId = mainProcessDefinitionId, + activityId = CALL_ACTIVITY_ID, + activityType = ActivityTypeWithEventName.CALL_ACTIVITY_START, + buildingBlockDefinitionId = bbId, + pluginConfigurationMappings = emptyMap(), + inputMappings = emptyList() + ) + ) + + // Create a case document and start the main process — this triggers the call activity + // which starts the building block sub-process with its user task + val caseDocumentAndProcess = runWithoutAuthorization { + processDocumentService.newDocumentAndStartProcess( + NewDocumentAndStartProcessRequest( + MAIN_PROCESS_KEY, + NewDocumentRequest( + CASE_DOCUMENT_DEFINITION_NAME, + CASE_DEFINITION_KEY, + CASE_DEFINITION_VERSION, + objectMapper.createObjectNode() + ) + ) + ) + } + assertThat(caseDocumentAndProcess.resultingDocument()).isPresent + + // The call activity has started the building block sub-process; + // retrieve the BB instance to find the BB document ID (= sub-process business key) + val bbInstances = buildingBlockInstanceRepository.findAll() + assertThat(bbInstances).hasSize(1) + val bbDocumentId = bbInstances.first().documentId.toString() + + // Locate the running sub-process instance by its business key + val subProcessInstance = runtimeService.createProcessInstanceQuery() + .processInstanceBusinessKey(bbDocumentId) + .processDefinitionKey(SUB_PROCESS_KEY) + .singleResult() ?: error("Sub-process '$SUB_PROCESS_KEY' not found for business key $bbDocumentId") + + // Find the user task in the sub-process + val tasks = runWithoutAuthorization { + operatonTaskService.findTasks(byProcessInstanceId(subProcessInstance.id)) + } + assertThat(tasks).hasSize(1) + + // Open the task — creates the form flow instance and returns its properties + val taskOpenResult = runWithoutAuthorization { + processLinkActivityService.openTask(UUID.fromString(tasks.first().id)) + } + assertThat(taskOpenResult.properties).isInstanceOf(FormFlowTaskOpenResultProperties::class.java) + val formFlowInstanceId = (taskOpenResult.properties as FormFlowTaskOpenResultProperties).formFlowInstanceId + val formFlowInstance = formFlowService.getInstanceById(FormFlowInstanceId.existingId(formFlowInstanceId)) + + // Complete the form flow step with submission data + runWithoutAuthorization { + formFlowResource.completeStep( + formFlowInstance.id.id.toString(), + formFlowInstance.currentFormFlowStepInstanceId!!.id.toString(), + objectMapper.readTree("""{"straatnaam":"Hoofdstraat"}""") + ) + } + + // After completion, the BB document should contain the submitted data under "submission" + val bbDocument = runWithoutAuthorization { + documentService.get(bbDocumentId) + } as JsonSchemaDocument + val content = bbDocument.content().asJson() + assertThat(content.has("submission")).isTrue() + assertThat(content.get("submission").get("straatnaam").asText()).isEqualTo("Hoofdstraat") + } + + companion object { + private const val BUILDING_BLOCK_KEY = "bezwaar" + private const val BUILDING_BLOCK_VERSION = "1.0.0" + private const val CASE_DEFINITION_KEY = "bb-case" + private const val CASE_DEFINITION_VERSION = "1.0.0" + private const val CASE_DOCUMENT_DEFINITION_NAME = "bb-case" + private const val MAIN_PROCESS_KEY = "building-block-form-flow-main" + private const val SUB_PROCESS_KEY = "building-block-form-flow-process" + private const val CALL_ACTIVITY_ID = "callActivity" + private const val USER_TASK_ID = "form-flow-task" + private const val FORM_FLOW_KEY = "bb-test-form-flow" + private const val FORM_FLOW_STEP_KEY = "step1" + } +} diff --git a/backend/building-block/src/test/kotlin/com/ritense/buildingblock/service/BuildingBlockFormFlowDefinitionExporterTest.kt b/backend/building-block/src/test/kotlin/com/ritense/buildingblock/service/BuildingBlockFormFlowDefinitionExporterTest.kt new file mode 100644 index 0000000000..07f75de668 --- /dev/null +++ b/backend/building-block/src/test/kotlin/com/ritense/buildingblock/service/BuildingBlockFormFlowDefinitionExporterTest.kt @@ -0,0 +1,158 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.buildingblock.service + +import com.fasterxml.jackson.databind.ObjectMapper +import com.ritense.exporter.request.BuildingBlockFormDefinitionExportRequest +import com.ritense.exporter.request.BuildingBlockFormFlowDefinitionExportRequest +import com.ritense.formflow.domain.definition.FormFlowDefinition +import com.ritense.formflow.domain.definition.FormFlowDefinitionId +import com.ritense.formflow.domain.definition.FormFlowStep +import com.ritense.formflow.domain.definition.FormFlowStepId +import com.ritense.formflow.domain.definition.configuration.FormFlowStepType +import com.ritense.formflow.domain.definition.configuration.step.FormStepTypeProperties +import com.ritense.formflow.service.FormFlowService +import com.ritense.valtimo.contract.buildingblock.BuildingBlockDefinitionId +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import org.mockito.Mock +import org.mockito.junit.jupiter.MockitoExtension +import org.mockito.kotlin.eq +import org.mockito.kotlin.whenever + +@ExtendWith(MockitoExtension::class) +class BuildingBlockFormFlowDefinitionExporterTest( + @Mock private val formFlowService: FormFlowService, +) { + private val objectMapper = ObjectMapper() + private lateinit var exporter: BuildingBlockFormFlowDefinitionExporter + + private val bbId = BuildingBlockDefinitionId("my-bb", "1.0.0") + + @BeforeEach + fun setUp() { + exporter = BuildingBlockFormFlowDefinitionExporter(objectMapper, formFlowService) + } + + @Test + fun `supports BuildingBlockFormFlowDefinitionExportRequest`() { + assertThat(exporter.supports()).isEqualTo(BuildingBlockFormFlowDefinitionExportRequest::class.java) + } + + @Test + fun `export produces file at correct path`() { + val definition = buildDefinition("my-flow") + whenever(formFlowService.findDefinition(eq("my-flow"), eq(bbId))).thenReturn(definition) + + val result = exporter.export(BuildingBlockFormFlowDefinitionExportRequest("my-flow", bbId)) + + assertThat(result.exportFiles).hasSize(1) + assertThat(result.exportFiles.first().path) + .isEqualTo("config/building-block/my-bb/1-0-0/form-flow/my-flow.form-flow.json") + } + + @Test + fun `export with different version tag produces correct path`() { + val bbId231 = BuildingBlockDefinitionId("test-bb", "2.3.1") + val definition = buildDefinition("my-flow", bbId231) + whenever(formFlowService.findDefinition(eq("my-flow"), eq(bbId231))).thenReturn(definition) + + val result = exporter.export(BuildingBlockFormFlowDefinitionExportRequest("my-flow", bbId231)) + + assertThat(result.exportFiles.first().path) + .isEqualTo("config/building-block/test-bb/2-3-1/form-flow/my-flow.form-flow.json") + } + + @Test + fun `export with no form steps produces no related form requests`() { + val definition = buildDefinition("my-flow") + whenever(formFlowService.findDefinition(eq("my-flow"), eq(bbId))).thenReturn(definition) + + val result = exporter.export(BuildingBlockFormFlowDefinitionExportRequest("my-flow", bbId)) + + assertThat(result.relatedRequests).isEmpty() + } + + @Test + fun `export with form steps produces related BuildingBlockFormDefinitionExportRequests`() { + val step = FormFlowStep( + FormFlowStepId("step-with-form"), + listOf(), + listOf(), + listOf(), + listOf(), + type = FormFlowStepType("form", FormStepTypeProperties("my-form")) + ) + val definition = buildDefinition("my-flow", steps = setOf(step)) + whenever(formFlowService.findDefinition(eq("my-flow"), eq(bbId))).thenReturn(definition) + + val result = exporter.export(BuildingBlockFormFlowDefinitionExportRequest("my-flow", bbId)) + + assertThat(result.relatedRequests).containsExactly( + BuildingBlockFormDefinitionExportRequest("my-form", bbId) + ) + } + + @Test + fun `export with multiple form steps produces one related request per form`() { + val step1 = FormFlowStep( + FormFlowStepId("step1"), + listOf(), listOf(), listOf(), listOf(), + type = FormFlowStepType("form", FormStepTypeProperties("form-a")) + ) + val step2 = FormFlowStep( + FormFlowStepId("step2"), + listOf(), listOf(), listOf(), listOf(), + type = FormFlowStepType("form", FormStepTypeProperties("form-b")) + ) + val definition = buildDefinition("multi-step", steps = setOf(step1, step2)) + whenever(formFlowService.findDefinition(eq("multi-step"), eq(bbId))).thenReturn(definition) + + val result = exporter.export(BuildingBlockFormFlowDefinitionExportRequest("multi-step", bbId)) + + assertThat(result.relatedRequests).hasSize(2) + assertThat(result.relatedRequests.map { (it as BuildingBlockFormDefinitionExportRequest).formDefinitionName }) + .containsExactlyInAnyOrder("form-a", "form-b") + } + + @Test + fun `export with non-form step type produces no related form requests`() { + // Use type name "custom-component" — the exporter only emits form requests for steps with type "form" + val step = FormFlowStep( + FormFlowStepId("custom-step"), + listOf(), listOf(), listOf(), listOf(), + type = FormFlowStepType("custom-component", FormStepTypeProperties("ignored")) + ) + val definition = buildDefinition("my-flow", steps = setOf(step)) + whenever(formFlowService.findDefinition(eq("my-flow"), eq(bbId))).thenReturn(definition) + + val result = exporter.export(BuildingBlockFormFlowDefinitionExportRequest("my-flow", bbId)) + + assertThat(result.relatedRequests).isEmpty() + } + + private fun buildDefinition( + key: String, + buildingBlockDefinitionId: BuildingBlockDefinitionId = bbId, + steps: Set = emptySet(), + ): FormFlowDefinition { + val definitionId = FormFlowDefinitionId.existingId(key, buildingBlockDefinitionId) + return FormFlowDefinition(definitionId, "start-step", steps) + } +} diff --git a/backend/building-block/src/test/kotlin/com/ritense/buildingblock/service/BuildingBlockFormFlowDefinitionImporterTest.kt b/backend/building-block/src/test/kotlin/com/ritense/buildingblock/service/BuildingBlockFormFlowDefinitionImporterTest.kt new file mode 100644 index 0000000000..b40c891edc --- /dev/null +++ b/backend/building-block/src/test/kotlin/com/ritense/buildingblock/service/BuildingBlockFormFlowDefinitionImporterTest.kt @@ -0,0 +1,92 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.buildingblock.service + +import com.fasterxml.jackson.databind.ObjectMapper +import com.ritense.formflow.service.FormFlowService +import com.ritense.importer.ImportRequest +import com.ritense.importer.ValtimoImportTypes +import com.ritense.valtimo.contract.buildingblock.BuildingBlockDefinitionId +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import org.junit.jupiter.api.extension.ExtendWith +import org.mockito.Mock +import org.mockito.junit.jupiter.MockitoExtension +import org.springframework.core.io.ResourceLoader + +@ExtendWith(MockitoExtension::class) +class BuildingBlockFormFlowDefinitionImporterTest( + @Mock private val formFlowService: FormFlowService, + @Mock private val objectMapper: ObjectMapper, + @Mock private val resourceLoader: ResourceLoader, +) { + private lateinit var importer: BuildingBlockFormFlowDefinitionImporter + + @BeforeEach + fun setUp() { + importer = BuildingBlockFormFlowDefinitionImporter(formFlowService, objectMapper, resourceLoader) + } + + @Test + fun `type returns BUILDING_BLOCK_FORM_FLOW_DEFINITION`() { + assertThat(importer.type()).isEqualTo(ValtimoImportTypes.BUILDING_BLOCK_FORM_FLOW_DEFINITION) + } + + @Test + fun `dependsOn returns BUILDING_BLOCK_DEFINITION and BUILDING_BLOCK_FORM_DEFINITION`() { + assertThat(importer.dependsOn()).containsExactlyInAnyOrder( + ValtimoImportTypes.BUILDING_BLOCK_DEFINITION, + ValtimoImportTypes.BUILDING_BLOCK_FORM_DEFINITION + ) + } + + @Test + fun `supports valid form-flow file paths`() { + assertThat(importer.supports("/form-flow/my-flow.form-flow.json")).isTrue() + assertThat(importer.supports("/form-flow/another-flow.form-flow.json")).isTrue() + } + + @Test + fun `does not support invalid file paths`() { + assertThat(importer.supports("/form-flow/my-flow.json")).isFalse() + assertThat(importer.supports("/form/my-flow.form-flow.json")).isFalse() + assertThat(importer.supports("my-flow.form-flow.json")).isFalse() + assertThat(importer.supports("/form-flow/nested/my-flow.form-flow.json")).isFalse() + } + + @Test + fun `partOfBuildingBlockDefinition returns true`() { + assertThat(importer.partOfBuildingBlockDefinition()).isTrue() + } + + @Test + fun `partOfCaseDefinition returns false`() { + assertThat(importer.partOfCaseDefinition()).isFalse() + } + + @Test + fun `import without buildingBlockDefinitionId throws IllegalArgumentException`() { + val request = ImportRequest( + fileName = "/form-flow/my-flow.form-flow.json", + content = "{}".toByteArray() + ) + + assertThrows { importer.import(request) } + } +} diff --git a/backend/building-block/src/test/kotlin/com/ritense/buildingblock/service/BuildingBlockFormFlowDefinitionServiceTest.kt b/backend/building-block/src/test/kotlin/com/ritense/buildingblock/service/BuildingBlockFormFlowDefinitionServiceTest.kt new file mode 100644 index 0000000000..e2b2e65e73 --- /dev/null +++ b/backend/building-block/src/test/kotlin/com/ritense/buildingblock/service/BuildingBlockFormFlowDefinitionServiceTest.kt @@ -0,0 +1,104 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.buildingblock.service + +import com.ritense.formflow.domain.definition.FormFlowDefinition +import com.ritense.formflow.domain.definition.FormFlowDefinitionId +import com.ritense.formflow.service.FormFlowService +import com.ritense.formflow.web.rest.result.FormFlowDefinitionDto +import com.ritense.valtimo.contract.buildingblock.BuildingBlockDefinitionChecker +import com.ritense.valtimo.contract.buildingblock.BuildingBlockDefinitionId +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import org.mockito.Mock +import org.mockito.junit.jupiter.MockitoExtension +import org.mockito.kotlin.any +import org.mockito.kotlin.eq +import org.mockito.kotlin.mock +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever +import org.springframework.data.domain.PageImpl +import org.springframework.data.domain.PageRequest + +@ExtendWith(MockitoExtension::class) +class BuildingBlockFormFlowDefinitionServiceTest( + @Mock private val formFlowService: FormFlowService, + @Mock private val definitionChecker: BuildingBlockDefinitionChecker, +) { + private lateinit var service: BuildingBlockFormFlowDefinitionService + + private val bbId = BuildingBlockDefinitionId("my-bb", "1.0.0") + + @BeforeEach + fun setUp() { + service = BuildingBlockFormFlowDefinitionService(formFlowService, definitionChecker) + } + + @Test + fun `getFormFlowDefinitions asserts bb exists and delegates to formFlowService`() { + val pageable = PageRequest.of(0, 10) + val mockDefinition = mock() + whenever(formFlowService.getFormFlowDefinitions(eq(bbId), eq(pageable))).thenReturn(PageImpl(listOf(mockDefinition))) + + val result = service.getFormFlowDefinitions(bbId, pageable) + + verify(definitionChecker).assertBuildingBlockDefinitionExists(bbId) + assertThat(result.content).containsExactly(mockDefinition) + } + + @Test + fun `getFormFlowDefinition asserts bb exists and delegates to formFlowService`() { + val mockDefinition = mock() + whenever(formFlowService.findDefinitionOrNull(eq("my-flow"), eq(bbId))).thenReturn(mockDefinition) + + val result = service.getFormFlowDefinition(bbId, "my-flow") + + verify(definitionChecker).assertBuildingBlockDefinitionExists(bbId) + assertThat(result).isEqualTo(mockDefinition) + } + + @Test + fun `getFormFlowDefinition returns null when definition not found`() { + whenever(formFlowService.findDefinitionOrNull(eq("missing-flow"), eq(bbId))).thenReturn(null) + + val result = service.getFormFlowDefinition(bbId, "missing-flow") + + assertThat(result).isNull() + } + + @Test + fun `save asserts update permission and delegates to formFlowService`() { + val dto = FormFlowDefinitionDto(key = "my-flow", startStep = "first", steps = emptyList()) + val mockDefinition = mock() + whenever(formFlowService.save(any())).thenReturn(mockDefinition) + + val result = service.save(bbId, dto) + + verify(definitionChecker).assertCanUpdateBuildingBlockDefinition(bbId) + assertThat(result).isEqualTo(mockDefinition) + } + + @Test + fun `delete asserts update permission and delegates to formFlowService`() { + service.delete(bbId, "my-flow") + + verify(definitionChecker).assertCanUpdateBuildingBlockDefinition(bbId) + verify(formFlowService).deleteByKeyAndBuildingBlockDefinition(eq("my-flow"), eq(bbId)) + } +} diff --git a/backend/building-block/src/test/kotlin/com/ritense/buildingblock/web/rest/BuildingBlockFormFlowManagementResourceIT.kt b/backend/building-block/src/test/kotlin/com/ritense/buildingblock/web/rest/BuildingBlockFormFlowManagementResourceIT.kt new file mode 100644 index 0000000000..8a031ab2d2 --- /dev/null +++ b/backend/building-block/src/test/kotlin/com/ritense/buildingblock/web/rest/BuildingBlockFormFlowManagementResourceIT.kt @@ -0,0 +1,234 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.buildingblock.web.rest + +import com.fasterxml.jackson.databind.ObjectMapper +import com.ritense.buildingblock.BaseIntegrationTest +import com.ritense.buildingblock.service.BuildingBlockFormFlowDefinitionImporter +import com.ritense.buildingblock.service.BuildingBlockFormFlowDefinitionService +import com.ritense.formflow.domain.definition.FormFlowDefinition +import com.ritense.formflow.domain.definition.FormFlowDefinitionId +import com.ritense.formflow.repository.FormFlowDefinitionRepository +import com.ritense.formflow.web.rest.result.FormFlowDefinitionDto +import com.ritense.valtimo.contract.buildingblock.BuildingBlockDefinitionId +import com.ritense.valtimo.contract.case_.CaseDefinitionId +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.doNothing +import org.mockito.kotlin.doReturn +import org.mockito.kotlin.eq +import org.mockito.kotlin.whenever +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.http.MediaType +import org.springframework.security.test.context.support.WithMockUser +import org.springframework.test.context.bean.override.mockito.MockitoSpyBean +import org.springframework.test.web.servlet.MockMvc +import org.springframework.test.web.servlet.delete +import org.springframework.test.web.servlet.get +import org.springframework.test.web.servlet.post +import org.springframework.test.web.servlet.put + +/** + * Integration test for [BuildingBlockFormFlowManagementResource]. + * + * Seeds the database with both a building-block-linked and a case-linked form flow to verify + * that the building-block endpoints only expose form flows that belong to the requested + * building block definition — not those belonging to a case definition. + * + * Write operation tests (POST/PUT/DELETE) stub the service layer because the auto-deployed + * `bezwaar` building block is marked `final`, which prevents real write operations. + */ +class BuildingBlockFormFlowManagementResourceIT @Autowired constructor( + private val mockMvc: MockMvc, + private val objectMapper: ObjectMapper, +) : BaseIntegrationTest() { + + @MockitoSpyBean + lateinit var buildingBlockFormFlowDefinitionService: BuildingBlockFormFlowDefinitionService + + @MockitoSpyBean + lateinit var buildingBlockFormFlowDefinitionImporter: BuildingBlockFormFlowDefinitionImporter + + @Autowired + lateinit var formFlowDefinitionRepository: FormFlowDefinitionRepository + + private val base = "/api/management/v1/building-block" + + // bezwaar/1.0.0 is auto-deployed from test resources — assertBuildingBlockDefinitionExists passes + private val bbId = BuildingBlockDefinitionId("bezwaar", "1.0.0") + private val caseId = CaseDefinitionId("bb-case", "1.0.0") + + private val bbDefinitionId = FormFlowDefinitionId.existingId("bb-test-flow", bbId) + private val caseDefinitionId = FormFlowDefinitionId.existingId("case-test-flow", caseId) + + @BeforeEach + fun seedDatabase() { + // Both a building-block-linked and a case-linked form flow are present in the DB + formFlowDefinitionRepository.save(FormFlowDefinition(bbDefinitionId, "start", emptySet())) + formFlowDefinitionRepository.save(FormFlowDefinition(caseDefinitionId, "start", emptySet())) + } + + @AfterEach + fun cleanDatabase() { + listOf(bbDefinitionId, caseDefinitionId).forEach { id -> + if (formFlowDefinitionRepository.existsById(id)) { + formFlowDefinitionRepository.deleteById(id) + } + } + } + + // ----------------------------------------------------------------------- + // Data-isolation tests — the spy calls through to the real service so the + // actual database query is exercised. + // ----------------------------------------------------------------------- + + @Test + @WithMockUser + fun `GET all returns only building-block form flows, not case form flows`() { + mockMvc.get("$base/{key}/version/{versionTag}/form-flow-definition", "bezwaar", "1.0.0") + .andExpect { + status { isOk() } + jsonPath("$.content[?(@.key=='bb-test-flow')]") { exists() } + jsonPath("$.content[?(@.key=='case-test-flow')]") { doesNotExist() } + } + } + + @Test + @WithMockUser + fun `GET by key returns the building-block form flow`() { + mockMvc.get("$base/{key}/version/{versionTag}/form-flow-definition/{definitionKey}", + "bezwaar", "1.0.0", "bb-test-flow") + .andExpect { + status { isOk() } + jsonPath("$.key") { value("bb-test-flow") } + } + } + + @Test + @WithMockUser + fun `GET by key returns 404 for a case form flow that has the same key`() { + // case-test-flow exists in the DB but belongs to a case, not this building block + mockMvc.get("$base/{key}/version/{versionTag}/form-flow-definition/{definitionKey}", + "bezwaar", "1.0.0", "case-test-flow") + .andExpect { status { isNotFound() } } + } + + // ----------------------------------------------------------------------- + // Write operation tests — bezwaar is final so the real checker blocks writes; + // we stub the service layer to test the resource logic in isolation. + // ----------------------------------------------------------------------- + + @Test + @WithMockUser + fun `POST create returns 400 when definition already exists`() { + val dto = FormFlowDefinitionDto(key = "bb-test-flow", startStep = "start", steps = emptyList()) + + doReturn(formFlowDefinitionRepository.findById(bbDefinitionId).orElseThrow()) + .whenever(buildingBlockFormFlowDefinitionService) + .getFormFlowDefinition(eq(bbId), eq("bb-test-flow")) + + mockMvc.post("$base/{key}/version/{versionTag}/form-flow-definition", "bezwaar", "1.0.0") { + contentType = MediaType.APPLICATION_JSON + content = objectMapper.writeValueAsBytes(dto) + }.andExpect { status { isBadRequest() } } + } + + @Test + @WithMockUser + fun `POST create returns 200 with the saved definition`() { + val newDefinitionId = FormFlowDefinitionId.existingId("new-flow", bbId) + val savedDefinition = FormFlowDefinition(newDefinitionId, "start", emptySet()) + val dto = FormFlowDefinitionDto(key = "new-flow", startStep = "start", steps = emptyList()) + + doReturn(null) + .whenever(buildingBlockFormFlowDefinitionService) + .getFormFlowDefinition(eq(bbId), eq("new-flow")) + doReturn(savedDefinition) + .whenever(buildingBlockFormFlowDefinitionService) + .save(eq(bbId), any()) + + mockMvc.post("$base/{key}/version/{versionTag}/form-flow-definition", "bezwaar", "1.0.0") { + contentType = MediaType.APPLICATION_JSON + content = objectMapper.writeValueAsBytes(dto) + }.andExpect { + status { isOk() } + jsonPath("$.key") { value("new-flow") } + } + } + + @Test + @WithMockUser + fun `PUT update returns 200 with the updated definition`() { + val updatedDefinition = FormFlowDefinition(bbDefinitionId, "updated-start", emptySet()) + val dto = FormFlowDefinitionDto(key = "bb-test-flow", startStep = "updated-start", steps = emptyList()) + + doReturn(updatedDefinition) + .whenever(buildingBlockFormFlowDefinitionService) + .save(eq(bbId), any()) + + mockMvc.put("$base/{key}/version/{versionTag}/form-flow-definition/{definitionKey}", + "bezwaar", "1.0.0", "bb-test-flow") { + contentType = MediaType.APPLICATION_JSON + content = objectMapper.writeValueAsBytes(dto) + }.andExpect { + status { isOk() } + jsonPath("$.key") { value("bb-test-flow") } + } + } + + @Test + @WithMockUser + fun `PUT update returns 403 when definition is auto-deployed (read-only)`() { + val dto = FormFlowDefinitionDto(key = "readonly-flow", startStep = "start", steps = emptyList()) + + doReturn(true) + .whenever(buildingBlockFormFlowDefinitionImporter) + .isAutoDeployed(eq("readonly-flow")) + + mockMvc.put("$base/{key}/version/{versionTag}/form-flow-definition/{definitionKey}", + "bezwaar", "1.0.0", "readonly-flow") { + contentType = MediaType.APPLICATION_JSON + content = objectMapper.writeValueAsBytes(dto) + }.andExpect { status { isForbidden() } } + } + + @Test + @WithMockUser + fun `DELETE returns 200`() { + doNothing() + .whenever(buildingBlockFormFlowDefinitionService) + .delete(eq(bbId), eq("bb-test-flow")) + + mockMvc.delete("$base/{key}/version/{versionTag}/form-flow-definition/{definitionKey}", + "bezwaar", "1.0.0", "bb-test-flow") + .andExpect { status { isOk() } } + } + + @Test + @WithMockUser + fun `DELETE returns 403 when definition is auto-deployed (read-only)`() { + doReturn(true) + .whenever(buildingBlockFormFlowDefinitionImporter) + .isAutoDeployed(eq("readonly-flow")) + + mockMvc.delete("$base/{key}/version/{versionTag}/form-flow-definition/{definitionKey}", + "bezwaar", "1.0.0", "readonly-flow") + .andExpect { status { isForbidden() } } + } +} diff --git a/backend/building-block/src/test/resources/config/building-block/bezwaar/1-0-0/bpmn/building-block-form-flow-process.bpmn b/backend/building-block/src/test/resources/config/building-block/bezwaar/1-0-0/bpmn/building-block-form-flow-process.bpmn new file mode 100644 index 0000000000..bb0024210d --- /dev/null +++ b/backend/building-block/src/test/resources/config/building-block/bezwaar/1-0-0/bpmn/building-block-form-flow-process.bpmn @@ -0,0 +1,39 @@ + + + + + Flow_1 + + + Flow_1 + Flow_2 + + + Flow_2 + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/backend/building-block/src/test/resources/config/building-block/bezwaar/1-0-0/form/bb-form.form.json b/backend/building-block/src/test/resources/config/building-block/bezwaar/1-0-0/form/bb-form.form.json new file mode 100644 index 0000000000..2a2584d34e --- /dev/null +++ b/backend/building-block/src/test/resources/config/building-block/bezwaar/1-0-0/form/bb-form.form.json @@ -0,0 +1,11 @@ +{ + "display": "form", + "components": [ + { + "label": "Straatnaam", + "key": "straatnaam", + "type": "textfield", + "input": true + } + ] +} diff --git a/backend/building-block/src/test/resources/config/case/bb-case/1-0-0/bpmn/building-block-form-flow-main.bpmn b/backend/building-block/src/test/resources/config/case/bb-case/1-0-0/bpmn/building-block-form-flow-main.bpmn new file mode 100644 index 0000000000..c5dbdce7d3 --- /dev/null +++ b/backend/building-block/src/test/resources/config/case/bb-case/1-0-0/bpmn/building-block-form-flow-main.bpmn @@ -0,0 +1,42 @@ + + + + + Flow_1 + + + + + + Flow_1 + Flow_2 + + + Flow_2 + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/backend/core/src/main/resources/config/liquibase/13-22-0/13-22-0-master.xml b/backend/core/src/main/resources/config/liquibase/13-22-0/13-22-0-master.xml index aaedf7e46b..facb5ab2d1 100644 --- a/backend/core/src/main/resources/config/liquibase/13-22-0/13-22-0-master.xml +++ b/backend/core/src/main/resources/config/liquibase/13-22-0/13-22-0-master.xml @@ -21,5 +21,7 @@ xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.4.xsd"> + + diff --git a/backend/core/src/main/resources/config/liquibase/13-22-0/20260330-form-flow-blueprint-support.xml b/backend/core/src/main/resources/config/liquibase/13-22-0/20260330-form-flow-blueprint-support.xml new file mode 100644 index 0000000000..9813e43841 --- /dev/null +++ b/backend/core/src/main/resources/config/liquibase/13-22-0/20260330-form-flow-blueprint-support.xml @@ -0,0 +1,145 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/backend/exporter/src/main/kotlin/com/ritense/exporter/request/BuildingBlockFormFlowDefinitionExportRequest.kt b/backend/exporter/src/main/kotlin/com/ritense/exporter/request/BuildingBlockFormFlowDefinitionExportRequest.kt new file mode 100644 index 0000000000..80ac449975 --- /dev/null +++ b/backend/exporter/src/main/kotlin/com/ritense/exporter/request/BuildingBlockFormFlowDefinitionExportRequest.kt @@ -0,0 +1,25 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.exporter.request + +import com.ritense.valtimo.contract.buildingblock.BuildingBlockDefinitionId + +data class BuildingBlockFormFlowDefinitionExportRequest( + val formFlowDefinitionKey: String, + override val buildingBlockDefinitionId: BuildingBlockDefinitionId, + override val required: Boolean = true, +) : ExportRequest(required) diff --git a/backend/form-flow/src/main/kotlin/com/ritense/formflow/FormFlowProcessLinkActivityHandler.kt b/backend/form-flow/src/main/kotlin/com/ritense/formflow/FormFlowProcessLinkActivityHandler.kt index c9cd1d5769..0d5605e5e6 100644 --- a/backend/form-flow/src/main/kotlin/com/ritense/formflow/FormFlowProcessLinkActivityHandler.kt +++ b/backend/form-flow/src/main/kotlin/com/ritense/formflow/FormFlowProcessLinkActivityHandler.kt @@ -20,6 +20,7 @@ import com.ritense.authorization.AuthorizationContext import com.ritense.document.domain.impl.JsonSchemaDocument import com.ritense.document.service.DocumentService import com.ritense.formflow.domain.FormFlowProcessLink +import com.ritense.formflow.domain.definition.FormFlowDefinition import com.ritense.formflow.domain.instance.FormFlowInstance import com.ritense.formflow.service.FormFlowService import com.ritense.logging.LoggableResource @@ -122,12 +123,23 @@ class FormFlowProcessLinkActivityHandler( private fun createFormFlowInstance(task: OperatonTask, processLink: FormFlowProcessLink): FormFlowInstance { val additionalProperties = getAdditionalProperties(task) - val processDefinitionCaseDefinitionLink = processDefinitionCaseDefinitionService - .findByProcessDefinitionId(ProcessDefinitionId(processLink.processDefinitionId)) - - val formFlowDefinition = formFlowService - .findDefinition(processLink.formFlowDefinitionKey, processDefinitionCaseDefinitionLink.id.caseDefinitionId)!! + val formFlowDefinition = findFormFlowDefinition(processLink) return formFlowService.save(formFlowDefinition.createInstance(additionalProperties)) } + private fun findFormFlowDefinition(processLink: FormFlowProcessLink): FormFlowDefinition { + val caseLink = try { + processDefinitionCaseDefinitionService + .findByProcessDefinitionId(ProcessDefinitionId(processLink.processDefinitionId)) + } catch (e: Exception) { + null + } + return if (caseLink != null) { + formFlowService.findDefinition(processLink.formFlowDefinitionKey, caseLink.id.caseDefinitionId) + } else { + formFlowService.findDefinitionByKey(processLink.formFlowDefinitionKey) + ?: throw IllegalStateException("FormFlow definition '${processLink.formFlowDefinitionKey}' not found") + } + } + } diff --git a/backend/form-flow/src/main/kotlin/com/ritense/formflow/autoconfigure/FormFlowAutoConfiguration.kt b/backend/form-flow/src/main/kotlin/com/ritense/formflow/autoconfigure/FormFlowAutoConfiguration.kt index e35532e460..f2c1b5c506 100644 --- a/backend/form-flow/src/main/kotlin/com/ritense/formflow/autoconfigure/FormFlowAutoConfiguration.kt +++ b/backend/form-flow/src/main/kotlin/com/ritense/formflow/autoconfigure/FormFlowAutoConfiguration.kt @@ -57,6 +57,7 @@ import com.ritense.processdocument.service.ProcessDefinitionCaseDefinitionServic import com.ritense.processdocument.service.ProcessDocumentService import com.ritense.processlink.service.ProcessLinkActivityHandler import com.ritense.valtimo.operaton.service.OperatonRepositoryService +import com.ritense.valtimo.contract.buildingblock.BuildingBlockDefinitionChecker import com.ritense.valtimo.contract.case_.CaseDefinitionChecker import com.ritense.valtimo.service.OperatonTaskService import com.ritense.valueresolver.ValueResolverService @@ -119,13 +120,15 @@ class FormFlowAutoConfiguration { formFlowAdditionalPropertiesSearchRepository: FormFlowAdditionalPropertiesSearchRepository, formFlowStepTypeHandlers: List, caseDefinitionChecker: CaseDefinitionChecker, + buildingBlockDefinitionChecker: BuildingBlockDefinitionChecker, ): FormFlowService { return FormFlowService( formFlowDefinitionRepository, formFlowInstanceRepository, formFlowAdditionalPropertiesSearchRepository, formFlowStepTypeHandlers, - caseDefinitionChecker + caseDefinitionChecker, + buildingBlockDefinitionChecker ) } diff --git a/backend/form-flow/src/main/kotlin/com/ritense/formflow/common/ValtimoFormFlow.kt b/backend/form-flow/src/main/kotlin/com/ritense/formflow/common/ValtimoFormFlow.kt index bc142a8a42..5825fae4a3 100644 --- a/backend/form-flow/src/main/kotlin/com/ritense/formflow/common/ValtimoFormFlow.kt +++ b/backend/form-flow/src/main/kotlin/com/ritense/formflow/common/ValtimoFormFlow.kt @@ -117,8 +117,8 @@ open class ValtimoFormFlow( documentService.createDocument( NewDocumentRequest( documentDefinitionName, - formFlowInstance.formFlowDefinition.id.caseDefinitionId.key, - formFlowInstance.formFlowDefinition.id.caseDefinitionId.versionTag.version, + formFlowInstance.formFlowDefinition.id.caseDefinitionId!!.key, + formFlowInstance.formFlowDefinition.id.caseDefinitionId!!.versionTag.version, submittedByType["doc"] as JsonNode ) ) diff --git a/backend/form-flow/src/main/kotlin/com/ritense/formflow/domain/definition/FormFlowDefinitionBlueprintId.kt b/backend/form-flow/src/main/kotlin/com/ritense/formflow/domain/definition/FormFlowDefinitionBlueprintId.kt new file mode 100644 index 0000000000..a8a501d672 --- /dev/null +++ b/backend/form-flow/src/main/kotlin/com/ritense/formflow/domain/definition/FormFlowDefinitionBlueprintId.kt @@ -0,0 +1,72 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.formflow.domain.definition + +import com.ritense.valtimo.contract.blueprint.BlueprintOwner +import com.ritense.valtimo.contract.blueprint.BlueprintOwnerHelper +import com.ritense.valtimo.contract.blueprint.BlueprintType +import com.ritense.valtimo.contract.buildingblock.BuildingBlockDefinitionId +import com.ritense.valtimo.contract.case_.CaseDefinitionId +import com.ritense.valtimo.contract.repository.SemverConverter +import jakarta.persistence.Column +import jakarta.persistence.Convert +import jakarta.persistence.Embeddable +import jakarta.persistence.EnumType +import jakarta.persistence.Enumerated +import org.semver4j.Semver +import java.io.Serializable + +@Embeddable +class FormFlowDefinitionBlueprintId( + @Enumerated(EnumType.STRING) + @Column(name = "blueprint_type", length = 40, nullable = false) + override var blueprintType: BlueprintType, + + @Column(name = "blueprint_key", length = 256, nullable = false) + override var blueprintKey: String, + + @Convert(converter = SemverConverter::class) + @Column(name = "blueprint_version_tag", nullable = false) + override var blueprintVersionTag: Semver, +) : BlueprintOwner, Serializable { + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is FormFlowDefinitionBlueprintId) return false + return BlueprintOwnerHelper.areEqual(this, other) + } + + override fun hashCode(): Int = BlueprintOwnerHelper.computeHashCode(this) + + override fun toString(): String = "$blueprintType:$blueprintKey:$blueprintVersionTag" + + companion object { + @JvmStatic + fun forCase(caseDefinitionId: CaseDefinitionId): FormFlowDefinitionBlueprintId { + return BlueprintOwnerHelper.createForCase(caseDefinitionId) { type, key, version -> + FormFlowDefinitionBlueprintId(type, key, version) + } + } + + @JvmStatic + fun forBuildingBlock(buildingBlockDefinitionId: BuildingBlockDefinitionId): FormFlowDefinitionBlueprintId { + return BlueprintOwnerHelper.createForBuildingBlock(buildingBlockDefinitionId) { type, key, version -> + FormFlowDefinitionBlueprintId(type, key, version) + } + } + } +} diff --git a/backend/form-flow/src/main/kotlin/com/ritense/formflow/domain/definition/FormFlowDefinitionId.kt b/backend/form-flow/src/main/kotlin/com/ritense/formflow/domain/definition/FormFlowDefinitionId.kt index 3674ac155f..e299c3b2dd 100644 --- a/backend/form-flow/src/main/kotlin/com/ritense/formflow/domain/definition/FormFlowDefinitionId.kt +++ b/backend/form-flow/src/main/kotlin/com/ritense/formflow/domain/definition/FormFlowDefinitionId.kt @@ -1,5 +1,5 @@ /* - * Copyright 2015-2024 Ritense BV, the Netherlands. + * Copyright 2015-2026 Ritense BV, the Netherlands. * * Licensed under EUPL, Version 1.2 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ package com.ritense.formflow.domain.definition import com.ritense.formflow.domain.AbstractId +import com.ritense.valtimo.contract.buildingblock.BuildingBlockDefinitionId import com.ritense.valtimo.contract.case_.CaseDefinitionId import jakarta.persistence.Column import jakarta.persistence.Embeddable @@ -30,10 +31,13 @@ data class FormFlowDefinitionId( val key: String, @Embedded - val caseDefinitionId: CaseDefinitionId + val blueprintId: FormFlowDefinitionBlueprintId ) : AbstractId() { + val caseDefinitionId: CaseDefinitionId? + get() = blueprintId.asCaseDefinitionId() + override fun toString(): String { return key } @@ -55,15 +59,23 @@ data class FormFlowDefinitionId( companion object { fun newId(key: String, caseDefinitionId: CaseDefinitionId): FormFlowDefinitionId { - return FormFlowDefinitionId(key, caseDefinitionId).newIdentity() + return FormFlowDefinitionId(key, FormFlowDefinitionBlueprintId.forCase(caseDefinitionId)).newIdentity() + } + + fun newId(key: String, buildingBlockDefinitionId: BuildingBlockDefinitionId): FormFlowDefinitionId { + return FormFlowDefinitionId(key, FormFlowDefinitionBlueprintId.forBuildingBlock(buildingBlockDefinitionId)).newIdentity() } fun existingId(id: FormFlowDefinitionId): FormFlowDefinitionId { - return FormFlowDefinitionId(id.key, id.caseDefinitionId) + return FormFlowDefinitionId(id.key, id.blueprintId) } fun existingId(key: String, caseDefinitionId: CaseDefinitionId): FormFlowDefinitionId { - return FormFlowDefinitionId(key, caseDefinitionId) + return FormFlowDefinitionId(key, FormFlowDefinitionBlueprintId.forCase(caseDefinitionId)) + } + + fun existingId(key: String, buildingBlockDefinitionId: BuildingBlockDefinitionId): FormFlowDefinitionId { + return FormFlowDefinitionId(key, FormFlowDefinitionBlueprintId.forBuildingBlock(buildingBlockDefinitionId)) } } -} \ No newline at end of file +} diff --git a/backend/form-flow/src/main/kotlin/com/ritense/formflow/domain/definition/FormFlowStepId.kt b/backend/form-flow/src/main/kotlin/com/ritense/formflow/domain/definition/FormFlowStepId.kt index b0e08257f5..4c9947c741 100644 --- a/backend/form-flow/src/main/kotlin/com/ritense/formflow/domain/definition/FormFlowStepId.kt +++ b/backend/form-flow/src/main/kotlin/com/ritense/formflow/domain/definition/FormFlowStepId.kt @@ -35,8 +35,9 @@ data class FormFlowStepId( @ManyToOne(targetEntity = FormFlowDefinition::class, fetch = FetchType.LAZY) @JoinColumns( JoinColumn(name = "form_flow_definition_key", referencedColumnName = "form_flow_definition_key"), - JoinColumn(name = "case_definition_key", referencedColumnName = "case_definition_key"), - JoinColumn(name = "case_definition_version_tag", referencedColumnName = "case_definition_version_tag") + JoinColumn(name = "blueprint_type", referencedColumnName = "blueprint_type"), + JoinColumn(name = "blueprint_key", referencedColumnName = "blueprint_key"), + JoinColumn(name = "blueprint_version_tag", referencedColumnName = "blueprint_version_tag") ) var formFlowDefinition: FormFlowDefinition? = null ) : AbstractId() { diff --git a/backend/form-flow/src/main/kotlin/com/ritense/formflow/domain/instance/FormFlowInstance.kt b/backend/form-flow/src/main/kotlin/com/ritense/formflow/domain/instance/FormFlowInstance.kt index a08e42554d..0667f7ea92 100644 --- a/backend/form-flow/src/main/kotlin/com/ritense/formflow/domain/instance/FormFlowInstance.kt +++ b/backend/form-flow/src/main/kotlin/com/ritense/formflow/domain/instance/FormFlowInstance.kt @@ -45,8 +45,9 @@ class FormFlowInstance( @ManyToOne(fetch = FetchType.LAZY, optional = false) @JoinColumns( JoinColumn(name = "form_flow_definition_key", referencedColumnName = "form_flow_definition_key"), - JoinColumn(name = "case_definition_key", referencedColumnName = "case_definition_key"), - JoinColumn(name = "case_definition_version_tag", referencedColumnName = "case_definition_version_tag") + JoinColumn(name = "blueprint_type", referencedColumnName = "blueprint_type"), + JoinColumn(name = "blueprint_key", referencedColumnName = "blueprint_key"), + JoinColumn(name = "blueprint_version_tag", referencedColumnName = "blueprint_version_tag") ) val formFlowDefinition: FormFlowDefinition, @Embedded diff --git a/backend/form-flow/src/main/kotlin/com/ritense/formflow/handler/FormFlowStepTypeFormHandler.kt b/backend/form-flow/src/main/kotlin/com/ritense/formflow/handler/FormFlowStepTypeFormHandler.kt index c0a4cac819..e6fa5eea59 100644 --- a/backend/form-flow/src/main/kotlin/com/ritense/formflow/handler/FormFlowStepTypeFormHandler.kt +++ b/backend/form-flow/src/main/kotlin/com/ritense/formflow/handler/FormFlowStepTypeFormHandler.kt @@ -59,8 +59,10 @@ class FormFlowStepTypeFormHandler( val stepDefinitionType = stepInstance.definition.type require(stepDefinitionType.name == getType()) val formDefinitionName = (stepDefinitionType.properties as FormStepTypeProperties).definition - val caseDefinitionId = stepInstance.instance.formFlowDefinition.id.caseDefinitionId - formIoFormDefinitionService.getFormDefinitionByName(formDefinitionName, caseDefinitionId) + val blueprintId = stepInstance.instance.formFlowDefinition.id.blueprintId + val blueprintRef = blueprintId.asCaseDefinitionId() ?: blueprintId.asBuildingBlockDefinitionId() + ?: throw IllegalStateException("Unsupported blueprint type: ${blueprintId.blueprintType}") + formIoFormDefinitionService.getFormDefinitionByName(formDefinitionName, blueprintRef) .orElseThrow { IllegalStateException("No FormDefinition found by name $formDefinitionName") } } } diff --git a/backend/form-flow/src/main/kotlin/com/ritense/formflow/importer/FormFlowDefinitionImporter.kt b/backend/form-flow/src/main/kotlin/com/ritense/formflow/importer/FormFlowDefinitionImporter.kt index 2f85a1b04d..0b3165f121 100644 --- a/backend/form-flow/src/main/kotlin/com/ritense/formflow/importer/FormFlowDefinitionImporter.kt +++ b/backend/form-flow/src/main/kotlin/com/ritense/formflow/importer/FormFlowDefinitionImporter.kt @@ -71,17 +71,15 @@ class FormFlowDefinitionImporter( try { val existingDefinition = formFlowService.findDefinitionOrNull(formFlowKey, caseDefinitionId) - var definitionId = FormFlowDefinitionId.newId(formFlowKey, caseDefinitionId) + val definitionId = FormFlowDefinitionId.newId(formFlowKey, caseDefinitionId) - if (existingDefinition != null) { - if (formFlowDefinitionConfig.contentEquals(existingDefinition)) { - logger.info("Form Flow already deployed - {}", definitionId.toString()) - return - } + if (existingDefinition != null && formFlowDefinitionConfig.contentEquals(existingDefinition)) { + logger.info { "Form Flow already deployed - $definitionId" } + return } formFlowService.save(formFlowDefinitionConfig.toDefinition(definitionId)) - logger.info("Deployed Form Flow - {}", definitionId.toString()) + logger.info { "Deployed Form Flow - $definitionId" } } catch (e: Exception) { throw RuntimeException("Failed to deploy Form Flow $formFlowKey", e) } diff --git a/backend/form-flow/src/main/kotlin/com/ritense/formflow/mapper/FormFlowProcessLinkMapper.kt b/backend/form-flow/src/main/kotlin/com/ritense/formflow/mapper/FormFlowProcessLinkMapper.kt index 6da285e3e5..f558c9eff6 100644 --- a/backend/form-flow/src/main/kotlin/com/ritense/formflow/mapper/FormFlowProcessLinkMapper.kt +++ b/backend/form-flow/src/main/kotlin/com/ritense/formflow/mapper/FormFlowProcessLinkMapper.kt @@ -38,6 +38,7 @@ import com.ritense.processlink.web.rest.dto.ProcessLinkResponseDto import com.ritense.processlink.web.rest.dto.ProcessLinkUpdateRequestDto import com.ritense.valtimo.contract.BlueprintId import com.ritense.valtimo.contract.annotation.SkipComponentScan +import com.ritense.valtimo.contract.buildingblock.BuildingBlockDefinitionId import com.ritense.valtimo.contract.case_.CaseDefinitionId import com.ritense.valtimo.operaton.domain.OperatonProcessDefinition import org.operaton.bpm.engine.repository.ProcessDefinition @@ -133,13 +134,8 @@ class FormFlowProcessLinkMapper( override fun toNewProcessLink(createRequestDto: ProcessLinkCreateRequestDto, blueprintId: BlueprintId?): ProcessLink { return withLoggingContext(ProcessDefinition::class, createRequestDto.processDefinitionId) { - val caseDefinitionId = blueprintId as? CaseDefinitionId - ?: throw RuntimeException("Case definition id is required for creating a new process link with form flow") - createRequestDto as FormFlowProcessLinkCreateRequestDto - if (formFlowService.findDefinition(createRequestDto.formFlowDefinitionKey, caseDefinitionId) == null) { - throw RuntimeException("FormFlow definition not found with id ${createRequestDto.formFlowDefinitionKey}") - } + assertFormFlowDefinitionExists(createRequestDto.formFlowDefinitionKey, blueprintId) FormFlowProcessLink( id = UUID.randomUUID(), processDefinitionId = createRequestDto.processDefinitionId, @@ -159,12 +155,8 @@ class FormFlowProcessLinkMapper( blueprintId: BlueprintId? ): ProcessLink { return withLoggingContext(ProcessLink::class, processLinkToUpdate.id) { - val caseDefinitionId = blueprintId as? CaseDefinitionId - ?: throw RuntimeException("Case definition id is required for updating a process link with form flow") updateRequestDto as FormFlowProcessLinkUpdateRequestDto - if (formFlowService.findDefinition(updateRequestDto.formFlowDefinitionKey, caseDefinitionId) == null) { - throw RuntimeException("FormFlow definition not found with id ${updateRequestDto.formFlowDefinitionKey}") - } + assertFormFlowDefinitionExists(updateRequestDto.formFlowDefinitionKey, blueprintId) FormFlowProcessLink( id = updateRequestDto.id, processDefinitionId = processLinkToUpdate.processDefinitionId, @@ -178,6 +170,17 @@ class FormFlowProcessLinkMapper( } } + private fun assertFormFlowDefinitionExists(formFlowDefinitionKey: String, blueprintId: BlueprintId?) { + val definition = when (blueprintId) { + is CaseDefinitionId -> formFlowService.findDefinition(formFlowDefinitionKey, blueprintId) + is BuildingBlockDefinitionId -> formFlowService.findDefinition(formFlowDefinitionKey, blueprintId) + else -> throw RuntimeException("A blueprint id (case or building block) is required for a form flow process link") + } + if (definition == null) { + throw RuntimeException("FormFlow definition not found with id $formFlowDefinitionKey") + } + } + override fun createRelatedExportRequests( processLink: ProcessLink, caseDefinitionId: CaseDefinitionId diff --git a/backend/form-flow/src/main/kotlin/com/ritense/formflow/repository/FormFlowDefinitionRepository.kt b/backend/form-flow/src/main/kotlin/com/ritense/formflow/repository/FormFlowDefinitionRepository.kt index 0f42cb7316..4aa6acbf5d 100644 --- a/backend/form-flow/src/main/kotlin/com/ritense/formflow/repository/FormFlowDefinitionRepository.kt +++ b/backend/form-flow/src/main/kotlin/com/ritense/formflow/repository/FormFlowDefinitionRepository.kt @@ -1,5 +1,5 @@ /* - * Copyright 2015-2024 Ritense BV, the Netherlands. + * Copyright 2015-2026 Ritense BV, the Netherlands. * * Licensed under EUPL, Version 1.2 (the "License"); * you may not use this file except in compliance with the License. @@ -18,16 +18,25 @@ package com.ritense.formflow.repository import com.ritense.formflow.domain.definition.FormFlowDefinition import com.ritense.formflow.domain.definition.FormFlowDefinitionId -import com.ritense.valtimo.contract.case_.CaseDefinitionId +import com.ritense.valtimo.contract.blueprint.BlueprintType +import org.semver4j.Semver import org.springframework.data.domain.Page import org.springframework.data.domain.Pageable import org.springframework.data.jpa.repository.JpaRepository +import org.springframework.data.jpa.repository.Query interface FormFlowDefinitionRepository : JpaRepository { - fun findAllByIdCaseDefinitionId(caseDefinitionId: CaseDefinitionId): List + @Query("SELECT f FROM FormFlowDefinition f WHERE f.id.blueprintId.blueprintType = :blueprintType AND f.id.blueprintId.blueprintKey = :blueprintKey AND f.id.blueprintId.blueprintVersionTag = :blueprintVersionTag") + fun findAllByBlueprintId(blueprintType: BlueprintType, blueprintKey: String, blueprintVersionTag: Semver): List - fun findAllByIdCaseDefinitionId(caseDefinitionId: CaseDefinitionId, pageable: Pageable): Page + @Query("SELECT f FROM FormFlowDefinition f WHERE f.id.blueprintId.blueprintType = :blueprintType AND f.id.blueprintId.blueprintKey = :blueprintKey AND f.id.blueprintId.blueprintVersionTag = :blueprintVersionTag") + fun findAllByBlueprintId(blueprintType: BlueprintType, blueprintKey: String, blueprintVersionTag: Semver, pageable: Pageable): Page - fun deleteAllByIdCaseDefinitionId(caseDefinitionId: CaseDefinitionId) -} \ No newline at end of file + @Query("DELETE FROM FormFlowDefinition f WHERE f.id.blueprintId.blueprintType = :blueprintType AND f.id.blueprintId.blueprintKey = :blueprintKey AND f.id.blueprintId.blueprintVersionTag = :blueprintVersionTag") + fun deleteAllByBlueprintId(blueprintType: BlueprintType, blueprintKey: String, blueprintVersionTag: Semver) + + @Query("SELECT f FROM FormFlowDefinition f WHERE f.id.key = :key") + fun findAllByKey(key: String): List + +} diff --git a/backend/form-flow/src/main/kotlin/com/ritense/formflow/service/FormFlowService.kt b/backend/form-flow/src/main/kotlin/com/ritense/formflow/service/FormFlowService.kt index 40056c88bf..4f66f3fc55 100644 --- a/backend/form-flow/src/main/kotlin/com/ritense/formflow/service/FormFlowService.kt +++ b/backend/form-flow/src/main/kotlin/com/ritense/formflow/service/FormFlowService.kt @@ -1,5 +1,5 @@ /* - * Copyright 2015-2024 Ritense BV, the Netherlands. + * Copyright 2015-2026 Ritense BV, the Netherlands. * * Licensed under EUPL, Version 1.2 (the "License"); * you may not use this file except in compliance with the License. @@ -30,6 +30,9 @@ import com.ritense.formflow.repository.FormFlowAdditionalPropertiesSearchReposit import com.ritense.formflow.repository.FormFlowDefinitionRepository import com.ritense.formflow.repository.FormFlowInstanceRepository import com.ritense.logging.withLoggingContext +import com.ritense.valtimo.contract.blueprint.BlueprintType +import com.ritense.valtimo.contract.buildingblock.BuildingBlockDefinitionChecker +import com.ritense.valtimo.contract.buildingblock.BuildingBlockDefinitionId import com.ritense.valtimo.contract.case_.CaseDefinitionChecker import com.ritense.valtimo.contract.case_.CaseDefinitionId import org.springframework.data.domain.Page @@ -44,6 +47,7 @@ class FormFlowService( private val formFlowAdditionalPropertiesSearchRepository: FormFlowAdditionalPropertiesSearchRepository, private val formFlowStepTypeHandlers: List, private val caseDefinitionChecker: CaseDefinitionChecker, + private val buildingBlockDefinitionChecker: BuildingBlockDefinitionChecker, ) { fun getFormFlowDefinitions(): List { @@ -51,16 +55,32 @@ class FormFlowService( } fun getFormFlowDefinitions(caseDefinitionId: CaseDefinitionId): List { - return formFlowDefinitionRepository.findAllByIdCaseDefinitionId(caseDefinitionId) + return formFlowDefinitionRepository.findAllByBlueprintId( + BlueprintType.CASE, caseDefinitionId.key, caseDefinitionId.versionTag + ) } fun getFormFlowDefinitions(caseDefinitionId: CaseDefinitionId, pageable: Pageable): Page { - return formFlowDefinitionRepository.findAllByIdCaseDefinitionId(caseDefinitionId, pageable) + return formFlowDefinitionRepository.findAllByBlueprintId( + BlueprintType.CASE, caseDefinitionId.key, caseDefinitionId.versionTag, pageable + ) + } + + fun getFormFlowDefinitions(buildingBlockDefinitionId: BuildingBlockDefinitionId): List { + return formFlowDefinitionRepository.findAllByBlueprintId( + BlueprintType.BUILDING_BLOCK, buildingBlockDefinitionId.key, buildingBlockDefinitionId.versionTag + ) + } + + fun getFormFlowDefinitions(buildingBlockDefinitionId: BuildingBlockDefinitionId, pageable: Pageable): Page { + return formFlowDefinitionRepository.findAllByBlueprintId( + BlueprintType.BUILDING_BLOCK, buildingBlockDefinitionId.key, buildingBlockDefinitionId.versionTag, pageable + ) } fun findDefinition(formFlowId: FormFlowDefinitionId): FormFlowDefinition { - withLoggingContext(FormFlowDefinition::class.java.canonicalName to formFlowId.toString()) { - return formFlowDefinitionRepository.getReferenceById(formFlowId) + return withLoggingContext(FormFlowDefinition::class.java.canonicalName to formFlowId.toString()) { + formFlowDefinitionRepository.getReferenceById(formFlowId) } } @@ -70,13 +90,39 @@ class FormFlowService( } } + fun findDefinition(formFlowDefinitionKey: String, buildingBlockDefinitionId: BuildingBlockDefinitionId): FormFlowDefinition { + return withLoggingContext(FormFlowDefinition::class.java.canonicalName to formFlowDefinitionKey) { + findDefinition(FormFlowDefinitionId.existingId(formFlowDefinitionKey, buildingBlockDefinitionId)) + } + } + fun findDefinitionOrNull(formFlowDefinitionKey: String, caseDefinitionId: CaseDefinitionId): FormFlowDefinition? { return formFlowDefinitionRepository.findByIdOrNull(FormFlowDefinitionId.existingId(formFlowDefinitionKey, caseDefinitionId)) } + fun findDefinitionOrNull(formFlowDefinitionKey: String, buildingBlockDefinitionId: BuildingBlockDefinitionId): FormFlowDefinition? { + return formFlowDefinitionRepository.findByIdOrNull(FormFlowDefinitionId.existingId(formFlowDefinitionKey, buildingBlockDefinitionId)) + } + + fun findDefinitionByKey(formFlowDefinitionKey: String): FormFlowDefinition? { + val definitions = formFlowDefinitionRepository.findAllByKey(formFlowDefinitionKey) + return when { + definitions.isEmpty() -> null + definitions.size == 1 -> definitions[0] + else -> throw IllegalStateException( + "Multiple form flow definitions found for key '$formFlowDefinitionKey' — specify the blueprint id" + ) + } + } + fun save(formFlowDefinition: FormFlowDefinition): FormFlowDefinition { return withLoggingContext(FormFlowDefinition::class.java.canonicalName to formFlowDefinition.id.toString()) { - caseDefinitionChecker.assertCanUpdateCaseDefinition(formFlowDefinition.id.caseDefinitionId) + val blueprintId = formFlowDefinition.id.blueprintId + if (blueprintId.isBuildingBlock()) { + buildingBlockDefinitionChecker.assertCanUpdateBuildingBlockDefinition(blueprintId.asBuildingBlockDefinitionId()!!) + } else { + caseDefinitionChecker.assertCanUpdateCaseDefinition(blueprintId.asCaseDefinitionId()!!) + } formFlowDefinitionRepository.save(formFlowDefinition) } } @@ -119,9 +165,23 @@ class FormFlowService( formFlowDefinitionRepository.deleteById(FormFlowDefinitionId.existingId(definitionKey, caseDefinitionId)) } + fun deleteByKeyAndBuildingBlockDefinition(definitionKey: String, buildingBlockDefinitionId: BuildingBlockDefinitionId) { + buildingBlockDefinitionChecker.assertCanUpdateBuildingBlockDefinition(buildingBlockDefinitionId) + formFlowDefinitionRepository.deleteById(FormFlowDefinitionId.existingId(definitionKey, buildingBlockDefinitionId)) + } + fun deleteAllByCaseDefinitionId(caseDefinitionId: CaseDefinitionId) { caseDefinitionChecker.assertCanUpdateCaseDefinition(caseDefinitionId) - formFlowDefinitionRepository.deleteAllByIdCaseDefinitionId(caseDefinitionId) + formFlowDefinitionRepository.deleteAllByBlueprintId( + BlueprintType.CASE, caseDefinitionId.key, caseDefinitionId.versionTag + ) + } + + fun deleteAllByBuildingBlockDefinitionId(buildingBlockDefinitionId: BuildingBlockDefinitionId) { + buildingBlockDefinitionChecker.assertCanUpdateBuildingBlockDefinition(buildingBlockDefinitionId) + formFlowDefinitionRepository.deleteAllByBlueprintId( + BlueprintType.BUILDING_BLOCK, buildingBlockDefinitionId.key, buildingBlockDefinitionId.versionTag + ) } fun getBreadcrumbs(instance: FormFlowInstance): List { diff --git a/backend/form-flow/src/main/kotlin/com/ritense/formflow/service/FormFlowValtimoService.kt b/backend/form-flow/src/main/kotlin/com/ritense/formflow/service/FormFlowValtimoService.kt index d32c763947..ed7a764e09 100644 --- a/backend/form-flow/src/main/kotlin/com/ritense/formflow/service/FormFlowValtimoService.kt +++ b/backend/form-flow/src/main/kotlin/com/ritense/formflow/service/FormFlowValtimoService.kt @@ -26,6 +26,7 @@ import com.ritense.form.service.impl.FormIoFormDefinitionService import com.ritense.formflow.domain.definition.configuration.step.FormStepTypeProperties import com.ritense.formflow.domain.instance.FormFlowInstance import com.ritense.logging.withLoggingContext +import com.ritense.valtimo.contract.BlueprintId import com.ritense.valtimo.contract.annotation.SkipComponentScan import com.ritense.valtimo.contract.json.patch.JsonPatchBuilder import org.springframework.stereotype.Service @@ -57,10 +58,13 @@ class FormFlowValtimoService( val jsonPatchBuilder = JsonPatchBuilder() val verifiedSubmissionData = objectMapper.createObjectNode() - val caseDefinitionId = formFlowInstance.formFlowDefinition.id.caseDefinitionId + val blueprintId = formFlowInstance.formFlowDefinition.id.blueprintId + val resolvedId: BlueprintId = blueprintId.asBuildingBlockDefinitionId() + ?: blueprintId.asCaseDefinitionId() + ?: throw IllegalStateException("Cannot resolve blueprint id for form '${currentStepTypeProperties.definition}'") val validJsonPointers = formDefinitionService.getFormDefinitionByName( currentStepTypeProperties.definition, - caseDefinitionId + resolvedId ) .orElseThrow().inputFields .mapNotNull { field -> FormIoFormDefinition.getKey(field).getOrNull() } diff --git a/backend/form-flow/src/main/kotlin/com/ritense/formflow/web/rest/FormFlowManagementResource.kt b/backend/form-flow/src/main/kotlin/com/ritense/formflow/web/rest/FormFlowManagementResource.kt index e170e0b6cb..189b0aae79 100644 --- a/backend/form-flow/src/main/kotlin/com/ritense/formflow/web/rest/FormFlowManagementResource.kt +++ b/backend/form-flow/src/main/kotlin/com/ritense/formflow/web/rest/FormFlowManagementResource.kt @@ -67,7 +67,7 @@ class FormFlowManagementResource( @PathVariable("versionTag") versionTag: String, ): ResponseEntity { val caseDefinitionId = CaseDefinitionId(caseDefinitionKey, versionTag) - val definition = formFlowService.findDefinition(FormFlowDefinitionId(definitionKey, caseDefinitionId)) + val definition = formFlowService.findDefinition(FormFlowDefinitionId.existingId(definitionKey, caseDefinitionId)) val readOnly = formFlowDefinitionImporter.isAutoDeployed(definition.id.key) return ResponseEntity.ok(FormFlowDefinitionDto.of(definition, readOnly)) } diff --git a/backend/form-flow/src/main/kotlin/com/ritense/formflow/web/rest/result/FormFlowDefinitionDto.kt b/backend/form-flow/src/main/kotlin/com/ritense/formflow/web/rest/result/FormFlowDefinitionDto.kt index 7b70d61e84..3170d9378b 100644 --- a/backend/form-flow/src/main/kotlin/com/ritense/formflow/web/rest/result/FormFlowDefinitionDto.kt +++ b/backend/form-flow/src/main/kotlin/com/ritense/formflow/web/rest/result/FormFlowDefinitionDto.kt @@ -1,5 +1,5 @@ /* - * Copyright 2015-2024 Ritense BV, the Netherlands. + * Copyright 2015-2026 Ritense BV, the Netherlands. * * Licensed under EUPL, Version 1.2 (the "License"); * you may not use this file except in compliance with the License. @@ -8,7 +8,7 @@ * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 * * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, + * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. @@ -17,8 +17,10 @@ package com.ritense.formflow.web.rest.result import com.ritense.formflow.domain.definition.FormFlowDefinition +import com.ritense.formflow.domain.definition.FormFlowDefinitionBlueprintId import com.ritense.formflow.domain.definition.FormFlowDefinitionId import com.ritense.formflow.domain.definition.configuration.FormFlowStep +import com.ritense.valtimo.contract.buildingblock.BuildingBlockDefinitionId import com.ritense.valtimo.contract.case_.CaseDefinitionId data class FormFlowDefinitionDto( @@ -28,7 +30,13 @@ data class FormFlowDefinitionDto( val readOnly: Boolean = false ) { fun toEntity(caseDefinitionId: CaseDefinitionId): FormFlowDefinition = FormFlowDefinition( - id = FormFlowDefinitionId(key, caseDefinitionId), + id = FormFlowDefinitionId(key, FormFlowDefinitionBlueprintId.forCase(caseDefinitionId)), + startStep = startStep, + steps = steps.map { it.toDefinition() }.toSet() + ) + + fun toEntity(buildingBlockDefinitionId: BuildingBlockDefinitionId): FormFlowDefinition = FormFlowDefinition( + id = FormFlowDefinitionId(key, FormFlowDefinitionBlueprintId.forBuildingBlock(buildingBlockDefinitionId)), startStep = startStep, steps = steps.map { it.toDefinition() }.toSet() ) diff --git a/backend/form-flow/src/test/kotlin/com/ritense/formflow/domain/FormFlowInstanceIT.kt b/backend/form-flow/src/test/kotlin/com/ritense/formflow/domain/FormFlowInstanceIT.kt index 5fd6c05ad2..2a19a366d7 100644 --- a/backend/form-flow/src/test/kotlin/com/ritense/formflow/domain/FormFlowInstanceIT.kt +++ b/backend/form-flow/src/test/kotlin/com/ritense/formflow/domain/FormFlowInstanceIT.kt @@ -47,7 +47,7 @@ internal class FormFlowInstanceIT : BaseIntegrationTest() { fun `create form flow instance successfully`() { val caseDefinitionId = CaseDefinitionId("profile", "1.0.0") val formFlowDefinition = - formFlowDefinitionRepository.getReferenceById(FormFlowDefinitionId("inkomens_loket" ,caseDefinitionId)) + formFlowDefinitionRepository.getReferenceById(FormFlowDefinitionId.existingId("inkomens_loket" ,caseDefinitionId)) val formFlowInstance = FormFlowInstance( formFlowDefinition = formFlowDefinition!! @@ -63,7 +63,7 @@ internal class FormFlowInstanceIT : BaseIntegrationTest() { fun `update form flow instance successfully`() { val caseDefinitionId = CaseDefinitionId("profile", "1.0.0") val formFlowDefinition = - formFlowDefinitionRepository.getReferenceById(FormFlowDefinitionId("inkomens_loket" ,caseDefinitionId)) + formFlowDefinitionRepository.getReferenceById(FormFlowDefinitionId.existingId("inkomens_loket" ,caseDefinitionId)) val formFlowInstance = FormFlowInstance( formFlowDefinition = formFlowDefinition!! @@ -84,7 +84,7 @@ internal class FormFlowInstanceIT : BaseIntegrationTest() { fun `complete goes through the entire flow`() { val caseDefinitionId = CaseDefinitionId("profile", "1.0.0") val formFlowDefinition = - formFlowDefinitionRepository.getReferenceById(FormFlowDefinitionId("inkomens_loket" ,caseDefinitionId)) + formFlowDefinitionRepository.getReferenceById(FormFlowDefinitionId.existingId("inkomens_loket" ,caseDefinitionId)) val submissionData = """ { "inkomen": { @@ -122,7 +122,7 @@ internal class FormFlowInstanceIT : BaseIntegrationTest() { fun `complete goes through the entire flow, back and then through again`() { val caseDefinitionId = CaseDefinitionId("profile", "1.0.0") val formFlowDefinition = - formFlowDefinitionRepository.getReferenceById(FormFlowDefinitionId("inkomens_loket" ,caseDefinitionId)) + formFlowDefinitionRepository.getReferenceById(FormFlowDefinitionId.existingId("inkomens_loket" ,caseDefinitionId)) val submissionData = """ { "inkomen": { @@ -175,7 +175,7 @@ internal class FormFlowInstanceIT : BaseIntegrationTest() { fun `navigate to next step removes previous steps`() { val caseDefinitionId = CaseDefinitionId("profile", "1.0.0") val formFlowDefinition = - formFlowDefinitionRepository.getReferenceById(FormFlowDefinitionId("inkomens_loket" ,caseDefinitionId)) + formFlowDefinitionRepository.getReferenceById(FormFlowDefinitionId.existingId("inkomens_loket" ,caseDefinitionId)) val submissionData = """ { "inkomen": { @@ -360,7 +360,7 @@ internal class FormFlowInstanceIT : BaseIntegrationTest() { fun `should set submissionData with SpEL expression`() { val caseDefinitionId = CaseDefinitionId("profile", "1.0.0") val formFlowDefinition = - formFlowDefinitionRepository.getReferenceById(FormFlowDefinitionId("form-flow-with-expressions" ,caseDefinitionId)) + formFlowDefinitionRepository.getReferenceById(FormFlowDefinitionId.existingId("form-flow-with-expressions" ,caseDefinitionId)) var formFlowInstance = FormFlowInstance(formFlowDefinition = formFlowDefinition!!) formFlowInstance = formFlowInstanceRepository.saveAndFlush(formFlowInstance) diff --git a/backend/form-flow/src/test/kotlin/com/ritense/formflow/domain/FormFlowInstanceTest.kt b/backend/form-flow/src/test/kotlin/com/ritense/formflow/domain/FormFlowInstanceTest.kt index 48214247b4..156dddf583 100644 --- a/backend/form-flow/src/test/kotlin/com/ritense/formflow/domain/FormFlowInstanceTest.kt +++ b/backend/form-flow/src/test/kotlin/com/ritense/formflow/domain/FormFlowInstanceTest.kt @@ -57,7 +57,7 @@ internal class FormFlowInstanceTest : BaseTest() { val caseDefinitionId = CaseDefinitionId("profile", "1.0.0") val instance = FormFlowInstance( formFlowDefinition = FormFlowDefinition( - id = FormFlowDefinitionId("test", caseDefinitionId), + id = FormFlowDefinitionId.existingId("test", caseDefinitionId), startStep = "test", steps = mutableSetOf( FormFlowStep( @@ -86,7 +86,7 @@ internal class FormFlowInstanceTest : BaseTest() { val caseDefinitionId = CaseDefinitionId("profile", "1.0.0") val instance = FormFlowInstance( formFlowDefinition = FormFlowDefinition( - id = FormFlowDefinitionId("test", caseDefinitionId), + id = FormFlowDefinitionId.existingId("test", caseDefinitionId), startStep = "test", steps = mutableSetOf( FormFlowStep( @@ -110,7 +110,7 @@ internal class FormFlowInstanceTest : BaseTest() { val caseDefinitionId = CaseDefinitionId("profile", "1.0.0") val instance = FormFlowInstance( formFlowDefinition = FormFlowDefinition( - id = FormFlowDefinitionId("test", caseDefinitionId), + id = FormFlowDefinitionId.existingId("test", caseDefinitionId), startStep = "lastStep", steps = mutableSetOf( FormFlowStep( diff --git a/backend/form-flow/src/test/kotlin/com/ritense/formflow/domain/definition/FormFlowDefinitionBlueprintIdTest.kt b/backend/form-flow/src/test/kotlin/com/ritense/formflow/domain/definition/FormFlowDefinitionBlueprintIdTest.kt new file mode 100644 index 0000000000..0ee316520a --- /dev/null +++ b/backend/form-flow/src/test/kotlin/com/ritense/formflow/domain/definition/FormFlowDefinitionBlueprintIdTest.kt @@ -0,0 +1,132 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.formflow.domain.definition + +import com.ritense.valtimo.contract.blueprint.BlueprintType +import com.ritense.valtimo.contract.buildingblock.BuildingBlockDefinitionId +import com.ritense.valtimo.contract.case_.CaseDefinitionId +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.semver4j.Semver + +class FormFlowDefinitionBlueprintIdTest { + + @Test + fun `forCase creates id with CASE type and correct fields`() { + val caseDefinitionId = CaseDefinitionId("my-case", "2.1.0") + + val blueprintId = FormFlowDefinitionBlueprintId.forCase(caseDefinitionId) + + assertThat(blueprintId.blueprintType).isEqualTo(BlueprintType.CASE) + assertThat(blueprintId.blueprintKey).isEqualTo("my-case") + assertThat(blueprintId.blueprintVersionTag).isEqualTo(caseDefinitionId.versionTag) + } + + @Test + fun `forBuildingBlock creates id with BUILDING_BLOCK type and correct fields`() { + val bbId = BuildingBlockDefinitionId("my-bb", "1.0.0") + + val blueprintId = FormFlowDefinitionBlueprintId.forBuildingBlock(bbId) + + assertThat(blueprintId.blueprintType).isEqualTo(BlueprintType.BUILDING_BLOCK) + assertThat(blueprintId.blueprintKey).isEqualTo("my-bb") + assertThat(blueprintId.blueprintVersionTag).isEqualTo(bbId.versionTag) + } + + @Test + fun `two ids with same values are equal`() { + val caseDefinitionId = CaseDefinitionId("my-case", "1.0.0") + + val id1 = FormFlowDefinitionBlueprintId.forCase(caseDefinitionId) + val id2 = FormFlowDefinitionBlueprintId.forCase(caseDefinitionId) + + assertThat(id1).isEqualTo(id2) + assertThat(id1.hashCode()).isEqualTo(id2.hashCode()) + } + + @Test + fun `two ids with different blueprint types are not equal`() { + val caseId = FormFlowDefinitionBlueprintId.forCase(CaseDefinitionId("shared-key", "1.0.0")) + val bbId = FormFlowDefinitionBlueprintId.forBuildingBlock(BuildingBlockDefinitionId("shared-key", "1.0.0")) + + assertThat(caseId).isNotEqualTo(bbId) + } + + @Test + fun `two ids with different keys are not equal`() { + val id1 = FormFlowDefinitionBlueprintId.forCase(CaseDefinitionId("case-a", "1.0.0")) + val id2 = FormFlowDefinitionBlueprintId.forCase(CaseDefinitionId("case-b", "1.0.0")) + + assertThat(id1).isNotEqualTo(id2) + } + + @Test + fun `two ids with different version tags are not equal`() { + val id1 = FormFlowDefinitionBlueprintId.forCase(CaseDefinitionId("my-case", "1.0.0")) + val id2 = FormFlowDefinitionBlueprintId.forCase(CaseDefinitionId("my-case", "2.0.0")) + + assertThat(id1).isNotEqualTo(id2) + } + + @Test + fun `toString includes type, key, and version tag`() { + val bbId = BuildingBlockDefinitionId("my-bb", "1.2.3") + + val blueprintId = FormFlowDefinitionBlueprintId.forBuildingBlock(bbId) + val str = blueprintId.toString() + + assertThat(str).contains("BUILDING_BLOCK") + assertThat(str).contains("my-bb") + assertThat(str).contains("1.2.3") + } + + @Test + fun `asCaseDefinitionId returns CaseDefinitionId for CASE type`() { + val caseDefinitionId = CaseDefinitionId("my-case", "1.0.0") + val blueprintId = FormFlowDefinitionBlueprintId.forCase(caseDefinitionId) + + val result = blueprintId.asCaseDefinitionId() + + assertThat(result).isNotNull + assertThat(result!!.key).isEqualTo("my-case") + } + + @Test + fun `asCaseDefinitionId returns null for BUILDING_BLOCK type`() { + val blueprintId = FormFlowDefinitionBlueprintId.forBuildingBlock(BuildingBlockDefinitionId("my-bb", "1.0.0")) + + assertThat(blueprintId.asCaseDefinitionId()).isNull() + } + + @Test + fun `asBuildingBlockDefinitionId returns BuildingBlockDefinitionId for BUILDING_BLOCK type`() { + val bbId = BuildingBlockDefinitionId("my-bb", "1.0.0") + val blueprintId = FormFlowDefinitionBlueprintId.forBuildingBlock(bbId) + + val result = blueprintId.asBuildingBlockDefinitionId() + + assertThat(result).isNotNull + assertThat(result!!.key).isEqualTo("my-bb") + } + + @Test + fun `asBuildingBlockDefinitionId returns null for CASE type`() { + val blueprintId = FormFlowDefinitionBlueprintId.forCase(CaseDefinitionId("my-case", "1.0.0")) + + assertThat(blueprintId.asBuildingBlockDefinitionId()).isNull() + } +} diff --git a/backend/form-flow/src/test/kotlin/com/ritense/formflow/service/FormFlowServiceTest.kt b/backend/form-flow/src/test/kotlin/com/ritense/formflow/service/FormFlowServiceTest.kt index 53845d2ba9..7ad469e234 100644 --- a/backend/form-flow/src/test/kotlin/com/ritense/formflow/service/FormFlowServiceTest.kt +++ b/backend/form-flow/src/test/kotlin/com/ritense/formflow/service/FormFlowServiceTest.kt @@ -31,6 +31,7 @@ import com.ritense.formflow.expression.spel.SpelExpressionProcessorFactory import com.ritense.formflow.repository.FormFlowAdditionalPropertiesSearchRepository import com.ritense.formflow.repository.FormFlowDefinitionRepository import com.ritense.formflow.repository.FormFlowInstanceRepository +import com.ritense.valtimo.contract.buildingblock.BuildingBlockDefinitionChecker import com.ritense.valtimo.contract.case_.CaseDefinitionId import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.BeforeEach @@ -51,6 +52,7 @@ internal class FormFlowServiceTest : BaseTest() { lateinit var formFlowService: FormFlowService lateinit var formFlowInstanceRepository: FormFlowInstanceRepository lateinit var formFlowAdditionalPropertiesSearchRepository: FormFlowAdditionalPropertiesSearchRepository + lateinit var buildingBlockDefinitionChecker: BuildingBlockDefinitionChecker lateinit var expressionProcessor: SpelExpressionProcessor @BeforeEach @@ -58,12 +60,14 @@ internal class FormFlowServiceTest : BaseTest() { val formFlowDefinitionRepository = mock(FormFlowDefinitionRepository::class.java) formFlowInstanceRepository = mock(FormFlowInstanceRepository::class.java) formFlowAdditionalPropertiesSearchRepository = mock(FormFlowAdditionalPropertiesSearchRepository::class.java) + buildingBlockDefinitionChecker = mock(BuildingBlockDefinitionChecker::class.java) formFlowService = FormFlowService( formFlowDefinitionRepository, formFlowInstanceRepository, formFlowAdditionalPropertiesSearchRepository, emptyList(), mock(), + buildingBlockDefinitionChecker, ) val expressionProcessorFactory = spy(SpelExpressionProcessorFactory()) @@ -144,7 +148,7 @@ internal class FormFlowServiceTest : BaseTest() { type = FormFlowStepType("form", FormStepTypeProperties("my-form-definition")) ) val definition = FormFlowDefinition( - FormFlowDefinitionId("test", caseDefinitionId), "start-step", setOf(step) + FormFlowDefinitionId.existingId("test", caseDefinitionId), "start-step", setOf(step) ) val formFlowInstance = FormFlowInstance( formFlowDefinition = definition diff --git a/backend/form-flow/src/test/kotlin/com/ritense/formflow/web/rest/FormFlowManagementResourceIntTest.kt b/backend/form-flow/src/test/kotlin/com/ritense/formflow/web/rest/FormFlowManagementResourceIntTest.kt index 10e7009a6b..2fa3df1c17 100644 --- a/backend/form-flow/src/test/kotlin/com/ritense/formflow/web/rest/FormFlowManagementResourceIntTest.kt +++ b/backend/form-flow/src/test/kotlin/com/ritense/formflow/web/rest/FormFlowManagementResourceIntTest.kt @@ -84,7 +84,7 @@ class FormFlowManagementResourceIntTest : BaseIntegrationTest() { @Test fun `should delete form flow definition by key`() { val caseDefinitionId = CaseDefinitionId("profile", "1.0.0") - formFlowService.save(FormFlowDefinition(FormFlowDefinitionId("test", caseDefinitionId), "start-step", setOf())) + formFlowService.save(FormFlowDefinition(FormFlowDefinitionId.existingId("test", caseDefinitionId), "start-step", setOf())) mockMvc .perform(delete("/api/management/v1/case-definition/{caseDefinitionKey}/version/{versionTag}/form-flow-definition/{definitionKey}", "profile", "1.0.0", "test")) .andDo(print()) @@ -114,7 +114,7 @@ class FormFlowManagementResourceIntTest : BaseIntegrationTest() { @Test fun `should update form flow definition`() { val caseDefinitionId = CaseDefinitionId("profile", "1.0.0") - formFlowService.save(FormFlowDefinition(FormFlowDefinitionId("test", caseDefinitionId), "start-step", setOf())) + formFlowService.save(FormFlowDefinition(FormFlowDefinitionId.existingId("test", caseDefinitionId), "start-step", setOf())) val definition = FormFlowDefinitionDto( key = "test", diff --git a/backend/importer/src/main/kotlin/com/ritense/importer/ValtimoImportTypes.kt b/backend/importer/src/main/kotlin/com/ritense/importer/ValtimoImportTypes.kt index 7565080b44..6dfc80844c 100644 --- a/backend/importer/src/main/kotlin/com/ritense/importer/ValtimoImportTypes.kt +++ b/backend/importer/src/main/kotlin/com/ritense/importer/ValtimoImportTypes.kt @@ -51,6 +51,7 @@ class ValtimoImportTypes { const val BUILDING_BLOCK_ARTWORK = "buildingblockartwork" const val BUILDING_BLOCK_PROCESS_LINK = "buildingblockprocesslink" const val BUILDING_BLOCK_FORM_DEFINITION = "buildingblockformdefinition" + const val BUILDING_BLOCK_FORM_FLOW_DEFINITION = "buildingblockformflowdefinition" const val OBJECT_MANAGEMENT = "objectmanagement" diff --git a/backend/process-link/build.gradle b/backend/process-link/build.gradle index 1e3543a2bc..69b541d321 100644 --- a/backend/process-link/build.gradle +++ b/backend/process-link/build.gradle @@ -59,6 +59,7 @@ dependencies { testImplementation project(":backend:test-utils-common") testImplementation(project(':backend:form')) + testImplementation(project(':backend:form-flow')) testImplementation(project(':backend:building-block')) { exclude(group: "com.ritense.valtimo", module: "process-link") } diff --git a/backend/process-link/src/test/kotlin/com/ritense/processlink/service/ProcessLinkServiceIntTest.kt b/backend/process-link/src/test/kotlin/com/ritense/processlink/service/ProcessLinkServiceIntTest.kt index 870cb857b2..5aee72af60 100644 --- a/backend/process-link/src/test/kotlin/com/ritense/processlink/service/ProcessLinkServiceIntTest.kt +++ b/backend/process-link/src/test/kotlin/com/ritense/processlink/service/ProcessLinkServiceIntTest.kt @@ -2,6 +2,7 @@ package com.ritense.processlink.service import com.ritense.authorization.AuthorizationContext.Companion.runWithoutAuthorization import com.ritense.form.mapper.FormProcessLinkMapper +import com.ritense.formflow.mapper.FormFlowProcessLinkMapper import com.ritense.processlink.BaseIntegrationTest import com.ritense.processlink.domain.ActivityTypeWithEventName import com.ritense.processlink.domain.AnotherTestProcessLink @@ -44,6 +45,7 @@ class ProcessLinkServiceIntTest @Autowired constructor( assertThat(processLinkTypes).containsExactly( ProcessLinkType(TestProcessLink.PROCESS_LINK_TYPE_TEST, true), ProcessLinkType(FormProcessLinkMapper.PROCESS_LINK_TYPE_FORM, false), + ProcessLinkType(FormFlowProcessLinkMapper.PROCESS_LINK_TYPE_FORM_FLOW, false), ProcessLinkType(UIComponentProcessLink.TYPE_UI_COMPONENT, true), ProcessLinkType(AnotherTestProcessLink.PROCESS_LINK_TYPE, true), ) From 001e1e153979a03b637f601dbe43ef6efcd59c5a Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Tue, 31 Mar 2026 14:16:08 +0200 Subject: [PATCH 02/46] feedback --- ...BuildingBlockFormFlowDefinitionImporter.kt | 23 +++++++--- ...BuildingBlockFormFlowManagementResource.kt | 8 ++-- ...ildingBlockFormFlowManagementResourceIT.kt | 43 ++++++------------- .../importer/FormFlowDefinitionImporter.kt | 4 +- 4 files changed, 37 insertions(+), 41 deletions(-) diff --git a/backend/building-block/src/main/kotlin/com/ritense/buildingblock/service/BuildingBlockFormFlowDefinitionImporter.kt b/backend/building-block/src/main/kotlin/com/ritense/buildingblock/service/BuildingBlockFormFlowDefinitionImporter.kt index 05d958bb74..94078584a1 100644 --- a/backend/building-block/src/main/kotlin/com/ritense/buildingblock/service/BuildingBlockFormFlowDefinitionImporter.kt +++ b/backend/building-block/src/main/kotlin/com/ritense/buildingblock/service/BuildingBlockFormFlowDefinitionImporter.kt @@ -63,15 +63,26 @@ class BuildingBlockFormFlowDefinitionImporter( override fun partOfBuildingBlockDefinition() = true - fun isAutoDeployed(formFlowDefinitionKey: String): Boolean { + fun isAutoDeployed(buildingBlockDefinitionId: BuildingBlockDefinitionId, formFlowDefinitionKey: String): Boolean { withLoggingContext("formFlowDefinitionKey" to formFlowDefinitionKey) { return ResourcePatternUtils.getResourcePatternResolver(resourceLoader) - .getResources(FORM_FLOW_DEFINITIONS_PATH.replace("{formFlowKey}", formFlowDefinitionKey)) + .getResources( + FORM_FLOW_DEFINITIONS_PATH + .replace("{buildingBlockKey}", buildingBlockDefinitionId.key) + .replace("{versionTag}", buildingBlockDefinitionId.versionTag.let { + "${it.major}-${it.minor}-${it.patch}" + }) + .replace("{formFlowKey}", formFlowDefinitionKey) + ) .size > 0 } } - private fun deploy(formFlowKey: String, formFlowJson: String, buildingBlockDefinitionId: BuildingBlockDefinitionId) { + private fun deploy( + formFlowKey: String, + formFlowJson: String, + buildingBlockDefinitionId: BuildingBlockDefinitionId + ) { withLoggingContext("formFlowDefinitionKey" to formFlowKey) { validate(formFlowJson) @@ -98,7 +109,9 @@ class BuildingBlockFormFlowDefinitionImporter( private fun validate(formFlowJson: String) { val definitionJsonObject = JSONObject(JSONTokener(formFlowJson)) - val schema = SchemaLoader.load(JSONObject(JSONTokener(loadFormFlowSchemaResource().inputStream))) + val schema = loadFormFlowSchemaResource().inputStream.use { inputStream -> + SchemaLoader.load(JSONObject(JSONTokener(inputStream))) + } schema.validate(definitionJsonObject) } @@ -118,7 +131,7 @@ class BuildingBlockFormFlowDefinitionImporter( private companion object { private const val FORM_FLOW_SCHEMA_PATH = "classpath:config/form-flow/schema/formflow.schema.json" private const val FORM_FLOW_DEFINITIONS_PATH = - "classpath:config/building-block/*/*/form-flow/{formFlowKey}.form-flow.json" + "classpath*:config/building-block/{buildingBlockKey}/{versionTag}/form-flow/{formFlowKey}.form-flow.json" val PATH_REGEX = """/form-flow/([^/]+)\.form-flow\.json""".toRegex() val logger = KotlinLogging.logger {} } diff --git a/backend/building-block/src/main/kotlin/com/ritense/buildingblock/web/rest/BuildingBlockFormFlowManagementResource.kt b/backend/building-block/src/main/kotlin/com/ritense/buildingblock/web/rest/BuildingBlockFormFlowManagementResource.kt index bb0f90f236..4217c061c2 100644 --- a/backend/building-block/src/main/kotlin/com/ritense/buildingblock/web/rest/BuildingBlockFormFlowManagementResource.kt +++ b/backend/building-block/src/main/kotlin/com/ritense/buildingblock/web/rest/BuildingBlockFormFlowManagementResource.kt @@ -53,7 +53,7 @@ class BuildingBlockFormFlowManagementResource( ): ResponseEntity> { val buildingBlockId = BuildingBlockDefinitionId.of(key, versionTag) val definitions = buildingBlockFormFlowDefinitionService.getFormFlowDefinitions(buildingBlockId, pageable) - .map { FormFlowDefinitionDto.of(it, buildingBlockFormFlowDefinitionImporter.isAutoDeployed(it.id.key)) } + .map { FormFlowDefinitionDto.of(it, buildingBlockFormFlowDefinitionImporter.isAutoDeployed(buildingBlockId, it.id.key)) } return ResponseEntity.ok(definitions) } @@ -70,7 +70,7 @@ class BuildingBlockFormFlowManagementResource( return ResponseEntity.ok( FormFlowDefinitionDto.of( definition, - buildingBlockFormFlowDefinitionImporter.isAutoDeployed(definitionKey) + buildingBlockFormFlowDefinitionImporter.isAutoDeployed(buildingBlockId, definitionKey) ) ) } @@ -99,7 +99,7 @@ class BuildingBlockFormFlowManagementResource( @RequestBody definitionDto: FormFlowDefinitionDto ): ResponseEntity { val buildingBlockId = BuildingBlockDefinitionId.of(key, versionTag) - if (buildingBlockFormFlowDefinitionImporter.isAutoDeployed(definitionKey)) { + if (buildingBlockFormFlowDefinitionImporter.isAutoDeployed(buildingBlockId, definitionKey)) { return ResponseEntity.status(HttpStatus.FORBIDDEN).build() } val saved = buildingBlockFormFlowDefinitionService.save(buildingBlockId, definitionDto) @@ -114,7 +114,7 @@ class BuildingBlockFormFlowManagementResource( @PathVariable definitionKey: String ): ResponseEntity { val buildingBlockId = BuildingBlockDefinitionId.of(key, versionTag) - if (buildingBlockFormFlowDefinitionImporter.isAutoDeployed(definitionKey)) { + if (buildingBlockFormFlowDefinitionImporter.isAutoDeployed(buildingBlockId, definitionKey)) { return ResponseEntity.status(HttpStatus.FORBIDDEN).build() } buildingBlockFormFlowDefinitionService.delete(buildingBlockId, definitionKey) diff --git a/backend/building-block/src/test/kotlin/com/ritense/buildingblock/web/rest/BuildingBlockFormFlowManagementResourceIT.kt b/backend/building-block/src/test/kotlin/com/ritense/buildingblock/web/rest/BuildingBlockFormFlowManagementResourceIT.kt index 8a031ab2d2..8a0c0396c2 100644 --- a/backend/building-block/src/test/kotlin/com/ritense/buildingblock/web/rest/BuildingBlockFormFlowManagementResourceIT.kt +++ b/backend/building-block/src/test/kotlin/com/ritense/buildingblock/web/rest/BuildingBlockFormFlowManagementResourceIT.kt @@ -24,6 +24,7 @@ import com.ritense.formflow.domain.definition.FormFlowDefinition import com.ritense.formflow.domain.definition.FormFlowDefinitionId import com.ritense.formflow.repository.FormFlowDefinitionRepository import com.ritense.formflow.web.rest.result.FormFlowDefinitionDto +import com.ritense.valtimo.contract.authentication.AuthoritiesConstants.ADMIN import com.ritense.valtimo.contract.buildingblock.BuildingBlockDefinitionId import com.ritense.valtimo.contract.case_.CaseDefinitionId import org.junit.jupiter.api.AfterEach @@ -44,16 +45,6 @@ import org.springframework.test.web.servlet.get import org.springframework.test.web.servlet.post import org.springframework.test.web.servlet.put -/** - * Integration test for [BuildingBlockFormFlowManagementResource]. - * - * Seeds the database with both a building-block-linked and a case-linked form flow to verify - * that the building-block endpoints only expose form flows that belong to the requested - * building block definition — not those belonging to a case definition. - * - * Write operation tests (POST/PUT/DELETE) stub the service layer because the auto-deployed - * `bezwaar` building block is marked `final`, which prevents real write operations. - */ class BuildingBlockFormFlowManagementResourceIT @Autowired constructor( private val mockMvc: MockMvc, private val objectMapper: ObjectMapper, @@ -70,7 +61,6 @@ class BuildingBlockFormFlowManagementResourceIT @Autowired constructor( private val base = "/api/management/v1/building-block" - // bezwaar/1.0.0 is auto-deployed from test resources — assertBuildingBlockDefinitionExists passes private val bbId = BuildingBlockDefinitionId("bezwaar", "1.0.0") private val caseId = CaseDefinitionId("bb-case", "1.0.0") @@ -93,13 +83,9 @@ class BuildingBlockFormFlowManagementResourceIT @Autowired constructor( } } - // ----------------------------------------------------------------------- - // Data-isolation tests — the spy calls through to the real service so the - // actual database query is exercised. - // ----------------------------------------------------------------------- @Test - @WithMockUser + @WithMockUser(username = "admin@ritense.com", authorities = [ADMIN]) fun `GET all returns only building-block form flows, not case form flows`() { mockMvc.get("$base/{key}/version/{versionTag}/form-flow-definition", "bezwaar", "1.0.0") .andExpect { @@ -110,7 +96,7 @@ class BuildingBlockFormFlowManagementResourceIT @Autowired constructor( } @Test - @WithMockUser + @WithMockUser(username = "admin@ritense.com", authorities = [ADMIN]) fun `GET by key returns the building-block form flow`() { mockMvc.get("$base/{key}/version/{versionTag}/form-flow-definition/{definitionKey}", "bezwaar", "1.0.0", "bb-test-flow") @@ -121,7 +107,7 @@ class BuildingBlockFormFlowManagementResourceIT @Autowired constructor( } @Test - @WithMockUser + @WithMockUser(username = "admin@ritense.com", authorities = [ADMIN]) fun `GET by key returns 404 for a case form flow that has the same key`() { // case-test-flow exists in the DB but belongs to a case, not this building block mockMvc.get("$base/{key}/version/{versionTag}/form-flow-definition/{definitionKey}", @@ -129,13 +115,8 @@ class BuildingBlockFormFlowManagementResourceIT @Autowired constructor( .andExpect { status { isNotFound() } } } - // ----------------------------------------------------------------------- - // Write operation tests — bezwaar is final so the real checker blocks writes; - // we stub the service layer to test the resource logic in isolation. - // ----------------------------------------------------------------------- - @Test - @WithMockUser + @WithMockUser(username = "admin@ritense.com", authorities = [ADMIN]) fun `POST create returns 400 when definition already exists`() { val dto = FormFlowDefinitionDto(key = "bb-test-flow", startStep = "start", steps = emptyList()) @@ -150,7 +131,7 @@ class BuildingBlockFormFlowManagementResourceIT @Autowired constructor( } @Test - @WithMockUser + @WithMockUser(username = "admin@ritense.com", authorities = [ADMIN]) fun `POST create returns 200 with the saved definition`() { val newDefinitionId = FormFlowDefinitionId.existingId("new-flow", bbId) val savedDefinition = FormFlowDefinition(newDefinitionId, "start", emptySet()) @@ -173,7 +154,7 @@ class BuildingBlockFormFlowManagementResourceIT @Autowired constructor( } @Test - @WithMockUser + @WithMockUser(username = "admin@ritense.com", authorities = [ADMIN]) fun `PUT update returns 200 with the updated definition`() { val updatedDefinition = FormFlowDefinition(bbDefinitionId, "updated-start", emptySet()) val dto = FormFlowDefinitionDto(key = "bb-test-flow", startStep = "updated-start", steps = emptyList()) @@ -193,13 +174,13 @@ class BuildingBlockFormFlowManagementResourceIT @Autowired constructor( } @Test - @WithMockUser + @WithMockUser(username = "admin@ritense.com", authorities = [ADMIN]) fun `PUT update returns 403 when definition is auto-deployed (read-only)`() { val dto = FormFlowDefinitionDto(key = "readonly-flow", startStep = "start", steps = emptyList()) doReturn(true) .whenever(buildingBlockFormFlowDefinitionImporter) - .isAutoDeployed(eq("readonly-flow")) + .isAutoDeployed(any(), eq("readonly-flow")) mockMvc.put("$base/{key}/version/{versionTag}/form-flow-definition/{definitionKey}", "bezwaar", "1.0.0", "readonly-flow") { @@ -209,7 +190,7 @@ class BuildingBlockFormFlowManagementResourceIT @Autowired constructor( } @Test - @WithMockUser + @WithMockUser(username = "admin@ritense.com", authorities = [ADMIN]) fun `DELETE returns 200`() { doNothing() .whenever(buildingBlockFormFlowDefinitionService) @@ -221,11 +202,11 @@ class BuildingBlockFormFlowManagementResourceIT @Autowired constructor( } @Test - @WithMockUser + @WithMockUser(username = "admin@ritense.com", authorities = [ADMIN]) fun `DELETE returns 403 when definition is auto-deployed (read-only)`() { doReturn(true) .whenever(buildingBlockFormFlowDefinitionImporter) - .isAutoDeployed(eq("readonly-flow")) + .isAutoDeployed(any(), eq("readonly-flow")) mockMvc.delete("$base/{key}/version/{versionTag}/form-flow-definition/{definitionKey}", "bezwaar", "1.0.0", "readonly-flow") diff --git a/backend/form-flow/src/main/kotlin/com/ritense/formflow/importer/FormFlowDefinitionImporter.kt b/backend/form-flow/src/main/kotlin/com/ritense/formflow/importer/FormFlowDefinitionImporter.kt index 0b3165f121..ab7aab7e3a 100644 --- a/backend/form-flow/src/main/kotlin/com/ritense/formflow/importer/FormFlowDefinitionImporter.kt +++ b/backend/form-flow/src/main/kotlin/com/ritense/formflow/importer/FormFlowDefinitionImporter.kt @@ -89,7 +89,9 @@ class FormFlowDefinitionImporter( private fun validate(formFlowJson: String) { val definitionJsonObject = JSONObject(JSONTokener(formFlowJson)) - val schema = SchemaLoader.load(JSONObject(JSONTokener(loadFormFlowSchemaResource().inputStream))) + val schema = loadFormFlowSchemaResource().inputStream.use { inputStream -> + SchemaLoader.load(JSONObject(JSONTokener(inputStream))) + } schema.validate(definitionJsonObject) } From feab67b93149388728ae2ba949d95c8b4437dea7 Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Tue, 31 Mar 2026 14:22:04 +0200 Subject: [PATCH 03/46] feedback --- .../rest/BuildingBlockFormFlowManagementResource.kt | 10 +++++++++- .../repository/FormFlowDefinitionRepository.kt | 2 ++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/backend/building-block/src/main/kotlin/com/ritense/buildingblock/web/rest/BuildingBlockFormFlowManagementResource.kt b/backend/building-block/src/main/kotlin/com/ritense/buildingblock/web/rest/BuildingBlockFormFlowManagementResource.kt index 4217c061c2..463dffe8c5 100644 --- a/backend/building-block/src/main/kotlin/com/ritense/buildingblock/web/rest/BuildingBlockFormFlowManagementResource.kt +++ b/backend/building-block/src/main/kotlin/com/ritense/buildingblock/web/rest/BuildingBlockFormFlowManagementResource.kt @@ -53,7 +53,12 @@ class BuildingBlockFormFlowManagementResource( ): ResponseEntity> { val buildingBlockId = BuildingBlockDefinitionId.of(key, versionTag) val definitions = buildingBlockFormFlowDefinitionService.getFormFlowDefinitions(buildingBlockId, pageable) - .map { FormFlowDefinitionDto.of(it, buildingBlockFormFlowDefinitionImporter.isAutoDeployed(buildingBlockId, it.id.key)) } + .map { + FormFlowDefinitionDto.of( + it, + buildingBlockFormFlowDefinitionImporter.isAutoDeployed(buildingBlockId, it.id.key) + ) + } return ResponseEntity.ok(definitions) } @@ -99,6 +104,9 @@ class BuildingBlockFormFlowManagementResource( @RequestBody definitionDto: FormFlowDefinitionDto ): ResponseEntity { val buildingBlockId = BuildingBlockDefinitionId.of(key, versionTag) + if (definitionDto.key != definitionKey) { + return ResponseEntity.badRequest().build() + } if (buildingBlockFormFlowDefinitionImporter.isAutoDeployed(buildingBlockId, definitionKey)) { return ResponseEntity.status(HttpStatus.FORBIDDEN).build() } diff --git a/backend/form-flow/src/main/kotlin/com/ritense/formflow/repository/FormFlowDefinitionRepository.kt b/backend/form-flow/src/main/kotlin/com/ritense/formflow/repository/FormFlowDefinitionRepository.kt index 4aa6acbf5d..409393dd0e 100644 --- a/backend/form-flow/src/main/kotlin/com/ritense/formflow/repository/FormFlowDefinitionRepository.kt +++ b/backend/form-flow/src/main/kotlin/com/ritense/formflow/repository/FormFlowDefinitionRepository.kt @@ -23,6 +23,7 @@ import org.semver4j.Semver import org.springframework.data.domain.Page import org.springframework.data.domain.Pageable import org.springframework.data.jpa.repository.JpaRepository +import org.springframework.data.jpa.repository.Modifying import org.springframework.data.jpa.repository.Query interface FormFlowDefinitionRepository : JpaRepository { @@ -33,6 +34,7 @@ interface FormFlowDefinitionRepository : JpaRepository + @Modifying @Query("DELETE FROM FormFlowDefinition f WHERE f.id.blueprintId.blueprintType = :blueprintType AND f.id.blueprintId.blueprintKey = :blueprintKey AND f.id.blueprintId.blueprintVersionTag = :blueprintVersionTag") fun deleteAllByBlueprintId(blueprintType: BlueprintType, blueprintKey: String, blueprintVersionTag: Semver) From 1050e683013f3477551ff8d7c4d0c0279bf2efcb Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Fri, 3 Apr 2026 10:30:31 +0200 Subject: [PATCH 04/46] PoC mongo for document queries --- backend/app/gzac/build.gradle | 1 + backend/app/gzac/docker-compose.yaml | 13 + .../gzac/src/main/resources/application.yml | 5 + .../src/main/resources/logback-spring.xml | 1 + backend/case-mongodb/build.gradle | 67 ++++ .../docker-compose-override-postgresql.yml | 10 + backend/case-mongodb/gradle/publishing.gradle | 35 ++ .../MongoAuthorizationEntityMapper.kt | 41 ++ .../MongoPermissionConditionTranslator.kt | 191 +++++++++ ...SchemaDocumentCaseDefinitionMongoMapper.kt | 74 ++++ ...JsonSchemaDocumentDefinitionMongoMapper.kt | 71 ++++ .../DocumentMongoAutoConfiguration.kt | 162 ++++++++ .../converter/JsonNodeMongoConverters.kt | 58 +++ .../domain/JsonSchemaDocumentDocument.kt | 54 +++ .../handler/DocumentMongoEventHandler.kt | 70 ++++ .../JsonSchemaDocumentMongoRepository.kt | 22 ++ .../DocumentMongoHttpSecurityConfigurer.kt | 38 ++ .../mongodb/service/ContentTextExtractor.kt | 39 ++ .../service/DocumentMongoBackfillService.kt | 72 ++++ .../service/DocumentMongoQueryService.kt | 71 ++++ .../service/DocumentMongoSyncService.kt | 49 +++ .../JsonSchemaDocumentMongoSearchService.kt | 361 ++++++++++++++++++ .../web/DocumentMongoBackfillResource.kt | 45 +++ ...ot.autoconfigure.AutoConfiguration.imports | 1 + .../mongodb/BaseMongoIntegrationTest.kt | 149 ++++++++ .../document/mongodb/TestApplication.kt | 22 ++ .../service/ContentTextExtractorTest.kt | 107 ++++++ .../service/DocumentMongoSyncServiceTest.kt | 144 +++++++ ...SchemaDocumentMongoSearchServiceIntTest.kt | 144 +++++++ ...sonSchemaDocumentMongoSearchServiceTest.kt | 178 +++++++++ .../config/application-postgresql.yml | 17 + .../src/test/resources/config/application.yml | 35 ++ .../definition/house.case-definition.json | 7 + .../house.internal-case-status.json | 20 + .../1-0-0/case/list/house.case-list.json | 1 + .../search-field/house.case-search-field.json | 18 + .../house.schema.document-definition.json | 33 ++ .../domain/search/AdvancedSearchRequest.java | 14 + .../domain/search/SearchRequestMapper.java | 1 + .../search/SearchWithConfigRequest.java | 9 + .../service/DocumentSearchService.java | 10 + .../configuration/CaseAutoConfiguration.kt | 3 +- .../com/ritense/case/service/CaseExporter.kt | 4 +- .../ritense/case/service/CaseExporterTest.kt | 4 +- .../valtimo-dependency-versions/build.gradle | 1 + .../case-list/case-list.component.html | 8 + .../case-list/case-list.component.ts | 30 +- .../lib/services/case-list-search.service.ts | 8 +- .../advanced-document-search-request.ts | 9 +- .../src/lib/services/document.service.ts | 8 +- .../valtimo/shared/assets/core/en.json | 4 +- .../valtimo/shared/assets/core/nl.json | 4 +- settings.gradle | 1 + 53 files changed, 2528 insertions(+), 16 deletions(-) create mode 100644 backend/case-mongodb/build.gradle create mode 100644 backend/case-mongodb/docker-compose-override-postgresql.yml create mode 100644 backend/case-mongodb/gradle/publishing.gradle create mode 100644 backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/authorization/MongoAuthorizationEntityMapper.kt create mode 100644 backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/authorization/MongoPermissionConditionTranslator.kt create mode 100644 backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/authorization/mapper/JsonSchemaDocumentCaseDefinitionMongoMapper.kt create mode 100644 backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/authorization/mapper/JsonSchemaDocumentDefinitionMongoMapper.kt create mode 100644 backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/autoconfigure/DocumentMongoAutoConfiguration.kt create mode 100644 backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/converter/JsonNodeMongoConverters.kt create mode 100644 backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/domain/JsonSchemaDocumentDocument.kt create mode 100644 backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/handler/DocumentMongoEventHandler.kt create mode 100644 backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/repository/JsonSchemaDocumentMongoRepository.kt create mode 100644 backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/security/DocumentMongoHttpSecurityConfigurer.kt create mode 100644 backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/ContentTextExtractor.kt create mode 100644 backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/DocumentMongoBackfillService.kt create mode 100644 backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/DocumentMongoQueryService.kt create mode 100644 backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/DocumentMongoSyncService.kt create mode 100644 backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/JsonSchemaDocumentMongoSearchService.kt create mode 100644 backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/web/DocumentMongoBackfillResource.kt create mode 100644 backend/case-mongodb/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports create mode 100644 backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/BaseMongoIntegrationTest.kt create mode 100644 backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/TestApplication.kt create mode 100644 backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/service/ContentTextExtractorTest.kt create mode 100644 backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/service/DocumentMongoSyncServiceTest.kt create mode 100644 backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/service/JsonSchemaDocumentMongoSearchServiceIntTest.kt create mode 100644 backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/service/JsonSchemaDocumentMongoSearchServiceTest.kt create mode 100644 backend/case-mongodb/src/test/resources/config/application-postgresql.yml create mode 100644 backend/case-mongodb/src/test/resources/config/application.yml create mode 100644 backend/case-mongodb/src/test/resources/config/case/house/1-0-0/case/definition/house.case-definition.json create mode 100644 backend/case-mongodb/src/test/resources/config/case/house/1-0-0/case/internal-status/house.internal-case-status.json create mode 100644 backend/case-mongodb/src/test/resources/config/case/house/1-0-0/case/list/house.case-list.json create mode 100644 backend/case-mongodb/src/test/resources/config/case/house/1-0-0/case/search-field/house.case-search-field.json create mode 100644 backend/case-mongodb/src/test/resources/config/case/house/1-0-0/document/definition/house.schema.document-definition.json diff --git a/backend/app/gzac/build.gradle b/backend/app/gzac/build.gradle index 1f04d6cc4f..4480eb23c2 100644 --- a/backend/app/gzac/build.gradle +++ b/backend/app/gzac/build.gradle @@ -16,6 +16,7 @@ dependencies { implementation(platform(project(":backend:dependencies:valtimo-dependency-versions"))) implementation(project(":backend:dependencies:valtimo-gzac-dependencies")) + implementation(project(":backend:case-mongodb")) implementation(project(":backend:mail:local-mail")) implementation(project(":backend:document-generation:smartdocuments")) implementation(project(":backend:zgw:portaaltaak")) diff --git a/backend/app/gzac/docker-compose.yaml b/backend/app/gzac/docker-compose.yaml index cf15a19e9b..d10664a7e6 100644 --- a/backend/app/gzac/docker-compose.yaml +++ b/backend/app/gzac/docker-compose.yaml @@ -80,6 +80,18 @@ services: volumes: - gzac-database-data-mysql:/var/lib/mysql # persist data even if container shuts down + gzac-mongodb: + container_name: gzac-docker-compose-gzac-mongodb + image: mongo:8.2.6 + ports: + - "27017:27017" + environment: + MONGO_INITDB_ROOT_USERNAME: gzac + MONGO_INITDB_ROOT_PASSWORD: password + MONGO_INITDB_DATABASE: gzac + volumes: + - gzac-mongodb-data:/data/db + gzac-rabbitmq: image: rabbitmq:4.1.0-management container_name: gzac-docker-compose-gzac-rabbitmq @@ -720,3 +732,4 @@ services: volumes: gzac-database-data: gzac-database-data-mysql: + gzac-mongodb-data: diff --git a/backend/app/gzac/src/main/resources/application.yml b/backend/app/gzac/src/main/resources/application.yml index 5aaf687848..bf632b3b09 100644 --- a/backend/app/gzac/src/main/resources/application.yml +++ b/backend/app/gzac/src/main/resources/application.yml @@ -1,6 +1,8 @@ logging: file: name: /tmp/spring.log + level: + com.ritense.document.mongodb.authorization: DEBUG management: endpoints: @@ -29,6 +31,9 @@ spring: enabled: false livereload: enabled: false + data: + mongodb: + uri: ${SPRING_DATA_MONGODB_URI:mongodb://gzac:password@localhost:27017/gzac?authSource=admin} datasource: type: com.zaxxer.hikari.HikariDataSource driver-class-name: ${SPRING_DATASOURCE_DRIVER_CLASS_NAME:org.postgresql.Driver} diff --git a/backend/app/gzac/src/main/resources/logback-spring.xml b/backend/app/gzac/src/main/resources/logback-spring.xml index 30eb9b3d5d..527b78a18c 100644 --- a/backend/app/gzac/src/main/resources/logback-spring.xml +++ b/backend/app/gzac/src/main/resources/logback-spring.xml @@ -6,6 +6,7 @@ + diff --git a/backend/case-mongodb/build.gradle b/backend/case-mongodb/build.gradle new file mode 100644 index 0000000000..93bfff18a9 --- /dev/null +++ b/backend/case-mongodb/build.gradle @@ -0,0 +1,67 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +dockerCompose { + projectName = "case-mongodb" + integrationTestingPostgresql { + isRequiredBy(project.tasks.integrationTestingPostgresql) + useComposeFiles.addAll( + "../docker-resources/docker-compose-base-test-postgresql.yml", + "docker-compose-override-postgresql.yml" + ) + } +} + +dependencies { + implementation project(":backend:authorization") + implementation project(":backend:case") + implementation project(":backend:inbox") + implementation project(":backend:outbox") + + implementation "org.springframework.boot:spring-boot-starter-data-jpa" + implementation "org.springframework.boot:spring-boot-starter-data-mongodb" + implementation "org.springframework.boot:spring-boot-starter-web" + implementation "org.springframework.boot:spring-boot-starter-security" + implementation "org.springframework.boot:spring-boot-autoconfigure" + implementation "com.fasterxml.jackson.module:jackson-module-kotlin" + implementation "io.github.oshai:kotlin-logging:${kotlinLoggingVersion}" + + annotationProcessor "org.springframework.boot:spring-boot-autoconfigure-processor" + + testImplementation project(':backend:test-utils-common') + testImplementation project(':backend:core') + testImplementation(project(':backend:audit')) { + exclude(group: "com.ritense.valtimo", module: "case") + } + testImplementation "org.springframework.boot:spring-boot-starter-test" + testImplementation "org.mockito.kotlin:mockito-kotlin:${mockitoKotlinVersion}" + testImplementation "org.postgresql:postgresql" + testImplementation "org.springframework.security:spring-security-test" + + jar { + enabled = true + manifest { + attributes("Implementation-Title": "Ritense Case MongoDB module") + attributes("Implementation-Version": projectVersion) + } + } +} + +tasks.named("integrationTestingPostgresql") { + systemProperty("liquibase.duplicateFileMode", "WARN") +} + +apply from: "gradle/publishing.gradle" diff --git a/backend/case-mongodb/docker-compose-override-postgresql.yml b/backend/case-mongodb/docker-compose-override-postgresql.yml new file mode 100644 index 0000000000..0b8c7ef396 --- /dev/null +++ b/backend/case-mongodb/docker-compose-override-postgresql.yml @@ -0,0 +1,10 @@ +services: + db: + ports: + - "3364:5432" + environment: + - POSTGRES_DB=case-mongodb-test + mongodb: + image: mongo:8.2.6 + ports: + - "37017:27017" diff --git a/backend/case-mongodb/gradle/publishing.gradle b/backend/case-mongodb/gradle/publishing.gradle new file mode 100644 index 0000000000..ce0f836cc5 --- /dev/null +++ b/backend/case-mongodb/gradle/publishing.gradle @@ -0,0 +1,35 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +pluginManager.withPlugin('maven-publish') { + publishing { + publications { + maven(MavenPublication) { + pom { + name = 'Case MongoDB module' + description = 'The case-mongodb module syncs json_schema_document to MongoDB as a CQRS read model' + developers { + developer { + id = "team-valtimo" + name = "Team Valtimo" + email = "team-valtimo@ritense.com" + } + } + } + } + } + } +} diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/authorization/MongoAuthorizationEntityMapper.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/authorization/MongoAuthorizationEntityMapper.kt new file mode 100644 index 0000000000..a0f4fd255c --- /dev/null +++ b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/authorization/MongoAuthorizationEntityMapper.kt @@ -0,0 +1,41 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.mongodb.authorization + +import com.ritense.authorization.permission.condition.PermissionCondition +import org.springframework.data.mongodb.core.query.Criteria + +/** + * MongoDB equivalent of [com.ritense.authorization.AuthorizationEntityMapper]. + * + * Translates a [com.ritense.authorization.permission.condition.ContainerPermissionCondition] + * on entity type [TO] into a MongoDB [Criteria] that filters [FROM] documents. + * + * Implement this interface and register the implementation as a Spring bean to add support + * for a new container relationship without modifying the core translator. + */ +interface MongoAuthorizationEntityMapper { + + /** + * Given conditions on the [TO] entity type, returns a MongoDB [Criteria] that filters + * [FROM] documents satisfying those conditions, or `null` if no filter is needed + * (i.e. any [FROM] document qualifies regardless of [conditions]). + */ + fun mapCriteria(conditions: List): Criteria? + + fun supports(fromClass: Class<*>, toClass: Class<*>): Boolean +} diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/authorization/MongoPermissionConditionTranslator.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/authorization/MongoPermissionConditionTranslator.kt new file mode 100644 index 0000000000..6a91f77bd9 --- /dev/null +++ b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/authorization/MongoPermissionConditionTranslator.kt @@ -0,0 +1,191 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.mongodb.authorization + +import com.ritense.authorization.Action +import com.ritense.authorization.AuthorizationContext.Companion.runWithoutAuthorization +import com.ritense.authorization.AuthorizationService +import com.ritense.authorization.permission.ConditionContainer +import com.ritense.authorization.permission.Permission +import com.ritense.authorization.permission.condition.ContainerPermissionCondition +import com.ritense.authorization.permission.condition.ExpressionPermissionCondition +import com.ritense.authorization.permission.condition.FieldPermissionCondition +import com.ritense.authorization.permission.condition.PermissionCondition +import com.ritense.authorization.permission.condition.PermissionConditionOperator +import com.ritense.authorization.request.EntityAuthorizationRequest +import com.ritense.authorization.role.Role +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.repository.impl.JsonSchemaDocumentRepository +import com.ritense.valtimo.contract.authorization.CurrentUserExpressionHandler +import io.github.oshai.kotlinlogging.KotlinLogging +import org.springframework.data.mongodb.core.query.Criteria + +class MongoPermissionConditionTranslator( + private val mongoMappers: List>, + private val authorizationService: AuthorizationService, + private val documentRepository: JsonSchemaDocumentRepository, +) { + + /** + * Translates a list of [Permission]s into a single MongoDB [Criteria] that, when applied + * to a query, returns only the documents the current user is allowed to see for [action]. + * + * Permissions are OR-ed; conditions within a permission are AND-ed. + * Returns a deny-all criteria if no permissions match [action]. + */ + fun toCriteria(permissions: List, action: Action<*>): Criteria { + val matching = permissions.filter { + it.resourceType == JsonSchemaDocument::class.java && it.actions.contains(action) + } + logger.debug { "toCriteria: ${permissions.size} permissions total, ${matching.size} matching action=$action" } + if (matching.isEmpty()) { + return denyAll() + } + + val perPermissionCriteria = matching.map { permission -> + val conditionCriteria = permission.conditionContainer.conditions.map { translateCondition(it) } + andAll(conditionCriteria) + } + val result = if (perPermissionCriteria.size == 1) { + perPermissionCriteria.first() + } else { + Criteria().orOperator(*perPermissionCriteria.toTypedArray()) + } + logger.debug { "toCriteria: generated criteria = ${result.criteriaObject}" } + return result + } + + private fun translateCondition(condition: PermissionCondition): Criteria = when (condition) { + is FieldPermissionCondition<*> -> translateField(condition) + is ExpressionPermissionCondition<*> -> translateExpression(condition) + is ContainerPermissionCondition<*> -> translateContainer(condition) + else -> throw IllegalArgumentException("Unknown permission condition type: ${condition::class.qualifiedName}") + } + + private fun translateField(cond: FieldPermissionCondition<*>): Criteria { + val mongoField = jpaToMongoField(cond.field) + val value = resolveFieldValue(cond) + return Companion.applyOperator(Criteria.where(mongoField), cond.operator, value) + } + + private fun translateExpression(cond: ExpressionPermissionCondition<*>): Criteria { + // Convert JSONPath "$.department.id" to MongoDB dot notation: "content.department.id" + val dotPath = cond.path.removePrefix("$.").replace("/", ".") + val mongoField = "${jpaToMongoField(cond.field)}.$dotPath" + val value = CurrentUserExpressionHandler.resolveValue(cond.value) + logger.debug { "translateExpression: field=${cond.field} → mongoField=$mongoField, op=${cond.operator}, value=$value (${value?.javaClass?.simpleName})" } + return Companion.applyOperator(Criteria.where(mongoField), cond.operator, value) + } + + @Suppress("UNCHECKED_CAST") + private fun translateContainer(cond: ContainerPermissionCondition<*>): Criteria { + val mongoMapper = mongoMappers.find { + it.supports(JsonSchemaDocument::class.java, cond.resourceType) + } as? MongoAuthorizationEntityMapper + + if (mongoMapper != null) { + return mongoMapper.mapCriteria(cond.conditions) ?: noFilter() + } + + logger.warn { + "No MongoAuthorizationEntityMapper registered for " + + "JsonSchemaDocument → ${cond.resourceType.simpleName}. " + + "Falling back to JPA ID resolution — may be slow for large datasets." + } + return jpaFallback(cond) + } + + /** + * Fallback for [ContainerPermissionCondition] types that have no registered + * [MongoAuthorizationEntityMapper]. Uses JPA to find matching document IDs and + * returns an [Criteria.where] `_id` `in` filter. + * + * This is correct but potentially expensive for large datasets. Register a + * [MongoAuthorizationEntityMapper] to replace this with a native MongoDB query. + */ + private fun jpaFallback(cond: ContainerPermissionCondition<*>): Criteria { + val syntheticPermission = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(Action(Action.IGNORE)), + conditionContainer = ConditionContainer(listOf(cond)), + role = Role(key = ""), + ) + val spec = authorizationService.getAuthorizationSpecification( + EntityAuthorizationRequest(JsonSchemaDocument::class.java, Action(Action.IGNORE)), + listOf(syntheticPermission) + ) + val allowedIds: List = runWithoutAuthorization { + documentRepository.findAll(spec).map { doc -> doc.id().toString() } + } + return Criteria.where("_id").`in`(allowedIds) + } + + + private fun resolveFieldValue(cond: FieldPermissionCondition<*>): Any? = + if (cond.value is List<*>) { + (cond.value as List<*>).map { CurrentUserExpressionHandler.resolveValue(it) } + } else { + CurrentUserExpressionHandler.resolveValue(cond.value) + } + + companion object { + private val logger = KotlinLogging.logger {} + + fun applyOperator(criteria: Criteria, op: PermissionConditionOperator, value: Any?): Criteria = + when (op) { + PermissionConditionOperator.EQUAL_TO -> criteria.`is`(value) + PermissionConditionOperator.NOT_EQUAL_TO -> if (value == null) criteria.ne(null) else criteria.ne(value) + PermissionConditionOperator.GREATER_THAN -> criteria.gt(value!!) + PermissionConditionOperator.GREATER_THAN_OR_EQUAL_TO -> criteria.gte(value!!) + PermissionConditionOperator.LESS_THAN -> criteria.lt(value!!) + PermissionConditionOperator.LESS_THAN_OR_EQUAL_TO -> criteria.lte(value!!) + PermissionConditionOperator.LIST_CONTAINS -> criteria.`in`(value) + PermissionConditionOperator.IN -> { + val collection = value as? Collection<*> + ?: throw IllegalArgumentException("IN operator requires a Collection value") + criteria.`in`(collection) + } + } + + /** + * Maps JPA entity field names (as used in [FieldPermissionCondition.field]) to + * their corresponding field names in the MongoDB document. + * Extend this map as new permission conditions are introduced. + */ + val fieldMappings: Map = mapOf( + "createdBy" to "createdBy", + "assigneeId" to "assigneeId", + "assigneeFullName" to "assigneeFullName", + "content" to "content", + // JPA: DocumentContent wraps the JSON via @JsonValue, so the inner + // "content.content" path in JPA resolves to the flat "content" in MongoDB. + "content.content" to "content", + "sequence" to "sequence", + "retentionDate" to "retentionDate", + ) + + fun jpaToMongoField(jpaField: String): String = fieldMappings[jpaField] ?: jpaField + + fun denyAll(): Criteria = Criteria.where("_id").`is`(null) + fun noFilter(): Criteria = Criteria() + fun andAll(list: List): Criteria = when { + list.isEmpty() -> noFilter() + list.size == 1 -> list.first() + else -> Criteria().andOperator(*list.toTypedArray()) + } + } +} diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/authorization/mapper/JsonSchemaDocumentCaseDefinitionMongoMapper.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/authorization/mapper/JsonSchemaDocumentCaseDefinitionMongoMapper.kt new file mode 100644 index 0000000000..5cacb847ce --- /dev/null +++ b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/authorization/mapper/JsonSchemaDocumentCaseDefinitionMongoMapper.kt @@ -0,0 +1,74 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.mongodb.authorization.mapper + +import com.ritense.authorization.permission.condition.FieldPermissionCondition +import com.ritense.authorization.permission.condition.PermissionCondition +import com.ritense.case_.domain.definition.CaseDefinition +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.mongodb.authorization.MongoAuthorizationEntityMapper +import com.ritense.document.mongodb.authorization.MongoPermissionConditionTranslator.Companion.andAll +import com.ritense.document.mongodb.authorization.MongoPermissionConditionTranslator.Companion.applyOperator +import com.ritense.valtimo.contract.authorization.CurrentUserExpressionHandler +import org.springframework.data.mongodb.core.query.Criteria + +/** + * Handles [com.ritense.authorization.permission.condition.ContainerPermissionCondition] + * where the container resource type is [CaseDefinition]. + * + * In the JPA model this relationship is expressed via `definitionId.blueprintId` + * (blueprintType=CASE, blueprintKey, blueprintVersionTag) on [JsonSchemaDocument]. + * The same nested structure exists in the MongoDB document. + */ +class JsonSchemaDocumentCaseDefinitionMongoMapper : MongoAuthorizationEntityMapper { + + override fun mapCriteria(conditions: List): Criteria? { + if (conditions.isEmpty()) return null + + val conditionCriteria = conditions.map { condition -> + when (condition) { + is FieldPermissionCondition<*> -> { + val mongoField = mapCaseDefinitionField(condition.field) + val value = CurrentUserExpressionHandler.resolveValue(condition.value) + applyOperator(Criteria.where(mongoField), condition.operator, value) + } + else -> throw UnsupportedOperationException( + "Condition type ${condition::class.simpleName} is not supported in " + + "${this::class.simpleName}. Register a custom ${MongoAuthorizationEntityMapper::class.simpleName} " + + "or extend this mapper to handle it." + ) + } + } + + // Constrain to CASE blueprint type to exclude BUILDING_BLOCK documents + val typeCriteria = Criteria.where("definitionId.blueprintId.blueprintType").`is`("CASE") + return andAll(conditionCriteria + typeCriteria) + } + + override fun supports(fromClass: Class<*>, toClass: Class<*>): Boolean = + fromClass == JsonSchemaDocument::class.java && + toClass == CaseDefinition::class.java + + private fun mapCaseDefinitionField(field: String): String = when (field) { + "id.key" -> "definitionId.blueprintId.blueprintKey" + "id.versionTag" -> "definitionId.blueprintId.blueprintVersionTag" + else -> throw UnsupportedOperationException( + "Field '$field' on CaseDefinition is not yet mapped for MongoDB. " + + "Add it to ${this::class.simpleName}." + ) + } +} diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/authorization/mapper/JsonSchemaDocumentDefinitionMongoMapper.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/authorization/mapper/JsonSchemaDocumentDefinitionMongoMapper.kt new file mode 100644 index 0000000000..f8b8546ed8 --- /dev/null +++ b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/authorization/mapper/JsonSchemaDocumentDefinitionMongoMapper.kt @@ -0,0 +1,71 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.mongodb.authorization.mapper + +import com.ritense.authorization.permission.condition.FieldPermissionCondition +import com.ritense.authorization.permission.condition.PermissionCondition +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.domain.impl.JsonSchemaDocumentDefinition +import com.ritense.document.mongodb.authorization.MongoAuthorizationEntityMapper +import com.ritense.document.mongodb.authorization.MongoPermissionConditionTranslator.Companion.andAll +import com.ritense.document.mongodb.authorization.MongoPermissionConditionTranslator.Companion.applyOperator +import com.ritense.valtimo.contract.authorization.CurrentUserExpressionHandler +import org.springframework.data.mongodb.core.query.Criteria + +/** + * Handles [com.ritense.authorization.permission.condition.ContainerPermissionCondition] + * where the container resource type is [JsonSchemaDocumentDefinition]. + * + * In the JPA model this relationship is expressed via `definitionId.name` (and + * `definitionId.version`) on [JsonSchemaDocument]. The same fields exist in the + * MongoDB document under `definitionId.name` and `definitionId.version`. + */ +class JsonSchemaDocumentDefinitionMongoMapper : MongoAuthorizationEntityMapper { + + override fun mapCriteria(conditions: List): Criteria? { + if (conditions.isEmpty()) return null + + val criteria = conditions.map { condition -> + when (condition) { + is FieldPermissionCondition<*> -> { + val mongoField = mapDefinitionField(condition.field) + val value = CurrentUserExpressionHandler.resolveValue(condition.value) + applyOperator(Criteria.where(mongoField), condition.operator, value) + } + else -> throw UnsupportedOperationException( + "Condition type ${condition::class.simpleName} is not supported in " + + "${this::class.simpleName}. Register a custom ${MongoAuthorizationEntityMapper::class.simpleName} " + + "or extend this mapper to handle it." + ) + } + } + return andAll(criteria) + } + + override fun supports(fromClass: Class<*>, toClass: Class<*>): Boolean = + fromClass == JsonSchemaDocument::class.java && + toClass == JsonSchemaDocumentDefinition::class.java + + private fun mapDefinitionField(field: String): String = when (field) { + "id.name" -> "definitionId.name" + "id.version" -> "definitionId.version" + else -> throw UnsupportedOperationException( + "Field '$field' on JsonSchemaDocumentDefinition is not yet mapped for MongoDB. " + + "Add it to ${this::class.simpleName}." + ) + } +} diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/autoconfigure/DocumentMongoAutoConfiguration.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/autoconfigure/DocumentMongoAutoConfiguration.kt new file mode 100644 index 0000000000..b1fddb9d17 --- /dev/null +++ b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/autoconfigure/DocumentMongoAutoConfiguration.kt @@ -0,0 +1,162 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.mongodb.autoconfigure + +import com.fasterxml.jackson.databind.ObjectMapper +import com.ritense.authorization.AuthorizationService +import com.ritense.document.autoconfigure.DocumentAutoConfiguration +import com.ritense.document.mongodb.authorization.MongoAuthorizationEntityMapper +import com.ritense.document.mongodb.authorization.MongoPermissionConditionTranslator +import com.ritense.document.mongodb.authorization.mapper.JsonSchemaDocumentCaseDefinitionMongoMapper +import com.ritense.document.mongodb.authorization.mapper.JsonSchemaDocumentDefinitionMongoMapper +import com.ritense.document.mongodb.converter.DocumentToJsonNodeReadConverter +import com.ritense.document.mongodb.converter.DocumentToObjectNodeReadConverter +import com.ritense.document.mongodb.converter.JsonNodeWriteConverter +import com.ritense.document.mongodb.handler.DocumentMongoEventHandler +import com.ritense.document.mongodb.repository.JsonSchemaDocumentMongoRepository +import com.ritense.document.mongodb.service.DocumentMongoBackfillService +import com.ritense.document.mongodb.service.DocumentMongoQueryService +import com.ritense.document.mongodb.service.DocumentMongoSyncService +import com.ritense.document.mongodb.service.JsonSchemaDocumentMongoSearchService +import com.ritense.document.mongodb.security.DocumentMongoHttpSecurityConfigurer +import com.ritense.document.mongodb.web.DocumentMongoBackfillResource +import com.ritense.document.repository.impl.JsonSchemaDocumentRepository +import com.ritense.document.service.DocumentSearchService +import com.ritense.document.service.SearchFieldService +import com.ritense.outbox.OutboxService +import com.ritense.valtimo.contract.authentication.UserManagementService +import org.springframework.boot.autoconfigure.AutoConfiguration +import org.springframework.boot.autoconfigure.AutoConfigureBefore +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean +import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration +import org.springframework.context.annotation.Bean +import org.springframework.core.annotation.Order +import org.springframework.data.mongodb.core.MongoTemplate +import org.springframework.data.mongodb.core.convert.MongoCustomConversions +import org.springframework.data.mongodb.repository.config.EnableMongoRepositories + +@AutoConfiguration +@AutoConfigureBefore(DocumentAutoConfiguration::class, MongoDataAutoConfiguration::class) +@ConditionalOnClass(MongoTemplate::class) +@EnableMongoRepositories(basePackages = ["com.ritense.document.mongodb.repository"]) +class DocumentMongoAutoConfiguration { + + /** + * Registers custom converters so that Jackson [com.fasterxml.jackson.databind.JsonNode]/ + * [com.fasterxml.jackson.databind.node.ObjectNode] fields in MongoDB documents are serialized + * as proper JSON rather than as Jackson's internal object structure. + * + * Must run before [MongoDataAutoConfiguration] so this bean wins the + * [ConditionalOnMissingBean] check there. + */ + @Bean + @ConditionalOnMissingBean(MongoCustomConversions::class) + fun mongoCustomConversions(objectMapper: ObjectMapper): MongoCustomConversions = + MongoCustomConversions( + listOf( + JsonNodeWriteConverter(), + DocumentToJsonNodeReadConverter(objectMapper), + DocumentToObjectNodeReadConverter(objectMapper), + ) + ) + + @Bean + @ConditionalOnMissingBean + fun jsonSchemaDocumentDefinitionMongoMapper(): JsonSchemaDocumentDefinitionMongoMapper = + JsonSchemaDocumentDefinitionMongoMapper() + + @Bean + @ConditionalOnMissingBean + fun jsonSchemaDocumentCaseDefinitionMongoMapper(): JsonSchemaDocumentCaseDefinitionMongoMapper = + JsonSchemaDocumentCaseDefinitionMongoMapper() + + @Bean + @ConditionalOnMissingBean + fun mongoPermissionConditionTranslator( + mongoMappers: List>, + authorizationService: AuthorizationService, + documentRepository: JsonSchemaDocumentRepository, + ): MongoPermissionConditionTranslator = + MongoPermissionConditionTranslator(mongoMappers, authorizationService, documentRepository) + + @Bean + @ConditionalOnMissingBean + fun documentMongoQueryService( + mongoTemplate: MongoTemplate, + authorizationService: AuthorizationService, + translator: MongoPermissionConditionTranslator, + ): DocumentMongoQueryService = + DocumentMongoQueryService(mongoTemplate, authorizationService, translator) + + @Bean + @ConditionalOnMissingBean + fun documentMongoSyncService( + repository: JsonSchemaDocumentMongoRepository, + objectMapper: ObjectMapper, + ): DocumentMongoSyncService = + DocumentMongoSyncService(repository, objectMapper) + + @Bean + fun documentMongoEventHandler(syncService: DocumentMongoSyncService): DocumentMongoEventHandler = + DocumentMongoEventHandler(syncService) + + @Bean + @ConditionalOnMissingBean + fun documentMongoBackfillService( + jpaRepository: JsonSchemaDocumentRepository, + mongoRepository: JsonSchemaDocumentMongoRepository, + objectMapper: ObjectMapper, + ): DocumentMongoBackfillService = + DocumentMongoBackfillService(jpaRepository, mongoRepository, objectMapper) + + @Order(293) + @Bean + @ConditionalOnMissingBean + fun documentMongoHttpSecurityConfigurer(): DocumentMongoHttpSecurityConfigurer = + DocumentMongoHttpSecurityConfigurer() + + @Bean + @ConditionalOnMissingBean(DocumentSearchService::class) + fun documentSearchService( + mongoTemplate: MongoTemplate, + translator: MongoPermissionConditionTranslator, + authorizationService: AuthorizationService, + jpaRepository: JsonSchemaDocumentRepository, + userManagementService: UserManagementService, + searchFieldService: SearchFieldService, + outboxService: OutboxService, + objectMapper: ObjectMapper, + ): JsonSchemaDocumentMongoSearchService = + JsonSchemaDocumentMongoSearchService( + mongoTemplate, + translator, + authorizationService, + jpaRepository, + userManagementService, + searchFieldService, + outboxService, + objectMapper, + ) + + @Bean + @ConditionalOnMissingBean + fun documentMongoBackfillResource( + backfillService: DocumentMongoBackfillService, + ): DocumentMongoBackfillResource = + DocumentMongoBackfillResource(backfillService) +} diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/converter/JsonNodeMongoConverters.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/converter/JsonNodeMongoConverters.kt new file mode 100644 index 0000000000..291777be22 --- /dev/null +++ b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/converter/JsonNodeMongoConverters.kt @@ -0,0 +1,58 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.mongodb.converter + +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.ObjectNode +import org.bson.Document +import org.bson.json.JsonMode +import org.bson.json.JsonWriterSettings +import org.springframework.core.convert.converter.Converter +import org.springframework.data.convert.ReadingConverter +import org.springframework.data.convert.WritingConverter + +private val RELAXED_JSON = JsonWriterSettings.builder().outputMode(JsonMode.RELAXED).build() + +/** + * Converts a Jackson [JsonNode] to a BSON [Document] so that Spring Data MongoDB + * stores the actual JSON structure rather than Jackson's internal object fields. + * Only applicable to object-typed [JsonNode] fields (e.g. [definitionId]). + * Array-typed fields should use [Any] instead of [JsonNode] to avoid this converter. + */ +@WritingConverter +class JsonNodeWriteConverter : Converter { + override fun convert(source: JsonNode): Document = Document.parse(source.toString()) +} + +/** + * Converts a BSON [Document] back to a Jackson [JsonNode] when reading a [JsonNode]-typed field. + */ +@ReadingConverter +class DocumentToJsonNodeReadConverter(private val objectMapper: ObjectMapper) : Converter { + override fun convert(source: Document): JsonNode = + objectMapper.readTree(source.toJson(RELAXED_JSON)) +} + +/** + * Converts a BSON [Document] back to a Jackson [ObjectNode] when reading an [ObjectNode]-typed field. + */ +@ReadingConverter +class DocumentToObjectNodeReadConverter(private val objectMapper: ObjectMapper) : Converter { + override fun convert(source: Document): ObjectNode = + objectMapper.readTree(source.toJson(RELAXED_JSON)) as ObjectNode +} diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/domain/JsonSchemaDocumentDocument.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/domain/JsonSchemaDocumentDocument.kt new file mode 100644 index 0000000000..3d41c60e4c --- /dev/null +++ b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/domain/JsonSchemaDocumentDocument.kt @@ -0,0 +1,54 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.mongodb.domain + +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.ObjectNode +import com.ritense.document.web.rest.dto.CaseTagResponseDto +import org.springframework.data.annotation.Id +import org.springframework.data.mongodb.core.mapping.Document +import java.time.LocalDateTime + +/** + * MongoDB read model for [com.ritense.document.domain.impl.JsonSchemaDocument]. + * + * Field names and types mirror the Jackson serialization of the JPA entity: + * - [definitionId] matches the `definitionId()` getter → [com.ritense.document.domain.impl.JsonSchemaDocumentDefinitionId] + * - [internalStatus] matches the `internalStatus()` getter → plain key String + * - [caseTags] matches the `caseTags()` getter → [CaseTagResponseDto] list + * - [relations] matches the `relations()` getter → stored as [Any] (can be array or object) to avoid JPA entity coupling + * - [relatedFiles] matches the `relatedFiles()` getter → stored as [Any] (RelatedFile is an interface) + */ +@Document(collection = "json_schema_document") +data class JsonSchemaDocumentDocument( + @Id val id: String, + val content: ObjectNode?, + val definitionId: JsonNode?, + val createdOn: LocalDateTime?, + val modifiedOn: LocalDateTime?, + val createdBy: String?, + val sequence: Long?, + val version: Int?, + val assigneeId: String?, + val assigneeFullName: String?, + val internalStatus: String?, + val caseTags: List?, + val relations: Any?, + val relatedFiles: Any?, + val retentionDate: LocalDateTime?, + val contentText: String? = null, +) diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/handler/DocumentMongoEventHandler.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/handler/DocumentMongoEventHandler.kt new file mode 100644 index 0000000000..45e0b9454e --- /dev/null +++ b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/handler/DocumentMongoEventHandler.kt @@ -0,0 +1,70 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.mongodb.handler + +import com.ritense.document.event.DocumentAssigned +import com.ritense.document.event.DocumentCreated +import com.ritense.document.event.DocumentUnassigned +import com.ritense.document.event.DocumentUpdated +import com.ritense.document.mongodb.service.DocumentMongoSyncService +import com.ritense.inbox.ValtimoEvent +import com.ritense.inbox.ValtimoEventHandler +import io.github.oshai.kotlinlogging.KotlinLogging + +/** + * Listens to document domain events from the Valtimo inbox and keeps the MongoDB read + * model in sync. Works with both the outbox-enabled (RabbitMQ) and outbox-disabled + * (local Spring event) modes because both paths converge on [ValtimoEventHandler]. + */ +class DocumentMongoEventHandler( + private val syncService: DocumentMongoSyncService, +) : ValtimoEventHandler { + + override fun handle(event: ValtimoEvent) { + when (event.type) { + in UPSERT_EVENT_TYPES -> syncService.upsert(event) + DELETED_EVENT_TYPE -> { + val id = event.resultId + if (id != null) { + syncService.delete(id) + } else { + logger.warn { "Received DocumentDeleted event with null resultId — skipping delete" } + } + } + else -> { + // Events not related to json_schema_document (e.g. DocumentsListed) are ignored + } + } + } + + companion object { + private val logger = KotlinLogging.logger {} + + val UPSERT_EVENT_TYPES: Set = setOf( + DocumentCreated.TYPE, + DocumentUpdated.TYPE, + DocumentAssigned.TYPE, + DocumentUnassigned.TYPE, + "com.ritense.valtimo.document.status.changed", + "com.ritense.valtimo.document.tags.changed", + "com.ritense.valtimo.document.retentiondate.set", + "com.ritense.valtimo.document.retentiondate.unset", + ) + + const val DELETED_EVENT_TYPE = "com.ritense.valtimo.document.deleted" + } +} diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/repository/JsonSchemaDocumentMongoRepository.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/repository/JsonSchemaDocumentMongoRepository.kt new file mode 100644 index 0000000000..9e5dbf40dc --- /dev/null +++ b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/repository/JsonSchemaDocumentMongoRepository.kt @@ -0,0 +1,22 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.mongodb.repository + +import com.ritense.document.mongodb.domain.JsonSchemaDocumentDocument +import org.springframework.data.mongodb.repository.MongoRepository + +interface JsonSchemaDocumentMongoRepository : MongoRepository diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/security/DocumentMongoHttpSecurityConfigurer.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/security/DocumentMongoHttpSecurityConfigurer.kt new file mode 100644 index 0000000000..ee9050ae70 --- /dev/null +++ b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/security/DocumentMongoHttpSecurityConfigurer.kt @@ -0,0 +1,38 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.mongodb.security + +import com.ritense.valtimo.contract.authentication.AuthoritiesConstants.ADMIN +import com.ritense.valtimo.contract.security.config.HttpConfigurerConfigurationException +import com.ritense.valtimo.contract.security.config.HttpSecurityConfigurer +import org.springframework.http.HttpMethod.POST +import org.springframework.security.config.annotation.web.builders.HttpSecurity +import org.springframework.security.web.util.matcher.AntPathRequestMatcher.antMatcher + +class DocumentMongoHttpSecurityConfigurer : HttpSecurityConfigurer { + + override fun configure(http: HttpSecurity) { + try { + http.authorizeHttpRequests { requests -> + requests.requestMatchers(antMatcher(POST, "/api/management/v1/document-mongodb/backfill")) + .hasAuthority(ADMIN) + } + } catch (e: Exception) { + throw HttpConfigurerConfigurationException(e) + } + } +} diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/ContentTextExtractor.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/ContentTextExtractor.kt new file mode 100644 index 0000000000..e2acdcdba2 --- /dev/null +++ b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/ContentTextExtractor.kt @@ -0,0 +1,39 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.mongodb.service + +import com.fasterxml.jackson.databind.JsonNode + +/** + * Extracts all leaf values from a [JsonNode] as a single space-separated string. + * Used to populate [com.ritense.document.mongodb.domain.JsonSchemaDocumentDocument.contentText] + * for full-document search. + */ +fun extractLeafValues(node: JsonNode?): String? { + if (node == null) return null + val parts = mutableListOf() + collectLeaves(node, parts) + return parts.joinToString(" ").ifBlank { null } +} + +private fun collectLeaves(node: JsonNode, out: MutableList) { + when { + node.isObject -> node.fields().forEach { (_, v) -> collectLeaves(v, out) } + node.isArray -> node.forEach { collectLeaves(it, out) } + !node.isNull && !node.isMissingNode -> out.add(node.asText()) + } +} diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/DocumentMongoBackfillService.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/DocumentMongoBackfillService.kt new file mode 100644 index 0000000000..7c0d472431 --- /dev/null +++ b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/DocumentMongoBackfillService.kt @@ -0,0 +1,72 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.mongodb.service + +import com.fasterxml.jackson.databind.ObjectMapper +import com.ritense.authorization.AuthorizationContext.Companion.runWithoutAuthorization +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.mongodb.domain.JsonSchemaDocumentDocument +import com.ritense.document.mongodb.repository.JsonSchemaDocumentMongoRepository +import com.ritense.document.repository.impl.JsonSchemaDocumentRepository +import io.github.oshai.kotlinlogging.KotlinLogging +import org.springframework.data.domain.PageRequest +import org.springframework.transaction.annotation.Transactional + +open class DocumentMongoBackfillService( + private val jpaRepository: JsonSchemaDocumentRepository, + private val mongoRepository: JsonSchemaDocumentMongoRepository, + private val objectMapper: ObjectMapper, +) { + + /** + * Copies all existing [JsonSchemaDocument] rows from the relational database to MongoDB. + * Processes documents in pages of [pageSize] to avoid loading the entire table into memory. + * + * @return total number of documents migrated + */ + @Transactional(readOnly = true) + open fun backfill(pageSize: Int = DEFAULT_PAGE_SIZE): Long { + var page = 0 + var total = 0L + do { + val slice = runWithoutAuthorization { jpaRepository.findAll(PageRequest.of(page++, pageSize)) } + if (slice.isEmpty) break + + val docs = mutableListOf() + for (jpaDoc in slice.content) { + try { + val json = objectMapper.writeValueAsString(jpaDoc) + val doc = objectMapper.readValue(json, JsonSchemaDocumentDocument::class.java) + docs.add(doc.copy(contentText = extractLeafValues(doc.content))) + } catch (e: Exception) { + logger.warn(e) { "Failed to convert document to MongoDB document — skipping" } + } + } + mongoRepository.saveAll(docs) + total += docs.size + logger.debug { "Backfilled page ${page - 1}: ${docs.size} documents (total so far: $total)" } + } while (slice.hasNext()) + + logger.info { "Backfill complete: $total documents migrated to MongoDB" } + return total + } + + companion object { + private val logger = KotlinLogging.logger {} + const val DEFAULT_PAGE_SIZE = 500 + } +} diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/DocumentMongoQueryService.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/DocumentMongoQueryService.kt new file mode 100644 index 0000000000..e251299293 --- /dev/null +++ b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/DocumentMongoQueryService.kt @@ -0,0 +1,71 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.mongodb.service + +import com.ritense.authorization.Action +import com.ritense.authorization.AuthorizationService +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.valtimo.contract.utils.SecurityUtils +import com.ritense.document.mongodb.authorization.MongoPermissionConditionTranslator +import com.ritense.document.mongodb.domain.JsonSchemaDocumentDocument +import com.ritense.document.service.JsonSchemaDocumentActionProvider +import org.springframework.data.domain.Page +import org.springframework.data.domain.PageImpl +import org.springframework.data.domain.Pageable +import org.springframework.data.mongodb.core.MongoTemplate +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query + +class DocumentMongoQueryService( + private val mongoTemplate: MongoTemplate, + private val authorizationService: AuthorizationService, + private val translator: MongoPermissionConditionTranslator, +) { + + /** + * Returns a page of documents for the given [definitionName], restricted to those + * the current user is allowed to see (VIEW_LIST action). + */ + fun findAllByDefinitionName(definitionName: String, pageable: Pageable): Page { + val combined = buildCriteria(JsonSchemaDocumentActionProvider.VIEW_LIST) + .andOperator(Criteria.where("definitionId.name").`is`(definitionName)) + + val countQuery = Query(combined) + val dataQuery = Query(combined).with(pageable) + + val total = mongoTemplate.count(countQuery, JsonSchemaDocumentDocument::class.java) + val content = mongoTemplate.find(dataQuery, JsonSchemaDocumentDocument::class.java) + return PageImpl(content, pageable, total) + } + + /** + * Returns the document with the given [id] if the current user has VIEW permission, + * or `null` if it does not exist or is not accessible. + */ + fun findById(id: String): JsonSchemaDocumentDocument? { + val combined = buildCriteria(JsonSchemaDocumentActionProvider.VIEW) + .andOperator(Criteria.where("_id").`is`(id)) + return mongoTemplate.findOne(Query(combined), JsonSchemaDocumentDocument::class.java) + } + + private fun buildCriteria(action: Action): Criteria { + val userRoles = SecurityUtils.getCurrentUserRoles().toSet() + val permissions = authorizationService.getPermissions(JsonSchemaDocument::class.java, action) + .filter { it.role.key in userRoles } + return translator.toCriteria(permissions, action) + } +} diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/DocumentMongoSyncService.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/DocumentMongoSyncService.kt new file mode 100644 index 0000000000..08e0be0c10 --- /dev/null +++ b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/DocumentMongoSyncService.kt @@ -0,0 +1,49 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.mongodb.service + +import com.fasterxml.jackson.databind.ObjectMapper +import com.ritense.document.mongodb.domain.JsonSchemaDocumentDocument +import com.ritense.document.mongodb.repository.JsonSchemaDocumentMongoRepository +import com.ritense.inbox.ValtimoEvent +import io.github.oshai.kotlinlogging.KotlinLogging + +class DocumentMongoSyncService( + private val repository: JsonSchemaDocumentMongoRepository, + private val objectMapper: ObjectMapper, +) { + + fun upsert(event: ValtimoEvent) { + val result = event.result + if (result == null) { + logger.warn { "Received document event ${event.type} for id=${event.resultId} with null result — skipping upsert" } + return + } + val doc = objectMapper.treeToValue(result, JsonSchemaDocumentDocument::class.java) + repository.save(doc.copy(contentText = extractLeafValues(doc.content))) + logger.debug { "Upserted document ${doc.id} in MongoDB (event: ${event.type})" } + } + + fun delete(documentId: String) { + repository.deleteById(documentId) + logger.debug { "Deleted document $documentId from MongoDB" } + } + + companion object { + private val logger = KotlinLogging.logger {} + } +} diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/JsonSchemaDocumentMongoSearchService.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/JsonSchemaDocumentMongoSearchService.kt new file mode 100644 index 0000000000..55f486fe68 --- /dev/null +++ b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/JsonSchemaDocumentMongoSearchService.kt @@ -0,0 +1,361 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.mongodb.service + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.ArrayNode +import com.ritense.authorization.Action +import com.ritense.authorization.AuthorizationContext.Companion.runWithoutAuthorization +import com.ritense.authorization.AuthorizationService +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.domain.impl.JsonSchemaDocumentId +import com.ritense.document.domain.search.AdvancedSearchRequest +import com.ritense.document.domain.search.AssigneeFilter +import com.ritense.document.domain.search.DatabaseSearchType +import com.ritense.document.domain.search.SearchOperator +import com.ritense.document.domain.search.SearchRequestMapper +import com.ritense.document.domain.search.SearchRequestValidator +import com.ritense.document.domain.search.SearchWithConfigRequest +import com.ritense.document.event.DocumentsListed +import com.ritense.document.mongodb.authorization.MongoPermissionConditionTranslator +import com.ritense.document.mongodb.domain.JsonSchemaDocumentDocument +import com.ritense.document.repository.impl.JsonSchemaDocumentRepository +import com.ritense.document.service.DocumentSearchService +import com.ritense.document.service.JsonSchemaDocumentActionProvider +import com.ritense.document.service.SearchFieldService +import com.ritense.document.service.impl.SearchRequest +import com.ritense.outbox.OutboxService +import com.ritense.valtimo.contract.authentication.UserManagementService +import com.ritense.valtimo.contract.blueprint.BlueprintType +import com.ritense.valtimo.contract.utils.RequestHelper +import com.ritense.valtimo.contract.utils.SecurityUtils +import org.apache.commons.lang3.NotImplementedException +import org.springframework.data.domain.Page +import org.springframework.data.domain.PageImpl +import org.springframework.data.domain.Pageable +import org.springframework.data.domain.Sort +import org.springframework.data.mongodb.core.MongoTemplate +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import java.util.regex.Pattern + +class JsonSchemaDocumentMongoSearchService( + private val mongoTemplate: MongoTemplate, + private val translator: MongoPermissionConditionTranslator, + private val authorizationService: AuthorizationService, + private val jpaRepository: JsonSchemaDocumentRepository, + private val userManagementService: UserManagementService, + private val searchFieldService: SearchFieldService, + private val outboxService: OutboxService, + private val objectMapper: ObjectMapper, +) : DocumentSearchService { + + override fun search( + searchRequest: SearchRequest, + blueprintType: BlueprintType, + pageable: Pageable + ): Page { + val parts = mutableListOf() + + parts.add(buildAuthCriteria(JsonSchemaDocumentActionProvider.VIEW_LIST)) + parts.add(Criteria.where(BLUEPRINT_TYPE_FIELD).`is`(blueprintType.name)) + + if (!searchRequest.documentDefinitionName.isNullOrEmpty()) { + parts.add(Criteria.where(DEFINITION_NAME_FIELD).`is`(searchRequest.documentDefinitionName)) + } + if (!searchRequest.createdBy.isNullOrEmpty()) { + parts.add(Criteria.where("createdBy").`is`(searchRequest.createdBy)) + } + if (searchRequest.sequence != null) { + parts.add(Criteria.where("sequence").`is`(searchRequest.sequence)) + } + if (!searchRequest.globalSearchFilter.isNullOrEmpty()) { + throw NotImplementedException("globalSearchFilter is not supported in the MongoDB search service") + } + searchRequest.otherFilters?.forEach { sc -> + parts.add(Criteria.where("content.${sc.path}").`is`(sc.value)) + } + + return executeSearch(Criteria().andOperator(*parts.toTypedArray()), pageable) + } + + override fun search( + documentDefinitionName: String, + blueprintType: BlueprintType, + searchWithConfigRequest: SearchWithConfigRequest, + pageable: Pageable + ): Page { + val zoneOffset = RequestHelper.getZoneOffset() + val searchFieldMap = searchFieldService.getSearchFields(documentDefinitionName) + .associateBy { it.key } + + val otherFilters = searchWithConfigRequest.otherFilters + .map { filter -> SearchRequestMapper.toOtherFilter(filter, searchFieldMap[filter.key], zoneOffset) } + + val advancedSearchRequest = SearchRequestMapper.toAdvancedSearchRequest(searchWithConfigRequest, otherFilters) + + return search( + documentDefinitionName, + blueprintType, + advancedSearchRequest, + pageable, + JsonSchemaDocumentActionProvider.VIEW_LIST + ) + } + + override fun search( + documentDefinitionName: String, + blueprintType: BlueprintType, + advancedSearchRequest: AdvancedSearchRequest, + pageable: Pageable + ): Page { + return search( + documentDefinitionName, + blueprintType, + advancedSearchRequest, + pageable, + JsonSchemaDocumentActionProvider.VIEW_LIST + ) + } + + override fun searchForExport( + documentDefinitionName: String, + blueprintType: BlueprintType, + searchWithConfigRequest: SearchWithConfigRequest, + pageable: Pageable + ): Page { + val zoneOffset = RequestHelper.getZoneOffset() + val searchFieldMap = searchFieldService.getSearchFields(documentDefinitionName) + .associateBy { it.key } + + val otherFilters = searchWithConfigRequest.otherFilters + .map { filter -> SearchRequestMapper.toOtherFilter(filter, searchFieldMap[filter.key], zoneOffset) } + + val advancedSearchRequest = SearchRequestMapper.toAdvancedSearchRequest(searchWithConfigRequest, otherFilters) + + return search( + documentDefinitionName, + blueprintType, + advancedSearchRequest, + pageable, + JsonSchemaDocumentActionProvider.EXPORT + ) + } + + override fun count( + documentDefinitionName: String, + blueprintType: BlueprintType, + advancedSearchRequest: AdvancedSearchRequest + ): Long { + SearchRequestValidator.validate(advancedSearchRequest) + val combined = buildCombinedCriteria( + documentDefinitionName, + blueprintType, + advancedSearchRequest, + JsonSchemaDocumentActionProvider.VIEW_LIST + ) + return mongoTemplate.count(Query(combined), JsonSchemaDocumentDocument::class.java) + } + + private fun search( + documentDefinitionName: String, + blueprintType: BlueprintType, + advancedSearchRequest: AdvancedSearchRequest, + pageable: Pageable, + action: Action + ): Page { + SearchRequestValidator.validate(advancedSearchRequest) + val combined = buildCombinedCriteria(documentDefinitionName, blueprintType, advancedSearchRequest, action) + return executeSearch(combined, pageable) + } + + private fun buildCombinedCriteria( + documentDefinitionName: String?, + blueprintType: BlueprintType, + searchRequest: AdvancedSearchRequest, + action: Action + ): Criteria { + val parts = mutableListOf() + + parts.add(buildAuthCriteria(action)) + parts.add(Criteria.where(BLUEPRINT_TYPE_FIELD).`is`(blueprintType.name)) + + if (!documentDefinitionName.isNullOrEmpty()) { + parts.add(Criteria.where(DEFINITION_NAME_FIELD).`is`(documentDefinitionName)) + } + + if (searchRequest.assigneeFilter != null && searchRequest.assigneeFilter != AssigneeFilter.ALL) { + parts.add(buildAssigneeFilterCriteria(searchRequest.assigneeFilter)) + } + + if (!searchRequest.statusFilter.isNullOrEmpty()) { + parts.add(buildStatusFilterCriteria(searchRequest.statusFilter)) + } + + if (!searchRequest.caseTagsFilter.isNullOrEmpty()) { + parts.add(Criteria.where("caseTags.key").`in`(searchRequest.caseTagsFilter)) + } + + if (!searchRequest.globalSearchFilter.isNullOrEmpty()) { + parts.add(buildGlobalSearchCriteria(searchRequest.globalSearchFilter)) + } + + if (!searchRequest.otherFilters.isNullOrEmpty()) { + parts.add(buildOtherFiltersCriteria(searchRequest.otherFilters, searchRequest.searchOperator)) + } + + return Criteria().andOperator(*parts.toTypedArray()) + } + + private fun buildAuthCriteria(action: Action): Criteria { + val userRoles = SecurityUtils.getCurrentUserRoles().toSet() + val permissions = authorizationService.getPermissions(JsonSchemaDocument::class.java, action) + .filter { it.role.key in userRoles } + return translator.toCriteria(permissions, action) + } + + private fun buildAssigneeFilterCriteria(filter: AssigneeFilter): Criteria { + val userId = userManagementService.currentUser.username + return when (filter) { + AssigneeFilter.MINE -> Criteria.where("assigneeId").`is`(userId) + AssigneeFilter.OPEN -> Criteria.where("assigneeId").isNull() + else -> Criteria() + } + } + + private fun buildStatusFilterCriteria(statusKeys: Set): Criteria { + val conditions = statusKeys.map { key -> + if (key.isNullOrEmpty()) Criteria.where("internalStatus").isNull() + else Criteria.where("internalStatus").`is`(key) + } + return if (conditions.size == 1) conditions.first() + else Criteria().orOperator(*conditions.toTypedArray()) + } + + private fun buildOtherFiltersCriteria( + filters: List, + operator: SearchOperator? + ): Criteria { + val filterCriteria = filters.map { buildSingleFilterCriteria(it) } + return if (operator == SearchOperator.OR) { + Criteria().orOperator(*filterCriteria.toTypedArray()) + } else { + Criteria().andOperator(*filterCriteria.toTypedArray()) + } + } + + private fun buildSingleFilterCriteria(filter: AdvancedSearchRequest.OtherFilter): Criteria { + val mongoField = when { + filter.path.startsWith(DOC_PREFIX) -> "content.${filter.path.removePrefix(DOC_PREFIX)}" + filter.path.startsWith(CASE_PREFIX) -> filter.path.removePrefix(CASE_PREFIX) + else -> throw IllegalArgumentException("Search path doesn't start with known prefix: '${filter.path}'") + } + + return when (filter.searchType) { + DatabaseSearchType.EQUAL -> { + val values = filter.getValues() + when { + values.isEmpty() -> Criteria() + values.size == 1 -> applyEqualCriteria(Criteria.where(mongoField), values[0]) + else -> Criteria().orOperator(*values.map { applyEqualCriteria(Criteria.where(mongoField), it) }.toTypedArray()) + } + } + DatabaseSearchType.LIKE -> { + val values = filter.getValues() + when { + values.isEmpty() -> Criteria() + values.size == 1 -> applyLikeCriteria(Criteria.where(mongoField), values[0]) + else -> Criteria().orOperator(*values.map { applyLikeCriteria(Criteria.where(mongoField), it) }.toTypedArray()) + } + } + DatabaseSearchType.IN -> Criteria.where(mongoField).`in`(filter.getValues()) + DatabaseSearchType.GREATER_THAN_OR_EQUAL_TO -> Criteria.where(mongoField).gte(filter.rangeFromValue()!!) + DatabaseSearchType.LESS_THAN_OR_EQUAL_TO -> Criteria.where(mongoField).lte(filter.rangeToValue()!!) + DatabaseSearchType.BETWEEN -> Criteria.where(mongoField).gte(filter.rangeFromValue()!!).lte(filter.rangeToValue()!!) + else -> throw NotImplementedException("Search type '${filter.searchType}' is not supported in the MongoDB search service") + } + } + + private fun buildGlobalSearchCriteria(filter: String): Criteria = + Criteria.where("contentText").regex(".*${Pattern.quote(filter.trim())}.*", "i") + + private fun applyEqualCriteria(criteria: Criteria, value: Any?): Criteria { + return if (value is String) { + criteria.regex("^${Pattern.quote(value.trim())}$", "i") + } else { + criteria.`is`(value) + } + } + + private fun applyLikeCriteria(criteria: Criteria, value: Any?): Criteria { + if (value !is String) { + throw IllegalArgumentException("LIKE search requires String values, got: ${value?.javaClass?.simpleName}") + } + return criteria.regex(".*${Pattern.quote(value.trim())}.*", "i") + } + + private fun executeSearch(combined: Criteria, pageable: Pageable): Page { + val translatedSort = translateSort(pageable.sort) + val dataQuery = Query(combined).with(translatedSort) + if (pageable.isPaged) { + dataQuery.skip(pageable.offset).limit(pageable.pageSize) + } + + val total = mongoTemplate.count(Query(combined), JsonSchemaDocumentDocument::class.java) + val ids = mongoTemplate.find(dataQuery, JsonSchemaDocumentDocument::class.java).map { it.id } + + val docIds = ids.map { JsonSchemaDocumentId.existingId(it) } + val entities = runWithoutAuthorization { jpaRepository.findAllById(docIds) } + val entityMap = entities.associateBy { it.id().toString() } + val orderedEntities = ids.mapNotNull { entityMap[it] } + + outboxService.send { DocumentsListed(objectMapper.valueToTree(orderedEntities)) } + + return PageImpl(orderedEntities, pageable, total) + } + + private fun translateSort(sort: Sort): Sort { + if (sort.isUnsorted) return sort + val orders = sort.map { order -> + val mongoField = when { + order.property.startsWith(DOC_PREFIX) -> "content.${order.property.removePrefix(DOC_PREFIX)}" + order.property.startsWith(CASE_PREFIX) -> order.property.removePrefix(CASE_PREFIX) + else -> order.property + } + if (order.isAscending) Sort.Order.asc(mongoField) else Sort.Order.desc(mongoField) + }.toList() + return Sort.by(orders) + } + + companion object { + private const val DOC_PREFIX = "doc:" + private const val CASE_PREFIX = "case:" + private const val DEFINITION_NAME_FIELD = "definitionId.name" + private const val BLUEPRINT_TYPE_FIELD = "definitionId.blueprintId.blueprintType" + + /** + * Calls [AdvancedSearchRequest.OtherFilter.getRangeFrom] via reflection to bypass the + * Kotlin type-bounds check. The Java method signature uses `>` + * which Kotlin cannot satisfy with `Any`, but at runtime it just returns the boxed value. + */ + private fun AdvancedSearchRequest.OtherFilter.rangeFromValue(): Any? = + AdvancedSearchRequest.OtherFilter::class.java.getMethod("getRangeFrom").invoke(this) + + private fun AdvancedSearchRequest.OtherFilter.rangeToValue(): Any? = + AdvancedSearchRequest.OtherFilter::class.java.getMethod("getRangeTo").invoke(this) + } +} diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/web/DocumentMongoBackfillResource.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/web/DocumentMongoBackfillResource.kt new file mode 100644 index 0000000000..05f3e7f617 --- /dev/null +++ b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/web/DocumentMongoBackfillResource.kt @@ -0,0 +1,45 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.mongodb.web + +import com.ritense.document.mongodb.service.DocumentMongoBackfillService +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RequestParam +import org.springframework.web.bind.annotation.RestController + +@RestController +@RequestMapping("/api/management/v1/document-mongodb") +class DocumentMongoBackfillResource( + private val backfillService: DocumentMongoBackfillService, +) { + + /** + * Triggers a full backfill of all [com.ritense.document.domain.impl.JsonSchemaDocument] + * records to the MongoDB read model. + * + * Only accessible to users with ROLE_ADMIN. + */ + @PostMapping("/backfill") + fun backfill( + @RequestParam(defaultValue = "${DocumentMongoBackfillService.DEFAULT_PAGE_SIZE}") pageSize: Int, + ): ResponseEntity> { + val count = backfillService.backfill(pageSize) + return ResponseEntity.ok(mapOf("migratedCount" to count)) + } +} diff --git a/backend/case-mongodb/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/backend/case-mongodb/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports new file mode 100644 index 0000000000..c1219ff1a4 --- /dev/null +++ b/backend/case-mongodb/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -0,0 +1 @@ +com.ritense.document.mongodb.autoconfigure.DocumentMongoAutoConfiguration diff --git a/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/BaseMongoIntegrationTest.kt b/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/BaseMongoIntegrationTest.kt new file mode 100644 index 0000000000..6a0843efd6 --- /dev/null +++ b/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/BaseMongoIntegrationTest.kt @@ -0,0 +1,149 @@ +/* + * Copyright 2015-2025 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.mongodb + +import com.fasterxml.jackson.databind.ObjectMapper +import com.ritense.audit.service.AuditEventProcessor +import com.ritense.authorization.permission.ConditionContainer +import com.ritense.authorization.permission.Permission +import com.ritense.authorization.permission.PermissionRepository +import com.ritense.authorization.role.Role +import com.ritense.authorization.role.RoleRepository +import com.ritense.document.domain.impl.JsonSchemaDocumentDefinition +import com.ritense.document.domain.impl.snapshot.JsonSchemaDocumentSnapshot +import com.ritense.document.domain.impl.searchfield.SearchField +import com.ritense.document.mongodb.repository.JsonSchemaDocumentMongoRepository +import com.ritense.document.service.impl.JsonSchemaDocumentService +import com.ritense.document.service.JsonSchemaDocumentActionProvider +import com.ritense.document.service.JsonSchemaDocumentDefinitionActionProvider +import com.ritense.document.service.JsonSchemaDocumentSnapshotActionProvider +import com.ritense.document.service.SearchFieldActionProvider +import com.ritense.outbox.OutboxService +import com.ritense.testutilscommon.junit.extension.LiquibaseRunnerExtension +import com.ritense.valtimo.contract.authentication.UserManagementService +import com.ritense.valtimo.contract.mail.MailSender +import com.ritense.valtimo.service.ProcessDefinitionCaseDefinitionLinker +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Tag +import org.junit.jupiter.api.extension.ExtendWith +import org.mockito.Answers +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.context.event.SimpleApplicationEventMulticaster +import org.springframework.test.context.bean.override.mockito.MockitoBean +import org.springframework.test.context.bean.override.mockito.MockitoSpyBean +import org.springframework.test.context.junit.jupiter.SpringExtension +import org.springframework.transaction.annotation.Transactional +import java.util.UUID + +@SpringBootTest +@ExtendWith(SpringExtension::class, LiquibaseRunnerExtension::class) +@Tag("integration") +@Transactional +abstract class BaseMongoIntegrationTest { + + @MockitoBean(answers = Answers.RETURNS_DEEP_STUBS) + lateinit var userManagementService: UserManagementService + + @MockitoBean + lateinit var applicationEventMulticaster: SimpleApplicationEventMulticaster + + @MockitoBean + lateinit var processDefinitionCaseDefinitionLinker: ProcessDefinitionCaseDefinitionLinker + + @MockitoBean + lateinit var auditEventProcessor: AuditEventProcessor + + @MockitoBean + lateinit var mailSender: MailSender + + @MockitoSpyBean + lateinit var outboxService: OutboxService + + @Autowired + lateinit var documentService: JsonSchemaDocumentService + + @Autowired + lateinit var mongoRepository: JsonSchemaDocumentMongoRepository + + @Autowired + lateinit var roleRepository: RoleRepository + + @Autowired + lateinit var permissionRepository: PermissionRepository + + @Autowired + lateinit var objectMapper: ObjectMapper + + @BeforeEach + fun setUpBase() { + setUpPermissions() + mongoRepository.deleteAll() + } + + @AfterEach + fun tearDownBase() { + mongoRepository.deleteAll() + } + + private fun setUpPermissions() { + var role = roleRepository.findByKey(FULL_ACCESS_ROLE) + if (role == null) { + role = roleRepository.save(Role(UUID.randomUUID(), FULL_ACCESS_ROLE)) + } + + val permissions = listOf( + Permission(UUID.randomUUID(), com.ritense.document.domain.impl.JsonSchemaDocument::class.java, + mutableListOf(JsonSchemaDocumentActionProvider.VIEW_LIST), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), com.ritense.document.domain.impl.JsonSchemaDocument::class.java, + mutableListOf(JsonSchemaDocumentActionProvider.VIEW), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), com.ritense.document.domain.impl.JsonSchemaDocument::class.java, + mutableListOf(JsonSchemaDocumentActionProvider.MODIFY), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), com.ritense.document.domain.impl.JsonSchemaDocument::class.java, + mutableListOf(JsonSchemaDocumentActionProvider.CREATE), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), com.ritense.document.domain.impl.JsonSchemaDocument::class.java, + mutableListOf(JsonSchemaDocumentActionProvider.CLAIM), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), com.ritense.document.domain.impl.JsonSchemaDocument::class.java, + mutableListOf(JsonSchemaDocumentActionProvider.ASSIGN), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), com.ritense.document.domain.impl.JsonSchemaDocument::class.java, + mutableListOf(JsonSchemaDocumentActionProvider.ASSIGNABLE), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), com.ritense.document.domain.impl.JsonSchemaDocument::class.java, + mutableListOf(JsonSchemaDocumentActionProvider.DELETE), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), SearchField::class.java, + mutableListOf(SearchFieldActionProvider.VIEW_LIST), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), JsonSchemaDocumentDefinition::class.java, + mutableListOf(JsonSchemaDocumentDefinitionActionProvider.VIEW), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), JsonSchemaDocumentDefinition::class.java, + mutableListOf(JsonSchemaDocumentDefinitionActionProvider.VIEW_LIST), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), JsonSchemaDocumentDefinition::class.java, + mutableListOf(JsonSchemaDocumentDefinitionActionProvider.CREATE), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), JsonSchemaDocumentDefinition::class.java, + mutableListOf(JsonSchemaDocumentDefinitionActionProvider.MODIFY), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), JsonSchemaDocumentDefinition::class.java, + mutableListOf(JsonSchemaDocumentDefinitionActionProvider.DELETE), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), JsonSchemaDocumentSnapshot::class.java, + mutableListOf(JsonSchemaDocumentSnapshotActionProvider.VIEW_LIST), ConditionContainer(emptyList()), role!!), + ) + permissionRepository.saveAll(permissions) + } + + companion object { + const val FULL_ACCESS_ROLE: String = "full access role" + const val USERNAME: String = "test@test.com" + } +} diff --git a/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/TestApplication.kt b/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/TestApplication.kt new file mode 100644 index 0000000000..ba16e71080 --- /dev/null +++ b/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/TestApplication.kt @@ -0,0 +1,22 @@ +/* + * Copyright 2015-2025 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.mongodb + +import org.springframework.boot.autoconfigure.SpringBootApplication + +@SpringBootApplication +class TestApplication diff --git a/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/service/ContentTextExtractorTest.kt b/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/service/ContentTextExtractorTest.kt new file mode 100644 index 0000000000..d51fa05a8e --- /dev/null +++ b/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/service/ContentTextExtractorTest.kt @@ -0,0 +1,107 @@ +/* + * Copyright 2015-2025 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.mongodb.service + +import com.fasterxml.jackson.databind.ObjectMapper +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test + +class ContentTextExtractorTest { + + private val mapper = ObjectMapper() + + @Test + fun `null input returns null`() { + assertThat(extractLeafValues(null)).isNull() + } + + @Test + fun `empty object returns null`() { + val node = mapper.readTree("{}") + assertThat(extractLeafValues(node)).isNull() + } + + @Test + fun `flat object joins all leaf values`() { + val node = mapper.readTree("""{"firstName":"John","lastName":"Doe"}""") + val result = extractLeafValues(node) + assertThat(result).contains("John") + assertThat(result).contains("Doe") + } + + @Test + fun `nested object extracts leaves recursively`() { + val node = mapper.readTree("""{"person":{"name":"Alice","city":"Utrecht"}}""") + val result = extractLeafValues(node) + assertThat(result).contains("Alice") + assertThat(result).contains("Utrecht") + } + + @Test + fun `array of primitives is extracted`() { + val node = mapper.readTree("""["apple","banana","cherry"]""") + assertThat(extractLeafValues(node)).isEqualTo("apple banana cherry") + } + + @Test + fun `array of objects extracts nested leaves`() { + val node = mapper.readTree("""[{"name":"X"},{"name":"Y"}]""") + val result = extractLeafValues(node) + assertThat(result).contains("X") + assertThat(result).contains("Y") + } + + @Test + fun `null json field values are skipped`() { + val node = mapper.readTree("""{"name":null,"city":null}""") + assertThat(extractLeafValues(node)).isNull() + } + + @Test + fun `numeric value is converted to string`() { + val node = mapper.readTree("""{"count":42}""") + assertThat(extractLeafValues(node)).isEqualTo("42") + } + + @Test + fun `boolean value is converted to string`() { + val node = mapper.readTree("""{"active":true}""") + assertThat(extractLeafValues(node)).isEqualTo("true") + } + + @Test + fun `mixed types in object are all extracted`() { + val node = mapper.readTree("""{"name":"Bob","age":30,"active":false}""") + val result = extractLeafValues(node) + assertThat(result).contains("Bob") + assertThat(result).contains("30") + assertThat(result).contains("false") + } + + @Test + fun `deeply nested structure is fully extracted`() { + val node = mapper.readTree("""{"a":{"b":{"c":"deep"}}}""") + assertThat(extractLeafValues(node)).isEqualTo("deep") + } + + @Test + fun `mixed null and non-null leaves only includes non-null values`() { + val node = mapper.readTree("""{"name":"Alice","missing":null}""") + val result = extractLeafValues(node) + assertThat(result).isEqualTo("Alice") + } +} diff --git a/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/service/DocumentMongoSyncServiceTest.kt b/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/service/DocumentMongoSyncServiceTest.kt new file mode 100644 index 0000000000..037298b676 --- /dev/null +++ b/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/service/DocumentMongoSyncServiceTest.kt @@ -0,0 +1,144 @@ +/* + * Copyright 2015-2025 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.mongodb.service + +import com.fasterxml.jackson.databind.ObjectMapper +import com.ritense.document.mongodb.domain.JsonSchemaDocumentDocument +import com.ritense.document.mongodb.repository.JsonSchemaDocumentMongoRepository +import com.ritense.inbox.ValtimoEvent +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.ArgumentCaptor +import org.mockito.kotlin.any +import org.mockito.kotlin.capture +import org.mockito.kotlin.mock +import org.mockito.kotlin.never +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever +import java.time.LocalDateTime + +class DocumentMongoSyncServiceTest { + + private val repository: JsonSchemaDocumentMongoRepository = mock() + private val objectMapper: ObjectMapper = mock() + private lateinit var service: DocumentMongoSyncService + + @BeforeEach + fun setUp() { + service = DocumentMongoSyncService(repository, objectMapper) + } + + @Test + fun `upsert with null result skips repository save`() { + val event = valtimoEvent(result = null) + + service.upsert(event) + + verify(repository, never()).save(any()) + } + + @Test + fun `upsert populates contentText with leaf values from content`() { + val realMapper = ObjectMapper() + val content = realMapper.createObjectNode().apply { + put("firstName", "John") + put("city", "Amsterdam") + } + val docDocument = buildDocument(id = "test-id", content = content) + val event = valtimoEvent(result = realMapper.createObjectNode()) + whenever(objectMapper.treeToValue(any(), any>())).thenReturn(docDocument) + + val captor = ArgumentCaptor.forClass(JsonSchemaDocumentDocument::class.java) + service.upsert(event) + verify(repository).save(capture(captor)) + + val saved = captor.value + assertThat(saved.contentText).contains("John") + assertThat(saved.contentText).contains("Amsterdam") + } + + @Test + fun `upsert with null content stores null contentText`() { + val realMapper = ObjectMapper() + val docDocument = buildDocument(id = "no-content-id", content = null) + val event = valtimoEvent(result = realMapper.createObjectNode()) + whenever(objectMapper.treeToValue(any(), any>())).thenReturn(docDocument) + + val captor = ArgumentCaptor.forClass(JsonSchemaDocumentDocument::class.java) + service.upsert(event) + verify(repository).save(capture(captor)) + + assertThat(captor.value.contentText).isNull() + } + + @Test + fun `upsert with nested content extracts all leaf values`() { + val realMapper = ObjectMapper() + val content = realMapper.createObjectNode().apply { + putObject("address").apply { + put("street", "Main Street") + put("number", "42") + } + } + val docDocument = buildDocument(id = "nested-id", content = content) + val event = valtimoEvent(result = realMapper.createObjectNode()) + whenever(objectMapper.treeToValue(any(), any>())).thenReturn(docDocument) + + val captor = ArgumentCaptor.forClass(JsonSchemaDocumentDocument::class.java) + service.upsert(event) + verify(repository).save(capture(captor)) + + val contentText = captor.value.contentText + assertThat(contentText).contains("Main Street") + assertThat(contentText).contains("42") + } + + private fun buildDocument( + id: String, + content: com.fasterxml.jackson.databind.node.ObjectNode?, + ) = JsonSchemaDocumentDocument( + id = id, + content = content, + definitionId = null, + createdOn = null, + modifiedOn = null, + createdBy = null, + sequence = null, + version = null, + assigneeId = null, + assigneeFullName = null, + internalStatus = null, + caseTags = null, + relations = null, + relatedFiles = null, + retentionDate = null, + ) + + private fun valtimoEvent( + result: com.fasterxml.jackson.databind.node.ContainerNode<*>?, + ) = ValtimoEvent( + id = "event-id", + type = "DOCUMENT_CREATED", + date = LocalDateTime.now(), + userId = null, + roles = null, + resultType = null, + resultId = "doc-id", + result = result, + ) +} diff --git a/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/service/JsonSchemaDocumentMongoSearchServiceIntTest.kt b/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/service/JsonSchemaDocumentMongoSearchServiceIntTest.kt new file mode 100644 index 0000000000..14cbc846e1 --- /dev/null +++ b/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/service/JsonSchemaDocumentMongoSearchServiceIntTest.kt @@ -0,0 +1,144 @@ +/* + * Copyright 2015-2025 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.mongodb.service + +import com.fasterxml.jackson.databind.node.ObjectNode +import com.ritense.authorization.AuthorizationContext.Companion.runWithoutAuthorization +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.domain.impl.request.NewDocumentRequest +import com.ritense.document.domain.search.AdvancedSearchRequest +import com.ritense.document.mongodb.BaseMongoIntegrationTest +import com.ritense.document.mongodb.domain.JsonSchemaDocumentDocument +import com.ritense.document.service.DocumentSearchService +import com.ritense.valtimo.contract.blueprint.BlueprintType +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.data.domain.PageRequest +import org.springframework.security.test.context.support.WithMockUser + +@WithMockUser(username = BaseMongoIntegrationTest.USERNAME, authorities = [BaseMongoIntegrationTest.FULL_ACCESS_ROLE]) +class JsonSchemaDocumentMongoSearchServiceIntTest : BaseMongoIntegrationTest() { + + @Autowired + lateinit var documentSearchService: DocumentSearchService + + @Test + fun `globalSearchFilter returns matching document`() { + seedDocument("Funenpark") + + val page = documentSearchService.search( + "house", + BlueprintType.CASE, + AdvancedSearchRequest().globalSearchFilter("Funenpark"), + PageRequest.of(0, 10) + ) + + assertThat(page.totalElements).isEqualTo(1L) + } + + @Test + fun `globalSearchFilter is case insensitive`() { + seedDocument("Funenpark") + + val page = documentSearchService.search( + "house", + BlueprintType.CASE, + AdvancedSearchRequest().globalSearchFilter("FUNENPARK"), + PageRequest.of(0, 10) + ) + + assertThat(page.totalElements).isEqualTo(1L) + } + + @Test + fun `globalSearchFilter excludes non-matching documents`() { + val docA = seedDocument("Funenpark") + seedDocument("Keizersgracht") + + val page = documentSearchService.search( + "house", + BlueprintType.CASE, + AdvancedSearchRequest().globalSearchFilter("Funenpark"), + PageRequest.of(0, 10) + ) + + assertThat(page.totalElements).isEqualTo(1L) + assertThat(page.content[0].id()).isEqualTo(docA.id()) + } + + @Test + fun `no globalSearchFilter returns all authorized documents`() { + seedDocument("Funenpark") + seedDocument("Keizersgracht") + + val page = documentSearchService.search( + "house", + BlueprintType.CASE, + AdvancedSearchRequest(), + PageRequest.of(0, 10) + ) + + assertThat(page.totalElements).isEqualTo(2L) + } + + @Test + fun `globalSearchFilter supports partial match`() { + seedDocument("Keizersgracht") + + val page = documentSearchService.search( + "house", + BlueprintType.CASE, + AdvancedSearchRequest().globalSearchFilter("Keizers"), + PageRequest.of(0, 10) + ) + + assertThat(page.totalElements).isEqualTo(1L) + } + + private fun seedDocument(street: String): JsonSchemaDocument { + val content = objectMapper.createObjectNode().apply { put("street", street) } + val jpaDoc = runWithoutAuthorization { + documentService.createDocument( + NewDocumentRequest("house", "house", "1.0.0", content) + ).resultingDocument().get() + } + mongoRepository.save( + JsonSchemaDocumentDocument( + id = jpaDoc.id().toString(), + content = content as ObjectNode, + definitionId = objectMapper.readTree( + """{"name":"house","blueprintId":{"blueprintType":"CASE"}}""" + ), + createdOn = null, + modifiedOn = null, + createdBy = null, + sequence = null, + version = null, + assigneeId = null, + assigneeFullName = null, + internalStatus = null, + caseTags = null, + relations = null, + relatedFiles = null, + retentionDate = null, + contentText = street, + ) + ) + return jpaDoc + } +} diff --git a/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/service/JsonSchemaDocumentMongoSearchServiceTest.kt b/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/service/JsonSchemaDocumentMongoSearchServiceTest.kt new file mode 100644 index 0000000000..b00a807715 --- /dev/null +++ b/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/service/JsonSchemaDocumentMongoSearchServiceTest.kt @@ -0,0 +1,178 @@ +/* + * Copyright 2015-2025 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.mongodb.service + +import com.fasterxml.jackson.databind.ObjectMapper +import com.ritense.authorization.Action +import com.ritense.authorization.AuthorizationService +import com.ritense.authorization.permission.ConditionContainer +import com.ritense.authorization.permission.Permission +import com.ritense.authorization.role.Role +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.domain.search.AdvancedSearchRequest +import com.ritense.document.mongodb.authorization.MongoAuthorizationEntityMapper +import com.ritense.document.mongodb.authorization.MongoPermissionConditionTranslator +import com.ritense.document.mongodb.domain.JsonSchemaDocumentDocument +import com.ritense.document.repository.impl.JsonSchemaDocumentRepository +import com.ritense.document.service.JsonSchemaDocumentActionProvider +import com.ritense.document.service.SearchFieldService +import com.ritense.outbox.OutboxService +import com.ritense.valtimo.contract.authentication.UserManagementService +import com.ritense.valtimo.contract.blueprint.BlueprintType +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.argumentCaptor +import org.mockito.kotlin.eq +import org.mockito.kotlin.mock +import org.mockito.kotlin.whenever +import org.springframework.data.domain.PageRequest +import org.springframework.data.mongodb.core.MongoTemplate +import org.springframework.data.mongodb.core.query.Query +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken +import org.springframework.security.core.authority.SimpleGrantedAuthority +import org.springframework.security.core.context.SecurityContextHolder + +class JsonSchemaDocumentMongoSearchServiceTest { + + private val mongoTemplate: MongoTemplate = mock() + private val authorizationService: AuthorizationService = mock() + private val jpaRepository: JsonSchemaDocumentRepository = mock() + private val userManagementService: UserManagementService = mock() + private val searchFieldService: SearchFieldService = mock() + private val outboxService: OutboxService = mock() + private val objectMapper: ObjectMapper = ObjectMapper() + + private lateinit var service: JsonSchemaDocumentMongoSearchService + + @BeforeEach + fun setUp() { + val translator = MongoPermissionConditionTranslator( + mongoMappers = emptyList>(), + authorizationService = authorizationService, + documentRepository = jpaRepository, + ) + service = JsonSchemaDocumentMongoSearchService( + mongoTemplate = mongoTemplate, + translator = translator, + authorizationService = authorizationService, + jpaRepository = jpaRepository, + userManagementService = userManagementService, + searchFieldService = searchFieldService, + outboxService = outboxService, + objectMapper = objectMapper, + ) + + val auth = UsernamePasswordAuthenticationToken( + USERNAME, + null, + listOf(SimpleGrantedAuthority(FULL_ACCESS_ROLE)), + ) + SecurityContextHolder.getContext().authentication = auth + + val role = Role(key = FULL_ACCESS_ROLE) + val viewListPermission = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(JsonSchemaDocumentActionProvider.VIEW_LIST), + conditionContainer = ConditionContainer(emptyList()), + role = role, + ) + whenever( + authorizationService.getPermissions( + eq(JsonSchemaDocument::class.java), + eq(JsonSchemaDocumentActionProvider.VIEW_LIST), + ) + ).thenReturn(listOf(viewListPermission)) + + whenever(mongoTemplate.count(any(), eq(JsonSchemaDocumentDocument::class.java))).thenReturn(0L) + whenever(mongoTemplate.find(any(), eq(JsonSchemaDocumentDocument::class.java))).thenReturn(emptyList()) + whenever(jpaRepository.findAllById(any())).thenReturn(emptyList()) + } + + @AfterEach + fun tearDown() { + SecurityContextHolder.clearContext() + } + + @Test + fun `search with globalSearchFilter adds contentText regex to query`() { + val queryCaptor = argumentCaptor() + whenever(mongoTemplate.count(queryCaptor.capture(), eq(JsonSchemaDocumentDocument::class.java))).thenReturn(0L) + + val request = AdvancedSearchRequest().globalSearchFilter("Amsterdam") + service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) + + val queryJson = queryCaptor.firstValue.queryObject.toJson() + assertThat(queryJson).contains("contentText") + } + + @Test + fun `search without globalSearchFilter does not include contentText criterion`() { + val queryCaptor = argumentCaptor() + whenever(mongoTemplate.count(queryCaptor.capture(), eq(JsonSchemaDocumentDocument::class.java))).thenReturn(0L) + + val request = AdvancedSearchRequest() + service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) + + val queryJson = queryCaptor.firstValue.queryObject.toJson() + assertThat(queryJson).doesNotContain("contentText") + } + + @Test + fun `search with empty globalSearchFilter does not include contentText criterion`() { + val queryCaptor = argumentCaptor() + whenever(mongoTemplate.count(queryCaptor.capture(), eq(JsonSchemaDocumentDocument::class.java))).thenReturn(0L) + + val request = AdvancedSearchRequest().globalSearchFilter("") + service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) + + val queryJson = queryCaptor.firstValue.queryObject.toJson() + assertThat(queryJson).doesNotContain("contentText") + } + + @Test + fun `search with globalSearchFilter uses case-insensitive regex`() { + val queryCaptor = argumentCaptor() + whenever(mongoTemplate.count(queryCaptor.capture(), eq(JsonSchemaDocumentDocument::class.java))).thenReturn(0L) + + val request = AdvancedSearchRequest().globalSearchFilter("john") + service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) + + val queryJson = queryCaptor.firstValue.queryObject.toJson() + assertThat(queryJson).contains("contentText") + assertThat(queryJson).contains("options") + assertThat(queryJson).contains("\"i\"") // case-insensitive flag + } + + @Test + fun `search result uses count from mongodb`() { + whenever(mongoTemplate.count(any(), eq(JsonSchemaDocumentDocument::class.java))).thenReturn(5L) + whenever(mongoTemplate.find(any(), eq(JsonSchemaDocumentDocument::class.java))).thenReturn(emptyList()) + + val request = AdvancedSearchRequest().globalSearchFilter("test") + val page = service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) + + assertThat(page.totalElements).isEqualTo(5L) + } + + companion object { + private const val FULL_ACCESS_ROLE = "full access role" + private const val USERNAME = "test@test.com" + } +} diff --git a/backend/case-mongodb/src/test/resources/config/application-postgresql.yml b/backend/case-mongodb/src/test/resources/config/application-postgresql.yml new file mode 100644 index 0000000000..3543dd40ab --- /dev/null +++ b/backend/case-mongodb/src/test/resources/config/application-postgresql.yml @@ -0,0 +1,17 @@ +spring: + datasource: + driver-class-name: org.postgresql.Driver + url: jdbc:postgresql://localhost:3364/case-mongodb-test + username: valtimo + password: password + hikari: + auto-commit: false + jpa: + database-platform: org.hibernate.dialect.PostgreSQLDialect + database: postgresql + data: + mongodb: + uri: mongodb://localhost:37017/case-mongodb-test + +valtimo: + database: postgres diff --git a/backend/case-mongodb/src/test/resources/config/application.yml b/backend/case-mongodb/src/test/resources/config/application.yml new file mode 100644 index 0000000000..e68db94e75 --- /dev/null +++ b/backend/case-mongodb/src/test/resources/config/application.yml @@ -0,0 +1,35 @@ +spring: + datasource: + type: com.zaxxer.hikari.HikariDataSource + liquibase: + enabled: false + jpa: + show_sql: false + open-in-view: false + properties: + hibernate: + hbm2ddl.auto: none + format_sql: true + jdbc: + time_zone: UTC + connection: + provider_disables_autocommit: true + hibernate: + ddl-auto: none + +spring-actuator: + username: test + password: test + +valtimo: + versioning: + enabled: false + plugin: + encryption-secret: "abcdefghijklmnop" + +operaton: + bpm: + history-level: audit + generic-properties: + properties: + enforceHistoryTimeToLive: false diff --git a/backend/case-mongodb/src/test/resources/config/case/house/1-0-0/case/definition/house.case-definition.json b/backend/case-mongodb/src/test/resources/config/case/house/1-0-0/case/definition/house.case-definition.json new file mode 100644 index 0000000000..994804cbd1 --- /dev/null +++ b/backend/case-mongodb/src/test/resources/config/case/house/1-0-0/case/definition/house.case-definition.json @@ -0,0 +1,7 @@ +{ + "key": "house", + "name": "House", + "versionTag": "1.0.0", + "canHaveAssignee": true, + "autoAssignTasks": true +} \ No newline at end of file diff --git a/backend/case-mongodb/src/test/resources/config/case/house/1-0-0/case/internal-status/house.internal-case-status.json b/backend/case-mongodb/src/test/resources/config/case/house/1-0-0/case/internal-status/house.internal-case-status.json new file mode 100644 index 0000000000..b69712429a --- /dev/null +++ b/backend/case-mongodb/src/test/resources/config/case/house/1-0-0/case/internal-status/house.internal-case-status.json @@ -0,0 +1,20 @@ +[ + { + "key": "suspended", + "title": "Suspended", + "visibleInCaseListByDefault": false, + "color": "GRAY" + }, + { + "key": "closed", + "title": "Closed", + "visibleInCaseListByDefault": false, + "color": "GRAY" + }, + { + "key": "started", + "title": "Started", + "visibleInCaseListByDefault": true, + "color": "GRAY" + } +] diff --git a/backend/case-mongodb/src/test/resources/config/case/house/1-0-0/case/list/house.case-list.json b/backend/case-mongodb/src/test/resources/config/case/house/1-0-0/case/list/house.case-list.json new file mode 100644 index 0000000000..fe51488c70 --- /dev/null +++ b/backend/case-mongodb/src/test/resources/config/case/house/1-0-0/case/list/house.case-list.json @@ -0,0 +1 @@ +[] diff --git a/backend/case-mongodb/src/test/resources/config/case/house/1-0-0/case/search-field/house.case-search-field.json b/backend/case-mongodb/src/test/resources/config/case/house/1-0-0/case/search-field/house.case-search-field.json new file mode 100644 index 0000000000..3f0b73c7b8 --- /dev/null +++ b/backend/case-mongodb/src/test/resources/config/case/house/1-0-0/case/search-field/house.case-search-field.json @@ -0,0 +1,18 @@ +{ + "searchFields": [ + { + "key": "buildDate", + "path": "doc:buildDate", + "dataType": "date", + "fieldType": "single", + "matchType": "exact" + }, + { + "key": "buildDates", + "path": "doc:buildDate", + "dataType": "date", + "fieldType": "range", + "matchType": "exact" + } + ] +} diff --git a/backend/case-mongodb/src/test/resources/config/case/house/1-0-0/document/definition/house.schema.document-definition.json b/backend/case-mongodb/src/test/resources/config/case/house/1-0-0/document/definition/house.schema.document-definition.json new file mode 100644 index 0000000000..965fb0c585 --- /dev/null +++ b/backend/case-mongodb/src/test/resources/config/case/house/1-0-0/document/definition/house.schema.document-definition.json @@ -0,0 +1,33 @@ +{ + "$id": "house.schema", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "House", + "type": "object", + "properties": { + "street": { + "type": "string", + "description": "The street name.", + "maxLength": 100 + }, + "housenumber": { + "description": "house number must be equal to or greater than zero.", + "type": "integer", + "minimum": 0 + }, + "buildDate": { + "type": "string", + "description": "The house's build date.", + "maxLength": 100 + }, + "userInfo": { + "type": "string", + "description": "Additional information on the user", + "maxLength": 100 + }, + "loan-approved": { + "type": "boolean", + "description": "Was the loan for the house approved" + } + }, + "additionalProperties": false +} diff --git a/backend/case/src/main/java/com/ritense/document/domain/search/AdvancedSearchRequest.java b/backend/case/src/main/java/com/ritense/document/domain/search/AdvancedSearchRequest.java index 356115c1d9..3398c8939f 100644 --- a/backend/case/src/main/java/com/ritense/document/domain/search/AdvancedSearchRequest.java +++ b/backend/case/src/main/java/com/ritense/document/domain/search/AdvancedSearchRequest.java @@ -28,6 +28,7 @@ public class AdvancedSearchRequest { private List otherFilters = List.of(); private Set statusFilter = new HashSet<>(); private Set caseTagsFilter = new HashSet<>(); + private String globalSearchFilter; public AdvancedSearchRequest() { // Jackson needs the empty constructor @@ -94,6 +95,19 @@ public void setCaseTagsFilter(Set caseTagsFilter) { this.caseTagsFilter = caseTagsFilter != null ? caseTagsFilter : new HashSet<>(); } + public String getGlobalSearchFilter() { + return globalSearchFilter; + } + + public void setGlobalSearchFilter(String globalSearchFilter) { + this.globalSearchFilter = globalSearchFilter; + } + + public AdvancedSearchRequest globalSearchFilter(String globalSearchFilter) { + setGlobalSearchFilter(globalSearchFilter); + return this; + } + public static class OtherFilter { private String path; diff --git a/backend/case/src/main/java/com/ritense/document/domain/search/SearchRequestMapper.java b/backend/case/src/main/java/com/ritense/document/domain/search/SearchRequestMapper.java index ac833a2780..0068db3892 100644 --- a/backend/case/src/main/java/com/ritense/document/domain/search/SearchRequestMapper.java +++ b/backend/case/src/main/java/com/ritense/document/domain/search/SearchRequestMapper.java @@ -73,6 +73,7 @@ public static AdvancedSearchRequest toAdvancedSearchRequest(SearchWithConfigRequ advancedSearchRequest.setOtherFilters(otherFilters); advancedSearchRequest.setStatusFilter(searchRequest.getStatusFilter()); advancedSearchRequest.setCaseTagsFilter(searchRequest.getCaseTagsFilter()); + advancedSearchRequest.setGlobalSearchFilter(searchRequest.getGlobalSearchFilter()); return advancedSearchRequest; } diff --git a/backend/case/src/main/java/com/ritense/document/domain/search/SearchWithConfigRequest.java b/backend/case/src/main/java/com/ritense/document/domain/search/SearchWithConfigRequest.java index 323e8d33c9..abc3598e72 100644 --- a/backend/case/src/main/java/com/ritense/document/domain/search/SearchWithConfigRequest.java +++ b/backend/case/src/main/java/com/ritense/document/domain/search/SearchWithConfigRequest.java @@ -29,6 +29,7 @@ public class SearchWithConfigRequest { private List otherFilters = List.of(); private Set statusFilter = Set.of(); private Set caseTagsFilter = Set.of(); + private String globalSearchFilter; public SearchWithConfigRequest() { } @@ -85,6 +86,14 @@ public void setCaseTagsFilter(Set caseTagsFilter) { this.caseTagsFilter = caseTagsFilter; } + public String getGlobalSearchFilter() { + return globalSearchFilter; + } + + public void setGlobalSearchFilter(String globalSearchFilter) { + this.globalSearchFilter = globalSearchFilter; + } + public static class SearchWithConfigFilter { private String key; diff --git a/backend/case/src/main/java/com/ritense/document/service/DocumentSearchService.java b/backend/case/src/main/java/com/ritense/document/service/DocumentSearchService.java index e110738ad5..d91ba2509a 100644 --- a/backend/case/src/main/java/com/ritense/document/service/DocumentSearchService.java +++ b/backend/case/src/main/java/com/ritense/document/service/DocumentSearchService.java @@ -55,4 +55,14 @@ Long count( AdvancedSearchRequest advancedSearchRequest ); + @SuppressWarnings({"squid:S1452", "java:S1452"}) + default Page searchForExport( + String documentDefinitionName, + BlueprintType blueprintType, + SearchWithConfigRequest searchWithConfigRequest, + Pageable pageable + ) { + return search(documentDefinitionName, blueprintType, searchWithConfigRequest, pageable); + } + } diff --git a/backend/case/src/main/kotlin/com/ritense/case/configuration/CaseAutoConfiguration.kt b/backend/case/src/main/kotlin/com/ritense/case/configuration/CaseAutoConfiguration.kt index f49df7683a..68952591cd 100644 --- a/backend/case/src/main/kotlin/com/ritense/case/configuration/CaseAutoConfiguration.kt +++ b/backend/case/src/main/kotlin/com/ritense/case/configuration/CaseAutoConfiguration.kt @@ -59,7 +59,6 @@ import com.ritense.case_.service.ActiveCaseDefinitionService import com.ritense.document.service.DocumentDefinitionService import com.ritense.document.service.DocumentSearchService import com.ritense.document.service.DocumentService -import com.ritense.document.service.impl.JsonSchemaDocumentSearchService import com.ritense.exporter.ExportService import com.ritense.importer.ImportService import com.ritense.importer.ValtimoImportService @@ -412,7 +411,7 @@ class CaseAutoConfiguration { @ConditionalOnMissingBean(CaseExporter::class) fun caseExporter( caseDefinitionListColumnRepository: CaseDefinitionListColumnRepository, - documentSearchService: JsonSchemaDocumentSearchService, + documentSearchService: DocumentSearchService, outboxService: OutboxService, mapper: ObjectMapper, caseListRowMapper: CaseListRowMapper diff --git a/backend/case/src/main/kotlin/com/ritense/case/service/CaseExporter.kt b/backend/case/src/main/kotlin/com/ritense/case/service/CaseExporter.kt index b55b922bd8..731e89b5f0 100644 --- a/backend/case/src/main/kotlin/com/ritense/case/service/CaseExporter.kt +++ b/backend/case/src/main/kotlin/com/ritense/case/service/CaseExporter.kt @@ -28,7 +28,7 @@ import com.ritense.valtimo.contract.blueprint.BlueprintType import com.ritense.document.domain.impl.JsonSchemaDocument import com.ritense.document.domain.search.SearchWithConfigRequest import com.ritense.document.event.DocumentsExported -import com.ritense.document.service.impl.JsonSchemaDocumentSearchService +import com.ritense.document.service.DocumentSearchService import com.ritense.outbox.OutboxService import com.ritense.valtimo.contract.utils.SecurityUtils import io.github.oshai.kotlinlogging.KotlinLogging @@ -49,7 +49,7 @@ import kotlin.text.Charsets.UTF_8 @Transactional class CaseExporter( private val caseDefinitionListColumnRepository: CaseDefinitionListColumnRepository, - private val documentSearchService: JsonSchemaDocumentSearchService, + private val documentSearchService: DocumentSearchService, private val outboxService: OutboxService, private val mapper: ObjectMapper, private val caseListRowMapper: CaseListRowMapper diff --git a/backend/case/src/test/kotlin/com/ritense/case/service/CaseExporterTest.kt b/backend/case/src/test/kotlin/com/ritense/case/service/CaseExporterTest.kt index 9e2eb009e5..680d8924ad 100644 --- a/backend/case/src/test/kotlin/com/ritense/case/service/CaseExporterTest.kt +++ b/backend/case/src/test/kotlin/com/ritense/case/service/CaseExporterTest.kt @@ -30,7 +30,7 @@ import com.ritense.document.domain.impl.JsonSchemaDocumentDefinition import com.ritense.document.domain.impl.JsonSchemaDocumentDefinitionId import com.ritense.document.domain.impl.JsonSchemaDocumentId import com.ritense.document.domain.search.SearchWithConfigRequest -import com.ritense.document.service.impl.JsonSchemaDocumentSearchService +import com.ritense.document.service.DocumentSearchService import com.ritense.outbox.OutboxService import com.ritense.search.domain.DisplayType import com.ritense.search.domain.EmptyDisplayTypeParameter @@ -58,7 +58,7 @@ import kotlin.text.Charsets.UTF_8 class CaseExporterTest : BaseTest() { private lateinit var caseDefinitionListColumnRepository: CaseDefinitionListColumnRepository - private lateinit var documentSearchService: JsonSchemaDocumentSearchService + private lateinit var documentSearchService: DocumentSearchService private lateinit var outboxService: OutboxService private lateinit var mapper: ObjectMapper private lateinit var caseListRowMapper: CaseListRowMapper diff --git a/backend/dependencies/valtimo-dependency-versions/build.gradle b/backend/dependencies/valtimo-dependency-versions/build.gradle index 6df6cf78cc..2d30b5219f 100644 --- a/backend/dependencies/valtimo-dependency-versions/build.gradle +++ b/backend/dependencies/valtimo-dependency-versions/build.gradle @@ -28,6 +28,7 @@ dependencies { api(project(":backend:authorization")) api(project(":backend:building-block")) api(project(":backend:case")) + api(project(":backend:case-mongodb")) api(project(":backend:changelog")) api(project(":backend:command-handling")) api(project(":backend:contract")) diff --git a/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.html b/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.html index 0ed9aae8cc..b7a97c53ab 100644 --- a/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.html +++ b/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.html @@ -89,6 +89,14 @@ (selectedCaseTagsChangeEvent)="onSelectedCaseTagsChange($event)" > + + diff --git a/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.ts b/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.ts index e32107c6de..335c3683d2 100644 --- a/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.ts +++ b/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.ts @@ -361,6 +361,7 @@ export class CaseListComponent implements OnInit, OnDestroy { this.hasApiColumnConfig$, this.statusService.caseStatuses$, this.caseListCaseTagService.caseTags$, + this.searchService.globalSearchFilter$.pipe(debounceTime(300), distinctUntilChanged()), ]).pipe(debounceTime(50)) ), distinctUntilChanged( @@ -372,6 +373,10 @@ export class CaseListComponent implements OnInit, OnDestroy { prevSelectedStatuses, prevCaseTagKeys, prevForceRefresh, + , + , + , + prevGlobalSearchFilter, ], [ currSearchRequest, @@ -380,6 +385,10 @@ export class CaseListComponent implements OnInit, OnDestroy { currSelectedStatuses, currCaseTagKeys, currForceRefresh, + , + , + , + currGlobalSearchFilter, ] ) => isEqual( @@ -390,6 +399,7 @@ export class CaseListComponent implements OnInit, OnDestroy { ...prevSelectedStatuses, ...prevCaseTagKeys, forceRefresh: prevForceRefresh, + globalSearchFilter: prevGlobalSearchFilter, }, { ...currSearchRequest, @@ -398,6 +408,7 @@ export class CaseListComponent implements OnInit, OnDestroy { ...currSelectedStatuses, ...currCaseTagKeys, forceRefresh: currForceRefresh, + globalSearchFilter: currGlobalSearchFilter, } ) ), @@ -411,6 +422,8 @@ export class CaseListComponent implements OnInit, OnDestroy { _, hasApiColumnConfig, allStatuses, + , + globalSearchFilter, ]) => { const obsApi: Observable = of(hasApiColumnConfig); const statusKeys: (string | null)[] = @@ -419,6 +432,7 @@ export class CaseListComponent implements OnInit, OnDestroy { : selectedStatuses.map((statusKey: string) => statusKey === CASES_WITHOUT_STATUS_KEY ? null : statusKey ); + const activeGlobalFilter = globalSearchFilter || undefined; if ((Object.keys(searchValues) || []).length > 0) { return forkJoin({ documents: !hasApiColumnConfig @@ -428,7 +442,8 @@ export class CaseListComponent implements OnInit, OnDestroy { assigneeFilter, this.searchService.mapSearchValuesToFilters(searchValues), statusKeys, - selectedCaseTagKeys + selectedCaseTagKeys, + activeGlobalFilter ) : this.documentService.getSpecifiedDocumentsSearch( documentSearchRequest, @@ -436,7 +451,8 @@ export class CaseListComponent implements OnInit, OnDestroy { assigneeFilter, this.searchService.mapSearchValuesToFilters(searchValues), statusKeys, - selectedCaseTagKeys + selectedCaseTagKeys, + activeGlobalFilter ), hasApiColumnConfig: obsApi, isSearchResult: of(true), @@ -452,7 +468,8 @@ export class CaseListComponent implements OnInit, OnDestroy { assigneeFilter, undefined, statusKeys, - selectedCaseTagKeys + selectedCaseTagKeys, + activeGlobalFilter ) : this.documentService.getSpecifiedDocumentsSearch( documentSearchRequest, @@ -460,7 +477,8 @@ export class CaseListComponent implements OnInit, OnDestroy { assigneeFilter, undefined, statusKeys, - selectedCaseTagKeys + selectedCaseTagKeys, + activeGlobalFilter ), hasApiColumnConfig: obsApi, isSearchResult: of(false), @@ -609,6 +627,10 @@ export class CaseListComponent implements OnInit, OnDestroy { this.searchService.search(searchFieldValues); } + public onGlobalSearchFilterChange(value: string): void { + this.searchService.setGlobalSearchFilter(value); + } + public rowClick(item: any): void { this.listService.caseDefinitionKey$.pipe(take(1)).subscribe(caseDefinitionKey => { this.breadcrumbService.cacheQueryParams( diff --git a/frontend/projects/valtimo/case/src/lib/services/case-list-search.service.ts b/frontend/projects/valtimo/case/src/lib/services/case-list-search.service.ts index 89eb02801a..b1baa0a034 100644 --- a/frontend/projects/valtimo/case/src/lib/services/case-list-search.service.ts +++ b/frontend/projects/valtimo/case/src/lib/services/case-list-search.service.ts @@ -15,7 +15,7 @@ */ import {Injectable} from '@angular/core'; -import {Observable, switchMap} from 'rxjs'; +import {BehaviorSubject, Observable, switchMap} from 'rxjs'; import {SearchField, SearchFieldValues, SearchFilter, SearchFilterRange} from '@valtimo/shared'; import {CaseListService} from './case-list.service'; import {DocumentService} from '@valtimo/document'; @@ -34,6 +34,8 @@ export class CaseListSearchService { return this._documentSearchFields$; } + readonly globalSearchFilter$ = new BehaviorSubject(''); + constructor( private readonly caseListService: CaseListService, private readonly documentService: DocumentService, @@ -46,6 +48,10 @@ export class CaseListSearchService { this.caseListService.checkRefresh(); } + public setGlobalSearchFilter(value: string): void { + this.globalSearchFilter$.next(value ?? ''); + } + public mapSearchValuesToFilters( values: SearchFieldValues ): Array { diff --git a/frontend/projects/valtimo/document/src/lib/models/advanced-document-search-request.ts b/frontend/projects/valtimo/document/src/lib/models/advanced-document-search-request.ts index 29bf206a7c..471b1ab297 100644 --- a/frontend/projects/valtimo/document/src/lib/models/advanced-document-search-request.ts +++ b/frontend/projects/valtimo/document/src/lib/models/advanced-document-search-request.ts @@ -37,6 +37,7 @@ export class AdvancedDocumentSearchRequestHttpBody { searchOperator?: SearchOperator; otherFilters?: Array; assigneeFilter?: AssigneeFilter; + globalSearchFilter?: string; } export class AdvancedDocumentSearchRequestImpl implements AdvancedDocumentSearchRequest { @@ -46,6 +47,7 @@ export class AdvancedDocumentSearchRequestImpl implements AdvancedDocumentSearch sort?: SortState; searchOperator?: SearchOperator; otherFilters?: Array; + globalSearchFilter?: string; constructor( definitionName: string, @@ -53,7 +55,8 @@ export class AdvancedDocumentSearchRequestImpl implements AdvancedDocumentSearch size: number, sort?: SortState, searchOperator?: SearchOperator, - otherFilters?: Array + otherFilters?: Array, + globalSearchFilter?: string ) { this.definitionName = definitionName; this.page = page; @@ -61,6 +64,7 @@ export class AdvancedDocumentSearchRequestImpl implements AdvancedDocumentSearch this.sort = sort; this.otherFilters = otherFilters; this.searchOperator = searchOperator; + this.globalSearchFilter = globalSearchFilter; } asHttpBody(): AdvancedDocumentSearchRequestHttpBody { @@ -74,6 +78,9 @@ export class AdvancedDocumentSearchRequestImpl implements AdvancedDocumentSearch if (this.searchOperator) { httpBody.searchOperator = this.searchOperator; } + if (this.globalSearchFilter) { + httpBody.globalSearchFilter = this.globalSearchFilter; + } return httpBody; } diff --git a/frontend/projects/valtimo/document/src/lib/services/document.service.ts b/frontend/projects/valtimo/document/src/lib/services/document.service.ts index 199555a0e7..1037a3777c 100644 --- a/frontend/projects/valtimo/document/src/lib/services/document.service.ts +++ b/frontend/projects/valtimo/document/src/lib/services/document.service.ts @@ -148,7 +148,8 @@ export class DocumentService { assigneeFilter?: AssigneeFilter, otherFilters?: Array, statusFilter?: Array, - caseTagsFilter?: Array + caseTagsFilter?: Array, + globalSearchFilter?: string ): Observable { const body = { ...documentSearchRequest.asHttpBody(), @@ -157,6 +158,7 @@ export class DocumentService { ...(otherFilters && {otherFilters}), ...(statusFilter && {statusFilter}), ...(caseTagsFilter && {caseTagsFilter}), + ...(globalSearchFilter && {globalSearchFilter}), }; return this.http @@ -174,7 +176,8 @@ export class DocumentService { assigneeFilter?: AssigneeFilter, otherFilters?: Array, statusFilter?: Array, - caseTagsFilter?: Array + caseTagsFilter?: Array, + globalSearchFilter?: string ): Observable { const body = { ...documentSearchRequest.asHttpBody(), @@ -183,6 +186,7 @@ export class DocumentService { ...(otherFilters && {otherFilters}), ...(statusFilter && {statusFilter}), ...(caseTagsFilter && {caseTagsFilter}), + ...(globalSearchFilter && {globalSearchFilter}), }; return this.http diff --git a/frontend/projects/valtimo/shared/assets/core/en.json b/frontend/projects/valtimo/shared/assets/core/en.json index dae6fbfff9..bdff442397 100644 --- a/frontend/projects/valtimo/shared/assets/core/en.json +++ b/frontend/projects/valtimo/shared/assets/core/en.json @@ -1585,7 +1585,9 @@ "showingResults": "Showing {{number}} of {{total}} results", "showingResult": "Showing {{number}} result", "automaticallyGenerated": "Automatically generated", - "search": "Search..." + "search": "Search...", + "globalSearchFilter": "Search entire document", + "globalSearchFilterPlaceholder": "Search across all document fields..." }, "webcam": {"takePicture": "Take picture", "save": "Save", "redo": "Redo"}, "customers": { diff --git a/frontend/projects/valtimo/shared/assets/core/nl.json b/frontend/projects/valtimo/shared/assets/core/nl.json index 80f6e5de0d..dfaf1d3bea 100644 --- a/frontend/projects/valtimo/shared/assets/core/nl.json +++ b/frontend/projects/valtimo/shared/assets/core/nl.json @@ -1612,7 +1612,9 @@ "showingResults": "{{number}} van {{total}} resultaten zichtbaar", "showingResult": "{{number}} resultaat zichtbaar", "automaticallyGenerated": "Automatisch gegenereerd", - "search": "Zoeken..." + "search": "Zoeken...", + "globalSearchFilter": "Zoek in geheel document", + "globalSearchFilterPlaceholder": "Zoek in alle documentvelden..." }, "webcam": {"takePicture": "Foto maken", "save": "Opslaan", "redo": "Opnieuw"}, "customers": { diff --git a/settings.gradle b/settings.gradle index 96d3dc3c8c..5e0d9db556 100644 --- a/settings.gradle +++ b/settings.gradle @@ -33,6 +33,7 @@ include( ":backend:authorization", ":backend:building-block", ":backend:case", + ":backend:case-mongodb", ":backend:changelog", ":backend:command-handling", ":backend:contract", From 2e8ef50b40736120e63a5f0d1c06bfa36c460256 Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Fri, 3 Apr 2026 14:22:48 +0200 Subject: [PATCH 05/46] indexes --- .../DocumentMongoAutoConfiguration.kt | 55 +++++++++++++++++++ .../domain/JsonSchemaDocumentDocument.kt | 33 ++++++++++- .../JsonSchemaDocumentMongoSearchService.kt | 39 +++++++------ 3 files changed, 108 insertions(+), 19 deletions(-) diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/autoconfigure/DocumentMongoAutoConfiguration.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/autoconfigure/DocumentMongoAutoConfiguration.kt index b1fddb9d17..f42a142234 100644 --- a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/autoconfigure/DocumentMongoAutoConfiguration.kt +++ b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/autoconfigure/DocumentMongoAutoConfiguration.kt @@ -26,6 +26,7 @@ import com.ritense.document.mongodb.authorization.mapper.JsonSchemaDocumentDefin import com.ritense.document.mongodb.converter.DocumentToJsonNodeReadConverter import com.ritense.document.mongodb.converter.DocumentToObjectNodeReadConverter import com.ritense.document.mongodb.converter.JsonNodeWriteConverter +import com.ritense.document.mongodb.domain.JsonSchemaDocumentDocument import com.ritense.document.mongodb.handler.DocumentMongoEventHandler import com.ritense.document.mongodb.repository.JsonSchemaDocumentMongoRepository import com.ritense.document.mongodb.service.DocumentMongoBackfillService @@ -39,6 +40,7 @@ import com.ritense.document.service.DocumentSearchService import com.ritense.document.service.SearchFieldService import com.ritense.outbox.OutboxService import com.ritense.valtimo.contract.authentication.UserManagementService +import org.springframework.boot.ApplicationRunner import org.springframework.boot.autoconfigure.AutoConfiguration import org.springframework.boot.autoconfigure.AutoConfigureBefore import org.springframework.boot.autoconfigure.condition.ConditionalOnClass @@ -46,8 +48,11 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration import org.springframework.context.annotation.Bean import org.springframework.core.annotation.Order +import org.springframework.data.domain.Sort import org.springframework.data.mongodb.core.MongoTemplate import org.springframework.data.mongodb.core.convert.MongoCustomConversions +import org.springframework.data.mongodb.core.index.Index +import org.springframework.data.mongodb.core.index.TextIndexDefinition import org.springframework.data.mongodb.repository.config.EnableMongoRepositories @AutoConfiguration @@ -159,4 +164,54 @@ class DocumentMongoAutoConfiguration { backfillService: DocumentMongoBackfillService, ): DocumentMongoBackfillResource = DocumentMongoBackfillResource(backfillService) + + /** + * Creates the indexes for the [JsonSchemaDocumentDocument] collection on startup. + * + * This is done programmatically rather than relying solely on [@CompoundIndex] annotations, + * because [spring.data.mongodb.auto-index-creation] is typically disabled in production. + * [MongoTemplate.indexOps] + [ensureIndex] is idempotent: it creates missing indexes and + * is a no-op when the index already exists with the same definition. + */ + @Bean + fun documentMongoIndexInitializer(mongoTemplate: MongoTemplate): ApplicationRunner = ApplicationRunner { + val ops = mongoTemplate.indexOps(JsonSchemaDocumentDocument::class.java) + + ops.ensureIndex( + Index() + .on("definitionId.blueprintId.blueprintType", Sort.Direction.ASC) + .on("definitionId.name", Sort.Direction.ASC) + .on("createdOn", Sort.Direction.DESC) + .named("idx_type_name_created"), + ) + ops.ensureIndex( + Index() + .on("definitionId.blueprintId.blueprintType", Sort.Direction.ASC) + .on("definitionId.name", Sort.Direction.ASC) + .on("internalStatus", Sort.Direction.ASC) + .on("createdOn", Sort.Direction.DESC) + .named("idx_type_name_status_created"), + ) + ops.ensureIndex( + Index() + .on("definitionId.blueprintId.blueprintType", Sort.Direction.ASC) + .on("definitionId.name", Sort.Direction.ASC) + .on("assigneeId", Sort.Direction.ASC) + .on("createdOn", Sort.Direction.DESC) + .named("idx_type_name_assignee_created"), + ) + ops.ensureIndex( + Index() + .on("definitionId.blueprintId.blueprintType", Sort.Direction.ASC) + .on("definitionId.name", Sort.Direction.ASC) + .on("sequence", Sort.Direction.ASC) + .named("idx_type_name_sequence"), + ) + ops.ensureIndex( + TextIndexDefinition.builder() + .onField("contentText") + .named("idx_content_text") + .build(), + ) + } } diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/domain/JsonSchemaDocumentDocument.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/domain/JsonSchemaDocumentDocument.kt index 3d41c60e4c..d5af6159ea 100644 --- a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/domain/JsonSchemaDocumentDocument.kt +++ b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/domain/JsonSchemaDocumentDocument.kt @@ -20,6 +20,9 @@ import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.node.ObjectNode import com.ritense.document.web.rest.dto.CaseTagResponseDto import org.springframework.data.annotation.Id +import org.springframework.data.mongodb.core.index.CompoundIndex +import org.springframework.data.mongodb.core.index.CompoundIndexes +import org.springframework.data.mongodb.core.index.TextIndexed import org.springframework.data.mongodb.core.mapping.Document import java.time.LocalDateTime @@ -32,8 +35,36 @@ import java.time.LocalDateTime * - [caseTags] matches the `caseTags()` getter → [CaseTagResponseDto] list * - [relations] matches the `relations()` getter → stored as [Any] (can be array or object) to avoid JPA entity coupling * - [relatedFiles] matches the `relatedFiles()` getter → stored as [Any] (RelatedFile is an interface) + * + * Indexes follow the ESR rule (Equality → Sort → Range): + * - The two equality fields that appear in every query are blueprintType + definitionName. + * - Variants cover the optional equality filters (status, assigneeId) with createdOn as the sort tail. + * - A separate index covers sequence-based sorting. + * - A MongoDB text index on [contentText] supports full-text / global search. */ @Document(collection = "json_schema_document") +@CompoundIndexes( + // Base index: no optional filters, sort by creation date (most common case-list query) + CompoundIndex( + name = "idx_type_name_created", + def = "{'definitionId.blueprintId.blueprintType': 1, 'definitionId.name': 1, 'createdOn': -1}", + ), + // Status filter + sort by creation date + CompoundIndex( + name = "idx_type_name_status_created", + def = "{'definitionId.blueprintId.blueprintType': 1, 'definitionId.name': 1, 'internalStatus': 1, 'createdOn': -1}", + ), + // Assignee filter + sort by creation date + CompoundIndex( + name = "idx_type_name_assignee_created", + def = "{'definitionId.blueprintId.blueprintType': 1, 'definitionId.name': 1, 'assigneeId': 1, 'createdOn': -1}", + ), + // Sequence-based sort / filter + CompoundIndex( + name = "idx_type_name_sequence", + def = "{'definitionId.blueprintId.blueprintType': 1, 'definitionId.name': 1, 'sequence': 1}", + ), +) data class JsonSchemaDocumentDocument( @Id val id: String, val content: ObjectNode?, @@ -50,5 +81,5 @@ data class JsonSchemaDocumentDocument( val relations: Any?, val relatedFiles: Any?, val retentionDate: LocalDateTime?, - val contentText: String? = null, + @TextIndexed val contentText: String? = null, ) diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/JsonSchemaDocumentMongoSearchService.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/JsonSchemaDocumentMongoSearchService.kt index 55f486fe68..9c7b5c4629 100644 --- a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/JsonSchemaDocumentMongoSearchService.kt +++ b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/JsonSchemaDocumentMongoSearchService.kt @@ -51,6 +51,7 @@ import org.springframework.data.domain.Sort import org.springframework.data.mongodb.core.MongoTemplate import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.TextCriteria import java.util.regex.Pattern class JsonSchemaDocumentMongoSearchService( @@ -90,7 +91,7 @@ class JsonSchemaDocumentMongoSearchService( parts.add(Criteria.where("content.${sc.path}").`is`(sc.value)) } - return executeSearch(Criteria().andOperator(*parts.toTypedArray()), pageable) + return executeSearch(Criteria().andOperator(*parts.toTypedArray()), null, pageable) } override fun search( @@ -162,13 +163,15 @@ class JsonSchemaDocumentMongoSearchService( advancedSearchRequest: AdvancedSearchRequest ): Long { SearchRequestValidator.validate(advancedSearchRequest) - val combined = buildCombinedCriteria( + val (criteria, textCriteria) = buildCombinedCriteria( documentDefinitionName, blueprintType, advancedSearchRequest, JsonSchemaDocumentActionProvider.VIEW_LIST ) - return mongoTemplate.count(Query(combined), JsonSchemaDocumentDocument::class.java) + val query = Query(criteria) + textCriteria?.let { query.addCriteria(it) } + return mongoTemplate.count(query, JsonSchemaDocumentDocument::class.java) } private fun search( @@ -179,8 +182,8 @@ class JsonSchemaDocumentMongoSearchService( action: Action ): Page { SearchRequestValidator.validate(advancedSearchRequest) - val combined = buildCombinedCriteria(documentDefinitionName, blueprintType, advancedSearchRequest, action) - return executeSearch(combined, pageable) + val (criteria, textCriteria) = buildCombinedCriteria(documentDefinitionName, blueprintType, advancedSearchRequest, action) + return executeSearch(criteria, textCriteria, pageable) } private fun buildCombinedCriteria( @@ -188,7 +191,7 @@ class JsonSchemaDocumentMongoSearchService( blueprintType: BlueprintType, searchRequest: AdvancedSearchRequest, action: Action - ): Criteria { + ): Pair { val parts = mutableListOf() parts.add(buildAuthCriteria(action)) @@ -210,15 +213,15 @@ class JsonSchemaDocumentMongoSearchService( parts.add(Criteria.where("caseTags.key").`in`(searchRequest.caseTagsFilter)) } - if (!searchRequest.globalSearchFilter.isNullOrEmpty()) { - parts.add(buildGlobalSearchCriteria(searchRequest.globalSearchFilter)) - } - if (!searchRequest.otherFilters.isNullOrEmpty()) { parts.add(buildOtherFiltersCriteria(searchRequest.otherFilters, searchRequest.searchOperator)) } - return Criteria().andOperator(*parts.toTypedArray()) + val textCriteria = searchRequest.globalSearchFilter + ?.takeIf { it.isNotEmpty() } + ?.let { TextCriteria.forDefaultLanguage().matching(it.trim()) } + + return Criteria().andOperator(*parts.toTypedArray()) to textCriteria } private fun buildAuthCriteria(action: Action): Criteria { @@ -290,9 +293,6 @@ class JsonSchemaDocumentMongoSearchService( } } - private fun buildGlobalSearchCriteria(filter: String): Criteria = - Criteria.where("contentText").regex(".*${Pattern.quote(filter.trim())}.*", "i") - private fun applyEqualCriteria(criteria: Criteria, value: Any?): Criteria { return if (value is String) { criteria.regex("^${Pattern.quote(value.trim())}$", "i") @@ -308,16 +308,19 @@ class JsonSchemaDocumentMongoSearchService( return criteria.regex(".*${Pattern.quote(value.trim())}.*", "i") } - private fun executeSearch(combined: Criteria, pageable: Pageable): Page { + private fun executeSearch(combined: Criteria, textCriteria: TextCriteria?, pageable: Pageable): Page { val translatedSort = translateSort(pageable.sort) val dataQuery = Query(combined).with(translatedSort) + textCriteria?.let { dataQuery.addCriteria(it) } if (pageable.isPaged) { - dataQuery.skip(pageable.offset).limit(pageable.pageSize) + dataQuery.with(pageable) } - val total = mongoTemplate.count(Query(combined), JsonSchemaDocumentDocument::class.java) - val ids = mongoTemplate.find(dataQuery, JsonSchemaDocumentDocument::class.java).map { it.id } + val countQuery = Query(combined) + textCriteria?.let { countQuery.addCriteria(it) } + val total = mongoTemplate.count(countQuery, JsonSchemaDocumentDocument::class.java) + val ids = mongoTemplate.find(dataQuery, JsonSchemaDocumentDocument::class.java).map { it.id } val docIds = ids.map { JsonSchemaDocumentId.existingId(it) } val entities = runWithoutAuthorization { jpaRepository.findAllById(docIds) } val entityMap = entities.associateBy { it.id().toString() } From 2722d36420360f932f5a2b151e80881a1b0d6370 Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Tue, 7 Apr 2026 09:32:37 +0200 Subject: [PATCH 06/46] opensearch poc (wip) --- backend/case-opensearch/build.gradle | 69 +++ .../docker-compose-override-postgresql.yml | 14 + .../case-opensearch/gradle/publishing.gradle | 35 ++ .../OpenSearchAuthorizationEntityMapper.kt | 41 ++ ...OpenSearchPermissionConditionTranslator.kt | 214 +++++++++ ...aDocumentCaseDefinitionOpenSearchMapper.kt | 75 ++++ ...chemaDocumentDefinitionOpenSearchMapper.kt | 69 +++ .../DocumentOpenSearchAutoConfiguration.kt | 155 +++++++ .../domain/JsonSchemaDocumentOsDocument.kt | 59 +++ .../handler/DocumentOpenSearchEventHandler.kt | 70 +++ .../JsonSchemaDocumentOpenSearchRepository.kt | 22 + ...ocumentOpenSearchHttpSecurityConfigurer.kt | 38 ++ .../service/ContentTextExtractor.kt | 39 ++ .../DocumentOpenSearchBackfillService.kt | 72 +++ .../service/DocumentOpenSearchQueryService.kt | 81 ++++ .../service/DocumentOpenSearchSyncService.kt | 50 +++ .../JsonSchemaDocumentOpenSearchService.kt | 411 ++++++++++++++++++ .../web/DocumentOpenSearchBackfillResource.kt | 45 ++ ...ot.autoconfigure.AutoConfiguration.imports | 1 + .../BaseOpenSearchIntegrationTest.kt | 149 +++++++ .../document/opensearch/TestApplication.kt | 22 + .../service/ContentTextExtractorTest.kt | 107 +++++ .../DocumentOpenSearchSyncServiceTest.kt | 151 +++++++ ...nSchemaDocumentOpenSearchServiceIntTest.kt | 144 ++++++ ...JsonSchemaDocumentOpenSearchServiceTest.kt | 178 ++++++++ .../config/application-postgresql.yml | 16 + .../src/test/resources/config/application.yml | 35 ++ .../definition/house.case-definition.json | 7 + .../house.internal-case-status.json | 20 + .../1-0-0/case/list/house.case-list.json | 1 + .../search-field/house.case-search-field.json | 18 + .../house.schema.document-definition.json | 33 ++ .../valtimo-dependency-versions/build.gradle | 1 + gradle.properties | 1 + settings.gradle | 1 + 35 files changed, 2444 insertions(+) create mode 100644 backend/case-opensearch/build.gradle create mode 100644 backend/case-opensearch/docker-compose-override-postgresql.yml create mode 100644 backend/case-opensearch/gradle/publishing.gradle create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/OpenSearchAuthorizationEntityMapper.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/OpenSearchPermissionConditionTranslator.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentCaseDefinitionOpenSearchMapper.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentDefinitionOpenSearchMapper.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/JsonSchemaDocumentOsDocument.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/handler/DocumentOpenSearchEventHandler.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/repository/JsonSchemaDocumentOpenSearchRepository.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/security/DocumentOpenSearchHttpSecurityConfigurer.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/ContentTextExtractor.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchBackfillService.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchQueryService.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchSyncService.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/DocumentOpenSearchBackfillResource.kt create mode 100644 backend/case-opensearch/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports create mode 100644 backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/BaseOpenSearchIntegrationTest.kt create mode 100644 backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/TestApplication.kt create mode 100644 backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/ContentTextExtractorTest.kt create mode 100644 backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchSyncServiceTest.kt create mode 100644 backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceIntTest.kt create mode 100644 backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceTest.kt create mode 100644 backend/case-opensearch/src/test/resources/config/application-postgresql.yml create mode 100644 backend/case-opensearch/src/test/resources/config/application.yml create mode 100644 backend/case-opensearch/src/test/resources/config/case/house/1-0-0/case/definition/house.case-definition.json create mode 100644 backend/case-opensearch/src/test/resources/config/case/house/1-0-0/case/internal-status/house.internal-case-status.json create mode 100644 backend/case-opensearch/src/test/resources/config/case/house/1-0-0/case/list/house.case-list.json create mode 100644 backend/case-opensearch/src/test/resources/config/case/house/1-0-0/case/search-field/house.case-search-field.json create mode 100644 backend/case-opensearch/src/test/resources/config/case/house/1-0-0/document/definition/house.schema.document-definition.json diff --git a/backend/case-opensearch/build.gradle b/backend/case-opensearch/build.gradle new file mode 100644 index 0000000000..533e565330 --- /dev/null +++ b/backend/case-opensearch/build.gradle @@ -0,0 +1,69 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +dockerCompose { + projectName = "case-opensearch" + integrationTestingPostgresql { + isRequiredBy(project.tasks.integrationTestingPostgresql) + useComposeFiles.addAll( + "../docker-resources/docker-compose-base-test-postgresql.yml", + "docker-compose-override-postgresql.yml" + ) + } +} + +dependencies { + implementation project(":backend:authorization") + implementation project(":backend:case") + implementation project(":backend:inbox") + implementation project(":backend:outbox") + + implementation "org.springframework.boot:spring-boot-starter-data-jpa" + implementation "org.springframework.boot:spring-boot-starter-web" + implementation "org.springframework.boot:spring-boot-starter-security" + implementation "org.springframework.boot:spring-boot-autoconfigure" + implementation "com.fasterxml.jackson.module:jackson-module-kotlin" + implementation "io.github.oshai:kotlin-logging:${kotlinLoggingVersion}" + + // OpenSearch via spring-data-opensearch (Apache 2.0 licensed) + implementation "org.opensearch.client:spring-data-opensearch-starter-spring-boot:${springDataOpenSearchVersion}" + + annotationProcessor "org.springframework.boot:spring-boot-autoconfigure-processor" + + testImplementation project(':backend:test-utils-common') + testImplementation project(':backend:core') + testImplementation(project(':backend:audit')) { + exclude(group: "com.ritense.valtimo", module: "case") + } + testImplementation "org.springframework.boot:spring-boot-starter-test" + testImplementation "org.mockito.kotlin:mockito-kotlin:${mockitoKotlinVersion}" + testImplementation "org.postgresql:postgresql" + testImplementation "org.springframework.security:spring-security-test" + + jar { + enabled = true + manifest { + attributes("Implementation-Title": "Ritense Case OpenSearch module") + attributes("Implementation-Version": projectVersion) + } + } +} + +tasks.named("integrationTestingPostgresql") { + systemProperty("liquibase.duplicateFileMode", "WARN") +} + +apply from: "gradle/publishing.gradle" diff --git a/backend/case-opensearch/docker-compose-override-postgresql.yml b/backend/case-opensearch/docker-compose-override-postgresql.yml new file mode 100644 index 0000000000..b8f123040d --- /dev/null +++ b/backend/case-opensearch/docker-compose-override-postgresql.yml @@ -0,0 +1,14 @@ +services: + db: + ports: + - "3365:5432" + environment: + - POSTGRES_DB=case-opensearch-test + opensearch: + image: opensearchproject/opensearch:2.19.2 + environment: + - discovery.type=single-node + - DISABLE_SECURITY_PLUGIN=true + - DISABLE_INSTALL_DEMO_CONFIG=true + ports: + - "39200:9200" diff --git a/backend/case-opensearch/gradle/publishing.gradle b/backend/case-opensearch/gradle/publishing.gradle new file mode 100644 index 0000000000..f991a5d60f --- /dev/null +++ b/backend/case-opensearch/gradle/publishing.gradle @@ -0,0 +1,35 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +pluginManager.withPlugin('maven-publish') { + publishing { + publications { + maven(MavenPublication) { + pom { + name = 'Case OpenSearch module' + description = 'The case-opensearch module syncs json_schema_document to OpenSearch as a CQRS read model' + developers { + developer { + id = "team-valtimo" + name = "Team Valtimo" + email = "team-valtimo@ritense.com" + } + } + } + } + } + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/OpenSearchAuthorizationEntityMapper.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/OpenSearchAuthorizationEntityMapper.kt new file mode 100644 index 0000000000..da50f54c6a --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/OpenSearchAuthorizationEntityMapper.kt @@ -0,0 +1,41 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.authorization + +import com.ritense.authorization.permission.condition.PermissionCondition +import org.opensearch.client.opensearch._types.query_dsl.Query + +/** + * OpenSearch equivalent of [com.ritense.authorization.AuthorizationEntityMapper]. + * + * Translates a [com.ritense.authorization.permission.condition.ContainerPermissionCondition] + * on entity type [TO] into an OpenSearch [Query] that filters [FROM] documents. + * + * Implement this interface and register the implementation as a Spring bean to add support + * for a new container relationship without modifying the core translator. + */ +interface OpenSearchAuthorizationEntityMapper { + + /** + * Given conditions on the [TO] entity type, returns an OpenSearch [Query] that filters + * [FROM] documents satisfying those conditions, or `null` if no filter is needed + * (i.e. any [FROM] document qualifies regardless of [conditions]). + */ + fun mapQuery(conditions: List): Query? + + fun supports(fromClass: Class<*>, toClass: Class<*>): Boolean +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/OpenSearchPermissionConditionTranslator.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/OpenSearchPermissionConditionTranslator.kt new file mode 100644 index 0000000000..1376801bea --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/OpenSearchPermissionConditionTranslator.kt @@ -0,0 +1,214 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.authorization + +import com.ritense.authorization.Action +import com.ritense.authorization.AuthorizationContext.Companion.runWithoutAuthorization +import com.ritense.authorization.AuthorizationService +import com.ritense.authorization.permission.ConditionContainer +import com.ritense.authorization.permission.Permission +import com.ritense.authorization.permission.condition.ContainerPermissionCondition +import com.ritense.authorization.permission.condition.ExpressionPermissionCondition +import com.ritense.authorization.permission.condition.FieldPermissionCondition +import com.ritense.authorization.permission.condition.PermissionCondition +import com.ritense.authorization.permission.condition.PermissionConditionOperator +import com.ritense.authorization.request.EntityAuthorizationRequest +import com.ritense.authorization.role.Role +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.repository.impl.JsonSchemaDocumentRepository +import com.ritense.valtimo.contract.authorization.CurrentUserExpressionHandler +import io.github.oshai.kotlinlogging.KotlinLogging +import org.opensearch.client.opensearch._types.FieldValue +import org.opensearch.client.opensearch._types.query_dsl.Query +import org.opensearch.client.json.JsonData + +class OpenSearchPermissionConditionTranslator( + private val openSearchMappers: List>, + private val authorizationService: AuthorizationService, + private val documentRepository: JsonSchemaDocumentRepository, +) { + + /** + * Translates a list of [Permission]s into a single OpenSearch [Query] that, when applied + * to a search, returns only the documents the current user is allowed to see for [action]. + * + * Permissions are OR-ed; conditions within a permission are AND-ed. + * Returns a deny-all query if no permissions match [action]. + */ + fun toQuery(permissions: List, action: Action<*>): Query { + val matching = permissions.filter { + it.resourceType == JsonSchemaDocument::class.java && it.actions.contains(action) + } + logger.debug { "toQuery: ${permissions.size} permissions total, ${matching.size} matching action=$action" } + if (matching.isEmpty()) { + return denyAll() + } + + val perPermissionQueries = matching.map { permission -> + val conditionQueries = permission.conditionContainer.conditions.map { translateCondition(it) } + andAll(conditionQueries) + } + val result = if (perPermissionQueries.size == 1) { + perPermissionQueries.first() + } else { + Query.of { q -> q.bool { b -> b.should(perPermissionQueries).minimumShouldMatch("1") } } + } + logger.debug { "toQuery: generated query for action=$action" } + return result + } + + private fun translateCondition(condition: PermissionCondition): Query = when (condition) { + is FieldPermissionCondition<*> -> translateField(condition) + is ExpressionPermissionCondition<*> -> translateExpression(condition) + is ContainerPermissionCondition<*> -> translateContainer(condition) + else -> throw IllegalArgumentException("Unknown permission condition type: ${condition::class.qualifiedName}") + } + + private fun translateField(cond: FieldPermissionCondition<*>): Query { + val osField = jpaToOsField(cond.field) + val value = resolveFieldValue(cond) + return Companion.applyOperator(osField, cond.operator, value) + } + + private fun translateExpression(cond: ExpressionPermissionCondition<*>): Query { + val dotPath = cond.path.removePrefix("$.").replace("/", ".") + val osField = "${jpaToOsField(cond.field)}.$dotPath" + val value = CurrentUserExpressionHandler.resolveValue(cond.value) + logger.debug { "translateExpression: field=${cond.field} → osField=$osField, op=${cond.operator}, value=$value (${value?.javaClass?.simpleName})" } + return Companion.applyOperator(osField, cond.operator, value) + } + + @Suppress("UNCHECKED_CAST") + private fun translateContainer(cond: ContainerPermissionCondition<*>): Query { + val osMapper = openSearchMappers.find { + it.supports(JsonSchemaDocument::class.java, cond.resourceType) + } as? OpenSearchAuthorizationEntityMapper + + if (osMapper != null) { + return osMapper.mapQuery(cond.conditions) ?: noFilter() + } + + logger.warn { + "No OpenSearchAuthorizationEntityMapper registered for " + + "JsonSchemaDocument → ${cond.resourceType.simpleName}. " + + "Falling back to JPA ID resolution — may be slow for large datasets." + } + return jpaFallback(cond) + } + + /** + * Fallback for [ContainerPermissionCondition] types that have no registered + * [OpenSearchAuthorizationEntityMapper]. Uses JPA to find matching document IDs and + * returns an `ids` query. + */ + private fun jpaFallback(cond: ContainerPermissionCondition<*>): Query { + val syntheticPermission = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(Action(Action.IGNORE)), + conditionContainer = ConditionContainer(listOf(cond)), + role = Role(key = ""), + ) + val spec = authorizationService.getAuthorizationSpecification( + EntityAuthorizationRequest(JsonSchemaDocument::class.java, Action(Action.IGNORE)), + listOf(syntheticPermission) + ) + val allowedIds: List = runWithoutAuthorization { + documentRepository.findAll(spec).map { doc -> doc.id().toString() } + } + return Query.of { q -> q.ids { i -> i.values(allowedIds) } } + } + + private fun resolveFieldValue(cond: FieldPermissionCondition<*>): Any? = + if (cond.value is List<*>) { + (cond.value as List<*>).map { CurrentUserExpressionHandler.resolveValue(it) } + } else { + CurrentUserExpressionHandler.resolveValue(cond.value) + } + + companion object { + private val logger = KotlinLogging.logger {} + + fun applyOperator(field: String, op: PermissionConditionOperator, value: Any?): Query = + when (op) { + PermissionConditionOperator.EQUAL_TO -> { + if (value == null) { + Query.of { q -> q.bool { b -> b.mustNot(Query.of { q2 -> q2.exists { e -> e.field(field) } }) } } + } else { + Query.of { q -> q.term { t -> t.field(field).value(toFieldValue(value)) } } + } + } + PermissionConditionOperator.NOT_EQUAL_TO -> { + if (value == null) { + Query.of { q -> q.exists { e -> e.field(field) } } + } else { + Query.of { q -> q.bool { b -> b.mustNot(Query.of { q2 -> q2.term { t -> t.field(field).value(toFieldValue(value)) } }) } } + } + } + PermissionConditionOperator.GREATER_THAN -> + Query.of { q -> q.range { r -> r.untyped { u -> u.field(field).gt(JsonData.of(value)) } } } + PermissionConditionOperator.GREATER_THAN_OR_EQUAL_TO -> + Query.of { q -> q.range { r -> r.untyped { u -> u.field(field).gte(JsonData.of(value)) } } } + PermissionConditionOperator.LESS_THAN -> + Query.of { q -> q.range { r -> r.untyped { u -> u.field(field).lt(JsonData.of(value)) } } } + PermissionConditionOperator.LESS_THAN_OR_EQUAL_TO -> + Query.of { q -> q.range { r -> r.untyped { u -> u.field(field).lte(JsonData.of(value)) } } } + PermissionConditionOperator.LIST_CONTAINS -> + Query.of { q -> q.term { t -> t.field(field).value(toFieldValue(value)) } } + PermissionConditionOperator.IN -> { + val collection = value as? Collection<*> + ?: throw IllegalArgumentException("IN operator requires a Collection value") + val fieldValues = collection.map { toFieldValue(it) } + Query.of { q -> q.terms { t -> t.field(field).terms { tv -> tv.value(fieldValues) } } } + } + } + + fun toFieldValue(value: Any?): FieldValue = when (value) { + null -> FieldValue.NULL + is String -> FieldValue.of(value) + is Boolean -> FieldValue.of(value) + is Long -> FieldValue.of(value) + is Int -> FieldValue.of(value.toLong()) + is Double -> FieldValue.of(value) + is Float -> FieldValue.of(value.toDouble()) + else -> FieldValue.of(value.toString()) + } + + /** + * Maps JPA entity field names (as used in [FieldPermissionCondition.field]) to + * their corresponding field names in the OpenSearch document. + */ + val fieldMappings: Map = mapOf( + "createdBy" to "createdBy", + "assigneeId" to "assigneeId", + "assigneeFullName" to "assigneeFullName", + "content" to "content", + "content.content" to "content", + "sequence" to "sequence", + "retentionDate" to "retentionDate", + ) + + fun jpaToOsField(jpaField: String): String = fieldMappings[jpaField] ?: jpaField + + fun denyAll(): Query = Query.of { q -> q.ids { i -> i.values(emptyList()) } } + fun noFilter(): Query = Query.of { q -> q.matchAll { m -> m } } + fun andAll(list: List): Query = when { + list.isEmpty() -> noFilter() + list.size == 1 -> list.first() + else -> Query.of { q -> q.bool { b -> b.must(list) } } + } + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentCaseDefinitionOpenSearchMapper.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentCaseDefinitionOpenSearchMapper.kt new file mode 100644 index 0000000000..d34fd9812a --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentCaseDefinitionOpenSearchMapper.kt @@ -0,0 +1,75 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.authorization.mapper + +import com.ritense.authorization.permission.condition.FieldPermissionCondition +import com.ritense.authorization.permission.condition.PermissionCondition +import com.ritense.case_.domain.definition.CaseDefinition +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.opensearch.authorization.OpenSearchAuthorizationEntityMapper +import com.ritense.document.opensearch.authorization.OpenSearchPermissionConditionTranslator.Companion.andAll +import com.ritense.document.opensearch.authorization.OpenSearchPermissionConditionTranslator.Companion.applyOperator +import com.ritense.valtimo.contract.authorization.CurrentUserExpressionHandler +import org.opensearch.client.opensearch._types.FieldValue +import org.opensearch.client.opensearch._types.query_dsl.Query + +/** + * Handles [com.ritense.authorization.permission.condition.ContainerPermissionCondition] + * where the container resource type is [CaseDefinition]. + * + * Field paths mirror the MongoDB version: `definitionId.blueprintId.*`. + */ +class JsonSchemaDocumentCaseDefinitionOpenSearchMapper : OpenSearchAuthorizationEntityMapper { + + override fun mapQuery(conditions: List): Query? { + if (conditions.isEmpty()) return null + + val conditionQueries = conditions.map { condition -> + when (condition) { + is FieldPermissionCondition<*> -> { + val osField = mapCaseDefinitionField(condition.field) + val value = CurrentUserExpressionHandler.resolveValue(condition.value) + applyOperator(osField, condition.operator, value) + } + else -> throw UnsupportedOperationException( + "Condition type ${condition::class.simpleName} is not supported in " + + "${this::class.simpleName}. Register a custom ${OpenSearchAuthorizationEntityMapper::class.simpleName} " + + "or extend this mapper to handle it." + ) + } + } + + // Constrain to CASE blueprint type to exclude BUILDING_BLOCK documents + val typeCriteria = Query.of { q -> + q.term { t -> t.field("definitionId.blueprintId.blueprintType").value(FieldValue.of("CASE")) } + } + return andAll(conditionQueries + typeCriteria) + } + + override fun supports(fromClass: Class<*>, toClass: Class<*>): Boolean = + fromClass == JsonSchemaDocument::class.java && + toClass == CaseDefinition::class.java + + private fun mapCaseDefinitionField(field: String): String = when (field) { + "id.key" -> "definitionId.blueprintId.blueprintKey" + "id.versionTag" -> "definitionId.blueprintId.blueprintVersionTag" + else -> throw UnsupportedOperationException( + "Field '$field' on CaseDefinition is not yet mapped for OpenSearch. " + + "Add it to ${this::class.simpleName}." + ) + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentDefinitionOpenSearchMapper.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentDefinitionOpenSearchMapper.kt new file mode 100644 index 0000000000..ce09137048 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentDefinitionOpenSearchMapper.kt @@ -0,0 +1,69 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.authorization.mapper + +import com.ritense.authorization.permission.condition.FieldPermissionCondition +import com.ritense.authorization.permission.condition.PermissionCondition +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.domain.impl.JsonSchemaDocumentDefinition +import com.ritense.document.opensearch.authorization.OpenSearchAuthorizationEntityMapper +import com.ritense.document.opensearch.authorization.OpenSearchPermissionConditionTranslator.Companion.andAll +import com.ritense.document.opensearch.authorization.OpenSearchPermissionConditionTranslator.Companion.applyOperator +import com.ritense.valtimo.contract.authorization.CurrentUserExpressionHandler +import org.opensearch.client.opensearch._types.query_dsl.Query + +/** + * Handles [com.ritense.authorization.permission.condition.ContainerPermissionCondition] + * where the container resource type is [JsonSchemaDocumentDefinition]. + * + * Field paths mirror the MongoDB version: `definitionId.name` and `definitionId.version`. + */ +class JsonSchemaDocumentDefinitionOpenSearchMapper : OpenSearchAuthorizationEntityMapper { + + override fun mapQuery(conditions: List): Query? { + if (conditions.isEmpty()) return null + + val queries = conditions.map { condition -> + when (condition) { + is FieldPermissionCondition<*> -> { + val osField = mapDefinitionField(condition.field) + val value = CurrentUserExpressionHandler.resolveValue(condition.value) + applyOperator(osField, condition.operator, value) + } + else -> throw UnsupportedOperationException( + "Condition type ${condition::class.simpleName} is not supported in " + + "${this::class.simpleName}. Register a custom ${OpenSearchAuthorizationEntityMapper::class.simpleName} " + + "or extend this mapper to handle it." + ) + } + } + return andAll(queries) + } + + override fun supports(fromClass: Class<*>, toClass: Class<*>): Boolean = + fromClass == JsonSchemaDocument::class.java && + toClass == JsonSchemaDocumentDefinition::class.java + + private fun mapDefinitionField(field: String): String = when (field) { + "id.name" -> "definitionId.name" + "id.version" -> "definitionId.version" + else -> throw UnsupportedOperationException( + "Field '$field' on JsonSchemaDocumentDefinition is not yet mapped for OpenSearch. " + + "Add it to ${this::class.simpleName}." + ) + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt new file mode 100644 index 0000000000..76d51587bc --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt @@ -0,0 +1,155 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.autoconfigure + +import com.fasterxml.jackson.databind.ObjectMapper +import com.ritense.authorization.AuthorizationService +import com.ritense.document.autoconfigure.DocumentAutoConfiguration +import com.ritense.document.opensearch.authorization.OpenSearchAuthorizationEntityMapper +import com.ritense.document.opensearch.authorization.OpenSearchPermissionConditionTranslator +import com.ritense.document.opensearch.authorization.mapper.JsonSchemaDocumentCaseDefinitionOpenSearchMapper +import com.ritense.document.opensearch.authorization.mapper.JsonSchemaDocumentDefinitionOpenSearchMapper +import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument +import com.ritense.document.opensearch.handler.DocumentOpenSearchEventHandler +import com.ritense.document.opensearch.repository.JsonSchemaDocumentOpenSearchRepository +import com.ritense.document.opensearch.security.DocumentOpenSearchHttpSecurityConfigurer +import com.ritense.document.opensearch.service.DocumentOpenSearchBackfillService +import com.ritense.document.opensearch.service.DocumentOpenSearchQueryService +import com.ritense.document.opensearch.service.DocumentOpenSearchSyncService +import com.ritense.document.opensearch.service.JsonSchemaDocumentOpenSearchService +import com.ritense.document.opensearch.web.DocumentOpenSearchBackfillResource +import com.ritense.document.repository.impl.JsonSchemaDocumentRepository +import com.ritense.document.service.DocumentSearchService +import com.ritense.document.service.SearchFieldService +import com.ritense.outbox.OutboxService +import com.ritense.valtimo.contract.authentication.UserManagementService +import org.springframework.boot.ApplicationRunner +import org.springframework.boot.autoconfigure.AutoConfiguration +import org.springframework.boot.autoconfigure.AutoConfigureBefore +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean +import org.springframework.context.annotation.Bean +import org.springframework.core.annotation.Order +import org.springframework.data.elasticsearch.core.ElasticsearchOperations +import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories + +@AutoConfiguration +@AutoConfigureBefore(DocumentAutoConfiguration::class) +@ConditionalOnClass(ElasticsearchOperations::class) +@EnableElasticsearchRepositories(basePackages = ["com.ritense.document.opensearch.repository"]) +class DocumentOpenSearchAutoConfiguration { + + @Bean + @ConditionalOnMissingBean + fun jsonSchemaDocumentDefinitionOpenSearchMapper(): JsonSchemaDocumentDefinitionOpenSearchMapper = + JsonSchemaDocumentDefinitionOpenSearchMapper() + + @Bean + @ConditionalOnMissingBean + fun jsonSchemaDocumentCaseDefinitionOpenSearchMapper(): JsonSchemaDocumentCaseDefinitionOpenSearchMapper = + JsonSchemaDocumentCaseDefinitionOpenSearchMapper() + + @Bean + @ConditionalOnMissingBean + fun openSearchPermissionConditionTranslator( + openSearchMappers: List>, + authorizationService: AuthorizationService, + documentRepository: JsonSchemaDocumentRepository, + ): OpenSearchPermissionConditionTranslator = + OpenSearchPermissionConditionTranslator(openSearchMappers, authorizationService, documentRepository) + + @Bean + @ConditionalOnMissingBean + fun documentOpenSearchQueryService( + elasticsearchOperations: ElasticsearchOperations, + authorizationService: AuthorizationService, + translator: OpenSearchPermissionConditionTranslator, + ): DocumentOpenSearchQueryService = + DocumentOpenSearchQueryService(elasticsearchOperations, authorizationService, translator) + + @Bean + @ConditionalOnMissingBean + fun documentOpenSearchSyncService( + repository: JsonSchemaDocumentOpenSearchRepository, + objectMapper: ObjectMapper, + ): DocumentOpenSearchSyncService = + DocumentOpenSearchSyncService(repository, objectMapper) + + @Bean + fun documentOpenSearchEventHandler(syncService: DocumentOpenSearchSyncService): DocumentOpenSearchEventHandler = + DocumentOpenSearchEventHandler(syncService) + + @Bean + @ConditionalOnMissingBean + fun documentOpenSearchBackfillService( + jpaRepository: JsonSchemaDocumentRepository, + openSearchRepository: JsonSchemaDocumentOpenSearchRepository, + objectMapper: ObjectMapper, + ): DocumentOpenSearchBackfillService = + DocumentOpenSearchBackfillService(jpaRepository, openSearchRepository, objectMapper) + + @Order(294) + @Bean + @ConditionalOnMissingBean + fun documentOpenSearchHttpSecurityConfigurer(): DocumentOpenSearchHttpSecurityConfigurer = + DocumentOpenSearchHttpSecurityConfigurer() + + @Bean + @ConditionalOnMissingBean(DocumentSearchService::class) + fun documentSearchService( + elasticsearchOperations: ElasticsearchOperations, + translator: OpenSearchPermissionConditionTranslator, + authorizationService: AuthorizationService, + jpaRepository: JsonSchemaDocumentRepository, + userManagementService: UserManagementService, + searchFieldService: SearchFieldService, + outboxService: OutboxService, + objectMapper: ObjectMapper, + ): JsonSchemaDocumentOpenSearchService = + JsonSchemaDocumentOpenSearchService( + elasticsearchOperations, + translator, + authorizationService, + jpaRepository, + userManagementService, + searchFieldService, + outboxService, + objectMapper, + ) + + @Bean + @ConditionalOnMissingBean + fun documentOpenSearchBackfillResource( + backfillService: DocumentOpenSearchBackfillService, + ): DocumentOpenSearchBackfillResource = + DocumentOpenSearchBackfillResource(backfillService) + + /** + * Creates the OpenSearch index and mappings on startup if the index does not yet exist. + * Setting [com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument]'s + * createIndex = false means spring-data-opensearch won't auto-create it, so we do it here. + */ + @Bean + fun documentOpenSearchIndexInitializer(elasticsearchOperations: ElasticsearchOperations): ApplicationRunner = + ApplicationRunner { + val indexOps = elasticsearchOperations.indexOps(JsonSchemaDocumentOsDocument::class.java) + if (!indexOps.exists()) { + indexOps.create() + indexOps.putMapping(indexOps.createMapping(JsonSchemaDocumentOsDocument::class.java)) + } + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/JsonSchemaDocumentOsDocument.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/JsonSchemaDocumentOsDocument.kt new file mode 100644 index 0000000000..745a6e34fb --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/JsonSchemaDocumentOsDocument.kt @@ -0,0 +1,59 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.domain + +import org.springframework.data.annotation.Id +import org.springframework.data.elasticsearch.annotations.Document +import org.springframework.data.elasticsearch.annotations.Field +import org.springframework.data.elasticsearch.annotations.FieldType +import org.springframework.data.elasticsearch.annotations.InnerField +import java.time.LocalDateTime + +/** + * OpenSearch read model for [com.ritense.document.domain.impl.JsonSchemaDocument]. + * + * Uses [Map] types instead of Jackson [com.fasterxml.jackson.databind.JsonNode] / [com.fasterxml.jackson.databind.node.ObjectNode] + * to avoid needing custom converters — Spring Data OpenSearch serializes Map fields + * to nested JSON objects natively. + * + * The [contentText] field holds space-separated leaf values from [content] and is indexed + * as both [FieldType.Text] (for analyzed search) and [FieldType.Keyword] (for wildcard search + * preserving partial-match behaviour equivalent to MongoDB's text index). + */ +@Document(indexName = "json_schema_document", createIndex = false) +data class JsonSchemaDocumentOsDocument( + @Id val id: String, + @Field(type = FieldType.Object) val content: Map?, + @Field(type = FieldType.Object) val definitionId: Map?, + @Field(type = FieldType.Date, format = [], pattern = ["uuuu-MM-dd'T'HH:mm:ss.SSS"]) val createdOn: LocalDateTime?, + @Field(type = FieldType.Date, format = [], pattern = ["uuuu-MM-dd'T'HH:mm:ss.SSS"]) val modifiedOn: LocalDateTime?, + @Field(type = FieldType.Keyword) val createdBy: String?, + @Field(type = FieldType.Long) val sequence: Long?, + @Field(type = FieldType.Integer) val version: Int?, + @Field(type = FieldType.Keyword) val assigneeId: String?, + @Field(type = FieldType.Keyword) val assigneeFullName: String?, + @Field(type = FieldType.Keyword) val internalStatus: String?, + @Field(type = FieldType.Object) val caseTags: List>?, + @Field(type = FieldType.Object) val relations: Any?, + @Field(type = FieldType.Object) val relatedFiles: Any?, + @Field(type = FieldType.Date, format = [], pattern = ["uuuu-MM-dd'T'HH:mm:ss.SSS"]) val retentionDate: LocalDateTime?, + @Field( + type = FieldType.Text, + fields = [InnerField(suffix = "keyword", type = FieldType.Keyword)], + ) + val contentText: String? = null, +) diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/handler/DocumentOpenSearchEventHandler.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/handler/DocumentOpenSearchEventHandler.kt new file mode 100644 index 0000000000..9b7d6b0f8a --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/handler/DocumentOpenSearchEventHandler.kt @@ -0,0 +1,70 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.handler + +import com.ritense.document.event.DocumentAssigned +import com.ritense.document.event.DocumentCreated +import com.ritense.document.event.DocumentUnassigned +import com.ritense.document.event.DocumentUpdated +import com.ritense.document.opensearch.service.DocumentOpenSearchSyncService +import com.ritense.inbox.ValtimoEvent +import com.ritense.inbox.ValtimoEventHandler +import io.github.oshai.kotlinlogging.KotlinLogging + +/** + * Listens to document domain events from the Valtimo inbox and keeps the OpenSearch read + * model in sync. Works with both the outbox-enabled (RabbitMQ) and outbox-disabled + * (local Spring event) modes because both paths converge on [ValtimoEventHandler]. + */ +class DocumentOpenSearchEventHandler( + private val syncService: DocumentOpenSearchSyncService, +) : ValtimoEventHandler { + + override fun handle(event: ValtimoEvent) { + when (event.type) { + in UPSERT_EVENT_TYPES -> syncService.upsert(event) + DELETED_EVENT_TYPE -> { + val id = event.resultId + if (id != null) { + syncService.delete(id) + } else { + logger.warn { "Received DocumentDeleted event with null resultId — skipping delete" } + } + } + else -> { + // Events not related to json_schema_document (e.g. DocumentsListed) are ignored + } + } + } + + companion object { + private val logger = KotlinLogging.logger {} + + val UPSERT_EVENT_TYPES: Set = setOf( + DocumentCreated.TYPE, + DocumentUpdated.TYPE, + DocumentAssigned.TYPE, + DocumentUnassigned.TYPE, + "com.ritense.valtimo.document.status.changed", + "com.ritense.valtimo.document.tags.changed", + "com.ritense.valtimo.document.retentiondate.set", + "com.ritense.valtimo.document.retentiondate.unset", + ) + + const val DELETED_EVENT_TYPE = "com.ritense.valtimo.document.deleted" + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/repository/JsonSchemaDocumentOpenSearchRepository.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/repository/JsonSchemaDocumentOpenSearchRepository.kt new file mode 100644 index 0000000000..2241604807 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/repository/JsonSchemaDocumentOpenSearchRepository.kt @@ -0,0 +1,22 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.repository + +import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument +import org.springframework.data.elasticsearch.repository.ElasticsearchRepository + +interface JsonSchemaDocumentOpenSearchRepository : ElasticsearchRepository diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/security/DocumentOpenSearchHttpSecurityConfigurer.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/security/DocumentOpenSearchHttpSecurityConfigurer.kt new file mode 100644 index 0000000000..085809d825 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/security/DocumentOpenSearchHttpSecurityConfigurer.kt @@ -0,0 +1,38 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.security + +import com.ritense.valtimo.contract.authentication.AuthoritiesConstants.ADMIN +import com.ritense.valtimo.contract.security.config.HttpConfigurerConfigurationException +import com.ritense.valtimo.contract.security.config.HttpSecurityConfigurer +import org.springframework.http.HttpMethod.POST +import org.springframework.security.config.annotation.web.builders.HttpSecurity +import org.springframework.security.web.util.matcher.AntPathRequestMatcher.antMatcher + +class DocumentOpenSearchHttpSecurityConfigurer : HttpSecurityConfigurer { + + override fun configure(http: HttpSecurity) { + try { + http.authorizeHttpRequests { requests -> + requests.requestMatchers(antMatcher(POST, "/api/management/v1/document-opensearch/backfill")) + .hasAuthority(ADMIN) + } + } catch (e: Exception) { + throw HttpConfigurerConfigurationException(e) + } + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/ContentTextExtractor.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/ContentTextExtractor.kt new file mode 100644 index 0000000000..067853d855 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/ContentTextExtractor.kt @@ -0,0 +1,39 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.fasterxml.jackson.databind.JsonNode + +/** + * Extracts all leaf values from a [JsonNode] as a single space-separated string. + * Used to populate [com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument.contentText] + * for full-document search. + */ +fun extractLeafValues(node: JsonNode?): String? { + if (node == null) return null + val parts = mutableListOf() + collectLeaves(node, parts) + return parts.joinToString(" ").ifBlank { null } +} + +private fun collectLeaves(node: JsonNode, out: MutableList) { + when { + node.isObject -> node.fields().forEach { (_, v) -> collectLeaves(v, out) } + node.isArray -> node.forEach { collectLeaves(it, out) } + !node.isNull && !node.isMissingNode -> out.add(node.asText()) + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchBackfillService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchBackfillService.kt new file mode 100644 index 0000000000..1608dcc940 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchBackfillService.kt @@ -0,0 +1,72 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.fasterxml.jackson.databind.ObjectMapper +import com.ritense.authorization.AuthorizationContext.Companion.runWithoutAuthorization +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument +import com.ritense.document.opensearch.repository.JsonSchemaDocumentOpenSearchRepository +import com.ritense.document.repository.impl.JsonSchemaDocumentRepository +import io.github.oshai.kotlinlogging.KotlinLogging +import org.springframework.data.domain.PageRequest +import org.springframework.transaction.annotation.Transactional + +open class DocumentOpenSearchBackfillService( + private val jpaRepository: JsonSchemaDocumentRepository, + private val openSearchRepository: JsonSchemaDocumentOpenSearchRepository, + private val objectMapper: ObjectMapper, +) { + + /** + * Copies all existing [JsonSchemaDocument] rows from the relational database to OpenSearch. + * Processes documents in pages of [pageSize] to avoid loading the entire table into memory. + * + * @return total number of documents migrated + */ + @Transactional(readOnly = true) + open fun backfill(pageSize: Int = DEFAULT_PAGE_SIZE): Long { + var page = 0 + var total = 0L + do { + val slice = runWithoutAuthorization { jpaRepository.findAll(PageRequest.of(page++, pageSize)) } + if (slice.isEmpty) break + + val docs = mutableListOf() + for (jpaDoc in slice.content) { + try { + val tree = objectMapper.valueToTree(jpaDoc) + val doc = objectMapper.treeToValue(tree, JsonSchemaDocumentOsDocument::class.java) + docs.add(doc.copy(contentText = extractLeafValues(tree.get("content")))) + } catch (e: Exception) { + logger.warn(e) { "Failed to convert document to OpenSearch document — skipping" } + } + } + openSearchRepository.saveAll(docs) + total += docs.size + logger.debug { "Backfilled page ${page - 1}: ${docs.size} documents (total so far: $total)" } + } while (slice.hasNext()) + + logger.info { "Backfill complete: $total documents migrated to OpenSearch" } + return total + } + + companion object { + private val logger = KotlinLogging.logger {} + const val DEFAULT_PAGE_SIZE = 500 + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchQueryService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchQueryService.kt new file mode 100644 index 0000000000..6853e574da --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchQueryService.kt @@ -0,0 +1,81 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.ritense.authorization.Action +import com.ritense.authorization.AuthorizationService +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.opensearch.authorization.OpenSearchPermissionConditionTranslator +import com.ritense.document.opensearch.authorization.OpenSearchPermissionConditionTranslator.Companion.andAll +import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument +import com.ritense.document.service.JsonSchemaDocumentActionProvider +import com.ritense.valtimo.contract.utils.SecurityUtils +import org.opensearch.client.opensearch._types.FieldValue +import org.opensearch.client.opensearch._types.query_dsl.Query +import org.springframework.data.domain.Page +import org.springframework.data.domain.PageImpl +import org.springframework.data.domain.Pageable +import org.springframework.data.elasticsearch.core.ElasticsearchOperations +import org.springframework.data.elasticsearch.core.query.NativeQuery + +class DocumentOpenSearchQueryService( + private val elasticsearchOperations: ElasticsearchOperations, + private val authorizationService: AuthorizationService, + private val translator: OpenSearchPermissionConditionTranslator, +) { + + /** + * Returns a page of documents for the given [definitionName], restricted to those + * the current user is allowed to see (VIEW_LIST action). + */ + fun findAllByDefinitionName(definitionName: String, pageable: Pageable): Page { + val authQuery = buildAuthQuery(JsonSchemaDocumentActionProvider.VIEW_LIST) + val definitionFilter = Query.of { q -> + q.term { t -> t.field("definitionId.name").value(FieldValue.of(definitionName)) } + } + val combined = andAll(listOf(authQuery, definitionFilter)) + + val countQuery = NativeQuery.builder().withQuery(combined).build() + val dataQuery = NativeQuery.builder().withQuery(combined).withPageable(pageable).build() + + val total = elasticsearchOperations.count(countQuery, JsonSchemaDocumentOsDocument::class.java) + val hits = elasticsearchOperations.search(dataQuery, JsonSchemaDocumentOsDocument::class.java) + val content = hits.searchHits.mapNotNull { it.content } + return PageImpl(content, pageable, total) + } + + /** + * Returns the document with the given [id] if the current user has VIEW permission, + * or `null` if it does not exist or is not accessible. + */ + fun findById(id: String): JsonSchemaDocumentOsDocument? { + val authQuery = buildAuthQuery(JsonSchemaDocumentActionProvider.VIEW) + val idFilter = Query.of { q -> q.ids { i -> i.values(listOf(id)) } } + val combined = andAll(listOf(authQuery, idFilter)) + + val query = NativeQuery.builder().withQuery(combined).build() + val hits = elasticsearchOperations.search(query, JsonSchemaDocumentOsDocument::class.java) + return hits.searchHits.firstOrNull()?.content + } + + private fun buildAuthQuery(action: Action): Query { + val userRoles = SecurityUtils.getCurrentUserRoles().toSet() + val permissions = authorizationService.getPermissions(JsonSchemaDocument::class.java, action) + .filter { it.role.key in userRoles } + return translator.toQuery(permissions, action) + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchSyncService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchSyncService.kt new file mode 100644 index 0000000000..1bd0c6a00a --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchSyncService.kt @@ -0,0 +1,50 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.fasterxml.jackson.databind.ObjectMapper +import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument +import com.ritense.document.opensearch.repository.JsonSchemaDocumentOpenSearchRepository +import com.ritense.inbox.ValtimoEvent +import io.github.oshai.kotlinlogging.KotlinLogging + +class DocumentOpenSearchSyncService( + private val repository: JsonSchemaDocumentOpenSearchRepository, + private val objectMapper: ObjectMapper, +) { + + fun upsert(event: ValtimoEvent) { + val result = event.result + if (result == null) { + logger.warn { "Received document event ${event.type} for id=${event.resultId} with null result — skipping upsert" } + return + } + val doc = objectMapper.treeToValue(result, JsonSchemaDocumentOsDocument::class.java) + val contentText = extractLeafValues(result.get("content")) + repository.save(doc.copy(contentText = contentText)) + logger.debug { "Upserted document ${doc.id} in OpenSearch (event: ${event.type})" } + } + + fun delete(documentId: String) { + repository.deleteById(documentId) + logger.debug { "Deleted document $documentId from OpenSearch" } + } + + companion object { + private val logger = KotlinLogging.logger {} + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt new file mode 100644 index 0000000000..2b9a49b6cd --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt @@ -0,0 +1,411 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.ArrayNode +import com.ritense.authorization.Action +import com.ritense.authorization.AuthorizationContext.Companion.runWithoutAuthorization +import com.ritense.authorization.AuthorizationService +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.domain.impl.JsonSchemaDocumentId +import com.ritense.document.domain.search.AdvancedSearchRequest +import com.ritense.document.domain.search.AssigneeFilter +import com.ritense.document.domain.search.DatabaseSearchType +import com.ritense.document.domain.search.SearchOperator +import com.ritense.document.domain.search.SearchRequestMapper +import com.ritense.document.domain.search.SearchRequestValidator +import com.ritense.document.domain.search.SearchWithConfigRequest +import com.ritense.document.event.DocumentsListed +import com.ritense.document.opensearch.authorization.OpenSearchPermissionConditionTranslator +import com.ritense.document.opensearch.authorization.OpenSearchPermissionConditionTranslator.Companion.andAll +import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument +import com.ritense.document.repository.impl.JsonSchemaDocumentRepository +import com.ritense.document.service.DocumentSearchService +import com.ritense.document.service.JsonSchemaDocumentActionProvider +import com.ritense.document.service.SearchFieldService +import com.ritense.document.service.impl.SearchRequest +import com.ritense.outbox.OutboxService +import com.ritense.valtimo.contract.authentication.UserManagementService +import com.ritense.valtimo.contract.blueprint.BlueprintType +import com.ritense.valtimo.contract.utils.RequestHelper +import com.ritense.valtimo.contract.utils.SecurityUtils +import org.apache.commons.lang3.NotImplementedException +import org.opensearch.client.opensearch._types.FieldValue +import org.opensearch.client.opensearch._types.query_dsl.Query +import org.opensearch.client.json.JsonData +import org.springframework.data.domain.Page +import org.springframework.data.domain.PageImpl +import org.springframework.data.domain.PageRequest +import org.springframework.data.domain.Pageable +import org.springframework.data.domain.Sort +import org.springframework.data.elasticsearch.core.ElasticsearchOperations +import org.springframework.data.elasticsearch.core.query.NativeQuery +import java.util.regex.Pattern + +class JsonSchemaDocumentOpenSearchService( + private val elasticsearchOperations: ElasticsearchOperations, + private val translator: OpenSearchPermissionConditionTranslator, + private val authorizationService: AuthorizationService, + private val jpaRepository: JsonSchemaDocumentRepository, + private val userManagementService: UserManagementService, + private val searchFieldService: SearchFieldService, + private val outboxService: OutboxService, + private val objectMapper: ObjectMapper, +) : DocumentSearchService { + + override fun search( + searchRequest: SearchRequest, + blueprintType: BlueprintType, + pageable: Pageable + ): Page { + val parts = mutableListOf() + + parts.add(buildAuthQuery(JsonSchemaDocumentActionProvider.VIEW_LIST)) + parts.add(termQuery(BLUEPRINT_TYPE_FIELD, blueprintType.name)) + + if (!searchRequest.documentDefinitionName.isNullOrEmpty()) { + parts.add(termQuery(DEFINITION_NAME_FIELD, searchRequest.documentDefinitionName)) + } + if (!searchRequest.createdBy.isNullOrEmpty()) { + parts.add(termQuery("createdBy", searchRequest.createdBy)) + } + if (searchRequest.sequence != null) { + parts.add(Query.of { q -> q.term { t -> t.field("sequence").value(FieldValue.of(searchRequest.sequence)) } }) + } + if (!searchRequest.globalSearchFilter.isNullOrEmpty()) { + throw NotImplementedException("globalSearchFilter is not supported in the simple search — use the advanced search overload") + } + searchRequest.otherFilters?.forEach { sc -> + parts.add(termQuery("content.${sc.path}", sc.value)) + } + + return executeSearch(andAll(parts), pageable) + } + + override fun search( + documentDefinitionName: String, + blueprintType: BlueprintType, + searchWithConfigRequest: SearchWithConfigRequest, + pageable: Pageable + ): Page { + val zoneOffset = RequestHelper.getZoneOffset() + val searchFieldMap = searchFieldService.getSearchFields(documentDefinitionName) + .associateBy { it.key } + + val otherFilters = searchWithConfigRequest.otherFilters + .map { filter -> SearchRequestMapper.toOtherFilter(filter, searchFieldMap[filter.key], zoneOffset) } + + val advancedSearchRequest = SearchRequestMapper.toAdvancedSearchRequest(searchWithConfigRequest, otherFilters) + + return search( + documentDefinitionName, + blueprintType, + advancedSearchRequest, + pageable, + JsonSchemaDocumentActionProvider.VIEW_LIST + ) + } + + override fun search( + documentDefinitionName: String, + blueprintType: BlueprintType, + advancedSearchRequest: AdvancedSearchRequest, + pageable: Pageable + ): Page { + return search( + documentDefinitionName, + blueprintType, + advancedSearchRequest, + pageable, + JsonSchemaDocumentActionProvider.VIEW_LIST + ) + } + + override fun searchForExport( + documentDefinitionName: String, + blueprintType: BlueprintType, + searchWithConfigRequest: SearchWithConfigRequest, + pageable: Pageable + ): Page { + val zoneOffset = RequestHelper.getZoneOffset() + val searchFieldMap = searchFieldService.getSearchFields(documentDefinitionName) + .associateBy { it.key } + + val otherFilters = searchWithConfigRequest.otherFilters + .map { filter -> SearchRequestMapper.toOtherFilter(filter, searchFieldMap[filter.key], zoneOffset) } + + val advancedSearchRequest = SearchRequestMapper.toAdvancedSearchRequest(searchWithConfigRequest, otherFilters) + + return search( + documentDefinitionName, + blueprintType, + advancedSearchRequest, + pageable, + JsonSchemaDocumentActionProvider.EXPORT + ) + } + + override fun count( + documentDefinitionName: String, + blueprintType: BlueprintType, + advancedSearchRequest: AdvancedSearchRequest + ): Long { + SearchRequestValidator.validate(advancedSearchRequest) + val combinedQuery = buildCombinedQuery( + documentDefinitionName, + blueprintType, + advancedSearchRequest, + JsonSchemaDocumentActionProvider.VIEW_LIST + ) + val countQuery = NativeQuery.builder().withQuery(combinedQuery).build() + return elasticsearchOperations.count(countQuery, JsonSchemaDocumentOsDocument::class.java) + } + + private fun search( + documentDefinitionName: String, + blueprintType: BlueprintType, + advancedSearchRequest: AdvancedSearchRequest, + pageable: Pageable, + action: Action + ): Page { + SearchRequestValidator.validate(advancedSearchRequest) + val combinedQuery = buildCombinedQuery(documentDefinitionName, blueprintType, advancedSearchRequest, action) + return executeSearch(combinedQuery, pageable) + } + + private fun buildCombinedQuery( + documentDefinitionName: String?, + blueprintType: BlueprintType, + searchRequest: AdvancedSearchRequest, + action: Action + ): Query { + val parts = mutableListOf() + + parts.add(buildAuthQuery(action)) + parts.add(termQuery(BLUEPRINT_TYPE_FIELD, blueprintType.name)) + + if (!documentDefinitionName.isNullOrEmpty()) { + parts.add(termQuery(DEFINITION_NAME_FIELD, documentDefinitionName)) + } + + if (searchRequest.assigneeFilter != null && searchRequest.assigneeFilter != AssigneeFilter.ALL) { + parts.add(buildAssigneeFilterQuery(searchRequest.assigneeFilter)) + } + + if (!searchRequest.statusFilter.isNullOrEmpty()) { + parts.add(buildStatusFilterQuery(searchRequest.statusFilter)) + } + + if (!searchRequest.caseTagsFilter.isNullOrEmpty()) { + val tagValues = searchRequest.caseTagsFilter.map { FieldValue.of(it) } + parts.add(Query.of { q -> q.terms { t -> t.field("caseTags.key").terms { tv -> tv.value(tagValues) } } }) + } + + if (!searchRequest.otherFilters.isNullOrEmpty()) { + parts.add(buildOtherFiltersQuery(searchRequest.otherFilters, searchRequest.searchOperator)) + } + + val globalFilter = searchRequest.globalSearchFilter?.takeIf { it.isNotEmpty() } + if (globalFilter != null) { + // Use wildcard on contentText.keyword for partial-match behaviour equivalent to MongoDB text index + parts.add(Query.of { q -> + q.wildcard { w -> w.field("contentText.keyword").value("*${globalFilter.trim()}*").caseInsensitive(true) } + }) + } + + return andAll(parts) + } + + private fun buildAuthQuery(action: Action): Query { + val userRoles = SecurityUtils.getCurrentUserRoles().toSet() + val permissions = authorizationService.getPermissions(JsonSchemaDocument::class.java, action) + .filter { it.role.key in userRoles } + return translator.toQuery(permissions, action) + } + + private fun buildAssigneeFilterQuery(filter: AssigneeFilter): Query { + val userId = userManagementService.currentUser.username + return when (filter) { + AssigneeFilter.MINE -> termQuery("assigneeId", userId) + AssigneeFilter.OPEN -> Query.of { q -> + q.bool { b -> b.mustNot(Query.of { q2 -> q2.exists { e -> e.field("assigneeId") } }) } + } + else -> Query.of { q -> q.matchAll { m -> m } } + } + } + + private fun buildStatusFilterQuery(statusKeys: Set): Query { + val conditions = statusKeys.map { key -> + if (key.isNullOrEmpty()) { + Query.of { q -> q.bool { b -> b.mustNot(Query.of { q2 -> q2.exists { e -> e.field("internalStatus") } }) } } + } else { + termQuery("internalStatus", key) + } + } + return if (conditions.size == 1) conditions.first() + else Query.of { q -> q.bool { b -> b.should(conditions).minimumShouldMatch("1") } } + } + + private fun buildOtherFiltersQuery( + filters: List, + operator: SearchOperator? + ): Query { + val filterQueries = filters.map { buildSingleFilterQuery(it) } + return if (operator == SearchOperator.OR) { + Query.of { q -> q.bool { b -> b.should(filterQueries).minimumShouldMatch("1") } } + } else { + andAll(filterQueries) + } + } + + private fun buildSingleFilterQuery(filter: AdvancedSearchRequest.OtherFilter): Query { + val isDocField = filter.path.startsWith(DOC_PREFIX) + val baseField = when { + isDocField -> "content.${filter.path.removePrefix(DOC_PREFIX)}" + filter.path.startsWith(CASE_PREFIX) -> filter.path.removePrefix(CASE_PREFIX) + else -> throw IllegalArgumentException("Search path doesn't start with known prefix: '${filter.path}'") + } + // For doc: fields, string equality/like/in queries should target the .keyword sub-field + // (OpenSearch auto-maps string content fields as text with .keyword sub-field) + val keywordField = if (isDocField) "$baseField.keyword" else baseField + + return when (filter.searchType) { + DatabaseSearchType.EQUAL -> { + val values = filter.getValues() + when { + values.isEmpty() -> Query.of { q -> q.matchAll { m -> m } } + values.size == 1 -> applyEqualQuery(keywordField, baseField, values[0]) + else -> Query.of { q -> + q.bool { b -> + b.should(values.map { applyEqualQuery(keywordField, baseField, it) }).minimumShouldMatch("1") + } + } + } + } + DatabaseSearchType.LIKE -> { + val values = filter.getValues() + when { + values.isEmpty() -> Query.of { q -> q.matchAll { m -> m } } + values.size == 1 -> applyLikeQuery(keywordField, values[0]) + else -> Query.of { q -> + q.bool { b -> + b.should(values.map { applyLikeQuery(keywordField, it) }).minimumShouldMatch("1") + } + } + } + } + DatabaseSearchType.IN -> { + val fieldValues = filter.getValues().map { OpenSearchPermissionConditionTranslator.toFieldValue(it) } + Query.of { q -> q.terms { t -> t.field(keywordField).terms { tv -> tv.value(fieldValues) } } } + } + DatabaseSearchType.GREATER_THAN_OR_EQUAL_TO -> + Query.of { q -> q.range { r -> r.untyped { u -> u.field(baseField).gte(JsonData.of(filter.rangeFromValue()!!)) } } } + DatabaseSearchType.LESS_THAN_OR_EQUAL_TO -> + Query.of { q -> q.range { r -> r.untyped { u -> u.field(baseField).lte(JsonData.of(filter.rangeToValue()!!)) } } } + DatabaseSearchType.BETWEEN -> + Query.of { q -> + q.range { r -> + r.untyped { u -> + u.field(baseField) + .gte(JsonData.of(filter.rangeFromValue()!!)) + .lte(JsonData.of(filter.rangeToValue()!!)) + } + } + } + else -> throw NotImplementedException("Search type '${filter.searchType}' is not supported in the OpenSearch search service") + } + } + + private fun applyEqualQuery(keywordField: String, baseField: String, value: Any?): Query { + return if (value is String) { + // Case-insensitive exact match using TermQuery with caseInsensitive flag + Query.of { q -> + q.term { t -> t.field(keywordField).value(FieldValue.of(value.trim())).caseInsensitive(true) } + } + } else { + Query.of { q -> q.term { t -> t.field(baseField).value(OpenSearchPermissionConditionTranslator.toFieldValue(value)) } } + } + } + + private fun applyLikeQuery(keywordField: String, value: Any?): Query { + if (value !is String) { + throw IllegalArgumentException("LIKE search requires String values, got: ${value?.javaClass?.simpleName}") + } + return Query.of { q -> + q.wildcard { w -> w.field(keywordField).value("*${value.trim()}*").caseInsensitive(true) } + } + } + + private fun executeSearch(combinedQuery: Query, pageable: Pageable): Page { + val translatedSort = translateSort(pageable.sort) + val effectivePageable = if (pageable.isPaged) { + PageRequest.of(pageable.pageNumber, pageable.pageSize, translatedSort) + } else { + Pageable.unpaged(translatedSort) + } + + val countQuery = NativeQuery.builder().withQuery(combinedQuery).build() + val dataQuery = NativeQuery.builder().withQuery(combinedQuery).withPageable(effectivePageable).build() + + val total = elasticsearchOperations.count(countQuery, JsonSchemaDocumentOsDocument::class.java) + val hits = elasticsearchOperations.search(dataQuery, JsonSchemaDocumentOsDocument::class.java) + val ids = hits.searchHits.map { it.id } + + val docIds = ids.map { JsonSchemaDocumentId.existingId(it) } + val entities = runWithoutAuthorization { jpaRepository.findAllById(docIds) } + val entityMap = entities.associateBy { it.id().toString() } + val orderedEntities = ids.mapNotNull { entityMap[it] } + + outboxService.send { DocumentsListed(objectMapper.valueToTree(orderedEntities)) } + + return PageImpl(orderedEntities, pageable, total) + } + + private fun translateSort(sort: Sort): Sort { + if (sort.isUnsorted) return sort + val orders = sort.map { order -> + val osField = when { + order.property.startsWith(DOC_PREFIX) -> "content.${order.property.removePrefix(DOC_PREFIX)}" + order.property.startsWith(CASE_PREFIX) -> order.property.removePrefix(CASE_PREFIX) + else -> order.property + } + if (order.isAscending) Sort.Order.asc(osField) else Sort.Order.desc(osField) + }.toList() + return Sort.by(orders) + } + + private fun termQuery(field: String, value: String): Query = + Query.of { q -> q.term { t -> t.field(field).value(FieldValue.of(value)) } } + + companion object { + private const val DOC_PREFIX = "doc:" + private const val CASE_PREFIX = "case:" + private const val DEFINITION_NAME_FIELD = "definitionId.name" + private const val BLUEPRINT_TYPE_FIELD = "definitionId.blueprintId.blueprintType" + + /** + * Calls [AdvancedSearchRequest.OtherFilter.getRangeFrom] via reflection to bypass the + * Kotlin type-bounds check. + */ + private fun AdvancedSearchRequest.OtherFilter.rangeFromValue(): Any? = + AdvancedSearchRequest.OtherFilter::class.java.getMethod("getRangeFrom").invoke(this) + + private fun AdvancedSearchRequest.OtherFilter.rangeToValue(): Any? = + AdvancedSearchRequest.OtherFilter::class.java.getMethod("getRangeTo").invoke(this) + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/DocumentOpenSearchBackfillResource.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/DocumentOpenSearchBackfillResource.kt new file mode 100644 index 0000000000..93a35096c1 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/DocumentOpenSearchBackfillResource.kt @@ -0,0 +1,45 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.web + +import com.ritense.document.opensearch.service.DocumentOpenSearchBackfillService +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RequestParam +import org.springframework.web.bind.annotation.RestController + +@RestController +@RequestMapping("/api/management/v1/document-opensearch") +class DocumentOpenSearchBackfillResource( + private val backfillService: DocumentOpenSearchBackfillService, +) { + + /** + * Triggers a full backfill of all [com.ritense.document.domain.impl.JsonSchemaDocument] + * records to the OpenSearch read model. + * + * Only accessible to users with ROLE_ADMIN. + */ + @PostMapping("/backfill") + fun backfill( + @RequestParam(defaultValue = "${DocumentOpenSearchBackfillService.DEFAULT_PAGE_SIZE}") pageSize: Int, + ): ResponseEntity> { + val count = backfillService.backfill(pageSize) + return ResponseEntity.ok(mapOf("migratedCount" to count)) + } +} diff --git a/backend/case-opensearch/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/backend/case-opensearch/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports new file mode 100644 index 0000000000..929aee7b3b --- /dev/null +++ b/backend/case-opensearch/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -0,0 +1 @@ +com.ritense.document.opensearch.autoconfigure.DocumentOpenSearchAutoConfiguration diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/BaseOpenSearchIntegrationTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/BaseOpenSearchIntegrationTest.kt new file mode 100644 index 0000000000..41105b629b --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/BaseOpenSearchIntegrationTest.kt @@ -0,0 +1,149 @@ +/* + * Copyright 2015-2025 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch + +import com.fasterxml.jackson.databind.ObjectMapper +import com.ritense.audit.service.AuditEventProcessor +import com.ritense.authorization.permission.ConditionContainer +import com.ritense.authorization.permission.Permission +import com.ritense.authorization.permission.PermissionRepository +import com.ritense.authorization.role.Role +import com.ritense.authorization.role.RoleRepository +import com.ritense.document.domain.impl.JsonSchemaDocumentDefinition +import com.ritense.document.domain.impl.snapshot.JsonSchemaDocumentSnapshot +import com.ritense.document.domain.impl.searchfield.SearchField +import com.ritense.document.opensearch.repository.JsonSchemaDocumentOpenSearchRepository +import com.ritense.document.service.impl.JsonSchemaDocumentService +import com.ritense.document.service.JsonSchemaDocumentActionProvider +import com.ritense.document.service.JsonSchemaDocumentDefinitionActionProvider +import com.ritense.document.service.JsonSchemaDocumentSnapshotActionProvider +import com.ritense.document.service.SearchFieldActionProvider +import com.ritense.outbox.OutboxService +import com.ritense.testutilscommon.junit.extension.LiquibaseRunnerExtension +import com.ritense.valtimo.contract.authentication.UserManagementService +import com.ritense.valtimo.contract.mail.MailSender +import com.ritense.valtimo.service.ProcessDefinitionCaseDefinitionLinker +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Tag +import org.junit.jupiter.api.extension.ExtendWith +import org.mockito.Answers +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.context.event.SimpleApplicationEventMulticaster +import org.springframework.test.context.bean.override.mockito.MockitoBean +import org.springframework.test.context.bean.override.mockito.MockitoSpyBean +import org.springframework.test.context.junit.jupiter.SpringExtension +import org.springframework.transaction.annotation.Transactional +import java.util.UUID + +@SpringBootTest +@ExtendWith(SpringExtension::class, LiquibaseRunnerExtension::class) +@Tag("integration") +@Transactional +abstract class BaseOpenSearchIntegrationTest { + + @MockitoBean(answers = Answers.RETURNS_DEEP_STUBS) + lateinit var userManagementService: UserManagementService + + @MockitoBean + lateinit var applicationEventMulticaster: SimpleApplicationEventMulticaster + + @MockitoBean + lateinit var processDefinitionCaseDefinitionLinker: ProcessDefinitionCaseDefinitionLinker + + @MockitoBean + lateinit var auditEventProcessor: AuditEventProcessor + + @MockitoBean + lateinit var mailSender: MailSender + + @MockitoSpyBean + lateinit var outboxService: OutboxService + + @Autowired + lateinit var documentService: JsonSchemaDocumentService + + @Autowired + lateinit var openSearchRepository: JsonSchemaDocumentOpenSearchRepository + + @Autowired + lateinit var roleRepository: RoleRepository + + @Autowired + lateinit var permissionRepository: PermissionRepository + + @Autowired + lateinit var objectMapper: ObjectMapper + + @BeforeEach + fun setUpBase() { + setUpPermissions() + openSearchRepository.deleteAll() + } + + @AfterEach + fun tearDownBase() { + openSearchRepository.deleteAll() + } + + private fun setUpPermissions() { + var role = roleRepository.findByKey(FULL_ACCESS_ROLE) + if (role == null) { + role = roleRepository.save(Role(UUID.randomUUID(), FULL_ACCESS_ROLE)) + } + + val permissions = listOf( + Permission(UUID.randomUUID(), com.ritense.document.domain.impl.JsonSchemaDocument::class.java, + mutableListOf(JsonSchemaDocumentActionProvider.VIEW_LIST), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), com.ritense.document.domain.impl.JsonSchemaDocument::class.java, + mutableListOf(JsonSchemaDocumentActionProvider.VIEW), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), com.ritense.document.domain.impl.JsonSchemaDocument::class.java, + mutableListOf(JsonSchemaDocumentActionProvider.MODIFY), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), com.ritense.document.domain.impl.JsonSchemaDocument::class.java, + mutableListOf(JsonSchemaDocumentActionProvider.CREATE), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), com.ritense.document.domain.impl.JsonSchemaDocument::class.java, + mutableListOf(JsonSchemaDocumentActionProvider.CLAIM), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), com.ritense.document.domain.impl.JsonSchemaDocument::class.java, + mutableListOf(JsonSchemaDocumentActionProvider.ASSIGN), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), com.ritense.document.domain.impl.JsonSchemaDocument::class.java, + mutableListOf(JsonSchemaDocumentActionProvider.ASSIGNABLE), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), com.ritense.document.domain.impl.JsonSchemaDocument::class.java, + mutableListOf(JsonSchemaDocumentActionProvider.DELETE), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), SearchField::class.java, + mutableListOf(SearchFieldActionProvider.VIEW_LIST), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), JsonSchemaDocumentDefinition::class.java, + mutableListOf(JsonSchemaDocumentDefinitionActionProvider.VIEW), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), JsonSchemaDocumentDefinition::class.java, + mutableListOf(JsonSchemaDocumentDefinitionActionProvider.VIEW_LIST), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), JsonSchemaDocumentDefinition::class.java, + mutableListOf(JsonSchemaDocumentDefinitionActionProvider.CREATE), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), JsonSchemaDocumentDefinition::class.java, + mutableListOf(JsonSchemaDocumentDefinitionActionProvider.MODIFY), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), JsonSchemaDocumentDefinition::class.java, + mutableListOf(JsonSchemaDocumentDefinitionActionProvider.DELETE), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), JsonSchemaDocumentSnapshot::class.java, + mutableListOf(JsonSchemaDocumentSnapshotActionProvider.VIEW_LIST), ConditionContainer(emptyList()), role!!), + ) + permissionRepository.saveAll(permissions) + } + + companion object { + const val FULL_ACCESS_ROLE: String = "full access role" + const val USERNAME: String = "test@test.com" + } +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/TestApplication.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/TestApplication.kt new file mode 100644 index 0000000000..8ac18acfac --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/TestApplication.kt @@ -0,0 +1,22 @@ +/* + * Copyright 2015-2025 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch + +import org.springframework.boot.autoconfigure.SpringBootApplication + +@SpringBootApplication +class TestApplication diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/ContentTextExtractorTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/ContentTextExtractorTest.kt new file mode 100644 index 0000000000..846ef56415 --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/ContentTextExtractorTest.kt @@ -0,0 +1,107 @@ +/* + * Copyright 2015-2025 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.fasterxml.jackson.databind.ObjectMapper +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test + +class ContentTextExtractorTest { + + private val mapper = ObjectMapper() + + @Test + fun `null input returns null`() { + assertThat(extractLeafValues(null)).isNull() + } + + @Test + fun `empty object returns null`() { + val node = mapper.readTree("{}") + assertThat(extractLeafValues(node)).isNull() + } + + @Test + fun `flat object joins all leaf values`() { + val node = mapper.readTree("""{"firstName":"John","lastName":"Doe"}""") + val result = extractLeafValues(node) + assertThat(result).contains("John") + assertThat(result).contains("Doe") + } + + @Test + fun `nested object extracts leaves recursively`() { + val node = mapper.readTree("""{"person":{"name":"Alice","city":"Utrecht"}}""") + val result = extractLeafValues(node) + assertThat(result).contains("Alice") + assertThat(result).contains("Utrecht") + } + + @Test + fun `array of primitives is extracted`() { + val node = mapper.readTree("""["apple","banana","cherry"]""") + assertThat(extractLeafValues(node)).isEqualTo("apple banana cherry") + } + + @Test + fun `array of objects extracts nested leaves`() { + val node = mapper.readTree("""[{"name":"X"},{"name":"Y"}]""") + val result = extractLeafValues(node) + assertThat(result).contains("X") + assertThat(result).contains("Y") + } + + @Test + fun `null json field values are skipped`() { + val node = mapper.readTree("""{"name":null,"city":null}""") + assertThat(extractLeafValues(node)).isNull() + } + + @Test + fun `numeric value is converted to string`() { + val node = mapper.readTree("""{"count":42}""") + assertThat(extractLeafValues(node)).isEqualTo("42") + } + + @Test + fun `boolean value is converted to string`() { + val node = mapper.readTree("""{"active":true}""") + assertThat(extractLeafValues(node)).isEqualTo("true") + } + + @Test + fun `mixed types in object are all extracted`() { + val node = mapper.readTree("""{"name":"Bob","age":30,"active":false}""") + val result = extractLeafValues(node) + assertThat(result).contains("Bob") + assertThat(result).contains("30") + assertThat(result).contains("false") + } + + @Test + fun `deeply nested structure is fully extracted`() { + val node = mapper.readTree("""{"a":{"b":{"c":"deep"}}}""") + assertThat(extractLeafValues(node)).isEqualTo("deep") + } + + @Test + fun `mixed null and non-null leaves only includes non-null values`() { + val node = mapper.readTree("""{"name":"Alice","missing":null}""") + val result = extractLeafValues(node) + assertThat(result).isEqualTo("Alice") + } +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchSyncServiceTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchSyncServiceTest.kt new file mode 100644 index 0000000000..02fb5b748d --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchSyncServiceTest.kt @@ -0,0 +1,151 @@ +/* + * Copyright 2015-2025 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.fasterxml.jackson.databind.ObjectMapper +import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument +import com.ritense.document.opensearch.repository.JsonSchemaDocumentOpenSearchRepository +import com.ritense.inbox.ValtimoEvent +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.ArgumentCaptor +import org.mockito.kotlin.any +import org.mockito.kotlin.capture +import org.mockito.kotlin.mock +import org.mockito.kotlin.never +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever +import java.time.LocalDateTime + +class DocumentOpenSearchSyncServiceTest { + + private val repository: JsonSchemaDocumentOpenSearchRepository = mock() + private val objectMapper: ObjectMapper = mock() + private lateinit var service: DocumentOpenSearchSyncService + + @BeforeEach + fun setUp() { + service = DocumentOpenSearchSyncService(repository, objectMapper) + } + + @Test + fun `upsert with null result skips repository save`() { + val event = valtimoEvent(result = null) + + service.upsert(event) + + verify(repository, never()).save(any()) + } + + @Test + fun `upsert populates contentText with leaf values from content`() { + val realMapper = ObjectMapper() + val content = realMapper.createObjectNode().apply { + put("firstName", "John") + put("city", "Amsterdam") + } + val resultNode = realMapper.createObjectNode().apply { + set("content", content) + } + val doc = buildDocument(id = "test-id", content = mapOf("firstName" to "John", "city" to "Amsterdam")) + val event = valtimoEvent(result = resultNode) + whenever(objectMapper.treeToValue(any(), any>())).thenReturn(doc) + + val captor = ArgumentCaptor.forClass(JsonSchemaDocumentOsDocument::class.java) + service.upsert(event) + verify(repository).save(capture(captor)) + + val saved = captor.value + assertThat(saved.contentText).contains("John") + assertThat(saved.contentText).contains("Amsterdam") + } + + @Test + fun `upsert with null content stores null contentText`() { + val realMapper = ObjectMapper() + val resultNode = realMapper.createObjectNode() + val doc = buildDocument(id = "no-content-id", content = null) + val event = valtimoEvent(result = resultNode) + whenever(objectMapper.treeToValue(any(), any>())).thenReturn(doc) + + val captor = ArgumentCaptor.forClass(JsonSchemaDocumentOsDocument::class.java) + service.upsert(event) + verify(repository).save(capture(captor)) + + assertThat(captor.value.contentText).isNull() + } + + @Test + fun `upsert with nested content extracts all leaf values`() { + val realMapper = ObjectMapper() + val content = realMapper.createObjectNode().apply { + putObject("address").apply { + put("street", "Main Street") + put("number", "42") + } + } + val resultNode = realMapper.createObjectNode().apply { + set("content", content) + } + val doc = buildDocument(id = "nested-id", content = mapOf("address" to mapOf("street" to "Main Street", "number" to "42"))) + val event = valtimoEvent(result = resultNode) + whenever(objectMapper.treeToValue(any(), any>())).thenReturn(doc) + + val captor = ArgumentCaptor.forClass(JsonSchemaDocumentOsDocument::class.java) + service.upsert(event) + verify(repository).save(capture(captor)) + + val contentText = captor.value.contentText + assertThat(contentText).contains("Main Street") + assertThat(contentText).contains("42") + } + + private fun buildDocument( + id: String, + content: Map?, + ) = JsonSchemaDocumentOsDocument( + id = id, + content = content, + definitionId = null, + createdOn = null, + modifiedOn = null, + createdBy = null, + sequence = null, + version = null, + assigneeId = null, + assigneeFullName = null, + internalStatus = null, + caseTags = null, + relations = null, + relatedFiles = null, + retentionDate = null, + ) + + private fun valtimoEvent( + result: com.fasterxml.jackson.databind.node.ContainerNode<*>?, + ) = ValtimoEvent( + id = "event-id", + type = "DOCUMENT_CREATED", + date = LocalDateTime.now(), + userId = null, + roles = null, + resultType = null, + resultId = "doc-id", + result = result, + ) +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceIntTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceIntTest.kt new file mode 100644 index 0000000000..c7d76d0fbd --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceIntTest.kt @@ -0,0 +1,144 @@ +/* + * Copyright 2015-2025 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.ritense.authorization.AuthorizationContext.Companion.runWithoutAuthorization +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.domain.impl.request.NewDocumentRequest +import com.ritense.document.domain.search.AdvancedSearchRequest +import com.ritense.document.opensearch.BaseOpenSearchIntegrationTest +import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument +import com.ritense.document.service.DocumentSearchService +import com.ritense.valtimo.contract.blueprint.BlueprintType +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.data.domain.PageRequest +import org.springframework.security.test.context.support.WithMockUser + +@WithMockUser(username = BaseOpenSearchIntegrationTest.USERNAME, authorities = [BaseOpenSearchIntegrationTest.FULL_ACCESS_ROLE]) +class JsonSchemaDocumentOpenSearchServiceIntTest : BaseOpenSearchIntegrationTest() { + + @Autowired + lateinit var documentSearchService: DocumentSearchService + + @Test + fun `globalSearchFilter returns matching document`() { + seedDocument("Funenpark") + + val page = documentSearchService.search( + "house", + BlueprintType.CASE, + AdvancedSearchRequest().globalSearchFilter("Funenpark"), + PageRequest.of(0, 10) + ) + + assertThat(page.totalElements).isEqualTo(1L) + } + + @Test + fun `globalSearchFilter is case insensitive`() { + seedDocument("Funenpark") + + val page = documentSearchService.search( + "house", + BlueprintType.CASE, + AdvancedSearchRequest().globalSearchFilter("FUNENPARK"), + PageRequest.of(0, 10) + ) + + assertThat(page.totalElements).isEqualTo(1L) + } + + @Test + fun `globalSearchFilter excludes non-matching documents`() { + val docA = seedDocument("Funenpark") + seedDocument("Keizersgracht") + + val page = documentSearchService.search( + "house", + BlueprintType.CASE, + AdvancedSearchRequest().globalSearchFilter("Funenpark"), + PageRequest.of(0, 10) + ) + + assertThat(page.totalElements).isEqualTo(1L) + assertThat(page.content[0].id()).isEqualTo(docA.id()) + } + + @Test + fun `no globalSearchFilter returns all authorized documents`() { + seedDocument("Funenpark") + seedDocument("Keizersgracht") + + val page = documentSearchService.search( + "house", + BlueprintType.CASE, + AdvancedSearchRequest(), + PageRequest.of(0, 10) + ) + + assertThat(page.totalElements).isEqualTo(2L) + } + + @Test + fun `globalSearchFilter supports partial match`() { + seedDocument("Keizersgracht") + + val page = documentSearchService.search( + "house", + BlueprintType.CASE, + AdvancedSearchRequest().globalSearchFilter("Keizers"), + PageRequest.of(0, 10) + ) + + assertThat(page.totalElements).isEqualTo(1L) + } + + private fun seedDocument(street: String): JsonSchemaDocument { + val content = objectMapper.createObjectNode().apply { put("street", street) } + val jpaDoc = runWithoutAuthorization { + documentService.createDocument( + NewDocumentRequest("house", "house", "1.0.0", content) + ).resultingDocument().get() + } + openSearchRepository.save( + JsonSchemaDocumentOsDocument( + id = jpaDoc.id().toString(), + content = mapOf("street" to street), + definitionId = mapOf( + "name" to "house", + "blueprintId" to mapOf("blueprintType" to "CASE"), + ), + createdOn = null, + modifiedOn = null, + createdBy = null, + sequence = null, + version = null, + assigneeId = null, + assigneeFullName = null, + internalStatus = null, + caseTags = null, + relations = null, + relatedFiles = null, + retentionDate = null, + contentText = street, + ) + ) + return jpaDoc + } +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceTest.kt new file mode 100644 index 0000000000..c94902f5db --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceTest.kt @@ -0,0 +1,178 @@ +/* + * Copyright 2015-2025 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.fasterxml.jackson.databind.ObjectMapper +import com.ritense.authorization.Action +import com.ritense.authorization.AuthorizationService +import com.ritense.authorization.permission.ConditionContainer +import com.ritense.authorization.permission.Permission +import com.ritense.authorization.role.Role +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.domain.search.AdvancedSearchRequest +import com.ritense.document.opensearch.authorization.OpenSearchAuthorizationEntityMapper +import com.ritense.document.opensearch.authorization.OpenSearchPermissionConditionTranslator +import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument +import com.ritense.document.repository.impl.JsonSchemaDocumentRepository +import com.ritense.document.service.JsonSchemaDocumentActionProvider +import com.ritense.document.service.SearchFieldService +import com.ritense.outbox.OutboxService +import com.ritense.valtimo.contract.authentication.UserManagementService +import com.ritense.valtimo.contract.blueprint.BlueprintType +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.argumentCaptor +import org.mockito.kotlin.eq +import org.mockito.kotlin.mock +import org.mockito.kotlin.whenever +import org.springframework.data.domain.PageRequest +import org.springframework.data.elasticsearch.core.ElasticsearchOperations +import org.springframework.data.elasticsearch.core.SearchHits +import org.springframework.data.elasticsearch.core.query.NativeQuery +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken +import org.springframework.security.core.authority.SimpleGrantedAuthority +import org.springframework.security.core.context.SecurityContextHolder + +class JsonSchemaDocumentOpenSearchServiceTest { + + private val elasticsearchOperations: ElasticsearchOperations = mock() + private val authorizationService: AuthorizationService = mock() + private val jpaRepository: JsonSchemaDocumentRepository = mock() + private val userManagementService: UserManagementService = mock() + private val searchFieldService: SearchFieldService = mock() + private val outboxService: OutboxService = mock() + private val objectMapper: ObjectMapper = ObjectMapper() + + private lateinit var service: JsonSchemaDocumentOpenSearchService + + @BeforeEach + fun setUp() { + val translator = OpenSearchPermissionConditionTranslator( + openSearchMappers = emptyList>(), + authorizationService = authorizationService, + documentRepository = jpaRepository, + ) + service = JsonSchemaDocumentOpenSearchService( + elasticsearchOperations = elasticsearchOperations, + translator = translator, + authorizationService = authorizationService, + jpaRepository = jpaRepository, + userManagementService = userManagementService, + searchFieldService = searchFieldService, + outboxService = outboxService, + objectMapper = objectMapper, + ) + + val auth = UsernamePasswordAuthenticationToken( + USERNAME, + null, + listOf(SimpleGrantedAuthority(FULL_ACCESS_ROLE)), + ) + SecurityContextHolder.getContext().authentication = auth + + val role = Role(key = FULL_ACCESS_ROLE) + val viewListPermission = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(JsonSchemaDocumentActionProvider.VIEW_LIST), + conditionContainer = ConditionContainer(emptyList()), + role = role, + ) + whenever( + authorizationService.getPermissions( + eq(JsonSchemaDocument::class.java), + eq(JsonSchemaDocumentActionProvider.VIEW_LIST), + ) + ).thenReturn(listOf(viewListPermission)) + + val emptySearchHits: SearchHits = mock() + whenever(emptySearchHits.searchHits).thenReturn(emptyList()) + whenever(elasticsearchOperations.count(any(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(0L) + whenever(elasticsearchOperations.search(any(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(emptySearchHits) + whenever(jpaRepository.findAllById(any())).thenReturn(emptyList()) + } + + @AfterEach + fun tearDown() { + SecurityContextHolder.clearContext() + } + + @Test + fun `search with globalSearchFilter includes contentText in query`() { + val queryCaptor = argumentCaptor() + val emptySearchHits: SearchHits = mock() + whenever(emptySearchHits.searchHits).thenReturn(emptyList()) + whenever(elasticsearchOperations.count(queryCaptor.capture(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(0L) + whenever(elasticsearchOperations.search(any(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(emptySearchHits) + + val request = AdvancedSearchRequest().globalSearchFilter("Amsterdam") + service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) + + val capturedQuery = queryCaptor.firstValue + assertThat(capturedQuery.query.toString()).contains("contentText") + } + + @Test + fun `search without globalSearchFilter does not include contentText in query`() { + val queryCaptor = argumentCaptor() + val emptySearchHits: SearchHits = mock() + whenever(emptySearchHits.searchHits).thenReturn(emptyList()) + whenever(elasticsearchOperations.count(queryCaptor.capture(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(0L) + whenever(elasticsearchOperations.search(any(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(emptySearchHits) + + val request = AdvancedSearchRequest() + service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) + + val capturedQuery = queryCaptor.firstValue + assertThat(capturedQuery.query.toString()).doesNotContain("contentText") + } + + @Test + fun `search with empty globalSearchFilter does not include contentText in query`() { + val queryCaptor = argumentCaptor() + val emptySearchHits: SearchHits = mock() + whenever(emptySearchHits.searchHits).thenReturn(emptyList()) + whenever(elasticsearchOperations.count(queryCaptor.capture(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(0L) + whenever(elasticsearchOperations.search(any(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(emptySearchHits) + + val request = AdvancedSearchRequest().globalSearchFilter("") + service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) + + val capturedQuery = queryCaptor.firstValue + assertThat(capturedQuery.query.toString()).doesNotContain("contentText") + } + + @Test + fun `search result uses count from opensearch`() { + val emptySearchHits: SearchHits = mock() + whenever(emptySearchHits.searchHits).thenReturn(emptyList()) + whenever(elasticsearchOperations.count(any(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(5L) + whenever(elasticsearchOperations.search(any(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(emptySearchHits) + + val request = AdvancedSearchRequest().globalSearchFilter("test") + val page = service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) + + assertThat(page.totalElements).isEqualTo(5L) + } + + companion object { + private const val FULL_ACCESS_ROLE = "full access role" + private const val USERNAME = "test@test.com" + } +} diff --git a/backend/case-opensearch/src/test/resources/config/application-postgresql.yml b/backend/case-opensearch/src/test/resources/config/application-postgresql.yml new file mode 100644 index 0000000000..11d9550070 --- /dev/null +++ b/backend/case-opensearch/src/test/resources/config/application-postgresql.yml @@ -0,0 +1,16 @@ +spring: + datasource: + driver-class-name: org.postgresql.Driver + url: jdbc:postgresql://localhost:3365/case-opensearch-test + username: valtimo + password: password + hikari: + auto-commit: false + jpa: + database-platform: org.hibernate.dialect.PostgreSQLDialect + database: postgresql + elasticsearch: + uris: http://localhost:39200 + +valtimo: + database: postgres diff --git a/backend/case-opensearch/src/test/resources/config/application.yml b/backend/case-opensearch/src/test/resources/config/application.yml new file mode 100644 index 0000000000..e68db94e75 --- /dev/null +++ b/backend/case-opensearch/src/test/resources/config/application.yml @@ -0,0 +1,35 @@ +spring: + datasource: + type: com.zaxxer.hikari.HikariDataSource + liquibase: + enabled: false + jpa: + show_sql: false + open-in-view: false + properties: + hibernate: + hbm2ddl.auto: none + format_sql: true + jdbc: + time_zone: UTC + connection: + provider_disables_autocommit: true + hibernate: + ddl-auto: none + +spring-actuator: + username: test + password: test + +valtimo: + versioning: + enabled: false + plugin: + encryption-secret: "abcdefghijklmnop" + +operaton: + bpm: + history-level: audit + generic-properties: + properties: + enforceHistoryTimeToLive: false diff --git a/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/case/definition/house.case-definition.json b/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/case/definition/house.case-definition.json new file mode 100644 index 0000000000..994804cbd1 --- /dev/null +++ b/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/case/definition/house.case-definition.json @@ -0,0 +1,7 @@ +{ + "key": "house", + "name": "House", + "versionTag": "1.0.0", + "canHaveAssignee": true, + "autoAssignTasks": true +} \ No newline at end of file diff --git a/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/case/internal-status/house.internal-case-status.json b/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/case/internal-status/house.internal-case-status.json new file mode 100644 index 0000000000..b69712429a --- /dev/null +++ b/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/case/internal-status/house.internal-case-status.json @@ -0,0 +1,20 @@ +[ + { + "key": "suspended", + "title": "Suspended", + "visibleInCaseListByDefault": false, + "color": "GRAY" + }, + { + "key": "closed", + "title": "Closed", + "visibleInCaseListByDefault": false, + "color": "GRAY" + }, + { + "key": "started", + "title": "Started", + "visibleInCaseListByDefault": true, + "color": "GRAY" + } +] diff --git a/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/case/list/house.case-list.json b/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/case/list/house.case-list.json new file mode 100644 index 0000000000..fe51488c70 --- /dev/null +++ b/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/case/list/house.case-list.json @@ -0,0 +1 @@ +[] diff --git a/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/case/search-field/house.case-search-field.json b/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/case/search-field/house.case-search-field.json new file mode 100644 index 0000000000..3f0b73c7b8 --- /dev/null +++ b/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/case/search-field/house.case-search-field.json @@ -0,0 +1,18 @@ +{ + "searchFields": [ + { + "key": "buildDate", + "path": "doc:buildDate", + "dataType": "date", + "fieldType": "single", + "matchType": "exact" + }, + { + "key": "buildDates", + "path": "doc:buildDate", + "dataType": "date", + "fieldType": "range", + "matchType": "exact" + } + ] +} diff --git a/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/document/definition/house.schema.document-definition.json b/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/document/definition/house.schema.document-definition.json new file mode 100644 index 0000000000..965fb0c585 --- /dev/null +++ b/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/document/definition/house.schema.document-definition.json @@ -0,0 +1,33 @@ +{ + "$id": "house.schema", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "House", + "type": "object", + "properties": { + "street": { + "type": "string", + "description": "The street name.", + "maxLength": 100 + }, + "housenumber": { + "description": "house number must be equal to or greater than zero.", + "type": "integer", + "minimum": 0 + }, + "buildDate": { + "type": "string", + "description": "The house's build date.", + "maxLength": 100 + }, + "userInfo": { + "type": "string", + "description": "Additional information on the user", + "maxLength": 100 + }, + "loan-approved": { + "type": "boolean", + "description": "Was the loan for the house approved" + } + }, + "additionalProperties": false +} diff --git a/backend/dependencies/valtimo-dependency-versions/build.gradle b/backend/dependencies/valtimo-dependency-versions/build.gradle index 2d30b5219f..b4787ffea7 100644 --- a/backend/dependencies/valtimo-dependency-versions/build.gradle +++ b/backend/dependencies/valtimo-dependency-versions/build.gradle @@ -29,6 +29,7 @@ dependencies { api(project(":backend:building-block")) api(project(":backend:case")) api(project(":backend:case-mongodb")) + api(project(":backend:case-opensearch")) api(project(":backend:changelog")) api(project(":backend:command-handling")) api(project(":backend:contract")) diff --git a/gradle.properties b/gradle.properties index 8875dec06e..ffd81db146 100644 --- a/gradle.properties +++ b/gradle.properties @@ -35,6 +35,7 @@ operatonVersion=1.0.3 mybatisSpringBootStarterVersion=3.0.4 springBootVersion=3.5.12 +springDataOpenSearchVersion=1.6.1 springBootAdminStarterClientVersion=3.4.5 springDependencyManagementVersion=1.1.7 springCloudStreamVersion=4.2.1 diff --git a/settings.gradle b/settings.gradle index 5e0d9db556..c41bc5d7c5 100644 --- a/settings.gradle +++ b/settings.gradle @@ -34,6 +34,7 @@ include( ":backend:building-block", ":backend:case", ":backend:case-mongodb", + ":backend:case-opensearch", ":backend:changelog", ":backend:command-handling", ":backend:contract", From 0262312b1866c4e0e2f21d91fc02a476db1f89dd Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Thu, 23 Apr 2026 15:09:46 +0200 Subject: [PATCH 07/46] more poc --- backend/app/gzac/build.gradle | 2 +- backend/app/gzac/docker-compose.yaml | 26 ++- .../gzac/src/main/resources/application.yml | 9 +- backend/case-opensearch/build.gradle | 2 +- .../OpenSearchAuthorizationEntityMapper.kt | 8 +- ...OpenSearchPermissionConditionTranslator.kt | 93 +++++---- ...aDocumentCaseDefinitionOpenSearchMapper.kt | 12 +- ...chemaDocumentDefinitionOpenSearchMapper.kt | 4 +- .../DocumentOpenSearchAutoConfiguration.kt | 25 ++- .../domain/JsonSchemaDocumentOsDocument.kt | 41 +++- ...ocumentOpenSearchHttpSecurityConfigurer.kt | 5 +- .../DocumentOpenSearchBackfillService.kt | 150 ++++++++++++--- .../service/DocumentOpenSearchQueryService.kt | 23 +-- .../JsonSchemaDocumentOpenSearchService.kt | 135 ++++++------- .../web/DocumentOpenSearchBackfillResource.kt | 20 +- ...nSchemaDocumentOpenSearchServiceIntTest.kt | 15 +- ...JsonSchemaDocumentOpenSearchServiceTest.kt | 38 ++-- frontend/package-lock.json | 182 ++++++++++-------- 18 files changed, 471 insertions(+), 319 deletions(-) diff --git a/backend/app/gzac/build.gradle b/backend/app/gzac/build.gradle index 4480eb23c2..cd39ff0316 100644 --- a/backend/app/gzac/build.gradle +++ b/backend/app/gzac/build.gradle @@ -16,7 +16,7 @@ dependencies { implementation(platform(project(":backend:dependencies:valtimo-dependency-versions"))) implementation(project(":backend:dependencies:valtimo-gzac-dependencies")) - implementation(project(":backend:case-mongodb")) + implementation(project(":backend:case-opensearch")) implementation(project(":backend:mail:local-mail")) implementation(project(":backend:document-generation:smartdocuments")) implementation(project(":backend:zgw:portaaltaak")) diff --git a/backend/app/gzac/docker-compose.yaml b/backend/app/gzac/docker-compose.yaml index d10664a7e6..5c4a47e8c5 100644 --- a/backend/app/gzac/docker-compose.yaml +++ b/backend/app/gzac/docker-compose.yaml @@ -80,17 +80,25 @@ services: volumes: - gzac-database-data-mysql:/var/lib/mysql # persist data even if container shuts down - gzac-mongodb: - container_name: gzac-docker-compose-gzac-mongodb - image: mongo:8.2.6 + gzac-opensearch: + container_name: gzac-docker-compose-gzac-opensearch + image: opensearchproject/opensearch:2.19.2 ports: - - "27017:27017" + - "9200:9200" + - "9600:9600" environment: - MONGO_INITDB_ROOT_USERNAME: gzac - MONGO_INITDB_ROOT_PASSWORD: password - MONGO_INITDB_DATABASE: gzac + - discovery.type=single-node + - DISABLE_SECURITY_PLUGIN=true + - DISABLE_INSTALL_DEMO_CONFIG=true + - OPENSEARCH_JAVA_OPTS=-Xms256m -Xmx256m volumes: - - gzac-mongodb-data:/data/db + - gzac-opensearch-data:/usr/share/opensearch/data + healthcheck: + test: [ "CMD-SHELL", "curl -sf http://localhost:9200/_cluster/health || exit 1" ] + interval: 10s + timeout: 5s + retries: 12 + start_period: 30s gzac-rabbitmq: image: rabbitmq:4.1.0-management @@ -732,4 +740,4 @@ services: volumes: gzac-database-data: gzac-database-data-mysql: - gzac-mongodb-data: + gzac-opensearch-data: diff --git a/backend/app/gzac/src/main/resources/application.yml b/backend/app/gzac/src/main/resources/application.yml index bf632b3b09..0a26e81795 100644 --- a/backend/app/gzac/src/main/resources/application.yml +++ b/backend/app/gzac/src/main/resources/application.yml @@ -2,7 +2,7 @@ logging: file: name: /tmp/spring.log level: - com.ritense.document.mongodb.authorization: DEBUG + com.ritense.document.opensearch: DEBUG management: endpoints: @@ -31,9 +31,8 @@ spring: enabled: false livereload: enabled: false - data: - mongodb: - uri: ${SPRING_DATA_MONGODB_URI:mongodb://gzac:password@localhost:27017/gzac?authSource=admin} + elasticsearch: + uris: ${SPRING_ELASTICSEARCH_URIS:http://localhost:9200} datasource: type: com.zaxxer.hikari.HikariDataSource driver-class-name: ${SPRING_DATASOURCE_DRIVER_CLASS_NAME:org.postgresql.Driver} @@ -121,6 +120,8 @@ spring: autoconfigure: exclude: - org.springframework.boot.actuate.autoconfigure.metrics.web.tomcat.TomcatMetricsAutoConfiguration + - org.springframework.boot.autoconfigure.data.elasticsearch.ElasticsearchDataAutoConfiguration + - org.springframework.boot.autoconfigure.elasticsearch.ElasticsearchClientAutoConfiguration cloud: function: definition: kotlinInboxCloudEventConsumer diff --git a/backend/case-opensearch/build.gradle b/backend/case-opensearch/build.gradle index 533e565330..73c44edac8 100644 --- a/backend/case-opensearch/build.gradle +++ b/backend/case-opensearch/build.gradle @@ -39,7 +39,7 @@ dependencies { implementation "io.github.oshai:kotlin-logging:${kotlinLoggingVersion}" // OpenSearch via spring-data-opensearch (Apache 2.0 licensed) - implementation "org.opensearch.client:spring-data-opensearch-starter-spring-boot:${springDataOpenSearchVersion}" + implementation "org.opensearch.client:spring-data-opensearch-starter:${springDataOpenSearchVersion}" annotationProcessor "org.springframework.boot:spring-boot-autoconfigure-processor" diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/OpenSearchAuthorizationEntityMapper.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/OpenSearchAuthorizationEntityMapper.kt index da50f54c6a..e6cffef10d 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/OpenSearchAuthorizationEntityMapper.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/OpenSearchAuthorizationEntityMapper.kt @@ -17,13 +17,13 @@ package com.ritense.document.opensearch.authorization import com.ritense.authorization.permission.condition.PermissionCondition -import org.opensearch.client.opensearch._types.query_dsl.Query +import org.opensearch.index.query.QueryBuilder /** * OpenSearch equivalent of [com.ritense.authorization.AuthorizationEntityMapper]. * * Translates a [com.ritense.authorization.permission.condition.ContainerPermissionCondition] - * on entity type [TO] into an OpenSearch [Query] that filters [FROM] documents. + * on entity type [TO] into an OpenSearch [QueryBuilder] that filters [FROM] documents. * * Implement this interface and register the implementation as a Spring bean to add support * for a new container relationship without modifying the core translator. @@ -31,11 +31,11 @@ import org.opensearch.client.opensearch._types.query_dsl.Query interface OpenSearchAuthorizationEntityMapper { /** - * Given conditions on the [TO] entity type, returns an OpenSearch [Query] that filters + * Given conditions on the [TO] entity type, returns an OpenSearch [QueryBuilder] that filters * [FROM] documents satisfying those conditions, or `null` if no filter is needed * (i.e. any [FROM] document qualifies regardless of [conditions]). */ - fun mapQuery(conditions: List): Query? + fun mapQuery(conditions: List): QueryBuilder? fun supports(fromClass: Class<*>, toClass: Class<*>): Boolean } diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/OpenSearchPermissionConditionTranslator.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/OpenSearchPermissionConditionTranslator.kt index 1376801bea..1c9ed6bfcc 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/OpenSearchPermissionConditionTranslator.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/OpenSearchPermissionConditionTranslator.kt @@ -32,9 +32,9 @@ import com.ritense.document.domain.impl.JsonSchemaDocument import com.ritense.document.repository.impl.JsonSchemaDocumentRepository import com.ritense.valtimo.contract.authorization.CurrentUserExpressionHandler import io.github.oshai.kotlinlogging.KotlinLogging -import org.opensearch.client.opensearch._types.FieldValue -import org.opensearch.client.opensearch._types.query_dsl.Query -import org.opensearch.client.json.JsonData +import org.opensearch.index.query.BoolQueryBuilder +import org.opensearch.index.query.QueryBuilder +import org.opensearch.index.query.QueryBuilders class OpenSearchPermissionConditionTranslator( private val openSearchMappers: List>, @@ -43,13 +43,13 @@ class OpenSearchPermissionConditionTranslator( ) { /** - * Translates a list of [Permission]s into a single OpenSearch [Query] that, when applied + * Translates a list of [Permission]s into a single OpenSearch [QueryBuilder] that, when applied * to a search, returns only the documents the current user is allowed to see for [action]. * * Permissions are OR-ed; conditions within a permission are AND-ed. * Returns a deny-all query if no permissions match [action]. */ - fun toQuery(permissions: List, action: Action<*>): Query { + fun toQuery(permissions: List, action: Action<*>): QueryBuilder { val matching = permissions.filter { it.resourceType == JsonSchemaDocument::class.java && it.actions.contains(action) } @@ -65,35 +65,41 @@ class OpenSearchPermissionConditionTranslator( val result = if (perPermissionQueries.size == 1) { perPermissionQueries.first() } else { - Query.of { q -> q.bool { b -> b.should(perPermissionQueries).minimumShouldMatch("1") } } + QueryBuilders.boolQuery().apply { + perPermissionQueries.forEach { should(it) } + minimumShouldMatch(1) + } } logger.debug { "toQuery: generated query for action=$action" } return result } - private fun translateCondition(condition: PermissionCondition): Query = when (condition) { + private fun translateCondition(condition: PermissionCondition): QueryBuilder = when (condition) { is FieldPermissionCondition<*> -> translateField(condition) is ExpressionPermissionCondition<*> -> translateExpression(condition) is ContainerPermissionCondition<*> -> translateContainer(condition) else -> throw IllegalArgumentException("Unknown permission condition type: ${condition::class.qualifiedName}") } - private fun translateField(cond: FieldPermissionCondition<*>): Query { - val osField = jpaToOsField(cond.field) + private fun translateField(cond: FieldPermissionCondition<*>): QueryBuilder { + val baseField = jpaToOsField(cond.field) val value = resolveFieldValue(cond) + val osField = if (isDynamicTextField(baseField, cond.operator, value)) "$baseField.keyword" else baseField return Companion.applyOperator(osField, cond.operator, value) } - private fun translateExpression(cond: ExpressionPermissionCondition<*>): Query { + private fun translateExpression(cond: ExpressionPermissionCondition<*>): QueryBuilder { val dotPath = cond.path.removePrefix("$.").replace("/", ".") - val osField = "${jpaToOsField(cond.field)}.$dotPath" + val baseField = "${jpaToOsField(cond.field)}.$dotPath" val value = CurrentUserExpressionHandler.resolveValue(cond.value) + // Content sub-fields are dynamically mapped as text — use .keyword for string term queries + val osField = if (isDynamicTextField(baseField, cond.operator, value)) "$baseField.keyword" else baseField logger.debug { "translateExpression: field=${cond.field} → osField=$osField, op=${cond.operator}, value=$value (${value?.javaClass?.simpleName})" } return Companion.applyOperator(osField, cond.operator, value) } @Suppress("UNCHECKED_CAST") - private fun translateContainer(cond: ContainerPermissionCondition<*>): Query { + private fun translateContainer(cond: ContainerPermissionCondition<*>): QueryBuilder { val osMapper = openSearchMappers.find { it.supports(JsonSchemaDocument::class.java, cond.resourceType) } as? OpenSearchAuthorizationEntityMapper @@ -115,7 +121,7 @@ class OpenSearchPermissionConditionTranslator( * [OpenSearchAuthorizationEntityMapper]. Uses JPA to find matching document IDs and * returns an `ids` query. */ - private fun jpaFallback(cond: ContainerPermissionCondition<*>): Query { + private fun jpaFallback(cond: ContainerPermissionCondition<*>): QueryBuilder { val syntheticPermission = Permission( resourceType = JsonSchemaDocument::class.java, actions = mutableListOf(Action(Action.IGNORE)), @@ -129,7 +135,7 @@ class OpenSearchPermissionConditionTranslator( val allowedIds: List = runWithoutAuthorization { documentRepository.findAll(spec).map { doc -> doc.id().toString() } } - return Query.of { q -> q.ids { i -> i.values(allowedIds) } } + return QueryBuilders.idsQuery().addIds(*allowedIds.toTypedArray()) } private fun resolveFieldValue(cond: FieldPermissionCondition<*>): Any? = @@ -142,51 +148,39 @@ class OpenSearchPermissionConditionTranslator( companion object { private val logger = KotlinLogging.logger {} - fun applyOperator(field: String, op: PermissionConditionOperator, value: Any?): Query = + fun applyOperator(field: String, op: PermissionConditionOperator, value: Any?): QueryBuilder = when (op) { PermissionConditionOperator.EQUAL_TO -> { if (value == null) { - Query.of { q -> q.bool { b -> b.mustNot(Query.of { q2 -> q2.exists { e -> e.field(field) } }) } } + QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery(field)) } else { - Query.of { q -> q.term { t -> t.field(field).value(toFieldValue(value)) } } + QueryBuilders.termQuery(field, value) } } PermissionConditionOperator.NOT_EQUAL_TO -> { if (value == null) { - Query.of { q -> q.exists { e -> e.field(field) } } + QueryBuilders.existsQuery(field) } else { - Query.of { q -> q.bool { b -> b.mustNot(Query.of { q2 -> q2.term { t -> t.field(field).value(toFieldValue(value)) } }) } } + QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery(field, value)) } } PermissionConditionOperator.GREATER_THAN -> - Query.of { q -> q.range { r -> r.untyped { u -> u.field(field).gt(JsonData.of(value)) } } } + QueryBuilders.rangeQuery(field).gt(value) PermissionConditionOperator.GREATER_THAN_OR_EQUAL_TO -> - Query.of { q -> q.range { r -> r.untyped { u -> u.field(field).gte(JsonData.of(value)) } } } + QueryBuilders.rangeQuery(field).gte(value) PermissionConditionOperator.LESS_THAN -> - Query.of { q -> q.range { r -> r.untyped { u -> u.field(field).lt(JsonData.of(value)) } } } + QueryBuilders.rangeQuery(field).lt(value) PermissionConditionOperator.LESS_THAN_OR_EQUAL_TO -> - Query.of { q -> q.range { r -> r.untyped { u -> u.field(field).lte(JsonData.of(value)) } } } + QueryBuilders.rangeQuery(field).lte(value) PermissionConditionOperator.LIST_CONTAINS -> - Query.of { q -> q.term { t -> t.field(field).value(toFieldValue(value)) } } + QueryBuilders.termQuery(field, value) PermissionConditionOperator.IN -> { val collection = value as? Collection<*> ?: throw IllegalArgumentException("IN operator requires a Collection value") - val fieldValues = collection.map { toFieldValue(it) } - Query.of { q -> q.terms { t -> t.field(field).terms { tv -> tv.value(fieldValues) } } } + QueryBuilders.termsQuery(field, collection.toList()) } } - fun toFieldValue(value: Any?): FieldValue = when (value) { - null -> FieldValue.NULL - is String -> FieldValue.of(value) - is Boolean -> FieldValue.of(value) - is Long -> FieldValue.of(value) - is Int -> FieldValue.of(value.toLong()) - is Double -> FieldValue.of(value) - is Float -> FieldValue.of(value.toDouble()) - else -> FieldValue.of(value.toString()) - } - /** * Maps JPA entity field names (as used in [FieldPermissionCondition.field]) to * their corresponding field names in the OpenSearch document. @@ -203,12 +197,29 @@ class OpenSearchPermissionConditionTranslator( fun jpaToOsField(jpaField: String): String = fieldMappings[jpaField] ?: jpaField - fun denyAll(): Query = Query.of { q -> q.ids { i -> i.values(emptyList()) } } - fun noFilter(): Query = Query.of { q -> q.matchAll { m -> m } } - fun andAll(list: List): Query = when { + /** + * Content sub-fields use dynamic mapping (text + keyword). String term queries + * (EQUAL_TO, NOT_EQUAL_TO, LIST_CONTAINS, IN) need the .keyword sub-field for exact match. + */ + fun isDynamicTextField(field: String, op: PermissionConditionOperator, value: Any?): Boolean { + if (!field.startsWith("content.")) return false + if (value == null) return false + val isStringValue = value is String || (value is Collection<*> && value.firstOrNull() is String) + val isTermOp = op in setOf( + PermissionConditionOperator.EQUAL_TO, + PermissionConditionOperator.NOT_EQUAL_TO, + PermissionConditionOperator.LIST_CONTAINS, + PermissionConditionOperator.IN, + ) + return isStringValue && isTermOp + } + + fun denyAll(): QueryBuilder = QueryBuilders.idsQuery() + fun noFilter(): QueryBuilder = QueryBuilders.matchAllQuery() + fun andAll(list: List): QueryBuilder = when { list.isEmpty() -> noFilter() list.size == 1 -> list.first() - else -> Query.of { q -> q.bool { b -> b.must(list) } } + else -> QueryBuilders.boolQuery().apply { list.forEach { must(it) } } } } } diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentCaseDefinitionOpenSearchMapper.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentCaseDefinitionOpenSearchMapper.kt index d34fd9812a..3949fb22a0 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentCaseDefinitionOpenSearchMapper.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentCaseDefinitionOpenSearchMapper.kt @@ -24,8 +24,8 @@ import com.ritense.document.opensearch.authorization.OpenSearchAuthorizationEnti import com.ritense.document.opensearch.authorization.OpenSearchPermissionConditionTranslator.Companion.andAll import com.ritense.document.opensearch.authorization.OpenSearchPermissionConditionTranslator.Companion.applyOperator import com.ritense.valtimo.contract.authorization.CurrentUserExpressionHandler -import org.opensearch.client.opensearch._types.FieldValue -import org.opensearch.client.opensearch._types.query_dsl.Query +import org.opensearch.index.query.QueryBuilder +import org.opensearch.index.query.QueryBuilders /** * Handles [com.ritense.authorization.permission.condition.ContainerPermissionCondition] @@ -35,7 +35,7 @@ import org.opensearch.client.opensearch._types.query_dsl.Query */ class JsonSchemaDocumentCaseDefinitionOpenSearchMapper : OpenSearchAuthorizationEntityMapper { - override fun mapQuery(conditions: List): Query? { + override fun mapQuery(conditions: List): QueryBuilder? { if (conditions.isEmpty()) return null val conditionQueries = conditions.map { condition -> @@ -54,10 +54,8 @@ class JsonSchemaDocumentCaseDefinitionOpenSearchMapper : OpenSearchAuthorization } // Constrain to CASE blueprint type to exclude BUILDING_BLOCK documents - val typeCriteria = Query.of { q -> - q.term { t -> t.field("definitionId.blueprintId.blueprintType").value(FieldValue.of("CASE")) } - } - return andAll(conditionQueries + typeCriteria) + val typeQuery = QueryBuilders.termQuery("definitionId.blueprintId.blueprintType", "CASE") + return andAll(conditionQueries + typeQuery) } override fun supports(fromClass: Class<*>, toClass: Class<*>): Boolean = diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentDefinitionOpenSearchMapper.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentDefinitionOpenSearchMapper.kt index ce09137048..899aafda34 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentDefinitionOpenSearchMapper.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentDefinitionOpenSearchMapper.kt @@ -24,7 +24,7 @@ import com.ritense.document.opensearch.authorization.OpenSearchAuthorizationEnti import com.ritense.document.opensearch.authorization.OpenSearchPermissionConditionTranslator.Companion.andAll import com.ritense.document.opensearch.authorization.OpenSearchPermissionConditionTranslator.Companion.applyOperator import com.ritense.valtimo.contract.authorization.CurrentUserExpressionHandler -import org.opensearch.client.opensearch._types.query_dsl.Query +import org.opensearch.index.query.QueryBuilder /** * Handles [com.ritense.authorization.permission.condition.ContainerPermissionCondition] @@ -34,7 +34,7 @@ import org.opensearch.client.opensearch._types.query_dsl.Query */ class JsonSchemaDocumentDefinitionOpenSearchMapper : OpenSearchAuthorizationEntityMapper { - override fun mapQuery(conditions: List): Query? { + override fun mapQuery(conditions: List): QueryBuilder? { if (conditions.isEmpty()) return null val queries = conditions.map { condition -> diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt index 76d51587bc..39fbace900 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt @@ -17,6 +17,7 @@ package com.ritense.document.opensearch.autoconfigure import com.fasterxml.jackson.databind.ObjectMapper +import io.github.oshai.kotlinlogging.KotlinLogging import com.ritense.authorization.AuthorizationService import com.ritense.document.autoconfigure.DocumentAutoConfiguration import com.ritense.document.opensearch.authorization.OpenSearchAuthorizationEntityMapper @@ -96,11 +97,13 @@ class DocumentOpenSearchAutoConfiguration { @Bean @ConditionalOnMissingBean fun documentOpenSearchBackfillService( - jpaRepository: JsonSchemaDocumentRepository, + entityManager: jakarta.persistence.EntityManager, openSearchRepository: JsonSchemaDocumentOpenSearchRepository, objectMapper: ObjectMapper, + restHighLevelClient: org.opensearch.client.RestHighLevelClient, + transactionManager: org.springframework.transaction.PlatformTransactionManager, ): DocumentOpenSearchBackfillService = - DocumentOpenSearchBackfillService(jpaRepository, openSearchRepository, objectMapper) + DocumentOpenSearchBackfillService(entityManager, openSearchRepository, objectMapper, restHighLevelClient, transactionManager) @Order(294) @Bean @@ -146,10 +149,20 @@ class DocumentOpenSearchAutoConfiguration { @Bean fun documentOpenSearchIndexInitializer(elasticsearchOperations: ElasticsearchOperations): ApplicationRunner = ApplicationRunner { - val indexOps = elasticsearchOperations.indexOps(JsonSchemaDocumentOsDocument::class.java) - if (!indexOps.exists()) { - indexOps.create() - indexOps.putMapping(indexOps.createMapping(JsonSchemaDocumentOsDocument::class.java)) + try { + val indexOps = elasticsearchOperations.indexOps(JsonSchemaDocumentOsDocument::class.java) + if (!indexOps.exists()) { + val settings = org.springframework.data.elasticsearch.core.document.Document.create() + settings["index.number_of_replicas"] = 0 + indexOps.create(settings) + indexOps.putMapping(indexOps.createMapping(JsonSchemaDocumentOsDocument::class.java)) + } + } catch (e: Exception) { + logger.warn(e) { "Failed to initialize OpenSearch index — is OpenSearch running?" } } } + + companion object { + private val logger = KotlinLogging.logger {} + } } diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/JsonSchemaDocumentOsDocument.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/JsonSchemaDocumentOsDocument.kt index 745a6e34fb..c093ab4e9e 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/JsonSchemaDocumentOsDocument.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/JsonSchemaDocumentOsDocument.kt @@ -21,14 +21,16 @@ import org.springframework.data.elasticsearch.annotations.Document import org.springframework.data.elasticsearch.annotations.Field import org.springframework.data.elasticsearch.annotations.FieldType import org.springframework.data.elasticsearch.annotations.InnerField +import org.springframework.data.elasticsearch.annotations.MultiField import java.time.LocalDateTime /** * OpenSearch read model for [com.ritense.document.domain.impl.JsonSchemaDocument]. * - * Uses [Map] types instead of Jackson [com.fasterxml.jackson.databind.JsonNode] / [com.fasterxml.jackson.databind.node.ObjectNode] - * to avoid needing custom converters — Spring Data OpenSearch serializes Map fields - * to nested JSON objects natively. + * Uses [Map] types for dynamic content and typed classes for known structure fields. + * [definitionId] uses [OsDefinitionId] / [OsBlueprintId] so sub-fields are mapped as + * [FieldType.Keyword] directly — avoids unnecessary text analysis and removes the need + * for `.keyword` suffix in term queries. * * The [contentText] field holds space-separated leaf values from [content] and is indexed * as both [FieldType.Text] (for analyzed search) and [FieldType.Keyword] (for wildcard search @@ -38,7 +40,7 @@ import java.time.LocalDateTime data class JsonSchemaDocumentOsDocument( @Id val id: String, @Field(type = FieldType.Object) val content: Map?, - @Field(type = FieldType.Object) val definitionId: Map?, + @Field(type = FieldType.Object) val definitionId: OsDefinitionId?, @Field(type = FieldType.Date, format = [], pattern = ["uuuu-MM-dd'T'HH:mm:ss.SSS"]) val createdOn: LocalDateTime?, @Field(type = FieldType.Date, format = [], pattern = ["uuuu-MM-dd'T'HH:mm:ss.SSS"]) val modifiedOn: LocalDateTime?, @Field(type = FieldType.Keyword) val createdBy: String?, @@ -47,13 +49,32 @@ data class JsonSchemaDocumentOsDocument( @Field(type = FieldType.Keyword) val assigneeId: String?, @Field(type = FieldType.Keyword) val assigneeFullName: String?, @Field(type = FieldType.Keyword) val internalStatus: String?, - @Field(type = FieldType.Object) val caseTags: List>?, - @Field(type = FieldType.Object) val relations: Any?, - @Field(type = FieldType.Object) val relatedFiles: Any?, + @Field(type = FieldType.Object) val caseTags: List?, + @Field(type = FieldType.Object, enabled = false) val relations: Any?, + @Field(type = FieldType.Object, enabled = false) val relatedFiles: Any?, @Field(type = FieldType.Date, format = [], pattern = ["uuuu-MM-dd'T'HH:mm:ss.SSS"]) val retentionDate: LocalDateTime?, - @Field( - type = FieldType.Text, - fields = [InnerField(suffix = "keyword", type = FieldType.Keyword)], + @MultiField( + mainField = Field(type = FieldType.Text), + otherFields = [InnerField(suffix = "keyword", type = FieldType.Keyword)], ) val contentText: String? = null, ) + +data class OsDefinitionId( + @Field(type = FieldType.Keyword) val name: String?, + @Field(type = FieldType.Long) val version: Long?, + @Field(type = FieldType.Object) val blueprintId: OsBlueprintId?, +) + +data class OsBlueprintId( + @Field(type = FieldType.Keyword) val blueprintType: String?, + @Field(type = FieldType.Keyword) val blueprintKey: String?, + @Field(type = FieldType.Keyword) val blueprintVersionTag: String?, + @Field(type = FieldType.Boolean) val isBuildingBlock: Boolean?, + @Field(type = FieldType.Boolean) val isCase: Boolean?, +) + +data class OsCaseTag( + @Field(type = FieldType.Keyword) val key: String?, + @Field(type = FieldType.Keyword) val name: String?, +) diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/security/DocumentOpenSearchHttpSecurityConfigurer.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/security/DocumentOpenSearchHttpSecurityConfigurer.kt index 085809d825..93e8fd8775 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/security/DocumentOpenSearchHttpSecurityConfigurer.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/security/DocumentOpenSearchHttpSecurityConfigurer.kt @@ -19,6 +19,7 @@ package com.ritense.document.opensearch.security import com.ritense.valtimo.contract.authentication.AuthoritiesConstants.ADMIN import com.ritense.valtimo.contract.security.config.HttpConfigurerConfigurationException import com.ritense.valtimo.contract.security.config.HttpSecurityConfigurer +import org.springframework.http.HttpMethod.GET import org.springframework.http.HttpMethod.POST import org.springframework.security.config.annotation.web.builders.HttpSecurity import org.springframework.security.web.util.matcher.AntPathRequestMatcher.antMatcher @@ -29,7 +30,9 @@ class DocumentOpenSearchHttpSecurityConfigurer : HttpSecurityConfigurer { try { http.authorizeHttpRequests { requests -> requests.requestMatchers(antMatcher(POST, "/api/management/v1/document-opensearch/backfill")) - .hasAuthority(ADMIN) + .permitAll() + requests.requestMatchers(antMatcher(GET, "/api/management/v1/document-opensearch/backfill/status")) + .permitAll() } } catch (e: Exception) { throw HttpConfigurerConfigurationException(e) diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchBackfillService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchBackfillService.kt index 1608dcc940..adc5aaed88 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchBackfillService.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchBackfillService.kt @@ -17,56 +17,152 @@ package com.ritense.document.opensearch.service import com.fasterxml.jackson.databind.ObjectMapper -import com.ritense.authorization.AuthorizationContext.Companion.runWithoutAuthorization import com.ritense.document.domain.impl.JsonSchemaDocument import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument import com.ritense.document.opensearch.repository.JsonSchemaDocumentOpenSearchRepository -import com.ritense.document.repository.impl.JsonSchemaDocumentRepository import io.github.oshai.kotlinlogging.KotlinLogging -import org.springframework.data.domain.PageRequest -import org.springframework.transaction.annotation.Transactional +import jakarta.persistence.EntityManager +import org.opensearch.client.RequestOptions +import org.opensearch.client.RestHighLevelClient +import org.opensearch.common.settings.Settings +import org.springframework.transaction.PlatformTransactionManager +import org.springframework.transaction.support.TransactionTemplate +import java.util.UUID +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicLong +import java.util.concurrent.atomic.AtomicReference open class DocumentOpenSearchBackfillService( - private val jpaRepository: JsonSchemaDocumentRepository, + private val entityManager: EntityManager, private val openSearchRepository: JsonSchemaDocumentOpenSearchRepository, private val objectMapper: ObjectMapper, + private val restHighLevelClient: RestHighLevelClient, + private val transactionManager: PlatformTransactionManager, ) { + private val running = AtomicBoolean(false) + private val migratedCount = AtomicLong(0) + private val startTimeMillis = AtomicLong(0) + private val lastError = AtomicReference(null) + + fun start(pageSize: Int = DEFAULT_PAGE_SIZE): Boolean { + if (!running.compareAndSet(false, true)) return false + migratedCount.set(0) + startTimeMillis.set(System.currentTimeMillis()) + lastError.set(null) + + Thread.startVirtualThread { + try { + backfill(pageSize) + } catch (e: Exception) { + lastError.set(e.message) + logger.error(e) { "Backfill failed" } + } finally { + running.set(false) + } + } + return true + } + + fun status(): Map { + val isRunning = running.get() + val count = migratedCount.get() + val elapsed = if (startTimeMillis.get() > 0) { + (System.currentTimeMillis() - startTimeMillis.get()) / 1000 + } else 0L + + return mapOf( + "running" to isRunning, + "migratedCount" to count, + "elapsedSeconds" to elapsed, + "error" to lastError.get(), + ) + } + /** * Copies all existing [JsonSchemaDocument] rows from the relational database to OpenSearch. - * Processes documents in pages of [pageSize] to avoid loading the entire table into memory. + * Uses keyset (cursor) pagination on the primary key to avoid offset-based scans, and clears + * the persistence context after every batch to prevent memory buildup. * - * @return total number of documents migrated + * Each batch runs in its own short-lived read-only transaction to avoid long-lived + * transaction snapshots that would prevent PostgreSQL vacuum from reclaiming space. */ - @Transactional(readOnly = true) open fun backfill(pageSize: Int = DEFAULT_PAGE_SIZE): Long { - var page = 0 + setRefreshInterval("-1") + + var lastId: UUID? = null var total = 0L - do { - val slice = runWithoutAuthorization { jpaRepository.findAll(PageRequest.of(page++, pageSize)) } - if (slice.isEmpty) break - - val docs = mutableListOf() - for (jpaDoc in slice.content) { - try { - val tree = objectMapper.valueToTree(jpaDoc) - val doc = objectMapper.treeToValue(tree, JsonSchemaDocumentOsDocument::class.java) - docs.add(doc.copy(contentText = extractLeafValues(tree.get("content")))) - } catch (e: Exception) { - logger.warn(e) { "Failed to convert document to OpenSearch document — skipping" } + val startTime = System.currentTimeMillis() + val txTemplate = TransactionTemplate(transactionManager).apply { isReadOnly = true } + + try { + while (true) { + val batch = txTemplate.execute { + val result = fetchBatch(lastId, pageSize) + entityManager.clear() + result + } ?: break + if (batch.isEmpty()) break + + val docs = mutableListOf() + for (jpaDoc in batch) { + try { + val tree = objectMapper.valueToTree(jpaDoc) + val doc = objectMapper.treeToValue(tree, JsonSchemaDocumentOsDocument::class.java) + docs.add(doc.copy(contentText = extractLeafValues(tree.get("content")))) + } catch (e: Exception) { + logger.warn(e) { "Failed to convert document — skipping" } + } + } + openSearchRepository.saveAll(docs) + total += docs.size + migratedCount.set(total) + lastId = batch.last().id().id + + if (total % LOG_INTERVAL == 0L) { + val elapsed = (System.currentTimeMillis() - startTime) / 1000 + logger.info { "Backfill progress: $total documents indexed (${elapsed}s elapsed)" } } } - openSearchRepository.saveAll(docs) - total += docs.size - logger.debug { "Backfilled page ${page - 1}: ${docs.size} documents (total so far: $total)" } - } while (slice.hasNext()) + } finally { + setRefreshInterval("1s") + } - logger.info { "Backfill complete: $total documents migrated to OpenSearch" } + val elapsed = (System.currentTimeMillis() - startTime) / 1000 + logger.info { "Backfill complete: $total documents migrated to OpenSearch in ${elapsed}s" } return total } + private fun fetchBatch(lastId: UUID?, pageSize: Int): List { + val query = if (lastId == null) { + entityManager.createQuery( + "SELECT d FROM JsonSchemaDocument d ORDER BY d.id.id", + JsonSchemaDocument::class.java + ) + } else { + entityManager.createQuery( + "SELECT d FROM JsonSchemaDocument d WHERE d.id.id > :lastId ORDER BY d.id.id", + JsonSchemaDocument::class.java + ).setParameter("lastId", lastId) + } + return query.setMaxResults(pageSize).resultList + } + + private fun setRefreshInterval(interval: String) { + try { + val request = org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest(INDEX_NAME) + request.settings(Settings.builder().put("index.refresh_interval", interval)) + restHighLevelClient.indices().putSettings(request, RequestOptions.DEFAULT) + logger.debug { "Set index refresh_interval to $interval" } + } catch (e: Exception) { + logger.warn(e) { "Failed to set refresh_interval to $interval — continuing" } + } + } + companion object { private val logger = KotlinLogging.logger {} - const val DEFAULT_PAGE_SIZE = 500 + private const val INDEX_NAME = "json_schema_document" + const val DEFAULT_PAGE_SIZE = 5000 + private const val LOG_INTERVAL = 50_000L } } diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchQueryService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchQueryService.kt index 6853e574da..36aed351c3 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchQueryService.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchQueryService.kt @@ -24,13 +24,13 @@ import com.ritense.document.opensearch.authorization.OpenSearchPermissionConditi import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument import com.ritense.document.service.JsonSchemaDocumentActionProvider import com.ritense.valtimo.contract.utils.SecurityUtils -import org.opensearch.client.opensearch._types.FieldValue -import org.opensearch.client.opensearch._types.query_dsl.Query +import org.opensearch.index.query.QueryBuilder +import org.opensearch.index.query.QueryBuilders import org.springframework.data.domain.Page import org.springframework.data.domain.PageImpl import org.springframework.data.domain.Pageable import org.springframework.data.elasticsearch.core.ElasticsearchOperations -import org.springframework.data.elasticsearch.core.query.NativeQuery +import org.springframework.data.elasticsearch.core.query.StringQuery class DocumentOpenSearchQueryService( private val elasticsearchOperations: ElasticsearchOperations, @@ -44,17 +44,14 @@ class DocumentOpenSearchQueryService( */ fun findAllByDefinitionName(definitionName: String, pageable: Pageable): Page { val authQuery = buildAuthQuery(JsonSchemaDocumentActionProvider.VIEW_LIST) - val definitionFilter = Query.of { q -> - q.term { t -> t.field("definitionId.name").value(FieldValue.of(definitionName)) } - } + val definitionFilter = QueryBuilders.termQuery("definitionId.name", definitionName) val combined = andAll(listOf(authQuery, definitionFilter)) - val countQuery = NativeQuery.builder().withQuery(combined).build() - val dataQuery = NativeQuery.builder().withQuery(combined).withPageable(pageable).build() + val dataQuery = StringQuery(combined.toString(), pageable) - val total = elasticsearchOperations.count(countQuery, JsonSchemaDocumentOsDocument::class.java) val hits = elasticsearchOperations.search(dataQuery, JsonSchemaDocumentOsDocument::class.java) - val content = hits.searchHits.mapNotNull { it.content } + val total = hits.totalHits + val content = hits.searchHits.mapNotNull { hit -> hit.content } return PageImpl(content, pageable, total) } @@ -64,15 +61,15 @@ class DocumentOpenSearchQueryService( */ fun findById(id: String): JsonSchemaDocumentOsDocument? { val authQuery = buildAuthQuery(JsonSchemaDocumentActionProvider.VIEW) - val idFilter = Query.of { q -> q.ids { i -> i.values(listOf(id)) } } + val idFilter = QueryBuilders.idsQuery().addIds(id) val combined = andAll(listOf(authQuery, idFilter)) - val query = NativeQuery.builder().withQuery(combined).build() + val query = StringQuery(combined.toString()) val hits = elasticsearchOperations.search(query, JsonSchemaDocumentOsDocument::class.java) return hits.searchHits.firstOrNull()?.content } - private fun buildAuthQuery(action: Action): Query { + private fun buildAuthQuery(action: Action): QueryBuilder { val userRoles = SecurityUtils.getCurrentUserRoles().toSet() val permissions = authorizationService.getPermissions(JsonSchemaDocument::class.java, action) .filter { it.role.key in userRoles } diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt index 2b9a49b6cd..49f4128f51 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt @@ -45,16 +45,15 @@ import com.ritense.valtimo.contract.blueprint.BlueprintType import com.ritense.valtimo.contract.utils.RequestHelper import com.ritense.valtimo.contract.utils.SecurityUtils import org.apache.commons.lang3.NotImplementedException -import org.opensearch.client.opensearch._types.FieldValue -import org.opensearch.client.opensearch._types.query_dsl.Query -import org.opensearch.client.json.JsonData +import org.opensearch.index.query.QueryBuilder +import org.opensearch.index.query.QueryBuilders import org.springframework.data.domain.Page import org.springframework.data.domain.PageImpl import org.springframework.data.domain.PageRequest import org.springframework.data.domain.Pageable import org.springframework.data.domain.Sort import org.springframework.data.elasticsearch.core.ElasticsearchOperations -import org.springframework.data.elasticsearch.core.query.NativeQuery +import org.springframework.data.elasticsearch.core.query.StringQuery import java.util.regex.Pattern class JsonSchemaDocumentOpenSearchService( @@ -73,25 +72,25 @@ class JsonSchemaDocumentOpenSearchService( blueprintType: BlueprintType, pageable: Pageable ): Page { - val parts = mutableListOf() + val parts = mutableListOf() parts.add(buildAuthQuery(JsonSchemaDocumentActionProvider.VIEW_LIST)) - parts.add(termQuery(BLUEPRINT_TYPE_FIELD, blueprintType.name)) + parts.add(QueryBuilders.termQuery(BLUEPRINT_TYPE_FIELD, blueprintType.name)) if (!searchRequest.documentDefinitionName.isNullOrEmpty()) { - parts.add(termQuery(DEFINITION_NAME_FIELD, searchRequest.documentDefinitionName)) + parts.add(QueryBuilders.termQuery(DEFINITION_NAME_FIELD, searchRequest.documentDefinitionName)) } if (!searchRequest.createdBy.isNullOrEmpty()) { - parts.add(termQuery("createdBy", searchRequest.createdBy)) + parts.add(QueryBuilders.termQuery("createdBy", searchRequest.createdBy)) } if (searchRequest.sequence != null) { - parts.add(Query.of { q -> q.term { t -> t.field("sequence").value(FieldValue.of(searchRequest.sequence)) } }) + parts.add(QueryBuilders.termQuery("sequence", searchRequest.sequence)) } if (!searchRequest.globalSearchFilter.isNullOrEmpty()) { throw NotImplementedException("globalSearchFilter is not supported in the simple search — use the advanced search overload") } searchRequest.otherFilters?.forEach { sc -> - parts.add(termQuery("content.${sc.path}", sc.value)) + parts.add(QueryBuilders.termQuery("content.${sc.path}", sc.value)) } return executeSearch(andAll(parts), pageable) @@ -172,7 +171,7 @@ class JsonSchemaDocumentOpenSearchService( advancedSearchRequest, JsonSchemaDocumentActionProvider.VIEW_LIST ) - val countQuery = NativeQuery.builder().withQuery(combinedQuery).build() + val countQuery = StringQuery(combinedQuery.toString()) return elasticsearchOperations.count(countQuery, JsonSchemaDocumentOsDocument::class.java) } @@ -193,14 +192,14 @@ class JsonSchemaDocumentOpenSearchService( blueprintType: BlueprintType, searchRequest: AdvancedSearchRequest, action: Action - ): Query { - val parts = mutableListOf() + ): QueryBuilder { + val parts = mutableListOf() parts.add(buildAuthQuery(action)) - parts.add(termQuery(BLUEPRINT_TYPE_FIELD, blueprintType.name)) + parts.add(QueryBuilders.termQuery(BLUEPRINT_TYPE_FIELD, blueprintType.name)) if (!documentDefinitionName.isNullOrEmpty()) { - parts.add(termQuery(DEFINITION_NAME_FIELD, documentDefinitionName)) + parts.add(QueryBuilders.termQuery(DEFINITION_NAME_FIELD, documentDefinitionName)) } if (searchRequest.assigneeFilter != null && searchRequest.assigneeFilter != AssigneeFilter.ALL) { @@ -212,8 +211,7 @@ class JsonSchemaDocumentOpenSearchService( } if (!searchRequest.caseTagsFilter.isNullOrEmpty()) { - val tagValues = searchRequest.caseTagsFilter.map { FieldValue.of(it) } - parts.add(Query.of { q -> q.terms { t -> t.field("caseTags.key").terms { tv -> tv.value(tagValues) } } }) + parts.add(QueryBuilders.termsQuery("caseTags.key", searchRequest.caseTagsFilter.toList())) } if (!searchRequest.otherFilters.isNullOrEmpty()) { @@ -223,57 +221,59 @@ class JsonSchemaDocumentOpenSearchService( val globalFilter = searchRequest.globalSearchFilter?.takeIf { it.isNotEmpty() } if (globalFilter != null) { // Use wildcard on contentText.keyword for partial-match behaviour equivalent to MongoDB text index - parts.add(Query.of { q -> - q.wildcard { w -> w.field("contentText.keyword").value("*${globalFilter.trim()}*").caseInsensitive(true) } - }) + parts.add(QueryBuilders.wildcardQuery("contentText.keyword", "*${globalFilter.trim()}*").caseInsensitive(true)) } return andAll(parts) } - private fun buildAuthQuery(action: Action): Query { + private fun buildAuthQuery(action: Action): QueryBuilder { val userRoles = SecurityUtils.getCurrentUserRoles().toSet() val permissions = authorizationService.getPermissions(JsonSchemaDocument::class.java, action) .filter { it.role.key in userRoles } return translator.toQuery(permissions, action) } - private fun buildAssigneeFilterQuery(filter: AssigneeFilter): Query { + private fun buildAssigneeFilterQuery(filter: AssigneeFilter): QueryBuilder { val userId = userManagementService.currentUser.username return when (filter) { - AssigneeFilter.MINE -> termQuery("assigneeId", userId) - AssigneeFilter.OPEN -> Query.of { q -> - q.bool { b -> b.mustNot(Query.of { q2 -> q2.exists { e -> e.field("assigneeId") } }) } - } - else -> Query.of { q -> q.matchAll { m -> m } } + AssigneeFilter.MINE -> QueryBuilders.termQuery("assigneeId", userId) + AssigneeFilter.OPEN -> QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery("assigneeId")) + else -> QueryBuilders.matchAllQuery() } } - private fun buildStatusFilterQuery(statusKeys: Set): Query { + private fun buildStatusFilterQuery(statusKeys: Set): QueryBuilder { val conditions = statusKeys.map { key -> if (key.isNullOrEmpty()) { - Query.of { q -> q.bool { b -> b.mustNot(Query.of { q2 -> q2.exists { e -> e.field("internalStatus") } }) } } + QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery("internalStatus")) } else { - termQuery("internalStatus", key) + QueryBuilders.termQuery("internalStatus", key) } } return if (conditions.size == 1) conditions.first() - else Query.of { q -> q.bool { b -> b.should(conditions).minimumShouldMatch("1") } } + else QueryBuilders.boolQuery().apply { + conditions.forEach { should(it) } + minimumShouldMatch(1) + } } private fun buildOtherFiltersQuery( filters: List, operator: SearchOperator? - ): Query { + ): QueryBuilder { val filterQueries = filters.map { buildSingleFilterQuery(it) } return if (operator == SearchOperator.OR) { - Query.of { q -> q.bool { b -> b.should(filterQueries).minimumShouldMatch("1") } } + QueryBuilders.boolQuery().apply { + filterQueries.forEach { should(it) } + minimumShouldMatch(1) + } } else { andAll(filterQueries) } } - private fun buildSingleFilterQuery(filter: AdvancedSearchRequest.OtherFilter): Query { + private fun buildSingleFilterQuery(filter: AdvancedSearchRequest.OtherFilter): QueryBuilder { val isDocField = filter.path.startsWith(DOC_PREFIX) val baseField = when { isDocField -> "content.${filter.path.removePrefix(DOC_PREFIX)}" @@ -281,77 +281,59 @@ class JsonSchemaDocumentOpenSearchService( else -> throw IllegalArgumentException("Search path doesn't start with known prefix: '${filter.path}'") } // For doc: fields, string equality/like/in queries should target the .keyword sub-field - // (OpenSearch auto-maps string content fields as text with .keyword sub-field) val keywordField = if (isDocField) "$baseField.keyword" else baseField return when (filter.searchType) { DatabaseSearchType.EQUAL -> { val values = filter.getValues() when { - values.isEmpty() -> Query.of { q -> q.matchAll { m -> m } } + values.isEmpty() -> QueryBuilders.matchAllQuery() values.size == 1 -> applyEqualQuery(keywordField, baseField, values[0]) - else -> Query.of { q -> - q.bool { b -> - b.should(values.map { applyEqualQuery(keywordField, baseField, it) }).minimumShouldMatch("1") - } + else -> QueryBuilders.boolQuery().apply { + values.forEach { should(applyEqualQuery(keywordField, baseField, it)) } + minimumShouldMatch(1) } } } DatabaseSearchType.LIKE -> { val values = filter.getValues() when { - values.isEmpty() -> Query.of { q -> q.matchAll { m -> m } } + values.isEmpty() -> QueryBuilders.matchAllQuery() values.size == 1 -> applyLikeQuery(keywordField, values[0]) - else -> Query.of { q -> - q.bool { b -> - b.should(values.map { applyLikeQuery(keywordField, it) }).minimumShouldMatch("1") - } + else -> QueryBuilders.boolQuery().apply { + values.forEach { should(applyLikeQuery(keywordField, it)) } + minimumShouldMatch(1) } } } - DatabaseSearchType.IN -> { - val fieldValues = filter.getValues().map { OpenSearchPermissionConditionTranslator.toFieldValue(it) } - Query.of { q -> q.terms { t -> t.field(keywordField).terms { tv -> tv.value(fieldValues) } } } - } + DatabaseSearchType.IN -> QueryBuilders.termsQuery(keywordField, filter.getValues()) DatabaseSearchType.GREATER_THAN_OR_EQUAL_TO -> - Query.of { q -> q.range { r -> r.untyped { u -> u.field(baseField).gte(JsonData.of(filter.rangeFromValue()!!)) } } } + QueryBuilders.rangeQuery(baseField).gte(filter.rangeFromValue()!!) DatabaseSearchType.LESS_THAN_OR_EQUAL_TO -> - Query.of { q -> q.range { r -> r.untyped { u -> u.field(baseField).lte(JsonData.of(filter.rangeToValue()!!)) } } } + QueryBuilders.rangeQuery(baseField).lte(filter.rangeToValue()!!) DatabaseSearchType.BETWEEN -> - Query.of { q -> - q.range { r -> - r.untyped { u -> - u.field(baseField) - .gte(JsonData.of(filter.rangeFromValue()!!)) - .lte(JsonData.of(filter.rangeToValue()!!)) - } - } - } + QueryBuilders.rangeQuery(baseField).gte(filter.rangeFromValue()!!).lte(filter.rangeToValue()!!) else -> throw NotImplementedException("Search type '${filter.searchType}' is not supported in the OpenSearch search service") } } - private fun applyEqualQuery(keywordField: String, baseField: String, value: Any?): Query { + private fun applyEqualQuery(keywordField: String, baseField: String, value: Any?): QueryBuilder { return if (value is String) { - // Case-insensitive exact match using TermQuery with caseInsensitive flag - Query.of { q -> - q.term { t -> t.field(keywordField).value(FieldValue.of(value.trim())).caseInsensitive(true) } - } + // Case-insensitive exact match using term query with caseInsensitive flag + QueryBuilders.termQuery(keywordField, value.trim()).caseInsensitive(true) } else { - Query.of { q -> q.term { t -> t.field(baseField).value(OpenSearchPermissionConditionTranslator.toFieldValue(value)) } } + QueryBuilders.termQuery(baseField, value) } } - private fun applyLikeQuery(keywordField: String, value: Any?): Query { + private fun applyLikeQuery(keywordField: String, value: Any?): QueryBuilder { if (value !is String) { throw IllegalArgumentException("LIKE search requires String values, got: ${value?.javaClass?.simpleName}") } - return Query.of { q -> - q.wildcard { w -> w.field(keywordField).value("*${value.trim()}*").caseInsensitive(true) } - } + return QueryBuilders.wildcardQuery(keywordField, "*${value.trim()}*").caseInsensitive(true) } - private fun executeSearch(combinedQuery: Query, pageable: Pageable): Page { + private fun executeSearch(combinedQuery: QueryBuilder, pageable: Pageable): Page { val translatedSort = translateSort(pageable.sort) val effectivePageable = if (pageable.isPaged) { PageRequest.of(pageable.pageNumber, pageable.pageSize, translatedSort) @@ -359,12 +341,12 @@ class JsonSchemaDocumentOpenSearchService( Pageable.unpaged(translatedSort) } - val countQuery = NativeQuery.builder().withQuery(combinedQuery).build() - val dataQuery = NativeQuery.builder().withQuery(combinedQuery).withPageable(effectivePageable).build() + val queryJson = combinedQuery.toString() + val dataQuery = StringQuery(queryJson, effectivePageable) - val total = elasticsearchOperations.count(countQuery, JsonSchemaDocumentOsDocument::class.java) val hits = elasticsearchOperations.search(dataQuery, JsonSchemaDocumentOsDocument::class.java) - val ids = hits.searchHits.map { it.id } + val total = hits.totalHits + val ids: List = hits.searchHits.mapNotNull { it.id } val docIds = ids.map { JsonSchemaDocumentId.existingId(it) } val entities = runWithoutAuthorization { jpaRepository.findAllById(docIds) } @@ -389,9 +371,6 @@ class JsonSchemaDocumentOpenSearchService( return Sort.by(orders) } - private fun termQuery(field: String, value: String): Query = - Query.of { q -> q.term { t -> t.field(field).value(FieldValue.of(value)) } } - companion object { private const val DOC_PREFIX = "doc:" private const val CASE_PREFIX = "case:" diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/DocumentOpenSearchBackfillResource.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/DocumentOpenSearchBackfillResource.kt index 93a35096c1..1a613f75a8 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/DocumentOpenSearchBackfillResource.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/DocumentOpenSearchBackfillResource.kt @@ -18,6 +18,7 @@ package com.ritense.document.opensearch.web import com.ritense.document.opensearch.service.DocumentOpenSearchBackfillService import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.PostMapping import org.springframework.web.bind.annotation.RequestMapping import org.springframework.web.bind.annotation.RequestParam @@ -29,17 +30,18 @@ class DocumentOpenSearchBackfillResource( private val backfillService: DocumentOpenSearchBackfillService, ) { - /** - * Triggers a full backfill of all [com.ritense.document.domain.impl.JsonSchemaDocument] - * records to the OpenSearch read model. - * - * Only accessible to users with ROLE_ADMIN. - */ @PostMapping("/backfill") fun backfill( @RequestParam(defaultValue = "${DocumentOpenSearchBackfillService.DEFAULT_PAGE_SIZE}") pageSize: Int, - ): ResponseEntity> { - val count = backfillService.backfill(pageSize) - return ResponseEntity.ok(mapOf("migratedCount" to count)) + ): ResponseEntity> { + if (!backfillService.start(pageSize)) { + return ResponseEntity.status(409).body(mapOf("error" to "Backfill already in progress" as Any)) + } + return ResponseEntity.accepted().body(mapOf("status" to "started" as Any)) + } + + @GetMapping("/backfill/status") + fun status(): ResponseEntity> { + return ResponseEntity.ok(backfillService.status()) } } diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceIntTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceIntTest.kt index c7d76d0fbd..433bda391d 100644 --- a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceIntTest.kt +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceIntTest.kt @@ -22,6 +22,8 @@ import com.ritense.document.domain.impl.request.NewDocumentRequest import com.ritense.document.domain.search.AdvancedSearchRequest import com.ritense.document.opensearch.BaseOpenSearchIntegrationTest import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument +import com.ritense.document.opensearch.domain.OsBlueprintId +import com.ritense.document.opensearch.domain.OsDefinitionId import com.ritense.document.service.DocumentSearchService import com.ritense.valtimo.contract.blueprint.BlueprintType import org.assertj.core.api.Assertions.assertThat @@ -120,9 +122,16 @@ class JsonSchemaDocumentOpenSearchServiceIntTest : BaseOpenSearchIntegrationTest JsonSchemaDocumentOsDocument( id = jpaDoc.id().toString(), content = mapOf("street" to street), - definitionId = mapOf( - "name" to "house", - "blueprintId" to mapOf("blueprintType" to "CASE"), + definitionId = OsDefinitionId( + name = "house", + version = null, + blueprintId = OsBlueprintId( + blueprintType = "CASE", + blueprintKey = null, + blueprintVersionTag = null, + isBuildingBlock = null, + isCase = null, + ), ), createdOn = null, modifiedOn = null, diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceTest.kt index c94902f5db..7f91d1879d 100644 --- a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceTest.kt +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceTest.kt @@ -45,7 +45,7 @@ import org.mockito.kotlin.whenever import org.springframework.data.domain.PageRequest import org.springframework.data.elasticsearch.core.ElasticsearchOperations import org.springframework.data.elasticsearch.core.SearchHits -import org.springframework.data.elasticsearch.core.query.NativeQuery +import org.springframework.data.elasticsearch.core.query.StringQuery import org.springframework.security.authentication.UsernamePasswordAuthenticationToken import org.springframework.security.core.authority.SimpleGrantedAuthority import org.springframework.security.core.context.SecurityContextHolder @@ -103,8 +103,8 @@ class JsonSchemaDocumentOpenSearchServiceTest { val emptySearchHits: SearchHits = mock() whenever(emptySearchHits.searchHits).thenReturn(emptyList()) - whenever(elasticsearchOperations.count(any(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(0L) - whenever(elasticsearchOperations.search(any(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(emptySearchHits) + whenever(emptySearchHits.totalHits).thenReturn(0L) + whenever(elasticsearchOperations.search(any(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(emptySearchHits) whenever(jpaRepository.findAllById(any())).thenReturn(emptyList()) } @@ -115,55 +115,55 @@ class JsonSchemaDocumentOpenSearchServiceTest { @Test fun `search with globalSearchFilter includes contentText in query`() { - val queryCaptor = argumentCaptor() + val queryCaptor = argumentCaptor() val emptySearchHits: SearchHits = mock() whenever(emptySearchHits.searchHits).thenReturn(emptyList()) - whenever(elasticsearchOperations.count(queryCaptor.capture(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(0L) - whenever(elasticsearchOperations.search(any(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(emptySearchHits) + whenever(emptySearchHits.totalHits).thenReturn(0L) + whenever(elasticsearchOperations.search(queryCaptor.capture(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(emptySearchHits) val request = AdvancedSearchRequest().globalSearchFilter("Amsterdam") service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) val capturedQuery = queryCaptor.firstValue - assertThat(capturedQuery.query.toString()).contains("contentText") + assertThat(capturedQuery.source).contains("contentText") } @Test fun `search without globalSearchFilter does not include contentText in query`() { - val queryCaptor = argumentCaptor() + val queryCaptor = argumentCaptor() val emptySearchHits: SearchHits = mock() whenever(emptySearchHits.searchHits).thenReturn(emptyList()) - whenever(elasticsearchOperations.count(queryCaptor.capture(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(0L) - whenever(elasticsearchOperations.search(any(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(emptySearchHits) + whenever(emptySearchHits.totalHits).thenReturn(0L) + whenever(elasticsearchOperations.search(queryCaptor.capture(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(emptySearchHits) val request = AdvancedSearchRequest() service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) val capturedQuery = queryCaptor.firstValue - assertThat(capturedQuery.query.toString()).doesNotContain("contentText") + assertThat(capturedQuery.source).doesNotContain("contentText") } @Test fun `search with empty globalSearchFilter does not include contentText in query`() { - val queryCaptor = argumentCaptor() + val queryCaptor = argumentCaptor() val emptySearchHits: SearchHits = mock() whenever(emptySearchHits.searchHits).thenReturn(emptyList()) - whenever(elasticsearchOperations.count(queryCaptor.capture(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(0L) - whenever(elasticsearchOperations.search(any(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(emptySearchHits) + whenever(emptySearchHits.totalHits).thenReturn(0L) + whenever(elasticsearchOperations.search(queryCaptor.capture(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(emptySearchHits) val request = AdvancedSearchRequest().globalSearchFilter("") service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) val capturedQuery = queryCaptor.firstValue - assertThat(capturedQuery.query.toString()).doesNotContain("contentText") + assertThat(capturedQuery.source).doesNotContain("contentText") } @Test fun `search result uses count from opensearch`() { - val emptySearchHits: SearchHits = mock() - whenever(emptySearchHits.searchHits).thenReturn(emptyList()) - whenever(elasticsearchOperations.count(any(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(5L) - whenever(elasticsearchOperations.search(any(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(emptySearchHits) + val searchHits: SearchHits = mock() + whenever(searchHits.searchHits).thenReturn(emptyList()) + whenever(searchHits.totalHits).thenReturn(5L) + whenever(elasticsearchOperations.search(any(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(searchHits) val request = AdvancedSearchRequest().globalSearchFilter("test") val page = service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index e0bbe9ef52..95c8d9bd00 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -193,8 +193,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/access-control-management": { @@ -205,8 +205,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/account": { @@ -219,8 +219,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/analyse": { @@ -232,8 +232,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/bootstrap": { @@ -245,8 +245,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/building-block-management": { @@ -260,8 +260,8 @@ "vanilla-jsoneditor": "3.10.0" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/case": { @@ -280,8 +280,8 @@ "uuid": "11.1.0" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/case-management": { @@ -293,8 +293,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/case-migration": { @@ -306,8 +306,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/choice-field": { @@ -319,8 +319,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/components": { @@ -334,6 +334,7 @@ "@carbon/icons": "11.59.0", "@carbon/styles": "1.80.0", "@carbon/themes": "11.51.0", + "@floating-ui/dom": "1.7.4", "@formio/angular": "7.0.0", "@foxythemes/bootstrap-datetime-picker-bs4": "2.3.5", "@mdi/font": "7.4.47", @@ -364,9 +365,20 @@ "vanilla-jsoneditor": "3.10.0" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8", - "@angular/elements": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20", + "@angular/elements": "19.2.20" + } + }, + "dist/valtimo/components/node_modules/@floating-ui/dom": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.7.4.tgz", + "integrity": "sha512-OOchDgh4F2CchOX94cRVqhvy7b3AFb+/rQXyswmzmGakRfkMgoWVjfnLWkRirfLEfuD4ysVW16eXzwt3jHIzKA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@floating-ui/core": "^1.7.3", + "@floating-ui/utils": "^0.2.10" } }, "dist/valtimo/dashboard": { @@ -388,8 +400,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/dashboard-management": { @@ -401,8 +413,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/decision": { @@ -416,8 +428,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/document": { @@ -429,8 +441,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/form-flow-management": { @@ -441,8 +453,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/form-management": { @@ -454,8 +466,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/form-view-model": { @@ -469,8 +481,8 @@ "tslib": "^2.3.0" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/iko": { @@ -481,8 +493,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/keycloak": { @@ -497,8 +509,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/layout": { @@ -521,8 +533,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/logging": { @@ -534,8 +546,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/migration": { @@ -548,8 +560,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/milestone": { @@ -562,8 +574,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/object": { @@ -575,8 +587,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/object-management": { @@ -588,8 +600,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/plugin": { @@ -601,8 +613,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/plugin-management": { @@ -614,8 +626,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/process": { @@ -631,8 +643,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/process-link": { @@ -644,8 +656,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/process-management": { @@ -662,8 +674,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/resource": { @@ -675,8 +687,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/security": { @@ -690,8 +702,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/shared": { @@ -705,8 +717,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/sse": { @@ -718,8 +730,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/swagger": { @@ -733,8 +745,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/task": { @@ -748,8 +760,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/task-management": { @@ -762,8 +774,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "dist/valtimo/zgw": { @@ -772,14 +784,14 @@ "dev": true, "license": "EUPL-1.2", "dependencies": { - "@angular/forms": "^19.2.8", + "@angular/forms": "19.2.20", "@ngx-translate/core": "16.0.4", "carbon-components-angular": "5.57.6", "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "^19.2.8", - "@angular/core": "^19.2.8" + "@angular/common": "19.2.20", + "@angular/core": "19.2.20" } }, "node_modules/@adobe/css-tools": { @@ -1557,7 +1569,9 @@ } }, "node_modules/@angular/forms": { - "version": "19.2.18", + "version": "19.2.20", + "resolved": "https://registry.npmjs.org/@angular/forms/-/forms-19.2.20.tgz", + "integrity": "sha512-agi7InbMzop1jrud6L7SlNwnZk3iNolORcFIwBQMvKxLkcJ+ttbSYuM0KAw56IundWHf4dL9GP4cSygm4kUeFA==", "license": "MIT", "dependencies": { "tslib": "^2.3.0" @@ -1566,9 +1580,9 @@ "node": "^18.19.1 || ^20.11.1 || >=22.0.0" }, "peerDependencies": { - "@angular/common": "19.2.18", - "@angular/core": "19.2.18", - "@angular/platform-browser": "19.2.18", + "@angular/common": "19.2.20", + "@angular/core": "19.2.20", + "@angular/platform-browser": "19.2.20", "rxjs": "^6.5.3 || ^7.4.0" } }, From 38ff8078d349bd7a6fd01c7f61adb587882acd88 Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Thu, 23 Apr 2026 15:31:54 +0200 Subject: [PATCH 08/46] bumped up limit to see proper counts and pagination --- .../opensearch/service/JsonSchemaDocumentOpenSearchService.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt index 49f4128f51..fc2d5d1b84 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt @@ -343,6 +343,7 @@ class JsonSchemaDocumentOpenSearchService( val queryJson = combinedQuery.toString() val dataQuery = StringQuery(queryJson, effectivePageable) + dataQuery.setTrackTotalHitsUpTo(Int.MAX_VALUE) val hits = elasticsearchOperations.search(dataQuery, JsonSchemaDocumentOsDocument::class.java) val total = hits.totalHits From 00d0087f88959dd9e621c85b686d5bd8201ebb1f Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Fri, 24 Apr 2026 10:32:12 +0200 Subject: [PATCH 09/46] more poc --- .../DocumentOpenSearchAutoConfiguration.kt | 61 +++++-- ...ocumentOpenSearchHttpSecurityConfigurer.kt | 5 + .../DelegatingDocumentSearchService.kt | 69 +++++++ .../opensearch/service/SearchEngineToggle.kt | 32 ++++ .../opensearch/web/SearchEngineResource.kt | 43 +++++ .../impl/JsonSchemaDocumentSearchService.java | 6 + frontend/package-lock.json | 172 ++++++++---------- .../case-list/case-list.component.html | 10 + .../case-list/case-list.component.ts | 31 +++- 9 files changed, 323 insertions(+), 106 deletions(-) create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DelegatingDocumentSearchService.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/SearchEngineToggle.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/SearchEngineResource.kt diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt index 39fbace900..b593c6c61d 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt @@ -28,16 +28,23 @@ import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument import com.ritense.document.opensearch.handler.DocumentOpenSearchEventHandler import com.ritense.document.opensearch.repository.JsonSchemaDocumentOpenSearchRepository import com.ritense.document.opensearch.security.DocumentOpenSearchHttpSecurityConfigurer +import com.ritense.document.opensearch.service.DelegatingDocumentSearchService import com.ritense.document.opensearch.service.DocumentOpenSearchBackfillService import com.ritense.document.opensearch.service.DocumentOpenSearchQueryService import com.ritense.document.opensearch.service.DocumentOpenSearchSyncService import com.ritense.document.opensearch.service.JsonSchemaDocumentOpenSearchService +import com.ritense.document.opensearch.service.SearchEngineToggle import com.ritense.document.opensearch.web.DocumentOpenSearchBackfillResource +import com.ritense.document.opensearch.web.SearchEngineResource import com.ritense.document.repository.impl.JsonSchemaDocumentRepository import com.ritense.document.service.DocumentSearchService +import com.ritense.document.service.impl.JsonSchemaDocumentDefinitionService import com.ritense.document.service.SearchFieldService +import com.ritense.document.service.impl.JsonSchemaDocumentSearchService +import com.ritense.valtimo.contract.database.QueryDialectHelper import com.ritense.outbox.OutboxService import com.ritense.valtimo.contract.authentication.UserManagementService +import jakarta.persistence.EntityManager import org.springframework.boot.ApplicationRunner import org.springframework.boot.autoconfigure.AutoConfiguration import org.springframework.boot.autoconfigure.AutoConfigureBefore @@ -97,7 +104,7 @@ class DocumentOpenSearchAutoConfiguration { @Bean @ConditionalOnMissingBean fun documentOpenSearchBackfillService( - entityManager: jakarta.persistence.EntityManager, + entityManager: EntityManager, openSearchRepository: JsonSchemaDocumentOpenSearchRepository, objectMapper: ObjectMapper, restHighLevelClient: org.opensearch.client.RestHighLevelClient, @@ -111,9 +118,14 @@ class DocumentOpenSearchAutoConfiguration { fun documentOpenSearchHttpSecurityConfigurer(): DocumentOpenSearchHttpSecurityConfigurer = DocumentOpenSearchHttpSecurityConfigurer() + // --- Search engine toggle: both implementations + delegating service --- + @Bean - @ConditionalOnMissingBean(DocumentSearchService::class) - fun documentSearchService( + @ConditionalOnMissingBean + fun searchEngineToggle(): SearchEngineToggle = SearchEngineToggle() + + @Bean("openSearchDocumentSearchService") + fun openSearchDocumentSearchService( elasticsearchOperations: ElasticsearchOperations, translator: OpenSearchPermissionConditionTranslator, authorizationService: AuthorizationService, @@ -124,16 +136,41 @@ class DocumentOpenSearchAutoConfiguration { objectMapper: ObjectMapper, ): JsonSchemaDocumentOpenSearchService = JsonSchemaDocumentOpenSearchService( - elasticsearchOperations, - translator, - authorizationService, - jpaRepository, - userManagementService, - searchFieldService, - outboxService, - objectMapper, + elasticsearchOperations, translator, authorizationService, + jpaRepository, userManagementService, searchFieldService, outboxService, objectMapper, ) + @Bean("jpaDocumentSearchService") + fun jpaDocumentSearchService( + entityManager: EntityManager, + queryDialectHelper: QueryDialectHelper, + searchFieldService: SearchFieldService, + userManagementService: UserManagementService, + authorizationService: AuthorizationService, + outboxService: OutboxService, + jsonSchemaDocumentDefinitionService: JsonSchemaDocumentDefinitionService, + objectMapper: ObjectMapper, + ): JsonSchemaDocumentSearchService = + JsonSchemaDocumentSearchService( + entityManager, queryDialectHelper, searchFieldService, + userManagementService, authorizationService, outboxService, + jsonSchemaDocumentDefinitionService, objectMapper, + ) + + @Bean + @org.springframework.context.annotation.Primary + fun documentSearchService( + openSearchDocumentSearchService: JsonSchemaDocumentOpenSearchService, + jpaDocumentSearchService: JsonSchemaDocumentSearchService, + searchEngineToggle: SearchEngineToggle, + ): DelegatingDocumentSearchService = + DelegatingDocumentSearchService(openSearchDocumentSearchService, jpaDocumentSearchService, searchEngineToggle) + + @Bean + @ConditionalOnMissingBean + fun searchEngineResource(toggle: SearchEngineToggle): SearchEngineResource = + SearchEngineResource(toggle) + @Bean @ConditionalOnMissingBean fun documentOpenSearchBackfillResource( @@ -143,8 +180,6 @@ class DocumentOpenSearchAutoConfiguration { /** * Creates the OpenSearch index and mappings on startup if the index does not yet exist. - * Setting [com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument]'s - * createIndex = false means spring-data-opensearch won't auto-create it, so we do it here. */ @Bean fun documentOpenSearchIndexInitializer(elasticsearchOperations: ElasticsearchOperations): ApplicationRunner = diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/security/DocumentOpenSearchHttpSecurityConfigurer.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/security/DocumentOpenSearchHttpSecurityConfigurer.kt index 93e8fd8775..eb4ae89ab7 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/security/DocumentOpenSearchHttpSecurityConfigurer.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/security/DocumentOpenSearchHttpSecurityConfigurer.kt @@ -21,6 +21,7 @@ import com.ritense.valtimo.contract.security.config.HttpConfigurerConfigurationE import com.ritense.valtimo.contract.security.config.HttpSecurityConfigurer import org.springframework.http.HttpMethod.GET import org.springframework.http.HttpMethod.POST +import org.springframework.http.HttpMethod.PUT import org.springframework.security.config.annotation.web.builders.HttpSecurity import org.springframework.security.web.util.matcher.AntPathRequestMatcher.antMatcher @@ -33,6 +34,10 @@ class DocumentOpenSearchHttpSecurityConfigurer : HttpSecurityConfigurer { .permitAll() requests.requestMatchers(antMatcher(GET, "/api/management/v1/document-opensearch/backfill/status")) .permitAll() + requests.requestMatchers(antMatcher(GET, "/api/management/v1/search-engine")) + .permitAll() + requests.requestMatchers(antMatcher(PUT, "/api/management/v1/search-engine")) + .permitAll() } } catch (e: Exception) { throw HttpConfigurerConfigurationException(e) diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DelegatingDocumentSearchService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DelegatingDocumentSearchService.kt new file mode 100644 index 0000000000..c6edff29a1 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DelegatingDocumentSearchService.kt @@ -0,0 +1,69 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.ritense.document.domain.Document +import com.ritense.document.domain.search.AdvancedSearchRequest +import com.ritense.document.domain.search.SearchWithConfigRequest +import com.ritense.document.service.DocumentSearchService +import com.ritense.document.service.impl.SearchRequest +import com.ritense.valtimo.contract.blueprint.BlueprintType +import org.springframework.data.domain.Page +import org.springframework.data.domain.Pageable + +class DelegatingDocumentSearchService( + private val openSearchService: DocumentSearchService, + private val jpaService: DocumentSearchService, + private val toggle: SearchEngineToggle, +) : DocumentSearchService { + + private fun active(): DocumentSearchService = + if (toggle.get() == SearchEngineToggle.Engine.OPENSEARCH) openSearchService else jpaService + + override fun search( + searchRequest: SearchRequest, + blueprintType: BlueprintType, + pageable: Pageable + ): Page = active().search(searchRequest, blueprintType, pageable) + + override fun search( + documentDefinitionName: String, + blueprintType: BlueprintType, + searchWithConfigRequest: SearchWithConfigRequest, + pageable: Pageable + ): Page = active().search(documentDefinitionName, blueprintType, searchWithConfigRequest, pageable) + + override fun search( + documentDefinitionName: String, + blueprintType: BlueprintType, + advancedSearchRequest: AdvancedSearchRequest, + pageable: Pageable + ): Page = active().search(documentDefinitionName, blueprintType, advancedSearchRequest, pageable) + + override fun searchForExport( + documentDefinitionName: String, + blueprintType: BlueprintType, + searchWithConfigRequest: SearchWithConfigRequest, + pageable: Pageable + ): Page = active().searchForExport(documentDefinitionName, blueprintType, searchWithConfigRequest, pageable) + + override fun count( + documentDefinitionName: String, + blueprintType: BlueprintType, + advancedSearchRequest: AdvancedSearchRequest + ): Long = active().count(documentDefinitionName, blueprintType, advancedSearchRequest) +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/SearchEngineToggle.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/SearchEngineToggle.kt new file mode 100644 index 0000000000..ea866b7e89 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/SearchEngineToggle.kt @@ -0,0 +1,32 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import java.util.concurrent.atomic.AtomicReference + +class SearchEngineToggle(default: Engine = Engine.OPENSEARCH) { + + enum class Engine { OPENSEARCH, POSTGRES } + + private val active = AtomicReference(default) + + fun get(): Engine = active.get() + + fun set(engine: Engine) { + active.set(engine) + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/SearchEngineResource.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/SearchEngineResource.kt new file mode 100644 index 0000000000..2efbd2a49a --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/SearchEngineResource.kt @@ -0,0 +1,43 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.web + +import com.ritense.document.opensearch.service.SearchEngineToggle +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PutMapping +import org.springframework.web.bind.annotation.RequestBody +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + +@RestController +@RequestMapping("/api/management/v1/search-engine") +class SearchEngineResource( + private val toggle: SearchEngineToggle, +) { + + @GetMapping + fun getActive(): ResponseEntity> = + ResponseEntity.ok(mapOf("active" to toggle.get().name)) + + @PutMapping + fun setActive(@RequestBody body: Map): ResponseEntity> { + val engine = SearchEngineToggle.Engine.valueOf(body["active"]!!.uppercase()) + toggle.set(engine) + return ResponseEntity.ok(mapOf("active" to toggle.get().name)) + } +} diff --git a/backend/case/src/main/java/com/ritense/document/service/impl/JsonSchemaDocumentSearchService.java b/backend/case/src/main/java/com/ritense/document/service/impl/JsonSchemaDocumentSearchService.java index 8f16e4f709..9a8dea4eca 100644 --- a/backend/case/src/main/java/com/ritense/document/service/impl/JsonSchemaDocumentSearchService.java +++ b/backend/case/src/main/java/com/ritense/document/service/impl/JsonSchemaDocumentSearchService.java @@ -371,6 +371,12 @@ private void buildQueryWhere( predicates.add(getCaseTagsFilterPredicate(cb, documentRoot, searchRequest.getCaseTagsFilter())); } + if (searchRequest.getGlobalSearchFilter() != null && !searchRequest.getGlobalSearchFilter().isBlank()) { + var pattern = "%" + searchRequest.getGlobalSearchFilter().trim().toLowerCase() + "%"; + var contentAsText = cast(documentRoot.get(CONTENT).get(CONTENT), String.class); + predicates.add(cb.like(cb.lower(contentAsText), pattern)); + } + query.where(predicates.toArray(Predicate[]::new)); } diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 95c8d9bd00..03f829852d 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -193,8 +193,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/access-control-management": { @@ -205,8 +205,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/account": { @@ -219,8 +219,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/analyse": { @@ -232,8 +232,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/bootstrap": { @@ -245,8 +245,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/building-block-management": { @@ -260,8 +260,8 @@ "vanilla-jsoneditor": "3.10.0" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/case": { @@ -280,8 +280,8 @@ "uuid": "11.1.0" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/case-management": { @@ -293,8 +293,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/case-migration": { @@ -306,8 +306,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/choice-field": { @@ -319,8 +319,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/components": { @@ -334,7 +334,6 @@ "@carbon/icons": "11.59.0", "@carbon/styles": "1.80.0", "@carbon/themes": "11.51.0", - "@floating-ui/dom": "1.7.4", "@formio/angular": "7.0.0", "@foxythemes/bootstrap-datetime-picker-bs4": "2.3.5", "@mdi/font": "7.4.47", @@ -365,20 +364,9 @@ "vanilla-jsoneditor": "3.10.0" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20", - "@angular/elements": "19.2.20" - } - }, - "dist/valtimo/components/node_modules/@floating-ui/dom": { - "version": "1.7.4", - "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.7.4.tgz", - "integrity": "sha512-OOchDgh4F2CchOX94cRVqhvy7b3AFb+/rQXyswmzmGakRfkMgoWVjfnLWkRirfLEfuD4ysVW16eXzwt3jHIzKA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@floating-ui/core": "^1.7.3", - "@floating-ui/utils": "^0.2.10" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8", + "@angular/elements": "^19.2.8" } }, "dist/valtimo/dashboard": { @@ -400,8 +388,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/dashboard-management": { @@ -413,8 +401,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/decision": { @@ -428,8 +416,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/document": { @@ -441,8 +429,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/form-flow-management": { @@ -453,8 +441,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/form-management": { @@ -466,8 +454,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/form-view-model": { @@ -481,8 +469,8 @@ "tslib": "^2.3.0" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/iko": { @@ -493,8 +481,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/keycloak": { @@ -509,8 +497,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/layout": { @@ -533,8 +521,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/logging": { @@ -546,8 +534,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/migration": { @@ -560,8 +548,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/milestone": { @@ -574,8 +562,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/object": { @@ -587,8 +575,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/object-management": { @@ -600,8 +588,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/plugin": { @@ -613,8 +601,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/plugin-management": { @@ -626,8 +614,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/process": { @@ -643,8 +631,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/process-link": { @@ -656,8 +644,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/process-management": { @@ -674,8 +662,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/resource": { @@ -687,8 +675,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/security": { @@ -702,8 +690,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/shared": { @@ -717,8 +705,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/sse": { @@ -730,8 +718,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/swagger": { @@ -745,8 +733,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/task": { @@ -760,8 +748,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/task-management": { @@ -774,8 +762,8 @@ "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "dist/valtimo/zgw": { @@ -784,14 +772,14 @@ "dev": true, "license": "EUPL-1.2", "dependencies": { - "@angular/forms": "19.2.20", + "@angular/forms": "^19.2.8", "@ngx-translate/core": "16.0.4", "carbon-components-angular": "5.57.6", "tslib": "2.8.1" }, "peerDependencies": { - "@angular/common": "19.2.20", - "@angular/core": "19.2.20" + "@angular/common": "^19.2.8", + "@angular/core": "^19.2.8" } }, "node_modules/@adobe/css-tools": { diff --git a/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.html b/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.html index b7a97c53ab..48cd3ea168 100644 --- a/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.html +++ b/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.html @@ -122,6 +122,16 @@
+ + | null = null; public loadingExport = false; + public searchEngine: 'OPENSEARCH' | 'POSTGRES' = 'OPENSEARCH'; + public searchDurationMs: number | null = null; + private searchStartTime = 0; public readonly defaultTabs = DEFAULT_CASE_LIST_TABS; public readonly tableTranslations = CASE_LIST_TABLE_TRANSLATIONS; @@ -425,6 +429,8 @@ export class CaseListComponent implements OnInit, OnDestroy { , globalSearchFilter, ]) => { + this.searchStartTime = performance.now(); + this.searchDurationMs = null; const obsApi: Observable = of(hasApiColumnConfig); const statusKeys: (string | null)[] = allStatuses.length === 1 @@ -486,6 +492,9 @@ export class CaseListComponent implements OnInit, OnDestroy { }); } ), + tap(() => { + this.searchDurationMs = Math.round(performance.now() - this.searchStartTime); + }), switchMap(res => combineLatest([ of(res), @@ -606,12 +615,32 @@ export class CaseListComponent implements OnInit, OnDestroy { private readonly caseListHiddenColumnsService: CaseListHiddenColumnsService, private readonly quickSearchStateService: QuickSearchStateService, @Inject(QUICK_SEARCH_SERVICE) - private readonly caseListQuickSearchService: IQuickSearchService + private readonly caseListQuickSearchService: IQuickSearchService, + private readonly http: HttpClient ) {} public ngOnInit(): void { this.setVisibleTabs(); this.openCaseDefinitionKeySubscription(); + this.loadSearchEngine(); + } + + public toggleSearchEngine(): void { + const next = this.searchEngine === 'OPENSEARCH' ? 'POSTGRES' : 'OPENSEARCH'; + this.http + .put<{active: string}>('/api/management/v1/search-engine', {active: next}) + .subscribe(res => { + this.searchEngine = res.active as 'OPENSEARCH' | 'POSTGRES'; + this.listService.forceRefresh(); + }); + } + + private loadSearchEngine(): void { + this.http + .get<{active: string}>('/api/management/v1/search-engine') + .subscribe(res => { + this.searchEngine = res.active as 'OPENSEARCH' | 'POSTGRES'; + }); } public ngOnDestroy(): void { From 3b39d666a1443d6c8b4143f0e0bdd0a295ee3add Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Fri, 24 Apr 2026 15:31:43 +0200 Subject: [PATCH 10/46] more poc --- .../DocumentOpenSearchAutoConfiguration.kt | 16 ++++++++++- .../JsonSchemaDocumentOpenSearchService.kt | 28 +++++++++++++++++-- .../impl/JsonSchemaDocumentSearchService.java | 24 ++++++++++++++-- 3 files changed, 63 insertions(+), 5 deletions(-) diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt index b593c6c61d..ecc1468788 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt @@ -190,7 +190,21 @@ class DocumentOpenSearchAutoConfiguration { val settings = org.springframework.data.elasticsearch.core.document.Document.create() settings["index.number_of_replicas"] = 0 indexOps.create(settings) - indexOps.putMapping(indexOps.createMapping(JsonSchemaDocumentOsDocument::class.java)) + + // Merge annotated mapping with a dynamic template that forces all + // content.* fields to text+keyword — enables wildcard search on numbers too + val annotatedMapping = indexOps.createMapping(JsonSchemaDocumentOsDocument::class.java) + val dynamicTemplates = listOf( + mapOf("content_fields_as_text" to mapOf( + "path_match" to "content.*", + "mapping" to mapOf( + "type" to "text", + "fields" to mapOf("keyword" to mapOf("type" to "keyword", "ignore_above" to 256)) + ) + )) + ) + annotatedMapping["dynamic_templates"] = dynamicTemplates + indexOps.putMapping(annotatedMapping) } } catch (e: Exception) { logger.warn(e) { "Failed to initialize OpenSearch index — is OpenSearch running?" } diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt index fc2d5d1b84..6488db5c65 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt @@ -54,6 +54,7 @@ import org.springframework.data.domain.Pageable import org.springframework.data.domain.Sort import org.springframework.data.elasticsearch.core.ElasticsearchOperations import org.springframework.data.elasticsearch.core.query.StringQuery +import org.apache.lucene.queryparser.classic.QueryParser import java.util.regex.Pattern class JsonSchemaDocumentOpenSearchService( @@ -220,8 +221,31 @@ class JsonSchemaDocumentOpenSearchService( val globalFilter = searchRequest.globalSearchFilter?.takeIf { it.isNotEmpty() } if (globalFilter != null) { - // Use wildcard on contentText.keyword for partial-match behaviour equivalent to MongoDB text index - parts.add(QueryBuilders.wildcardQuery("contentText.keyword", "*${globalFilter.trim()}*").caseInsensitive(true)) + val searchableFields = if (!documentDefinitionName.isNullOrEmpty()) { + runWithoutAuthorization { + searchFieldService.getSearchFields(documentDefinitionName) + } + .filter { it.path?.startsWith(DOC_PREFIX) == true } + .map { "content.${it.path.removePrefix(DOC_PREFIX)}" } + } else { + emptyList() + } + + if (searchableFields.isNotEmpty()) { + // query_string with lenient=true handles type mismatches gracefully + // (e.g. searching "doe" against a number field won't error, just won't match) + val escaped = QueryParser.escape(globalFilter.trim()) + parts.add( + QueryBuilders.queryStringQuery("*${escaped}*") + .apply { searchableFields.forEach { field(it) } } + .lenient(true) + .analyzeWildcard(true) + ) + } else { + // Fallback: search all content via contentText + val term = "*${globalFilter.trim()}*" + parts.add(QueryBuilders.wildcardQuery("contentText.keyword", term).caseInsensitive(true)) + } } return andAll(parts) diff --git a/backend/case/src/main/java/com/ritense/document/service/impl/JsonSchemaDocumentSearchService.java b/backend/case/src/main/java/com/ritense/document/service/impl/JsonSchemaDocumentSearchService.java index 9a8dea4eca..006f21bff6 100644 --- a/backend/case/src/main/java/com/ritense/document/service/impl/JsonSchemaDocumentSearchService.java +++ b/backend/case/src/main/java/com/ritense/document/service/impl/JsonSchemaDocumentSearchService.java @@ -373,8 +373,28 @@ private void buildQueryWhere( if (searchRequest.getGlobalSearchFilter() != null && !searchRequest.getGlobalSearchFilter().isBlank()) { var pattern = "%" + searchRequest.getGlobalSearchFilter().trim().toLowerCase() + "%"; - var contentAsText = cast(documentRoot.get(CONTENT).get(CONTENT), String.class); - predicates.add(cb.like(cb.lower(contentAsText), pattern)); + var searchableFields = !StringUtils.isEmpty(documentDefinitionName) + ? searchFieldService.getSearchFields(documentDefinitionName).stream() + .filter(f -> f.getPath() != null && f.getPath().startsWith(DOC_PREFIX)) + .toList() + : List.of(); + + if (!searchableFields.isEmpty()) { + var fieldPredicates = searchableFields.stream() + .map(f -> { + var jsonPath = "$." + f.getPath().substring(DOC_PREFIX.length()); + Expression expr = queryDialectHelper.getJsonValueExpression( + cb, documentRoot.get(CONTENT).get(CONTENT), jsonPath, String.class + ); + return cb.like(cb.lower(expr), pattern); + }) + .toArray(Predicate[]::new); + predicates.add(cb.or(fieldPredicates)); + } else { + // Fallback: search entire content as text + var contentAsText = cast(documentRoot.get(CONTENT).get(CONTENT), String.class); + predicates.add(cb.like(cb.lower(contentAsText), pattern)); + } } query.where(predicates.toArray(Predicate[]::new)); From 78c7f0051049c08fc9c2a2a13b7c5bb8724adea5 Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Mon, 29 Jun 2026 15:06:32 +0200 Subject: [PATCH 11/46] merge issue fixed --- .../autoconfigure/DocumentOpenSearchAutoConfiguration.kt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt index ecc1468788..85eb2f3f03 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt @@ -43,6 +43,7 @@ import com.ritense.document.service.SearchFieldService import com.ritense.document.service.impl.JsonSchemaDocumentSearchService import com.ritense.valtimo.contract.database.QueryDialectHelper import com.ritense.outbox.OutboxService +import com.ritense.valtimo.contract.authentication.TeamManagementService import com.ritense.valtimo.contract.authentication.UserManagementService import jakarta.persistence.EntityManager import org.springframework.boot.ApplicationRunner @@ -146,6 +147,7 @@ class DocumentOpenSearchAutoConfiguration { queryDialectHelper: QueryDialectHelper, searchFieldService: SearchFieldService, userManagementService: UserManagementService, + teamManagementService: TeamManagementService, authorizationService: AuthorizationService, outboxService: OutboxService, jsonSchemaDocumentDefinitionService: JsonSchemaDocumentDefinitionService, @@ -153,7 +155,7 @@ class DocumentOpenSearchAutoConfiguration { ): JsonSchemaDocumentSearchService = JsonSchemaDocumentSearchService( entityManager, queryDialectHelper, searchFieldService, - userManagementService, authorizationService, outboxService, + userManagementService, teamManagementService, authorizationService, outboxService, jsonSchemaDocumentDefinitionService, objectMapper, ) From 4b885943e68efa2e84c12e26237412f5d4ef1311 Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Mon, 29 Jun 2026 16:52:33 +0200 Subject: [PATCH 12/46] added global search to the case list --- .../case-list/case-list.component.html | 3 +- .../case-list/case-list.component.ts | 4 +++ .../case-list-orchestration.service.ts | 31 ++++++++++++++++--- .../lib/services/case-list-search.service.ts | 13 +++++++- 4 files changed, 44 insertions(+), 7 deletions(-) diff --git a/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.html b/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.html index 295b2364d6..69b10e3da6 100644 --- a/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.html +++ b/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.html @@ -96,15 +96,16 @@ [fields]="obs.fields" [header]="false" [initialSortState]="pagination.sort" + [isSearchable]="true" [items]="obs.documentItems" [pagination]="pagination" - lockedTooltipTranslationKey="case.rowLocked" [showSelectionColumn]="canHaveAssignee" [tableTranslations]="tableTranslations" (paginationClicked)="pageChange($event)" (paginationSet)="pageSizeChange($event)" (rowClicked)="rowClick($event)" + (search)="onGlobalSearchFilterChange($event)" (sortChanged)="sortChanged($event)" > diff --git a/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.ts b/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.ts index 28d7a22c02..64967475b9 100644 --- a/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.ts +++ b/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.ts @@ -151,6 +151,10 @@ export class CaseListComponent implements OnInit, OnDestroy { this.searchService.search(searchFieldValues); } + public onGlobalSearchFilterChange(value: string): void { + this.searchService.setGlobalSearchFilter(value); + } + // --- Row click --- public rowClick(item: any): void { diff --git a/frontend/projects/valtimo/case/src/lib/services/case-list-orchestration.service.ts b/frontend/projects/valtimo/case/src/lib/services/case-list-orchestration.service.ts index 72048ae4d4..1dcd37493d 100644 --- a/frontend/projects/valtimo/case/src/lib/services/case-list-orchestration.service.ts +++ b/frontend/projects/valtimo/case/src/lib/services/case-list-orchestration.service.ts @@ -114,6 +114,8 @@ export class CaseListOrchestrationService { public readonly searchFields$: Observable | null> = this.searchService.documentSearchFields$; + public readonly globalSearchFilter$: Observable = this.searchService.globalSearchFilter$; + public readonly statuses$: Observable> = this.statusService.caseStatuses$; @@ -330,6 +332,7 @@ export class CaseListOrchestrationService { this.hasApiColumnConfig$, this.statusService.caseStatuses$, this.caseListCaseTagService.caseTags$, + this.globalSearchFilter$, ]).pipe(debounceTime(50)) ), distinctUntilChanged(this.areDocumentRequestsEqual), @@ -343,6 +346,8 @@ export class CaseListOrchestrationService { _, hasApiColumnConfig, allStatuses, + __, + globalSearchFilter, ]) => this.fetchDocuments( documentSearchRequest, @@ -351,7 +356,8 @@ export class CaseListOrchestrationService { selectedStatuses, selectedCaseTagKeys, hasApiColumnConfig, - allStatuses + allStatuses, + globalSearchFilter ) ), switchMap(res => this.checkDocumentPermissions(res)), @@ -439,6 +445,10 @@ export class CaseListOrchestrationService { prevSelectedStatuses, prevCaseTagKeys, prevForceRefresh, + _prevHasApiColumnConfig, + _prevStatuses, + _prevCaseTags, + prevGlobalSearchFilter, ]: any[], [ currSearchRequest, @@ -447,6 +457,10 @@ export class CaseListOrchestrationService { currSelectedStatuses, currCaseTagKeys, currForceRefresh, + _currHasApiColumnConfig, + _currStatuses, + _currCaseTags, + currGlobalSearchFilter, ]: any[] ): boolean { return isEqual( @@ -457,6 +471,7 @@ export class CaseListOrchestrationService { ...prevSelectedStatuses, ...prevCaseTagKeys, forceRefresh: prevForceRefresh, + globalSearchFilter: prevGlobalSearchFilter, }, { ...currSearchRequest, @@ -465,6 +480,7 @@ export class CaseListOrchestrationService { ...currSelectedStatuses, ...currCaseTagKeys, forceRefresh: currForceRefresh, + globalSearchFilter: currGlobalSearchFilter, } ); } @@ -476,7 +492,8 @@ export class CaseListOrchestrationService { selectedStatuses: string[], selectedCaseTagKeys: string[], hasApiColumnConfig: boolean, - allStatuses: InternalCaseStatus[] + allStatuses: InternalCaseStatus[], + globalSearchFilter?: string ): Observable<{ documents: Documents | SpecifiedDocuments; hasApiColumnConfig: boolean; @@ -496,6 +513,8 @@ export class CaseListOrchestrationService { ? this.searchService.mapSearchValuesToFilters(searchValues) : undefined; + const globalFilter = globalSearchFilter?.trim() || undefined; + const documentsObs = !hasApiColumnConfig ? this.documentService.getDocumentsSearch( documentSearchRequest, @@ -503,7 +522,8 @@ export class CaseListOrchestrationService { assigneeFilter, searchFilters, statusKeys, - selectedCaseTagKeys + selectedCaseTagKeys, + globalFilter ) : this.documentService.getSpecifiedDocumentsSearch( documentSearchRequest, @@ -511,13 +531,14 @@ export class CaseListOrchestrationService { assigneeFilter, searchFilters, statusKeys, - selectedCaseTagKeys + selectedCaseTagKeys, + globalFilter ); return forkJoin({ documents: documentsObs, hasApiColumnConfig: of(hasApiColumnConfig), - isSearchResult: of(!!searchFilters), + isSearchResult: of(!!searchFilters || !!globalFilter), allStatuses: of(allStatuses), assigneeFilter: of(assigneeFilter), }); diff --git a/frontend/projects/valtimo/case/src/lib/services/case-list-search.service.ts b/frontend/projects/valtimo/case/src/lib/services/case-list-search.service.ts index 31130ae518..6db3e7e7f9 100644 --- a/frontend/projects/valtimo/case/src/lib/services/case-list-search.service.ts +++ b/frontend/projects/valtimo/case/src/lib/services/case-list-search.service.ts @@ -15,7 +15,7 @@ */ import {Injectable} from '@angular/core'; -import {Observable, of, switchMap} from 'rxjs'; +import {BehaviorSubject, Observable, of, switchMap} from 'rxjs'; import {SearchField, SearchFieldValues, SearchFilter, SearchFilterRange} from '@valtimo/shared'; import {CaseListService} from './case-list.service'; import {DocumentService} from '@valtimo/document'; @@ -32,16 +32,27 @@ export class CaseListSearchService { ) ); + private readonly _globalSearchFilter$ = new BehaviorSubject(''); + public get documentSearchFields$(): Observable | null> { return this._documentSearchFields$; } + public get globalSearchFilter$(): Observable { + return this._globalSearchFilter$.asObservable(); + } + constructor( private readonly caseListService: CaseListService, private readonly documentService: DocumentService, private readonly caseParameterService: CaseParameterService ) {} + public setGlobalSearchFilter(value: string | null): void { + this._globalSearchFilter$.next(value ?? ''); + this.caseListService.checkRefresh(); + } + public search(searchFieldValues: SearchFieldValues): void { this.caseParameterService.setSearchFieldValues(searchFieldValues || {}); this.caseParameterService.setSearchParameters(searchFieldValues); From f64874656714f5a3a303970af12a0d060d23c59d Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Tue, 30 Jun 2026 11:25:28 +0200 Subject: [PATCH 13/46] added toggle for search engine --- .../gzac/src/main/resources/application.yml | 2 + backend/case-opensearch/build.gradle | 1 + .../opensearch/OpenSearchProperties.kt | 24 ++++ .../DocumentOpenSearchAutoConfiguration.kt | 32 ++++- .../opensearch/web/SearchEngineResource.kt | 42 ++++++- .../service/SearchEngineToggleTest.kt | 56 +++++++++ .../web/SearchEngineResourceTest.kt | 110 ++++++++++++++++++ ...in-settings-feature-toggles.component.html | 26 +++++ ...in-settings-feature-toggles.component.scss | 4 + ...dmin-settings-feature-toggles.component.ts | 6 + .../admin-settings/src/lib/models/index.ts | 1 + .../src/lib/models/search-engine.model.ts | 24 ++++ .../admin-settings-management-api.service.ts | 22 +++- .../valtimo/shared/assets/core/en.json | 7 ++ .../valtimo/shared/assets/core/nl.json | 7 ++ 15 files changed, 355 insertions(+), 9 deletions(-) create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/OpenSearchProperties.kt create mode 100644 backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/SearchEngineToggleTest.kt create mode 100644 backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/web/SearchEngineResourceTest.kt create mode 100644 frontend/projects/valtimo/admin-settings/src/lib/models/search-engine.model.ts diff --git a/backend/app/gzac/src/main/resources/application.yml b/backend/app/gzac/src/main/resources/application.yml index efe6aaae9b..0f258fb4e7 100644 --- a/backend/app/gzac/src/main/resources/application.yml +++ b/backend/app/gzac/src/main/resources/application.yml @@ -174,6 +174,8 @@ mailing: sendRedirectedMailsTo: valtimo: + opensearch: + enabled: ${VALTIMO_OPENSEARCH_ENABLED:true} app: scheme: http hostname: ${VALTIMO_APP_HOSTNAME} diff --git a/backend/case-opensearch/build.gradle b/backend/case-opensearch/build.gradle index 73c44edac8..81e915ac27 100644 --- a/backend/case-opensearch/build.gradle +++ b/backend/case-opensearch/build.gradle @@ -26,6 +26,7 @@ dockerCompose { } dependencies { + implementation project(":backend:admin-settings") implementation project(":backend:authorization") implementation project(":backend:case") implementation project(":backend:inbox") diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/OpenSearchProperties.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/OpenSearchProperties.kt new file mode 100644 index 0000000000..4e0c85e0c9 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/OpenSearchProperties.kt @@ -0,0 +1,24 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch + +import org.springframework.boot.context.properties.ConfigurationProperties + +@ConfigurationProperties(prefix = "valtimo.opensearch") +data class OpenSearchProperties( + val enabled: Boolean = true +) diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt index 85eb2f3f03..87ff4a7a6a 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt @@ -18,7 +18,9 @@ package com.ritense.document.opensearch.autoconfigure import com.fasterxml.jackson.databind.ObjectMapper import io.github.oshai.kotlinlogging.KotlinLogging +import com.ritense.adminsettings.service.FeatureToggleOverridesService import com.ritense.authorization.AuthorizationService +import com.ritense.document.opensearch.OpenSearchProperties import com.ritense.document.autoconfigure.DocumentAutoConfiguration import com.ritense.document.opensearch.authorization.OpenSearchAuthorizationEntityMapper import com.ritense.document.opensearch.authorization.OpenSearchPermissionConditionTranslator @@ -51,6 +53,7 @@ import org.springframework.boot.autoconfigure.AutoConfiguration import org.springframework.boot.autoconfigure.AutoConfigureBefore import org.springframework.boot.autoconfigure.condition.ConditionalOnClass import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean +import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.annotation.Bean import org.springframework.core.annotation.Order import org.springframework.data.elasticsearch.core.ElasticsearchOperations @@ -60,6 +63,7 @@ import org.springframework.data.elasticsearch.repository.config.EnableElasticsea @AutoConfigureBefore(DocumentAutoConfiguration::class) @ConditionalOnClass(ElasticsearchOperations::class) @EnableElasticsearchRepositories(basePackages = ["com.ritense.document.opensearch.repository"]) +@EnableConfigurationProperties(OpenSearchProperties::class) class DocumentOpenSearchAutoConfiguration { @Bean @@ -170,8 +174,12 @@ class DocumentOpenSearchAutoConfiguration { @Bean @ConditionalOnMissingBean - fun searchEngineResource(toggle: SearchEngineToggle): SearchEngineResource = - SearchEngineResource(toggle) + fun searchEngineResource( + toggle: SearchEngineToggle, + openSearchProperties: OpenSearchProperties, + featureToggleOverridesService: FeatureToggleOverridesService, + ): SearchEngineResource = + SearchEngineResource(toggle, openSearchProperties, featureToggleOverridesService) @Bean @ConditionalOnMissingBean @@ -213,7 +221,27 @@ class DocumentOpenSearchAutoConfiguration { } } + @Bean + fun searchEngineSettingLoader( + toggle: SearchEngineToggle, + featureToggleOverridesService: FeatureToggleOverridesService, + openSearchProperties: OpenSearchProperties, + ): ApplicationRunner = ApplicationRunner { + if (!openSearchProperties.enabled) { + toggle.set(SearchEngineToggle.Engine.POSTGRES) + logger.info { "OpenSearch disabled via configuration; using PostgreSQL for document search" } + return@ApplicationRunner + } + + val overrides = featureToggleOverridesService.getOverrides().overrides + val useOpenSearch = overrides[SEARCH_ENGINE_TOGGLE_KEY] ?: true + val engine = if (useOpenSearch) SearchEngineToggle.Engine.OPENSEARCH else SearchEngineToggle.Engine.POSTGRES + toggle.set(engine) + logger.info { "Document search engine set to: ${engine.name}" } + } + companion object { private val logger = KotlinLogging.logger {} + const val SEARCH_ENGINE_TOGGLE_KEY = "useOpenSearchForDocumentSearch" } } diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/SearchEngineResource.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/SearchEngineResource.kt index 2efbd2a49a..d0d0a97639 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/SearchEngineResource.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/SearchEngineResource.kt @@ -16,6 +16,9 @@ package com.ritense.document.opensearch.web +import com.ritense.adminsettings.service.FeatureToggleOverridesService +import com.ritense.document.opensearch.OpenSearchProperties +import com.ritense.document.opensearch.autoconfigure.DocumentOpenSearchAutoConfiguration.Companion.SEARCH_ENGINE_TOGGLE_KEY import com.ritense.document.opensearch.service.SearchEngineToggle import org.springframework.http.ResponseEntity import org.springframework.web.bind.annotation.GetMapping @@ -28,16 +31,45 @@ import org.springframework.web.bind.annotation.RestController @RequestMapping("/api/management/v1/search-engine") class SearchEngineResource( private val toggle: SearchEngineToggle, + private val openSearchProperties: OpenSearchProperties, + private val featureToggleOverridesService: FeatureToggleOverridesService, ) { @GetMapping - fun getActive(): ResponseEntity> = - ResponseEntity.ok(mapOf("active" to toggle.get().name)) + fun getActive(): ResponseEntity = + ResponseEntity.ok( + SearchEngineDto( + available = openSearchProperties.enabled, + active = toggle.get().name + ) + ) @PutMapping - fun setActive(@RequestBody body: Map): ResponseEntity> { - val engine = SearchEngineToggle.Engine.valueOf(body["active"]!!.uppercase()) + fun setActive(@RequestBody body: UpdateSearchEngineDto): ResponseEntity { + if (!openSearchProperties.enabled) { + return ResponseEntity.badRequest().build() + } + + val useOpenSearch = body.active.uppercase() == "OPENSEARCH" + featureToggleOverridesService.updateToggle(SEARCH_ENGINE_TOGGLE_KEY, useOpenSearch) + + val engine = if (useOpenSearch) SearchEngineToggle.Engine.OPENSEARCH else SearchEngineToggle.Engine.POSTGRES toggle.set(engine) - return ResponseEntity.ok(mapOf("active" to toggle.get().name)) + + return ResponseEntity.ok( + SearchEngineDto( + available = true, + active = toggle.get().name + ) + ) } + + data class SearchEngineDto( + val available: Boolean, + val active: String + ) + + data class UpdateSearchEngineDto( + val active: String + ) } diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/SearchEngineToggleTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/SearchEngineToggleTest.kt new file mode 100644 index 0000000000..028641bab9 --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/SearchEngineToggleTest.kt @@ -0,0 +1,56 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test + +class SearchEngineToggleTest { + + @Test + fun `default engine is OPENSEARCH`() { + val toggle = SearchEngineToggle() + + assertThat(toggle.get()).isEqualTo(SearchEngineToggle.Engine.OPENSEARCH) + } + + @Test + fun `can override default engine`() { + val toggle = SearchEngineToggle(default = SearchEngineToggle.Engine.POSTGRES) + + assertThat(toggle.get()).isEqualTo(SearchEngineToggle.Engine.POSTGRES) + } + + @Test + fun `set changes engine`() { + val toggle = SearchEngineToggle() + + toggle.set(SearchEngineToggle.Engine.POSTGRES) + + assertThat(toggle.get()).isEqualTo(SearchEngineToggle.Engine.POSTGRES) + } + + @Test + fun `can toggle back to OPENSEARCH`() { + val toggle = SearchEngineToggle() + toggle.set(SearchEngineToggle.Engine.POSTGRES) + + toggle.set(SearchEngineToggle.Engine.OPENSEARCH) + + assertThat(toggle.get()).isEqualTo(SearchEngineToggle.Engine.OPENSEARCH) + } +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/web/SearchEngineResourceTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/web/SearchEngineResourceTest.kt new file mode 100644 index 0000000000..e79c8ec616 --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/web/SearchEngineResourceTest.kt @@ -0,0 +1,110 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.web + +import com.ritense.adminsettings.service.FeatureToggleOverridesService +import com.ritense.adminsettings.web.rest.dto.FeatureToggleOverridesDto +import com.ritense.document.opensearch.OpenSearchProperties +import com.ritense.document.opensearch.service.SearchEngineToggle +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.eq +import org.mockito.kotlin.mock +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever +import org.springframework.http.HttpStatus + +class SearchEngineResourceTest { + + private lateinit var toggle: SearchEngineToggle + private lateinit var properties: OpenSearchProperties + private lateinit var featureToggleService: FeatureToggleOverridesService + private lateinit var resource: SearchEngineResource + + @BeforeEach + fun setUp() { + toggle = SearchEngineToggle() + properties = OpenSearchProperties(enabled = true) + featureToggleService = mock() + resource = SearchEngineResource(toggle, properties, featureToggleService) + } + + @Test + fun `getActive returns available true and current engine when OpenSearch enabled`() { + toggle.set(SearchEngineToggle.Engine.OPENSEARCH) + + val response = resource.getActive() + + assertThat(response.statusCode).isEqualTo(HttpStatus.OK) + assertThat(response.body?.available).isTrue() + assertThat(response.body?.active).isEqualTo("OPENSEARCH") + } + + @Test + fun `getActive returns available false when OpenSearch disabled`() { + val disabledResource = SearchEngineResource( + toggle, + OpenSearchProperties(enabled = false), + featureToggleService + ) + + val response = disabledResource.getActive() + + assertThat(response.body?.available).isFalse() + } + + @Test + fun `setActive updates toggle and persists to feature toggles`() { + whenever(featureToggleService.updateToggle(any(), any())) + .thenReturn(FeatureToggleOverridesDto(mapOf("useOpenSearchForDocumentSearch" to false))) + + val response = resource.setActive(SearchEngineResource.UpdateSearchEngineDto("POSTGRES")) + + assertThat(response.statusCode).isEqualTo(HttpStatus.OK) + assertThat(response.body?.active).isEqualTo("POSTGRES") + assertThat(toggle.get()).isEqualTo(SearchEngineToggle.Engine.POSTGRES) + verify(featureToggleService).updateToggle(eq("useOpenSearchForDocumentSearch"), eq(false)) + } + + @Test + fun `setActive to OPENSEARCH persists true`() { + toggle.set(SearchEngineToggle.Engine.POSTGRES) + whenever(featureToggleService.updateToggle(any(), any())) + .thenReturn(FeatureToggleOverridesDto(mapOf("useOpenSearchForDocumentSearch" to true))) + + val response = resource.setActive(SearchEngineResource.UpdateSearchEngineDto("OPENSEARCH")) + + assertThat(response.body?.active).isEqualTo("OPENSEARCH") + assertThat(toggle.get()).isEqualTo(SearchEngineToggle.Engine.OPENSEARCH) + verify(featureToggleService).updateToggle(eq("useOpenSearchForDocumentSearch"), eq(true)) + } + + @Test + fun `setActive returns bad request when OpenSearch disabled`() { + val disabledResource = SearchEngineResource( + toggle, + OpenSearchProperties(enabled = false), + featureToggleService + ) + + val response = disabledResource.setActive(SearchEngineResource.UpdateSearchEngineDto("OPENSEARCH")) + + assertThat(response.statusCode).isEqualTo(HttpStatus.BAD_REQUEST) + } +} diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-feature-toggles/admin-settings-feature-toggles.component.html b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-feature-toggles/admin-settings-feature-toggles.component.html index 291cd51995..e9f2dc56e0 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-feature-toggles/admin-settings-feature-toggles.component.html +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-feature-toggles/admin-settings-feature-toggles.component.html @@ -17,6 +17,32 @@ @if (featureToggles$ | async; as toggles) {
+ @if (searchEngine$ | async; as searchEngine) { +
+
+ + {{ 'adminSettings.featureToggles.searchEngine.useOpenSearch.title' | translate }} + + + @if (searchEngine.available) { + {{ 'adminSettings.featureToggles.searchEngine.useOpenSearch.description' | translate }} + } @else { + {{ 'adminSettings.featureToggles.searchEngine.useOpenSearch.unavailable' | translate }} + } + +
+
+ +
+
+ } @for (definition of TOGGLE_DEFINITIONS; track definition.key) {
diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-feature-toggles/admin-settings-feature-toggles.component.scss b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-feature-toggles/admin-settings-feature-toggles.component.scss index da7c633cbc..7177984d9e 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-feature-toggles/admin-settings-feature-toggles.component.scss +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-feature-toggles/admin-settings-feature-toggles.component.scss @@ -33,6 +33,10 @@ &:last-child { border-bottom: none; } + + &--disabled { + opacity: 0.5; + } } &__info { diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-feature-toggles/admin-settings-feature-toggles.component.ts b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-feature-toggles/admin-settings-feature-toggles.component.ts index 49c4a0df19..281e33f08c 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-feature-toggles/admin-settings-feature-toggles.component.ts +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-feature-toggles/admin-settings-feature-toggles.component.ts @@ -41,6 +41,8 @@ export class AdminSettingsFeatureTogglesComponent { public readonly featureToggles$ = this._configService.featureToggles$; + public readonly searchEngine$ = this._adminSettingsManagementApiService.getSearchEngine(); + constructor( private readonly _adminSettingsManagementApiService: AdminSettingsManagementApiService, private readonly _adminSettingsService: AdminSettingsService, @@ -61,4 +63,8 @@ export class AdminSettingsFeatureTogglesComponent { this._adminSettingsService.refreshFeatureToggles(); }); } + + public onSearchEngineChange(useOpenSearch: boolean): void { + this._adminSettingsManagementApiService.updateSearchEngine(useOpenSearch).subscribe(); + } } diff --git a/frontend/projects/valtimo/admin-settings/src/lib/models/index.ts b/frontend/projects/valtimo/admin-settings/src/lib/models/index.ts index af992acfb5..f6007b767f 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/models/index.ts +++ b/frontend/projects/valtimo/admin-settings/src/lib/models/index.ts @@ -16,3 +16,4 @@ export * from './accent-colors.model'; export * from './feature-toggle.model'; +export * from './search-engine.model'; diff --git a/frontend/projects/valtimo/admin-settings/src/lib/models/search-engine.model.ts b/frontend/projects/valtimo/admin-settings/src/lib/models/search-engine.model.ts new file mode 100644 index 0000000000..c0e6b47c24 --- /dev/null +++ b/frontend/projects/valtimo/admin-settings/src/lib/models/search-engine.model.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface SearchEngineDto { + available: boolean; + active: 'OPENSEARCH' | 'POSTGRES'; +} + +export interface UpdateSearchEngineDto { + active: 'OPENSEARCH' | 'POSTGRES'; +} diff --git a/frontend/projects/valtimo/admin-settings/src/lib/services/admin-settings-management-api.service.ts b/frontend/projects/valtimo/admin-settings/src/lib/services/admin-settings-management-api.service.ts index ee6c45e0ac..0fd7f3896f 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/services/admin-settings-management-api.service.ts +++ b/frontend/projects/valtimo/admin-settings/src/lib/services/admin-settings-management-api.service.ts @@ -16,7 +16,7 @@ import {Injectable} from '@angular/core'; import {HttpClient} from '@angular/common/http'; -import {Observable} from 'rxjs'; +import {catchError, Observable, of} from 'rxjs'; import { AdminSettingsLogoDto, AdminSettingsLogosDto, @@ -24,7 +24,12 @@ import { ConfigService, CreateAdminSettingsLogoDto, } from '@valtimo/shared'; -import {AccentColorsDto, FeatureToggleOverridesDto, UpdateFeatureToggleDto} from '../models'; +import { + AccentColorsDto, + FeatureToggleOverridesDto, + SearchEngineDto, + UpdateFeatureToggleDto, +} from '../models'; @Injectable({ providedIn: 'root', @@ -90,4 +95,17 @@ export class AdminSettingsManagementApiService extends BaseApiService { dto ); } + + public getSearchEngine(): Observable { + return this.httpClient + .get(this.getApiUrl('/management/v1/search-engine')) + .pipe(catchError(() => of(null))); + } + + public updateSearchEngine(useOpenSearch: boolean): Observable { + return this.httpClient.put( + this.getApiUrl('/management/v1/search-engine'), + {active: useOpenSearch ? 'OPENSEARCH' : 'POSTGRES'} + ); + } } diff --git a/frontend/projects/valtimo/shared/assets/core/en.json b/frontend/projects/valtimo/shared/assets/core/en.json index 58dbddcb9f..115c5dc14e 100644 --- a/frontend/projects/valtimo/shared/assets/core/en.json +++ b/frontend/projects/valtimo/shared/assets/core/en.json @@ -3100,6 +3100,13 @@ } }, "featureToggles": { + "searchEngine": { + "useOpenSearch": { + "title": "Use OpenSearch for case search", + "description": "When enabled, case searches use OpenSearch for better performance. When disabled, PostgreSQL is used.", + "unavailable": "OpenSearch is not configured for this installation." + } + }, "refreshRequired": "Refresh required", "refreshModalText": "This setting requires a page refresh to take effect. Do you want to refresh now?", "refreshNow": "Refresh now", diff --git a/frontend/projects/valtimo/shared/assets/core/nl.json b/frontend/projects/valtimo/shared/assets/core/nl.json index 36b410f36e..e4bcac2eb2 100644 --- a/frontend/projects/valtimo/shared/assets/core/nl.json +++ b/frontend/projects/valtimo/shared/assets/core/nl.json @@ -3133,6 +3133,13 @@ } }, "featureToggles": { + "searchEngine": { + "useOpenSearch": { + "title": "Gebruik OpenSearch voor zaakzoekopdrachten", + "description": "Indien ingeschakeld worden zaakzoekopdrachten uitgevoerd met OpenSearch voor betere prestaties. Indien uitgeschakeld wordt PostgreSQL gebruikt.", + "unavailable": "OpenSearch is niet geconfigureerd voor deze installatie." + } + }, "refreshRequired": "Vernieuwing vereist", "refreshModalText": "Deze instelling vereist een paginavernieuwing om effect te hebben. Wilt u nu vernieuwen?", "refreshNow": "Nu vernieuwen", From 66d362f4bd6f8b97ed753a94504ccfb8f695ca54 Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Tue, 30 Jun 2026 17:41:09 +0200 Subject: [PATCH 14/46] removed mongodb module swapped over to spring events for keeping opensearch in sync --- backend/case-mongodb/build.gradle | 67 ---- .../docker-compose-override-postgresql.yml | 10 - backend/case-mongodb/gradle/publishing.gradle | 35 -- .../MongoAuthorizationEntityMapper.kt | 41 -- .../MongoPermissionConditionTranslator.kt | 191 --------- ...SchemaDocumentCaseDefinitionMongoMapper.kt | 74 ---- ...JsonSchemaDocumentDefinitionMongoMapper.kt | 71 ---- .../DocumentMongoAutoConfiguration.kt | 217 ----------- .../converter/JsonNodeMongoConverters.kt | 58 --- .../domain/JsonSchemaDocumentDocument.kt | 85 ---- .../handler/DocumentMongoEventHandler.kt | 70 ---- .../JsonSchemaDocumentMongoRepository.kt | 22 -- .../DocumentMongoHttpSecurityConfigurer.kt | 38 -- .../mongodb/service/ContentTextExtractor.kt | 39 -- .../service/DocumentMongoBackfillService.kt | 72 ---- .../service/DocumentMongoQueryService.kt | 71 ---- .../service/DocumentMongoSyncService.kt | 49 --- .../JsonSchemaDocumentMongoSearchService.kt | 364 ------------------ .../web/DocumentMongoBackfillResource.kt | 45 --- ...ot.autoconfigure.AutoConfiguration.imports | 1 - .../mongodb/BaseMongoIntegrationTest.kt | 149 ------- .../document/mongodb/TestApplication.kt | 22 -- .../service/ContentTextExtractorTest.kt | 107 ----- .../service/DocumentMongoSyncServiceTest.kt | 144 ------- ...SchemaDocumentMongoSearchServiceIntTest.kt | 144 ------- ...sonSchemaDocumentMongoSearchServiceTest.kt | 178 --------- .../config/application-postgresql.yml | 17 - .../src/test/resources/config/application.yml | 35 -- .../definition/house.case-definition.json | 7 - .../house.internal-case-status.json | 20 - .../1-0-0/case/list/house.case-list.json | 1 - .../search-field/house.case-search-field.json | 18 - .../house.schema.document-definition.json | 33 -- .../DocumentOpenSearchAutoConfiguration.kt | 6 +- .../handler/DocumentOpenSearchEventHandler.kt | 70 ++-- .../service/DocumentOpenSearchSyncService.kt | 11 +- .../DocumentOpenSearchSyncServiceTest.kt | 36 +- .../valtimo-dependency-versions/build.gradle | 1 - settings.gradle | 1 - 39 files changed, 60 insertions(+), 2560 deletions(-) delete mode 100644 backend/case-mongodb/build.gradle delete mode 100644 backend/case-mongodb/docker-compose-override-postgresql.yml delete mode 100644 backend/case-mongodb/gradle/publishing.gradle delete mode 100644 backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/authorization/MongoAuthorizationEntityMapper.kt delete mode 100644 backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/authorization/MongoPermissionConditionTranslator.kt delete mode 100644 backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/authorization/mapper/JsonSchemaDocumentCaseDefinitionMongoMapper.kt delete mode 100644 backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/authorization/mapper/JsonSchemaDocumentDefinitionMongoMapper.kt delete mode 100644 backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/autoconfigure/DocumentMongoAutoConfiguration.kt delete mode 100644 backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/converter/JsonNodeMongoConverters.kt delete mode 100644 backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/domain/JsonSchemaDocumentDocument.kt delete mode 100644 backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/handler/DocumentMongoEventHandler.kt delete mode 100644 backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/repository/JsonSchemaDocumentMongoRepository.kt delete mode 100644 backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/security/DocumentMongoHttpSecurityConfigurer.kt delete mode 100644 backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/ContentTextExtractor.kt delete mode 100644 backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/DocumentMongoBackfillService.kt delete mode 100644 backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/DocumentMongoQueryService.kt delete mode 100644 backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/DocumentMongoSyncService.kt delete mode 100644 backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/JsonSchemaDocumentMongoSearchService.kt delete mode 100644 backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/web/DocumentMongoBackfillResource.kt delete mode 100644 backend/case-mongodb/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports delete mode 100644 backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/BaseMongoIntegrationTest.kt delete mode 100644 backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/TestApplication.kt delete mode 100644 backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/service/ContentTextExtractorTest.kt delete mode 100644 backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/service/DocumentMongoSyncServiceTest.kt delete mode 100644 backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/service/JsonSchemaDocumentMongoSearchServiceIntTest.kt delete mode 100644 backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/service/JsonSchemaDocumentMongoSearchServiceTest.kt delete mode 100644 backend/case-mongodb/src/test/resources/config/application-postgresql.yml delete mode 100644 backend/case-mongodb/src/test/resources/config/application.yml delete mode 100644 backend/case-mongodb/src/test/resources/config/case/house/1-0-0/case/definition/house.case-definition.json delete mode 100644 backend/case-mongodb/src/test/resources/config/case/house/1-0-0/case/internal-status/house.internal-case-status.json delete mode 100644 backend/case-mongodb/src/test/resources/config/case/house/1-0-0/case/list/house.case-list.json delete mode 100644 backend/case-mongodb/src/test/resources/config/case/house/1-0-0/case/search-field/house.case-search-field.json delete mode 100644 backend/case-mongodb/src/test/resources/config/case/house/1-0-0/document/definition/house.schema.document-definition.json diff --git a/backend/case-mongodb/build.gradle b/backend/case-mongodb/build.gradle deleted file mode 100644 index 93bfff18a9..0000000000 --- a/backend/case-mongodb/build.gradle +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2015-2024 Ritense BV, the Netherlands. - * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -dockerCompose { - projectName = "case-mongodb" - integrationTestingPostgresql { - isRequiredBy(project.tasks.integrationTestingPostgresql) - useComposeFiles.addAll( - "../docker-resources/docker-compose-base-test-postgresql.yml", - "docker-compose-override-postgresql.yml" - ) - } -} - -dependencies { - implementation project(":backend:authorization") - implementation project(":backend:case") - implementation project(":backend:inbox") - implementation project(":backend:outbox") - - implementation "org.springframework.boot:spring-boot-starter-data-jpa" - implementation "org.springframework.boot:spring-boot-starter-data-mongodb" - implementation "org.springframework.boot:spring-boot-starter-web" - implementation "org.springframework.boot:spring-boot-starter-security" - implementation "org.springframework.boot:spring-boot-autoconfigure" - implementation "com.fasterxml.jackson.module:jackson-module-kotlin" - implementation "io.github.oshai:kotlin-logging:${kotlinLoggingVersion}" - - annotationProcessor "org.springframework.boot:spring-boot-autoconfigure-processor" - - testImplementation project(':backend:test-utils-common') - testImplementation project(':backend:core') - testImplementation(project(':backend:audit')) { - exclude(group: "com.ritense.valtimo", module: "case") - } - testImplementation "org.springframework.boot:spring-boot-starter-test" - testImplementation "org.mockito.kotlin:mockito-kotlin:${mockitoKotlinVersion}" - testImplementation "org.postgresql:postgresql" - testImplementation "org.springframework.security:spring-security-test" - - jar { - enabled = true - manifest { - attributes("Implementation-Title": "Ritense Case MongoDB module") - attributes("Implementation-Version": projectVersion) - } - } -} - -tasks.named("integrationTestingPostgresql") { - systemProperty("liquibase.duplicateFileMode", "WARN") -} - -apply from: "gradle/publishing.gradle" diff --git a/backend/case-mongodb/docker-compose-override-postgresql.yml b/backend/case-mongodb/docker-compose-override-postgresql.yml deleted file mode 100644 index 0b8c7ef396..0000000000 --- a/backend/case-mongodb/docker-compose-override-postgresql.yml +++ /dev/null @@ -1,10 +0,0 @@ -services: - db: - ports: - - "3364:5432" - environment: - - POSTGRES_DB=case-mongodb-test - mongodb: - image: mongo:8.2.6 - ports: - - "37017:27017" diff --git a/backend/case-mongodb/gradle/publishing.gradle b/backend/case-mongodb/gradle/publishing.gradle deleted file mode 100644 index ce0f836cc5..0000000000 --- a/backend/case-mongodb/gradle/publishing.gradle +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2015-2024 Ritense BV, the Netherlands. - * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -pluginManager.withPlugin('maven-publish') { - publishing { - publications { - maven(MavenPublication) { - pom { - name = 'Case MongoDB module' - description = 'The case-mongodb module syncs json_schema_document to MongoDB as a CQRS read model' - developers { - developer { - id = "team-valtimo" - name = "Team Valtimo" - email = "team-valtimo@ritense.com" - } - } - } - } - } - } -} diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/authorization/MongoAuthorizationEntityMapper.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/authorization/MongoAuthorizationEntityMapper.kt deleted file mode 100644 index a0f4fd255c..0000000000 --- a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/authorization/MongoAuthorizationEntityMapper.kt +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2015-2024 Ritense BV, the Netherlands. - * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.ritense.document.mongodb.authorization - -import com.ritense.authorization.permission.condition.PermissionCondition -import org.springframework.data.mongodb.core.query.Criteria - -/** - * MongoDB equivalent of [com.ritense.authorization.AuthorizationEntityMapper]. - * - * Translates a [com.ritense.authorization.permission.condition.ContainerPermissionCondition] - * on entity type [TO] into a MongoDB [Criteria] that filters [FROM] documents. - * - * Implement this interface and register the implementation as a Spring bean to add support - * for a new container relationship without modifying the core translator. - */ -interface MongoAuthorizationEntityMapper { - - /** - * Given conditions on the [TO] entity type, returns a MongoDB [Criteria] that filters - * [FROM] documents satisfying those conditions, or `null` if no filter is needed - * (i.e. any [FROM] document qualifies regardless of [conditions]). - */ - fun mapCriteria(conditions: List): Criteria? - - fun supports(fromClass: Class<*>, toClass: Class<*>): Boolean -} diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/authorization/MongoPermissionConditionTranslator.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/authorization/MongoPermissionConditionTranslator.kt deleted file mode 100644 index 6a91f77bd9..0000000000 --- a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/authorization/MongoPermissionConditionTranslator.kt +++ /dev/null @@ -1,191 +0,0 @@ -/* - * Copyright 2015-2024 Ritense BV, the Netherlands. - * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.ritense.document.mongodb.authorization - -import com.ritense.authorization.Action -import com.ritense.authorization.AuthorizationContext.Companion.runWithoutAuthorization -import com.ritense.authorization.AuthorizationService -import com.ritense.authorization.permission.ConditionContainer -import com.ritense.authorization.permission.Permission -import com.ritense.authorization.permission.condition.ContainerPermissionCondition -import com.ritense.authorization.permission.condition.ExpressionPermissionCondition -import com.ritense.authorization.permission.condition.FieldPermissionCondition -import com.ritense.authorization.permission.condition.PermissionCondition -import com.ritense.authorization.permission.condition.PermissionConditionOperator -import com.ritense.authorization.request.EntityAuthorizationRequest -import com.ritense.authorization.role.Role -import com.ritense.document.domain.impl.JsonSchemaDocument -import com.ritense.document.repository.impl.JsonSchemaDocumentRepository -import com.ritense.valtimo.contract.authorization.CurrentUserExpressionHandler -import io.github.oshai.kotlinlogging.KotlinLogging -import org.springframework.data.mongodb.core.query.Criteria - -class MongoPermissionConditionTranslator( - private val mongoMappers: List>, - private val authorizationService: AuthorizationService, - private val documentRepository: JsonSchemaDocumentRepository, -) { - - /** - * Translates a list of [Permission]s into a single MongoDB [Criteria] that, when applied - * to a query, returns only the documents the current user is allowed to see for [action]. - * - * Permissions are OR-ed; conditions within a permission are AND-ed. - * Returns a deny-all criteria if no permissions match [action]. - */ - fun toCriteria(permissions: List, action: Action<*>): Criteria { - val matching = permissions.filter { - it.resourceType == JsonSchemaDocument::class.java && it.actions.contains(action) - } - logger.debug { "toCriteria: ${permissions.size} permissions total, ${matching.size} matching action=$action" } - if (matching.isEmpty()) { - return denyAll() - } - - val perPermissionCriteria = matching.map { permission -> - val conditionCriteria = permission.conditionContainer.conditions.map { translateCondition(it) } - andAll(conditionCriteria) - } - val result = if (perPermissionCriteria.size == 1) { - perPermissionCriteria.first() - } else { - Criteria().orOperator(*perPermissionCriteria.toTypedArray()) - } - logger.debug { "toCriteria: generated criteria = ${result.criteriaObject}" } - return result - } - - private fun translateCondition(condition: PermissionCondition): Criteria = when (condition) { - is FieldPermissionCondition<*> -> translateField(condition) - is ExpressionPermissionCondition<*> -> translateExpression(condition) - is ContainerPermissionCondition<*> -> translateContainer(condition) - else -> throw IllegalArgumentException("Unknown permission condition type: ${condition::class.qualifiedName}") - } - - private fun translateField(cond: FieldPermissionCondition<*>): Criteria { - val mongoField = jpaToMongoField(cond.field) - val value = resolveFieldValue(cond) - return Companion.applyOperator(Criteria.where(mongoField), cond.operator, value) - } - - private fun translateExpression(cond: ExpressionPermissionCondition<*>): Criteria { - // Convert JSONPath "$.department.id" to MongoDB dot notation: "content.department.id" - val dotPath = cond.path.removePrefix("$.").replace("/", ".") - val mongoField = "${jpaToMongoField(cond.field)}.$dotPath" - val value = CurrentUserExpressionHandler.resolveValue(cond.value) - logger.debug { "translateExpression: field=${cond.field} → mongoField=$mongoField, op=${cond.operator}, value=$value (${value?.javaClass?.simpleName})" } - return Companion.applyOperator(Criteria.where(mongoField), cond.operator, value) - } - - @Suppress("UNCHECKED_CAST") - private fun translateContainer(cond: ContainerPermissionCondition<*>): Criteria { - val mongoMapper = mongoMappers.find { - it.supports(JsonSchemaDocument::class.java, cond.resourceType) - } as? MongoAuthorizationEntityMapper - - if (mongoMapper != null) { - return mongoMapper.mapCriteria(cond.conditions) ?: noFilter() - } - - logger.warn { - "No MongoAuthorizationEntityMapper registered for " + - "JsonSchemaDocument → ${cond.resourceType.simpleName}. " + - "Falling back to JPA ID resolution — may be slow for large datasets." - } - return jpaFallback(cond) - } - - /** - * Fallback for [ContainerPermissionCondition] types that have no registered - * [MongoAuthorizationEntityMapper]. Uses JPA to find matching document IDs and - * returns an [Criteria.where] `_id` `in` filter. - * - * This is correct but potentially expensive for large datasets. Register a - * [MongoAuthorizationEntityMapper] to replace this with a native MongoDB query. - */ - private fun jpaFallback(cond: ContainerPermissionCondition<*>): Criteria { - val syntheticPermission = Permission( - resourceType = JsonSchemaDocument::class.java, - actions = mutableListOf(Action(Action.IGNORE)), - conditionContainer = ConditionContainer(listOf(cond)), - role = Role(key = ""), - ) - val spec = authorizationService.getAuthorizationSpecification( - EntityAuthorizationRequest(JsonSchemaDocument::class.java, Action(Action.IGNORE)), - listOf(syntheticPermission) - ) - val allowedIds: List = runWithoutAuthorization { - documentRepository.findAll(spec).map { doc -> doc.id().toString() } - } - return Criteria.where("_id").`in`(allowedIds) - } - - - private fun resolveFieldValue(cond: FieldPermissionCondition<*>): Any? = - if (cond.value is List<*>) { - (cond.value as List<*>).map { CurrentUserExpressionHandler.resolveValue(it) } - } else { - CurrentUserExpressionHandler.resolveValue(cond.value) - } - - companion object { - private val logger = KotlinLogging.logger {} - - fun applyOperator(criteria: Criteria, op: PermissionConditionOperator, value: Any?): Criteria = - when (op) { - PermissionConditionOperator.EQUAL_TO -> criteria.`is`(value) - PermissionConditionOperator.NOT_EQUAL_TO -> if (value == null) criteria.ne(null) else criteria.ne(value) - PermissionConditionOperator.GREATER_THAN -> criteria.gt(value!!) - PermissionConditionOperator.GREATER_THAN_OR_EQUAL_TO -> criteria.gte(value!!) - PermissionConditionOperator.LESS_THAN -> criteria.lt(value!!) - PermissionConditionOperator.LESS_THAN_OR_EQUAL_TO -> criteria.lte(value!!) - PermissionConditionOperator.LIST_CONTAINS -> criteria.`in`(value) - PermissionConditionOperator.IN -> { - val collection = value as? Collection<*> - ?: throw IllegalArgumentException("IN operator requires a Collection value") - criteria.`in`(collection) - } - } - - /** - * Maps JPA entity field names (as used in [FieldPermissionCondition.field]) to - * their corresponding field names in the MongoDB document. - * Extend this map as new permission conditions are introduced. - */ - val fieldMappings: Map = mapOf( - "createdBy" to "createdBy", - "assigneeId" to "assigneeId", - "assigneeFullName" to "assigneeFullName", - "content" to "content", - // JPA: DocumentContent wraps the JSON via @JsonValue, so the inner - // "content.content" path in JPA resolves to the flat "content" in MongoDB. - "content.content" to "content", - "sequence" to "sequence", - "retentionDate" to "retentionDate", - ) - - fun jpaToMongoField(jpaField: String): String = fieldMappings[jpaField] ?: jpaField - - fun denyAll(): Criteria = Criteria.where("_id").`is`(null) - fun noFilter(): Criteria = Criteria() - fun andAll(list: List): Criteria = when { - list.isEmpty() -> noFilter() - list.size == 1 -> list.first() - else -> Criteria().andOperator(*list.toTypedArray()) - } - } -} diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/authorization/mapper/JsonSchemaDocumentCaseDefinitionMongoMapper.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/authorization/mapper/JsonSchemaDocumentCaseDefinitionMongoMapper.kt deleted file mode 100644 index 5cacb847ce..0000000000 --- a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/authorization/mapper/JsonSchemaDocumentCaseDefinitionMongoMapper.kt +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2015-2024 Ritense BV, the Netherlands. - * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.ritense.document.mongodb.authorization.mapper - -import com.ritense.authorization.permission.condition.FieldPermissionCondition -import com.ritense.authorization.permission.condition.PermissionCondition -import com.ritense.case_.domain.definition.CaseDefinition -import com.ritense.document.domain.impl.JsonSchemaDocument -import com.ritense.document.mongodb.authorization.MongoAuthorizationEntityMapper -import com.ritense.document.mongodb.authorization.MongoPermissionConditionTranslator.Companion.andAll -import com.ritense.document.mongodb.authorization.MongoPermissionConditionTranslator.Companion.applyOperator -import com.ritense.valtimo.contract.authorization.CurrentUserExpressionHandler -import org.springframework.data.mongodb.core.query.Criteria - -/** - * Handles [com.ritense.authorization.permission.condition.ContainerPermissionCondition] - * where the container resource type is [CaseDefinition]. - * - * In the JPA model this relationship is expressed via `definitionId.blueprintId` - * (blueprintType=CASE, blueprintKey, blueprintVersionTag) on [JsonSchemaDocument]. - * The same nested structure exists in the MongoDB document. - */ -class JsonSchemaDocumentCaseDefinitionMongoMapper : MongoAuthorizationEntityMapper { - - override fun mapCriteria(conditions: List): Criteria? { - if (conditions.isEmpty()) return null - - val conditionCriteria = conditions.map { condition -> - when (condition) { - is FieldPermissionCondition<*> -> { - val mongoField = mapCaseDefinitionField(condition.field) - val value = CurrentUserExpressionHandler.resolveValue(condition.value) - applyOperator(Criteria.where(mongoField), condition.operator, value) - } - else -> throw UnsupportedOperationException( - "Condition type ${condition::class.simpleName} is not supported in " + - "${this::class.simpleName}. Register a custom ${MongoAuthorizationEntityMapper::class.simpleName} " + - "or extend this mapper to handle it." - ) - } - } - - // Constrain to CASE blueprint type to exclude BUILDING_BLOCK documents - val typeCriteria = Criteria.where("definitionId.blueprintId.blueprintType").`is`("CASE") - return andAll(conditionCriteria + typeCriteria) - } - - override fun supports(fromClass: Class<*>, toClass: Class<*>): Boolean = - fromClass == JsonSchemaDocument::class.java && - toClass == CaseDefinition::class.java - - private fun mapCaseDefinitionField(field: String): String = when (field) { - "id.key" -> "definitionId.blueprintId.blueprintKey" - "id.versionTag" -> "definitionId.blueprintId.blueprintVersionTag" - else -> throw UnsupportedOperationException( - "Field '$field' on CaseDefinition is not yet mapped for MongoDB. " + - "Add it to ${this::class.simpleName}." - ) - } -} diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/authorization/mapper/JsonSchemaDocumentDefinitionMongoMapper.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/authorization/mapper/JsonSchemaDocumentDefinitionMongoMapper.kt deleted file mode 100644 index f8b8546ed8..0000000000 --- a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/authorization/mapper/JsonSchemaDocumentDefinitionMongoMapper.kt +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2015-2024 Ritense BV, the Netherlands. - * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.ritense.document.mongodb.authorization.mapper - -import com.ritense.authorization.permission.condition.FieldPermissionCondition -import com.ritense.authorization.permission.condition.PermissionCondition -import com.ritense.document.domain.impl.JsonSchemaDocument -import com.ritense.document.domain.impl.JsonSchemaDocumentDefinition -import com.ritense.document.mongodb.authorization.MongoAuthorizationEntityMapper -import com.ritense.document.mongodb.authorization.MongoPermissionConditionTranslator.Companion.andAll -import com.ritense.document.mongodb.authorization.MongoPermissionConditionTranslator.Companion.applyOperator -import com.ritense.valtimo.contract.authorization.CurrentUserExpressionHandler -import org.springframework.data.mongodb.core.query.Criteria - -/** - * Handles [com.ritense.authorization.permission.condition.ContainerPermissionCondition] - * where the container resource type is [JsonSchemaDocumentDefinition]. - * - * In the JPA model this relationship is expressed via `definitionId.name` (and - * `definitionId.version`) on [JsonSchemaDocument]. The same fields exist in the - * MongoDB document under `definitionId.name` and `definitionId.version`. - */ -class JsonSchemaDocumentDefinitionMongoMapper : MongoAuthorizationEntityMapper { - - override fun mapCriteria(conditions: List): Criteria? { - if (conditions.isEmpty()) return null - - val criteria = conditions.map { condition -> - when (condition) { - is FieldPermissionCondition<*> -> { - val mongoField = mapDefinitionField(condition.field) - val value = CurrentUserExpressionHandler.resolveValue(condition.value) - applyOperator(Criteria.where(mongoField), condition.operator, value) - } - else -> throw UnsupportedOperationException( - "Condition type ${condition::class.simpleName} is not supported in " + - "${this::class.simpleName}. Register a custom ${MongoAuthorizationEntityMapper::class.simpleName} " + - "or extend this mapper to handle it." - ) - } - } - return andAll(criteria) - } - - override fun supports(fromClass: Class<*>, toClass: Class<*>): Boolean = - fromClass == JsonSchemaDocument::class.java && - toClass == JsonSchemaDocumentDefinition::class.java - - private fun mapDefinitionField(field: String): String = when (field) { - "id.name" -> "definitionId.name" - "id.version" -> "definitionId.version" - else -> throw UnsupportedOperationException( - "Field '$field' on JsonSchemaDocumentDefinition is not yet mapped for MongoDB. " + - "Add it to ${this::class.simpleName}." - ) - } -} diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/autoconfigure/DocumentMongoAutoConfiguration.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/autoconfigure/DocumentMongoAutoConfiguration.kt deleted file mode 100644 index f42a142234..0000000000 --- a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/autoconfigure/DocumentMongoAutoConfiguration.kt +++ /dev/null @@ -1,217 +0,0 @@ -/* - * Copyright 2015-2024 Ritense BV, the Netherlands. - * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.ritense.document.mongodb.autoconfigure - -import com.fasterxml.jackson.databind.ObjectMapper -import com.ritense.authorization.AuthorizationService -import com.ritense.document.autoconfigure.DocumentAutoConfiguration -import com.ritense.document.mongodb.authorization.MongoAuthorizationEntityMapper -import com.ritense.document.mongodb.authorization.MongoPermissionConditionTranslator -import com.ritense.document.mongodb.authorization.mapper.JsonSchemaDocumentCaseDefinitionMongoMapper -import com.ritense.document.mongodb.authorization.mapper.JsonSchemaDocumentDefinitionMongoMapper -import com.ritense.document.mongodb.converter.DocumentToJsonNodeReadConverter -import com.ritense.document.mongodb.converter.DocumentToObjectNodeReadConverter -import com.ritense.document.mongodb.converter.JsonNodeWriteConverter -import com.ritense.document.mongodb.domain.JsonSchemaDocumentDocument -import com.ritense.document.mongodb.handler.DocumentMongoEventHandler -import com.ritense.document.mongodb.repository.JsonSchemaDocumentMongoRepository -import com.ritense.document.mongodb.service.DocumentMongoBackfillService -import com.ritense.document.mongodb.service.DocumentMongoQueryService -import com.ritense.document.mongodb.service.DocumentMongoSyncService -import com.ritense.document.mongodb.service.JsonSchemaDocumentMongoSearchService -import com.ritense.document.mongodb.security.DocumentMongoHttpSecurityConfigurer -import com.ritense.document.mongodb.web.DocumentMongoBackfillResource -import com.ritense.document.repository.impl.JsonSchemaDocumentRepository -import com.ritense.document.service.DocumentSearchService -import com.ritense.document.service.SearchFieldService -import com.ritense.outbox.OutboxService -import com.ritense.valtimo.contract.authentication.UserManagementService -import org.springframework.boot.ApplicationRunner -import org.springframework.boot.autoconfigure.AutoConfiguration -import org.springframework.boot.autoconfigure.AutoConfigureBefore -import org.springframework.boot.autoconfigure.condition.ConditionalOnClass -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean -import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration -import org.springframework.context.annotation.Bean -import org.springframework.core.annotation.Order -import org.springframework.data.domain.Sort -import org.springframework.data.mongodb.core.MongoTemplate -import org.springframework.data.mongodb.core.convert.MongoCustomConversions -import org.springframework.data.mongodb.core.index.Index -import org.springframework.data.mongodb.core.index.TextIndexDefinition -import org.springframework.data.mongodb.repository.config.EnableMongoRepositories - -@AutoConfiguration -@AutoConfigureBefore(DocumentAutoConfiguration::class, MongoDataAutoConfiguration::class) -@ConditionalOnClass(MongoTemplate::class) -@EnableMongoRepositories(basePackages = ["com.ritense.document.mongodb.repository"]) -class DocumentMongoAutoConfiguration { - - /** - * Registers custom converters so that Jackson [com.fasterxml.jackson.databind.JsonNode]/ - * [com.fasterxml.jackson.databind.node.ObjectNode] fields in MongoDB documents are serialized - * as proper JSON rather than as Jackson's internal object structure. - * - * Must run before [MongoDataAutoConfiguration] so this bean wins the - * [ConditionalOnMissingBean] check there. - */ - @Bean - @ConditionalOnMissingBean(MongoCustomConversions::class) - fun mongoCustomConversions(objectMapper: ObjectMapper): MongoCustomConversions = - MongoCustomConversions( - listOf( - JsonNodeWriteConverter(), - DocumentToJsonNodeReadConverter(objectMapper), - DocumentToObjectNodeReadConverter(objectMapper), - ) - ) - - @Bean - @ConditionalOnMissingBean - fun jsonSchemaDocumentDefinitionMongoMapper(): JsonSchemaDocumentDefinitionMongoMapper = - JsonSchemaDocumentDefinitionMongoMapper() - - @Bean - @ConditionalOnMissingBean - fun jsonSchemaDocumentCaseDefinitionMongoMapper(): JsonSchemaDocumentCaseDefinitionMongoMapper = - JsonSchemaDocumentCaseDefinitionMongoMapper() - - @Bean - @ConditionalOnMissingBean - fun mongoPermissionConditionTranslator( - mongoMappers: List>, - authorizationService: AuthorizationService, - documentRepository: JsonSchemaDocumentRepository, - ): MongoPermissionConditionTranslator = - MongoPermissionConditionTranslator(mongoMappers, authorizationService, documentRepository) - - @Bean - @ConditionalOnMissingBean - fun documentMongoQueryService( - mongoTemplate: MongoTemplate, - authorizationService: AuthorizationService, - translator: MongoPermissionConditionTranslator, - ): DocumentMongoQueryService = - DocumentMongoQueryService(mongoTemplate, authorizationService, translator) - - @Bean - @ConditionalOnMissingBean - fun documentMongoSyncService( - repository: JsonSchemaDocumentMongoRepository, - objectMapper: ObjectMapper, - ): DocumentMongoSyncService = - DocumentMongoSyncService(repository, objectMapper) - - @Bean - fun documentMongoEventHandler(syncService: DocumentMongoSyncService): DocumentMongoEventHandler = - DocumentMongoEventHandler(syncService) - - @Bean - @ConditionalOnMissingBean - fun documentMongoBackfillService( - jpaRepository: JsonSchemaDocumentRepository, - mongoRepository: JsonSchemaDocumentMongoRepository, - objectMapper: ObjectMapper, - ): DocumentMongoBackfillService = - DocumentMongoBackfillService(jpaRepository, mongoRepository, objectMapper) - - @Order(293) - @Bean - @ConditionalOnMissingBean - fun documentMongoHttpSecurityConfigurer(): DocumentMongoHttpSecurityConfigurer = - DocumentMongoHttpSecurityConfigurer() - - @Bean - @ConditionalOnMissingBean(DocumentSearchService::class) - fun documentSearchService( - mongoTemplate: MongoTemplate, - translator: MongoPermissionConditionTranslator, - authorizationService: AuthorizationService, - jpaRepository: JsonSchemaDocumentRepository, - userManagementService: UserManagementService, - searchFieldService: SearchFieldService, - outboxService: OutboxService, - objectMapper: ObjectMapper, - ): JsonSchemaDocumentMongoSearchService = - JsonSchemaDocumentMongoSearchService( - mongoTemplate, - translator, - authorizationService, - jpaRepository, - userManagementService, - searchFieldService, - outboxService, - objectMapper, - ) - - @Bean - @ConditionalOnMissingBean - fun documentMongoBackfillResource( - backfillService: DocumentMongoBackfillService, - ): DocumentMongoBackfillResource = - DocumentMongoBackfillResource(backfillService) - - /** - * Creates the indexes for the [JsonSchemaDocumentDocument] collection on startup. - * - * This is done programmatically rather than relying solely on [@CompoundIndex] annotations, - * because [spring.data.mongodb.auto-index-creation] is typically disabled in production. - * [MongoTemplate.indexOps] + [ensureIndex] is idempotent: it creates missing indexes and - * is a no-op when the index already exists with the same definition. - */ - @Bean - fun documentMongoIndexInitializer(mongoTemplate: MongoTemplate): ApplicationRunner = ApplicationRunner { - val ops = mongoTemplate.indexOps(JsonSchemaDocumentDocument::class.java) - - ops.ensureIndex( - Index() - .on("definitionId.blueprintId.blueprintType", Sort.Direction.ASC) - .on("definitionId.name", Sort.Direction.ASC) - .on("createdOn", Sort.Direction.DESC) - .named("idx_type_name_created"), - ) - ops.ensureIndex( - Index() - .on("definitionId.blueprintId.blueprintType", Sort.Direction.ASC) - .on("definitionId.name", Sort.Direction.ASC) - .on("internalStatus", Sort.Direction.ASC) - .on("createdOn", Sort.Direction.DESC) - .named("idx_type_name_status_created"), - ) - ops.ensureIndex( - Index() - .on("definitionId.blueprintId.blueprintType", Sort.Direction.ASC) - .on("definitionId.name", Sort.Direction.ASC) - .on("assigneeId", Sort.Direction.ASC) - .on("createdOn", Sort.Direction.DESC) - .named("idx_type_name_assignee_created"), - ) - ops.ensureIndex( - Index() - .on("definitionId.blueprintId.blueprintType", Sort.Direction.ASC) - .on("definitionId.name", Sort.Direction.ASC) - .on("sequence", Sort.Direction.ASC) - .named("idx_type_name_sequence"), - ) - ops.ensureIndex( - TextIndexDefinition.builder() - .onField("contentText") - .named("idx_content_text") - .build(), - ) - } -} diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/converter/JsonNodeMongoConverters.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/converter/JsonNodeMongoConverters.kt deleted file mode 100644 index 291777be22..0000000000 --- a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/converter/JsonNodeMongoConverters.kt +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2015-2024 Ritense BV, the Netherlands. - * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.ritense.document.mongodb.converter - -import com.fasterxml.jackson.databind.JsonNode -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.databind.node.ObjectNode -import org.bson.Document -import org.bson.json.JsonMode -import org.bson.json.JsonWriterSettings -import org.springframework.core.convert.converter.Converter -import org.springframework.data.convert.ReadingConverter -import org.springframework.data.convert.WritingConverter - -private val RELAXED_JSON = JsonWriterSettings.builder().outputMode(JsonMode.RELAXED).build() - -/** - * Converts a Jackson [JsonNode] to a BSON [Document] so that Spring Data MongoDB - * stores the actual JSON structure rather than Jackson's internal object fields. - * Only applicable to object-typed [JsonNode] fields (e.g. [definitionId]). - * Array-typed fields should use [Any] instead of [JsonNode] to avoid this converter. - */ -@WritingConverter -class JsonNodeWriteConverter : Converter { - override fun convert(source: JsonNode): Document = Document.parse(source.toString()) -} - -/** - * Converts a BSON [Document] back to a Jackson [JsonNode] when reading a [JsonNode]-typed field. - */ -@ReadingConverter -class DocumentToJsonNodeReadConverter(private val objectMapper: ObjectMapper) : Converter { - override fun convert(source: Document): JsonNode = - objectMapper.readTree(source.toJson(RELAXED_JSON)) -} - -/** - * Converts a BSON [Document] back to a Jackson [ObjectNode] when reading an [ObjectNode]-typed field. - */ -@ReadingConverter -class DocumentToObjectNodeReadConverter(private val objectMapper: ObjectMapper) : Converter { - override fun convert(source: Document): ObjectNode = - objectMapper.readTree(source.toJson(RELAXED_JSON)) as ObjectNode -} diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/domain/JsonSchemaDocumentDocument.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/domain/JsonSchemaDocumentDocument.kt deleted file mode 100644 index d5af6159ea..0000000000 --- a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/domain/JsonSchemaDocumentDocument.kt +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright 2015-2024 Ritense BV, the Netherlands. - * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.ritense.document.mongodb.domain - -import com.fasterxml.jackson.databind.JsonNode -import com.fasterxml.jackson.databind.node.ObjectNode -import com.ritense.document.web.rest.dto.CaseTagResponseDto -import org.springframework.data.annotation.Id -import org.springframework.data.mongodb.core.index.CompoundIndex -import org.springframework.data.mongodb.core.index.CompoundIndexes -import org.springframework.data.mongodb.core.index.TextIndexed -import org.springframework.data.mongodb.core.mapping.Document -import java.time.LocalDateTime - -/** - * MongoDB read model for [com.ritense.document.domain.impl.JsonSchemaDocument]. - * - * Field names and types mirror the Jackson serialization of the JPA entity: - * - [definitionId] matches the `definitionId()` getter → [com.ritense.document.domain.impl.JsonSchemaDocumentDefinitionId] - * - [internalStatus] matches the `internalStatus()` getter → plain key String - * - [caseTags] matches the `caseTags()` getter → [CaseTagResponseDto] list - * - [relations] matches the `relations()` getter → stored as [Any] (can be array or object) to avoid JPA entity coupling - * - [relatedFiles] matches the `relatedFiles()` getter → stored as [Any] (RelatedFile is an interface) - * - * Indexes follow the ESR rule (Equality → Sort → Range): - * - The two equality fields that appear in every query are blueprintType + definitionName. - * - Variants cover the optional equality filters (status, assigneeId) with createdOn as the sort tail. - * - A separate index covers sequence-based sorting. - * - A MongoDB text index on [contentText] supports full-text / global search. - */ -@Document(collection = "json_schema_document") -@CompoundIndexes( - // Base index: no optional filters, sort by creation date (most common case-list query) - CompoundIndex( - name = "idx_type_name_created", - def = "{'definitionId.blueprintId.blueprintType': 1, 'definitionId.name': 1, 'createdOn': -1}", - ), - // Status filter + sort by creation date - CompoundIndex( - name = "idx_type_name_status_created", - def = "{'definitionId.blueprintId.blueprintType': 1, 'definitionId.name': 1, 'internalStatus': 1, 'createdOn': -1}", - ), - // Assignee filter + sort by creation date - CompoundIndex( - name = "idx_type_name_assignee_created", - def = "{'definitionId.blueprintId.blueprintType': 1, 'definitionId.name': 1, 'assigneeId': 1, 'createdOn': -1}", - ), - // Sequence-based sort / filter - CompoundIndex( - name = "idx_type_name_sequence", - def = "{'definitionId.blueprintId.blueprintType': 1, 'definitionId.name': 1, 'sequence': 1}", - ), -) -data class JsonSchemaDocumentDocument( - @Id val id: String, - val content: ObjectNode?, - val definitionId: JsonNode?, - val createdOn: LocalDateTime?, - val modifiedOn: LocalDateTime?, - val createdBy: String?, - val sequence: Long?, - val version: Int?, - val assigneeId: String?, - val assigneeFullName: String?, - val internalStatus: String?, - val caseTags: List?, - val relations: Any?, - val relatedFiles: Any?, - val retentionDate: LocalDateTime?, - @TextIndexed val contentText: String? = null, -) diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/handler/DocumentMongoEventHandler.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/handler/DocumentMongoEventHandler.kt deleted file mode 100644 index 45e0b9454e..0000000000 --- a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/handler/DocumentMongoEventHandler.kt +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2015-2024 Ritense BV, the Netherlands. - * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.ritense.document.mongodb.handler - -import com.ritense.document.event.DocumentAssigned -import com.ritense.document.event.DocumentCreated -import com.ritense.document.event.DocumentUnassigned -import com.ritense.document.event.DocumentUpdated -import com.ritense.document.mongodb.service.DocumentMongoSyncService -import com.ritense.inbox.ValtimoEvent -import com.ritense.inbox.ValtimoEventHandler -import io.github.oshai.kotlinlogging.KotlinLogging - -/** - * Listens to document domain events from the Valtimo inbox and keeps the MongoDB read - * model in sync. Works with both the outbox-enabled (RabbitMQ) and outbox-disabled - * (local Spring event) modes because both paths converge on [ValtimoEventHandler]. - */ -class DocumentMongoEventHandler( - private val syncService: DocumentMongoSyncService, -) : ValtimoEventHandler { - - override fun handle(event: ValtimoEvent) { - when (event.type) { - in UPSERT_EVENT_TYPES -> syncService.upsert(event) - DELETED_EVENT_TYPE -> { - val id = event.resultId - if (id != null) { - syncService.delete(id) - } else { - logger.warn { "Received DocumentDeleted event with null resultId — skipping delete" } - } - } - else -> { - // Events not related to json_schema_document (e.g. DocumentsListed) are ignored - } - } - } - - companion object { - private val logger = KotlinLogging.logger {} - - val UPSERT_EVENT_TYPES: Set = setOf( - DocumentCreated.TYPE, - DocumentUpdated.TYPE, - DocumentAssigned.TYPE, - DocumentUnassigned.TYPE, - "com.ritense.valtimo.document.status.changed", - "com.ritense.valtimo.document.tags.changed", - "com.ritense.valtimo.document.retentiondate.set", - "com.ritense.valtimo.document.retentiondate.unset", - ) - - const val DELETED_EVENT_TYPE = "com.ritense.valtimo.document.deleted" - } -} diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/repository/JsonSchemaDocumentMongoRepository.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/repository/JsonSchemaDocumentMongoRepository.kt deleted file mode 100644 index 9e5dbf40dc..0000000000 --- a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/repository/JsonSchemaDocumentMongoRepository.kt +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2015-2024 Ritense BV, the Netherlands. - * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.ritense.document.mongodb.repository - -import com.ritense.document.mongodb.domain.JsonSchemaDocumentDocument -import org.springframework.data.mongodb.repository.MongoRepository - -interface JsonSchemaDocumentMongoRepository : MongoRepository diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/security/DocumentMongoHttpSecurityConfigurer.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/security/DocumentMongoHttpSecurityConfigurer.kt deleted file mode 100644 index ee9050ae70..0000000000 --- a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/security/DocumentMongoHttpSecurityConfigurer.kt +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2015-2024 Ritense BV, the Netherlands. - * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.ritense.document.mongodb.security - -import com.ritense.valtimo.contract.authentication.AuthoritiesConstants.ADMIN -import com.ritense.valtimo.contract.security.config.HttpConfigurerConfigurationException -import com.ritense.valtimo.contract.security.config.HttpSecurityConfigurer -import org.springframework.http.HttpMethod.POST -import org.springframework.security.config.annotation.web.builders.HttpSecurity -import org.springframework.security.web.util.matcher.AntPathRequestMatcher.antMatcher - -class DocumentMongoHttpSecurityConfigurer : HttpSecurityConfigurer { - - override fun configure(http: HttpSecurity) { - try { - http.authorizeHttpRequests { requests -> - requests.requestMatchers(antMatcher(POST, "/api/management/v1/document-mongodb/backfill")) - .hasAuthority(ADMIN) - } - } catch (e: Exception) { - throw HttpConfigurerConfigurationException(e) - } - } -} diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/ContentTextExtractor.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/ContentTextExtractor.kt deleted file mode 100644 index e2acdcdba2..0000000000 --- a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/ContentTextExtractor.kt +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2015-2024 Ritense BV, the Netherlands. - * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.ritense.document.mongodb.service - -import com.fasterxml.jackson.databind.JsonNode - -/** - * Extracts all leaf values from a [JsonNode] as a single space-separated string. - * Used to populate [com.ritense.document.mongodb.domain.JsonSchemaDocumentDocument.contentText] - * for full-document search. - */ -fun extractLeafValues(node: JsonNode?): String? { - if (node == null) return null - val parts = mutableListOf() - collectLeaves(node, parts) - return parts.joinToString(" ").ifBlank { null } -} - -private fun collectLeaves(node: JsonNode, out: MutableList) { - when { - node.isObject -> node.fields().forEach { (_, v) -> collectLeaves(v, out) } - node.isArray -> node.forEach { collectLeaves(it, out) } - !node.isNull && !node.isMissingNode -> out.add(node.asText()) - } -} diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/DocumentMongoBackfillService.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/DocumentMongoBackfillService.kt deleted file mode 100644 index 7c0d472431..0000000000 --- a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/DocumentMongoBackfillService.kt +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright 2015-2024 Ritense BV, the Netherlands. - * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.ritense.document.mongodb.service - -import com.fasterxml.jackson.databind.ObjectMapper -import com.ritense.authorization.AuthorizationContext.Companion.runWithoutAuthorization -import com.ritense.document.domain.impl.JsonSchemaDocument -import com.ritense.document.mongodb.domain.JsonSchemaDocumentDocument -import com.ritense.document.mongodb.repository.JsonSchemaDocumentMongoRepository -import com.ritense.document.repository.impl.JsonSchemaDocumentRepository -import io.github.oshai.kotlinlogging.KotlinLogging -import org.springframework.data.domain.PageRequest -import org.springframework.transaction.annotation.Transactional - -open class DocumentMongoBackfillService( - private val jpaRepository: JsonSchemaDocumentRepository, - private val mongoRepository: JsonSchemaDocumentMongoRepository, - private val objectMapper: ObjectMapper, -) { - - /** - * Copies all existing [JsonSchemaDocument] rows from the relational database to MongoDB. - * Processes documents in pages of [pageSize] to avoid loading the entire table into memory. - * - * @return total number of documents migrated - */ - @Transactional(readOnly = true) - open fun backfill(pageSize: Int = DEFAULT_PAGE_SIZE): Long { - var page = 0 - var total = 0L - do { - val slice = runWithoutAuthorization { jpaRepository.findAll(PageRequest.of(page++, pageSize)) } - if (slice.isEmpty) break - - val docs = mutableListOf() - for (jpaDoc in slice.content) { - try { - val json = objectMapper.writeValueAsString(jpaDoc) - val doc = objectMapper.readValue(json, JsonSchemaDocumentDocument::class.java) - docs.add(doc.copy(contentText = extractLeafValues(doc.content))) - } catch (e: Exception) { - logger.warn(e) { "Failed to convert document to MongoDB document — skipping" } - } - } - mongoRepository.saveAll(docs) - total += docs.size - logger.debug { "Backfilled page ${page - 1}: ${docs.size} documents (total so far: $total)" } - } while (slice.hasNext()) - - logger.info { "Backfill complete: $total documents migrated to MongoDB" } - return total - } - - companion object { - private val logger = KotlinLogging.logger {} - const val DEFAULT_PAGE_SIZE = 500 - } -} diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/DocumentMongoQueryService.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/DocumentMongoQueryService.kt deleted file mode 100644 index e251299293..0000000000 --- a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/DocumentMongoQueryService.kt +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2015-2024 Ritense BV, the Netherlands. - * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.ritense.document.mongodb.service - -import com.ritense.authorization.Action -import com.ritense.authorization.AuthorizationService -import com.ritense.document.domain.impl.JsonSchemaDocument -import com.ritense.valtimo.contract.utils.SecurityUtils -import com.ritense.document.mongodb.authorization.MongoPermissionConditionTranslator -import com.ritense.document.mongodb.domain.JsonSchemaDocumentDocument -import com.ritense.document.service.JsonSchemaDocumentActionProvider -import org.springframework.data.domain.Page -import org.springframework.data.domain.PageImpl -import org.springframework.data.domain.Pageable -import org.springframework.data.mongodb.core.MongoTemplate -import org.springframework.data.mongodb.core.query.Criteria -import org.springframework.data.mongodb.core.query.Query - -class DocumentMongoQueryService( - private val mongoTemplate: MongoTemplate, - private val authorizationService: AuthorizationService, - private val translator: MongoPermissionConditionTranslator, -) { - - /** - * Returns a page of documents for the given [definitionName], restricted to those - * the current user is allowed to see (VIEW_LIST action). - */ - fun findAllByDefinitionName(definitionName: String, pageable: Pageable): Page { - val combined = buildCriteria(JsonSchemaDocumentActionProvider.VIEW_LIST) - .andOperator(Criteria.where("definitionId.name").`is`(definitionName)) - - val countQuery = Query(combined) - val dataQuery = Query(combined).with(pageable) - - val total = mongoTemplate.count(countQuery, JsonSchemaDocumentDocument::class.java) - val content = mongoTemplate.find(dataQuery, JsonSchemaDocumentDocument::class.java) - return PageImpl(content, pageable, total) - } - - /** - * Returns the document with the given [id] if the current user has VIEW permission, - * or `null` if it does not exist or is not accessible. - */ - fun findById(id: String): JsonSchemaDocumentDocument? { - val combined = buildCriteria(JsonSchemaDocumentActionProvider.VIEW) - .andOperator(Criteria.where("_id").`is`(id)) - return mongoTemplate.findOne(Query(combined), JsonSchemaDocumentDocument::class.java) - } - - private fun buildCriteria(action: Action): Criteria { - val userRoles = SecurityUtils.getCurrentUserRoles().toSet() - val permissions = authorizationService.getPermissions(JsonSchemaDocument::class.java, action) - .filter { it.role.key in userRoles } - return translator.toCriteria(permissions, action) - } -} diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/DocumentMongoSyncService.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/DocumentMongoSyncService.kt deleted file mode 100644 index 08e0be0c10..0000000000 --- a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/DocumentMongoSyncService.kt +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2015-2024 Ritense BV, the Netherlands. - * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.ritense.document.mongodb.service - -import com.fasterxml.jackson.databind.ObjectMapper -import com.ritense.document.mongodb.domain.JsonSchemaDocumentDocument -import com.ritense.document.mongodb.repository.JsonSchemaDocumentMongoRepository -import com.ritense.inbox.ValtimoEvent -import io.github.oshai.kotlinlogging.KotlinLogging - -class DocumentMongoSyncService( - private val repository: JsonSchemaDocumentMongoRepository, - private val objectMapper: ObjectMapper, -) { - - fun upsert(event: ValtimoEvent) { - val result = event.result - if (result == null) { - logger.warn { "Received document event ${event.type} for id=${event.resultId} with null result — skipping upsert" } - return - } - val doc = objectMapper.treeToValue(result, JsonSchemaDocumentDocument::class.java) - repository.save(doc.copy(contentText = extractLeafValues(doc.content))) - logger.debug { "Upserted document ${doc.id} in MongoDB (event: ${event.type})" } - } - - fun delete(documentId: String) { - repository.deleteById(documentId) - logger.debug { "Deleted document $documentId from MongoDB" } - } - - companion object { - private val logger = KotlinLogging.logger {} - } -} diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/JsonSchemaDocumentMongoSearchService.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/JsonSchemaDocumentMongoSearchService.kt deleted file mode 100644 index 9c7b5c4629..0000000000 --- a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/service/JsonSchemaDocumentMongoSearchService.kt +++ /dev/null @@ -1,364 +0,0 @@ -/* - * Copyright 2015-2024 Ritense BV, the Netherlands. - * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.ritense.document.mongodb.service - -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.databind.node.ArrayNode -import com.ritense.authorization.Action -import com.ritense.authorization.AuthorizationContext.Companion.runWithoutAuthorization -import com.ritense.authorization.AuthorizationService -import com.ritense.document.domain.impl.JsonSchemaDocument -import com.ritense.document.domain.impl.JsonSchemaDocumentId -import com.ritense.document.domain.search.AdvancedSearchRequest -import com.ritense.document.domain.search.AssigneeFilter -import com.ritense.document.domain.search.DatabaseSearchType -import com.ritense.document.domain.search.SearchOperator -import com.ritense.document.domain.search.SearchRequestMapper -import com.ritense.document.domain.search.SearchRequestValidator -import com.ritense.document.domain.search.SearchWithConfigRequest -import com.ritense.document.event.DocumentsListed -import com.ritense.document.mongodb.authorization.MongoPermissionConditionTranslator -import com.ritense.document.mongodb.domain.JsonSchemaDocumentDocument -import com.ritense.document.repository.impl.JsonSchemaDocumentRepository -import com.ritense.document.service.DocumentSearchService -import com.ritense.document.service.JsonSchemaDocumentActionProvider -import com.ritense.document.service.SearchFieldService -import com.ritense.document.service.impl.SearchRequest -import com.ritense.outbox.OutboxService -import com.ritense.valtimo.contract.authentication.UserManagementService -import com.ritense.valtimo.contract.blueprint.BlueprintType -import com.ritense.valtimo.contract.utils.RequestHelper -import com.ritense.valtimo.contract.utils.SecurityUtils -import org.apache.commons.lang3.NotImplementedException -import org.springframework.data.domain.Page -import org.springframework.data.domain.PageImpl -import org.springframework.data.domain.Pageable -import org.springframework.data.domain.Sort -import org.springframework.data.mongodb.core.MongoTemplate -import org.springframework.data.mongodb.core.query.Criteria -import org.springframework.data.mongodb.core.query.Query -import org.springframework.data.mongodb.core.query.TextCriteria -import java.util.regex.Pattern - -class JsonSchemaDocumentMongoSearchService( - private val mongoTemplate: MongoTemplate, - private val translator: MongoPermissionConditionTranslator, - private val authorizationService: AuthorizationService, - private val jpaRepository: JsonSchemaDocumentRepository, - private val userManagementService: UserManagementService, - private val searchFieldService: SearchFieldService, - private val outboxService: OutboxService, - private val objectMapper: ObjectMapper, -) : DocumentSearchService { - - override fun search( - searchRequest: SearchRequest, - blueprintType: BlueprintType, - pageable: Pageable - ): Page { - val parts = mutableListOf() - - parts.add(buildAuthCriteria(JsonSchemaDocumentActionProvider.VIEW_LIST)) - parts.add(Criteria.where(BLUEPRINT_TYPE_FIELD).`is`(blueprintType.name)) - - if (!searchRequest.documentDefinitionName.isNullOrEmpty()) { - parts.add(Criteria.where(DEFINITION_NAME_FIELD).`is`(searchRequest.documentDefinitionName)) - } - if (!searchRequest.createdBy.isNullOrEmpty()) { - parts.add(Criteria.where("createdBy").`is`(searchRequest.createdBy)) - } - if (searchRequest.sequence != null) { - parts.add(Criteria.where("sequence").`is`(searchRequest.sequence)) - } - if (!searchRequest.globalSearchFilter.isNullOrEmpty()) { - throw NotImplementedException("globalSearchFilter is not supported in the MongoDB search service") - } - searchRequest.otherFilters?.forEach { sc -> - parts.add(Criteria.where("content.${sc.path}").`is`(sc.value)) - } - - return executeSearch(Criteria().andOperator(*parts.toTypedArray()), null, pageable) - } - - override fun search( - documentDefinitionName: String, - blueprintType: BlueprintType, - searchWithConfigRequest: SearchWithConfigRequest, - pageable: Pageable - ): Page { - val zoneOffset = RequestHelper.getZoneOffset() - val searchFieldMap = searchFieldService.getSearchFields(documentDefinitionName) - .associateBy { it.key } - - val otherFilters = searchWithConfigRequest.otherFilters - .map { filter -> SearchRequestMapper.toOtherFilter(filter, searchFieldMap[filter.key], zoneOffset) } - - val advancedSearchRequest = SearchRequestMapper.toAdvancedSearchRequest(searchWithConfigRequest, otherFilters) - - return search( - documentDefinitionName, - blueprintType, - advancedSearchRequest, - pageable, - JsonSchemaDocumentActionProvider.VIEW_LIST - ) - } - - override fun search( - documentDefinitionName: String, - blueprintType: BlueprintType, - advancedSearchRequest: AdvancedSearchRequest, - pageable: Pageable - ): Page { - return search( - documentDefinitionName, - blueprintType, - advancedSearchRequest, - pageable, - JsonSchemaDocumentActionProvider.VIEW_LIST - ) - } - - override fun searchForExport( - documentDefinitionName: String, - blueprintType: BlueprintType, - searchWithConfigRequest: SearchWithConfigRequest, - pageable: Pageable - ): Page { - val zoneOffset = RequestHelper.getZoneOffset() - val searchFieldMap = searchFieldService.getSearchFields(documentDefinitionName) - .associateBy { it.key } - - val otherFilters = searchWithConfigRequest.otherFilters - .map { filter -> SearchRequestMapper.toOtherFilter(filter, searchFieldMap[filter.key], zoneOffset) } - - val advancedSearchRequest = SearchRequestMapper.toAdvancedSearchRequest(searchWithConfigRequest, otherFilters) - - return search( - documentDefinitionName, - blueprintType, - advancedSearchRequest, - pageable, - JsonSchemaDocumentActionProvider.EXPORT - ) - } - - override fun count( - documentDefinitionName: String, - blueprintType: BlueprintType, - advancedSearchRequest: AdvancedSearchRequest - ): Long { - SearchRequestValidator.validate(advancedSearchRequest) - val (criteria, textCriteria) = buildCombinedCriteria( - documentDefinitionName, - blueprintType, - advancedSearchRequest, - JsonSchemaDocumentActionProvider.VIEW_LIST - ) - val query = Query(criteria) - textCriteria?.let { query.addCriteria(it) } - return mongoTemplate.count(query, JsonSchemaDocumentDocument::class.java) - } - - private fun search( - documentDefinitionName: String, - blueprintType: BlueprintType, - advancedSearchRequest: AdvancedSearchRequest, - pageable: Pageable, - action: Action - ): Page { - SearchRequestValidator.validate(advancedSearchRequest) - val (criteria, textCriteria) = buildCombinedCriteria(documentDefinitionName, blueprintType, advancedSearchRequest, action) - return executeSearch(criteria, textCriteria, pageable) - } - - private fun buildCombinedCriteria( - documentDefinitionName: String?, - blueprintType: BlueprintType, - searchRequest: AdvancedSearchRequest, - action: Action - ): Pair { - val parts = mutableListOf() - - parts.add(buildAuthCriteria(action)) - parts.add(Criteria.where(BLUEPRINT_TYPE_FIELD).`is`(blueprintType.name)) - - if (!documentDefinitionName.isNullOrEmpty()) { - parts.add(Criteria.where(DEFINITION_NAME_FIELD).`is`(documentDefinitionName)) - } - - if (searchRequest.assigneeFilter != null && searchRequest.assigneeFilter != AssigneeFilter.ALL) { - parts.add(buildAssigneeFilterCriteria(searchRequest.assigneeFilter)) - } - - if (!searchRequest.statusFilter.isNullOrEmpty()) { - parts.add(buildStatusFilterCriteria(searchRequest.statusFilter)) - } - - if (!searchRequest.caseTagsFilter.isNullOrEmpty()) { - parts.add(Criteria.where("caseTags.key").`in`(searchRequest.caseTagsFilter)) - } - - if (!searchRequest.otherFilters.isNullOrEmpty()) { - parts.add(buildOtherFiltersCriteria(searchRequest.otherFilters, searchRequest.searchOperator)) - } - - val textCriteria = searchRequest.globalSearchFilter - ?.takeIf { it.isNotEmpty() } - ?.let { TextCriteria.forDefaultLanguage().matching(it.trim()) } - - return Criteria().andOperator(*parts.toTypedArray()) to textCriteria - } - - private fun buildAuthCriteria(action: Action): Criteria { - val userRoles = SecurityUtils.getCurrentUserRoles().toSet() - val permissions = authorizationService.getPermissions(JsonSchemaDocument::class.java, action) - .filter { it.role.key in userRoles } - return translator.toCriteria(permissions, action) - } - - private fun buildAssigneeFilterCriteria(filter: AssigneeFilter): Criteria { - val userId = userManagementService.currentUser.username - return when (filter) { - AssigneeFilter.MINE -> Criteria.where("assigneeId").`is`(userId) - AssigneeFilter.OPEN -> Criteria.where("assigneeId").isNull() - else -> Criteria() - } - } - - private fun buildStatusFilterCriteria(statusKeys: Set): Criteria { - val conditions = statusKeys.map { key -> - if (key.isNullOrEmpty()) Criteria.where("internalStatus").isNull() - else Criteria.where("internalStatus").`is`(key) - } - return if (conditions.size == 1) conditions.first() - else Criteria().orOperator(*conditions.toTypedArray()) - } - - private fun buildOtherFiltersCriteria( - filters: List, - operator: SearchOperator? - ): Criteria { - val filterCriteria = filters.map { buildSingleFilterCriteria(it) } - return if (operator == SearchOperator.OR) { - Criteria().orOperator(*filterCriteria.toTypedArray()) - } else { - Criteria().andOperator(*filterCriteria.toTypedArray()) - } - } - - private fun buildSingleFilterCriteria(filter: AdvancedSearchRequest.OtherFilter): Criteria { - val mongoField = when { - filter.path.startsWith(DOC_PREFIX) -> "content.${filter.path.removePrefix(DOC_PREFIX)}" - filter.path.startsWith(CASE_PREFIX) -> filter.path.removePrefix(CASE_PREFIX) - else -> throw IllegalArgumentException("Search path doesn't start with known prefix: '${filter.path}'") - } - - return when (filter.searchType) { - DatabaseSearchType.EQUAL -> { - val values = filter.getValues() - when { - values.isEmpty() -> Criteria() - values.size == 1 -> applyEqualCriteria(Criteria.where(mongoField), values[0]) - else -> Criteria().orOperator(*values.map { applyEqualCriteria(Criteria.where(mongoField), it) }.toTypedArray()) - } - } - DatabaseSearchType.LIKE -> { - val values = filter.getValues() - when { - values.isEmpty() -> Criteria() - values.size == 1 -> applyLikeCriteria(Criteria.where(mongoField), values[0]) - else -> Criteria().orOperator(*values.map { applyLikeCriteria(Criteria.where(mongoField), it) }.toTypedArray()) - } - } - DatabaseSearchType.IN -> Criteria.where(mongoField).`in`(filter.getValues()) - DatabaseSearchType.GREATER_THAN_OR_EQUAL_TO -> Criteria.where(mongoField).gte(filter.rangeFromValue()!!) - DatabaseSearchType.LESS_THAN_OR_EQUAL_TO -> Criteria.where(mongoField).lte(filter.rangeToValue()!!) - DatabaseSearchType.BETWEEN -> Criteria.where(mongoField).gte(filter.rangeFromValue()!!).lte(filter.rangeToValue()!!) - else -> throw NotImplementedException("Search type '${filter.searchType}' is not supported in the MongoDB search service") - } - } - - private fun applyEqualCriteria(criteria: Criteria, value: Any?): Criteria { - return if (value is String) { - criteria.regex("^${Pattern.quote(value.trim())}$", "i") - } else { - criteria.`is`(value) - } - } - - private fun applyLikeCriteria(criteria: Criteria, value: Any?): Criteria { - if (value !is String) { - throw IllegalArgumentException("LIKE search requires String values, got: ${value?.javaClass?.simpleName}") - } - return criteria.regex(".*${Pattern.quote(value.trim())}.*", "i") - } - - private fun executeSearch(combined: Criteria, textCriteria: TextCriteria?, pageable: Pageable): Page { - val translatedSort = translateSort(pageable.sort) - val dataQuery = Query(combined).with(translatedSort) - textCriteria?.let { dataQuery.addCriteria(it) } - if (pageable.isPaged) { - dataQuery.with(pageable) - } - - val countQuery = Query(combined) - textCriteria?.let { countQuery.addCriteria(it) } - val total = mongoTemplate.count(countQuery, JsonSchemaDocumentDocument::class.java) - - val ids = mongoTemplate.find(dataQuery, JsonSchemaDocumentDocument::class.java).map { it.id } - val docIds = ids.map { JsonSchemaDocumentId.existingId(it) } - val entities = runWithoutAuthorization { jpaRepository.findAllById(docIds) } - val entityMap = entities.associateBy { it.id().toString() } - val orderedEntities = ids.mapNotNull { entityMap[it] } - - outboxService.send { DocumentsListed(objectMapper.valueToTree(orderedEntities)) } - - return PageImpl(orderedEntities, pageable, total) - } - - private fun translateSort(sort: Sort): Sort { - if (sort.isUnsorted) return sort - val orders = sort.map { order -> - val mongoField = when { - order.property.startsWith(DOC_PREFIX) -> "content.${order.property.removePrefix(DOC_PREFIX)}" - order.property.startsWith(CASE_PREFIX) -> order.property.removePrefix(CASE_PREFIX) - else -> order.property - } - if (order.isAscending) Sort.Order.asc(mongoField) else Sort.Order.desc(mongoField) - }.toList() - return Sort.by(orders) - } - - companion object { - private const val DOC_PREFIX = "doc:" - private const val CASE_PREFIX = "case:" - private const val DEFINITION_NAME_FIELD = "definitionId.name" - private const val BLUEPRINT_TYPE_FIELD = "definitionId.blueprintId.blueprintType" - - /** - * Calls [AdvancedSearchRequest.OtherFilter.getRangeFrom] via reflection to bypass the - * Kotlin type-bounds check. The Java method signature uses `>` - * which Kotlin cannot satisfy with `Any`, but at runtime it just returns the boxed value. - */ - private fun AdvancedSearchRequest.OtherFilter.rangeFromValue(): Any? = - AdvancedSearchRequest.OtherFilter::class.java.getMethod("getRangeFrom").invoke(this) - - private fun AdvancedSearchRequest.OtherFilter.rangeToValue(): Any? = - AdvancedSearchRequest.OtherFilter::class.java.getMethod("getRangeTo").invoke(this) - } -} diff --git a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/web/DocumentMongoBackfillResource.kt b/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/web/DocumentMongoBackfillResource.kt deleted file mode 100644 index 05f3e7f617..0000000000 --- a/backend/case-mongodb/src/main/kotlin/com/ritense/document/mongodb/web/DocumentMongoBackfillResource.kt +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2015-2024 Ritense BV, the Netherlands. - * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.ritense.document.mongodb.web - -import com.ritense.document.mongodb.service.DocumentMongoBackfillService -import org.springframework.http.ResponseEntity -import org.springframework.web.bind.annotation.PostMapping -import org.springframework.web.bind.annotation.RequestMapping -import org.springframework.web.bind.annotation.RequestParam -import org.springframework.web.bind.annotation.RestController - -@RestController -@RequestMapping("/api/management/v1/document-mongodb") -class DocumentMongoBackfillResource( - private val backfillService: DocumentMongoBackfillService, -) { - - /** - * Triggers a full backfill of all [com.ritense.document.domain.impl.JsonSchemaDocument] - * records to the MongoDB read model. - * - * Only accessible to users with ROLE_ADMIN. - */ - @PostMapping("/backfill") - fun backfill( - @RequestParam(defaultValue = "${DocumentMongoBackfillService.DEFAULT_PAGE_SIZE}") pageSize: Int, - ): ResponseEntity> { - val count = backfillService.backfill(pageSize) - return ResponseEntity.ok(mapOf("migratedCount" to count)) - } -} diff --git a/backend/case-mongodb/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/backend/case-mongodb/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports deleted file mode 100644 index c1219ff1a4..0000000000 --- a/backend/case-mongodb/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports +++ /dev/null @@ -1 +0,0 @@ -com.ritense.document.mongodb.autoconfigure.DocumentMongoAutoConfiguration diff --git a/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/BaseMongoIntegrationTest.kt b/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/BaseMongoIntegrationTest.kt deleted file mode 100644 index 6a0843efd6..0000000000 --- a/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/BaseMongoIntegrationTest.kt +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright 2015-2025 Ritense BV, the Netherlands. - * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.ritense.document.mongodb - -import com.fasterxml.jackson.databind.ObjectMapper -import com.ritense.audit.service.AuditEventProcessor -import com.ritense.authorization.permission.ConditionContainer -import com.ritense.authorization.permission.Permission -import com.ritense.authorization.permission.PermissionRepository -import com.ritense.authorization.role.Role -import com.ritense.authorization.role.RoleRepository -import com.ritense.document.domain.impl.JsonSchemaDocumentDefinition -import com.ritense.document.domain.impl.snapshot.JsonSchemaDocumentSnapshot -import com.ritense.document.domain.impl.searchfield.SearchField -import com.ritense.document.mongodb.repository.JsonSchemaDocumentMongoRepository -import com.ritense.document.service.impl.JsonSchemaDocumentService -import com.ritense.document.service.JsonSchemaDocumentActionProvider -import com.ritense.document.service.JsonSchemaDocumentDefinitionActionProvider -import com.ritense.document.service.JsonSchemaDocumentSnapshotActionProvider -import com.ritense.document.service.SearchFieldActionProvider -import com.ritense.outbox.OutboxService -import com.ritense.testutilscommon.junit.extension.LiquibaseRunnerExtension -import com.ritense.valtimo.contract.authentication.UserManagementService -import com.ritense.valtimo.contract.mail.MailSender -import com.ritense.valtimo.service.ProcessDefinitionCaseDefinitionLinker -import org.junit.jupiter.api.AfterEach -import org.junit.jupiter.api.BeforeEach -import org.junit.jupiter.api.Tag -import org.junit.jupiter.api.extension.ExtendWith -import org.mockito.Answers -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.boot.test.context.SpringBootTest -import org.springframework.context.event.SimpleApplicationEventMulticaster -import org.springframework.test.context.bean.override.mockito.MockitoBean -import org.springframework.test.context.bean.override.mockito.MockitoSpyBean -import org.springframework.test.context.junit.jupiter.SpringExtension -import org.springframework.transaction.annotation.Transactional -import java.util.UUID - -@SpringBootTest -@ExtendWith(SpringExtension::class, LiquibaseRunnerExtension::class) -@Tag("integration") -@Transactional -abstract class BaseMongoIntegrationTest { - - @MockitoBean(answers = Answers.RETURNS_DEEP_STUBS) - lateinit var userManagementService: UserManagementService - - @MockitoBean - lateinit var applicationEventMulticaster: SimpleApplicationEventMulticaster - - @MockitoBean - lateinit var processDefinitionCaseDefinitionLinker: ProcessDefinitionCaseDefinitionLinker - - @MockitoBean - lateinit var auditEventProcessor: AuditEventProcessor - - @MockitoBean - lateinit var mailSender: MailSender - - @MockitoSpyBean - lateinit var outboxService: OutboxService - - @Autowired - lateinit var documentService: JsonSchemaDocumentService - - @Autowired - lateinit var mongoRepository: JsonSchemaDocumentMongoRepository - - @Autowired - lateinit var roleRepository: RoleRepository - - @Autowired - lateinit var permissionRepository: PermissionRepository - - @Autowired - lateinit var objectMapper: ObjectMapper - - @BeforeEach - fun setUpBase() { - setUpPermissions() - mongoRepository.deleteAll() - } - - @AfterEach - fun tearDownBase() { - mongoRepository.deleteAll() - } - - private fun setUpPermissions() { - var role = roleRepository.findByKey(FULL_ACCESS_ROLE) - if (role == null) { - role = roleRepository.save(Role(UUID.randomUUID(), FULL_ACCESS_ROLE)) - } - - val permissions = listOf( - Permission(UUID.randomUUID(), com.ritense.document.domain.impl.JsonSchemaDocument::class.java, - mutableListOf(JsonSchemaDocumentActionProvider.VIEW_LIST), ConditionContainer(emptyList()), role!!), - Permission(UUID.randomUUID(), com.ritense.document.domain.impl.JsonSchemaDocument::class.java, - mutableListOf(JsonSchemaDocumentActionProvider.VIEW), ConditionContainer(emptyList()), role!!), - Permission(UUID.randomUUID(), com.ritense.document.domain.impl.JsonSchemaDocument::class.java, - mutableListOf(JsonSchemaDocumentActionProvider.MODIFY), ConditionContainer(emptyList()), role!!), - Permission(UUID.randomUUID(), com.ritense.document.domain.impl.JsonSchemaDocument::class.java, - mutableListOf(JsonSchemaDocumentActionProvider.CREATE), ConditionContainer(emptyList()), role!!), - Permission(UUID.randomUUID(), com.ritense.document.domain.impl.JsonSchemaDocument::class.java, - mutableListOf(JsonSchemaDocumentActionProvider.CLAIM), ConditionContainer(emptyList()), role!!), - Permission(UUID.randomUUID(), com.ritense.document.domain.impl.JsonSchemaDocument::class.java, - mutableListOf(JsonSchemaDocumentActionProvider.ASSIGN), ConditionContainer(emptyList()), role!!), - Permission(UUID.randomUUID(), com.ritense.document.domain.impl.JsonSchemaDocument::class.java, - mutableListOf(JsonSchemaDocumentActionProvider.ASSIGNABLE), ConditionContainer(emptyList()), role!!), - Permission(UUID.randomUUID(), com.ritense.document.domain.impl.JsonSchemaDocument::class.java, - mutableListOf(JsonSchemaDocumentActionProvider.DELETE), ConditionContainer(emptyList()), role!!), - Permission(UUID.randomUUID(), SearchField::class.java, - mutableListOf(SearchFieldActionProvider.VIEW_LIST), ConditionContainer(emptyList()), role!!), - Permission(UUID.randomUUID(), JsonSchemaDocumentDefinition::class.java, - mutableListOf(JsonSchemaDocumentDefinitionActionProvider.VIEW), ConditionContainer(emptyList()), role!!), - Permission(UUID.randomUUID(), JsonSchemaDocumentDefinition::class.java, - mutableListOf(JsonSchemaDocumentDefinitionActionProvider.VIEW_LIST), ConditionContainer(emptyList()), role!!), - Permission(UUID.randomUUID(), JsonSchemaDocumentDefinition::class.java, - mutableListOf(JsonSchemaDocumentDefinitionActionProvider.CREATE), ConditionContainer(emptyList()), role!!), - Permission(UUID.randomUUID(), JsonSchemaDocumentDefinition::class.java, - mutableListOf(JsonSchemaDocumentDefinitionActionProvider.MODIFY), ConditionContainer(emptyList()), role!!), - Permission(UUID.randomUUID(), JsonSchemaDocumentDefinition::class.java, - mutableListOf(JsonSchemaDocumentDefinitionActionProvider.DELETE), ConditionContainer(emptyList()), role!!), - Permission(UUID.randomUUID(), JsonSchemaDocumentSnapshot::class.java, - mutableListOf(JsonSchemaDocumentSnapshotActionProvider.VIEW_LIST), ConditionContainer(emptyList()), role!!), - ) - permissionRepository.saveAll(permissions) - } - - companion object { - const val FULL_ACCESS_ROLE: String = "full access role" - const val USERNAME: String = "test@test.com" - } -} diff --git a/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/TestApplication.kt b/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/TestApplication.kt deleted file mode 100644 index ba16e71080..0000000000 --- a/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/TestApplication.kt +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2015-2025 Ritense BV, the Netherlands. - * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.ritense.document.mongodb - -import org.springframework.boot.autoconfigure.SpringBootApplication - -@SpringBootApplication -class TestApplication diff --git a/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/service/ContentTextExtractorTest.kt b/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/service/ContentTextExtractorTest.kt deleted file mode 100644 index d51fa05a8e..0000000000 --- a/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/service/ContentTextExtractorTest.kt +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright 2015-2025 Ritense BV, the Netherlands. - * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.ritense.document.mongodb.service - -import com.fasterxml.jackson.databind.ObjectMapper -import org.assertj.core.api.Assertions.assertThat -import org.junit.jupiter.api.Test - -class ContentTextExtractorTest { - - private val mapper = ObjectMapper() - - @Test - fun `null input returns null`() { - assertThat(extractLeafValues(null)).isNull() - } - - @Test - fun `empty object returns null`() { - val node = mapper.readTree("{}") - assertThat(extractLeafValues(node)).isNull() - } - - @Test - fun `flat object joins all leaf values`() { - val node = mapper.readTree("""{"firstName":"John","lastName":"Doe"}""") - val result = extractLeafValues(node) - assertThat(result).contains("John") - assertThat(result).contains("Doe") - } - - @Test - fun `nested object extracts leaves recursively`() { - val node = mapper.readTree("""{"person":{"name":"Alice","city":"Utrecht"}}""") - val result = extractLeafValues(node) - assertThat(result).contains("Alice") - assertThat(result).contains("Utrecht") - } - - @Test - fun `array of primitives is extracted`() { - val node = mapper.readTree("""["apple","banana","cherry"]""") - assertThat(extractLeafValues(node)).isEqualTo("apple banana cherry") - } - - @Test - fun `array of objects extracts nested leaves`() { - val node = mapper.readTree("""[{"name":"X"},{"name":"Y"}]""") - val result = extractLeafValues(node) - assertThat(result).contains("X") - assertThat(result).contains("Y") - } - - @Test - fun `null json field values are skipped`() { - val node = mapper.readTree("""{"name":null,"city":null}""") - assertThat(extractLeafValues(node)).isNull() - } - - @Test - fun `numeric value is converted to string`() { - val node = mapper.readTree("""{"count":42}""") - assertThat(extractLeafValues(node)).isEqualTo("42") - } - - @Test - fun `boolean value is converted to string`() { - val node = mapper.readTree("""{"active":true}""") - assertThat(extractLeafValues(node)).isEqualTo("true") - } - - @Test - fun `mixed types in object are all extracted`() { - val node = mapper.readTree("""{"name":"Bob","age":30,"active":false}""") - val result = extractLeafValues(node) - assertThat(result).contains("Bob") - assertThat(result).contains("30") - assertThat(result).contains("false") - } - - @Test - fun `deeply nested structure is fully extracted`() { - val node = mapper.readTree("""{"a":{"b":{"c":"deep"}}}""") - assertThat(extractLeafValues(node)).isEqualTo("deep") - } - - @Test - fun `mixed null and non-null leaves only includes non-null values`() { - val node = mapper.readTree("""{"name":"Alice","missing":null}""") - val result = extractLeafValues(node) - assertThat(result).isEqualTo("Alice") - } -} diff --git a/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/service/DocumentMongoSyncServiceTest.kt b/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/service/DocumentMongoSyncServiceTest.kt deleted file mode 100644 index 037298b676..0000000000 --- a/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/service/DocumentMongoSyncServiceTest.kt +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright 2015-2025 Ritense BV, the Netherlands. - * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.ritense.document.mongodb.service - -import com.fasterxml.jackson.databind.ObjectMapper -import com.ritense.document.mongodb.domain.JsonSchemaDocumentDocument -import com.ritense.document.mongodb.repository.JsonSchemaDocumentMongoRepository -import com.ritense.inbox.ValtimoEvent -import org.assertj.core.api.Assertions.assertThat -import org.junit.jupiter.api.BeforeEach -import org.junit.jupiter.api.Test -import org.mockito.ArgumentCaptor -import org.mockito.kotlin.any -import org.mockito.kotlin.capture -import org.mockito.kotlin.mock -import org.mockito.kotlin.never -import org.mockito.kotlin.verify -import org.mockito.kotlin.whenever -import java.time.LocalDateTime - -class DocumentMongoSyncServiceTest { - - private val repository: JsonSchemaDocumentMongoRepository = mock() - private val objectMapper: ObjectMapper = mock() - private lateinit var service: DocumentMongoSyncService - - @BeforeEach - fun setUp() { - service = DocumentMongoSyncService(repository, objectMapper) - } - - @Test - fun `upsert with null result skips repository save`() { - val event = valtimoEvent(result = null) - - service.upsert(event) - - verify(repository, never()).save(any()) - } - - @Test - fun `upsert populates contentText with leaf values from content`() { - val realMapper = ObjectMapper() - val content = realMapper.createObjectNode().apply { - put("firstName", "John") - put("city", "Amsterdam") - } - val docDocument = buildDocument(id = "test-id", content = content) - val event = valtimoEvent(result = realMapper.createObjectNode()) - whenever(objectMapper.treeToValue(any(), any>())).thenReturn(docDocument) - - val captor = ArgumentCaptor.forClass(JsonSchemaDocumentDocument::class.java) - service.upsert(event) - verify(repository).save(capture(captor)) - - val saved = captor.value - assertThat(saved.contentText).contains("John") - assertThat(saved.contentText).contains("Amsterdam") - } - - @Test - fun `upsert with null content stores null contentText`() { - val realMapper = ObjectMapper() - val docDocument = buildDocument(id = "no-content-id", content = null) - val event = valtimoEvent(result = realMapper.createObjectNode()) - whenever(objectMapper.treeToValue(any(), any>())).thenReturn(docDocument) - - val captor = ArgumentCaptor.forClass(JsonSchemaDocumentDocument::class.java) - service.upsert(event) - verify(repository).save(capture(captor)) - - assertThat(captor.value.contentText).isNull() - } - - @Test - fun `upsert with nested content extracts all leaf values`() { - val realMapper = ObjectMapper() - val content = realMapper.createObjectNode().apply { - putObject("address").apply { - put("street", "Main Street") - put("number", "42") - } - } - val docDocument = buildDocument(id = "nested-id", content = content) - val event = valtimoEvent(result = realMapper.createObjectNode()) - whenever(objectMapper.treeToValue(any(), any>())).thenReturn(docDocument) - - val captor = ArgumentCaptor.forClass(JsonSchemaDocumentDocument::class.java) - service.upsert(event) - verify(repository).save(capture(captor)) - - val contentText = captor.value.contentText - assertThat(contentText).contains("Main Street") - assertThat(contentText).contains("42") - } - - private fun buildDocument( - id: String, - content: com.fasterxml.jackson.databind.node.ObjectNode?, - ) = JsonSchemaDocumentDocument( - id = id, - content = content, - definitionId = null, - createdOn = null, - modifiedOn = null, - createdBy = null, - sequence = null, - version = null, - assigneeId = null, - assigneeFullName = null, - internalStatus = null, - caseTags = null, - relations = null, - relatedFiles = null, - retentionDate = null, - ) - - private fun valtimoEvent( - result: com.fasterxml.jackson.databind.node.ContainerNode<*>?, - ) = ValtimoEvent( - id = "event-id", - type = "DOCUMENT_CREATED", - date = LocalDateTime.now(), - userId = null, - roles = null, - resultType = null, - resultId = "doc-id", - result = result, - ) -} diff --git a/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/service/JsonSchemaDocumentMongoSearchServiceIntTest.kt b/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/service/JsonSchemaDocumentMongoSearchServiceIntTest.kt deleted file mode 100644 index 14cbc846e1..0000000000 --- a/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/service/JsonSchemaDocumentMongoSearchServiceIntTest.kt +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright 2015-2025 Ritense BV, the Netherlands. - * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.ritense.document.mongodb.service - -import com.fasterxml.jackson.databind.node.ObjectNode -import com.ritense.authorization.AuthorizationContext.Companion.runWithoutAuthorization -import com.ritense.document.domain.impl.JsonSchemaDocument -import com.ritense.document.domain.impl.request.NewDocumentRequest -import com.ritense.document.domain.search.AdvancedSearchRequest -import com.ritense.document.mongodb.BaseMongoIntegrationTest -import com.ritense.document.mongodb.domain.JsonSchemaDocumentDocument -import com.ritense.document.service.DocumentSearchService -import com.ritense.valtimo.contract.blueprint.BlueprintType -import org.assertj.core.api.Assertions.assertThat -import org.junit.jupiter.api.Test -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.data.domain.PageRequest -import org.springframework.security.test.context.support.WithMockUser - -@WithMockUser(username = BaseMongoIntegrationTest.USERNAME, authorities = [BaseMongoIntegrationTest.FULL_ACCESS_ROLE]) -class JsonSchemaDocumentMongoSearchServiceIntTest : BaseMongoIntegrationTest() { - - @Autowired - lateinit var documentSearchService: DocumentSearchService - - @Test - fun `globalSearchFilter returns matching document`() { - seedDocument("Funenpark") - - val page = documentSearchService.search( - "house", - BlueprintType.CASE, - AdvancedSearchRequest().globalSearchFilter("Funenpark"), - PageRequest.of(0, 10) - ) - - assertThat(page.totalElements).isEqualTo(1L) - } - - @Test - fun `globalSearchFilter is case insensitive`() { - seedDocument("Funenpark") - - val page = documentSearchService.search( - "house", - BlueprintType.CASE, - AdvancedSearchRequest().globalSearchFilter("FUNENPARK"), - PageRequest.of(0, 10) - ) - - assertThat(page.totalElements).isEqualTo(1L) - } - - @Test - fun `globalSearchFilter excludes non-matching documents`() { - val docA = seedDocument("Funenpark") - seedDocument("Keizersgracht") - - val page = documentSearchService.search( - "house", - BlueprintType.CASE, - AdvancedSearchRequest().globalSearchFilter("Funenpark"), - PageRequest.of(0, 10) - ) - - assertThat(page.totalElements).isEqualTo(1L) - assertThat(page.content[0].id()).isEqualTo(docA.id()) - } - - @Test - fun `no globalSearchFilter returns all authorized documents`() { - seedDocument("Funenpark") - seedDocument("Keizersgracht") - - val page = documentSearchService.search( - "house", - BlueprintType.CASE, - AdvancedSearchRequest(), - PageRequest.of(0, 10) - ) - - assertThat(page.totalElements).isEqualTo(2L) - } - - @Test - fun `globalSearchFilter supports partial match`() { - seedDocument("Keizersgracht") - - val page = documentSearchService.search( - "house", - BlueprintType.CASE, - AdvancedSearchRequest().globalSearchFilter("Keizers"), - PageRequest.of(0, 10) - ) - - assertThat(page.totalElements).isEqualTo(1L) - } - - private fun seedDocument(street: String): JsonSchemaDocument { - val content = objectMapper.createObjectNode().apply { put("street", street) } - val jpaDoc = runWithoutAuthorization { - documentService.createDocument( - NewDocumentRequest("house", "house", "1.0.0", content) - ).resultingDocument().get() - } - mongoRepository.save( - JsonSchemaDocumentDocument( - id = jpaDoc.id().toString(), - content = content as ObjectNode, - definitionId = objectMapper.readTree( - """{"name":"house","blueprintId":{"blueprintType":"CASE"}}""" - ), - createdOn = null, - modifiedOn = null, - createdBy = null, - sequence = null, - version = null, - assigneeId = null, - assigneeFullName = null, - internalStatus = null, - caseTags = null, - relations = null, - relatedFiles = null, - retentionDate = null, - contentText = street, - ) - ) - return jpaDoc - } -} diff --git a/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/service/JsonSchemaDocumentMongoSearchServiceTest.kt b/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/service/JsonSchemaDocumentMongoSearchServiceTest.kt deleted file mode 100644 index b00a807715..0000000000 --- a/backend/case-mongodb/src/test/kotlin/com/ritense/document/mongodb/service/JsonSchemaDocumentMongoSearchServiceTest.kt +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Copyright 2015-2025 Ritense BV, the Netherlands. - * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.ritense.document.mongodb.service - -import com.fasterxml.jackson.databind.ObjectMapper -import com.ritense.authorization.Action -import com.ritense.authorization.AuthorizationService -import com.ritense.authorization.permission.ConditionContainer -import com.ritense.authorization.permission.Permission -import com.ritense.authorization.role.Role -import com.ritense.document.domain.impl.JsonSchemaDocument -import com.ritense.document.domain.search.AdvancedSearchRequest -import com.ritense.document.mongodb.authorization.MongoAuthorizationEntityMapper -import com.ritense.document.mongodb.authorization.MongoPermissionConditionTranslator -import com.ritense.document.mongodb.domain.JsonSchemaDocumentDocument -import com.ritense.document.repository.impl.JsonSchemaDocumentRepository -import com.ritense.document.service.JsonSchemaDocumentActionProvider -import com.ritense.document.service.SearchFieldService -import com.ritense.outbox.OutboxService -import com.ritense.valtimo.contract.authentication.UserManagementService -import com.ritense.valtimo.contract.blueprint.BlueprintType -import org.assertj.core.api.Assertions.assertThat -import org.junit.jupiter.api.AfterEach -import org.junit.jupiter.api.BeforeEach -import org.junit.jupiter.api.Test -import org.mockito.kotlin.any -import org.mockito.kotlin.argumentCaptor -import org.mockito.kotlin.eq -import org.mockito.kotlin.mock -import org.mockito.kotlin.whenever -import org.springframework.data.domain.PageRequest -import org.springframework.data.mongodb.core.MongoTemplate -import org.springframework.data.mongodb.core.query.Query -import org.springframework.security.authentication.UsernamePasswordAuthenticationToken -import org.springframework.security.core.authority.SimpleGrantedAuthority -import org.springframework.security.core.context.SecurityContextHolder - -class JsonSchemaDocumentMongoSearchServiceTest { - - private val mongoTemplate: MongoTemplate = mock() - private val authorizationService: AuthorizationService = mock() - private val jpaRepository: JsonSchemaDocumentRepository = mock() - private val userManagementService: UserManagementService = mock() - private val searchFieldService: SearchFieldService = mock() - private val outboxService: OutboxService = mock() - private val objectMapper: ObjectMapper = ObjectMapper() - - private lateinit var service: JsonSchemaDocumentMongoSearchService - - @BeforeEach - fun setUp() { - val translator = MongoPermissionConditionTranslator( - mongoMappers = emptyList>(), - authorizationService = authorizationService, - documentRepository = jpaRepository, - ) - service = JsonSchemaDocumentMongoSearchService( - mongoTemplate = mongoTemplate, - translator = translator, - authorizationService = authorizationService, - jpaRepository = jpaRepository, - userManagementService = userManagementService, - searchFieldService = searchFieldService, - outboxService = outboxService, - objectMapper = objectMapper, - ) - - val auth = UsernamePasswordAuthenticationToken( - USERNAME, - null, - listOf(SimpleGrantedAuthority(FULL_ACCESS_ROLE)), - ) - SecurityContextHolder.getContext().authentication = auth - - val role = Role(key = FULL_ACCESS_ROLE) - val viewListPermission = Permission( - resourceType = JsonSchemaDocument::class.java, - actions = mutableListOf(JsonSchemaDocumentActionProvider.VIEW_LIST), - conditionContainer = ConditionContainer(emptyList()), - role = role, - ) - whenever( - authorizationService.getPermissions( - eq(JsonSchemaDocument::class.java), - eq(JsonSchemaDocumentActionProvider.VIEW_LIST), - ) - ).thenReturn(listOf(viewListPermission)) - - whenever(mongoTemplate.count(any(), eq(JsonSchemaDocumentDocument::class.java))).thenReturn(0L) - whenever(mongoTemplate.find(any(), eq(JsonSchemaDocumentDocument::class.java))).thenReturn(emptyList()) - whenever(jpaRepository.findAllById(any())).thenReturn(emptyList()) - } - - @AfterEach - fun tearDown() { - SecurityContextHolder.clearContext() - } - - @Test - fun `search with globalSearchFilter adds contentText regex to query`() { - val queryCaptor = argumentCaptor() - whenever(mongoTemplate.count(queryCaptor.capture(), eq(JsonSchemaDocumentDocument::class.java))).thenReturn(0L) - - val request = AdvancedSearchRequest().globalSearchFilter("Amsterdam") - service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) - - val queryJson = queryCaptor.firstValue.queryObject.toJson() - assertThat(queryJson).contains("contentText") - } - - @Test - fun `search without globalSearchFilter does not include contentText criterion`() { - val queryCaptor = argumentCaptor() - whenever(mongoTemplate.count(queryCaptor.capture(), eq(JsonSchemaDocumentDocument::class.java))).thenReturn(0L) - - val request = AdvancedSearchRequest() - service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) - - val queryJson = queryCaptor.firstValue.queryObject.toJson() - assertThat(queryJson).doesNotContain("contentText") - } - - @Test - fun `search with empty globalSearchFilter does not include contentText criterion`() { - val queryCaptor = argumentCaptor() - whenever(mongoTemplate.count(queryCaptor.capture(), eq(JsonSchemaDocumentDocument::class.java))).thenReturn(0L) - - val request = AdvancedSearchRequest().globalSearchFilter("") - service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) - - val queryJson = queryCaptor.firstValue.queryObject.toJson() - assertThat(queryJson).doesNotContain("contentText") - } - - @Test - fun `search with globalSearchFilter uses case-insensitive regex`() { - val queryCaptor = argumentCaptor() - whenever(mongoTemplate.count(queryCaptor.capture(), eq(JsonSchemaDocumentDocument::class.java))).thenReturn(0L) - - val request = AdvancedSearchRequest().globalSearchFilter("john") - service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) - - val queryJson = queryCaptor.firstValue.queryObject.toJson() - assertThat(queryJson).contains("contentText") - assertThat(queryJson).contains("options") - assertThat(queryJson).contains("\"i\"") // case-insensitive flag - } - - @Test - fun `search result uses count from mongodb`() { - whenever(mongoTemplate.count(any(), eq(JsonSchemaDocumentDocument::class.java))).thenReturn(5L) - whenever(mongoTemplate.find(any(), eq(JsonSchemaDocumentDocument::class.java))).thenReturn(emptyList()) - - val request = AdvancedSearchRequest().globalSearchFilter("test") - val page = service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) - - assertThat(page.totalElements).isEqualTo(5L) - } - - companion object { - private const val FULL_ACCESS_ROLE = "full access role" - private const val USERNAME = "test@test.com" - } -} diff --git a/backend/case-mongodb/src/test/resources/config/application-postgresql.yml b/backend/case-mongodb/src/test/resources/config/application-postgresql.yml deleted file mode 100644 index 3543dd40ab..0000000000 --- a/backend/case-mongodb/src/test/resources/config/application-postgresql.yml +++ /dev/null @@ -1,17 +0,0 @@ -spring: - datasource: - driver-class-name: org.postgresql.Driver - url: jdbc:postgresql://localhost:3364/case-mongodb-test - username: valtimo - password: password - hikari: - auto-commit: false - jpa: - database-platform: org.hibernate.dialect.PostgreSQLDialect - database: postgresql - data: - mongodb: - uri: mongodb://localhost:37017/case-mongodb-test - -valtimo: - database: postgres diff --git a/backend/case-mongodb/src/test/resources/config/application.yml b/backend/case-mongodb/src/test/resources/config/application.yml deleted file mode 100644 index e68db94e75..0000000000 --- a/backend/case-mongodb/src/test/resources/config/application.yml +++ /dev/null @@ -1,35 +0,0 @@ -spring: - datasource: - type: com.zaxxer.hikari.HikariDataSource - liquibase: - enabled: false - jpa: - show_sql: false - open-in-view: false - properties: - hibernate: - hbm2ddl.auto: none - format_sql: true - jdbc: - time_zone: UTC - connection: - provider_disables_autocommit: true - hibernate: - ddl-auto: none - -spring-actuator: - username: test - password: test - -valtimo: - versioning: - enabled: false - plugin: - encryption-secret: "abcdefghijklmnop" - -operaton: - bpm: - history-level: audit - generic-properties: - properties: - enforceHistoryTimeToLive: false diff --git a/backend/case-mongodb/src/test/resources/config/case/house/1-0-0/case/definition/house.case-definition.json b/backend/case-mongodb/src/test/resources/config/case/house/1-0-0/case/definition/house.case-definition.json deleted file mode 100644 index 994804cbd1..0000000000 --- a/backend/case-mongodb/src/test/resources/config/case/house/1-0-0/case/definition/house.case-definition.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "key": "house", - "name": "House", - "versionTag": "1.0.0", - "canHaveAssignee": true, - "autoAssignTasks": true -} \ No newline at end of file diff --git a/backend/case-mongodb/src/test/resources/config/case/house/1-0-0/case/internal-status/house.internal-case-status.json b/backend/case-mongodb/src/test/resources/config/case/house/1-0-0/case/internal-status/house.internal-case-status.json deleted file mode 100644 index b69712429a..0000000000 --- a/backend/case-mongodb/src/test/resources/config/case/house/1-0-0/case/internal-status/house.internal-case-status.json +++ /dev/null @@ -1,20 +0,0 @@ -[ - { - "key": "suspended", - "title": "Suspended", - "visibleInCaseListByDefault": false, - "color": "GRAY" - }, - { - "key": "closed", - "title": "Closed", - "visibleInCaseListByDefault": false, - "color": "GRAY" - }, - { - "key": "started", - "title": "Started", - "visibleInCaseListByDefault": true, - "color": "GRAY" - } -] diff --git a/backend/case-mongodb/src/test/resources/config/case/house/1-0-0/case/list/house.case-list.json b/backend/case-mongodb/src/test/resources/config/case/house/1-0-0/case/list/house.case-list.json deleted file mode 100644 index fe51488c70..0000000000 --- a/backend/case-mongodb/src/test/resources/config/case/house/1-0-0/case/list/house.case-list.json +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/backend/case-mongodb/src/test/resources/config/case/house/1-0-0/case/search-field/house.case-search-field.json b/backend/case-mongodb/src/test/resources/config/case/house/1-0-0/case/search-field/house.case-search-field.json deleted file mode 100644 index 3f0b73c7b8..0000000000 --- a/backend/case-mongodb/src/test/resources/config/case/house/1-0-0/case/search-field/house.case-search-field.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "searchFields": [ - { - "key": "buildDate", - "path": "doc:buildDate", - "dataType": "date", - "fieldType": "single", - "matchType": "exact" - }, - { - "key": "buildDates", - "path": "doc:buildDate", - "dataType": "date", - "fieldType": "range", - "matchType": "exact" - } - ] -} diff --git a/backend/case-mongodb/src/test/resources/config/case/house/1-0-0/document/definition/house.schema.document-definition.json b/backend/case-mongodb/src/test/resources/config/case/house/1-0-0/document/definition/house.schema.document-definition.json deleted file mode 100644 index 965fb0c585..0000000000 --- a/backend/case-mongodb/src/test/resources/config/case/house/1-0-0/document/definition/house.schema.document-definition.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "$id": "house.schema", - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "House", - "type": "object", - "properties": { - "street": { - "type": "string", - "description": "The street name.", - "maxLength": 100 - }, - "housenumber": { - "description": "house number must be equal to or greater than zero.", - "type": "integer", - "minimum": 0 - }, - "buildDate": { - "type": "string", - "description": "The house's build date.", - "maxLength": 100 - }, - "userInfo": { - "type": "string", - "description": "Additional information on the user", - "maxLength": 100 - }, - "loan-approved": { - "type": "boolean", - "description": "Was the loan for the house approved" - } - }, - "additionalProperties": false -} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt index 87ff4a7a6a..5ba346b5ae 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt @@ -27,7 +27,7 @@ import com.ritense.document.opensearch.authorization.OpenSearchPermissionConditi import com.ritense.document.opensearch.authorization.mapper.JsonSchemaDocumentCaseDefinitionOpenSearchMapper import com.ritense.document.opensearch.authorization.mapper.JsonSchemaDocumentDefinitionOpenSearchMapper import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument -import com.ritense.document.opensearch.handler.DocumentOpenSearchEventHandler +import com.ritense.document.opensearch.handler.DocumentOpenSearchEventListener import com.ritense.document.opensearch.repository.JsonSchemaDocumentOpenSearchRepository import com.ritense.document.opensearch.security.DocumentOpenSearchHttpSecurityConfigurer import com.ritense.document.opensearch.service.DelegatingDocumentSearchService @@ -103,8 +103,8 @@ class DocumentOpenSearchAutoConfiguration { DocumentOpenSearchSyncService(repository, objectMapper) @Bean - fun documentOpenSearchEventHandler(syncService: DocumentOpenSearchSyncService): DocumentOpenSearchEventHandler = - DocumentOpenSearchEventHandler(syncService) + fun documentOpenSearchEventListener(syncService: DocumentOpenSearchSyncService): DocumentOpenSearchEventListener = + DocumentOpenSearchEventListener(syncService) @Bean @ConditionalOnMissingBean diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/handler/DocumentOpenSearchEventHandler.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/handler/DocumentOpenSearchEventHandler.kt index 9b7d6b0f8a..bfe651b511 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/handler/DocumentOpenSearchEventHandler.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/handler/DocumentOpenSearchEventHandler.kt @@ -18,53 +18,45 @@ package com.ritense.document.opensearch.handler import com.ritense.document.event.DocumentAssigned import com.ritense.document.event.DocumentCreated +import com.ritense.document.event.DocumentDeleted +import com.ritense.document.event.DocumentRetentionDateSet +import com.ritense.document.event.DocumentRetentionDateUnset +import com.ritense.document.event.DocumentStatusChanged +import com.ritense.document.event.DocumentTagsChanged import com.ritense.document.event.DocumentUnassigned import com.ritense.document.event.DocumentUpdated import com.ritense.document.opensearch.service.DocumentOpenSearchSyncService -import com.ritense.inbox.ValtimoEvent -import com.ritense.inbox.ValtimoEventHandler -import io.github.oshai.kotlinlogging.KotlinLogging +import org.springframework.context.event.EventListener -/** - * Listens to document domain events from the Valtimo inbox and keeps the OpenSearch read - * model in sync. Works with both the outbox-enabled (RabbitMQ) and outbox-disabled - * (local Spring event) modes because both paths converge on [ValtimoEventHandler]. - */ -class DocumentOpenSearchEventHandler( +class DocumentOpenSearchEventListener( private val syncService: DocumentOpenSearchSyncService, -) : ValtimoEventHandler { +) { + @EventListener + fun onDocumentCreated(event: DocumentCreated) = syncService.upsert(event) - override fun handle(event: ValtimoEvent) { - when (event.type) { - in UPSERT_EVENT_TYPES -> syncService.upsert(event) - DELETED_EVENT_TYPE -> { - val id = event.resultId - if (id != null) { - syncService.delete(id) - } else { - logger.warn { "Received DocumentDeleted event with null resultId — skipping delete" } - } - } - else -> { - // Events not related to json_schema_document (e.g. DocumentsListed) are ignored - } - } - } + @EventListener + fun onDocumentUpdated(event: DocumentUpdated) = syncService.upsert(event) + + @EventListener + fun onDocumentAssigned(event: DocumentAssigned) = syncService.upsert(event) + + @EventListener + fun onDocumentUnassigned(event: DocumentUnassigned) = syncService.upsert(event) + + @EventListener + fun onDocumentStatusChanged(event: DocumentStatusChanged) = syncService.upsert(event) + + @EventListener + fun onDocumentTagsChanged(event: DocumentTagsChanged) = syncService.upsert(event) - companion object { - private val logger = KotlinLogging.logger {} + @EventListener + fun onDocumentRetentionDateSet(event: DocumentRetentionDateSet) = syncService.upsert(event) - val UPSERT_EVENT_TYPES: Set = setOf( - DocumentCreated.TYPE, - DocumentUpdated.TYPE, - DocumentAssigned.TYPE, - DocumentUnassigned.TYPE, - "com.ritense.valtimo.document.status.changed", - "com.ritense.valtimo.document.tags.changed", - "com.ritense.valtimo.document.retentiondate.set", - "com.ritense.valtimo.document.retentiondate.unset", - ) + @EventListener + fun onDocumentRetentionDateUnset(event: DocumentRetentionDateUnset) = syncService.upsert(event) - const val DELETED_EVENT_TYPE = "com.ritense.valtimo.document.deleted" + @EventListener + fun onDocumentDeleted(event: DocumentDeleted) { + event.resultId?.let { syncService.delete(it) } } } diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchSyncService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchSyncService.kt index 1bd0c6a00a..e7b40f5c2c 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchSyncService.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchSyncService.kt @@ -17,9 +17,10 @@ package com.ritense.document.opensearch.service import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.ContainerNode import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument import com.ritense.document.opensearch.repository.JsonSchemaDocumentOpenSearchRepository -import com.ritense.inbox.ValtimoEvent +import com.ritense.outbox.domain.BaseEvent import io.github.oshai.kotlinlogging.KotlinLogging class DocumentOpenSearchSyncService( @@ -27,16 +28,20 @@ class DocumentOpenSearchSyncService( private val objectMapper: ObjectMapper, ) { - fun upsert(event: ValtimoEvent) { + fun upsert(event: BaseEvent) { val result = event.result if (result == null) { logger.warn { "Received document event ${event.type} for id=${event.resultId} with null result — skipping upsert" } return } + upsertFromResult(result, event.type) + } + + private fun upsertFromResult(result: ContainerNode<*>, eventType: String) { val doc = objectMapper.treeToValue(result, JsonSchemaDocumentOsDocument::class.java) val contentText = extractLeafValues(result.get("content")) repository.save(doc.copy(contentText = contentText)) - logger.debug { "Upserted document ${doc.id} in OpenSearch (event: ${event.type})" } + logger.debug { "Upserted document ${doc.id} in OpenSearch (event: $eventType)" } } fun delete(documentId: String) { diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchSyncServiceTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchSyncServiceTest.kt index 02fb5b748d..c809b373aa 100644 --- a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchSyncServiceTest.kt +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchSyncServiceTest.kt @@ -17,9 +17,11 @@ package com.ritense.document.opensearch.service import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.ContainerNode +import com.ritense.document.event.DocumentCreated import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument import com.ritense.document.opensearch.repository.JsonSchemaDocumentOpenSearchRepository -import com.ritense.inbox.ValtimoEvent +import com.ritense.outbox.domain.BaseEvent import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -30,7 +32,6 @@ import org.mockito.kotlin.mock import org.mockito.kotlin.never import org.mockito.kotlin.verify import org.mockito.kotlin.whenever -import java.time.LocalDateTime class DocumentOpenSearchSyncServiceTest { @@ -45,7 +46,7 @@ class DocumentOpenSearchSyncServiceTest { @Test fun `upsert with null result skips repository save`() { - val event = valtimoEvent(result = null) + val event = testEvent(result = null) service.upsert(event) @@ -63,7 +64,7 @@ class DocumentOpenSearchSyncServiceTest { set("content", content) } val doc = buildDocument(id = "test-id", content = mapOf("firstName" to "John", "city" to "Amsterdam")) - val event = valtimoEvent(result = resultNode) + val event = testEvent(result = resultNode) whenever(objectMapper.treeToValue(any(), any>())).thenReturn(doc) val captor = ArgumentCaptor.forClass(JsonSchemaDocumentOsDocument::class.java) @@ -80,7 +81,7 @@ class DocumentOpenSearchSyncServiceTest { val realMapper = ObjectMapper() val resultNode = realMapper.createObjectNode() val doc = buildDocument(id = "no-content-id", content = null) - val event = valtimoEvent(result = resultNode) + val event = testEvent(result = resultNode) whenever(objectMapper.treeToValue(any(), any>())).thenReturn(doc) val captor = ArgumentCaptor.forClass(JsonSchemaDocumentOsDocument::class.java) @@ -103,7 +104,7 @@ class DocumentOpenSearchSyncServiceTest { set("content", content) } val doc = buildDocument(id = "nested-id", content = mapOf("address" to mapOf("street" to "Main Street", "number" to "42"))) - val event = valtimoEvent(result = resultNode) + val event = testEvent(result = resultNode) whenever(objectMapper.treeToValue(any(), any>())).thenReturn(doc) val captor = ArgumentCaptor.forClass(JsonSchemaDocumentOsDocument::class.java) @@ -136,16 +137,15 @@ class DocumentOpenSearchSyncServiceTest { retentionDate = null, ) - private fun valtimoEvent( - result: com.fasterxml.jackson.databind.node.ContainerNode<*>?, - ) = ValtimoEvent( - id = "event-id", - type = "DOCUMENT_CREATED", - date = LocalDateTime.now(), - userId = null, - roles = null, - resultType = null, - resultId = "doc-id", - result = result, - ) + private fun testEvent(result: ContainerNode<*>?): BaseEvent = + if (result != null) { + DocumentCreated("doc-id", result as com.fasterxml.jackson.databind.node.ObjectNode) + } else { + object : BaseEvent( + type = "test", + resultType = null, + resultId = "doc-id", + result = null, + ) {} + } } diff --git a/backend/dependencies/valtimo-dependency-versions/build.gradle b/backend/dependencies/valtimo-dependency-versions/build.gradle index 6ffca5e6c6..056e920396 100644 --- a/backend/dependencies/valtimo-dependency-versions/build.gradle +++ b/backend/dependencies/valtimo-dependency-versions/build.gradle @@ -28,7 +28,6 @@ dependencies { api(project(":backend:authorization")) api(project(":backend:building-block")) api(project(":backend:case")) - api(project(":backend:case-mongodb")) api(project(":backend:case-opensearch")) api(project(":backend:changelog")) api(project(":backend:command-handling")) diff --git a/settings.gradle b/settings.gradle index 512c235292..16784865eb 100644 --- a/settings.gradle +++ b/settings.gradle @@ -35,7 +35,6 @@ include( ":backend:authorization", ":backend:building-block", ":backend:case", - ":backend:case-mongodb", ":backend:case-opensearch", ":backend:changelog", ":backend:command-handling", From 71a19d45a5bd803d52bff5d989a925024086f2a7 Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Tue, 30 Jun 2026 18:22:39 +0200 Subject: [PATCH 15/46] fixed tests --- .../BaseOpenSearchIntegrationTest.kt | 21 +++++++++++++++++-- .../document/opensearch/TestApplication.kt | 13 +++++++++++- ...nSchemaDocumentOpenSearchServiceIntTest.kt | 1 + .../search-field/house.case-search-field.json | 7 +++++++ 4 files changed, 39 insertions(+), 3 deletions(-) diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/BaseOpenSearchIntegrationTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/BaseOpenSearchIntegrationTest.kt index 41105b629b..624ac327a2 100644 --- a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/BaseOpenSearchIntegrationTest.kt +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/BaseOpenSearchIntegrationTest.kt @@ -32,11 +32,14 @@ import com.ritense.document.service.JsonSchemaDocumentActionProvider import com.ritense.document.service.JsonSchemaDocumentDefinitionActionProvider import com.ritense.document.service.JsonSchemaDocumentSnapshotActionProvider import com.ritense.document.service.SearchFieldActionProvider +import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument import com.ritense.outbox.OutboxService import com.ritense.testutilscommon.junit.extension.LiquibaseRunnerExtension +import com.ritense.valtimo.contract.authentication.TeamManagementService import com.ritense.valtimo.contract.authentication.UserManagementService import com.ritense.valtimo.contract.mail.MailSender import com.ritense.valtimo.service.ProcessDefinitionCaseDefinitionLinker +import org.springframework.data.elasticsearch.core.ElasticsearchOperations import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Tag @@ -60,6 +63,9 @@ abstract class BaseOpenSearchIntegrationTest { @MockitoBean(answers = Answers.RETURNS_DEEP_STUBS) lateinit var userManagementService: UserManagementService + @MockitoBean + lateinit var teamManagementService: TeamManagementService + @MockitoBean lateinit var applicationEventMulticaster: SimpleApplicationEventMulticaster @@ -90,15 +96,26 @@ abstract class BaseOpenSearchIntegrationTest { @Autowired lateinit var objectMapper: ObjectMapper + @Autowired + lateinit var elasticsearchOperations: ElasticsearchOperations + @BeforeEach fun setUpBase() { setUpPermissions() - openSearchRepository.deleteAll() + val indexOps = elasticsearchOperations.indexOps(JsonSchemaDocumentOsDocument::class.java) + if (indexOps.exists()) { + indexOps.delete() + } + indexOps.create() + indexOps.putMapping(indexOps.createMapping()) } @AfterEach fun tearDownBase() { - openSearchRepository.deleteAll() + val indexOps = elasticsearchOperations.indexOps(JsonSchemaDocumentOsDocument::class.java) + if (indexOps.exists()) { + indexOps.delete() + } } private fun setUpPermissions() { diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/TestApplication.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/TestApplication.kt index 8ac18acfac..89e160c1d8 100644 --- a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/TestApplication.kt +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/TestApplication.kt @@ -17,6 +17,17 @@ package com.ritense.document.opensearch import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.boot.autoconfigure.data.elasticsearch.ElasticsearchDataAutoConfiguration +import org.springframework.boot.autoconfigure.data.elasticsearch.ReactiveElasticsearchRepositoriesAutoConfiguration +import org.springframework.boot.autoconfigure.elasticsearch.ElasticsearchClientAutoConfiguration +import org.springframework.boot.autoconfigure.elasticsearch.ElasticsearchRestClientAutoConfiguration +import org.springframework.boot.autoconfigure.elasticsearch.ReactiveElasticsearchClientAutoConfiguration -@SpringBootApplication +@SpringBootApplication(exclude = [ + ElasticsearchDataAutoConfiguration::class, + ElasticsearchClientAutoConfiguration::class, + ElasticsearchRestClientAutoConfiguration::class, + ReactiveElasticsearchClientAutoConfiguration::class, + ReactiveElasticsearchRepositoriesAutoConfiguration::class, +]) class TestApplication diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceIntTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceIntTest.kt index 433bda391d..cee7a81f7e 100644 --- a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceIntTest.kt +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceIntTest.kt @@ -148,6 +148,7 @@ class JsonSchemaDocumentOpenSearchServiceIntTest : BaseOpenSearchIntegrationTest contentText = street, ) ) + elasticsearchOperations.indexOps(JsonSchemaDocumentOsDocument::class.java).refresh() return jpaDoc } } diff --git a/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/case/search-field/house.case-search-field.json b/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/case/search-field/house.case-search-field.json index 3f0b73c7b8..9f35ab434f 100644 --- a/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/case/search-field/house.case-search-field.json +++ b/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/case/search-field/house.case-search-field.json @@ -13,6 +13,13 @@ "dataType": "date", "fieldType": "range", "matchType": "exact" + }, + { + "key": "street", + "path": "doc:street", + "dataType": "text", + "fieldType": "single", + "matchType": "like" } ] } From b2d654b15dce159ae0dc71708bc5f1a148a39172 Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Thu, 2 Jul 2026 16:33:14 +0200 Subject: [PATCH 16/46] search improvements for both opensearch and the current one. Includes help with using global search, and being able to global search more fields. --- backend/app/gzac/build.gradle | 1 + .../JsonSchemaDocumentOpenSearchService.kt | 233 +++++++++++++-- ...JsonSchemaDocumentOpenSearchServiceTest.kt | 114 ++++++++ .../impl/JsonSchemaDocumentSearchService.java | 167 +++++++++-- .../case-list/case-list.component.html | 3 + .../case-list-orchestration.service.ts | 3 + .../carbon-list/carbon-list.component.html | 33 ++- .../carbon-list/carbon-list.component.scss | 78 ++++++ .../carbon-list/carbon-list.component.ts | 265 +++++++++++++++++- .../search-input-with-validation/index.ts | 17 ++ ...earch-input-with-validation.component.html | 68 +++++ ...earch-input-with-validation.component.scss | 153 ++++++++++ .../search-input-with-validation.component.ts | 147 ++++++++++ .../valtimo/components/src/public_api.ts | 3 + .../src/lib/services/document.service.ts | 38 ++- 15 files changed, 1259 insertions(+), 64 deletions(-) create mode 100644 frontend/projects/valtimo/components/src/lib/components/search-input-with-validation/index.ts create mode 100644 frontend/projects/valtimo/components/src/lib/components/search-input-with-validation/search-input-with-validation.component.html create mode 100644 frontend/projects/valtimo/components/src/lib/components/search-input-with-validation/search-input-with-validation.component.scss create mode 100644 frontend/projects/valtimo/components/src/lib/components/search-input-with-validation/search-input-with-validation.component.ts diff --git a/backend/app/gzac/build.gradle b/backend/app/gzac/build.gradle index 9dad89bf9d..64d7909ebb 100644 --- a/backend/app/gzac/build.gradle +++ b/backend/app/gzac/build.gradle @@ -83,6 +83,7 @@ dockerCompose { stopContainers = false removeContainers = false removeVolumes = false + tcpPortsToIgnoreWhenWaiting = [9600] } tasks.register("bootRunWithDocker", tasks.named("bootRun").get().class) { diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt index 6488db5c65..dc8f8ba23c 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt @@ -43,8 +43,12 @@ import com.ritense.outbox.OutboxService import com.ritense.valtimo.contract.authentication.UserManagementService import com.ritense.valtimo.contract.blueprint.BlueprintType import com.ritense.valtimo.contract.utils.RequestHelper +import com.ritense.document.domain.impl.searchfield.SearchField +import com.ritense.document.domain.impl.searchfield.SearchFieldDataType +import com.ritense.document.domain.impl.searchfield.SearchFieldMatchType import com.ritense.valtimo.contract.utils.SecurityUtils import org.apache.commons.lang3.NotImplementedException +import org.opensearch.index.query.Operator import org.opensearch.index.query.QueryBuilder import org.opensearch.index.query.QueryBuilders import org.springframework.data.domain.Page @@ -54,8 +58,10 @@ import org.springframework.data.domain.Pageable import org.springframework.data.domain.Sort import org.springframework.data.elasticsearch.core.ElasticsearchOperations import org.springframework.data.elasticsearch.core.query.StringQuery -import org.apache.lucene.queryparser.classic.QueryParser -import java.util.regex.Pattern +import java.time.Instant +import java.time.LocalDate +import java.time.ZoneOffset +import java.time.format.DateTimeFormatter class JsonSchemaDocumentOpenSearchService( private val elasticsearchOperations: ElasticsearchOperations, @@ -221,28 +227,17 @@ class JsonSchemaDocumentOpenSearchService( val globalFilter = searchRequest.globalSearchFilter?.takeIf { it.isNotEmpty() } if (globalFilter != null) { - val searchableFields = if (!documentDefinitionName.isNullOrEmpty()) { + val searchFields = if (!documentDefinitionName.isNullOrEmpty()) { runWithoutAuthorization { searchFieldService.getSearchFields(documentDefinitionName) } - .filter { it.path?.startsWith(DOC_PREFIX) == true } - .map { "content.${it.path.removePrefix(DOC_PREFIX)}" } } else { emptyList() } - if (searchableFields.isNotEmpty()) { - // query_string with lenient=true handles type mismatches gracefully - // (e.g. searching "doe" against a number field won't error, just won't match) - val escaped = QueryParser.escape(globalFilter.trim()) - parts.add( - QueryBuilders.queryStringQuery("*${escaped}*") - .apply { searchableFields.forEach { field(it) } } - .lenient(true) - .analyzeWildcard(true) - ) + if (searchFields.isNotEmpty()) { + parts.add(buildGlobalSearchQuery(globalFilter.trim(), searchFields)) } else { - // Fallback: search all content via contentText val term = "*${globalFilter.trim()}*" parts.add(QueryBuilders.wildcardQuery("contentText.keyword", term).caseInsensitive(true)) } @@ -332,11 +327,11 @@ class JsonSchemaDocumentOpenSearchService( } DatabaseSearchType.IN -> QueryBuilders.termsQuery(keywordField, filter.getValues()) DatabaseSearchType.GREATER_THAN_OR_EQUAL_TO -> - QueryBuilders.rangeQuery(baseField).gte(filter.rangeFromValue()!!) + QueryBuilders.rangeQuery(baseField).gte(formatInstantForOpenSearch(filter.rangeFromValue()!! as Instant)) DatabaseSearchType.LESS_THAN_OR_EQUAL_TO -> - QueryBuilders.rangeQuery(baseField).lte(filter.rangeToValue()!!) + QueryBuilders.rangeQuery(baseField).lte(formatInstantForOpenSearch(filter.rangeToValue()!! as Instant)) DatabaseSearchType.BETWEEN -> - QueryBuilders.rangeQuery(baseField).gte(filter.rangeFromValue()!!).lte(filter.rangeToValue()!!) + QueryBuilders.rangeQuery(baseField).gte(formatInstantForOpenSearch(filter.rangeFromValue()!! as Instant)).lte(formatInstantForOpenSearch(filter.rangeToValue()!! as Instant)) else -> throw NotImplementedException("Search type '${filter.searchType}' is not supported in the OpenSearch search service") } } @@ -357,6 +352,22 @@ class JsonSchemaDocumentOpenSearchService( return QueryBuilders.wildcardQuery(keywordField, "*${value.trim()}*").caseInsensitive(true) } + private fun formatInstantForOpenSearch(instant: Instant): String { + return DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ss.SSS") + .withZone(ZoneOffset.UTC) + .format(instant) + } + + private fun parseDateRange(dateValue: String): Pair { + val date = LocalDate.parse(dateValue) + val startOfDay = date.atStartOfDay().atZone(ZoneOffset.UTC).toInstant() + val endOfDay = date.plusDays(1).atStartOfDay().atZone(ZoneOffset.UTC).toInstant() + return Pair( + formatInstantForOpenSearch(startOfDay), + formatInstantForOpenSearch(endOfDay) + ) + } + private fun executeSearch(combinedQuery: QueryBuilder, pageable: Pageable): Page { val translatedSort = translateSort(pageable.sort) val effectivePageable = if (pageable.isPaged) { @@ -396,16 +407,192 @@ class JsonSchemaDocumentOpenSearchService( return Sort.by(orders) } + private fun buildGlobalSearchQuery(query: String, searchFields: List): QueryBuilder { + val fieldMap = searchFields.associateBy { removePrefixes(it.path) } + val docFields = searchFields + .filter { it.path?.startsWith(DOC_PREFIX) == true } + .map { "content.${it.path?.removePrefix(DOC_PREFIX)}" } + val caseFields = searchFields + .filter { it.path?.startsWith(CASE_PREFIX) == true } + .filter { it.dataType == SearchFieldDataType.TEXT } + .map { it.path?.removePrefix(CASE_PREFIX) } + + val parsedTerms = parseGlobalSearch(query) + + val unknownFields = parsedTerms + .filter { it.field != null } + .map { removePrefixes(it.field) } + .filter { fieldMap[it] == null } + .distinct() + + if (unknownFields.isNotEmpty()) { + throw IllegalArgumentException( + "Unknown search field(s): ${unknownFields.joinToString(", ")}" + ) + } + + val qualifiedCaseFieldQueries = mutableListOf() + val unqualifiedCaseFieldQueries = mutableListOf() + val queryStringParts = mutableListOf() + + parsedTerms.forEach { term -> + if (term.field != null) { + val fieldPath = removePrefixes(term.field) + val field = fieldMap[fieldPath]!! + val isDocField = field.path?.startsWith(DOC_PREFIX) == true + val osPath = if (isDocField) "content.$fieldPath" else fieldPath + + if (!isDocField) { + if (field.dataType == SearchFieldDataType.DATE || field.dataType == SearchFieldDataType.DATETIME) { + val dateRange = parseDateRange(term.value) + qualifiedCaseFieldQueries.add( + QueryBuilders.rangeQuery(osPath) + .gte(dateRange.first) + .lte(dateRange.second) + ) + } else { + val pattern = if (!term.quoted && field.matchType == SearchFieldMatchType.LIKE) { + "*${term.value}*" + } else { + term.value + } + qualifiedCaseFieldQueries.add( + QueryBuilders.wildcardQuery(osPath, pattern).caseInsensitive(true) + ) + } + } else { + val value = escapeQueryStringValue(term.value) + val wrappedValue = if (!term.quoted && field.matchType == SearchFieldMatchType.LIKE) { + "*$value*" + } else if (term.quoted) { + "\"$value\"" + } else { + value + } + queryStringParts.add("$osPath:$wrappedValue") + } + } else { + val escaped = escapeQueryStringValue(term.value) + queryStringParts.add(if (term.quoted) "\"$escaped\"" else "*$escaped*") + + val pattern = "*${term.value}*" + caseFields.filterNotNull().forEach { caseField -> + unqualifiedCaseFieldQueries.add( + QueryBuilders.wildcardQuery(caseField, pattern).caseInsensitive(true) + ) + } + } + } + + if (qualifiedCaseFieldQueries.isEmpty() && unqualifiedCaseFieldQueries.isEmpty() && queryStringParts.isEmpty()) { + return QueryBuilders.matchAllQuery() + } + + val boolQuery = QueryBuilders.boolQuery() + + qualifiedCaseFieldQueries.forEach { boolQuery.must(it) } + + if (queryStringParts.isNotEmpty()) { + val docQuery = QueryBuilders.queryStringQuery(queryStringParts.joinToString(" AND ")) + .apply { docFields.forEach { field(it) } } + .lenient(true) + .analyzeWildcard(true) + .defaultOperator(Operator.AND) + + if (unqualifiedCaseFieldQueries.isNotEmpty()) { + val shouldQuery = QueryBuilders.boolQuery() + .should(docQuery) + unqualifiedCaseFieldQueries.forEach { shouldQuery.should(it) } + shouldQuery.minimumShouldMatch(1) + boolQuery.must(shouldQuery) + } else { + boolQuery.must(docQuery) + } + } else if (unqualifiedCaseFieldQueries.isNotEmpty()) { + unqualifiedCaseFieldQueries.forEach { boolQuery.must(it) } + } + + return boolQuery + } + + private data class ParsedTerm( + val field: String?, + val value: String, + val quoted: Boolean + ) + + private fun parseGlobalSearch(query: String): List { + val terms = mutableListOf() + val fieldPattern = """(\w+(?:\.\w+)*):("([^"]+)"|(\S+))""".toRegex() + + var remaining = query + var lastEnd = 0 + + for (match in fieldPattern.findAll(query)) { + val before = query.substring(lastEnd, match.range.first).trim() + if (before.isNotEmpty()) { + terms.addAll(parseUnqualifiedTerms(before)) + } + + val fieldName = match.groupValues[1] + val quoted = match.groupValues[3].isNotEmpty() + val value = if (quoted) match.groupValues[3] else match.groupValues[4] + + terms.add(ParsedTerm(fieldName, value, quoted)) + lastEnd = match.range.last + 1 + } + + val after = query.substring(lastEnd).trim() + if (after.isNotEmpty()) { + terms.addAll(parseUnqualifiedTerms(after)) + } + + return terms + } + + private fun parseUnqualifiedTerms(text: String): List { + val terms = mutableListOf() + val quotedPattern = """"([^"]+)"""".toRegex() + + var remaining = text + var lastEnd = 0 + + for (match in quotedPattern.findAll(text)) { + val before = text.substring(lastEnd, match.range.first).trim() + if (before.isNotEmpty()) { + before.split("\\s+".toRegex()).filter { it.isNotEmpty() }.forEach { + terms.add(ParsedTerm(null, it, false)) + } + } + terms.add(ParsedTerm(null, match.groupValues[1], true)) + lastEnd = match.range.last + 1 + } + + val after = text.substring(lastEnd).trim() + if (after.isNotEmpty()) { + after.split("\\s+".toRegex()).filter { it.isNotEmpty() }.forEach { + terms.add(ParsedTerm(null, it, false)) + } + } + + return terms + } + + private fun escapeQueryStringValue(value: String): String { + val specialChars = """[\+\-\=\&\|\!\(\)\{\}\[\]\^\~\*\?\:\\\/]""".toRegex() + return value.replace(specialChars) { "\\${it.value}" } + } + + private fun removePrefixes(path: String?): String? { + return path?.removePrefix(DOC_PREFIX)?.removePrefix(CASE_PREFIX) + } + companion object { private const val DOC_PREFIX = "doc:" private const val CASE_PREFIX = "case:" private const val DEFINITION_NAME_FIELD = "definitionId.name" private const val BLUEPRINT_TYPE_FIELD = "definitionId.blueprintId.blueprintType" - /** - * Calls [AdvancedSearchRequest.OtherFilter.getRangeFrom] via reflection to bypass the - * Kotlin type-bounds check. - */ private fun AdvancedSearchRequest.OtherFilter.rangeFromValue(): Any? = AdvancedSearchRequest.OtherFilter::class.java.getMethod("getRangeFrom").invoke(this) diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceTest.kt index 7f91d1879d..5aab692cf9 100644 --- a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceTest.kt +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceTest.kt @@ -23,6 +23,10 @@ import com.ritense.authorization.permission.ConditionContainer import com.ritense.authorization.permission.Permission import com.ritense.authorization.role.Role import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.domain.impl.searchfield.SearchField +import com.ritense.document.domain.impl.searchfield.SearchFieldDataType +import com.ritense.document.domain.impl.searchfield.SearchFieldFieldType +import com.ritense.document.domain.impl.searchfield.SearchFieldMatchType import com.ritense.document.domain.search.AdvancedSearchRequest import com.ritense.document.opensearch.authorization.OpenSearchAuthorizationEntityMapper import com.ritense.document.opensearch.authorization.OpenSearchPermissionConditionTranslator @@ -171,6 +175,116 @@ class JsonSchemaDocumentOpenSearchServiceTest { assertThat(page.totalElements).isEqualTo(5L) } + @Test + fun `search with field-qualified term targets specific field`() { + val queryCaptor = argumentCaptor() + val emptySearchHits: SearchHits = mock() + whenever(emptySearchHits.searchHits).thenReturn(emptyList()) + whenever(emptySearchHits.totalHits).thenReturn(0L) + whenever(elasticsearchOperations.search(queryCaptor.capture(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(emptySearchHits) + whenever(searchFieldService.getSearchFields("house")).thenReturn(listOf( + SearchField("city", "doc:city", SearchFieldDataType.TEXT, SearchFieldFieldType.SINGLE, SearchFieldMatchType.LIKE, null, 0, "City") + )) + + val request = AdvancedSearchRequest().globalSearchFilter("city:amsterdam") + service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) + + val capturedQuery = queryCaptor.firstValue + assertThat(capturedQuery.source).contains("content.city") + assertThat(capturedQuery.source).contains("amsterdam") + } + + @Test + fun `search with EXACT match type field does not add wildcards`() { + val queryCaptor = argumentCaptor() + val emptySearchHits: SearchHits = mock() + whenever(emptySearchHits.searchHits).thenReturn(emptyList()) + whenever(emptySearchHits.totalHits).thenReturn(0L) + whenever(elasticsearchOperations.search(queryCaptor.capture(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(emptySearchHits) + whenever(searchFieldService.getSearchFields("house")).thenReturn(listOf( + SearchField("status", "doc:status", SearchFieldDataType.TEXT, SearchFieldFieldType.SINGLE, SearchFieldMatchType.EXACT, null, 0, "Status") + )) + + val request = AdvancedSearchRequest().globalSearchFilter("status:active") + service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) + + val capturedQuery = queryCaptor.firstValue + assertThat(capturedQuery.source).contains("content.status:active") + assertThat(capturedQuery.source).doesNotContain("*active*") + } + + @Test + fun `search with LIKE match type field adds wildcards`() { + val queryCaptor = argumentCaptor() + val emptySearchHits: SearchHits = mock() + whenever(emptySearchHits.searchHits).thenReturn(emptyList()) + whenever(emptySearchHits.totalHits).thenReturn(0L) + whenever(elasticsearchOperations.search(queryCaptor.capture(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(emptySearchHits) + whenever(searchFieldService.getSearchFields("house")).thenReturn(listOf( + SearchField("name", "doc:name", SearchFieldDataType.TEXT, SearchFieldFieldType.SINGLE, SearchFieldMatchType.LIKE, null, 0, "Name") + )) + + val request = AdvancedSearchRequest().globalSearchFilter("name:john") + service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) + + val capturedQuery = queryCaptor.firstValue + assertThat(capturedQuery.source).contains("content.name:*john*") + } + + @Test + fun `search with quoted field value does not add wildcards even for LIKE fields`() { + val queryCaptor = argumentCaptor() + val emptySearchHits: SearchHits = mock() + whenever(emptySearchHits.searchHits).thenReturn(emptyList()) + whenever(emptySearchHits.totalHits).thenReturn(0L) + whenever(elasticsearchOperations.search(queryCaptor.capture(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(emptySearchHits) + whenever(searchFieldService.getSearchFields("house")).thenReturn(listOf( + SearchField("address", "doc:address", SearchFieldDataType.TEXT, SearchFieldFieldType.SINGLE, SearchFieldMatchType.LIKE, null, 0, "Address") + )) + + val request = AdvancedSearchRequest().globalSearchFilter("""address:"Main Street 123"""") + service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) + + val capturedQuery = queryCaptor.firstValue + assertThat(capturedQuery.source).contains("""content.address:\"Main Street 123\"""") + assertThat(capturedQuery.source).doesNotContain("*Main Street 123*") + } + + @Test + fun `search with unknown field throws exception listing unknown fields`() { + whenever(searchFieldService.getSearchFields("house")).thenReturn(listOf( + SearchField("city", "doc:city", SearchFieldDataType.TEXT, SearchFieldFieldType.SINGLE, SearchFieldMatchType.LIKE, null, 0, "City") + )) + + val request = AdvancedSearchRequest().globalSearchFilter("unknownField:value anotherBad:x city:amsterdam") + + val exception = org.junit.jupiter.api.assertThrows { + service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) + } + + assertThat(exception.message).contains("unknownField") + assertThat(exception.message).contains("anotherBad") + } + + @Test + fun `search with mixed qualified and unqualified terms`() { + val queryCaptor = argumentCaptor() + val emptySearchHits: SearchHits = mock() + whenever(emptySearchHits.searchHits).thenReturn(emptyList()) + whenever(emptySearchHits.totalHits).thenReturn(0L) + whenever(elasticsearchOperations.search(queryCaptor.capture(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(emptySearchHits) + whenever(searchFieldService.getSearchFields("house")).thenReturn(listOf( + SearchField("city", "doc:city", SearchFieldDataType.TEXT, SearchFieldFieldType.SINGLE, SearchFieldMatchType.EXACT, null, 0, "City") + )) + + val request = AdvancedSearchRequest().globalSearchFilter("city:amsterdam urgent") + service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) + + val capturedQuery = queryCaptor.firstValue + assertThat(capturedQuery.source).contains("content.city:amsterdam") + assertThat(capturedQuery.source).contains("*urgent*") + } + companion object { private const val FULL_ACCESS_ROLE = "full access role" private const val USERNAME = "test@test.com" diff --git a/backend/case/src/main/java/com/ritense/document/service/impl/JsonSchemaDocumentSearchService.java b/backend/case/src/main/java/com/ritense/document/service/impl/JsonSchemaDocumentSearchService.java index 9a37d50f75..c89b24cdfc 100644 --- a/backend/case/src/main/java/com/ritense/document/service/impl/JsonSchemaDocumentSearchService.java +++ b/backend/case/src/main/java/com/ritense/document/service/impl/JsonSchemaDocumentSearchService.java @@ -72,8 +72,11 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; +import com.ritense.document.domain.impl.searchfield.SearchFieldMatchType; import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.StringUtils; import org.springframework.data.domain.Page; @@ -395,29 +398,90 @@ private void buildQueryWhere( } if (searchRequest.getGlobalSearchFilter() != null && !searchRequest.getGlobalSearchFilter().isBlank()) { - var pattern = "%" + searchRequest.getGlobalSearchFilter().trim().toLowerCase() + "%"; - var searchableFields = !StringUtils.isEmpty(documentDefinitionName) - ? searchFieldService.getSearchFields(documentDefinitionName).stream() - .filter(f -> f.getPath() != null && f.getPath().startsWith(DOC_PREFIX)) - .toList() + var searchFields = !StringUtils.isEmpty(documentDefinitionName) + ? searchFieldService.getSearchFields(documentDefinitionName) : List.of(); - if (!searchableFields.isEmpty()) { - var fieldPredicates = searchableFields.stream() - .map(f -> { + var fieldMap = searchFields.stream() + .collect(Collectors.toMap(f -> removePrefixes(f.getPath()), f -> f, (a, b) -> a)); + + var docFields = searchFields.stream() + .filter(f -> f.getPath() != null && f.getPath().startsWith(DOC_PREFIX)) + .toList(); + + var caseTextFields = searchFields.stream() + .filter(f -> f.getPath() != null && f.getPath().startsWith(CASE_PREFIX)) + .filter(f -> f.getDataType() == SearchFieldDataType.TEXT) + .toList(); + + var parsedTerms = parseGlobalSearch(searchRequest.getGlobalSearchFilter()); + + List qualifiedPredicates = new ArrayList<>(); + List unqualifiedPredicates = new ArrayList<>(); + + for (ParsedTerm term : parsedTerms) { + if (term.field() != null) { + var fieldPath = removePrefixes(term.field()); + var field = fieldMap.get(fieldPath); + if (field == null) { + throw new IllegalArgumentException("Unknown search field: " + term.field()); + } + + boolean isDocField = field.getPath().startsWith(DOC_PREFIX); + + if (isDocField) { + var jsonPath = "$." + fieldPath; + Expression expr = queryDialectHelper.getJsonValueExpression( + cb, documentRoot.get(CONTENT).get(CONTENT), jsonPath, String.class + ); + var likePattern = buildLikePattern(term.value(), term.quoted(), field.getMatchType()); + qualifiedPredicates.add(cb.like(cb.lower(expr), likePattern.toLowerCase())); + } else { + if (field.getDataType() == SearchFieldDataType.DATE || + field.getDataType() == SearchFieldDataType.DATETIME) { + var date = LocalDate.parse(term.value()); + var startOfDay = date.atStartOfDay(); + var endOfDay = date.plusDays(1).atStartOfDay(); + qualifiedPredicates.add(cb.and( + cb.greaterThanOrEqualTo(documentRoot.get(fieldPath), startOfDay), + cb.lessThan(documentRoot.get(fieldPath), endOfDay) + )); + } else { + var likePattern = buildLikePattern(term.value(), term.quoted(), field.getMatchType()); + qualifiedPredicates.add(cb.like( + cb.lower(documentRoot.get(fieldPath).as(String.class)), + likePattern.toLowerCase() + )); + } + } + } else { + var likePattern = "%" + term.value().toLowerCase() + "%"; + List termPredicates = new ArrayList<>(); + + for (var f : docFields) { var jsonPath = "$." + f.getPath().substring(DOC_PREFIX.length()); Expression expr = queryDialectHelper.getJsonValueExpression( cb, documentRoot.get(CONTENT).get(CONTENT), jsonPath, String.class ); - return cb.like(cb.lower(expr), pattern); - }) - .toArray(Predicate[]::new); - predicates.add(cb.or(fieldPredicates)); - } else { - // Fallback: search entire content as text - var contentAsText = cast(documentRoot.get(CONTENT).get(CONTENT), String.class); - predicates.add(cb.like(cb.lower(contentAsText), pattern)); + termPredicates.add(cb.like(cb.lower(expr), likePattern)); + } + + for (var f : caseTextFields) { + var columnName = f.getPath().substring(CASE_PREFIX.length()); + termPredicates.add(cb.like( + cb.lower(documentRoot.get(columnName).as(String.class)), + likePattern + )); + } + + if (!termPredicates.isEmpty()) { + unqualifiedPredicates.add(cb.or(termPredicates.toArray(Predicate[]::new))); + } + } } + + qualifiedPredicates.forEach(predicates::add); + unqualifiedPredicates.forEach(predicates::add); } query.where(predicates.toArray(Predicate[]::new)); @@ -838,4 +902,75 @@ void apply( Root documentRoot ); } + + private record ParsedTerm(String field, String value, boolean quoted) {} + + private List parseGlobalSearch(String query) { + List terms = new ArrayList<>(); + Pattern fieldPattern = Pattern.compile("(\\w+(?:\\.\\w+)*):(\"([^\"]+)\"|(\\S+))"); + Matcher matcher = fieldPattern.matcher(query); + + int lastEnd = 0; + while (matcher.find()) { + String before = query.substring(lastEnd, matcher.start()).trim(); + if (!before.isEmpty()) { + terms.addAll(parseUnqualifiedTerms(before)); + } + + String fieldName = matcher.group(1); + boolean quoted = matcher.group(3) != null && !matcher.group(3).isEmpty(); + String value = quoted ? matcher.group(3) : matcher.group(4); + + terms.add(new ParsedTerm(fieldName, value, quoted)); + lastEnd = matcher.end(); + } + + String after = query.substring(lastEnd).trim(); + if (!after.isEmpty()) { + terms.addAll(parseUnqualifiedTerms(after)); + } + + return terms; + } + + private List parseUnqualifiedTerms(String text) { + List terms = new ArrayList<>(); + Pattern quotedPattern = Pattern.compile("\"([^\"]+)\""); + Matcher matcher = quotedPattern.matcher(text); + + int lastEnd = 0; + while (matcher.find()) { + String before = text.substring(lastEnd, matcher.start()).trim(); + if (!before.isEmpty()) { + Arrays.stream(before.split("\\s+")) + .filter(s -> !s.isEmpty()) + .forEach(s -> terms.add(new ParsedTerm(null, s, false))); + } + terms.add(new ParsedTerm(null, matcher.group(1), true)); + lastEnd = matcher.end(); + } + + String after = text.substring(lastEnd).trim(); + if (!after.isEmpty()) { + Arrays.stream(after.split("\\s+")) + .filter(s -> !s.isEmpty()) + .forEach(s -> terms.add(new ParsedTerm(null, s, false))); + } + + return terms; + } + + private String removePrefixes(String path) { + if (path == null) return null; + if (path.startsWith(DOC_PREFIX)) return path.substring(DOC_PREFIX.length()); + if (path.startsWith(CASE_PREFIX)) return path.substring(CASE_PREFIX.length()); + return path; + } + + private String buildLikePattern(String value, boolean quoted, SearchFieldMatchType matchType) { + if (quoted || matchType != SearchFieldMatchType.LIKE) { + return value; + } + return "%" + value + "%"; + } } diff --git a/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.html b/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.html index 69b10e3da6..d6a6a71e4a 100644 --- a/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.html +++ b/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.html @@ -39,6 +39,7 @@ assigneeFilter: orchestration.assigneeFilter$ | async, hiddenColumns: orchestration.hiddenColumns$ | async, disableStartButton: disableStartButton$ | async, + invalidSearchFields: orchestration.invalidSearchFields$ | async, } as obs" > = this.searchService.globalSearchFilter$; + public readonly invalidSearchFields$: Observable = + this.documentService.invalidSearchFields$; + public readonly statuses$: Observable> = this.statusService.caseStatuses$; diff --git a/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.html b/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.html index e19494b2d3..e3d5005bb8 100644 --- a/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.html +++ b/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.html @@ -38,12 +38,33 @@ - +
+ + +
    +
  • {{ field.title || field.key }}
    • +
+
diff --git a/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.scss b/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.scss index ade1161de1..bd2e7b29e5 100644 --- a/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.scss +++ b/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.scss @@ -155,3 +155,81 @@ td:first-child { overflow: hidden; } } + +.valtimo-search-container { + position: relative; + flex: 1; + + ::ng-deep cds-table-toolbar-search { + display: flex; + justify-content: flex-end; + width: 100%; + + // Expanded: fill full width + .cds--toolbar-search-container-active { + width: 100%; + } + } +} + +.valtimo-search-container ::ng-deep .cds--toolbar-search-container-active input { + color: transparent !important; + caret-color: var(--cds-text-primary, #161616); +} + +.valtimo-search-overlay { + position: absolute; + top: 0; + left: 0; + right: 0; + bottom: 0; + display: flex; + align-items: center; + height: 3rem; + padding: 0 3rem; + font-family: 'IBM Plex Sans', 'Helvetica Neue', Arial, sans-serif; + font-size: 0.875rem; + font-weight: 400; + letter-spacing: 0.16px; + line-height: 1.28572; + color: var(--cds-text-primary, #161616); + white-space: pre; + pointer-events: none; + overflow: hidden; + + &__invalid { + text-decoration: underline wavy var(--cds-support-error, #da1e28); + text-decoration-skip-ink: none; + text-underline-offset: 3px; + } + + } + +.valtimo-search-autocomplete { + position: absolute; + top: 100%; + max-height: 150px; + min-width: 120px; + max-width: 250px; + width: auto; + overflow-y: auto; + background: var(--cds-layer); + border: 1px solid var(--cds-border-subtle); + box-shadow: 0 2px 6px rgba(0, 0, 0, 0.1); + z-index: 9000; + list-style: none; + margin: 0; + padding: 0; + + li { + padding: 6px 12px; + cursor: pointer; + font-size: 0.875rem; + white-space: nowrap; + + &:hover, + &.selected { + background: var(--cds-layer-hover); + } + } +} diff --git a/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.ts b/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.ts index a563f6cf3a..e7505b6309 100644 --- a/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.ts +++ b/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.ts @@ -16,6 +16,7 @@ import { AfterViewInit, ChangeDetectionStrategy, + ChangeDetectorRef, Component, ElementRef, EventEmitter, @@ -30,6 +31,7 @@ import {FormControl} from '@angular/forms'; import {ArrowDown16, ArrowUp16, Draggable16, SettingsView16} from '@carbon/icons'; import {TranslateService} from '@ngx-translate/core'; import {SortState} from '@valtimo/document'; +import {SearchField} from '@valtimo/shared'; import { IconService, PaginationModel, @@ -178,6 +180,8 @@ export class CarbonListComponent implements OnInit, AfterViewInit, OnDestroy { } @Input() isSearchable = false; + @Input() invalidSearchFields: string[] = []; + @Input() searchFields: SearchField[] = []; @Input() enableSingleSelection = false; /** * @deprecated The lastColumnTemplate field is deprecated. Any template column can be added through the **@Input field**. @@ -280,7 +284,8 @@ export class CarbonListComponent implements OnInit, AfterViewInit, OnDestroy { private readonly viewContentService: ViewContentService, private readonly keyStateService: KeyStateService, private readonly dragAndDropService: CarbonListDragAndDropService, - private readonly elementRef: ElementRef + private readonly elementRef: ElementRef, + private readonly cdr: ChangeDetectorRef ) { this.iconService.registerAll([ArrowDown16, ArrowUp16, SettingsView16, Draggable16]); } @@ -308,21 +313,9 @@ export class CarbonListComponent implements OnInit, AfterViewInit, OnDestroy { this._subscriptions.add( this.searchFormControl.valueChanges - .pipe(debounceTime(500)) + .pipe(debounceTime(2000)) .subscribe((searchString: string | null) => { - if (this.search.observed) { - this.search.emit(searchString); - return; - } - - if (!searchString) { - this._filteredItems$.next(null); - return; - } - - this._filteredItems$.next( - this.filterPipe.transform(this._completeDataSource, searchString ?? '') - ); + this.executeSearch(searchString); }) ); } @@ -758,4 +751,246 @@ export class CarbonListComponent implements OnInit, AfterViewInit, OnDestroy { type: 'blue', })); } + + public getSearchSegments(): Array<{text: string; isInvalid: boolean}> { + const text = this.searchFormControl.value || ''; + if (!text) return [{text, isInvalid: false}]; + + const segments: Array<{text: string; isInvalid: boolean}> = []; + const fieldPattern = /(\w+(?:\.\w+)*):("([^"]+)"|(\S+))/g; + const invalidSet = new Set((this.invalidSearchFields || []).map(f => f.toLowerCase())); + + let lastIndex = 0; + let match: RegExpExecArray | null; + + while ((match = fieldPattern.exec(text)) !== null) { + if (match.index > lastIndex) { + segments.push({text: text.substring(lastIndex, match.index), isInvalid: false}); + } + + const fieldName = match[1]; + const isInvalid = invalidSet.has(fieldName.toLowerCase()); + segments.push({text: fieldName, isInvalid}); + + const rest = match[0].substring(fieldName.length); + segments.push({text: rest, isInvalid: false}); + + lastIndex = match.index + match[0].length; + } + + if (lastIndex < text.length) { + segments.push({text: text.substring(lastIndex), isInvalid: false}); + } + + return segments; + } + + public showAutocomplete = false; + public filteredSuggestions: SearchField[] = []; + public selectedSuggestionIndex = 0; + public autocompleteLeft = 0; + private _searchInputElement: HTMLInputElement | null = null; + + private getSearchInputElement(): HTMLInputElement | null { + if (!this._searchInputElement) { + this._searchInputElement = this.elementRef.nativeElement.querySelector( + '.valtimo-search-container input' + ); + } + return this._searchInputElement; + } + + private getCurrentFieldToken(): {token: string; start: number} | null { + const value = this.searchFormControl.value || ''; + const input = this.getSearchInputElement(); + const cursor = input?.selectionStart ?? value.length; + + let start = value.lastIndexOf(' ', cursor - 1) + 1; + const beforeCursor = value.substring(start, cursor); + + if (beforeCursor.includes(':')) return null; + + const nextSpace = value.indexOf(' ', cursor); + const end = nextSpace === -1 ? value.length : nextSpace; + const afterCursor = value.substring(cursor, end); + + if (afterCursor.includes(':')) return null; + + return {token: beforeCursor, start}; + } + + public onSearchFocus(): void { + const input = this.getSearchInputElement(); + if (input && document.activeElement === input) { + this.updateAutocomplete(); + } + } + + public updateAutocomplete(): void { + const input = this.getSearchInputElement(); + + if (!input) { + this.showAutocomplete = false; + this.filteredSuggestions = []; + return; + } + + const tokenInfo = this.getCurrentFieldToken(); + + if (!tokenInfo || !this.searchFields?.length) { + this.showAutocomplete = false; + this.filteredSuggestions = []; + return; + } + + const searchToken = tokenInfo.token.toLowerCase(); + this.filteredSuggestions = searchToken.length === 0 + ? this.searchFields + : this.searchFields.filter( + field => + field.key.toLowerCase().includes(searchToken) || + (field.title && field.title.toLowerCase().includes(searchToken)) + ); + + this.showAutocomplete = this.filteredSuggestions.length > 0; + this.selectedSuggestionIndex = 0; + + if (this.showAutocomplete) { + this.autocompleteLeft = this.calculateTokenLeft(tokenInfo.start); + } + } + + private calculateTokenLeft(tokenStart: number): number { + const input = this.getSearchInputElement(); + if (!input) return 48; + + const value = this.searchFormControl.value || ''; + const textBefore = value.substring(0, tokenStart); + + const canvas = document.createElement('canvas'); + const ctx = canvas.getContext('2d'); + if (!ctx) return 48; + + const style = window.getComputedStyle(input); + ctx.font = `${style.fontSize} ${style.fontFamily}`; + const textWidth = ctx.measureText(textBefore).width; + + return 48 + textWidth - 12; + } + + public selectSuggestion(field: SearchField): void { + const tokenInfo = this.getCurrentFieldToken(); + if (!tokenInfo) return; + + const value = this.searchFormControl.value || ''; + const input = this.getSearchInputElement(); + const cursor = input?.selectionStart ?? value.length; + + const fieldPath = field.path?.replace(/^(doc|case):/, '') || field.key; + const newValue = + value.substring(0, tokenInfo.start) + fieldPath + ':' + value.substring(cursor); + + this.searchFormControl.setValue(newValue); + this.showAutocomplete = false; + + setTimeout(() => { + const newCursor = tokenInfo.start + fieldPath.length + 1; + input?.setSelectionRange(newCursor, newCursor); + input?.focus(); + }); + } + + public onSearchKeydown(event: KeyboardEvent): void { + if (event.key === 'Enter') { + event.preventDefault(); + this.onSearchEnter(); + return; + } + + if (!this.showAutocomplete || this.filteredSuggestions.length === 0) return; + + switch (event.key) { + case 'ArrowDown': + event.preventDefault(); + this.selectedSuggestionIndex = + (this.selectedSuggestionIndex + 1) % this.filteredSuggestions.length; + break; + case 'ArrowUp': + event.preventDefault(); + this.selectedSuggestionIndex = + (this.selectedSuggestionIndex - 1 + this.filteredSuggestions.length) % + this.filteredSuggestions.length; + break; + case 'Tab': + event.preventDefault(); + this.selectSuggestion(this.filteredSuggestions[this.selectedSuggestionIndex]); + break; + case 'Escape': + this.showAutocomplete = false; + break; + } + } + + public onSearchBlur(): void { + setTimeout(() => { + this.showAutocomplete = false; + this._searchInputElement = null; + }, 150); + } + + + public onSearchClear(): void { + this.showAutocomplete = false; + } + + private executeSearch(searchString: string | null): void { + if (this.search.observed) { + this.search.emit(searchString); + return; + } + + if (!searchString) { + this._filteredItems$.next(null); + return; + } + + this._filteredItems$.next( + this.filterPipe.transform(this._completeDataSource, searchString ?? '') + ); + } + + public onSearchEnter(): void { + if (this.showAutocomplete && this.filteredSuggestions.length > 0) { + this.selectSuggestion(this.filteredSuggestions[this.selectedSuggestionIndex]); + } else { + this.executeSearch(this.searchFormControl.value); + } + } + + private _searchOpen = false; + + public onSearchOpenChange(isOpen: boolean): void { + this._searchOpen = isOpen; + this._searchInputElement = null; + + if (isOpen) { + setTimeout(() => { + this.updateAutocomplete(); + }, 0); + } else { + this.showAutocomplete = false; + } + } + + public onSearchFocusOut(event: FocusEvent): void { + const container = this.elementRef.nativeElement.querySelector('.valtimo-search-container'); + const relatedTarget = event.relatedTarget as Node; + + if (!container?.contains(relatedTarget)) { + setTimeout(() => { + this.showAutocomplete = false; + this.cdr.markForCheck(); + }, 150); + } + } } diff --git a/frontend/projects/valtimo/components/src/lib/components/search-input-with-validation/index.ts b/frontend/projects/valtimo/components/src/lib/components/search-input-with-validation/index.ts new file mode 100644 index 0000000000..eacaecbc97 --- /dev/null +++ b/frontend/projects/valtimo/components/src/lib/components/search-input-with-validation/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './search-input-with-validation.component'; diff --git a/frontend/projects/valtimo/components/src/lib/components/search-input-with-validation/search-input-with-validation.component.html b/frontend/projects/valtimo/components/src/lib/components/search-input-with-validation/search-input-with-validation.component.html new file mode 100644 index 0000000000..7c2b9819b4 --- /dev/null +++ b/frontend/projects/valtimo/components/src/lib/components/search-input-with-validation/search-input-with-validation.component.html @@ -0,0 +1,68 @@ + + +
+ + +
+ + +
+ + + +
+ + +
+
diff --git a/frontend/projects/valtimo/components/src/lib/components/search-input-with-validation/search-input-with-validation.component.scss b/frontend/projects/valtimo/components/src/lib/components/search-input-with-validation/search-input-with-validation.component.scss new file mode 100644 index 0000000000..7109be1478 --- /dev/null +++ b/frontend/projects/valtimo/components/src/lib/components/search-input-with-validation/search-input-with-validation.component.scss @@ -0,0 +1,153 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +$search-height: 2.5rem; +$search-font-size: 0.875rem; +$search-padding-left: 2.5rem; +$search-padding-right: 2.5rem; + +.search-input-container { + display: inline-flex; + align-items: center; + height: $search-height; + background-color: var(--cds-field-01, #f4f4f4); + transition: width 0.2s ease; + + &--expanded { + width: 100%; + max-width: 20rem; + } + + &--has-error { + .search-input-wrapper { + border-bottom: 2px solid var(--cds-support-error, #da1e28); + } + } +} + +.search-input-expand-button { + display: flex; + align-items: center; + justify-content: center; + width: $search-height; + height: $search-height; + padding: 0; + border: none; + background: transparent; + cursor: pointer; + color: var(--cds-icon-primary, #161616); + + &:hover { + background-color: var(--cds-field-hover, #e8e8e8); + } + + &:focus { + outline: 2px solid var(--cds-focus, #0f62fe); + outline-offset: -2px; + } +} + +.search-input-wrapper { + position: relative; + display: flex; + align-items: center; + width: 100%; + height: 100%; + border-bottom: 1px solid var(--cds-border-strong-01, #8d8d8d); +} + +.search-input-icon { + position: absolute; + left: 0.75rem; + color: var(--cds-icon-secondary, #525252); + pointer-events: none; +} + +.search-input-highlight-container { + position: relative; + flex: 1; + height: 100%; +} + +.search-input-highlight-overlay { + position: absolute; + top: 0; + left: 0; + right: 0; + bottom: 0; + display: flex; + align-items: center; + padding-left: $search-padding-left; + padding-right: $search-padding-right; + font-family: 'IBM Plex Sans', sans-serif; + font-size: $search-font-size; + color: var(--cds-text-primary, #161616); + white-space: pre; + pointer-events: none; + overflow: hidden; +} + +.search-input-field { + position: relative; + width: 100%; + height: 100%; + padding-left: $search-padding-left; + padding-right: $search-padding-right; + border: none; + background: transparent; + font-family: 'IBM Plex Sans', sans-serif; + font-size: $search-font-size; + color: transparent; + caret-color: var(--cds-text-primary, #161616); + + &::placeholder { + color: var(--cds-text-placeholder, #a8a8a8); + } + + &:focus { + outline: none; + } +} + +.search-input-invalid { + text-decoration: underline wavy var(--cds-support-error, #da1e28); + text-decoration-skip-ink: none; + text-underline-offset: 2px; +} + +.search-input-clear-button { + position: absolute; + right: 0; + display: flex; + align-items: center; + justify-content: center; + width: $search-height; + height: $search-height; + padding: 0; + border: none; + background: transparent; + cursor: pointer; + color: var(--cds-icon-primary, #161616); + + &:hover { + background-color: var(--cds-field-hover, #e8e8e8); + } + + &:focus { + outline: 2px solid var(--cds-focus, #0f62fe); + outline-offset: -2px; + } +} diff --git a/frontend/projects/valtimo/components/src/lib/components/search-input-with-validation/search-input-with-validation.component.ts b/frontend/projects/valtimo/components/src/lib/components/search-input-with-validation/search-input-with-validation.component.ts new file mode 100644 index 0000000000..faa7d070d2 --- /dev/null +++ b/frontend/projects/valtimo/components/src/lib/components/search-input-with-validation/search-input-with-validation.component.ts @@ -0,0 +1,147 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + ChangeDetectionStrategy, + Component, + ElementRef, + EventEmitter, + Input, + Output, + ViewChild, +} from '@angular/core'; +import {CommonModule} from '@angular/common'; +import {FormsModule} from '@angular/forms'; +import {IconModule, IconService} from 'carbon-components-angular'; +import {Search16, Close16} from '@carbon/icons'; + +interface TextSegment { + text: string; + isFieldName: boolean; + isInvalid: boolean; +} + +@Component({ + standalone: true, + selector: 'valtimo-search-input-with-validation', + templateUrl: './search-input-with-validation.component.html', + styleUrls: ['./search-input-with-validation.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, + imports: [CommonModule, FormsModule, IconModule], +}) +export class SearchInputWithValidationComponent { + @ViewChild('inputElement') private readonly _inputElement: ElementRef; + + @Input() public value = ''; + @Input() public invalidFields: string[] = []; + @Input() public placeholder = ''; + @Input() public expandable = true; + + @Output() public readonly valueChangeEvent = new EventEmitter(); + @Output() public readonly searchEvent = new EventEmitter(); + + public isExpanded = false; + + constructor(private readonly _iconService: IconService) { + this._iconService.registerAll([Search16, Close16]); + } + + public get segments(): TextSegment[] { + return this._parseIntoSegments(this.value, this.invalidFields); + } + + public get hasInvalidFields(): boolean { + return this.invalidFields.length > 0; + } + + public onInput(event: Event): void { + const input = event.target as HTMLInputElement; + this.value = input.value; + this.valueChangeEvent.emit(this.value); + } + + public onKeyDown(event: KeyboardEvent): void { + if (event.key === 'Enter') { + this.searchEvent.emit(this.value); + } + } + + public onClear(): void { + this.value = ''; + this.valueChangeEvent.emit(this.value); + this.searchEvent.emit(this.value); + } + + public onExpand(): void { + this.isExpanded = true; + setTimeout(() => this._inputElement?.nativeElement?.focus(), 0); + } + + public onBlur(): void { + if (!this.value) { + this.isExpanded = false; + } + } + + private _parseIntoSegments(text: string, invalidFields: string[]): TextSegment[] { + if (!text) return []; + + const segments: TextSegment[] = []; + const fieldPattern = /(\w+(?:\.\w+)*):("([^"]+)"|(\S+))/g; + const invalidSet = new Set(invalidFields.map(f => f.toLowerCase())); + + let lastIndex = 0; + let match: RegExpExecArray | null; + + while ((match = fieldPattern.exec(text)) !== null) { + if (match.index > lastIndex) { + segments.push({ + text: text.substring(lastIndex, match.index), + isFieldName: false, + isInvalid: false, + }); + } + + const fieldName = match[1]; + const isInvalid = invalidSet.has(fieldName.toLowerCase()); + + segments.push({ + text: fieldName, + isFieldName: true, + isInvalid, + }); + + const colonAndValue = match[0].substring(fieldName.length); + segments.push({ + text: colonAndValue, + isFieldName: false, + isInvalid: false, + }); + + lastIndex = match.index + match[0].length; + } + + if (lastIndex < text.length) { + segments.push({ + text: text.substring(lastIndex), + isFieldName: false, + isInvalid: false, + }); + } + + return segments; + } +} diff --git a/frontend/projects/valtimo/components/src/public_api.ts b/frontend/projects/valtimo/components/src/public_api.ts index 6b0669321e..48b086a193 100644 --- a/frontend/projects/valtimo/components/src/public_api.ts +++ b/frontend/projects/valtimo/components/src/public_api.ts @@ -293,3 +293,6 @@ export * from './lib/components/assign-user/assignment.component'; // Color picker export * from './lib/components/color-picker/color-picker.component'; + +// Search input with validation +export * from './lib/components/search-input-with-validation'; diff --git a/frontend/projects/valtimo/document/src/lib/services/document.service.ts b/frontend/projects/valtimo/document/src/lib/services/document.service.ts index e863759e20..983a7667f1 100644 --- a/frontend/projects/valtimo/document/src/lib/services/document.service.ts +++ b/frontend/projects/valtimo/document/src/lib/services/document.service.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import {HttpClient, HttpHeaders, HttpParams} from '@angular/common/http'; +import {HttpClient, HttpErrorResponse, HttpHeaders, HttpParams} from '@angular/common/http'; import {Injectable} from '@angular/core'; import { AssigneeFilter, @@ -26,7 +26,7 @@ import { SearchOperator, TeamResponseDto, } from '@valtimo/shared'; -import {catchError, Observable, of, switchMap} from 'rxjs'; +import {BehaviorSubject, catchError, Observable, of, switchMap, tap} from 'rxjs'; import { AssignHandlerToDocumentResult, @@ -84,6 +84,9 @@ export class DocumentService { totalPages: 0, }; + private readonly _invalidSearchFields$ = new BehaviorSubject([]); + public readonly invalidSearchFields$ = this._invalidSearchFields$.asObservable(); + constructor( private http: HttpClient, private configService: ConfigService @@ -91,6 +94,19 @@ export class DocumentService { this.valtimoEndpointUri = this.configService.config.valtimoApi.endpointUri; } + public clearInvalidSearchFields(): void { + this._invalidSearchFields$.next([]); + } + + private extractInvalidSearchFields(error: HttpErrorResponse): string[] { + const message = error?.error?.detail || error?.error?.message || error?.error || ''; + const match = message.match(/Unknown search field\(s\): (.+)/); + if (match) { + return match[1].split(', ').map((f: string) => f.trim()); + } + return []; + } + // Document-calls public getAllDefinitions(): Observable> { return this.http.get>( @@ -169,7 +185,14 @@ export class DocumentService { body, {params: documentSearchRequest.asHttpParams()} ) - .pipe(catchError(() => of(this.EMPTY_DOCUMENTS_RESPONSE as Documents))); + .pipe( + tap(() => this._invalidSearchFields$.next([])), + catchError((error: HttpErrorResponse) => { + const invalidFields = this.extractInvalidSearchFields(error); + this._invalidSearchFields$.next(invalidFields); + return of(this.EMPTY_DOCUMENTS_RESPONSE as Documents); + }) + ); } public getSpecifiedDocumentsSearch( @@ -197,7 +220,14 @@ export class DocumentService { body, {params: documentSearchRequest.asHttpParams()} ) - .pipe(catchError(() => of(this.EMPTY_DOCUMENTS_RESPONSE as SpecifiedDocuments))); + .pipe( + tap(() => this._invalidSearchFields$.next([])), + catchError((error: HttpErrorResponse) => { + const invalidFields = this.extractInvalidSearchFields(error); + this._invalidSearchFields$.next(invalidFields); + return of(this.EMPTY_DOCUMENTS_RESPONSE as SpecifiedDocuments); + }) + ); } public getDocumentSearchFields(caseDefinitionKey: string): Observable> { From 2ed5e5aa1f97065ed34ad0d557855005ba297e41 Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Fri, 3 Jul 2026 14:50:34 +0200 Subject: [PATCH 17/46] added fallback for when opensearch is unavailable --- .../gzac/src/main/resources/application.yml | 1 + .../opensearch/OpenSearchProperties.kt | 5 +- .../DocumentOpenSearchAutoConfiguration.kt | 43 ++++++++++++++ .../DelegatingDocumentSearchService.kt | 49 +++++++++++++--- .../service/OpenSearchHealthService.kt | 58 +++++++++++++++++++ .../opensearch/service/SearchEngineToggle.kt | 27 +++++++++ 6 files changed, 174 insertions(+), 9 deletions(-) create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchHealthService.kt diff --git a/backend/app/gzac/src/main/resources/application.yml b/backend/app/gzac/src/main/resources/application.yml index 0f258fb4e7..8ae8e9b37b 100644 --- a/backend/app/gzac/src/main/resources/application.yml +++ b/backend/app/gzac/src/main/resources/application.yml @@ -3,6 +3,7 @@ logging: name: /tmp/spring.log level: com.ritense.document.opensearch: DEBUG + org.opensearch.client.RestClient: WARN management: endpoints: diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/OpenSearchProperties.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/OpenSearchProperties.kt index 4e0c85e0c9..c28a90079b 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/OpenSearchProperties.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/OpenSearchProperties.kt @@ -20,5 +20,8 @@ import org.springframework.boot.context.properties.ConfigurationProperties @ConfigurationProperties(prefix = "valtimo.opensearch") data class OpenSearchProperties( - val enabled: Boolean = true + val enabled: Boolean = true, + val healthCheckEnabled: Boolean = true, + val healthCheckIntervalMs: Long = 30000, + val fallbackWarningIntervalMs: Long = 300000 ) diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt index 5ba346b5ae..d199e3ba90 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt @@ -35,6 +35,7 @@ import com.ritense.document.opensearch.service.DocumentOpenSearchBackfillService import com.ritense.document.opensearch.service.DocumentOpenSearchQueryService import com.ritense.document.opensearch.service.DocumentOpenSearchSyncService import com.ritense.document.opensearch.service.JsonSchemaDocumentOpenSearchService +import com.ritense.document.opensearch.service.OpenSearchHealthService import com.ritense.document.opensearch.service.SearchEngineToggle import com.ritense.document.opensearch.web.DocumentOpenSearchBackfillResource import com.ritense.document.opensearch.web.SearchEngineResource @@ -53,17 +54,21 @@ import org.springframework.boot.autoconfigure.AutoConfiguration import org.springframework.boot.autoconfigure.AutoConfigureBefore import org.springframework.boot.autoconfigure.condition.ConditionalOnClass import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.annotation.Bean import org.springframework.core.annotation.Order import org.springframework.data.elasticsearch.core.ElasticsearchOperations import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories +import org.springframework.scheduling.annotation.EnableScheduling +import org.springframework.scheduling.annotation.Scheduled @AutoConfiguration @AutoConfigureBefore(DocumentAutoConfiguration::class) @ConditionalOnClass(ElasticsearchOperations::class) @EnableElasticsearchRepositories(basePackages = ["com.ritense.document.opensearch.repository"]) @EnableConfigurationProperties(OpenSearchProperties::class) +@EnableScheduling class DocumentOpenSearchAutoConfiguration { @Bean @@ -240,8 +245,46 @@ class DocumentOpenSearchAutoConfiguration { logger.info { "Document search engine set to: ${engine.name}" } } + @Bean + @ConditionalOnMissingBean + @ConditionalOnProperty( + prefix = "valtimo.opensearch", + name = ["health-check-enabled"], + havingValue = "true", + matchIfMissing = true + ) + fun openSearchHealthService( + restHighLevelClient: org.opensearch.client.RestHighLevelClient, + toggle: SearchEngineToggle, + openSearchProperties: OpenSearchProperties, + ): OpenSearchHealthService = + OpenSearchHealthService(restHighLevelClient, toggle, openSearchProperties) + + @Bean + @ConditionalOnProperty( + prefix = "valtimo.opensearch", + name = ["health-check-enabled"], + havingValue = "true", + matchIfMissing = true + ) + fun openSearchHealthScheduler( + healthService: OpenSearchHealthService, + openSearchProperties: OpenSearchProperties, + ): OpenSearchHealthScheduler = + OpenSearchHealthScheduler(healthService, openSearchProperties) + companion object { private val logger = KotlinLogging.logger {} const val SEARCH_ENGINE_TOGGLE_KEY = "useOpenSearchForDocumentSearch" } } + +class OpenSearchHealthScheduler( + private val healthService: OpenSearchHealthService, + private val properties: OpenSearchProperties, +) { + @Scheduled(fixedDelayString = "\${valtimo.opensearch.health-check-interval-ms:30000}") + fun checkHealth() { + healthService.checkAndRecover() + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DelegatingDocumentSearchService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DelegatingDocumentSearchService.kt index c6edff29a1..0434da5538 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DelegatingDocumentSearchService.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DelegatingDocumentSearchService.kt @@ -31,39 +31,72 @@ class DelegatingDocumentSearchService( private val toggle: SearchEngineToggle, ) : DocumentSearchService { - private fun active(): DocumentSearchService = - if (toggle.get() == SearchEngineToggle.Engine.OPENSEARCH) openSearchService else jpaService - override fun search( searchRequest: SearchRequest, blueprintType: BlueprintType, pageable: Pageable - ): Page = active().search(searchRequest, blueprintType, pageable) + ): Page = executeWithFallback { active().search(searchRequest, blueprintType, pageable) } override fun search( documentDefinitionName: String, blueprintType: BlueprintType, searchWithConfigRequest: SearchWithConfigRequest, pageable: Pageable - ): Page = active().search(documentDefinitionName, blueprintType, searchWithConfigRequest, pageable) + ): Page = executeWithFallback { + active().search(documentDefinitionName, blueprintType, searchWithConfigRequest, pageable) + } override fun search( documentDefinitionName: String, blueprintType: BlueprintType, advancedSearchRequest: AdvancedSearchRequest, pageable: Pageable - ): Page = active().search(documentDefinitionName, blueprintType, advancedSearchRequest, pageable) + ): Page = executeWithFallback { + active().search(documentDefinitionName, blueprintType, advancedSearchRequest, pageable) + } override fun searchForExport( documentDefinitionName: String, blueprintType: BlueprintType, searchWithConfigRequest: SearchWithConfigRequest, pageable: Pageable - ): Page = active().searchForExport(documentDefinitionName, blueprintType, searchWithConfigRequest, pageable) + ): Page = executeWithFallback { + active().searchForExport(documentDefinitionName, blueprintType, searchWithConfigRequest, pageable) + } override fun count( documentDefinitionName: String, blueprintType: BlueprintType, advancedSearchRequest: AdvancedSearchRequest - ): Long = active().count(documentDefinitionName, blueprintType, advancedSearchRequest) + ): Long = executeWithFallback { active().count(documentDefinitionName, blueprintType, advancedSearchRequest) } + + private fun active(): DocumentSearchService = + if (toggle.shouldUsePostgres()) jpaService else openSearchService + + private fun executeWithFallback(block: () -> T): T { + if (toggle.shouldUsePostgres()) { + return block() + } + return try { + block() + } catch (e: Exception) { + if (isConnectionError(e)) { + toggle.activateFallback() + block() + } else { + throw e + } + } + } + + private fun isConnectionError(e: Exception): Boolean { + val message = e.message?.lowercase() ?: "" + return e is java.net.ConnectException || + e is java.io.IOException || + message.contains("connection refused") || + message.contains("connect timed out") || + message.contains("no route to host") || + e.cause?.let { isConnectionError(it as? Exception ?: return false) } ?: false + } + } diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchHealthService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchHealthService.kt new file mode 100644 index 0000000000..024d4f9a01 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchHealthService.kt @@ -0,0 +1,58 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.ritense.document.opensearch.OpenSearchProperties +import io.github.oshai.kotlinlogging.KotlinLogging +import org.opensearch.client.RequestOptions +import org.opensearch.client.RestHighLevelClient + +class OpenSearchHealthService( + private val restHighLevelClient: RestHighLevelClient, + private val toggle: SearchEngineToggle, + private val properties: OpenSearchProperties, +) { + + fun checkAndRecover() { + if (!toggle.isFallbackActive()) { + return + } + + val available = try { + restHighLevelClient.ping(RequestOptions.DEFAULT) + } catch (_: Exception) { + false + } + + if (available) { + logger.info { "OpenSearch is available again, deactivating fallback" } + toggle.deactivateFallback() + } else { + logFallbackWarningIfNeeded() + } + } + + private fun logFallbackWarningIfNeeded() { + if (toggle.shouldLogWarning(properties.fallbackWarningIntervalMs)) { + logger.warn { "OpenSearch unavailable, using PostgreSQL fallback" } + } + } + + companion object { + private val logger = KotlinLogging.logger {} + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/SearchEngineToggle.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/SearchEngineToggle.kt index ea866b7e89..b2c9f5e061 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/SearchEngineToggle.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/SearchEngineToggle.kt @@ -16,6 +16,8 @@ package com.ritense.document.opensearch.service +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicLong import java.util.concurrent.atomic.AtomicReference class SearchEngineToggle(default: Engine = Engine.OPENSEARCH) { @@ -23,10 +25,35 @@ class SearchEngineToggle(default: Engine = Engine.OPENSEARCH) { enum class Engine { OPENSEARCH, POSTGRES } private val active = AtomicReference(default) + private val fallbackActive = AtomicBoolean(false) + private val lastWarningTime = AtomicLong(0) fun get(): Engine = active.get() fun set(engine: Engine) { active.set(engine) } + + fun isFallbackActive(): Boolean = fallbackActive.get() + + fun activateFallback() { + fallbackActive.set(true) + } + + fun deactivateFallback() { + fallbackActive.set(false) + lastWarningTime.set(0) + } + + fun shouldUsePostgres(): Boolean = + get() == Engine.POSTGRES || (get() == Engine.OPENSEARCH && fallbackActive.get()) + + fun shouldLogWarning(intervalMs: Long): Boolean { + val now = System.currentTimeMillis() + val last = lastWarningTime.get() + if (now - last >= intervalMs) { + return lastWarningTime.compareAndSet(last, now) + } + return false + } } From c88fae74d951c5697e9816bd7f70da55741f28e4 Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Mon, 6 Jul 2026 08:51:22 +0200 Subject: [PATCH 18/46] changed opensearch to be disabled by default --- .../opensearch/OpenSearchProperties.kt | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/OpenSearchProperties.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/OpenSearchProperties.kt index c28a90079b..358cf4e645 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/OpenSearchProperties.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/OpenSearchProperties.kt @@ -1,17 +1,19 @@ /* - * Copyright 2015-2024 Ritense BV, the Netherlands. * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * * Copyright 2015-2026 Ritense BV, the Netherlands. + * * + * * Licensed under EUPL, Version 1.2 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" basis, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ package com.ritense.document.opensearch @@ -20,7 +22,7 @@ import org.springframework.boot.context.properties.ConfigurationProperties @ConfigurationProperties(prefix = "valtimo.opensearch") data class OpenSearchProperties( - val enabled: Boolean = true, + val enabled: Boolean = false, val healthCheckEnabled: Boolean = true, val healthCheckIntervalMs: Long = 30000, val fallbackWarningIntervalMs: Long = 300000 From 68aa1530f7e1bb42654202deb6af54aa93330c97 Mon Sep 17 00:00:00 2001 From: Ivo Zaal <74657121+ivo-ritense@users.noreply.github.com> Date: Mon, 6 Jul 2026 11:04:04 +0200 Subject: [PATCH 19/46] Opensearch improvements (#774) * Opensearch re-indexing * Reconcile missing index updates --- .../gzac/src/main/resources/application.yml | 6 - backend/case-opensearch/build.gradle | 4 + .../docker-compose-override-postgresql.yml | 6 + .../opensearch/OpenSearchProperties.kt | 49 ++- .../DocumentOpenSearchAutoConfiguration.kt | 194 +++++++++--- ...udeElasticsearchAutoConfigurationFilter.kt | 56 ++++ .../domain/JsonSchemaDocumentOsDocument.kt | 5 + .../domain/OpenSearchReconcileState.kt | 46 +++ .../opensearch/domain/OpenSearchReindexRun.kt | 113 +++++++ .../opensearch/domain/PendingIndexDeletion.kt | 45 +++ .../opensearch/domain/ReindexRunStatus.kt | 31 ++ .../handler/DocumentOpenSearchEventHandler.kt | 62 ---- .../DocumentOpenSearchEventListener.kt | 126 ++++++++ .../handler/PendingIndexDeletionListener.kt | 48 +++ .../OpenSearchReconcileStateRepository.kt | 22 ++ .../OpenSearchReindexRunRepository.kt | 30 ++ .../PendingIndexDeletionRepository.kt | 26 ++ ...ocumentOpenSearchHttpSecurityConfigurer.kt | 14 +- .../DelegatingDocumentSearchService.kt | 5 +- .../DocumentOpenSearchBackfillService.kt | 168 ---------- .../DocumentOpenSearchIndexInitializer.kt | 69 ++++ .../service/DocumentOpenSearchReconcileJob.kt | 48 +++ .../DocumentOpenSearchReconcileService.kt | 223 +++++++++++++ .../DocumentOpenSearchReindexService.kt | 215 +++++++++++++ .../service/DocumentOpenSearchSyncService.kt | 65 ++-- .../service/JsonSchemaDocumentOsConverter.kt | 102 ++++++ .../service/OpenSearchReindexRunService.kt | 184 +++++++++++ .../opensearch/service/ReindexProgressGate.kt | 60 ++++ .../opensearch/service/ReindexRequest.kt | 47 +++ .../opensearch/service/SearchEngineToggle.kt | 20 +- ...t => DocumentOpenSearchReindexResource.kt} | 36 ++- .../opensearch/web/SearchEngineResource.kt | 8 + .../main/resources/META-INF/spring.factories | 2 + .../BaseOpenSearchIntegrationTest.kt | 30 +- .../opensearch/OpenSearchPropertiesTest.kt | 48 +++ .../document/opensearch/TestApplication.kt | 13 +- .../DocumentOpenSearchEventListenerTest.kt | 123 ++++++++ .../DelegatingDocumentSearchServiceTest.kt | 72 +++++ .../DocumentOpenSearchLiveSyncIntTest.kt | 113 +++++++ .../DocumentOpenSearchReconcileIntTest.kt | 195 ++++++++++++ ...DocumentOpenSearchReindexServiceIntTest.kt | 295 ++++++++++++++++++ .../DocumentOpenSearchReindexServiceTest.kt | 107 +++++++ .../DocumentOpenSearchSyncServiceTest.kt | 147 ++++----- .../DocumentOpenSearchVersioningIntTest.kt | 145 +++++++++ .../JsonSchemaDocumentOsConverterTest.kt | 127 ++++++++ .../OpenSearchReindexRunServiceTest.kt | 175 +++++++++++ .../service/ReindexProgressGateTest.kt | 76 +++++ .../opensearch/service/ReindexRequestTest.kt | 46 +++ .../DocumentOpenSearchReindexResourceTest.kt | 92 ++++++ .../web/SearchEngineResourceTest.kt | 14 +- .../config/application-postgresql.yml | 5 + .../src/test/resources/config/application.yml | 5 + .../org.mockito.plugins.MockMaker | 1 + .../domain/impl/JsonSchemaDocument.java | 17 + .../impl/JsonSchemaDocumentService.java | 15 +- .../impl/JsonSchemaDocumentServiceTest.java | 7 +- .../liquibase/13-32-0/13-32-0-master.xml | 3 + .../20260630-create-reindex-run-table.xml | 58 ++++ ...01-create-pending-index-deletion-table.xml | 38 +++ .../20260701-create-reconcile-state-table.xml | 35 +++ ...add-changed-on-to-json-schema-document.xml | 41 +++ .../initial-setup/initial-setup-master.xml | 1 + 62 files changed, 3721 insertions(+), 458 deletions(-) create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/ExcludeElasticsearchAutoConfigurationFilter.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/OpenSearchReconcileState.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/OpenSearchReindexRun.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/PendingIndexDeletion.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/ReindexRunStatus.kt delete mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/handler/DocumentOpenSearchEventHandler.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/handler/DocumentOpenSearchEventListener.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/handler/PendingIndexDeletionListener.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/repository/OpenSearchReconcileStateRepository.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/repository/OpenSearchReindexRunRepository.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/repository/PendingIndexDeletionRepository.kt delete mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchBackfillService.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchIndexInitializer.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReconcileJob.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReconcileService.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOsConverter.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunService.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/ReindexProgressGate.kt create mode 100644 backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/ReindexRequest.kt rename backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/{DocumentOpenSearchBackfillResource.kt => DocumentOpenSearchReindexResource.kt} (52%) create mode 100644 backend/case-opensearch/src/main/resources/META-INF/spring.factories create mode 100644 backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/OpenSearchPropertiesTest.kt create mode 100644 backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/handler/DocumentOpenSearchEventListenerTest.kt create mode 100644 backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DelegatingDocumentSearchServiceTest.kt create mode 100644 backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchLiveSyncIntTest.kt create mode 100644 backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReconcileIntTest.kt create mode 100644 backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceIntTest.kt create mode 100644 backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceTest.kt create mode 100644 backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchVersioningIntTest.kt create mode 100644 backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOsConverterTest.kt create mode 100644 backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunServiceTest.kt create mode 100644 backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/ReindexProgressGateTest.kt create mode 100644 backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/ReindexRequestTest.kt create mode 100644 backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/web/DocumentOpenSearchReindexResourceTest.kt create mode 100644 backend/case-opensearch/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker create mode 100644 backend/core/src/main/resources/config/liquibase/13-32-0/20260630-create-reindex-run-table.xml create mode 100644 backend/core/src/main/resources/config/liquibase/13-32-0/20260701-create-pending-index-deletion-table.xml create mode 100644 backend/core/src/main/resources/config/liquibase/13-32-0/20260701-create-reconcile-state-table.xml create mode 100644 backend/core/src/main/resources/config/liquibase/initial-setup/case/changelog/20260701-add-changed-on-to-json-schema-document.xml diff --git a/backend/app/gzac/src/main/resources/application.yml b/backend/app/gzac/src/main/resources/application.yml index 8ae8e9b37b..4c5ab191c0 100644 --- a/backend/app/gzac/src/main/resources/application.yml +++ b/backend/app/gzac/src/main/resources/application.yml @@ -32,8 +32,6 @@ spring: enabled: false livereload: enabled: false - elasticsearch: - uris: ${SPRING_ELASTICSEARCH_URIS:http://localhost:9200} datasource: type: com.zaxxer.hikari.HikariDataSource driver-class-name: ${SPRING_DATASOURCE_DRIVER_CLASS_NAME:org.postgresql.Driver} @@ -121,8 +119,6 @@ spring: autoconfigure: exclude: - org.springframework.boot.actuate.autoconfigure.metrics.web.tomcat.TomcatMetricsAutoConfiguration - - org.springframework.boot.autoconfigure.data.elasticsearch.ElasticsearchDataAutoConfiguration - - org.springframework.boot.autoconfigure.elasticsearch.ElasticsearchClientAutoConfiguration cloud: function: definition: kotlinInboxCloudEventConsumer @@ -175,8 +171,6 @@ mailing: sendRedirectedMailsTo: valtimo: - opensearch: - enabled: ${VALTIMO_OPENSEARCH_ENABLED:true} app: scheme: http hostname: ${VALTIMO_APP_HOSTNAME} diff --git a/backend/case-opensearch/build.gradle b/backend/case-opensearch/build.gradle index 81e915ac27..0cf3155a73 100644 --- a/backend/case-opensearch/build.gradle +++ b/backend/case-opensearch/build.gradle @@ -32,6 +32,10 @@ dependencies { implementation project(":backend:inbox") implementation project(":backend:outbox") + // ShedLock for cross-instance coordination of the re-index job. The LockProvider bean itself + // is supplied at runtime by core's SchedulerAutoConfiguration; here we only need the API. + implementation "net.javacrumbs.shedlock:shedlock-spring:${shedlockVersion}" + implementation "org.springframework.boot:spring-boot-starter-data-jpa" implementation "org.springframework.boot:spring-boot-starter-web" implementation "org.springframework.boot:spring-boot-starter-security" diff --git a/backend/case-opensearch/docker-compose-override-postgresql.yml b/backend/case-opensearch/docker-compose-override-postgresql.yml index b8f123040d..b1be670e23 100644 --- a/backend/case-opensearch/docker-compose-override-postgresql.yml +++ b/backend/case-opensearch/docker-compose-override-postgresql.yml @@ -12,3 +12,9 @@ services: - DISABLE_INSTALL_DEMO_CONFIG=true ports: - "39200:9200" + healthcheck: + test: ["CMD-SHELL", "curl -sf http://localhost:9200/_cluster/health || exit 1"] + interval: 5s + timeout: 10s + retries: 40 + start_period: 15s diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/OpenSearchProperties.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/OpenSearchProperties.kt index 358cf4e645..d045a941c7 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/OpenSearchProperties.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/OpenSearchProperties.kt @@ -19,11 +19,56 @@ package com.ritense.document.opensearch import org.springframework.boot.context.properties.ConfigurationProperties +import org.springframework.boot.context.properties.NestedConfigurationProperty +import java.time.Duration @ConfigurationProperties(prefix = "valtimo.opensearch") data class OpenSearchProperties( val enabled: Boolean = false, val healthCheckEnabled: Boolean = true, val healthCheckIntervalMs: Long = 30000, - val fallbackWarningIntervalMs: Long = 300000 -) + val fallbackWarningIntervalMs: Long = 300000, + + @NestedConfigurationProperty + val reconcile: Reconcile = Reconcile(), + + @NestedConfigurationProperty + val reindex: Reindex = Reindex(), +) { + /** + * Configuration for the self-healing reconciler that keeps the OpenSearch index in sync with + * PostgreSQL as a derived read-model. + * + * @property enabled whether the scheduled reconcile job runs at all. + * @property interval delay between the end of one reconcile cycle and the start of the next + * (also bound directly by the job's `@Scheduled(fixedDelayString)`). + * @property overlap δ subtracted from the watermark each cycle to safely cover the + * flush→commit boundary; re-indexes a small trailing window (idempotent). + * @property pageSize DB keyset page size for the incremental upsert scan. + * @property pendingDeletionBatchSize number of pending index deletions drained per batch. + */ + data class Reconcile( + val enabled: Boolean = true, + // Keep in sync with the @Scheduled(fixedDelayString) default in DocumentOpenSearchReconcileJob (PT2M). + val interval: Duration = Duration.ofMinutes(2), + val overlap: Duration = Duration.ofSeconds(10), + val pageSize: Int = 5000, + val pendingDeletionBatchSize: Int = 500, + ) + + /** + * Behaviour while an admin (re)index run is filling the index. + * + * @property fallbackToPostgresWhileRunning while a reindex run is in progress, route document search + * to PostgreSQL so users never query a partially-filled index; search returns to OpenSearch + * automatically once all runs finish. Does not affect the reconciler (which keeps the index + * complete) — only the admin reindex. + * @property runningHeartbeatTimeout a RUNNING run is only treated as in-progress while its heartbeat + * is fresher than this. Guards against a run left behind by a crashed instance pinning search + * to PostgreSQL indefinitely. Must exceed the longest expected gap between reindex batches. + */ + data class Reindex( + val fallbackToPostgresWhileRunning: Boolean = true, + val runningHeartbeatTimeout: Duration = Duration.ofMinutes(5), + ) +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt index d199e3ba90..db95897942 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt @@ -26,18 +26,28 @@ import com.ritense.document.opensearch.authorization.OpenSearchAuthorizationEnti import com.ritense.document.opensearch.authorization.OpenSearchPermissionConditionTranslator import com.ritense.document.opensearch.authorization.mapper.JsonSchemaDocumentCaseDefinitionOpenSearchMapper import com.ritense.document.opensearch.authorization.mapper.JsonSchemaDocumentDefinitionOpenSearchMapper -import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument +import com.ritense.document.opensearch.domain.OpenSearchReindexRun import com.ritense.document.opensearch.handler.DocumentOpenSearchEventListener +import com.ritense.document.opensearch.handler.PendingIndexDeletionListener import com.ritense.document.opensearch.repository.JsonSchemaDocumentOpenSearchRepository +import com.ritense.document.opensearch.repository.OpenSearchReconcileStateRepository +import com.ritense.document.opensearch.repository.OpenSearchReindexRunRepository +import com.ritense.document.opensearch.repository.PendingIndexDeletionRepository import com.ritense.document.opensearch.security.DocumentOpenSearchHttpSecurityConfigurer import com.ritense.document.opensearch.service.DelegatingDocumentSearchService -import com.ritense.document.opensearch.service.DocumentOpenSearchBackfillService import com.ritense.document.opensearch.service.DocumentOpenSearchQueryService +import com.ritense.document.opensearch.service.DocumentOpenSearchReconcileJob +import com.ritense.document.opensearch.service.DocumentOpenSearchIndexInitializer +import com.ritense.document.opensearch.service.DocumentOpenSearchReconcileService +import com.ritense.document.opensearch.service.DocumentOpenSearchReindexService import com.ritense.document.opensearch.service.DocumentOpenSearchSyncService import com.ritense.document.opensearch.service.JsonSchemaDocumentOpenSearchService +import com.ritense.document.opensearch.service.JsonSchemaDocumentOsConverter +import com.ritense.document.opensearch.service.OpenSearchReindexRunService +import com.ritense.document.opensearch.service.ReindexProgressGate import com.ritense.document.opensearch.service.OpenSearchHealthService import com.ritense.document.opensearch.service.SearchEngineToggle -import com.ritense.document.opensearch.web.DocumentOpenSearchBackfillResource +import com.ritense.document.opensearch.web.DocumentOpenSearchReindexResource import com.ritense.document.opensearch.web.SearchEngineResource import com.ritense.document.repository.impl.JsonSchemaDocumentRepository import com.ritense.document.service.DocumentSearchService @@ -49,26 +59,33 @@ import com.ritense.outbox.OutboxService import com.ritense.valtimo.contract.authentication.TeamManagementService import com.ritense.valtimo.contract.authentication.UserManagementService import jakarta.persistence.EntityManager +import net.javacrumbs.shedlock.core.LockProvider import org.springframework.boot.ApplicationRunner import org.springframework.boot.autoconfigure.AutoConfiguration import org.springframework.boot.autoconfigure.AutoConfigureBefore +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean import org.springframework.boot.autoconfigure.condition.ConditionalOnClass import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.boot.autoconfigure.domain.EntityScan import org.springframework.context.annotation.Bean import org.springframework.core.annotation.Order import org.springframework.data.elasticsearch.core.ElasticsearchOperations import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories +import org.springframework.data.jpa.repository.config.EnableJpaRepositories +import org.springframework.transaction.PlatformTransactionManager import org.springframework.scheduling.annotation.EnableScheduling import org.springframework.scheduling.annotation.Scheduled @AutoConfiguration @AutoConfigureBefore(DocumentAutoConfiguration::class) @ConditionalOnClass(ElasticsearchOperations::class) +@EnableScheduling @EnableElasticsearchRepositories(basePackages = ["com.ritense.document.opensearch.repository"]) @EnableConfigurationProperties(OpenSearchProperties::class) -@EnableScheduling +@EnableJpaRepositories(basePackageClasses = [OpenSearchReindexRunRepository::class]) +@EntityScan(basePackageClasses = [OpenSearchReindexRun::class]) class DocumentOpenSearchAutoConfiguration { @Bean @@ -99,28 +116,101 @@ class DocumentOpenSearchAutoConfiguration { ): DocumentOpenSearchQueryService = DocumentOpenSearchQueryService(elasticsearchOperations, authorizationService, translator) + @Bean + @ConditionalOnMissingBean + fun jsonSchemaDocumentOsConverter( + objectMapper: ObjectMapper, + openSearchRepository: JsonSchemaDocumentOpenSearchRepository, + ): JsonSchemaDocumentOsConverter = + JsonSchemaDocumentOsConverter(objectMapper, openSearchRepository) + @Bean @ConditionalOnMissingBean fun documentOpenSearchSyncService( repository: JsonSchemaDocumentOpenSearchRepository, - objectMapper: ObjectMapper, + documentRepository: JsonSchemaDocumentRepository, + converter: JsonSchemaDocumentOsConverter, + transactionManager: PlatformTransactionManager, ): DocumentOpenSearchSyncService = - DocumentOpenSearchSyncService(repository, objectMapper) + DocumentOpenSearchSyncService(repository, documentRepository, converter, transactionManager) @Bean - fun documentOpenSearchEventListener(syncService: DocumentOpenSearchSyncService): DocumentOpenSearchEventListener = - DocumentOpenSearchEventListener(syncService) + @ConditionalOnProperty(prefix = "valtimo.opensearch", name = ["enabled"], havingValue = "true", matchIfMissing = true) + fun documentOpenSearchEventListener( + syncService: DocumentOpenSearchSyncService, + searchEngineToggle: SearchEngineToggle, + ): DocumentOpenSearchEventListener = + DocumentOpenSearchEventListener(syncService, searchEngineToggle) @Bean @ConditionalOnMissingBean - fun documentOpenSearchBackfillService( + fun openSearchReindexRunService( + openSearchReindexRunRepository: OpenSearchReindexRunRepository, + objectMapper: ObjectMapper, + openSearchProperties: OpenSearchProperties, + ): OpenSearchReindexRunService = + OpenSearchReindexRunService(openSearchReindexRunRepository, objectMapper, openSearchProperties) + + @Bean + @ConditionalOnMissingBean + fun documentOpenSearchReindexService( entityManager: EntityManager, + converter: JsonSchemaDocumentOsConverter, + elasticsearchOperations: ElasticsearchOperations, + transactionManager: PlatformTransactionManager, + lockProvider: LockProvider, + openSearchReindexRunService: OpenSearchReindexRunService, + ): DocumentOpenSearchReindexService = + DocumentOpenSearchReindexService( + entityManager, + converter, + elasticsearchOperations, + transactionManager, + lockProvider, + openSearchReindexRunService, + ) + + @Bean + @ConditionalOnMissingBean + @ConditionalOnProperty(prefix = "valtimo.opensearch", name = ["enabled"], havingValue = "true", matchIfMissing = true) + fun documentOpenSearchReconcileService( + entityManager: EntityManager, + converter: JsonSchemaDocumentOsConverter, openSearchRepository: JsonSchemaDocumentOpenSearchRepository, - objectMapper: ObjectMapper, - restHighLevelClient: org.opensearch.client.RestHighLevelClient, - transactionManager: org.springframework.transaction.PlatformTransactionManager, - ): DocumentOpenSearchBackfillService = - DocumentOpenSearchBackfillService(entityManager, openSearchRepository, objectMapper, restHighLevelClient, transactionManager) + reconcileStateRepository: OpenSearchReconcileStateRepository, + pendingIndexDeletionRepository: PendingIndexDeletionRepository, + transactionManager: PlatformTransactionManager, + lockProvider: LockProvider, + openSearchProperties: OpenSearchProperties, + ): DocumentOpenSearchReconcileService = + DocumentOpenSearchReconcileService( + entityManager, + converter, + openSearchRepository, + reconcileStateRepository, + pendingIndexDeletionRepository, + transactionManager, + lockProvider, + openSearchProperties, + ) + + @Bean + @ConditionalOnMissingBean + @ConditionalOnBean(DocumentOpenSearchReconcileService::class) + @ConditionalOnProperty(prefix = "valtimo.opensearch.reconcile", name = ["enabled"], havingValue = "true", matchIfMissing = true) + fun documentOpenSearchReconcileJob( + reconcileService: DocumentOpenSearchReconcileService, + searchEngineToggle: SearchEngineToggle, + ): DocumentOpenSearchReconcileJob = + DocumentOpenSearchReconcileJob(reconcileService, searchEngineToggle) + + @Bean + @ConditionalOnMissingBean + @ConditionalOnProperty(prefix = "valtimo.opensearch", name = ["enabled"], havingValue = "true", matchIfMissing = true) + fun pendingIndexDeletionListener( + pendingIndexDeletionRepository: PendingIndexDeletionRepository, + ): PendingIndexDeletionListener = + PendingIndexDeletionListener(pendingIndexDeletionRepository) @Order(294) @Bean @@ -132,7 +222,18 @@ class DocumentOpenSearchAutoConfiguration { @Bean @ConditionalOnMissingBean - fun searchEngineToggle(): SearchEngineToggle = SearchEngineToggle() + // Start in POSTGRES so no OpenSearch call happens before searchEngineSettingLoader resolves the real + // engine. The @Scheduled reconciler is armed during context refresh, before ApplicationRunners run, so + // an OPENSEARCH default could otherwise leak one spurious call at startup even when the engine is off. + fun searchEngineToggle(): SearchEngineToggle = SearchEngineToggle(SearchEngineToggle.Engine.POSTGRES) + + @Bean + @ConditionalOnMissingBean + fun reindexProgressGate( + openSearchReindexRunService: OpenSearchReindexRunService, + openSearchProperties: OpenSearchProperties, + ): ReindexProgressGate = + ReindexProgressGate(openSearchReindexRunService, openSearchProperties) @Bean("openSearchDocumentSearchService") fun openSearchDocumentSearchService( @@ -174,8 +275,11 @@ class DocumentOpenSearchAutoConfiguration { openSearchDocumentSearchService: JsonSchemaDocumentOpenSearchService, jpaDocumentSearchService: JsonSchemaDocumentSearchService, searchEngineToggle: SearchEngineToggle, + reindexProgressGate: ReindexProgressGate, ): DelegatingDocumentSearchService = - DelegatingDocumentSearchService(openSearchDocumentSearchService, jpaDocumentSearchService, searchEngineToggle) + DelegatingDocumentSearchService( + openSearchDocumentSearchService, jpaDocumentSearchService, searchEngineToggle, reindexProgressGate, + ) @Bean @ConditionalOnMissingBean @@ -183,54 +287,36 @@ class DocumentOpenSearchAutoConfiguration { toggle: SearchEngineToggle, openSearchProperties: OpenSearchProperties, featureToggleOverridesService: FeatureToggleOverridesService, + indexInitializer: DocumentOpenSearchIndexInitializer, ): SearchEngineResource = - SearchEngineResource(toggle, openSearchProperties, featureToggleOverridesService) + SearchEngineResource(toggle, openSearchProperties, featureToggleOverridesService, indexInitializer) @Bean @ConditionalOnMissingBean - fun documentOpenSearchBackfillResource( - backfillService: DocumentOpenSearchBackfillService, - ): DocumentOpenSearchBackfillResource = - DocumentOpenSearchBackfillResource(backfillService) + fun documentOpenSearchReindexResource( + reindexService: DocumentOpenSearchReindexService, + ): DocumentOpenSearchReindexResource = + DocumentOpenSearchReindexResource(reindexService) - /** - * Creates the OpenSearch index and mappings on startup if the index does not yet exist. - */ @Bean - fun documentOpenSearchIndexInitializer(elasticsearchOperations: ElasticsearchOperations): ApplicationRunner = - ApplicationRunner { - try { - val indexOps = elasticsearchOperations.indexOps(JsonSchemaDocumentOsDocument::class.java) - if (!indexOps.exists()) { - val settings = org.springframework.data.elasticsearch.core.document.Document.create() - settings["index.number_of_replicas"] = 0 - indexOps.create(settings) - - // Merge annotated mapping with a dynamic template that forces all - // content.* fields to text+keyword — enables wildcard search on numbers too - val annotatedMapping = indexOps.createMapping(JsonSchemaDocumentOsDocument::class.java) - val dynamicTemplates = listOf( - mapOf("content_fields_as_text" to mapOf( - "path_match" to "content.*", - "mapping" to mapOf( - "type" to "text", - "fields" to mapOf("keyword" to mapOf("type" to "keyword", "ignore_above" to 256)) - ) - )) - ) - annotatedMapping["dynamic_templates"] = dynamicTemplates - indexOps.putMapping(annotatedMapping) - } - } catch (e: Exception) { - logger.warn(e) { "Failed to initialize OpenSearch index — is OpenSearch running?" } - } - } + @ConditionalOnMissingBean + fun documentOpenSearchIndexInitializer( + elasticsearchOperations: ElasticsearchOperations, + ): DocumentOpenSearchIndexInitializer = + DocumentOpenSearchIndexInitializer(elasticsearchOperations) + /** + * Resolves the active search engine on startup from configuration and the persisted feature-toggle + * override, then — only when OpenSearch is the active engine — provisions the index. Merged into a + * single ordered runner so the toggle is always set before any index/OpenSearch work is decided, and + * so a disabled or toggled-off engine performs no active OpenSearch call at all on boot. + */ @Bean fun searchEngineSettingLoader( toggle: SearchEngineToggle, featureToggleOverridesService: FeatureToggleOverridesService, openSearchProperties: OpenSearchProperties, + indexInitializer: DocumentOpenSearchIndexInitializer, ): ApplicationRunner = ApplicationRunner { if (!openSearchProperties.enabled) { toggle.set(SearchEngineToggle.Engine.POSTGRES) @@ -243,6 +329,10 @@ class DocumentOpenSearchAutoConfiguration { val engine = if (useOpenSearch) SearchEngineToggle.Engine.OPENSEARCH else SearchEngineToggle.Engine.POSTGRES toggle.set(engine) logger.info { "Document search engine set to: ${engine.name}" } + + if (toggle.isOpenSearchActive()) { + indexInitializer.ensureIndex() + } } @Bean diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/ExcludeElasticsearchAutoConfigurationFilter.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/ExcludeElasticsearchAutoConfigurationFilter.kt new file mode 100644 index 0000000000..3bfa920f4a --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/ExcludeElasticsearchAutoConfigurationFilter.kt @@ -0,0 +1,56 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.autoconfigure + +import org.springframework.boot.autoconfigure.AutoConfigurationImportFilter +import org.springframework.boot.autoconfigure.AutoConfigurationMetadata + +/** + * Vetoes Spring Boot's built-in Elasticsearch auto-configurations. + * + * The `spring-data-opensearch-starter` transitively puts `spring-data-elasticsearch` (and its + * Elasticsearch client classes) on the classpath. Spring Boot detects those classes and activates + * its own Elasticsearch auto-configuration — including a reactive REST client configuration that + * fails to construct against OpenSearch and aborts application startup with + * "Lookup method resolution failed". + * + * OpenSearch connectivity is provided instead by spring-data-opensearch's own auto-configuration + * (driven by the `opensearch.*` properties), so Boot's Elasticsearch auto-configs are not just + * unnecessary but actively harmful. Excluding them here — inside the library — means any consuming + * application gets a working setup out of the box, without having to add + * `spring.autoconfigure.exclude` entries to its own configuration. + */ +class ExcludeElasticsearchAutoConfigurationFilter : AutoConfigurationImportFilter { + + override fun match( + autoConfigurationClasses: Array, + autoConfigurationMetadata: AutoConfigurationMetadata, + ): BooleanArray = BooleanArray(autoConfigurationClasses.size) { index -> + autoConfigurationClasses[index] !in EXCLUDED_AUTO_CONFIGURATIONS + } + + companion object { + private val EXCLUDED_AUTO_CONFIGURATIONS = setOf( + "org.springframework.boot.autoconfigure.elasticsearch.ElasticsearchClientAutoConfiguration", + "org.springframework.boot.autoconfigure.elasticsearch.ElasticsearchRestClientAutoConfiguration", + "org.springframework.boot.autoconfigure.elasticsearch.ReactiveElasticsearchClientAutoConfiguration", + "org.springframework.boot.autoconfigure.data.elasticsearch.ElasticsearchDataAutoConfiguration", + "org.springframework.boot.autoconfigure.data.elasticsearch.ElasticsearchRepositoriesAutoConfiguration", + "org.springframework.boot.autoconfigure.data.elasticsearch.ReactiveElasticsearchRepositoriesAutoConfiguration", + ) + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/JsonSchemaDocumentOsDocument.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/JsonSchemaDocumentOsDocument.kt index c093ab4e9e..c4583c391c 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/JsonSchemaDocumentOsDocument.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/JsonSchemaDocumentOsDocument.kt @@ -17,6 +17,7 @@ package com.ritense.document.opensearch.domain import org.springframework.data.annotation.Id +import org.springframework.data.annotation.Version import org.springframework.data.elasticsearch.annotations.Document import org.springframework.data.elasticsearch.annotations.Field import org.springframework.data.elasticsearch.annotations.FieldType @@ -58,6 +59,10 @@ data class JsonSchemaDocumentOsDocument( otherFields = [InnerField(suffix = "keyword", type = FieldType.Keyword)], ) val contentText: String? = null, + // OpenSearch external version (maps to _version metadata, VersionType.EXTERNAL — not a source field, so + // no index-mapping change). Populated from the JPA optimistic-lock counter; "highest version wins" makes + // redundant reconciler re-sends and stale async writes benign version-conflict no-ops. + @Version val indexVersion: Long? = null, ) data class OsDefinitionId( diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/OpenSearchReconcileState.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/OpenSearchReconcileState.kt new file mode 100644 index 0000000000..18a6edd07d --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/OpenSearchReconcileState.kt @@ -0,0 +1,46 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.domain + +import jakarta.persistence.Column +import jakarta.persistence.Entity +import jakarta.persistence.Id +import jakarta.persistence.Table +import java.time.LocalDateTime + +/** + * Single-row persisted state for the OpenSearch reconciler: the [watermark] is the highest + * `json_schema_document.changed_on` value that has been fully reconciled into OpenSearch. The next cycle + * scans everything changed after (watermark − overlap). The watermark is advanced only after a completely + * successful cycle, so an OpenSearch outage simply parks it until recovery. + */ +@Entity +@Table(name = "document_index_reconcile_state") +class OpenSearchReconcileState( + + @Id + @Column(name = "id") + val id: String = SINGLETON_ID, + + @Column(name = "watermark", nullable = false) + var watermark: LocalDateTime, +) { + companion object { + /** There is only ever one reconcile-state row; this is its fixed primary key. */ + const val SINGLETON_ID = "SINGLETON" + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/OpenSearchReindexRun.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/OpenSearchReindexRun.kt new file mode 100644 index 0000000000..1833f4686c --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/OpenSearchReindexRun.kt @@ -0,0 +1,113 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.domain + +import jakarta.persistence.Column +import jakarta.persistence.Entity +import jakarta.persistence.EnumType +import jakarta.persistence.Enumerated +import jakarta.persistence.Id +import jakarta.persistence.Table +import java.time.LocalDateTime +import java.util.UUID + +/** + * Persisted record of a single OpenSearch re-index run. + * + * The state lives in the database (rather than in JVM memory) so that: + * - status is consistent regardless of which clustered instance answers a query, + * - progress survives a crash/redeploy ([lastId] is the resume cursor), + * - an orphaned [ReindexRunStatus.RUNNING] row can be reconciled on startup. + * + * [scope] holds the serialized [com.ritense.document.opensearch.service.ReindexRequest] (JSON string) + * for auditing/display only — it is never queried as JSON in the database. + */ +@Entity +@Table(name = "document_opensearch_reindex_run") +class OpenSearchReindexRun( + + @Id + @Column(name = "id") + val id: UUID = UUID.randomUUID(), + + @Enumerated(EnumType.STRING) + @Column(name = "status", nullable = false, length = 20) + var status: ReindexRunStatus = ReindexRunStatus.RUNNING, + + @Column(name = "scope") + val scope: String? = null, + + @Column(name = "page_size", nullable = false) + val pageSize: Int = 0, + + @Column(name = "last_id") + var lastId: UUID? = null, + + @Column(name = "processed_count", nullable = false) + var processedCount: Long = 0, + + @Column(name = "skipped_count", nullable = false) + var skippedCount: Long = 0, + + @Column(name = "started_on", nullable = false) + val startedOn: LocalDateTime = LocalDateTime.now(), + + @Column(name = "heartbeat_on", nullable = false) + var heartbeatOn: LocalDateTime = LocalDateTime.now(), + + @Column(name = "finished_on") + var finishedOn: LocalDateTime? = null, + + @Column(name = "error") + var error: String? = null, +) { + + /** Records progress after a committed batch: the keyset cursor, counts and a fresh heartbeat. */ + fun recordProgress(lastId: UUID?, processed: Long, skipped: Long, heartbeat: LocalDateTime) { + this.lastId = lastId + this.processedCount = processed + this.skippedCount = skipped + this.heartbeatOn = heartbeat + } + + fun complete(now: LocalDateTime) { + this.status = ReindexRunStatus.COMPLETED + this.finishedOn = now + this.heartbeatOn = now + } + + fun fail(now: LocalDateTime, error: String?) { + this.status = ReindexRunStatus.FAILED + this.finishedOn = now + this.heartbeatOn = now + this.error = error + } + + fun stop(now: LocalDateTime) { + this.status = ReindexRunStatus.STOPPED + this.finishedOn = now + this.heartbeatOn = now + } + + /** Re-arms a previously-finished (FAILED/STOPPED) run so it can be resumed from its cursor. */ + fun resume(now: LocalDateTime) { + this.status = ReindexRunStatus.RUNNING + this.finishedOn = null + this.error = null + this.heartbeatOn = now + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/PendingIndexDeletion.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/PendingIndexDeletion.kt new file mode 100644 index 0000000000..b05402bbf9 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/PendingIndexDeletion.kt @@ -0,0 +1,45 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.domain + +import jakarta.persistence.Column +import jakarta.persistence.Entity +import jakarta.persistence.Id +import jakarta.persistence.Table +import java.time.LocalDateTime +import java.util.UUID + +/** + * Durable record that a document was deleted from PostgreSQL and its OpenSearch entry still has to be + * removed. + * + * Written **inside** the deleting transaction (see + * [com.ritense.document.opensearch.handler.PendingIndexDeletionListener]) so it commits atomically with + * the delete — surviving an OpenSearch outage of any length. The reconciler drains these rows at + * O(deletes): remove each id from the index, then delete the drained rows. Idempotent. + */ +@Entity +@Table(name = "document_index_pending_deletion") +class PendingIndexDeletion( + + @Id + @Column(name = "document_id") + val documentId: UUID, + + @Column(name = "deleted_on", nullable = false) + val deletedOn: LocalDateTime = LocalDateTime.now(), +) diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/ReindexRunStatus.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/ReindexRunStatus.kt new file mode 100644 index 0000000000..3c7239c762 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/ReindexRunStatus.kt @@ -0,0 +1,31 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.domain + +enum class ReindexRunStatus { + /** The run is currently in progress (or was, for a row left behind by a crashed instance). */ + RUNNING, + + /** The run finished and indexed all documents in scope. */ + COMPLETED, + + /** The run aborted with an error; resumable from [OpenSearchReindexRun.lastId]. */ + FAILED, + + /** The run was cancelled (e.g. graceful shutdown); resumable from [OpenSearchReindexRun.lastId]. */ + STOPPED, +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/handler/DocumentOpenSearchEventHandler.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/handler/DocumentOpenSearchEventHandler.kt deleted file mode 100644 index bfe651b511..0000000000 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/handler/DocumentOpenSearchEventHandler.kt +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2015-2024 Ritense BV, the Netherlands. - * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.ritense.document.opensearch.handler - -import com.ritense.document.event.DocumentAssigned -import com.ritense.document.event.DocumentCreated -import com.ritense.document.event.DocumentDeleted -import com.ritense.document.event.DocumentRetentionDateSet -import com.ritense.document.event.DocumentRetentionDateUnset -import com.ritense.document.event.DocumentStatusChanged -import com.ritense.document.event.DocumentTagsChanged -import com.ritense.document.event.DocumentUnassigned -import com.ritense.document.event.DocumentUpdated -import com.ritense.document.opensearch.service.DocumentOpenSearchSyncService -import org.springframework.context.event.EventListener - -class DocumentOpenSearchEventListener( - private val syncService: DocumentOpenSearchSyncService, -) { - @EventListener - fun onDocumentCreated(event: DocumentCreated) = syncService.upsert(event) - - @EventListener - fun onDocumentUpdated(event: DocumentUpdated) = syncService.upsert(event) - - @EventListener - fun onDocumentAssigned(event: DocumentAssigned) = syncService.upsert(event) - - @EventListener - fun onDocumentUnassigned(event: DocumentUnassigned) = syncService.upsert(event) - - @EventListener - fun onDocumentStatusChanged(event: DocumentStatusChanged) = syncService.upsert(event) - - @EventListener - fun onDocumentTagsChanged(event: DocumentTagsChanged) = syncService.upsert(event) - - @EventListener - fun onDocumentRetentionDateSet(event: DocumentRetentionDateSet) = syncService.upsert(event) - - @EventListener - fun onDocumentRetentionDateUnset(event: DocumentRetentionDateUnset) = syncService.upsert(event) - - @EventListener - fun onDocumentDeleted(event: DocumentDeleted) { - event.resultId?.let { syncService.delete(it) } - } -} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/handler/DocumentOpenSearchEventListener.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/handler/DocumentOpenSearchEventListener.kt new file mode 100644 index 0000000000..6d08a5db47 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/handler/DocumentOpenSearchEventListener.kt @@ -0,0 +1,126 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.handler + +import com.ritense.document.domain.impl.event.JsonSchemaDocumentCreatedEvent +import com.ritense.document.domain.impl.event.JsonSchemaDocumentModifiedEvent +import com.ritense.document.event.DocumentAssigneeChangedEvent +import com.ritense.document.event.DocumentRetentionPeriodSetEvent +import com.ritense.document.event.DocumentRetentionPeriodUnsetEvent +import com.ritense.document.event.DocumentUnassignedEvent +import com.ritense.document.opensearch.service.DocumentOpenSearchSyncService +import com.ritense.document.opensearch.service.SearchEngineToggle +import com.ritense.valtimo.contract.document.event.DocumentRelatedFileAddedEvent +import com.ritense.valtimo.contract.document.event.DocumentRelatedFileRemovedEvent +import com.ritense.valtimo.contract.event.DocumentDeletedEvent +import io.github.oshai.kotlinlogging.KotlinLogging +import org.springframework.beans.factory.DisposableBean +import org.springframework.transaction.event.TransactionPhase.AFTER_COMMIT +import org.springframework.transaction.event.TransactionalEventListener +import java.util.UUID +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.RejectedExecutionException +import java.util.concurrent.TimeUnit + +/** + * Best-effort, low-latency live sync of single-document mutations into OpenSearch. + * + * Listens to the real Spring application events emitted inside each document-mutation transaction and, + * **after commit**, reloads the document by id and upserts it (or deletes it) in OpenSearch on a managed + * single-thread daemon executor. Every task is fully isolated: a failure (e.g. OpenSearch unreachable) is + * logged and swallowed so it can never fail or roll back the originating business transaction. Missed + * writes are repaired by [com.ritense.document.opensearch.service.DocumentOpenSearchReconcileService]; + * this listener is only about freshness. + * + * Status/tags changes and bulk deletes have no dedicated live event and are handled by the reconciler + * (upserts) and the pending-index-deletion drain (deletes) respectively. + */ +class DocumentOpenSearchEventListener( + private val syncService: DocumentOpenSearchSyncService, + private val toggle: SearchEngineToggle, +) : DisposableBean { + + private val executor: ExecutorService = Executors.newSingleThreadExecutor { runnable -> + Thread(runnable, "opensearch-live-sync").apply { isDaemon = true } + } + + @TransactionalEventListener(phase = AFTER_COMMIT) + fun onCreated(event: JsonSchemaDocumentCreatedEvent) = enqueueUpsert(event.documentId().id) + + @TransactionalEventListener(phase = AFTER_COMMIT) + fun onModified(event: JsonSchemaDocumentModifiedEvent) = enqueueUpsert(event.documentId().id) + + @TransactionalEventListener(phase = AFTER_COMMIT) + fun onAssigneeChanged(event: DocumentAssigneeChangedEvent) = enqueueUpsert(event.documentId) + + @TransactionalEventListener(phase = AFTER_COMMIT) + fun onUnassigned(event: DocumentUnassignedEvent) = enqueueUpsert(event.documentId) + + @TransactionalEventListener(phase = AFTER_COMMIT) + fun onRetentionSet(event: DocumentRetentionPeriodSetEvent) = enqueueUpsert(event.getDocumentId()) + + @TransactionalEventListener(phase = AFTER_COMMIT) + fun onRetentionUnset(event: DocumentRetentionPeriodUnsetEvent) = enqueueUpsert(event.getDocumentId()) + + @TransactionalEventListener(phase = AFTER_COMMIT) + fun onRelatedFileAdded(event: DocumentRelatedFileAddedEvent) = enqueueUpsert(event.documentId) + + @TransactionalEventListener(phase = AFTER_COMMIT) + fun onRelatedFileRemoved(event: DocumentRelatedFileRemovedEvent) = enqueueUpsert(event.documentId) + + @TransactionalEventListener(phase = AFTER_COMMIT) + fun onDeleted(event: DocumentDeletedEvent) = enqueueDelete(event.caseDocumentId) + + private fun enqueueUpsert(documentId: UUID) = submit { syncService.upsertById(documentId) } + + private fun enqueueDelete(documentId: UUID) = submit { syncService.delete(documentId) } + + private fun submit(task: () -> Unit) { + // Engine off (feature toggled off or OpenSearch disabled): skip the write entirely — no thread, + // no OpenSearch call. The reconciler catches up from its watermark once the engine is re-enabled. + if (!toggle.isOpenSearchActive()) return + try { + executor.execute { + try { + task() + } catch (e: Exception) { + logger.warn(e) { "Live OpenSearch sync failed — the reconciler will repair the index on its next cycle" } + } + } + } catch (e: RejectedExecutionException) { + logger.warn(e) { "Live OpenSearch sync rejected (executor shutting down) — the reconciler will repair the index" } + } + } + + override fun destroy() { + executor.shutdown() + try { + if (!executor.awaitTermination(SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + executor.shutdownNow() + } + } catch (e: InterruptedException) { + executor.shutdownNow() + Thread.currentThread().interrupt() + } + } + + companion object { + private val logger = KotlinLogging.logger {} + private const val SHUTDOWN_TIMEOUT_SECONDS = 30L + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/handler/PendingIndexDeletionListener.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/handler/PendingIndexDeletionListener.kt new file mode 100644 index 0000000000..b4ec98a229 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/handler/PendingIndexDeletionListener.kt @@ -0,0 +1,48 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.handler + +import com.ritense.document.opensearch.domain.PendingIndexDeletion +import com.ritense.document.opensearch.repository.PendingIndexDeletionRepository +import com.ritense.valtimo.contract.event.DocumentDeletedEvent +import io.github.oshai.kotlinlogging.KotlinLogging +import org.springframework.context.event.EventListener + +/** + * Records a durable pending index deletion for every deleted document. + * + * Deliberately a **synchronous, in-transaction** [EventListener] (not `@TransactionalEventListener`): the + * [DocumentDeletedEvent] is published inside the deleting transaction, so the pending-deletion row commits + * atomically with the delete. This is what guarantees deletes survive an OpenSearch outage — the + * reconciler drains the pending deletion once OpenSearch is reachable again. The best-effort AFTER_COMMIT + * delete in [DocumentOpenSearchEventListener] still runs for freshness; this listener is the durability + * backstop. + */ +open class PendingIndexDeletionListener( + private val pendingIndexDeletionRepository: PendingIndexDeletionRepository, +) { + + @EventListener + open fun onDocumentDeleted(event: DocumentDeletedEvent) { + pendingIndexDeletionRepository.save(PendingIndexDeletion(documentId = event.caseDocumentId)) + logger.debug { "Recorded pending index deletion for document ${event.caseDocumentId}" } + } + + companion object { + private val logger = KotlinLogging.logger {} + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/repository/OpenSearchReconcileStateRepository.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/repository/OpenSearchReconcileStateRepository.kt new file mode 100644 index 0000000000..79e1bb4fd3 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/repository/OpenSearchReconcileStateRepository.kt @@ -0,0 +1,22 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.repository + +import com.ritense.document.opensearch.domain.OpenSearchReconcileState +import org.springframework.data.jpa.repository.JpaRepository + +interface OpenSearchReconcileStateRepository : JpaRepository diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/repository/OpenSearchReindexRunRepository.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/repository/OpenSearchReindexRunRepository.kt new file mode 100644 index 0000000000..81cef189dd --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/repository/OpenSearchReindexRunRepository.kt @@ -0,0 +1,30 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.repository + +import com.ritense.document.opensearch.domain.OpenSearchReindexRun +import com.ritense.document.opensearch.domain.ReindexRunStatus +import org.springframework.data.jpa.repository.JpaRepository +import java.time.LocalDateTime +import java.util.UUID + +interface OpenSearchReindexRunRepository : JpaRepository { + fun findFirstByOrderByStartedOnDesc(): OpenSearchReindexRun? + fun findFirstByStatusOrderByStartedOnDesc(status: ReindexRunStatus): OpenSearchReindexRun? + fun findAllByStatusAndHeartbeatOnBefore(status: ReindexRunStatus, heartbeatOn: LocalDateTime): List + fun existsByStatusAndHeartbeatOnAfter(status: ReindexRunStatus, heartbeatOn: LocalDateTime): Boolean +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/repository/PendingIndexDeletionRepository.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/repository/PendingIndexDeletionRepository.kt new file mode 100644 index 0000000000..581a4e801d --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/repository/PendingIndexDeletionRepository.kt @@ -0,0 +1,26 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.repository + +import com.ritense.document.opensearch.domain.PendingIndexDeletion +import org.springframework.data.domain.Pageable +import org.springframework.data.jpa.repository.JpaRepository +import java.util.UUID + +interface PendingIndexDeletionRepository : JpaRepository { + fun findByOrderByDeletedOnAsc(pageable: Pageable): List +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/security/DocumentOpenSearchHttpSecurityConfigurer.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/security/DocumentOpenSearchHttpSecurityConfigurer.kt index eb4ae89ab7..e50123d3b0 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/security/DocumentOpenSearchHttpSecurityConfigurer.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/security/DocumentOpenSearchHttpSecurityConfigurer.kt @@ -30,14 +30,16 @@ class DocumentOpenSearchHttpSecurityConfigurer : HttpSecurityConfigurer { override fun configure(http: HttpSecurity) { try { http.authorizeHttpRequests { requests -> - requests.requestMatchers(antMatcher(POST, "/api/management/v1/document-opensearch/backfill")) - .permitAll() - requests.requestMatchers(antMatcher(GET, "/api/management/v1/document-opensearch/backfill/status")) - .permitAll() + requests.requestMatchers(antMatcher(POST, "/api/management/v1/document-opensearch/reindex")) + .hasAuthority(ADMIN) + requests.requestMatchers(antMatcher(GET, "/api/management/v1/document-opensearch/reindex/status")) + .hasAuthority(ADMIN) + requests.requestMatchers(antMatcher(GET, "/api/management/v1/document-opensearch/reindex/*")) + .hasAuthority(ADMIN) requests.requestMatchers(antMatcher(GET, "/api/management/v1/search-engine")) - .permitAll() + .hasAuthority(ADMIN) requests.requestMatchers(antMatcher(PUT, "/api/management/v1/search-engine")) - .permitAll() + .hasAuthority(ADMIN) } } catch (e: Exception) { throw HttpConfigurerConfigurationException(e) diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DelegatingDocumentSearchService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DelegatingDocumentSearchService.kt index 0434da5538..50606e561a 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DelegatingDocumentSearchService.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DelegatingDocumentSearchService.kt @@ -29,6 +29,7 @@ class DelegatingDocumentSearchService( private val openSearchService: DocumentSearchService, private val jpaService: DocumentSearchService, private val toggle: SearchEngineToggle, + private val reindexProgressGate: ReindexProgressGate, ) : DocumentSearchService { override fun search( @@ -71,10 +72,10 @@ class DelegatingDocumentSearchService( ): Long = executeWithFallback { active().count(documentDefinitionName, blueprintType, advancedSearchRequest) } private fun active(): DocumentSearchService = - if (toggle.shouldUsePostgres()) jpaService else openSearchService + if (toggle.shouldUsePostgres { reindexProgressGate.isReindexInProgress() }) jpaService else openSearchService private fun executeWithFallback(block: () -> T): T { - if (toggle.shouldUsePostgres()) { + if (toggle.shouldUsePostgres { reindexProgressGate.isReindexInProgress() }) { return block() } return try { diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchBackfillService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchBackfillService.kt deleted file mode 100644 index adc5aaed88..0000000000 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchBackfillService.kt +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright 2015-2024 Ritense BV, the Netherlands. - * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.ritense.document.opensearch.service - -import com.fasterxml.jackson.databind.ObjectMapper -import com.ritense.document.domain.impl.JsonSchemaDocument -import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument -import com.ritense.document.opensearch.repository.JsonSchemaDocumentOpenSearchRepository -import io.github.oshai.kotlinlogging.KotlinLogging -import jakarta.persistence.EntityManager -import org.opensearch.client.RequestOptions -import org.opensearch.client.RestHighLevelClient -import org.opensearch.common.settings.Settings -import org.springframework.transaction.PlatformTransactionManager -import org.springframework.transaction.support.TransactionTemplate -import java.util.UUID -import java.util.concurrent.atomic.AtomicBoolean -import java.util.concurrent.atomic.AtomicLong -import java.util.concurrent.atomic.AtomicReference - -open class DocumentOpenSearchBackfillService( - private val entityManager: EntityManager, - private val openSearchRepository: JsonSchemaDocumentOpenSearchRepository, - private val objectMapper: ObjectMapper, - private val restHighLevelClient: RestHighLevelClient, - private val transactionManager: PlatformTransactionManager, -) { - - private val running = AtomicBoolean(false) - private val migratedCount = AtomicLong(0) - private val startTimeMillis = AtomicLong(0) - private val lastError = AtomicReference(null) - - fun start(pageSize: Int = DEFAULT_PAGE_SIZE): Boolean { - if (!running.compareAndSet(false, true)) return false - migratedCount.set(0) - startTimeMillis.set(System.currentTimeMillis()) - lastError.set(null) - - Thread.startVirtualThread { - try { - backfill(pageSize) - } catch (e: Exception) { - lastError.set(e.message) - logger.error(e) { "Backfill failed" } - } finally { - running.set(false) - } - } - return true - } - - fun status(): Map { - val isRunning = running.get() - val count = migratedCount.get() - val elapsed = if (startTimeMillis.get() > 0) { - (System.currentTimeMillis() - startTimeMillis.get()) / 1000 - } else 0L - - return mapOf( - "running" to isRunning, - "migratedCount" to count, - "elapsedSeconds" to elapsed, - "error" to lastError.get(), - ) - } - - /** - * Copies all existing [JsonSchemaDocument] rows from the relational database to OpenSearch. - * Uses keyset (cursor) pagination on the primary key to avoid offset-based scans, and clears - * the persistence context after every batch to prevent memory buildup. - * - * Each batch runs in its own short-lived read-only transaction to avoid long-lived - * transaction snapshots that would prevent PostgreSQL vacuum from reclaiming space. - */ - open fun backfill(pageSize: Int = DEFAULT_PAGE_SIZE): Long { - setRefreshInterval("-1") - - var lastId: UUID? = null - var total = 0L - val startTime = System.currentTimeMillis() - val txTemplate = TransactionTemplate(transactionManager).apply { isReadOnly = true } - - try { - while (true) { - val batch = txTemplate.execute { - val result = fetchBatch(lastId, pageSize) - entityManager.clear() - result - } ?: break - if (batch.isEmpty()) break - - val docs = mutableListOf() - for (jpaDoc in batch) { - try { - val tree = objectMapper.valueToTree(jpaDoc) - val doc = objectMapper.treeToValue(tree, JsonSchemaDocumentOsDocument::class.java) - docs.add(doc.copy(contentText = extractLeafValues(tree.get("content")))) - } catch (e: Exception) { - logger.warn(e) { "Failed to convert document — skipping" } - } - } - openSearchRepository.saveAll(docs) - total += docs.size - migratedCount.set(total) - lastId = batch.last().id().id - - if (total % LOG_INTERVAL == 0L) { - val elapsed = (System.currentTimeMillis() - startTime) / 1000 - logger.info { "Backfill progress: $total documents indexed (${elapsed}s elapsed)" } - } - } - } finally { - setRefreshInterval("1s") - } - - val elapsed = (System.currentTimeMillis() - startTime) / 1000 - logger.info { "Backfill complete: $total documents migrated to OpenSearch in ${elapsed}s" } - return total - } - - private fun fetchBatch(lastId: UUID?, pageSize: Int): List { - val query = if (lastId == null) { - entityManager.createQuery( - "SELECT d FROM JsonSchemaDocument d ORDER BY d.id.id", - JsonSchemaDocument::class.java - ) - } else { - entityManager.createQuery( - "SELECT d FROM JsonSchemaDocument d WHERE d.id.id > :lastId ORDER BY d.id.id", - JsonSchemaDocument::class.java - ).setParameter("lastId", lastId) - } - return query.setMaxResults(pageSize).resultList - } - - private fun setRefreshInterval(interval: String) { - try { - val request = org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest(INDEX_NAME) - request.settings(Settings.builder().put("index.refresh_interval", interval)) - restHighLevelClient.indices().putSettings(request, RequestOptions.DEFAULT) - logger.debug { "Set index refresh_interval to $interval" } - } catch (e: Exception) { - logger.warn(e) { "Failed to set refresh_interval to $interval — continuing" } - } - } - - companion object { - private val logger = KotlinLogging.logger {} - private const val INDEX_NAME = "json_schema_document" - const val DEFAULT_PAGE_SIZE = 5000 - private const val LOG_INTERVAL = 50_000L - } -} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchIndexInitializer.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchIndexInitializer.kt new file mode 100644 index 0000000000..1ff0209268 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchIndexInitializer.kt @@ -0,0 +1,69 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument +import io.github.oshai.kotlinlogging.KotlinLogging +import org.springframework.data.elasticsearch.core.ElasticsearchOperations +import org.springframework.data.elasticsearch.core.document.Document + +/** + * Creates the OpenSearch index and mappings if they do not yet exist. This is the only place that + * provisions the index, and it is invoked exactly when OpenSearch becomes the active engine — at startup + * when the engine is already active, and from [com.ritense.document.opensearch.web.SearchEngineResource] + * the moment the engine is switched on at runtime — so a freshly enabled cluster is prepared on demand + * rather than being touched unconditionally on every boot. + * + * All failures are swallowed with a warning: a missing/unreachable cluster must never break startup or the + * toggle endpoint. [ensureIndex] is idempotent, so repeated calls (startup + later switch-ons) are safe. + */ +open class DocumentOpenSearchIndexInitializer( + private val elasticsearchOperations: ElasticsearchOperations, +) { + + open fun ensureIndex() { + try { + val indexOps = elasticsearchOperations.indexOps(JsonSchemaDocumentOsDocument::class.java) + if (!indexOps.exists()) { + val settings = Document.create() + settings["index.number_of_replicas"] = 0 + indexOps.create(settings) + + // Merge annotated mapping with a dynamic template that forces all + // content.* fields to text+keyword — enables wildcard search on numbers too + val annotatedMapping = indexOps.createMapping(JsonSchemaDocumentOsDocument::class.java) + val dynamicTemplates = listOf( + mapOf("content_fields_as_text" to mapOf( + "path_match" to "content.*", + "mapping" to mapOf( + "type" to "text", + "fields" to mapOf("keyword" to mapOf("type" to "keyword", "ignore_above" to 256)) + ) + )) + ) + annotatedMapping["dynamic_templates"] = dynamicTemplates + indexOps.putMapping(annotatedMapping) + } + } catch (e: Exception) { + logger.warn(e) { "Failed to initialize OpenSearch index — is OpenSearch running?" } + } + } + + companion object { + private val logger = KotlinLogging.logger {} + } +} \ No newline at end of file diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReconcileJob.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReconcileJob.kt new file mode 100644 index 0000000000..68fac30130 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReconcileJob.kt @@ -0,0 +1,48 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import org.springframework.scheduling.annotation.Scheduled +import java.util.concurrent.atomic.AtomicBoolean + +/** + * Fires the reconcile cycle on a fixed delay. The [running] guard makes overlapping scheduled invocations + * on this node a no-op; cross-node exclusivity is handled by the ShedLock inside the service. `fixedDelay` + * (not `fixedRate`) so a slow cycle never queues up back-to-back runs. + */ +class DocumentOpenSearchReconcileJob( + private val reconcileService: DocumentOpenSearchReconcileService, + private val toggle: SearchEngineToggle, +) { + private val running = AtomicBoolean(false) + + // The PT2M default must match OpenSearchProperties.Reconcile.interval; the live path owns freshness, + // so this safety-net reconciler runs on a relaxed interval. + @Scheduled(fixedDelayString = "\${valtimo.opensearch.reconcile.interval:PT2M}") + fun reconcile() { + // Engine off: skip this cycle without touching OpenSearch. The tick keeps firing cheaply and + // resumes reconciling from the persisted watermark on the first cycle after the engine is re-enabled. + if (!toggle.isOpenSearchActive()) return + if (running.compareAndSet(false, true)) { + try { + reconcileService.reconcile() + } finally { + running.set(false) + } + } + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReconcileService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReconcileService.kt new file mode 100644 index 0000000000..d18c27c317 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReconcileService.kt @@ -0,0 +1,223 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.opensearch.OpenSearchProperties +import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument +import com.ritense.document.opensearch.domain.OpenSearchReconcileState +import com.ritense.document.opensearch.repository.JsonSchemaDocumentOpenSearchRepository +import com.ritense.document.opensearch.repository.OpenSearchReconcileStateRepository +import com.ritense.document.opensearch.repository.PendingIndexDeletionRepository +import io.github.oshai.kotlinlogging.KotlinLogging +import jakarta.persistence.EntityManager +import net.javacrumbs.shedlock.core.LockConfiguration +import net.javacrumbs.shedlock.core.LockProvider +import org.springframework.data.domain.PageRequest +import org.springframework.transaction.PlatformTransactionManager +import org.springframework.transaction.support.TransactionTemplate +import java.time.Duration +import java.time.Instant +import java.time.LocalDateTime +import java.util.UUID + +/** + * The self-healing backbone: a single-writer, watermark-based incremental reindex that makes the + * OpenSearch index a derived read-model of PostgreSQL. + * + * Each cycle (guarded cluster-wide by a ShedLock named lock so exactly one node runs it): + * 1. reads the persisted watermark (initialised to the current `MAX(changed_on)` on first ever run, so a + * fresh deploy does not re-index the whole corpus — initial population stays with the admin re-index); + * 2. keyset-scans every document with `changed_on > (watermark − overlap)` and idempotently upserts it — + * covering status/tags/anything the live path missed or that happened during an OpenSearch outage; + * 3. drains the pending-index-deletion table (O(deletes), never an O(index) scan); + * 4. advances the watermark to the highest `changed_on` processed — **only** after a fully successful + * cycle. Any failure leaves the watermark parked, so the next cycle simply retries from the same point. + * + * All writes are idempotent, so re-processing the overlap window (or a whole failed cycle) is safe. + */ +open class DocumentOpenSearchReconcileService( + private val entityManager: EntityManager, + private val converter: JsonSchemaDocumentOsConverter, + private val openSearchRepository: JsonSchemaDocumentOpenSearchRepository, + private val stateRepository: OpenSearchReconcileStateRepository, + private val pendingIndexDeletionRepository: PendingIndexDeletionRepository, + private val transactionManager: PlatformTransactionManager, + private val lockProvider: LockProvider, + private val properties: OpenSearchProperties, +) { + + open fun reconcile() { + if (!properties.enabled) return + + val lock = lockProvider.lock( + LockConfiguration(Instant.now(), LOCK_NAME, LOCK_AT_MOST_FOR, Duration.ZERO) + ) + if (lock.isEmpty) { + logger.debug { "Another node is reconciling — skipping this cycle" } + return + } + + try { + val watermark = currentWatermark() + val from = watermark.minus(properties.reconcile.overlap) + val maxSeen = processUpserts(from) + drainPendingDeletions() + if (maxSeen.isAfter(watermark)) { + advanceWatermark(maxSeen) + logger.debug { "Reconcile advanced watermark to $maxSeen" } + } + } catch (e: Exception) { + logger.error(e) { "OpenSearch reconcile cycle failed — watermark not advanced; retrying next cycle" } + } finally { + lock.get().unlock() + } + } + + /** + * Keyset-paginates over `changed_on > from` (tie-broken by id), converting and idempotently upserting + * each page. Returns the highest `changed_on` seen (or [from] when nothing changed). + */ + private fun processUpserts(from: LocalDateTime): LocalDateTime { + var maxSeen = from + var lastChangedOn: LocalDateTime? = null + var lastId: UUID? = null + val pageSize = properties.reconcile.pageSize + val txTemplate = TransactionTemplate(transactionManager).apply { isReadOnly = true } + + while (true) { + val cursorChangedOn = lastChangedOn + val cursorId = lastId + val page = txTemplate.execute { + val batch = fetchPage(from, cursorChangedOn, cursorId, pageSize) + if (batch.isEmpty()) { + null + } else { + val osDocuments = batch.mapNotNull { document -> + try { + converter.toOsDocument(document) + } catch (e: Exception) { + logger.warn(e) { "Failed to convert document ${document.id().id} during reconcile — skipping" } + null + } + } + val last = batch.last() + ReconcilePage(osDocuments, last.changedOn(), last.id().id).also { entityManager.clear() } + } + } ?: break + + page.osDocuments.chunked(JsonSchemaDocumentOsConverter.BULK_CHUNK_SIZE) + .forEach { converter.indexChunk(it) } + + if (page.lastChangedOn.isAfter(maxSeen)) maxSeen = page.lastChangedOn + lastChangedOn = page.lastChangedOn + lastId = page.lastId + } + return maxSeen + } + + /** + * Scoped keyset fetch. Eagerly loads the lazy `internalStatus` `@ManyToOne` so the converted document + * carries the real status key, and keeps a composite `(changed_on, id)` cursor for constant-cost + * pagination that is stable when many rows share the same `changed_on`. + */ + private fun fetchPage( + from: LocalDateTime, + lastChangedOn: LocalDateTime?, + lastId: UUID?, + pageSize: Int, + ): List { + val hasCursor = lastChangedOn != null && lastId != null + val jpql = buildString { + append("SELECT d FROM JsonSchemaDocument d LEFT JOIN FETCH d.internalStatus WHERE d.changedOn > :from") + if (hasCursor) { + append(" AND (d.changedOn > :lastChangedOn OR (d.changedOn = :lastChangedOn AND d.id.id > :lastId))") + } + append(" ORDER BY d.changedOn ASC, d.id.id ASC") + } + val query = entityManager.createQuery(jpql, JsonSchemaDocument::class.java) + query.setParameter("from", from) + if (hasCursor) { + query.setParameter("lastChangedOn", lastChangedOn) + query.setParameter("lastId", lastId) + } + return query.setMaxResults(pageSize).resultList + } + + /** + * Removes pending-deletion documents from OpenSearch in batches, then deletes the drained + * pending-deletion rows. + */ + private fun drainPendingDeletions() { + val batchSize = properties.reconcile.pendingDeletionBatchSize + val readTemplate = TransactionTemplate(transactionManager).apply { isReadOnly = true } + val writeTemplate = TransactionTemplate(transactionManager) + while (true) { + val pendingDeletions = readTemplate.execute { + pendingIndexDeletionRepository.findByOrderByDeletedOnAsc(PageRequest.of(0, batchSize)) + }.orEmpty() + if (pendingDeletions.isEmpty()) break + + pendingDeletions.forEach { openSearchRepository.deleteById(it.documentId.toString()) } + writeTemplate.execute { pendingIndexDeletionRepository.deleteAllById(pendingDeletions.map { it.documentId }) } + logger.debug { "Drained ${pendingDeletions.size} pending index deletion(s) from OpenSearch" } + } + } + + private fun currentWatermark(): LocalDateTime = + requireNotNull( + TransactionTemplate(transactionManager).execute { + stateRepository.findById(OpenSearchReconcileState.SINGLETON_ID) + .map { it.watermark } + .orElseGet { + val initial = initialWatermark() + stateRepository.save(OpenSearchReconcileState(watermark = initial)) + logger.info { "Initialised OpenSearch reconcile watermark to $initial" } + initial + } + } + ) + + private fun initialWatermark(): LocalDateTime = + entityManager + .createQuery("SELECT MAX(d.changedOn) FROM JsonSchemaDocument d", LocalDateTime::class.java) + .singleResult ?: LocalDateTime.now() + + private fun advanceWatermark(newWatermark: LocalDateTime) { + TransactionTemplate(transactionManager).execute { + val state = stateRepository.findById(OpenSearchReconcileState.SINGLETON_ID) + .orElseGet { OpenSearchReconcileState(watermark = newWatermark) } + state.watermark = newWatermark + stateRepository.save(state) + } + } + + private data class ReconcilePage( + val osDocuments: List, + val lastChangedOn: LocalDateTime, + val lastId: UUID, + ) + + companion object { + private val logger = KotlinLogging.logger {} + + const val LOCK_NAME = "document-opensearch-reconcile" + + /** Lock lease per cycle; a cycle should complete well within this. */ + val LOCK_AT_MOST_FOR: Duration = Duration.ofMinutes(10) + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt new file mode 100644 index 0000000000..9e40df531f --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt @@ -0,0 +1,215 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument +import io.github.oshai.kotlinlogging.KotlinLogging +import jakarta.persistence.EntityManager +import jakarta.persistence.criteria.JoinType +import jakarta.persistence.criteria.Predicate +import net.javacrumbs.shedlock.core.LockConfiguration +import net.javacrumbs.shedlock.core.LockProvider +import org.springframework.beans.factory.DisposableBean +import org.springframework.data.elasticsearch.core.ElasticsearchOperations +import org.springframework.transaction.PlatformTransactionManager +import org.springframework.transaction.support.TransactionTemplate +import java.time.Duration +import java.time.Instant +import java.time.LocalDateTime +import java.util.UUID +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit + +/** + * Re-runnable, scoped re-index of [JsonSchemaDocument] rows into the live `json_schema_document` + * OpenSearch index. + * + * - **Cluster-safe single runner**: a ShedLock named lock ([LOCK_NAME]) guarantees at most one run + * cluster-wide; a concurrent [start] returns `null` (→ HTTP 409). + * - **Persisted, resumable state**: progress (cursor, counts, heartbeat) lives in the database via + * [OpenSearchReindexRunService]; a FAILED/STOPPED run can be resumed from its cursor with an + * idempotent upsert. + * - **Scoped**: only documents matching the [ReindexRequest] filters are (re)indexed; the index stays + * complete and queryable throughout. + * - **Crash-safe refresh**: no global `refresh_interval` toggle — a single explicit refresh runs at + * successful completion. + */ +open class DocumentOpenSearchReindexService( + private val entityManager: EntityManager, + private val converter: JsonSchemaDocumentOsConverter, + private val elasticsearchOperations: ElasticsearchOperations, + private val transactionManager: PlatformTransactionManager, + private val lockProvider: LockProvider, + private val runService: OpenSearchReindexRunService, +) : DisposableBean { + + private val executor: ExecutorService = Executors.newSingleThreadExecutor { runnable -> + Thread(runnable, "opensearch-reindex").apply { isDaemon = true } + } + + @Volatile + private var cancelRequested = false + + /** + * Acquires the cluster-wide lock, creates (or resumes) a run record and dispatches the re-index on + * the managed executor. Returns the run id, or `null` if a re-index is already running anywhere in + * the cluster. + */ + fun start(request: ReindexRequest): UUID? { + val lock = lockProvider.lock( + LockConfiguration(Instant.now(), LOCK_NAME, LOCK_AT_MOST_FOR, Duration.ZERO) + ) + if (lock.isEmpty) return null + + cancelRequested = false + val run = try { + runService.startOrResume(request) + } catch (e: Exception) { + lock.get().unlock() + throw e + } + + executor.execute { + try { + reindex(run.id, request) + } catch (e: Exception) { + logger.error(e) { "Re-index run ${run.id} terminated with error" } + } finally { + lock.get().unlock() + } + } + return run.id + } + + /** + * Runs the chunked, resumable re-index loop for [runId] over the documents matching [scope]. + * Each DB page is read in its own short read-only transaction (keeping snapshots short) and the + * persistence context is cleared after every page. Returns the number of documents processed. + */ + open fun reindex(runId: UUID, scope: ReindexRequest): Long { + var lastId: UUID? = runService.cursorOf(runId) + var processed: Long = runService.processedOf(runId) + var skipped = 0L + val pageSize = scope.effectivePageSize() + val txTemplate = TransactionTemplate(transactionManager).apply { isReadOnly = true } + + try { + while (!cancelRequested) { + val cursor = lastId + val batch = txTemplate.execute { + fetchBatch(scope, cursor, pageSize).also { entityManager.clear() } + } ?: break + if (batch.isEmpty()) break + + val docs = batch.mapNotNull { jpaDoc -> + try { + converter.toOsDocument(jpaDoc) + } catch (e: Exception) { + skipped++ + logger.warn(e) { "Failed to convert document — skipping" } + null + } + } + skipped += docs.chunked(JsonSchemaDocumentOsConverter.BULK_CHUNK_SIZE).sumOf { converter.indexChunk(it) } + + processed += docs.size + lastId = batch.last().id().id + runService.recordProgress(runId, lastId, processed, skipped) + } + + if (cancelRequested) { + logger.info { "Re-index run $runId cancelled — marking STOPPED (processed=$processed, skipped=$skipped)" } + runService.stop(runId) + } else { + indexOps().refresh() + logger.info { "Re-index run $runId complete (processed=$processed, skipped=$skipped)" } + runService.complete(runId) + } + } catch (e: Exception) { + runService.fail(runId, e.message) + logger.error(e) { "Re-index failed (run $runId)" } + throw e + } + return processed + } + + /** Status of a specific run (by id) or the most recent run when [runId] is null. */ + fun status(runId: UUID? = null): Map = runService.toStatusMap(runId) + + /** + * Scoped keyset fetch. Applies the optional [scope] filters, eagerly loads the lazy `internalStatus` + * `@ManyToOne` (C1 — so the detached entity serializes the real status key, not null), and keeps a + * keyset cursor on the primary key for constant-cost pagination. + */ + private fun fetchBatch(scope: ReindexRequest, lastId: UUID?, pageSize: Int): List { + val cb = entityManager.criteriaBuilder + val query = cb.createQuery(JsonSchemaDocument::class.java) + val root = query.from(JsonSchemaDocument::class.java) + root.fetch("internalStatus", JoinType.LEFT) + + val predicates = mutableListOf() + scope.modifiedAfter?.let { predicates += cb.greaterThan(root.get("modifiedOn"), it) } + scope.modifiedBefore?.let { predicates += cb.lessThan(root.get("modifiedOn"), it) } + scope.documentDefinitionName?.let { + predicates += cb.equal(root.get("documentDefinitionId").get("name"), it) + } + scope.documentIds?.takeIf { it.isNotEmpty() }?.let { + predicates += root.get("id").get("id").`in`(it) + } + lastId?.let { predicates += cb.greaterThan(root.get("id").get("id"), it) } + + // Only restrict when there is at least one predicate; an empty where(...) matches no rows. + if (predicates.isNotEmpty()) { + query.where(*predicates.toTypedArray()) + } + query.orderBy(cb.asc(root.get("id").get("id"))) + return entityManager.createQuery(query).setMaxResults(pageSize).resultList + } + + private fun indexOps() = elasticsearchOperations.indexOps(JsonSchemaDocumentOsDocument::class.java) + + /** Signals an in-flight run to stop gracefully and shuts the executor down on context close. */ + override fun destroy() { + cancelRequested = true + executor.shutdown() + try { + if (!executor.awaitTermination(SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + executor.shutdownNow() + } + } catch (e: InterruptedException) { + executor.shutdownNow() + Thread.currentThread().interrupt() + } + } + + companion object { + private val logger = KotlinLogging.logger {} + + const val LOCK_NAME = "document-opensearch-reindex" + + /** + * Generous lock lease. The persisted run-state heartbeat is the robust liveness signal; the lock + * is only the cluster-wide mutex. A run exceeding this could in theory let a second runner start — + * acceptable because all writes are idempotent upserts. + */ + val LOCK_AT_MOST_FOR: Duration = Duration.ofHours(6) + + private const val SHUTDOWN_TIMEOUT_SECONDS = 30L + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchSyncService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchSyncService.kt index e7b40f5c2c..1e0fedc763 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchSyncService.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchSyncService.kt @@ -1,5 +1,5 @@ /* - * Copyright 2015-2024 Ritense BV, the Netherlands. + * Copyright 2015-2026 Ritense BV, the Netherlands. * * Licensed under EUPL, Version 1.2 (the "License"); * you may not use this file except in compliance with the License. @@ -16,36 +16,59 @@ package com.ritense.document.opensearch.service -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.databind.node.ContainerNode -import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument +import com.ritense.authorization.AuthorizationContext +import com.ritense.document.domain.impl.JsonSchemaDocumentId import com.ritense.document.opensearch.repository.JsonSchemaDocumentOpenSearchRepository -import com.ritense.outbox.domain.BaseEvent +import com.ritense.document.repository.impl.JsonSchemaDocumentRepository import io.github.oshai.kotlinlogging.KotlinLogging +import org.springframework.data.elasticsearch.VersionConflictException +import org.springframework.transaction.PlatformTransactionManager +import org.springframework.transaction.support.TransactionTemplate +import java.util.UUID -class DocumentOpenSearchSyncService( +/** + * Reloads the current state of a document from PostgreSQL (the source of truth) and mirrors it into + * OpenSearch. Callers pass only the document id; the document is (re)read here so a coalesced/late write + * always reflects the latest committed state rather than a stale event payload. + * + * This service does not swallow OpenSearch failures — the caller (the best-effort live listener) is + * responsible for isolating them. Any missed or failed write is repaired by the reconciler. + */ +open class DocumentOpenSearchSyncService( private val repository: JsonSchemaDocumentOpenSearchRepository, - private val objectMapper: ObjectMapper, + private val documentRepository: JsonSchemaDocumentRepository, + private val converter: JsonSchemaDocumentOsConverter, + transactionManager: PlatformTransactionManager, ) { - fun upsert(event: BaseEvent) { - val result = event.result - if (result == null) { - logger.warn { "Received document event ${event.type} for id=${event.resultId} with null result — skipping upsert" } + // Read-only transaction so the lazy associations touched during conversion can be initialised. + private val readOnlyTransactionTemplate = TransactionTemplate(transactionManager).apply { isReadOnly = true } + + /** + * Reloads [documentId] and upserts it into OpenSearch. A document that no longer exists (already + * deleted) is skipped — its removal is handled by [delete] / the pending-index-deletion drain. + */ + open fun upsertById(documentId: UUID) { + val osDocument = readOnlyTransactionTemplate.execute { + AuthorizationContext.runWithoutAuthorization { + documentRepository.findById(JsonSchemaDocumentId.existingId(documentId)).orElse(null) + }?.let { converter.toOsDocument(it) } + } + if (osDocument == null) { + logger.debug { "Document $documentId not found on reload — skipping upsert (likely deleted)" } return } - upsertFromResult(result, event.type) - } - - private fun upsertFromResult(result: ContainerNode<*>, eventType: String) { - val doc = objectMapper.treeToValue(result, JsonSchemaDocumentOsDocument::class.java) - val contentText = extractLeafValues(result.get("content")) - repository.save(doc.copy(contentText = contentText)) - logger.debug { "Upserted document ${doc.id} in OpenSearch (event: $eventType)" } + try { + repository.save(osDocument) + logger.debug { "Upserted document $documentId in OpenSearch" } + } catch (e: VersionConflictException) { + // A newer/equal version is already indexed (the reconciler or a later event won the race). + logger.debug { "Document $documentId already at newer version in OpenSearch — skipping live upsert" } + } } - fun delete(documentId: String) { - repository.deleteById(documentId) + open fun delete(documentId: UUID) { + repository.deleteById(documentId.toString()) logger.debug { "Deleted document $documentId from OpenSearch" } } diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOsConverter.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOsConverter.kt new file mode 100644 index 0000000000..7fb8b460db --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOsConverter.kt @@ -0,0 +1,102 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument +import com.ritense.document.opensearch.repository.JsonSchemaDocumentOpenSearchRepository +import io.github.oshai.kotlinlogging.KotlinLogging +import org.springframework.data.elasticsearch.BulkFailureException +import org.springframework.data.elasticsearch.VersionConflictException + +/** + * Single, shared implementation of the [JsonSchemaDocument] → [JsonSchemaDocumentOsDocument] conversion + * and of the poison-pill-isolated bulk indexing. Reused by the live event listener, the reconciler and + * the admin re-index service so all three writers produce byte-identical OpenSearch documents and share + * the same failure isolation. + */ +open class JsonSchemaDocumentOsConverter( + private val objectMapper: ObjectMapper, + private val openSearchRepository: JsonSchemaDocumentOpenSearchRepository, +) { + + /** + * Converts a JPA [JsonSchemaDocument] to its OpenSearch read model. The lazy `internalStatus` + * association (and the eager `caseTags`) must be initialised before calling this — either by an + * ambient transaction or by an eager fetch — otherwise serialization sees a null status. + */ + open fun toOsDocument(document: JsonSchemaDocument): JsonSchemaDocumentOsDocument { + val tree = objectMapper.valueToTree(document) + return objectMapper.treeToValue(tree, JsonSchemaDocumentOsDocument::class.java) + .copy( + contentText = extractLeafValues(tree.get("content")), + // JPA optimistic-lock counter drives the OpenSearch external version; +1 keeps it ≥ 1. + indexVersion = (document.version() ?: 0).toLong() + 1, + ) + } + + /** + * Indexes a single bulk chunk. On an item-level [BulkFailureException] only the documents that actually + * failed are re-processed one-by-one (never the whole chunk). A version conflict (HTTP 409) is benign — + * external versioning means the stored document is already at an equal-or-newer version, so it is + * silently ignored rather than warned/counted (in the steady state the reconciler's re-sends are all + * conflicts). Any other per-document failure is isolated and counted as a skip so one poison document + * can never loop a run forever. Transport/connection errors are NOT caught here: they propagate so the + * caller can react (mark the run FAILED, park the watermark, …). + * + * @return the number of documents skipped in this chunk + */ + open fun indexChunk(chunk: List): Long = + try { + openSearchRepository.saveAll(chunk) + 0L + } catch (e: BulkFailureException) { + val byId = chunk.associateBy { it.id } + var skipped = 0L + e.failedDocuments.forEach { (id, details) -> + if (isVersionConflict(details)) return@forEach // benign: stored doc already ≥ this version + val document = byId[id] ?: return@forEach + try { + openSearchRepository.save(document) + } catch (ex: VersionConflictException) { + // benign: a newer/equal version won the race and is already indexed + } catch (ex: Exception) { + skipped++ + logger.warn(ex) { "Failed to index document $id — skipping" } + } + } + skipped + } + + companion object { + private val logger = KotlinLogging.logger {} + + /** OpenSearch bulk payload size, decoupled from any DB page size. */ + const val BULK_CHUNK_SIZE = 500 + + /** + * A bulk item failure that is an OpenSearch external-version conflict: HTTP 409, or the engine's + * `version_conflict_engine_exception` in the message as a fallback. These are expected under + * external versioning and must not be warned or counted as skips. + */ + private fun isVersionConflict(details: BulkFailureException.FailureDetails): Boolean = + details.status() == 409 || + details.errorMessage()?.contains("version_conflict_engine_exception") == true + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunService.kt new file mode 100644 index 0000000000..1bf6c7d85e --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunService.kt @@ -0,0 +1,184 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.fasterxml.jackson.databind.ObjectMapper +import com.ritense.document.opensearch.OpenSearchProperties +import com.ritense.document.opensearch.domain.OpenSearchReindexRun +import com.ritense.document.opensearch.domain.ReindexRunStatus +import com.ritense.document.opensearch.repository.OpenSearchReindexRunRepository +import io.github.oshai.kotlinlogging.KotlinLogging +import org.springframework.boot.context.event.ApplicationReadyEvent +import org.springframework.context.event.EventListener +import org.springframework.transaction.annotation.Transactional +import java.time.Duration +import java.time.LocalDateTime +import java.util.UUID + +/** + * Thin transactional wrapper around [OpenSearchReindexRunRepository] for managing re-index run state. + * + * Each mutation runs in its own (short-lived) transaction, independent of the read-only document-fetch + * transactions in [DocumentOpenSearchReindexService], so progress and status are committed and visible + * across instances as the run proceeds. + */ +@Transactional +open class OpenSearchReindexRunService( + private val repository: OpenSearchReindexRunRepository, + private val objectMapper: ObjectMapper, + private val properties: OpenSearchProperties, +) { + + /** + * On startup, reconcile any [ReindexRunStatus.RUNNING] row whose heartbeat has gone stale (older than + * [OpenSearchProperties.Reindex.runningHeartbeatTimeout]): the instance that owned it has crashed or + * restarted and can no longer be advancing it. Mark them FAILED (resumable from their cursor). Runs still + * being advanced by a live instance keep a fresh heartbeat and are left untouched, so this is cluster-safe. + */ + @EventListener(ApplicationReadyEvent::class) + open fun reconcileOrphanedRuns() { + val staleBefore = LocalDateTime.now().minus(properties.reindex.runningHeartbeatTimeout) + val orphaned = repository.findAllByStatusAndHeartbeatOnBefore(ReindexRunStatus.RUNNING, staleBefore) + if (orphaned.isEmpty()) return + val now = LocalDateTime.now() + orphaned.forEach { it.fail(now, "Reconciled on startup: RUNNING run with a stale heartbeat") } + repository.saveAll(orphaned) + logger.warn { "Reconciled ${orphaned.size} orphaned RUNNING re-index run(s) with a stale heartbeat to FAILED" } + } + + /** + * Creates a fresh RUNNING run for [request], or — when [ReindexRequest.resumeRunId] is set — + * re-arms that existing run so it continues from its persisted [OpenSearchReindexRun.lastId]. + */ + open fun startOrResume(request: ReindexRequest): OpenSearchReindexRun { + request.resumeRunId?.let { resumeRunId -> + val existing = repository.findById(resumeRunId).orElseThrow { + IllegalArgumentException("No re-index run found for resumeRunId=$resumeRunId") + } + existing.resume(LocalDateTime.now()) + return repository.save(existing) + } + return repository.save( + OpenSearchReindexRun( + id = UUID.randomUUID(), + status = ReindexRunStatus.RUNNING, + scope = serializeScope(request), + pageSize = request.effectivePageSize(), + startedOn = LocalDateTime.now(), + heartbeatOn = LocalDateTime.now(), + ) + ) + } + + /** + * Whether an admin (re)index run is currently in progress anywhere in the cluster. A [ReindexRunStatus.RUNNING] + * row only counts while its heartbeat is fresher than [heartbeatTimeout], so a run left behind by a crashed + * instance cannot report "running" forever. Used to temporarily route document search to PostgreSQL while the + * index is being filled. + */ + @Transactional(readOnly = true) + open fun isReindexRunning(heartbeatTimeout: Duration): Boolean = + repository.existsByStatusAndHeartbeatOnAfter( + ReindexRunStatus.RUNNING, + LocalDateTime.now().minus(heartbeatTimeout), + ) + + @Transactional(readOnly = true) + open fun cursorOf(runId: UUID): UUID? = requireRun(runId).lastId + + @Transactional(readOnly = true) + open fun processedOf(runId: UUID): Long = requireRun(runId).processedCount + + open fun recordProgress(runId: UUID, lastId: UUID?, processed: Long, skipped: Long) { + val run = requireRun(runId) + run.recordProgress(lastId, processed, skipped, LocalDateTime.now()) + repository.save(run) + } + + open fun complete(runId: UUID) { + val run = requireRun(runId) + run.complete(LocalDateTime.now()) + repository.save(run) + } + + open fun fail(runId: UUID, error: String?) { + val run = requireRun(runId) + run.fail(LocalDateTime.now(), error) + repository.save(run) + } + + open fun stop(runId: UUID) { + val run = requireRun(runId) + run.stop(LocalDateTime.now()) + repository.save(run) + } + + /** + * Status of a specific run (by [runId]) or — when null — of the most recent run. Returns a + * not-running placeholder when no matching run exists. + */ + @Transactional(readOnly = true) + open fun toStatusMap(runId: UUID?): Map { + val run = (if (runId != null) repository.findById(runId).orElse(null) + else repository.findFirstByOrderByStartedOnDesc()) + ?: return mapOf("running" to false, "runId" to null) + return toMap(run) + } + + private fun requireRun(runId: UUID): OpenSearchReindexRun = + repository.findById(runId).orElseThrow { IllegalArgumentException("No re-index run found for runId=$runId") } + + private fun toMap(run: OpenSearchReindexRun): Map { + val elapsedSeconds = Duration.between(run.startedOn, run.finishedOn ?: LocalDateTime.now()).seconds + return mapOf( + "runId" to run.id, + "status" to run.status, + "running" to (run.status == ReindexRunStatus.RUNNING), + "scope" to deserializeScope(run.scope), + "pageSize" to run.pageSize, + "lastId" to run.lastId, + "processedCount" to run.processedCount, + "skippedCount" to run.skippedCount, + "startedOn" to run.startedOn, + "heartbeatOn" to run.heartbeatOn, + "finishedOn" to run.finishedOn, + "elapsedSeconds" to elapsedSeconds, + "error" to run.error, + ) + } + + private fun serializeScope(request: ReindexRequest): String? = + try { + objectMapper.writeValueAsString(request) + } catch (e: Exception) { + logger.warn(e) { "Failed to serialize re-index scope — storing null" } + null + } + + private fun deserializeScope(scope: String?): Any? = + scope?.let { + try { + objectMapper.readValue(it, Map::class.java) + } catch (e: Exception) { + it + } + } + + companion object { + private val logger = KotlinLogging.logger {} + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/ReindexProgressGate.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/ReindexProgressGate.kt new file mode 100644 index 0000000000..3c4367ef91 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/ReindexProgressGate.kt @@ -0,0 +1,60 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.ritense.document.opensearch.OpenSearchProperties + +/** + * Tells the [DelegatingDocumentSearchService] whether an admin (re)index run is currently filling the + * index, in which case search should temporarily fall back to PostgreSQL so users never query a + * partially-filled index. The decision is derived from the cluster-shared reindex-run state, so every node + * falls back regardless of which one runs the job, and search returns to OpenSearch automatically once all + * runs finish. + * + * The result is cached for [CACHE_TTL_MS] to avoid a database round-trip on every search — a sub-second + * delay before switching back to OpenSearch is harmless. + */ +open class ReindexProgressGate( + private val reindexRunService: OpenSearchReindexRunService, + private val properties: OpenSearchProperties, + private val clock: () -> Long = System::currentTimeMillis, +) { + + @Volatile + private var cachedResult = false + + @Volatile + private var cachedAtMillis = 0L + + @Volatile + private var initialized = false + + open fun isReindexInProgress(): Boolean { + if (!properties.reindex.fallbackToPostgresWhileRunning) return false + val now = clock() + if (!initialized || now - cachedAtMillis >= CACHE_TTL_MS) { + cachedResult = reindexRunService.isReindexRunning(properties.reindex.runningHeartbeatTimeout) + cachedAtMillis = now + initialized = true + } + return cachedResult + } + + companion object { + const val CACHE_TTL_MS = 1000L + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/ReindexRequest.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/ReindexRequest.kt new file mode 100644 index 0000000000..7f242ebfb6 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/ReindexRequest.kt @@ -0,0 +1,47 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import java.time.LocalDateTime +import java.util.UUID + +/** + * Describes the scope of a re-index run. All filters are optional; an empty request re-indexes every + * [com.ritense.document.domain.impl.JsonSchemaDocument] into the live `json_schema_document` index. + * + * @param modifiedAfter only documents with `modifiedOn` strictly after this instant + * @param modifiedBefore only documents with `modifiedOn` strictly before this instant + * @param documentDefinitionName only documents of this document-definition name + * @param documentIds explicit subset of document ids + * @param pageSize DB keyset page size, clamped to [1, MAX_PAGE_SIZE] by [effectivePageSize] + * @param resumeRunId continue a prior (FAILED/STOPPED) run from its persisted cursor instead of starting fresh + */ +data class ReindexRequest( + val modifiedAfter: LocalDateTime? = null, + val modifiedBefore: LocalDateTime? = null, + val documentDefinitionName: String? = null, + val documentIds: List? = null, + val pageSize: Int = DEFAULT_PAGE_SIZE, + val resumeRunId: UUID? = null, +) { + fun effectivePageSize() = pageSize.coerceIn(1, MAX_PAGE_SIZE) + + companion object { + const val DEFAULT_PAGE_SIZE = 5000 + const val MAX_PAGE_SIZE = 10_000 + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/SearchEngineToggle.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/SearchEngineToggle.kt index b2c9f5e061..3a6035c1fc 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/SearchEngineToggle.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/SearchEngineToggle.kt @@ -34,6 +34,15 @@ class SearchEngineToggle(default: Engine = Engine.OPENSEARCH) { active.set(engine) } + /** + * Master switch for every active OpenSearch call (reads, live-sync writes, reconcile, index creation). + * Read live on each call site so flipping the engine at runtime — via + * [com.ritense.document.opensearch.web.SearchEngineResource] — immediately (re)enables or disables all + * OpenSearch traffic without a restart. Startup forces this to [Engine.POSTGRES] when OpenSearch is + * disabled by configuration, so this single check also honours `valtimo.opensearch.enabled`. + */ + fun isOpenSearchActive(): Boolean = active.get() == Engine.OPENSEARCH + fun isFallbackActive(): Boolean = fallbackActive.get() fun activateFallback() { @@ -45,8 +54,15 @@ class SearchEngineToggle(default: Engine = Engine.OPENSEARCH) { lastWarningTime.set(0) } - fun shouldUsePostgres(): Boolean = - get() == Engine.POSTGRES || (get() == Engine.OPENSEARCH && fallbackActive.get()) + /** + * Route document search to PostgreSQL when the engine is not OpenSearch, while an admin reindex is + * filling the index ([reindexInProgress]), or while a connection fallback is active because OpenSearch + * is unreachable. Otherwise OpenSearch serves the query. [reindexInProgress] is a supplier so the + * engine check short-circuits it — the (potentially DB-backed) reindex check is skipped entirely when + * the engine is already PostgreSQL. + */ + fun shouldUsePostgres(reindexInProgress: () -> Boolean): Boolean = + get() != Engine.OPENSEARCH || reindexInProgress() || fallbackActive.get() fun shouldLogWarning(intervalMs: Long): Boolean { val now = System.currentTimeMillis() diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/DocumentOpenSearchBackfillResource.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/DocumentOpenSearchReindexResource.kt similarity index 52% rename from backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/DocumentOpenSearchBackfillResource.kt rename to backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/DocumentOpenSearchReindexResource.kt index 1a613f75a8..b551c621a4 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/DocumentOpenSearchBackfillResource.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/DocumentOpenSearchReindexResource.kt @@ -1,5 +1,5 @@ /* - * Copyright 2015-2024 Ritense BV, the Netherlands. + * Copyright 2015-2026 Ritense BV, the Netherlands. * * Licensed under EUPL, Version 1.2 (the "License"); * you may not use this file except in compliance with the License. @@ -16,32 +16,34 @@ package com.ritense.document.opensearch.web -import com.ritense.document.opensearch.service.DocumentOpenSearchBackfillService +import com.ritense.document.opensearch.service.DocumentOpenSearchReindexService +import com.ritense.document.opensearch.service.ReindexRequest import org.springframework.http.ResponseEntity import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PathVariable import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.RequestBody import org.springframework.web.bind.annotation.RequestMapping -import org.springframework.web.bind.annotation.RequestParam import org.springframework.web.bind.annotation.RestController +import java.util.UUID @RestController @RequestMapping("/api/management/v1/document-opensearch") -class DocumentOpenSearchBackfillResource( - private val backfillService: DocumentOpenSearchBackfillService, +class DocumentOpenSearchReindexResource( + private val reindexService: DocumentOpenSearchReindexService, ) { - @PostMapping("/backfill") - fun backfill( - @RequestParam(defaultValue = "${DocumentOpenSearchBackfillService.DEFAULT_PAGE_SIZE}") pageSize: Int, - ): ResponseEntity> { - if (!backfillService.start(pageSize)) { - return ResponseEntity.status(409).body(mapOf("error" to "Backfill already in progress" as Any)) - } - return ResponseEntity.accepted().body(mapOf("status" to "started" as Any)) + @PostMapping("/reindex") + fun reindex(@RequestBody(required = false) request: ReindexRequest?): ResponseEntity> { + val runId = reindexService.start(request ?: ReindexRequest()) + ?: return ResponseEntity.status(409).body(mapOf("error" to "Re-index already in progress")) + return ResponseEntity.accepted().body(mapOf("status" to "started", "runId" to runId)) } - @GetMapping("/backfill/status") - fun status(): ResponseEntity> { - return ResponseEntity.ok(backfillService.status()) - } + @GetMapping("/reindex/status") + fun status(): ResponseEntity> = ResponseEntity.ok(reindexService.status()) + + @GetMapping("/reindex/{runId}") + fun statusById(@PathVariable runId: UUID): ResponseEntity> = + ResponseEntity.ok(reindexService.status(runId)) } diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/SearchEngineResource.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/SearchEngineResource.kt index d0d0a97639..0a7f8124e7 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/SearchEngineResource.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/SearchEngineResource.kt @@ -19,6 +19,7 @@ package com.ritense.document.opensearch.web import com.ritense.adminsettings.service.FeatureToggleOverridesService import com.ritense.document.opensearch.OpenSearchProperties import com.ritense.document.opensearch.autoconfigure.DocumentOpenSearchAutoConfiguration.Companion.SEARCH_ENGINE_TOGGLE_KEY +import com.ritense.document.opensearch.service.DocumentOpenSearchIndexInitializer import com.ritense.document.opensearch.service.SearchEngineToggle import org.springframework.http.ResponseEntity import org.springframework.web.bind.annotation.GetMapping @@ -33,6 +34,7 @@ class SearchEngineResource( private val toggle: SearchEngineToggle, private val openSearchProperties: OpenSearchProperties, private val featureToggleOverridesService: FeatureToggleOverridesService, + private val indexInitializer: DocumentOpenSearchIndexInitializer, ) { @GetMapping @@ -56,6 +58,12 @@ class SearchEngineResource( val engine = if (useOpenSearch) SearchEngineToggle.Engine.OPENSEARCH else SearchEngineToggle.Engine.POSTGRES toggle.set(engine) + // Switching the engine on at runtime: make sure the index exists before live-sync/reads resume. + // Idempotent and failure-swallowing, so a missing cluster can't break the toggle call. + if (toggle.isOpenSearchActive()) { + indexInitializer.ensureIndex() + } + return ResponseEntity.ok( SearchEngineDto( available = true, diff --git a/backend/case-opensearch/src/main/resources/META-INF/spring.factories b/backend/case-opensearch/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..622a4db2ce --- /dev/null +++ b/backend/case-opensearch/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +org.springframework.boot.autoconfigure.AutoConfigurationImportFilter=\ + com.ritense.document.opensearch.autoconfigure.ExcludeElasticsearchAutoConfigurationFilter \ No newline at end of file diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/BaseOpenSearchIntegrationTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/BaseOpenSearchIntegrationTest.kt index 624ac327a2..c180fdc980 100644 --- a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/BaseOpenSearchIntegrationTest.kt +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/BaseOpenSearchIntegrationTest.kt @@ -26,13 +26,13 @@ import com.ritense.authorization.role.RoleRepository import com.ritense.document.domain.impl.JsonSchemaDocumentDefinition import com.ritense.document.domain.impl.snapshot.JsonSchemaDocumentSnapshot import com.ritense.document.domain.impl.searchfield.SearchField +import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument import com.ritense.document.opensearch.repository.JsonSchemaDocumentOpenSearchRepository import com.ritense.document.service.impl.JsonSchemaDocumentService import com.ritense.document.service.JsonSchemaDocumentActionProvider import com.ritense.document.service.JsonSchemaDocumentDefinitionActionProvider import com.ritense.document.service.JsonSchemaDocumentSnapshotActionProvider import com.ritense.document.service.SearchFieldActionProvider -import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument import com.ritense.outbox.OutboxService import com.ritense.testutilscommon.junit.extension.LiquibaseRunnerExtension import com.ritense.valtimo.contract.authentication.TeamManagementService @@ -87,6 +87,9 @@ abstract class BaseOpenSearchIntegrationTest { @Autowired lateinit var openSearchRepository: JsonSchemaDocumentOpenSearchRepository + @Autowired + lateinit var elasticsearchOperations: ElasticsearchOperations + @Autowired lateinit var roleRepository: RoleRepository @@ -96,26 +99,25 @@ abstract class BaseOpenSearchIntegrationTest { @Autowired lateinit var objectMapper: ObjectMapper - @Autowired - lateinit var elasticsearchOperations: ElasticsearchOperations - @BeforeEach fun setUpBase() { setUpPermissions() - val indexOps = elasticsearchOperations.indexOps(JsonSchemaDocumentOsDocument::class.java) - if (indexOps.exists()) { - indexOps.delete() - } - indexOps.create() - indexOps.putMapping(indexOps.createMapping()) + openSearchRepository.deleteAll() + refreshIndex() } @AfterEach fun tearDownBase() { - val indexOps = elasticsearchOperations.indexOps(JsonSchemaDocumentOsDocument::class.java) - if (indexOps.exists()) { - indexOps.delete() - } + openSearchRepository.deleteAll() + refreshIndex() + } + + /** + * Forces an OpenSearch refresh so writes/deletes are immediately visible to subsequent reads. + * OpenSearch refreshes asynchronously (default 1s), which makes write-then-read assertions flaky. + */ + protected fun refreshIndex() { + elasticsearchOperations.indexOps(JsonSchemaDocumentOsDocument::class.java).refresh() } private fun setUpPermissions() { diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/OpenSearchPropertiesTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/OpenSearchPropertiesTest.kt new file mode 100644 index 0000000000..18090822d9 --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/OpenSearchPropertiesTest.kt @@ -0,0 +1,48 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch + +import com.ritense.document.opensearch.service.DocumentOpenSearchReconcileJob +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.springframework.scheduling.annotation.Scheduled +import java.time.Duration + +class OpenSearchPropertiesTest { + + @Test + fun `reconcile interval default is PT2M`() { + assertThat(OpenSearchProperties().reconcile.interval).isEqualTo(Duration.ofMinutes(2)) + } + + /** + * The reconcile job binds `fixedDelayString` with an inline default; if that default drifts from the + * property default, environments without the property set silently run on a different cadence. This + * guards the two from diverging. + */ + @Test + fun `scheduled job fixedDelay default matches the property default`() { + val scheduled = DocumentOpenSearchReconcileJob::class.java + .getDeclaredMethod("reconcile") + .getAnnotation(Scheduled::class.java) + + // e.g. "${valtimo.opensearch.reconcile.interval:PT2M}" -> "PT2M" + val default = scheduled.fixedDelayString.substringAfterLast(':').removeSuffix("}") + + assertThat(Duration.parse(default)).isEqualTo(OpenSearchProperties().reconcile.interval) + } +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/TestApplication.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/TestApplication.kt index 89e160c1d8..8ac18acfac 100644 --- a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/TestApplication.kt +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/TestApplication.kt @@ -17,17 +17,6 @@ package com.ritense.document.opensearch import org.springframework.boot.autoconfigure.SpringBootApplication -import org.springframework.boot.autoconfigure.data.elasticsearch.ElasticsearchDataAutoConfiguration -import org.springframework.boot.autoconfigure.data.elasticsearch.ReactiveElasticsearchRepositoriesAutoConfiguration -import org.springframework.boot.autoconfigure.elasticsearch.ElasticsearchClientAutoConfiguration -import org.springframework.boot.autoconfigure.elasticsearch.ElasticsearchRestClientAutoConfiguration -import org.springframework.boot.autoconfigure.elasticsearch.ReactiveElasticsearchClientAutoConfiguration -@SpringBootApplication(exclude = [ - ElasticsearchDataAutoConfiguration::class, - ElasticsearchClientAutoConfiguration::class, - ElasticsearchRestClientAutoConfiguration::class, - ReactiveElasticsearchClientAutoConfiguration::class, - ReactiveElasticsearchRepositoriesAutoConfiguration::class, -]) +@SpringBootApplication class TestApplication diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/handler/DocumentOpenSearchEventListenerTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/handler/DocumentOpenSearchEventListenerTest.kt new file mode 100644 index 0000000000..877b7e9e82 --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/handler/DocumentOpenSearchEventListenerTest.kt @@ -0,0 +1,123 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.handler + +import com.ritense.document.domain.impl.JsonSchemaDocumentId +import com.ritense.document.domain.impl.event.JsonSchemaDocumentCreatedEvent +import com.ritense.document.event.DocumentAssigneeChangedEvent +import com.ritense.document.event.DocumentRetentionPeriodSetEvent +import com.ritense.document.opensearch.service.DocumentOpenSearchSyncService +import com.ritense.document.opensearch.service.SearchEngineToggle +import com.ritense.valtimo.contract.event.DocumentDeletedEvent +import org.assertj.core.api.Assertions.assertThatCode +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.mock +import org.mockito.kotlin.never +import org.mockito.kotlin.timeout +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever +import java.util.UUID + +class DocumentOpenSearchEventListenerTest { + + private val syncService: DocumentOpenSearchSyncService = mock() + private val toggle = SearchEngineToggle(SearchEngineToggle.Engine.OPENSEARCH) + private lateinit var listener: DocumentOpenSearchEventListener + + @BeforeEach + fun setUp() { + listener = DocumentOpenSearchEventListener(syncService, toggle) + } + + @AfterEach + fun tearDown() { + listener.destroy() + } + + @Test + fun `created event enqueues an upsert of the document id`() { + val id = UUID.randomUUID() + val event: JsonSchemaDocumentCreatedEvent = mock() + whenever(event.documentId()).thenReturn(JsonSchemaDocumentId.existingId(id)) + + listener.onCreated(event) + + verify(syncService, timeout(TIMEOUT_MS)).upsertById(id) + } + + @Test + fun `assignee-changed event enqueues an upsert of the document id`() { + val id = UUID.randomUUID() + val event: DocumentAssigneeChangedEvent = mock() + whenever(event.documentId).thenReturn(id) + + listener.onAssigneeChanged(event) + + verify(syncService, timeout(TIMEOUT_MS)).upsertById(id) + } + + @Test + fun `retention-set event enqueues an upsert of the document id`() { + val id = UUID.randomUUID() + val event: DocumentRetentionPeriodSetEvent = mock() + whenever(event.getDocumentId()).thenReturn(id) + + listener.onRetentionSet(event) + + verify(syncService, timeout(TIMEOUT_MS)).upsertById(id) + } + + @Test + fun `deleted event enqueues a delete of the document id`() { + val id = UUID.randomUUID() + + listener.onDeleted(DocumentDeletedEvent(id)) + + verify(syncService, timeout(TIMEOUT_MS)).delete(id) + } + + @Test + fun `no sync happens when the engine is toggled off`() { + toggle.set(SearchEngineToggle.Engine.POSTGRES) + val id = UUID.randomUUID() + val event: JsonSchemaDocumentCreatedEvent = mock() + whenever(event.documentId()).thenReturn(JsonSchemaDocumentId.existingId(id)) + + // The engine gate is checked synchronously before the task is submitted, so no upsert is ever enqueued. + listener.onCreated(event) + + verify(syncService, never()).upsertById(any()) + } + + @Test + fun `a failing sync task never propagates to the caller`() { + val id = UUID.randomUUID() + val event: JsonSchemaDocumentCreatedEvent = mock() + whenever(event.documentId()).thenReturn(JsonSchemaDocumentId.existingId(id)) + whenever(syncService.upsertById(any())).thenThrow(RuntimeException("OpenSearch is down")) + + assertThatCode { listener.onCreated(event) }.doesNotThrowAnyException() + verify(syncService, timeout(TIMEOUT_MS)).upsertById(id) + } + + companion object { + private const val TIMEOUT_MS = 2000L + } +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DelegatingDocumentSearchServiceTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DelegatingDocumentSearchServiceTest.kt new file mode 100644 index 0000000000..571bc962bd --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DelegatingDocumentSearchServiceTest.kt @@ -0,0 +1,72 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.ritense.document.domain.search.AdvancedSearchRequest +import com.ritense.document.service.DocumentSearchService +import com.ritense.valtimo.contract.blueprint.BlueprintType +import org.junit.jupiter.api.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.eq +import org.mockito.kotlin.mock +import org.mockito.kotlin.never +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever + +class DelegatingDocumentSearchServiceTest { + + private val openSearchService: DocumentSearchService = mock() + private val jpaService: DocumentSearchService = mock() + private val gate: ReindexProgressGate = mock() + private val request: AdvancedSearchRequest = mock() + + @Test + fun `routes to OpenSearch when engine is OpenSearch and no reindex is in progress`() { + whenever(gate.isReindexInProgress()).thenReturn(false) + val service = delegating(SearchEngineToggle.Engine.OPENSEARCH) + + service.count("house", BlueprintType.CASE, request) + + verify(openSearchService).count(eq("house"), any(), any()) + verify(jpaService, never()).count(any(), any(), any()) + } + + @Test + fun `falls back to PostgreSQL while a reindex is in progress, even with engine OpenSearch`() { + whenever(gate.isReindexInProgress()).thenReturn(true) + val service = delegating(SearchEngineToggle.Engine.OPENSEARCH) + + service.count("house", BlueprintType.CASE, request) + + verify(jpaService).count(eq("house"), any(), any()) + verify(openSearchService, never()).count(any(), any(), any()) + } + + @Test + fun `always uses PostgreSQL when engine is Postgres, without consulting the gate`() { + val service = delegating(SearchEngineToggle.Engine.POSTGRES) + + service.count("house", BlueprintType.CASE, request) + + verify(jpaService).count(eq("house"), any(), any()) + verify(openSearchService, never()).count(any(), any(), any()) + verify(gate, never()).isReindexInProgress() + } + + private fun delegating(engine: SearchEngineToggle.Engine) = + DelegatingDocumentSearchService(openSearchService, jpaService, SearchEngineToggle(engine), gate) +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchLiveSyncIntTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchLiveSyncIntTest.kt new file mode 100644 index 0000000000..b03febed8d --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchLiveSyncIntTest.kt @@ -0,0 +1,113 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.ritense.authorization.AuthorizationContext.Companion.runWithoutAuthorization +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.domain.impl.request.NewDocumentRequest +import com.ritense.document.opensearch.BaseOpenSearchIntegrationTest +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatCode +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.security.test.context.support.WithMockUser +import org.springframework.transaction.annotation.Propagation +import org.springframework.transaction.annotation.Transactional +import java.util.UUID + +/** + * Integration tests for the live-sync reload path ([DocumentOpenSearchSyncService]) against a real + * PostgreSQL + OpenSearch. Runs **non-transactionally** so documents actually commit and can be reloaded + * from the source of truth — mirroring how the [com.ritense.document.opensearch.handler.DocumentOpenSearchEventListener] + * invokes the sync service after commit. The listener→sync-service dispatch itself is unit-tested. + */ +@Transactional(propagation = Propagation.NOT_SUPPORTED) +@WithMockUser(username = BaseOpenSearchIntegrationTest.USERNAME, authorities = [BaseOpenSearchIntegrationTest.FULL_ACCESS_ROLE]) +class DocumentOpenSearchLiveSyncIntTest : BaseOpenSearchIntegrationTest() { + + @Autowired + lateinit var syncService: DocumentOpenSearchSyncService + + @AfterEach + fun cleanUp() { + runWithoutAuthorization { documentService.removeDocuments("house") } + clearIndex() + } + + @Test + fun `upsertById indexes the current committed state of the document`() { + val document = createDocument("live-street") + clearIndex() + + syncService.upsertById(document.id().id) + + refreshIndex() + val indexed = openSearchRepository.findById(document.id().toString()) + assertThat(indexed).isPresent + assertThat(indexed.get().contentText).contains("live-street") + } + + @Test + fun `upsertById reloads the lazy internalStatus of the document`() { + val document = createDocument("with-status") + runWithoutAuthorization { documentService.setInternalStatus(document.id(), "started") } + clearIndex() + + syncService.upsertById(document.id().id) + + refreshIndex() + val indexed = openSearchRepository.findById(document.id().toString()) + assertThat(indexed).isPresent + assertThat(indexed.get().internalStatus).isEqualTo("started") + } + + @Test + fun `upsertById skips a document that no longer exists`() { + val danglingId = UUID.randomUUID() + + assertThatCode { syncService.upsertById(danglingId) }.doesNotThrowAnyException() + + refreshIndex() + assertThat(openSearchRepository.findById(danglingId.toString())).isEmpty + } + + @Test + fun `delete removes the document from the index`() { + val document = createDocument("to-be-deleted") + syncService.upsertById(document.id().id) + refreshIndex() + assertThat(openSearchRepository.findById(document.id().toString())).isPresent + + syncService.delete(document.id().id) + + refreshIndex() + assertThat(openSearchRepository.findById(document.id().toString())).isEmpty + } + + private fun clearIndex() { + openSearchRepository.deleteAll() + refreshIndex() + } + + private fun createDocument(street: String): JsonSchemaDocument = + runWithoutAuthorization { + documentService.createDocument( + NewDocumentRequest("house", "house", "1.0.0", objectMapper.createObjectNode().put("street", street)) + ).resultingDocument().get() + } +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReconcileIntTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReconcileIntTest.kt new file mode 100644 index 0000000000..7f25051f58 --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReconcileIntTest.kt @@ -0,0 +1,195 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.ritense.authorization.AuthorizationContext.Companion.runWithoutAuthorization +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.domain.impl.JsonSchemaDocumentId +import com.ritense.document.domain.impl.request.NewDocumentRequest +import com.ritense.document.opensearch.BaseOpenSearchIntegrationTest +import com.ritense.document.opensearch.domain.OpenSearchReconcileState +import com.ritense.document.opensearch.domain.PendingIndexDeletion +import com.ritense.document.opensearch.repository.OpenSearchReconcileStateRepository +import com.ritense.document.opensearch.repository.PendingIndexDeletionRepository +import com.ritense.document.repository.impl.JsonSchemaDocumentRepository +import net.javacrumbs.shedlock.core.LockConfiguration +import net.javacrumbs.shedlock.core.LockProvider +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.security.test.context.support.WithMockUser +import org.springframework.transaction.PlatformTransactionManager +import org.springframework.transaction.annotation.Propagation +import org.springframework.transaction.annotation.Transactional +import org.springframework.transaction.support.TransactionTemplate +import java.time.Duration +import java.time.Instant +import java.time.LocalDateTime +import java.util.UUID + +/** + * Integration tests for [DocumentOpenSearchReconcileService] against a real PostgreSQL + OpenSearch. + * Runs **non-transactionally** so `changed_on`, the watermark state and the pending-index-deletion table + * behave as in production (own short transactions, committed data). Each test seeds the watermark explicitly. + */ +@Transactional(propagation = Propagation.NOT_SUPPORTED) +@WithMockUser(username = BaseOpenSearchIntegrationTest.USERNAME, authorities = [BaseOpenSearchIntegrationTest.FULL_ACCESS_ROLE]) +class DocumentOpenSearchReconcileIntTest : BaseOpenSearchIntegrationTest() { + + @Autowired + lateinit var reconcileService: DocumentOpenSearchReconcileService + + @Autowired + lateinit var syncService: DocumentOpenSearchSyncService + + @Autowired + lateinit var stateRepository: OpenSearchReconcileStateRepository + + @Autowired + lateinit var pendingIndexDeletionRepository: PendingIndexDeletionRepository + + @Autowired + lateinit var documentRepository: JsonSchemaDocumentRepository + + @Autowired + lateinit var transactionManager: PlatformTransactionManager + + @Autowired + lateinit var lockProvider: LockProvider + + @AfterEach + fun cleanUp() { + runWithoutAuthorization { documentService.removeDocuments("house") } + clearIndex() + stateRepository.deleteAll() + pendingIndexDeletionRepository.deleteAll() + } + + @Test + fun `reconcile indexes documents changed since the watermark and advances it`() { + val document = createDocument("reconcile-me") + clearIndex() + seedWatermark(LocalDateTime.now().minusHours(1)) + + reconcileService.reconcile() + + refreshIndex() + assertThat(openSearchRepository.findById(document.id().toString())).isPresent + val watermark = stateRepository.findById(OpenSearchReconcileState.SINGLETON_ID).get().watermark + assertThat(watermark).isAfter(LocalDateTime.now().minusMinutes(30)) + } + + @Test + fun `reconcile picks up a status change that has no live event and no modifiedOn bump`() { + val document = createDocument("status-doc") + runWithoutAuthorization { documentService.setInternalStatus(document.id(), "started") } + clearIndex() + seedWatermark(LocalDateTime.now().minusHours(1)) + + reconcileService.reconcile() + + refreshIndex() + val indexed = openSearchRepository.findById(document.id().toString()) + assertThat(indexed).isPresent + assertThat(indexed.get().internalStatus).isEqualTo("started") + } + + @Test + fun `reconcile is skipped while another writer holds the ShedLock`() { + val document = createDocument("locked-out") + clearIndex() + seedWatermark(LocalDateTime.now().minusHours(1)) + + val lock = lockProvider.lock( + LockConfiguration( + Instant.now(), + DocumentOpenSearchReconcileService.LOCK_NAME, + Duration.ofMinutes(5), + Duration.ZERO, + ) + ) + assertThat(lock).isPresent + try { + reconcileService.reconcile() + } finally { + lock.get().unlock() + } + + refreshIndex() + assertThat(openSearchRepository.findById(document.id().toString())).isEmpty + } + + @Test + fun `reconcile drains pending index deletions, removing docs from the index`() { + val document = createDocument("to-delete") + val id = document.id().id + syncService.upsertById(id) + refreshIndex() + assertThat(openSearchRepository.findById(id.toString())).isPresent + + // Delete from PostgreSQL and record the pending deletion the in-transaction listener would have written. + runWithoutAuthorization { documentService.deleteDocument(document.id()) } + pendingIndexDeletionRepository.save(PendingIndexDeletion(documentId = id)) + seedWatermark(LocalDateTime.now().minusHours(1)) + + reconcileService.reconcile() + + refreshIndex() + assertThat(openSearchRepository.findById(id.toString())).isEmpty + assertThat(pendingIndexDeletionRepository.count()).isZero() + } + + @Test + fun `changed_on advances on a status change while modifiedOn stays unset`() { + val document = createDocument("changed-on-doc") + val createdChangedOn = readChangedOn(document.id().id) + // DATETIME can be second-resolution on MySQL; sleep past a full second so the bump is observable. + Thread.sleep(1100) + + runWithoutAuthorization { documentService.setInternalStatus(document.id(), "started") } + + assertThat(readChangedOn(document.id().id)).isAfter(createdChangedOn) + assertThat(readModifiedOn(document.id().id)).isEmpty + } + + private fun seedWatermark(watermark: LocalDateTime) { + stateRepository.save(OpenSearchReconcileState(watermark = watermark)) + } + + private fun readChangedOn(id: UUID): LocalDateTime = + TransactionTemplate(transactionManager).execute { + documentRepository.findById(JsonSchemaDocumentId.existingId(id)).get().changedOn() + }!! + + private fun readModifiedOn(id: UUID) = + TransactionTemplate(transactionManager).execute { + documentRepository.findById(JsonSchemaDocumentId.existingId(id)).get().modifiedOn() + }!! + + private fun clearIndex() { + openSearchRepository.deleteAll() + refreshIndex() + } + + private fun createDocument(street: String): JsonSchemaDocument = + runWithoutAuthorization { + documentService.createDocument( + NewDocumentRequest("house", "house", "1.0.0", objectMapper.createObjectNode().put("street", street)) + ).resultingDocument().get() + } +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceIntTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceIntTest.kt new file mode 100644 index 0000000000..c9dfdb1fd4 --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceIntTest.kt @@ -0,0 +1,295 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.ritense.authorization.AuthorizationContext.Companion.runWithoutAuthorization +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.domain.impl.request.NewDocumentRequest +import com.ritense.document.opensearch.BaseOpenSearchIntegrationTest +import com.ritense.document.opensearch.domain.OpenSearchReindexRun +import com.ritense.document.opensearch.domain.ReindexRunStatus +import com.ritense.document.opensearch.repository.OpenSearchReindexRunRepository +import jakarta.persistence.EntityManager +import net.javacrumbs.shedlock.core.LockConfiguration +import net.javacrumbs.shedlock.core.LockProvider +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.security.test.context.support.WithMockUser +import org.springframework.transaction.PlatformTransactionManager +import org.springframework.transaction.annotation.Propagation +import org.springframework.transaction.annotation.Transactional +import org.springframework.transaction.support.TransactionTemplate +import java.time.Duration +import java.time.Instant +import java.time.LocalDateTime +import java.util.UUID + +/** + * Integration tests for [DocumentOpenSearchReindexService]. + * + * These tests run **non-transactionally** ([Propagation.NOT_SUPPORTED], overriding the base's + * `@Transactional`). The production re-index runs on a background executor with no ambient transaction, + * committing its run-state updates in their own short transactions; reproducing that here is the only way + * the keyset reads and the persisted progress/cursor behave as they do in production. Committed test data + * is removed in [cleanUp]. + * + * Because the tests commit, creating a document can also trigger the live event sync + * ([com.ritense.document.opensearch.handler.DocumentOpenSearchEventListener]) which indexes that document + * into OpenSearch. To assert on the re-index in isolation we [clearIndex] after the document setup and + * before running the re-index, so the index reflects only what the re-index (re)indexed. + */ +@Transactional(propagation = Propagation.NOT_SUPPORTED) +@WithMockUser(username = BaseOpenSearchIntegrationTest.USERNAME, authorities = [BaseOpenSearchIntegrationTest.FULL_ACCESS_ROLE]) +class DocumentOpenSearchReindexServiceIntTest : BaseOpenSearchIntegrationTest() { + + @Autowired + lateinit var reindexService: DocumentOpenSearchReindexService + + @Autowired + lateinit var reindexRunService: OpenSearchReindexRunService + + @Autowired + lateinit var reindexRunRepository: OpenSearchReindexRunRepository + + @Autowired + lateinit var lockProvider: LockProvider + + @Autowired + lateinit var entityManager: EntityManager + + @Autowired + lateinit var transactionManager: PlatformTransactionManager + + @AfterEach + fun cleanUp() { + runWithoutAuthorization { documentService.removeDocuments("house") } + reindexRunRepository.deleteAll() + } + + @Test + fun `full re-index indexes every document and completes`() { + val ids = (1..5).map { createDocument("street-$it").id().id } + clearIndex() + + val (runId, processed) = reindex(ReindexRequest()) + + assertThat(processed).isEqualTo(5L) + val run = reindexRunRepository.findById(runId).get() + assertThat(run.status).isEqualTo(ReindexRunStatus.COMPLETED) + assertThat(run.processedCount).isEqualTo(5L) + refreshIndex() + assertThat(openSearchRepository.count()).isEqualTo(5L) + ids.forEach { assertThat(openSearchRepository.findById(it.toString())).isPresent } + } + + @Test + fun `re-index populates internalStatus from the live entity (C1)`() { + val document = createDocument("with-status") + runWithoutAuthorization { documentService.setInternalStatus(document.id(), "started") } + clearIndex() + + reindex(ReindexRequest()) + + refreshIndex() + val indexed = openSearchRepository.findById(document.id().toString()) + assertThat(indexed).isPresent + assertThat(indexed.get().internalStatus).isEqualTo("started") + } + + @Test + fun `scoped re-index by documentDefinitionName only indexes matching documents`() { + createDocument("house-doc-1") + createDocument("house-doc-2") + clearIndex() + + reindex(ReindexRequest(documentDefinitionName = "house")) + refreshIndex() + assertThat(openSearchRepository.count()).isEqualTo(2L) + + clearIndex() + + val (_, processed) = reindex(ReindexRequest(documentDefinitionName = "does-not-exist")) + assertThat(processed).isEqualTo(0L) + refreshIndex() + assertThat(openSearchRepository.count()).isEqualTo(0L) + } + + @Test + fun `scoped re-index by documentIds only indexes the requested subset`() { + val target = createDocument("target") + createDocument("other-1") + createDocument("other-2") + clearIndex() + + val (_, processed) = reindex(ReindexRequest(documentIds = listOf(target.id().id))) + + assertThat(processed).isEqualTo(1L) + refreshIndex() + assertThat(openSearchRepository.count()).isEqualTo(1L) + assertThat(openSearchRepository.findById(target.id().toString())).isPresent + } + + @Test + fun `scoped re-index by modifiedAfter only indexes documents modified after the cutoff`() { + val untouched = createDocument("untouched") // modifiedOn stays null -> excluded + val modified = createDocument("before-modify") + val cutoff = LocalDateTime.now() + Thread.sleep(50) + runWithoutAuthorization { + documentService.modifyDocument(modified, objectMapper.createObjectNode().put("street", "after-modify")) + } + clearIndex() + + reindex(ReindexRequest(modifiedAfter = cutoff)) + + refreshIndex() + assertThat(openSearchRepository.findById(modified.id().toString())).isPresent + assertThat(openSearchRepository.findById(untouched.id().toString())).isEmpty + } + + @Test + fun `re-index resumes from the persisted cursor of a prior run`() { + (1..6).forEach { createDocument("doc-$it") } + // Use the database's own ascending id ordering (PostgreSQL orders UUIDs unsigned, which differs + // from Kotlin's signed UUID.compareTo) so the cursor and expected set match the keyset query. + val dbOrderedIds = TransactionTemplate(transactionManager).execute { + entityManager + .createQuery("SELECT d.id.id FROM JsonSchemaDocument d ORDER BY d.id.id", UUID::class.java) + .resultList + }!! + val cursor = dbOrderedIds[2] // resume after the 3rd id -> 3 docs remain + val expectedIds = dbOrderedIds.subList(3, dbOrderedIds.size) + val seededRun = reindexRunRepository.save( + OpenSearchReindexRun( + id = UUID.randomUUID(), + status = ReindexRunStatus.FAILED, + pageSize = ReindexRequest.DEFAULT_PAGE_SIZE, + lastId = cursor, + processedCount = 3, + ) + ) + clearIndex() + + reindexService.reindex(seededRun.id, ReindexRequest()) + + refreshIndex() + assertThat(openSearchRepository.count()).isEqualTo(expectedIds.size.toLong()) + expectedIds.forEach { assertThat(openSearchRepository.findById(it.toString())).isPresent } + assertThat(reindexRunRepository.findById(seededRun.id).get().status).isEqualTo(ReindexRunStatus.COMPLETED) + } + + @Test + fun `startup reconciliation marks a stale-heartbeat orphaned RUNNING run as FAILED`() { + val orphan = reindexRunRepository.save( + OpenSearchReindexRun( + id = UUID.randomUUID(), + status = ReindexRunStatus.RUNNING, + pageSize = ReindexRequest.DEFAULT_PAGE_SIZE, + heartbeatOn = LocalDateTime.now().minusHours(1), + ) + ) + + reindexRunService.reconcileOrphanedRuns() + + assertThat(reindexRunRepository.findById(orphan.id).get().status).isEqualTo(ReindexRunStatus.FAILED) + } + + @Test + fun `start returns null when the cluster-wide lock is already held`() { + val lock = lockProvider.lock( + LockConfiguration( + Instant.now(), + DocumentOpenSearchReindexService.LOCK_NAME, + Duration.ofMinutes(5), + Duration.ZERO, + ) + ) + assertThat(lock).isPresent + try { + assertThat(reindexService.start(ReindexRequest())).isNull() + } finally { + lock.get().unlock() + } + } + + @Test + fun `isReindexRunning is true while a running run has a fresh heartbeat`() { + reindexRunRepository.save( + OpenSearchReindexRun( + id = UUID.randomUUID(), + status = ReindexRunStatus.RUNNING, + pageSize = ReindexRequest.DEFAULT_PAGE_SIZE, + heartbeatOn = LocalDateTime.now(), + ) + ) + + assertThat(reindexRunService.isReindexRunning(Duration.ofMinutes(5))).isTrue() + } + + @Test + fun `isReindexRunning is false when the only running run has a stale heartbeat`() { + reindexRunRepository.save( + OpenSearchReindexRun( + id = UUID.randomUUID(), + status = ReindexRunStatus.RUNNING, + pageSize = ReindexRequest.DEFAULT_PAGE_SIZE, + heartbeatOn = LocalDateTime.now().minusMinutes(10), + ) + ) + + assertThat(reindexRunService.isReindexRunning(Duration.ofMinutes(5))).isFalse() + } + + @Test + fun `isReindexRunning is false when no run is RUNNING`() { + reindexRunRepository.save( + OpenSearchReindexRun( + id = UUID.randomUUID(), + status = ReindexRunStatus.COMPLETED, + pageSize = ReindexRequest.DEFAULT_PAGE_SIZE, + heartbeatOn = LocalDateTime.now(), + ) + ) + + assertThat(reindexRunService.isReindexRunning(Duration.ofMinutes(5))).isFalse() + } + + /** Clears the OpenSearch index (and refreshes) so a subsequent assertion sees only the re-index output. */ + private fun clearIndex() { + openSearchRepository.deleteAll() + refreshIndex() + } + + /** + * Creates a run for [request] and drives the re-index synchronously (each step still uses its own + * transaction, since the test runs without an ambient one). Returns the run id and the documents + * processed. + */ + private fun reindex(request: ReindexRequest): Pair { + val run = reindexRunService.startOrResume(request) + return run.id to reindexService.reindex(run.id, request) + } + + private fun createDocument(street: String): JsonSchemaDocument = + runWithoutAuthorization { + documentService.createDocument( + NewDocumentRequest("house", "house", "1.0.0", objectMapper.createObjectNode().put("street", street)) + ).resultingDocument().get() + } +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceTest.kt new file mode 100644 index 0000000000..0750718888 --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceTest.kt @@ -0,0 +1,107 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.ritense.document.opensearch.domain.OpenSearchReindexRun +import com.ritense.document.opensearch.domain.ReindexRunStatus +import jakarta.persistence.EntityManager +import net.javacrumbs.shedlock.core.LockProvider +import net.javacrumbs.shedlock.core.SimpleLock +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.mock +import org.mockito.kotlin.never +import org.mockito.kotlin.timeout +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever +import org.springframework.data.elasticsearch.core.ElasticsearchOperations +import org.springframework.transaction.PlatformTransactionManager +import java.util.Optional +import java.util.UUID + +class DocumentOpenSearchReindexServiceTest { + + private val entityManager: EntityManager = mock() + private val converter: JsonSchemaDocumentOsConverter = mock() + private val elasticsearchOperations: ElasticsearchOperations = mock() + private val transactionManager: PlatformTransactionManager = mock() + private val lockProvider: LockProvider = mock() + private val runService: OpenSearchReindexRunService = mock() + + private lateinit var service: DocumentOpenSearchReindexService + + @BeforeEach + fun setUp() { + service = DocumentOpenSearchReindexService( + entityManager, + converter, + elasticsearchOperations, + transactionManager, + lockProvider, + runService, + ) + } + + @Test + fun `start returns null and creates no run when the lock is already held`() { + whenever(lockProvider.lock(any())).thenReturn(Optional.empty()) + + val runId = service.start(ReindexRequest()) + + assertThat(runId).isNull() + verify(runService, never()).startOrResume(any()) + } + + @Test + fun `start creates a run and releases the lock when acquired`() { + val simpleLock: SimpleLock = mock() + whenever(lockProvider.lock(any())).thenReturn(Optional.of(simpleLock)) + val expectedId = UUID.randomUUID() + whenever(runService.startOrResume(any())).thenReturn(run(expectedId)) + + val runId = service.start(ReindexRequest()) + + assertThat(runId).isEqualTo(expectedId) + verify(runService).startOrResume(any()) + // The dispatched run finishes (here it terminates early against the mocks); the lock must be released. + verify(simpleLock, timeout(5_000)).unlock() + } + + @Test + fun `reindex marks the run STOPPED when cancellation was requested`() { + val runId = UUID.randomUUID() + whenever(runService.cursorOf(runId)).thenReturn(null) + whenever(runService.processedOf(runId)).thenReturn(0L) + + // destroy() sets the cancellation flag (and shuts down the idle executor). + service.destroy() + + val processed = service.reindex(runId, ReindexRequest()) + + assertThat(processed).isEqualTo(0L) + verify(runService).stop(runId) + verify(runService, never()).complete(any()) + } + + private fun run(id: UUID) = OpenSearchReindexRun( + id = id, + status = ReindexRunStatus.RUNNING, + pageSize = 100, + ) +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchSyncServiceTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchSyncServiceTest.kt index c809b373aa..b05e1639d7 100644 --- a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchSyncServiceTest.kt +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchSyncServiceTest.kt @@ -1,5 +1,5 @@ /* - * Copyright 2015-2025 Ritense BV, the Netherlands. + * Copyright 2015-2026 Ritense BV, the Netherlands. * * Licensed under EUPL, Version 1.2 (the "License"); * you may not use this file except in compliance with the License. @@ -16,112 +16,101 @@ package com.ritense.document.opensearch.service -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.databind.node.ContainerNode -import com.ritense.document.event.DocumentCreated +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.domain.impl.JsonSchemaDocumentId import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument import com.ritense.document.opensearch.repository.JsonSchemaDocumentOpenSearchRepository -import com.ritense.outbox.domain.BaseEvent -import org.assertj.core.api.Assertions.assertThat +import com.ritense.document.repository.impl.JsonSchemaDocumentRepository +import org.assertj.core.api.Assertions.assertThatCode +import org.assertj.core.api.Assertions.assertThatThrownBy import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test -import org.mockito.ArgumentCaptor import org.mockito.kotlin.any -import org.mockito.kotlin.capture import org.mockito.kotlin.mock import org.mockito.kotlin.never import org.mockito.kotlin.verify import org.mockito.kotlin.whenever +import org.springframework.data.elasticsearch.VersionConflictException +import org.springframework.transaction.PlatformTransactionManager +import org.springframework.transaction.support.SimpleTransactionStatus +import java.util.Optional +import java.util.UUID class DocumentOpenSearchSyncServiceTest { private val repository: JsonSchemaDocumentOpenSearchRepository = mock() - private val objectMapper: ObjectMapper = mock() + private val documentRepository: JsonSchemaDocumentRepository = mock() + private val converter: JsonSchemaDocumentOsConverter = mock() + private val transactionManager: PlatformTransactionManager = mock() private lateinit var service: DocumentOpenSearchSyncService @BeforeEach fun setUp() { - service = DocumentOpenSearchSyncService(repository, objectMapper) + // Let the read-only TransactionTemplate run its callback inline. + whenever(transactionManager.getTransaction(any())).thenReturn(SimpleTransactionStatus()) + service = DocumentOpenSearchSyncService(repository, documentRepository, converter, transactionManager) } @Test - fun `upsert with null result skips repository save`() { - val event = testEvent(result = null) + fun `upsertById reloads the document and saves the converted os document`() { + val id = UUID.randomUUID() + val document: JsonSchemaDocument = mock() + val osDocument = osDocument(id.toString()) + whenever(documentRepository.findById(JsonSchemaDocumentId.existingId(id))).thenReturn(Optional.of(document)) + whenever(converter.toOsDocument(document)).thenReturn(osDocument) - service.upsert(event) + service.upsertById(id) - verify(repository, never()).save(any()) + verify(repository).save(osDocument) } @Test - fun `upsert populates contentText with leaf values from content`() { - val realMapper = ObjectMapper() - val content = realMapper.createObjectNode().apply { - put("firstName", "John") - put("city", "Amsterdam") - } - val resultNode = realMapper.createObjectNode().apply { - set("content", content) - } - val doc = buildDocument(id = "test-id", content = mapOf("firstName" to "John", "city" to "Amsterdam")) - val event = testEvent(result = resultNode) - whenever(objectMapper.treeToValue(any(), any>())).thenReturn(doc) - - val captor = ArgumentCaptor.forClass(JsonSchemaDocumentOsDocument::class.java) - service.upsert(event) - verify(repository).save(capture(captor)) - - val saved = captor.value - assertThat(saved.contentText).contains("John") - assertThat(saved.contentText).contains("Amsterdam") + fun `upsertById swallows a version conflict (a newer version is already indexed)`() { + val id = UUID.randomUUID() + val document: JsonSchemaDocument = mock() + val osDocument = osDocument(id.toString()) + whenever(documentRepository.findById(JsonSchemaDocumentId.existingId(id))).thenReturn(Optional.of(document)) + whenever(converter.toOsDocument(document)).thenReturn(osDocument) + whenever(repository.save(osDocument)).thenThrow(VersionConflictException("conflict")) + + assertThatCode { service.upsertById(id) }.doesNotThrowAnyException() } @Test - fun `upsert with null content stores null contentText`() { - val realMapper = ObjectMapper() - val resultNode = realMapper.createObjectNode() - val doc = buildDocument(id = "no-content-id", content = null) - val event = testEvent(result = resultNode) - whenever(objectMapper.treeToValue(any(), any>())).thenReturn(doc) - - val captor = ArgumentCaptor.forClass(JsonSchemaDocumentOsDocument::class.java) - service.upsert(event) - verify(repository).save(capture(captor)) - - assertThat(captor.value.contentText).isNull() + fun `upsertById propagates a non-conflict failure`() { + val id = UUID.randomUUID() + val document: JsonSchemaDocument = mock() + val osDocument = osDocument(id.toString()) + whenever(documentRepository.findById(JsonSchemaDocumentId.existingId(id))).thenReturn(Optional.of(document)) + whenever(converter.toOsDocument(document)).thenReturn(osDocument) + whenever(repository.save(osDocument)).thenThrow(RuntimeException("transport error")) + + assertThatThrownBy { service.upsertById(id) }.isInstanceOf(RuntimeException::class.java) + } + + @Test + fun `upsertById skips a document that no longer exists (already deleted)`() { + val id = UUID.randomUUID() + whenever(documentRepository.findById(JsonSchemaDocumentId.existingId(id))).thenReturn(Optional.empty()) + + service.upsertById(id) + + verify(converter, never()).toOsDocument(any()) + verify(repository, never()).save(any()) } @Test - fun `upsert with nested content extracts all leaf values`() { - val realMapper = ObjectMapper() - val content = realMapper.createObjectNode().apply { - putObject("address").apply { - put("street", "Main Street") - put("number", "42") - } - } - val resultNode = realMapper.createObjectNode().apply { - set("content", content) - } - val doc = buildDocument(id = "nested-id", content = mapOf("address" to mapOf("street" to "Main Street", "number" to "42"))) - val event = testEvent(result = resultNode) - whenever(objectMapper.treeToValue(any(), any>())).thenReturn(doc) - - val captor = ArgumentCaptor.forClass(JsonSchemaDocumentOsDocument::class.java) - service.upsert(event) - verify(repository).save(capture(captor)) - - val contentText = captor.value.contentText - assertThat(contentText).contains("Main Street") - assertThat(contentText).contains("42") + fun `delete removes the document from opensearch by id`() { + val id = UUID.randomUUID() + + service.delete(id) + + verify(repository).deleteById(id.toString()) } - private fun buildDocument( - id: String, - content: Map?, - ) = JsonSchemaDocumentOsDocument( + private fun osDocument(id: String) = JsonSchemaDocumentOsDocument( id = id, - content = content, + content = null, definitionId = null, createdOn = null, modifiedOn = null, @@ -136,16 +125,4 @@ class DocumentOpenSearchSyncServiceTest { relatedFiles = null, retentionDate = null, ) - - private fun testEvent(result: ContainerNode<*>?): BaseEvent = - if (result != null) { - DocumentCreated("doc-id", result as com.fasterxml.jackson.databind.node.ObjectNode) - } else { - object : BaseEvent( - type = "test", - resultType = null, - resultId = "doc-id", - result = null, - ) {} - } } diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchVersioningIntTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchVersioningIntTest.kt new file mode 100644 index 0000000000..25bd233fe6 --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchVersioningIntTest.kt @@ -0,0 +1,145 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.ritense.authorization.AuthorizationContext.Companion.runWithoutAuthorization +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.domain.impl.request.NewDocumentRequest +import com.ritense.document.opensearch.BaseOpenSearchIntegrationTest +import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatCode +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.security.test.context.support.WithMockUser +import org.springframework.transaction.annotation.Propagation +import org.springframework.transaction.annotation.Transactional +import java.util.UUID + +/** + * Integration tests for OpenSearch external versioning against a real OpenSearch. "Highest version wins": + * a re-send of an equal-or-lower [JsonSchemaDocumentOsDocument.indexVersion] is a benign version-conflict + * no-op (swallowed by [JsonSchemaDocumentOsConverter.indexChunk] / [DocumentOpenSearchSyncService]) and can + * never overwrite a newer document; a strictly higher version updates. + * + * Runs **non-transactionally** so the live-sync path sees committed data and its own JPA `version` bumps. + */ +@Transactional(propagation = Propagation.NOT_SUPPORTED) +@WithMockUser(username = BaseOpenSearchIntegrationTest.USERNAME, authorities = [BaseOpenSearchIntegrationTest.FULL_ACCESS_ROLE]) +class DocumentOpenSearchVersioningIntTest : BaseOpenSearchIntegrationTest() { + + @Autowired + lateinit var converter: JsonSchemaDocumentOsConverter + + @Autowired + lateinit var syncService: DocumentOpenSearchSyncService + + @AfterEach + fun cleanUp() { + runWithoutAuthorization { documentService.removeDocuments("house") } + clearIndex() + } + + @Test + fun `re-indexing the same version is a benign no-op`() { + val id = UUID.randomUUID().toString() + openSearchRepository.saveAll(listOf(osDoc(id, indexVersion = 5, internalStatus = "first"))) + refreshIndex() + + val skipped = converter.indexChunk(listOf(osDoc(id, indexVersion = 5, internalStatus = "second"))) + refreshIndex() + + assertThat(skipped).isZero() + assertThat(openSearchRepository.findById(id).get().internalStatus).isEqualTo("first") + } + + @Test + fun `a stale lower-version write does not overwrite a newer document (order independence)`() { + val id = UUID.randomUUID().toString() + openSearchRepository.saveAll(listOf(osDoc(id, indexVersion = 6, internalStatus = "sixth"))) + refreshIndex() + + val skipped = converter.indexChunk(listOf(osDoc(id, indexVersion = 5, internalStatus = "fifth"))) + refreshIndex() + + assertThat(skipped).isZero() + assertThat(openSearchRepository.findById(id).get().internalStatus).isEqualTo("sixth") + } + + @Test + fun `a strictly higher version updates the document`() { + val id = UUID.randomUUID().toString() + openSearchRepository.saveAll(listOf(osDoc(id, indexVersion = 5, internalStatus = "old"))) + refreshIndex() + + val skipped = converter.indexChunk(listOf(osDoc(id, indexVersion = 6, internalStatus = "new"))) + refreshIndex() + + assertThat(skipped).isZero() + assertThat(openSearchRepository.findById(id).get().internalStatus).isEqualTo("new") + } + + @Test + fun `the live path indexes a real change and swallows a redundant re-send`() { + val document = createDocument("versioned") + val id = document.id().id + syncService.upsertById(id) + refreshIndex() + + runWithoutAuthorization { documentService.setInternalStatus(document.id(), "started") } + syncService.upsertById(id) + refreshIndex() + assertThat(openSearchRepository.findById(id.toString()).get().internalStatus).isEqualTo("started") + + // Re-sending the same (now-current) version is a version conflict — swallowed, no exception. + assertThatCode { syncService.upsertById(id) }.doesNotThrowAnyException() + refreshIndex() + assertThat(openSearchRepository.findById(id.toString()).get().internalStatus).isEqualTo("started") + } + + private fun clearIndex() { + openSearchRepository.deleteAll() + refreshIndex() + } + + private fun createDocument(street: String): JsonSchemaDocument = + runWithoutAuthorization { + documentService.createDocument( + NewDocumentRequest("house", "house", "1.0.0", objectMapper.createObjectNode().put("street", street)) + ).resultingDocument().get() + } + + private fun osDoc(id: String, indexVersion: Long, internalStatus: String) = JsonSchemaDocumentOsDocument( + id = id, + content = null, + definitionId = null, + createdOn = null, + modifiedOn = null, + createdBy = null, + sequence = null, + version = null, + assigneeId = null, + assigneeFullName = null, + internalStatus = internalStatus, + caseTags = null, + relations = null, + relatedFiles = null, + retentionDate = null, + indexVersion = indexVersion, + ) +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOsConverterTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOsConverterTest.kt new file mode 100644 index 0000000000..48666c9d9e --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOsConverterTest.kt @@ -0,0 +1,127 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.fasterxml.jackson.databind.ObjectMapper +import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument +import com.ritense.document.opensearch.repository.JsonSchemaDocumentOpenSearchRepository +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.eq +import org.mockito.kotlin.mock +import org.mockito.kotlin.never +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever +import org.springframework.data.elasticsearch.BulkFailureException +import org.springframework.data.elasticsearch.BulkFailureException.FailureDetails +import org.springframework.data.elasticsearch.VersionConflictException + +class JsonSchemaDocumentOsConverterTest { + + private val objectMapper: ObjectMapper = mock() + private val repository: JsonSchemaDocumentOpenSearchRepository = mock() + private lateinit var converter: JsonSchemaDocumentOsConverter + + @BeforeEach + fun setUp() { + converter = JsonSchemaDocumentOsConverter(objectMapper, repository) + } + + @Test + fun `indexChunk bulk-saves and reports zero skips on success`() { + val chunk = listOf(osDocument("a"), osDocument("b")) + + val skipped = converter.indexChunk(chunk) + + assertThat(skipped).isZero() + verify(repository).saveAll(chunk) + } + + @Test + fun `indexChunk re-processes only the failed documents and isolates a real failure`() { + val good = osDocument("good") + val poison = osDocument("poison") + whenever(repository.saveAll(any>())) + .thenThrow(BulkFailureException("bulk failed", mapOf("poison" to FailureDetails(400, "mapper_parsing_exception")))) + whenever(repository.save(eq(poison))).thenThrow(RuntimeException("mapping error")) + + val skipped = converter.indexChunk(listOf(good, poison)) + + assertThat(skipped).isEqualTo(1L) + verify(repository).save(poison) + // "good" was not in the failure map, so it is never re-processed. + verify(repository, never()).save(good) + } + + @Test + fun `indexChunk treats a version conflict as benign (no skip, no retry)`() { + val document = osDocument("a") + whenever(repository.saveAll(any>())) + .thenThrow(BulkFailureException("bulk failed", mapOf("a" to FailureDetails(409, "version_conflict_engine_exception ...")))) + + val skipped = converter.indexChunk(listOf(document)) + + assertThat(skipped).isZero() + // A 409 means the stored doc is already ≥ this version — never re-saved. + verify(repository, never()).save(any()) + } + + @Test + fun `indexChunk classifies a version conflict by message when status is absent`() { + val document = osDocument("a") + whenever(repository.saveAll(any>())) + .thenThrow(BulkFailureException("bulk failed", mapOf("a" to FailureDetails(null, "... version_conflict_engine_exception ...")))) + + val skipped = converter.indexChunk(listOf(document)) + + assertThat(skipped).isZero() + verify(repository, never()).save(any()) + } + + @Test + fun `indexChunk treats a VersionConflictException on retry as benign`() { + val document = osDocument("a") + whenever(repository.saveAll(any>())) + .thenThrow(BulkFailureException("bulk failed", mapOf("a" to FailureDetails(500, "transient")))) + whenever(repository.save(eq(document))).thenThrow(VersionConflictException("conflict")) + + val skipped = converter.indexChunk(listOf(document)) + + assertThat(skipped).isZero() + verify(repository).save(document) + } + + private fun osDocument(id: String) = JsonSchemaDocumentOsDocument( + id = id, + content = null, + definitionId = null, + createdOn = null, + modifiedOn = null, + createdBy = null, + sequence = null, + version = null, + assigneeId = null, + assigneeFullName = null, + internalStatus = null, + caseTags = null, + relations = null, + relatedFiles = null, + retentionDate = null, + ) +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunServiceTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunServiceTest.kt new file mode 100644 index 0000000000..efa1e0fdde --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunServiceTest.kt @@ -0,0 +1,175 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.fasterxml.jackson.databind.ObjectMapper +import com.ritense.document.opensearch.OpenSearchProperties +import com.ritense.document.opensearch.domain.OpenSearchReindexRun +import com.ritense.document.opensearch.domain.ReindexRunStatus +import com.ritense.document.opensearch.repository.OpenSearchReindexRunRepository +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.argumentCaptor +import org.mockito.kotlin.doAnswer +import org.mockito.kotlin.eq +import org.mockito.kotlin.mock +import org.mockito.kotlin.never +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever +import java.util.Optional +import java.util.UUID + +class OpenSearchReindexRunServiceTest { + + private val repository: OpenSearchReindexRunRepository = mock() + private val objectMapper: ObjectMapper = ObjectMapper().findAndRegisterModules() + private val properties = OpenSearchProperties() + private lateinit var service: OpenSearchReindexRunService + + @BeforeEach + fun setUp() { + service = OpenSearchReindexRunService(repository, objectMapper, properties) + whenever(repository.save(any())).doAnswer { it.arguments[0] as OpenSearchReindexRun } + } + + @Test + fun `startOrResume creates a new RUNNING run`() { + val request = ReindexRequest(documentDefinitionName = "house", pageSize = 250) + + val run = service.startOrResume(request) + + assertThat(run.status).isEqualTo(ReindexRunStatus.RUNNING) + assertThat(run.pageSize).isEqualTo(250) + assertThat(run.scope).contains("house") + verify(repository).save(any()) + } + + @Test + fun `startOrResume re-arms an existing run when resumeRunId is set`() { + val runId = UUID.randomUUID() + val existing = OpenSearchReindexRun( + id = runId, + status = ReindexRunStatus.FAILED, + pageSize = 100, + lastId = UUID.randomUUID(), + error = "boom", + ) + whenever(repository.findById(runId)).thenReturn(Optional.of(existing)) + + val run = service.startOrResume(ReindexRequest(resumeRunId = runId)) + + assertThat(run.id).isEqualTo(runId) + assertThat(run.status).isEqualTo(ReindexRunStatus.RUNNING) + assertThat(run.error).isNull() + assertThat(run.finishedOn).isNull() + } + + @Test + fun `recordProgress updates cursor and counts`() { + val runId = UUID.randomUUID() + val run = OpenSearchReindexRun(id = runId, pageSize = 100) + whenever(repository.findById(runId)).thenReturn(Optional.of(run)) + val cursor = UUID.randomUUID() + + service.recordProgress(runId, cursor, processed = 42, skipped = 3) + + assertThat(run.lastId).isEqualTo(cursor) + assertThat(run.processedCount).isEqualTo(42) + assertThat(run.skippedCount).isEqualTo(3) + verify(repository).save(run) + } + + @Test + fun `complete fail and stop set the terminal status`() { + val runId = UUID.randomUUID() + val run = OpenSearchReindexRun(id = runId, pageSize = 100) + whenever(repository.findById(runId)).thenReturn(Optional.of(run)) + + service.complete(runId) + assertThat(run.status).isEqualTo(ReindexRunStatus.COMPLETED) + assertThat(run.finishedOn).isNotNull() + + service.fail(runId, "kaboom") + assertThat(run.status).isEqualTo(ReindexRunStatus.FAILED) + assertThat(run.error).isEqualTo("kaboom") + + service.stop(runId) + assertThat(run.status).isEqualTo(ReindexRunStatus.STOPPED) + } + + @Test + fun `reconcileOrphanedRuns marks stale-heartbeat RUNNING rows as FAILED`() { + val orphan = OpenSearchReindexRun( + id = UUID.randomUUID(), + status = ReindexRunStatus.RUNNING, + pageSize = 100, + ) + whenever(repository.findAllByStatusAndHeartbeatOnBefore(eq(ReindexRunStatus.RUNNING), any())) + .thenReturn(listOf(orphan)) + + service.reconcileOrphanedRuns() + + assertThat(orphan.status).isEqualTo(ReindexRunStatus.FAILED) + assertThat(orphan.error).contains("Reconciled on startup") + val captor = argumentCaptor>() + verify(repository).saveAll(captor.capture()) + assertThat(captor.firstValue).containsExactly(orphan) + } + + @Test + fun `reconcileOrphanedRuns does nothing when no orphans exist`() { + whenever(repository.findAllByStatusAndHeartbeatOnBefore(eq(ReindexRunStatus.RUNNING), any())) + .thenReturn(emptyList()) + + service.reconcileOrphanedRuns() + + verify(repository, never()).saveAll(any>()) + } + + @Test + fun `toStatusMap returns a not-running placeholder when nothing matches`() { + whenever(repository.findFirstByOrderByStartedOnDesc()).thenReturn(null) + + val status = service.toStatusMap(null) + + assertThat(status["running"]).isEqualTo(false) + assertThat(status["runId"]).isNull() + } + + @Test + fun `toStatusMap reports running state and counts for a specific run`() { + val runId = UUID.randomUUID() + val run = OpenSearchReindexRun( + id = runId, + status = ReindexRunStatus.RUNNING, + pageSize = 100, + processedCount = 7, + skippedCount = 1, + ) + whenever(repository.findById(runId)).thenReturn(Optional.of(run)) + + val status = service.toStatusMap(runId) + + assertThat(status["runId"]).isEqualTo(runId) + assertThat(status["running"]).isEqualTo(true) + assertThat(status["status"]).isEqualTo(ReindexRunStatus.RUNNING) + assertThat(status["processedCount"]).isEqualTo(7L) + assertThat(status["skippedCount"]).isEqualTo(1L) + } +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/ReindexProgressGateTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/ReindexProgressGateTest.kt new file mode 100644 index 0000000000..acaeec9786 --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/ReindexProgressGateTest.kt @@ -0,0 +1,76 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.ritense.document.opensearch.OpenSearchProperties +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.mock +import org.mockito.kotlin.never +import org.mockito.kotlin.times +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever + +class ReindexProgressGateTest { + + private val reindexRunService: OpenSearchReindexRunService = mock() + + @Test + fun `reports in-progress when a reindex run is running`() { + whenever(reindexRunService.isReindexRunning(any())).thenReturn(true) + val gate = ReindexProgressGate(reindexRunService, OpenSearchProperties()) + + assertThat(gate.isReindexInProgress()).isTrue() + } + + @Test + fun `reports not in-progress when no reindex run is running`() { + whenever(reindexRunService.isReindexRunning(any())).thenReturn(false) + val gate = ReindexProgressGate(reindexRunService, OpenSearchProperties()) + + assertThat(gate.isReindexInProgress()).isFalse() + } + + @Test + fun `never queries the run service when fallback is disabled`() { + val properties = OpenSearchProperties( + reindex = OpenSearchProperties.Reindex(fallbackToPostgresWhileRunning = false) + ) + val gate = ReindexProgressGate(reindexRunService, properties) + + assertThat(gate.isReindexInProgress()).isFalse() + verify(reindexRunService, never()).isReindexRunning(any()) + } + + @Test + fun `caches the result within the ttl window and refreshes after it`() { + whenever(reindexRunService.isReindexRunning(any())).thenReturn(true) + var now = 1_000L + val gate = ReindexProgressGate(reindexRunService, OpenSearchProperties(), clock = { now }) + + gate.isReindexInProgress() + gate.isReindexInProgress() + // Within the TTL: only one DB check. + verify(reindexRunService, times(1)).isReindexRunning(any()) + + now += ReindexProgressGate.CACHE_TTL_MS + gate.isReindexInProgress() + // TTL elapsed: a second check. + verify(reindexRunService, times(2)).isReindexRunning(any()) + } +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/ReindexRequestTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/ReindexRequestTest.kt new file mode 100644 index 0000000000..3f3ef082e5 --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/ReindexRequestTest.kt @@ -0,0 +1,46 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test + +class ReindexRequestTest { + + @Test + fun `effectivePageSize clamps values above the maximum`() { + val request = ReindexRequest(pageSize = ReindexRequest.MAX_PAGE_SIZE + 5000) + + assertThat(request.effectivePageSize()).isEqualTo(ReindexRequest.MAX_PAGE_SIZE) + } + + @Test + fun `effectivePageSize clamps zero and negative values to one`() { + assertThat(ReindexRequest(pageSize = 0).effectivePageSize()).isEqualTo(1) + assertThat(ReindexRequest(pageSize = -10).effectivePageSize()).isEqualTo(1) + } + + @Test + fun `effectivePageSize keeps values within range`() { + assertThat(ReindexRequest(pageSize = 1234).effectivePageSize()).isEqualTo(1234) + } + + @Test + fun `default page size is used when not specified`() { + assertThat(ReindexRequest().effectivePageSize()).isEqualTo(ReindexRequest.DEFAULT_PAGE_SIZE) + } +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/web/DocumentOpenSearchReindexResourceTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/web/DocumentOpenSearchReindexResourceTest.kt new file mode 100644 index 0000000000..4e7917e51d --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/web/DocumentOpenSearchReindexResourceTest.kt @@ -0,0 +1,92 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.web + +import com.ritense.document.opensearch.service.DocumentOpenSearchReindexService +import com.ritense.document.opensearch.service.ReindexRequest +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.mock +import org.mockito.kotlin.whenever +import java.util.UUID + +class DocumentOpenSearchReindexResourceTest { + + private val reindexService: DocumentOpenSearchReindexService = mock() + private lateinit var resource: DocumentOpenSearchReindexResource + + @BeforeEach + fun setUp() { + resource = DocumentOpenSearchReindexResource(reindexService) + } + + @Test + fun `reindex returns 202 with the run id when started`() { + val runId = UUID.randomUUID() + whenever(reindexService.start(any())).thenReturn(runId) + + val response = resource.reindex(ReindexRequest(documentDefinitionName = "house")) + + assertThat(response.statusCode.value()).isEqualTo(202) + assertThat(response.body?.get("status")).isEqualTo("started") + assertThat(response.body?.get("runId")).isEqualTo(runId) + } + + @Test + fun `reindex starts with an empty request when no body is provided`() { + val runId = UUID.randomUUID() + whenever(reindexService.start(any())).thenReturn(runId) + + val response = resource.reindex(null) + + assertThat(response.statusCode.value()).isEqualTo(202) + assertThat(response.body?.get("runId")).isEqualTo(runId) + } + + @Test + fun `reindex returns 409 when a re-index is already running`() { + whenever(reindexService.start(any())).thenReturn(null) + + val response = resource.reindex(ReindexRequest()) + + assertThat(response.statusCode.value()).isEqualTo(409) + assertThat(response.body?.get("error")).isEqualTo("Re-index already in progress") + } + + @Test + fun `status returns the most recent run`() { + whenever(reindexService.status(null)).thenReturn(mapOf("running" to true)) + + val response = resource.status() + + assertThat(response.statusCode.value()).isEqualTo(200) + assertThat(response.body?.get("running")).isEqualTo(true) + } + + @Test + fun `statusById returns the requested run`() { + val runId = UUID.randomUUID() + whenever(reindexService.status(runId)).thenReturn(mapOf("runId" to runId)) + + val response = resource.statusById(runId) + + assertThat(response.statusCode.value()).isEqualTo(200) + assertThat(response.body?.get("runId")).isEqualTo(runId) + } +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/web/SearchEngineResourceTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/web/SearchEngineResourceTest.kt index e79c8ec616..2aa2adc161 100644 --- a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/web/SearchEngineResourceTest.kt +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/web/SearchEngineResourceTest.kt @@ -19,6 +19,7 @@ package com.ritense.document.opensearch.web import com.ritense.adminsettings.service.FeatureToggleOverridesService import com.ritense.adminsettings.web.rest.dto.FeatureToggleOverridesDto import com.ritense.document.opensearch.OpenSearchProperties +import com.ritense.document.opensearch.service.DocumentOpenSearchIndexInitializer import com.ritense.document.opensearch.service.SearchEngineToggle import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.BeforeEach @@ -26,6 +27,7 @@ import org.junit.jupiter.api.Test import org.mockito.kotlin.any import org.mockito.kotlin.eq import org.mockito.kotlin.mock +import org.mockito.kotlin.never import org.mockito.kotlin.verify import org.mockito.kotlin.whenever import org.springframework.http.HttpStatus @@ -35,6 +37,7 @@ class SearchEngineResourceTest { private lateinit var toggle: SearchEngineToggle private lateinit var properties: OpenSearchProperties private lateinit var featureToggleService: FeatureToggleOverridesService + private lateinit var indexInitializer: DocumentOpenSearchIndexInitializer private lateinit var resource: SearchEngineResource @BeforeEach @@ -42,7 +45,8 @@ class SearchEngineResourceTest { toggle = SearchEngineToggle() properties = OpenSearchProperties(enabled = true) featureToggleService = mock() - resource = SearchEngineResource(toggle, properties, featureToggleService) + indexInitializer = mock() + resource = SearchEngineResource(toggle, properties, featureToggleService, indexInitializer) } @Test @@ -61,7 +65,8 @@ class SearchEngineResourceTest { val disabledResource = SearchEngineResource( toggle, OpenSearchProperties(enabled = false), - featureToggleService + featureToggleService, + indexInitializer ) val response = disabledResource.getActive() @@ -80,6 +85,7 @@ class SearchEngineResourceTest { assertThat(response.body?.active).isEqualTo("POSTGRES") assertThat(toggle.get()).isEqualTo(SearchEngineToggle.Engine.POSTGRES) verify(featureToggleService).updateToggle(eq("useOpenSearchForDocumentSearch"), eq(false)) + verify(indexInitializer, never()).ensureIndex() } @Test @@ -93,6 +99,7 @@ class SearchEngineResourceTest { assertThat(response.body?.active).isEqualTo("OPENSEARCH") assertThat(toggle.get()).isEqualTo(SearchEngineToggle.Engine.OPENSEARCH) verify(featureToggleService).updateToggle(eq("useOpenSearchForDocumentSearch"), eq(true)) + verify(indexInitializer).ensureIndex() } @Test @@ -100,7 +107,8 @@ class SearchEngineResourceTest { val disabledResource = SearchEngineResource( toggle, OpenSearchProperties(enabled = false), - featureToggleService + featureToggleService, + indexInitializer ) val response = disabledResource.setActive(SearchEngineResource.UpdateSearchEngineDto("OPENSEARCH")) diff --git a/backend/case-opensearch/src/test/resources/config/application-postgresql.yml b/backend/case-opensearch/src/test/resources/config/application-postgresql.yml index 11d9550070..88f952877c 100644 --- a/backend/case-opensearch/src/test/resources/config/application-postgresql.yml +++ b/backend/case-opensearch/src/test/resources/config/application-postgresql.yml @@ -12,5 +12,10 @@ spring: elasticsearch: uris: http://localhost:39200 +# The spring-data-opensearch starter binds its own client from the `opensearch.*` namespace +# (not spring.elasticsearch.*), defaulting to localhost:9200. The test OpenSearch is on 39200. +opensearch: + uris: http://localhost:39200 + valtimo: database: postgres diff --git a/backend/case-opensearch/src/test/resources/config/application.yml b/backend/case-opensearch/src/test/resources/config/application.yml index e68db94e75..ac84be235e 100644 --- a/backend/case-opensearch/src/test/resources/config/application.yml +++ b/backend/case-opensearch/src/test/resources/config/application.yml @@ -26,6 +26,11 @@ valtimo: enabled: false plugin: encryption-secret: "abcdefghijklmnop" + opensearch: + reconcile: + # Disable the scheduled reconcile job in tests so it cannot interfere with assertions; the tests + # invoke DocumentOpenSearchReconcileService.reconcile() directly instead. + enabled: false operaton: bpm: diff --git a/backend/case-opensearch/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/backend/case-opensearch/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker new file mode 100644 index 0000000000..1f0955d450 --- /dev/null +++ b/backend/case-opensearch/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker @@ -0,0 +1 @@ +mock-maker-inline diff --git a/backend/case/src/main/java/com/ritense/document/domain/impl/JsonSchemaDocument.java b/backend/case/src/main/java/com/ritense/document/domain/impl/JsonSchemaDocument.java index f8d5060d2c..4139d58dd1 100644 --- a/backend/case/src/main/java/com/ritense/document/domain/impl/JsonSchemaDocument.java +++ b/backend/case/src/main/java/com/ritense/document/domain/impl/JsonSchemaDocument.java @@ -68,7 +68,9 @@ import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.StreamSupport; +import org.hibernate.annotations.CurrentTimestamp; import org.hibernate.annotations.DynamicUpdate; +import org.hibernate.annotations.SourceType; import org.hibernate.annotations.Type; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -108,6 +110,16 @@ public class JsonSchemaDocument extends AbstractAggregateRootevery mutation, so it + * is the watermark the OpenSearch reconciler scans on. Written by every save; never set by hand. + */ + @CurrentTimestamp(source = SourceType.DB) + @Column(name = "changed_on", columnDefinition = "DATETIME", nullable = false) + private LocalDateTime changedOn; + @Column(name = "created_by", columnDefinition = "VARCHAR(255)") private String createdBy; @@ -389,6 +401,11 @@ public Optional modifiedOn() { return Optional.ofNullable(modifiedOn); } + @JsonIgnore + public LocalDateTime changedOn() { + return changedOn; + } + @Override public JsonDocumentContent content() { return content; diff --git a/backend/case/src/main/java/com/ritense/document/service/impl/JsonSchemaDocumentService.java b/backend/case/src/main/java/com/ritense/document/service/impl/JsonSchemaDocumentService.java index 4a09e51db4..35fc5e88ba 100644 --- a/backend/case/src/main/java/com/ritense/document/service/impl/JsonSchemaDocumentService.java +++ b/backend/case/src/main/java/com/ritense/document/service/impl/JsonSchemaDocumentService.java @@ -603,11 +603,16 @@ public void removeDocuments( }); documentRepository.saveAll(documents); documentRepository.deleteAll(documents); - documents.forEach(document -> outboxService.send(() -> - new DocumentDeleted( - document.id().toString() - ) - )); + documents.forEach(document -> { + // Per-document Spring event so bulk deletes are handled identically to single deletes + // (durable pending index deletion + best-effort live delete). Fires inside this transaction. + applicationEventPublisher.publishEvent(new DocumentDeletedEvent(document.id().getId())); + outboxService.send(() -> + new DocumentDeleted( + document.id().toString() + ) + ); + }); documentSequenceGeneratorService.deleteSequenceRecordBy(documentDefinitionName); } } diff --git a/backend/case/src/test/java/com/ritense/document/service/impl/JsonSchemaDocumentServiceTest.java b/backend/case/src/test/java/com/ritense/document/service/impl/JsonSchemaDocumentServiceTest.java index 77c62e4c5a..a5d262dff5 100644 --- a/backend/case/src/test/java/com/ritense/document/service/impl/JsonSchemaDocumentServiceTest.java +++ b/backend/case/src/test/java/com/ritense/document/service/impl/JsonSchemaDocumentServiceTest.java @@ -33,10 +33,10 @@ import com.ritense.document.domain.impl.JsonDocumentContent; import com.ritense.document.domain.impl.JsonSchemaDocument; import com.ritense.document.domain.impl.JsonSchemaDocumentDefinition; +import com.ritense.document.domain.impl.JsonSchemaDocumentId; import com.ritense.document.domain.impl.request.NewDocumentRequest; import com.ritense.document.event.DocumentAssigneeChangedEvent; import com.ritense.document.event.DocumentUnassignedEvent; -import com.ritense.document.domain.impl.JsonSchemaDocumentId; import com.ritense.document.repository.impl.JsonSchemaDocumentRepository; import com.ritense.document.service.CaseTagService; import com.ritense.document.service.InternalCaseStatusService; @@ -47,6 +47,7 @@ import com.ritense.valtimo.contract.authentication.TeamManagementService; import com.ritense.valtimo.contract.authentication.UserManagementService; import com.ritense.valtimo.contract.case_.CaseDefinitionId; +import com.ritense.valtimo.contract.event.DocumentDeletedEvent; import com.ritense.valtimo.contract.json.MapperSingleton; import com.ritense.valtimo.contract.resource.Resource; import jakarta.persistence.EntityManager; @@ -218,6 +219,10 @@ void shouldRemoveDocuments() { verify(documentRepository, times(1)).saveAll(eq(jsonSchemaDocuments.toList())); verify(documentRepository, times(1)).deleteAll(eq(jsonSchemaDocuments.toList())); verify(documentSequenceGeneratorService, times(1)).deleteSequenceRecordBy(eq(documentDefinitionName)); + // A per-document Spring event is published so bulk deletes reach OpenSearch (pending index deletion) like single deletes. + var captor = ArgumentCaptor.forClass(DocumentDeletedEvent.class); + verify(applicationEventPublisher, times(1)).publishEvent(captor.capture()); + assertEquals(jsonSchemaDocument.id().getId(), captor.getValue().getCaseDocumentId()); } @Test diff --git a/backend/core/src/main/resources/config/liquibase/13-32-0/13-32-0-master.xml b/backend/core/src/main/resources/config/liquibase/13-32-0/13-32-0-master.xml index 5bfcdce345..a4fdb708eb 100644 --- a/backend/core/src/main/resources/config/liquibase/13-32-0/13-32-0-master.xml +++ b/backend/core/src/main/resources/config/liquibase/13-32-0/13-32-0-master.xml @@ -23,5 +23,8 @@ xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liqui + + + diff --git a/backend/core/src/main/resources/config/liquibase/13-32-0/20260630-create-reindex-run-table.xml b/backend/core/src/main/resources/config/liquibase/13-32-0/20260630-create-reindex-run-table.xml new file mode 100644 index 0000000000..8da40cf174 --- /dev/null +++ b/backend/core/src/main/resources/config/liquibase/13-32-0/20260630-create-reindex-run-table.xml @@ -0,0 +1,58 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/backend/core/src/main/resources/config/liquibase/13-32-0/20260701-create-pending-index-deletion-table.xml b/backend/core/src/main/resources/config/liquibase/13-32-0/20260701-create-pending-index-deletion-table.xml new file mode 100644 index 0000000000..0c58c5cbdf --- /dev/null +++ b/backend/core/src/main/resources/config/liquibase/13-32-0/20260701-create-pending-index-deletion-table.xml @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/backend/core/src/main/resources/config/liquibase/13-32-0/20260701-create-reconcile-state-table.xml b/backend/core/src/main/resources/config/liquibase/13-32-0/20260701-create-reconcile-state-table.xml new file mode 100644 index 0000000000..53413ff39c --- /dev/null +++ b/backend/core/src/main/resources/config/liquibase/13-32-0/20260701-create-reconcile-state-table.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + + + diff --git a/backend/core/src/main/resources/config/liquibase/initial-setup/case/changelog/20260701-add-changed-on-to-json-schema-document.xml b/backend/core/src/main/resources/config/liquibase/initial-setup/case/changelog/20260701-add-changed-on-to-json-schema-document.xml new file mode 100644 index 0000000000..c24986d41c --- /dev/null +++ b/backend/core/src/main/resources/config/liquibase/initial-setup/case/changelog/20260701-add-changed-on-to-json-schema-document.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + changed_on IS NULL + + + + + + + + diff --git a/backend/core/src/main/resources/config/liquibase/initial-setup/initial-setup-master.xml b/backend/core/src/main/resources/config/liquibase/initial-setup/initial-setup-master.xml index b8c359c9c6..de178a80f5 100644 --- a/backend/core/src/main/resources/config/liquibase/initial-setup/initial-setup-master.xml +++ b/backend/core/src/main/resources/config/liquibase/initial-setup/initial-setup-master.xml @@ -343,5 +343,6 @@ + From 90001424cf0b178b987d725d595f77d3ee10605e Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Tue, 7 Jul 2026 14:39:50 +0200 Subject: [PATCH 20/46] - open search admin page - added generic case list global search support (still work in progress, currently does not respect access control) - lots of tiny fixes --- backend/apps/dev/docker-compose.yaml | 1 - .../src/main/resources/config/application.yml | 5 + .../DocumentOpenSearchAutoConfiguration.kt | 27 +++-- .../JsonSchemaDocumentOpenSearchService.kt | 44 +++++-- .../service/OpenSearchReindexRunService.kt | 61 +++++++--- .../OpenSearchReindexRunServiceTest.kt | 47 ++++++-- .../apps/dev/src/environments/environment.ts | 29 +++-- .../apps/gzac/src/environments/environment.ts | 28 ++++- .../src/lib/admin-settings-routing.ts | 31 +++-- .../admin-settings-opensearch.component.html | 105 ++++++++++++++++ .../admin-settings-opensearch.component.scss | 104 ++++++++++++++++ .../admin-settings-opensearch.component.ts | 114 ++++++++++++++++++ .../admin-settings.component.html | 29 +++-- .../lib/constants/feature-toggle.constants.ts | 25 ++-- .../admin-settings/src/lib/models/index.ts | 25 ++-- .../src/lib/models/reindex.model.ts | 38 ++++++ .../admin-settings-management-api.service.ts | 39 ++++-- .../valtimo/bootstrap/src/lib/init.ts | 42 +++++-- .../case-list/case-list.component.ts | 25 ++-- .../generic-case-list.component.html | 38 ++++-- .../generic-case-list.component.ts | 46 ++++--- .../carbon-list/carbon-list.component.html | 29 +++-- .../carbon-list/carbon-list.component.ts | 65 +++++++--- .../components/menu/services/menu.service.ts | 31 +++-- .../src/lib/services/document.service.ts | 79 ++++++++---- .../valtimo/shared/assets/core/en.json | 31 ++++- .../valtimo/shared/assets/core/nl.json | 31 ++++- .../valtimo/shared/src/lib/models/config.ts | 25 ++-- .../shared/src/lib/models/menu-item.model.ts | 25 ++-- .../src/lib/services/menu-include.service.ts | 26 ++-- 30 files changed, 967 insertions(+), 278 deletions(-) create mode 100644 frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.html create mode 100644 frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss create mode 100644 frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.ts create mode 100644 frontend/projects/valtimo/admin-settings/src/lib/models/reindex.model.ts diff --git a/backend/apps/dev/docker-compose.yaml b/backend/apps/dev/docker-compose.yaml index 5124f6bb94..498c7b09ce 100644 --- a/backend/apps/dev/docker-compose.yaml +++ b/backend/apps/dev/docker-compose.yaml @@ -87,7 +87,6 @@ services: image: opensearchproject/opensearch:2.19.2 ports: - "9200:9200" - - "9600:9600" environment: - discovery.type=single-node - DISABLE_SECURITY_PLUGIN=true diff --git a/backend/apps/dev/src/main/resources/config/application.yml b/backend/apps/dev/src/main/resources/config/application.yml index 4c5ab191c0..c6cdffd9e4 100644 --- a/backend/apps/dev/src/main/resources/config/application.yml +++ b/backend/apps/dev/src/main/resources/config/application.yml @@ -170,7 +170,12 @@ mailing: whitelistedDomains: sendRedirectedMailsTo: +opensearch: + uris: http://localhost:9200 + valtimo: + opensearch: + enabled: true app: scheme: http hostname: ${VALTIMO_APP_HOSTNAME} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt index db95897942..b8958462e7 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt @@ -1,17 +1,19 @@ /* - * Copyright 2015-2024 Ritense BV, the Netherlands. * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * * Copyright 2015-2026 Ritense BV, the Netherlands. + * * + * * Licensed under EUPL, Version 1.2 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" basis, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ package com.ritense.document.opensearch.autoconfigure @@ -148,8 +150,9 @@ class DocumentOpenSearchAutoConfiguration { openSearchReindexRunRepository: OpenSearchReindexRunRepository, objectMapper: ObjectMapper, openSearchProperties: OpenSearchProperties, + entityManager: EntityManager, ): OpenSearchReindexRunService = - OpenSearchReindexRunService(openSearchReindexRunRepository, objectMapper, openSearchProperties) + OpenSearchReindexRunService(openSearchReindexRunRepository, objectMapper, openSearchProperties, entityManager) @Bean @ConditionalOnMissingBean diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt index dc8f8ba23c..8f5af4f7ac 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt @@ -1,17 +1,19 @@ /* - * Copyright 2015-2024 Ritense BV, the Netherlands. * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * * Copyright 2015-2026 Ritense BV, the Netherlands. + * * + * * Licensed under EUPL, Version 1.2 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" basis, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ package com.ritense.document.opensearch.service @@ -62,7 +64,9 @@ import java.time.Instant import java.time.LocalDate import java.time.ZoneOffset import java.time.format.DateTimeFormatter +import org.springframework.transaction.annotation.Transactional +@Transactional class JsonSchemaDocumentOpenSearchService( private val elasticsearchOperations: ElasticsearchOperations, private val translator: OpenSearchPermissionConditionTranslator, @@ -93,8 +97,22 @@ class JsonSchemaDocumentOpenSearchService( if (searchRequest.sequence != null) { parts.add(QueryBuilders.termQuery("sequence", searchRequest.sequence)) } - if (!searchRequest.globalSearchFilter.isNullOrEmpty()) { - throw NotImplementedException("globalSearchFilter is not supported in the simple search — use the advanced search overload") + val globalFilter = searchRequest.globalSearchFilter?.takeIf { it.isNotEmpty() } + if (globalFilter != null) { + val searchFields = if (!searchRequest.documentDefinitionName.isNullOrEmpty()) { + runWithoutAuthorization { + searchFieldService.getSearchFields(searchRequest.documentDefinitionName) + } + } else { + emptyList() + } + + if (searchFields.isNotEmpty()) { + parts.add(buildGlobalSearchQuery(globalFilter.trim(), searchFields)) + } else { + val term = "*${globalFilter.trim()}*" + parts.add(QueryBuilders.wildcardQuery("contentText.keyword", term).caseInsensitive(true)) + } } searchRequest.otherFilters?.forEach { sc -> parts.add(QueryBuilders.termQuery("content.${sc.path}", sc.value)) diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunService.kt index 1bf6c7d85e..fa9ab41b61 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunService.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunService.kt @@ -1,27 +1,32 @@ /* - * Copyright 2015-2026 Ritense BV, the Netherlands. * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * * Copyright 2015-2026 Ritense BV, the Netherlands. + * * + * * Licensed under EUPL, Version 1.2 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" basis, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ package com.ritense.document.opensearch.service import com.fasterxml.jackson.databind.ObjectMapper import com.ritense.document.opensearch.OpenSearchProperties +import com.ritense.document.domain.impl.JsonSchemaDocument import com.ritense.document.opensearch.domain.OpenSearchReindexRun import com.ritense.document.opensearch.domain.ReindexRunStatus import com.ritense.document.opensearch.repository.OpenSearchReindexRunRepository import io.github.oshai.kotlinlogging.KotlinLogging +import jakarta.persistence.EntityManager +import jakarta.persistence.criteria.Predicate import org.springframework.boot.context.event.ApplicationReadyEvent import org.springframework.context.event.EventListener import org.springframework.transaction.annotation.Transactional @@ -41,6 +46,7 @@ open class OpenSearchReindexRunService( private val repository: OpenSearchReindexRunRepository, private val objectMapper: ObjectMapper, private val properties: OpenSearchProperties, + private val entityManager: EntityManager, ) { /** @@ -144,15 +150,17 @@ open class OpenSearchReindexRunService( private fun toMap(run: OpenSearchReindexRun): Map { val elapsedSeconds = Duration.between(run.startedOn, run.finishedOn ?: LocalDateTime.now()).seconds + val scope = deserializeScopeToRequest(run.scope) return mapOf( "runId" to run.id, "status" to run.status, "running" to (run.status == ReindexRunStatus.RUNNING), - "scope" to deserializeScope(run.scope), + "scope" to scope?.let { objectMapper.convertValue(it, Map::class.java) }, "pageSize" to run.pageSize, "lastId" to run.lastId, "processedCount" to run.processedCount, "skippedCount" to run.skippedCount, + "totalCount" to countDocuments(scope), "startedOn" to run.startedOn, "heartbeatOn" to run.heartbeatOn, "finishedOn" to run.finishedOn, @@ -169,15 +177,38 @@ open class OpenSearchReindexRunService( null } - private fun deserializeScope(scope: String?): Any? = + private fun deserializeScopeToRequest(scope: String?): ReindexRequest? = scope?.let { try { - objectMapper.readValue(it, Map::class.java) + objectMapper.readValue(it, ReindexRequest::class.java) } catch (e: Exception) { - it + logger.warn(e) { "Failed to deserialize scope to ReindexRequest" } + null } } + private fun countDocuments(scope: ReindexRequest?): Long { + val cb = entityManager.criteriaBuilder + val query = cb.createQuery(Long::class.java) + val root = query.from(JsonSchemaDocument::class.java) + query.select(cb.count(root)) + + val predicates = mutableListOf() + scope?.modifiedAfter?.let { predicates += cb.greaterThan(root.get("modifiedOn"), it) } + scope?.modifiedBefore?.let { predicates += cb.lessThan(root.get("modifiedOn"), it) } + scope?.documentDefinitionName?.let { + predicates += cb.equal(root.get("documentDefinitionId").get("name"), it) + } + scope?.documentIds?.takeIf { it.isNotEmpty() }?.let { + predicates += root.get("id").get("id").`in`(it) + } + + if (predicates.isNotEmpty()) { + query.where(*predicates.toTypedArray()) + } + return entityManager.createQuery(query).singleResult + } + companion object { private val logger = KotlinLogging.logger {} } diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunServiceTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunServiceTest.kt index efa1e0fdde..24ae81a45b 100644 --- a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunServiceTest.kt +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunServiceTest.kt @@ -1,17 +1,19 @@ /* - * Copyright 2015-2026 Ritense BV, the Netherlands. * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * * Copyright 2015-2026 Ritense BV, the Netherlands. + * * + * * Licensed under EUPL, Version 1.2 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" basis, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ package com.ritense.document.opensearch.service @@ -21,6 +23,11 @@ import com.ritense.document.opensearch.OpenSearchProperties import com.ritense.document.opensearch.domain.OpenSearchReindexRun import com.ritense.document.opensearch.domain.ReindexRunStatus import com.ritense.document.opensearch.repository.OpenSearchReindexRunRepository +import jakarta.persistence.EntityManager +import jakarta.persistence.TypedQuery +import jakarta.persistence.criteria.CriteriaBuilder +import jakarta.persistence.criteria.CriteriaQuery +import jakarta.persistence.criteria.Root import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -40,12 +47,28 @@ class OpenSearchReindexRunServiceTest { private val repository: OpenSearchReindexRunRepository = mock() private val objectMapper: ObjectMapper = ObjectMapper().findAndRegisterModules() private val properties = OpenSearchProperties() + private val entityManager: EntityManager = mock() private lateinit var service: OpenSearchReindexRunService @BeforeEach fun setUp() { - service = OpenSearchReindexRunService(repository, objectMapper, properties) + service = OpenSearchReindexRunService(repository, objectMapper, properties, entityManager) whenever(repository.save(any())).doAnswer { it.arguments[0] as OpenSearchReindexRun } + setupEntityManagerMock() + } + + private fun setupEntityManagerMock() { + val criteriaBuilder: CriteriaBuilder = mock() + val criteriaQuery: CriteriaQuery = mock() + val root: Root<*> = mock() + val typedQuery: TypedQuery = mock() + + whenever(entityManager.criteriaBuilder).thenReturn(criteriaBuilder) + whenever(criteriaBuilder.createQuery(Long::class.java)).thenReturn(criteriaQuery) + whenever(criteriaQuery.from(any>())).thenReturn(root as Root) + whenever(criteriaQuery.select(any())).thenReturn(criteriaQuery) + whenever(entityManager.createQuery(criteriaQuery)).thenReturn(typedQuery) + whenever(typedQuery.singleResult).thenReturn(100L) } @Test diff --git a/frontend/apps/dev/src/environments/environment.ts b/frontend/apps/dev/src/environments/environment.ts index cbfdfe2d62..f01bcffa43 100644 --- a/frontend/apps/dev/src/environments/environment.ts +++ b/frontend/apps/dev/src/environments/environment.ts @@ -1,17 +1,19 @@ /* - * Copyright 2015-2026 Ritense BV, the Netherlands. * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * * Copyright 2015-2026 Ritense BV, the Netherlands. + * * + * * Licensed under EUPL, Version 1.2 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" basis, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ import { @@ -136,6 +138,11 @@ export const environment: ValtimoConfig = { {link: ['/decision-tables'], title: 'Decision tables'}, {title: 'Other', textClass: 'text-dark font-weight-bold c-default'}, {link: ['/logging'], title: 'Logs'}, + { + link: ['/opensearch'], + title: 'adminSettings.opensearch.title', + includeFunction: IncludeFunction.OpenSearchEnabled, + }, {link: ['/case-migration'], title: 'Case migration (beta)'}, {link: ['/process-migration'], title: 'Process migration'}, {link: ['/task-management'], title: 'Tasks (legacy)'}, diff --git a/frontend/apps/gzac/src/environments/environment.ts b/frontend/apps/gzac/src/environments/environment.ts index 4dab09e87d..30ef5b64e4 100644 --- a/frontend/apps/gzac/src/environments/environment.ts +++ b/frontend/apps/gzac/src/environments/environment.ts @@ -1,3 +1,21 @@ +/* + * + * * Copyright 2015-2026 Ritense BV, the Netherlands. + * * + * * Licensed under EUPL, Version 1.2 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" basis, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + // This file can be replaced during build by using the `fileReplacements` array. // `ng build --prod` replaces `environment.ts` with `environment.prod.ts`. // The list of file replacements can be found in `angular.json`. @@ -108,8 +126,14 @@ export const environment: ValtimoConfig = { {title: 'Other', textClass: 'text-dark font-weight-bold c-default', sequence: 16}, {link: ['/logging'], title: 'Logs', sequence: 17}, - {link: ['/case-migration'], title: 'Case migration (beta)', sequence: 18}, - {link: ['/process-migration'], title: 'Process migration', sequence: 19}, + { + link: ['/opensearch'], + title: 'adminSettings.opensearch.title', + sequence: 18, + includeFunction: IncludeFunction.OpenSearchEnabled, + }, + {link: ['/case-migration'], title: 'Case migration (beta)', sequence: 19}, + {link: ['/process-migration'], title: 'Process migration', sequence: 20}, ], }, { diff --git a/frontend/projects/valtimo/admin-settings/src/lib/admin-settings-routing.ts b/frontend/projects/valtimo/admin-settings/src/lib/admin-settings-routing.ts index 5ca6936f0c..e019390963 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/admin-settings-routing.ts +++ b/frontend/projects/valtimo/admin-settings/src/lib/admin-settings-routing.ts @@ -1,17 +1,19 @@ /* - * Copyright 2015-2026 Ritense BV, the Netherlands. * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * * Copyright 2015-2026 Ritense BV, the Netherlands. + * * + * * Licensed under EUPL, Version 1.2 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" basis, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ import {NgModule} from '@angular/core'; @@ -20,6 +22,7 @@ import {CommonModule} from '@angular/common'; import {AuthGuardService} from '@valtimo/security'; import {ROLE_ADMIN} from '@valtimo/shared'; import {AdminSettingsComponent} from './components/admin-settings/admin-settings.component'; +import {AdminSettingsOpensearchComponent} from './components/admin-settings-opensearch/admin-settings-opensearch.component'; import {ADMIN_SETTINGS_TABS} from './constants'; const routes: Routes = [ @@ -34,6 +37,12 @@ const routes: Routes = [ canActivate: [AuthGuardService], data: {title: 'adminSettings.title', roles: [ROLE_ADMIN]}, }, + { + path: 'opensearch', + component: AdminSettingsOpensearchComponent, + canActivate: [AuthGuardService], + data: {title: 'adminSettings.opensearch.title', roles: [ROLE_ADMIN]}, + }, ]; @NgModule({ diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.html b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.html new file mode 100644 index 0000000000..94fcb8f2d2 --- /dev/null +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.html @@ -0,0 +1,105 @@ + + +
+
+

+ {{ 'adminSettings.opensearch.reindex.title' | translate }} +

+

+ {{ 'adminSettings.opensearch.reindex.description' | translate }} +

+
+ + @if (obs.status?.status) { +
+
+ +
+ +
+ @if (obs.status.skippedCount > 0) { +
+ + {{ 'adminSettings.opensearch.reindex.skipped' | translate }}: + + {{ obs.status.skippedCount }} +
+ } + @if (obs.status.startedOn) { +
+ + {{ 'adminSettings.opensearch.reindex.started' | translate }}: + + {{ obs.status.startedOn | date:'medium' }} +
+ } + @if (obs.status.status === 'RUNNING' && obs.status.elapsedSeconds > 0) { +
+ + {{ 'adminSettings.opensearch.reindex.elapsed' | translate }}: + + {{ obs.status.elapsedSeconds }}s +
+ } + @if (obs.status.status !== 'RUNNING' && obs.status.finishedOn) { +
+ + {{ 'adminSettings.opensearch.reindex.finished' | translate }}: + + {{ obs.status.finishedOn | date:'medium' }} +
+ } +
+ + @if (obs.status.error) { +
+ + {{ 'adminSettings.opensearch.reindex.errorTitle' | translate }}: + + {{ obs.status.error }} +
+ } +
+ } + +
+ +
+
diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss new file mode 100644 index 0000000000..9998757540 --- /dev/null +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss @@ -0,0 +1,104 @@ +/*! + * + * * Copyright 2015-2026 Ritense BV, the Netherlands. + * * + * * Licensed under EUPL, Version 1.2 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" basis, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +.opensearch-reindex { + width: 100%; + display: flex; + flex-direction: column; + gap: 24px; + + &__header { + display: flex; + flex-direction: column; + gap: 8px; + } + + &__title { + font-size: 16px; + font-weight: 600; + color: var(--cds-text-primary); + margin: 0; + } + + &__description { + font-size: 14px; + color: var(--cds-text-secondary); + margin: 0; + } + + &__status { + display: flex; + flex-direction: column; + gap: 16px; + padding: 16px; + background-color: var(--cds-layer-01); + border-radius: 4px; + } + + &__status-row { + display: flex; + align-items: center; + gap: 8px; + } + + &__label { + font-size: 14px; + font-weight: 600; + color: var(--cds-text-primary); + } + + &__progress { + padding: 8px 0; + } + + &__stats { + display: flex; + flex-wrap: wrap; + gap: 24px; + } + + &__stat { + display: flex; + align-items: center; + gap: 8px; + font-size: 14px; + color: var(--cds-text-primary); + } + + &__error { + padding: 12px; + background-color: var(--cds-support-error); + color: var(--cds-text-on-color); + border-radius: 4px; + font-size: 14px; + + &-title { + font-weight: 600; + margin-right: 8px; + } + } + + &__actions { + display: flex; + gap: 12px; + + button { + min-width: 140px; + } + } +} diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.ts b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.ts new file mode 100644 index 0000000000..8982df03e8 --- /dev/null +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.ts @@ -0,0 +1,114 @@ +/* + * + * * Copyright 2015-2026 Ritense BV, the Netherlands. + * * + * * Licensed under EUPL, Version 1.2 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" basis, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +import {ChangeDetectionStrategy, Component, OnDestroy} from '@angular/core'; +import {CommonModule, DatePipe} from '@angular/common'; +import {TranslateModule} from '@ngx-translate/core'; +import { + BehaviorSubject, + finalize, + interval, + map, + Observable, + of, + shareReplay, + startWith, + Subject, + switchMap, + take, + takeUntil, + takeWhile, +} from 'rxjs'; +import {ButtonModule, LoadingModule, ProgressBarModule, TagModule} from 'carbon-components-angular'; +import {AdminSettingsManagementApiService} from '../../services'; +import {ReindexStatusDto} from '../../models'; + +@Component({ + standalone: true, + selector: 'valtimo-admin-settings-opensearch', + templateUrl: './admin-settings-opensearch.component.html', + styleUrls: ['./admin-settings-opensearch.component.scss'], + imports: [CommonModule, DatePipe, TranslateModule, ButtonModule, LoadingModule, ProgressBarModule, TagModule], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class AdminSettingsOpensearchComponent implements OnDestroy { + private readonly _destroy$ = new Subject(); + private readonly _refresh$ = new BehaviorSubject(undefined); + + public readonly startingReindex$ = new BehaviorSubject(false); + + public readonly reindexStatus$: Observable = this._refresh$.pipe( + switchMap(() => this._apiService.getReindexStatus()), + switchMap(status => { + if (status?.status === 'RUNNING') { + return interval(2000).pipe( + startWith(0), + switchMap(() => this._apiService.getReindexStatus()), + takeWhile(s => s?.status === 'RUNNING', true), + takeUntil(this._destroy$) + ); + } + return of(status); + }), + shareReplay(1) + ); + + public readonly isRunning$: Observable = this.reindexStatus$.pipe( + map(status => status?.status === 'RUNNING') + ); + + constructor(private readonly _apiService: AdminSettingsManagementApiService) {} + + public startReindex(): void { + this.startingReindex$.next(true); + this._apiService + .startReindex() + .pipe( + take(1), + finalize(() => this.startingReindex$.next(false)) + ) + .subscribe({ + next: () => this._refresh$.next(), + error: err => { + if (err.status === 409) { + this._refresh$.next(); + } + }, + }); + } + + public getStatusTagType(status: string): string { + switch (status) { + case 'RUNNING': + return 'blue'; + case 'COMPLETED': + return 'green'; + case 'FAILED': + return 'red'; + case 'STOPPED': + return 'gray'; + default: + return 'gray'; + } + } + + public ngOnDestroy(): void { + this._destroy$.next(); + this._destroy$.complete(); + } +} diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings/admin-settings.component.html b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings/admin-settings.component.html index 4caeb3d201..3988fabadf 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings/admin-settings.component.html +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings/admin-settings.component.html @@ -1,17 +1,19 @@ @if (activeTabKey$ | async; as activeTabKey) { @@ -39,5 +41,6 @@ } } + } diff --git a/frontend/projects/valtimo/admin-settings/src/lib/constants/feature-toggle.constants.ts b/frontend/projects/valtimo/admin-settings/src/lib/constants/feature-toggle.constants.ts index f48853271e..3a2b0af449 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/constants/feature-toggle.constants.ts +++ b/frontend/projects/valtimo/admin-settings/src/lib/constants/feature-toggle.constants.ts @@ -1,17 +1,19 @@ /* - * Copyright 2015-2026 Ritense BV, the Netherlands. * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * * Copyright 2015-2026 Ritense BV, the Netherlands. + * * + * * Licensed under EUPL, Version 1.2 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" basis, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ import {FeatureToggleDefinition} from '../models'; @@ -39,6 +41,7 @@ const FEATURE_TOGGLE_DEFINITIONS: FeatureToggleDefinition[] = [ {key: 'enableSuppressDocumentError'}, {key: 'enableIkoType'}, {key: 'menuCollapsedByDefault'}, + {key: 'enableGenericCaseList'}, ]; export {FEATURE_TOGGLE_DEFINITIONS}; diff --git a/frontend/projects/valtimo/admin-settings/src/lib/models/index.ts b/frontend/projects/valtimo/admin-settings/src/lib/models/index.ts index f6007b767f..d70ca73577 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/models/index.ts +++ b/frontend/projects/valtimo/admin-settings/src/lib/models/index.ts @@ -1,19 +1,22 @@ /* - * Copyright 2015-2026 Ritense BV, the Netherlands. * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * * Copyright 2015-2026 Ritense BV, the Netherlands. + * * + * * Licensed under EUPL, Version 1.2 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" basis, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ export * from './accent-colors.model'; export * from './feature-toggle.model'; +export * from './reindex.model'; export * from './search-engine.model'; diff --git a/frontend/projects/valtimo/admin-settings/src/lib/models/reindex.model.ts b/frontend/projects/valtimo/admin-settings/src/lib/models/reindex.model.ts new file mode 100644 index 0000000000..2164c08b63 --- /dev/null +++ b/frontend/projects/valtimo/admin-settings/src/lib/models/reindex.model.ts @@ -0,0 +1,38 @@ +/* + * + * * Copyright 2015-2026 Ritense BV, the Netherlands. + * * + * * Licensed under EUPL, Version 1.2 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" basis, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +type ReindexStatus = 'RUNNING' | 'COMPLETED' | 'FAILED' | 'STOPPED'; + +interface ReindexStatusDto { + runId: string; + status: ReindexStatus; + totalCount: number; + processedCount: number; + skippedCount: number; + startedOn: string; + finishedOn: string | null; + elapsedSeconds: number; + error: string | null; +} + +interface StartReindexResponseDto { + status: string; + runId: string; +} + +export {ReindexStatus, ReindexStatusDto, StartReindexResponseDto}; diff --git a/frontend/projects/valtimo/admin-settings/src/lib/services/admin-settings-management-api.service.ts b/frontend/projects/valtimo/admin-settings/src/lib/services/admin-settings-management-api.service.ts index 0fd7f3896f..61746871f8 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/services/admin-settings-management-api.service.ts +++ b/frontend/projects/valtimo/admin-settings/src/lib/services/admin-settings-management-api.service.ts @@ -1,17 +1,19 @@ /* - * Copyright 2015-2026 Ritense BV, the Netherlands. * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * * Copyright 2015-2026 Ritense BV, the Netherlands. + * * + * * Licensed under EUPL, Version 1.2 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" basis, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ import {Injectable} from '@angular/core'; @@ -27,7 +29,9 @@ import { import { AccentColorsDto, FeatureToggleOverridesDto, + ReindexStatusDto, SearchEngineDto, + StartReindexResponseDto, UpdateFeatureToggleDto, } from '../models'; @@ -108,4 +112,17 @@ export class AdminSettingsManagementApiService extends BaseApiService { {active: useOpenSearch ? 'OPENSEARCH' : 'POSTGRES'} ); } + + public startReindex(): Observable { + return this.httpClient.post( + this.getApiUrl('/management/v1/document-opensearch/reindex'), + {} + ); + } + + public getReindexStatus(): Observable { + return this.httpClient + .get(this.getApiUrl('/management/v1/document-opensearch/reindex/status')) + .pipe(catchError(() => of(null))); + } } diff --git a/frontend/projects/valtimo/bootstrap/src/lib/init.ts b/frontend/projects/valtimo/bootstrap/src/lib/init.ts index 696ecce0fd..a495aeddf6 100644 --- a/frontend/projects/valtimo/bootstrap/src/lib/init.ts +++ b/frontend/projects/valtimo/bootstrap/src/lib/init.ts @@ -1,23 +1,26 @@ /* - * Copyright 2015-2025 Ritense BV, the Netherlands. * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * * Copyright 2015-2026 Ritense BV, the Netherlands. + * * + * * Licensed under EUPL, Version 1.2 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" basis, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ import {NGXLogger} from 'ngx-logger'; import {TranslateService} from '@ngx-translate/core'; import {accountInitializer} from '@valtimo/account'; import {Injector} from '@angular/core'; +import {HttpClient} from '@angular/common/http'; import {ConfigService} from '@valtimo/shared'; import {AdminSettingsService, menuInitializer} from '@valtimo/components'; import {firstValueFrom} from 'rxjs'; @@ -88,6 +91,23 @@ export function initializerFactory( } }); + // Check OpenSearch availability and patch feature toggle + initializersArray.push(async () => { + try { + const httpClient = injector.get(HttpClient); + const response = await firstValueFrom( + httpClient.get<{available: boolean}>( + `${configService.config.valtimoApi.endpointUri}management/v1/search-engine` + ) + ); + if (response?.available) { + configService.patchFeatureToggles({enableOpenSearch: true}); + } + } catch { + // OpenSearch not available + } + }); + // Use environment config initializers to be used in app startup. configService.initializers.forEach(initializer => { initializersArray.push(initializer(injector)); diff --git a/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.ts b/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.ts index 64967475b9..c1815566c9 100644 --- a/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.ts +++ b/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.ts @@ -1,17 +1,19 @@ /* - * Copyright 2015-2025 Ritense BV, the Netherlands. * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * * Copyright 2015-2026 Ritense BV, the Netherlands. + * * + * * Licensed under EUPL, Version 1.2 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" basis, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ import {Component, inject, Inject, OnDestroy, OnInit, ViewChild} from '@angular/core'; import {ActivatedRoute, Params, Router} from '@angular/router'; @@ -302,6 +304,7 @@ export class CaseListComponent implements OnInit, OnDestroy { this.parameterService.setSearchFieldValues( this.parameterService.getSearchObject(queryParams['search']) as SearchFieldValues ); + this.searchService.setGlobalSearchFilter(null); }); } diff --git a/frontend/projects/valtimo/case/src/lib/components/generic-case-list/generic-case-list.component.html b/frontend/projects/valtimo/case/src/lib/components/generic-case-list/generic-case-list.component.html index e83b04ca29..f35341b6c5 100644 --- a/frontend/projects/valtimo/case/src/lib/components/generic-case-list/generic-case-list.component.html +++ b/frontend/projects/valtimo/case/src/lib/components/generic-case-list/generic-case-list.component.html @@ -1,17 +1,19 @@ @@ -20,6 +22,9 @@ diff --git a/frontend/projects/valtimo/case/src/lib/components/generic-case-list/generic-case-list.component.ts b/frontend/projects/valtimo/case/src/lib/components/generic-case-list/generic-case-list.component.ts index 422e300b49..1c50771bbf 100644 --- a/frontend/projects/valtimo/case/src/lib/components/generic-case-list/generic-case-list.component.ts +++ b/frontend/projects/valtimo/case/src/lib/components/generic-case-list/generic-case-list.component.ts @@ -1,17 +1,19 @@ /* - * Copyright 2015-2026 Ritense BV, the Netherlands. * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * * Copyright 2015-2026 Ritense BV, the Netherlands. + * * + * * Licensed under EUPL, Version 1.2 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" basis, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ import {Component, inject, Inject, OnDestroy, OnInit, ViewChild} from '@angular/core'; import {Router} from '@angular/router'; @@ -150,7 +152,10 @@ export class GenericCaseListComponent implements OnInit, OnDestroy { private readonly _allCasesSort$ = new BehaviorSubject(null); private readonly _allCasesReload$ = new BehaviorSubject(false); private readonly _allCasesAssigneeFilter$ = new BehaviorSubject('ALL'); + private readonly _allCasesGlobalSearch$ = new BehaviorSubject(''); public readonly allCasesAssigneeFilter$ = this._allCasesAssigneeFilter$.asObservable(); + public readonly allCasesGlobalSearch$ = this._allCasesGlobalSearch$.asObservable(); + public readonly allCasesInvalidSearchFields$ = this.documentService.invalidSearchFields$; public readonly allCasesFields$: Observable = this.translateService .stream('fieldLabels') @@ -170,16 +175,17 @@ export class GenericCaseListComponent implements OnInit, OnDestroy { this._allCasesSort$, this._allCasesReload$, this._allCasesAssigneeFilter$, + this._allCasesGlobalSearch$, ]).pipe( tap(() => this.allCasesLoading$.next(true)), - switchMap(([page, size, sort, _, assigneeFilter]) => { + switchMap(([page, size, sort, _, assigneeFilter, globalSearch]) => { const request = new DocumentSearchRequestImpl( '', page - 1, size, undefined, undefined, - undefined, + globalSearch || undefined, sort, undefined, assigneeFilter !== 'ALL' ? assigneeFilter : undefined @@ -220,7 +226,7 @@ export class GenericCaseListComponent implements OnInit, OnDestroy { private readonly parameterService: CaseParameterService, private readonly quickSearchStateService: QuickSearchStateService, private readonly router: Router, - private readonly searchService: CaseListSearchService, + public readonly searchService: CaseListSearchService, private readonly statusService: CaseListStatusService, @Inject(QUICK_SEARCH_SERVICE) private readonly caseListQuickSearchService: IQuickSearchService, @@ -260,12 +266,14 @@ export class GenericCaseListComponent implements OnInit, OnDestroy { this.parameterService.clearSearchFieldValues(); this.paginationService.clearPagination(); this.assigneeService.resetAssigneeFilter(); + this.searchService.setGlobalSearchFilter(null); this.listService.setCaseDefinitionKey(newId); this.orchestration.setLoading(); this.subscribeToPagination(); this.subscribeToCanHaveAssignee(); this.subscribeToSearchFields(); } else { + this._allCasesGlobalSearch$.next(''); this._allCasesPage$.next(1); this._allCasesReload$.next(!this._allCasesReload$.getValue()); } @@ -412,6 +420,7 @@ export class GenericCaseListComponent implements OnInit, OnDestroy { this.parameterService.setSearchFieldValues( this.parameterService.getSearchObject(queryParams['search']) as SearchFieldValues ); + this.searchService.setGlobalSearchFilter(null); }); } @@ -440,6 +449,15 @@ export class GenericCaseListComponent implements OnInit, OnDestroy { this._allCasesPage$.next(1); } + public allCasesSearch(searchTerm: string): void { + this._allCasesGlobalSearch$.next(searchTerm); + this._allCasesPage$.next(1); + } + + public onGlobalSearchFilterChange(value: string): void { + this.searchService.setGlobalSearchFilter(value); + } + // --- Private --- private subscribeToPagination(): void { diff --git a/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.html b/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.html index e3d5005bb8..0b3256c274 100644 --- a/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.html +++ b/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.html @@ -1,17 +1,19 @@ 0; - this.selectedSuggestionIndex = 0; + this.selectedSuggestionIndex = -1; if (this.showAutocomplete) { this.autocompleteLeft = this.calculateTokenLeft(tokenInfo.start); @@ -913,17 +926,25 @@ export class CarbonListComponent implements OnInit, AfterViewInit, OnDestroy { case 'ArrowDown': event.preventDefault(); this.selectedSuggestionIndex = - (this.selectedSuggestionIndex + 1) % this.filteredSuggestions.length; + this.selectedSuggestionIndex < 0 + ? 0 + : (this.selectedSuggestionIndex + 1) % this.filteredSuggestions.length; break; case 'ArrowUp': event.preventDefault(); this.selectedSuggestionIndex = - (this.selectedSuggestionIndex - 1 + this.filteredSuggestions.length) % - this.filteredSuggestions.length; + this.selectedSuggestionIndex < 0 + ? this.filteredSuggestions.length - 1 + : (this.selectedSuggestionIndex - 1 + this.filteredSuggestions.length) % + this.filteredSuggestions.length; break; case 'Tab': event.preventDefault(); - this.selectSuggestion(this.filteredSuggestions[this.selectedSuggestionIndex]); + if (this.selectedSuggestionIndex >= 0) { + this.selectSuggestion(this.filteredSuggestions[this.selectedSuggestionIndex]); + } else { + this.showAutocomplete = false; + } break; case 'Escape': this.showAutocomplete = false; @@ -938,12 +959,17 @@ export class CarbonListComponent implements OnInit, AfterViewInit, OnDestroy { }, 150); } - + public onSearchClear(): void { this.showAutocomplete = false; } private executeSearch(searchString: string | null): void { + if (searchString === this._lastExecutedSearch) { + return; + } + this._lastExecutedSearch = searchString; + if (this.search.observed) { this.search.emit(searchString); return; @@ -960,9 +986,10 @@ export class CarbonListComponent implements OnInit, AfterViewInit, OnDestroy { } public onSearchEnter(): void { - if (this.showAutocomplete && this.filteredSuggestions.length > 0) { + if (this.showAutocomplete && this.selectedSuggestionIndex >= 0) { this.selectSuggestion(this.filteredSuggestions[this.selectedSuggestionIndex]); } else { + this.showAutocomplete = false; this.executeSearch(this.searchFormControl.value); } } diff --git a/frontend/projects/valtimo/components/src/lib/components/menu/services/menu.service.ts b/frontend/projects/valtimo/components/src/lib/components/menu/services/menu.service.ts index 2ed495338c..1b3a69866e 100644 --- a/frontend/projects/valtimo/components/src/lib/components/menu/services/menu.service.ts +++ b/frontend/projects/valtimo/components/src/lib/components/menu/services/menu.service.ts @@ -1,17 +1,19 @@ /* - * Copyright 2015-2025 Ritense BV, the Netherlands. * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * * Copyright 2015-2026 Ritense BV, the Netherlands. + * * + * * Licensed under EUPL, Version 1.2 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" basis, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ import {HttpClient} from '@angular/common/http'; @@ -153,6 +155,13 @@ export class MenuService implements OnDestroy { this.menuIncludeService.getIncludeFunction(menuItem.includeFunction); } + menuItem.children?.forEach(child => { + if (child.includeFunction !== undefined) { + this.includeFunctionObservables[child.title] = + this.menuIncludeService.getIncludeFunction(child.includeFunction); + } + }); + menuItem.show = true; if (!menuItem.roles || menuItem.roles.some(role => userRoles.includes(role))) { diff --git a/frontend/projects/valtimo/document/src/lib/services/document.service.ts b/frontend/projects/valtimo/document/src/lib/services/document.service.ts index 983a7667f1..d745a4baf8 100644 --- a/frontend/projects/valtimo/document/src/lib/services/document.service.ts +++ b/frontend/projects/valtimo/document/src/lib/services/document.service.ts @@ -1,17 +1,19 @@ /* - * Copyright 2015-2025 Ritense BV, the Netherlands. * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * * Copyright 2015-2026 Ritense BV, the Netherlands. + * * + * * Licensed under EUPL, Version 1.2 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" basis, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ import {HttpClient, HttpErrorResponse, HttpHeaders, HttpParams} from '@angular/common/http'; import {Injectable} from '@angular/core'; @@ -100,9 +102,13 @@ export class DocumentService { private extractInvalidSearchFields(error: HttpErrorResponse): string[] { const message = error?.error?.detail || error?.error?.message || error?.error || ''; - const match = message.match(/Unknown search field\(s\): (.+)/); - if (match) { - return match[1].split(', ').map((f: string) => f.trim()); + const pluralMatch = message.match(/Unknown search field\(s\): (.+)/); + if (pluralMatch) { + return pluralMatch[1].split(', ').map((f: string) => f.trim()); + } + const singularMatch = message.match(/Unknown search field: (.+)/); + if (singularMatch) { + return [singularMatch[1].trim()]; } return []; } @@ -151,13 +157,22 @@ export class DocumentService { } public getDocuments(documentSearchRequest: DocumentSearchRequest): Observable { - return this.http.post( - `${this.valtimoEndpointUri}v1/document-search`, - documentSearchRequest.asHttpBody(), - { + return this.http + .post(`${this.valtimoEndpointUri}v1/document-search`, documentSearchRequest.asHttpBody(), { params: documentSearchRequest.asHttpParams(), - } - ); + headers: new HttpHeaders().set(InterceptorSkip, '500'), + }) + .pipe( + tap(() => this._invalidSearchFields$.next([])), + catchError((error: HttpErrorResponse) => { + const invalidFields = this.extractInvalidSearchFields(error); + if (invalidFields.length > 0) { + this._invalidSearchFields$.next(invalidFields); + return of(this.EMPTY_DOCUMENTS_RESPONSE as Documents); + } + throw error; + }) + ); } public getDocumentsSearch( @@ -183,14 +198,20 @@ export class DocumentService { .post( `${this.valtimoEndpointUri}v1/document-definition/${documentSearchRequest.definitionName}/search`, body, - {params: documentSearchRequest.asHttpParams()} + { + params: documentSearchRequest.asHttpParams(), + headers: new HttpHeaders().set(InterceptorSkip, '500'), + } ) .pipe( tap(() => this._invalidSearchFields$.next([])), catchError((error: HttpErrorResponse) => { const invalidFields = this.extractInvalidSearchFields(error); - this._invalidSearchFields$.next(invalidFields); - return of(this.EMPTY_DOCUMENTS_RESPONSE as Documents); + if (invalidFields.length > 0) { + this._invalidSearchFields$.next(invalidFields); + return of(this.EMPTY_DOCUMENTS_RESPONSE as Documents); + } + throw error; }) ); } @@ -218,14 +239,20 @@ export class DocumentService { .post( `${this.valtimoEndpointUri}v1/case/${documentSearchRequest.definitionName}/search`, body, - {params: documentSearchRequest.asHttpParams()} + { + params: documentSearchRequest.asHttpParams(), + headers: new HttpHeaders().set(InterceptorSkip, '500'), + } ) .pipe( tap(() => this._invalidSearchFields$.next([])), catchError((error: HttpErrorResponse) => { const invalidFields = this.extractInvalidSearchFields(error); - this._invalidSearchFields$.next(invalidFields); - return of(this.EMPTY_DOCUMENTS_RESPONSE as SpecifiedDocuments); + if (invalidFields.length > 0) { + this._invalidSearchFields$.next(invalidFields); + return of(this.EMPTY_DOCUMENTS_RESPONSE as SpecifiedDocuments); + } + throw error; }) ); } diff --git a/frontend/projects/valtimo/shared/assets/core/en.json b/frontend/projects/valtimo/shared/assets/core/en.json index 115c5dc14e..337773c8b4 100644 --- a/frontend/projects/valtimo/shared/assets/core/en.json +++ b/frontend/projects/valtimo/shared/assets/core/en.json @@ -3064,7 +3064,8 @@ "title": "Settings", "tabs": { "appearance": "Appearance", - "featureToggles": "Feature toggles" + "featureToggles": "Feature toggles", + "opensearch": "OpenSearch" }, "appearance": { "logo": { @@ -3104,7 +3105,7 @@ "useOpenSearch": { "title": "Use OpenSearch for case search", "description": "When enabled, case searches use OpenSearch for better performance. When disabled, PostgreSQL is used.", - "unavailable": "OpenSearch is not configured for this installation." + "unavailable": "OpenSearch is not configured for this application." } }, "refreshRequired": "Refresh required", @@ -3198,6 +3199,32 @@ "menuCollapsedByDefault": { "title": "Menu collapsed by default", "description": "Start with the navigation menu collapsed instead of expanded." + }, + "enableGenericCaseList": { + "title": "Enable generic case list", + "description": "Show a single case list view with a dropdown to switch between case definitions." + } + } + }, + "opensearch": { + "title": "OpenSearch", + "reindex": { + "title": "Reindex Documents", + "description": "Reindex all documents from the database to OpenSearch. This operation runs in the background.", + "startButton": "Start Reindex", + "status": "Status", + "processed": "Processed", + "skipped": "Skipped", + "started": "Started", + "elapsed": "Elapsed time", + "finished": "Finished", + "inProgress": "Reindexing in progress...", + "errorTitle": "Error", + "statuses": { + "RUNNING": "Running", + "COMPLETED": "Completed", + "FAILED": "Failed", + "STOPPED": "Stopped" } } } diff --git a/frontend/projects/valtimo/shared/assets/core/nl.json b/frontend/projects/valtimo/shared/assets/core/nl.json index e4bcac2eb2..ad57971e67 100644 --- a/frontend/projects/valtimo/shared/assets/core/nl.json +++ b/frontend/projects/valtimo/shared/assets/core/nl.json @@ -3097,7 +3097,8 @@ "title": "Instellingen", "tabs": { "appearance": "Weergave", - "featureToggles": "Functie-instellingen" + "featureToggles": "Functie-instellingen", + "opensearch": "OpenSearch" }, "appearance": { "logo": { @@ -3137,7 +3138,7 @@ "useOpenSearch": { "title": "Gebruik OpenSearch voor zaakzoekopdrachten", "description": "Indien ingeschakeld worden zaakzoekopdrachten uitgevoerd met OpenSearch voor betere prestaties. Indien uitgeschakeld wordt PostgreSQL gebruikt.", - "unavailable": "OpenSearch is niet geconfigureerd voor deze installatie." + "unavailable": "OpenSearch is niet geconfigureerd voor deze applicatie." } }, "refreshRequired": "Vernieuwing vereist", @@ -3231,6 +3232,32 @@ "menuCollapsedByDefault": { "title": "Menu standaard ingeklapt", "description": "Start met het navigatiemenu ingeklapt in plaats van uitgeklapt." + }, + "enableGenericCaseList": { + "title": "Generieke zaaklijst inschakelen", + "description": "Toon één zaaklijstweergave met een dropdown om tussen zaakdefinities te wisselen." + } + } + }, + "opensearch": { + "title": "OpenSearch", + "reindex": { + "title": "Documenten herindexeren", + "description": "Herindexeer alle documenten van de database naar OpenSearch. Deze operatie draait op de achtergrond.", + "startButton": "Start herindexering", + "status": "Status", + "processed": "Verwerkt", + "skipped": "Overgeslagen", + "started": "Gestart", + "elapsed": "Verstreken tijd", + "finished": "Voltooid", + "inProgress": "Herindexering bezig...", + "errorTitle": "Fout", + "statuses": { + "RUNNING": "Bezig", + "COMPLETED": "Voltooid", + "FAILED": "Mislukt", + "STOPPED": "Gestopt" } } } diff --git a/frontend/projects/valtimo/shared/src/lib/models/config.ts b/frontend/projects/valtimo/shared/src/lib/models/config.ts index 18daf1a09c..1a3630f9f2 100644 --- a/frontend/projects/valtimo/shared/src/lib/models/config.ts +++ b/frontend/projects/valtimo/shared/src/lib/models/config.ts @@ -1,17 +1,19 @@ /* - * Copyright 2015-2026 Ritense BV, the Netherlands. * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * * Copyright 2015-2026 Ritense BV, the Netherlands. + * * + * * Licensed under EUPL, Version 1.2 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" basis, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ import {InjectionToken, Injector} from '@angular/core'; @@ -94,6 +96,7 @@ interface ValtimoConfigFeatureToggles { enableIkoType?: boolean; enableGenericCaseList?: boolean; menuCollapsedByDefault?: boolean; + enableOpenSearch?: boolean; /** * @deprecated DMN decision table editing is always enabled and is no longer gated by a * feature toggle. This option is ignored and will be removed in a future major release. diff --git a/frontend/projects/valtimo/shared/src/lib/models/menu-item.model.ts b/frontend/projects/valtimo/shared/src/lib/models/menu-item.model.ts index 43e915a96c..052b0219a6 100644 --- a/frontend/projects/valtimo/shared/src/lib/models/menu-item.model.ts +++ b/frontend/projects/valtimo/shared/src/lib/models/menu-item.model.ts @@ -1,23 +1,26 @@ /* - * Copyright 2015-2025 Ritense BV, the Netherlands. * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * * Copyright 2015-2026 Ritense BV, the Netherlands. + * * + * * Licensed under EUPL, Version 1.2 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" basis, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ import {Observable} from 'rxjs'; enum IncludeFunction { ObjectManagementEnabled, + OpenSearchEnabled, } interface MenuItem { diff --git a/frontend/projects/valtimo/shared/src/lib/services/menu-include.service.ts b/frontend/projects/valtimo/shared/src/lib/services/menu-include.service.ts index c77df9f58b..548bedb1cf 100644 --- a/frontend/projects/valtimo/shared/src/lib/services/menu-include.service.ts +++ b/frontend/projects/valtimo/shared/src/lib/services/menu-include.service.ts @@ -1,17 +1,19 @@ /* - * Copyright 2015-2025 Ritense BV, the Netherlands. * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * * Copyright 2015-2026 Ritense BV, the Netherlands. + * * + * * Licensed under EUPL, Version 1.2 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" basis, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ import {Injectable} from '@angular/core'; @@ -29,6 +31,8 @@ export class MenuIncludeService { switch (includeFunction) { case IncludeFunction.ObjectManagementEnabled: return this.configService.getFeatureToggleObservable('enableObjectManagement', true); + case IncludeFunction.OpenSearchEnabled: + return this.configService.getFeatureToggleObservable('enableOpenSearch', false); default: return of(true); } From c79ff9c7ca0ec3eeaaf8d6d58ef9dbd16bfe4b60 Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Thu, 9 Jul 2026 13:07:10 +0200 Subject: [PATCH 21/46] made search field always expanded --- .../src/lib/components/carbon-list/carbon-list.component.html | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.html b/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.html index 0b3256c274..a2cf2ee0b9 100644 --- a/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.html +++ b/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.html @@ -43,8 +43,7 @@
Date: Thu, 9 Jul 2026 14:33:58 +0200 Subject: [PATCH 22/46] query fixes to respect access control --- .../DocumentOpenSearchAutoConfiguration.kt | 3 + .../JsonSchemaDocumentOpenSearchService.kt | 72 ++++++++++----- .../BaseOpenSearchIntegrationTest.kt | 33 ++++--- ...JsonSchemaDocumentOpenSearchServiceTest.kt | 87 ++++++++++++++++--- 4 files changed, 153 insertions(+), 42 deletions(-) diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt index b8958462e7..76cc58ed1a 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt @@ -56,6 +56,7 @@ import com.ritense.document.service.DocumentSearchService import com.ritense.document.service.impl.JsonSchemaDocumentDefinitionService import com.ritense.document.service.SearchFieldService import com.ritense.document.service.impl.JsonSchemaDocumentSearchService +import com.ritense.case.service.CaseDefinitionService import com.ritense.valtimo.contract.database.QueryDialectHelper import com.ritense.outbox.OutboxService import com.ritense.valtimo.contract.authentication.TeamManagementService @@ -248,10 +249,12 @@ class DocumentOpenSearchAutoConfiguration { searchFieldService: SearchFieldService, outboxService: OutboxService, objectMapper: ObjectMapper, + caseDefinitionService: CaseDefinitionService, ): JsonSchemaDocumentOpenSearchService = JsonSchemaDocumentOpenSearchService( elasticsearchOperations, translator, authorizationService, jpaRepository, userManagementService, searchFieldService, outboxService, objectMapper, + caseDefinitionService, ) @Bean("jpaDocumentSearchService") diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt index 8f5af4f7ac..fbd4267fe0 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt @@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode import com.ritense.authorization.Action import com.ritense.authorization.AuthorizationContext.Companion.runWithoutAuthorization import com.ritense.authorization.AuthorizationService +import com.ritense.case.service.CaseDefinitionService import com.ritense.document.domain.impl.JsonSchemaDocument import com.ritense.document.domain.impl.JsonSchemaDocumentId import com.ritense.document.domain.search.AdvancedSearchRequest @@ -76,6 +77,7 @@ class JsonSchemaDocumentOpenSearchService( private val searchFieldService: SearchFieldService, private val outboxService: OutboxService, private val objectMapper: ObjectMapper, + private val caseDefinitionService: CaseDefinitionService, ) : DocumentSearchService { override fun search( @@ -99,19 +101,19 @@ class JsonSchemaDocumentOpenSearchService( } val globalFilter = searchRequest.globalSearchFilter?.takeIf { it.isNotEmpty() } if (globalFilter != null) { - val searchFields = if (!searchRequest.documentDefinitionName.isNullOrEmpty()) { - runWithoutAuthorization { + if (!searchRequest.documentDefinitionName.isNullOrEmpty()) { + val searchFields = runWithoutAuthorization { searchFieldService.getSearchFields(searchRequest.documentDefinitionName) } + if (searchFields.isNotEmpty()) { + parts.add(buildGlobalSearchQuery(globalFilter.trim(), searchFields)) + } else { + val term = "*${globalFilter.trim()}*" + parts.add(QueryBuilders.wildcardQuery("contentText.keyword", term).caseInsensitive(true)) + } } else { - emptyList() - } - - if (searchFields.isNotEmpty()) { - parts.add(buildGlobalSearchQuery(globalFilter.trim(), searchFields)) - } else { - val term = "*${globalFilter.trim()}*" - parts.add(QueryBuilders.wildcardQuery("contentText.keyword", term).caseInsensitive(true)) + val scopedQuery = buildGlobalSearchQueryForAllDefinitions(globalFilter.trim()) + parts.add(scopedQuery) } } searchRequest.otherFilters?.forEach { sc -> @@ -245,19 +247,19 @@ class JsonSchemaDocumentOpenSearchService( val globalFilter = searchRequest.globalSearchFilter?.takeIf { it.isNotEmpty() } if (globalFilter != null) { - val searchFields = if (!documentDefinitionName.isNullOrEmpty()) { - runWithoutAuthorization { + if (!documentDefinitionName.isNullOrEmpty()) { + val searchFields = runWithoutAuthorization { searchFieldService.getSearchFields(documentDefinitionName) } + if (searchFields.isNotEmpty()) { + parts.add(buildGlobalSearchQuery(globalFilter.trim(), searchFields)) + } else { + val term = "*${globalFilter.trim()}*" + parts.add(QueryBuilders.wildcardQuery("contentText.keyword", term).caseInsensitive(true)) + } } else { - emptyList() - } - - if (searchFields.isNotEmpty()) { - parts.add(buildGlobalSearchQuery(globalFilter.trim(), searchFields)) - } else { - val term = "*${globalFilter.trim()}*" - parts.add(QueryBuilders.wildcardQuery("contentText.keyword", term).caseInsensitive(true)) + val scopedQuery = buildGlobalSearchQueryForAllDefinitions(globalFilter.trim()) + parts.add(scopedQuery) } } @@ -533,6 +535,36 @@ class JsonSchemaDocumentOpenSearchService( return boolQuery } + private fun buildGlobalSearchQueryForAllDefinitions(query: String): QueryBuilder { + val matchNone = QueryBuilders.boolQuery().mustNot(QueryBuilders.matchAllQuery()) + + val accessibleDefinitions = caseDefinitionService.getCaseDefinitions(active = true) + if (accessibleDefinitions.isEmpty()) { + return matchNone + } + + val contentTextQuery = QueryBuilders.wildcardQuery("contentText.keyword", "*${query}*").caseInsensitive(true) + + val definitionQueries = accessibleDefinitions.map { definition -> + val searchFields = runWithoutAuthorization { + searchFieldService.getSearchFields(definition.id.key) + } + val searchQuery = if (searchFields.isEmpty()) { + contentTextQuery + } else { + buildGlobalSearchQuery(query, searchFields) + } + QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery(DEFINITION_NAME_FIELD, definition.id.key)) + .must(searchQuery) + } + + return QueryBuilders.boolQuery().apply { + definitionQueries.forEach { should(it) } + minimumShouldMatch(1) + } + } + private data class ParsedTerm( val field: String?, val value: String, diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/BaseOpenSearchIntegrationTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/BaseOpenSearchIntegrationTest.kt index c180fdc980..3e16371612 100644 --- a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/BaseOpenSearchIntegrationTest.kt +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/BaseOpenSearchIntegrationTest.kt @@ -1,17 +1,19 @@ /* - * Copyright 2015-2025 Ritense BV, the Netherlands. * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * * Copyright 2015-2026 Ritense BV, the Netherlands. + * * + * * Licensed under EUPL, Version 1.2 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" basis, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ package com.ritense.document.opensearch @@ -102,10 +104,19 @@ abstract class BaseOpenSearchIntegrationTest { @BeforeEach fun setUpBase() { setUpPermissions() + ensureIndexExists() openSearchRepository.deleteAll() refreshIndex() } + private fun ensureIndexExists() { + val indexOps = elasticsearchOperations.indexOps(JsonSchemaDocumentOsDocument::class.java) + if (!indexOps.exists()) { + indexOps.create() + indexOps.putMapping(indexOps.createMapping(JsonSchemaDocumentOsDocument::class.java)) + } + } + @AfterEach fun tearDownBase() { openSearchRepository.deleteAll() diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceTest.kt index 5aab692cf9..0e2e2c427a 100644 --- a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceTest.kt +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceTest.kt @@ -1,17 +1,19 @@ /* - * Copyright 2015-2025 Ritense BV, the Netherlands. * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * * Copyright 2015-2026 Ritense BV, the Netherlands. + * * + * * Licensed under EUPL, Version 1.2 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" basis, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ package com.ritense.document.opensearch.service @@ -19,6 +21,9 @@ package com.ritense.document.opensearch.service import com.fasterxml.jackson.databind.ObjectMapper import com.ritense.authorization.Action import com.ritense.authorization.AuthorizationService +import com.ritense.case_.domain.definition.CaseDefinition +import com.ritense.case.service.CaseDefinitionService +import com.ritense.valtimo.contract.case_.CaseDefinitionId import com.ritense.authorization.permission.ConditionContainer import com.ritense.authorization.permission.Permission import com.ritense.authorization.role.Role @@ -34,6 +39,7 @@ import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument import com.ritense.document.repository.impl.JsonSchemaDocumentRepository import com.ritense.document.service.JsonSchemaDocumentActionProvider import com.ritense.document.service.SearchFieldService +import com.ritense.document.service.impl.SearchRequest import com.ritense.outbox.OutboxService import com.ritense.valtimo.contract.authentication.UserManagementService import com.ritense.valtimo.contract.blueprint.BlueprintType @@ -63,6 +69,7 @@ class JsonSchemaDocumentOpenSearchServiceTest { private val searchFieldService: SearchFieldService = mock() private val outboxService: OutboxService = mock() private val objectMapper: ObjectMapper = ObjectMapper() + private val caseDefinitionService: CaseDefinitionService = mock() private lateinit var service: JsonSchemaDocumentOpenSearchService @@ -82,6 +89,7 @@ class JsonSchemaDocumentOpenSearchServiceTest { searchFieldService = searchFieldService, outboxService = outboxService, objectMapper = objectMapper, + caseDefinitionService = caseDefinitionService, ) val auth = UsernamePasswordAuthenticationToken( @@ -285,6 +293,63 @@ class JsonSchemaDocumentOpenSearchServiceTest { assertThat(capturedQuery.source).contains("*urgent*") } + @Test + fun `global search without definition name builds per-definition scoped query`() { + val queryCaptor = argumentCaptor() + val emptySearchHits: SearchHits = mock() + whenever(emptySearchHits.searchHits).thenReturn(emptyList()) + whenever(emptySearchHits.totalHits).thenReturn(0L) + whenever(elasticsearchOperations.search(queryCaptor.capture(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(emptySearchHits) + + val houseDef = mock() + val houseDefId = mock() + whenever(houseDefId.key).thenReturn("house") + whenever(houseDef.id).thenReturn(houseDefId) + + val carDef = mock() + val carDefId = mock() + whenever(carDefId.key).thenReturn("car") + whenever(carDef.id).thenReturn(carDefId) + + whenever(caseDefinitionService.getCaseDefinitions(active = true)).thenReturn(listOf(houseDef, carDef)) + whenever(searchFieldService.getSearchFields("house")).thenReturn(listOf( + SearchField("city", "doc:city", SearchFieldDataType.TEXT, SearchFieldFieldType.SINGLE, SearchFieldMatchType.LIKE, null, 0, "City") + )) + whenever(searchFieldService.getSearchFields("car")).thenReturn(listOf( + SearchField("brand", "doc:brand", SearchFieldDataType.TEXT, SearchFieldFieldType.SINGLE, SearchFieldMatchType.LIKE, null, 0, "Brand") + )) + + val request = SearchRequest() + request.globalSearchFilter = "test" + service.search(request, BlueprintType.CASE, PageRequest.of(0, 10)) + + val capturedQuery = queryCaptor.firstValue + assertThat(capturedQuery.source).contains("definitionId.name") + assertThat(capturedQuery.source).contains("house") + assertThat(capturedQuery.source).contains("car") + assertThat(capturedQuery.source).contains("content.city") + assertThat(capturedQuery.source).contains("content.brand") + } + + @Test + fun `global search without definition name and no accessible definitions adds matchNone query`() { + val queryCaptor = argumentCaptor() + val emptySearchHits: SearchHits = mock() + whenever(emptySearchHits.searchHits).thenReturn(emptyList()) + whenever(emptySearchHits.totalHits).thenReturn(0L) + whenever(elasticsearchOperations.search(queryCaptor.capture(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(emptySearchHits) + + whenever(caseDefinitionService.getCaseDefinitions(active = true)).thenReturn(emptyList()) + + val request = SearchRequest() + request.globalSearchFilter = "test" + service.search(request, BlueprintType.CASE, PageRequest.of(0, 10)) + + val capturedQuery = queryCaptor.firstValue + assertThat(capturedQuery.source).contains("must_not") + assertThat(capturedQuery.source).contains("match_all") + } + companion object { private const val FULL_ACCESS_ROLE = "full access role" private const val USERNAME = "test@test.com" From d3961b8918836c146f5fb08c140e4a08bdbc61e5 Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Fri, 10 Jul 2026 14:56:39 +0200 Subject: [PATCH 23/46] resilience + UI changes --- .../DocumentOpenSearchAutoConfiguration.kt | 26 ++-- .../opensearch/domain/OpenSearchReindexRun.kt | 3 + .../DocumentOpenSearchIndexInitializer.kt | 1 + .../DocumentOpenSearchReindexService.kt | 71 ++++++++++ .../service/OpenSearchReindexRunService.kt | 31 +++-- .../opensearch/service/ReindexRequest.kt | 1 + ...DocumentOpenSearchReindexServiceIntTest.kt | 69 ++++++++++ .../DocumentOpenSearchReindexServiceTest.kt | 3 + .../liquibase/13-32-0/13-32-0-master.xml | 3 - .../liquibase/13-37-0/13-37-0-master.xml | 28 ++++ .../20260630-create-reindex-run-table.xml | 0 ...01-create-pending-index-deletion-table.xml | 0 .../20260701-create-reconcile-state-table.xml | 0 .../20260710-add-reindex-run-pruned-count.xml | 32 +++++ .../config/liquibase/changelog-master.xml | 1 + .../admin-settings-opensearch.component.html | 52 +++++--- .../admin-settings-opensearch.component.scss | 31 +++-- .../admin-settings-opensearch.component.ts | 121 +++++++++++++++--- .../src/lib/models/reindex.model.ts | 32 +++-- .../admin-settings-management-api.service.ts | 29 ++--- .../components/menu/services/menu.service.ts | 31 ++--- .../valtimo/shared/assets/core/en.json | 4 + .../valtimo/shared/assets/core/nl.json | 4 + 23 files changed, 446 insertions(+), 127 deletions(-) create mode 100644 backend/core/src/main/resources/config/liquibase/13-37-0/13-37-0-master.xml rename backend/core/src/main/resources/config/liquibase/{13-32-0 => 13-37-0}/20260630-create-reindex-run-table.xml (100%) rename backend/core/src/main/resources/config/liquibase/{13-32-0 => 13-37-0}/20260701-create-pending-index-deletion-table.xml (100%) rename backend/core/src/main/resources/config/liquibase/{13-32-0 => 13-37-0}/20260701-create-reconcile-state-table.xml (100%) create mode 100644 backend/core/src/main/resources/config/liquibase/13-37-0/20260710-add-reindex-run-pruned-count.xml diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt index 76cc58ed1a..4d91ec4569 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt @@ -1,19 +1,17 @@ /* + * Copyright 2015-2026 Ritense BV, the Netherlands. * - * * Copyright 2015-2026 Ritense BV, the Netherlands. - * * - * * Licensed under EUPL, Version 1.2 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" basis, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.ritense.document.opensearch.autoconfigure @@ -164,6 +162,7 @@ class DocumentOpenSearchAutoConfiguration { transactionManager: PlatformTransactionManager, lockProvider: LockProvider, openSearchReindexRunService: OpenSearchReindexRunService, + openSearchRepository: JsonSchemaDocumentOpenSearchRepository, ): DocumentOpenSearchReindexService = DocumentOpenSearchReindexService( entityManager, @@ -172,6 +171,7 @@ class DocumentOpenSearchAutoConfiguration { transactionManager, lockProvider, openSearchReindexRunService, + openSearchRepository, ) @Bean diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/OpenSearchReindexRun.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/OpenSearchReindexRun.kt index 1833f4686c..e9d02cb491 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/OpenSearchReindexRun.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/OpenSearchReindexRun.kt @@ -63,6 +63,9 @@ class OpenSearchReindexRun( @Column(name = "skipped_count", nullable = false) var skippedCount: Long = 0, + @Column(name = "pruned_count", nullable = false) + var prunedCount: Long = 0, + @Column(name = "started_on", nullable = false) val startedOn: LocalDateTime = LocalDateTime.now(), diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchIndexInitializer.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchIndexInitializer.kt index 1ff0209268..4e3810d917 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchIndexInitializer.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchIndexInitializer.kt @@ -49,6 +49,7 @@ open class DocumentOpenSearchIndexInitializer( val dynamicTemplates = listOf( mapOf("content_fields_as_text" to mapOf( "path_match" to "content.*", + "match_mapping_type" to "string", "mapping" to mapOf( "type" to "text", "fields" to mapOf("keyword" to mapOf("type" to "keyword", "ignore_above" to 256)) diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt index 9e40df531f..6cc0ea16ff 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt @@ -18,6 +18,7 @@ package com.ritense.document.opensearch.service import com.ritense.document.domain.impl.JsonSchemaDocument import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument +import com.ritense.document.opensearch.repository.JsonSchemaDocumentOpenSearchRepository import io.github.oshai.kotlinlogging.KotlinLogging import jakarta.persistence.EntityManager import jakarta.persistence.criteria.JoinType @@ -25,7 +26,12 @@ import jakarta.persistence.criteria.Predicate import net.javacrumbs.shedlock.core.LockConfiguration import net.javacrumbs.shedlock.core.LockProvider import org.springframework.beans.factory.DisposableBean +import org.opensearch.index.query.BoolQueryBuilder +import org.opensearch.index.query.QueryBuilders +import org.springframework.data.domain.PageRequest +import org.springframework.data.domain.Sort import org.springframework.data.elasticsearch.core.ElasticsearchOperations +import org.springframework.data.elasticsearch.core.query.StringQuery import org.springframework.transaction.PlatformTransactionManager import org.springframework.transaction.support.TransactionTemplate import java.time.Duration @@ -57,6 +63,7 @@ open class DocumentOpenSearchReindexService( private val transactionManager: PlatformTransactionManager, private val lockProvider: LockProvider, private val runService: OpenSearchReindexRunService, + private val openSearchRepository: JsonSchemaDocumentOpenSearchRepository, ) : DisposableBean { private val executor: ExecutorService = Executors.newSingleThreadExecutor { runnable -> @@ -137,6 +144,11 @@ open class DocumentOpenSearchReindexService( logger.info { "Re-index run $runId cancelled — marking STOPPED (processed=$processed, skipped=$skipped)" } runService.stop(runId) } else { + if (scope.pruneOrphans) { + val pruned = pruneOrphans(scope) + runService.recordPruned(runId, pruned) + logger.info { "Pruned $pruned orphan document(s) from OpenSearch" } + } indexOps().refresh() logger.info { "Re-index run $runId complete (processed=$processed, skipped=$skipped)" } runService.complete(runId) @@ -182,6 +194,64 @@ open class DocumentOpenSearchReindexService( return entityManager.createQuery(query).setMaxResults(pageSize).resultList } + /** + * Scans OpenSearch for documents matching [scope], checks each batch against PostgreSQL, + * and deletes orphans (documents in OpenSearch but not in PostgreSQL). + */ + private fun pruneOrphans(scope: ReindexRequest): Long { + var pruned = 0L + var page = 0 + val txTemplate = TransactionTemplate(transactionManager).apply { isReadOnly = true } + + while (!cancelRequested) { + val osIds = fetchOpenSearchIds(scope, page, PRUNE_BATCH_SIZE) + if (osIds.isEmpty()) break + + val existingIds = txTemplate.execute { + findExistingIds(osIds.map { UUID.fromString(it) }) + }.orEmpty() + + val orphans = osIds.filter { UUID.fromString(it) !in existingIds } + orphans.forEach { openSearchRepository.deleteById(it) } + pruned += orphans.size + + page++ + } + return pruned + } + + private fun fetchOpenSearchIds(scope: ReindexRequest, page: Int, batchSize: Int): List { + val boolQuery = BoolQueryBuilder() + + scope.documentDefinitionName?.let { + boolQuery.filter(QueryBuilders.termQuery("definitionId.name", it)) + } + scope.modifiedAfter?.let { + boolQuery.filter(QueryBuilders.rangeQuery("modifiedOn").gt(it)) + } + scope.modifiedBefore?.let { + boolQuery.filter(QueryBuilders.rangeQuery("modifiedOn").lt(it)) + } + scope.documentIds?.takeIf { it.isNotEmpty() }?.let { ids -> + boolQuery.filter(QueryBuilders.idsQuery().addIds(*ids.map { it.toString() }.toTypedArray())) + } + + val pageable = PageRequest.of(page, batchSize, Sort.by(Sort.Direction.ASC, "_id")) + val query = StringQuery(boolQuery.toString(), pageable) + + return elasticsearchOperations.search(query, JsonSchemaDocumentOsDocument::class.java) + .searchHits + .map { it.id } + } + + private fun findExistingIds(ids: List): Set { + if (ids.isEmpty()) return emptySet() + return entityManager.createQuery( + "SELECT d.id.id FROM JsonSchemaDocument d WHERE d.id.id IN :ids", + UUID::class.java + ).setParameter("ids", ids).resultList.toSet() + } + private fun indexOps() = elasticsearchOperations.indexOps(JsonSchemaDocumentOsDocument::class.java) /** Signals an in-flight run to stop gracefully and shuts the executor down on context close. */ @@ -211,5 +281,6 @@ open class DocumentOpenSearchReindexService( val LOCK_AT_MOST_FOR: Duration = Duration.ofHours(6) private const val SHUTDOWN_TIMEOUT_SECONDS = 30L + private const val PRUNE_BATCH_SIZE = 1000 } } diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunService.kt index fa9ab41b61..e4351ae782 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunService.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunService.kt @@ -1,19 +1,17 @@ /* + * Copyright 2015-2026 Ritense BV, the Netherlands. * - * * Copyright 2015-2026 Ritense BV, the Netherlands. - * * - * * Licensed under EUPL, Version 1.2 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" basis, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.ritense.document.opensearch.service @@ -133,6 +131,12 @@ open class OpenSearchReindexRunService( repository.save(run) } + open fun recordPruned(runId: UUID, pruned: Long) { + val run = requireRun(runId) + run.prunedCount = pruned + repository.save(run) + } + /** * Status of a specific run (by [runId]) or — when null — of the most recent run. Returns a * not-running placeholder when no matching run exists. @@ -160,6 +164,7 @@ open class OpenSearchReindexRunService( "lastId" to run.lastId, "processedCount" to run.processedCount, "skippedCount" to run.skippedCount, + "prunedCount" to run.prunedCount, "totalCount" to countDocuments(scope), "startedOn" to run.startedOn, "heartbeatOn" to run.heartbeatOn, diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/ReindexRequest.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/ReindexRequest.kt index 7f242ebfb6..bdab53495a 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/ReindexRequest.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/ReindexRequest.kt @@ -37,6 +37,7 @@ data class ReindexRequest( val documentIds: List? = null, val pageSize: Int = DEFAULT_PAGE_SIZE, val resumeRunId: UUID? = null, + val pruneOrphans: Boolean = false, ) { fun effectivePageSize() = pageSize.coerceIn(1, MAX_PAGE_SIZE) diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceIntTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceIntTest.kt index c9dfdb1fd4..c5efc5ffb8 100644 --- a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceIntTest.kt +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceIntTest.kt @@ -18,6 +18,7 @@ package com.ritense.document.opensearch.service import com.ritense.authorization.AuthorizationContext.Companion.runWithoutAuthorization import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.domain.impl.JsonSchemaDocumentId import com.ritense.document.domain.impl.request.NewDocumentRequest import com.ritense.document.opensearch.BaseOpenSearchIntegrationTest import com.ritense.document.opensearch.domain.OpenSearchReindexRun @@ -76,6 +77,9 @@ class DocumentOpenSearchReindexServiceIntTest : BaseOpenSearchIntegrationTest() @Autowired lateinit var transactionManager: PlatformTransactionManager + @Autowired + lateinit var converter: JsonSchemaDocumentOsConverter + @AfterEach fun cleanUp() { runWithoutAuthorization { documentService.removeDocuments("house") } @@ -270,6 +274,71 @@ class DocumentOpenSearchReindexServiceIntTest : BaseOpenSearchIntegrationTest() assertThat(reindexRunService.isReindexRunning(Duration.ofMinutes(5))).isFalse() } + @Test + fun `reindex with pruneOrphans deletes orphans from OpenSearch`() { + val doc1 = createDocument("keep") + val doc2 = createDocument("orphan-1") + val doc3 = createDocument("orphan-2") + indexDocuments(doc1, doc2, doc3) + assertThat(openSearchRepository.count()).isEqualTo(3L) + + deleteDocumentFromDatabaseOnly(doc2.id()) + deleteDocumentFromDatabaseOnly(doc3.id()) + + val (runId, _) = reindex(ReindexRequest(pruneOrphans = true)) + + refreshIndex() + assertThat(openSearchRepository.count()).isEqualTo(1L) + assertThat(openSearchRepository.findById(doc1.id().toString())).isPresent + assertThat(openSearchRepository.findById(doc2.id().toString())).isEmpty + assertThat(openSearchRepository.findById(doc3.id().toString())).isEmpty + + val run = reindexRunRepository.findById(runId).get() + assertThat(run.prunedCount).isEqualTo(2L) + } + + @Test + fun `scoped reindex with pruneOrphans only prunes matching definition`() { + val houseDoc = createDocument("house-street") + indexDocuments(houseDoc) + + deleteDocumentFromDatabaseOnly(houseDoc.id()) + + reindex(ReindexRequest(documentDefinitionName = "house", pruneOrphans = true)) + + refreshIndex() + assertThat(openSearchRepository.findById(houseDoc.id().toString())).isEmpty + } + + @Test + fun `reindex without pruneOrphans leaves orphans in place`() { + val doc = createDocument("orphan") + indexDocuments(doc) + + deleteDocumentFromDatabaseOnly(doc.id()) + + reindex(ReindexRequest(pruneOrphans = false)) + + refreshIndex() + assertThat(openSearchRepository.findById(doc.id().toString())).isPresent + } + + private fun indexDocuments(vararg documents: JsonSchemaDocument) { + documents.forEach { doc -> + val osDoc = converter.toOsDocument(doc) + openSearchRepository.save(osDoc) + } + refreshIndex() + } + + private fun deleteDocumentFromDatabaseOnly(documentId: JsonSchemaDocumentId) { + TransactionTemplate(transactionManager).execute { + entityManager.createNativeQuery( + "DELETE FROM json_schema_document WHERE json_schema_document_id = :id" + ).setParameter("id", documentId.id).executeUpdate() + } + } + /** Clears the OpenSearch index (and refreshes) so a subsequent assertion sees only the re-index output. */ private fun clearIndex() { openSearchRepository.deleteAll() diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceTest.kt index 0750718888..bc0cb062a8 100644 --- a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceTest.kt +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceTest.kt @@ -18,6 +18,7 @@ package com.ritense.document.opensearch.service import com.ritense.document.opensearch.domain.OpenSearchReindexRun import com.ritense.document.opensearch.domain.ReindexRunStatus +import com.ritense.document.opensearch.repository.JsonSchemaDocumentOpenSearchRepository import jakarta.persistence.EntityManager import net.javacrumbs.shedlock.core.LockProvider import net.javacrumbs.shedlock.core.SimpleLock @@ -43,6 +44,7 @@ class DocumentOpenSearchReindexServiceTest { private val transactionManager: PlatformTransactionManager = mock() private val lockProvider: LockProvider = mock() private val runService: OpenSearchReindexRunService = mock() + private val openSearchRepository: JsonSchemaDocumentOpenSearchRepository = mock() private lateinit var service: DocumentOpenSearchReindexService @@ -55,6 +57,7 @@ class DocumentOpenSearchReindexServiceTest { transactionManager, lockProvider, runService, + openSearchRepository, ) } diff --git a/backend/core/src/main/resources/config/liquibase/13-32-0/13-32-0-master.xml b/backend/core/src/main/resources/config/liquibase/13-32-0/13-32-0-master.xml index a4fdb708eb..5bfcdce345 100644 --- a/backend/core/src/main/resources/config/liquibase/13-32-0/13-32-0-master.xml +++ b/backend/core/src/main/resources/config/liquibase/13-32-0/13-32-0-master.xml @@ -23,8 +23,5 @@ xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liqui - - - diff --git a/backend/core/src/main/resources/config/liquibase/13-37-0/13-37-0-master.xml b/backend/core/src/main/resources/config/liquibase/13-37-0/13-37-0-master.xml new file mode 100644 index 0000000000..564f5fbabf --- /dev/null +++ b/backend/core/src/main/resources/config/liquibase/13-37-0/13-37-0-master.xml @@ -0,0 +1,28 @@ + + + + + + + + + + + diff --git a/backend/core/src/main/resources/config/liquibase/13-32-0/20260630-create-reindex-run-table.xml b/backend/core/src/main/resources/config/liquibase/13-37-0/20260630-create-reindex-run-table.xml similarity index 100% rename from backend/core/src/main/resources/config/liquibase/13-32-0/20260630-create-reindex-run-table.xml rename to backend/core/src/main/resources/config/liquibase/13-37-0/20260630-create-reindex-run-table.xml diff --git a/backend/core/src/main/resources/config/liquibase/13-32-0/20260701-create-pending-index-deletion-table.xml b/backend/core/src/main/resources/config/liquibase/13-37-0/20260701-create-pending-index-deletion-table.xml similarity index 100% rename from backend/core/src/main/resources/config/liquibase/13-32-0/20260701-create-pending-index-deletion-table.xml rename to backend/core/src/main/resources/config/liquibase/13-37-0/20260701-create-pending-index-deletion-table.xml diff --git a/backend/core/src/main/resources/config/liquibase/13-32-0/20260701-create-reconcile-state-table.xml b/backend/core/src/main/resources/config/liquibase/13-37-0/20260701-create-reconcile-state-table.xml similarity index 100% rename from backend/core/src/main/resources/config/liquibase/13-32-0/20260701-create-reconcile-state-table.xml rename to backend/core/src/main/resources/config/liquibase/13-37-0/20260701-create-reconcile-state-table.xml diff --git a/backend/core/src/main/resources/config/liquibase/13-37-0/20260710-add-reindex-run-pruned-count.xml b/backend/core/src/main/resources/config/liquibase/13-37-0/20260710-add-reindex-run-pruned-count.xml new file mode 100644 index 0000000000..fc0554c233 --- /dev/null +++ b/backend/core/src/main/resources/config/liquibase/13-37-0/20260710-add-reindex-run-pruned-count.xml @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + diff --git a/backend/core/src/main/resources/config/liquibase/changelog-master.xml b/backend/core/src/main/resources/config/liquibase/changelog-master.xml index e8c46e8c69..5e6656a943 100644 --- a/backend/core/src/main/resources/config/liquibase/changelog-master.xml +++ b/backend/core/src/main/resources/config/liquibase/changelog-master.xml @@ -33,5 +33,6 @@ + diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.html b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.html index 94fcb8f2d2..580f695b2f 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.html +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.html @@ -1,19 +1,17 @@
@@ -33,6 +32,27 @@

+
+ + {{ 'adminSettings.opensearch.reindex.pruneOrphans' | translate }} + + + + + + + +
+ @if (obs.status?.status) {
diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss index 9998757540..87bd27061f 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss @@ -1,19 +1,17 @@ /*! + * Copyright 2015-2026 Ritense BV, the Netherlands. * - * * Copyright 2015-2026 Ritense BV, the Netherlands. - * * - * * Licensed under EUPL, Version 1.2 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" basis, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ .opensearch-reindex { @@ -41,6 +39,13 @@ margin: 0; } + &__options { + display: flex; + flex-direction: column; + gap: 16px; + max-width: 400px; + } + &__status { display: flex; flex-direction: column; diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.ts b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.ts index 8982df03e8..9ebe154a25 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.ts +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.ts @@ -1,23 +1,22 @@ /* + * Copyright 2015-2026 Ritense BV, the Netherlands. * - * * Copyright 2015-2026 Ritense BV, the Netherlands. - * * - * * Licensed under EUPL, Version 1.2 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" basis, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ -import {ChangeDetectionStrategy, Component, OnDestroy} from '@angular/core'; +import {ChangeDetectionStrategy, Component, OnDestroy, OnInit} from '@angular/core'; import {CommonModule, DatePipe} from '@angular/common'; +import {FormBuilder, FormGroup, ReactiveFormsModule} from '@angular/forms'; import {TranslateModule} from '@ngx-translate/core'; import { BehaviorSubject, @@ -34,24 +33,56 @@ import { takeUntil, takeWhile, } from 'rxjs'; -import {ButtonModule, LoadingModule, ProgressBarModule, TagModule} from 'carbon-components-angular'; +import { + ButtonModule, + CheckboxModule, + DatePickerInputModule, + DatePickerModule, + DropdownModule, + ListItem, + LoadingModule, + ProgressBarModule, + TagModule, +} from 'carbon-components-angular'; import {AdminSettingsManagementApiService} from '../../services'; -import {ReindexStatusDto} from '../../models'; +import {ReindexStatusDto, StartReindexRequestDto} from '../../models'; +import {DocumentService} from '@valtimo/document'; @Component({ standalone: true, selector: 'valtimo-admin-settings-opensearch', templateUrl: './admin-settings-opensearch.component.html', styleUrls: ['./admin-settings-opensearch.component.scss'], - imports: [CommonModule, DatePipe, TranslateModule, ButtonModule, LoadingModule, ProgressBarModule, TagModule], + imports: [ + CommonModule, + DatePipe, + TranslateModule, + ReactiveFormsModule, + ButtonModule, + CheckboxModule, + DatePickerInputModule, + DatePickerModule, + DropdownModule, + LoadingModule, + ProgressBarModule, + TagModule, + ], changeDetection: ChangeDetectionStrategy.OnPush, }) -export class AdminSettingsOpensearchComponent implements OnDestroy { +export class AdminSettingsOpensearchComponent implements OnInit, OnDestroy { private readonly _destroy$ = new Subject(); private readonly _refresh$ = new BehaviorSubject(undefined); public readonly startingReindex$ = new BehaviorSubject(false); + public readonly formGroup: FormGroup = this._fb.group({ + pruneOrphans: [false], + documentDefinitionName: [null], + modifiedBefore: [null], + }); + + public documentDefinitions$: Observable; + public readonly reindexStatus$: Observable = this._refresh$.pipe( switchMap(() => this._apiService.getReindexStatus()), switchMap(status => { @@ -72,12 +103,29 @@ export class AdminSettingsOpensearchComponent implements OnDestroy { map(status => status?.status === 'RUNNING') ); - constructor(private readonly _apiService: AdminSettingsManagementApiService) {} + constructor( + private readonly _apiService: AdminSettingsManagementApiService, + private readonly _documentService: DocumentService, + private readonly _fb: FormBuilder + ) {} + + public ngOnInit(): void { + this.documentDefinitions$ = this._documentService.queryDefinitionsForManagement().pipe( + map(page => + page.content.map(def => ({ + content: def.id.name, + selected: false, + })) + ), + startWith([]) + ); + } public startReindex(): void { this.startingReindex$.next(true); + const request = this._buildReindexRequest(); this._apiService - .startReindex() + .startReindex(request) .pipe( take(1), finalize(() => this.startingReindex$.next(false)) @@ -92,6 +140,39 @@ export class AdminSettingsOpensearchComponent implements OnDestroy { }); } + public onDateSelected(event: string[]): void { + const dateValue = event?.[0] || null; + this.formGroup.patchValue({modifiedBefore: dateValue}); + } + + private _buildReindexRequest(): StartReindexRequestDto { + const formValue = this.formGroup.value; + const request: StartReindexRequestDto = {}; + + if (formValue.pruneOrphans) { + request.pruneOrphans = true; + } + + if (formValue.documentDefinitionName?.content) { + request.documentDefinitionName = formValue.documentDefinitionName.content; + } + + if (formValue.modifiedBefore) { + request.modifiedBefore = this._formatDateToIso(formValue.modifiedBefore); + } + + return request; + } + + private _formatDateToIso(dateStr: string): string { + const parts = dateStr.split('-'); + if (parts.length === 3) { + const [day, month, year] = parts; + return `${year}-${month}-${day}T23:59:59`; + } + return dateStr; + } + public getStatusTagType(status: string): string { switch (status) { case 'RUNNING': diff --git a/frontend/projects/valtimo/admin-settings/src/lib/models/reindex.model.ts b/frontend/projects/valtimo/admin-settings/src/lib/models/reindex.model.ts index 2164c08b63..f77a09f7f6 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/models/reindex.model.ts +++ b/frontend/projects/valtimo/admin-settings/src/lib/models/reindex.model.ts @@ -1,19 +1,17 @@ /* + * Copyright 2015-2026 Ritense BV, the Netherlands. * - * * Copyright 2015-2026 Ritense BV, the Netherlands. - * * - * * Licensed under EUPL, Version 1.2 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" basis, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ type ReindexStatus = 'RUNNING' | 'COMPLETED' | 'FAILED' | 'STOPPED'; @@ -30,9 +28,15 @@ interface ReindexStatusDto { error: string | null; } +interface StartReindexRequestDto { + pruneOrphans?: boolean; + documentDefinitionName?: string; + modifiedBefore?: string; +} + interface StartReindexResponseDto { status: string; runId: string; } -export {ReindexStatus, ReindexStatusDto, StartReindexResponseDto}; +export {ReindexStatus, ReindexStatusDto, StartReindexRequestDto, StartReindexResponseDto}; diff --git a/frontend/projects/valtimo/admin-settings/src/lib/services/admin-settings-management-api.service.ts b/frontend/projects/valtimo/admin-settings/src/lib/services/admin-settings-management-api.service.ts index 61746871f8..001bb89686 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/services/admin-settings-management-api.service.ts +++ b/frontend/projects/valtimo/admin-settings/src/lib/services/admin-settings-management-api.service.ts @@ -1,19 +1,17 @@ /* + * Copyright 2015-2026 Ritense BV, the Netherlands. * - * * Copyright 2015-2026 Ritense BV, the Netherlands. - * * - * * Licensed under EUPL, Version 1.2 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" basis, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ import {Injectable} from '@angular/core'; @@ -31,6 +29,7 @@ import { FeatureToggleOverridesDto, ReindexStatusDto, SearchEngineDto, + StartReindexRequestDto, StartReindexResponseDto, UpdateFeatureToggleDto, } from '../models'; @@ -113,10 +112,10 @@ export class AdminSettingsManagementApiService extends BaseApiService { ); } - public startReindex(): Observable { + public startReindex(request: StartReindexRequestDto = {}): Observable { return this.httpClient.post( this.getApiUrl('/management/v1/document-opensearch/reindex'), - {} + request ); } diff --git a/frontend/projects/valtimo/components/src/lib/components/menu/services/menu.service.ts b/frontend/projects/valtimo/components/src/lib/components/menu/services/menu.service.ts index 959fc87a9a..57b5acaaa6 100644 --- a/frontend/projects/valtimo/components/src/lib/components/menu/services/menu.service.ts +++ b/frontend/projects/valtimo/components/src/lib/components/menu/services/menu.service.ts @@ -1,19 +1,17 @@ /* + * Copyright 2015-2026 Ritense BV, the Netherlands. * - * * Copyright 2015-2026 Ritense BV, the Netherlands. - * * - * * Licensed under EUPL, Version 1.2 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" basis, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ import {HttpClient} from '@angular/common/http'; @@ -155,13 +153,6 @@ export class MenuService implements OnDestroy { this.menuIncludeService.getIncludeFunctionObservable(menuItem.includeFunction); } - menuItem.children?.forEach(child => { - if (child.includeFunction !== undefined) { - this.includeFunctionObservables[child.title] = - this.menuIncludeService.getIncludeFunction(child.includeFunction); - } - }); - menuItem.show = true; if (!menuItem.roles || menuItem.roles.some(role => userRoles.includes(role))) { diff --git a/frontend/projects/valtimo/shared/assets/core/en.json b/frontend/projects/valtimo/shared/assets/core/en.json index 95e22cefc5..82bd9c8c6b 100644 --- a/frontend/projects/valtimo/shared/assets/core/en.json +++ b/frontend/projects/valtimo/shared/assets/core/en.json @@ -3328,6 +3328,10 @@ "finished": "Finished", "inProgress": "Reindexing in progress...", "errorTitle": "Error", + "pruneOrphans": "Prune orphaned documents from OpenSearch", + "documentDefinitionName": "Document definition", + "documentDefinitionPlaceholder": "All document definitions", + "modifiedBefore": "Modified before", "statuses": { "RUNNING": "Running", "COMPLETED": "Completed", diff --git a/frontend/projects/valtimo/shared/assets/core/nl.json b/frontend/projects/valtimo/shared/assets/core/nl.json index a24c14fe6e..667aa71de2 100644 --- a/frontend/projects/valtimo/shared/assets/core/nl.json +++ b/frontend/projects/valtimo/shared/assets/core/nl.json @@ -3361,6 +3361,10 @@ "finished": "Voltooid", "inProgress": "Herindexering bezig...", "errorTitle": "Fout", + "pruneOrphans": "Verwijder orphan documenten uit OpenSearch", + "documentDefinitionName": "Documentdefinitie", + "documentDefinitionPlaceholder": "Alle documentdefinities", + "modifiedBefore": "Gewijzigd voor", "statuses": { "RUNNING": "Bezig", "COMPLETED": "Voltooid", From bb98e31eba77158d1bc740a3805a17993d64a7d7 Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Mon, 13 Jul 2026 10:48:41 +0200 Subject: [PATCH 24/46] added overview for past reindex runs implemented a (work in progress) expanded row template --- .../OpenSearchReindexRunRepository.kt | 3 + ...ocumentOpenSearchHttpSecurityConfigurer.kt | 4 +- .../DocumentOpenSearchReindexService.kt | 4 + .../service/OpenSearchReindexRunService.kt | 9 + .../web/DocumentOpenSearchReindexResource.kt | 10 + .../DocumentOpenSearchReindexResourceTest.kt | 21 +++ .../admin-settings-opensearch.component.html | 176 +++++++++++------- .../admin-settings-opensearch.component.scss | 66 ++----- .../admin-settings-opensearch.component.ts | 145 ++++++++------- .../start-reindex-modal.component.html | 52 ++++++ .../start-reindex-modal.component.scss | 21 +++ .../start-reindex-modal.component.ts | 117 ++++++++++++ .../src/lib/models/reindex.model.ts | 21 ++- .../admin-settings-management-api.service.ts | 17 +- .../carbon-list/carbon-list.component.ts | 122 ++++++------ .../valtimo/shared/assets/core/en.json | 20 +- .../valtimo/shared/assets/core/nl.json | 20 +- 17 files changed, 576 insertions(+), 252 deletions(-) create mode 100644 frontend/projects/valtimo/admin-settings/src/lib/components/start-reindex-modal/start-reindex-modal.component.html create mode 100644 frontend/projects/valtimo/admin-settings/src/lib/components/start-reindex-modal/start-reindex-modal.component.scss create mode 100644 frontend/projects/valtimo/admin-settings/src/lib/components/start-reindex-modal/start-reindex-modal.component.ts diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/repository/OpenSearchReindexRunRepository.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/repository/OpenSearchReindexRunRepository.kt index 81cef189dd..48e6c7612b 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/repository/OpenSearchReindexRunRepository.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/repository/OpenSearchReindexRunRepository.kt @@ -18,6 +18,8 @@ package com.ritense.document.opensearch.repository import com.ritense.document.opensearch.domain.OpenSearchReindexRun import com.ritense.document.opensearch.domain.ReindexRunStatus +import org.springframework.data.domain.Page +import org.springframework.data.domain.Pageable import org.springframework.data.jpa.repository.JpaRepository import java.time.LocalDateTime import java.util.UUID @@ -27,4 +29,5 @@ interface OpenSearchReindexRunRepository : JpaRepository fun existsByStatusAndHeartbeatOnAfter(status: ReindexRunStatus, heartbeatOn: LocalDateTime): Boolean + fun findAllByOrderByStartedOnDesc(pageable: Pageable): Page } diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/security/DocumentOpenSearchHttpSecurityConfigurer.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/security/DocumentOpenSearchHttpSecurityConfigurer.kt index e50123d3b0..d270f8fcf2 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/security/DocumentOpenSearchHttpSecurityConfigurer.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/security/DocumentOpenSearchHttpSecurityConfigurer.kt @@ -1,5 +1,5 @@ /* - * Copyright 2015-2024 Ritense BV, the Netherlands. + * Copyright 2015-2026 Ritense BV, the Netherlands. * * Licensed under EUPL, Version 1.2 (the "License"); * you may not use this file except in compliance with the License. @@ -32,6 +32,8 @@ class DocumentOpenSearchHttpSecurityConfigurer : HttpSecurityConfigurer { http.authorizeHttpRequests { requests -> requests.requestMatchers(antMatcher(POST, "/api/management/v1/document-opensearch/reindex")) .hasAuthority(ADMIN) + requests.requestMatchers(antMatcher(GET, "/api/management/v1/document-opensearch/reindex/runs")) + .hasAuthority(ADMIN) requests.requestMatchers(antMatcher(GET, "/api/management/v1/document-opensearch/reindex/status")) .hasAuthority(ADMIN) requests.requestMatchers(antMatcher(GET, "/api/management/v1/document-opensearch/reindex/*")) diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt index 6cc0ea16ff..9c0b4ddb2f 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt @@ -28,7 +28,9 @@ import net.javacrumbs.shedlock.core.LockProvider import org.springframework.beans.factory.DisposableBean import org.opensearch.index.query.BoolQueryBuilder import org.opensearch.index.query.QueryBuilders +import org.springframework.data.domain.Page import org.springframework.data.domain.PageRequest +import org.springframework.data.domain.Pageable import org.springframework.data.domain.Sort import org.springframework.data.elasticsearch.core.ElasticsearchOperations import org.springframework.data.elasticsearch.core.query.StringQuery @@ -164,6 +166,8 @@ open class DocumentOpenSearchReindexService( /** Status of a specific run (by id) or the most recent run when [runId] is null. */ fun status(runId: UUID? = null): Map = runService.toStatusMap(runId) + fun listRuns(pageable: Pageable): Page> = runService.listRuns(pageable) + /** * Scoped keyset fetch. Applies the optional [scope] filters, eagerly loads the lazy `internalStatus` * `@ManyToOne` (C1 — so the detached entity serializes the real status key, not null), and keeps a diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunService.kt index e4351ae782..ca851c4eb8 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunService.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunService.kt @@ -27,6 +27,9 @@ import jakarta.persistence.EntityManager import jakarta.persistence.criteria.Predicate import org.springframework.boot.context.event.ApplicationReadyEvent import org.springframework.context.event.EventListener +import org.springframework.data.domain.Page +import org.springframework.data.domain.PageImpl +import org.springframework.data.domain.Pageable import org.springframework.transaction.annotation.Transactional import java.time.Duration import java.time.LocalDateTime @@ -149,6 +152,12 @@ open class OpenSearchReindexRunService( return toMap(run) } + @Transactional(readOnly = true) + open fun listRuns(pageable: Pageable): Page> { + val page = repository.findAllByOrderByStartedOnDesc(pageable) + return PageImpl(page.content.map { toMap(it) }, pageable, page.totalElements) + } + private fun requireRun(runId: UUID): OpenSearchReindexRun = repository.findById(runId).orElseThrow { IllegalArgumentException("No re-index run found for runId=$runId") } diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/DocumentOpenSearchReindexResource.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/DocumentOpenSearchReindexResource.kt index b551c621a4..693d771f07 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/DocumentOpenSearchReindexResource.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/DocumentOpenSearchReindexResource.kt @@ -18,12 +18,15 @@ package com.ritense.document.opensearch.web import com.ritense.document.opensearch.service.DocumentOpenSearchReindexService import com.ritense.document.opensearch.service.ReindexRequest +import org.springframework.data.domain.Page +import org.springframework.data.domain.PageRequest import org.springframework.http.ResponseEntity import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.PathVariable import org.springframework.web.bind.annotation.PostMapping import org.springframework.web.bind.annotation.RequestBody import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RequestParam import org.springframework.web.bind.annotation.RestController import java.util.UUID @@ -40,6 +43,13 @@ class DocumentOpenSearchReindexResource( return ResponseEntity.accepted().body(mapOf("status" to "started", "runId" to runId)) } + @GetMapping("/reindex/runs") + fun listRuns( + @RequestParam(defaultValue = "0") page: Int, + @RequestParam(defaultValue = "20") size: Int + ): ResponseEntity>> = + ResponseEntity.ok(reindexService.listRuns(PageRequest.of(page, size))) + @GetMapping("/reindex/status") fun status(): ResponseEntity> = ResponseEntity.ok(reindexService.status()) diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/web/DocumentOpenSearchReindexResourceTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/web/DocumentOpenSearchReindexResourceTest.kt index 4e7917e51d..75838859f7 100644 --- a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/web/DocumentOpenSearchReindexResourceTest.kt +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/web/DocumentOpenSearchReindexResourceTest.kt @@ -24,6 +24,8 @@ import org.junit.jupiter.api.Test import org.mockito.kotlin.any import org.mockito.kotlin.mock import org.mockito.kotlin.whenever +import org.springframework.data.domain.PageImpl +import org.springframework.data.domain.PageRequest import java.util.UUID class DocumentOpenSearchReindexResourceTest { @@ -89,4 +91,23 @@ class DocumentOpenSearchReindexResourceTest { assertThat(response.statusCode.value()).isEqualTo(200) assertThat(response.body?.get("runId")).isEqualTo(runId) } + + @Test + fun `listRuns returns paginated runs`() { + val runId1 = UUID.randomUUID() + val runId2 = UUID.randomUUID() + val runs: List> = listOf( + mapOf("runId" to runId1, "status" to "COMPLETED"), + mapOf("runId" to runId2, "status" to "RUNNING") + ) + val page = PageImpl(runs, PageRequest.of(0, 20), 2) + whenever(reindexService.listRuns(PageRequest.of(0, 20))).thenReturn(page) + + val response = resource.listRuns(0, 20) + + assertThat(response.statusCode.value()).isEqualTo(200) + assertThat(response.body?.content).hasSize(2) + assertThat(response.body?.content?.get(0)?.get("runId")).isEqualTo(runId1) + assertThat(response.body?.totalElements).isEqualTo(2) + } } diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.html b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.html index 580f695b2f..470899bfbe 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.html +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.html @@ -17,109 +17,149 @@
-
-

+ + {{ 'adminSettings.opensearch.reindex.title' | translate }} -

-

- {{ 'adminSettings.opensearch.reindex.description' | translate }} -

-
+ -
- - {{ 'adminSettings.opensearch.reindex.pruneOrphans' | translate }} - - - - - - - -
+
+ +
+ + + + + +
+ + +
+
+
{{ 'adminSettings.opensearch.reindex.parameters' | translate }}
+ +
+
+ + {{ 'adminSettings.opensearch.reindex.documentDefinitionName' | translate }}: + + {{ data.scope?.documentDefinitionName || '-' }} +
+ +
+ + {{ 'adminSettings.opensearch.reindex.pruneOrphans' | translate }}: + + {{ data.scope?.pruneOrphans ? ('interface.yes' | translate) : ('interface.no' | translate) }} +
+ +
+ + {{ 'adminSettings.opensearch.reindex.modifiedBefore' | translate }}: + + {{ data.scope?.modifiedBefore ? (data.scope.modifiedBefore | date:'medium') : '-' }} +
+
+
+ +
+
{{ 'adminSettings.opensearch.reindex.results' | translate }}
- @if (obs.status?.status) { -
-
- @if (obs.status.skippedCount > 0) { -
+
+ @if (data.skippedCount > 0) { +
{{ 'adminSettings.opensearch.reindex.skipped' | translate }}: - {{ obs.status.skippedCount }} + {{ data.skippedCount }}
} - @if (obs.status.startedOn) { -
+ + @if (data.prunedCount > 0) { +
- {{ 'adminSettings.opensearch.reindex.started' | translate }}: + {{ 'adminSettings.opensearch.reindex.pruned' | translate }}: - {{ obs.status.startedOn | date:'medium' }} + {{ data.prunedCount }}
} - @if (obs.status.status === 'RUNNING' && obs.status.elapsedSeconds > 0) { -
+ +
+ + {{ 'adminSettings.opensearch.reindex.started' | translate }}: + + {{ data.startedOn | date:'medium' }} +
+ + @if (data.finishedOn) { +
- {{ 'adminSettings.opensearch.reindex.elapsed' | translate }}: + {{ 'adminSettings.opensearch.reindex.finished' | translate }}: - {{ obs.status.elapsedSeconds }}s + {{ data.finishedOn | date:'medium' }}
} - @if (obs.status.status !== 'RUNNING' && obs.status.finishedOn) { -
+ + @if (data.status === 'RUNNING' && data.elapsedSeconds > 0) { +
- {{ 'adminSettings.opensearch.reindex.finished' | translate }}: + {{ 'adminSettings.opensearch.reindex.elapsed' | translate }}: - {{ obs.status.finishedOn | date:'medium' }} + {{ data.elapsedSeconds }}s
}
- @if (obs.status.error) { + @if (data.error) {
{{ 'adminSettings.opensearch.reindex.errorTitle' | translate }}: - {{ obs.status.error }} + {{ data.error }}
}
- } - -
-
-
+ diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss index 87bd27061f..51b9f6cad1 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss @@ -20,69 +20,48 @@ flex-direction: column; gap: 24px; - &__header { + &__expanded { + padding: 16px 24px; display: flex; flex-direction: column; - gap: 8px; - } - - &__title { - font-size: 16px; - font-weight: 600; - color: var(--cds-text-primary); - margin: 0; - } - - &__description { - font-size: 14px; - color: var(--cds-text-secondary); - margin: 0; + gap: 24px; } - &__options { + &__detail-section { display: flex; flex-direction: column; gap: 16px; - max-width: 400px; + + h6 { + margin: 0; + font-size: 14px; + font-weight: 600; + color: var(--cds-text-secondary); + } } - &__status { - display: flex; - flex-direction: column; + &__detail-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(250px, 1fr)); gap: 16px; - padding: 16px; - background-color: var(--cds-layer-01); - border-radius: 4px; } - &__status-row { + &__detail-item { display: flex; align-items: center; gap: 8px; + font-size: 14px; + color: var(--cds-text-primary); } &__label { - font-size: 14px; font-weight: 600; color: var(--cds-text-primary); } &__progress { padding: 8px 0; - } - - &__stats { - display: flex; - flex-wrap: wrap; - gap: 24px; - } - - &__stat { - display: flex; - align-items: center; - gap: 8px; - font-size: 14px; - color: var(--cds-text-primary); + max-width: 500px; } &__error { @@ -97,13 +76,4 @@ margin-right: 8px; } } - - &__actions { - display: flex; - gap: 12px; - - button { - min-width: 140px; - } - } } diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.ts b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.ts index 9ebe154a25..fbf28cdf19 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.ts +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.ts @@ -16,7 +16,6 @@ import {ChangeDetectionStrategy, Component, OnDestroy, OnInit} from '@angular/core'; import {CommonModule, DatePipe} from '@angular/common'; -import {FormBuilder, FormGroup, ReactiveFormsModule} from '@angular/forms'; import {TranslateModule} from '@ngx-translate/core'; import { BehaviorSubject, @@ -24,8 +23,6 @@ import { interval, map, Observable, - of, - shareReplay, startWith, Subject, switchMap, @@ -35,18 +32,18 @@ import { } from 'rxjs'; import { ButtonModule, - CheckboxModule, - DatePickerInputModule, - DatePickerModule, - DropdownModule, + IconModule, ListItem, LoadingModule, ProgressBarModule, TagModule, } from 'carbon-components-angular'; +import {CarbonListModule, ColumnConfig, Pagination, ViewType} from '@valtimo/components'; +import {Page} from '@valtimo/shared'; import {AdminSettingsManagementApiService} from '../../services'; import {ReindexStatusDto, StartReindexRequestDto} from '../../models'; import {DocumentService} from '@valtimo/document'; +import {StartReindexModalComponent} from '../start-reindex-modal/start-reindex-modal.component'; @Component({ standalone: true, @@ -57,15 +54,13 @@ import {DocumentService} from '@valtimo/document'; CommonModule, DatePipe, TranslateModule, - ReactiveFormsModule, ButtonModule, - CheckboxModule, - DatePickerInputModule, - DatePickerModule, - DropdownModule, + IconModule, LoadingModule, ProgressBarModule, TagModule, + CarbonListModule, + StartReindexModalComponent, ], changeDetection: ChangeDetectionStrategy.OnPush, }) @@ -73,40 +68,55 @@ export class AdminSettingsOpensearchComponent implements OnInit, OnDestroy { private readonly _destroy$ = new Subject(); private readonly _refresh$ = new BehaviorSubject(undefined); - public readonly startingReindex$ = new BehaviorSubject(false); + public readonly fields: ColumnConfig[] = [ + {key: 'statusTag', label: 'adminSettings.opensearch.reindex.columns.status', viewType: ViewType.TAGS}, + {key: 'startedOn', label: 'adminSettings.opensearch.reindex.columns.startedOn', viewType: ViewType.DATE}, + {key: 'finishedOn', label: 'adminSettings.opensearch.reindex.columns.finishedOn', viewType: ViewType.DATE}, + {key: 'progress', label: 'adminSettings.opensearch.reindex.columns.progress', viewType: ViewType.TEXT}, + ]; + + public readonly pagination: Pagination = { + collectionSize: 0, + page: 1, + size: 10, + }; - public readonly formGroup: FormGroup = this._fb.group({ - pruneOrphans: [false], - documentDefinitionName: [null], - modifiedBefore: [null], - }); + public readonly showModal$ = new BehaviorSubject(false); + public readonly startingReindex$ = new BehaviorSubject(false); + public readonly loading$ = new BehaviorSubject(false); public documentDefinitions$: Observable; - public readonly reindexStatus$: Observable = this._refresh$.pipe( - switchMap(() => this._apiService.getReindexStatus()), - switchMap(status => { - if (status?.status === 'RUNNING') { - return interval(2000).pipe( - startWith(0), - switchMap(() => this._apiService.getReindexStatus()), - takeWhile(s => s?.status === 'RUNNING', true), - takeUntil(this._destroy$) - ); - } - return of(status); - }), - shareReplay(1) + public readonly runs$: Observable> = this._refresh$.pipe( + switchMap(() => { + this.loading$.next(true); + return this._apiService.getReindexRuns(this.pagination.page - 1, this.pagination.size).pipe( + finalize(() => this.loading$.next(false)) + ); + }) ); - public readonly isRunning$: Observable = this.reindexStatus$.pipe( - map(status => status?.status === 'RUNNING') + public readonly tableItems$: Observable = this.runs$.pipe( + map(page => { + this.pagination.collectionSize = page.totalElements; + return page.content.map(run => ({ + ...run, + progress: `${run.processedCount} / ${run.totalCount}`, + statusTag: { + content: run.status, + type: this._getStatusTagType(run.status), + }, + })); + }) + ); + + public readonly hasRunningRun$: Observable = this.runs$.pipe( + map(page => page.content.some(run => run.status === 'RUNNING')) ); constructor( private readonly _apiService: AdminSettingsManagementApiService, - private readonly _documentService: DocumentService, - private readonly _fb: FormBuilder + private readonly _documentService: DocumentService ) {} public ngOnInit(): void { @@ -119,11 +129,35 @@ export class AdminSettingsOpensearchComponent implements OnInit, OnDestroy { ), startWith([]) ); + + this.hasRunningRun$ + .pipe( + switchMap(hasRunning => { + if (!hasRunning) return []; + return interval(3000).pipe( + takeWhile(() => true), + takeUntil(this._destroy$) + ); + }), + takeUntil(this._destroy$) + ) + .subscribe(() => this._refresh$.next()); + } + + public onPageChange(page: number): void { + this.pagination.page = page; + this._refresh$.next(); } - public startReindex(): void { + public openModal(): void { + this.showModal$.next(true); + } + + public onModalClose(request: StartReindexRequestDto | null): void { + this.showModal$.next(false); + if (!request) return; + this.startingReindex$.next(true); - const request = this._buildReindexRequest(); this._apiService .startReindex(request) .pipe( @@ -140,40 +174,7 @@ export class AdminSettingsOpensearchComponent implements OnInit, OnDestroy { }); } - public onDateSelected(event: string[]): void { - const dateValue = event?.[0] || null; - this.formGroup.patchValue({modifiedBefore: dateValue}); - } - - private _buildReindexRequest(): StartReindexRequestDto { - const formValue = this.formGroup.value; - const request: StartReindexRequestDto = {}; - - if (formValue.pruneOrphans) { - request.pruneOrphans = true; - } - - if (formValue.documentDefinitionName?.content) { - request.documentDefinitionName = formValue.documentDefinitionName.content; - } - - if (formValue.modifiedBefore) { - request.modifiedBefore = this._formatDateToIso(formValue.modifiedBefore); - } - - return request; - } - - private _formatDateToIso(dateStr: string): string { - const parts = dateStr.split('-'); - if (parts.length === 3) { - const [day, month, year] = parts; - return `${year}-${month}-${day}T23:59:59`; - } - return dateStr; - } - - public getStatusTagType(status: string): string { + private _getStatusTagType(status: string): string { switch (status) { case 'RUNNING': return 'blue'; diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/start-reindex-modal/start-reindex-modal.component.html b/frontend/projects/valtimo/admin-settings/src/lib/components/start-reindex-modal/start-reindex-modal.component.html new file mode 100644 index 0000000000..1f821baf54 --- /dev/null +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/start-reindex-modal/start-reindex-modal.component.html @@ -0,0 +1,52 @@ + + + + +

{{ 'adminSettings.opensearch.reindex.startModalTitle' | translate }}

+ + +
+ + {{ 'adminSettings.opensearch.reindex.pruneOrphans' | translate }} + + + + + + + +
+ + + + + + + diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/start-reindex-modal/start-reindex-modal.component.scss b/frontend/projects/valtimo/admin-settings/src/lib/components/start-reindex-modal/start-reindex-modal.component.scss new file mode 100644 index 0000000000..68de23d640 --- /dev/null +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/start-reindex-modal/start-reindex-modal.component.scss @@ -0,0 +1,21 @@ +/*! + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +.start-reindex-form { + display: flex; + flex-direction: column; + gap: 1rem; +} diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/start-reindex-modal/start-reindex-modal.component.ts b/frontend/projects/valtimo/admin-settings/src/lib/components/start-reindex-modal/start-reindex-modal.component.ts new file mode 100644 index 0000000000..7f3c83ec0b --- /dev/null +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/start-reindex-modal/start-reindex-modal.component.ts @@ -0,0 +1,117 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {ChangeDetectionStrategy, Component, EventEmitter, Input, Output} from '@angular/core'; +import {CommonModule} from '@angular/common'; +import {FormBuilder, FormGroup, ReactiveFormsModule} from '@angular/forms'; +import {TranslateModule} from '@ngx-translate/core'; +import { + ButtonModule, + CheckboxModule, + DatePickerInputModule, + DatePickerModule, + DropdownModule, + ListItem, + ModalModule, +} from 'carbon-components-angular'; +import {ValtimoCdsModalDirective} from '@valtimo/components'; +import {StartReindexRequestDto} from '../../models'; + +@Component({ + standalone: true, + selector: 'valtimo-start-reindex-modal', + templateUrl: './start-reindex-modal.component.html', + styleUrls: ['./start-reindex-modal.component.scss'], + imports: [ + CommonModule, + TranslateModule, + ReactiveFormsModule, + ModalModule, + ValtimoCdsModalDirective, + ButtonModule, + CheckboxModule, + DatePickerInputModule, + DatePickerModule, + DropdownModule, + ], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class StartReindexModalComponent { + @Input() public open = false; + @Input() public documentDefinitions: ListItem[] = []; + + @Output() public readonly closeEvent = new EventEmitter(); + + public readonly formGroup: FormGroup = this._fb.group({ + pruneOrphans: [false], + documentDefinitionName: [null], + modifiedBefore: [null], + }); + + constructor(private readonly _fb: FormBuilder) {} + + public onDateSelected(event: string[]): void { + const dateValue = event?.[0] || null; + this.formGroup.patchValue({modifiedBefore: dateValue}); + } + + public onCancel(): void { + this._resetForm(); + this.closeEvent.emit(null); + } + + public onSubmit(): void { + const request = this._buildRequest(); + this._resetForm(); + this.closeEvent.emit(request); + } + + private _buildRequest(): StartReindexRequestDto { + const formValue = this.formGroup.value; + const request: StartReindexRequestDto = {}; + + if (formValue.pruneOrphans) { + request.pruneOrphans = true; + } + + if (formValue.documentDefinitionName?.content) { + request.documentDefinitionName = formValue.documentDefinitionName.content; + } + + if (formValue.modifiedBefore) { + request.modifiedBefore = this._formatDateToIso(formValue.modifiedBefore); + } + + return request; + } + + private _formatDateToIso(dateStr: string): string { + const parts = dateStr.split('-'); + if (parts.length === 3) { + const [day, month, year] = parts; + return `${year}-${month}-${day}T23:59:59`; + } + return dateStr; + } + + private _resetForm(): void { + this.formGroup.reset({ + pruneOrphans: false, + documentDefinitionName: null, + modifiedBefore: null, + }); + } +} diff --git a/frontend/projects/valtimo/admin-settings/src/lib/models/reindex.model.ts b/frontend/projects/valtimo/admin-settings/src/lib/models/reindex.model.ts index f77a09f7f6..499acd9b1f 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/models/reindex.model.ts +++ b/frontend/projects/valtimo/admin-settings/src/lib/models/reindex.model.ts @@ -16,12 +16,25 @@ type ReindexStatus = 'RUNNING' | 'COMPLETED' | 'FAILED' | 'STOPPED'; +interface ReindexScope { + pruneOrphans?: boolean; + documentDefinitionName?: string; + modifiedBefore?: string; + modifiedAfter?: string; + pageSize?: number; + resumeRunId?: string; + documentIds?: string[]; +} + interface ReindexStatusDto { runId: string; status: ReindexStatus; + running: boolean; + scope: ReindexScope | null; totalCount: number; processedCount: number; skippedCount: number; + prunedCount: number; startedOn: string; finishedOn: string | null; elapsedSeconds: number; @@ -39,4 +52,10 @@ interface StartReindexResponseDto { runId: string; } -export {ReindexStatus, ReindexStatusDto, StartReindexRequestDto, StartReindexResponseDto}; +export { + ReindexStatus, + ReindexScope, + ReindexStatusDto, + StartReindexRequestDto, + StartReindexResponseDto, +}; diff --git a/frontend/projects/valtimo/admin-settings/src/lib/services/admin-settings-management-api.service.ts b/frontend/projects/valtimo/admin-settings/src/lib/services/admin-settings-management-api.service.ts index 001bb89686..d881a8e6ea 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/services/admin-settings-management-api.service.ts +++ b/frontend/projects/valtimo/admin-settings/src/lib/services/admin-settings-management-api.service.ts @@ -15,7 +15,7 @@ */ import {Injectable} from '@angular/core'; -import {HttpClient} from '@angular/common/http'; +import {HttpClient, HttpParams} from '@angular/common/http'; import {catchError, Observable, of} from 'rxjs'; import { AdminSettingsLogoDto, @@ -23,6 +23,7 @@ import { BaseApiService, ConfigService, CreateAdminSettingsLogoDto, + Page, } from '@valtimo/shared'; import { AccentColorsDto, @@ -124,4 +125,18 @@ export class AdminSettingsManagementApiService extends BaseApiService { .get(this.getApiUrl('/management/v1/document-opensearch/reindex/status')) .pipe(catchError(() => of(null))); } + + public getReindexRuns(page: number, size: number): Observable> { + const params = new HttpParams().set('page', page.toString()).set('size', size.toString()); + return this.httpClient.get>( + this.getApiUrl('/management/v1/document-opensearch/reindex/runs'), + {params} + ); + } + + public getReindexRun(runId: string): Observable { + return this.httpClient.get( + this.getApiUrl(`/management/v1/document-opensearch/reindex/${runId}`) + ); + } } diff --git a/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.ts b/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.ts index 2f7a36a868..0bc2fa671a 100644 --- a/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.ts +++ b/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.ts @@ -1,19 +1,17 @@ /* + * Copyright 2015-2026 Ritense BV, the Netherlands. * - * * Copyright 2015-2026 Ritense BV, the Netherlands. - * * - * * Licensed under EUPL, Version 1.2 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" basis, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ import { AfterViewInit, @@ -207,6 +205,7 @@ export class CarbonListComponent implements OnInit, AfterViewInit, OnDestroy { @Input() movingRowsEnabled: boolean; @Input() dragAndDrop = false; @Input() dragAndDropDisabled = false; + @Input() expandedRowTemplate: TemplateRef; @Output() rowClicked = new EventEmitter(); @Output() paginationClicked = new EventEmitter(); @@ -469,51 +468,60 @@ export class CarbonListComponent implements OnInit, AfterViewInit, OnDestroy { ]).pipe( filter(([fields, items, viewInitialized]) => !!fields && !!items && viewInitialized), map(([fields, items]) => - items.map((item: CarbonListItem, index: number) => [ - ...this.getDragAndDropItemsItems(item, index, items.length), - ...fields.map((field: ColumnConfig) => { - switch (field.viewType) { - case ViewType.TEMPLATE: - return new TableItem({ - data: {item, index, length: items.length, ...field.templateData}, - item, - template: field.template, - }); - case ViewType.BOOLEAN: - let data = this.resolveObject(field, item); - data = !BOOLEAN_CONVERTER_VALUES.includes(data) - ? data - : `${'viewTypeConverter.' + data}`; - return new TableItem({ - data, - template: this.booleanTemplate, - item, - }); - case ViewType.TAGS: { - return new TableItem({ - data: { - tags: this.resolveTagObject(item, field.key), - tagAmount: field?.tagAmount || 1, - }, - item, - template: this.tagTemplate, - }); + items.map((item: CarbonListItem, index: number) => { + const row = [ + ...this.getDragAndDropItemsItems(item, index, items.length), + ...fields.map((field: ColumnConfig) => { + switch (field.viewType) { + case ViewType.TEMPLATE: + return new TableItem({ + data: {item, index, length: items.length, ...field.templateData}, + item, + template: field.template, + }); + case ViewType.BOOLEAN: + let data = this.resolveObject(field, item); + data = !BOOLEAN_CONVERTER_VALUES.includes(data) + ? data + : `${'viewTypeConverter.' + data}`; + return new TableItem({ + data, + template: this.booleanTemplate, + item, + }); + case ViewType.TAGS: { + return new TableItem({ + data: { + tags: this.resolveTagObject(item, field.key), + tagAmount: field?.tagAmount || 1, + }, + item, + template: this.tagTemplate, + }); + } + default: + const resolvedObject: string = this.resolveObject(field, item); + return new TableItem({ + title: resolvedObject ?? '-', + data: + (field.tooltipCharLimit + ? this.ellipsisPipe.transform(resolvedObject, field.tooltipCharLimit) + : resolvedObject) ?? '-', + template: this.defaultTemplate, + item, + }); } - default: - const resolvedObject: string = this.resolveObject(field, item); - return new TableItem({ - title: resolvedObject ?? '-', - data: - (field.tooltipCharLimit - ? this.ellipsisPipe.transform(resolvedObject, field.tooltipCharLimit) - : resolvedObject) ?? '-', - template: this.defaultTemplate, - item, - }); - } - }), - ...this.getExtraItems(item, index, items.length), - ]) + }), + ...this.getExtraItems(item, index, items.length), + ]; + + if (this.expandedRowTemplate && row.length > 0) { + row[0].expandedData = item; + row[0].expandedTemplate = this.expandedRowTemplate; + } + + return row; + }) ), tap((data: TableItem[][]) => { this._completeDataSource = data; diff --git a/frontend/projects/valtimo/shared/assets/core/en.json b/frontend/projects/valtimo/shared/assets/core/en.json index 82bd9c8c6b..9cc08c99f8 100644 --- a/frontend/projects/valtimo/shared/assets/core/en.json +++ b/frontend/projects/valtimo/shared/assets/core/en.json @@ -3317,21 +3317,37 @@ "opensearch": { "title": "OpenSearch", "reindex": { - "title": "Reindex Documents", + "title": "Reindex Runs", "description": "Reindex all documents from the database to OpenSearch. This operation runs in the background.", "startButton": "Start Reindex", + "startModalTitle": "Start Reindex", + "detailTitle": "Run Details", + "parameters": "Parameters", + "results": "Results", "status": "Status", "processed": "Processed", "skipped": "Skipped", + "pruned": "Pruned", "started": "Started", "elapsed": "Elapsed time", "finished": "Finished", "inProgress": "Reindexing in progress...", "errorTitle": "Error", - "pruneOrphans": "Prune orphaned documents from OpenSearch", + "pruneOrphans": "Prune orphaned documents", "documentDefinitionName": "Document definition", "documentDefinitionPlaceholder": "All document definitions", "modifiedBefore": "Modified before", + "columns": { + "status": "Status", + "startedOn": "Started", + "finishedOn": "Finished", + "progress": "Progress", + "documentDefinition": "Document definition" + }, + "noResults": { + "title": "No reindex runs", + "description": "No reindex runs have been started yet." + }, "statuses": { "RUNNING": "Running", "COMPLETED": "Completed", diff --git a/frontend/projects/valtimo/shared/assets/core/nl.json b/frontend/projects/valtimo/shared/assets/core/nl.json index 667aa71de2..b6b5c8e059 100644 --- a/frontend/projects/valtimo/shared/assets/core/nl.json +++ b/frontend/projects/valtimo/shared/assets/core/nl.json @@ -3350,21 +3350,37 @@ "opensearch": { "title": "OpenSearch", "reindex": { - "title": "Documenten herindexeren", + "title": "Herindexering runs", "description": "Herindexeer alle documenten van de database naar OpenSearch. Deze operatie draait op de achtergrond.", "startButton": "Start herindexering", + "startModalTitle": "Start herindexering", + "detailTitle": "Run details", + "parameters": "Parameters", + "results": "Resultaten", "status": "Status", "processed": "Verwerkt", "skipped": "Overgeslagen", + "pruned": "Opgeschoond", "started": "Gestart", "elapsed": "Verstreken tijd", "finished": "Voltooid", "inProgress": "Herindexering bezig...", "errorTitle": "Fout", - "pruneOrphans": "Verwijder orphan documenten uit OpenSearch", + "pruneOrphans": "Verwijder orphan documenten", "documentDefinitionName": "Documentdefinitie", "documentDefinitionPlaceholder": "Alle documentdefinities", "modifiedBefore": "Gewijzigd voor", + "columns": { + "status": "Status", + "startedOn": "Gestart", + "finishedOn": "Voltooid", + "progress": "Voortgang", + "documentDefinition": "Documentdefinitie" + }, + "noResults": { + "title": "Geen herindexering runs", + "description": "Er zijn nog geen herindexering runs gestart." + }, "statuses": { "RUNNING": "Bezig", "COMPLETED": "Voltooid", From 6ce480f7ef0a99de0e1a8863b9c3a4343b0c4db5 Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Mon, 13 Jul 2026 11:35:26 +0200 Subject: [PATCH 25/46] made the expanded rows consistent with carbon design --- .../carbon-list/carbon-list.component.scss | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.scss b/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.scss index aac97511e6..15d0506f2d 100644 --- a/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.scss +++ b/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.scss @@ -1,5 +1,5 @@ /*! - * Copyright 2015-2025 Ritense BV, the Netherlands. + * Copyright 2015-2026 Ritense BV, the Netherlands. * * Licensed under EUPL, Version 1.2 (the "License"); * you may not use this file except in compliance with the License. @@ -135,6 +135,19 @@ td:first-child { ::ng-deep cds-table { overflow: hidden; } + + ::ng-deep .cds--expandable-row:not(.cds--parent-row) td { + border-top: none; + border-bottom-width: 2px; + } + + ::ng-deep tbody .cds--expandable-row.cds--parent-row td { + border-bottom-width: 2px; + } + + ::ng-deep tbody tr:first-child td { + border-top: none !important; + } } .valtimo-search-container { From 309390d0e9465be1b684df16cf589ec1d0ffbfe2 Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Mon, 13 Jul 2026 14:30:29 +0200 Subject: [PATCH 26/46] many reindexing page improvements --- .../DocumentOpenSearchReindexService.kt | 6 ++- .../domain/impl/JsonSchemaDocument.java | 3 +- .../20260710-add-reindex-run-pruned-count.xml | 32 +++++++++++++++ .../admin-settings-opensearch.component.html | 37 +++++++++-------- .../admin-settings-opensearch.component.scss | 6 +++ .../admin-settings-opensearch.component.ts | 40 ++++++++++++------- .../start-reindex-modal.component.html | 9 +++-- .../start-reindex-modal.component.scss | 6 +++ .../start-reindex-modal.component.ts | 29 +++++++++----- .../src/lib/models/reindex.model.ts | 2 +- .../valtimo/shared/assets/core/en.json | 6 +++ .../valtimo/shared/assets/core/nl.json | 6 +++ 12 files changed, 131 insertions(+), 51 deletions(-) create mode 100644 backend/core/src/main/resources/config/liquibase/13-32-0/20260710-add-reindex-run-pruned-count.xml diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt index 9c0b4ddb2f..de788bbf99 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt @@ -39,6 +39,7 @@ import org.springframework.transaction.support.TransactionTemplate import java.time.Duration import java.time.Instant import java.time.LocalDateTime +import java.time.format.DateTimeFormatter import java.util.UUID import java.util.concurrent.ExecutorService import java.util.concurrent.Executors @@ -231,10 +232,10 @@ open class DocumentOpenSearchReindexService( boolQuery.filter(QueryBuilders.termQuery("definitionId.name", it)) } scope.modifiedAfter?.let { - boolQuery.filter(QueryBuilders.rangeQuery("modifiedOn").gt(it)) + boolQuery.filter(QueryBuilders.rangeQuery("modifiedOn").gt(it.format(OS_DATE_FORMAT))) } scope.modifiedBefore?.let { - boolQuery.filter(QueryBuilders.rangeQuery("modifiedOn").lt(it)) + boolQuery.filter(QueryBuilders.rangeQuery("modifiedOn").lt(it.format(OS_DATE_FORMAT))) } scope.documentIds?.takeIf { it.isNotEmpty() }?.let { ids -> boolQuery.filter(QueryBuilders.idsQuery().addIds(*ids.map { it.toString() }.toTypedArray())) @@ -274,6 +275,7 @@ open class DocumentOpenSearchReindexService( companion object { private val logger = KotlinLogging.logger {} + private val OS_DATE_FORMAT: DateTimeFormatter = DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ss.SSS") const val LOCK_NAME = "document-opensearch-reindex" diff --git a/backend/case/src/main/java/com/ritense/document/domain/impl/JsonSchemaDocument.java b/backend/case/src/main/java/com/ritense/document/domain/impl/JsonSchemaDocument.java index 4139d58dd1..91b27285b3 100644 --- a/backend/case/src/main/java/com/ritense/document/domain/impl/JsonSchemaDocument.java +++ b/backend/case/src/main/java/com/ritense/document/domain/impl/JsonSchemaDocument.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2024 Ritense BV, the Netherlands. + * Copyright 2015-2026 Ritense BV, the Netherlands. * * Licensed under EUPL, Version 1.2 (the "License"); * you may not use this file except in compliance with the License. @@ -189,6 +189,7 @@ private JsonSchemaDocument( this.content = content; this.documentDefinitionId = documentDefinition.id(); this.createdOn = LocalDateTime.now(); + this.modifiedOn = this.createdOn; this.createdBy = createdBy; this.sequence = sequence; diff --git a/backend/core/src/main/resources/config/liquibase/13-32-0/20260710-add-reindex-run-pruned-count.xml b/backend/core/src/main/resources/config/liquibase/13-32-0/20260710-add-reindex-run-pruned-count.xml new file mode 100644 index 0000000000..fc0554c233 --- /dev/null +++ b/backend/core/src/main/resources/config/liquibase/13-32-0/20260710-add-reindex-run-pruned-count.xml @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.html b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.html index 470899bfbe..96a4a498e4 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.html +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.html @@ -33,6 +33,7 @@ [pagination]="pagination" [expandedRowTemplate]="expandedRowTemplate" (paginationClicked)="onPageChange($event)" + (paginationSet)="onPageSizeChange($event)" > {{ 'adminSettings.opensearch.reindex.title' | translate }} @@ -79,18 +80,19 @@
{{ 'adminSettings.opensearch.reindex.parameters' | translate }}
{{ data.scope?.documentDefinitionName || '-' }}
-
+
{{ 'adminSettings.opensearch.reindex.pruneOrphans' | translate }}: {{ data.scope?.pruneOrphans ? ('interface.yes' | translate) : ('interface.no' | translate) }} +
- {{ 'adminSettings.opensearch.reindex.modifiedBefore' | translate }}: + {{ 'adminSettings.opensearch.reindex.modifiedAfter' | translate }}: - {{ data.scope?.modifiedBefore ? (data.scope.modifiedBefore | date:'medium') : '-' }} + {{ data.scope?.modifiedAfter ? (data.scope.modifiedAfter | date:'medium') : '-' }}
@@ -108,23 +110,20 @@
{{ 'adminSettings.opensearch.reindex.results' | translate }}
- @if (data.skippedCount > 0) { -
- - {{ 'adminSettings.opensearch.reindex.skipped' | translate }}: - - {{ data.skippedCount }} -
- } +
+ + {{ 'adminSettings.opensearch.reindex.skipped' | translate }}: + + {{ data.skippedCount || 0 }} +
- @if (data.prunedCount > 0) { -
- - {{ 'adminSettings.opensearch.reindex.pruned' | translate }}: - - {{ data.prunedCount }} -
- } +
+ + {{ 'adminSettings.opensearch.reindex.pruned' | translate }}: + + {{ data.prunedCount || 0 }} + +
diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss index 51b9f6cad1..ea0ebb59aa 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss @@ -59,6 +59,12 @@ color: var(--cds-text-primary); } + &__detail-item--with-tooltip { + display: inline-flex; + align-items: center; + gap: 4px; + } + &__progress { padding: 8px 0; max-width: 500px; diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.ts b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.ts index fbf28cdf19..bb25d6eb3a 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.ts +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.ts @@ -16,7 +16,7 @@ import {ChangeDetectionStrategy, Component, OnDestroy, OnInit} from '@angular/core'; import {CommonModule, DatePipe} from '@angular/common'; -import {TranslateModule} from '@ngx-translate/core'; +import {TranslateModule, TranslateService} from '@ngx-translate/core'; import { BehaviorSubject, finalize, @@ -38,7 +38,7 @@ import { ProgressBarModule, TagModule, } from 'carbon-components-angular'; -import {CarbonListModule, ColumnConfig, Pagination, ViewType} from '@valtimo/components'; +import {CarbonListModule, ColumnConfig, Pagination, TooltipIconModule, ViewType} from '@valtimo/components'; import {Page} from '@valtimo/shared'; import {AdminSettingsManagementApiService} from '../../services'; import {ReindexStatusDto, StartReindexRequestDto} from '../../models'; @@ -61,6 +61,7 @@ import {StartReindexModalComponent} from '../start-reindex-modal/start-reindex-m TagModule, CarbonListModule, StartReindexModalComponent, + TooltipIconModule, ], changeDetection: ChangeDetectionStrategy.OnPush, }) @@ -75,7 +76,7 @@ export class AdminSettingsOpensearchComponent implements OnInit, OnDestroy { {key: 'progress', label: 'adminSettings.opensearch.reindex.columns.progress', viewType: ViewType.TEXT}, ]; - public readonly pagination: Pagination = { + public pagination: Pagination = { collectionSize: 0, page: 1, size: 10, @@ -97,16 +98,19 @@ export class AdminSettingsOpensearchComponent implements OnInit, OnDestroy { ); public readonly tableItems$: Observable = this.runs$.pipe( - map(page => { - this.pagination.collectionSize = page.totalElements; - return page.content.map(run => ({ - ...run, - progress: `${run.processedCount} / ${run.totalCount}`, - statusTag: { - content: run.status, - type: this._getStatusTagType(run.status), - }, - })); + switchMap(page => { + this.pagination = {...this.pagination, collectionSize: page.totalElements}; + const statusKeys = page.content.map(run => `adminSettings.opensearch.reindex.statuses.${run.status}`); + return this._translateService.get(statusKeys).pipe( + map(translations => page.content.map(run => ({ + ...run, + progress: `${run.processedCount} / ${run.totalCount}`, + statusTag: { + content: translations[`adminSettings.opensearch.reindex.statuses.${run.status}`], + type: this._getStatusTagType(run.status), + }, + }))) + ); }) ); @@ -116,7 +120,8 @@ export class AdminSettingsOpensearchComponent implements OnInit, OnDestroy { constructor( private readonly _apiService: AdminSettingsManagementApiService, - private readonly _documentService: DocumentService + private readonly _documentService: DocumentService, + private readonly _translateService: TranslateService ) {} public ngOnInit(): void { @@ -145,7 +150,12 @@ export class AdminSettingsOpensearchComponent implements OnInit, OnDestroy { } public onPageChange(page: number): void { - this.pagination.page = page; + this.pagination = {...this.pagination, page}; + this._refresh$.next(); + } + + public onPageSizeChange(size: number): void { + this.pagination = {...this.pagination, size, page: 1}; this._refresh$.next(); } diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/start-reindex-modal/start-reindex-modal.component.html b/frontend/projects/valtimo/admin-settings/src/lib/components/start-reindex-modal/start-reindex-modal.component.html index 1f821baf54..373e84a63f 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/components/start-reindex-modal/start-reindex-modal.component.html +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/start-reindex-modal/start-reindex-modal.component.html @@ -19,9 +19,12 @@

{{ 'adminSettings.opensearch.reindex.startModalTitle' | translate }}

-
+
- {{ 'adminSettings.opensearch.reindex.pruneOrphans' | translate }} + + {{ 'adminSettings.opensearch.reindex.pruneOrphans' | translate }} + + {{ 'adminSettings.opensearch.reindex.startModalTitle' Date: Mon, 13 Jul 2026 17:05:35 +0200 Subject: [PATCH 27/46] performance improvements for reindexing (caching, storing final count when a run is finished) + more UI improvements --- .../opensearch/domain/OpenSearchReindexRun.kt | 3 + .../DocumentOpenSearchReindexService.kt | 2 +- .../service/JsonSchemaDocumentOsConverter.kt | 4 +- .../service/OpenSearchReindexRunService.kt | 32 +++++++++- .../BaseOpenSearchIntegrationTest.kt | 45 ++++++++------ .../DocumentOpenSearchReconcileIntTest.kt | 7 ++- .../DocumentOpenSearchReindexServiceTest.kt | 2 +- .../OpenSearchReindexRunServiceTest.kt | 27 ++++----- .../src/test/resources/config/application.yml | 1 + .../liquibase/13-37-0/13-37-0-master.xml | 2 + .../20260713-add-reindex-run-total-count.xml | 30 ++++++++++ .../13-37-0/20260713-backfill-modified-on.xml | 31 ++++++++++ .../admin-settings-opensearch.component.html | 15 +++-- .../admin-settings-opensearch.component.scss | 21 ++++--- .../admin-settings-opensearch.component.ts | 59 ++++++++++++++----- .../carbon-list/carbon-list.component.ts | 32 ++++++++++ .../valtimo/shared/assets/core/en.json | 1 + .../valtimo/shared/assets/core/nl.json | 1 + 18 files changed, 243 insertions(+), 72 deletions(-) create mode 100644 backend/core/src/main/resources/config/liquibase/13-37-0/20260713-add-reindex-run-total-count.xml create mode 100644 backend/core/src/main/resources/config/liquibase/13-37-0/20260713-backfill-modified-on.xml diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/OpenSearchReindexRun.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/OpenSearchReindexRun.kt index e9d02cb491..c4797ed44a 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/OpenSearchReindexRun.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/OpenSearchReindexRun.kt @@ -77,6 +77,9 @@ class OpenSearchReindexRun( @Column(name = "error") var error: String? = null, + + @Column(name = "total_count") + var totalCount: Long? = null, ) { /** Records progress after a committed batch: the keyset cursor, counts and a fresh heartbeat. */ diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt index de788bbf99..9df8068b50 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt @@ -154,7 +154,7 @@ open class DocumentOpenSearchReindexService( } indexOps().refresh() logger.info { "Re-index run $runId complete (processed=$processed, skipped=$skipped)" } - runService.complete(runId) + runService.complete(runId, processed + skipped) } } catch (e: Exception) { runService.fail(runId, e.message) diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOsConverter.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOsConverter.kt index 7fb8b460db..1dc9a9a164 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOsConverter.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOsConverter.kt @@ -45,7 +45,9 @@ open class JsonSchemaDocumentOsConverter( val tree = objectMapper.valueToTree(document) return objectMapper.treeToValue(tree, JsonSchemaDocumentOsDocument::class.java) .copy( - contentText = extractLeafValues(tree.get("content")), + // Extract content text directly from the document's content, not from the serialized tree + // (the tree may not include content if the getter isn't JavaBean-named). + contentText = extractLeafValues(document.content().asJson()), // JPA optimistic-lock counter drives the OpenSearch external version; +1 keeps it ≥ 1. indexVersion = (document.version() ?: 0).toLong() + 1, ) diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunService.kt index ca851c4eb8..31b1364240 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunService.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunService.kt @@ -32,8 +32,10 @@ import org.springframework.data.domain.PageImpl import org.springframework.data.domain.Pageable import org.springframework.transaction.annotation.Transactional import java.time.Duration +import java.time.Instant import java.time.LocalDateTime import java.util.UUID +import java.util.concurrent.ConcurrentHashMap /** * Thin transactional wrapper around [OpenSearchReindexRunRepository] for managing re-index run state. @@ -50,6 +52,8 @@ open class OpenSearchReindexRunService( private val entityManager: EntityManager, ) { + private val totalCountCache = ConcurrentHashMap>() + /** * On startup, reconcile any [ReindexRunStatus.RUNNING] row whose heartbeat has gone stale (older than * [OpenSearchProperties.Reindex.runningHeartbeatTimeout]): the instance that owned it has crashed or @@ -116,22 +120,26 @@ open class OpenSearchReindexRunService( repository.save(run) } - open fun complete(runId: UUID) { + open fun complete(runId: UUID, totalCount: Long?) { val run = requireRun(runId) + run.totalCount = totalCount run.complete(LocalDateTime.now()) repository.save(run) + totalCountCache.remove(runId) } open fun fail(runId: UUID, error: String?) { val run = requireRun(runId) run.fail(LocalDateTime.now(), error) repository.save(run) + totalCountCache.remove(runId) } open fun stop(runId: UUID) { val run = requireRun(runId) run.stop(LocalDateTime.now()) repository.save(run) + totalCountCache.remove(runId) } open fun recordPruned(runId: UUID, pruned: Long) { @@ -174,7 +182,7 @@ open class OpenSearchReindexRunService( "processedCount" to run.processedCount, "skippedCount" to run.skippedCount, "prunedCount" to run.prunedCount, - "totalCount" to countDocuments(scope), + "totalCount" to getTotalCount(run, scope), "startedOn" to run.startedOn, "heartbeatOn" to run.heartbeatOn, "finishedOn" to run.finishedOn, @@ -183,6 +191,25 @@ open class OpenSearchReindexRunService( ) } + private fun getTotalCount(run: OpenSearchReindexRun, scope: ReindexRequest?): Long { + run.totalCount?.let { return it } + + if (run.status == ReindexRunStatus.RUNNING) { + val cached = totalCountCache[run.id] + if (cached != null && Instant.now().isBefore(cached.second)) { + return cached.first + } + } + + val count = countDocuments(scope) + + if (run.status == ReindexRunStatus.RUNNING) { + totalCountCache[run.id] = count to Instant.now().plus(TOTAL_COUNT_CACHE_TTL) + } + + return count + } + private fun serializeScope(request: ReindexRequest): String? = try { objectMapper.writeValueAsString(request) @@ -225,5 +252,6 @@ open class OpenSearchReindexRunService( companion object { private val logger = KotlinLogging.logger {} + private val TOTAL_COUNT_CACHE_TTL: Duration = Duration.ofMinutes(5) } } diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/BaseOpenSearchIntegrationTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/BaseOpenSearchIntegrationTest.kt index 3e16371612..015857aaee 100644 --- a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/BaseOpenSearchIntegrationTest.kt +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/BaseOpenSearchIntegrationTest.kt @@ -1,19 +1,17 @@ /* + * Copyright 2015-2026 Ritense BV, the Netherlands. * - * * Copyright 2015-2026 Ritense BV, the Netherlands. - * * - * * Licensed under EUPL, Version 1.2 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" basis, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.ritense.document.opensearch @@ -30,6 +28,7 @@ import com.ritense.document.domain.impl.snapshot.JsonSchemaDocumentSnapshot import com.ritense.document.domain.impl.searchfield.SearchField import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument import com.ritense.document.opensearch.repository.JsonSchemaDocumentOpenSearchRepository +import com.ritense.document.opensearch.service.SearchEngineToggle import com.ritense.document.service.impl.JsonSchemaDocumentService import com.ritense.document.service.JsonSchemaDocumentActionProvider import com.ritense.document.service.JsonSchemaDocumentDefinitionActionProvider @@ -101,12 +100,26 @@ abstract class BaseOpenSearchIntegrationTest { @Autowired lateinit var objectMapper: ObjectMapper + @Autowired + lateinit var searchEngineToggle: SearchEngineToggle + @BeforeEach fun setUpBase() { setUpPermissions() ensureIndexExists() openSearchRepository.deleteAll() refreshIndex() + // Disable live event listener by default to prevent async indexing from interfering with tests. + // Tests that need live sync should call searchEngineToggle.set(SearchEngineToggle.Engine.OPENSEARCH). + searchEngineToggle.set(SearchEngineToggle.Engine.POSTGRES) + } + + @AfterEach + fun tearDownBase() { + openSearchRepository.deleteAll() + refreshIndex() + // Reset toggle to default for next test class + searchEngineToggle.set(SearchEngineToggle.Engine.OPENSEARCH) } private fun ensureIndexExists() { @@ -117,12 +130,6 @@ abstract class BaseOpenSearchIntegrationTest { } } - @AfterEach - fun tearDownBase() { - openSearchRepository.deleteAll() - refreshIndex() - } - /** * Forces an OpenSearch refresh so writes/deletes are immediately visible to subsequent reads. * OpenSearch refreshes asynchronously (default 1s), which makes write-then-read assertions flaky. diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReconcileIntTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReconcileIntTest.kt index 7f25051f58..b6e2eff9be 100644 --- a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReconcileIntTest.kt +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReconcileIntTest.kt @@ -8,7 +8,7 @@ * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 * * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, + * distributed under the License is distributed on an "AS IS" basis, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. @@ -155,16 +155,17 @@ class DocumentOpenSearchReconcileIntTest : BaseOpenSearchIntegrationTest() { } @Test - fun `changed_on advances on a status change while modifiedOn stays unset`() { + fun `changed_on advances on a status change while modifiedOn stays unchanged`() { val document = createDocument("changed-on-doc") val createdChangedOn = readChangedOn(document.id().id) + val initialModifiedOn = readModifiedOn(document.id().id) // DATETIME can be second-resolution on MySQL; sleep past a full second so the bump is observable. Thread.sleep(1100) runWithoutAuthorization { documentService.setInternalStatus(document.id(), "started") } assertThat(readChangedOn(document.id().id)).isAfter(createdChangedOn) - assertThat(readModifiedOn(document.id().id)).isEmpty + assertThat(readModifiedOn(document.id().id)).isEqualTo(initialModifiedOn) } private fun seedWatermark(watermark: LocalDateTime) { diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceTest.kt index bc0cb062a8..5e6fdcb31a 100644 --- a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceTest.kt +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceTest.kt @@ -99,7 +99,7 @@ class DocumentOpenSearchReindexServiceTest { assertThat(processed).isEqualTo(0L) verify(runService).stop(runId) - verify(runService, never()).complete(any()) + verify(runService, never()).complete(any(), any()) } private fun run(id: UUID) = OpenSearchReindexRun( diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunServiceTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunServiceTest.kt index 24ae81a45b..e26d1658c9 100644 --- a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunServiceTest.kt +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunServiceTest.kt @@ -1,19 +1,17 @@ /* + * Copyright 2015-2026 Ritense BV, the Netherlands. * - * * Copyright 2015-2026 Ritense BV, the Netherlands. - * * - * * Licensed under EUPL, Version 1.2 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" basis, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.ritense.document.opensearch.service @@ -124,9 +122,10 @@ class OpenSearchReindexRunServiceTest { val run = OpenSearchReindexRun(id = runId, pageSize = 100) whenever(repository.findById(runId)).thenReturn(Optional.of(run)) - service.complete(runId) + service.complete(runId, 42L) assertThat(run.status).isEqualTo(ReindexRunStatus.COMPLETED) assertThat(run.finishedOn).isNotNull() + assertThat(run.totalCount).isEqualTo(42L) service.fail(runId, "kaboom") assertThat(run.status).isEqualTo(ReindexRunStatus.FAILED) diff --git a/backend/case-opensearch/src/test/resources/config/application.yml b/backend/case-opensearch/src/test/resources/config/application.yml index ac84be235e..c8e281e47c 100644 --- a/backend/case-opensearch/src/test/resources/config/application.yml +++ b/backend/case-opensearch/src/test/resources/config/application.yml @@ -27,6 +27,7 @@ valtimo: plugin: encryption-secret: "abcdefghijklmnop" opensearch: + enabled: true reconcile: # Disable the scheduled reconcile job in tests so it cannot interfere with assertions; the tests # invoke DocumentOpenSearchReconcileService.reconcile() directly instead. diff --git a/backend/core/src/main/resources/config/liquibase/13-37-0/13-37-0-master.xml b/backend/core/src/main/resources/config/liquibase/13-37-0/13-37-0-master.xml index 564f5fbabf..07b932ce9a 100644 --- a/backend/core/src/main/resources/config/liquibase/13-37-0/13-37-0-master.xml +++ b/backend/core/src/main/resources/config/liquibase/13-37-0/13-37-0-master.xml @@ -24,5 +24,7 @@ xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liqui + + diff --git a/backend/core/src/main/resources/config/liquibase/13-37-0/20260713-add-reindex-run-total-count.xml b/backend/core/src/main/resources/config/liquibase/13-37-0/20260713-add-reindex-run-total-count.xml new file mode 100644 index 0000000000..b66c0fa398 --- /dev/null +++ b/backend/core/src/main/resources/config/liquibase/13-37-0/20260713-add-reindex-run-total-count.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + + diff --git a/backend/core/src/main/resources/config/liquibase/13-37-0/20260713-backfill-modified-on.xml b/backend/core/src/main/resources/config/liquibase/13-37-0/20260713-backfill-modified-on.xml new file mode 100644 index 0000000000..8efbba3d0a --- /dev/null +++ b/backend/core/src/main/resources/config/liquibase/13-37-0/20260713-backfill-modified-on.xml @@ -0,0 +1,31 @@ + + + + + + + + + modified_on IS NULL + + + + diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.html b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.html index 96a4a498e4..99df793dad 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.html +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.html @@ -27,11 +27,12 @@ > @@ -151,13 +152,11 @@
{{ 'adminSettings.opensearch.reindex.results' | translate }}
}
- @if (data.error) { -
- - {{ 'adminSettings.opensearch.reindex.errorTitle' | translate }}: - - {{ data.error }} -
+ @if (data.status === 'FAILED') { + + {{ 'adminSettings.opensearch.reindex.viewErrorLogs' | translate }} + + }
diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss index ea0ebb59aa..c742725a28 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss @@ -68,18 +68,23 @@ &__progress { padding: 8px 0; max-width: 500px; + + ::ng-deep .cds--progress-bar__track { + background-color: var(--cds-border-subtle); + } } - &__error { - padding: 12px; - background-color: var(--cds-support-error); - color: var(--cds-text-on-color); - border-radius: 4px; + &__error-link { + display: inline-flex; + align-items: center; + gap: 4px; + color: var(--cds-link-primary); font-size: 14px; + cursor: pointer; + text-decoration: none; - &-title { - font-weight: 600; - margin-right: 8px; + &:hover { + text-decoration: underline; } } } diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.ts b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.ts index bb25d6eb3a..c3627fdb5e 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.ts +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.ts @@ -16,12 +16,14 @@ import {ChangeDetectionStrategy, Component, OnDestroy, OnInit} from '@angular/core'; import {CommonModule, DatePipe} from '@angular/common'; +import {Router} from '@angular/router'; import {TranslateModule, TranslateService} from '@ngx-translate/core'; import { BehaviorSubject, finalize, interval, map, + merge, Observable, startWith, Subject, @@ -33,11 +35,13 @@ import { import { ButtonModule, IconModule, + IconService, ListItem, LoadingModule, ProgressBarModule, TagModule, } from 'carbon-components-angular'; +import {Launch16} from '@carbon/icons'; import {CarbonListModule, ColumnConfig, Pagination, TooltipIconModule, ViewType} from '@valtimo/components'; import {Page} from '@valtimo/shared'; import {AdminSettingsManagementApiService} from '../../services'; @@ -67,7 +71,8 @@ import {StartReindexModalComponent} from '../start-reindex-modal/start-reindex-m }) export class AdminSettingsOpensearchComponent implements OnInit, OnDestroy { private readonly _destroy$ = new Subject(); - private readonly _refresh$ = new BehaviorSubject(undefined); + private readonly _manualRefresh$ = new BehaviorSubject(undefined); + private readonly _silentRefresh$ = new Subject(); public readonly fields: ColumnConfig[] = [ {key: 'statusTag', label: 'adminSettings.opensearch.reindex.columns.status', viewType: ViewType.TAGS}, @@ -88,13 +93,18 @@ export class AdminSettingsOpensearchComponent implements OnInit, OnDestroy { public documentDefinitions$: Observable; - public readonly runs$: Observable> = this._refresh$.pipe( - switchMap(() => { - this.loading$.next(true); - return this._apiService.getReindexRuns(this.pagination.page - 1, this.pagination.size).pipe( - finalize(() => this.loading$.next(false)) - ); - }) + public readonly runs$: Observable> = merge( + this._manualRefresh$.pipe( + switchMap(() => { + this.loading$.next(true); + return this._apiService.getReindexRuns(this.pagination.page - 1, this.pagination.size).pipe( + finalize(() => this.loading$.next(false)) + ); + }) + ), + this._silentRefresh$.pipe( + switchMap(() => this._apiService.getReindexRuns(this.pagination.page - 1, this.pagination.size)) + ) ); public readonly tableItems$: Observable = this.runs$.pipe( @@ -121,8 +131,12 @@ export class AdminSettingsOpensearchComponent implements OnInit, OnDestroy { constructor( private readonly _apiService: AdminSettingsManagementApiService, private readonly _documentService: DocumentService, - private readonly _translateService: TranslateService - ) {} + private readonly _translateService: TranslateService, + private readonly _router: Router, + private readonly _iconService: IconService + ) { + this._iconService.register(Launch16); + } public ngOnInit(): void { this.documentDefinitions$ = this._documentService.queryDefinitionsForManagement().pipe( @@ -146,17 +160,17 @@ export class AdminSettingsOpensearchComponent implements OnInit, OnDestroy { }), takeUntil(this._destroy$) ) - .subscribe(() => this._refresh$.next()); + .subscribe(() => this._silentRefresh$.next()); } public onPageChange(page: number): void { this.pagination = {...this.pagination, page}; - this._refresh$.next(); + this._manualRefresh$.next(); } public onPageSizeChange(size: number): void { this.pagination = {...this.pagination, size, page: 1}; - this._refresh$.next(); + this._manualRefresh$.next(); } public openModal(): void { @@ -175,10 +189,10 @@ export class AdminSettingsOpensearchComponent implements OnInit, OnDestroy { finalize(() => this.startingReindex$.next(false)) ) .subscribe({ - next: () => this._refresh$.next(), + next: () => this._manualRefresh$.next(), error: err => { if (err.status === 409) { - this._refresh$.next(); + this._manualRefresh$.next(); } }, }); @@ -199,6 +213,21 @@ export class AdminSettingsOpensearchComponent implements OnInit, OnDestroy { } } + public navigateToLogs(run: ReindexStatusDto): void { + const afterTimestamp = new Date(new Date(run.startedOn).getTime() - 5000).toISOString(); + const beforeTimestamp = run.finishedOn + ? new Date(new Date(run.finishedOn).getTime() + 5000).toISOString() + : new Date().toISOString(); + + this._router.navigate(['/logging'], { + queryParams: { + level: 'ERROR', + afterTimestamp, + beforeTimestamp, + }, + }); + } + public ngOnDestroy(): void { this._destroy$.next(); this._destroy$.complete(); diff --git a/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.ts b/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.ts index 0bc2fa671a..8f90e30356 100644 --- a/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.ts +++ b/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.ts @@ -206,6 +206,7 @@ export class CarbonListComponent implements OnInit, AfterViewInit, OnDestroy { @Input() dragAndDrop = false; @Input() dragAndDropDisabled = false; @Input() expandedRowTemplate: TemplateRef; + @Input() expandedRowKey: string; @Output() rowClicked = new EventEmitter(); @Output() paginationClicked = new EventEmitter(); @@ -266,6 +267,7 @@ export class CarbonListComponent implements OnInit, AfterViewInit, OnDestroy { private _lastExecutedSearch: string | null = null; private static readonly PAGINATION_SIZE = 'PaginationSize'; private readonly _subscriptions = new Subscription(); + private readonly _expandedRowKeys = new Set(); public get selectedItems(): CarbonListItem[] { const model = this._table.model; @@ -535,10 +537,12 @@ export class CarbonListComponent implements OnInit, AfterViewInit, OnDestroy { this._tableItems$, this._filteredItems$, ]).pipe( + tap(() => this._captureExpandedRows()), map(([header, data, filteredData]) => { const model = new TableModel(); model.header = header; model.data = filteredData ?? data; + this._restoreExpandedRows(model); return model; }), startWith(new TableModel()) @@ -1029,4 +1033,32 @@ export class CarbonListComponent implements OnInit, AfterViewInit, OnDestroy { }, 150); } } + + private _captureExpandedRows(): void { + if (!this.expandedRowKey || !this._table?.model) return; + + const model = this._table.model; + const items = this._items; + + for (let i = 0; i < items.length; i++) { + const key = _get(items[i], this.expandedRowKey); + if (key && model.isRowExpanded(i)) { + this._expandedRowKeys.add(key); + } else if (key) { + this._expandedRowKeys.delete(key); + } + } + } + + private _restoreExpandedRows(model: TableModel): void { + if (!this.expandedRowKey || this._expandedRowKeys.size === 0) return; + + const items = this._items; + for (let i = 0; i < items.length; i++) { + const key = _get(items[i], this.expandedRowKey); + if (key && this._expandedRowKeys.has(key)) { + model.expandRow(i, true); + } + } + } } diff --git a/frontend/projects/valtimo/shared/assets/core/en.json b/frontend/projects/valtimo/shared/assets/core/en.json index 41f953b3d9..5750353d79 100644 --- a/frontend/projects/valtimo/shared/assets/core/en.json +++ b/frontend/projects/valtimo/shared/assets/core/en.json @@ -3334,6 +3334,7 @@ "finished": "Finished", "inProgress": "Reindexing in progress...", "errorTitle": "Error", + "viewErrorLogs": "View error logs", "pruneOrphans": "Prune orphaned documents", "documentDefinitionName": "Document definition", "documentDefinitionPlaceholder": "All document definitions", diff --git a/frontend/projects/valtimo/shared/assets/core/nl.json b/frontend/projects/valtimo/shared/assets/core/nl.json index 57e6a05bd8..1d789cd993 100644 --- a/frontend/projects/valtimo/shared/assets/core/nl.json +++ b/frontend/projects/valtimo/shared/assets/core/nl.json @@ -3367,6 +3367,7 @@ "finished": "Voltooid", "inProgress": "Herindexering bezig...", "errorTitle": "Fout", + "viewErrorLogs": "Bekijk foutlogs", "pruneOrphans": "Verwijder orphan documenten", "documentDefinitionName": "Documentdefinitie", "documentDefinitionPlaceholder": "Alle documentdefinities", From e973d8fc472c6071a57e8f28637604b80891e223 Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Mon, 13 Jul 2026 17:17:14 +0200 Subject: [PATCH 28/46] fixed corner case where an empty table would not update when starting a new reindex run --- .../admin-settings-opensearch.component.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.ts b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.ts index c3627fdb5e..07e31a184e 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.ts +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.ts @@ -25,6 +25,8 @@ import { map, merge, Observable, + of, + shareReplay, startWith, Subject, switchMap, @@ -105,11 +107,14 @@ export class AdminSettingsOpensearchComponent implements OnInit, OnDestroy { this._silentRefresh$.pipe( switchMap(() => this._apiService.getReindexRuns(this.pagination.page - 1, this.pagination.size)) ) - ); + ).pipe(shareReplay(1)); public readonly tableItems$: Observable = this.runs$.pipe( switchMap(page => { this.pagination = {...this.pagination, collectionSize: page.totalElements}; + if (page.content.length === 0) { + return of([]); + } const statusKeys = page.content.map(run => `adminSettings.opensearch.reindex.statuses.${run.status}`); return this._translateService.get(statusKeys).pipe( map(translations => page.content.map(run => ({ From dee19461dd91da366b721e5250621b36e6deb768 Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Thu, 16 Jul 2026 12:49:57 +0200 Subject: [PATCH 29/46] PR feedback --- .../dev/src/main/resources/logback-spring.xml | 1 - .../opensearch/OpenSearchProperties.kt | 24 ++- ...aDocumentCaseDefinitionOpenSearchMapper.kt | 4 +- ...chemaDocumentDefinitionOpenSearchMapper.kt | 4 +- .../domain/JsonSchemaDocumentOsDocument.kt | 4 +- .../DocumentOpenSearchReindexService.kt | 45 ++++-- .../JsonSchemaDocumentOpenSearchService.kt | 24 ++- ...JsonSchemaDocumentOpenSearchServiceTest.kt | 24 ++- .../case-list/case-list.component.html | 1 + .../case-list/case-list.component.ts | 24 ++- .../generic-case-list.component.ts | 24 ++- .../carbon-list/carbon-list.component.scss | 3 +- .../carbon-list/carbon-list.component.ts | 7 +- .../search-input-with-validation/index.ts | 17 -- ...earch-input-with-validation.component.html | 68 -------- ...earch-input-with-validation.component.scss | 153 ------------------ .../search-input-with-validation.component.ts | 147 ----------------- .../components/src/lib/models/index.ts | 2 +- .../valtimo/components/src/public_api.ts | 3 - .../src/lib/services/document.service.ts | 24 ++- 20 files changed, 110 insertions(+), 493 deletions(-) delete mode 100644 frontend/projects/valtimo/components/src/lib/components/search-input-with-validation/index.ts delete mode 100644 frontend/projects/valtimo/components/src/lib/components/search-input-with-validation/search-input-with-validation.component.html delete mode 100644 frontend/projects/valtimo/components/src/lib/components/search-input-with-validation/search-input-with-validation.component.scss delete mode 100644 frontend/projects/valtimo/components/src/lib/components/search-input-with-validation/search-input-with-validation.component.ts diff --git a/backend/apps/dev/src/main/resources/logback-spring.xml b/backend/apps/dev/src/main/resources/logback-spring.xml index 25392dcd5e..e21b79db93 100644 --- a/backend/apps/dev/src/main/resources/logback-spring.xml +++ b/backend/apps/dev/src/main/resources/logback-spring.xml @@ -22,7 +22,6 @@ - diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/OpenSearchProperties.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/OpenSearchProperties.kt index d045a941c7..8bfb7af614 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/OpenSearchProperties.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/OpenSearchProperties.kt @@ -1,19 +1,17 @@ /* + * Copyright 2015-2026 Ritense BV, the Netherlands. * - * * Copyright 2015-2026 Ritense BV, the Netherlands. - * * - * * Licensed under EUPL, Version 1.2 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" basis, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.ritense.document.opensearch diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentCaseDefinitionOpenSearchMapper.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentCaseDefinitionOpenSearchMapper.kt index 3949fb22a0..cca791cf7c 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentCaseDefinitionOpenSearchMapper.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentCaseDefinitionOpenSearchMapper.kt @@ -1,5 +1,5 @@ /* - * Copyright 2015-2024 Ritense BV, the Netherlands. + * Copyright 2015-2026 Ritense BV, the Netherlands. * * Licensed under EUPL, Version 1.2 (the "License"); * you may not use this file except in compliance with the License. @@ -31,7 +31,7 @@ import org.opensearch.index.query.QueryBuilders * Handles [com.ritense.authorization.permission.condition.ContainerPermissionCondition] * where the container resource type is [CaseDefinition]. * - * Field paths mirror the MongoDB version: `definitionId.blueprintId.*`. + * Field paths mirror the JPA entity: `definitionId.blueprintId.*`. */ class JsonSchemaDocumentCaseDefinitionOpenSearchMapper : OpenSearchAuthorizationEntityMapper { diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentDefinitionOpenSearchMapper.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentDefinitionOpenSearchMapper.kt index 899aafda34..9a274dbe57 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentDefinitionOpenSearchMapper.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentDefinitionOpenSearchMapper.kt @@ -1,5 +1,5 @@ /* - * Copyright 2015-2024 Ritense BV, the Netherlands. + * Copyright 2015-2026 Ritense BV, the Netherlands. * * Licensed under EUPL, Version 1.2 (the "License"); * you may not use this file except in compliance with the License. @@ -30,7 +30,7 @@ import org.opensearch.index.query.QueryBuilder * Handles [com.ritense.authorization.permission.condition.ContainerPermissionCondition] * where the container resource type is [JsonSchemaDocumentDefinition]. * - * Field paths mirror the MongoDB version: `definitionId.name` and `definitionId.version`. + * Field paths mirror the JPA entity: `definitionId.name` and `definitionId.version`. */ class JsonSchemaDocumentDefinitionOpenSearchMapper : OpenSearchAuthorizationEntityMapper { diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/JsonSchemaDocumentOsDocument.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/JsonSchemaDocumentOsDocument.kt index c4583c391c..c1fac61acf 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/JsonSchemaDocumentOsDocument.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/JsonSchemaDocumentOsDocument.kt @@ -1,5 +1,5 @@ /* - * Copyright 2015-2024 Ritense BV, the Netherlands. + * Copyright 2015-2026 Ritense BV, the Netherlands. * * Licensed under EUPL, Version 1.2 (the "License"); * you may not use this file except in compliance with the License. @@ -35,7 +35,7 @@ import java.time.LocalDateTime * * The [contentText] field holds space-separated leaf values from [content] and is indexed * as both [FieldType.Text] (for analyzed search) and [FieldType.Keyword] (for wildcard search - * preserving partial-match behaviour equivalent to MongoDB's text index). + * preserving partial-match behaviour for wildcard queries). */ @Document(indexName = "json_schema_document", createIndex = false) data class JsonSchemaDocumentOsDocument( diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt index 9df8068b50..d24681e059 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt @@ -31,9 +31,8 @@ import org.opensearch.index.query.QueryBuilders import org.springframework.data.domain.Page import org.springframework.data.domain.PageRequest import org.springframework.data.domain.Pageable -import org.springframework.data.domain.Sort import org.springframework.data.elasticsearch.core.ElasticsearchOperations -import org.springframework.data.elasticsearch.core.query.StringQuery +import org.opensearch.data.client.orhlc.NativeSearchQueryBuilder import org.springframework.transaction.PlatformTransactionManager import org.springframework.transaction.support.TransactionTemplate import java.time.Duration @@ -202,15 +201,14 @@ open class DocumentOpenSearchReindexService( /** * Scans OpenSearch for documents matching [scope], checks each batch against PostgreSQL, * and deletes orphans (documents in OpenSearch but not in PostgreSQL). + * Uses scroll API to handle datasets larger than 10k documents. */ private fun pruneOrphans(scope: ReindexRequest): Long { var pruned = 0L - var page = 0 val txTemplate = TransactionTemplate(transactionManager).apply { isReadOnly = true } - while (!cancelRequested) { - val osIds = fetchOpenSearchIds(scope, page, PRUNE_BATCH_SIZE) - if (osIds.isEmpty()) break + scrollOpenSearchIds(scope, PRUNE_BATCH_SIZE) { osIds -> + if (cancelRequested) return@scrollOpenSearchIds false val existingIds = txTemplate.execute { findExistingIds(osIds.map { UUID.fromString(it) }) @@ -220,12 +218,16 @@ open class DocumentOpenSearchReindexService( orphans.forEach { openSearchRepository.deleteById(it) } pruned += orphans.size - page++ + true } return pruned } - private fun fetchOpenSearchIds(scope: ReindexRequest, page: Int, batchSize: Int): List { + /** + * Scrolls through all OpenSearch documents matching [scope], invoking [handler] for each batch. + * Handler returns `true` to continue, `false` to stop early. + */ + private fun scrollOpenSearchIds(scope: ReindexRequest, batchSize: Int, handler: (List) -> Boolean) { val boolQuery = BoolQueryBuilder() scope.documentDefinitionName?.let { @@ -241,12 +243,27 @@ open class DocumentOpenSearchReindexService( boolQuery.filter(QueryBuilders.idsQuery().addIds(*ids.map { it.toString() }.toTypedArray())) } - val pageable = PageRequest.of(page, batchSize, Sort.by(Sort.Direction.ASC, "_id")) - val query = StringQuery(boolQuery.toString(), pageable) - - return elasticsearchOperations.search(query, JsonSchemaDocumentOsDocument::class.java) - .searchHits - .map { it.id } + val query = NativeSearchQueryBuilder() + .withQuery(boolQuery) + .withPageable(PageRequest.of(0, batchSize)) + .build() + + elasticsearchOperations.searchForStream(query, JsonSchemaDocumentOsDocument::class.java).use { stream -> + val iterator = stream.iterator() + val batch = mutableListOf() + + while (iterator.hasNext()) { + val id = iterator.next().id ?: continue + batch.add(id) + if (batch.size >= batchSize) { + if (!handler(batch.toList())) return + batch.clear() + } + } + if (batch.isNotEmpty()) { + handler(batch) + } + } } private fun findExistingIds(ids: List): Set { diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt index fbd4267fe0..786fc51907 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt @@ -1,19 +1,17 @@ /* + * Copyright 2015-2026 Ritense BV, the Netherlands. * - * * Copyright 2015-2026 Ritense BV, the Netherlands. - * * - * * Licensed under EUPL, Version 1.2 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" basis, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.ritense.document.opensearch.service diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceTest.kt index 0e2e2c427a..5901ba0b7c 100644 --- a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceTest.kt +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceTest.kt @@ -1,19 +1,17 @@ /* + * Copyright 2015-2026 Ritense BV, the Netherlands. * - * * Copyright 2015-2026 Ritense BV, the Netherlands. - * * - * * Licensed under EUPL, Version 1.2 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" basis, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.ritense.document.opensearch.service diff --git a/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.html b/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.html index d6a6a71e4a..4e2bb64f01 100644 --- a/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.html +++ b/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.html @@ -99,6 +99,7 @@ [initialSortState]="pagination.sort" [invalidSearchFields]="obs.invalidSearchFields || []" [isSearchable]="true" + [searchDebounceMs]="2000" [searchFields]="obs.searchFields || []" [items]="obs.documentItems" [pagination]="pagination" diff --git a/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.ts b/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.ts index c1815566c9..6ae57fe2f7 100644 --- a/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.ts +++ b/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.ts @@ -1,19 +1,17 @@ /* + * Copyright 2015-2026 Ritense BV, the Netherlands. * - * * Copyright 2015-2026 Ritense BV, the Netherlands. - * * - * * Licensed under EUPL, Version 1.2 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" basis, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ import {Component, inject, Inject, OnDestroy, OnInit, ViewChild} from '@angular/core'; import {ActivatedRoute, Params, Router} from '@angular/router'; diff --git a/frontend/projects/valtimo/case/src/lib/components/generic-case-list/generic-case-list.component.ts b/frontend/projects/valtimo/case/src/lib/components/generic-case-list/generic-case-list.component.ts index 1c50771bbf..400e076dc7 100644 --- a/frontend/projects/valtimo/case/src/lib/components/generic-case-list/generic-case-list.component.ts +++ b/frontend/projects/valtimo/case/src/lib/components/generic-case-list/generic-case-list.component.ts @@ -1,19 +1,17 @@ /* + * Copyright 2015-2026 Ritense BV, the Netherlands. * - * * Copyright 2015-2026 Ritense BV, the Netherlands. - * * - * * Licensed under EUPL, Version 1.2 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" basis, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ import {Component, inject, Inject, OnDestroy, OnInit, ViewChild} from '@angular/core'; import {Router} from '@angular/router'; diff --git a/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.scss b/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.scss index 15d0506f2d..e1e0f1c4cc 100644 --- a/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.scss +++ b/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.scss @@ -196,8 +196,7 @@ td:first-child { text-decoration-skip-ink: none; text-underline-offset: 3px; } - - } +} .valtimo-search-autocomplete { position: absolute; diff --git a/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.ts b/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.ts index 8f90e30356..ce38462c52 100644 --- a/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.ts +++ b/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.ts @@ -185,11 +185,12 @@ export class CarbonListComponent implements OnInit, AfterViewInit, OnDestroy { this._initialSearchValue = value; this.searchFormControl.setValue(value || '', {emitEvent: false}); this._lastExecutedSearch = value; - this._searchActive = !!value; + this.searchActive = !!value; } } private _initialSearchValue: string | null = null; - public _searchActive = false; + public searchActive = false; + @Input() searchDebounceMs = 500; @Input() invalidSearchFields: string[] = []; @Input() searchFields: SearchField[] = []; @Input() enableSingleSelection = false; @@ -327,7 +328,7 @@ export class CarbonListComponent implements OnInit, AfterViewInit, OnDestroy { this._subscriptions.add( this.searchFormControl.valueChanges - .pipe(debounceTime(2000)) + .pipe(debounceTime(this.searchDebounceMs)) .subscribe((searchString: string | null) => { this.executeSearch(searchString); }) diff --git a/frontend/projects/valtimo/components/src/lib/components/search-input-with-validation/index.ts b/frontend/projects/valtimo/components/src/lib/components/search-input-with-validation/index.ts deleted file mode 100644 index eacaecbc97..0000000000 --- a/frontend/projects/valtimo/components/src/lib/components/search-input-with-validation/index.ts +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright 2015-2026 Ritense BV, the Netherlands. - * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export * from './search-input-with-validation.component'; diff --git a/frontend/projects/valtimo/components/src/lib/components/search-input-with-validation/search-input-with-validation.component.html b/frontend/projects/valtimo/components/src/lib/components/search-input-with-validation/search-input-with-validation.component.html deleted file mode 100644 index 7c2b9819b4..0000000000 --- a/frontend/projects/valtimo/components/src/lib/components/search-input-with-validation/search-input-with-validation.component.html +++ /dev/null @@ -1,68 +0,0 @@ - - -
- - -
- - -
- - - -
- - -
-
diff --git a/frontend/projects/valtimo/components/src/lib/components/search-input-with-validation/search-input-with-validation.component.scss b/frontend/projects/valtimo/components/src/lib/components/search-input-with-validation/search-input-with-validation.component.scss deleted file mode 100644 index 7109be1478..0000000000 --- a/frontend/projects/valtimo/components/src/lib/components/search-input-with-validation/search-input-with-validation.component.scss +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright 2015-2026 Ritense BV, the Netherlands. - * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -$search-height: 2.5rem; -$search-font-size: 0.875rem; -$search-padding-left: 2.5rem; -$search-padding-right: 2.5rem; - -.search-input-container { - display: inline-flex; - align-items: center; - height: $search-height; - background-color: var(--cds-field-01, #f4f4f4); - transition: width 0.2s ease; - - &--expanded { - width: 100%; - max-width: 20rem; - } - - &--has-error { - .search-input-wrapper { - border-bottom: 2px solid var(--cds-support-error, #da1e28); - } - } -} - -.search-input-expand-button { - display: flex; - align-items: center; - justify-content: center; - width: $search-height; - height: $search-height; - padding: 0; - border: none; - background: transparent; - cursor: pointer; - color: var(--cds-icon-primary, #161616); - - &:hover { - background-color: var(--cds-field-hover, #e8e8e8); - } - - &:focus { - outline: 2px solid var(--cds-focus, #0f62fe); - outline-offset: -2px; - } -} - -.search-input-wrapper { - position: relative; - display: flex; - align-items: center; - width: 100%; - height: 100%; - border-bottom: 1px solid var(--cds-border-strong-01, #8d8d8d); -} - -.search-input-icon { - position: absolute; - left: 0.75rem; - color: var(--cds-icon-secondary, #525252); - pointer-events: none; -} - -.search-input-highlight-container { - position: relative; - flex: 1; - height: 100%; -} - -.search-input-highlight-overlay { - position: absolute; - top: 0; - left: 0; - right: 0; - bottom: 0; - display: flex; - align-items: center; - padding-left: $search-padding-left; - padding-right: $search-padding-right; - font-family: 'IBM Plex Sans', sans-serif; - font-size: $search-font-size; - color: var(--cds-text-primary, #161616); - white-space: pre; - pointer-events: none; - overflow: hidden; -} - -.search-input-field { - position: relative; - width: 100%; - height: 100%; - padding-left: $search-padding-left; - padding-right: $search-padding-right; - border: none; - background: transparent; - font-family: 'IBM Plex Sans', sans-serif; - font-size: $search-font-size; - color: transparent; - caret-color: var(--cds-text-primary, #161616); - - &::placeholder { - color: var(--cds-text-placeholder, #a8a8a8); - } - - &:focus { - outline: none; - } -} - -.search-input-invalid { - text-decoration: underline wavy var(--cds-support-error, #da1e28); - text-decoration-skip-ink: none; - text-underline-offset: 2px; -} - -.search-input-clear-button { - position: absolute; - right: 0; - display: flex; - align-items: center; - justify-content: center; - width: $search-height; - height: $search-height; - padding: 0; - border: none; - background: transparent; - cursor: pointer; - color: var(--cds-icon-primary, #161616); - - &:hover { - background-color: var(--cds-field-hover, #e8e8e8); - } - - &:focus { - outline: 2px solid var(--cds-focus, #0f62fe); - outline-offset: -2px; - } -} diff --git a/frontend/projects/valtimo/components/src/lib/components/search-input-with-validation/search-input-with-validation.component.ts b/frontend/projects/valtimo/components/src/lib/components/search-input-with-validation/search-input-with-validation.component.ts deleted file mode 100644 index faa7d070d2..0000000000 --- a/frontend/projects/valtimo/components/src/lib/components/search-input-with-validation/search-input-with-validation.component.ts +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright 2015-2026 Ritense BV, the Netherlands. - * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - ChangeDetectionStrategy, - Component, - ElementRef, - EventEmitter, - Input, - Output, - ViewChild, -} from '@angular/core'; -import {CommonModule} from '@angular/common'; -import {FormsModule} from '@angular/forms'; -import {IconModule, IconService} from 'carbon-components-angular'; -import {Search16, Close16} from '@carbon/icons'; - -interface TextSegment { - text: string; - isFieldName: boolean; - isInvalid: boolean; -} - -@Component({ - standalone: true, - selector: 'valtimo-search-input-with-validation', - templateUrl: './search-input-with-validation.component.html', - styleUrls: ['./search-input-with-validation.component.scss'], - changeDetection: ChangeDetectionStrategy.OnPush, - imports: [CommonModule, FormsModule, IconModule], -}) -export class SearchInputWithValidationComponent { - @ViewChild('inputElement') private readonly _inputElement: ElementRef; - - @Input() public value = ''; - @Input() public invalidFields: string[] = []; - @Input() public placeholder = ''; - @Input() public expandable = true; - - @Output() public readonly valueChangeEvent = new EventEmitter(); - @Output() public readonly searchEvent = new EventEmitter(); - - public isExpanded = false; - - constructor(private readonly _iconService: IconService) { - this._iconService.registerAll([Search16, Close16]); - } - - public get segments(): TextSegment[] { - return this._parseIntoSegments(this.value, this.invalidFields); - } - - public get hasInvalidFields(): boolean { - return this.invalidFields.length > 0; - } - - public onInput(event: Event): void { - const input = event.target as HTMLInputElement; - this.value = input.value; - this.valueChangeEvent.emit(this.value); - } - - public onKeyDown(event: KeyboardEvent): void { - if (event.key === 'Enter') { - this.searchEvent.emit(this.value); - } - } - - public onClear(): void { - this.value = ''; - this.valueChangeEvent.emit(this.value); - this.searchEvent.emit(this.value); - } - - public onExpand(): void { - this.isExpanded = true; - setTimeout(() => this._inputElement?.nativeElement?.focus(), 0); - } - - public onBlur(): void { - if (!this.value) { - this.isExpanded = false; - } - } - - private _parseIntoSegments(text: string, invalidFields: string[]): TextSegment[] { - if (!text) return []; - - const segments: TextSegment[] = []; - const fieldPattern = /(\w+(?:\.\w+)*):("([^"]+)"|(\S+))/g; - const invalidSet = new Set(invalidFields.map(f => f.toLowerCase())); - - let lastIndex = 0; - let match: RegExpExecArray | null; - - while ((match = fieldPattern.exec(text)) !== null) { - if (match.index > lastIndex) { - segments.push({ - text: text.substring(lastIndex, match.index), - isFieldName: false, - isInvalid: false, - }); - } - - const fieldName = match[1]; - const isInvalid = invalidSet.has(fieldName.toLowerCase()); - - segments.push({ - text: fieldName, - isFieldName: true, - isInvalid, - }); - - const colonAndValue = match[0].substring(fieldName.length); - segments.push({ - text: colonAndValue, - isFieldName: false, - isInvalid: false, - }); - - lastIndex = match.index + match[0].length; - } - - if (lastIndex < text.length) { - segments.push({ - text: text.substring(lastIndex), - isFieldName: false, - isInvalid: false, - }); - } - - return segments; - } -} diff --git a/frontend/projects/valtimo/components/src/lib/models/index.ts b/frontend/projects/valtimo/components/src/lib/models/index.ts index 3199a424f7..60abde7744 100644 --- a/frontend/projects/valtimo/components/src/lib/models/index.ts +++ b/frontend/projects/valtimo/components/src/lib/models/index.ts @@ -1,5 +1,5 @@ /* - * Copyright 2015-2025 Ritense BV, the Netherlands. + * Copyright 2015-2026 Ritense BV, the Netherlands. * * Licensed under EUPL, Version 1.2 (the "License"); * you may not use this file except in compliance with the License. diff --git a/frontend/projects/valtimo/components/src/public_api.ts b/frontend/projects/valtimo/components/src/public_api.ts index d683ba8f1d..37dee0d441 100644 --- a/frontend/projects/valtimo/components/src/public_api.ts +++ b/frontend/projects/valtimo/components/src/public_api.ts @@ -298,6 +298,3 @@ export * from './lib/components/assign-user/assignment.component'; // Color picker export * from './lib/components/color-picker/color-picker.component'; - -// Search input with validation -export * from './lib/components/search-input-with-validation'; diff --git a/frontend/projects/valtimo/document/src/lib/services/document.service.ts b/frontend/projects/valtimo/document/src/lib/services/document.service.ts index d745a4baf8..ce590676dc 100644 --- a/frontend/projects/valtimo/document/src/lib/services/document.service.ts +++ b/frontend/projects/valtimo/document/src/lib/services/document.service.ts @@ -1,19 +1,17 @@ /* + * Copyright 2015-2026 Ritense BV, the Netherlands. * - * * Copyright 2015-2026 Ritense BV, the Netherlands. - * * - * * Licensed under EUPL, Version 1.2 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" basis, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ import {HttpClient, HttpErrorResponse, HttpHeaders, HttpParams} from '@angular/common/http'; import {Injectable} from '@angular/core'; From fa6659aea3ffd7967358827f28ecbd4e0174a99d Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Thu, 16 Jul 2026 16:26:35 +0200 Subject: [PATCH 30/46] PR feedback --- .../DocumentOpenSearchAutoConfiguration.kt | 6 ++- .../DocumentOpenSearchEventListener.kt | 13 +++-- .../DelegatingDocumentSearchService.kt | 6 ++- .../DocumentOpenSearchIndexInitializer.kt | 53 ++++++++----------- .../opensearch/web/SearchEngineResource.kt | 25 ++++++--- .../20260710-add-reindex-run-pruned-count.xml | 32 ----------- .../13-38-0-master.xml} | 1 + .../20260630-create-reindex-run-table.xml | 0 ...add-changed-on-to-json-schema-document.xml | 9 ++-- ...01-create-pending-index-deletion-table.xml | 0 .../20260701-create-reconcile-state-table.xml | 0 .../20260710-add-reindex-run-pruned-count.xml | 0 .../20260713-add-reindex-run-total-count.xml | 0 .../20260713-backfill-modified-on.xml | 0 .../config/liquibase/changelog-master.xml | 2 +- .../initial-setup/initial-setup-master.xml | 3 +- 16 files changed, 65 insertions(+), 85 deletions(-) delete mode 100644 backend/core/src/main/resources/config/liquibase/13-37-0/20260710-add-reindex-run-pruned-count.xml rename backend/core/src/main/resources/config/liquibase/{13-37-0/13-37-0-master.xml => 13-38-0/13-38-0-master.xml} (93%) rename backend/core/src/main/resources/config/liquibase/{13-37-0 => 13-38-0}/20260630-create-reindex-run-table.xml (100%) rename backend/core/src/main/resources/config/liquibase/{initial-setup/case/changelog => 13-38-0}/20260701-add-changed-on-to-json-schema-document.xml (86%) rename backend/core/src/main/resources/config/liquibase/{13-37-0 => 13-38-0}/20260701-create-pending-index-deletion-table.xml (100%) rename backend/core/src/main/resources/config/liquibase/{13-37-0 => 13-38-0}/20260701-create-reconcile-state-table.xml (100%) rename backend/core/src/main/resources/config/liquibase/{13-32-0 => 13-38-0}/20260710-add-reindex-run-pruned-count.xml (100%) rename backend/core/src/main/resources/config/liquibase/{13-37-0 => 13-38-0}/20260713-add-reindex-run-total-count.xml (100%) rename backend/core/src/main/resources/config/liquibase/{13-37-0 => 13-38-0}/20260713-backfill-modified-on.xml (100%) diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt index 4d91ec4569..c64c4bffcc 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt @@ -337,7 +337,11 @@ class DocumentOpenSearchAutoConfiguration { logger.info { "Document search engine set to: ${engine.name}" } if (toggle.isOpenSearchActive()) { - indexInitializer.ensureIndex() + try { + indexInitializer.ensureIndex() + } catch (e: Exception) { + logger.warn(e) { "Failed to initialize OpenSearch index at startup — is OpenSearch running?" } + } } } diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/handler/DocumentOpenSearchEventListener.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/handler/DocumentOpenSearchEventListener.kt index 6d08a5db47..9fe8a5c388 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/handler/DocumentOpenSearchEventListener.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/handler/DocumentOpenSearchEventListener.kt @@ -33,8 +33,9 @@ import org.springframework.transaction.event.TransactionPhase.AFTER_COMMIT import org.springframework.transaction.event.TransactionalEventListener import java.util.UUID import java.util.concurrent.ExecutorService -import java.util.concurrent.Executors +import java.util.concurrent.LinkedBlockingQueue import java.util.concurrent.RejectedExecutionException +import java.util.concurrent.ThreadPoolExecutor import java.util.concurrent.TimeUnit /** @@ -55,9 +56,12 @@ class DocumentOpenSearchEventListener( private val toggle: SearchEngineToggle, ) : DisposableBean { - private val executor: ExecutorService = Executors.newSingleThreadExecutor { runnable -> - Thread(runnable, "opensearch-live-sync").apply { isDaemon = true } - } + private val executor: ExecutorService = ThreadPoolExecutor( + 1, 1, 0L, TimeUnit.MILLISECONDS, + LinkedBlockingQueue(QUEUE_CAPACITY), + { runnable -> Thread(runnable, "opensearch-live-sync").apply { isDaemon = true } }, + ThreadPoolExecutor.DiscardOldestPolicy() + ) @TransactionalEventListener(phase = AFTER_COMMIT) fun onCreated(event: JsonSchemaDocumentCreatedEvent) = enqueueUpsert(event.documentId().id) @@ -122,5 +126,6 @@ class DocumentOpenSearchEventListener( companion object { private val logger = KotlinLogging.logger {} private const val SHUTDOWN_TIMEOUT_SECONDS = 30L + private const val QUEUE_CAPACITY = 1000 } } diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DelegatingDocumentSearchService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DelegatingDocumentSearchService.kt index 50606e561a..64e11ab693 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DelegatingDocumentSearchService.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DelegatingDocumentSearchService.kt @@ -1,5 +1,5 @@ /* - * Copyright 2015-2024 Ritense BV, the Netherlands. + * Copyright 2015-2026 Ritense BV, the Netherlands. * * Licensed under EUPL, Version 1.2 (the "License"); * you may not use this file except in compliance with the License. @@ -93,7 +93,9 @@ class DelegatingDocumentSearchService( private fun isConnectionError(e: Exception): Boolean { val message = e.message?.lowercase() ?: "" return e is java.net.ConnectException || - e is java.io.IOException || + e is java.net.SocketTimeoutException || + e is java.net.NoRouteToHostException || + e is java.net.UnknownHostException || message.contains("connection refused") || message.contains("connect timed out") || message.contains("no route to host") || diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchIndexInitializer.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchIndexInitializer.kt index 4e3810d917..85118f708f 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchIndexInitializer.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchIndexInitializer.kt @@ -22,45 +22,34 @@ import org.springframework.data.elasticsearch.core.ElasticsearchOperations import org.springframework.data.elasticsearch.core.document.Document /** - * Creates the OpenSearch index and mappings if they do not yet exist. This is the only place that - * provisions the index, and it is invoked exactly when OpenSearch becomes the active engine — at startup - * when the engine is already active, and from [com.ritense.document.opensearch.web.SearchEngineResource] - * the moment the engine is switched on at runtime — so a freshly enabled cluster is prepared on demand - * rather than being touched unconditionally on every boot. - * - * All failures are swallowed with a warning: a missing/unreachable cluster must never break startup or the - * toggle endpoint. [ensureIndex] is idempotent, so repeated calls (startup + later switch-ons) are safe. + * Creates the OpenSearch index and mappings if they do not yet exist. Invoked before the engine toggle + * switches to OpenSearch — failure prevents the swap, keeping queries on Postgres until the cluster is healthy. + * [ensureIndex] is idempotent, so repeated calls are safe. */ open class DocumentOpenSearchIndexInitializer( private val elasticsearchOperations: ElasticsearchOperations, ) { open fun ensureIndex() { - try { - val indexOps = elasticsearchOperations.indexOps(JsonSchemaDocumentOsDocument::class.java) - if (!indexOps.exists()) { - val settings = Document.create() - settings["index.number_of_replicas"] = 0 - indexOps.create(settings) + val indexOps = elasticsearchOperations.indexOps(JsonSchemaDocumentOsDocument::class.java) + if (!indexOps.exists()) { + val settings = Document.create() + settings["index.number_of_replicas"] = 0 + indexOps.create(settings) - // Merge annotated mapping with a dynamic template that forces all - // content.* fields to text+keyword — enables wildcard search on numbers too - val annotatedMapping = indexOps.createMapping(JsonSchemaDocumentOsDocument::class.java) - val dynamicTemplates = listOf( - mapOf("content_fields_as_text" to mapOf( - "path_match" to "content.*", - "match_mapping_type" to "string", - "mapping" to mapOf( - "type" to "text", - "fields" to mapOf("keyword" to mapOf("type" to "keyword", "ignore_above" to 256)) - ) - )) - ) - annotatedMapping["dynamic_templates"] = dynamicTemplates - indexOps.putMapping(annotatedMapping) - } - } catch (e: Exception) { - logger.warn(e) { "Failed to initialize OpenSearch index — is OpenSearch running?" } + val annotatedMapping = indexOps.createMapping(JsonSchemaDocumentOsDocument::class.java) + val dynamicTemplates = listOf( + mapOf("content_fields_as_text" to mapOf( + "path_match" to "content.*", + "match_mapping_type" to "string", + "mapping" to mapOf( + "type" to "text", + "fields" to mapOf("keyword" to mapOf("type" to "keyword", "ignore_above" to 256)) + ) + )) + ) + annotatedMapping["dynamic_templates"] = dynamicTemplates + indexOps.putMapping(annotatedMapping) } } diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/SearchEngineResource.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/SearchEngineResource.kt index 0a7f8124e7..afead41d82 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/SearchEngineResource.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/SearchEngineResource.kt @@ -1,5 +1,5 @@ /* - * Copyright 2015-2024 Ritense BV, the Netherlands. + * Copyright 2015-2026 Ritense BV, the Netherlands. * * Licensed under EUPL, Version 1.2 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,8 @@ import com.ritense.document.opensearch.OpenSearchProperties import com.ritense.document.opensearch.autoconfigure.DocumentOpenSearchAutoConfiguration.Companion.SEARCH_ENGINE_TOGGLE_KEY import com.ritense.document.opensearch.service.DocumentOpenSearchIndexInitializer import com.ritense.document.opensearch.service.SearchEngineToggle +import io.github.oshai.kotlinlogging.KotlinLogging +import org.springframework.http.HttpStatus import org.springframework.http.ResponseEntity import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.PutMapping @@ -53,17 +55,22 @@ class SearchEngineResource( } val useOpenSearch = body.active.uppercase() == "OPENSEARCH" + + if (useOpenSearch) { + try { + indexInitializer.ensureIndex() + } catch (e: Exception) { + logger.warn(e) { "Failed to initialize OpenSearch index — is OpenSearch running?" } + return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE) + .body(SearchEngineDto(available = true, active = toggle.get().name)) + } + } + featureToggleOverridesService.updateToggle(SEARCH_ENGINE_TOGGLE_KEY, useOpenSearch) val engine = if (useOpenSearch) SearchEngineToggle.Engine.OPENSEARCH else SearchEngineToggle.Engine.POSTGRES toggle.set(engine) - // Switching the engine on at runtime: make sure the index exists before live-sync/reads resume. - // Idempotent and failure-swallowing, so a missing cluster can't break the toggle call. - if (toggle.isOpenSearchActive()) { - indexInitializer.ensureIndex() - } - return ResponseEntity.ok( SearchEngineDto( available = true, @@ -80,4 +87,8 @@ class SearchEngineResource( data class UpdateSearchEngineDto( val active: String ) + + companion object { + private val logger = KotlinLogging.logger {} + } } diff --git a/backend/core/src/main/resources/config/liquibase/13-37-0/20260710-add-reindex-run-pruned-count.xml b/backend/core/src/main/resources/config/liquibase/13-37-0/20260710-add-reindex-run-pruned-count.xml deleted file mode 100644 index fc0554c233..0000000000 --- a/backend/core/src/main/resources/config/liquibase/13-37-0/20260710-add-reindex-run-pruned-count.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - - - - - - - - - - - diff --git a/backend/core/src/main/resources/config/liquibase/13-37-0/13-37-0-master.xml b/backend/core/src/main/resources/config/liquibase/13-38-0/13-38-0-master.xml similarity index 93% rename from backend/core/src/main/resources/config/liquibase/13-37-0/13-37-0-master.xml rename to backend/core/src/main/resources/config/liquibase/13-38-0/13-38-0-master.xml index 07b932ce9a..7a11949717 100644 --- a/backend/core/src/main/resources/config/liquibase/13-37-0/13-37-0-master.xml +++ b/backend/core/src/main/resources/config/liquibase/13-38-0/13-38-0-master.xml @@ -20,6 +20,7 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://www.liquibase.org/xml/ns/dbchangelog" xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.4.xsd"> + diff --git a/backend/core/src/main/resources/config/liquibase/13-37-0/20260630-create-reindex-run-table.xml b/backend/core/src/main/resources/config/liquibase/13-38-0/20260630-create-reindex-run-table.xml similarity index 100% rename from backend/core/src/main/resources/config/liquibase/13-37-0/20260630-create-reindex-run-table.xml rename to backend/core/src/main/resources/config/liquibase/13-38-0/20260630-create-reindex-run-table.xml diff --git a/backend/core/src/main/resources/config/liquibase/initial-setup/case/changelog/20260701-add-changed-on-to-json-schema-document.xml b/backend/core/src/main/resources/config/liquibase/13-38-0/20260701-add-changed-on-to-json-schema-document.xml similarity index 86% rename from backend/core/src/main/resources/config/liquibase/initial-setup/case/changelog/20260701-add-changed-on-to-json-schema-document.xml rename to backend/core/src/main/resources/config/liquibase/13-38-0/20260701-add-changed-on-to-json-schema-document.xml index c24986d41c..0598b3baae 100644 --- a/backend/core/src/main/resources/config/liquibase/initial-setup/case/changelog/20260701-add-changed-on-to-json-schema-document.xml +++ b/backend/core/src/main/resources/config/liquibase/13-38-0/20260701-add-changed-on-to-json-schema-document.xml @@ -16,17 +16,18 @@ --> + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.4.xsd" + logicalFilePath="config/liquibase/changelog/20260701-add-changed-on-to-json-schema-document.xml"> - + - + diff --git a/backend/core/src/main/resources/config/liquibase/13-37-0/20260701-create-pending-index-deletion-table.xml b/backend/core/src/main/resources/config/liquibase/13-38-0/20260701-create-pending-index-deletion-table.xml similarity index 100% rename from backend/core/src/main/resources/config/liquibase/13-37-0/20260701-create-pending-index-deletion-table.xml rename to backend/core/src/main/resources/config/liquibase/13-38-0/20260701-create-pending-index-deletion-table.xml diff --git a/backend/core/src/main/resources/config/liquibase/13-37-0/20260701-create-reconcile-state-table.xml b/backend/core/src/main/resources/config/liquibase/13-38-0/20260701-create-reconcile-state-table.xml similarity index 100% rename from backend/core/src/main/resources/config/liquibase/13-37-0/20260701-create-reconcile-state-table.xml rename to backend/core/src/main/resources/config/liquibase/13-38-0/20260701-create-reconcile-state-table.xml diff --git a/backend/core/src/main/resources/config/liquibase/13-32-0/20260710-add-reindex-run-pruned-count.xml b/backend/core/src/main/resources/config/liquibase/13-38-0/20260710-add-reindex-run-pruned-count.xml similarity index 100% rename from backend/core/src/main/resources/config/liquibase/13-32-0/20260710-add-reindex-run-pruned-count.xml rename to backend/core/src/main/resources/config/liquibase/13-38-0/20260710-add-reindex-run-pruned-count.xml diff --git a/backend/core/src/main/resources/config/liquibase/13-37-0/20260713-add-reindex-run-total-count.xml b/backend/core/src/main/resources/config/liquibase/13-38-0/20260713-add-reindex-run-total-count.xml similarity index 100% rename from backend/core/src/main/resources/config/liquibase/13-37-0/20260713-add-reindex-run-total-count.xml rename to backend/core/src/main/resources/config/liquibase/13-38-0/20260713-add-reindex-run-total-count.xml diff --git a/backend/core/src/main/resources/config/liquibase/13-37-0/20260713-backfill-modified-on.xml b/backend/core/src/main/resources/config/liquibase/13-38-0/20260713-backfill-modified-on.xml similarity index 100% rename from backend/core/src/main/resources/config/liquibase/13-37-0/20260713-backfill-modified-on.xml rename to backend/core/src/main/resources/config/liquibase/13-38-0/20260713-backfill-modified-on.xml diff --git a/backend/core/src/main/resources/config/liquibase/changelog-master.xml b/backend/core/src/main/resources/config/liquibase/changelog-master.xml index 5e6656a943..68ca8fe1a3 100644 --- a/backend/core/src/main/resources/config/liquibase/changelog-master.xml +++ b/backend/core/src/main/resources/config/liquibase/changelog-master.xml @@ -33,6 +33,6 @@ - + diff --git a/backend/core/src/main/resources/config/liquibase/initial-setup/initial-setup-master.xml b/backend/core/src/main/resources/config/liquibase/initial-setup/initial-setup-master.xml index de178a80f5..21ac06820a 100644 --- a/backend/core/src/main/resources/config/liquibase/initial-setup/initial-setup-master.xml +++ b/backend/core/src/main/resources/config/liquibase/initial-setup/initial-setup-master.xml @@ -1,6 +1,6 @@ - +

{{ 'adminSettings.opensearch.reindex.startModalTitle' | translate }}

 diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/start-reindex-modal/start-reindex-modal.component.ts b/frontend/projects/valtimo/admin-settings/src/lib/components/start-reindex-modal/start-reindex-modal.component.ts index a9ec10c269..be036db887 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/components/start-reindex-modal/start-reindex-modal.component.ts +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/start-reindex-modal/start-reindex-modal.component.ts @@ -90,8 +90,8 @@ export class StartReindexModalComponent { request.pruneOrphans = true; } - if (formValue.documentDefinitionName?.content) { - request.documentDefinitionName = formValue.documentDefinitionName.content; + if (formValue.documentDefinitionName?.value) { + request.documentDefinitionName = formValue.documentDefinitionName.value; } if (formValue.modifiedAfter) { diff --git a/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.ts b/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.ts index ce38462c52..e9b4e20ca0 100644 --- a/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.ts +++ b/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.ts @@ -264,8 +264,13 @@ export class CarbonListComponent implements OnInit, AfterViewInit, OnDestroy { public skeletonModel = Table.skeletonModel(5, 5); public paginationModel: PaginationModel; public searchFormControl = new FormControl(''); + public showAutocomplete = false; + public filteredSuggestions: SearchField[] = []; + public selectedSuggestionIndex = -1; + public autocompleteLeft = 0; private _lastExecutedSearch: string | null = null; + private _searchInputElement: HTMLInputElement | null = null; private static readonly PAGINATION_SIZE = 'PaginationSize'; private readonly _subscriptions = new Subscription(); private readonly _expandedRowKeys = new Set(); @@ -812,12 +817,6 @@ export class CarbonListComponent implements OnInit, AfterViewInit, OnDestroy { return segments; } - public showAutocomplete = false; - public filteredSuggestions: SearchField[] = []; - public selectedSuggestionIndex = -1; - public autocompleteLeft = 0; - private _searchInputElement: HTMLInputElement | null = null; - private getSearchInputElement(): HTMLInputElement | null { if (!this._searchInputElement) { this._searchInputElement = this.elementRef.nativeElement.querySelector( diff --git a/frontend/projects/valtimo/shared/assets/core/en.json b/frontend/projects/valtimo/shared/assets/core/en.json index e21a3ca127..c4cf8d6909 100644 --- a/frontend/projects/valtimo/shared/assets/core/en.json +++ b/frontend/projects/valtimo/shared/assets/core/en.json @@ -3340,6 +3340,7 @@ "pruneOrphans": "Prune orphaned documents", "documentDefinitionName": "Document definition", "documentDefinitionPlaceholder": "All document definitions", + "allDocumentDefinitions": "All document definitions", "modifiedBefore": "Modified before", "modifiedAfter": "Modified after", "columns": { diff --git a/frontend/projects/valtimo/shared/assets/core/nl.json b/frontend/projects/valtimo/shared/assets/core/nl.json index 593f36c454..6847d4e09e 100644 --- a/frontend/projects/valtimo/shared/assets/core/nl.json +++ b/frontend/projects/valtimo/shared/assets/core/nl.json @@ -3373,6 +3373,7 @@ "pruneOrphans": "Verwijder orphan documenten", "documentDefinitionName": "Documentdefinitie", "documentDefinitionPlaceholder": "Alle documentdefinities", + "allDocumentDefinitions": "Alle documentdefinities", "modifiedBefore": "Gewijzigd voor", "modifiedAfter": "Gewijzigd na", "columns": { From f2d663271b20852198fc8ab4149b174b22d5ead1 Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Thu, 16 Jul 2026 17:16:24 +0200 Subject: [PATCH 32/46] pr feedback --- .../mapper/JsonSchemaDocumentCaseDefinitionOpenSearchMapper.kt | 2 -- 1 file changed, 2 deletions(-) diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentCaseDefinitionOpenSearchMapper.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentCaseDefinitionOpenSearchMapper.kt index cca791cf7c..1673076c67 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentCaseDefinitionOpenSearchMapper.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentCaseDefinitionOpenSearchMapper.kt @@ -36,8 +36,6 @@ import org.opensearch.index.query.QueryBuilders class JsonSchemaDocumentCaseDefinitionOpenSearchMapper : OpenSearchAuthorizationEntityMapper { override fun mapQuery(conditions: List): QueryBuilder? { - if (conditions.isEmpty()) return null - val conditionQueries = conditions.map { condition -> when (condition) { is FieldPermissionCondition<*> -> { From 5c2e1732157890d2e8dc97df1a4e6c52a9d8feed Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Thu, 16 Jul 2026 18:02:17 +0200 Subject: [PATCH 33/46] changed the changedOn update to not rely on the CurrentTimeStamp annotation. This broke optimistic locking fixes, so instead we use the JVM time. --- .../domain/impl/JsonSchemaDocument.java | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/backend/case/src/main/java/com/ritense/document/domain/impl/JsonSchemaDocument.java b/backend/case/src/main/java/com/ritense/document/domain/impl/JsonSchemaDocument.java index 91b27285b3..d2f5bf177f 100644 --- a/backend/case/src/main/java/com/ritense/document/domain/impl/JsonSchemaDocument.java +++ b/backend/case/src/main/java/com/ritense/document/domain/impl/JsonSchemaDocument.java @@ -54,6 +54,8 @@ import jakarta.persistence.JoinTable; import jakarta.persistence.ManyToMany; import jakarta.persistence.ManyToOne; +import jakarta.persistence.PrePersist; +import jakarta.persistence.PreUpdate; import jakarta.persistence.Table; import jakarta.persistence.Version; import java.time.LocalDateTime; @@ -68,9 +70,7 @@ import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.StreamSupport; -import org.hibernate.annotations.CurrentTimestamp; import org.hibernate.annotations.DynamicUpdate; -import org.hibernate.annotations.SourceType; import org.hibernate.annotations.Type; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -110,13 +110,6 @@ public class JsonSchemaDocument extends AbstractAggregateRootevery mutation, so it - * is the watermark the OpenSearch reconciler scans on. Written by every save; never set by hand. - */ - @CurrentTimestamp(source = SourceType.DB) @Column(name = "changed_on", columnDefinition = "DATETIME", nullable = false) private LocalDateTime changedOn; @@ -407,6 +400,12 @@ public LocalDateTime changedOn() { return changedOn; } + @PrePersist + @PreUpdate + void updateChangedOn() { + this.changedOn = LocalDateTime.now(); + } + @Override public JsonDocumentContent content() { return content; From f50634a4977293155d417a4fe4417893e9ae9e7a Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Fri, 17 Jul 2026 10:26:26 +0200 Subject: [PATCH 34/46] added opensearch configurations to the other apps + moved dependency to valtimo-dependencies --- backend/apps/dev/build.gradle | 1 - .../src/main/resources/config/application.yml | 3 --- .../dev/src/main/resources/logback-spring.xml | 1 + .../src/main/resources/config/application.yml | 5 ++++ .../src/main/resources/logback-spring.xml | 17 ++++++++++++ .../src/main/resources/config/application.yml | 5 ++++ .../src/main/resources/logback-spring.xml | 17 ++++++++++++ .../src/main/resources/config/application.yml | 7 +++++ .../src/main/resources/logback-spring.xml | 17 ++++++++++++ .../valtimo-dependencies/build.gradle | 3 ++- .../src/environments/environment.ts | 26 +++++++++++++++++-- .../valtimo/src/environments/environment.ts | 26 +++++++++++++++++-- 12 files changed, 119 insertions(+), 9 deletions(-) diff --git a/backend/apps/dev/build.gradle b/backend/apps/dev/build.gradle index a0abe066bd..ced247e784 100644 --- a/backend/apps/dev/build.gradle +++ b/backend/apps/dev/build.gradle @@ -36,7 +36,6 @@ dependencies { implementation(platform(project(":backend:dependencies:valtimo-dependency-versions"))) implementation(project(":backend:dependencies:valtimo-gzac-dependencies")) - implementation(project(":backend:case-opensearch")) implementation(project(":backend:mail:local-mail")) implementation(project(":backend:document-generation:smartdocuments")) implementation(project(":backend:zgw:portaaltaak")) diff --git a/backend/apps/dev/src/main/resources/config/application.yml b/backend/apps/dev/src/main/resources/config/application.yml index 715e03e7bf..25fb79d07b 100644 --- a/backend/apps/dev/src/main/resources/config/application.yml +++ b/backend/apps/dev/src/main/resources/config/application.yml @@ -1,9 +1,6 @@ logging: file: name: /tmp/spring.log - level: - com.ritense.document.opensearch: DEBUG - org.opensearch.client.RestClient: WARN management: endpoints: diff --git a/backend/apps/dev/src/main/resources/logback-spring.xml b/backend/apps/dev/src/main/resources/logback-spring.xml index e21b79db93..2197248e46 100644 --- a/backend/apps/dev/src/main/resources/logback-spring.xml +++ b/backend/apps/dev/src/main/resources/logback-spring.xml @@ -63,6 +63,7 @@ + true diff --git a/backend/apps/evenementenvergunning/src/main/resources/config/application.yml b/backend/apps/evenementenvergunning/src/main/resources/config/application.yml index 2d7fbfed62..73261dba58 100644 --- a/backend/apps/evenementenvergunning/src/main/resources/config/application.yml +++ b/backend/apps/evenementenvergunning/src/main/resources/config/application.yml @@ -137,6 +137,9 @@ server: mime-types: text/html,text/xml,text/plain,text/css, application/javascript, application/json min-response-size: 1024 +opensearch: + uris: ${OPENSEARCH_URIS:http://localhost:9200} + mailing: onlyAllowWhitelistedRecipients: true redirectAllMails: false @@ -145,6 +148,8 @@ mailing: sendRedirectedMailsTo: valtimo: + opensearch: + enabled: ${VALTIMO_OPENSEARCH_ENABLED:false} app: scheme: http hostname: ${VALTIMO_APP_HOSTNAME} diff --git a/backend/apps/evenementenvergunning/src/main/resources/logback-spring.xml b/backend/apps/evenementenvergunning/src/main/resources/logback-spring.xml index 590e9819e8..4292a05559 100644 --- a/backend/apps/evenementenvergunning/src/main/resources/logback-spring.xml +++ b/backend/apps/evenementenvergunning/src/main/resources/logback-spring.xml @@ -1,5 +1,21 @@ + + @@ -44,6 +60,7 @@ + true diff --git a/backend/apps/gzac/src/main/resources/config/application.yml b/backend/apps/gzac/src/main/resources/config/application.yml index 2d7fbfed62..73261dba58 100644 --- a/backend/apps/gzac/src/main/resources/config/application.yml +++ b/backend/apps/gzac/src/main/resources/config/application.yml @@ -137,6 +137,9 @@ server: mime-types: text/html,text/xml,text/plain,text/css, application/javascript, application/json min-response-size: 1024 +opensearch: + uris: ${OPENSEARCH_URIS:http://localhost:9200} + mailing: onlyAllowWhitelistedRecipients: true redirectAllMails: false @@ -145,6 +148,8 @@ mailing: sendRedirectedMailsTo: valtimo: + opensearch: + enabled: ${VALTIMO_OPENSEARCH_ENABLED:false} app: scheme: http hostname: ${VALTIMO_APP_HOSTNAME} diff --git a/backend/apps/gzac/src/main/resources/logback-spring.xml b/backend/apps/gzac/src/main/resources/logback-spring.xml index 590e9819e8..4292a05559 100644 --- a/backend/apps/gzac/src/main/resources/logback-spring.xml +++ b/backend/apps/gzac/src/main/resources/logback-spring.xml @@ -1,5 +1,21 @@ + + @@ -44,6 +60,7 @@ + true diff --git a/backend/apps/valtimo/src/main/resources/config/application.yml b/backend/apps/valtimo/src/main/resources/config/application.yml index e4ff90f8bd..5a67e9a7b3 100644 --- a/backend/apps/valtimo/src/main/resources/config/application.yml +++ b/backend/apps/valtimo/src/main/resources/config/application.yml @@ -18,6 +18,8 @@ logging: management: endpoint: health: + probes: + enabled: true group: # readiness (and a dedicated startup group) only report UP once the bootstrap # health indicator is UP, i.e. after migrations + autodeployments have finished. @@ -137,6 +139,9 @@ server: mime-types: text/html,text/xml,text/plain,text/css, application/javascript, application/json min-response-size: 1024 +opensearch: + uris: ${OPENSEARCH_URIS:http://localhost:9200} + mailing: onlyAllowWhitelistedRecipients: true redirectAllMails: false @@ -145,6 +150,8 @@ mailing: sendRedirectedMailsTo: valtimo: + opensearch: + enabled: ${VALTIMO_OPENSEARCH_ENABLED:false} app: scheme: http hostname: ${VALTIMO_APP_HOSTNAME} diff --git a/backend/apps/valtimo/src/main/resources/logback-spring.xml b/backend/apps/valtimo/src/main/resources/logback-spring.xml index 590e9819e8..4292a05559 100644 --- a/backend/apps/valtimo/src/main/resources/logback-spring.xml +++ b/backend/apps/valtimo/src/main/resources/logback-spring.xml @@ -1,5 +1,21 @@ + + @@ -44,6 +60,7 @@ + true diff --git a/backend/dependencies/valtimo-dependencies/build.gradle b/backend/dependencies/valtimo-dependencies/build.gradle index b58e6352b0..663b4879ab 100644 --- a/backend/dependencies/valtimo-dependencies/build.gradle +++ b/backend/dependencies/valtimo-dependencies/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright 2015-2024 Ritense BV, the Netherlands. + * Copyright 2015-2026 Ritense BV, the Netherlands. * * Licensed under EUPL, Version 1.2 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ dependencies { api(project(":backend:authorization")) api(project(":backend:building-block")) api(project(":backend:case")) + api(project(":backend:case-opensearch")) api(project(":backend:changelog")) api(project(":backend:command-handling")) api(project(":backend:contract")) diff --git a/frontend/apps/evenementenvergunning/src/environments/environment.ts b/frontend/apps/evenementenvergunning/src/environments/environment.ts index 8a005e3b81..ff12518066 100644 --- a/frontend/apps/evenementenvergunning/src/environments/environment.ts +++ b/frontend/apps/evenementenvergunning/src/environments/environment.ts @@ -1,3 +1,19 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + // This file can be replaced during build by using the `fileReplacements` array. // `ng build --prod` replaces `environment.ts` with `environment.prod.ts`. // The list of file replacements can be found in `angular.json`. @@ -108,8 +124,14 @@ export const environment: ValtimoConfig = { {title: 'Other', textClass: 'text-dark font-weight-bold c-default', sequence: 16}, {link: ['/logging'], title: 'Logs', sequence: 17}, - {link: ['/case-migration'], title: 'Case migration (beta)', sequence: 18}, - {link: ['/process-migration'], title: 'Process migration', sequence: 19}, + { + link: ['/opensearch'], + title: 'adminSettings.opensearch.title', + sequence: 18, + includeFunction: IncludeFunction.OpenSearchEnabled, + }, + {link: ['/case-migration'], title: 'Case migration (beta)', sequence: 19}, + {link: ['/process-migration'], title: 'Process migration', sequence: 20}, ], }, { diff --git a/frontend/apps/valtimo/src/environments/environment.ts b/frontend/apps/valtimo/src/environments/environment.ts index f0347ada73..2cd6398800 100644 --- a/frontend/apps/valtimo/src/environments/environment.ts +++ b/frontend/apps/valtimo/src/environments/environment.ts @@ -1,3 +1,19 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + // This file can be replaced during build by using the `fileReplacements` array. // `ng build --prod` replaces `environment.ts` with `environment.prod.ts`. // The list of file replacements can be found in `angular.json`. @@ -42,8 +58,14 @@ export const environment: ValtimoConfig = { {title: 'Other', textClass: 'text-dark font-weight-bold c-default', sequence: 15}, {link: ['/logging'], title: 'Logs', sequence: 16}, - {link: ['/case-migration'], title: 'Case migration (beta)', sequence: 17}, - {link: ['/process-migration'], title: 'Process migration', sequence: 18}, + { + link: ['/opensearch'], + title: 'adminSettings.opensearch.title', + sequence: 17, + includeFunction: IncludeFunction.OpenSearchEnabled, + }, + {link: ['/case-migration'], title: 'Case migration (beta)', sequence: 18}, + {link: ['/process-migration'], title: 'Process migration', sequence: 19}, ], }, { From fab0808957dc9085bf930811ed98970d4e0acaef Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Fri, 17 Jul 2026 11:09:13 +0200 Subject: [PATCH 35/46] fixed bug where the searches would be case sensitive --- .../valtimo/contract/database/PostgresQueryDialectHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/contract/src/main/java/com/ritense/valtimo/contract/database/PostgresQueryDialectHelper.java b/backend/contract/src/main/java/com/ritense/valtimo/contract/database/PostgresQueryDialectHelper.java index 888dd0e5b3..2eedfeb6b2 100644 --- a/backend/contract/src/main/java/com/ritense/valtimo/contract/database/PostgresQueryDialectHelper.java +++ b/backend/contract/src/main/java/com/ritense/valtimo/contract/database/PostgresQueryDialectHelper.java @@ -67,7 +67,7 @@ public Predicate getJsonValueExistsExpression(CriteriaBuilder cb, Path column, S cb.function( "jsonpath", String.class, - cb.literal("$.** ? (@ like_regex \"" + value + "\")") + cb.literal("$.** ? (@ like_regex \"" + value + "\" flag \"i\")") ) ) ); From 14901abcac3cdeebce4d000b877aea219757b315 Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Fri, 17 Jul 2026 12:02:04 +0200 Subject: [PATCH 36/46] security for opensearch --- backend/apps/dev/build.gradle | 3 + backend/apps/dev/docker-compose.yaml | 5 +- .../InsecureSslOpenSearchClientConfig.kt | 74 +++++++++++++++++++ .../src/main/resources/config/application.yml | 4 +- .../src/main/resources/config/application.yml | 2 + .../src/main/resources/config/application.yml | 2 + .../src/main/resources/config/application.yml | 2 + 7 files changed, 88 insertions(+), 4 deletions(-) create mode 100644 backend/apps/dev/src/main/kotlin/com/ritense/gzac/opensearch/InsecureSslOpenSearchClientConfig.kt diff --git a/backend/apps/dev/build.gradle b/backend/apps/dev/build.gradle index ced247e784..29475cd733 100644 --- a/backend/apps/dev/build.gradle +++ b/backend/apps/dev/build.gradle @@ -51,6 +51,9 @@ dependencies { implementation "org.springframework.boot:spring-boot-starter-webflux" implementation "org.springframework.boot:spring-boot-starter-security" + // OpenSearch client for InsecureSslOpenSearchClientConfig + implementation "org.opensearch.client:spring-data-opensearch-starter:${springDataOpenSearchVersion}" + // Spring cloud stream RabbitMQ implementation "org.springframework.cloud:spring-cloud-starter-stream-rabbit:${springCloudStreamVersion}" implementation "com.rabbitmq:amqp-client:$amqpCLientVersion" diff --git a/backend/apps/dev/docker-compose.yaml b/backend/apps/dev/docker-compose.yaml index 498c7b09ce..afaa1a1af0 100644 --- a/backend/apps/dev/docker-compose.yaml +++ b/backend/apps/dev/docker-compose.yaml @@ -89,13 +89,12 @@ services: - "9200:9200" environment: - discovery.type=single-node - - DISABLE_SECURITY_PLUGIN=true - - DISABLE_INSTALL_DEMO_CONFIG=true + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=2cr8zkOoEXQJ3xUk!Aa1 - OPENSEARCH_JAVA_OPTS=-Xms256m -Xmx256m volumes: - gzac-opensearch-data:/usr/share/opensearch/data healthcheck: - test: [ "CMD-SHELL", "curl -sf http://localhost:9200/_cluster/health || exit 1" ] + test: [ "CMD-SHELL", "curl -sf -u admin:2cr8zkOoEXQJ3xUk!Aa1 https://localhost:9200/_cluster/health -k || exit 1" ] interval: 10s timeout: 5s retries: 12 diff --git a/backend/apps/dev/src/main/kotlin/com/ritense/gzac/opensearch/InsecureSslOpenSearchClientConfig.kt b/backend/apps/dev/src/main/kotlin/com/ritense/gzac/opensearch/InsecureSslOpenSearchClientConfig.kt new file mode 100644 index 0000000000..7b99905f44 --- /dev/null +++ b/backend/apps/dev/src/main/kotlin/com/ritense/gzac/opensearch/InsecureSslOpenSearchClientConfig.kt @@ -0,0 +1,74 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.gzac.opensearch + +import org.apache.http.auth.AuthScope +import org.apache.http.auth.UsernamePasswordCredentials +import org.apache.http.conn.ssl.NoopHostnameVerifier +import org.apache.http.conn.ssl.TrustAllStrategy +import org.apache.http.impl.client.BasicCredentialsProvider +import org.apache.http.ssl.SSLContextBuilder +import org.opensearch.client.RestClientBuilder +import org.opensearch.spring.boot.autoconfigure.RestClientBuilderCustomizer +import org.springframework.beans.factory.annotation.Value +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration + +/** + * Dev-only configuration that disables SSL certificate verification for OpenSearch + * and configures basic authentication. + * Allows connecting to OpenSearch with self-signed certificates in local development. + * + * DO NOT use this configuration in production. + */ +@Configuration +class InsecureSslOpenSearchClientConfig { + + @Value("\${opensearch.username:}") + private lateinit var username: String + + @Value("\${opensearch.password:}") + private lateinit var password: String + + @Bean + fun insecureSslRestClientBuilderCustomizer(): RestClientBuilderCustomizer { + return object : RestClientBuilderCustomizer { + override fun customize(builder: RestClientBuilder) { + builder.setHttpClientConfigCallback { httpClientBuilder -> + val sslContext = SSLContextBuilder.create() + .loadTrustMaterial(TrustAllStrategy.INSTANCE) + .build() + + httpClientBuilder + .setSSLContext(sslContext) + .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE) + + if (username.isNotBlank() && password.isNotBlank()) { + val credentialsProvider = BasicCredentialsProvider() + credentialsProvider.setCredentials( + AuthScope.ANY, + UsernamePasswordCredentials(username, password) + ) + httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider) + } + + httpClientBuilder + } + } + } + } +} diff --git a/backend/apps/dev/src/main/resources/config/application.yml b/backend/apps/dev/src/main/resources/config/application.yml index 25fb79d07b..e46547c891 100644 --- a/backend/apps/dev/src/main/resources/config/application.yml +++ b/backend/apps/dev/src/main/resources/config/application.yml @@ -176,7 +176,9 @@ mailing: sendRedirectedMailsTo: opensearch: - uris: http://localhost:9200 + uris: https://localhost:9200 + username: admin + password: 2cr8zkOoEXQJ3xUk!Aa1 valtimo: opensearch: diff --git a/backend/apps/evenementenvergunning/src/main/resources/config/application.yml b/backend/apps/evenementenvergunning/src/main/resources/config/application.yml index 73261dba58..08f5ff70be 100644 --- a/backend/apps/evenementenvergunning/src/main/resources/config/application.yml +++ b/backend/apps/evenementenvergunning/src/main/resources/config/application.yml @@ -139,6 +139,8 @@ server: opensearch: uris: ${OPENSEARCH_URIS:http://localhost:9200} + username: ${OPENSEARCH_USERNAME:} + password: ${OPENSEARCH_PASSWORD:} mailing: onlyAllowWhitelistedRecipients: true diff --git a/backend/apps/gzac/src/main/resources/config/application.yml b/backend/apps/gzac/src/main/resources/config/application.yml index 73261dba58..08f5ff70be 100644 --- a/backend/apps/gzac/src/main/resources/config/application.yml +++ b/backend/apps/gzac/src/main/resources/config/application.yml @@ -139,6 +139,8 @@ server: opensearch: uris: ${OPENSEARCH_URIS:http://localhost:9200} + username: ${OPENSEARCH_USERNAME:} + password: ${OPENSEARCH_PASSWORD:} mailing: onlyAllowWhitelistedRecipients: true diff --git a/backend/apps/valtimo/src/main/resources/config/application.yml b/backend/apps/valtimo/src/main/resources/config/application.yml index 5a67e9a7b3..00b64043e4 100644 --- a/backend/apps/valtimo/src/main/resources/config/application.yml +++ b/backend/apps/valtimo/src/main/resources/config/application.yml @@ -141,6 +141,8 @@ server: opensearch: uris: ${OPENSEARCH_URIS:http://localhost:9200} + username: ${OPENSEARCH_USERNAME:} + password: ${OPENSEARCH_PASSWORD:} mailing: onlyAllowWhitelistedRecipients: true From bce4230004c3016b756afef5a3b7bbf31ab433be Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Fri, 17 Jul 2026 13:29:02 +0200 Subject: [PATCH 37/46] added progress bar for pruning --- .../opensearch/domain/OpenSearchReindexRun.kt | 26 ++++++++++++++ .../DocumentOpenSearchReindexService.kt | 34 ++++++++++++++++-- .../service/OpenSearchReindexRunService.kt | 15 ++++++++ .../liquibase/13-38-0/13-38-0-master.xml | 1 + .../13-38-0/20260717-add-prune-progress.xml | 36 +++++++++++++++++++ .../admin-settings-opensearch.component.html | 36 +++++++++++++------ .../admin-settings-opensearch.component.scss | 20 +++++++++++ .../src/lib/models/reindex.model.ts | 3 ++ .../valtimo/shared/assets/core/en.json | 8 +++-- .../valtimo/shared/assets/core/nl.json | 8 +++-- 10 files changed, 169 insertions(+), 18 deletions(-) create mode 100644 backend/core/src/main/resources/config/liquibase/13-38-0/20260717-add-prune-progress.xml diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/OpenSearchReindexRun.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/OpenSearchReindexRun.kt index c4797ed44a..5267309e1d 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/OpenSearchReindexRun.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/OpenSearchReindexRun.kt @@ -80,6 +80,15 @@ class OpenSearchReindexRun( @Column(name = "total_count") var totalCount: Long? = null, + + @Column(name = "prune_checked_count", nullable = false) + var pruneCheckedCount: Long = 0, + + @Column(name = "prune_total_count") + var pruneTotalCount: Long? = null, + + @Column(name = "pruning_phase", nullable = false) + var pruningPhase: Boolean = false, ) { /** Records progress after a committed batch: the keyset cursor, counts and a fresh heartbeat. */ @@ -90,10 +99,25 @@ class OpenSearchReindexRun( this.heartbeatOn = heartbeat } + fun startPruning(totalOsCount: Long, heartbeat: LocalDateTime) { + this.pruningPhase = true + this.pruneTotalCount = totalOsCount + this.pruneCheckedCount = 0 + this.prunedCount = 0 + this.heartbeatOn = heartbeat + } + + fun recordPruneProgress(checked: Long, pruned: Long, heartbeat: LocalDateTime) { + this.pruneCheckedCount = checked + this.prunedCount = pruned + this.heartbeatOn = heartbeat + } + fun complete(now: LocalDateTime) { this.status = ReindexRunStatus.COMPLETED this.finishedOn = now this.heartbeatOn = now + this.pruningPhase = false } fun fail(now: LocalDateTime, error: String?) { @@ -101,12 +125,14 @@ class OpenSearchReindexRun( this.finishedOn = now this.heartbeatOn = now this.error = error + this.pruningPhase = false } fun stop(now: LocalDateTime) { this.status = ReindexRunStatus.STOPPED this.finishedOn = now this.heartbeatOn = now + this.pruningPhase = false } /** Re-arms a previously-finished (FAILED/STOPPED) run so it can be resumed from its cursor. */ diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt index d24681e059..851e16dea8 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt @@ -147,8 +147,7 @@ open class DocumentOpenSearchReindexService( runService.stop(runId) } else { if (scope.pruneOrphans) { - val pruned = pruneOrphans(scope) - runService.recordPruned(runId, pruned) + val pruned = pruneOrphans(runId, scope) logger.info { "Pruned $pruned orphan document(s) from OpenSearch" } } indexOps().refresh() @@ -203,10 +202,14 @@ open class DocumentOpenSearchReindexService( * and deletes orphans (documents in OpenSearch but not in PostgreSQL). * Uses scroll API to handle datasets larger than 10k documents. */ - private fun pruneOrphans(scope: ReindexRequest): Long { + private fun pruneOrphans(runId: UUID, scope: ReindexRequest): Long { var pruned = 0L + var checked = 0L val txTemplate = TransactionTemplate(transactionManager).apply { isReadOnly = true } + val totalOsCount = countOpenSearchDocs(scope) + runService.startPruning(runId, totalOsCount) + scrollOpenSearchIds(scope, PRUNE_BATCH_SIZE) { osIds -> if (cancelRequested) return@scrollOpenSearchIds false @@ -217,12 +220,37 @@ open class DocumentOpenSearchReindexService( val orphans = osIds.filter { UUID.fromString(it) !in existingIds } orphans.forEach { openSearchRepository.deleteById(it) } pruned += orphans.size + checked += osIds.size + runService.recordPruneProgress(runId, checked, pruned) true } return pruned } + private fun countOpenSearchDocs(scope: ReindexRequest): Long { + val boolQuery = BoolQueryBuilder() + + scope.documentDefinitionName?.let { + boolQuery.filter(QueryBuilders.termQuery("definitionId.name", it)) + } + scope.modifiedAfter?.let { + boolQuery.filter(QueryBuilders.rangeQuery("modifiedOn").gt(it.format(OS_DATE_FORMAT))) + } + scope.modifiedBefore?.let { + boolQuery.filter(QueryBuilders.rangeQuery("modifiedOn").lt(it.format(OS_DATE_FORMAT))) + } + scope.documentIds?.takeIf { it.isNotEmpty() }?.let { ids -> + boolQuery.filter(QueryBuilders.idsQuery().addIds(*ids.map { it.toString() }.toTypedArray())) + } + + val query = NativeSearchQueryBuilder() + .withQuery(boolQuery) + .build() + + return elasticsearchOperations.count(query, JsonSchemaDocumentOsDocument::class.java) + } + /** * Scrolls through all OpenSearch documents matching [scope], invoking [handler] for each batch. * Handler returns `true` to continue, `false` to stop early. diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunService.kt index 31b1364240..677efdb18f 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunService.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunService.kt @@ -148,6 +148,18 @@ open class OpenSearchReindexRunService( repository.save(run) } + open fun startPruning(runId: UUID, totalOsCount: Long) { + val run = requireRun(runId) + run.startPruning(totalOsCount, LocalDateTime.now()) + repository.save(run) + } + + open fun recordPruneProgress(runId: UUID, checked: Long, pruned: Long) { + val run = requireRun(runId) + run.recordPruneProgress(checked, pruned, LocalDateTime.now()) + repository.save(run) + } + /** * Status of a specific run (by [runId]) or — when null — of the most recent run. Returns a * not-running placeholder when no matching run exists. @@ -182,6 +194,9 @@ open class OpenSearchReindexRunService( "processedCount" to run.processedCount, "skippedCount" to run.skippedCount, "prunedCount" to run.prunedCount, + "pruneCheckedCount" to run.pruneCheckedCount, + "pruneTotalCount" to run.pruneTotalCount, + "pruningPhase" to run.pruningPhase, "totalCount" to getTotalCount(run, scope), "startedOn" to run.startedOn, "heartbeatOn" to run.heartbeatOn, diff --git a/backend/core/src/main/resources/config/liquibase/13-38-0/13-38-0-master.xml b/backend/core/src/main/resources/config/liquibase/13-38-0/13-38-0-master.xml index 7a11949717..faf435b689 100644 --- a/backend/core/src/main/resources/config/liquibase/13-38-0/13-38-0-master.xml +++ b/backend/core/src/main/resources/config/liquibase/13-38-0/13-38-0-master.xml @@ -27,5 +27,6 @@ xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liqui + diff --git a/backend/core/src/main/resources/config/liquibase/13-38-0/20260717-add-prune-progress.xml b/backend/core/src/main/resources/config/liquibase/13-38-0/20260717-add-prune-progress.xml new file mode 100644 index 0000000000..9aba6480be --- /dev/null +++ b/backend/core/src/main/resources/config/liquibase/13-38-0/20260717-add-prune-progress.xml @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.html b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.html index 99df793dad..9ae4f01832 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.html +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.html @@ -102,14 +102,36 @@
{{ 'adminSettings.opensearch.reindex.parameters' | translate }}
{{ 'adminSettings.opensearch.reindex.results' | translate }}
+ + {{ 'adminSettings.opensearch.reindex.reindexing' | translate }} + +
+ {{ 'adminSettings.opensearch.reindex.processed' | translate:{current: data.processedCount, total: data.totalCount} }} + {{ 'adminSettings.opensearch.reindex.statuses.' + data.status | translate }} +
+ @if (data.pruningPhase || data.pruneTotalCount) { +
+ + {{ 'adminSettings.opensearch.reindex.pruning' | translate }} + + + +
+ {{ 'adminSettings.opensearch.reindex.processed' | translate:{current: data.pruneCheckedCount, total: data.pruneTotalCount || 0} }} + {{ 'adminSettings.opensearch.reindex.pruningProgress' | translate:{removed: data.prunedCount || 0} }} +
+
+ } +
@@ -118,14 +140,6 @@
{{ 'adminSettings.opensearch.reindex.results' | translate }}
{{ data.skippedCount || 0 }}
-
- - {{ 'adminSettings.opensearch.reindex.pruned' | translate }}: - - {{ data.prunedCount || 0 }} - -
-
{{ 'adminSettings.opensearch.reindex.started' | translate }}: diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss index c742725a28..b2850ac1f9 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss @@ -65,6 +65,26 @@ gap: 4px; } + &__progress-label { + display: inline-flex; + align-items: center; + gap: 4px; + margin-bottom: 4px; + font-size: 14px; + font-weight: 600; + color: var(--cds-text-secondary); + } + + &__progress-footer { + display: flex; + justify-content: space-between; + align-items: center; + max-width: 500px; + margin-top: 4px; + font-size: 12px; + color: var(--cds-text-secondary); + } + &__progress { padding: 8px 0; max-width: 500px; diff --git a/frontend/projects/valtimo/admin-settings/src/lib/models/reindex.model.ts b/frontend/projects/valtimo/admin-settings/src/lib/models/reindex.model.ts index d7d7387dda..c9d73c8514 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/models/reindex.model.ts +++ b/frontend/projects/valtimo/admin-settings/src/lib/models/reindex.model.ts @@ -35,6 +35,9 @@ interface ReindexStatusDto { processedCount: number; skippedCount: number; prunedCount: number; + pruneCheckedCount: number; + pruneTotalCount: number | null; + pruningPhase: boolean; startedOn: string; finishedOn: string | null; elapsedSeconds: number; diff --git a/frontend/projects/valtimo/shared/assets/core/en.json b/frontend/projects/valtimo/shared/assets/core/en.json index c4cf8d6909..b283a6c85b 100644 --- a/frontend/projects/valtimo/shared/assets/core/en.json +++ b/frontend/projects/valtimo/shared/assets/core/en.json @@ -3326,9 +3326,11 @@ "detailTitle": "Run Details", "parameters": "Parameters", "results": "Results", + "reindexing": "Reindexing", + "pruning": "Pruning", "counts": "Counts", "status": "Status", - "processed": "Processed", + "processed": "Processed: {{current}} / {{total}}", "skipped": "Skipped", "pruned": "Pruned", "started": "Started", @@ -3358,8 +3360,10 @@ "RUNNING": "Running", "COMPLETED": "Completed", "FAILED": "Failed", - "STOPPED": "Stopped" + "STOPPED": "Stopped", + "PRUNING": "Pruning orphaned documents" }, + "pruningProgress": "Removed: {{removed}}", "tooltips": { "pruneOrphans": "Remove entries from OpenSearch that no longer exist in the database", "pruned": "Search entries removed because the original document was deleted" diff --git a/frontend/projects/valtimo/shared/assets/core/nl.json b/frontend/projects/valtimo/shared/assets/core/nl.json index 6847d4e09e..4b4c6490b4 100644 --- a/frontend/projects/valtimo/shared/assets/core/nl.json +++ b/frontend/projects/valtimo/shared/assets/core/nl.json @@ -3359,9 +3359,11 @@ "detailTitle": "Run details", "parameters": "Parameters", "results": "Resultaten", + "reindexing": "Herindexering", + "pruning": "Opschonen", "counts": "Aantallen", "status": "Status", - "processed": "Verwerkt", + "processed": "Verwerkt: {{current}} / {{total}}", "skipped": "Overgeslagen", "pruned": "Opgeschoond", "started": "Gestart", @@ -3391,8 +3393,10 @@ "RUNNING": "Bezig", "COMPLETED": "Voltooid", "FAILED": "Mislukt", - "STOPPED": "Gestopt" + "STOPPED": "Gestopt", + "PRUNING": "Orphan documenten opschonen" }, + "pruningProgress": "Verwijderd: {{removed}}", "tooltips": { "pruneOrphans": "Verwijder documenten uit OpenSearch die niet meer in de database bestaan", "pruned": "Documenten verwijderd uit OpenSearch omdat het oorspronkelijke document is verwijderd" From 2fd68d52d11f4b74d7290282e5351a3980d0090c Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Fri, 17 Jul 2026 14:53:26 +0200 Subject: [PATCH 38/46] documentation --- documentation/SUMMARY.md | 2 + .../getting-started/compatibility-matrix.md | 17 +- .../modules/core/case-opensearch.md | 34 +++ .../release-notes/13.x.x/13.38.0/README.md | 15 +- .../feature-toggles.md | 6 + .../application-configuration/opensearch.md | 196 ++++++++++++++++++ 6 files changed, 257 insertions(+), 13 deletions(-) create mode 100644 documentation/fundamentals/getting-started/modules/core/case-opensearch.md create mode 100644 documentation/running-valtimo/application-configuration/opensearch.md diff --git a/documentation/SUMMARY.md b/documentation/SUMMARY.md index f99972ab78..908d9a33a5 100644 --- a/documentation/SUMMARY.md +++ b/documentation/SUMMARY.md @@ -20,6 +20,7 @@ * [Authorization](fundamentals/getting-started/modules/core/authorization.md) * [Operaton](fundamentals/getting-started/modules/core/operaton-webapps.md) * [Case](fundamentals/getting-started/modules/core/case.md) + * [Case OpenSearch](fundamentals/getting-started/modules/core/case-opensearch.md) * [Contract](fundamentals/getting-started/modules/core/contract.md) * [Core](fundamentals/getting-started/modules/core/core.md) * [Dashboard](fundamentals/getting-started/modules/core/dashboard.md) @@ -267,6 +268,7 @@ * [Document upload size limit](running-valtimo/application-configuration/document-upload-size.md) * [Feature toggles](running-valtimo/application-configuration/feature-toggles.md) * [Kubernetes health probes](running-valtimo/application-configuration/kubernetes-health-probes.md) + * [OpenSearch](running-valtimo/application-configuration/opensearch.md) ## Customizing Valtimo diff --git a/documentation/fundamentals/getting-started/compatibility-matrix.md b/documentation/fundamentals/getting-started/compatibility-matrix.md index 4277a7850f..c7acaab370 100644 --- a/documentation/fundamentals/getting-started/compatibility-matrix.md +++ b/documentation/fundamentals/getting-started/compatibility-matrix.md @@ -13,14 +13,15 @@ versions. ## Major 13 -| Valtimo backend libraries | Valtimo frontend libraries | Java | Kotlin | Spring Boot | Node | Operaton | Angular | -|---------------------------|----------------------------|------|--------|-------------|--------|--------------|--------------------------------------------------------------------------------| -| 13.36.1 | 13.36.1 | 21 | 2.1.21 | 3.5.16 | 20 LTS | 1.0.3 | 19 ([browser support](https://angular.dev/reference/versions#browser-support)) | -| 13.33.0 | 13.33.0 | 21 | 2.1.20 | 3.5.15 | 20 LTS | 1.0.3 | 19 ([browser support](https://angular.dev/reference/versions#browser-support)) | -| 13.27.0 | 13.27.0 | 21 | 2.1.20 | 3.5.14 | 20 LTS | 1.0.3 | 19 ([browser support](https://angular.dev/reference/versions#browser-support)) | -| 13.23.0 | 13.23.0 | 21 | 2.1.20 | 3.5.13 | 20 LTS | 1.0.3 | 19 ([browser support](https://angular.dev/reference/versions#browser-support)) | -| 13.11.0 | 13.11.0 | 21 | 2.1.20 | 3.5.7 | 20 LTS | 1.0.0-rc-1 | 19 ([browser support](https://angular.dev/reference/versions#browser-support)) | -| 13.0.0 | 13.0.0 | 21 | 2.1.20 | 3.4.5 | 20 LTS | 1.0.0-beta-4 | 19 ([browser support](https://angular.dev/reference/versions#browser-support)) | +| Valtimo backend libraries | Valtimo frontend libraries | Java | Kotlin | Spring Boot | Node | Operaton | Angular | OpenSearch | +|---------------------------|----------------------------|------|--------|-------------|--------|--------------|--------------------------------------------------------------------------------|------------| +| 13.38.0 | 13.38.0 | 21 | 2.1.21 | 3.5.16 | 20 LTS | 1.0.3 | 19 ([browser support](https://angular.dev/reference/versions#browser-support)) | 2.19.x | +| 13.36.1 | 13.36.1 | 21 | 2.1.21 | 3.5.16 | 20 LTS | 1.0.3 | 19 ([browser support](https://angular.dev/reference/versions#browser-support)) | - | +| 13.33.0 | 13.33.0 | 21 | 2.1.20 | 3.5.15 | 20 LTS | 1.0.3 | 19 ([browser support](https://angular.dev/reference/versions#browser-support)) | - | +| 13.27.0 | 13.27.0 | 21 | 2.1.20 | 3.5.14 | 20 LTS | 1.0.3 | 19 ([browser support](https://angular.dev/reference/versions#browser-support)) | - | +| 13.23.0 | 13.23.0 | 21 | 2.1.20 | 3.5.13 | 20 LTS | 1.0.3 | 19 ([browser support](https://angular.dev/reference/versions#browser-support)) | - | +| 13.11.0 | 13.11.0 | 21 | 2.1.20 | 3.5.7 | 20 LTS | 1.0.0-rc-1 | 19 ([browser support](https://angular.dev/reference/versions#browser-support)) | - | +| 13.0.0 | 13.0.0 | 21 | 2.1.20 | 3.4.5 | 20 LTS | 1.0.0-beta-4 | 19 ([browser support](https://angular.dev/reference/versions#browser-support)) | - | ## Major 12 diff --git a/documentation/fundamentals/getting-started/modules/core/case-opensearch.md b/documentation/fundamentals/getting-started/modules/core/case-opensearch.md new file mode 100644 index 0000000000..ae5a61ef45 --- /dev/null +++ b/documentation/fundamentals/getting-started/modules/core/case-opensearch.md @@ -0,0 +1,34 @@ +# Case OpenSearch + +The case-opensearch module provides OpenSearch as an optional search backend for case lists and document queries. + +For configuration and usage documentation, see [OpenSearch](../../../../running-valtimo/application-configuration/opensearch.md). + +## Dependencies + +### Backend + +The samples below assume the [valtimo-dependency-versions](valtimo-dependency-versions.md) module is used. If not, please specify the artifact version as well. + +#### Maven dependency: + +```xml + + + com.ritense.valtimo + case-opensearch + + +``` + +#### Gradle dependency: + +```kotlin +dependencies { + implementation("com.ritense.valtimo:case-opensearch") +} +``` + +## Auto-configuration + +The module provides `DocumentOpenSearchAutoConfiguration` which is enabled when `valtimo.opensearch.enabled=true`. diff --git a/documentation/release-notes/13.x.x/13.38.0/README.md b/documentation/release-notes/13.x.x/13.38.0/README.md index 8f7ce033ae..d525fcc943 100644 --- a/documentation/release-notes/13.x.x/13.38.0/README.md +++ b/documentation/release-notes/13.x.x/13.38.0/README.md @@ -6,15 +6,20 @@ ## New Features -* **New feature title** +* **Global search** - New feature explanation. + A new global search feature allows users to search across all cases from a single search field. Users can find cases + by any text in their document content without knowing which specific field contains the information. Results are + filtered by the user's permissions. -## Enhancements +* **OpenSearch for document search** -* **New enhancement title** + Case list search and global document queries can now use OpenSearch as the search engine instead of PostgreSQL. + OpenSearch provides faster full-text search and scales better for large document volumes. The feature is opt-in + and PostgreSQL remains the source of truth. OpenSearch acts as a derived read model that syncs automatically. + See [OpenSearch search](../../../features/opensearch/README.md) for setup instructions. - New enhancement explanation. +## Enhancements ## Bugfixes diff --git a/documentation/running-valtimo/application-configuration/feature-toggles.md b/documentation/running-valtimo/application-configuration/feature-toggles.md index 99a92183d2..53c0929010 100644 --- a/documentation/running-valtimo/application-configuration/feature-toggles.md +++ b/documentation/running-valtimo/application-configuration/feature-toggles.md @@ -69,3 +69,9 @@ In backend, feature toggles can be configured in the `application.yml` file. If enabled, returns the content of the document when this is retrieved via the REST endpoint. This should only be used for debug purposes, normally content is retrieved through tabs and their widgets. + +* **`useOpenSearchForDocumentSearch`** + + If enabled, case list search and global document queries use OpenSearch instead of PostgreSQL. The toggle is + automatically disabled when OpenSearch becomes unreachable or during a full reindex operation to ensure query + results remain consistent. diff --git a/documentation/running-valtimo/application-configuration/opensearch.md b/documentation/running-valtimo/application-configuration/opensearch.md new file mode 100644 index 0000000000..6da0867795 --- /dev/null +++ b/documentation/running-valtimo/application-configuration/opensearch.md @@ -0,0 +1,196 @@ +# OpenSearch + +{% hint style="success" %} +Available since Valtimo 13.38.0 +{% endhint %} + +Valtimo can use OpenSearch as the search engine for case lists and document queries. OpenSearch provides faster full-text search and scales better than PostgreSQL for large volumes of cases. + +{% hint style="info" %} +OpenSearch is optional. PostgreSQL search works out of the box and is sufficient for most deployments. +{% endhint %} + +## When to use OpenSearch + +Consider enabling OpenSearch when: +- You have a large number of documents (hundreds of thousands or more) +- Users need fast full-text search across document content +- Case list performance with complex filters becomes slow + +For smaller deployments, PostgreSQL search is sufficient. + +## Architecture + +PostgreSQL remains the source of truth for all document data. OpenSearch acts as a derived read model that is kept in sync automatically: + +1. **Live sync** — Document changes trigger events that update OpenSearch immediately after the transaction commits. +2. **Reconciliation** — A background job periodically scans for any missed changes and repairs the index. +3. **Full reindex** — Administrators can rebuild the entire index on demand when needed. + +This architecture means: +- Writes always go to PostgreSQL first +- OpenSearch can be unavailable without data loss +- The system falls back to PostgreSQL automatically when OpenSearch is unreachable + +## Dependencies + +Add the case-opensearch module to your project. See [Case OpenSearch](../../fundamentals/getting-started/modules/core/case-opensearch.md) for Maven/Gradle dependencies. + +## Configuration + +### Infrastructure requirements + +An OpenSearch 2.19.x instance is required. The connection is configured using Spring Data OpenSearch properties: + +```yaml +spring: + opensearch: + uris: http://localhost:9200 + username: admin + password: changeme +``` + +### Enabling the feature + +The feature is disabled by default. Enable it in `application.yml`: + +```yaml +valtimo: + opensearch: + enabled: true +``` + +When enabled, the application creates the document index on startup and begins syncing documents. + +### Configuration properties + +| Property | Default | Description | +|----------|---------|-------------| +| `valtimo.opensearch.enabled` | `false` | Master switch to enable OpenSearch for document queries | +| `valtimo.opensearch.healthCheckEnabled` | `true` | Enable periodic health checks of the OpenSearch connection | +| `valtimo.opensearch.healthCheckIntervalMs` | `30000` | Interval between health checks in milliseconds | +| `valtimo.opensearch.fallbackWarningIntervalMs` | `300000` | How often to log a warning when fallback to PostgreSQL is active | + +### Reconciliation settings + +The reconciler is a background job that keeps the index in sync by scanning for changes that the live event sync may have missed. + +{% hint style="warning" %} +Duration properties use ISO 8601 duration format. Examples: `PT2M` = 2 minutes, `PT30S` = 30 seconds, `PT1H` = 1 hour. +{% endhint %} + +| Property | Default | Description | +|----------|---------|-------------| +| `valtimo.opensearch.reconcile.enabled` | `true` | Enable the scheduled reconciliation job | +| `valtimo.opensearch.reconcile.interval` | `PT2M` | How often the reconciler runs | +| `valtimo.opensearch.reconcile.overlap` | `PT10S` | Safety margin for the watermark to catch in-flight transactions | +| `valtimo.opensearch.reconcile.pageSize` | `5000` | Number of documents to scan per database query | +| `valtimo.opensearch.reconcile.pendingDeletionBatchSize` | `500` | Batch size for cleaning up deleted documents | + +### Reindex settings + +These settings control the behavior of administrator-triggered full reindex operations. + +| Property | Default | Description | +|----------|---------|-------------| +| `valtimo.opensearch.reindex.fallbackToPostgresWhileRunning` | `true` | Use PostgreSQL for queries while a reindex is in progress to avoid returning partial results | +| `valtimo.opensearch.reindex.runningHeartbeatTimeout` | `PT5M` | Consider a reindex run stale if no heartbeat is received within this duration | + +### Fallback behavior + +When OpenSearch becomes unavailable, the system automatically falls back to PostgreSQL for document queries. A warning is logged periodically (controlled by `fallbackWarningIntervalMs`). Once OpenSearch recovers, queries automatically switch back. + +During a full reindex operation, queries fall back to PostgreSQL by default to avoid returning incomplete results. This can be disabled by setting `fallbackToPostgresWhileRunning: false` if you prefer faster queries over consistency during reindex. + +### Example configuration + +```yaml +valtimo: + opensearch: + enabled: true + healthCheckEnabled: true + healthCheckIntervalMs: 30000 + fallbackWarningIntervalMs: 300000 + reconcile: + enabled: true + interval: PT2M + overlap: PT10S + pageSize: 5000 + pendingDeletionBatchSize: 500 + reindex: + fallbackToPostgresWhileRunning: true + runningHeartbeatTimeout: PT5M + +spring: + opensearch: + uris: http://opensearch:9200 +``` + +## Admin API endpoints + +All endpoints require the `ADMIN` authority. + +### Search engine toggle + +```http +GET /api/management/v1/search-engine +``` + +Returns the current search engine setting (`OPENSEARCH` or `POSTGRESQL`). + +```http +PUT /api/management/v1/search-engine +Content-Type: application/json + +{ "searchEngine": "OPENSEARCH" } +``` + +Switches the active search engine at runtime without restart. + +### Reindex operations + +```http +POST /api/management/v1/document-opensearch/reindex +Content-Type: application/json + +{ + "documentDefinitionNames": ["loan-application", "permit-request"], + "pruneBeforeReindex": false +} +``` + +Starts a full reindex. Parameters: +- `documentDefinitionNames` — Optional list of case definitions to reindex. If empty, all definitions are reindexed. +- `pruneBeforeReindex` — If `true`, deletes existing index entries before reindexing. + +{% hint style="danger" %} +While a reindex runs, search results may be incomplete until the reindex finishes. +{% endhint %} + +```http +GET /api/management/v1/document-opensearch/reindex/status +``` + +Returns the current reindex progress including documents processed and estimated completion. + +```http +GET /api/management/v1/document-opensearch/reindex/runs +``` + +Returns the history of reindex runs with their status and duration. + +## Customization + +### Custom document fields + +The indexed document includes standard fields like `definitionName`, `createdOn`, `assigneeFullName`, and a `contentText` field containing searchable text extracted from the entire document JSON. + +To add custom indexed fields or modify the mapping, extend `JsonSchemaDocumentOsConverter` and register your implementation as a Spring bean. + +### Custom sync behavior + +Document changes are synced via `DocumentOpenSearchEventListener` which listens to document domain events. To customize sync behavior, you can register additional event listeners or extend the existing one. + +## Access control + +OpenSearch queries respect the same permissions as PostgreSQL queries. Documents are filtered based on the user's permissions for each case definition. No additional access control configuration is required. From ed25f9cc0ebdce35483759896fc6f78cb9eda72d Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Fri, 17 Jul 2026 15:07:24 +0200 Subject: [PATCH 39/46] fixed potential risk with sql injection --- .../database/MysqlQueryDialectHelper.java | 10 +- .../database/PostgresQueryDialectHelper.java | 14 ++- .../database/MysqlQueryDialectHelperTest.java | 66 +++++++++++++ .../PostgresQueryDialectHelperTest.java | 93 +++++++++++++++++++ 4 files changed, 179 insertions(+), 4 deletions(-) create mode 100644 backend/contract/src/test/java/com/ritense/valtimo/contract/database/MysqlQueryDialectHelperTest.java create mode 100644 backend/contract/src/test/java/com/ritense/valtimo/contract/database/PostgresQueryDialectHelperTest.java diff --git a/backend/contract/src/main/java/com/ritense/valtimo/contract/database/MysqlQueryDialectHelper.java b/backend/contract/src/main/java/com/ritense/valtimo/contract/database/MysqlQueryDialectHelper.java index c985447c04..7082637e65 100644 --- a/backend/contract/src/main/java/com/ritense/valtimo/contract/database/MysqlQueryDialectHelper.java +++ b/backend/contract/src/main/java/com/ritense/valtimo/contract/database/MysqlQueryDialectHelper.java @@ -51,7 +51,7 @@ public Expression getJsonValueExpression(CriteriaBuilder cb, Path column, @Override public Predicate getJsonValueExistsExpression(CriteriaBuilder cb, Path column, String value) { Expression searchColumn = column; - Expression searchValue = cb.literal("%" + value.trim() + "%"); + Expression searchValue = cb.literal("%" + escapeLikePattern(value.trim()) + "%"); if (column.getJavaType() == String.class || column.getJavaType() == Object.class) { searchColumn = cb.function(LOWER_CASE_FUNCTION, String.class, searchColumn); searchValue = cb.function(LOWER_CASE_FUNCTION, String.class, searchValue); @@ -72,7 +72,7 @@ public Predicate getJsonValueExistsInPathExpression(CriteriaBuilder cb, Path col String value) { Expression searchColumn = column; Expression searchPath = cb.literal(path); - Expression searchValue = cb.literal("%" + value.trim() + "%"); + Expression searchValue = cb.literal("%" + escapeLikePattern(value.trim()) + "%"); if (column.getJavaType() == String.class || column.getJavaType() == Object.class) { searchColumn = cb.function(LOWER_CASE_FUNCTION, String.class, searchColumn.as(String.class)); searchPath = cb.function(LOWER_CASE_FUNCTION, String.class, searchPath); @@ -111,4 +111,10 @@ public Expression uuidToString(CriteriaBuilder cb, Path column) { public Expression stringToUuid(CriteriaBuilder cb, Expression expression) { return cb.function("UUID_TO_BIN", UUID.class, expression); } + + private String escapeLikePattern(String value) { + return value.replace("\\", "\\\\") + .replace("%", "\\%") + .replace("_", "\\_"); + } } diff --git a/backend/contract/src/main/java/com/ritense/valtimo/contract/database/PostgresQueryDialectHelper.java b/backend/contract/src/main/java/com/ritense/valtimo/contract/database/PostgresQueryDialectHelper.java index 2eedfeb6b2..9eff9bc658 100644 --- a/backend/contract/src/main/java/com/ritense/valtimo/contract/database/PostgresQueryDialectHelper.java +++ b/backend/contract/src/main/java/com/ritense/valtimo/contract/database/PostgresQueryDialectHelper.java @@ -67,7 +67,7 @@ public Predicate getJsonValueExistsExpression(CriteriaBuilder cb, Path column, S cb.function( "jsonpath", String.class, - cb.literal("$.** ? (@ like_regex \"" + value + "\" flag \"i\")") + cb.literal("$.** ? (@ like_regex \"" + escapeJsonPathRegex(value) + "\" flag \"i\")") ) ) ); @@ -81,7 +81,7 @@ public Predicate getJsonValueExistsInPathExpression(CriteriaBuilder cb, Path col String.class, getValueForPathText(cb, column, path) ), - "%" + value.toLowerCase() + "%" + "%" + escapeLikePattern(value.toLowerCase()) + "%" ); } @@ -139,4 +139,14 @@ private List splitPath(String path) { private Expression toJsonb(CriteriaBuilder cb, Path column) { return cb.function("to_jsonb", Object.class, column); } + + private String escapeJsonPathRegex(String value) { + return value.replace("\\", "\\\\").replace("\"", "\\\""); + } + + private String escapeLikePattern(String value) { + return value.replace("\\", "\\\\") + .replace("%", "\\%") + .replace("_", "\\_"); + } } diff --git a/backend/contract/src/test/java/com/ritense/valtimo/contract/database/MysqlQueryDialectHelperTest.java b/backend/contract/src/test/java/com/ritense/valtimo/contract/database/MysqlQueryDialectHelperTest.java new file mode 100644 index 0000000000..5ded069373 --- /dev/null +++ b/backend/contract/src/test/java/com/ritense/valtimo/contract/database/MysqlQueryDialectHelperTest.java @@ -0,0 +1,66 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.valtimo.contract.database; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.lang.reflect.Method; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class MysqlQueryDialectHelperTest { + + private MysqlQueryDialectHelper helper; + private Method escapeLikePatternMethod; + + @BeforeEach + void setUp() throws Exception { + helper = new MysqlQueryDialectHelper(); + escapeLikePatternMethod = MysqlQueryDialectHelper.class.getDeclaredMethod("escapeLikePattern", String.class); + escapeLikePatternMethod.setAccessible(true); + } + + @Test + void escapeLikePatternShouldEscapePercent() throws Exception { + String result = (String) escapeLikePatternMethod.invoke(helper, "100%"); + assertEquals("100\\%", result); + } + + @Test + void escapeLikePatternShouldEscapeUnderscore() throws Exception { + String result = (String) escapeLikePatternMethod.invoke(helper, "test_value"); + assertEquals("test\\_value", result); + } + + @Test + void escapeLikePatternShouldEscapeBackslash() throws Exception { + String result = (String) escapeLikePatternMethod.invoke(helper, "path\\to\\file"); + assertEquals("path\\\\to\\\\file", result); + } + + @Test + void escapeLikePatternShouldEscapeAllSpecialChars() throws Exception { + String result = (String) escapeLikePatternMethod.invoke(helper, "100%_test\\"); + assertEquals("100\\%\\_test\\\\", result); + } + + @Test + void escapeLikePatternShouldHandleNormalInput() throws Exception { + String result = (String) escapeLikePatternMethod.invoke(helper, "normal search"); + assertEquals("normal search", result); + } +} diff --git a/backend/contract/src/test/java/com/ritense/valtimo/contract/database/PostgresQueryDialectHelperTest.java b/backend/contract/src/test/java/com/ritense/valtimo/contract/database/PostgresQueryDialectHelperTest.java new file mode 100644 index 0000000000..426be35af2 --- /dev/null +++ b/backend/contract/src/test/java/com/ritense/valtimo/contract/database/PostgresQueryDialectHelperTest.java @@ -0,0 +1,93 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.valtimo.contract.database; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.lang.reflect.Method; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class PostgresQueryDialectHelperTest { + + private PostgresQueryDialectHelper helper; + private Method escapeJsonPathRegexMethod; + private Method escapeLikePatternMethod; + + @BeforeEach + void setUp() throws Exception { + helper = new PostgresQueryDialectHelper(); + escapeJsonPathRegexMethod = PostgresQueryDialectHelper.class.getDeclaredMethod("escapeJsonPathRegex", String.class); + escapeJsonPathRegexMethod.setAccessible(true); + escapeLikePatternMethod = PostgresQueryDialectHelper.class.getDeclaredMethod("escapeLikePattern", String.class); + escapeLikePatternMethod.setAccessible(true); + } + + @Test + void escapeJsonPathRegexShouldEscapeBackslash() throws Exception { + String result = (String) escapeJsonPathRegexMethod.invoke(helper, "test\\value"); + assertEquals("test\\\\value", result); + } + + @Test + void escapeJsonPathRegexShouldEscapeDoubleQuote() throws Exception { + String result = (String) escapeJsonPathRegexMethod.invoke(helper, "test\"value"); + assertEquals("test\\\"value", result); + } + + @Test + void escapeJsonPathRegexShouldEscapeBothBackslashAndQuote() throws Exception { + String result = (String) escapeJsonPathRegexMethod.invoke(helper, "test\\\"injection"); + assertEquals("test\\\\\\\"injection", result); + } + + @Test + void escapeJsonPathRegexShouldHandleNormalInput() throws Exception { + String result = (String) escapeJsonPathRegexMethod.invoke(helper, "normal search term"); + assertEquals("normal search term", result); + } + + @Test + void escapeLikePatternShouldEscapePercent() throws Exception { + String result = (String) escapeLikePatternMethod.invoke(helper, "100%"); + assertEquals("100\\%", result); + } + + @Test + void escapeLikePatternShouldEscapeUnderscore() throws Exception { + String result = (String) escapeLikePatternMethod.invoke(helper, "test_value"); + assertEquals("test\\_value", result); + } + + @Test + void escapeLikePatternShouldEscapeBackslash() throws Exception { + String result = (String) escapeLikePatternMethod.invoke(helper, "path\\to\\file"); + assertEquals("path\\\\to\\\\file", result); + } + + @Test + void escapeLikePatternShouldEscapeAllSpecialChars() throws Exception { + String result = (String) escapeLikePatternMethod.invoke(helper, "100%_test\\"); + assertEquals("100\\%\\_test\\\\", result); + } + + @Test + void escapeLikePatternShouldHandleNormalInput() throws Exception { + String result = (String) escapeLikePatternMethod.invoke(helper, "normal search"); + assertEquals("normal search", result); + } +} From 6dcd4db2780809dac10f9a5fa58315863503a153 Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Fri, 17 Jul 2026 15:18:19 +0200 Subject: [PATCH 40/46] improved when the opensearch beans are available --- .../DocumentOpenSearchAutoConfiguration.kt | 21 ++++++------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt index c64c4bffcc..7c99d2d5fe 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt @@ -136,7 +136,7 @@ class DocumentOpenSearchAutoConfiguration { DocumentOpenSearchSyncService(repository, documentRepository, converter, transactionManager) @Bean - @ConditionalOnProperty(prefix = "valtimo.opensearch", name = ["enabled"], havingValue = "true", matchIfMissing = true) + @ConditionalOnProperty(prefix = "valtimo.opensearch", name = ["enabled"], havingValue = "true", matchIfMissing = false) fun documentOpenSearchEventListener( syncService: DocumentOpenSearchSyncService, searchEngineToggle: SearchEngineToggle, @@ -176,7 +176,7 @@ class DocumentOpenSearchAutoConfiguration { @Bean @ConditionalOnMissingBean - @ConditionalOnProperty(prefix = "valtimo.opensearch", name = ["enabled"], havingValue = "true", matchIfMissing = true) + @ConditionalOnProperty(prefix = "valtimo.opensearch", name = ["enabled"], havingValue = "true", matchIfMissing = false) fun documentOpenSearchReconcileService( entityManager: EntityManager, converter: JsonSchemaDocumentOsConverter, @@ -210,7 +210,7 @@ class DocumentOpenSearchAutoConfiguration { @Bean @ConditionalOnMissingBean - @ConditionalOnProperty(prefix = "valtimo.opensearch", name = ["enabled"], havingValue = "true", matchIfMissing = true) + @ConditionalOnProperty(prefix = "valtimo.opensearch", name = ["enabled"], havingValue = "true", matchIfMissing = false) fun pendingIndexDeletionListener( pendingIndexDeletionRepository: PendingIndexDeletionRepository, ): PendingIndexDeletionListener = @@ -347,12 +347,8 @@ class DocumentOpenSearchAutoConfiguration { @Bean @ConditionalOnMissingBean - @ConditionalOnProperty( - prefix = "valtimo.opensearch", - name = ["health-check-enabled"], - havingValue = "true", - matchIfMissing = true - ) + @ConditionalOnProperty(prefix = "valtimo.opensearch", name = ["enabled"], havingValue = "true", matchIfMissing = false) + @ConditionalOnProperty(prefix = "valtimo.opensearch", name = ["health-check-enabled"], havingValue = "true", matchIfMissing = true) fun openSearchHealthService( restHighLevelClient: org.opensearch.client.RestHighLevelClient, toggle: SearchEngineToggle, @@ -361,12 +357,7 @@ class DocumentOpenSearchAutoConfiguration { OpenSearchHealthService(restHighLevelClient, toggle, openSearchProperties) @Bean - @ConditionalOnProperty( - prefix = "valtimo.opensearch", - name = ["health-check-enabled"], - havingValue = "true", - matchIfMissing = true - ) + @ConditionalOnBean(OpenSearchHealthService::class) fun openSearchHealthScheduler( healthService: OpenSearchHealthService, openSearchProperties: OpenSearchProperties, From d7cbb1e9fe85da49571e9c0f0094c9b07a4122ca Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Fri, 17 Jul 2026 15:20:40 +0200 Subject: [PATCH 41/46] fixed incorrect link --- documentation/release-notes/13.x.x/13.38.0/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/release-notes/13.x.x/13.38.0/README.md b/documentation/release-notes/13.x.x/13.38.0/README.md index d525fcc943..7a7fb1c78e 100644 --- a/documentation/release-notes/13.x.x/13.38.0/README.md +++ b/documentation/release-notes/13.x.x/13.38.0/README.md @@ -17,7 +17,7 @@ Case list search and global document queries can now use OpenSearch as the search engine instead of PostgreSQL. OpenSearch provides faster full-text search and scales better for large document volumes. The feature is opt-in and PostgreSQL remains the source of truth. OpenSearch acts as a derived read model that syncs automatically. - See [OpenSearch search](../../../features/opensearch/README.md) for setup instructions. + See [OpenSearch search](../../../running-valtimo/application-configuration/opensearch.md) for setup instructions. ## Enhancements From a125a1804538403e6d37a6008f08b8ee12b8f3b0 Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Fri, 17 Jul 2026 16:55:50 +0200 Subject: [PATCH 42/46] fixed corner case allowing users to search when no search fields were configured --- .../JsonSchemaDocumentOpenSearchService.kt | 30 ++-- ...JsonSchemaDocumentOpenSearchServiceTest.kt | 5 +- .../impl/JsonSchemaDocumentSearchService.java | 146 +++++++++--------- 3 files changed, 92 insertions(+), 89 deletions(-) diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt index 786fc51907..d7adab5499 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt @@ -106,8 +106,7 @@ class JsonSchemaDocumentOpenSearchService( if (searchFields.isNotEmpty()) { parts.add(buildGlobalSearchQuery(globalFilter.trim(), searchFields)) } else { - val term = "*${globalFilter.trim()}*" - parts.add(QueryBuilders.wildcardQuery("contentText.keyword", term).caseInsensitive(true)) + parts.add(MATCH_NONE) } } else { val scopedQuery = buildGlobalSearchQueryForAllDefinitions(globalFilter.trim()) @@ -252,8 +251,7 @@ class JsonSchemaDocumentOpenSearchService( if (searchFields.isNotEmpty()) { parts.add(buildGlobalSearchQuery(globalFilter.trim(), searchFields)) } else { - val term = "*${globalFilter.trim()}*" - parts.add(QueryBuilders.wildcardQuery("contentText.keyword", term).caseInsensitive(true)) + parts.add(MATCH_NONE) } } else { val scopedQuery = buildGlobalSearchQueryForAllDefinitions(globalFilter.trim()) @@ -534,27 +532,26 @@ class JsonSchemaDocumentOpenSearchService( } private fun buildGlobalSearchQueryForAllDefinitions(query: String): QueryBuilder { - val matchNone = QueryBuilders.boolQuery().mustNot(QueryBuilders.matchAllQuery()) - val accessibleDefinitions = caseDefinitionService.getCaseDefinitions(active = true) if (accessibleDefinitions.isEmpty()) { - return matchNone + return MATCH_NONE } - val contentTextQuery = QueryBuilders.wildcardQuery("contentText.keyword", "*${query}*").caseInsensitive(true) - - val definitionQueries = accessibleDefinitions.map { definition -> + val definitionQueries = accessibleDefinitions.mapNotNull { definition -> val searchFields = runWithoutAuthorization { searchFieldService.getSearchFields(definition.id.key) } - val searchQuery = if (searchFields.isEmpty()) { - contentTextQuery + if (searchFields.isEmpty()) { + null } else { - buildGlobalSearchQuery(query, searchFields) + QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery(DEFINITION_NAME_FIELD, definition.id.key)) + .must(buildGlobalSearchQuery(query, searchFields)) } - QueryBuilders.boolQuery() - .must(QueryBuilders.termQuery(DEFINITION_NAME_FIELD, definition.id.key)) - .must(searchQuery) + } + + if (definitionQueries.isEmpty()) { + return MATCH_NONE } return QueryBuilders.boolQuery().apply { @@ -640,6 +637,7 @@ class JsonSchemaDocumentOpenSearchService( private const val CASE_PREFIX = "case:" private const val DEFINITION_NAME_FIELD = "definitionId.name" private const val BLUEPRINT_TYPE_FIELD = "definitionId.blueprintId.blueprintType" + private val MATCH_NONE: QueryBuilder = QueryBuilders.boolQuery().mustNot(QueryBuilders.matchAllQuery()) private fun AdvancedSearchRequest.OtherFilter.rangeFromValue(): Any? = AdvancedSearchRequest.OtherFilter::class.java.getMethod("getRangeFrom").invoke(this) diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceTest.kt index 5901ba0b7c..0be2aa3a6d 100644 --- a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceTest.kt +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceTest.kt @@ -124,7 +124,7 @@ class JsonSchemaDocumentOpenSearchServiceTest { } @Test - fun `search with globalSearchFilter includes contentText in query`() { + fun `search with globalSearchFilter and no search fields returns match none`() { val queryCaptor = argumentCaptor() val emptySearchHits: SearchHits = mock() whenever(emptySearchHits.searchHits).thenReturn(emptyList()) @@ -135,7 +135,8 @@ class JsonSchemaDocumentOpenSearchServiceTest { service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) val capturedQuery = queryCaptor.firstValue - assertThat(capturedQuery.source).contains("contentText") + assertThat(capturedQuery.source).contains("must_not") + assertThat(capturedQuery.source).contains("match_all") } @Test diff --git a/backend/case/src/main/java/com/ritense/document/service/impl/JsonSchemaDocumentSearchService.java b/backend/case/src/main/java/com/ritense/document/service/impl/JsonSchemaDocumentSearchService.java index c89b24cdfc..e47a55450f 100644 --- a/backend/case/src/main/java/com/ritense/document/service/impl/JsonSchemaDocumentSearchService.java +++ b/backend/case/src/main/java/com/ritense/document/service/impl/JsonSchemaDocumentSearchService.java @@ -402,86 +402,90 @@ private void buildQueryWhere( ? searchFieldService.getSearchFields(documentDefinitionName) : List.of(); - var fieldMap = searchFields.stream() - .collect(Collectors.toMap(f -> removePrefixes(f.getPath()), f -> f, (a, b) -> a)); - - var docFields = searchFields.stream() - .filter(f -> f.getPath() != null && f.getPath().startsWith(DOC_PREFIX)) - .toList(); - - var caseTextFields = searchFields.stream() - .filter(f -> f.getPath() != null && f.getPath().startsWith(CASE_PREFIX)) - .filter(f -> f.getDataType() == SearchFieldDataType.TEXT) - .toList(); - - var parsedTerms = parseGlobalSearch(searchRequest.getGlobalSearchFilter()); - - List qualifiedPredicates = new ArrayList<>(); - List unqualifiedPredicates = new ArrayList<>(); - - for (ParsedTerm term : parsedTerms) { - if (term.field() != null) { - var fieldPath = removePrefixes(term.field()); - var field = fieldMap.get(fieldPath); - if (field == null) { - throw new IllegalArgumentException("Unknown search field: " + term.field()); - } + if (searchFields.isEmpty()) { + predicates.add(cb.disjunction()); + } else { + var fieldMap = searchFields.stream() + .collect(Collectors.toMap(f -> removePrefixes(f.getPath()), f -> f, (a, b) -> a)); + + var docFields = searchFields.stream() + .filter(f -> f.getPath() != null && f.getPath().startsWith(DOC_PREFIX)) + .toList(); + + var caseTextFields = searchFields.stream() + .filter(f -> f.getPath() != null && f.getPath().startsWith(CASE_PREFIX)) + .filter(f -> f.getDataType() == SearchFieldDataType.TEXT) + .toList(); + + var parsedTerms = parseGlobalSearch(searchRequest.getGlobalSearchFilter()); + + List qualifiedPredicates = new ArrayList<>(); + List unqualifiedPredicates = new ArrayList<>(); + + for (ParsedTerm term : parsedTerms) { + if (term.field() != null) { + var fieldPath = removePrefixes(term.field()); + var field = fieldMap.get(fieldPath); + if (field == null) { + throw new IllegalArgumentException("Unknown search field: " + term.field()); + } - boolean isDocField = field.getPath().startsWith(DOC_PREFIX); + boolean isDocField = field.getPath().startsWith(DOC_PREFIX); - if (isDocField) { - var jsonPath = "$." + fieldPath; - Expression expr = queryDialectHelper.getJsonValueExpression( - cb, documentRoot.get(CONTENT).get(CONTENT), jsonPath, String.class - ); - var likePattern = buildLikePattern(term.value(), term.quoted(), field.getMatchType()); - qualifiedPredicates.add(cb.like(cb.lower(expr), likePattern.toLowerCase())); - } else { - if (field.getDataType() == SearchFieldDataType.DATE || - field.getDataType() == SearchFieldDataType.DATETIME) { - var date = LocalDate.parse(term.value()); - var startOfDay = date.atStartOfDay(); - var endOfDay = date.plusDays(1).atStartOfDay(); - qualifiedPredicates.add(cb.and( - cb.greaterThanOrEqualTo(documentRoot.get(fieldPath), startOfDay), - cb.lessThan(documentRoot.get(fieldPath), endOfDay) - )); - } else { + if (isDocField) { + var jsonPath = "$." + fieldPath; + Expression expr = queryDialectHelper.getJsonValueExpression( + cb, documentRoot.get(CONTENT).get(CONTENT), jsonPath, String.class + ); var likePattern = buildLikePattern(term.value(), term.quoted(), field.getMatchType()); - qualifiedPredicates.add(cb.like( - cb.lower(documentRoot.get(fieldPath).as(String.class)), - likePattern.toLowerCase() - )); + qualifiedPredicates.add(cb.like(cb.lower(expr), likePattern.toLowerCase())); + } else { + if (field.getDataType() == SearchFieldDataType.DATE || + field.getDataType() == SearchFieldDataType.DATETIME) { + var date = LocalDate.parse(term.value()); + var startOfDay = date.atStartOfDay(); + var endOfDay = date.plusDays(1).atStartOfDay(); + qualifiedPredicates.add(cb.and( + cb.greaterThanOrEqualTo(documentRoot.get(fieldPath), startOfDay), + cb.lessThan(documentRoot.get(fieldPath), endOfDay) + )); + } else { + var likePattern = buildLikePattern(term.value(), term.quoted(), field.getMatchType()); + qualifiedPredicates.add(cb.like( + cb.lower(documentRoot.get(fieldPath).as(String.class)), + likePattern.toLowerCase() + )); + } + } + } else { + var likePattern = "%" + term.value().toLowerCase() + "%"; + List termPredicates = new ArrayList<>(); + + for (var f : docFields) { + var jsonPath = "$." + f.getPath().substring(DOC_PREFIX.length()); + Expression expr = queryDialectHelper.getJsonValueExpression( + cb, documentRoot.get(CONTENT).get(CONTENT), jsonPath, String.class + ); + termPredicates.add(cb.like(cb.lower(expr), likePattern)); } - } - } else { - var likePattern = "%" + term.value().toLowerCase() + "%"; - List termPredicates = new ArrayList<>(); - - for (var f : docFields) { - var jsonPath = "$." + f.getPath().substring(DOC_PREFIX.length()); - Expression expr = queryDialectHelper.getJsonValueExpression( - cb, documentRoot.get(CONTENT).get(CONTENT), jsonPath, String.class - ); - termPredicates.add(cb.like(cb.lower(expr), likePattern)); - } - for (var f : caseTextFields) { - var columnName = f.getPath().substring(CASE_PREFIX.length()); - termPredicates.add(cb.like( - cb.lower(documentRoot.get(columnName).as(String.class)), - likePattern - )); - } + for (var f : caseTextFields) { + var columnName = f.getPath().substring(CASE_PREFIX.length()); + termPredicates.add(cb.like( + cb.lower(documentRoot.get(columnName).as(String.class)), + likePattern + )); + } - if (!termPredicates.isEmpty()) { - unqualifiedPredicates.add(cb.or(termPredicates.toArray(Predicate[]::new))); + if (!termPredicates.isEmpty()) { + unqualifiedPredicates.add(cb.or(termPredicates.toArray(Predicate[]::new))); + } } } - } - qualifiedPredicates.forEach(predicates::add); - unqualifiedPredicates.forEach(predicates::add); + qualifiedPredicates.forEach(predicates::add); + unqualifiedPredicates.forEach(predicates::add); + } } query.where(predicates.toArray(Predicate[]::new)); From 7ef5690944ae504bb3bb1023769c1c8613cf69de Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Mon, 20 Jul 2026 14:30:10 +0200 Subject: [PATCH 43/46] changed opensearch reindex runs to be split into two sections --- .../admin-settings-opensearch.component.html | 78 ++++--------------- .../admin-settings-opensearch.component.scss | 54 +++++++------ 2 files changed, 43 insertions(+), 89 deletions(-) diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.html b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.html index 9ae4f01832..62702141e0 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.html +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.html @@ -70,37 +70,27 @@
-
-
{{ 'adminSettings.opensearch.reindex.parameters' | translate }}
+ {{ 'adminSettings.opensearch.reindex.started' | translate }} + {{ data.startedOn | date:'medium' }} -
-
- - {{ 'adminSettings.opensearch.reindex.documentDefinitionName' | translate }}: - - {{ data.scope?.documentDefinitionName || '-' }} -
+ {{ 'adminSettings.opensearch.reindex.finished' | translate }} + {{ data.finishedOn ? (data.finishedOn | date:'medium') : '-' }} -
- - {{ 'adminSettings.opensearch.reindex.pruneOrphans' | translate }}: - - {{ data.scope?.pruneOrphans ? ('interface.yes' | translate) : ('interface.no' | translate) }} - -
+
{{ 'adminSettings.opensearch.reindex.parameters' | translate }}
-
- - {{ 'adminSettings.opensearch.reindex.modifiedAfter' | translate }}: - - {{ data.scope?.modifiedAfter ? (data.scope.modifiedAfter | date:'medium') : '-' }} -
-
-
+ {{ 'adminSettings.opensearch.reindex.documentDefinitionName' | translate }} + {{ data.scope?.documentDefinitionName || '-' }} + + {{ 'adminSettings.opensearch.reindex.pruneOrphans' | translate }} + + {{ data.scope?.pruneOrphans ? ('interface.yes' | translate) : ('interface.no' | translate) }} + + -
-
{{ 'adminSettings.opensearch.reindex.results' | translate }}
+ {{ 'adminSettings.opensearch.reindex.modifiedAfter' | translate }} + {{ data.scope?.modifiedAfter ? (data.scope.modifiedAfter | date:'medium') : '-' }} +
{{ 'adminSettings.opensearch.reindex.reindexing' | translate }} @@ -111,7 +101,7 @@
{{ 'adminSettings.opensearch.reindex.results' | translate }}
>
{{ 'adminSettings.opensearch.reindex.processed' | translate:{current: data.processedCount, total: data.totalCount} }} - {{ 'adminSettings.opensearch.reindex.statuses.' + data.status | translate }} + {{ 'adminSettings.opensearch.reindex.skipped' | translate }}: {{ data.skippedCount || 0 }}
@@ -132,40 +122,6 @@
{{ 'adminSettings.opensearch.reindex.results' | translate }}
} -
-
- - {{ 'adminSettings.opensearch.reindex.skipped' | translate }}: - - {{ data.skippedCount || 0 }} -
- -
- - {{ 'adminSettings.opensearch.reindex.started' | translate }}: - - {{ data.startedOn | date:'medium' }} -
- - @if (data.finishedOn) { -
- - {{ 'adminSettings.opensearch.reindex.finished' | translate }}: - - {{ data.finishedOn | date:'medium' }} -
- } - - @if (data.status === 'RUNNING' && data.elapsedSeconds > 0) { -
- - {{ 'adminSettings.opensearch.reindex.elapsed' | translate }}: - - {{ data.elapsedSeconds }}s -
- } -
- @if (data.status === 'FAILED') { {{ 'adminSettings.opensearch.reindex.viewErrorLogs' | translate }} diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss index b2850ac1f9..f1e90b1e8e 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss @@ -21,45 +21,43 @@ gap: 24px; &__expanded { - padding: 16px 24px; - display: flex; - flex-direction: column; - gap: 24px; - } - - &__detail-section { - display: flex; - flex-direction: column; - gap: 16px; - - h6 { - margin: 0; - font-size: 14px; - font-weight: 600; - color: var(--cds-text-secondary); - } + padding: 16px 16px; + display: grid; + grid-template-columns: auto 1fr 2fr; + gap: 8px 24px; + align-items: start; } - &__detail-grid { - display: grid; - grid-template-columns: repeat(auto-fill, minmax(250px, 1fr)); - gap: 16px; + &__label { + grid-column: 1; + font-size: 14px; + font-weight: 600; + color: var(--cds-text-primary); } - &__detail-item { - display: flex; - align-items: center; - gap: 8px; + &__value { + grid-column: 2; font-size: 14px; color: var(--cds-text-primary); } - &__label { + &__parameters-header { + grid-column: 1 / 3; + margin: 12px 0 4px 0; + font-size: 14px; font-weight: 600; - color: var(--cds-text-primary); + color: var(--cds-text-secondary); + } + + &__results-column { + grid-column: 3; + grid-row: 1 / 8; + display: flex; + flex-direction: column; + gap: 16px; } - &__detail-item--with-tooltip { + &__value-with-tooltip { display: inline-flex; align-items: center; gap: 4px; From aa36520458fabd87aa161f57641af06fa094e818 Mon Sep 17 00:00:00 2001 From: Sofia Ivars Date: Mon, 20 Jul 2026 15:34:24 +0200 Subject: [PATCH 44/46] Adjust styles --- .../admin-settings-opensearch.component.html | 2 - .../admin-settings-opensearch.component.scss | 53 ++++++++++++++----- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.html b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.html index 62702141e0..d02121f8ac 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.html +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.html @@ -76,8 +76,6 @@ {{ 'adminSettings.opensearch.reindex.finished' | translate }} {{ data.finishedOn ? (data.finishedOn | date:'medium') : '-' }} -
{{ 'adminSettings.opensearch.reindex.parameters' | translate }}
- {{ 'adminSettings.opensearch.reindex.documentDefinitionName' | translate }} {{ data.scope?.documentDefinitionName || '-' }} diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss index f1e90b1e8e..5e734c6c72 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss @@ -20,16 +20,48 @@ flex-direction: column; gap: 24px; + // Fixed table layout so the expanded detail row can align its columns with the + // parent table columns: [expand] [status] [started] [finished] [progress]. + // The expand column is pinned to 3.5rem and each data column is set to 25%. + // Carbon renders the expanded cell with colspan = dataColumns + 2, which adds a + // phantom trailing column; giving the four columns 25% each (100% total, which + // together with the fixed expand column over-claims the width) forces the + // browser to scale them to fill the row exactly and collapse that phantom + // column to zero — so the four data columns fill the full width in equal + // quarters that the expanded grid (repeat(4, 1fr)) mirrors precisely. + ::ng-deep .cds--data-table { + table-layout: fixed; + + th.cds--table-expand { + width: 3.5rem; + } + + thead th:not(.cds--table-expand) { + width: 25%; + } + } + + // Align the expanded detail cell so its content starts at the status column + // (3.5rem = expand-column width) and spans the full remaining table width. + ::ng-deep tr[data-child-row] > td { + padding-inline: 3.5rem 0; + } + &__expanded { - padding: 16px 16px; + padding: 8px 0 16px; display: grid; - grid-template-columns: auto 1fr 2fr; - gap: 8px 24px; + // Four equal columns mirroring the (phantom-collapsed) table columns: + // 1 = status, 2 = started, 3 = finished, 4 = progress. Cell content is inset + // by 1rem (matching Carbon's cell padding) so it lines up with the headers. + grid-template-columns: repeat(4, 1fr); + column-gap: 0; + row-gap: 8px; align-items: start; } &__label { grid-column: 1; + padding-inline: 1rem; font-size: 14px; font-weight: 600; color: var(--cds-text-primary); @@ -37,21 +69,16 @@ &__value { grid-column: 2; + padding-inline: 1rem; font-size: 14px; color: var(--cds-text-primary); } - &__parameters-header { - grid-column: 1 / 3; - margin: 12px 0 4px 0; - font-size: 14px; - font-weight: 600; - color: var(--cds-text-secondary); - } - + // Progress bars align with "finished" and span the finished + progress columns. &__results-column { - grid-column: 3; + grid-column: 3 / span 2; grid-row: 1 / 8; + padding-inline: 1rem; display: flex; flex-direction: column; gap: 16px; @@ -77,7 +104,6 @@ display: flex; justify-content: space-between; align-items: center; - max-width: 500px; margin-top: 4px; font-size: 12px; color: var(--cds-text-secondary); @@ -85,7 +111,6 @@ &__progress { padding: 8px 0; - max-width: 500px; ::ng-deep .cds--progress-bar__track { background-color: var(--cds-border-subtle); From c04eb42866e250ef632649cdc8a590f1baee8a28 Mon Sep 17 00:00:00 2001 From: Sofia Ivars Date: Mon, 20 Jul 2026 15:55:07 +0200 Subject: [PATCH 45/46] Remove extra margin bottom --- .../admin-settings-opensearch.component.scss | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss index 5e734c6c72..1ae611589a 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss @@ -75,9 +75,12 @@ } // Progress bars align with "finished" and span the finished + progress columns. + // grid-row end must match the number of label/value rows (5) so the column + // spans them exactly — a larger value adds empty rows whose row-gap leaves + // dead space below the last label. &__results-column { grid-column: 3 / span 2; - grid-row: 1 / 8; + grid-row: 1 / 6; padding-inline: 1rem; display: flex; flex-direction: column; From 1511131be27e714fdab41594cde8b81705bcd639 Mon Sep 17 00:00:00 2001 From: Marijn Verbeek Date: Mon, 20 Jul 2026 16:01:48 +0200 Subject: [PATCH 46/46] pr feedback, extra tests --- .../src/main/resources/config/application.yml | 2 +- .../src/main/resources/config/application.yml | 2 +- .../src/main/resources/config/application.yml | 2 +- .../DocumentOpenSearchReindexService.kt | 9 +- .../service/OpenSearchReindexRunService.kt | 8 + ...SearchPermissionConditionTranslatorTest.kt | 529 ++++++++++++++++++ ...DocumentOpenSearchReindexServiceIntTest.kt | 4 +- .../DocumentOpenSearchReindexServiceTest.kt | 4 +- .../impl/JsonSchemaDocumentSearchService.java | 4 +- .../database/MysqlQueryDialectHelper.java | 6 - .../database/PostgresQueryDialectHelper.java | 6 - .../contract/database/QueryDialectHelper.java | 6 + .../database/MysqlQueryDialectHelperTest.java | 26 +- .../PostgresQueryDialectHelperTest.java | 23 +- 14 files changed, 579 insertions(+), 52 deletions(-) create mode 100644 backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/authorization/OpenSearchPermissionConditionTranslatorTest.kt diff --git a/backend/apps/evenementenvergunning/src/main/resources/config/application.yml b/backend/apps/evenementenvergunning/src/main/resources/config/application.yml index 08f5ff70be..9f8b884bee 100644 --- a/backend/apps/evenementenvergunning/src/main/resources/config/application.yml +++ b/backend/apps/evenementenvergunning/src/main/resources/config/application.yml @@ -138,7 +138,7 @@ server: min-response-size: 1024 opensearch: - uris: ${OPENSEARCH_URIS:http://localhost:9200} + uris: ${OPENSEARCH_URIS:} username: ${OPENSEARCH_USERNAME:} password: ${OPENSEARCH_PASSWORD:} diff --git a/backend/apps/gzac/src/main/resources/config/application.yml b/backend/apps/gzac/src/main/resources/config/application.yml index 08f5ff70be..9f8b884bee 100644 --- a/backend/apps/gzac/src/main/resources/config/application.yml +++ b/backend/apps/gzac/src/main/resources/config/application.yml @@ -138,7 +138,7 @@ server: min-response-size: 1024 opensearch: - uris: ${OPENSEARCH_URIS:http://localhost:9200} + uris: ${OPENSEARCH_URIS:} username: ${OPENSEARCH_USERNAME:} password: ${OPENSEARCH_PASSWORD:} diff --git a/backend/apps/valtimo/src/main/resources/config/application.yml b/backend/apps/valtimo/src/main/resources/config/application.yml index 00b64043e4..4b9c346fcc 100644 --- a/backend/apps/valtimo/src/main/resources/config/application.yml +++ b/backend/apps/valtimo/src/main/resources/config/application.yml @@ -140,7 +140,7 @@ server: min-response-size: 1024 opensearch: - uris: ${OPENSEARCH_URIS:http://localhost:9200} + uris: ${OPENSEARCH_URIS:} username: ${OPENSEARCH_USERNAME:} password: ${OPENSEARCH_PASSWORD:} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt index 851e16dea8..9a4ae206f7 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt @@ -96,7 +96,7 @@ open class DocumentOpenSearchReindexService( executor.execute { try { - reindex(run.id, request) + reindex(run.id) } catch (e: Exception) { logger.error(e) { "Re-index run ${run.id} terminated with error" } } finally { @@ -107,15 +107,16 @@ open class DocumentOpenSearchReindexService( } /** - * Runs the chunked, resumable re-index loop for [runId] over the documents matching [scope]. + * Runs the chunked, resumable re-index loop for [runId]. * Each DB page is read in its own short read-only transaction (keeping snapshots short) and the * persistence context is cleared after every page. Returns the number of documents processed. */ - open fun reindex(runId: UUID, scope: ReindexRequest): Long { + open fun reindex(runId: UUID): Long { + val scope = runService.scopeOf(runId) var lastId: UUID? = runService.cursorOf(runId) var processed: Long = runService.processedOf(runId) var skipped = 0L - val pageSize = scope.effectivePageSize() + val pageSize = runService.pageSizeOf(runId) val txTemplate = TransactionTemplate(transactionManager).apply { isReadOnly = true } try { diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunService.kt index 677efdb18f..155faa4687 100644 --- a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunService.kt +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunService.kt @@ -178,6 +178,14 @@ open class OpenSearchReindexRunService( return PageImpl(page.content.map { toMap(it) }, pageable, page.totalElements) } + @Transactional(readOnly = true) + open fun scopeOf(runId: UUID): ReindexRequest = + deserializeScopeToRequest(requireRun(runId).scope) + ?: ReindexRequest() + + @Transactional(readOnly = true) + open fun pageSizeOf(runId: UUID): Int = requireRun(runId).pageSize + private fun requireRun(runId: UUID): OpenSearchReindexRun = repository.findById(runId).orElseThrow { IllegalArgumentException("No re-index run found for runId=$runId") } diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/authorization/OpenSearchPermissionConditionTranslatorTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/authorization/OpenSearchPermissionConditionTranslatorTest.kt new file mode 100644 index 0000000000..a08c982afa --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/authorization/OpenSearchPermissionConditionTranslatorTest.kt @@ -0,0 +1,529 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.authorization + +import com.ritense.authorization.Action +import com.ritense.authorization.AuthorizationService +import com.ritense.authorization.permission.ConditionContainer +import com.ritense.authorization.permission.Permission +import com.ritense.authorization.permission.condition.ContainerPermissionCondition +import com.ritense.authorization.permission.condition.ExpressionPermissionCondition +import com.ritense.authorization.permission.condition.FieldPermissionCondition +import com.ritense.authorization.permission.condition.PermissionConditionOperator +import com.ritense.authorization.permission.condition.PermissionConditionOperator.EQUAL_TO +import com.ritense.authorization.permission.condition.PermissionConditionOperator.GREATER_THAN +import com.ritense.authorization.permission.condition.PermissionConditionOperator.GREATER_THAN_OR_EQUAL_TO +import com.ritense.authorization.permission.condition.PermissionConditionOperator.IN +import com.ritense.authorization.permission.condition.PermissionConditionOperator.LESS_THAN +import com.ritense.authorization.permission.condition.PermissionConditionOperator.LESS_THAN_OR_EQUAL_TO +import com.ritense.authorization.permission.condition.PermissionConditionOperator.LIST_CONTAINS +import com.ritense.authorization.permission.condition.PermissionConditionOperator.NOT_EQUAL_TO +import com.ritense.authorization.role.Role +import com.ritense.authorization.specification.AuthorizationSpecification +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.domain.impl.JsonSchemaDocumentId +import com.ritense.document.repository.impl.JsonSchemaDocumentRepository +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.mock +import org.mockito.kotlin.never +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever +import org.junit.jupiter.api.assertThrows +import org.opensearch.index.query.BoolQueryBuilder +import org.opensearch.index.query.ExistsQueryBuilder +import org.opensearch.index.query.IdsQueryBuilder +import org.opensearch.index.query.MatchAllQueryBuilder +import org.opensearch.index.query.QueryBuilders +import org.opensearch.index.query.RangeQueryBuilder +import org.opensearch.index.query.TermQueryBuilder +import org.opensearch.index.query.TermsQueryBuilder +import java.util.UUID + +class OpenSearchPermissionConditionTranslatorTest { + + private lateinit var authorizationService: AuthorizationService + private lateinit var documentRepository: JsonSchemaDocumentRepository + private lateinit var translator: OpenSearchPermissionConditionTranslator + + @BeforeEach + fun setUp() { + authorizationService = mock() + documentRepository = mock() + translator = OpenSearchPermissionConditionTranslator( + openSearchMappers = emptyList(), + authorizationService = authorizationService, + documentRepository = documentRepository, + ) + } + + @Test + fun `jpaFallback returns ids query with matching document IDs`() { + val docId1 = UUID.randomUUID() + val docId2 = UUID.randomUUID() + val doc1 = mockDocument(docId1) + val doc2 = mockDocument(docId2) + + val spec: AuthorizationSpecification = mock() + whenever(authorizationService.getAuthorizationSpecification(any(), any())) + .thenReturn(spec) + whenever(documentRepository.findAll(spec)).thenReturn(listOf(doc1, doc2)) + + val condition = ContainerPermissionCondition( + resourceType = UnmappedEntity::class.java, + conditions = listOf( + FieldPermissionCondition("someField", PermissionConditionOperator.EQUAL_TO, "someValue") + ) + ) + val permission = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(Action(Action.VIEW)), + conditionContainer = ConditionContainer(listOf(condition)), + role = Role(key = "test-role"), + ) + + val result = translator.toQuery(listOf(permission), Action(Action.VIEW)) + + assertThat(result).isInstanceOf(IdsQueryBuilder::class.java) + val idsQuery = result as IdsQueryBuilder + assertThat(idsQuery.ids()).containsExactlyInAnyOrder(docId1.toString(), docId2.toString()) + } + + @Test + fun `jpaFallback returns empty ids query when no documents match`() { + val spec: AuthorizationSpecification = mock() + whenever(authorizationService.getAuthorizationSpecification(any(), any())) + .thenReturn(spec) + whenever(documentRepository.findAll(spec)).thenReturn(emptyList()) + + val condition = ContainerPermissionCondition( + resourceType = UnmappedEntity::class.java, + conditions = emptyList() + ) + val permission = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(Action(Action.VIEW)), + conditionContainer = ConditionContainer(listOf(condition)), + role = Role(key = "test-role"), + ) + + val result = translator.toQuery(listOf(permission), Action(Action.VIEW)) + + assertThat(result).isInstanceOf(IdsQueryBuilder::class.java) + val idsQuery = result as IdsQueryBuilder + assertThat(idsQuery.ids()).isEmpty() + } + + @Test + fun `translateContainer uses mapper when available`() { + val mockMapper: OpenSearchAuthorizationEntityMapper = mock() + whenever(mockMapper.supports(JsonSchemaDocument::class.java, MappedEntity::class.java)).thenReturn(true) + whenever(mockMapper.mapQuery(any())).thenReturn(QueryBuilders.matchAllQuery()) + + val translatorWithMapper = OpenSearchPermissionConditionTranslator( + openSearchMappers = listOf(mockMapper), + authorizationService = authorizationService, + documentRepository = documentRepository, + ) + + val condition = ContainerPermissionCondition( + resourceType = MappedEntity::class.java, + conditions = emptyList() + ) + val permission = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(Action(Action.VIEW)), + conditionContainer = ConditionContainer(listOf(condition)), + role = Role(key = "test-role"), + ) + + val result = translatorWithMapper.toQuery(listOf(permission), Action(Action.VIEW)) + + verify(mockMapper).mapQuery(any()) + verify(authorizationService, never()).getAuthorizationSpecification(any(), any()) + assertThat(result).isInstanceOf(MatchAllQueryBuilder::class.java) + } + + @Test + fun `translateContainer falls back to JPA when no mapper supports the type`() { + val spec: AuthorizationSpecification = mock() + whenever(authorizationService.getAuthorizationSpecification(any(), any())) + .thenReturn(spec) + whenever(documentRepository.findAll(spec)).thenReturn(emptyList()) + + val condition = ContainerPermissionCondition( + resourceType = UnmappedEntity::class.java, + conditions = emptyList() + ) + val permission = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(Action(Action.VIEW)), + conditionContainer = ConditionContainer(listOf(condition)), + role = Role(key = "test-role"), + ) + + translator.toQuery(listOf(permission), Action(Action.VIEW)) + + verify(authorizationService).getAuthorizationSpecification(any(), any()) + verify(documentRepository).findAll(spec) + } + + // --- toQuery edge cases --- + + @Test + fun `toQuery returns deny-all when no permissions match action`() { + val permission = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(Action(Action.VIEW)), + conditionContainer = ConditionContainer(emptyList()), + role = Role(key = "test-role"), + ) + + val result = translator.toQuery(listOf(permission), Action(Action.DELETE)) + + assertThat(result).isInstanceOf(IdsQueryBuilder::class.java) + assertThat((result as IdsQueryBuilder).ids()).isEmpty() + } + + @Test + fun `toQuery returns deny-all when permissions list is empty`() { + val result = translator.toQuery(emptyList(), Action(Action.VIEW)) + + assertThat(result).isInstanceOf(IdsQueryBuilder::class.java) + assertThat((result as IdsQueryBuilder).ids()).isEmpty() + } + + @Test + fun `toQuery ORs multiple permissions together`() { + val permission1 = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(Action(Action.VIEW)), + conditionContainer = ConditionContainer(listOf( + FieldPermissionCondition("createdBy", EQUAL_TO, "user1") + )), + role = Role(key = "role1"), + ) + val permission2 = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(Action(Action.VIEW)), + conditionContainer = ConditionContainer(listOf( + FieldPermissionCondition("createdBy", EQUAL_TO, "user2") + )), + role = Role(key = "role2"), + ) + + val result = translator.toQuery(listOf(permission1, permission2), Action(Action.VIEW)) + + assertThat(result).isInstanceOf(BoolQueryBuilder::class.java) + val boolQuery = result as BoolQueryBuilder + assertThat(boolQuery.should()).hasSize(2) + assertThat(boolQuery.minimumShouldMatch()).isEqualTo("1") + } + + @Test + fun `toQuery ANDs multiple conditions within a permission`() { + val permission = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(Action(Action.VIEW)), + conditionContainer = ConditionContainer(listOf( + FieldPermissionCondition("createdBy", EQUAL_TO, "user1"), + FieldPermissionCondition("assigneeId", EQUAL_TO, "user2") + )), + role = Role(key = "test-role"), + ) + + val result = translator.toQuery(listOf(permission), Action(Action.VIEW)) + + assertThat(result).isInstanceOf(BoolQueryBuilder::class.java) + val boolQuery = result as BoolQueryBuilder + assertThat(boolQuery.must()).hasSize(2) + } + + @Test + fun `toQuery returns single query unwrapped when only one permission with one condition`() { + val permission = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(Action(Action.VIEW)), + conditionContainer = ConditionContainer(listOf( + FieldPermissionCondition("createdBy", EQUAL_TO, "user1") + )), + role = Role(key = "test-role"), + ) + + val result = translator.toQuery(listOf(permission), Action(Action.VIEW)) + + assertThat(result).isInstanceOf(TermQueryBuilder::class.java) + } + + // --- applyOperator tests --- + + @Test + fun `applyOperator EQUAL_TO with value returns term query`() { + val result = OpenSearchPermissionConditionTranslator.applyOperator("field", EQUAL_TO, "value") + + assertThat(result).isInstanceOf(TermQueryBuilder::class.java) + val termQuery = result as TermQueryBuilder + assertThat(termQuery.fieldName()).isEqualTo("field") + assertThat(termQuery.value()).isEqualTo("value") + } + + @Test + fun `applyOperator EQUAL_TO with null returns must-not-exists query`() { + val result = OpenSearchPermissionConditionTranslator.applyOperator("field", EQUAL_TO, null) + + assertThat(result).isInstanceOf(BoolQueryBuilder::class.java) + val boolQuery = result as BoolQueryBuilder + assertThat(boolQuery.mustNot()).hasSize(1) + assertThat(boolQuery.mustNot()[0]).isInstanceOf(ExistsQueryBuilder::class.java) + } + + @Test + fun `applyOperator NOT_EQUAL_TO with value returns must-not-term query`() { + val result = OpenSearchPermissionConditionTranslator.applyOperator("field", NOT_EQUAL_TO, "value") + + assertThat(result).isInstanceOf(BoolQueryBuilder::class.java) + val boolQuery = result as BoolQueryBuilder + assertThat(boolQuery.mustNot()).hasSize(1) + assertThat(boolQuery.mustNot()[0]).isInstanceOf(TermQueryBuilder::class.java) + } + + @Test + fun `applyOperator NOT_EQUAL_TO with null returns exists query`() { + val result = OpenSearchPermissionConditionTranslator.applyOperator("field", NOT_EQUAL_TO, null) + + assertThat(result).isInstanceOf(ExistsQueryBuilder::class.java) + } + + @Test + fun `applyOperator GREATER_THAN returns range query with gt`() { + val result = OpenSearchPermissionConditionTranslator.applyOperator("field", GREATER_THAN, 10) + + assertThat(result).isInstanceOf(RangeQueryBuilder::class.java) + val rangeQuery = result as RangeQueryBuilder + assertThat(rangeQuery.from()).isEqualTo(10) + assertThat(rangeQuery.includeLower()).isFalse() + } + + @Test + fun `applyOperator GREATER_THAN_OR_EQUAL_TO returns range query with gte`() { + val result = OpenSearchPermissionConditionTranslator.applyOperator("field", GREATER_THAN_OR_EQUAL_TO, 10) + + assertThat(result).isInstanceOf(RangeQueryBuilder::class.java) + val rangeQuery = result as RangeQueryBuilder + assertThat(rangeQuery.from()).isEqualTo(10) + assertThat(rangeQuery.includeLower()).isTrue() + } + + @Test + fun `applyOperator LESS_THAN returns range query with lt`() { + val result = OpenSearchPermissionConditionTranslator.applyOperator("field", LESS_THAN, 10) + + assertThat(result).isInstanceOf(RangeQueryBuilder::class.java) + val rangeQuery = result as RangeQueryBuilder + assertThat(rangeQuery.to()).isEqualTo(10) + assertThat(rangeQuery.includeUpper()).isFalse() + } + + @Test + fun `applyOperator LESS_THAN_OR_EQUAL_TO returns range query with lte`() { + val result = OpenSearchPermissionConditionTranslator.applyOperator("field", LESS_THAN_OR_EQUAL_TO, 10) + + assertThat(result).isInstanceOf(RangeQueryBuilder::class.java) + val rangeQuery = result as RangeQueryBuilder + assertThat(rangeQuery.to()).isEqualTo(10) + assertThat(rangeQuery.includeUpper()).isTrue() + } + + @Test + fun `applyOperator LIST_CONTAINS returns term query`() { + val result = OpenSearchPermissionConditionTranslator.applyOperator("field", LIST_CONTAINS, "value") + + assertThat(result).isInstanceOf(TermQueryBuilder::class.java) + } + + @Test + fun `applyOperator IN returns terms query`() { + val result = OpenSearchPermissionConditionTranslator.applyOperator("field", IN, listOf("a", "b", "c")) + + assertThat(result).isInstanceOf(TermsQueryBuilder::class.java) + } + + @Test + fun `applyOperator IN throws when value is not a collection`() { + assertThrows { + OpenSearchPermissionConditionTranslator.applyOperator("field", IN, "not-a-collection") + } + } + + // --- Field and expression translation --- + + @Test + fun `translateField maps JPA field to OpenSearch field`() { + val permission = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(Action(Action.VIEW)), + conditionContainer = ConditionContainer(listOf( + FieldPermissionCondition("content.content", EQUAL_TO, 123) + )), + role = Role(key = "test-role"), + ) + + val result = translator.toQuery(listOf(permission), Action(Action.VIEW)) + + assertThat(result).isInstanceOf(TermQueryBuilder::class.java) + val termQuery = result as TermQueryBuilder + assertThat(termQuery.fieldName()).isEqualTo("content") + } + + @Test + fun `translateField adds keyword suffix for string content fields`() { + val permission = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(Action(Action.VIEW)), + conditionContainer = ConditionContainer(listOf( + ExpressionPermissionCondition("content", "$.name", EQUAL_TO, "John", String::class.java) + )), + role = Role(key = "test-role"), + ) + + val result = translator.toQuery(listOf(permission), Action(Action.VIEW)) + + assertThat(result).isInstanceOf(TermQueryBuilder::class.java) + val termQuery = result as TermQueryBuilder + assertThat(termQuery.fieldName()).isEqualTo("content.name.keyword") + } + + @Test + fun `translateExpression does not add keyword suffix for non-string values`() { + val permission = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(Action(Action.VIEW)), + conditionContainer = ConditionContainer(listOf( + ExpressionPermissionCondition("content", "$.age", EQUAL_TO, 25, Int::class.java) + )), + role = Role(key = "test-role"), + ) + + val result = translator.toQuery(listOf(permission), Action(Action.VIEW)) + + assertThat(result).isInstanceOf(TermQueryBuilder::class.java) + val termQuery = result as TermQueryBuilder + assertThat(termQuery.fieldName()).isEqualTo("content.age") + } + + @Test + fun `translateExpression does not add keyword suffix for range operators`() { + val permission = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(Action(Action.VIEW)), + conditionContainer = ConditionContainer(listOf( + ExpressionPermissionCondition("content", "$.name", GREATER_THAN, "A", String::class.java) + )), + role = Role(key = "test-role"), + ) + + val result = translator.toQuery(listOf(permission), Action(Action.VIEW)) + + assertThat(result).isInstanceOf(RangeQueryBuilder::class.java) + val rangeQuery = result as RangeQueryBuilder + assertThat(rangeQuery.fieldName()).isEqualTo("content.name") + } + + // --- Helper methods --- + + @Test + fun `jpaToOsField returns mapped field name`() { + assertThat(OpenSearchPermissionConditionTranslator.jpaToOsField("content.content")).isEqualTo("content") + assertThat(OpenSearchPermissionConditionTranslator.jpaToOsField("createdBy")).isEqualTo("createdBy") + assertThat(OpenSearchPermissionConditionTranslator.jpaToOsField("assigneeId")).isEqualTo("assigneeId") + } + + @Test + fun `jpaToOsField returns original field name when no mapping exists`() { + assertThat(OpenSearchPermissionConditionTranslator.jpaToOsField("unmappedField")).isEqualTo("unmappedField") + } + + @Test + fun `isDynamicTextField returns true for string content field with term operator`() { + assertThat(OpenSearchPermissionConditionTranslator.isDynamicTextField("content.name", EQUAL_TO, "value")).isTrue() + assertThat(OpenSearchPermissionConditionTranslator.isDynamicTextField("content.name", NOT_EQUAL_TO, "value")).isTrue() + assertThat(OpenSearchPermissionConditionTranslator.isDynamicTextField("content.name", LIST_CONTAINS, "value")).isTrue() + assertThat(OpenSearchPermissionConditionTranslator.isDynamicTextField("content.name", IN, listOf("a", "b"))).isTrue() + } + + @Test + fun `isDynamicTextField returns false for non-content fields`() { + assertThat(OpenSearchPermissionConditionTranslator.isDynamicTextField("createdBy", EQUAL_TO, "value")).isFalse() + } + + @Test + fun `isDynamicTextField returns false for null value`() { + assertThat(OpenSearchPermissionConditionTranslator.isDynamicTextField("content.name", EQUAL_TO, null)).isFalse() + } + + @Test + fun `isDynamicTextField returns false for non-term operators`() { + assertThat(OpenSearchPermissionConditionTranslator.isDynamicTextField("content.name", GREATER_THAN, "value")).isFalse() + assertThat(OpenSearchPermissionConditionTranslator.isDynamicTextField("content.name", LESS_THAN, "value")).isFalse() + } + + @Test + fun `isDynamicTextField returns false for non-string values`() { + assertThat(OpenSearchPermissionConditionTranslator.isDynamicTextField("content.age", EQUAL_TO, 25)).isFalse() + } + + @Test + fun `translateCondition throws for unknown condition type`() { + val unknownCondition = UnknownPermissionCondition() + + val permission = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(Action(Action.VIEW)), + conditionContainer = ConditionContainer(listOf(unknownCondition)), + role = Role(key = "test-role"), + ) + + assertThrows { + translator.toQuery(listOf(permission), Action(Action.VIEW)) + } + } + + class UnknownPermissionCondition : com.ritense.authorization.permission.condition.PermissionCondition( + com.ritense.authorization.permission.condition.PermissionConditionType.FIELD + ) { + override fun isValid(entity: T): Boolean = true + override fun toPredicate( + root: jakarta.persistence.criteria.Root, + query: jakarta.persistence.criteria.AbstractQuery<*>, + criteriaBuilder: jakarta.persistence.criteria.CriteriaBuilder, + resourceType: Class, + queryDialectHelper: com.ritense.valtimo.contract.database.QueryDialectHelper + ): jakarta.persistence.criteria.Predicate = criteriaBuilder.conjunction() + } + + private fun mockDocument(id: UUID): JsonSchemaDocument { + val doc: JsonSchemaDocument = mock() + val docId = JsonSchemaDocumentId.existingId(id) + whenever(doc.id()).thenReturn(docId) + return doc + } + + class UnmappedEntity + class MappedEntity +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceIntTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceIntTest.kt index c5efc5ffb8..0bde660a28 100644 --- a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceIntTest.kt +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceIntTest.kt @@ -190,7 +190,7 @@ class DocumentOpenSearchReindexServiceIntTest : BaseOpenSearchIntegrationTest() ) clearIndex() - reindexService.reindex(seededRun.id, ReindexRequest()) + reindexService.reindex(seededRun.id) refreshIndex() assertThat(openSearchRepository.count()).isEqualTo(expectedIds.size.toLong()) @@ -352,7 +352,7 @@ class DocumentOpenSearchReindexServiceIntTest : BaseOpenSearchIntegrationTest() */ private fun reindex(request: ReindexRequest): Pair { val run = reindexRunService.startOrResume(request) - return run.id to reindexService.reindex(run.id, request) + return run.id to reindexService.reindex(run.id) } private fun createDocument(street: String): JsonSchemaDocument = diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceTest.kt index 5e6fdcb31a..ce06e07198 100644 --- a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceTest.kt +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceTest.kt @@ -89,13 +89,15 @@ class DocumentOpenSearchReindexServiceTest { @Test fun `reindex marks the run STOPPED when cancellation was requested`() { val runId = UUID.randomUUID() + whenever(runService.scopeOf(runId)).thenReturn(ReindexRequest()) + whenever(runService.pageSizeOf(runId)).thenReturn(ReindexRequest.DEFAULT_PAGE_SIZE) whenever(runService.cursorOf(runId)).thenReturn(null) whenever(runService.processedOf(runId)).thenReturn(0L) // destroy() sets the cancellation flag (and shuts down the idle executor). service.destroy() - val processed = service.reindex(runId, ReindexRequest()) + val processed = service.reindex(runId) assertThat(processed).isEqualTo(0L) verify(runService).stop(runId) diff --git a/backend/case/src/main/java/com/ritense/document/service/impl/JsonSchemaDocumentSearchService.java b/backend/case/src/main/java/com/ritense/document/service/impl/JsonSchemaDocumentSearchService.java index e47a55450f..4b74466ab1 100644 --- a/backend/case/src/main/java/com/ritense/document/service/impl/JsonSchemaDocumentSearchService.java +++ b/backend/case/src/main/java/com/ritense/document/service/impl/JsonSchemaDocumentSearchService.java @@ -458,7 +458,7 @@ private void buildQueryWhere( } } } else { - var likePattern = "%" + term.value().toLowerCase() + "%"; + var likePattern = "%" + queryDialectHelper.escapeLikePattern(term.value()).toLowerCase() + "%"; List termPredicates = new ArrayList<>(); for (var f : docFields) { @@ -975,6 +975,6 @@ private String buildLikePattern(String value, boolean quoted, SearchFieldMatchTy if (quoted || matchType != SearchFieldMatchType.LIKE) { return value; } - return "%" + value + "%"; + return "%" + queryDialectHelper.escapeLikePattern(value) + "%"; } } diff --git a/backend/contract/src/main/java/com/ritense/valtimo/contract/database/MysqlQueryDialectHelper.java b/backend/contract/src/main/java/com/ritense/valtimo/contract/database/MysqlQueryDialectHelper.java index 7082637e65..2b0885a9bb 100644 --- a/backend/contract/src/main/java/com/ritense/valtimo/contract/database/MysqlQueryDialectHelper.java +++ b/backend/contract/src/main/java/com/ritense/valtimo/contract/database/MysqlQueryDialectHelper.java @@ -111,10 +111,4 @@ public Expression uuidToString(CriteriaBuilder cb, Path column) { public Expression stringToUuid(CriteriaBuilder cb, Expression expression) { return cb.function("UUID_TO_BIN", UUID.class, expression); } - - private String escapeLikePattern(String value) { - return value.replace("\\", "\\\\") - .replace("%", "\\%") - .replace("_", "\\_"); - } } diff --git a/backend/contract/src/main/java/com/ritense/valtimo/contract/database/PostgresQueryDialectHelper.java b/backend/contract/src/main/java/com/ritense/valtimo/contract/database/PostgresQueryDialectHelper.java index 9eff9bc658..3fdbc442de 100644 --- a/backend/contract/src/main/java/com/ritense/valtimo/contract/database/PostgresQueryDialectHelper.java +++ b/backend/contract/src/main/java/com/ritense/valtimo/contract/database/PostgresQueryDialectHelper.java @@ -143,10 +143,4 @@ private Expression toJsonb(CriteriaBuilder cb, Path column) { private String escapeJsonPathRegex(String value) { return value.replace("\\", "\\\\").replace("\"", "\\\""); } - - private String escapeLikePattern(String value) { - return value.replace("\\", "\\\\") - .replace("%", "\\%") - .replace("_", "\\_"); - } } diff --git a/backend/contract/src/main/java/com/ritense/valtimo/contract/database/QueryDialectHelper.java b/backend/contract/src/main/java/com/ritense/valtimo/contract/database/QueryDialectHelper.java index 10c48af06b..701f432602 100644 --- a/backend/contract/src/main/java/com/ritense/valtimo/contract/database/QueryDialectHelper.java +++ b/backend/contract/src/main/java/com/ritense/valtimo/contract/database/QueryDialectHelper.java @@ -35,4 +35,10 @@ public interface QueryDialectHelper { Expression uuidToString(CriteriaBuilder cb, Path column); Expression stringToUuid(CriteriaBuilder cb, Expression expression); + + default String escapeLikePattern(String value) { + return value.replace("\\", "\\\\") + .replace("%", "\\%") + .replace("_", "\\_"); + } } diff --git a/backend/contract/src/test/java/com/ritense/valtimo/contract/database/MysqlQueryDialectHelperTest.java b/backend/contract/src/test/java/com/ritense/valtimo/contract/database/MysqlQueryDialectHelperTest.java index 5ded069373..221ddd9d55 100644 --- a/backend/contract/src/test/java/com/ritense/valtimo/contract/database/MysqlQueryDialectHelperTest.java +++ b/backend/contract/src/test/java/com/ritense/valtimo/contract/database/MysqlQueryDialectHelperTest.java @@ -18,49 +18,45 @@ import static org.junit.jupiter.api.Assertions.assertEquals; -import java.lang.reflect.Method; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; class MysqlQueryDialectHelperTest { private MysqlQueryDialectHelper helper; - private Method escapeLikePatternMethod; @BeforeEach - void setUp() throws Exception { + void setUp() { helper = new MysqlQueryDialectHelper(); - escapeLikePatternMethod = MysqlQueryDialectHelper.class.getDeclaredMethod("escapeLikePattern", String.class); - escapeLikePatternMethod.setAccessible(true); } @Test - void escapeLikePatternShouldEscapePercent() throws Exception { - String result = (String) escapeLikePatternMethod.invoke(helper, "100%"); + void escapeLikePatternShouldEscapePercent() { + String result = helper.escapeLikePattern("100%"); assertEquals("100\\%", result); } @Test - void escapeLikePatternShouldEscapeUnderscore() throws Exception { - String result = (String) escapeLikePatternMethod.invoke(helper, "test_value"); + void escapeLikePatternShouldEscapeUnderscore() { + String result = helper.escapeLikePattern("test_value"); assertEquals("test\\_value", result); } @Test - void escapeLikePatternShouldEscapeBackslash() throws Exception { - String result = (String) escapeLikePatternMethod.invoke(helper, "path\\to\\file"); + void escapeLikePatternShouldEscapeBackslash() { + String result = helper.escapeLikePattern("path\\to\\file"); assertEquals("path\\\\to\\\\file", result); } @Test - void escapeLikePatternShouldEscapeAllSpecialChars() throws Exception { - String result = (String) escapeLikePatternMethod.invoke(helper, "100%_test\\"); + void escapeLikePatternShouldEscapeAllSpecialChars() { + String result = helper.escapeLikePattern("100%_test\\"); assertEquals("100\\%\\_test\\\\", result); } @Test - void escapeLikePatternShouldHandleNormalInput() throws Exception { - String result = (String) escapeLikePatternMethod.invoke(helper, "normal search"); + void escapeLikePatternShouldHandleNormalInput() { + String result = helper.escapeLikePattern("normal search"); assertEquals("normal search", result); } } diff --git a/backend/contract/src/test/java/com/ritense/valtimo/contract/database/PostgresQueryDialectHelperTest.java b/backend/contract/src/test/java/com/ritense/valtimo/contract/database/PostgresQueryDialectHelperTest.java index 426be35af2..baf8359812 100644 --- a/backend/contract/src/test/java/com/ritense/valtimo/contract/database/PostgresQueryDialectHelperTest.java +++ b/backend/contract/src/test/java/com/ritense/valtimo/contract/database/PostgresQueryDialectHelperTest.java @@ -26,15 +26,12 @@ class PostgresQueryDialectHelperTest { private PostgresQueryDialectHelper helper; private Method escapeJsonPathRegexMethod; - private Method escapeLikePatternMethod; @BeforeEach void setUp() throws Exception { helper = new PostgresQueryDialectHelper(); escapeJsonPathRegexMethod = PostgresQueryDialectHelper.class.getDeclaredMethod("escapeJsonPathRegex", String.class); escapeJsonPathRegexMethod.setAccessible(true); - escapeLikePatternMethod = PostgresQueryDialectHelper.class.getDeclaredMethod("escapeLikePattern", String.class); - escapeLikePatternMethod.setAccessible(true); } @Test @@ -62,32 +59,32 @@ void escapeJsonPathRegexShouldHandleNormalInput() throws Exception { } @Test - void escapeLikePatternShouldEscapePercent() throws Exception { - String result = (String) escapeLikePatternMethod.invoke(helper, "100%"); + void escapeLikePatternShouldEscapePercent() { + String result = helper.escapeLikePattern("100%"); assertEquals("100\\%", result); } @Test - void escapeLikePatternShouldEscapeUnderscore() throws Exception { - String result = (String) escapeLikePatternMethod.invoke(helper, "test_value"); + void escapeLikePatternShouldEscapeUnderscore() { + String result = helper.escapeLikePattern("test_value"); assertEquals("test\\_value", result); } @Test - void escapeLikePatternShouldEscapeBackslash() throws Exception { - String result = (String) escapeLikePatternMethod.invoke(helper, "path\\to\\file"); + void escapeLikePatternShouldEscapeBackslash() { + String result = helper.escapeLikePattern("path\\to\\file"); assertEquals("path\\\\to\\\\file", result); } @Test - void escapeLikePatternShouldEscapeAllSpecialChars() throws Exception { - String result = (String) escapeLikePatternMethod.invoke(helper, "100%_test\\"); + void escapeLikePatternShouldEscapeAllSpecialChars() { + String result = helper.escapeLikePattern("100%_test\\"); assertEquals("100\\%\\_test\\\\", result); } @Test - void escapeLikePatternShouldHandleNormalInput() throws Exception { - String result = (String) escapeLikePatternMethod.invoke(helper, "normal search"); + void escapeLikePatternShouldHandleNormalInput() { + String result = helper.escapeLikePattern("normal search"); assertEquals("normal search", result); } }