diff --git a/.gitignore b/.gitignore index 655e1560..eb0fd26a 100644 --- a/.gitignore +++ b/.gitignore @@ -32,9 +32,11 @@ dist/ .local/ *.local.* .vscode/ +.idea/ # AI context files (auto-generated) COPILOT.md .DS_Store GEMINI.md CVE/* -.todo/* \ No newline at end of file +.todo/* +.ai/ diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 00000000..902d1820 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,12 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Ignored default folder with query files +/queries/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml +# Jetbrains IDE +/.idea/ \ No newline at end of file diff --git a/routes/setup.js b/routes/setup.js index 5d76735c..ba915d4e 100644 --- a/routes/setup.js +++ b/routes/setup.js @@ -15,7 +15,6 @@ const path = require('path'); const crypto = require('crypto'); const { validateApiUrl, - validateCustomFieldValue, shouldQueueForOcrOnAiError, classifyOcrQueueReasonFromAiError, } = require('../services/serviceUtils'); @@ -25,6 +24,11 @@ const QRCode = require('qrcode'); const { isAuthenticated } = require('./auth.js'); const customService = require('../services/customService.js'); const mistralOcrService = require('../services/mistralOcrService'); +const { + updateCustomFieldsData, + updateDocumentTypeData, + updateCorrespondentData, +} = require('../services/dataProcessingUtils'); const quickstartService = require('../services/quickstartService'); const reconciliationService = require('../services/reconciliationService'); const { @@ -3244,116 +3248,9 @@ async function buildUpdateData(analysis, doc) { // Add created date regardless of settings as it's a core field updateData.created = analysis.document.document_date || doc.created; - // Only process document type if document type classification is activated - if ( - config.limitFunctions?.activateDocumentType !== 'no' && - analysis.document.document_type - ) { - try { - const documentType = await paperlessService.getOrCreateDocumentType( - analysis.document.document_type, - options - ); - if (documentType) { - updateData.document_type = documentType.id; - } - } catch (error) { - console.error(`[ERROR] Error processing document type:`, error); - } - } - - // Only process custom fields if custom fields detection is activated - if ( - config.limitFunctions?.activateCustomFields !== 'no' && - analysis.document.custom_fields - ) { - const customFields = analysis.document.custom_fields; - const processedFields = []; - const customFieldsForHistory = []; - - // Get existing custom fields - const existingFields = await paperlessService.getExistingCustomFields( - doc.id - ); - console.log(`[DEBUG] Found existing fields:`, existingFields); - - // Keep track of which fields we've processed to avoid duplicates - const processedFieldIds = new Set(); - - // First, add any new/updated fields - for (const customField of Object.values(customFields)) { - if (!customField || typeof customField !== 'object') { - console.log('[DEBUG] Skipping null/invalid custom field entry'); - continue; - } - - if ( - !customField.field_name || - customField.value === null || - customField.value === undefined || - String(customField.value).trim() === '' - ) { - console.log(`[DEBUG] Skipping empty/invalid custom field`); - continue; - } - - const fieldDetails = await paperlessService.findExistingCustomField( - customField.field_name - ); - if (fieldDetails?.id) { - const validation = validateCustomFieldValue( - customField.field_name, - customField.value, - fieldDetails.data_type - ); - if (validation.skip) { - if (validation.warn) console.warn(validation.warn); - continue; - } - processedFields.push({ - field: fieldDetails.id, - value: validation.value, - }); - customFieldsForHistory.push({ - field_name: customField.field_name, - value: validation.value, - }); - processedFieldIds.add(fieldDetails.id); - } - } - - // Then add any existing fields that weren't updated - for (const existingField of existingFields) { - if (!processedFieldIds.has(existingField.field)) { - processedFields.push(existingField); - } - } - - if (processedFields.length > 0) { - updateData.custom_fields = processedFields; - } - if (customFieldsForHistory.length > 0) { - updateData._customFieldsForHistory = customFieldsForHistory; - } - } - - // Only process correspondent if correspondent detection is activated - if ( - config.limitFunctions?.activateCorrespondents !== 'no' && - analysis.document.correspondent - ) { - try { - const correspondent = await paperlessService.getOrCreateCorrespondent( - analysis.document.correspondent, - options - ); - if (correspondent) { - updateData.correspondent = correspondent.id; - } - } catch (error) { - console.error(`[ERROR] Error processing correspondent:`, error); - } - } + await updateDocumentTypeData(analysis, updateData, config, paperlessService, options); + await updateCustomFieldsData(analysis, doc, updateData, config, paperlessService); + await updateCorrespondentData(analysis, updateData, config, paperlessService, options); // Always include language if provided as it's a core field if (analysis.document.language) { @@ -3378,16 +3275,30 @@ async function saveDocumentChanges(docId, updateData, analysis, originalData) { const origDocType = originalData.document_type ?? null; const origLanguage = originalData.language ?? null; + await documentModel.saveOriginalData( + docId, + originalTags, + originalCorrespondent, + originalTitle, + origDocType, + origLanguage + ); + + const updatedDocument = await paperlessService.updateDocument(docId, updateData); + if (!updatedDocument) { + throw new Error(`Paperless update failed for document ${docId}`); + } + + const currentCorrespondentId = updatedDocument.correspondent ?? updateData.correspondent; + const currentCorrespondent = currentCorrespondentId + ? await paperlessService.getCorrespondentNameById(currentCorrespondentId) + : null; + const currentDocumentTypeId = updatedDocument.document_type ?? updateData.document_type; + const currentDocumentType = currentDocumentTypeId + ? await paperlessService.getDocumentTypeNameById(currentDocumentTypeId) + : null; + await Promise.all([ - documentModel.saveOriginalData( - docId, - originalTags, - originalCorrespondent, - originalTitle, - origDocType, - origLanguage - ), - paperlessService.updateDocument(docId, updateData), documentModel.addProcessedDocument(docId, updateData.title), documentModel.addOpenAIMetrics( docId, @@ -3399,9 +3310,9 @@ async function saveDocumentChanges(docId, updateData, analysis, originalData) { docId, updateData.tags, updateData.title, - analysis.document.correspondent, + currentCorrespondent?.name ?? null, historyCustomFields, - historyDocTypeName, + currentDocumentType?.name ?? null, historyLanguage ), ]); @@ -8520,10 +8431,6 @@ router.post('/settings', express.json(), async (req, res) => { ...updatedConfig, }; - // The route has already validated the submitted values (and Paperless - // reachability when its settings changed). Skip saveConfig's built-in - // validateConfig so saving does not live-test Paperless/AI on every save. - await setupService.saveConfig(mergedConfig, { skipValidation: true }); try { for (const field of processedCustomFields) { await paperlessService.createCustomFieldSafely( @@ -8533,9 +8440,17 @@ router.post('/settings', express.json(), async (req, res) => { ); } } catch (error) { - console.log('[ERROR] Error creating custom fields:', error); + if (error.code === 'CUSTOM_FIELD_TYPE_MISMATCH') { + return res.status(400).json({ error: error.message }); + } + throw error; } + // The route has already validated the submitted values (and Paperless + // reachability when its settings changed). Skip saveConfig's built-in + // validateConfig so saving does not live-test Paperless/AI on every save. + await setupService.saveConfig(mergedConfig, { skipValidation: true }); + res.json({ success: true, message: 'Configuration saved successfully.', diff --git a/server.js b/server.js index 78edceb8..8947f5b5 100644 --- a/server.js +++ b/server.js @@ -23,7 +23,11 @@ const jwt = require('jsonwebtoken'); const Logger = require('./services/loggerService'); const { max } = require('date-fns'); const { - validateCustomFieldValue, + updateCustomFieldsData, + updateDocumentTypeData, + updateCorrespondentData +} = require('./services/dataProcessingUtils'); +const { shouldQueueForOcrOnAiError, classifyOcrQueueReasonFromAiError, isTimeoutError, @@ -684,88 +688,9 @@ async function buildUpdateData(analysis, doc) { // Add created date regardless of settings as it's a core field updateData.created = analysis.document.document_date || doc.created; - // Only process document type if document type classification is activated - if (config.limitFunctions?.activateDocumentType !== 'no' && analysis.document.document_type) { - try { - const documentType = await paperlessService.getOrCreateDocumentType(analysis.document.document_type, options); - if (documentType) { - updateData.document_type = documentType.id; - } - } catch (error) { - console.error(`[ERROR] Error processing document type: ${error.message}`); - console.debug(error); - } - } - - // Only process custom fields if custom fields detection is activated - if (config.limitFunctions?.activateCustomFields !== 'no' && analysis.document.custom_fields) { - const customFields = analysis.document.custom_fields; - const processedFields = []; - const customFieldsForHistory = []; - - // Get existing custom fields - const existingFields = await paperlessService.getExistingCustomFields(doc.id); - console.debug('Found existing fields:', existingFields); - - // Keep track of which fields we've processed to avoid duplicates - const processedFieldIds = new Set(); - - // First, add any new/updated fields - for (const key in customFields) { - const customField = customFields[key]; - - if (!customField.field_name || (customField.value === null || customField.value === undefined || String(customField.value).trim() === '')) { - console.debug('Skipping empty or invalid custom field'); - continue; - } - - const fieldDetails = await paperlessService.findExistingCustomField(customField.field_name); - if (fieldDetails?.id) { - const validation = validateCustomFieldValue(customField.field_name, customField.value, fieldDetails.data_type); - if (validation.skip) { - if (validation.warn) console.warn(validation.warn); - continue; - } - processedFields.push({ - field: fieldDetails.id, - value: validation.value - }); - // Capture name + validated value for history at the point where we have both - customFieldsForHistory.push({ - field_name: customField.field_name, - value: validation.value - }); - processedFieldIds.add(fieldDetails.id); - } - } - - // Then add any existing fields that weren't updated - for (const existingField of existingFields) { - if (!processedFieldIds.has(existingField.field)) { - processedFields.push(existingField); - } - } - - if (processedFields.length > 0) { - updateData.custom_fields = processedFields; - } - if (customFieldsForHistory.length > 0) { - updateData._customFieldsForHistory = customFieldsForHistory; - } - } - - // Only process correspondent if correspondent detection is activated - if (config.limitFunctions?.activateCorrespondents !== 'no' && analysis.document.correspondent) { - try { - const correspondent = await paperlessService.getOrCreateCorrespondent(analysis.document.correspondent, options); - if (correspondent) { - updateData.correspondent = correspondent.id; - } - } catch (error) { - console.error(`[ERROR] Error processing correspondent: ${error.message}`); - console.debug(error); - } - } + await updateDocumentTypeData(analysis, updateData, config, paperlessService, options); + await updateCustomFieldsData(analysis, doc, updateData, config, paperlessService); + await updateCorrespondentData(analysis, updateData, config, paperlessService, options); // Always include language if provided as it's a core field if (analysis.document.language) { @@ -794,15 +719,24 @@ async function saveDocumentChanges(docId, updateData, analysis, originalData) { throw new Error(`Paperless update failed for document ${docId}`); } + const currentCorrespondentId = updatedDocument.correspondent ?? updateData.correspondent; + const currentCorrespondent = currentCorrespondentId + ? await paperlessService.getCorrespondentNameById(currentCorrespondentId) + : null; + const currentDocumentTypeId = updatedDocument.document_type ?? updateData.document_type; + const currentDocumentType = currentDocumentTypeId + ? await paperlessService.getDocumentTypeNameById(currentDocumentTypeId) + : null; + const persistenceTasks = [ documentModel.addProcessedDocument(docId, updateData.title), documentModel.addToHistory( docId, updateData.tags, updateData.title, - analysis.document.correspondent, + currentCorrespondent?.name ?? null, historyCustomFields, - historyDocTypeName, + currentDocumentType?.name ?? null, historyLanguage ) ]; diff --git a/services/dataProcessingUtils.js b/services/dataProcessingUtils.js new file mode 100644 index 00000000..0b9f02d2 --- /dev/null +++ b/services/dataProcessingUtils.js @@ -0,0 +1,123 @@ +const { validateCustomFieldValue } = require('./serviceUtils'); + +async function updateCustomFieldsData(analysis, doc, updateData, config, paperlessService) { + // Only process custom fields if custom fields detection is activated + if ( + config.limitFunctions?.activateCustomFields === 'no' || + !analysis.document.custom_fields + ) { + console.log('[DEBUG] no processing of customFields.'); + return; + } + + const customFields = analysis.document.custom_fields; + const processedFields = []; + const customFieldsForHistory = []; + + console.log('[DEBUG] updateCustomFieldsData:', customFields); + + // Get existing custom fields + const existingFields = await paperlessService.getExistingCustomFields(doc.id); + console.debug('Found existing fields:', existingFields); + + // Keep track of which fields we've processed to avoid duplicates + const processedFieldIds = new Set(); + + // First, add any new/updated fields + for (const key in customFields) { + const customField = customFields[key]; + + if ( + !customField.field_name || + customField.value === null || + customField.value === undefined || + String(customField.value).trim() === '' + ) { + console.debug('Skipping empty or invalid custom field'); + continue; + } + + const fieldDetails = await paperlessService.findExistingCustomField( + customField.field_name + ); + if (fieldDetails?.id) { + const validation = validateCustomFieldValue( + customField.field_name, + customField.value, + fieldDetails.data_type + ); + if (validation.skip) { + if (validation.warn) console.warn(validation.warn); + continue; + } + processedFields.push({ + field: fieldDetails.id, + value: validation.value, + }); + // Capture name + validated value for history at the point where we have both + customFieldsForHistory.push({ + field_name: customField.field_name, + value: validation.value, + }); + processedFieldIds.add(fieldDetails.id); + } + } + + // Then add any existing fields that weren't updated + for (const existingField of existingFields) { + if (!processedFieldIds.has(existingField.field)) { + processedFields.push(existingField); + } + } + + if (processedFields.length > 0) { + updateData.custom_fields = processedFields; + } + if (customFieldsForHistory.length > 0) { + updateData._customFieldsForHistory = customFieldsForHistory; + } +} + +async function updateDocumentTypeData(analysis, updateData, config, paperlessService, options = {}) { + if (config.limitFunctions?.activateDocumentType === 'no' || !analysis.document.document_type) { + return; + } + + try { + const documentType = await paperlessService.getOrCreateDocumentType( + analysis.document.document_type, + options + ); + if (documentType) { + updateData.document_type = documentType.id; + } + } catch (error) { + console.error(`[ERROR] Error processing document type: ${error.message}`); + console.debug(error); + } +} + +async function updateCorrespondentData(analysis, updateData, config, paperlessService, options = {}) { + if (config.limitFunctions?.activateCorrespondents === 'no' || !analysis.document.correspondent) { + return; + } + + try { + const correspondent = await paperlessService.getOrCreateCorrespondent( + analysis.document.correspondent, + options + ); + if (correspondent) { + updateData.correspondent = correspondent.id; + } + } catch (error) { + console.error(`[ERROR] Error processing correspondent: ${error.message}`); + console.debug(error); + } +} + +module.exports = { + updateCustomFieldsData, + updateDocumentTypeData, + updateCorrespondentData +}; diff --git a/services/mistralOcrService.js b/services/mistralOcrService.js index 792c2cb2..28b3d3d4 100644 --- a/services/mistralOcrService.js +++ b/services/mistralOcrService.js @@ -13,6 +13,11 @@ const popplerService = require('./popplerService'); const documentModel = require('../models/document'); const AIServiceFactory = require('./aiServiceFactory'); const { isTimeoutError, buildTimeoutErrorMessage } = require('./serviceUtils'); +const { + updateCustomFieldsData, + updateDocumentTypeData, + updateCorrespondentData +} = require('./dataProcessingUtils'); class MistralOcrService { constructor() { @@ -956,8 +961,8 @@ class MistralOcrService { throw new Error(analysis.error); } - // Build update data (simplified – reuse paperlessService helpers) const updateData = {}; + const config = require('../config/config'); const options = { restrictToExistingTags: config.restrictToExistingTags === 'yes', @@ -967,41 +972,7 @@ class MistralOcrService { config.restrictToExistingDocumentTypes === 'yes', }; - if (config.limitFunctions?.activateTagging !== 'no') { - const { tagIds } = await PaperlessService.processTags( - analysis.document.tags, - options - ); - updateData.tags = tagIds; - } - if (config.limitFunctions?.activateTitle !== 'no') { - updateData.title = analysis.document.title || originalData.title; - } - updateData.created = - analysis.document.document_date || originalData.created; - if ( - config.limitFunctions?.activateDocumentType !== 'no' && - analysis.document.document_type - ) { - const dt = await PaperlessService.getOrCreateDocumentType( - analysis.document.document_type, - options - ); - if (dt) updateData.document_type = dt.id; - } - if ( - config.limitFunctions?.activateCorrespondents !== 'no' && - analysis.document.correspondent - ) { - const corr = await PaperlessService.getOrCreateCorrespondent( - analysis.document.correspondent, - options - ); - if (corr) updateData.correspondent = corr.id; - } - if (analysis.document.language) { - updateData.language = analysis.document.language; - } + await this.calculateNewDocumentMetadata(config, analysis, options, updateData, originalData); // Apply updates to Paperless const updatedDocument = await PaperlessService.updateDocument( @@ -1012,6 +983,15 @@ class MistralOcrService { throw new Error(`Paperless update failed for document ${documentId}`); } + const currentCorrespondentId = updatedDocument.correspondent ?? updateData.correspondent; + const currentCorrespondent = currentCorrespondentId + ? await PaperlessService.getCorrespondentNameById(currentCorrespondentId) + : null; + const currentDocumentTypeId = updatedDocument.document_type ?? updateData.document_type; + const currentDocumentType = currentDocumentTypeId + ? await PaperlessService.getDocumentTypeNameById(currentDocumentTypeId) + : null; + // Persist metrics & history if (analysis.metrics) { await documentModel.addOpenAIMetrics( @@ -1029,14 +1009,42 @@ class MistralOcrService { documentId, updateData.tags || [], updateData.title || originalData.title, - analysis.document.correspondent, + currentCorrespondent?.name ?? null, null, - analysis.document.document_type || null, + currentDocumentType?.name ?? null, analysis.document.language || null ); return analysis; } + + + async calculateNewDocumentMetadata(config, analysis, options, + updateData, originalData) { + if (config.limitFunctions?.activateTagging !== 'no') { + const { tagIds } = await PaperlessService.processTags( + analysis.document.tags, + options + ); + updateData.tags = tagIds; + } + if (config.limitFunctions?.activateTitle !== 'no') { + updateData.title = analysis.document.title || originalData.title; + } + + await updateDocumentTypeData(analysis, + updateData, config, PaperlessService, options); + await updateCorrespondentData(analysis, + updateData, config, PaperlessService, options); + await updateCustomFieldsData(analysis, + originalData, updateData, config, PaperlessService); + + updateData.created = + analysis.document.document_date || originalData.created; + + updateData.language = + analysis.document.language || updateData.language; + } } module.exports = new MistralOcrService(); diff --git a/services/ollamaService.js b/services/ollamaService.js index e81c9c07..53616489 100644 --- a/services/ollamaService.js +++ b/services/ollamaService.js @@ -591,8 +591,9 @@ class OllamaService { top_p: 0.9, repeat_penalty: 1.1, top_k: 7, - num_predict: 256, - num_ctx: numCtx + // limits make no sense. imagine a doc with 50 pages + num_predict: -1, + num_ctx: 1024 * 32 } }; @@ -619,10 +620,8 @@ class OllamaService { * @returns {Object} Parsed response */ _processOllamaResponse(responseData) { - // Check if we got a structured response or need to parse from text if (responseData.response && typeof responseData.response === 'object') { - // We got a structured response directly - console.log('Using structured output response'); + console.log('Using response JS object directly'); return { tags: Array.isArray(responseData.response.tags) ? responseData.response.tags : [], correspondent: responseData.response.correspondent || null, @@ -633,8 +632,7 @@ class OllamaService { custom_fields: responseData.response.custom_fields || null }; } else if (responseData.response) { - // Fall back to parsing from text response - console.log('Falling back to text response parsing'); + console.log('No JS object yet, parsing JSON'); return this._parseResponse(responseData.response); } else { throw new Error('No response data from Ollama API'); diff --git a/services/paperlessService.js b/services/paperlessService.js index 0b6b2e31..848ee3c2 100644 --- a/services/paperlessService.js +++ b/services/paperlessService.js @@ -326,11 +326,30 @@ class PaperlessService { } async createCustomFieldSafely(fieldName, fieldType, default_currency) { + const normalizedFieldType = String(fieldType || '').trim(); + + // Check an already existing field before attempting creation. Paperless + // returns 400 for duplicate names, so without this check a configured + // type mismatch (e.g. string vs. longtext) would be silently ignored. + const existingField = await this.findExistingCustomField(fieldName); + if (existingField) { + const existingFieldType = String(existingField.data_type || '').trim(); + if (existingFieldType !== normalizedFieldType) { + const error = new Error( + `Custom field "${fieldName}" has type "${existingFieldType}" in Paperless, ` + + `but configuration requests "${normalizedFieldType}".` + ); + error.code = 'CUSTOM_FIELD_TYPE_MISMATCH'; + throw error; + } + return existingField; + } + try { // Try to create the field first const response = await this.client.post('/custom_fields/', { name: fieldName, - data_type: fieldType, + data_type: normalizedFieldType, extra_data: { default_currency: default_currency || null } @@ -344,6 +363,15 @@ class PaperlessService { await this.refreshCustomFieldCache(); const existingField = await this.findExistingCustomField(fieldName); if (existingField) { + const existingFieldType = String(existingField.data_type || '').trim(); + if (existingFieldType !== normalizedFieldType) { + const mismatchError = new Error( + `Custom field "${fieldName}" has type "${existingFieldType}" in Paperless, ` + + `but configuration requests "${normalizedFieldType}".` + ); + mismatchError.code = 'CUSTOM_FIELD_TYPE_MISMATCH'; + throw mismatchError; + } return existingField; } } @@ -923,7 +951,7 @@ class PaperlessService { includeTagIds = await this.resolveTagIdsByName(includeTagNames); if (includeTagIds.length === 0) { - console.warn('[DEBUG] None of the specified tags were found'); + console.warn(`[DEBUG] None of the specified tags ${includeTagNames} were found`); return []; } @@ -1138,7 +1166,7 @@ class PaperlessService { } if (tagIds.length === 0) { - console.warn('[DEBUG] None of the specified tags were found'); + console.warn(`[DEBUG] None of the specified tags ${tagNames} were found`); return []; } @@ -1202,6 +1230,17 @@ class PaperlessService { return null; } } + + async getDocumentTypeNameById(documentTypeId) { + this.initialize(); + try { + const response = await this.client.get(`/document_types/${documentTypeId}/`); + return response.data; + } catch (error) { + console.error(`[ERROR] fetching document type ${documentTypeId}:`, error.message); + return null; + } + } async getTagNameById(tagId) { /** diff --git a/views/settings.ejs b/views/settings.ejs index b0294f77..a5ea6886 100644 --- a/views/settings.ejs +++ b/views/settings.ejs @@ -661,7 +661,8 @@