From 8d5f1ed6bcd67e5e6323367ff96e31d04c54b85c Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 7 Jul 2026 14:05:14 +0200 Subject: [PATCH 01/14] Update to Tika 4 --- .../elasticsearch/FsCrawlerTestOcrIT.java | 104 +++---- .../replace_meta_only.txt.json | 2 +- pom.xml | 3 +- tika/pom.xml | 1 + .../crawler/fs/tika/TikaDocParser.java | 6 +- .../crawler/fs/tika/TikaInstance.java | 196 +++++++----- .../crawler/fs/tika/TikaDocParserTest.java | 287 +++++++++--------- tika/src/test/resources/log4j2.xml | 2 +- 8 files changed, 329 insertions(+), 272 deletions(-) diff --git a/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestOcrIT.java b/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestOcrIT.java index 760774b7b..401b7c7b5 100644 --- a/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestOcrIT.java +++ b/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestOcrIT.java @@ -150,71 +150,71 @@ void ocr_disabled() throws Exception { Assertions.assertThat(raw) .as("test-ocr.jpg: raw metadata") .hasSize(64) - .containsKey("Component 1") - .containsKey("Component 2") - .containsKey("Component 3") + .containsEntry("ICC:Profile Connection Space", "XYZ") + .containsEntry("Number of Tables", "4 Huffman tables") .containsEntry("Compression Type", "Baseline") - .containsEntry("Content-Type", "image/jpeg") - .containsEntry("Data Precision", "8 bits") - .containsEntry("Exif IFD0:Orientation", "Top, left side (Horizontal / normal)") - .containsEntry("Exif IFD0:Resolution Unit", "Inch") - .containsKey("Exif IFD0:X Resolution") - .containsKey("Exif IFD0:Y Resolution") - .containsEntry("Exif IFD0:YCbCr Positioning", "Center of pixel array") - .containsEntry("Exif SubIFD:Color Space", "sRGB") - .containsEntry("Exif SubIFD:Components Configuration", "YCbCr") - .containsEntry("Exif SubIFD:Exif Image Height", "622 pixels") - .containsEntry("Exif SubIFD:Exif Image Width", "982 pixels") - .containsEntry("Exif SubIFD:Exif Version", "2.21") - .containsEntry("Exif SubIFD:FlashPix Version", "1.00") - .containsEntry("Exif SubIFD:Scene Capture Type", "Standard") - .containsKey("File Modified Date") - .hasEntrySatisfying( - "File Name", value -> Assertions.assertThat(value).startsWith("apache-tika-")) - .containsEntry("File Size", "41426 bytes") + .containsEntry("ICC:Profile Copyright", "Copyright Apple Inc., 2017") .containsKey("ICC:Apple Multi-language Profile Name") - .containsKey("ICC:Blue Colorant") - .containsKey("ICC:Blue Parametric TRC") - .containsKey("ICC:Blue TRC") - .containsKey("ICC:Chromatic Adaptation") + .containsKey("X-TIKA:Parsed-By-Full-Set") .containsEntry("ICC:Class", "Display Device") - .containsEntry("ICC:CMM Type", "appl") - .containsEntry("ICC:Color space", "RGB") - .containsEntry("ICC:Device manufacturer", "APPL") .containsKey("ICC:Green Colorant") - .containsKey("ICC:Green Parametric TRC") + .containsKey("ICC:Video Card Gamma") + .containsEntry("Number of Components", "3") + .containsKey("Component 2") + .containsKey("Component 1") + .containsEntry("Exif SubIFD:Exif Image Width", "982 pixels") + .containsEntry("ICC:Device manufacturer", "APPL") + .containsKey("Exif IFD0:X Resolution") + .containsEntry("tiff:ResolutionUnit", "Inch") + .containsEntry("ICC:Signature", "acsp") .containsKey("ICC:Green TRC") - .containsKey("ICC:Make And Model") .containsKey("ICC:Media White Point") - .containsKey("ICC:Native Display Information") - .containsEntry("ICC:Primary Platform", "Apple Computer, Inc.") - .containsEntry("ICC:Profile Connection Space", "XYZ") - .containsEntry("ICC:Profile Copyright", "Copyright Apple Inc., 2017") + .containsEntry("ICC:CMM Type", "appl") + .containsKey("Component 3") + .containsEntry("Exif SubIFD:Components Configuration", "YCbCr") + .containsEntry("tiff:BitsPerSample", "8") + .containsEntry("Exif IFD0:YCbCr Positioning", "Center of pixel array") + .containsEntry("X-TIKA:resourceName", "test-ocr.jpg") + .containsEntry("Exif IFD0:Orientation", "Top, left side (Horizontal / normal)") + .containsEntry("tiff:Orientation", "1") + .containsKey("ICC:Version") + .containsEntry("ICC:Profile Size", "3888") + .containsKey("X-TIKA:Parsed-By") + .containsKey("ICC:Blue Colorant") + .containsEntry("ICC:Tag Count", "17") + .containsEntry("tiff:YResolution", "144.0") + .containsEntry("Exif SubIFD:Scene Capture Type", "Standard") + .containsKey("ICC:Red TRC") + .containsEntry("Data Precision", "8 bits") + .containsEntry("tiff:ImageLength", "622") .containsKey("ICC:Profile Date/Time") + .containsKey("ICC:Blue Parametric TRC") + .containsEntry("Exif SubIFD:Color Space", "sRGB") .containsEntry("ICC:Profile Description", "Display") - .containsEntry("ICC:Profile Size", "3888") - .containsKey("ICC:Red Colorant") + .containsEntry("File Size", "41426 bytes") + .containsEntry("Exif SubIFD:Exif Version", "2.21") .containsKey("ICC:Red Parametric TRC") - .containsKey("ICC:Red TRC") - .containsEntry("ICC:Signature", "acsp") - .containsEntry("ICC:Tag Count", "17") - .containsKey("ICC:Version") - .containsKey("ICC:Video Card Gamma") + .hasEntrySatisfying( + "File Name", value -> Assertions.assertThat(value).startsWith("apache-tika-")) + .containsEntry("Exif IFD0:Resolution Unit", "Inch") + .containsEntry("ICC:Color space", "RGB") + .containsKey("ICC:Green Parametric TRC") + .containsEntry("Content-Type", "image/jpeg") + .containsKey("ICC:Blue TRC") .containsKey("ICC:XYZ values") + .containsKey("ICC:Native Display Information") + .containsKey("File Modified Date") + .containsEntry("tiff:XResolution", "144.0") .containsEntry("Image Height", "622 pixels") + .containsEntry("Exif SubIFD:FlashPix Version", "1.00") + .containsKey("ICC:Make And Model") + .containsEntry("Exif SubIFD:Exif Image Height", "622 pixels") .containsEntry("Image Width", "982 pixels") - .containsEntry("Number of Components", "3") - .containsEntry("Number of Tables", "4 Huffman tables") - .containsEntry("resourceName", "test-ocr.jpg") - .containsEntry("tiff:BitsPerSample", "8") - .containsEntry("tiff:ImageLength", "622") + .containsEntry("ICC:Primary Platform", "Apple Computer, Inc.") + .containsKey("ICC:Chromatic Adaptation") + .containsKey("ICC:Red Colorant") .containsEntry("tiff:ImageWidth", "982") - .containsEntry("tiff:Orientation", "1") - .containsEntry("tiff:ResolutionUnit", "Inch") - .containsEntry("tiff:XResolution", "144.0") - .containsEntry("tiff:YResolution", "144.0") - .containsKey("X-TIKA:Parsed-By") - .containsKey("X-TIKA:Parsed-By-Full-Set"); + .containsKey("Exif IFD0:Y Resolution"); // test-ocr.png: OCR disabled → no content String pngHitSource = findHitByFilename(searchResponse, "test-ocr.png").getSource(); diff --git a/integration-tests/src/test/resources-binary/tags/document_with_external_tags/replace_meta_only.txt.json b/integration-tests/src/test/resources-binary/tags/document_with_external_tags/replace_meta_only.txt.json index 617442d5d..4548718e1 100644 --- a/integration-tests/src/test/resources-binary/tags/document_with_external_tags/replace_meta_only.txt.json +++ b/integration-tests/src/test/resources-binary/tags/document_with_external_tags/replace_meta_only.txt.json @@ -1,7 +1,7 @@ { "meta":{ "raw":{ - "resourceName":"another-file-name.txt" + "X-TIKA:resourceName":"another-file-name.txt" } } } diff --git a/pom.xml b/pom.xml index aa5b142c0..c4ad8f8c0 100644 --- a/pom.xml +++ b/pom.xml @@ -69,7 +69,7 @@ 0.38.0 2.6 - 3.3.1 + 4.0.0-beta-1 3.1.4 - - - - - - application/xhtml+xml - - - diff --git a/docs/source/admin/fs/rest.rst b/docs/source/admin/fs/rest.rst index 4ecc1c210..7cf2771c7 100644 --- a/docs/source/admin/fs/rest.rst +++ b/docs/source/admin/fs/rest.rst @@ -151,9 +151,12 @@ You will get back your document as it has been stored by elasticsearch: }, "meta" : { "raw" : { - "X-Parsed-By" : "org.apache.tika.parser.DefaultParser", "Content-Encoding" : "ISO-8859-1", - "Content-Type" : "text/plain; charset=ISO-8859-1" + "Content-Type" : "text/plain; charset=ISO-8859-1", + "Content-Type-Magic-Detected" : "text/plain", + "X-TIKA:Parsed-By" : "org.apache.tika.parser.DefaultParser", + "X-TIKA:Parsed-By-Full-Set" : "org.apache.tika.parser.DefaultParser", + "X-TIKA:resourceName" : "test.txt" } }, "path" : { @@ -187,9 +190,12 @@ will give }, "meta" : { "raw" : { - "X-Parsed-By" : "org.apache.tika.parser.DefaultParser", "Content-Encoding" : "ISO-8859-1", - "Content-Type" : "text/plain; charset=ISO-8859-1" + "Content-Type" : "text/plain; charset=ISO-8859-1", + "Content-Type-Magic-Detected" : "text/plain", + "X-TIKA:Parsed-By" : "org.apache.tika.parser.DefaultParser", + "X-TIKA:Parsed-By-Full-Set" : "org.apache.tika.parser.DefaultParser", + "X-TIKA:resourceName" : "test.txt" } }, "path" : { diff --git a/docs/source/release/2.10.rst b/docs/source/release/2.10.rst index 5022c6387..43ad660b2 100644 --- a/docs/source/release/2.10.rst +++ b/docs/source/release/2.10.rst @@ -38,6 +38,8 @@ New * You can now remove a document in Elasticsearch using FSCrawler ``_document`` endpoint. See :ref:`rest-service`. Thanks to dadoonet. * Implement our own HTTP Client for Elasticsearch. Thanks to dadoonet. * Add option to set path to custom tika config file. See :ref:`local-fs-settings`. Thanks to iadcode. + **Note**: this option is temporarily disabled after the Apache Tika 4 upgrade, which removed the XML-based + configuration file mechanism. It is expected to return on top of Tika 4's JSON-based configuration. * Support for Index Templates. See :ref:`mappings`. Thanks to dadoonet. * Support for Aliases. You can now index to an alias. Thanks to dadoonet. * Support for Access Token and Api Keys instead of Basic Authentication. See :ref:`credentials`. Thanks to dadoonet. From 071728890aab1a120139a1ccc22c147c1b06500a Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 7 Jul 2026 16:08:41 +0200 Subject: [PATCH 09/14] =?UTF-8?q?fix(log):=20=F0=9F=90=9B=20change=20fscra?= =?UTF-8?q?wler.fs.client=20logger=20level=20from=20debug=20to=20info?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- elasticsearch-client/src/test/resources/log4j2.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elasticsearch-client/src/test/resources/log4j2.xml b/elasticsearch-client/src/test/resources/log4j2.xml index aea0bc66b..d30e01bce 100644 --- a/elasticsearch-client/src/test/resources/log4j2.xml +++ b/elasticsearch-client/src/test/resources/log4j2.xml @@ -9,7 +9,7 @@ - + From 7aedb4cfcb72d0d9fd4e7fbcdeafcd60c5829a6b Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 7 Jul 2026 17:48:47 +0200 Subject: [PATCH 10/14] =?UTF-8?q?fix(test):=20=F0=9F=90=9B=20update=20OCR?= =?UTF-8?q?=20IT=20raw=20metadata=20assertions=20for=20Tika=204=20img:=20n?= =?UTF-8?q?amespace?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Tika 4 namespaces all metadata-extractor image keys under "img:" (both the bare keys like "Number of Tables" and the "Exif IFD0:/SubIFD:" ones); values are unchanged, ICC:/tiff:/Content-*/X-TIKA: keys stay as-is - Bump hasSize(64) -> hasSize(66) (Tika 4 adds Content-Type-Magic-Detected and Content-Length) and add the img: prefixes, verified against real output - Sort the raw-metadata assertions case-insensitively for readability Co-Authored-By: Claude Opus 4.8 (1M context) --- .../elasticsearch/FsCrawlerTestOcrIT.java | 113 +++++++++--------- 1 file changed, 57 insertions(+), 56 deletions(-) diff --git a/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestOcrIT.java b/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestOcrIT.java index 401b7c7b5..c0aa2606b 100644 --- a/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestOcrIT.java +++ b/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestOcrIT.java @@ -149,72 +149,73 @@ void ocr_disabled() throws Exception { Map raw = JsonPath.read(jpgHitSource, "$.meta.raw"); Assertions.assertThat(raw) .as("test-ocr.jpg: raw metadata") - .hasSize(64) - .containsEntry("ICC:Profile Connection Space", "XYZ") - .containsEntry("Number of Tables", "4 Huffman tables") - .containsEntry("Compression Type", "Baseline") - .containsEntry("ICC:Profile Copyright", "Copyright Apple Inc., 2017") + .hasSize(66) + .containsEntry("Content-Type", "image/jpeg") + .containsEntry("Content-Type-Magic-Detected", "image/jpeg") .containsKey("ICC:Apple Multi-language Profile Name") - .containsKey("X-TIKA:Parsed-By-Full-Set") + .containsKey("ICC:Blue Colorant") + .containsKey("ICC:Blue Parametric TRC") + .containsKey("ICC:Blue TRC") + .containsKey("ICC:Chromatic Adaptation") .containsEntry("ICC:Class", "Display Device") - .containsKey("ICC:Green Colorant") - .containsKey("ICC:Video Card Gamma") - .containsEntry("Number of Components", "3") - .containsKey("Component 2") - .containsKey("Component 1") - .containsEntry("Exif SubIFD:Exif Image Width", "982 pixels") + .containsEntry("ICC:CMM Type", "appl") + .containsEntry("ICC:Color space", "RGB") .containsEntry("ICC:Device manufacturer", "APPL") - .containsKey("Exif IFD0:X Resolution") - .containsEntry("tiff:ResolutionUnit", "Inch") - .containsEntry("ICC:Signature", "acsp") + .containsKey("ICC:Green Colorant") + .containsKey("ICC:Green Parametric TRC") .containsKey("ICC:Green TRC") + .containsKey("ICC:Make And Model") .containsKey("ICC:Media White Point") - .containsEntry("ICC:CMM Type", "appl") - .containsKey("Component 3") - .containsEntry("Exif SubIFD:Components Configuration", "YCbCr") - .containsEntry("tiff:BitsPerSample", "8") - .containsEntry("Exif IFD0:YCbCr Positioning", "Center of pixel array") - .containsEntry("X-TIKA:resourceName", "test-ocr.jpg") - .containsEntry("Exif IFD0:Orientation", "Top, left side (Horizontal / normal)") - .containsEntry("tiff:Orientation", "1") - .containsKey("ICC:Version") - .containsEntry("ICC:Profile Size", "3888") - .containsKey("X-TIKA:Parsed-By") - .containsKey("ICC:Blue Colorant") - .containsEntry("ICC:Tag Count", "17") - .containsEntry("tiff:YResolution", "144.0") - .containsEntry("Exif SubIFD:Scene Capture Type", "Standard") - .containsKey("ICC:Red TRC") - .containsEntry("Data Precision", "8 bits") - .containsEntry("tiff:ImageLength", "622") + .containsKey("ICC:Native Display Information") + .containsEntry("ICC:Primary Platform", "Apple Computer, Inc.") + .containsEntry("ICC:Profile Connection Space", "XYZ") + .containsEntry("ICC:Profile Copyright", "Copyright Apple Inc., 2017") .containsKey("ICC:Profile Date/Time") - .containsKey("ICC:Blue Parametric TRC") - .containsEntry("Exif SubIFD:Color Space", "sRGB") .containsEntry("ICC:Profile Description", "Display") - .containsEntry("File Size", "41426 bytes") - .containsEntry("Exif SubIFD:Exif Version", "2.21") + .containsEntry("ICC:Profile Size", "3888") + .containsKey("ICC:Red Colorant") .containsKey("ICC:Red Parametric TRC") - .hasEntrySatisfying( - "File Name", value -> Assertions.assertThat(value).startsWith("apache-tika-")) - .containsEntry("Exif IFD0:Resolution Unit", "Inch") - .containsEntry("ICC:Color space", "RGB") - .containsKey("ICC:Green Parametric TRC") - .containsEntry("Content-Type", "image/jpeg") - .containsKey("ICC:Blue TRC") + .containsKey("ICC:Red TRC") + .containsEntry("ICC:Signature", "acsp") + .containsEntry("ICC:Tag Count", "17") + .containsKey("ICC:Version") + .containsKey("ICC:Video Card Gamma") .containsKey("ICC:XYZ values") - .containsKey("ICC:Native Display Information") - .containsKey("File Modified Date") - .containsEntry("tiff:XResolution", "144.0") - .containsEntry("Image Height", "622 pixels") - .containsEntry("Exif SubIFD:FlashPix Version", "1.00") - .containsKey("ICC:Make And Model") - .containsEntry("Exif SubIFD:Exif Image Height", "622 pixels") - .containsEntry("Image Width", "982 pixels") - .containsEntry("ICC:Primary Platform", "Apple Computer, Inc.") - .containsKey("ICC:Chromatic Adaptation") - .containsKey("ICC:Red Colorant") + .containsKey("img:Component 1") + .containsKey("img:Component 2") + .containsKey("img:Component 3") + .containsEntry("img:Compression Type", "Baseline") + .containsEntry("img:Data Precision", "8 bits") + .containsEntry("img:Exif IFD0:Orientation", "Top, left side (Horizontal / normal)") + .containsEntry("img:Exif IFD0:Resolution Unit", "Inch") + .containsKey("img:Exif IFD0:X Resolution") + .containsKey("img:Exif IFD0:Y Resolution") + .containsEntry("img:Exif IFD0:YCbCr Positioning", "Center of pixel array") + .containsEntry("img:Exif SubIFD:Color Space", "sRGB") + .containsEntry("img:Exif SubIFD:Components Configuration", "YCbCr") + .containsEntry("img:Exif SubIFD:Exif Image Height", "622 pixels") + .containsEntry("img:Exif SubIFD:Exif Image Width", "982 pixels") + .containsEntry("img:Exif SubIFD:Exif Version", "2.21") + .containsEntry("img:Exif SubIFD:FlashPix Version", "1.00") + .containsEntry("img:Exif SubIFD:Scene Capture Type", "Standard") + .containsKey("img:File Modified Date") + .hasEntrySatisfying( + "img:File Name", value -> Assertions.assertThat(value).startsWith("apache-tika-")) + .containsEntry("img:File Size", "41426 bytes") + .containsEntry("img:Image Height", "622 pixels") + .containsEntry("img:Image Width", "982 pixels") + .containsEntry("img:Number of Components", "3") + .containsEntry("img:Number of Tables", "4 Huffman tables") + .containsEntry("tiff:BitsPerSample", "8") + .containsEntry("tiff:ImageLength", "622") .containsEntry("tiff:ImageWidth", "982") - .containsKey("Exif IFD0:Y Resolution"); + .containsEntry("tiff:Orientation", "1") + .containsEntry("tiff:ResolutionUnit", "Inch") + .containsEntry("tiff:XResolution", "144.0") + .containsEntry("tiff:YResolution", "144.0") + .containsKey("X-TIKA:Parsed-By") + .containsKey("X-TIKA:Parsed-By-Full-Set") + .containsEntry("X-TIKA:resourceName", "test-ocr.jpg"); // test-ocr.png: OCR disabled → no content String pngHitSource = findHitByFilename(searchResponse, "test-ocr.png").getSource(); From 897c84e575d91f0b7e72d4545be0de34f39b3fae Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 7 Jul 2026 17:53:07 +0200 Subject: [PATCH 11/14] =?UTF-8?q?fix(test):=20=F0=9F=90=9B=20read=20X-TIKA?= =?UTF-8?q?:resourceName=20in=20REST=20IT=20for=20Tika=204?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Tika 4 renamed the resourceName metadata key to X-TIKA:resourceName; update the JsonPath in documentWithExternalTags to bracket notation for the new key Co-Authored-By: Claude Opus 4.8 (1M context) --- .../fs/test/integration/elasticsearch/FsCrawlerRestIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerRestIT.java b/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerRestIT.java index 3a5ee72ce..613d4a7f7 100644 --- a/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerRestIT.java +++ b/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerRestIT.java @@ -372,7 +372,7 @@ void documentWithExternalTags() throws Exception { checkDocument("replace_meta_only.txt", hit -> { Assertions.assertThat((String) JsonPath.read(hit.getSource(), "$.content")) .contains("This file content will be extracted"); - Assertions.assertThat((String) JsonPath.read(hit.getSource(), "$.meta.raw.resourceName")) + Assertions.assertThat((String) JsonPath.read(hit.getSource(), "$.meta.raw['X-TIKA:resourceName']")) .isEqualTo("another-file-name.txt"); Assertions.assertThatThrownBy(() -> JsonPath.read(hit.getSource(), "$.external")) .isInstanceOf(PathNotFoundException.class); From f4ed6c0bdb731d2e5a2810e4f4bab41669a1b981 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 7 Jul 2026 17:55:56 +0200 Subject: [PATCH 12/14] =?UTF-8?q?docs(tika):=20=F0=9F=93=9D=20document=20T?= =?UTF-8?q?ika=204=20meta.raw=20key=20namespace=20changes=20as=20breaking?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Note the img: namespace for image/EXIF metadata keys - Note resourceName -> X-TIKA:resourceName rename - Note new keys: Content-Type-Magic-Detected, X-TIKA:Parsed-By-Full-Set and the text encoding-detection keys (X-TIKA:detectedEncoding/encodingDetectionTrace/encodingDetector) Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/source/release/2.10.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/source/release/2.10.rst b/docs/source/release/2.10.rst index 43ad660b2..78fc53715 100644 --- a/docs/source/release/2.10.rst +++ b/docs/source/release/2.10.rst @@ -17,6 +17,18 @@ enough. Thanks to dadoonet. * We don't support anymore the ``elasticsearch.nodes.url`` setting. You need to use ``elasticsearch.urls`` instead. Thanks to dadoonet. * The ``_upload`` REST endpoint has been removed. Please now use the ``_document`` endpoint. Thanks to dadoonet. +* The Apache Tika 4 upgrade changes several ``meta.raw.*`` metadata key names. If you search, aggregate or + map these fields, you will need to update your queries and index templates accordingly: + + * Image and EXIF metadata (JPEG, PNG, TIFF, …) keys are now namespaced under ``img:``. For example + ``Number of Tables`` becomes ``img:Number of Tables`` and ``Exif IFD0:Orientation`` becomes + ``img:Exif IFD0:Orientation``. + * The resource name key ``resourceName`` is renamed to ``X-TIKA:resourceName``. + * New keys are now produced, such as ``Content-Type-Magic-Detected``, ``X-TIKA:Parsed-By-Full-Set`` and, + for text documents, the encoding detection keys ``X-TIKA:detectedEncoding``, + ``X-TIKA:encodingDetectionTrace`` and ``X-TIKA:encodingDetector``. + + Thanks to dadoonet. New --- From ad00c11c23bcf365cc26c9bcd3f92cc9c0e8ecf0 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 10 Jul 2026 17:19:26 +0200 Subject: [PATCH 13/14] =?UTF-8?q?fix(tika):=20=F0=9F=90=9B=20keep=20caller?= =?UTF-8?q?-owned=20streams=20open=20during=20Tika=204=20parsing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TikaInputStream.get(stream, metadata) closes the underlying stream on exit, which breaks FTP uploads where completePendingCommand() must run in the caller's close path. Use TemporaryResources so Tika can spool without closing provider streams. Co-authored-by: Cursor --- .../elasticsearch/crawler/fs/tika/TikaInstance.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tika/src/main/java/fr/pilato/elasticsearch/crawler/fs/tika/TikaInstance.java b/tika/src/main/java/fr/pilato/elasticsearch/crawler/fs/tika/TikaInstance.java index 42e763755..d0174b903 100644 --- a/tika/src/main/java/fr/pilato/elasticsearch/crawler/fs/tika/TikaInstance.java +++ b/tika/src/main/java/fr/pilato/elasticsearch/crawler/fs/tika/TikaInstance.java @@ -37,6 +37,7 @@ import org.apache.tika.exception.TikaException; import org.apache.tika.exception.WriteLimitReachedException; import org.apache.tika.exception.ZeroByteFileException; +import org.apache.tika.io.TemporaryResources; import org.apache.tika.io.TikaInputStream; import org.apache.tika.language.detect.LanguageDetector; import org.apache.tika.metadata.Metadata; @@ -275,9 +276,10 @@ private void initContext(Fs fs) { */ String extractText(int indexedChars, InputStream stream, Metadata metadata) throws IOException, TikaException { WriteOutContentHandler handler = new WriteOutContentHandler(indexedChars); - // Tika 4 requires a TikaInputStream for Parser.parse(). Wrapping here keeps the caller unaware of the - // change; closing the wrapper releases any temporary resources Tika spooled while detecting the type. - try (TikaInputStream tikaInputStream = TikaInputStream.get(stream, metadata)) { + // Tika 4 requires a TikaInputStream for Parser.parse(). Use TemporaryResources so Tika can spool/cache + // without closing the caller's stream (required for FTP/SFTP wrappers that finalize the transfer on close). + try (TemporaryResources tmp = new TemporaryResources()) { + TikaInputStream tikaInputStream = TikaInputStream.get(stream, tmp, metadata); parser.parse(tikaInputStream, new BodyContentHandler(handler), metadata, context); } catch (WriteLimitReachedException e) { String resourceName = metadata.get(TikaCoreProperties.RESOURCE_NAME_KEY); From 281d916434db43b0dc29b4acc6a6e156128e52ca Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 10 Jul 2026 20:25:06 +0200 Subject: [PATCH 14/14] =?UTF-8?q?fix(ocr):=20=F0=9F=90=9B=20change=20defau?= =?UTF-8?q?lt=20PDF=20strategy=20to=20'auto'=20for=20improved=20performanc?= =?UTF-8?q?e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/admin/fs/rest.rst | 2 +- docs/source/release/2.10.rst | 3 +++ docs/source/user/ocr.rst | 6 +++--- .../fr/pilato/elasticsearch/crawler/fs/settings/Ocr.java | 4 ++-- .../crawler/fs/settings/fscrawler-default.properties | 2 +- .../crawler/fs/settings/fscrawler-default.yaml | 2 +- .../crawler/fs/settings/FsSettingsLoaderTest.java | 2 +- .../pilato/elasticsearch/crawler/fs/tika/TikaInstance.java | 7 ++++--- 8 files changed, 16 insertions(+), 12 deletions(-) diff --git a/docs/source/admin/fs/rest.rst b/docs/source/admin/fs/rest.rst index 7cf2771c7..9f10191c2 100644 --- a/docs/source/admin/fs/rest.rst +++ b/docs/source/admin/fs/rest.rst @@ -78,7 +78,7 @@ It will give you a response similar to: "language" : "eng", "output_type" : "txt", "page_seg_mode" : 1, - "pdf_strategy" : "ocr_and_text", + "pdf_strategy" : "auto", "preserve_interword_spacing" : false }, "raw_metadata" : true, diff --git a/docs/source/release/2.10.rst b/docs/source/release/2.10.rst index 78fc53715..113e3c761 100644 --- a/docs/source/release/2.10.rst +++ b/docs/source/release/2.10.rst @@ -29,6 +29,9 @@ enough. Thanks to dadoonet. ``X-TIKA:encodingDetectionTrace`` and ``X-TIKA:encodingDetector``. Thanks to dadoonet. +* The default ``fs.ocr.pdf_strategy`` is now ``auto`` instead of ``ocr_and_text``. With ``auto``, OCR is skipped on + PDF pages that already contain more than 10 characters of text. If you relied on the previous behaviour and want + OCR on every PDF page, explicitly set ``fs.ocr.pdf_strategy`` to ``ocr_and_text``. Thanks to dadoonet. New --- diff --git a/docs/source/user/ocr.rst b/docs/source/user/ocr.rst index 83e65f5ff..fa564c232 100644 --- a/docs/source/user/ocr.rst +++ b/docs/source/user/ocr.rst @@ -30,7 +30,7 @@ Here is a list of OCR settings (under ``fs.ocr`` prefix): +---------------------------------------+-------------------------------------------------+------------------+-----------------------------------+ | ``fs.ocr.output_type`` | ``FSCRAWLER_FS_OCR_OUTPUT_TYPE`` | ``txt`` | `OCR Output Type`_ | +---------------------------------------+-------------------------------------------------+------------------+-----------------------------------+ -| ``fs.ocr.pdf_strategy`` | ``FSCRAWLER_FS_OCR_PDF_STRATEGY`` | ``ocr_and_text`` | `OCR PDF Strategy`_ | +| ``fs.ocr.pdf_strategy`` | ``FSCRAWLER_FS_OCR_PDF_STRATEGY`` | ``auto`` | `OCR PDF Strategy`_ | +---------------------------------------+-------------------------------------------------+------------------+-----------------------------------+ | ``fs.ocr.page_seg_mode`` | ``FSCRAWLER_FS_OCR_PAGE_SEG_MODE`` | ``01`` | `OCR Page Seg Mode`_ | +---------------------------------------+-------------------------------------------------+------------------+-----------------------------------+ @@ -165,8 +165,8 @@ Supported strategies are: * ``ocr_and_text``: OCR and text extraction is performed. -.. note:: When omitted, ``ocr_and_text`` value is used. If you have performance issues, it's worth using the ``auto`` option - instead as only documents with barely no text will go through the OCR process. +.. note:: When omitted, ``auto`` value is used. OCR is skipped on PDF pages that already contain more than + 10 characters of text. If you need OCR on every page regardless, set ``ocr_and_text`` or ``ocr_only``. OCR Page Seg Mode ----------------- diff --git a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Ocr.java b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Ocr.java index 7fa00cf79..7873a05c9 100644 --- a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Ocr.java +++ b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Ocr.java @@ -44,7 +44,7 @@ public class Ocr { @Nullable private String outputType; // Pdf OCR Strategy - @Config(defaultVal = "ocr_and_text") + @Config(defaultVal = "auto") private String pdfStrategy; // PDF Page Seg Mode @Config @@ -110,7 +110,7 @@ public void setPageSegMode(Integer pageSegMode) { } /** - * Get the PDF Strategy. Could be "no_ocr", "auto", "ocr_only" or "ocr_and_text" (default) + * Get the PDF Strategy. Could be "no_ocr", "auto" (default), "ocr_only" or "ocr_and_text" * * @return the PDF Strategy */ diff --git a/settings/src/main/resources/fr/pilato/elasticsearch/crawler/fs/settings/fscrawler-default.properties b/settings/src/main/resources/fr/pilato/elasticsearch/crawler/fs/settings/fscrawler-default.properties index 249cb38f0..5534299e0 100644 --- a/settings/src/main/resources/fr/pilato/elasticsearch/crawler/fs/settings/fscrawler-default.properties +++ b/settings/src/main/resources/fr/pilato/elasticsearch/crawler/fs/settings/fscrawler-default.properties @@ -25,7 +25,7 @@ fs.follow_symlinks=false fs.ocr.enabled=true fs.ocr.language=eng fs.ocr.output_type=txt -fs.ocr.pdf_strategy=ocr_and_text +fs.ocr.pdf_strategy=auto fs.ocr.page_seg_mode=1 fs.ocr.preserve_interword_spacing=false diff --git a/settings/src/main/resources/fr/pilato/elasticsearch/crawler/fs/settings/fscrawler-default.yaml b/settings/src/main/resources/fr/pilato/elasticsearch/crawler/fs/settings/fscrawler-default.yaml index f7563e929..a95a7e252 100644 --- a/settings/src/main/resources/fr/pilato/elasticsearch/crawler/fs/settings/fscrawler-default.yaml +++ b/settings/src/main/resources/fr/pilato/elasticsearch/crawler/fs/settings/fscrawler-default.yaml @@ -85,7 +85,7 @@ # optional: the OCR output type, either txt or hocr #output_type: txt # optional: the OCR strategy, either auto, no_ocr, ocr_only, ocr_and_text - #pdf_strategy: ocr_and_text + #pdf_strategy: auto # optional: the mode to use for page orientation from 0 to 13 #page_seg_mode: 1 # automatic mode # optional: if true, we will try to preserve interword spacing diff --git a/settings/src/test/java/fr/pilato/elasticsearch/crawler/fs/settings/FsSettingsLoaderTest.java b/settings/src/test/java/fr/pilato/elasticsearch/crawler/fs/settings/FsSettingsLoaderTest.java index fe1f6f1eb..1f99ae68a 100644 --- a/settings/src/test/java/fr/pilato/elasticsearch/crawler/fs/settings/FsSettingsLoaderTest.java +++ b/settings/src/test/java/fr/pilato/elasticsearch/crawler/fs/settings/FsSettingsLoaderTest.java @@ -230,7 +230,7 @@ private FsSettings generateExpectedDefaultFsSettings() { ocr.setEnabled(true); ocr.setLanguage("eng"); ocr.setOutputType("txt"); - ocr.setPdfStrategy("ocr_and_text"); + ocr.setPdfStrategy("auto"); ocr.setPageSegMode(1); fs.setOcr(ocr); expected.setFs(fs); diff --git a/tika/src/main/java/fr/pilato/elasticsearch/crawler/fs/tika/TikaInstance.java b/tika/src/main/java/fr/pilato/elasticsearch/crawler/fs/tika/TikaInstance.java index d0174b903..2c6d5ae7a 100644 --- a/tika/src/main/java/fr/pilato/elasticsearch/crawler/fs/tika/TikaInstance.java +++ b/tika/src/main/java/fr/pilato/elasticsearch/crawler/fs/tika/TikaInstance.java @@ -217,9 +217,10 @@ private static void setPdfOcrStrategy(PDFParser pdfParser, String strategy) { } /** - * Maps an FSCrawler PDF OCR strategy string to the Tika 4 {@link OcrConfig.Strategy} enum. The FSCrawler value - * {@code ocr_and_text} (the default) maps to {@link OcrConfig.Strategy#OCR_AND_TEXT_EXTRACTION}. Unknown values - * fall back to {@link OcrConfig.Strategy#AUTO}. + * Maps an FSCrawler PDF OCR strategy string to the Tika 4 {@link OcrConfig.Strategy} enum. When the strategy is + * null, {@link OcrConfig.Strategy#AUTO} is used (the FSCrawler default). The FSCrawler value {@code ocr_and_text} + * maps to {@link OcrConfig.Strategy#OCR_AND_TEXT_EXTRACTION}. Unknown values fall back to + * {@link OcrConfig.Strategy#AUTO}. * * @param strategy the FSCrawler strategy string (may be null) * @return the matching Tika strategy