diff --git a/docs/source/admin/fs/local-fs.rst b/docs/source/admin/fs/local-fs.rst index 3178f3777..741858cd7 100644 --- a/docs/source/admin/fs/local-fs.rst +++ b/docs/source/admin/fs/local-fs.rst @@ -512,42 +512,53 @@ For example, a PDF document could generate: .. code:: json { - "date" : "2016-07-07T08:37:42Z", - "pdf:PDFVersion" : "1.5", - "xmp:CreatorTool" : "Microsoft Word", - "Keywords" : "keyword1, keyword2", + "access_permission:assemble_document" : "true", + "access_permission:can_modify" : "true", + "access_permission:can_print" : "true", + "access_permission:can_print_faithful" : "true", + "access_permission:extract_content" : "true", + "access_permission:extract_for_accessibility" : "true", + "access_permission:fill_in_form" : "true", "access_permission:modify_annotations" : "true", - "access_permission:can_print_degraded" : "true", - "subject" : "Test Tika Object", + "Content-Length" : "101643", + "Content-Type" : "application/pdf", + "Content-Type-Magic-Detected" : "application/pdf", "dc:creator" : "David Pilato", + "dc:format" : "application/pdf; version=1.5", + "dc:language" : "en-US", + "dc:subject" : "keyword1, keyword2", + "dc:title" : "Test Tika title", "dcterms:created" : "2016-07-07T08:37:42Z", - "Last-Modified" : "2016-07-07T08:37:42Z", "dcterms:modified" : "2016-07-07T08:37:42Z", - "dc:format" : "application/pdf; version=1.5", - "title" : "Test Tika title", - "Last-Save-Date" : "2016-07-07T08:37:42Z", - "access_permission:fill_in_form" : "true", - "meta:save-date" : "2016-07-07T08:37:42Z", + "pdf:charsPerPage" : "42", + "pdf:containsDamagedFont" : "false", + "pdf:containsNonEmbeddedFont" : "false", + "pdf:docinfo:created" : "2016-07-07T08:37:42Z", + "pdf:docinfo:creator" : "David Pilato", + "pdf:docinfo:creator_tool" : "Microsoft Word", + "pdf:docinfo:keywords" : "keyword1, keyword2", + "pdf:docinfo:modified" : "2016-07-07T08:37:42Z", + "pdf:docinfo:subject" : "Test Tika Object", + "pdf:docinfo:title" : "Test Tika title", "pdf:encrypted" : "false", - "dc:title" : "Test Tika title", - "modified" : "2016-07-07T08:37:42Z", - "cp:subject" : "Test Tika Object", - "Content-Type" : "application/pdf", - "X-Parsed-By" : "org.apache.tika.parser.DefaultParser", - "creator" : "David Pilato", - "meta:author" : "David Pilato", - "dc:subject" : "keyword1, keyword2", - "meta:creation-date" : "2016-07-07T08:37:42Z", - "created" : "Thu Jul 07 10:37:42 CEST 2016", - "access_permission:extract_for_accessibility" : "true", - "access_permission:assemble_document" : "true", - "xmpTPg:NPages" : "2", - "Creation-Date" : "2016-07-07T08:37:42Z", - "access_permission:extract_content" : "true", - "access_permission:can_print" : "true", - "meta:keyword" : "keyword1, keyword2", - "Author" : "David Pilato", - "access_permission:can_modify" : "true" + "pdf:eofOffsets" : "101460", + "pdf:hasCollection" : "false", + "pdf:hasMarkedContent" : "true", + "pdf:hasXFA" : "false", + "pdf:hasXMP" : "false", + "pdf:incrementalUpdateCount" : "1", + "pdf:num3DAnnotations" : "0", + "pdf:ocrPageCount" : "0", + "pdf:overallPercentageUnmappedUnicodeChars" : "0.0", + "pdf:PDFVersion" : "1.5", + "pdf:totalUnmappedUnicodeChars" : "0", + "pdf:unmappedUnicodeCharsPerPage" : "0", + "X-TIKA:Parsed-By" : "org.apache.tika.parser.pdf.PDFParser", + "X-TIKA:Parsed-By-Full-Set" : "org.apache.tika.parser.pdf.PDFParser", + "X-TIKA:resourceName" : "test.pdf", + "X-TIKA:versionCount" : "1", + "xmp:CreatorTool" : "Microsoft Word", + "xmpTPg:NPages" : "2" } Where a MP3 file would generate: @@ -555,27 +566,26 @@ Where a MP3 file would generate: .. code:: json { - "xmpDM:genre" : "Vocal", - "X-Parsed-By" : "org.apache.tika.parser.DefaultParser", - "creator" : "David Pilato", + "channels" : "2", + "Content-Type" : "audio/mpeg", + "Content-Type-Magic-Detected" : "audio/mpeg", + "dc:creator" : "David Pilato", + "dc:title" : "Test Tika", + "samplerate" : "44100", + "version" : "MPEG 3 Layer III Version 1", + "X-TIKA:Parsed-By" : "org.apache.tika.parser.DefaultParser", + "X-TIKA:Parsed-By-Full-Set" : "org.apache.tika.parser.DefaultParser", + "X-TIKA:resourceName" : "test.mp3", "xmpDM:album" : "FS Crawler", - "xmpDM:trackNumber" : "1", - "xmpDM:releaseDate" : "2016", - "meta:author" : "David Pilato", "xmpDM:artist" : "David Pilato", - "dc:creator" : "David Pilato", - "xmpDM:audioCompressor" : "MP3", - "title" : "Test Tika", "xmpDM:audioChannelType" : "Stereo", - "version" : "MPEG 3 Layer III Version 1", - "xmpDM:logComment" : "Hello but reverted", + "xmpDM:audioCompressor" : "MP3", "xmpDM:audioSampleRate" : "44100", - "channels" : "2", - "dc:title" : "Test Tika", - "Author" : "David Pilato", - "xmpDM:duration" : "1018.775146484375", - "Content-Type" : "audio/mpeg", - "samplerate" : "44100" + "xmpDM:duration" : "1.0187751054763794", + "xmpDM:genre" : "Vocal", + "xmpDM:logComment" : "Hello but reverted", + "xmpDM:releaseDate" : "2016", + "xmpDM:trackNumber" : "1" } .. note:: @@ -879,31 +889,19 @@ Tika Config Path .. versionadded:: 2.10 -If you want to override the default tika parser configuration, you can set the path to a custom tika -configuration file, which will be used instead. +.. warning:: + + Since the upgrade to Apache Tika 4, ``fs.tika_config_path`` is **temporarily not supported**. Tika 4 + removed the XML-based Tika configuration file mechanism, and there is currently no drop-in replacement + to assemble a parser from a configuration file. If this setting is provided, FSCrawler fails fast with + a clear configuration error. Support is expected to return on top of Tika 4's new JSON-based + configuration. + +This setting used to let you override the default Tika parser configuration by pointing to a custom Tika +configuration file: .. code:: yaml name: "test" fs: tika_config_path: '/path/to/tikaConfig.xml' - -An example tika config file is shown below. See |Tika_configuring|_ for more information. - -.. code:: xml - - - - - - - - - - - - - application/xhtml+xml - - - diff --git a/docs/source/admin/fs/rest.rst b/docs/source/admin/fs/rest.rst index 4ecc1c210..9f10191c2 100644 --- a/docs/source/admin/fs/rest.rst +++ b/docs/source/admin/fs/rest.rst @@ -78,7 +78,7 @@ It will give you a response similar to: "language" : "eng", "output_type" : "txt", "page_seg_mode" : 1, - "pdf_strategy" : "ocr_and_text", + "pdf_strategy" : "auto", "preserve_interword_spacing" : false }, "raw_metadata" : true, @@ -151,9 +151,12 @@ You will get back your document as it has been stored by elasticsearch: }, "meta" : { "raw" : { - "X-Parsed-By" : "org.apache.tika.parser.DefaultParser", "Content-Encoding" : "ISO-8859-1", - "Content-Type" : "text/plain; charset=ISO-8859-1" + "Content-Type" : "text/plain; charset=ISO-8859-1", + "Content-Type-Magic-Detected" : "text/plain", + "X-TIKA:Parsed-By" : "org.apache.tika.parser.DefaultParser", + "X-TIKA:Parsed-By-Full-Set" : "org.apache.tika.parser.DefaultParser", + "X-TIKA:resourceName" : "test.txt" } }, "path" : { @@ -187,9 +190,12 @@ will give }, "meta" : { "raw" : { - "X-Parsed-By" : "org.apache.tika.parser.DefaultParser", "Content-Encoding" : "ISO-8859-1", - "Content-Type" : "text/plain; charset=ISO-8859-1" + "Content-Type" : "text/plain; charset=ISO-8859-1", + "Content-Type-Magic-Detected" : "text/plain", + "X-TIKA:Parsed-By" : "org.apache.tika.parser.DefaultParser", + "X-TIKA:Parsed-By-Full-Set" : "org.apache.tika.parser.DefaultParser", + "X-TIKA:resourceName" : "test.txt" } }, "path" : { diff --git a/docs/source/fscrawler.ini b/docs/source/fscrawler.ini index beabba034..70cb9a521 100644 --- a/docs/source/fscrawler.ini +++ b/docs/source/fscrawler.ini @@ -3,7 +3,7 @@ Version=2.10-SNAPSHOT JavaVersion=17 [3rdParty] -TikaVersion=3.3.1 +TikaVersion=4.0.0-beta-1 ElasticsearchVersion7=7.17.29 ElasticsearchVersion8=8.19.5 ElasticsearchVersion9=9.3.0 diff --git a/docs/source/release/2.10.rst b/docs/source/release/2.10.rst index 5022c6387..113e3c761 100644 --- a/docs/source/release/2.10.rst +++ b/docs/source/release/2.10.rst @@ -17,6 +17,21 @@ enough. Thanks to dadoonet. * We don't support anymore the ``elasticsearch.nodes.url`` setting. You need to use ``elasticsearch.urls`` instead. Thanks to dadoonet. * The ``_upload`` REST endpoint has been removed. Please now use the ``_document`` endpoint. Thanks to dadoonet. +* The Apache Tika 4 upgrade changes several ``meta.raw.*`` metadata key names. If you search, aggregate or + map these fields, you will need to update your queries and index templates accordingly: + + * Image and EXIF metadata (JPEG, PNG, TIFF, …) keys are now namespaced under ``img:``. For example + ``Number of Tables`` becomes ``img:Number of Tables`` and ``Exif IFD0:Orientation`` becomes + ``img:Exif IFD0:Orientation``. + * The resource name key ``resourceName`` is renamed to ``X-TIKA:resourceName``. + * New keys are now produced, such as ``Content-Type-Magic-Detected``, ``X-TIKA:Parsed-By-Full-Set`` and, + for text documents, the encoding detection keys ``X-TIKA:detectedEncoding``, + ``X-TIKA:encodingDetectionTrace`` and ``X-TIKA:encodingDetector``. + + Thanks to dadoonet. +* The default ``fs.ocr.pdf_strategy`` is now ``auto`` instead of ``ocr_and_text``. With ``auto``, OCR is skipped on + PDF pages that already contain more than 10 characters of text. If you relied on the previous behaviour and want + OCR on every PDF page, explicitly set ``fs.ocr.pdf_strategy`` to ``ocr_and_text``. Thanks to dadoonet. New --- @@ -38,6 +53,8 @@ New * You can now remove a document in Elasticsearch using FSCrawler ``_document`` endpoint. See :ref:`rest-service`. Thanks to dadoonet. * Implement our own HTTP Client for Elasticsearch. Thanks to dadoonet. * Add option to set path to custom tika config file. See :ref:`local-fs-settings`. Thanks to iadcode. + **Note**: this option is temporarily disabled after the Apache Tika 4 upgrade, which removed the XML-based + configuration file mechanism. It is expected to return on top of Tika 4's JSON-based configuration. * Support for Index Templates. See :ref:`mappings`. Thanks to dadoonet. * Support for Aliases. You can now index to an alias. Thanks to dadoonet. * Support for Access Token and Api Keys instead of Basic Authentication. See :ref:`credentials`. Thanks to dadoonet. diff --git a/docs/source/user/ocr.rst b/docs/source/user/ocr.rst index 83e65f5ff..fa564c232 100644 --- a/docs/source/user/ocr.rst +++ b/docs/source/user/ocr.rst @@ -30,7 +30,7 @@ Here is a list of OCR settings (under ``fs.ocr`` prefix): +---------------------------------------+-------------------------------------------------+------------------+-----------------------------------+ | ``fs.ocr.output_type`` | ``FSCRAWLER_FS_OCR_OUTPUT_TYPE`` | ``txt`` | `OCR Output Type`_ | +---------------------------------------+-------------------------------------------------+------------------+-----------------------------------+ -| ``fs.ocr.pdf_strategy`` | ``FSCRAWLER_FS_OCR_PDF_STRATEGY`` | ``ocr_and_text`` | `OCR PDF Strategy`_ | +| ``fs.ocr.pdf_strategy`` | ``FSCRAWLER_FS_OCR_PDF_STRATEGY`` | ``auto`` | `OCR PDF Strategy`_ | +---------------------------------------+-------------------------------------------------+------------------+-----------------------------------+ | ``fs.ocr.page_seg_mode`` | ``FSCRAWLER_FS_OCR_PAGE_SEG_MODE`` | ``01`` | `OCR Page Seg Mode`_ | +---------------------------------------+-------------------------------------------------+------------------+-----------------------------------+ @@ -165,8 +165,8 @@ Supported strategies are: * ``ocr_and_text``: OCR and text extraction is performed. -.. note:: When omitted, ``ocr_and_text`` value is used. If you have performance issues, it's worth using the ``auto`` option - instead as only documents with barely no text will go through the OCR process. +.. note:: When omitted, ``auto`` value is used. OCR is skipped on PDF pages that already contain more than + 10 characters of text. If you need OCR on every page regardless, set ``ocr_and_text`` or ``ocr_only``. OCR Page Seg Mode ----------------- diff --git a/elasticsearch-client/src/test/resources/log4j2.xml b/elasticsearch-client/src/test/resources/log4j2.xml index aea0bc66b..d30e01bce 100644 --- a/elasticsearch-client/src/test/resources/log4j2.xml +++ b/elasticsearch-client/src/test/resources/log4j2.xml @@ -9,7 +9,7 @@ - + diff --git a/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerImplAllDocumentsIT.java b/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerImplAllDocumentsIT.java index c720136d0..4ede6f4b6 100644 --- a/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerImplAllDocumentsIT.java +++ b/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerImplAllDocumentsIT.java @@ -43,13 +43,14 @@ import java.nio.file.Path; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.tika.parser.external.ExternalParser; +import org.apache.tika.exception.TikaConfigException; +import org.apache.tika.parser.ocr.TesseractOCRParser; import org.assertj.core.api.Assertions; -import org.assertj.core.api.Assumptions; import org.assertj.core.api.InstanceOfAssertFactories; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledIf; /** Test all type of documents we have */ @VerySlow @@ -253,14 +254,28 @@ void chineseContent369() throws ElasticsearchClientException { } @Test + @DisabledIf(value = "tesseractNotInstalled", disabledReason = "Tesseract is not installed") void ocr() throws ElasticsearchClientException { - Assumptions.assumeThat(ExternalParser.check("tesseract")) - .as("Tesseract is not installed so we are skipping this test") - .isTrue(); runSearch("test-ocr.png", "words"); runSearch("test-ocr.pdf", "words"); } + /** + * Condition for {@link #ocr()}. Tika 4 removed {@code ExternalParser.check()}, so we reuse the same Tesseract + * availability check as production code. Returns {@code true} (disabling the test) when Tesseract is not installed + * or cannot be configured. + * + * @return {@code true} when Tesseract is unavailable + */ + @SuppressWarnings("unused") + static boolean tesseractNotInstalled() { + try { + return !new TesseractOCRParser().hasTesseract(); + } catch (TikaConfigException e) { + return true; + } + } + @Test void shiftJisEncoding() throws ElasticsearchClientException { runSearch("issue-400-shiftjis.txt", "elasticsearch"); diff --git a/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerRestIT.java b/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerRestIT.java index 3a5ee72ce..613d4a7f7 100644 --- a/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerRestIT.java +++ b/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerRestIT.java @@ -372,7 +372,7 @@ void documentWithExternalTags() throws Exception { checkDocument("replace_meta_only.txt", hit -> { Assertions.assertThat((String) JsonPath.read(hit.getSource(), "$.content")) .contains("This file content will be extracted"); - Assertions.assertThat((String) JsonPath.read(hit.getSource(), "$.meta.raw.resourceName")) + Assertions.assertThat((String) JsonPath.read(hit.getSource(), "$.meta.raw['X-TIKA:resourceName']")) .isEqualTo("another-file-name.txt"); Assertions.assertThatThrownBy(() -> JsonPath.read(hit.getSource(), "$.external")) .isInstanceOf(PathNotFoundException.class); diff --git a/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestOcrIT.java b/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestOcrIT.java index 760774b7b..c0aa2606b 100644 --- a/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestOcrIT.java +++ b/integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestOcrIT.java @@ -149,29 +149,9 @@ void ocr_disabled() throws Exception { Map raw = JsonPath.read(jpgHitSource, "$.meta.raw"); Assertions.assertThat(raw) .as("test-ocr.jpg: raw metadata") - .hasSize(64) - .containsKey("Component 1") - .containsKey("Component 2") - .containsKey("Component 3") - .containsEntry("Compression Type", "Baseline") + .hasSize(66) .containsEntry("Content-Type", "image/jpeg") - .containsEntry("Data Precision", "8 bits") - .containsEntry("Exif IFD0:Orientation", "Top, left side (Horizontal / normal)") - .containsEntry("Exif IFD0:Resolution Unit", "Inch") - .containsKey("Exif IFD0:X Resolution") - .containsKey("Exif IFD0:Y Resolution") - .containsEntry("Exif IFD0:YCbCr Positioning", "Center of pixel array") - .containsEntry("Exif SubIFD:Color Space", "sRGB") - .containsEntry("Exif SubIFD:Components Configuration", "YCbCr") - .containsEntry("Exif SubIFD:Exif Image Height", "622 pixels") - .containsEntry("Exif SubIFD:Exif Image Width", "982 pixels") - .containsEntry("Exif SubIFD:Exif Version", "2.21") - .containsEntry("Exif SubIFD:FlashPix Version", "1.00") - .containsEntry("Exif SubIFD:Scene Capture Type", "Standard") - .containsKey("File Modified Date") - .hasEntrySatisfying( - "File Name", value -> Assertions.assertThat(value).startsWith("apache-tika-")) - .containsEntry("File Size", "41426 bytes") + .containsEntry("Content-Type-Magic-Detected", "image/jpeg") .containsKey("ICC:Apple Multi-language Profile Name") .containsKey("ICC:Blue Colorant") .containsKey("ICC:Blue Parametric TRC") @@ -201,11 +181,31 @@ void ocr_disabled() throws Exception { .containsKey("ICC:Version") .containsKey("ICC:Video Card Gamma") .containsKey("ICC:XYZ values") - .containsEntry("Image Height", "622 pixels") - .containsEntry("Image Width", "982 pixels") - .containsEntry("Number of Components", "3") - .containsEntry("Number of Tables", "4 Huffman tables") - .containsEntry("resourceName", "test-ocr.jpg") + .containsKey("img:Component 1") + .containsKey("img:Component 2") + .containsKey("img:Component 3") + .containsEntry("img:Compression Type", "Baseline") + .containsEntry("img:Data Precision", "8 bits") + .containsEntry("img:Exif IFD0:Orientation", "Top, left side (Horizontal / normal)") + .containsEntry("img:Exif IFD0:Resolution Unit", "Inch") + .containsKey("img:Exif IFD0:X Resolution") + .containsKey("img:Exif IFD0:Y Resolution") + .containsEntry("img:Exif IFD0:YCbCr Positioning", "Center of pixel array") + .containsEntry("img:Exif SubIFD:Color Space", "sRGB") + .containsEntry("img:Exif SubIFD:Components Configuration", "YCbCr") + .containsEntry("img:Exif SubIFD:Exif Image Height", "622 pixels") + .containsEntry("img:Exif SubIFD:Exif Image Width", "982 pixels") + .containsEntry("img:Exif SubIFD:Exif Version", "2.21") + .containsEntry("img:Exif SubIFD:FlashPix Version", "1.00") + .containsEntry("img:Exif SubIFD:Scene Capture Type", "Standard") + .containsKey("img:File Modified Date") + .hasEntrySatisfying( + "img:File Name", value -> Assertions.assertThat(value).startsWith("apache-tika-")) + .containsEntry("img:File Size", "41426 bytes") + .containsEntry("img:Image Height", "622 pixels") + .containsEntry("img:Image Width", "982 pixels") + .containsEntry("img:Number of Components", "3") + .containsEntry("img:Number of Tables", "4 Huffman tables") .containsEntry("tiff:BitsPerSample", "8") .containsEntry("tiff:ImageLength", "622") .containsEntry("tiff:ImageWidth", "982") @@ -214,7 +214,8 @@ void ocr_disabled() throws Exception { .containsEntry("tiff:XResolution", "144.0") .containsEntry("tiff:YResolution", "144.0") .containsKey("X-TIKA:Parsed-By") - .containsKey("X-TIKA:Parsed-By-Full-Set"); + .containsKey("X-TIKA:Parsed-By-Full-Set") + .containsEntry("X-TIKA:resourceName", "test-ocr.jpg"); // test-ocr.png: OCR disabled → no content String pngHitSource = findHitByFilename(searchResponse, "test-ocr.png").getSource(); diff --git a/integration-tests/src/test/resources-binary/tags/document_with_external_tags/replace_meta_only.txt.json b/integration-tests/src/test/resources-binary/tags/document_with_external_tags/replace_meta_only.txt.json index 617442d5d..4548718e1 100644 --- a/integration-tests/src/test/resources-binary/tags/document_with_external_tags/replace_meta_only.txt.json +++ b/integration-tests/src/test/resources-binary/tags/document_with_external_tags/replace_meta_only.txt.json @@ -1,7 +1,7 @@ { "meta":{ "raw":{ - "resourceName":"another-file-name.txt" + "X-TIKA:resourceName":"another-file-name.txt" } } } diff --git a/pom.xml b/pom.xml index aa5b142c0..c4ad8f8c0 100644 --- a/pom.xml +++ b/pom.xml @@ -69,7 +69,7 @@ 0.38.0 2.6 - 3.3.1 + 4.0.0-beta-1 3.1.4