diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index f249eec..da5d73d 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10", "3.11"] # , "3.12", "3.13"] + python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - name: Checkout uses: actions/checkout@v7 @@ -29,6 +29,6 @@ jobs: - name: Install dependencies run: >- python -m - pip install . pytest pytest-mock ja-ginza ja-ginza-electra + pip install . pytest pytest-mock ja-ginza ja-ginza-electra ja-ginza-bert-large - name: Run Tests run: pytest diff --git a/README.md b/README.md index fd7a087..e33806a 100644 --- a/README.md +++ b/README.md @@ -5,112 +5,93 @@   [![Downloads](https://pepy.tech/badge/ginza/week)](https://pepy.tech/project/ginza) -An Open Source Japanese NLP Library, based on Universal Dependencies - -***Please read the [Important changes](#ginza-520) before you upgrade GiNZA.*** - [日本語ページはこちら](https://megagonlabs.github.io/ginza/) -## License -GiNZA NLP Library and GiNZA Japanese Universal Dependencies Models are distributed under the -[MIT License](https://github.com/megagonlabs/ginza/blob/master/LICENSE). -You must agree and follow the MIT License to use GiNZA NLP Library and GiNZA Japanese Universal Dependencies Models. - -### Explosion / spaCy -spaCy is the key framework of GiNZA. - -[spaCy LICENSE PAGE](https://github.com/explosion/spaCy/blob/master/LICENSE) - -### Works Applications Enterprise / Sudachi/SudachiPy - SudachiDict - chiVe -SudachiPy provides high accuracies for tokenization and pos tagging. - -[Sudachi LICENSE PAGE](https://github.com/WorksApplications/Sudachi/blob/develop/LICENSE-2.0.txt), -[SudachiPy LICENSE PAGE](https://github.com/WorksApplications/SudachiPy/blob/develop/LICENSE), -[SudachiDict LEGAL PAGE](https://github.com/WorksApplications/SudachiDict/blob/develop/LEGAL), -[chiVe LICENSE PAGE](https://github.com/WorksApplications/chiVe/blob/master/LICENSE) - -### Hugging Face / transformers -The GiNZA v5 Transformers model (ja_ginza_electra) is trained by using Hugging Face Transformers as a framework for pretrained models. - -[transformers LICENSE PAGE](https://github.com/huggingface/transformers/blob/master/LICENSE) - -## Training Datasets - -### UD Japanese BCCWJ r2.8 -The parsing model of GiNZA v5 is trained on a part of -[UD Japanese BCCWJ](https://github.com/UniversalDependencies/UD_Japanese-BCCWJ) r2.8 -([Omura and Asahara:2018](https://www.aclweb.org/anthology/W18-6014/)). -This model is developed by National Institute for Japanese Language and Linguistics, and Megagon Labs. +An Open Source Japanese NLP Library, based on Universal Dependencies -### GSK2014-A (2019) BCCWJ edition -The named entity recognition model of GiNZA v5 is trained on a part of -[GSK2014-A](https://www.gsk.or.jp/catalog/gsk2014-a/) (2019) BCCWJ edition -([Hashimoto, Inui, and Murakami:2008](https://www.anlp.jp/proceedings/annual_meeting/2010/pdf_dir/C4-4.pdf)). -We use two of the named entity label systems, both -[Sekine's Extended Named Entity Hierarchy](http://liat-aip.sakura.ne.jp/ene/ene8/definition_jp/html/enedetail.html) -and extended [OntoNotes5](https://catalog.ldc.upenn.edu/docs/LDC2013T19/OntoNotes-Release-5.0.pdf). -This model is developed by National Institute for Japanese Language and Linguistics, and Megagon Labs. +***Please read the [Breaking Changes](#ginza-530) before you upgrade GiNZA.*** -### mC4 -The GiNZA v5 Transformers model (ja_ginza_electra) is trained by using [transformers-ud-japanese-electra-base-discriminator](https://huggingface.co/megagonlabs/transformers-ud-japanese-electra-base-discriminator) which is pretrained on more than 200 million Japanese sentences extracted from [mC4](https://huggingface.co/datasets/mc4). +The GiNZA parsing models are released as part of the results of a joint research project between Recruit Co., Ltd. and the National Institute for Japanese Language and Linguistics. -Contains information from mC4 which is made available under the ODC Attribution License. -``` -@article{2019t5, - author = {Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu}, - title = {Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer}, - journal = {arXiv e-prints}, - year = {2019}, - archivePrefix = {arXiv}, - eprint = {1910.10683}, -} -``` +Please also read the English documentation. +- [GiNZA Version 4.0: Improving Syntactic Structure Analysis Through Japanese Bunsetsu-Phrase Extraction API Integration](https://megagon.ai/ginza-version-4-0-improving-syntactic-structure-analysis-through-japanese-bunsetsu-phrase-extraction-api-integration/) - Megagon Labs Blog (2021.03) ## Runtime Environment -This project is developed with Python>=3.8 and pip for it. +From GiNZA v5.3.0, the runtime environment has changed to Python 3.10 or later. +We recommend using Python 3.12 because of simpler dependency installation process and higher throughput. We do not recommend to use Anaconda environment because the pip install step may not work properly. Please also see the Development Environment section below. + ### Runtime set up +> [!NOTE] +> The GiNZA dependency parsing can be significantly accelerated with a GPU. +> The hardware acceleration is enabled by default in Mac OS environments running on Apple Silicon. +> For details, see [3. Enabling the GPU](#3.-Enabling-the-GPU)。 + #### 1. Install GiNZA NLP Library with Transformer-based Model -Uninstall previous version of ginza and ja_ginza_electra packages: + +> Running the Transformers model requires at least 16GB of memory. +> If you have insufficient memory, please try the standard model described below. + +Install the Transformers model (`ja_ginza_electra` or `ja_ginza_bert_large`) by running one of the following commands: +(This will also install GiNZA and related Transformers libraries.) ```console -$ pip uninstall ginza ja_ginza_electra +$ pip install -U ja_ginza_electra ``` -Then, install the latest version of `ginza` and `ja_ginza_electra`: ```console -$ pip install -U ginza ja_ginza_electra +$ pip install -U ja_ginza_bert_large ``` -The package of `ja_ginza_electra` does not include `pytorch_model.bin` due to PyPI's archive size restrictions. -This large model file will be automatically downloaded at the first run time, and the locally cached file will be used for subsequent runs. +> [!NOTE] +> The model package installed by the above command does not include large transformer models or tokenizers. +> These large files are automatically downloaded from Hugging Face Hub on the first run, and the locally cached files are used for subsequent runs. +> The first time you run transformers model, it will take several seconds to tens of seconds to start up due to the initialization process. -If you need to install `ja_ginza_electra` along with `pytorch_model.bin` at the install time, you can specify direct link for GitHub release archive as follows: -```console -$ pip install -U ginza https://github.com/megagonlabs/ginza/releases/download/latest/ja_ginza_electra-latest-with-model.tar.gz -``` +#### 2. Install GiNZA NLP Library with Standard Model -If you hope to accelerate the transformers-based models by using GPUs with CUDA support, you can install `spacy` by specifying the CUDA version as follows: +Run the following command to install the standard model `ja_ginza`. +(GiNZA-related libraries will also be installed at the same time.) ```console -pip install -U "spacy[cuda117]" +$ pip install -U ja_ginza ``` -And you need to install a version of pytorch that is consistent with the CUDA version. - -#### 2. Install GiNZA NLP Library with Standard Model -Uninstall previous version: +#### 3. Enabling the GPU + +On Mac OS environments running on Apple Silicon, the hardware acceleration is automatically enabled under the following conditions: +- `ja_ginza` + - Acceleration by `thinc-apple-ops` + - Supports Python 3.10 to 3.12 (`thinc-apple-ops` is not supported in 3.13 and later) + - The throughput improvement effect from enabling `thinc-apple-ops` on MacBook Pro M4 Max 128GB is 2.6 times for Python 3.10-3.11 and 5.2 times for Python 3.12 in `ja_ginza`. +- `ja_ginza_electra` and `ja_ginza_large_bert` + - Acceleration by `torch.backends.mps` + - Supports Python 3.10 to 3.13 (some compilation environments required to build dependent libraries in Python 3.14) + - The throughput improvement effect from enabling `torch.backends.mps` on MacBook Pro M4 Max 128GB is 1.3 times for Python 3.10-13 in `ja_ginza_bert_large`. + +To enable NVIDIA GPU acceleration in a Linux OS environment, install CUDA on Linux, add the path to the CUDA libraries to the environment variable `LD_LIBRARY_PATH` like `export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH`, and then install the `ginza` package specifying the CUDA version in extras as follows: +- CUDA 11.x + - `$ pip install ginza[cuda11x]` +- CUDA 12.x + - `$ pip install ginza[cuda12x]` +- CUDA 13.x + - `$ pip install ginza[cuda13x]` + +The `ginza` command enables GPU acceleration by default if a GPU is available. + +The `-g` option of the `ginza` command allows you to specify the device number of the GPU to use (specifying `-1` disables GPU acceleration). ```console -$ pip uninstall ginza ja_ginza +$ ginza -g 0 ``` -Then, install the latest version of `ginza` and `ja_ginza`: +When running the `ginza` command without the `-g` option, the following init log is output if GPU acceleration is enabled: ```console -$ pip install -U ginza ja_ginza +$ ginza +GPU #0 enabled ``` - -When using Apple Silicon such as M1 or M2, you can accelerate the analysis process by installing `thinc-apple-ops`: +When executing standard model in Mac OS running on Apple Silicon, if hardware acceleration is enabled, the init log will be: ```console -$ pip install torch thinc-apple-ops +$ ginza +thinc-apple-ops enabled ``` ### Execute ginza command @@ -235,10 +216,96 @@ Please read the official documents to compile user dictionaries with `sudachipy` [SudachiPy - User defined Dictionary](https://github.com/WorksApplications/SudachiPy#user-defined-dictionary) [Sudachi User Dictionary Construction (Japanese Only)](https://github.com/WorksApplications/Sudachi/blob/develop/docs/user_dict.md) +## License +GiNZA NLP Library and GiNZA Japanese Universal Dependencies Models are distributed under the +[MIT License](https://github.com/megagonlabs/ginza/blob/master/LICENSE). +You must agree and follow the MIT License to use GiNZA NLP Library and GiNZA Japanese Universal Dependencies Models. + +### Explosion / spaCy +spaCy is the key framework of GiNZA. + +[spaCy LICENSE PAGE](https://github.com/explosion/spaCy/blob/master/LICENSE) + +### Works Applications Enterprise / Sudachi/SudachiPy - SudachiDict - chiVe +SudachiPy provides high accuracies for tokenization and pos tagging. + +[Sudachi LICENSE PAGE](https://github.com/WorksApplications/Sudachi/blob/develop/LICENSE-2.0.txt), +[SudachiPy LICENSE PAGE](https://github.com/WorksApplications/SudachiPy/blob/develop/LICENSE), +[SudachiDict LEGAL PAGE](https://github.com/WorksApplications/SudachiDict/blob/develop/LEGAL), +[chiVe LICENSE PAGE](https://github.com/WorksApplications/chiVe/blob/master/LICENSE) + +### Hugging Face / transformers +The GiNZA v5 transformer models (`ja_ginza_electra` and `ja_ginza_bert_large`) use Hugging Face Transformers as inference framework. + +[transformers LICENSE PAGE](https://github.com/huggingface/transformers/blob/master/LICENSE) + +## Training Datasets + +### UD Japanese BCCWJ r2.8 +The parsing model of GiNZA v5 is trained on a part of +[UD Japanese BCCWJ](https://github.com/UniversalDependencies/UD_Japanese-BCCWJ) r2.8 +([Omura and Asahara:2018](https://www.aclweb.org/anthology/W18-6014/)). +``` +@inproceedings{omura-asahara-2018-ud, + title = "{UD}-{J}apanese {BCCWJ}: {U}niversal {D}ependencies Annotation for the {B}alanced {C}orpus of {C}ontemporary {W}ritten {J}apanese", + author = "Omura, Mai and + Asahara, Masayuki", + booktitle = "Proceedings of the Second Workshop on Universal Dependencies ({UDW} 2018)", + month = nov, + year = "2018", + address = "Brussels, Belgium", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/W18-6014/", + doi = "10.18653/v1/W18-6014", + pages = "117--125" +} +``` + +### GSK2014-A (2019) BCCWJ edition +The named entity recognition model of GiNZA v5 is trained on a part of +[GSK2014-A](https://www.gsk.or.jp/catalog/gsk2014-a/) (2019) BCCWJ edition +([Hashimoto, Inui, and Murakami:2008](https://www.anlp.jp/proceedings/annual_meeting/2010/pdf_dir/C4-4.pdf)). +We use two of the named entity label systems, both +[Sekine's Extended Named Entity Hierarchy](http://liat-aip.sakura.ne.jp/ene/ene8/definition_jp/html/enedetail.html) +and extended [OntoNotes5](https://catalog.ldc.upenn.edu/docs/LDC2013T19/OntoNotes-Release-5.0.pdf). +This model is developed by National Institute for Japanese Language and Linguistics, and Megagon Labs. + +### mC4 +`ja_ginza_electra` is a fine-tuned model of [transformers-ud-japanese-electra-base-discriminator](https://huggingface.co/megagonlabs/transformers-ud-japanese-electra-base-discriminator) which is pretrained on more than 200 million Japanese sentences extracted from [mC4](https://huggingface.co/datasets/mc4). + +The mC4 is published under the ODC Attribution License. +``` +@article{2019t5, + author = {Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu}, + title = {Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer}, + journal = {arXiv e-prints}, + year = {2019}, + archivePrefix = {arXiv}, + eprint = {1910.10683}, +} +``` + ## Releases ### version 5.x +#### ginza-5.3.0 +- 2026-09-30, Grossular Garnet +- Breaking Changes + - We changed the supported Python version to 3.10 or later, and the supported spaCy version to 3.8.16 or later. + - We recommend using Python 3.12 because of simpler dependency installation process and higher throughput. + - Some compilation environments required to build dependent libraries in Python 3.14. + - The model package loading priority was changed to `ja_ginza_bert_large`, `ja_ginza_electra`, and `ja_ginza`. + - `ginza` command enables GPU acceleration by default under certain conditions. + - GPU acceleration can be disabled with `ginza -g -1`. +- New Features + - Enabling hardware acceleration by default in Mac OS environments with Apple Silicon + - The Python versions that can be used with `thinc-apple-ops` on Mac OS running Apple Silicon are 3.10 through 3.12. + - Official release of `ja_ginza_bert_large` + - [`ginza-transformers`](https://github.com/megagonlabs/ginza-transformers) was upgraded to v1.4.0. + - Changed the transformers component to obtain both model and tokenizer from Hugging Face Hub. + - [`spacy-transformers`](https://github.com/explosion/spacy-transformers) has a requirement of `torch>=1.8.0` and `transformers<4.53.3`. + #### ginza-5.2.1 - 2026-09-01 - This release is the final version that will work with Python 3.9 or earlier. @@ -254,7 +321,7 @@ Please read the official documents to compile user dictionaries with `sudachipy` - [#266](https://github.com/megagonlabs/ginza/pull/266) - [#273](https://github.com/megagonlabs/ginza/pull/273) #### ginza-5.2.0 -- 2024-03-31 +- 2024-03-31, Fluorite - Require python>=3.8 - Migrate to spaCy v3.7 - New functionality @@ -571,8 +638,8 @@ $ python -m spacy train ja ja_ginza-4.0.0 corpus/ja_ginza-ud-train.json corpus/j ``` ## Run tests -Ginza uses the pytest framework for testing, and you can run the tests via `setup.py` without install test requirements explicitly. -Some tests depends on the ginza default models (`ja-ginza`, `ja-ginza-electra`), so install them before the tests is needed. +GiNZA uses the pytest framework for testing, and you can run the tests via `setup.py` without install test requirements explicitly. +Some tests depends on the ginza parsing models, so install them before the tests is needed. ```console $ pip install ja-ginza ja-ginza-electra diff --git a/config/ja_ginza.cfg b/config/ja_ginza.cfg index ac267ba..b0be8f0 100644 --- a/config/ja_ginza.cfg +++ b/config/ja_ginza.cfg @@ -29,7 +29,7 @@ validate = false [components.bunsetu_recognizer] factory = "bunsetu_recognizer" -remain_bunsetu_suffix = true +remain_bunsetu_suffix = false [components.compound_splitter] factory = "compound_splitter" @@ -197,4 +197,4 @@ after_init = null [initialize.components] -[initialize.tokenizer] \ No newline at end of file +[initialize.tokenizer] diff --git a/config/ja_ginza.meta.json b/config/ja_ginza.meta.json index 1864604..a35435f 100644 --- a/config/ja_ginza.meta.json +++ b/config/ja_ginza.meta.json @@ -1,7 +1,7 @@ { "lang":"ja", "name":"ginza", - "version":"5.2.0", + "version":"5.3.0", "description":"Japanese multi-task CNN trained on UD-Japanese BCCWJ r2.8 + GSK2014-A(2019). Assigns word2vec token vectors. Components: tok2vec, parser, ner, morphologizer, atteribute_ruler, compound_splitter, bunsetu_recognizer.", "author":"Megagon Labs Tokyo.", "email":"ginza@megagon.ai", @@ -34,7 +34,7 @@ } ], "parent_package":"spacy", - "spacy_version":">=3.4.4,<4.0.0", + "spacy_version":">=3.8.16,<4.0.0", "spacy_git_version":"0fc3dee77", "vectors":{ "width":300, @@ -64,8 +64,1391 @@ "attribute_ruler" ], "requirements":[ - "sudachipy>=0.6.2,<0.7.0", + "sudachipy>=0.6.11,<0.7.0", "sudachidict_core>=20210802", - "ginza>=5.2.0,<5.3.0" - ] + "ginza>=5.3.0,<6.0.0" + ], + "labels":{ + "tok2vec":[ + + ], + "parser":[ + "ROOT", + "acl", + "acl_bunsetu", + "advcl", + "advcl_bunsetu", + "advmod", + "advmod_bunsetu", + "amod", + "amod_bunsetu", + "aux", + "aux_bunsetu", + "case", + "case_bunsetu", + "cc", + "cc_bunsetu", + "ccomp_bunsetu", + "compound", + "compound_bunsetu", + "cop", + "csubj_bunsetu", + "dep", + "dep_bunsetu", + "det_bunsetu", + "discourse_bunsetu", + "dislocated_bunsetu", + "fixed", + "mark", + "nmod", + "nmod_bunsetu", + "nsubj_bunsetu", + "nummod", + "obj_bunsetu", + "obl_bunsetu", + "punct", + "punct_bunsetu" + ], + "attribute_ruler":[ + + ], + "ner":[ + "Academic", + "Age", + "Aircraft", + "Airport", + "Amphibia", + "Amusement_Park", + "Animal_Disease", + "Animal_Part", + "Archaeological_Place_Other", + "Art_Other", + "Astral_Body_Other", + "Award", + "Bay", + "Bird", + "Book", + "Bridge", + "Broadcast_Program", + "Cabinet", + "Calorie", + "Canal", + "Car", + "Car_Stop", + "Character", + "City", + "Class", + "Clothing", + "Color_Other", + "Company", + "Company_Group", + "Compound", + "Conference", + "Constellation", + "Continental_Region", + "Corporation_Other", + "Country", + "Countx_Other", + "County", + "Culture", + "Date", + "Day_Of_Week", + "Disease_Other", + "Dish", + "Doctrine_Method_Other", + "Domestic_Region", + "Drug", + "Earthquake", + "Element", + "Email", + "Era", + "Ethnic_Group_Other", + "Event_Other", + "Facility_Other", + "Facility_Part", + "Family", + "Fish", + "Flora", + "Flora_Part", + "Food_Other", + "Frequency", + "Fungus", + "GOE_Other", + "GPE_Other", + "Game", + "Geological_Region_Other", + "God", + "Government", + "ID_Number", + "Incident_Other", + "Insect", + "Intensity", + "International_Organization", + "Island", + "Lake", + "Language_Other", + "Latitude_Longtitude", + "Law", + "Line_Other", + "Living_Thing_Other", + "Living_Thing_Part_Other", + "Location_Other", + "Magazine", + "Mammal", + "Material", + "Measurement_Other", + "Military", + "Mineral", + "Mollusc_Arthropod", + "Money", + "Money_Form", + "Mountain", + "Movement", + "Movie", + "Multiplication", + "Museum", + "Music", + "N_Animal", + "N_Country", + "N_Event", + "N_Facility", + "N_Flora", + "N_Location_Other", + "N_Natural_Object_Other", + "N_Organization", + "N_Person", + "N_Product", + "Name_Other", + "National_Language", + "Nationality", + "Natural_Disaster", + "Natural_Object_Other", + "Natural_Phenomenon_Other", + "Nature_Color", + "Newspaper", + "Numex_Other", + "Occasion_Other", + "Offense", + "Ordinal_Number", + "Organization_Other", + "Park", + "Percent", + "Period_Day", + "Period_Month", + "Period_Time", + "Period_Week", + "Period_Year", + "Periodx_Other", + "Person", + "Phone_Number", + "Physical_Extent", + "Plan", + "Planet", + "Point", + "Political_Organization_Other", + "Political_Party", + "Port", + "Position_Vocation", + "Postal_Address", + "Printing_Other", + "Pro_Sports_Organization", + "Product_Other", + "Province", + "Public_Institution", + "Railroad", + "Rank", + "Region_Other", + "Religion", + "Religious_Festival", + "Reptile", + "Research_Institute", + "River", + "Road", + "Rule_Other", + "School", + "School_Age", + "Sea", + "Ship", + "Show", + "Show_Organization", + "Spa", + "Space", + "Spaceship", + "Speed", + "Sport", + "Sports_Facility", + "Sports_League", + "Sports_Organization_Other", + "Station", + "Style", + "Temperature", + "Theater", + "Theory", + "Time", + "Time_Top_Other", + "Timex_Other", + "Title_Other", + "Train", + "Treaty", + "Tumulus", + "Tunnel", + "URL", + "Unit_Other", + "Vehicle_Other", + "Volume", + "War", + "Water_Route", + "Weapon", + "Weight", + "Worship_Place", + "Zoo" + ], + "morphologizer":[ + "POS=PUNCT", + "POS=NUM", + "POS=NOUN", + "POS=ADP", + "POS=AUX", + "POS=VERB", + "POS=CCONJ", + "POS=PART", + "POS=SCONJ", + "POS=SYM", + "POS=ADJ", + "POS=DET", + "POS=PRON", + "POS=PROPN", + "POS=ADV", + "POS=X", + "POS=INTJ" + ] + }, + "performance":{ + "dep_uas":0.9094703175, + "dep_las":0.8906500624, + "dep_las_per_type":{ + "compound":{ + "p":0.8789917474, + "r":0.8754969846, + "f":0.8772408854 + }, + "nummod":{ + "p":0.9176536943, + "r":0.9297142857, + "f":0.9236446211 + }, + "dep_bunsetu":{ + "p":0.5923009624, + "r":0.4596062458, + "f":0.5175840979 + }, + "obl_bunsetu":{ + "p":0.7597479993, + "r":0.7881303541, + "f":0.7736789631 + }, + "case":{ + "p":0.9777676951, + "r":0.9813793452, + "f":0.9795701912 + }, + "acl_bunsetu":{ + "p":0.8585409253, + "r":0.8310607148, + "f":0.8445773467 + }, + "nmod_bunsetu":{ + "p":0.8316480124, + "r":0.8108067429, + "f":0.8210951492 + }, + "root":{ + "p":0.8984536082, + "r":0.8987315665, + "f":0.8985925659 + }, + "fixed":{ + "p":0.9814903616, + "r":0.9711520213, + "f":0.976293823 + }, + "obj_bunsetu":{ + "p":0.929671666, + "r":0.9304259635, + "f":0.9300486618 + }, + "advcl_bunsetu":{ + "p":0.7171969968, + "r":0.7246237207, + "f":0.7208912314 + }, + "aux":{ + "p":0.9682417647, + "r":0.9665840123, + "f":0.9674121783 + }, + "nsubj_bunsetu":{ + "p":0.8207343413, + "r":0.8131603584, + "f":0.8169297951 + }, + "cc_bunsetu":{ + "p":0.762728146, + "r":0.7338262477, + "f":0.7479981159 + }, + "det_bunsetu":{ + "p":0.94644506, + "r":0.9490740741, + "f":0.9477577439 + }, + "advmod_bunsetu":{ + "p":0.7511450382, + "r":0.7472660996, + "f":0.7492005482 + }, + "cop":{ + "p":0.9463980881, + "r":0.9070680628, + "f":0.9263157895 + }, + "amod":{ + "p":0.6976744186, + "r":0.7114624506, + "f":0.7045009785 + }, + "dep":{ + "p":0.5112923463, + "r":0.6791666667, + "f":0.583392985 + }, + "nmod":{ + "p":0.8125, + "r":0.8227848101, + "f":0.8176100629 + }, + "mark":{ + "p":0.9562554681, + "r":0.9594605379, + "f":0.9578553219 + }, + "cc":{ + "p":0.7894736842, + "r":0.6, + "f":0.6818181818 + }, + "acl":{ + "p":1.0, + "r":0.625, + "f":0.7692307692 + }, + "compound_bunsetu":{ + "p":0.3410852713, + "r":0.2056074766, + "f":0.2565597668 + }, + "advcl":{ + "p":0.7407407407, + "r":0.7659574468, + "f":0.7531380753 + }, + "dislocated_bunsetu":{ + "p":0.5149700599, + "r":0.5341614907, + "f":0.5243902439 + }, + "ccomp_bunsetu":{ + "p":0.9301310044, + "r":0.8469184891, + "f":0.8865764828 + }, + "advmod":{ + "p":0.4523809524, + "r":0.3275862069, + "f":0.38 + }, + "csubj_bunsetu":{ + "p":0.786407767, + "r":0.7264573991, + "f":0.7552447552 + }, + "amod_bunsetu":{ + "p":0.9341317365, + "r":0.901734104, + "f":0.9176470588 + }, + "punct_bunsetu":{ + "p":0.6527777778, + "r":0.324137931, + "f":0.4331797235 + }, + "discourse_bunsetu":{ + "p":0.7083333333, + "r":0.2615384615, + "f":0.3820224719 + }, + "case_bunsetu":{ + "p":0.25, + "r":0.1538461538, + "f":0.1904761905 + }, + "aux_bunsetu":{ + "p":0.6666666667, + "r":0.1333333333, + "f":0.2222222222 + }, + "nummod_bunsetu":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "discourse":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "det":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "reparandum":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "reparandum_bunsetu":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "cop_bunsetu":{ + "p":0.0, + "r":0.0, + "f":0.0 + } + }, + "sents_p":0.8302567804, + "sents_r":0.8302567804, + "sents_f":0.8302567804, + "ents_f":0.554029304, + "ents_p":0.58374206, + "ents_r":0.5271948297, + "ents_per_type":{ + "Ordinal_Number":{ + "p":0.7529538131, + "r":0.7644492912, + "f":0.7586580087 + }, + "Nationality":{ + "p":0.7111111111, + "r":0.7804878049, + "f":0.7441860465 + }, + "Date":{ + "p":0.747582205, + "r":0.8052083333, + "f":0.7753259779 + }, + "Government":{ + "p":0.5654761905, + "r":0.5757575758, + "f":0.5705705706 + }, + "Law":{ + "p":0.3703703704, + "r":0.119047619, + "f":0.1801801802 + }, + "Doctrine_Method_Other":{ + "p":0.3051643192, + "r":0.1666666667, + "f":0.2155887231 + }, + "Time":{ + "p":0.5720524017, + "r":0.6064814815, + "f":0.5887640449 + }, + "Company":{ + "p":0.3211382114, + "r":0.3333333333, + "f":0.3271221532 + }, + "Occasion_Other":{ + "p":0.3, + "r":0.1621621622, + "f":0.2105263158 + }, + "Product_Other":{ + "p":0.325, + "r":0.2596291013, + "f":0.2886597938 + }, + "Money":{ + "p":0.6923076923, + "r":0.6428571429, + "f":0.6666666667 + }, + "Movement":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Plan":{ + "p":0.4, + "r":0.2556390977, + "f":0.3119266055 + }, + "Period_Time":{ + "p":0.8023255814, + "r":0.6571428571, + "f":0.722513089 + }, + "Public_Institution":{ + "p":0.4285714286, + "r":0.3, + "f":0.3529411765 + }, + "National_Language":{ + "p":0.8055555556, + "r":0.8787878788, + "f":0.8405797101 + }, + "Corporation_Other":{ + "p":0.2777777778, + "r":0.2325581395, + "f":0.253164557 + }, + "Natural_Disaster":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Position_Vocation":{ + "p":0.5828729282, + "r":0.5255292653, + "f":0.5527177472 + }, + "Organization_Other":{ + "p":0.2878787879, + "r":0.2602739726, + "f":0.273381295 + }, + "Country":{ + "p":0.8246445498, + "r":0.807424594, + "f":0.815943728 + }, + "Event_Other":{ + "p":0.4848484848, + "r":0.253968254, + "f":0.3333333333 + }, + "Political_Organization_Other":{ + "p":0.24, + "r":0.375, + "f":0.2926829268 + }, + "Incident_Other":{ + "p":0.2857142857, + "r":0.25, + "f":0.2666666667 + }, + "War":{ + "p":0.5, + "r":0.4, + "f":0.4444444444 + }, + "Military":{ + "p":0.25, + "r":0.1, + "f":0.1428571429 + }, + "International_Organization":{ + "p":0.4172661871, + "r":0.3391812865, + "f":0.3741935484 + }, + "Treaty":{ + "p":0.7636363636, + "r":0.375, + "f":0.502994012 + }, + "Continental_Region":{ + "p":0.5652173913, + "r":0.5652173913, + "f":0.5652173913 + }, + "Person":{ + "p":0.7507352941, + "r":0.6413316583, + "f":0.6917344173 + }, + "N_Person":{ + "p":0.8272425249, + "r":0.8217821782, + "f":0.8245033113 + }, + "Drug":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Conference":{ + "p":0.64, + "r":0.237037037, + "f":0.3459459459 + }, + "Period_Year":{ + "p":0.7639751553, + "r":0.7735849057, + "f":0.76875 + }, + "Earthquake":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "N_Product":{ + "p":0.3745318352, + "r":0.5291005291, + "f":0.4385964912 + }, + "GPE_Other":{ + "p":0.6, + "r":0.1525423729, + "f":0.2432432432 + }, + "N_Country":{ + "p":0.641509434, + "r":0.6666666667, + "f":0.6538461538 + }, + "Pro_Sports_Organization":{ + "p":0.1034482759, + "r":0.5, + "f":0.1714285714 + }, + "City":{ + "p":0.4060150376, + "r":0.5427135678, + "f":0.464516129 + }, + "Percent":{ + "p":0.8699186992, + "r":0.8794520548, + "f":0.8746594005 + }, + "Point":{ + "p":0.2058823529, + "r":0.1944444444, + "f":0.2 + }, + "Countx_Other":{ + "p":0.1145833333, + "r":0.1774193548, + "f":0.1392405063 + }, + "Printing_Other":{ + "p":0.747826087, + "r":0.6935483871, + "f":0.719665272 + }, + "N_Event":{ + "p":0.28, + "r":0.2333333333, + "f":0.2545454545 + }, + "Political_Party":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Fish":{ + "p":0.1, + "r":0.1111111111, + "f":0.1052631579 + }, + "Age":{ + "p":0.6516129032, + "r":0.5804597701, + "f":0.6139817629 + }, + "Physical_Extent":{ + "p":0.3571428571, + "r":0.5128205128, + "f":0.4210526316 + }, + "Broadcast_Program":{ + "p":0.2545454545, + "r":0.1386138614, + "f":0.1794871795 + }, + "Magazine":{ + "p":0.0588235294, + "r":0.0714285714, + "f":0.064516129 + }, + "Aircraft":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Weapon":{ + "p":0.5135135135, + "r":0.2923076923, + "f":0.3725490196 + }, + "Ship":{ + "p":0.1764705882, + "r":0.1304347826, + "f":0.15 + }, + "N_Organization":{ + "p":0.3214285714, + "r":0.1956521739, + "f":0.2432432432 + }, + "Research_Institute":{ + "p":0.8, + "r":0.2352941176, + "f":0.3636363636 + }, + "Weight":{ + "p":0.652173913, + "r":0.5555555556, + "f":0.6 + }, + "Game":{ + "p":0.0714285714, + "r":0.0377358491, + "f":0.049382716 + }, + "Car":{ + "p":0.3225806452, + "r":0.2272727273, + "f":0.2666666667 + }, + "God":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Province":{ + "p":0.4736842105, + "r":0.8076923077, + "f":0.5971563981 + }, + "Mollusc_Arthropod":{ + "p":0.1666666667, + "r":0.0714285714, + "f":0.1 + }, + "Numex_Other":{ + "p":0.2075471698, + "r":0.1803278689, + "f":0.1929824561 + }, + "Facility_Other":{ + "p":0.3333333333, + "r":0.1428571429, + "f":0.2 + }, + "Region_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Timex_Other":{ + "p":0.2, + "r":0.2, + "f":0.2 + }, + "Insect":{ + "p":0.1818181818, + "r":0.1666666667, + "f":0.1739130435 + }, + "Compound":{ + "p":0.55, + "r":0.5, + "f":0.5238095238 + }, + "Material":{ + "p":0.21875, + "r":0.2258064516, + "f":0.2222222222 + }, + "Natural_Phenomenon_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Element":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Rank":{ + "p":0.6593406593, + "r":0.6896551724, + "f":0.6741573034 + }, + "Class":{ + "p":0.2972972973, + "r":0.2682926829, + "f":0.2820512821 + }, + "Offense":{ + "p":0.5434782609, + "r":0.5747126437, + "f":0.5586592179 + }, + "Show_Organization":{ + "p":0.0317460317, + "r":0.064516129, + "f":0.0425531915 + }, + "Period_Month":{ + "p":0.4814814815, + "r":0.8387096774, + "f":0.6117647059 + }, + "School":{ + "p":0.3965517241, + "r":0.2839506173, + "f":0.3309352518 + }, + "School_Age":{ + "p":0.4838709677, + "r":0.4285714286, + "f":0.4545454545 + }, + "N_Facility":{ + "p":0.2666666667, + "r":0.1052631579, + "f":0.1509433962 + }, + "Music":{ + "p":0.0350877193, + "r":0.0526315789, + "f":0.0421052632 + }, + "Title_Other":{ + "p":0.8088737201, + "r":0.7953020134, + "f":0.8020304569 + }, + "Book":{ + "p":0.3037974684, + "r":0.2926829268, + "f":0.298136646 + }, + "Stock":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Name_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Food_Other":{ + "p":0.3333333333, + "r":0.2613636364, + "f":0.2929936306 + }, + "URL":{ + "p":0.8, + "r":0.8, + "f":0.8 + }, + "GOE_Other":{ + "p":0.2054794521, + "r":0.1578947368, + "f":0.1785714286 + }, + "Style":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "ID_Number":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Era":{ + "p":0.5555555556, + "r":0.8, + "f":0.6557377049 + }, + "Sports_Organization_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Frequency":{ + "p":0.597826087, + "r":0.7432432432, + "f":0.6626506024 + }, + "Rule_Other":{ + "p":0.5, + "r":0.0833333333, + "f":0.1428571429 + }, + "Color_Other":{ + "p":0.1428571429, + "r":0.25, + "f":0.1818181818 + }, + "Domestic_Region":{ + "p":0.6315789474, + "r":0.1666666667, + "f":0.2637362637 + }, + "Animal_Disease":{ + "p":0.5733333333, + "r":0.5657894737, + "f":0.5695364238 + }, + "Clothing":{ + "p":0.4308943089, + "r":0.6235294118, + "f":0.5096153846 + }, + "Flora":{ + "p":0.26, + "r":0.1940298507, + "f":0.2222222222 + }, + "Animal_Part":{ + "p":0.7411167513, + "r":0.6728110599, + "f":0.7053140097 + }, + "Mineral":{ + "p":0.25, + "r":0.2, + "f":0.2222222222 + }, + "Period_Day":{ + "p":0.5757575758, + "r":0.6440677966, + "f":0.608 + }, + "Facility_Part":{ + "p":0.0731707317, + "r":0.0652173913, + "f":0.0689655172 + }, + "N_Flora":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Mammal":{ + "p":0.7741935484, + "r":0.6545454545, + "f":0.7093596059 + }, + "N_Animal":{ + "p":0.3333333333, + "r":0.5, + "f":0.4 + }, + "Dish":{ + "p":0.5044247788, + "r":0.5377358491, + "f":0.5205479452 + }, + "Nature_Color":{ + "p":0.6585365854, + "r":0.6585365854, + "f":0.6585365854 + }, + "Flora_Part":{ + "p":0.5952380952, + "r":0.5102040816, + "f":0.5494505495 + }, + "Geological_Region_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Religious_Festival":{ + "p":0.2, + "r":0.1538461538, + "f":0.1739130435 + }, + "N_Location_Other":{ + "p":0.1, + "r":0.2142857143, + "f":0.1363636364 + }, + "Road":{ + "p":0.6666666667, + "r":0.3703703704, + "f":0.4761904762 + }, + "Money_Form":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Ethnic_Group_Other":{ + "p":0.4666666667, + "r":0.3043478261, + "f":0.3684210526 + }, + "Measurement_Other":{ + "p":0.25, + "r":0.2, + "f":0.2222222222 + }, + "Sea":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Amphibia":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Academic":{ + "p":0.5652173913, + "r":0.2708333333, + "f":0.3661971831 + }, + "N_Natural_Object_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Location_Other":{ + "p":0.1111111111, + "r":0.0833333333, + "f":0.0952380952 + }, + "Railroad":{ + "p":0.4545454545, + "r":0.3333333333, + "f":0.3846153846 + }, + "Port":{ + "p":0.0833333333, + "r":0.2, + "f":0.1176470588 + }, + "Station":{ + "p":0.5, + "r":0.4, + "f":0.4444444444 + }, + "Periodx_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Archaeological_Place_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Bridge":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "River":{ + "p":0.3333333333, + "r":0.6, + "f":0.4285714286 + }, + "Period_Week":{ + "p":0.8421052632, + "r":0.6956521739, + "f":0.7619047619 + }, + "Space":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Amusement_Park":{ + "p":0.5862068966, + "r":0.3469387755, + "f":0.4358974359 + }, + "Character":{ + "p":0.1111111111, + "r":0.0425531915, + "f":0.0615384615 + }, + "Day_Of_Week":{ + "p":0.62, + "r":0.6458333333, + "f":0.6326530612 + }, + "Worship_Place":{ + "p":0.4444444444, + "r":0.1538461538, + "f":0.2285714286 + }, + "Email":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Speed":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Volume":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Phone_Number":{ + "p":0.9333333333, + "r":1.0, + "f":0.9655172414 + }, + "Postal_Address":{ + "p":0.25, + "r":0.0526315789, + "f":0.0869565217 + }, + "Show":{ + "p":0.4, + "r":0.2, + "f":0.2666666667 + }, + "Constellation":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Theater":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Planet":{ + "p":0.6666666667, + "r":0.5, + "f":0.5714285714 + }, + "Religion":{ + "p":0.2, + "r":0.2222222222, + "f":0.2105263158 + }, + "Airport":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Train":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Island":{ + "p":0.375, + "r":1.0, + "f":0.5454545455 + }, + "Living_Thing_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Living_Thing_Part_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Canal":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Culture":{ + "p":0.25, + "r":0.1111111111, + "f":0.1538461538 + }, + "Bay":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Sport":{ + "p":0.4736842105, + "r":0.4, + "f":0.4337349398 + }, + "Movie":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Newspaper":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Award":{ + "p":0.4, + "r":0.25, + "f":0.3076923077 + }, + "Theory":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Family":{ + "p":0.5, + "r":0.2857142857, + "f":0.3636363636 + }, + "Multiplication":{ + "p":0.6666666667, + "r":0.4, + "f":0.5 + }, + "Vehicle_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Natural_Object_Other":{ + "p":1.0, + "r":0.5, + "f":0.6666666667 + }, + "Park":{ + "p":0.5, + "r":0.4, + "f":0.4444444444 + }, + "Mountain":{ + "p":0.2857142857, + "r":0.3333333333, + "f":0.3076923077 + }, + "Reptile":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Lake":{ + "p":1.0, + "r":1.0, + "f":1.0 + }, + "Sports_Facility":{ + "p":0.2857142857, + "r":0.0769230769, + "f":0.1212121212 + }, + "Spa":{ + "p":0.8333333333, + "r":0.625, + "f":0.7142857143 + }, + "Tunnel":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Art_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Market":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Bird":{ + "p":1.0, + "r":0.2777777778, + "f":0.4347826087 + }, + "Seismic_Intensity":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Spaceship":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Car_Stop":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Museum":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "County":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Company_Group":{ + "p":1.0, + "r":0.5, + "f":0.6666666667 + }, + "Stock_Index":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Picture":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Zoo":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Temperature":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Fungus":{ + "p":0.0, + "r":0.0, + "f":0.0 + } + }, + "pos_acc":0.9743697141, + "morph_micro_f":0.0, + "morph_per_feat":0.0, + "morph_acc":0.0, + "tag_acc":0.0, + "tok2vec_loss":204759.3473588387, + "parser_loss":8554.9184086516, + "ner_loss":1241.1498988768, + "morphologizer_loss":267.1441991855 + } } diff --git a/config/ja_ginza_bert_large.meta.json b/config/ja_ginza_bert_large.meta.json index d7973b8..1e88ad3 100644 --- a/config/ja_ginza_bert_large.meta.json +++ b/config/ja_ginza_bert_large.meta.json @@ -1,7 +1,7 @@ { "lang":"ja", "name":"ginza_bert_large", - "version":"5.2.0b1", + "version":"5.3.0", "description":"Japanese multi-task CNN trained on UD-Japanese BCCWJ r2.8 + GSK2014-A(2019) + transformers-ud-japanese-electra--base. Components: transformer, parser, atteribute_ruler, ner, morphologizer, compound_splitter, bunsetu_recognizer.", "author":"Megagon Labs Tokyo.", "email":"ginza@megagon.ai", @@ -27,13 +27,13 @@ "author":"Works Applications Enterprise Co., Ltd." }, { - "name":"cl-tohoku/bert-large-japanese-v2", - "url":"https://huggingface.co/cl-tohoku/bert-large-japanese-v2", + "name":"tohoku-nlp/bert-large-japanese-v2", + "url":"https://huggingface.co/tohoku-nlp/bert-large-japanese-v2", "license":"Apache License 2.0", "author":"Tohoku University" } ], - "spacy_version":">=3.6.1,<4.0.0", + "spacy_version":">=3.8.16,<4.0.0", "spacy_git_version":"458bc5f45", "pipeline":[ "transformer", @@ -63,11 +63,1368 @@ "mode":"default" }, "requirements":[ - "sudachipy>=0.6.7,<0.7.0", + "sudachipy>=0.6.11,<0.7.0", "sudachidict_core>=20230711", - "spacy-transformers>=1.2.5,<1.3.0", - "fugashi>=1.3.0", + "ginza-transformers>=1.4.0,<1.5.0", + "fugashi>=1.5.2", "unidic-lite>=1.0.8", - "ginza>=5.2.0,<5.3.0" - ] -} \ No newline at end of file + "ginza>=5.3.0,<6.0.0" + ], + "labels":{ + "transformer":[ + + ], + "parser":[ + "ROOT", + "acl", + "acl_bunsetu", + "advcl", + "advcl_bunsetu", + "advmod", + "advmod_bunsetu", + "amod", + "amod_bunsetu", + "aux", + "aux_bunsetu", + "case", + "case_bunsetu", + "cc", + "cc_bunsetu", + "ccomp_bunsetu", + "compound", + "compound_bunsetu", + "cop", + "csubj_bunsetu", + "dep", + "dep_bunsetu", + "det_bunsetu", + "discourse_bunsetu", + "dislocated_bunsetu", + "fixed", + "mark", + "nmod", + "nmod_bunsetu", + "nsubj_bunsetu", + "nummod", + "obj_bunsetu", + "obl_bunsetu", + "punct", + "punct_bunsetu" + ], + "attribute_ruler":[ + + ], + "ner":[ + "Academic", + "Age", + "Aircraft", + "Airport", + "Amphibia", + "Amusement_Park", + "Animal_Disease", + "Animal_Part", + "Archaeological_Place_Other", + "Art_Other", + "Astral_Body_Other", + "Award", + "Bay", + "Bird", + "Book", + "Bridge", + "Broadcast_Program", + "Cabinet", + "Calorie", + "Canal", + "Car", + "Car_Stop", + "Character", + "City", + "Class", + "Clothing", + "Color_Other", + "Company", + "Company_Group", + "Compound", + "Conference", + "Constellation", + "Continental_Region", + "Corporation_Other", + "Country", + "Countx_Other", + "County", + "Culture", + "Date", + "Day_Of_Week", + "Disease_Other", + "Dish", + "Doctrine_Method_Other", + "Domestic_Region", + "Drug", + "Earthquake", + "Element", + "Email", + "Era", + "Ethnic_Group_Other", + "Event_Other", + "Facility_Other", + "Facility_Part", + "Family", + "Fish", + "Flora", + "Flora_Part", + "Food_Other", + "Frequency", + "Fungus", + "GOE_Other", + "GPE_Other", + "Game", + "Geological_Region_Other", + "God", + "Government", + "ID_Number", + "Incident_Other", + "Insect", + "Intensity", + "International_Organization", + "Island", + "Lake", + "Language_Other", + "Latitude_Longtitude", + "Law", + "Line_Other", + "Living_Thing_Other", + "Living_Thing_Part_Other", + "Location_Other", + "Magazine", + "Mammal", + "Material", + "Measurement_Other", + "Military", + "Mineral", + "Mollusc_Arthropod", + "Money", + "Money_Form", + "Mountain", + "Movement", + "Movie", + "Multiplication", + "Museum", + "Music", + "N_Animal", + "N_Country", + "N_Event", + "N_Facility", + "N_Flora", + "N_Location_Other", + "N_Natural_Object_Other", + "N_Organization", + "N_Person", + "N_Product", + "Name_Other", + "National_Language", + "Nationality", + "Natural_Disaster", + "Natural_Object_Other", + "Natural_Phenomenon_Other", + "Nature_Color", + "Newspaper", + "Numex_Other", + "Occasion_Other", + "Offense", + "Ordinal_Number", + "Organization_Other", + "Park", + "Percent", + "Period_Day", + "Period_Month", + "Period_Time", + "Period_Week", + "Period_Year", + "Periodx_Other", + "Person", + "Phone_Number", + "Physical_Extent", + "Plan", + "Planet", + "Point", + "Political_Organization_Other", + "Political_Party", + "Port", + "Position_Vocation", + "Postal_Address", + "Printing_Other", + "Pro_Sports_Organization", + "Product_Other", + "Province", + "Public_Institution", + "Railroad", + "Rank", + "Region_Other", + "Religion", + "Religious_Festival", + "Reptile", + "Research_Institute", + "River", + "Road", + "Rule_Other", + "School", + "School_Age", + "Sea", + "Ship", + "Show", + "Show_Organization", + "Spa", + "Space", + "Spaceship", + "Speed", + "Sport", + "Sports_Facility", + "Sports_League", + "Sports_Organization_Other", + "Station", + "Style", + "Temperature", + "Theater", + "Theory", + "Time", + "Time_Top_Other", + "Timex_Other", + "Title_Other", + "Train", + "Treaty", + "Tumulus", + "Tunnel", + "URL", + "Unit_Other", + "Vehicle_Other", + "Volume", + "War", + "Water_Route", + "Weapon", + "Weight", + "Worship_Place", + "Zoo" + ], + "morphologizer":[ + "POS=PUNCT", + "POS=NUM", + "POS=NOUN", + "POS=ADP", + "POS=AUX", + "POS=VERB", + "POS=CCONJ", + "POS=PART", + "POS=SCONJ", + "POS=SYM", + "POS=ADJ", + "POS=DET", + "POS=PRON", + "POS=PROPN", + "POS=ADV", + "POS=X", + "POS=INTJ" + ] + }, + "performance":{ + "dep_uas":0.94479758, + "dep_las":0.9335430878, + "dep_las_per_type":{ + "compound":{ + "p":0.9342807191, + "r":0.9310252401, + "f":0.9326501387 + }, + "nummod":{ + "p":0.9681712963, + "r":0.956, + "f":0.9620471535 + }, + "dep_bunsetu":{ + "p":0.640839971, + "r":0.600814664, + "f":0.6201822004 + }, + "obl_bunsetu":{ + "p":0.8712101123, + "r":0.8704406959, + "f":0.8708252341 + }, + "case":{ + "p":0.9895465963, + "r":0.9867176036, + "f":0.9881300751 + }, + "acl_bunsetu":{ + "p":0.9200640652, + "r":0.9069900962, + "f":0.9134803036 + }, + "nmod_bunsetu":{ + "p":0.9012672005, + "r":0.8724413004, + "f":0.8866200145 + }, + "root":{ + "p":0.9354904982, + "r":0.9391564401, + "f":0.9373198847 + }, + "fixed":{ + "p":0.9822866345, + "r":0.984057696, + "f":0.9831713676 + }, + "obj_bunsetu":{ + "p":0.9617185909, + "r":0.9580121704, + "f":0.9598618027 + }, + "advcl_bunsetu":{ + "p":0.8284284523, + "r":0.8372065021, + "f":0.8327943466 + }, + "aux":{ + "p":0.9829827464, + "r":0.982170268, + "f":0.9825763393 + }, + "nsubj_bunsetu":{ + "p":0.8841766206, + "r":0.8811020463, + "f":0.8826366559 + }, + "cc_bunsetu":{ + "p":0.8548387097, + "r":0.8327171904, + "f":0.8436329588 + }, + "det_bunsetu":{ + "p":0.951627907, + "r":0.9472222222, + "f":0.9494199536 + }, + "advmod_bunsetu":{ + "p":0.8421210306, + "r":0.8538882139, + "f":0.8479638009 + }, + "cop":{ + "p":0.9657556799, + "r":0.9597513089, + "f":0.9627441326 + }, + "amod":{ + "p":0.8620689655, + "r":0.790513834, + "f":0.824742268 + }, + "dep":{ + "p":0.7030075188, + "r":0.7791666667, + "f":0.7391304348 + }, + "nmod":{ + "p":0.8563734291, + "r":0.8625678119, + "f":0.8594594595 + }, + "mark":{ + "p":0.9791816224, + "r":0.9796504668, + "f":0.9794159885 + }, + "acl":{ + "p":1.0, + "r":0.75, + "f":0.8571428571 + }, + "advcl":{ + "p":0.8553191489, + "r":0.8553191489, + "f":0.8553191489 + }, + "compound_bunsetu":{ + "p":0.6453900709, + "r":0.4252336449, + "f":0.5126760563 + }, + "dislocated_bunsetu":{ + "p":0.6928571429, + "r":0.602484472, + "f":0.6445182724 + }, + "ccomp_bunsetu":{ + "p":0.9179389313, + "r":0.9562624254, + "f":0.9367088608 + }, + "advmod":{ + "p":0.5454545455, + "r":0.5172413793, + "f":0.5309734513 + }, + "csubj_bunsetu":{ + "p":0.8584474886, + "r":0.8430493274, + "f":0.850678733 + }, + "amod_bunsetu":{ + "p":0.9545454545, + "r":0.9710982659, + "f":0.9627507163 + }, + "punct_bunsetu":{ + "p":0.8795180723, + "r":0.7551724138, + "f":0.8126159555 + }, + "discourse_bunsetu":{ + "p":0.5757575758, + "r":0.5846153846, + "f":0.5801526718 + }, + "case_bunsetu":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "cc":{ + "p":0.8823529412, + "r":0.6, + "f":0.7142857143 + }, + "aux_bunsetu":{ + "p":0.8571428571, + "r":0.4, + "f":0.5454545455 + }, + "nummod_bunsetu":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "discourse":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "det":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "reparandum":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "reparandum_bunsetu":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "cop_bunsetu":{ + "p":0.0, + "r":0.0, + "f":0.0 + } + }, + "sents_p":0.897278494, + "sents_r":0.9044034237, + "sents_f":0.9008268707, + "ents_f":0.7111680703, + "ents_p":0.7172612453, + "ents_r":0.705177547, + "ents_per_type":{ + "Ordinal_Number":{ + "p":0.8112840467, + "r":0.9094874591, + "f":0.8575835476 + }, + "Nationality":{ + "p":0.6346153846, + "r":0.8048780488, + "f":0.7096774194 + }, + "Date":{ + "p":0.8476755687, + "r":0.8927083333, + "f":0.8696093354 + }, + "Public_Institution":{ + "p":0.4857142857, + "r":0.5666666667, + "f":0.5230769231 + }, + "Government":{ + "p":0.5625, + "r":0.7636363636, + "f":0.64781491 + }, + "Law":{ + "p":0.8550724638, + "r":0.7023809524, + "f":0.7712418301 + }, + "Doctrine_Method_Other":{ + "p":0.4939759036, + "r":0.4205128205, + "f":0.4542936288 + }, + "Product_Other":{ + "p":0.5019505852, + "r":0.5506419401, + "f":0.525170068 + }, + "Event_Other":{ + "p":0.59375, + "r":0.3015873016, + "f":0.4 + }, + "Occasion_Other":{ + "p":0.2950819672, + "r":0.2432432432, + "f":0.2666666667 + }, + "Time":{ + "p":0.6604651163, + "r":0.6574074074, + "f":0.6589327146 + }, + "Movement":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Money":{ + "p":0.8136645963, + "r":0.7797619048, + "f":0.7963525836 + }, + "Plan":{ + "p":0.8037383178, + "r":0.6466165414, + "f":0.7166666667 + }, + "Period_Time":{ + "p":0.8045977011, + "r":0.6666666667, + "f":0.7291666667 + }, + "National_Language":{ + "p":0.8055555556, + "r":0.8787878788, + "f":0.8405797101 + }, + "Corporation_Other":{ + "p":0.435483871, + "r":0.6279069767, + "f":0.5142857143 + }, + "Natural_Disaster":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Position_Vocation":{ + "p":0.7450058754, + "r":0.7895392279, + "f":0.7666263603 + }, + "Organization_Other":{ + "p":0.5555555556, + "r":0.4109589041, + "f":0.4724409449 + }, + "Country":{ + "p":0.9013761468, + "r":0.9118329466, + "f":0.9065743945 + }, + "Political_Organization_Other":{ + "p":0.5384615385, + "r":0.4375, + "f":0.4827586207 + }, + "War":{ + "p":0.75, + "r":0.6, + "f":0.6666666667 + }, + "Military":{ + "p":0.4444444444, + "r":0.4, + "f":0.4210526316 + }, + "Treaty":{ + "p":0.8351648352, + "r":0.6785714286, + "f":0.7487684729 + }, + "Conference":{ + "p":0.6276595745, + "r":0.437037037, + "f":0.5152838428 + }, + "Continental_Region":{ + "p":0.5529411765, + "r":0.6811594203, + "f":0.6103896104 + }, + "N_Person":{ + "p":0.9124579125, + "r":0.8943894389, + "f":0.9033333333 + }, + "Class":{ + "p":0.4871794872, + "r":0.4634146341, + "f":0.475 + }, + "Person":{ + "p":0.8763227513, + "r":0.8322864322, + "f":0.8537371134 + }, + "Period_Year":{ + "p":0.8280254777, + "r":0.8176100629, + "f":0.8227848101 + }, + "N_Product":{ + "p":0.6282722513, + "r":0.6349206349, + "f":0.6315789474 + }, + "N_Country":{ + "p":0.7555555556, + "r":0.6666666667, + "f":0.7083333333 + }, + "Percent":{ + "p":0.9095890411, + "r":0.9095890411, + "f":0.9095890411 + }, + "City":{ + "p":0.6710526316, + "r":0.7688442211, + "f":0.7166276347 + }, + "International_Organization":{ + "p":0.6, + "r":0.5614035088, + "f":0.580060423 + }, + "Printing_Other":{ + "p":0.9009009009, + "r":0.8064516129, + "f":0.8510638298 + }, + "Countx_Other":{ + "p":0.1714285714, + "r":0.1935483871, + "f":0.1818181818 + }, + "N_Organization":{ + "p":0.6756756757, + "r":0.5434782609, + "f":0.6024096386 + }, + "Weight":{ + "p":0.7407407407, + "r":0.7407407407, + "f":0.7407407407 + }, + "Weapon":{ + "p":0.4912280702, + "r":0.4307692308, + "f":0.4590163934 + }, + "Research_Institute":{ + "p":0.3333333333, + "r":0.1176470588, + "f":0.1739130435 + }, + "Physical_Extent":{ + "p":0.6666666667, + "r":0.7692307692, + "f":0.7142857143 + }, + "Ship":{ + "p":1.0, + "r":0.652173913, + "f":0.7894736842 + }, + "Aircraft":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Rule_Other":{ + "p":0.08, + "r":0.0833333333, + "f":0.0816326531 + }, + "Company_Group":{ + "p":0.4, + "r":0.5, + "f":0.4444444444 + }, + "Measurement_Other":{ + "p":0.3225806452, + "r":0.5, + "f":0.3921568627 + }, + "Compound":{ + "p":0.5333333333, + "r":0.7272727273, + "f":0.6153846154 + }, + "Material":{ + "p":0.3548387097, + "r":0.3548387097, + "f":0.3548387097 + }, + "Natural_Phenomenon_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Facility_Other":{ + "p":0.0833333333, + "r":0.1428571429, + "f":0.1052631579 + }, + "Element":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Academic":{ + "p":0.6818181818, + "r":0.625, + "f":0.652173913 + }, + "Rank":{ + "p":0.7065217391, + "r":0.7471264368, + "f":0.7262569832 + }, + "Numex_Other":{ + "p":0.2112676056, + "r":0.2459016393, + "f":0.2272727273 + }, + "Period_Month":{ + "p":0.431372549, + "r":0.7096774194, + "f":0.5365853659 + }, + "School_Age":{ + "p":0.6571428571, + "r":0.6571428571, + "f":0.6571428571 + }, + "N_Facility":{ + "p":0.4857142857, + "r":0.4473684211, + "f":0.4657534247 + }, + "Company":{ + "p":0.652173913, + "r":0.6329113924, + "f":0.6423982869 + }, + "Age":{ + "p":0.8992248062, + "r":0.6666666667, + "f":0.7656765677 + }, + "Title_Other":{ + "p":0.8919860627, + "r":0.8590604027, + "f":0.8752136752 + }, + "School":{ + "p":0.935483871, + "r":0.7160493827, + "f":0.8111888112 + }, + "Stock":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "GOE_Other":{ + "p":0.5652173913, + "r":0.5473684211, + "f":0.5561497326 + }, + "URL":{ + "p":1.0, + "r":1.0, + "f":1.0 + }, + "Amusement_Park":{ + "p":0.6530612245, + "r":0.6530612245, + "f":0.6530612245 + }, + "Era":{ + "p":0.6774193548, + "r":0.84, + "f":0.75 + }, + "Region_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "GPE_Other":{ + "p":0.476744186, + "r":0.6949152542, + "f":0.5655172414 + }, + "Political_Party":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Offense":{ + "p":0.7323943662, + "r":0.5977011494, + "f":0.6582278481 + }, + "Incident_Other":{ + "p":0.5833333333, + "r":0.4375, + "f":0.5 + }, + "Animal_Disease":{ + "p":0.8529411765, + "r":0.7631578947, + "f":0.8055555556 + }, + "Animal_Part":{ + "p":0.8268792711, + "r":0.83640553, + "f":0.8316151203 + }, + "Mineral":{ + "p":0.5, + "r":0.2, + "f":0.2857142857 + }, + "Flora":{ + "p":0.4247787611, + "r":0.7164179104, + "f":0.5333333333 + }, + "Period_Day":{ + "p":0.6909090909, + "r":0.6440677966, + "f":0.6666666667 + }, + "Mammal":{ + "p":0.7843137255, + "r":0.7272727273, + "f":0.7547169811 + }, + "N_Animal":{ + "p":0.5, + "r":0.5, + "f":0.5 + }, + "Clothing":{ + "p":0.7473684211, + "r":0.8352941176, + "f":0.7888888889 + }, + "Food_Other":{ + "p":0.5409836066, + "r":0.375, + "f":0.4429530201 + }, + "Dish":{ + "p":0.7090909091, + "r":0.7358490566, + "f":0.7222222222 + }, + "Domestic_Region":{ + "p":0.7368421053, + "r":0.3888888889, + "f":0.5090909091 + }, + "Nature_Color":{ + "p":0.75, + "r":0.8048780488, + "f":0.7764705882 + }, + "Flora_Part":{ + "p":0.6875, + "r":0.6734693878, + "f":0.6804123711 + }, + "Car":{ + "p":0.8095238095, + "r":0.3863636364, + "f":0.5230769231 + }, + "Geological_Region_Other":{ + "p":0.4285714286, + "r":0.2142857143, + "f":0.2857142857 + }, + "Province":{ + "p":0.8441558442, + "r":0.8333333333, + "f":0.8387096774 + }, + "Railroad":{ + "p":0.4166666667, + "r":0.6666666667, + "f":0.5128205128 + }, + "Location_Other":{ + "p":0.1666666667, + "r":0.1666666667, + "f":0.1666666667 + }, + "Worship_Place":{ + "p":0.2820512821, + "r":0.4230769231, + "f":0.3384615385 + }, + "Road":{ + "p":0.7777777778, + "r":0.7777777778, + "f":0.7777777778 + }, + "Money_Form":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Ethnic_Group_Other":{ + "p":0.7692307692, + "r":0.4347826087, + "f":0.5555555556 + }, + "Religious_Festival":{ + "p":0.4615384615, + "r":0.4615384615, + "f":0.4615384615 + }, + "Culture":{ + "p":0.25, + "r":0.1111111111, + "f":0.1538461538 + }, + "Music":{ + "p":0.4642857143, + "r":0.3421052632, + "f":0.3939393939 + }, + "Port":{ + "p":0.25, + "r":0.2, + "f":0.2222222222 + }, + "Point":{ + "p":0.3529411765, + "r":0.3333333333, + "f":0.3428571429 + }, + "Periodx_Other":{ + "p":0.375, + "r":0.6, + "f":0.4615384615 + }, + "Vehicle_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Frequency":{ + "p":0.7971014493, + "r":0.7432432432, + "f":0.7692307692 + }, + "Facility_Part":{ + "p":0.2962962963, + "r":0.1739130435, + "f":0.2191780822 + }, + "Bridge":{ + "p":0.3333333333, + "r":0.5, + "f":0.4 + }, + "N_Location_Other":{ + "p":0.2666666667, + "r":0.2857142857, + "f":0.275862069 + }, + "River":{ + "p":1.0, + "r":0.8, + "f":0.8888888889 + }, + "N_Natural_Object_Other":{ + "p":0.2857142857, + "r":0.1666666667, + "f":0.2105263158 + }, + "Period_Week":{ + "p":0.7826086957, + "r":0.7826086957, + "f":0.7826086957 + }, + "Space":{ + "p":0.5, + "r":0.4, + "f":0.4444444444 + }, + "Constellation":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "N_Event":{ + "p":0.4782608696, + "r":0.3666666667, + "f":0.4150943396 + }, + "Day_Of_Week":{ + "p":0.7142857143, + "r":0.8333333333, + "f":0.7692307692 + }, + "Book":{ + "p":0.3644067797, + "r":0.5243902439, + "f":0.43 + }, + "Character":{ + "p":0.4444444444, + "r":0.3404255319, + "f":0.3855421687 + }, + "Name_Other":{ + "p":0.5897435897, + "r":0.4339622642, + "f":0.5 + }, + "Email":{ + "p":1.0, + "r":1.0, + "f":1.0 + }, + "Phone_Number":{ + "p":0.9333333333, + "r":1.0, + "f":0.9655172414 + }, + "Theater":{ + "p":0.375, + "r":0.375, + "f":0.375 + }, + "Fish":{ + "p":1.0, + "r":0.8888888889, + "f":0.9411764706 + }, + "Planet":{ + "p":0.75, + "r":0.75, + "f":0.75 + }, + "Game":{ + "p":0.5909090909, + "r":0.4905660377, + "f":0.5360824742 + }, + "Award":{ + "p":0.8888888889, + "r":1.0, + "f":0.9411764706 + }, + "Religion":{ + "p":0.8888888889, + "r":0.8888888889, + "f":0.8888888889 + }, + "Speed":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Airport":{ + "p":0.1, + "r":0.05, + "f":0.0666666667 + }, + "Train":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Living_Thing_Other":{ + "p":0.3333333333, + "r":0.25, + "f":0.2857142857 + }, + "Color_Other":{ + "p":0.4, + "r":0.5, + "f":0.4444444444 + }, + "Drug":{ + "p":0.5555555556, + "r":0.4166666667, + "f":0.4761904762 + }, + "Living_Thing_Part_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "N_Flora":{ + "p":0.3333333333, + "r":0.3333333333, + "f":0.3333333333 + }, + "Station":{ + "p":0.4242424242, + "r":0.56, + "f":0.4827586207 + }, + "Mountain":{ + "p":0.25, + "r":0.3333333333, + "f":0.2857142857 + }, + "Canal":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Bay":{ + "p":0.6666666667, + "r":1.0, + "f":0.8 + }, + "Sea":{ + "p":0.5, + "r":0.6666666667, + "f":0.5714285714 + }, + "Sport":{ + "p":0.5862068966, + "r":0.7555555556, + "f":0.6601941748 + }, + "God":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Show_Organization":{ + "p":0.45, + "r":0.2903225806, + "f":0.3529411765 + }, + "Magazine":{ + "p":0.55, + "r":0.7857142857, + "f":0.6470588235 + }, + "Newspaper":{ + "p":0.4285714286, + "r":0.2, + "f":0.2727272727 + }, + "Insect":{ + "p":0.8333333333, + "r":0.8333333333, + "f":0.8333333333 + }, + "Theory":{ + "p":0.5, + "r":0.4, + "f":0.4444444444 + }, + "Park":{ + "p":0.5714285714, + "r":0.8, + "f":0.6666666667 + }, + "Family":{ + "p":1.0, + "r":0.2857142857, + "f":0.4444444444 + }, + "County":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Multiplication":{ + "p":0.6666666667, + "r":0.4, + "f":0.5 + }, + "Bird":{ + "p":0.9166666667, + "r":0.6111111111, + "f":0.7333333333 + }, + "Timex_Other":{ + "p":0.5, + "r":0.2, + "f":0.2857142857 + }, + "Island":{ + "p":0.75, + "r":1.0, + "f":0.8571428571 + }, + "Natural_Object_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Mollusc_Arthropod":{ + "p":0.6666666667, + "r":0.5714285714, + "f":0.6153846154 + }, + "Postal_Address":{ + "p":0.9375, + "r":0.7894736842, + "f":0.8571428571 + }, + "Reptile":{ + "p":0.5, + "r":0.3333333333, + "f":0.4 + }, + "Lake":{ + "p":1.0, + "r":1.0, + "f":1.0 + }, + "Sports_Facility":{ + "p":0.1818181818, + "r":0.0769230769, + "f":0.1081081081 + }, + "Archaeological_Place_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Tunnel":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Art_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Broadcast_Program":{ + "p":0.75, + "r":0.297029703, + "f":0.4255319149 + }, + "Market":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Movie":{ + "p":0.1153846154, + "r":0.15, + "f":0.1304347826 + }, + "Seismic_Intensity":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Sports_Organization_Other":{ + "p":0.3333333333, + "r":0.2, + "f":0.25 + }, + "Spa":{ + "p":0.75, + "r":0.75, + "f":0.75 + }, + "Museum":{ + "p":0.75, + "r":1.0, + "f":0.8571428571 + }, + "Show":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Pro_Sports_Organization":{ + "p":0.4615384615, + "r":1.0, + "f":0.6315789474 + }, + "ID_Number":{ + "p":0.2857142857, + "r":0.3529411765, + "f":0.3157894737 + }, + "Spaceship":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Fungus":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Temperature":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Stock_Index":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Picture":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Zoo":{ + "p":0.0, + "r":0.0, + "f":0.0 + } + }, + "pos_acc":0.986660685, + "morph_micro_f":0.0, + "morph_per_feat":0.0, + "tag_acc":0.0, + "transformer_loss":863.1535821792, + "parser_loss":9468.2328317891, + "ner_loss":34.3299089293, + "morphologizer_loss":25.5198627768 + } +} diff --git a/config/ja_ginza_bert_large_analysis.cfg b/config/ja_ginza_bert_large_analysis.cfg index 8ad31bb..f2f3640 100644 --- a/config/ja_ginza_bert_large_analysis.cfg +++ b/config/ja_ginza_bert_large_analysis.cfg @@ -10,7 +10,7 @@ seed = 0 [nlp] lang = "ja" -pipeline = ["transformer","parser","attribute_ruler","ner","morphologizer","compound_splitter","bunsetu_recognizer"] +pipeline = ["transformer", "parser", "attribute_ruler", "ner", "morphologizer", "compound_splitter", "bunsetu_recognizer"] batch_size = 128 disabled = ["attribute_ruler"] before_creation = null @@ -27,6 +27,9 @@ split_mode = "C" factory = "attribute_ruler" validate = false +[components.attribute_ruler.scorer] +@scorers = "spacy.attribute_ruler_scorer.v1" + [components.bunsetu_recognizer] factory = "bunsetu_recognizer" remain_bunsetu_suffix = false @@ -39,7 +42,6 @@ split_mode = null factory = "morphologizer" extend = true overwrite = true -scorer = {"@scorers":"spacy.morphologizer_scorer.v1"} [components.morphologizer.model] @architectures = "spacy.Tagger.v2" @@ -49,14 +51,18 @@ normalize = false [components.morphologizer.model.tok2vec] @architectures = "spacy-transformers.TransformerListener.v1" grad_factor = 1.0 -pooling = {"@layers":"reduce_mean.v1"} upstream = "*" +[components.morphologizer.model.tok2vec.pooling] +@layers = "reduce_mean.v1" + +[components.morphologizer.scorer] +@scorers = "spacy.morphologizer_scorer.v1" + [components.ner] factory = "ner" incorrect_spans_key = null moves = null -scorer = {"@scorers":"spacy.ner_scorer.v1"} update_with_oracle_cut_size = 100 [components.ner.model] @@ -71,15 +77,19 @@ nO = null [components.ner.model.tok2vec] @architectures = "spacy-transformers.TransformerListener.v1" grad_factor = 1.0 -pooling = {"@layers":"reduce_mean.v1"} upstream = "*" +[components.ner.model.tok2vec.pooling] +@layers = "reduce_mean.v1" + +[components.ner.scorer] +@scorers = "spacy.ner_scorer.v1" + [components.parser] factory = "parser" learn_tokens = false min_action_freq = 30 moves = null -scorer = {"@scorers":"spacy.parser_scorer.v1"} update_with_oracle_cut_size = 100 [components.parser.model] @@ -94,17 +104,21 @@ nO = null [components.parser.model.tok2vec] @architectures = "spacy-transformers.TransformerListener.v1" grad_factor = 1.0 -pooling = {"@layers":"reduce_mean.v1"} upstream = "*" +[components.parser.model.tok2vec.pooling] +@layers = "reduce_mean.v1" + +[components.parser.scorer] +@scorers = "spacy.parser_scorer.v1" + [components.transformer] -factory = "transformer" +factory = "transformer_custom" max_batch_items = 4096 -set_extra_annotations = {"@annotation_setters":"spacy-transformers.null_annotation_setter.v1"} [components.transformer.model] @architectures = "spacy-transformers.TransformerModel.v3" -name = "cl-tohoku/bert-large-japanese-v2" +name = "megagonlabs/transformers-ud-japanese-bert-large-ginza" mixed_precision = false [components.transformer.model.get_spans] @@ -118,12 +132,17 @@ stride = 96 use_fast = false tokenizer_class = "BertJapaneseTokenizer" do_lower_case = false -word_tokenizer_type = mecab -subword_tokenizer_type = wordpiece -mecab_kwargs = {"mecab_dic":"unidic_lite"} +word_tokenizer_type = "mecab" +subword_tokenizer_type = "wordpiece" + +[components.transformer.model.tokenizer_config.mecab_kwargs] +mecab_dic = "unidic_lite" [components.transformer.model.transformer_config] +[components.transformer.set_extra_annotations] +@annotation_setters = "spacy-transformers.null_annotation_setter.v1" + [corpora] [corpora.dev] @@ -177,18 +196,15 @@ L2_is_weight_decay = true L2 = 0.01 grad_clip = 1.0 use_averages = false -eps = 0.00000001 +eps = 1e-08 [training.optimizer.learn_rate] @schedules = "warmup_linear.v1" warmup_steps = 250 total_steps = 20000 -initial_rate = 0.00005 +initial_rate = 5e-05 [training.score_weights] -pos_acc = 0.15 -morph_micro_f = 0.0 -morph_per_feat = null dep_uas = 0.25 dep_las = 0.25 dep_las_per_type = null @@ -199,6 +215,9 @@ ents_f = 0.25 ents_p = 0.0 ents_r = 0.0 ents_per_type = null +pos_acc = 0.15 +morph_micro_f = 0.0 +morph_per_feat = null tag_acc = 0.0 [pretraining] @@ -213,4 +232,4 @@ after_init = null [initialize.components] -[initialize.tokenizer] \ No newline at end of file +[initialize.tokenizer] diff --git a/config/ja_ginza_electra.analysis.cfg b/config/ja_ginza_electra.analysis.cfg index 17666e9..dd898d8 100644 --- a/config/ja_ginza_electra.analysis.cfg +++ b/config/ja_ginza_electra.analysis.cfg @@ -10,7 +10,7 @@ seed = 0 [nlp] lang = "ja" -pipeline = ["transformer","parser","attribute_ruler","ner","morphologizer","compound_splitter","bunsetu_recognizer"] +pipeline = ["transformer", "parser", "attribute_ruler", "ner", "morphologizer", "compound_splitter", "bunsetu_recognizer"] batch_size = 128 disabled = ["attribute_ruler"] before_creation = null @@ -25,9 +25,11 @@ split_mode = "C" [components.attribute_ruler] factory = "attribute_ruler" -scorer = {"@scorers":"spacy.attribute_ruler_scorer.v1"} validate = false +[components.attribute_ruler.scorer] +@scorers = "spacy.attribute_ruler_scorer.v1" + [components.bunsetu_recognizer] factory = "bunsetu_recognizer" remain_bunsetu_suffix = false @@ -40,7 +42,6 @@ split_mode = null factory = "morphologizer" extend = true overwrite = true -scorer = {"@scorers":"spacy.morphologizer_scorer.v1"} [components.morphologizer.model] @architectures = "spacy.Tagger.v1" @@ -49,14 +50,18 @@ nO = null [components.morphologizer.model.tok2vec] @architectures = "spacy-transformers.TransformerListener.v1" grad_factor = 1.0 -pooling = {"@layers":"reduce_mean.v1"} upstream = "*" +[components.morphologizer.model.tok2vec.pooling] +@layers = "reduce_mean.v1" + +[components.morphologizer.scorer] +@scorers = "spacy.morphologizer_scorer.v1" + [components.ner] factory = "ner" incorrect_spans_key = null moves = null -scorer = {"@scorers":"spacy.ner_scorer.v1"} update_with_oracle_cut_size = 100 [components.ner.model] @@ -71,15 +76,19 @@ nO = null [components.ner.model.tok2vec] @architectures = "spacy-transformers.TransformerListener.v1" grad_factor = 1.0 -pooling = {"@layers":"reduce_mean.v1"} upstream = "*" +[components.ner.model.tok2vec.pooling] +@layers = "reduce_mean.v1" + +[components.ner.scorer] +@scorers = "spacy.ner_scorer.v1" + [components.parser] factory = "parser" learn_tokens = false min_action_freq = 30 moves = null -scorer = {"@scorers":"spacy.parser_scorer.v1"} update_with_oracle_cut_size = 100 [components.parser.model] @@ -94,17 +103,21 @@ nO = null [components.parser.model.tok2vec] @architectures = "spacy-transformers.TransformerListener.v1" grad_factor = 1.0 -pooling = {"@layers":"reduce_mean.v1"} upstream = "*" +[components.parser.model.tok2vec.pooling] +@layers = "reduce_mean.v1" + +[components.parser.scorer] +@scorers = "spacy.parser_scorer.v1" + [components.transformer] factory = "transformer_custom" max_batch_items = 4096 -set_extra_annotations = {"@annotation_setters":"spacy-transformers.null_annotation_setter.v1"} [components.transformer.model] @architectures = "spacy-transformers.TransformerModel.v3" -name = "megagonlabs/transformers-ud-japanese-electra-base-ginza-510" +name = "megagonlabs/transformers-ud-japanese-electra-base-ginza" mixed_precision = false [components.transformer.model.get_spans] @@ -116,7 +129,7 @@ stride = 96 [components.transformer.model.tokenizer_config] use_fast = false -tokenizer_class = "sudachitra.tokenization_electra_sudachipy.ElectraSudachipyTokenizer" +tokenizer_class = "modeling.ElectraSudachipyTokenizer" do_lower_case = false do_word_tokenize = true do_subword_tokenize = true @@ -130,6 +143,9 @@ dict_type = "core" [components.transformer.model.transformer_config] +[components.transformer.set_extra_annotations] +@annotation_setters = "spacy-transformers.null_annotation_setter.v1" + [corpora] [corpora.dev] @@ -182,28 +198,29 @@ L2_is_weight_decay = true L2 = 0.01 grad_clip = 1.0 use_averages = false -eps = 0.00000001 +eps = 1e-08 [training.optimizer.learn_rate] @schedules = "warmup_linear.v1" warmup_steps = 250 total_steps = 50000 -initial_rate = 0.00005 +initial_rate = 5e-05 [training.score_weights] -dep_uas = 0.25 -dep_las = 0.25 +dep_uas = 0.17 +dep_las = 0.17 dep_las_per_type = null sents_p = null sents_r = null -sents_f = 0.1 -ents_f = 0.25 +sents_f = 0.07 +ents_f = 0.17 ents_p = 0.0 ents_r = 0.0 ents_per_type = null -pos_acc = 0.15 -morph_acc = 0.0 +pos_acc = 0.1 +morph_micro_f = 0.33 morph_per_feat = null +morph_acc = 0.0 tag_acc = 0.0 [pretraining] @@ -218,4 +235,4 @@ after_init = null [initialize.components] -[initialize.tokenizer] \ No newline at end of file +[initialize.tokenizer] diff --git a/config/ja_ginza_electra.meta.json b/config/ja_ginza_electra.meta.json index adcd30d..2818c67 100644 --- a/config/ja_ginza_electra.meta.json +++ b/config/ja_ginza_electra.meta.json @@ -1,7 +1,7 @@ { "lang":"ja", "name":"ginza_electra", - "version":"5.2.0", + "version":"5.3.0", "description":"Japanese multi-task CNN trained on UD-Japanese BCCWJ r2.8 + GSK2014-A(2019) + transformers-ud-japanese-electra--base. Components: transformer, parser, atteribute_ruler, ner, morphologizer, compound_splitter, bunsetu_recognizer.", "author":"Megagon Labs Tokyo.", "email":"ginza@megagon.ai", @@ -34,14 +34,14 @@ "author":"Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, & Peter J. Liu" }, { - "name":"megagonlabs/transformers-ud-japanese-electra-base-ginza-5.1.0", - "url":"https://huggingface.co/megagonlabs/transformers-ud-japanese-electra-base-ginza-5.1.0", + "name":"megagonlabs/transformers-ud-japanese-electra-base-ginza", + "url":"https://huggingface.co/megagonlabs/transformers-ud-japanese-electra-base-ginza", "license":"MIT Licence", "author":"Hiroshi Matsuda (Megagon Labs Tokyo, Recruit Co., Ltd.)" } ], "parent_package":"spacy", - "spacy_version":">=3.4.4,<4.0.0", + "spacy_version":">=3.8.16,<4.0.0", "spacy_git_version":"0fc3dee77", "pipeline":[ "transformer", @@ -61,20 +61,1399 @@ "compound_splitter", "bunsetu_recognizer" ], - "disabled": [ + "disabled":[ "attribute_ruler" ], "vectors":{ "width":0, "vectors":0, "keys":0, - "name":null + "name":null, + "mode":"default" }, - "requirements":[ - "sudachipy>=0.6.2,<0.7.0", + "requirements":[ + "sudachipy>=0.6.11,<0.7.0", "sudachidict_core>=20210802", - "sudachitra>=0.1.6,<0.2.0", - "ginza-transformers>=0.4.0,<0.5.0", - "ginza>=5.2.0,<5.3.0" - ] + "sudachitra>=0.1.9,<0.2.0", + "ginza-transformers>=1.4.0,<1.5.0", + "ginza>=5.3.0,<6.0.0" + ], + "labels":{ + "transformer":[ + + ], + "parser":[ + "ROOT", + "acl", + "acl_bunsetu", + "advcl", + "advcl_bunsetu", + "advmod", + "advmod_bunsetu", + "amod", + "amod_bunsetu", + "aux", + "aux_bunsetu", + "case", + "case_bunsetu", + "cc", + "cc_bunsetu", + "ccomp_bunsetu", + "compound", + "compound_bunsetu", + "cop", + "csubj_bunsetu", + "dep", + "dep_bunsetu", + "det_bunsetu", + "discourse_bunsetu", + "dislocated_bunsetu", + "fixed", + "mark", + "nmod", + "nmod_bunsetu", + "nsubj_bunsetu", + "nummod", + "obj_bunsetu", + "obl_bunsetu", + "punct", + "punct_bunsetu" + ], + "attribute_ruler":[ + + ], + "ner":[ + "Academic", + "Age", + "Aircraft", + "Airport", + "Amphibia", + "Amusement_Park", + "Animal_Disease", + "Animal_Part", + "Archaeological_Place_Other", + "Art_Other", + "Astral_Body_Other", + "Award", + "Bay", + "Bird", + "Book", + "Bridge", + "Broadcast_Program", + "Cabinet", + "Calorie", + "Canal", + "Car", + "Car_Stop", + "Character", + "City", + "Class", + "Clothing", + "Color_Other", + "Company", + "Company_Group", + "Compound", + "Conference", + "Constellation", + "Continental_Region", + "Corporation_Other", + "Country", + "Countx_Other", + "County", + "Culture", + "Date", + "Day_Of_Week", + "Disease_Other", + "Dish", + "Doctrine_Method_Other", + "Domestic_Region", + "Drug", + "Earthquake", + "Element", + "Email", + "Era", + "Ethnic_Group_Other", + "Event_Other", + "Facility_Other", + "Facility_Part", + "Family", + "Fish", + "Flora", + "Flora_Part", + "Food_Other", + "Frequency", + "Fungus", + "GOE_Other", + "GPE_Other", + "Game", + "Geological_Region_Other", + "God", + "Government", + "ID_Number", + "Incident_Other", + "Insect", + "Intensity", + "International_Organization", + "Island", + "Lake", + "Language_Other", + "Latitude_Longtitude", + "Law", + "Line_Other", + "Living_Thing_Other", + "Living_Thing_Part_Other", + "Location_Other", + "Magazine", + "Mammal", + "Material", + "Measurement_Other", + "Military", + "Mineral", + "Mollusc_Arthropod", + "Money", + "Money_Form", + "Mountain", + "Movement", + "Movie", + "Multiplication", + "Museum", + "Music", + "N_Animal", + "N_Country", + "N_Event", + "N_Facility", + "N_Flora", + "N_Location_Other", + "N_Natural_Object_Other", + "N_Organization", + "N_Person", + "N_Product", + "Name_Other", + "National_Language", + "Nationality", + "Natural_Disaster", + "Natural_Object_Other", + "Natural_Phenomenon_Other", + "Nature_Color", + "Newspaper", + "Numex_Other", + "Occasion_Other", + "Offense", + "Ordinal_Number", + "Organization_Other", + "Park", + "Percent", + "Period_Day", + "Period_Month", + "Period_Time", + "Period_Week", + "Period_Year", + "Periodx_Other", + "Person", + "Phone_Number", + "Physical_Extent", + "Plan", + "Planet", + "Point", + "Political_Organization_Other", + "Political_Party", + "Port", + "Position_Vocation", + "Postal_Address", + "Printing_Other", + "Pro_Sports_Organization", + "Product_Other", + "Province", + "Public_Institution", + "Railroad", + "Rank", + "Region_Other", + "Religion", + "Religious_Festival", + "Reptile", + "Research_Institute", + "River", + "Road", + "Rule_Other", + "School", + "School_Age", + "Sea", + "Ship", + "Show", + "Show_Organization", + "Spa", + "Space", + "Spaceship", + "Speed", + "Sport", + "Sports_Facility", + "Sports_League", + "Sports_Organization_Other", + "Station", + "Style", + "Temperature", + "Theater", + "Theory", + "Time", + "Time_Top_Other", + "Timex_Other", + "Title_Other", + "Train", + "Treaty", + "Tumulus", + "Tunnel", + "URL", + "Unit_Other", + "Vehicle_Other", + "Volume", + "War", + "Water_Route", + "Weapon", + "Weight", + "Worship_Place", + "Zoo" + ], + "morphologizer":[ + "POS=PUNCT", + "POS=NUM", + "POS=NOUN", + "POS=ADP", + "POS=AUX", + "POS=VERB", + "POS=CCONJ", + "POS=PART", + "POS=SCONJ", + "POS=SYM", + "POS=ADJ", + "POS=DET", + "POS=PRON", + "POS=PROPN", + "POS=ADV", + "POS=X", + "POS=INTJ" + ] + }, + "performance":{ + "dep_uas":0.9344858507, + "dep_las":0.9208774079, + "dep_las_per_type":{ + "compound":{ + "p":0.9204362596, + "r":0.9199017199, + "f":0.9201689121 + }, + "nummod":{ + "p":0.9615163699, + "r":0.9565714286, + "f":0.9590375251 + }, + "dep_bunsetu":{ + "p":0.6116788321, + "r":0.5689069925, + "f":0.5895181147 + }, + "obl_bunsetu":{ + "p":0.8426436577, + "r":0.8366157379, + "f":0.8396188788 + }, + "case":{ + "p":0.9886582767, + "r":0.9858068107, + "f":0.9872304847 + }, + "acl_bunsetu":{ + "p":0.8881370573, + "r":0.8854600258, + "f":0.8867965212 + }, + "nmod_bunsetu":{ + "p":0.8709554043, + "r":0.8568633353, + "f":0.8638519024 + }, + "root":{ + "p":0.9302156781, + "r":0.925131484, + "f":0.927666615 + }, + "fixed":{ + "p":0.9804979646, + "r":0.9828240653, + "f":0.981659637 + }, + "obj_bunsetu":{ + "p":0.948582996, + "r":0.9505070994, + "f":0.9495440729 + }, + "advcl_bunsetu":{ + "p":0.7841409692, + "r":0.7930162553, + "f":0.7885536398 + }, + "aux":{ + "p":0.9796376085, + "r":0.9799267918, + "f":0.9797821788 + }, + "nsubj_bunsetu":{ + "p":0.8594463854, + "r":0.8554232981, + "f":0.8574301227 + }, + "cc_bunsetu":{ + "p":0.7958412098, + "r":0.7781885397, + "f":0.7869158879 + }, + "det_bunsetu":{ + "p":0.9486709441, + "r":0.9583333333, + "f":0.9534776601 + }, + "advmod_bunsetu":{ + "p":0.8091693957, + "r":0.8256379101, + "f":0.8173207037 + }, + "cop":{ + "p":0.9489430894, + "r":0.9548429319, + "f":0.9518838689 + }, + "amod":{ + "p":0.8325991189, + "r":0.7470355731, + "f":0.7875 + }, + "dep":{ + "p":0.6311176041, + "r":0.72, + "f":0.6726352666 + }, + "nmod":{ + "p":0.8267857143, + "r":0.8372513562, + "f":0.8319856244 + }, + "mark":{ + "p":0.9788186396, + "r":0.9772564041, + "f":0.978036898 + }, + "acl":{ + "p":0.8571428571, + "r":0.75, + "f":0.8 + }, + "advcl":{ + "p":0.8212765957, + "r":0.8212765957, + "f":0.8212765957 + }, + "compound_bunsetu":{ + "p":0.4717948718, + "r":0.4299065421, + "f":0.4498777506 + }, + "dislocated_bunsetu":{ + "p":0.6575342466, + "r":0.5962732919, + "f":0.6254071661 + }, + "ccomp_bunsetu":{ + "p":0.9021956088, + "r":0.8986083499, + "f":0.9003984064 + }, + "advmod":{ + "p":0.7058823529, + "r":0.4137931034, + "f":0.5217391304 + }, + "csubj_bunsetu":{ + "p":0.8468468468, + "r":0.8430493274, + "f":0.8449438202 + }, + "amod_bunsetu":{ + "p":0.9488636364, + "r":0.9653179191, + "f":0.9570200573 + }, + "discourse_bunsetu":{ + "p":0.5294117647, + "r":0.5538461538, + "f":0.5413533835 + }, + "punct_bunsetu":{ + "p":0.7897196262, + "r":0.5827586207, + "f":0.6706349206 + }, + "case_bunsetu":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "cc":{ + "p":1.0, + "r":0.72, + "f":0.8372093023 + }, + "aux_bunsetu":{ + "p":1.0, + "r":0.0666666667, + "f":0.125 + }, + "nummod_bunsetu":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "discourse":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "det":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "reparandum":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "reparandum_bunsetu":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "cop_bunsetu":{ + "p":0.0, + "r":0.0, + "f":0.0 + } + }, + "sents_p":0.8865468072, + "sents_r":0.884809735, + "sents_f":0.8856774194, + "ents_f":0.6543236768, + "ents_p":0.6542286751, + "ents_r":0.654418706, + "ents_per_type":{ + "Ordinal_Number":{ + "p":0.7965056526, + "r":0.8451472192, + "f":0.8201058201 + }, + "Nationality":{ + "p":0.756097561, + "r":0.756097561, + "f":0.756097561 + }, + "Date":{ + "p":0.8418972332, + "r":0.8875, + "f":0.8640973631 + }, + "Government":{ + "p":0.6296296296, + "r":0.7212121212, + "f":0.6723163842 + }, + "National_Language":{ + "p":0.9090909091, + "r":0.9090909091, + "f":0.9090909091 + }, + "Law":{ + "p":0.6, + "r":0.6428571429, + "f":0.6206896552 + }, + "Doctrine_Method_Other":{ + "p":0.3910447761, + "r":0.3358974359, + "f":0.3613793103 + }, + "Product_Other":{ + "p":0.482381531, + "r":0.5663338088, + "f":0.5209973753 + }, + "Event_Other":{ + "p":0.4237288136, + "r":0.3968253968, + "f":0.4098360656 + }, + "Occasion_Other":{ + "p":0.2125, + "r":0.2297297297, + "f":0.2207792208 + }, + "Time":{ + "p":0.6255924171, + "r":0.6111111111, + "f":0.6182669789 + }, + "Movement":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Money":{ + "p":0.8169934641, + "r":0.744047619, + "f":0.7788161994 + }, + "Offense":{ + "p":0.575, + "r":0.5287356322, + "f":0.5508982036 + }, + "Plan":{ + "p":0.6043165468, + "r":0.6315789474, + "f":0.6176470588 + }, + "Period_Time":{ + "p":0.7582417582, + "r":0.6571428571, + "f":0.7040816327 + }, + "Public_Institution":{ + "p":0.9, + "r":0.6, + "f":0.72 + }, + "Academic":{ + "p":0.3544303797, + "r":0.5833333333, + "f":0.4409448819 + }, + "Corporation_Other":{ + "p":0.3389830508, + "r":0.4651162791, + "f":0.3921568627 + }, + "Natural_Disaster":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Position_Vocation":{ + "p":0.6678043231, + "r":0.7310087173, + "f":0.6979785969 + }, + "Organization_Other":{ + "p":0.2840909091, + "r":0.3424657534, + "f":0.3105590062 + }, + "Country":{ + "p":0.8941176471, + "r":0.8816705336, + "f":0.8878504673 + }, + "Company":{ + "p":0.6813186813, + "r":0.5232067511, + "f":0.5918854415 + }, + "Political_Organization_Other":{ + "p":0.2631578947, + "r":0.3125, + "f":0.2857142857 + }, + "War":{ + "p":0.5, + "r":0.6, + "f":0.5454545455 + }, + "Military":{ + "p":0.6956521739, + "r":0.5333333333, + "f":0.6037735849 + }, + "Treaty":{ + "p":0.8241758242, + "r":0.6696428571, + "f":0.7389162562 + }, + "Conference":{ + "p":0.5161290323, + "r":0.3555555556, + "f":0.4210526316 + }, + "Continental_Region":{ + "p":0.4516129032, + "r":0.6086956522, + "f":0.5185185185 + }, + "N_Person":{ + "p":0.8242811502, + "r":0.8514851485, + "f":0.8376623377 + }, + "Person":{ + "p":0.8418474458, + "r":0.7556532663, + "f":0.7964250248 + }, + "Period_Year":{ + "p":0.7898089172, + "r":0.7798742138, + "f":0.7848101266 + }, + "N_Product":{ + "p":0.6108108108, + "r":0.5978835979, + "f":0.6042780749 + }, + "N_Country":{ + "p":0.72, + "r":0.7058823529, + "f":0.7128712871 + }, + "Percent":{ + "p":0.9215686275, + "r":0.901369863, + "f":0.9113573407 + }, + "City":{ + "p":0.6703296703, + "r":0.6130653266, + "f":0.6404199475 + }, + "International_Organization":{ + "p":0.6357142857, + "r":0.5204678363, + "f":0.5723472669 + }, + "Incident_Other":{ + "p":0.2222222222, + "r":0.375, + "f":0.2790697674 + }, + "Point":{ + "p":0.2926829268, + "r":0.3333333333, + "f":0.3116883117 + }, + "Countx_Other":{ + "p":0.1384615385, + "r":0.1451612903, + "f":0.1417322835 + }, + "Printing_Other":{ + "p":0.9072164948, + "r":0.7096774194, + "f":0.7963800905 + }, + "N_Organization":{ + "p":0.4482758621, + "r":0.2826086957, + "f":0.3466666667 + }, + "N_Facility":{ + "p":0.2549019608, + "r":0.3421052632, + "f":0.2921348315 + }, + "Weight":{ + "p":0.75, + "r":0.6666666667, + "f":0.7058823529 + }, + "ID_Number":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Animal_Disease":{ + "p":0.6818181818, + "r":0.7894736842, + "f":0.7317073171 + }, + "Weapon":{ + "p":0.5531914894, + "r":0.4, + "f":0.4642857143 + }, + "Numex_Other":{ + "p":0.1428571429, + "r":0.131147541, + "f":0.1367521368 + }, + "Research_Institute":{ + "p":0.5, + "r":0.1176470588, + "f":0.1904761905 + }, + "Physical_Extent":{ + "p":0.5869565217, + "r":0.6923076923, + "f":0.6352941176 + }, + "Aircraft":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Ship":{ + "p":0.6086956522, + "r":0.6086956522, + "f":0.6086956522 + }, + "Game":{ + "p":0.5581395349, + "r":0.4528301887, + "f":0.5 + }, + "Periodx_Other":{ + "p":0.1764705882, + "r":0.6, + "f":0.2727272727 + }, + "Compound":{ + "p":0.5882352941, + "r":0.4545454545, + "f":0.5128205128 + }, + "Material":{ + "p":0.4642857143, + "r":0.4193548387, + "f":0.4406779661 + }, + "Natural_Phenomenon_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Element":{ + "p":0.5, + "r":0.1428571429, + "f":0.2222222222 + }, + "N_Location_Other":{ + "p":0.1142857143, + "r":0.2857142857, + "f":0.1632653061 + }, + "Facility_Other":{ + "p":0.0333333333, + "r":0.1428571429, + "f":0.0540540541 + }, + "Rank":{ + "p":0.6842105263, + "r":0.7471264368, + "f":0.7142857143 + }, + "Class":{ + "p":0.6129032258, + "r":0.4634146341, + "f":0.5277777778 + }, + "School_Age":{ + "p":0.44, + "r":0.6285714286, + "f":0.5176470588 + }, + "Period_Month":{ + "p":0.5102040816, + "r":0.8064516129, + "f":0.625 + }, + "GOE_Other":{ + "p":0.3820224719, + "r":0.3578947368, + "f":0.3695652174 + }, + "Title_Other":{ + "p":0.8908450704, + "r":0.8489932886, + "f":0.8694158076 + }, + "School":{ + "p":0.7272727273, + "r":0.3950617284, + "f":0.512 + }, + "Stock":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "URL":{ + "p":0.9166666667, + "r":0.9166666667, + "f":0.9166666667 + }, + "Email":{ + "p":0.6666666667, + "r":0.5, + "f":0.5714285714 + }, + "Mollusc_Arthropod":{ + "p":0.7272727273, + "r":0.5714285714, + "f":0.64 + }, + "Era":{ + "p":0.5128205128, + "r":0.8, + "f":0.625 + }, + "Rule_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Location_Other":{ + "p":0.1923076923, + "r":0.2083333333, + "f":0.2 + }, + "Award":{ + "p":0.3157894737, + "r":0.75, + "f":0.4444444444 + }, + "GPE_Other":{ + "p":0.5714285714, + "r":0.6779661017, + "f":0.6201550388 + }, + "Mammal":{ + "p":0.6481481481, + "r":0.6363636364, + "f":0.6422018349 + }, + "Drug":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Insect":{ + "p":0.25, + "r":0.25, + "f":0.25 + }, + "Clothing":{ + "p":0.6796116505, + "r":0.8235294118, + "f":0.7446808511 + }, + "Book":{ + "p":0.3048780488, + "r":0.3048780488, + "f":0.3048780488 + }, + "Mineral":{ + "p":0.1666666667, + "r":0.2, + "f":0.1818181818 + }, + "Animal_Part":{ + "p":0.7938596491, + "r":0.8341013825, + "f":0.8134831461 + }, + "Flora":{ + "p":0.3884297521, + "r":0.7014925373, + "f":0.5 + }, + "Period_Day":{ + "p":0.4915254237, + "r":0.4915254237, + "f":0.4915254237 + }, + "N_Animal":{ + "p":0.4285714286, + "r":0.5, + "f":0.4615384615 + }, + "Newspaper":{ + "p":0.2727272727, + "r":0.2, + "f":0.2307692308 + }, + "Fish":{ + "p":0.0975609756, + "r":0.4444444444, + "f":0.16 + }, + "Dish":{ + "p":0.6880733945, + "r":0.7075471698, + "f":0.6976744186 + }, + "Food_Other":{ + "p":0.5, + "r":0.2954545455, + "f":0.3714285714 + }, + "Facility_Part":{ + "p":0.1509433962, + "r":0.1739130435, + "f":0.1616161616 + }, + "Domestic_Region":{ + "p":0.641025641, + "r":0.3472222222, + "f":0.4504504505 + }, + "Nature_Color":{ + "p":0.7777777778, + "r":0.8536585366, + "f":0.8139534884 + }, + "Flora_Part":{ + "p":0.66, + "r":0.6734693878, + "f":0.6666666667 + }, + "Car":{ + "p":0.6, + "r":0.4090909091, + "f":0.4864864865 + }, + "Geological_Region_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Province":{ + "p":0.7804878049, + "r":0.8205128205, + "f":0.8 + }, + "Bird":{ + "p":0.3333333333, + "r":0.2777777778, + "f":0.303030303 + }, + "Park":{ + "p":0.1538461538, + "r":0.4, + "f":0.2222222222 + }, + "Road":{ + "p":0.5625, + "r":0.6666666667, + "f":0.6101694915 + }, + "Money_Form":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Ethnic_Group_Other":{ + "p":0.7222222222, + "r":0.5652173913, + "f":0.6341463415 + }, + "Name_Other":{ + "p":0.0740740741, + "r":0.0754716981, + "f":0.0747663551 + }, + "Age":{ + "p":0.8444444444, + "r":0.6551724138, + "f":0.7378640777 + }, + "Sport":{ + "p":0.6111111111, + "r":0.7333333333, + "f":0.6666666667 + }, + "Island":{ + "p":0.1875, + "r":1.0, + "f":0.3157894737 + }, + "Worship_Place":{ + "p":0.4761904762, + "r":0.3846153846, + "f":0.4255319149 + }, + "Show_Organization":{ + "p":0.1111111111, + "r":0.2258064516, + "f":0.1489361702 + }, + "Mountain":{ + "p":0.3333333333, + "r":0.5833333333, + "f":0.4242424242 + }, + "Religious_Festival":{ + "p":0.1176470588, + "r":0.1538461538, + "f":0.1333333333 + }, + "Music":{ + "p":0.1791044776, + "r":0.3157894737, + "f":0.2285714286 + }, + "Sea":{ + "p":0.1111111111, + "r":0.3333333333, + "f":0.1666666667 + }, + "River":{ + "p":0.2727272727, + "r":0.6, + "f":0.375 + }, + "God":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Port":{ + "p":0.1, + "r":0.2, + "f":0.1333333333 + }, + "Frequency":{ + "p":0.7272727273, + "r":0.6486486486, + "f":0.6857142857 + }, + "N_Natural_Object_Other":{ + "p":0.2222222222, + "r":0.1666666667, + "f":0.1904761905 + }, + "Bridge":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Vehicle_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Period_Week":{ + "p":0.8636363636, + "r":0.8260869565, + "f":0.8444444444 + }, + "Space":{ + "p":0.5, + "r":0.4, + "f":0.4444444444 + }, + "N_Event":{ + "p":0.3103448276, + "r":0.3, + "f":0.3050847458 + }, + "Day_Of_Week":{ + "p":0.7592592593, + "r":0.8541666667, + "f":0.8039215686 + }, + "Character":{ + "p":0.085106383, + "r":0.085106383, + "f":0.085106383 + }, + "Phone_Number":{ + "p":0.9333333333, + "r":1.0, + "f":0.9655172414 + }, + "Theater":{ + "p":0.4, + "r":0.5, + "f":0.4444444444 + }, + "Show":{ + "p":0.0769230769, + "r":0.1, + "f":0.0869565217 + }, + "Reptile":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Family":{ + "p":0.3333333333, + "r":0.2857142857, + "f":0.3076923077 + }, + "Speed":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Planet":{ + "p":0.5, + "r":0.5, + "f":0.5 + }, + "Religion":{ + "p":1.0, + "r":0.7777777778, + "f":0.875 + }, + "Airport":{ + "p":0.5, + "r":0.1, + "f":0.1666666667 + }, + "Railroad":{ + "p":0.5, + "r":0.3333333333, + "f":0.4 + }, + "Train":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Living_Thing_Other":{ + "p":0.375, + "r":0.75, + "f":0.5 + }, + "Color_Other":{ + "p":0.2380952381, + "r":0.625, + "f":0.3448275862 + }, + "Living_Thing_Part_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "N_Flora":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Volume":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Measurement_Other":{ + "p":0.5789473684, + "r":0.55, + "f":0.5641025641 + }, + "Canal":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Bay":{ + "p":0.3333333333, + "r":1.0, + "f":0.5 + }, + "Pro_Sports_Organization":{ + "p":0.1818181818, + "r":1.0, + "f":0.3076923077 + }, + "Amusement_Park":{ + "p":0.724137931, + "r":0.4285714286, + "f":0.5384615385 + }, + "Lake":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Magazine":{ + "p":0.2307692308, + "r":0.2142857143, + "f":0.2222222222 + }, + "Company_Group":{ + "p":0.2857142857, + "r":0.5, + "f":0.3636363636 + }, + "Theory":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Culture":{ + "p":0.2, + "r":0.1111111111, + "f":0.1428571429 + }, + "Movie":{ + "p":0.1071428571, + "r":0.15, + "f":0.125 + }, + "Spaceship":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Station":{ + "p":0.4242424242, + "r":0.56, + "f":0.4827586207 + }, + "Sports_Organization_Other":{ + "p":0.2857142857, + "r":0.2, + "f":0.2352941176 + }, + "Archaeological_Place_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Multiplication":{ + "p":0.6666666667, + "r":0.4, + "f":0.5 + }, + "County":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Fungus":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Natural_Object_Other":{ + "p":1.0, + "r":0.5, + "f":0.6666666667 + }, + "Postal_Address":{ + "p":1.0, + "r":0.8421052632, + "f":0.9142857143 + }, + "Region_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Line_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Tunnel":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Art_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Broadcast_Program":{ + "p":0.3703703704, + "r":0.198019802, + "f":0.2580645161 + }, + "Sports_League":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Market":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Seismic_Intensity":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Spa":{ + "p":0.5454545455, + "r":0.75, + "f":0.6315789474 + }, + "Sports_Facility":{ + "p":0.4285714286, + "r":0.2307692308, + "f":0.3 + }, + "Museum":{ + "p":0.4, + "r":0.6666666667, + "f":0.5 + }, + "Amphibia":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Temperature":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Stock_Index":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Timex_Other":{ + "p":1.0, + "r":0.2, + "f":0.3333333333 + }, + "Picture":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Constellation":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Zoo":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Political_Party":{ + "p":0.0, + "r":0.0, + "f":0.0 + } + }, + "pos_acc":0.985524188, + "morph_micro_f":0.0, + "morph_per_feat":0.0, + "morph_acc":0.0, + "tag_acc":0.9878354143, + "transformer_loss":15946.5850335878, + "parser_loss":9292.4742132145, + "ner_loss":275.138688962, + "morphologizer_loss":343.3875644008 + } } diff --git a/docs/bunsetu_api.md b/docs/bunsetu_api.md index 4d4189e..d58bf19 100644 --- a/docs/bunsetu_api.md +++ b/docs/bunsetu_api.md @@ -81,5 +81,6 @@ for frame, count in sorted(frames.items(), key=lambda t: -t[1]): 詳細な解説はこちらの記事をご覧ください。 -- [GiNZA version 4.0: 多言語依存構造解析技術への文節APIの統合 - Megagon Labs Blog](https://www.megagon.ai/jp/blog/ginza-version-4-0/) +- [GiNZA version 4.0: 多言語依存構造解析技術への文節APIの統合 - Megagon Labs Blog](https://megagon.ai/jp/ginza-version-4-0/) + - [English version](https://megagon.ai/ginza-version-4-0-improving-syntactic-structure-analysis-through-japanese-bunsetsu-phrase-extraction-api-integration/) - [GiNZA - Universal Dependenciesによる実用的日本語解析 - 自然言語処理 Volume 27 Number 3](https://www.jstage.jst.go.jp/article/jnlp/27/3/27_695/_article/-char/ja/) diff --git a/docs/index.md b/docs/index.md index d25d33b..e8bfe46 100644 --- a/docs/index.md +++ b/docs/index.md @@ -6,44 +6,34 @@   [![Downloads](https://pepy.tech/badge/ginza/week)](https://pepy.tech/project/ginza) +***GiNZAをアップグレードする際は下記の互換性情報を確認してください。*** + +GiNZAの解析モデルは、株式会社リクルートと国立国語研究所の共同研究プロジェクトの成果の一部として公開されています。 + ## What's new! +- `GiNZA v5.3.0 - Grossular Garnet` をリリースしました (2026.09.30) + - 動作環境を Python 3.10 以降に変更（インストールの簡単さと性能が両立する Python 3.12 の使用を推奨） + - `ja_ginza_bert_large` を正式にリリース + - Apple Siliconで動作するMac OS環境で自動的にハードウェアアクセラレーションを有効化するよう変更 + - thinc-apple-ops を適用可能なPythonのバージョンは 3.10 〜 3.12 です - `GiNZA v5.2.1`をリリースしました (2026.09.01) - - バグ改修を行いました。 - - このリリースがPython 3.9以前で動作する最後のバージョンとなります。 -- `GiNZA v5.2.0`をリリースしました (2024.03.31) + - バグ改修を行いました + - このリリースがPython 3.9以前で動作する最後のバージョンとなります +- `GiNZA v5.2.0 - Fluorite`をリリースしました (2024.03.31) - 日本語の節認定のためのAPIを追加 (experimental) -- [`ja_ginza_bert_large`のβ版を公開中](https://github.com/megagonlabs/ginza/releases/tag/v5.2.0) - - [cl-tohoku/bert-large-japanese-v2](https://huggingface.co/cl-tohoku/bert-large-japanese-v2)をベースモデルに採用 - - 精度が大幅に向上（LAS=0.938, UAS=0.949, UPOS=0.983, ENE=0.708） - - CUDAに対応し8GB以上のRAMを搭載したGPU環境、または、M1・M2などApple Silicon環境の利用を推奨 -- ginzaコマンドで日本語以外を含む全てのspaCyモデルが利用可能に - - `ginza -m en_core_web_md` の形でモデル名を指定することで[CoNLL-U](https://universaldependencies.org/format.html#syntactic-annotation)出力ツールとして利用可能 - [ginzaコマンドの解説ページ](https://megagonlabs.github.io/ginza/command_line_tool.html)の記述を拡充 - `ginza`コマンドで使用するGPUのgpu_idを`ginza -g 1`の形で指定可能に -***GiNZAをアップグレードする際は下記の互換性情報を確認してください。*** - -## GiNZA v5.2 互換性情報 -- Pythonの対応バージョンが3.8以上に変更されました -- spaCyの対応バージョンがv3.4.4に変更されました -- コマンドラインのconllu出力のmisc列にClauseHeadフィールドが追加されました - -## GiNZA v5.1 互換性情報 -- `ginza --require_gpu`および`ginza -g`オプションが引数にgpu_idを取るようになりました - - gpu_idに-1を指定(デフォルト)するとCPUのみを使用します -- v5.0以前の`ja_ginza`および`ja_ginza_electra`パケージはGiNZA v5.1で使用できません（旧バージョン向けパッケージは事前にアンインストールが必要です） - - `pip uninstall ginza ; pip uninstall ja_ginza ; pip uninstall ja_ginza_electra` -- transformersモデルの追加に伴いGiNZA v5.1インストール時は`ginza`パッケージとともに解析モデルパッケージを明示的に指定する必要があります - - 解析精度重視モデル (メモリ容量16GB以上を推奨) - - `pip install -U ginza ja_ginza_electra` - - 実行速度重視モデル - - `pip install -U ginza ja_ginza` -- トークンの活用/読み/正規形の保存先がTokenクラスのフィールドに変更されました。 - - `doc.user_data[“reading_forms”][token.i]` -> `token.morph.get(“Reading”)` - - `doc.user_data[“inflections”][token.i]` -> `token.morph.get(“Inflection”)` - - `force_using_normalized_form_as_lemma(True)` -> `token.norm_` -- 併せて[GiNZA v5.0.0 重要な変更](#ginza-500)についても確認してください +## GiNZA v5.3 互換性情報 +- Pythonの対応バージョンが3.10以上に変更されました + - インストールの簡単さと性能が両立する Python 3.12 の使用を推奨します + - Python 3.14では依存ライブラリのビルドのためにコンパイル環境を構築する必要があります +- spaCyの対応バージョンがv3.8.16以上に変更されました +- GiNZA v5.3.0より前のモデルパッケージはv5.3.0で使用できないため更新が必要です +- モデルの読み込み優先度を `ja_ginza_bert_large`, `ja_ginza_electra`, `ja_ginza` の順に変更しました +- `ginza`コマンドで一定の条件を満たす場合にデフォルトでGPUアクセラレーションが有効化されるようになりました + - GPUアクセラレーションは `ginza -g -1` で無効化できます ## GiNZA v5 新機能 @@ -76,80 +66,92 @@ GiNZA v5の解析精度は以前のバージョンから飛躍的な向上を遂 | Model | LAS | UAS | UPOS | ENE | | --- | --- | --- | --- | --- | -| *ja_ginza_bert_large (β版)* | *93.8* | *94.9* | *98.3* | *70.8* | +| *ja_ginza_bert_large* | *93.8* | *94.9* | *98.3* | *70.8* | | ja_ginza_electra | 92.3 | 93.7 | 98.1 | 61.3 | | ja_ginza (v5) | 89.2 | 91.1 | 97.0 | 53.9 | | ja_ginza (v4相当) | 89.0 | 91.0 | 95.1 | 53.1 | -`ja_ginza_electra`は`ja_ginza`に対して、5万ステップ学習時の依存関係ラベリング・単語依存構造解析の誤りを、以前のバージョンと比較して25%以上低減できました。 - -また従来型モデルの`ja_ginza`においても、処理パイプラインに`morphologizer`を追加することにより、以前のバージョンと比較してUD品詞推定の誤りを4割低減できました。 +`ja_ginza_bert_large`は`ja_ginza`に対して、依存関係ラベリング・単語依存構造解析の誤りを4割以上低減できました。 -[関根の拡張固有表現階層](http://liat-aip.sakura.ne.jp/ene/ene8/definition_jp/html/enedetail.html)を用いた拡張固有表現抽出精度(ENE)においても`ja_ginza_electra`は大幅な精度向上が得られています。GiNZAは関根の拡張固有表現階層にもとづく固有表現抽出結果を、spaCyで標準的に用いられる[OntoNotes5](https://catalog.ldc.upenn.edu/docs/LDC2013T19/OntoNotes-Release-5.0.pdf)にマッピング(変換表を適用)して出力しています。OntoNotes5は関根の拡張固有表現階層よりカテゴリ数が非常に少ない(粗い)ため、拡張固有表現をOntoNotes5体系にマッピングした場合の固有表現抽出精度は、拡張固有表現での数値より一般に高くなります。 +[関根の拡張固有表現階層](http://liat-aip.sakura.ne.jp/ene/ene8/definition_jp/html/enedetail.html)を用いた拡張固有表現抽出精度(ENE)においても`ja_ginza_bert_large`は大幅な精度向上が得られています。GiNZAは関根の拡張固有表現階層にもとづく固有表現抽出結果を、spaCyで標準的に用いられる[OntoNotes5](https://catalog.ldc.upenn.edu/docs/LDC2013T19/OntoNotes-Release-5.0.pdf)にマッピング(変換表を適用)して出力しています。OntoNotes5は関根の拡張固有表現階層よりカテゴリ数が非常に少ない(粗い)ため、拡張固有表現をOntoNotes5体系にマッピングした場合の固有表現抽出精度は、拡張固有表現での数値より一般に高くなります。 ※各モデルの学習と解析精度評価にはUD_Japanese-BCCWJ r2.8から新聞記事系のテキストを除外したものをSudachi辞書mode C(長単位)で再解析(retokenize)した上で、文節主辞情報を依存関係ラベルに組み合わせた状態のコーパスを用いています。 ## 実行環境 -GiNZAは Python 3.8以上（および対応するpip）で動作検証を行っています。 +GiNZAは Python 3.10以上で動作検証を行っています。 GiNZAをインストールする前に予めPython実行環境を構築してください。 -### 実行環境のセットアップ +### GiNZAのセットアップ -#### 1. GiNZA + Transformersモデル +> [!NOTE] +> GiNZAによる依存構造解析処理は、GPUで大幅に高速化することができます。 +> Apple Siliconで動作するMac OS環境では、デフォルトでハードウェアアクセラレーションが有効化されます。 +> 詳細は [3. GPUの有効化](#3.-GPUの有効化) を参照してください。 -※Transformersモデルの実行には16GB以上のメモリ容量が必要です。メモリ容量が不足する場合は後述の従来型モデルをお試しください。 +#### 1. Transformersモデル -旧バージョンのGiNZAをインストールしている場合は次のコマンドでアンインストールします。 -```console -$ pip uninstall ginza -``` +> [!NOTE] +> Transformersモデルの実行は、メモリ容量16GB以上の環境を推奨します。 +> メモリ容量が不足する場合は後述の従来型モデルをお試しください。 -旧バージョンの`ja_ginza`をインストールしている場合は次のコマンドでアンインストールします。 +次のいずれかのコマンドを実行して、Transformersモデル（`ja_ginza_electra` または `ja_ginza_bert_large`）をインストールします。 +（GiNZAおよびTransformers関連ライブラリも同時にインストールされます。） ```console -$ pip uninstall ja_ginza +$ pip install -U ja_ginza_electra ``` - -旧バージョンの`ja_ginza_electra`をインストールしている場合は次のコマンドでアンインストールします。 ```console -$ pip uninstall ja_ginza_electra +$ pip install -U ja_ginza_bert_electra ``` -次のコマンドを実行して最新のGiNZAおよびTransformersモデルをインストールします。 -```console -$ pip install -U ginza ja_ginza_electra -``` +> [!NOTE] +> 上記コマンドでインストールされるモデルパッケージには、容量の大きいTransformersモデルやトークナイザは含まれていません。 +> これらの大容量のファイルは初回実行時に自動的にHugging Face Hubからダウンロードされて、以降の実行時にはローカルにキャッシュされたファイルが使用されます。 +> 初回実行時は初期化処理のため起動に数秒〜数十秒かかります。 -上記コマンドでインストールされる`ja_ginza_electra`パッケージには大容量モデルファイルは含まれていません。大容量モデルファイルは初回実行時に自動的にダウンロードされて、以降の実行時にはローカルにキャッシュされたファイルが使用されます。 +#### 2. 従来型モデル -大容量モデルファイルを含めたインストールを行うには、次のようにGitHubのリリースアーカイブを指定します。 +次のコマンドを実行して、従来型モデル（`ja_ginza`）をインストールします。 +（GiNZA関連ライブラリも同時にインストールされます。） ```console -$ pip install -U ginza https://github.com/megagonlabs/ginza/releases/download/latest/ja_ginza_electra-latest-with-model.tar.gz +$ pip install -U ja_ginza ``` -GPUを利用してtransformersモデルを高速に実行するには、実行環境に応じたオプションを指定してspacyを上書きインストールします。 - -CUDA 11.7を使用する場合: +#### 3. GPUの有効化 + +Apple Siliconで動作するMac OS環境では、次の条件を満たす場合に自動でハードウェアアクセラレーションが有効化されます。 +- `ja_ginza` + - `thinc-apple-ops`によるacceleration + - Python 3.10 〜 3.12 に対応 (3.13以降は`thinc-apple-ops`が非対応) + - `ja_ginza`の`thinc-apple-ops`によるスループット向上効果は、MacBook Pro M4 Max 128GBにおいて、Python 3.10-3.11で2.6倍、Python 3.12で5.2倍です。 +- `ja_ginza_electra` および `ja_ginza_large_bert` + - `torch.backends.mps`によるacceleration + - Python 3.10 〜 3.13 に対応 (3.14では依存ライブラリのビルドのためにコンパイル環境の構築が必要) + - `ja_ginza_bert_large`の`torch.backends.mps`によるスループット向上効果は、MacBook Pro M4 Max 128GBにおいて、Python 3.10-3.13で1.3倍です。 + +Linux OS環境でNVIDIA GPUによるアクセラレーションを有効化するには、LinuxにCUDAをインストールし、`export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH` のように環境変数 `LD_LIBRARY_PATH` にCUDAライブラリのパスを追加した上で、次のようにCUDAバージョンをextrasに指定して`ginza`パッケージのインストールを行います。 +- CUDA 11.x + - `$ pip install ginza[cuda11x]` +- CUDA 12.x + - `$ pip install ginza[cuda12x]` +- CUDA 13.x + - `$ pip install ginza[cuda13x]` + +`ginza`コマンドは、GPUが利用可能な場合はデフォルトでGPUアクセラレーションを有効化します。 + +`ginza`コマンドの`-g`オプションで、使用するGPUのデバイス番号を指定することができます（`-1`を指定するとGPUアクセラレーションは無効化されます）。 ```console -$ pip install -U spacy[cuda117] +$ ginza -g 0 ``` -※ あわせてpytorchもCUDAと整合したバージョンをインストールする必要があります。 - -#### 2. GiNZA + 従来型モデル - -旧バージョンのGiNZAおよび`ja_ginza`をインストールしている場合は次のコマンドでアンインストールを実行します。 +`ginza`コマンドを`-g`オプションなしで実行する際、GPUアクセラレーションが有効化された場合は次のログが最初に出力されます。 ```console -$ pip uninstall ginza _ginza -``` - -次のコマンドを実行して最新のGiNZAと従来型モデルをインストールします。 -```console -$ pip install -U ginza ja_ginza +$ ginza +GPU #0 enabled ``` - -M1やM2などのMPSに対応したApple Silicon環境では、`thinc-apple-ops`を導入することで解析速度が向上します。 +Apple Siliconで動作するMac OS環境で従来型モデルを実行する際、ハードウェアアクセラレーションが有効化された場合は次のログが最初に出力されます。 ```console -$ pip install torch thinc-apple-ops +$ ginza +thinc-apple-ops enabled ``` ### ginzaコマンドによる解析処理の実行 @@ -170,7 +172,7 @@ $ ginza 9 。 。 PUNCT 補助記号-句点 _ 6 punct _ SpaceAfter=No|BunsetuBILabel=I|BunsetuPositionType=CONT|Reading=。|ClauseHead=6 ``` -実行環境に`ja_ginza_electra`と`ja_ginza`の両方のモデルがインストールされている場合、`ginza`コマンドは`ja_ginza_electra`を優先して使用します。同じ状況で`ja_ginza`を使用するには`ginza -m`オプションでモデル名を指定します。 +実行環境に複数のモデルパッケージがインストールされている場合、`ginza`コマンドは解析精度が高いものを優先して使用します。モデルパッケージを明示的に指定して使用するには`-m`オプションでモデルパッケージ名を指定します。 ```console # ginza -m ja_ginza ``` @@ -183,7 +185,7 @@ $ ginza ### Pythonコードによる解析処理の実行 次のコードは、Transformersモデルによる依存構造解析結果を文単位で出力します。 -従来型モデルを用いる場合は`ja_ginza_electra`を`ja_ginza`に置き換えます。 +他のモデルパッケージを用いる場合は`ja_ginza_electra`の部分を目的のモデルに置き換えます。 ```python import spacy nlp = spacy.load('ja_ginza_electra') @@ -227,9 +229,9 @@ for sent in doc.sents: ### 解説記事 -- [GiNZA Version 4.0: Improving Syntactic Structure Analysis Through Japanese Bunsetsu-Phrase Extraction API Integration](https://megagon.ai/blog/ginza-version-4-0-improving-syntactic-structure-analysis-through-japanese-bunsetsu-phrase-extraction-api-integration/) - Megagon Labs Blog (2021.03) -- [GiNZA version 4.0: 多言語依存構造解析技術への文節APIの統合](https://www.megagon.ai/jp/blog/ginza-version-4-0/) - Megagon Labs Blog (2020.09) -- [GiNZA: 日本語自然言語処理オープンソースライブラリ](https://www.megagon.ai/jp/projects/ginza-install-a-japanese-nlp-library-in-one-step/) - Megagon Labs (2019) +- [GiNZA version 4.0: 多言語依存構造解析技術への文節APIの統合](https://megagon.ai/jp/ginza-version-4-0/) - Megagon Labs Blog (2020.09) + - (英語版) [GiNZA Version 4.0: Improving Syntactic Structure Analysis Through Japanese Bunsetsu-Phrase Extraction API Integration](https://megagon.ai/ginza-version-4-0-improving-syntactic-structure-analysis-through-japanese-bunsetsu-phrase-extraction-api-integration/) +- [GiNZA: 日本語自然言語処理オープンソースライブラリ](https://megagon.ai/jp/ginza/) - Megagon Labs (2019) ## ライセンス GiNZA NLPライブラリおよびGiNZA日本語Universal Dependenciesモデルは @@ -250,7 +252,7 @@ GiNZAはトークン化（形態素解析）処理にSudachiPyを、単語ベク [chiVe LICENSE PAGE](https://github.com/WorksApplications/chiVe/blob/master/LICENSE) ### Hugging Face / transformers -GiNZA v5 Transformersモデル(ja_ginza_electra)はHugging Face社が提供するtransformersを事前学習フレームワークに用いています。 +GiNZA v5 TransformersモデルはHugging Face社が提供するtransformersを推論フレームワークに用いています。 [transformers LICENSE PAGE](https://github.com/huggingface/transformers/blob/master/LICENSE) @@ -261,7 +263,21 @@ GiNZA v5の依存構造解析モデルは [UD Japanese BCCWJ](https://github.com/UniversalDependencies/UD_Japanese-BCCWJ) r2.8 ([Omura and Asahara:2018](https://www.aclweb.org/anthology/W18-6014/)) から新聞系文書を除外して学習しています。 -GiNZA v5の依存構造解析モデルは国立国語研究所とMegagon Labsの共同研究成果です。 +``` +@inproceedings{omura-asahara-2018-ud, + title = "{UD}-{J}apanese {BCCWJ}: {U}niversal {D}ependencies Annotation for the {B}alanced {C}orpus of {C}ontemporary {W}ritten {J}apanese", + author = "Omura, Mai and + Asahara, Masayuki", + booktitle = "Proceedings of the Second Workshop on Universal Dependencies ({UDW} 2018)", + month = nov, + year = "2018", + address = "Brussels, Belgium", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/W18-6014/", + doi = "10.18653/v1/W18-6014", + pages = "117--125" +} +``` ### GSK2014-A (2019) BCCWJ版 GiNZA v5の固有表現抽出モデルは @@ -274,7 +290,7 @@ GiNZA v5の固有表現抽出モデルは GiNZA v5の固有表現抽出モデルは国立国語研究所とMegagon Labsの共同研究成果です。 ### mC4 -GiNZA v5 Transformersモデル(ja_ginza_electra)は、[mC4](https://huggingface.co/datasets/mc4)から抽出した日本語20億文以上を用いて事前学習した[transformers-ud-japanese-electra-base-discriminator](https://huggingface.co/megagonlabs/transformers-ud-japanese-electra-base-discriminator)を使用しています。 +`ja_ginza_electra`は、[mC4](https://huggingface.co/datasets/mc4)から抽出した日本語20億文以上を用いて事前学習した[transformers-ud-japanese-electra-base-discriminator](https://huggingface.co/megagonlabs/transformers-ud-japanese-electra-base-discriminator)を使用しています。 mC4はODC-BYライセンスの規約に基づいて事前学習データとして利用しています。 Contains information from mC4 which is made available under the ODC Attribution License. @@ -289,12 +305,28 @@ Contains information from mC4 which is made available under the ODC Attribution } ``` - - ## [リリース履歴](https://github.com/megagonlabs/ginza/releases) ### version 5.x +#### ginza-5.3.0 +- 2026-09-30, Grossular Garnet +- 重要な変更 + - 本リリースからサポート対象のPythonバージョンが3.10以上に、spaCyのバージョンが3.8.16以上に変更されました。 + - インストールの簡単さと性能が両立する Python 3.12 の使用を推奨します。 + - Python 3.14では依存ライブラリのビルドのためにコンパイル環境を構築する必要があります。 + - GiNZA v5.3.0より前のモデルパッケージはGiNZA v5.3.0以降で使用できません。 + - モデルパッケージの読み込み優先度を `ja_ginza_bert_large`, `ja_ginza_electra`, `ja_ginza` の順に変更しました。 + - `ginza`コマンドで一定の条件を満たす場合にデフォルトでGPUアクセラレーションが有効化されるようになりました。 + - GPUアクセラレーションは `ginza -g -1` で無効化できます。 +- 新機能 + - Apple Siliconで動作するMac OS環境で自動的にハードウェアアクセラレーションを有効化するよう変更しました。 + - Apple Siliconで動作するMac OSで thinc-apple-ops を適用可能なPythonのバージョンは 3.10 〜 3.12 です。 + - `ja_ginza_bert_large` を正式にリリースしました。 + - [`ginza-transformers`](https://github.com/megagonlabs/ginza-transformers) をv1.4.0にアップグレードしました。 + - transformers componentでmodelとtokenizerの両方をHugging Face Hubから取得する形に変更しました。 + - [`spacy-transformers`](https://github.com/explosion/spacy-transformers)のrequirementsにより `torch>=1.8.0, transformers<4.53.3` などの制限があります。 + #### ginza-5.2.1 - 2026-09-01 - このリリースがPython 3.9以前で動作する最後のバージョンとなります。 @@ -310,16 +342,20 @@ Contains information from mC4 which is made available under the ODC Attribution - [#266](https://github.com/megagonlabs/ginza/pull/266) - [#273](https://github.com/megagonlabs/ginza/pull/273) #### ginza-5.2.0 -- 2024-03-31 +- 2024-03-31, Fluorite - Require python>=3.8 - Migrate to spaCy v3.7 - New functionality - add Japanese clause recognition API (experimental) + - コマンドラインのconllu出力のmisc列にClauseHeadフィールドが追加されました #### ginza-5.1.3 - 2023-09-25 - Migrate to spaCy v3.6 -- Beta release of `ja_ginza_bert_large` +- [`ja_ginza_bert_large` β版を公開](https://github.com/megagonlabs/ginza/releases/tag/v5.2.0) + - [tohoku-nlp/bert-large-japanese-v2](https://huggingface.co/tohoku-nlp/bert-large-japanese-v2)をベースモデルに採用 + - 精度が大幅に向上（LAS=0.938, UAS=0.949, UPOS=0.983, ENE=0.708） + - CUDAに対応し8GB以上のRAMを搭載したGPU環境、または、M1・M2などApple Silicon環境の利用を推奨 #### ginza-5.1.2 - 2022-03-12 diff --git a/ginza/__init__.py b/ginza/__init__.py index 2ef3859..00c5059 100644 --- a/ginza/__init__.py +++ b/ginza/__init__.py @@ -49,6 +49,20 @@ ] +GINZA_MODEL_PACKAGES = ["ja_ginza_bert_large", "ja_ginza_electra", "ja_ginza"] + + +def default_model_name(): + from importlib import import_module + for model_package in GINZA_MODEL_PACKAGES: + try: + import_module(model_package) + return model_package + except ModuleNotFoundError: + pass + return None + + @Language.factory( "compound_splitter", requires=[], diff --git a/ginza/analyzer.py b/ginza/analyzer.py index e4b38a6..5499363 100644 --- a/ginza/analyzer.py +++ b/ginza/analyzer.py @@ -1,5 +1,4 @@ # coding: utf8 -import sys from typing import Iterable, Optional import thinc @@ -9,7 +8,7 @@ from spacy.language import Language from spacy.lang.ja import Japanese -from . import set_split_mode, inflection, reading_form, ent_label_ene, ent_label_ontonotes, bunsetu_bi_label, bunsetu_position_type, clause_head_i, _safe_lemma_, _safe_norm_ +from . import set_split_mode, inflection, reading_form, ent_label_ene, ent_label_ontonotes, bunsetu_bi_label, bunsetu_position_type, clause_head_i, _safe_lemma_, _safe_norm_, GINZA_MODEL_PACKAGES from .bunsetu_recognizer import bunsetu_available, bunsetu_head_list, bunsetu_phrase_span @@ -67,20 +66,10 @@ def set_nlp(self) -> None: if self.output_format in ["2", "mecab"]: nlp = try_sudachi_import(self.split_mode) else: - # Work-around for pickle error. Need to share model data. - if self.model_name_or_path: + try: nlp = spacy.load(self.model_name_or_path) - else: - try: - nlp = spacy.load("ja_ginza_electra") - except IOError as e: - try: - nlp = spacy.load("ja_ginza") - except IOError as e: - try: - nlp = spacy.load("ja_ginza_bert_large") - except IOError as e: - raise OSError("E050", 'You need to install "ja-ginza" or "ja-ginza-electra" by executing `pip install ja-ginza` or `pip install ja-ginza-electra`.') + except IOError: + raise OSError("E050", f"You need to install on or more model packages in {GINZA_MODEL_PACKAGES} by executing `pip install`.") if self.disable_sentencizer: nlp.add_pipe("disable_sentencizer", before="parser") diff --git a/ginza/command_line.py b/ginza/command_line.py index a2eadc8..bacbf05 100644 --- a/ginza/command_line.py +++ b/ginza/command_line.py @@ -8,13 +8,15 @@ from typing import Generator, Iterable, Optional, List import plac +from thinc.api import get_current_ops +from thinc.compat import has_cupy_gpu, has_torch_mps +from . import default_model_name, GINZA_MODEL_PACKAGES from .analyzer import Analyzer MINI_BATCH_SIZE = 100 MAX_BYTE_LENGTH = 49149 # defined in sudachi.rs https://github.com/megagonlabs/ginza/issues/242 -GINZA_MODEL_PATTERN = re.compile(r"^(ja_ginza|ja_ginza_electra)$") SPACY_MODEL_PATTERN = re.compile(r"^[a-z]{2}[-_].+[-_].+(sm|md|lg|trf)$") - +TRANSFORMERS_MODEL_PATTERN = re.compile(r"^(ja_ginza_electra|ja_ginza_bert_large|.+_trf)$") class _OutputWrapper: def __init__(self, output_path, output_format): @@ -59,7 +61,7 @@ def run( hash_comment: str = "print", output_path: Optional[Path] = None, output_format: str = "0", - require_gpu: int = -1, + require_gpu: Optional[int] = None, disable_sentencizer: bool = False, use_normalized_form: bool = False, parallel_level: int = 1, @@ -71,26 +73,6 @@ def run( file=sys.stderr ) - assert parallel_level == 1 or require_gpu == -1, "require_gpu not allowed for multi-processing. https://github.com/explosion/spaCy/issues/5507" - - if parallel_level <= 0: - level = max(1, cpu_count() + parallel_level) - if output_format in [2, "mecab"]: - if require_gpu >= 0: - print("GPU not used for mecab mode", file=sys.stderr) - require_gpu = False - elif parallel_level <= 0: - if require_gpu >= 0: - if level < 4: - print(f"GPU #{require_gpu} enabled: parallel_level' set to {level}", end="", file=sys.stderr) - else: - print(f"GPU #{require_gpu} enabled: parallel_level' set to {level} but seems it's too much", end="", file=sys.stderr) - else: - print(f"'parallel_level' set to {level}", file=sys.stderr) - elif require_gpu: - print(f"GPU #{require_gpu} enabled", file=sys.stderr) - parallel_level = level - assert model_path is None or ensure_model is None if ensure_model: ensure_model = ensure_model.replace("-", "_") @@ -98,7 +80,7 @@ def run( from importlib import import_module import_module(ensure_model) except ModuleNotFoundError: - if GINZA_MODEL_PATTERN.match(ensure_model): + if ensure_model in GINZA_MODEL_PACKAGES: print("Installing", ensure_model, file=sys.stderr) import pip pip.main(["install", ensure_model]) @@ -111,8 +93,36 @@ def run( else: raise OSError("E050", f'You need to install "{ensure_model}" before executing ginza.') model_name_or_path = ensure_model - else: + elif model_path: model_name_or_path = model_path + else: + model_name_or_path = default_model_name() + + if output_format in [2, "mecab"]: + if require_gpu is not None and require_gpu >= 0: + print(f"GPU not used in mecab mode.", file=sys.stderr) + require_gpu = -1 + elif require_gpu is None: + if has_torch_mps: + require_gpu = 0 + elif has_cupy_gpu: + require_gpu = 0 + else: + require_gpu = -1 + if require_gpu != -1: + if parallel_level == 1: + print(f"GPU #{require_gpu} enabled", file=sys.stderr) + else: + require_gpu = -1 + print(f"GPU disabled because 'parallel_level' set to {parallel_level}", file=sys.stderr) + elif get_current_ops().name == "apple": + print(f"thinc-apple-ops enabled", file=sys.stderr) + + assert parallel_level == 1 or require_gpu == -1, "require_gpu not allowed for multi-processing. https://github.com/explosion/spaCy/issues/5507" + + if parallel_level <= 0: + parallel_level = max(1, cpu_count() + parallel_level) + print(f"'parallel_level' set to {parallel_level}", file=sys.stderr) analyzer = Analyzer( model_name_or_path, @@ -366,7 +376,7 @@ def run_ginza( hash_comment="print", output_path=None, output_format="conllu", - require_gpu=-1, + require_gpu=None, use_normalized_form=False, disable_sentencizer=False, parallel=1, diff --git a/requirements.txt b/requirements.txt index f5c3af3..f1340f0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ -spacy>=3.4.4,<4.0.0 +spacy>=3.8.16,<4.0.0 plac>=1.3.3 -SudachiPy>=0.6.2,<0.7.0 +SudachiPy>=0.6.11,<0.7.0 SudachiDict-core>=20210802 +thinc-apple-ops>=1.0.0 ; sys_platform == 'darwin' and platform_machine == 'arm64' diff --git a/setup.py b/setup.py index 6ffc016..2119861 100644 --- a/setup.py +++ b/setup.py @@ -16,18 +16,24 @@ "ginzame = ginza.command_line:main_ginzame", ], }, - python_requires=">=3.8", + python_requires=">=3.10", install_requires=[ - "spacy>=3.4.4,<4.0.0", + "spacy>=3.8.16,<4.0.0", "plac>=1.3.3", - "SudachiPy>=0.6.2,<0.7.0", + "SudachiPy>=0.6.11,<0.7.0", "SudachiDict-core>=20210802", + "thinc-apple-ops>=1.0.0 ; sys_platform == 'darwin' and platform_machine == 'arm64' and python_version < '3.13'", ], + extras_require={ + "cuda11x": ["cupy-cuda11x"], + "cuda12x": ["cupy-cuda12x"], + "cuda13x": ["cupy-cuda13x"], + }, setup_requires=["pytest-runner"], tests_require=["pytest", "pytest-cov", "pytest-mock"], license="MIT", name="ginza", packages=find_packages(include=["ginza"]), url="https://github.com/megagonlabs/ginza", - version='5.2.1', + version='5.3.0', )