From 35f1cab465dd4bb6c5253080c07d23803eef4726 Mon Sep 17 00:00:00 2001 From: Hiroshi Matsuda Date: Wed, 2 Sep 2026 15:37:59 +0900 Subject: [PATCH 01/24] requirements --- .github/workflows/pytest.yml | 4 ++-- requirements.txt | 4 ++-- setup.py | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index f249eec..8013d5b 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10", "3.11"] # , "3.12", "3.13"] + python-version: ["3.10", "3.13"] steps: - name: Checkout uses: actions/checkout@v7 @@ -29,6 +29,6 @@ jobs: - name: Install dependencies run: >- python -m - pip install . pytest pytest-mock ja-ginza ja-ginza-electra + pip install . pytest pytest-mock ja-ginza ja-ginza-electra ja-ginza-bert-large - name: Run Tests run: pytest diff --git a/requirements.txt b/requirements.txt index f5c3af3..afba787 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -spacy>=3.4.4,<4.0.0 +spacy>=3.8.16,<4.0.0 plac>=1.3.3 -SudachiPy>=0.6.2,<0.7.0 +SudachiPy>=0.6.11,<0.7.0 SudachiDict-core>=20210802 diff --git a/setup.py b/setup.py index 6ffc016..2561bee 100644 --- a/setup.py +++ b/setup.py @@ -16,11 +16,11 @@ "ginzame = ginza.command_line:main_ginzame", ], }, - python_requires=">=3.8", + python_requires=">=3.10,<3.15", install_requires=[ - "spacy>=3.4.4,<4.0.0", + "spacy>=3.8.16,<4.0.0", "plac>=1.3.3", - "SudachiPy>=0.6.2,<0.7.0", + "SudachiPy>=0.6.11,<0.7.0", "SudachiDict-core>=20210802", ], setup_requires=["pytest-runner"], @@ -29,5 +29,5 @@ name="ginza", packages=find_packages(include=["ginza"]), url="https://github.com/megagonlabs/ginza", - version='5.2.1', + version='5.3.0', ) From 3cee9cfb2d055d279931e479957051681be27c86 Mon Sep 17 00:00:00 2001 From: Hiroshi Matsuda Date: Wed, 2 Sep 2026 15:57:35 +0900 Subject: [PATCH 02/24] downgrade python version up to 3.13 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2561bee..a4eed1e 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ "ginzame = ginza.command_line:main_ginzame", ], }, - python_requires=">=3.10,<3.15", + python_requires=">=3.10,<3.14", install_requires=[ "spacy>=3.8.16,<4.0.0", "plac>=1.3.3", From 372f30ade7538335c3ffda5eae053c744675ffe6 Mon Sep 17 00:00:00 2001 From: Hiroshi Matsuda Date: Fri, 4 Sep 2026 12:20:21 +0900 Subject: [PATCH 03/24] extras_require --- setup.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/setup.py b/setup.py index a4eed1e..8ad5946 100644 --- a/setup.py +++ b/setup.py @@ -23,6 +23,11 @@ "SudachiPy>=0.6.11,<0.7.0", "SudachiDict-core>=20210802", ], + extras_require={ + "apple": ["thinc[apple]"], + "cuda11x": ["thinc[cuda11x]"], + "cuda12x": ["thinc[cuda12x]"], + }, setup_requires=["pytest-runner"], tests_require=["pytest", "pytest-cov", "pytest-mock"], license="MIT", From 7a884768268ba2fc124cd58bac9d3a6f8b67c388 Mon Sep 17 00:00:00 2001 From: Hiroshi Matsuda Date: Fri, 4 Sep 2026 12:39:32 +0900 Subject: [PATCH 04/24] add cython>=3.3.0 --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 8ad5946..c95ccfb 100644 --- a/setup.py +++ b/setup.py @@ -25,8 +25,8 @@ ], extras_require={ "apple": ["thinc[apple]"], - "cuda11x": ["thinc[cuda11x]"], - "cuda12x": ["thinc[cuda12x]"], + "cuda11x": ["thinc[cuda11x]", "cython>=3.3.0"], + "cuda12x": ["thinc[cuda12x]", "cython>=3.3.0"], }, setup_requires=["pytest-runner"], tests_require=["pytest", "pytest-cov", "pytest-mock"], From e8aa38c1531ebae3157c2102d9a816807a462cdf Mon Sep 17 00:00:00 2001 From: Hiroshi Matsuda Date: Fri, 4 Sep 2026 12:43:38 +0900 Subject: [PATCH 05/24] change extras_require --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index c95ccfb..844419d 100644 --- a/setup.py +++ b/setup.py @@ -24,9 +24,9 @@ "SudachiDict-core>=20210802", ], extras_require={ - "apple": ["thinc[apple]"], - "cuda11x": ["thinc[cuda11x]", "cython>=3.3.0"], - "cuda12x": ["thinc[cuda12x]", "cython>=3.3.0"], + "apple": ["thinc-apple-ops"], + "cuda11x": ["cupy-cuda11x"], + "cuda12x": ["cupy-cuda12x"], }, setup_requires=["pytest-runner"], tests_require=["pytest", "pytest-cov", "pytest-mock"], From c4836d1cb3a5484fe8cd914c3f22e45c993d89c4 Mon Sep 17 00:00:00 2001 From: Hiroshi Matsuda Date: Fri, 4 Sep 2026 12:58:21 +0900 Subject: [PATCH 06/24] add empty cpu option --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 844419d..17ea13d 100644 --- a/setup.py +++ b/setup.py @@ -24,9 +24,10 @@ "SudachiDict-core>=20210802", ], extras_require={ + "cpu": [], "apple": ["thinc-apple-ops"], - "cuda11x": ["cupy-cuda11x"], - "cuda12x": ["cupy-cuda12x"], + "cuda11x": ["thinc", "cupy-cuda11x"], + "cuda12x": ["thinc", "cupy-cuda12x"], }, setup_requires=["pytest-runner"], tests_require=["pytest", "pytest-cov", "pytest-mock"], From f0fb0b351594dcc5c4e2651f02ee1e65316d983a Mon Sep 17 00:00:00 2001 From: Hiroshi Matsuda Date: Fri, 4 Sep 2026 13:46:16 +0900 Subject: [PATCH 07/24] drop thinc --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 17ea13d..5310520 100644 --- a/setup.py +++ b/setup.py @@ -26,8 +26,8 @@ extras_require={ "cpu": [], "apple": ["thinc-apple-ops"], - "cuda11x": ["thinc", "cupy-cuda11x"], - "cuda12x": ["thinc", "cupy-cuda12x"], + "cuda11x": ["cupy-cuda11x"], + "cuda12x": ["cupy-cuda12x"], }, setup_requires=["pytest-runner"], tests_require=["pytest", "pytest-cov", "pytest-mock"], From 322b821eb9a4b2dfc3d3df6aa265136095ea60df Mon Sep 17 00:00:00 2001 From: Hiroshi Matsuda Date: Sun, 6 Sep 2026 09:30:13 +0900 Subject: [PATCH 08/24] add thinc-apple-ops for darwin-arm64 environment --- requirements.txt | 1 + setup.py | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index afba787..f1340f0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ spacy>=3.8.16,<4.0.0 plac>=1.3.3 SudachiPy>=0.6.11,<0.7.0 SudachiDict-core>=20210802 +thinc-apple-ops>=1.0.0 ; sys_platform == 'darwin' and platform_machine == 'arm64' diff --git a/setup.py b/setup.py index 5310520..b7f7432 100644 --- a/setup.py +++ b/setup.py @@ -16,16 +16,15 @@ "ginzame = ginza.command_line:main_ginzame", ], }, - python_requires=">=3.10,<3.14", + python_requires=">=3.10", install_requires=[ "spacy>=3.8.16,<4.0.0", "plac>=1.3.3", "SudachiPy>=0.6.11,<0.7.0", "SudachiDict-core>=20210802", + "thinc-apple-ops>=1.0.0 ; sys_platform == 'darwin' and platform_machine == 'arm64'", ], extras_require={ - "cpu": [], - "apple": ["thinc-apple-ops"], "cuda11x": ["cupy-cuda11x"], "cuda12x": ["cupy-cuda12x"], }, From e1f14d3928f1bec3515674d0e6919ca5b384c98f Mon Sep 17 00:00:00 2001 From: Hiroshi Matsuda Date: Sun, 6 Sep 2026 10:26:44 +0900 Subject: [PATCH 09/24] update model configs --- config/ja_ginza.cfg | 4 +- config/ja_ginza.meta.json | 1393 +++++++++++++++++++++- config/ja_ginza_bert_large.meta.json | 1373 +++++++++++++++++++++- config/ja_ginza_bert_large_analysis.cfg | 57 +- config/ja_ginza_electra.analysis.cfg | 57 +- config/ja_ginza_electra.meta.json | 1403 ++++++++++++++++++++++- 6 files changed, 4221 insertions(+), 66 deletions(-) diff --git a/config/ja_ginza.cfg b/config/ja_ginza.cfg index ac267ba..b0be8f0 100644 --- a/config/ja_ginza.cfg +++ b/config/ja_ginza.cfg @@ -29,7 +29,7 @@ validate = false [components.bunsetu_recognizer] factory = "bunsetu_recognizer" -remain_bunsetu_suffix = true +remain_bunsetu_suffix = false [components.compound_splitter] factory = "compound_splitter" @@ -197,4 +197,4 @@ after_init = null [initialize.components] -[initialize.tokenizer] \ No newline at end of file +[initialize.tokenizer] diff --git a/config/ja_ginza.meta.json b/config/ja_ginza.meta.json index 1864604..a35435f 100644 --- a/config/ja_ginza.meta.json +++ b/config/ja_ginza.meta.json @@ -1,7 +1,7 @@ { "lang":"ja", "name":"ginza", - "version":"5.2.0", + "version":"5.3.0", "description":"Japanese multi-task CNN trained on UD-Japanese BCCWJ r2.8 + GSK2014-A(2019). Assigns word2vec token vectors. Components: tok2vec, parser, ner, morphologizer, atteribute_ruler, compound_splitter, bunsetu_recognizer.", "author":"Megagon Labs Tokyo.", "email":"ginza@megagon.ai", @@ -34,7 +34,7 @@ } ], "parent_package":"spacy", - "spacy_version":">=3.4.4,<4.0.0", + "spacy_version":">=3.8.16,<4.0.0", "spacy_git_version":"0fc3dee77", "vectors":{ "width":300, @@ -64,8 +64,1391 @@ "attribute_ruler" ], "requirements":[ - "sudachipy>=0.6.2,<0.7.0", + "sudachipy>=0.6.11,<0.7.0", "sudachidict_core>=20210802", - "ginza>=5.2.0,<5.3.0" - ] + "ginza>=5.3.0,<6.0.0" + ], + "labels":{ + "tok2vec":[ + + ], + "parser":[ + "ROOT", + "acl", + "acl_bunsetu", + "advcl", + "advcl_bunsetu", + "advmod", + "advmod_bunsetu", + "amod", + "amod_bunsetu", + "aux", + "aux_bunsetu", + "case", + "case_bunsetu", + "cc", + "cc_bunsetu", + "ccomp_bunsetu", + "compound", + "compound_bunsetu", + "cop", + "csubj_bunsetu", + "dep", + "dep_bunsetu", + "det_bunsetu", + "discourse_bunsetu", + "dislocated_bunsetu", + "fixed", + "mark", + "nmod", + "nmod_bunsetu", + "nsubj_bunsetu", + "nummod", + "obj_bunsetu", + "obl_bunsetu", + "punct", + "punct_bunsetu" + ], + "attribute_ruler":[ + + ], + "ner":[ + "Academic", + "Age", + "Aircraft", + "Airport", + "Amphibia", + "Amusement_Park", + "Animal_Disease", + "Animal_Part", + "Archaeological_Place_Other", + "Art_Other", + "Astral_Body_Other", + "Award", + "Bay", + "Bird", + "Book", + "Bridge", + "Broadcast_Program", + "Cabinet", + "Calorie", + "Canal", + "Car", + "Car_Stop", + "Character", + "City", + "Class", + "Clothing", + "Color_Other", + "Company", + "Company_Group", + "Compound", + "Conference", + "Constellation", + "Continental_Region", + "Corporation_Other", + "Country", + "Countx_Other", + "County", + "Culture", + "Date", + "Day_Of_Week", + "Disease_Other", + "Dish", + "Doctrine_Method_Other", + "Domestic_Region", + "Drug", + "Earthquake", + "Element", + "Email", + "Era", + "Ethnic_Group_Other", + "Event_Other", + "Facility_Other", + "Facility_Part", + "Family", + "Fish", + "Flora", + "Flora_Part", + "Food_Other", + "Frequency", + "Fungus", + "GOE_Other", + "GPE_Other", + "Game", + "Geological_Region_Other", + "God", + "Government", + "ID_Number", + "Incident_Other", + "Insect", + "Intensity", + "International_Organization", + "Island", + "Lake", + "Language_Other", + "Latitude_Longtitude", + "Law", + "Line_Other", + "Living_Thing_Other", + "Living_Thing_Part_Other", + "Location_Other", + "Magazine", + "Mammal", + "Material", + "Measurement_Other", + "Military", + "Mineral", + "Mollusc_Arthropod", + "Money", + "Money_Form", + "Mountain", + "Movement", + "Movie", + "Multiplication", + "Museum", + "Music", + "N_Animal", + "N_Country", + "N_Event", + "N_Facility", + "N_Flora", + "N_Location_Other", + "N_Natural_Object_Other", + "N_Organization", + "N_Person", + "N_Product", + "Name_Other", + "National_Language", + "Nationality", + "Natural_Disaster", + "Natural_Object_Other", + "Natural_Phenomenon_Other", + "Nature_Color", + "Newspaper", + "Numex_Other", + "Occasion_Other", + "Offense", + "Ordinal_Number", + "Organization_Other", + "Park", + "Percent", + "Period_Day", + "Period_Month", + "Period_Time", + "Period_Week", + "Period_Year", + "Periodx_Other", + "Person", + "Phone_Number", + "Physical_Extent", + "Plan", + "Planet", + "Point", + "Political_Organization_Other", + "Political_Party", + "Port", + "Position_Vocation", + "Postal_Address", + "Printing_Other", + "Pro_Sports_Organization", + "Product_Other", + "Province", + "Public_Institution", + "Railroad", + "Rank", + "Region_Other", + "Religion", + "Religious_Festival", + "Reptile", + "Research_Institute", + "River", + "Road", + "Rule_Other", + "School", + "School_Age", + "Sea", + "Ship", + "Show", + "Show_Organization", + "Spa", + "Space", + "Spaceship", + "Speed", + "Sport", + "Sports_Facility", + "Sports_League", + "Sports_Organization_Other", + "Station", + "Style", + "Temperature", + "Theater", + "Theory", + "Time", + "Time_Top_Other", + "Timex_Other", + "Title_Other", + "Train", + "Treaty", + "Tumulus", + "Tunnel", + "URL", + "Unit_Other", + "Vehicle_Other", + "Volume", + "War", + "Water_Route", + "Weapon", + "Weight", + "Worship_Place", + "Zoo" + ], + "morphologizer":[ + "POS=PUNCT", + "POS=NUM", + "POS=NOUN", + "POS=ADP", + "POS=AUX", + "POS=VERB", + "POS=CCONJ", + "POS=PART", + "POS=SCONJ", + "POS=SYM", + "POS=ADJ", + "POS=DET", + "POS=PRON", + "POS=PROPN", + "POS=ADV", + "POS=X", + "POS=INTJ" + ] + }, + "performance":{ + "dep_uas":0.9094703175, + "dep_las":0.8906500624, + "dep_las_per_type":{ + "compound":{ + "p":0.8789917474, + "r":0.8754969846, + "f":0.8772408854 + }, + "nummod":{ + "p":0.9176536943, + "r":0.9297142857, + "f":0.9236446211 + }, + "dep_bunsetu":{ + "p":0.5923009624, + "r":0.4596062458, + "f":0.5175840979 + }, + "obl_bunsetu":{ + "p":0.7597479993, + "r":0.7881303541, + "f":0.7736789631 + }, + "case":{ + "p":0.9777676951, + "r":0.9813793452, + "f":0.9795701912 + }, + "acl_bunsetu":{ + "p":0.8585409253, + "r":0.8310607148, + "f":0.8445773467 + }, + "nmod_bunsetu":{ + "p":0.8316480124, + "r":0.8108067429, + "f":0.8210951492 + }, + "root":{ + "p":0.8984536082, + "r":0.8987315665, + "f":0.8985925659 + }, + "fixed":{ + "p":0.9814903616, + "r":0.9711520213, + "f":0.976293823 + }, + "obj_bunsetu":{ + "p":0.929671666, + "r":0.9304259635, + "f":0.9300486618 + }, + "advcl_bunsetu":{ + "p":0.7171969968, + "r":0.7246237207, + "f":0.7208912314 + }, + "aux":{ + "p":0.9682417647, + "r":0.9665840123, + "f":0.9674121783 + }, + "nsubj_bunsetu":{ + "p":0.8207343413, + "r":0.8131603584, + "f":0.8169297951 + }, + "cc_bunsetu":{ + "p":0.762728146, + "r":0.7338262477, + "f":0.7479981159 + }, + "det_bunsetu":{ + "p":0.94644506, + "r":0.9490740741, + "f":0.9477577439 + }, + "advmod_bunsetu":{ + "p":0.7511450382, + "r":0.7472660996, + "f":0.7492005482 + }, + "cop":{ + "p":0.9463980881, + "r":0.9070680628, + "f":0.9263157895 + }, + "amod":{ + "p":0.6976744186, + "r":0.7114624506, + "f":0.7045009785 + }, + "dep":{ + "p":0.5112923463, + "r":0.6791666667, + "f":0.583392985 + }, + "nmod":{ + "p":0.8125, + "r":0.8227848101, + "f":0.8176100629 + }, + "mark":{ + "p":0.9562554681, + "r":0.9594605379, + "f":0.9578553219 + }, + "cc":{ + "p":0.7894736842, + "r":0.6, + "f":0.6818181818 + }, + "acl":{ + "p":1.0, + "r":0.625, + "f":0.7692307692 + }, + "compound_bunsetu":{ + "p":0.3410852713, + "r":0.2056074766, + "f":0.2565597668 + }, + "advcl":{ + "p":0.7407407407, + "r":0.7659574468, + "f":0.7531380753 + }, + "dislocated_bunsetu":{ + "p":0.5149700599, + "r":0.5341614907, + "f":0.5243902439 + }, + "ccomp_bunsetu":{ + "p":0.9301310044, + "r":0.8469184891, + "f":0.8865764828 + }, + "advmod":{ + "p":0.4523809524, + "r":0.3275862069, + "f":0.38 + }, + "csubj_bunsetu":{ + "p":0.786407767, + "r":0.7264573991, + "f":0.7552447552 + }, + "amod_bunsetu":{ + "p":0.9341317365, + "r":0.901734104, + "f":0.9176470588 + }, + "punct_bunsetu":{ + "p":0.6527777778, + "r":0.324137931, + "f":0.4331797235 + }, + "discourse_bunsetu":{ + "p":0.7083333333, + "r":0.2615384615, + "f":0.3820224719 + }, + "case_bunsetu":{ + "p":0.25, + "r":0.1538461538, + "f":0.1904761905 + }, + "aux_bunsetu":{ + "p":0.6666666667, + "r":0.1333333333, + "f":0.2222222222 + }, + "nummod_bunsetu":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "discourse":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "det":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "reparandum":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "reparandum_bunsetu":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "cop_bunsetu":{ + "p":0.0, + "r":0.0, + "f":0.0 + } + }, + "sents_p":0.8302567804, + "sents_r":0.8302567804, + "sents_f":0.8302567804, + "ents_f":0.554029304, + "ents_p":0.58374206, + "ents_r":0.5271948297, + "ents_per_type":{ + "Ordinal_Number":{ + "p":0.7529538131, + "r":0.7644492912, + "f":0.7586580087 + }, + "Nationality":{ + "p":0.7111111111, + "r":0.7804878049, + "f":0.7441860465 + }, + "Date":{ + "p":0.747582205, + "r":0.8052083333, + "f":0.7753259779 + }, + "Government":{ + "p":0.5654761905, + "r":0.5757575758, + "f":0.5705705706 + }, + "Law":{ + "p":0.3703703704, + "r":0.119047619, + "f":0.1801801802 + }, + "Doctrine_Method_Other":{ + "p":0.3051643192, + "r":0.1666666667, + "f":0.2155887231 + }, + "Time":{ + "p":0.5720524017, + "r":0.6064814815, + "f":0.5887640449 + }, + "Company":{ + "p":0.3211382114, + "r":0.3333333333, + "f":0.3271221532 + }, + "Occasion_Other":{ + "p":0.3, + "r":0.1621621622, + "f":0.2105263158 + }, + "Product_Other":{ + "p":0.325, + "r":0.2596291013, + "f":0.2886597938 + }, + "Money":{ + "p":0.6923076923, + "r":0.6428571429, + "f":0.6666666667 + }, + "Movement":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Plan":{ + "p":0.4, + "r":0.2556390977, + "f":0.3119266055 + }, + "Period_Time":{ + "p":0.8023255814, + "r":0.6571428571, + "f":0.722513089 + }, + "Public_Institution":{ + "p":0.4285714286, + "r":0.3, + "f":0.3529411765 + }, + "National_Language":{ + "p":0.8055555556, + "r":0.8787878788, + "f":0.8405797101 + }, + "Corporation_Other":{ + "p":0.2777777778, + "r":0.2325581395, + "f":0.253164557 + }, + "Natural_Disaster":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Position_Vocation":{ + "p":0.5828729282, + "r":0.5255292653, + "f":0.5527177472 + }, + "Organization_Other":{ + "p":0.2878787879, + "r":0.2602739726, + "f":0.273381295 + }, + "Country":{ + "p":0.8246445498, + "r":0.807424594, + "f":0.815943728 + }, + "Event_Other":{ + "p":0.4848484848, + "r":0.253968254, + "f":0.3333333333 + }, + "Political_Organization_Other":{ + "p":0.24, + "r":0.375, + "f":0.2926829268 + }, + "Incident_Other":{ + "p":0.2857142857, + "r":0.25, + "f":0.2666666667 + }, + "War":{ + "p":0.5, + "r":0.4, + "f":0.4444444444 + }, + "Military":{ + "p":0.25, + "r":0.1, + "f":0.1428571429 + }, + "International_Organization":{ + "p":0.4172661871, + "r":0.3391812865, + "f":0.3741935484 + }, + "Treaty":{ + "p":0.7636363636, + "r":0.375, + "f":0.502994012 + }, + "Continental_Region":{ + "p":0.5652173913, + "r":0.5652173913, + "f":0.5652173913 + }, + "Person":{ + "p":0.7507352941, + "r":0.6413316583, + "f":0.6917344173 + }, + "N_Person":{ + "p":0.8272425249, + "r":0.8217821782, + "f":0.8245033113 + }, + "Drug":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Conference":{ + "p":0.64, + "r":0.237037037, + "f":0.3459459459 + }, + "Period_Year":{ + "p":0.7639751553, + "r":0.7735849057, + "f":0.76875 + }, + "Earthquake":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "N_Product":{ + "p":0.3745318352, + "r":0.5291005291, + "f":0.4385964912 + }, + "GPE_Other":{ + "p":0.6, + "r":0.1525423729, + "f":0.2432432432 + }, + "N_Country":{ + "p":0.641509434, + "r":0.6666666667, + "f":0.6538461538 + }, + "Pro_Sports_Organization":{ + "p":0.1034482759, + "r":0.5, + "f":0.1714285714 + }, + "City":{ + "p":0.4060150376, + "r":0.5427135678, + "f":0.464516129 + }, + "Percent":{ + "p":0.8699186992, + "r":0.8794520548, + "f":0.8746594005 + }, + "Point":{ + "p":0.2058823529, + "r":0.1944444444, + "f":0.2 + }, + "Countx_Other":{ + "p":0.1145833333, + "r":0.1774193548, + "f":0.1392405063 + }, + "Printing_Other":{ + "p":0.747826087, + "r":0.6935483871, + "f":0.719665272 + }, + "N_Event":{ + "p":0.28, + "r":0.2333333333, + "f":0.2545454545 + }, + "Political_Party":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Fish":{ + "p":0.1, + "r":0.1111111111, + "f":0.1052631579 + }, + "Age":{ + "p":0.6516129032, + "r":0.5804597701, + "f":0.6139817629 + }, + "Physical_Extent":{ + "p":0.3571428571, + "r":0.5128205128, + "f":0.4210526316 + }, + "Broadcast_Program":{ + "p":0.2545454545, + "r":0.1386138614, + "f":0.1794871795 + }, + "Magazine":{ + "p":0.0588235294, + "r":0.0714285714, + "f":0.064516129 + }, + "Aircraft":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Weapon":{ + "p":0.5135135135, + "r":0.2923076923, + "f":0.3725490196 + }, + "Ship":{ + "p":0.1764705882, + "r":0.1304347826, + "f":0.15 + }, + "N_Organization":{ + "p":0.3214285714, + "r":0.1956521739, + "f":0.2432432432 + }, + "Research_Institute":{ + "p":0.8, + "r":0.2352941176, + "f":0.3636363636 + }, + "Weight":{ + "p":0.652173913, + "r":0.5555555556, + "f":0.6 + }, + "Game":{ + "p":0.0714285714, + "r":0.0377358491, + "f":0.049382716 + }, + "Car":{ + "p":0.3225806452, + "r":0.2272727273, + "f":0.2666666667 + }, + "God":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Province":{ + "p":0.4736842105, + "r":0.8076923077, + "f":0.5971563981 + }, + "Mollusc_Arthropod":{ + "p":0.1666666667, + "r":0.0714285714, + "f":0.1 + }, + "Numex_Other":{ + "p":0.2075471698, + "r":0.1803278689, + "f":0.1929824561 + }, + "Facility_Other":{ + "p":0.3333333333, + "r":0.1428571429, + "f":0.2 + }, + "Region_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Timex_Other":{ + "p":0.2, + "r":0.2, + "f":0.2 + }, + "Insect":{ + "p":0.1818181818, + "r":0.1666666667, + "f":0.1739130435 + }, + "Compound":{ + "p":0.55, + "r":0.5, + "f":0.5238095238 + }, + "Material":{ + "p":0.21875, + "r":0.2258064516, + "f":0.2222222222 + }, + "Natural_Phenomenon_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Element":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Rank":{ + "p":0.6593406593, + "r":0.6896551724, + "f":0.6741573034 + }, + "Class":{ + "p":0.2972972973, + "r":0.2682926829, + "f":0.2820512821 + }, + "Offense":{ + "p":0.5434782609, + "r":0.5747126437, + "f":0.5586592179 + }, + "Show_Organization":{ + "p":0.0317460317, + "r":0.064516129, + "f":0.0425531915 + }, + "Period_Month":{ + "p":0.4814814815, + "r":0.8387096774, + "f":0.6117647059 + }, + "School":{ + "p":0.3965517241, + "r":0.2839506173, + "f":0.3309352518 + }, + "School_Age":{ + "p":0.4838709677, + "r":0.4285714286, + "f":0.4545454545 + }, + "N_Facility":{ + "p":0.2666666667, + "r":0.1052631579, + "f":0.1509433962 + }, + "Music":{ + "p":0.0350877193, + "r":0.0526315789, + "f":0.0421052632 + }, + "Title_Other":{ + "p":0.8088737201, + "r":0.7953020134, + "f":0.8020304569 + }, + "Book":{ + "p":0.3037974684, + "r":0.2926829268, + "f":0.298136646 + }, + "Stock":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Name_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Food_Other":{ + "p":0.3333333333, + "r":0.2613636364, + "f":0.2929936306 + }, + "URL":{ + "p":0.8, + "r":0.8, + "f":0.8 + }, + "GOE_Other":{ + "p":0.2054794521, + "r":0.1578947368, + "f":0.1785714286 + }, + "Style":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "ID_Number":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Era":{ + "p":0.5555555556, + "r":0.8, + "f":0.6557377049 + }, + "Sports_Organization_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Frequency":{ + "p":0.597826087, + "r":0.7432432432, + "f":0.6626506024 + }, + "Rule_Other":{ + "p":0.5, + "r":0.0833333333, + "f":0.1428571429 + }, + "Color_Other":{ + "p":0.1428571429, + "r":0.25, + "f":0.1818181818 + }, + "Domestic_Region":{ + "p":0.6315789474, + "r":0.1666666667, + "f":0.2637362637 + }, + "Animal_Disease":{ + "p":0.5733333333, + "r":0.5657894737, + "f":0.5695364238 + }, + "Clothing":{ + "p":0.4308943089, + "r":0.6235294118, + "f":0.5096153846 + }, + "Flora":{ + "p":0.26, + "r":0.1940298507, + "f":0.2222222222 + }, + "Animal_Part":{ + "p":0.7411167513, + "r":0.6728110599, + "f":0.7053140097 + }, + "Mineral":{ + "p":0.25, + "r":0.2, + "f":0.2222222222 + }, + "Period_Day":{ + "p":0.5757575758, + "r":0.6440677966, + "f":0.608 + }, + "Facility_Part":{ + "p":0.0731707317, + "r":0.0652173913, + "f":0.0689655172 + }, + "N_Flora":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Mammal":{ + "p":0.7741935484, + "r":0.6545454545, + "f":0.7093596059 + }, + "N_Animal":{ + "p":0.3333333333, + "r":0.5, + "f":0.4 + }, + "Dish":{ + "p":0.5044247788, + "r":0.5377358491, + "f":0.5205479452 + }, + "Nature_Color":{ + "p":0.6585365854, + "r":0.6585365854, + "f":0.6585365854 + }, + "Flora_Part":{ + "p":0.5952380952, + "r":0.5102040816, + "f":0.5494505495 + }, + "Geological_Region_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Religious_Festival":{ + "p":0.2, + "r":0.1538461538, + "f":0.1739130435 + }, + "N_Location_Other":{ + "p":0.1, + "r":0.2142857143, + "f":0.1363636364 + }, + "Road":{ + "p":0.6666666667, + "r":0.3703703704, + "f":0.4761904762 + }, + "Money_Form":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Ethnic_Group_Other":{ + "p":0.4666666667, + "r":0.3043478261, + "f":0.3684210526 + }, + "Measurement_Other":{ + "p":0.25, + "r":0.2, + "f":0.2222222222 + }, + "Sea":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Amphibia":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Academic":{ + "p":0.5652173913, + "r":0.2708333333, + "f":0.3661971831 + }, + "N_Natural_Object_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Location_Other":{ + "p":0.1111111111, + "r":0.0833333333, + "f":0.0952380952 + }, + "Railroad":{ + "p":0.4545454545, + "r":0.3333333333, + "f":0.3846153846 + }, + "Port":{ + "p":0.0833333333, + "r":0.2, + "f":0.1176470588 + }, + "Station":{ + "p":0.5, + "r":0.4, + "f":0.4444444444 + }, + "Periodx_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Archaeological_Place_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Bridge":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "River":{ + "p":0.3333333333, + "r":0.6, + "f":0.4285714286 + }, + "Period_Week":{ + "p":0.8421052632, + "r":0.6956521739, + "f":0.7619047619 + }, + "Space":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Amusement_Park":{ + "p":0.5862068966, + "r":0.3469387755, + "f":0.4358974359 + }, + "Character":{ + "p":0.1111111111, + "r":0.0425531915, + "f":0.0615384615 + }, + "Day_Of_Week":{ + "p":0.62, + "r":0.6458333333, + "f":0.6326530612 + }, + "Worship_Place":{ + "p":0.4444444444, + "r":0.1538461538, + "f":0.2285714286 + }, + "Email":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Speed":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Volume":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Phone_Number":{ + "p":0.9333333333, + "r":1.0, + "f":0.9655172414 + }, + "Postal_Address":{ + "p":0.25, + "r":0.0526315789, + "f":0.0869565217 + }, + "Show":{ + "p":0.4, + "r":0.2, + "f":0.2666666667 + }, + "Constellation":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Theater":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Planet":{ + "p":0.6666666667, + "r":0.5, + "f":0.5714285714 + }, + "Religion":{ + "p":0.2, + "r":0.2222222222, + "f":0.2105263158 + }, + "Airport":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Train":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Island":{ + "p":0.375, + "r":1.0, + "f":0.5454545455 + }, + "Living_Thing_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Living_Thing_Part_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Canal":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Culture":{ + "p":0.25, + "r":0.1111111111, + "f":0.1538461538 + }, + "Bay":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Sport":{ + "p":0.4736842105, + "r":0.4, + "f":0.4337349398 + }, + "Movie":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Newspaper":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Award":{ + "p":0.4, + "r":0.25, + "f":0.3076923077 + }, + "Theory":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Family":{ + "p":0.5, + "r":0.2857142857, + "f":0.3636363636 + }, + "Multiplication":{ + "p":0.6666666667, + "r":0.4, + "f":0.5 + }, + "Vehicle_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Natural_Object_Other":{ + "p":1.0, + "r":0.5, + "f":0.6666666667 + }, + "Park":{ + "p":0.5, + "r":0.4, + "f":0.4444444444 + }, + "Mountain":{ + "p":0.2857142857, + "r":0.3333333333, + "f":0.3076923077 + }, + "Reptile":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Lake":{ + "p":1.0, + "r":1.0, + "f":1.0 + }, + "Sports_Facility":{ + "p":0.2857142857, + "r":0.0769230769, + "f":0.1212121212 + }, + "Spa":{ + "p":0.8333333333, + "r":0.625, + "f":0.7142857143 + }, + "Tunnel":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Art_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Market":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Bird":{ + "p":1.0, + "r":0.2777777778, + "f":0.4347826087 + }, + "Seismic_Intensity":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Spaceship":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Car_Stop":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Museum":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "County":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Company_Group":{ + "p":1.0, + "r":0.5, + "f":0.6666666667 + }, + "Stock_Index":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Picture":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Zoo":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Temperature":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Fungus":{ + "p":0.0, + "r":0.0, + "f":0.0 + } + }, + "pos_acc":0.9743697141, + "morph_micro_f":0.0, + "morph_per_feat":0.0, + "morph_acc":0.0, + "tag_acc":0.0, + "tok2vec_loss":204759.3473588387, + "parser_loss":8554.9184086516, + "ner_loss":1241.1498988768, + "morphologizer_loss":267.1441991855 + } } diff --git a/config/ja_ginza_bert_large.meta.json b/config/ja_ginza_bert_large.meta.json index d7973b8..cd8cea5 100644 --- a/config/ja_ginza_bert_large.meta.json +++ b/config/ja_ginza_bert_large.meta.json @@ -1,7 +1,7 @@ { "lang":"ja", "name":"ginza_bert_large", - "version":"5.2.0b1", + "version":"5.3.0", "description":"Japanese multi-task CNN trained on UD-Japanese BCCWJ r2.8 + GSK2014-A(2019) + transformers-ud-japanese-electra--base. Components: transformer, parser, atteribute_ruler, ner, morphologizer, compound_splitter, bunsetu_recognizer.", "author":"Megagon Labs Tokyo.", "email":"ginza@megagon.ai", @@ -33,7 +33,7 @@ "author":"Tohoku University" } ], - "spacy_version":">=3.6.1,<4.0.0", + "spacy_version":">=3.8.16,<4.0.0", "spacy_git_version":"458bc5f45", "pipeline":[ "transformer", @@ -63,11 +63,1368 @@ "mode":"default" }, "requirements":[ - "sudachipy>=0.6.7,<0.7.0", + "sudachipy>=0.6.11,<0.7.0", "sudachidict_core>=20230711", - "spacy-transformers>=1.2.5,<1.3.0", - "fugashi>=1.3.0", + "ginza-transformers>=1.4.0,<1.5.0", + "fugashi>=1.5.2", "unidic-lite>=1.0.8", - "ginza>=5.2.0,<5.3.0" - ] -} \ No newline at end of file + "ginza>=5.3.0,<6.0.0" + ], + "labels":{ + "transformer":[ + + ], + "parser":[ + "ROOT", + "acl", + "acl_bunsetu", + "advcl", + "advcl_bunsetu", + "advmod", + "advmod_bunsetu", + "amod", + "amod_bunsetu", + "aux", + "aux_bunsetu", + "case", + "case_bunsetu", + "cc", + "cc_bunsetu", + "ccomp_bunsetu", + "compound", + "compound_bunsetu", + "cop", + "csubj_bunsetu", + "dep", + "dep_bunsetu", + "det_bunsetu", + "discourse_bunsetu", + "dislocated_bunsetu", + "fixed", + "mark", + "nmod", + "nmod_bunsetu", + "nsubj_bunsetu", + "nummod", + "obj_bunsetu", + "obl_bunsetu", + "punct", + "punct_bunsetu" + ], + "attribute_ruler":[ + + ], + "ner":[ + "Academic", + "Age", + "Aircraft", + "Airport", + "Amphibia", + "Amusement_Park", + "Animal_Disease", + "Animal_Part", + "Archaeological_Place_Other", + "Art_Other", + "Astral_Body_Other", + "Award", + "Bay", + "Bird", + "Book", + "Bridge", + "Broadcast_Program", + "Cabinet", + "Calorie", + "Canal", + "Car", + "Car_Stop", + "Character", + "City", + "Class", + "Clothing", + "Color_Other", + "Company", + "Company_Group", + "Compound", + "Conference", + "Constellation", + "Continental_Region", + "Corporation_Other", + "Country", + "Countx_Other", + "County", + "Culture", + "Date", + "Day_Of_Week", + "Disease_Other", + "Dish", + "Doctrine_Method_Other", + "Domestic_Region", + "Drug", + "Earthquake", + "Element", + "Email", + "Era", + "Ethnic_Group_Other", + "Event_Other", + "Facility_Other", + "Facility_Part", + "Family", + "Fish", + "Flora", + "Flora_Part", + "Food_Other", + "Frequency", + "Fungus", + "GOE_Other", + "GPE_Other", + "Game", + "Geological_Region_Other", + "God", + "Government", + "ID_Number", + "Incident_Other", + "Insect", + "Intensity", + "International_Organization", + "Island", + "Lake", + "Language_Other", + "Latitude_Longtitude", + "Law", + "Line_Other", + "Living_Thing_Other", + "Living_Thing_Part_Other", + "Location_Other", + "Magazine", + "Mammal", + "Material", + "Measurement_Other", + "Military", + "Mineral", + "Mollusc_Arthropod", + "Money", + "Money_Form", + "Mountain", + "Movement", + "Movie", + "Multiplication", + "Museum", + "Music", + "N_Animal", + "N_Country", + "N_Event", + "N_Facility", + "N_Flora", + "N_Location_Other", + "N_Natural_Object_Other", + "N_Organization", + "N_Person", + "N_Product", + "Name_Other", + "National_Language", + "Nationality", + "Natural_Disaster", + "Natural_Object_Other", + "Natural_Phenomenon_Other", + "Nature_Color", + "Newspaper", + "Numex_Other", + "Occasion_Other", + "Offense", + "Ordinal_Number", + "Organization_Other", + "Park", + "Percent", + "Period_Day", + "Period_Month", + "Period_Time", + "Period_Week", + "Period_Year", + "Periodx_Other", + "Person", + "Phone_Number", + "Physical_Extent", + "Plan", + "Planet", + "Point", + "Political_Organization_Other", + "Political_Party", + "Port", + "Position_Vocation", + "Postal_Address", + "Printing_Other", + "Pro_Sports_Organization", + "Product_Other", + "Province", + "Public_Institution", + "Railroad", + "Rank", + "Region_Other", + "Religion", + "Religious_Festival", + "Reptile", + "Research_Institute", + "River", + "Road", + "Rule_Other", + "School", + "School_Age", + "Sea", + "Ship", + "Show", + "Show_Organization", + "Spa", + "Space", + "Spaceship", + "Speed", + "Sport", + "Sports_Facility", + "Sports_League", + "Sports_Organization_Other", + "Station", + "Style", + "Temperature", + "Theater", + "Theory", + "Time", + "Time_Top_Other", + "Timex_Other", + "Title_Other", + "Train", + "Treaty", + "Tumulus", + "Tunnel", + "URL", + "Unit_Other", + "Vehicle_Other", + "Volume", + "War", + "Water_Route", + "Weapon", + "Weight", + "Worship_Place", + "Zoo" + ], + "morphologizer":[ + "POS=PUNCT", + "POS=NUM", + "POS=NOUN", + "POS=ADP", + "POS=AUX", + "POS=VERB", + "POS=CCONJ", + "POS=PART", + "POS=SCONJ", + "POS=SYM", + "POS=ADJ", + "POS=DET", + "POS=PRON", + "POS=PROPN", + "POS=ADV", + "POS=X", + "POS=INTJ" + ] + }, + "performance":{ + "dep_uas":0.94479758, + "dep_las":0.9335430878, + "dep_las_per_type":{ + "compound":{ + "p":0.9342807191, + "r":0.9310252401, + "f":0.9326501387 + }, + "nummod":{ + "p":0.9681712963, + "r":0.956, + "f":0.9620471535 + }, + "dep_bunsetu":{ + "p":0.640839971, + "r":0.600814664, + "f":0.6201822004 + }, + "obl_bunsetu":{ + "p":0.8712101123, + "r":0.8704406959, + "f":0.8708252341 + }, + "case":{ + "p":0.9895465963, + "r":0.9867176036, + "f":0.9881300751 + }, + "acl_bunsetu":{ + "p":0.9200640652, + "r":0.9069900962, + "f":0.9134803036 + }, + "nmod_bunsetu":{ + "p":0.9012672005, + "r":0.8724413004, + "f":0.8866200145 + }, + "root":{ + "p":0.9354904982, + "r":0.9391564401, + "f":0.9373198847 + }, + "fixed":{ + "p":0.9822866345, + "r":0.984057696, + "f":0.9831713676 + }, + "obj_bunsetu":{ + "p":0.9617185909, + "r":0.9580121704, + "f":0.9598618027 + }, + "advcl_bunsetu":{ + "p":0.8284284523, + "r":0.8372065021, + "f":0.8327943466 + }, + "aux":{ + "p":0.9829827464, + "r":0.982170268, + "f":0.9825763393 + }, + "nsubj_bunsetu":{ + "p":0.8841766206, + "r":0.8811020463, + "f":0.8826366559 + }, + "cc_bunsetu":{ + "p":0.8548387097, + "r":0.8327171904, + "f":0.8436329588 + }, + "det_bunsetu":{ + "p":0.951627907, + "r":0.9472222222, + "f":0.9494199536 + }, + "advmod_bunsetu":{ + "p":0.8421210306, + "r":0.8538882139, + "f":0.8479638009 + }, + "cop":{ + "p":0.9657556799, + "r":0.9597513089, + "f":0.9627441326 + }, + "amod":{ + "p":0.8620689655, + "r":0.790513834, + "f":0.824742268 + }, + "dep":{ + "p":0.7030075188, + "r":0.7791666667, + "f":0.7391304348 + }, + "nmod":{ + "p":0.8563734291, + "r":0.8625678119, + "f":0.8594594595 + }, + "mark":{ + "p":0.9791816224, + "r":0.9796504668, + "f":0.9794159885 + }, + "acl":{ + "p":1.0, + "r":0.75, + "f":0.8571428571 + }, + "advcl":{ + "p":0.8553191489, + "r":0.8553191489, + "f":0.8553191489 + }, + "compound_bunsetu":{ + "p":0.6453900709, + "r":0.4252336449, + "f":0.5126760563 + }, + "dislocated_bunsetu":{ + "p":0.6928571429, + "r":0.602484472, + "f":0.6445182724 + }, + "ccomp_bunsetu":{ + "p":0.9179389313, + "r":0.9562624254, + "f":0.9367088608 + }, + "advmod":{ + "p":0.5454545455, + "r":0.5172413793, + "f":0.5309734513 + }, + "csubj_bunsetu":{ + "p":0.8584474886, + "r":0.8430493274, + "f":0.850678733 + }, + "amod_bunsetu":{ + "p":0.9545454545, + "r":0.9710982659, + "f":0.9627507163 + }, + "punct_bunsetu":{ + "p":0.8795180723, + "r":0.7551724138, + "f":0.8126159555 + }, + "discourse_bunsetu":{ + "p":0.5757575758, + "r":0.5846153846, + "f":0.5801526718 + }, + "case_bunsetu":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "cc":{ + "p":0.8823529412, + "r":0.6, + "f":0.7142857143 + }, + "aux_bunsetu":{ + "p":0.8571428571, + "r":0.4, + "f":0.5454545455 + }, + "nummod_bunsetu":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "discourse":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "det":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "reparandum":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "reparandum_bunsetu":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "cop_bunsetu":{ + "p":0.0, + "r":0.0, + "f":0.0 + } + }, + "sents_p":0.897278494, + "sents_r":0.9044034237, + "sents_f":0.9008268707, + "ents_f":0.7111680703, + "ents_p":0.7172612453, + "ents_r":0.705177547, + "ents_per_type":{ + "Ordinal_Number":{ + "p":0.8112840467, + "r":0.9094874591, + "f":0.8575835476 + }, + "Nationality":{ + "p":0.6346153846, + "r":0.8048780488, + "f":0.7096774194 + }, + "Date":{ + "p":0.8476755687, + "r":0.8927083333, + "f":0.8696093354 + }, + "Public_Institution":{ + "p":0.4857142857, + "r":0.5666666667, + "f":0.5230769231 + }, + "Government":{ + "p":0.5625, + "r":0.7636363636, + "f":0.64781491 + }, + "Law":{ + "p":0.8550724638, + "r":0.7023809524, + "f":0.7712418301 + }, + "Doctrine_Method_Other":{ + "p":0.4939759036, + "r":0.4205128205, + "f":0.4542936288 + }, + "Product_Other":{ + "p":0.5019505852, + "r":0.5506419401, + "f":0.525170068 + }, + "Event_Other":{ + "p":0.59375, + "r":0.3015873016, + "f":0.4 + }, + "Occasion_Other":{ + "p":0.2950819672, + "r":0.2432432432, + "f":0.2666666667 + }, + "Time":{ + "p":0.6604651163, + "r":0.6574074074, + "f":0.6589327146 + }, + "Movement":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Money":{ + "p":0.8136645963, + "r":0.7797619048, + "f":0.7963525836 + }, + "Plan":{ + "p":0.8037383178, + "r":0.6466165414, + "f":0.7166666667 + }, + "Period_Time":{ + "p":0.8045977011, + "r":0.6666666667, + "f":0.7291666667 + }, + "National_Language":{ + "p":0.8055555556, + "r":0.8787878788, + "f":0.8405797101 + }, + "Corporation_Other":{ + "p":0.435483871, + "r":0.6279069767, + "f":0.5142857143 + }, + "Natural_Disaster":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Position_Vocation":{ + "p":0.7450058754, + "r":0.7895392279, + "f":0.7666263603 + }, + "Organization_Other":{ + "p":0.5555555556, + "r":0.4109589041, + "f":0.4724409449 + }, + "Country":{ + "p":0.9013761468, + "r":0.9118329466, + "f":0.9065743945 + }, + "Political_Organization_Other":{ + "p":0.5384615385, + "r":0.4375, + "f":0.4827586207 + }, + "War":{ + "p":0.75, + "r":0.6, + "f":0.6666666667 + }, + "Military":{ + "p":0.4444444444, + "r":0.4, + "f":0.4210526316 + }, + "Treaty":{ + "p":0.8351648352, + "r":0.6785714286, + "f":0.7487684729 + }, + "Conference":{ + "p":0.6276595745, + "r":0.437037037, + "f":0.5152838428 + }, + "Continental_Region":{ + "p":0.5529411765, + "r":0.6811594203, + "f":0.6103896104 + }, + "N_Person":{ + "p":0.9124579125, + "r":0.8943894389, + "f":0.9033333333 + }, + "Class":{ + "p":0.4871794872, + "r":0.4634146341, + "f":0.475 + }, + "Person":{ + "p":0.8763227513, + "r":0.8322864322, + "f":0.8537371134 + }, + "Period_Year":{ + "p":0.8280254777, + "r":0.8176100629, + "f":0.8227848101 + }, + "N_Product":{ + "p":0.6282722513, + "r":0.6349206349, + "f":0.6315789474 + }, + "N_Country":{ + "p":0.7555555556, + "r":0.6666666667, + "f":0.7083333333 + }, + "Percent":{ + "p":0.9095890411, + "r":0.9095890411, + "f":0.9095890411 + }, + "City":{ + "p":0.6710526316, + "r":0.7688442211, + "f":0.7166276347 + }, + "International_Organization":{ + "p":0.6, + "r":0.5614035088, + "f":0.580060423 + }, + "Printing_Other":{ + "p":0.9009009009, + "r":0.8064516129, + "f":0.8510638298 + }, + "Countx_Other":{ + "p":0.1714285714, + "r":0.1935483871, + "f":0.1818181818 + }, + "N_Organization":{ + "p":0.6756756757, + "r":0.5434782609, + "f":0.6024096386 + }, + "Weight":{ + "p":0.7407407407, + "r":0.7407407407, + "f":0.7407407407 + }, + "Weapon":{ + "p":0.4912280702, + "r":0.4307692308, + "f":0.4590163934 + }, + "Research_Institute":{ + "p":0.3333333333, + "r":0.1176470588, + "f":0.1739130435 + }, + "Physical_Extent":{ + "p":0.6666666667, + "r":0.7692307692, + "f":0.7142857143 + }, + "Ship":{ + "p":1.0, + "r":0.652173913, + "f":0.7894736842 + }, + "Aircraft":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Rule_Other":{ + "p":0.08, + "r":0.0833333333, + "f":0.0816326531 + }, + "Company_Group":{ + "p":0.4, + "r":0.5, + "f":0.4444444444 + }, + "Measurement_Other":{ + "p":0.3225806452, + "r":0.5, + "f":0.3921568627 + }, + "Compound":{ + "p":0.5333333333, + "r":0.7272727273, + "f":0.6153846154 + }, + "Material":{ + "p":0.3548387097, + "r":0.3548387097, + "f":0.3548387097 + }, + "Natural_Phenomenon_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Facility_Other":{ + "p":0.0833333333, + "r":0.1428571429, + "f":0.1052631579 + }, + "Element":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Academic":{ + "p":0.6818181818, + "r":0.625, + "f":0.652173913 + }, + "Rank":{ + "p":0.7065217391, + "r":0.7471264368, + "f":0.7262569832 + }, + "Numex_Other":{ + "p":0.2112676056, + "r":0.2459016393, + "f":0.2272727273 + }, + "Period_Month":{ + "p":0.431372549, + "r":0.7096774194, + "f":0.5365853659 + }, + "School_Age":{ + "p":0.6571428571, + "r":0.6571428571, + "f":0.6571428571 + }, + "N_Facility":{ + "p":0.4857142857, + "r":0.4473684211, + "f":0.4657534247 + }, + "Company":{ + "p":0.652173913, + "r":0.6329113924, + "f":0.6423982869 + }, + "Age":{ + "p":0.8992248062, + "r":0.6666666667, + "f":0.7656765677 + }, + "Title_Other":{ + "p":0.8919860627, + "r":0.8590604027, + "f":0.8752136752 + }, + "School":{ + "p":0.935483871, + "r":0.7160493827, + "f":0.8111888112 + }, + "Stock":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "GOE_Other":{ + "p":0.5652173913, + "r":0.5473684211, + "f":0.5561497326 + }, + "URL":{ + "p":1.0, + "r":1.0, + "f":1.0 + }, + "Amusement_Park":{ + "p":0.6530612245, + "r":0.6530612245, + "f":0.6530612245 + }, + "Era":{ + "p":0.6774193548, + "r":0.84, + "f":0.75 + }, + "Region_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "GPE_Other":{ + "p":0.476744186, + "r":0.6949152542, + "f":0.5655172414 + }, + "Political_Party":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Offense":{ + "p":0.7323943662, + "r":0.5977011494, + "f":0.6582278481 + }, + "Incident_Other":{ + "p":0.5833333333, + "r":0.4375, + "f":0.5 + }, + "Animal_Disease":{ + "p":0.8529411765, + "r":0.7631578947, + "f":0.8055555556 + }, + "Animal_Part":{ + "p":0.8268792711, + "r":0.83640553, + "f":0.8316151203 + }, + "Mineral":{ + "p":0.5, + "r":0.2, + "f":0.2857142857 + }, + "Flora":{ + "p":0.4247787611, + "r":0.7164179104, + "f":0.5333333333 + }, + "Period_Day":{ + "p":0.6909090909, + "r":0.6440677966, + "f":0.6666666667 + }, + "Mammal":{ + "p":0.7843137255, + "r":0.7272727273, + "f":0.7547169811 + }, + "N_Animal":{ + "p":0.5, + "r":0.5, + "f":0.5 + }, + "Clothing":{ + "p":0.7473684211, + "r":0.8352941176, + "f":0.7888888889 + }, + "Food_Other":{ + "p":0.5409836066, + "r":0.375, + "f":0.4429530201 + }, + "Dish":{ + "p":0.7090909091, + "r":0.7358490566, + "f":0.7222222222 + }, + "Domestic_Region":{ + "p":0.7368421053, + "r":0.3888888889, + "f":0.5090909091 + }, + "Nature_Color":{ + "p":0.75, + "r":0.8048780488, + "f":0.7764705882 + }, + "Flora_Part":{ + "p":0.6875, + "r":0.6734693878, + "f":0.6804123711 + }, + "Car":{ + "p":0.8095238095, + "r":0.3863636364, + "f":0.5230769231 + }, + "Geological_Region_Other":{ + "p":0.4285714286, + "r":0.2142857143, + "f":0.2857142857 + }, + "Province":{ + "p":0.8441558442, + "r":0.8333333333, + "f":0.8387096774 + }, + "Railroad":{ + "p":0.4166666667, + "r":0.6666666667, + "f":0.5128205128 + }, + "Location_Other":{ + "p":0.1666666667, + "r":0.1666666667, + "f":0.1666666667 + }, + "Worship_Place":{ + "p":0.2820512821, + "r":0.4230769231, + "f":0.3384615385 + }, + "Road":{ + "p":0.7777777778, + "r":0.7777777778, + "f":0.7777777778 + }, + "Money_Form":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Ethnic_Group_Other":{ + "p":0.7692307692, + "r":0.4347826087, + "f":0.5555555556 + }, + "Religious_Festival":{ + "p":0.4615384615, + "r":0.4615384615, + "f":0.4615384615 + }, + "Culture":{ + "p":0.25, + "r":0.1111111111, + "f":0.1538461538 + }, + "Music":{ + "p":0.4642857143, + "r":0.3421052632, + "f":0.3939393939 + }, + "Port":{ + "p":0.25, + "r":0.2, + "f":0.2222222222 + }, + "Point":{ + "p":0.3529411765, + "r":0.3333333333, + "f":0.3428571429 + }, + "Periodx_Other":{ + "p":0.375, + "r":0.6, + "f":0.4615384615 + }, + "Vehicle_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Frequency":{ + "p":0.7971014493, + "r":0.7432432432, + "f":0.7692307692 + }, + "Facility_Part":{ + "p":0.2962962963, + "r":0.1739130435, + "f":0.2191780822 + }, + "Bridge":{ + "p":0.3333333333, + "r":0.5, + "f":0.4 + }, + "N_Location_Other":{ + "p":0.2666666667, + "r":0.2857142857, + "f":0.275862069 + }, + "River":{ + "p":1.0, + "r":0.8, + "f":0.8888888889 + }, + "N_Natural_Object_Other":{ + "p":0.2857142857, + "r":0.1666666667, + "f":0.2105263158 + }, + "Period_Week":{ + "p":0.7826086957, + "r":0.7826086957, + "f":0.7826086957 + }, + "Space":{ + "p":0.5, + "r":0.4, + "f":0.4444444444 + }, + "Constellation":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "N_Event":{ + "p":0.4782608696, + "r":0.3666666667, + "f":0.4150943396 + }, + "Day_Of_Week":{ + "p":0.7142857143, + "r":0.8333333333, + "f":0.7692307692 + }, + "Book":{ + "p":0.3644067797, + "r":0.5243902439, + "f":0.43 + }, + "Character":{ + "p":0.4444444444, + "r":0.3404255319, + "f":0.3855421687 + }, + "Name_Other":{ + "p":0.5897435897, + "r":0.4339622642, + "f":0.5 + }, + "Email":{ + "p":1.0, + "r":1.0, + "f":1.0 + }, + "Phone_Number":{ + "p":0.9333333333, + "r":1.0, + "f":0.9655172414 + }, + "Theater":{ + "p":0.375, + "r":0.375, + "f":0.375 + }, + "Fish":{ + "p":1.0, + "r":0.8888888889, + "f":0.9411764706 + }, + "Planet":{ + "p":0.75, + "r":0.75, + "f":0.75 + }, + "Game":{ + "p":0.5909090909, + "r":0.4905660377, + "f":0.5360824742 + }, + "Award":{ + "p":0.8888888889, + "r":1.0, + "f":0.9411764706 + }, + "Religion":{ + "p":0.8888888889, + "r":0.8888888889, + "f":0.8888888889 + }, + "Speed":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Airport":{ + "p":0.1, + "r":0.05, + "f":0.0666666667 + }, + "Train":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Living_Thing_Other":{ + "p":0.3333333333, + "r":0.25, + "f":0.2857142857 + }, + "Color_Other":{ + "p":0.4, + "r":0.5, + "f":0.4444444444 + }, + "Drug":{ + "p":0.5555555556, + "r":0.4166666667, + "f":0.4761904762 + }, + "Living_Thing_Part_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "N_Flora":{ + "p":0.3333333333, + "r":0.3333333333, + "f":0.3333333333 + }, + "Station":{ + "p":0.4242424242, + "r":0.56, + "f":0.4827586207 + }, + "Mountain":{ + "p":0.25, + "r":0.3333333333, + "f":0.2857142857 + }, + "Canal":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Bay":{ + "p":0.6666666667, + "r":1.0, + "f":0.8 + }, + "Sea":{ + "p":0.5, + "r":0.6666666667, + "f":0.5714285714 + }, + "Sport":{ + "p":0.5862068966, + "r":0.7555555556, + "f":0.6601941748 + }, + "God":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Show_Organization":{ + "p":0.45, + "r":0.2903225806, + "f":0.3529411765 + }, + "Magazine":{ + "p":0.55, + "r":0.7857142857, + "f":0.6470588235 + }, + "Newspaper":{ + "p":0.4285714286, + "r":0.2, + "f":0.2727272727 + }, + "Insect":{ + "p":0.8333333333, + "r":0.8333333333, + "f":0.8333333333 + }, + "Theory":{ + "p":0.5, + "r":0.4, + "f":0.4444444444 + }, + "Park":{ + "p":0.5714285714, + "r":0.8, + "f":0.6666666667 + }, + "Family":{ + "p":1.0, + "r":0.2857142857, + "f":0.4444444444 + }, + "County":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Multiplication":{ + "p":0.6666666667, + "r":0.4, + "f":0.5 + }, + "Bird":{ + "p":0.9166666667, + "r":0.6111111111, + "f":0.7333333333 + }, + "Timex_Other":{ + "p":0.5, + "r":0.2, + "f":0.2857142857 + }, + "Island":{ + "p":0.75, + "r":1.0, + "f":0.8571428571 + }, + "Natural_Object_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Mollusc_Arthropod":{ + "p":0.6666666667, + "r":0.5714285714, + "f":0.6153846154 + }, + "Postal_Address":{ + "p":0.9375, + "r":0.7894736842, + "f":0.8571428571 + }, + "Reptile":{ + "p":0.5, + "r":0.3333333333, + "f":0.4 + }, + "Lake":{ + "p":1.0, + "r":1.0, + "f":1.0 + }, + "Sports_Facility":{ + "p":0.1818181818, + "r":0.0769230769, + "f":0.1081081081 + }, + "Archaeological_Place_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Tunnel":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Art_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Broadcast_Program":{ + "p":0.75, + "r":0.297029703, + "f":0.4255319149 + }, + "Market":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Movie":{ + "p":0.1153846154, + "r":0.15, + "f":0.1304347826 + }, + "Seismic_Intensity":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Sports_Organization_Other":{ + "p":0.3333333333, + "r":0.2, + "f":0.25 + }, + "Spa":{ + "p":0.75, + "r":0.75, + "f":0.75 + }, + "Museum":{ + "p":0.75, + "r":1.0, + "f":0.8571428571 + }, + "Show":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Pro_Sports_Organization":{ + "p":0.4615384615, + "r":1.0, + "f":0.6315789474 + }, + "ID_Number":{ + "p":0.2857142857, + "r":0.3529411765, + "f":0.3157894737 + }, + "Spaceship":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Fungus":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Temperature":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Stock_Index":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Picture":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Zoo":{ + "p":0.0, + "r":0.0, + "f":0.0 + } + }, + "pos_acc":0.986660685, + "morph_micro_f":0.0, + "morph_per_feat":0.0, + "tag_acc":0.0, + "transformer_loss":863.1535821792, + "parser_loss":9468.2328317891, + "ner_loss":34.3299089293, + "morphologizer_loss":25.5198627768 + } +} diff --git a/config/ja_ginza_bert_large_analysis.cfg b/config/ja_ginza_bert_large_analysis.cfg index 8ad31bb..f2f3640 100644 --- a/config/ja_ginza_bert_large_analysis.cfg +++ b/config/ja_ginza_bert_large_analysis.cfg @@ -10,7 +10,7 @@ seed = 0 [nlp] lang = "ja" -pipeline = ["transformer","parser","attribute_ruler","ner","morphologizer","compound_splitter","bunsetu_recognizer"] +pipeline = ["transformer", "parser", "attribute_ruler", "ner", "morphologizer", "compound_splitter", "bunsetu_recognizer"] batch_size = 128 disabled = ["attribute_ruler"] before_creation = null @@ -27,6 +27,9 @@ split_mode = "C" factory = "attribute_ruler" validate = false +[components.attribute_ruler.scorer] +@scorers = "spacy.attribute_ruler_scorer.v1" + [components.bunsetu_recognizer] factory = "bunsetu_recognizer" remain_bunsetu_suffix = false @@ -39,7 +42,6 @@ split_mode = null factory = "morphologizer" extend = true overwrite = true -scorer = {"@scorers":"spacy.morphologizer_scorer.v1"} [components.morphologizer.model] @architectures = "spacy.Tagger.v2" @@ -49,14 +51,18 @@ normalize = false [components.morphologizer.model.tok2vec] @architectures = "spacy-transformers.TransformerListener.v1" grad_factor = 1.0 -pooling = {"@layers":"reduce_mean.v1"} upstream = "*" +[components.morphologizer.model.tok2vec.pooling] +@layers = "reduce_mean.v1" + +[components.morphologizer.scorer] +@scorers = "spacy.morphologizer_scorer.v1" + [components.ner] factory = "ner" incorrect_spans_key = null moves = null -scorer = {"@scorers":"spacy.ner_scorer.v1"} update_with_oracle_cut_size = 100 [components.ner.model] @@ -71,15 +77,19 @@ nO = null [components.ner.model.tok2vec] @architectures = "spacy-transformers.TransformerListener.v1" grad_factor = 1.0 -pooling = {"@layers":"reduce_mean.v1"} upstream = "*" +[components.ner.model.tok2vec.pooling] +@layers = "reduce_mean.v1" + +[components.ner.scorer] +@scorers = "spacy.ner_scorer.v1" + [components.parser] factory = "parser" learn_tokens = false min_action_freq = 30 moves = null -scorer = {"@scorers":"spacy.parser_scorer.v1"} update_with_oracle_cut_size = 100 [components.parser.model] @@ -94,17 +104,21 @@ nO = null [components.parser.model.tok2vec] @architectures = "spacy-transformers.TransformerListener.v1" grad_factor = 1.0 -pooling = {"@layers":"reduce_mean.v1"} upstream = "*" +[components.parser.model.tok2vec.pooling] +@layers = "reduce_mean.v1" + +[components.parser.scorer] +@scorers = "spacy.parser_scorer.v1" + [components.transformer] -factory = "transformer" +factory = "transformer_custom" max_batch_items = 4096 -set_extra_annotations = {"@annotation_setters":"spacy-transformers.null_annotation_setter.v1"} [components.transformer.model] @architectures = "spacy-transformers.TransformerModel.v3" -name = "cl-tohoku/bert-large-japanese-v2" +name = "megagonlabs/transformers-ud-japanese-bert-large-ginza" mixed_precision = false [components.transformer.model.get_spans] @@ -118,12 +132,17 @@ stride = 96 use_fast = false tokenizer_class = "BertJapaneseTokenizer" do_lower_case = false -word_tokenizer_type = mecab -subword_tokenizer_type = wordpiece -mecab_kwargs = {"mecab_dic":"unidic_lite"} +word_tokenizer_type = "mecab" +subword_tokenizer_type = "wordpiece" + +[components.transformer.model.tokenizer_config.mecab_kwargs] +mecab_dic = "unidic_lite" [components.transformer.model.transformer_config] +[components.transformer.set_extra_annotations] +@annotation_setters = "spacy-transformers.null_annotation_setter.v1" + [corpora] [corpora.dev] @@ -177,18 +196,15 @@ L2_is_weight_decay = true L2 = 0.01 grad_clip = 1.0 use_averages = false -eps = 0.00000001 +eps = 1e-08 [training.optimizer.learn_rate] @schedules = "warmup_linear.v1" warmup_steps = 250 total_steps = 20000 -initial_rate = 0.00005 +initial_rate = 5e-05 [training.score_weights] -pos_acc = 0.15 -morph_micro_f = 0.0 -morph_per_feat = null dep_uas = 0.25 dep_las = 0.25 dep_las_per_type = null @@ -199,6 +215,9 @@ ents_f = 0.25 ents_p = 0.0 ents_r = 0.0 ents_per_type = null +pos_acc = 0.15 +morph_micro_f = 0.0 +morph_per_feat = null tag_acc = 0.0 [pretraining] @@ -213,4 +232,4 @@ after_init = null [initialize.components] -[initialize.tokenizer] \ No newline at end of file +[initialize.tokenizer] diff --git a/config/ja_ginza_electra.analysis.cfg b/config/ja_ginza_electra.analysis.cfg index 17666e9..dd898d8 100644 --- a/config/ja_ginza_electra.analysis.cfg +++ b/config/ja_ginza_electra.analysis.cfg @@ -10,7 +10,7 @@ seed = 0 [nlp] lang = "ja" -pipeline = ["transformer","parser","attribute_ruler","ner","morphologizer","compound_splitter","bunsetu_recognizer"] +pipeline = ["transformer", "parser", "attribute_ruler", "ner", "morphologizer", "compound_splitter", "bunsetu_recognizer"] batch_size = 128 disabled = ["attribute_ruler"] before_creation = null @@ -25,9 +25,11 @@ split_mode = "C" [components.attribute_ruler] factory = "attribute_ruler" -scorer = {"@scorers":"spacy.attribute_ruler_scorer.v1"} validate = false +[components.attribute_ruler.scorer] +@scorers = "spacy.attribute_ruler_scorer.v1" + [components.bunsetu_recognizer] factory = "bunsetu_recognizer" remain_bunsetu_suffix = false @@ -40,7 +42,6 @@ split_mode = null factory = "morphologizer" extend = true overwrite = true -scorer = {"@scorers":"spacy.morphologizer_scorer.v1"} [components.morphologizer.model] @architectures = "spacy.Tagger.v1" @@ -49,14 +50,18 @@ nO = null [components.morphologizer.model.tok2vec] @architectures = "spacy-transformers.TransformerListener.v1" grad_factor = 1.0 -pooling = {"@layers":"reduce_mean.v1"} upstream = "*" +[components.morphologizer.model.tok2vec.pooling] +@layers = "reduce_mean.v1" + +[components.morphologizer.scorer] +@scorers = "spacy.morphologizer_scorer.v1" + [components.ner] factory = "ner" incorrect_spans_key = null moves = null -scorer = {"@scorers":"spacy.ner_scorer.v1"} update_with_oracle_cut_size = 100 [components.ner.model] @@ -71,15 +76,19 @@ nO = null [components.ner.model.tok2vec] @architectures = "spacy-transformers.TransformerListener.v1" grad_factor = 1.0 -pooling = {"@layers":"reduce_mean.v1"} upstream = "*" +[components.ner.model.tok2vec.pooling] +@layers = "reduce_mean.v1" + +[components.ner.scorer] +@scorers = "spacy.ner_scorer.v1" + [components.parser] factory = "parser" learn_tokens = false min_action_freq = 30 moves = null -scorer = {"@scorers":"spacy.parser_scorer.v1"} update_with_oracle_cut_size = 100 [components.parser.model] @@ -94,17 +103,21 @@ nO = null [components.parser.model.tok2vec] @architectures = "spacy-transformers.TransformerListener.v1" grad_factor = 1.0 -pooling = {"@layers":"reduce_mean.v1"} upstream = "*" +[components.parser.model.tok2vec.pooling] +@layers = "reduce_mean.v1" + +[components.parser.scorer] +@scorers = "spacy.parser_scorer.v1" + [components.transformer] factory = "transformer_custom" max_batch_items = 4096 -set_extra_annotations = {"@annotation_setters":"spacy-transformers.null_annotation_setter.v1"} [components.transformer.model] @architectures = "spacy-transformers.TransformerModel.v3" -name = "megagonlabs/transformers-ud-japanese-electra-base-ginza-510" +name = "megagonlabs/transformers-ud-japanese-electra-base-ginza" mixed_precision = false [components.transformer.model.get_spans] @@ -116,7 +129,7 @@ stride = 96 [components.transformer.model.tokenizer_config] use_fast = false -tokenizer_class = "sudachitra.tokenization_electra_sudachipy.ElectraSudachipyTokenizer" +tokenizer_class = "modeling.ElectraSudachipyTokenizer" do_lower_case = false do_word_tokenize = true do_subword_tokenize = true @@ -130,6 +143,9 @@ dict_type = "core" [components.transformer.model.transformer_config] +[components.transformer.set_extra_annotations] +@annotation_setters = "spacy-transformers.null_annotation_setter.v1" + [corpora] [corpora.dev] @@ -182,28 +198,29 @@ L2_is_weight_decay = true L2 = 0.01 grad_clip = 1.0 use_averages = false -eps = 0.00000001 +eps = 1e-08 [training.optimizer.learn_rate] @schedules = "warmup_linear.v1" warmup_steps = 250 total_steps = 50000 -initial_rate = 0.00005 +initial_rate = 5e-05 [training.score_weights] -dep_uas = 0.25 -dep_las = 0.25 +dep_uas = 0.17 +dep_las = 0.17 dep_las_per_type = null sents_p = null sents_r = null -sents_f = 0.1 -ents_f = 0.25 +sents_f = 0.07 +ents_f = 0.17 ents_p = 0.0 ents_r = 0.0 ents_per_type = null -pos_acc = 0.15 -morph_acc = 0.0 +pos_acc = 0.1 +morph_micro_f = 0.33 morph_per_feat = null +morph_acc = 0.0 tag_acc = 0.0 [pretraining] @@ -218,4 +235,4 @@ after_init = null [initialize.components] -[initialize.tokenizer] \ No newline at end of file +[initialize.tokenizer] diff --git a/config/ja_ginza_electra.meta.json b/config/ja_ginza_electra.meta.json index adcd30d..2818c67 100644 --- a/config/ja_ginza_electra.meta.json +++ b/config/ja_ginza_electra.meta.json @@ -1,7 +1,7 @@ { "lang":"ja", "name":"ginza_electra", - "version":"5.2.0", + "version":"5.3.0", "description":"Japanese multi-task CNN trained on UD-Japanese BCCWJ r2.8 + GSK2014-A(2019) + transformers-ud-japanese-electra--base. Components: transformer, parser, atteribute_ruler, ner, morphologizer, compound_splitter, bunsetu_recognizer.", "author":"Megagon Labs Tokyo.", "email":"ginza@megagon.ai", @@ -34,14 +34,14 @@ "author":"Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, & Peter J. Liu" }, { - "name":"megagonlabs/transformers-ud-japanese-electra-base-ginza-5.1.0", - "url":"https://huggingface.co/megagonlabs/transformers-ud-japanese-electra-base-ginza-5.1.0", + "name":"megagonlabs/transformers-ud-japanese-electra-base-ginza", + "url":"https://huggingface.co/megagonlabs/transformers-ud-japanese-electra-base-ginza", "license":"MIT Licence", "author":"Hiroshi Matsuda (Megagon Labs Tokyo, Recruit Co., Ltd.)" } ], "parent_package":"spacy", - "spacy_version":">=3.4.4,<4.0.0", + "spacy_version":">=3.8.16,<4.0.0", "spacy_git_version":"0fc3dee77", "pipeline":[ "transformer", @@ -61,20 +61,1399 @@ "compound_splitter", "bunsetu_recognizer" ], - "disabled": [ + "disabled":[ "attribute_ruler" ], "vectors":{ "width":0, "vectors":0, "keys":0, - "name":null + "name":null, + "mode":"default" }, - "requirements":[ - "sudachipy>=0.6.2,<0.7.0", + "requirements":[ + "sudachipy>=0.6.11,<0.7.0", "sudachidict_core>=20210802", - "sudachitra>=0.1.6,<0.2.0", - "ginza-transformers>=0.4.0,<0.5.0", - "ginza>=5.2.0,<5.3.0" - ] + "sudachitra>=0.1.9,<0.2.0", + "ginza-transformers>=1.4.0,<1.5.0", + "ginza>=5.3.0,<6.0.0" + ], + "labels":{ + "transformer":[ + + ], + "parser":[ + "ROOT", + "acl", + "acl_bunsetu", + "advcl", + "advcl_bunsetu", + "advmod", + "advmod_bunsetu", + "amod", + "amod_bunsetu", + "aux", + "aux_bunsetu", + "case", + "case_bunsetu", + "cc", + "cc_bunsetu", + "ccomp_bunsetu", + "compound", + "compound_bunsetu", + "cop", + "csubj_bunsetu", + "dep", + "dep_bunsetu", + "det_bunsetu", + "discourse_bunsetu", + "dislocated_bunsetu", + "fixed", + "mark", + "nmod", + "nmod_bunsetu", + "nsubj_bunsetu", + "nummod", + "obj_bunsetu", + "obl_bunsetu", + "punct", + "punct_bunsetu" + ], + "attribute_ruler":[ + + ], + "ner":[ + "Academic", + "Age", + "Aircraft", + "Airport", + "Amphibia", + "Amusement_Park", + "Animal_Disease", + "Animal_Part", + "Archaeological_Place_Other", + "Art_Other", + "Astral_Body_Other", + "Award", + "Bay", + "Bird", + "Book", + "Bridge", + "Broadcast_Program", + "Cabinet", + "Calorie", + "Canal", + "Car", + "Car_Stop", + "Character", + "City", + "Class", + "Clothing", + "Color_Other", + "Company", + "Company_Group", + "Compound", + "Conference", + "Constellation", + "Continental_Region", + "Corporation_Other", + "Country", + "Countx_Other", + "County", + "Culture", + "Date", + "Day_Of_Week", + "Disease_Other", + "Dish", + "Doctrine_Method_Other", + "Domestic_Region", + "Drug", + "Earthquake", + "Element", + "Email", + "Era", + "Ethnic_Group_Other", + "Event_Other", + "Facility_Other", + "Facility_Part", + "Family", + "Fish", + "Flora", + "Flora_Part", + "Food_Other", + "Frequency", + "Fungus", + "GOE_Other", + "GPE_Other", + "Game", + "Geological_Region_Other", + "God", + "Government", + "ID_Number", + "Incident_Other", + "Insect", + "Intensity", + "International_Organization", + "Island", + "Lake", + "Language_Other", + "Latitude_Longtitude", + "Law", + "Line_Other", + "Living_Thing_Other", + "Living_Thing_Part_Other", + "Location_Other", + "Magazine", + "Mammal", + "Material", + "Measurement_Other", + "Military", + "Mineral", + "Mollusc_Arthropod", + "Money", + "Money_Form", + "Mountain", + "Movement", + "Movie", + "Multiplication", + "Museum", + "Music", + "N_Animal", + "N_Country", + "N_Event", + "N_Facility", + "N_Flora", + "N_Location_Other", + "N_Natural_Object_Other", + "N_Organization", + "N_Person", + "N_Product", + "Name_Other", + "National_Language", + "Nationality", + "Natural_Disaster", + "Natural_Object_Other", + "Natural_Phenomenon_Other", + "Nature_Color", + "Newspaper", + "Numex_Other", + "Occasion_Other", + "Offense", + "Ordinal_Number", + "Organization_Other", + "Park", + "Percent", + "Period_Day", + "Period_Month", + "Period_Time", + "Period_Week", + "Period_Year", + "Periodx_Other", + "Person", + "Phone_Number", + "Physical_Extent", + "Plan", + "Planet", + "Point", + "Political_Organization_Other", + "Political_Party", + "Port", + "Position_Vocation", + "Postal_Address", + "Printing_Other", + "Pro_Sports_Organization", + "Product_Other", + "Province", + "Public_Institution", + "Railroad", + "Rank", + "Region_Other", + "Religion", + "Religious_Festival", + "Reptile", + "Research_Institute", + "River", + "Road", + "Rule_Other", + "School", + "School_Age", + "Sea", + "Ship", + "Show", + "Show_Organization", + "Spa", + "Space", + "Spaceship", + "Speed", + "Sport", + "Sports_Facility", + "Sports_League", + "Sports_Organization_Other", + "Station", + "Style", + "Temperature", + "Theater", + "Theory", + "Time", + "Time_Top_Other", + "Timex_Other", + "Title_Other", + "Train", + "Treaty", + "Tumulus", + "Tunnel", + "URL", + "Unit_Other", + "Vehicle_Other", + "Volume", + "War", + "Water_Route", + "Weapon", + "Weight", + "Worship_Place", + "Zoo" + ], + "morphologizer":[ + "POS=PUNCT", + "POS=NUM", + "POS=NOUN", + "POS=ADP", + "POS=AUX", + "POS=VERB", + "POS=CCONJ", + "POS=PART", + "POS=SCONJ", + "POS=SYM", + "POS=ADJ", + "POS=DET", + "POS=PRON", + "POS=PROPN", + "POS=ADV", + "POS=X", + "POS=INTJ" + ] + }, + "performance":{ + "dep_uas":0.9344858507, + "dep_las":0.9208774079, + "dep_las_per_type":{ + "compound":{ + "p":0.9204362596, + "r":0.9199017199, + "f":0.9201689121 + }, + "nummod":{ + "p":0.9615163699, + "r":0.9565714286, + "f":0.9590375251 + }, + "dep_bunsetu":{ + "p":0.6116788321, + "r":0.5689069925, + "f":0.5895181147 + }, + "obl_bunsetu":{ + "p":0.8426436577, + "r":0.8366157379, + "f":0.8396188788 + }, + "case":{ + "p":0.9886582767, + "r":0.9858068107, + "f":0.9872304847 + }, + "acl_bunsetu":{ + "p":0.8881370573, + "r":0.8854600258, + "f":0.8867965212 + }, + "nmod_bunsetu":{ + "p":0.8709554043, + "r":0.8568633353, + "f":0.8638519024 + }, + "root":{ + "p":0.9302156781, + "r":0.925131484, + "f":0.927666615 + }, + "fixed":{ + "p":0.9804979646, + "r":0.9828240653, + "f":0.981659637 + }, + "obj_bunsetu":{ + "p":0.948582996, + "r":0.9505070994, + "f":0.9495440729 + }, + "advcl_bunsetu":{ + "p":0.7841409692, + "r":0.7930162553, + "f":0.7885536398 + }, + "aux":{ + "p":0.9796376085, + "r":0.9799267918, + "f":0.9797821788 + }, + "nsubj_bunsetu":{ + "p":0.8594463854, + "r":0.8554232981, + "f":0.8574301227 + }, + "cc_bunsetu":{ + "p":0.7958412098, + "r":0.7781885397, + "f":0.7869158879 + }, + "det_bunsetu":{ + "p":0.9486709441, + "r":0.9583333333, + "f":0.9534776601 + }, + "advmod_bunsetu":{ + "p":0.8091693957, + "r":0.8256379101, + "f":0.8173207037 + }, + "cop":{ + "p":0.9489430894, + "r":0.9548429319, + "f":0.9518838689 + }, + "amod":{ + "p":0.8325991189, + "r":0.7470355731, + "f":0.7875 + }, + "dep":{ + "p":0.6311176041, + "r":0.72, + "f":0.6726352666 + }, + "nmod":{ + "p":0.8267857143, + "r":0.8372513562, + "f":0.8319856244 + }, + "mark":{ + "p":0.9788186396, + "r":0.9772564041, + "f":0.978036898 + }, + "acl":{ + "p":0.8571428571, + "r":0.75, + "f":0.8 + }, + "advcl":{ + "p":0.8212765957, + "r":0.8212765957, + "f":0.8212765957 + }, + "compound_bunsetu":{ + "p":0.4717948718, + "r":0.4299065421, + "f":0.4498777506 + }, + "dislocated_bunsetu":{ + "p":0.6575342466, + "r":0.5962732919, + "f":0.6254071661 + }, + "ccomp_bunsetu":{ + "p":0.9021956088, + "r":0.8986083499, + "f":0.9003984064 + }, + "advmod":{ + "p":0.7058823529, + "r":0.4137931034, + "f":0.5217391304 + }, + "csubj_bunsetu":{ + "p":0.8468468468, + "r":0.8430493274, + "f":0.8449438202 + }, + "amod_bunsetu":{ + "p":0.9488636364, + "r":0.9653179191, + "f":0.9570200573 + }, + "discourse_bunsetu":{ + "p":0.5294117647, + "r":0.5538461538, + "f":0.5413533835 + }, + "punct_bunsetu":{ + "p":0.7897196262, + "r":0.5827586207, + "f":0.6706349206 + }, + "case_bunsetu":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "cc":{ + "p":1.0, + "r":0.72, + "f":0.8372093023 + }, + "aux_bunsetu":{ + "p":1.0, + "r":0.0666666667, + "f":0.125 + }, + "nummod_bunsetu":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "discourse":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "det":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "reparandum":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "reparandum_bunsetu":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "cop_bunsetu":{ + "p":0.0, + "r":0.0, + "f":0.0 + } + }, + "sents_p":0.8865468072, + "sents_r":0.884809735, + "sents_f":0.8856774194, + "ents_f":0.6543236768, + "ents_p":0.6542286751, + "ents_r":0.654418706, + "ents_per_type":{ + "Ordinal_Number":{ + "p":0.7965056526, + "r":0.8451472192, + "f":0.8201058201 + }, + "Nationality":{ + "p":0.756097561, + "r":0.756097561, + "f":0.756097561 + }, + "Date":{ + "p":0.8418972332, + "r":0.8875, + "f":0.8640973631 + }, + "Government":{ + "p":0.6296296296, + "r":0.7212121212, + "f":0.6723163842 + }, + "National_Language":{ + "p":0.9090909091, + "r":0.9090909091, + "f":0.9090909091 + }, + "Law":{ + "p":0.6, + "r":0.6428571429, + "f":0.6206896552 + }, + "Doctrine_Method_Other":{ + "p":0.3910447761, + "r":0.3358974359, + "f":0.3613793103 + }, + "Product_Other":{ + "p":0.482381531, + "r":0.5663338088, + "f":0.5209973753 + }, + "Event_Other":{ + "p":0.4237288136, + "r":0.3968253968, + "f":0.4098360656 + }, + "Occasion_Other":{ + "p":0.2125, + "r":0.2297297297, + "f":0.2207792208 + }, + "Time":{ + "p":0.6255924171, + "r":0.6111111111, + "f":0.6182669789 + }, + "Movement":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Money":{ + "p":0.8169934641, + "r":0.744047619, + "f":0.7788161994 + }, + "Offense":{ + "p":0.575, + "r":0.5287356322, + "f":0.5508982036 + }, + "Plan":{ + "p":0.6043165468, + "r":0.6315789474, + "f":0.6176470588 + }, + "Period_Time":{ + "p":0.7582417582, + "r":0.6571428571, + "f":0.7040816327 + }, + "Public_Institution":{ + "p":0.9, + "r":0.6, + "f":0.72 + }, + "Academic":{ + "p":0.3544303797, + "r":0.5833333333, + "f":0.4409448819 + }, + "Corporation_Other":{ + "p":0.3389830508, + "r":0.4651162791, + "f":0.3921568627 + }, + "Natural_Disaster":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Position_Vocation":{ + "p":0.6678043231, + "r":0.7310087173, + "f":0.6979785969 + }, + "Organization_Other":{ + "p":0.2840909091, + "r":0.3424657534, + "f":0.3105590062 + }, + "Country":{ + "p":0.8941176471, + "r":0.8816705336, + "f":0.8878504673 + }, + "Company":{ + "p":0.6813186813, + "r":0.5232067511, + "f":0.5918854415 + }, + "Political_Organization_Other":{ + "p":0.2631578947, + "r":0.3125, + "f":0.2857142857 + }, + "War":{ + "p":0.5, + "r":0.6, + "f":0.5454545455 + }, + "Military":{ + "p":0.6956521739, + "r":0.5333333333, + "f":0.6037735849 + }, + "Treaty":{ + "p":0.8241758242, + "r":0.6696428571, + "f":0.7389162562 + }, + "Conference":{ + "p":0.5161290323, + "r":0.3555555556, + "f":0.4210526316 + }, + "Continental_Region":{ + "p":0.4516129032, + "r":0.6086956522, + "f":0.5185185185 + }, + "N_Person":{ + "p":0.8242811502, + "r":0.8514851485, + "f":0.8376623377 + }, + "Person":{ + "p":0.8418474458, + "r":0.7556532663, + "f":0.7964250248 + }, + "Period_Year":{ + "p":0.7898089172, + "r":0.7798742138, + "f":0.7848101266 + }, + "N_Product":{ + "p":0.6108108108, + "r":0.5978835979, + "f":0.6042780749 + }, + "N_Country":{ + "p":0.72, + "r":0.7058823529, + "f":0.7128712871 + }, + "Percent":{ + "p":0.9215686275, + "r":0.901369863, + "f":0.9113573407 + }, + "City":{ + "p":0.6703296703, + "r":0.6130653266, + "f":0.6404199475 + }, + "International_Organization":{ + "p":0.6357142857, + "r":0.5204678363, + "f":0.5723472669 + }, + "Incident_Other":{ + "p":0.2222222222, + "r":0.375, + "f":0.2790697674 + }, + "Point":{ + "p":0.2926829268, + "r":0.3333333333, + "f":0.3116883117 + }, + "Countx_Other":{ + "p":0.1384615385, + "r":0.1451612903, + "f":0.1417322835 + }, + "Printing_Other":{ + "p":0.9072164948, + "r":0.7096774194, + "f":0.7963800905 + }, + "N_Organization":{ + "p":0.4482758621, + "r":0.2826086957, + "f":0.3466666667 + }, + "N_Facility":{ + "p":0.2549019608, + "r":0.3421052632, + "f":0.2921348315 + }, + "Weight":{ + "p":0.75, + "r":0.6666666667, + "f":0.7058823529 + }, + "ID_Number":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Animal_Disease":{ + "p":0.6818181818, + "r":0.7894736842, + "f":0.7317073171 + }, + "Weapon":{ + "p":0.5531914894, + "r":0.4, + "f":0.4642857143 + }, + "Numex_Other":{ + "p":0.1428571429, + "r":0.131147541, + "f":0.1367521368 + }, + "Research_Institute":{ + "p":0.5, + "r":0.1176470588, + "f":0.1904761905 + }, + "Physical_Extent":{ + "p":0.5869565217, + "r":0.6923076923, + "f":0.6352941176 + }, + "Aircraft":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Ship":{ + "p":0.6086956522, + "r":0.6086956522, + "f":0.6086956522 + }, + "Game":{ + "p":0.5581395349, + "r":0.4528301887, + "f":0.5 + }, + "Periodx_Other":{ + "p":0.1764705882, + "r":0.6, + "f":0.2727272727 + }, + "Compound":{ + "p":0.5882352941, + "r":0.4545454545, + "f":0.5128205128 + }, + "Material":{ + "p":0.4642857143, + "r":0.4193548387, + "f":0.4406779661 + }, + "Natural_Phenomenon_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Element":{ + "p":0.5, + "r":0.1428571429, + "f":0.2222222222 + }, + "N_Location_Other":{ + "p":0.1142857143, + "r":0.2857142857, + "f":0.1632653061 + }, + "Facility_Other":{ + "p":0.0333333333, + "r":0.1428571429, + "f":0.0540540541 + }, + "Rank":{ + "p":0.6842105263, + "r":0.7471264368, + "f":0.7142857143 + }, + "Class":{ + "p":0.6129032258, + "r":0.4634146341, + "f":0.5277777778 + }, + "School_Age":{ + "p":0.44, + "r":0.6285714286, + "f":0.5176470588 + }, + "Period_Month":{ + "p":0.5102040816, + "r":0.8064516129, + "f":0.625 + }, + "GOE_Other":{ + "p":0.3820224719, + "r":0.3578947368, + "f":0.3695652174 + }, + "Title_Other":{ + "p":0.8908450704, + "r":0.8489932886, + "f":0.8694158076 + }, + "School":{ + "p":0.7272727273, + "r":0.3950617284, + "f":0.512 + }, + "Stock":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "URL":{ + "p":0.9166666667, + "r":0.9166666667, + "f":0.9166666667 + }, + "Email":{ + "p":0.6666666667, + "r":0.5, + "f":0.5714285714 + }, + "Mollusc_Arthropod":{ + "p":0.7272727273, + "r":0.5714285714, + "f":0.64 + }, + "Era":{ + "p":0.5128205128, + "r":0.8, + "f":0.625 + }, + "Rule_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Location_Other":{ + "p":0.1923076923, + "r":0.2083333333, + "f":0.2 + }, + "Award":{ + "p":0.3157894737, + "r":0.75, + "f":0.4444444444 + }, + "GPE_Other":{ + "p":0.5714285714, + "r":0.6779661017, + "f":0.6201550388 + }, + "Mammal":{ + "p":0.6481481481, + "r":0.6363636364, + "f":0.6422018349 + }, + "Drug":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Insect":{ + "p":0.25, + "r":0.25, + "f":0.25 + }, + "Clothing":{ + "p":0.6796116505, + "r":0.8235294118, + "f":0.7446808511 + }, + "Book":{ + "p":0.3048780488, + "r":0.3048780488, + "f":0.3048780488 + }, + "Mineral":{ + "p":0.1666666667, + "r":0.2, + "f":0.1818181818 + }, + "Animal_Part":{ + "p":0.7938596491, + "r":0.8341013825, + "f":0.8134831461 + }, + "Flora":{ + "p":0.3884297521, + "r":0.7014925373, + "f":0.5 + }, + "Period_Day":{ + "p":0.4915254237, + "r":0.4915254237, + "f":0.4915254237 + }, + "N_Animal":{ + "p":0.4285714286, + "r":0.5, + "f":0.4615384615 + }, + "Newspaper":{ + "p":0.2727272727, + "r":0.2, + "f":0.2307692308 + }, + "Fish":{ + "p":0.0975609756, + "r":0.4444444444, + "f":0.16 + }, + "Dish":{ + "p":0.6880733945, + "r":0.7075471698, + "f":0.6976744186 + }, + "Food_Other":{ + "p":0.5, + "r":0.2954545455, + "f":0.3714285714 + }, + "Facility_Part":{ + "p":0.1509433962, + "r":0.1739130435, + "f":0.1616161616 + }, + "Domestic_Region":{ + "p":0.641025641, + "r":0.3472222222, + "f":0.4504504505 + }, + "Nature_Color":{ + "p":0.7777777778, + "r":0.8536585366, + "f":0.8139534884 + }, + "Flora_Part":{ + "p":0.66, + "r":0.6734693878, + "f":0.6666666667 + }, + "Car":{ + "p":0.6, + "r":0.4090909091, + "f":0.4864864865 + }, + "Geological_Region_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Province":{ + "p":0.7804878049, + "r":0.8205128205, + "f":0.8 + }, + "Bird":{ + "p":0.3333333333, + "r":0.2777777778, + "f":0.303030303 + }, + "Park":{ + "p":0.1538461538, + "r":0.4, + "f":0.2222222222 + }, + "Road":{ + "p":0.5625, + "r":0.6666666667, + "f":0.6101694915 + }, + "Money_Form":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Ethnic_Group_Other":{ + "p":0.7222222222, + "r":0.5652173913, + "f":0.6341463415 + }, + "Name_Other":{ + "p":0.0740740741, + "r":0.0754716981, + "f":0.0747663551 + }, + "Age":{ + "p":0.8444444444, + "r":0.6551724138, + "f":0.7378640777 + }, + "Sport":{ + "p":0.6111111111, + "r":0.7333333333, + "f":0.6666666667 + }, + "Island":{ + "p":0.1875, + "r":1.0, + "f":0.3157894737 + }, + "Worship_Place":{ + "p":0.4761904762, + "r":0.3846153846, + "f":0.4255319149 + }, + "Show_Organization":{ + "p":0.1111111111, + "r":0.2258064516, + "f":0.1489361702 + }, + "Mountain":{ + "p":0.3333333333, + "r":0.5833333333, + "f":0.4242424242 + }, + "Religious_Festival":{ + "p":0.1176470588, + "r":0.1538461538, + "f":0.1333333333 + }, + "Music":{ + "p":0.1791044776, + "r":0.3157894737, + "f":0.2285714286 + }, + "Sea":{ + "p":0.1111111111, + "r":0.3333333333, + "f":0.1666666667 + }, + "River":{ + "p":0.2727272727, + "r":0.6, + "f":0.375 + }, + "God":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Port":{ + "p":0.1, + "r":0.2, + "f":0.1333333333 + }, + "Frequency":{ + "p":0.7272727273, + "r":0.6486486486, + "f":0.6857142857 + }, + "N_Natural_Object_Other":{ + "p":0.2222222222, + "r":0.1666666667, + "f":0.1904761905 + }, + "Bridge":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Vehicle_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Period_Week":{ + "p":0.8636363636, + "r":0.8260869565, + "f":0.8444444444 + }, + "Space":{ + "p":0.5, + "r":0.4, + "f":0.4444444444 + }, + "N_Event":{ + "p":0.3103448276, + "r":0.3, + "f":0.3050847458 + }, + "Day_Of_Week":{ + "p":0.7592592593, + "r":0.8541666667, + "f":0.8039215686 + }, + "Character":{ + "p":0.085106383, + "r":0.085106383, + "f":0.085106383 + }, + "Phone_Number":{ + "p":0.9333333333, + "r":1.0, + "f":0.9655172414 + }, + "Theater":{ + "p":0.4, + "r":0.5, + "f":0.4444444444 + }, + "Show":{ + "p":0.0769230769, + "r":0.1, + "f":0.0869565217 + }, + "Reptile":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Family":{ + "p":0.3333333333, + "r":0.2857142857, + "f":0.3076923077 + }, + "Speed":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Planet":{ + "p":0.5, + "r":0.5, + "f":0.5 + }, + "Religion":{ + "p":1.0, + "r":0.7777777778, + "f":0.875 + }, + "Airport":{ + "p":0.5, + "r":0.1, + "f":0.1666666667 + }, + "Railroad":{ + "p":0.5, + "r":0.3333333333, + "f":0.4 + }, + "Train":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Living_Thing_Other":{ + "p":0.375, + "r":0.75, + "f":0.5 + }, + "Color_Other":{ + "p":0.2380952381, + "r":0.625, + "f":0.3448275862 + }, + "Living_Thing_Part_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "N_Flora":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Volume":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Measurement_Other":{ + "p":0.5789473684, + "r":0.55, + "f":0.5641025641 + }, + "Canal":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Bay":{ + "p":0.3333333333, + "r":1.0, + "f":0.5 + }, + "Pro_Sports_Organization":{ + "p":0.1818181818, + "r":1.0, + "f":0.3076923077 + }, + "Amusement_Park":{ + "p":0.724137931, + "r":0.4285714286, + "f":0.5384615385 + }, + "Lake":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Magazine":{ + "p":0.2307692308, + "r":0.2142857143, + "f":0.2222222222 + }, + "Company_Group":{ + "p":0.2857142857, + "r":0.5, + "f":0.3636363636 + }, + "Theory":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Culture":{ + "p":0.2, + "r":0.1111111111, + "f":0.1428571429 + }, + "Movie":{ + "p":0.1071428571, + "r":0.15, + "f":0.125 + }, + "Spaceship":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Station":{ + "p":0.4242424242, + "r":0.56, + "f":0.4827586207 + }, + "Sports_Organization_Other":{ + "p":0.2857142857, + "r":0.2, + "f":0.2352941176 + }, + "Archaeological_Place_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Multiplication":{ + "p":0.6666666667, + "r":0.4, + "f":0.5 + }, + "County":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Fungus":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Natural_Object_Other":{ + "p":1.0, + "r":0.5, + "f":0.6666666667 + }, + "Postal_Address":{ + "p":1.0, + "r":0.8421052632, + "f":0.9142857143 + }, + "Region_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Line_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Tunnel":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Art_Other":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Broadcast_Program":{ + "p":0.3703703704, + "r":0.198019802, + "f":0.2580645161 + }, + "Sports_League":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Market":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Seismic_Intensity":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Spa":{ + "p":0.5454545455, + "r":0.75, + "f":0.6315789474 + }, + "Sports_Facility":{ + "p":0.4285714286, + "r":0.2307692308, + "f":0.3 + }, + "Museum":{ + "p":0.4, + "r":0.6666666667, + "f":0.5 + }, + "Amphibia":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Temperature":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Stock_Index":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Timex_Other":{ + "p":1.0, + "r":0.2, + "f":0.3333333333 + }, + "Picture":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Constellation":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Zoo":{ + "p":0.0, + "r":0.0, + "f":0.0 + }, + "Political_Party":{ + "p":0.0, + "r":0.0, + "f":0.0 + } + }, + "pos_acc":0.985524188, + "morph_micro_f":0.0, + "morph_per_feat":0.0, + "morph_acc":0.0, + "tag_acc":0.9878354143, + "transformer_loss":15946.5850335878, + "parser_loss":9292.4742132145, + "ner_loss":275.138688962, + "morphologizer_loss":343.3875644008 + } } From a54d8025af54d9497f6b1362ee572c5b8ca12fee Mon Sep 17 00:00:00 2001 From: Hiroshi Matsuda Date: Mon, 7 Sep 2026 21:17:38 +0900 Subject: [PATCH 10/24] update model url --- config/ja_ginza_bert_large.meta.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/ja_ginza_bert_large.meta.json b/config/ja_ginza_bert_large.meta.json index cd8cea5..1e88ad3 100644 --- a/config/ja_ginza_bert_large.meta.json +++ b/config/ja_ginza_bert_large.meta.json @@ -27,8 +27,8 @@ "author":"Works Applications Enterprise Co., Ltd." }, { - "name":"cl-tohoku/bert-large-japanese-v2", - "url":"https://huggingface.co/cl-tohoku/bert-large-japanese-v2", + "name":"tohoku-nlp/bert-large-japanese-v2", + "url":"https://huggingface.co/tohoku-nlp/bert-large-japanese-v2", "license":"Apache License 2.0", "author":"Tohoku University" } From b0661d70b8465354bfab89e208d4369aa20589c6 Mon Sep 17 00:00:00 2001 From: Hiroshi Matsuda Date: Fri, 11 Sep 2026 17:29:27 +0900 Subject: [PATCH 11/24] bert_large as the highest priority --- ginza/analyzer.py | 8 ++++---- ginza/command_line.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ginza/analyzer.py b/ginza/analyzer.py index e4b38a6..cea4bca 100644 --- a/ginza/analyzer.py +++ b/ginza/analyzer.py @@ -72,15 +72,15 @@ def set_nlp(self) -> None: nlp = spacy.load(self.model_name_or_path) else: try: - nlp = spacy.load("ja_ginza_electra") + nlp = spacy.load("ja_ginza_bert_large") except IOError as e: try: - nlp = spacy.load("ja_ginza") + nlp = spacy.load("ja_ginza_electra") except IOError as e: try: - nlp = spacy.load("ja_ginza_bert_large") + nlp = spacy.load("ja_ginza") except IOError as e: - raise OSError("E050", 'You need to install "ja-ginza" or "ja-ginza-electra" by executing `pip install ja-ginza` or `pip install ja-ginza-electra`.') + raise OSError("E050", 'You need to install "ja-ginza", "ja-ginza-electra", or "ja-ginza-bert-large" by executing `pip install`.') if self.disable_sentencizer: nlp.add_pipe("disable_sentencizer", before="parser") diff --git a/ginza/command_line.py b/ginza/command_line.py index a2eadc8..c675e1b 100644 --- a/ginza/command_line.py +++ b/ginza/command_line.py @@ -12,7 +12,7 @@ MINI_BATCH_SIZE = 100 MAX_BYTE_LENGTH = 49149 # defined in sudachi.rs https://github.com/megagonlabs/ginza/issues/242 -GINZA_MODEL_PATTERN = re.compile(r"^(ja_ginza|ja_ginza_electra)$") +GINZA_MODEL_PATTERN = re.compile(r"^(ja_ginza|ja_ginza_electra|ja_ginza_bert_large)$") SPACY_MODEL_PATTERN = re.compile(r"^[a-z]{2}[-_].+[-_].+(sm|md|lg|trf)$") From 51c096e2b216631c41588647885c72a76c6ef5da Mon Sep 17 00:00:00 2001 From: Hiroshi Matsuda Date: Sun, 13 Sep 2026 03:43:49 +0900 Subject: [PATCH 12/24] GPU auto enable --- .github/workflows/pytest.yml | 2 +- ginza/command_line.py | 55 +++++++++++++++++++++--------------- setup.py | 3 +- 3 files changed, 35 insertions(+), 25 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 8013d5b..da5d73d 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10", "3.13"] + python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - name: Checkout uses: actions/checkout@v7 diff --git a/ginza/command_line.py b/ginza/command_line.py index c675e1b..5c5f826 100644 --- a/ginza/command_line.py +++ b/ginza/command_line.py @@ -8,13 +8,15 @@ from typing import Generator, Iterable, Optional, List import plac +from thinc.api import get_current_ops +from thinc.compat import has_cupy_gpu from .analyzer import Analyzer MINI_BATCH_SIZE = 100 MAX_BYTE_LENGTH = 49149 # defined in sudachi.rs https://github.com/megagonlabs/ginza/issues/242 GINZA_MODEL_PATTERN = re.compile(r"^(ja_ginza|ja_ginza_electra|ja_ginza_bert_large)$") SPACY_MODEL_PATTERN = re.compile(r"^[a-z]{2}[-_].+[-_].+(sm|md|lg|trf)$") - +TRANSFORMERS_MODEL_PATTERN = re.compile(r"^(ja_ginza_electra|ja_ginza_bert_large|.+_trf)$") class _OutputWrapper: def __init__(self, output_path, output_format): @@ -59,7 +61,7 @@ def run( hash_comment: str = "print", output_path: Optional[Path] = None, output_format: str = "0", - require_gpu: int = -1, + require_gpu: Optional[int] = None, disable_sentencizer: bool = False, use_normalized_form: bool = False, parallel_level: int = 1, @@ -71,26 +73,6 @@ def run( file=sys.stderr ) - assert parallel_level == 1 or require_gpu == -1, "require_gpu not allowed for multi-processing. https://github.com/explosion/spaCy/issues/5507" - - if parallel_level <= 0: - level = max(1, cpu_count() + parallel_level) - if output_format in [2, "mecab"]: - if require_gpu >= 0: - print("GPU not used for mecab mode", file=sys.stderr) - require_gpu = False - elif parallel_level <= 0: - if require_gpu >= 0: - if level < 4: - print(f"GPU #{require_gpu} enabled: parallel_level' set to {level}", end="", file=sys.stderr) - else: - print(f"GPU #{require_gpu} enabled: parallel_level' set to {level} but seems it's too much", end="", file=sys.stderr) - else: - print(f"'parallel_level' set to {level}", file=sys.stderr) - elif require_gpu: - print(f"GPU #{require_gpu} enabled", file=sys.stderr) - parallel_level = level - assert model_path is None or ensure_model is None if ensure_model: ensure_model = ensure_model.replace("-", "_") @@ -114,6 +96,33 @@ def run( else: model_name_or_path = model_path + if output_format in [2, "mecab"]: + if require_gpu is not None and require_gpu >= 0: + print(f"GPU is disabled in mecab mode.", file=sys.stderr) + require_gpu = -1 + elif require_gpu is None: + if get_current_ops().name == "apple": + if TRANSFORMERS_MODEL_PATTERN.match(model_name_or_path.rstrip("/")): + require_gpu = 0 + else: + require_gpu = -1 + elif has_cupy_gpu: + require_gpu = 0 + else: + require_gpu = -1 + if require_gpu != -1: + if parallel_level == 1: + print(f"GPU #{require_gpu} enabled", file=sys.stderr) + else: + require_gpu = -1 + print(f"GPU disabled because 'parallel_level' set to {parallel_level}", file=sys.stderr) + + assert parallel_level == 1 or require_gpu == -1, "require_gpu not allowed for multi-processing. https://github.com/explosion/spaCy/issues/5507" + + if parallel_level <= 0: + parallel_level = max(1, cpu_count() + parallel_level) + print(f"'parallel_level' set to {parallel_level}", file=sys.stderr) + analyzer = Analyzer( model_name_or_path, split_mode, @@ -366,7 +375,7 @@ def run_ginza( hash_comment="print", output_path=None, output_format="conllu", - require_gpu=-1, + require_gpu=None, use_normalized_form=False, disable_sentencizer=False, parallel=1, diff --git a/setup.py b/setup.py index b7f7432..2119861 100644 --- a/setup.py +++ b/setup.py @@ -22,11 +22,12 @@ "plac>=1.3.3", "SudachiPy>=0.6.11,<0.7.0", "SudachiDict-core>=20210802", - "thinc-apple-ops>=1.0.0 ; sys_platform == 'darwin' and platform_machine == 'arm64'", + "thinc-apple-ops>=1.0.0 ; sys_platform == 'darwin' and platform_machine == 'arm64' and python_version < '3.13'", ], extras_require={ "cuda11x": ["cupy-cuda11x"], "cuda12x": ["cupy-cuda12x"], + "cuda13x": ["cupy-cuda13x"], }, setup_requires=["pytest-runner"], tests_require=["pytest", "pytest-cov", "pytest-mock"], From 4a0d57461eed98d3df14761a3d5e64b10161df79 Mon Sep 17 00:00:00 2001 From: Hiroshi Matsuda Date: Sun, 13 Sep 2026 04:19:41 +0900 Subject: [PATCH 13/24] improve --- ginza/analyzer.py | 19 ++++--------------- ginza/command_line.py | 10 ++++++---- 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/ginza/analyzer.py b/ginza/analyzer.py index cea4bca..5499363 100644 --- a/ginza/analyzer.py +++ b/ginza/analyzer.py @@ -1,5 +1,4 @@ # coding: utf8 -import sys from typing import Iterable, Optional import thinc @@ -9,7 +8,7 @@ from spacy.language import Language from spacy.lang.ja import Japanese -from . import set_split_mode, inflection, reading_form, ent_label_ene, ent_label_ontonotes, bunsetu_bi_label, bunsetu_position_type, clause_head_i, _safe_lemma_, _safe_norm_ +from . import set_split_mode, inflection, reading_form, ent_label_ene, ent_label_ontonotes, bunsetu_bi_label, bunsetu_position_type, clause_head_i, _safe_lemma_, _safe_norm_, GINZA_MODEL_PACKAGES from .bunsetu_recognizer import bunsetu_available, bunsetu_head_list, bunsetu_phrase_span @@ -67,20 +66,10 @@ def set_nlp(self) -> None: if self.output_format in ["2", "mecab"]: nlp = try_sudachi_import(self.split_mode) else: - # Work-around for pickle error. Need to share model data. - if self.model_name_or_path: + try: nlp = spacy.load(self.model_name_or_path) - else: - try: - nlp = spacy.load("ja_ginza_bert_large") - except IOError as e: - try: - nlp = spacy.load("ja_ginza_electra") - except IOError as e: - try: - nlp = spacy.load("ja_ginza") - except IOError as e: - raise OSError("E050", 'You need to install "ja-ginza", "ja-ginza-electra", or "ja-ginza-bert-large" by executing `pip install`.') + except IOError: + raise OSError("E050", f"You need to install on or more model packages in {GINZA_MODEL_PACKAGES} by executing `pip install`.") if self.disable_sentencizer: nlp.add_pipe("disable_sentencizer", before="parser") diff --git a/ginza/command_line.py b/ginza/command_line.py index 5c5f826..4de596c 100644 --- a/ginza/command_line.py +++ b/ginza/command_line.py @@ -10,11 +10,11 @@ import plac from thinc.api import get_current_ops from thinc.compat import has_cupy_gpu +from . import default_model_name, GINZA_MODEL_PACKAGES from .analyzer import Analyzer MINI_BATCH_SIZE = 100 MAX_BYTE_LENGTH = 49149 # defined in sudachi.rs https://github.com/megagonlabs/ginza/issues/242 -GINZA_MODEL_PATTERN = re.compile(r"^(ja_ginza|ja_ginza_electra|ja_ginza_bert_large)$") SPACY_MODEL_PATTERN = re.compile(r"^[a-z]{2}[-_].+[-_].+(sm|md|lg|trf)$") TRANSFORMERS_MODEL_PATTERN = re.compile(r"^(ja_ginza_electra|ja_ginza_bert_large|.+_trf)$") @@ -80,7 +80,7 @@ def run( from importlib import import_module import_module(ensure_model) except ModuleNotFoundError: - if GINZA_MODEL_PATTERN.match(ensure_model): + if ensure_model in GINZA_MODEL_PACKAGES: print("Installing", ensure_model, file=sys.stderr) import pip pip.main(["install", ensure_model]) @@ -93,8 +93,10 @@ def run( else: raise OSError("E050", f'You need to install "{ensure_model}" before executing ginza.') model_name_or_path = ensure_model - else: + elif model_path: model_name_or_path = model_path + else: + model_name_or_path = default_model_name() if output_format in [2, "mecab"]: if require_gpu is not None and require_gpu >= 0: @@ -102,7 +104,7 @@ def run( require_gpu = -1 elif require_gpu is None: if get_current_ops().name == "apple": - if TRANSFORMERS_MODEL_PATTERN.match(model_name_or_path.rstrip("/")): + if model_name_or_path and TRANSFORMERS_MODEL_PATTERN.match(model_name_or_path.rstrip("/")): require_gpu = 0 else: require_gpu = -1 From d215e83856a0175be731c11de5dac5b783ecd0ff Mon Sep 17 00:00:00 2001 From: Hiroshi Matsuda Date: Sun, 13 Sep 2026 04:20:01 +0900 Subject: [PATCH 14/24] improve --- ginza/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ginza/__init__.py b/ginza/__init__.py index 2ef3859..00c5059 100644 --- a/ginza/__init__.py +++ b/ginza/__init__.py @@ -49,6 +49,20 @@ ] +GINZA_MODEL_PACKAGES = ["ja_ginza_bert_large", "ja_ginza_electra", "ja_ginza"] + + +def default_model_name(): + from importlib import import_module + for model_package in GINZA_MODEL_PACKAGES: + try: + import_module(model_package) + return model_package + except ModuleNotFoundError: + pass + return None + + @Language.factory( "compound_splitter", requires=[], From 2c67a31a18590eeac5dd1b4d27b76ba14caa3a50 Mon Sep 17 00:00:00 2001 From: Hiroshi Matsuda Date: Sun, 13 Sep 2026 04:45:50 +0900 Subject: [PATCH 15/24] update md files --- README.md | 217 +++++++++++++++++++++++++++----------------- docs/bunsetu_api.md | 2 +- docs/index.md | 188 +++++++++++++++++++++----------------- 3 files changed, 242 insertions(+), 165 deletions(-) diff --git a/README.md b/README.md index fd7a087..239f1ac 100644 --- a/README.md +++ b/README.md @@ -5,112 +5,80 @@   [![Downloads](https://pepy.tech/badge/ginza/week)](https://pepy.tech/project/ginza) -An Open Source Japanese NLP Library, based on Universal Dependencies - -***Please read the [Important changes](#ginza-520) before you upgrade GiNZA.*** - [日本語ページはこちら](https://megagonlabs.github.io/ginza/) -## License -GiNZA NLP Library and GiNZA Japanese Universal Dependencies Models are distributed under the -[MIT License](https://github.com/megagonlabs/ginza/blob/master/LICENSE). -You must agree and follow the MIT License to use GiNZA NLP Library and GiNZA Japanese Universal Dependencies Models. - -### Explosion / spaCy -spaCy is the key framework of GiNZA. - -[spaCy LICENSE PAGE](https://github.com/explosion/spaCy/blob/master/LICENSE) - -### Works Applications Enterprise / Sudachi/SudachiPy - SudachiDict - chiVe -SudachiPy provides high accuracies for tokenization and pos tagging. - -[Sudachi LICENSE PAGE](https://github.com/WorksApplications/Sudachi/blob/develop/LICENSE-2.0.txt), -[SudachiPy LICENSE PAGE](https://github.com/WorksApplications/SudachiPy/blob/develop/LICENSE), -[SudachiDict LEGAL PAGE](https://github.com/WorksApplications/SudachiDict/blob/develop/LEGAL), -[chiVe LICENSE PAGE](https://github.com/WorksApplications/chiVe/blob/master/LICENSE) - -### Hugging Face / transformers -The GiNZA v5 Transformers model (ja_ginza_electra) is trained by using Hugging Face Transformers as a framework for pretrained models. - -[transformers LICENSE PAGE](https://github.com/huggingface/transformers/blob/master/LICENSE) - -## Training Datasets - -### UD Japanese BCCWJ r2.8 -The parsing model of GiNZA v5 is trained on a part of -[UD Japanese BCCWJ](https://github.com/UniversalDependencies/UD_Japanese-BCCWJ) r2.8 -([Omura and Asahara:2018](https://www.aclweb.org/anthology/W18-6014/)). -This model is developed by National Institute for Japanese Language and Linguistics, and Megagon Labs. - -### GSK2014-A (2019) BCCWJ edition -The named entity recognition model of GiNZA v5 is trained on a part of -[GSK2014-A](https://www.gsk.or.jp/catalog/gsk2014-a/) (2019) BCCWJ edition -([Hashimoto, Inui, and Murakami:2008](https://www.anlp.jp/proceedings/annual_meeting/2010/pdf_dir/C4-4.pdf)). -We use two of the named entity label systems, both -[Sekine's Extended Named Entity Hierarchy](http://liat-aip.sakura.ne.jp/ene/ene8/definition_jp/html/enedetail.html) -and extended [OntoNotes5](https://catalog.ldc.upenn.edu/docs/LDC2013T19/OntoNotes-Release-5.0.pdf). -This model is developed by National Institute for Japanese Language and Linguistics, and Megagon Labs. +An Open Source Japanese NLP Library, based on Universal Dependencies -### mC4 -The GiNZA v5 Transformers model (ja_ginza_electra) is trained by using [transformers-ud-japanese-electra-base-discriminator](https://huggingface.co/megagonlabs/transformers-ud-japanese-electra-base-discriminator) which is pretrained on more than 200 million Japanese sentences extracted from [mC4](https://huggingface.co/datasets/mc4). +***Please read the [Breaking Changes](#ginza-530) before you upgrade GiNZA.*** -Contains information from mC4 which is made available under the ODC Attribution License. -``` -@article{2019t5, - author = {Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu}, - title = {Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer}, - journal = {arXiv e-prints}, - year = {2019}, - archivePrefix = {arXiv}, - eprint = {1910.10683}, -} -``` +The GiNZA parsing models are released as part of the results of a joint research project between Recruit Co., Ltd. and the National Institute for Japanese Language and Linguistics. ## Runtime Environment -This project is developed with Python>=3.8 and pip for it. +From GiNZA v5.3.0, the runtime environment has changed to Python 3.10 or later. +We recommend using Python 3.10–3.12 because installation errors may occur with some dependent libraries. We do not recommend to use Anaconda environment because the pip install step may not work properly. Please also see the Development Environment section below. + ### Runtime set up +> [!NOTE] +> The GiNZA dependency parsing can be significantly accelerated with a GPU. +> GPU acceleration is enabled by default in Mac OS environments running on Apple Silicon. +> For details, see [3. Enabling the GPU](#3.-Enbling-the-GPU)。 + #### 1. Install GiNZA NLP Library with Transformer-based Model -Uninstall previous version of ginza and ja_ginza_electra packages: -```console -$ pip uninstall ginza ja_ginza_electra -``` -Then, install the latest version of `ginza` and `ja_ginza_electra`: -```console -$ pip install -U ginza ja_ginza_electra -``` -The package of `ja_ginza_electra` does not include `pytorch_model.bin` due to PyPI's archive size restrictions. -This large model file will be automatically downloaded at the first run time, and the locally cached file will be used for subsequent runs. +> Running the Transformers model requires at least 16GB of memory. +> If you have insufficient memory, please try the standard model described below. -If you need to install `ja_ginza_electra` along with `pytorch_model.bin` at the install time, you can specify direct link for GitHub release archive as follows: +Install the Transformers model (`ja_ginza_electra` or `ja_ginza_bert_large`) by running one of the following commands: +(This will also install GiNZA and related Transformers libraries.) ```console -$ pip install -U ginza https://github.com/megagonlabs/ginza/releases/download/latest/ja_ginza_electra-latest-with-model.tar.gz +$ pip install -U ja_ginza_electra ``` - -If you hope to accelerate the transformers-based models by using GPUs with CUDA support, you can install `spacy` by specifying the CUDA version as follows: ```console -pip install -U "spacy[cuda117]" +$ pip install -U ja_ginza_bert_large ``` -And you need to install a version of pytorch that is consistent with the CUDA version. +> [!NOTE] +> The model package installed by the above command does not include large transformer models or tokenizers. +> These large files are automatically downloaded from Hugging Face Hub on the first run, and the locally cached files are used for subsequent runs. #### 2. Install GiNZA NLP Library with Standard Model -Uninstall previous version: + +Run the following command to install the standard model `ja_ginza`. +(GiNZA-related libraries will also be installed at the same time.) ```console -$ pip uninstall ginza ja_ginza +$ pip install -U ja_ginza ``` -Then, install the latest version of `ginza` and `ja_ginza`: + +#### 3. Enabling the GPU + +On Mac OS environments running on Apple Silicon, GPU acceleration is automatically enabled under the following conditions: +- `ja_ginza` + - Python 3.10 to 3.12 (thinc-apple-ops is not supported in 3.13 and later) +- `ja_ginza_electra` and `ja_ginza_large_bert` + - Python 3.10 or later (GPU acceleration is only enabled for `transformers` in 3.13 and later) + +To enable NVIDIA GPU acceleration in a Linux OS environment, install CUDA on Linux, add the path to the CUDA libraries to the environment variable `LD_LIBRARY_PATH` like `export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH`, and then install the `ginza` package specifying the CUDA version in extras as follows: +- CUDA 11.x + - `$ pip install ginza[cuda11x]` +- CUDA 12.x + - `$ pip install ginza[cuda12x]` +- CUDA 13.x + - `$ pip install ginza[cuda13x]` + +The `ginza` command enables GPU acceleration by default if a GPU is available. + +The `-g` option of the `ginza` command allows you to specify the device number of the GPU to use (specifying `-1` disables GPU acceleration). ```console -$ pip install -U ginza ja_ginza +$ ginza -g 0 ``` - -When using Apple Silicon such as M1 or M2, you can accelerate the analysis process by installing `thinc-apple-ops`: +When running the `ginza` command without the `-g` option, the following log is output if GPU acceleration is enabled: ```console -$ pip install torch thinc-apple-ops +$ ginza +GPU #0 enabled ``` ### Execute ginza command @@ -235,10 +203,95 @@ Please read the official documents to compile user dictionaries with `sudachipy` [SudachiPy - User defined Dictionary](https://github.com/WorksApplications/SudachiPy#user-defined-dictionary) [Sudachi User Dictionary Construction (Japanese Only)](https://github.com/WorksApplications/Sudachi/blob/develop/docs/user_dict.md) +## License +GiNZA NLP Library and GiNZA Japanese Universal Dependencies Models are distributed under the +[MIT License](https://github.com/megagonlabs/ginza/blob/master/LICENSE). +You must agree and follow the MIT License to use GiNZA NLP Library and GiNZA Japanese Universal Dependencies Models. + +### Explosion / spaCy +spaCy is the key framework of GiNZA. + +[spaCy LICENSE PAGE](https://github.com/explosion/spaCy/blob/master/LICENSE) + +### Works Applications Enterprise / Sudachi/SudachiPy - SudachiDict - chiVe +SudachiPy provides high accuracies for tokenization and pos tagging. + +[Sudachi LICENSE PAGE](https://github.com/WorksApplications/Sudachi/blob/develop/LICENSE-2.0.txt), +[SudachiPy LICENSE PAGE](https://github.com/WorksApplications/SudachiPy/blob/develop/LICENSE), +[SudachiDict LEGAL PAGE](https://github.com/WorksApplications/SudachiDict/blob/develop/LEGAL), +[chiVe LICENSE PAGE](https://github.com/WorksApplications/chiVe/blob/master/LICENSE) + +### Hugging Face / transformers +The GiNZA v5 transformer models (`ja_ginza_electra` and `ja_ginza_bert_large`) use Hugging Face Transformers as inference framework. + +[transformers LICENSE PAGE](https://github.com/huggingface/transformers/blob/master/LICENSE) + +## Training Datasets + +### UD Japanese BCCWJ r2.8 +The parsing model of GiNZA v5 is trained on a part of +[UD Japanese BCCWJ](https://github.com/UniversalDependencies/UD_Japanese-BCCWJ) r2.8 +([Omura and Asahara:2018](https://www.aclweb.org/anthology/W18-6014/)). +``` +@inproceedings{omura-asahara-2018-ud, + title = "{UD}-{J}apanese {BCCWJ}: {U}niversal {D}ependencies Annotation for the {B}alanced {C}orpus of {C}ontemporary {W}ritten {J}apanese", + author = "Omura, Mai and + Asahara, Masayuki", + booktitle = "Proceedings of the Second Workshop on Universal Dependencies ({UDW} 2018)", + month = nov, + year = "2018", + address = "Brussels, Belgium", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/W18-6014/", + doi = "10.18653/v1/W18-6014", + pages = "117--125" +} +``` + +### GSK2014-A (2019) BCCWJ edition +The named entity recognition model of GiNZA v5 is trained on a part of +[GSK2014-A](https://www.gsk.or.jp/catalog/gsk2014-a/) (2019) BCCWJ edition +([Hashimoto, Inui, and Murakami:2008](https://www.anlp.jp/proceedings/annual_meeting/2010/pdf_dir/C4-4.pdf)). +We use two of the named entity label systems, both +[Sekine's Extended Named Entity Hierarchy](http://liat-aip.sakura.ne.jp/ene/ene8/definition_jp/html/enedetail.html) +and extended [OntoNotes5](https://catalog.ldc.upenn.edu/docs/LDC2013T19/OntoNotes-Release-5.0.pdf). +This model is developed by National Institute for Japanese Language and Linguistics, and Megagon Labs. + +### mC4 +`ja_ginza_electra` is a fine-tuned model of [transformers-ud-japanese-electra-base-discriminator](https://huggingface.co/megagonlabs/transformers-ud-japanese-electra-base-discriminator) which is pretrained on more than 200 million Japanese sentences extracted from [mC4](https://huggingface.co/datasets/mc4). + +The mC4 is published under the ODC Attribution License. +``` +@article{2019t5, + author = {Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu}, + title = {Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer}, + journal = {arXiv e-prints}, + year = {2019}, + archivePrefix = {arXiv}, + eprint = {1910.10683}, +} +``` + ## Releases ### version 5.x +#### ginza-5.3.0 +- 2026-09-30 +- Breaking Changes + - We changed the supported Python version to 3.10 or later, and the supported spaCy version to 3.8.16 or later. + - The recommended environment is Python 3.10 to 3.13. + - In a Python 3.14 environment, the Rust compiler is required to build some dependent libraries. + - The model package loading priority was changed to `ja_ginza_bert_large`, `ja_ginza_electra`, and `ja_ginza`. + - `ginza` command enables GPU acceleration by default under certain conditions. + - GPU acceleration can be disabled with `ginza -g -1`. +- New Features + - Enabling GPU acceleration by default in Mac OS environments with Apple Silicon + - Official release of `ja_ginza_bert_large` + - [`ginza-transformers`](https://github.com/megagonlabs/ginza-transformers) was upgraded to v1.4.0. + - Changed the transformers component to obtain both model and tokenizer from Hugging Face Hub. + - [`spacy-transformers`](https://github.com/explosion/spacy-transformers) has a requirement of `torch>=1.8.0` and `transformers<4.53.3`. + #### ginza-5.2.1 - 2026-09-01 - This release is the final version that will work with Python 3.9 or earlier. @@ -571,8 +624,8 @@ $ python -m spacy train ja ja_ginza-4.0.0 corpus/ja_ginza-ud-train.json corpus/j ``` ## Run tests -Ginza uses the pytest framework for testing, and you can run the tests via `setup.py` without install test requirements explicitly. -Some tests depends on the ginza default models (`ja-ginza`, `ja-ginza-electra`), so install them before the tests is needed. +GiNZA uses the pytest framework for testing, and you can run the tests via `setup.py` without install test requirements explicitly. +Some tests depends on the ginza parsing models, so install them before the tests is needed. ```console $ pip install ja-ginza ja-ginza-electra diff --git a/docs/bunsetu_api.md b/docs/bunsetu_api.md index 4d4189e..aba75bc 100644 --- a/docs/bunsetu_api.md +++ b/docs/bunsetu_api.md @@ -81,5 +81,5 @@ for frame, count in sorted(frames.items(), key=lambda t: -t[1]): 詳細な解説はこちらの記事をご覧ください。 -- [GiNZA version 4.0: 多言語依存構造解析技術への文節APIの統合 - Megagon Labs Blog](https://www.megagon.ai/jp/blog/ginza-version-4-0/) +- [GiNZA version 4.0: 多言語依存構造解析技術への文節APIの統合 - Megagon Labs Blog](https://megagon.ai/jp/ginza-version-4-0/) - [GiNZA - Universal Dependenciesによる実用的日本語解析 - 自然言語処理 Volume 27 Number 3](https://www.jstage.jst.go.jp/article/jnlp/27/3/27_695/_article/-char/ja/) diff --git a/docs/index.md b/docs/index.md index d25d33b..0e44b8c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -6,44 +6,33 @@   [![Downloads](https://pepy.tech/badge/ginza/week)](https://pepy.tech/project/ginza) +***GiNZAをアップグレードする際は下記の互換性情報を確認してください。*** + +GiNZAの解析モデルは、株式会社リクルートと国立国語研究所の共同研究プロジェクトの成果の一部として公開されています。 + ## What's new! +- `GiNZA v5.3.0`をリリースしました (2026.09.30) + - 動作環境を Python 3.10 以降に変更（Python 3.10 ~ 3.12 を推奨） + - `ja_ginza_bert_large` を正式にリリース + - Apple Silicon搭載のMac OS環境で自動的にGPUアクセラレーションを有効化するよう変更 - `GiNZA v5.2.1`をリリースしました (2026.09.01) - - バグ改修を行いました。 - - このリリースがPython 3.9以前で動作する最後のバージョンとなります。 + - バグ改修を行いました + - このリリースがPython 3.9以前で動作する最後のバージョンとなります - `GiNZA v5.2.0`をリリースしました (2024.03.31) - 日本語の節認定のためのAPIを追加 (experimental) -- [`ja_ginza_bert_large`のβ版を公開中](https://github.com/megagonlabs/ginza/releases/tag/v5.2.0) - - [cl-tohoku/bert-large-japanese-v2](https://huggingface.co/cl-tohoku/bert-large-japanese-v2)をベースモデルに採用 - - 精度が大幅に向上（LAS=0.938, UAS=0.949, UPOS=0.983, ENE=0.708） - - CUDAに対応し8GB以上のRAMを搭載したGPU環境、または、M1・M2などApple Silicon環境の利用を推奨 -- ginzaコマンドで日本語以外を含む全てのspaCyモデルが利用可能に - - `ginza -m en_core_web_md` の形でモデル名を指定することで[CoNLL-U](https://universaldependencies.org/format.html#syntactic-annotation)出力ツールとして利用可能 - [ginzaコマンドの解説ページ](https://megagonlabs.github.io/ginza/command_line_tool.html)の記述を拡充 - `ginza`コマンドで使用するGPUのgpu_idを`ginza -g 1`の形で指定可能に -***GiNZAをアップグレードする際は下記の互換性情報を確認してください。*** - -## GiNZA v5.2 互換性情報 -- Pythonの対応バージョンが3.8以上に変更されました -- spaCyの対応バージョンがv3.4.4に変更されました -- コマンドラインのconllu出力のmisc列にClauseHeadフィールドが追加されました - -## GiNZA v5.1 互換性情報 -- `ginza --require_gpu`および`ginza -g`オプションが引数にgpu_idを取るようになりました - - gpu_idに-1を指定(デフォルト)するとCPUのみを使用します -- v5.0以前の`ja_ginza`および`ja_ginza_electra`パケージはGiNZA v5.1で使用できません（旧バージョン向けパッケージは事前にアンインストールが必要です） - - `pip uninstall ginza ; pip uninstall ja_ginza ; pip uninstall ja_ginza_electra` -- transformersモデルの追加に伴いGiNZA v5.1インストール時は`ginza`パッケージとともに解析モデルパッケージを明示的に指定する必要があります - - 解析精度重視モデル (メモリ容量16GB以上を推奨) - - `pip install -U ginza ja_ginza_electra` - - 実行速度重視モデル - - `pip install -U ginza ja_ginza` -- トークンの活用/読み/正規形の保存先がTokenクラスのフィールドに変更されました。 - - `doc.user_data[“reading_forms”][token.i]` -> `token.morph.get(“Reading”)` - - `doc.user_data[“inflections”][token.i]` -> `token.morph.get(“Inflection”)` - - `force_using_normalized_form_as_lemma(True)` -> `token.norm_` -- 併せて[GiNZA v5.0.0 重要な変更](#ginza-500)についても確認してください +## GiNZA v5.3 互換性情報 +- Pythonの対応バージョンが3.10以上に変更されました + - 推奨動作環境は Python 3.10 ~ 3.13 です + - Python 3.14では一部の依存ライブラリのビルドにRustコンパイラが必要です +- spaCyの対応バージョンがv3.8.16以上に変更されました +- GiNZA v5.3.0より前のモデルパケージはv5.3.0で使用できないため更新が必要です +- モデルの読み込み優先度を `ja_ginza_bert_large`, `ja_ginza_electra`, `ja_ginza` の順に変更しました +- `ginza`コマンドで一定の条件を満たす場合にデフォルトでGPUアクセラレーションが有効化されるようになりました + - GPUアクセラレーションは `ginza -g -1` で無効化できます ## GiNZA v5 新機能 @@ -76,80 +65,82 @@ GiNZA v5の解析精度は以前のバージョンから飛躍的な向上を遂 | Model | LAS | UAS | UPOS | ENE | | --- | --- | --- | --- | --- | -| *ja_ginza_bert_large (β版)* | *93.8* | *94.9* | *98.3* | *70.8* | +| *ja_ginza_bert_larg* | *93.8* | *94.9* | *98.3* | *70.8* | | ja_ginza_electra | 92.3 | 93.7 | 98.1 | 61.3 | | ja_ginza (v5) | 89.2 | 91.1 | 97.0 | 53.9 | | ja_ginza (v4相当) | 89.0 | 91.0 | 95.1 | 53.1 | -`ja_ginza_electra`は`ja_ginza`に対して、5万ステップ学習時の依存関係ラベリング・単語依存構造解析の誤りを、以前のバージョンと比較して25%以上低減できました。 +`ja_ginza_bert_large`は`ja_ginza`に対して、依存関係ラベリング・単語依存構造解析の誤りを4割以上低減できました。 -また従来型モデルの`ja_ginza`においても、処理パイプラインに`morphologizer`を追加することにより、以前のバージョンと比較してUD品詞推定の誤りを4割低減できました。 - -[関根の拡張固有表現階層](http://liat-aip.sakura.ne.jp/ene/ene8/definition_jp/html/enedetail.html)を用いた拡張固有表現抽出精度(ENE)においても`ja_ginza_electra`は大幅な精度向上が得られています。GiNZAは関根の拡張固有表現階層にもとづく固有表現抽出結果を、spaCyで標準的に用いられる[OntoNotes5](https://catalog.ldc.upenn.edu/docs/LDC2013T19/OntoNotes-Release-5.0.pdf)にマッピング(変換表を適用)して出力しています。OntoNotes5は関根の拡張固有表現階層よりカテゴリ数が非常に少ない(粗い)ため、拡張固有表現をOntoNotes5体系にマッピングした場合の固有表現抽出精度は、拡張固有表現での数値より一般に高くなります。 +[関根の拡張固有表現階層](http://liat-aip.sakura.ne.jp/ene/ene8/definition_jp/html/enedetail.html)を用いた拡張固有表現抽出精度(ENE)においても`ja_ginza_bert_large`は大幅な精度向上が得られています。GiNZAは関根の拡張固有表現階層にもとづく固有表現抽出結果を、spaCyで標準的に用いられる[OntoNotes5](https://catalog.ldc.upenn.edu/docs/LDC2013T19/OntoNotes-Release-5.0.pdf)にマッピング(変換表を適用)して出力しています。OntoNotes5は関根の拡張固有表現階層よりカテゴリ数が非常に少ない(粗い)ため、拡張固有表現をOntoNotes5体系にマッピングした場合の固有表現抽出精度は、拡張固有表現での数値より一般に高くなります。 ※各モデルの学習と解析精度評価にはUD_Japanese-BCCWJ r2.8から新聞記事系のテキストを除外したものをSudachi辞書mode C(長単位)で再解析(retokenize)した上で、文節主辞情報を依存関係ラベルに組み合わせた状態のコーパスを用いています。 ## 実行環境 -GiNZAは Python 3.8以上（および対応するpip）で動作検証を行っています。 +GiNZAは Python 3.10以上で動作検証を行っています。 GiNZAをインストールする前に予めPython実行環境を構築してください。 -### 実行環境のセットアップ +### GiNZAのセットアップ -#### 1. GiNZA + Transformersモデル +> [!NOTE] +> GiNZAによる依存構造解析処理は、GPUで大幅に高速化することができます。 +> Apple Siliconで動作するMac OS環境では、デフォルトでGPUアクセラレーションが有効化されます。 +> 詳細は [3. GPUの有効化](#3.-GPUの有効化) を参照してください。 -※Transformersモデルの実行には16GB以上のメモリ容量が必要です。メモリ容量が不足する場合は後述の従来型モデルをお試しください。 +#### 1. Transformersモデル -旧バージョンのGiNZAをインストールしている場合は次のコマンドでアンインストールします。 -```console -$ pip uninstall ginza -``` +> [!NOTE] +> Transformersモデルの実行は、メモリ容量16GB以上の環境を推奨します。 +> メモリ容量が不足する場合は後述の従来型モデルをお試しください。 -旧バージョンの`ja_ginza`をインストールしている場合は次のコマンドでアンインストールします。 +次のいずれかのコマンドを実行して、Transformersモデル（`ja_ginza_electra` または `ja_ginza_bert_large`）をインストールします。 +（GiNZAおよびTransformers関連ライブラリも同時にインストールされます。） ```console -$ pip uninstall ja_ginza +$ pip install -U ja_ginza_electra ``` - -旧バージョンの`ja_ginza_electra`をインストールしている場合は次のコマンドでアンインストールします。 ```console -$ pip uninstall ja_ginza_electra +$ pip install -U ja_ginza_bert_electra ``` -次のコマンドを実行して最新のGiNZAおよびTransformersモデルをインストールします。 -```console -$ pip install -U ginza ja_ginza_electra -``` +> [!NOTE] +> 上記コマンドでインストールされるモデルパッケージには、容量の大きいtransformerモデルやトークナイザは含まれていません。 +> これらの大容量のファイルは初回実行時に自動的にHugging Face Hubからダウンロードされて、以降の実行時にはローカルにキャッシュされたファイルが使用されます。 -上記コマンドでインストールされる`ja_ginza_electra`パッケージには大容量モデルファイルは含まれていません。大容量モデルファイルは初回実行時に自動的にダウンロードされて、以降の実行時にはローカルにキャッシュされたファイルが使用されます。 +#### 2. 従来型モデル -大容量モデルファイルを含めたインストールを行うには、次のようにGitHubのリリースアーカイブを指定します。 +次のコマンドを実行して、従来型モデル（`ja_ginza`）をインストールします。 +（GiNZA関連ライブラリも同時にインストールされます。） ```console -$ pip install -U ginza https://github.com/megagonlabs/ginza/releases/download/latest/ja_ginza_electra-latest-with-model.tar.gz +$ pip install -U ja_ginza ``` -GPUを利用してtransformersモデルを高速に実行するには、実行環境に応じたオプションを指定してspacyを上書きインストールします。 +#### 3. GPUの有効化 -CUDA 11.7を使用する場合: -```console -$ pip install -U spacy[cuda117] -``` -※ あわせてpytorchもCUDAと整合したバージョンをインストールする必要があります。 +Apple Siliconで動作するMac OS環境では、次の条件を満たす場合に自動でGPUアクセラレーションが有効化されます。 +- `ja_ginza` + - Python 3.10 〜 3.12 (3.13以降は`thinc-apple-ops`が非対応) +- `ja_ginza_electra` および `ja_ginza_large_bert` + - Python 3.10 以降 (3.13以降は`thinc-apple-ops`が非対応のため`transformers`のみGPUが有効化される) -#### 2. GiNZA + 従来型モデル +Linux OS環境でNVIDIA GPUによるアクセラレーションを有効化するには、LinuxにCUDAをインストールし、`export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH` のように環境変数 `LD_LIBRARY_PATH` にCUDAライブラリのパスを追加した上で、次のようにCUDAバージョンをextrasに指定して`ginza`パッケージのインストールを行います。 +- CUDA 11.x + - `$ pip install ginza[cuda11x]` +- CUDA 12.x + - `$ pip install ginza[cuda12x]` +- CUDA 13.x + - `$ pip install ginza[cuda13x]` -旧バージョンのGiNZAおよび`ja_ginza`をインストールしている場合は次のコマンドでアンインストールを実行します。 -```console -$ pip uninstall ginza _ginza -``` +`ginza`コマンドは、GPUが利用可能な場合はデフォルトでGPUアクセラレーションを有効化します。 -次のコマンドを実行して最新のGiNZAと従来型モデルをインストールします。 +`ginza`コマンドの`-g`オプションで、使用するGPUのデバイス番号を指定することができます（`-1`を指定するとGPUアクセラレーションは無効化されます）。 ```console -$ pip install -U ginza ja_ginza +$ ginza -g 0 ``` - -M1やM2などのMPSに対応したApple Silicon環境では、`thinc-apple-ops`を導入することで解析速度が向上します。 +`ginza`コマンドを`-g`オプションなしで実行する際、GPUアクセラレーションが有効化された場合は次のログが出力されます。 ```console -$ pip install torch thinc-apple-ops +$ ginza +GPU #0 enabled ``` ### ginzaコマンドによる解析処理の実行 @@ -170,7 +161,7 @@ $ ginza 9 。 。 PUNCT 補助記号-句点 _ 6 punct _ SpaceAfter=No|BunsetuBILabel=I|BunsetuPositionType=CONT|Reading=。|ClauseHead=6 ``` -実行環境に`ja_ginza_electra`と`ja_ginza`の両方のモデルがインストールされている場合、`ginza`コマンドは`ja_ginza_electra`を優先して使用します。同じ状況で`ja_ginza`を使用するには`ginza -m`オプションでモデル名を指定します。 +実行環境に複数のモデルパッケージがインストールされている場合、`ginza`コマンドは解析精度が高いものを優先して使用します。モデルパッケージを明示的に指定して使用するには`-m`オプションでモデルパッケージ名を指定します。 ```console # ginza -m ja_ginza ``` @@ -183,7 +174,7 @@ $ ginza ### Pythonコードによる解析処理の実行 次のコードは、Transformersモデルによる依存構造解析結果を文単位で出力します。 -従来型モデルを用いる場合は`ja_ginza_electra`を`ja_ginza`に置き換えます。 +他のモデルパッケージを用いる場合は`ja_ginza_electra`の部分を目的のモデルに置き換えます。 ```python import spacy nlp = spacy.load('ja_ginza_electra') @@ -227,9 +218,9 @@ for sent in doc.sents: ### 解説記事 -- [GiNZA Version 4.0: Improving Syntactic Structure Analysis Through Japanese Bunsetsu-Phrase Extraction API Integration](https://megagon.ai/blog/ginza-version-4-0-improving-syntactic-structure-analysis-through-japanese-bunsetsu-phrase-extraction-api-integration/) - Megagon Labs Blog (2021.03) -- [GiNZA version 4.0: 多言語依存構造解析技術への文節APIの統合](https://www.megagon.ai/jp/blog/ginza-version-4-0/) - Megagon Labs Blog (2020.09) -- [GiNZA: 日本語自然言語処理オープンソースライブラリ](https://www.megagon.ai/jp/projects/ginza-install-a-japanese-nlp-library-in-one-step/) - Megagon Labs (2019) +- [GiNZA Version 4.0: Improving Syntactic Structure Analysis Through Japanese Bunsetsu-Phrase Extraction API Integration](https://megagon.ai/en/ginza-version-4-0/) - Megagon Labs Blog (2021.03) +- [GiNZA version 4.0: 多言語依存構造解析技術への文節APIの統合](https://megagon.ai/jp/ginza-version-4-0/) - Megagon Labs Blog (2020.09) +- [GiNZA: 日本語自然言語処理オープンソースライブラリ](https://megagon.ai/jp/ginza/) - Megagon Labs (2019) ## ライセンス GiNZA NLPライブラリおよびGiNZA日本語Universal Dependenciesモデルは @@ -250,7 +241,7 @@ GiNZAはトークン化（形態素解析）処理にSudachiPyを、単語ベク [chiVe LICENSE PAGE](https://github.com/WorksApplications/chiVe/blob/master/LICENSE) ### Hugging Face / transformers -GiNZA v5 Transformersモデル(ja_ginza_electra)はHugging Face社が提供するtransformersを事前学習フレームワークに用いています。 +GiNZA v5 TransformersモデルはHugging Face社が提供するtransformersを推論フレームワークに用いています。 [transformers LICENSE PAGE](https://github.com/huggingface/transformers/blob/master/LICENSE) @@ -261,7 +252,21 @@ GiNZA v5の依存構造解析モデルは [UD Japanese BCCWJ](https://github.com/UniversalDependencies/UD_Japanese-BCCWJ) r2.8 ([Omura and Asahara:2018](https://www.aclweb.org/anthology/W18-6014/)) から新聞系文書を除外して学習しています。 -GiNZA v5の依存構造解析モデルは国立国語研究所とMegagon Labsの共同研究成果です。 +``` +@inproceedings{omura-asahara-2018-ud, + title = "{UD}-{J}apanese {BCCWJ}: {U}niversal {D}ependencies Annotation for the {B}alanced {C}orpus of {C}ontemporary {W}ritten {J}apanese", + author = "Omura, Mai and + Asahara, Masayuki", + booktitle = "Proceedings of the Second Workshop on Universal Dependencies ({UDW} 2018)", + month = nov, + year = "2018", + address = "Brussels, Belgium", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/W18-6014/", + doi = "10.18653/v1/W18-6014", + pages = "117--125" +} +``` ### GSK2014-A (2019) BCCWJ版 GiNZA v5の固有表現抽出モデルは @@ -274,7 +279,7 @@ GiNZA v5の固有表現抽出モデルは GiNZA v5の固有表現抽出モデルは国立国語研究所とMegagon Labsの共同研究成果です。 ### mC4 -GiNZA v5 Transformersモデル(ja_ginza_electra)は、[mC4](https://huggingface.co/datasets/mc4)から抽出した日本語20億文以上を用いて事前学習した[transformers-ud-japanese-electra-base-discriminator](https://huggingface.co/megagonlabs/transformers-ud-japanese-electra-base-discriminator)を使用しています。 +`ja_ginza_electra`は、[mC4](https://huggingface.co/datasets/mc4)から抽出した日本語20億文以上を用いて事前学習した[transformers-ud-japanese-electra-base-discriminator](https://huggingface.co/megagonlabs/transformers-ud-japanese-electra-base-discriminator)を使用しています。 mC4はODC-BYライセンスの規約に基づいて事前学習データとして利用しています。 Contains information from mC4 which is made available under the ODC Attribution License. @@ -289,12 +294,27 @@ Contains information from mC4 which is made available under the ODC Attribution } ``` - - ## [リリース履歴](https://github.com/megagonlabs/ginza/releases) ### version 5.x +#### ginza-5.3.0 +- 2026-09-30 +- 重要な変更 + - 本リリースからサポート対象のPythonバージョンが3.10以上に、spaCyのバージョンが3.8.16以上に変更されました。 + - 推奨動作環境は Python 3.10 ~ 3.12 です。 + - Python 3.14では一部の依存ライブラリのビルドにRustコンパイラが必要です。 + - GiNZA v5.3.0より前のモデルパケージはGiNZA v5.3.0以降で使用できません。 + - モデルパッケージの読み込み優先度を `ja_ginza_bert_large`, `ja_ginza_electra`, `ja_ginza` の順に変更しました。 + - `ginza`コマンドで一定の条件を満たす場合にデフォルトでGPUアクセラレーションが有効化されるようになりました。 + - GPUアクセラレーションは `ginza -g -1` で無効化できます。 +- 新機能 + - Apple Silicon搭載のMac OS環境で自動的にGPUアクセラレーションを有効化するよう変更しました。 + - `ja_ginza_bert_large` を正式にリリースしました。 + - [`ginza-transformers`](https://github.com/megagonlabs/ginza-transformers) をv1.4.0にアップグレードしました。 + - transformers componentでmodelとtokenizerの両方をHugging Face Hubから取得する形に変更しました。 + - [`spacy-transformers`](https://github.com/explosion/spacy-transformers)のrequirementsにより `torch>=1.8.0, transformers<4.53.3` などの制限があります。 + #### ginza-5.2.1 - 2026-09-01 - このリリースがPython 3.9以前で動作する最後のバージョンとなります。 @@ -315,11 +335,15 @@ Contains information from mC4 which is made available under the ODC Attribution - Migrate to spaCy v3.7 - New functionality - add Japanese clause recognition API (experimental) + - コマンドラインのconllu出力のmisc列にClauseHeadフィールドが追加されました #### ginza-5.1.3 - 2023-09-25 - Migrate to spaCy v3.6 -- Beta release of `ja_ginza_bert_large` +- [`ja_ginza_bert_large` β版を公開](https://github.com/megagonlabs/ginza/releases/tag/v5.2.0) + - [tohoku-nlp/bert-large-japanese-v2](https://huggingface.co/tohoku-nlp/bert-large-japanese-v2)をベースモデルに採用 + - 精度が大幅に向上（LAS=0.938, UAS=0.949, UPOS=0.983, ENE=0.708） + - CUDAに対応し8GB以上のRAMを搭載したGPU環境、または、M1・M2などApple Silicon環境の利用を推奨 #### ginza-5.1.2 - 2022-03-12 From a3eeddff456563ac7b042b8a4a0a12d37593a107 Mon Sep 17 00:00:00 2001 From: Hiroshi Matsuda Date: Sun, 13 Sep 2026 16:23:13 +0900 Subject: [PATCH 16/24] use has_torch_mps instead of get_current_ops() --- README.md | 4 +++- docs/index.md | 13 ++++++++----- ginza/command_line.py | 12 ++++-------- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 239f1ac..ee83f87 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The GiNZA parsing models are released as part of the results of a joint research ## Runtime Environment From GiNZA v5.3.0, the runtime environment has changed to Python 3.10 or later. -We recommend using Python 3.10–3.12 because installation errors may occur with some dependent libraries. +We recommend using Python 3.10–3.13 because installation errors may occur with some dependent libraries. We do not recommend to use Anaconda environment because the pip install step may not work properly. Please also see the Development Environment section below. @@ -44,6 +44,7 @@ $ pip install -U ja_ginza_bert_large > [!NOTE] > The model package installed by the above command does not include large transformer models or tokenizers. > These large files are automatically downloaded from Hugging Face Hub on the first run, and the locally cached files are used for subsequent runs. +> The first time you run transformers model, it will take several seconds to tens of seconds to start up due to the initialization process. #### 2. Install GiNZA NLP Library with Standard Model @@ -287,6 +288,7 @@ The mC4 is published under the ODC Attribution License. - GPU acceleration can be disabled with `ginza -g -1`. - New Features - Enabling GPU acceleration by default in Mac OS environments with Apple Silicon + - The Python versions that can be used with `thinc-apple-ops` on Mac OS running Apple Silicon are 3.10 through 3.12. - Official release of `ja_ginza_bert_large` - [`ginza-transformers`](https://github.com/megagonlabs/ginza-transformers) was upgraded to v1.4.0. - Changed the transformers component to obtain both model and tokenizer from Hugging Face Hub. diff --git a/docs/index.md b/docs/index.md index 0e44b8c..1b322ca 100644 --- a/docs/index.md +++ b/docs/index.md @@ -13,9 +13,10 @@ GiNZAの解析モデルは、株式会社リクルートと国立国語研究所 ## What's new! - `GiNZA v5.3.0`をリリースしました (2026.09.30) - - 動作環境を Python 3.10 以降に変更（Python 3.10 ~ 3.12 を推奨） + - 動作環境を Python 3.10 以降に変更（Python 3.10 ~ 3.13 の使用を推奨） - `ja_ginza_bert_large` を正式にリリース - Apple Silicon搭載のMac OS環境で自動的にGPUアクセラレーションを有効化するよう変更 + - thinc-apple-ops を適用可能なPythonのバージョンは 3.10 〜 3.12 です - `GiNZA v5.2.1`をリリースしました (2026.09.01) - バグ改修を行いました - このリリースがPython 3.9以前で動作する最後のバージョンとなります @@ -85,7 +86,7 @@ GiNZAをインストールする前に予めPython実行環境を構築してく > [!NOTE] > GiNZAによる依存構造解析処理は、GPUで大幅に高速化することができます。 -> Apple Siliconで動作するMac OS環境では、デフォルトでGPUアクセラレーションが有効化されます。 +> Apple Silicon搭載のMac OS環境では、デフォルトでGPUアクセラレーションが有効化されます。 > 詳細は [3. GPUの有効化](#3.-GPUの有効化) を参照してください。 #### 1. Transformersモデル @@ -104,8 +105,9 @@ $ pip install -U ja_ginza_bert_electra ``` > [!NOTE] -> 上記コマンドでインストールされるモデルパッケージには、容量の大きいtransformerモデルやトークナイザは含まれていません。 +> 上記コマンドでインストールされるモデルパッケージには、容量の大きいTransformersモデルやトークナイザは含まれていません。 > これらの大容量のファイルは初回実行時に自動的にHugging Face Hubからダウンロードされて、以降の実行時にはローカルにキャッシュされたファイルが使用されます。 +> 初回実行時は初期化処理のため起動に数秒〜数十秒かかります。 #### 2. 従来型モデル @@ -117,7 +119,7 @@ $ pip install -U ja_ginza #### 3. GPUの有効化 -Apple Siliconで動作するMac OS環境では、次の条件を満たす場合に自動でGPUアクセラレーションが有効化されます。 +Apple Silicon搭載のMac OS環境では、次の条件を満たす場合に自動でGPUアクセラレーションが有効化されます。 - `ja_ginza` - Python 3.10 〜 3.12 (3.13以降は`thinc-apple-ops`が非対応) - `ja_ginza_electra` および `ja_ginza_large_bert` @@ -302,7 +304,7 @@ Contains information from mC4 which is made available under the ODC Attribution - 2026-09-30 - 重要な変更 - 本リリースからサポート対象のPythonバージョンが3.10以上に、spaCyのバージョンが3.8.16以上に変更されました。 - - 推奨動作環境は Python 3.10 ~ 3.12 です。 + - 推奨動作環境は Python 3.10 〜 3.13 です。 - Python 3.14では一部の依存ライブラリのビルドにRustコンパイラが必要です。 - GiNZA v5.3.0より前のモデルパケージはGiNZA v5.3.0以降で使用できません。 - モデルパッケージの読み込み優先度を `ja_ginza_bert_large`, `ja_ginza_electra`, `ja_ginza` の順に変更しました。 @@ -310,6 +312,7 @@ Contains information from mC4 which is made available under the ODC Attribution - GPUアクセラレーションは `ginza -g -1` で無効化できます。 - 新機能 - Apple Silicon搭載のMac OS環境で自動的にGPUアクセラレーションを有効化するよう変更しました。 + - Apple Silicon搭載のMac OSで thinc-apple-ops を適用可能なPythonのバージョンは 3.10 〜 3.12 です。 - `ja_ginza_bert_large` を正式にリリースしました。 - [`ginza-transformers`](https://github.com/megagonlabs/ginza-transformers) をv1.4.0にアップグレードしました。 - transformers componentでmodelとtokenizerの両方をHugging Face Hubから取得する形に変更しました。 diff --git a/ginza/command_line.py b/ginza/command_line.py index 4de596c..e4da6ac 100644 --- a/ginza/command_line.py +++ b/ginza/command_line.py @@ -8,8 +8,7 @@ from typing import Generator, Iterable, Optional, List import plac -from thinc.api import get_current_ops -from thinc.compat import has_cupy_gpu +from thinc.compat import has_cupy_gpu, has_torch_mps from . import default_model_name, GINZA_MODEL_PACKAGES from .analyzer import Analyzer @@ -100,14 +99,11 @@ def run( if output_format in [2, "mecab"]: if require_gpu is not None and require_gpu >= 0: - print(f"GPU is disabled in mecab mode.", file=sys.stderr) + print(f"GPU not used in mecab mode.", file=sys.stderr) require_gpu = -1 elif require_gpu is None: - if get_current_ops().name == "apple": - if model_name_or_path and TRANSFORMERS_MODEL_PATTERN.match(model_name_or_path.rstrip("/")): - require_gpu = 0 - else: - require_gpu = -1 + if has_torch_mps: + require_gpu = 0 elif has_cupy_gpu: require_gpu = 0 else: From 0d78296c195e3045adb49d6c3b4427b247a7abda Mon Sep 17 00:00:00 2001 From: Hiroshi Matsuda Date: Sun, 13 Sep 2026 16:32:31 +0900 Subject: [PATCH 17/24] fix typos --- README.md | 2 +- docs/index.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ee83f87..c7bfb91 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Please also see the Development Environment section below. > [!NOTE] > The GiNZA dependency parsing can be significantly accelerated with a GPU. > GPU acceleration is enabled by default in Mac OS environments running on Apple Silicon. -> For details, see [3. Enabling the GPU](#3.-Enbling-the-GPU)。 +> For details, see [3. Enabling the GPU](#3.-Enabling-the-GPU)。 #### 1. Install GiNZA NLP Library with Transformer-based Model diff --git a/docs/index.md b/docs/index.md index 1b322ca..3af17f3 100644 --- a/docs/index.md +++ b/docs/index.md @@ -66,7 +66,7 @@ GiNZA v5の解析精度は以前のバージョンから飛躍的な向上を遂 | Model | LAS | UAS | UPOS | ENE | | --- | --- | --- | --- | --- | -| *ja_ginza_bert_larg* | *93.8* | *94.9* | *98.3* | *70.8* | +| *ja_ginza_bert_large* | *93.8* | *94.9* | *98.3* | *70.8* | | ja_ginza_electra | 92.3 | 93.7 | 98.1 | 61.3 | | ja_ginza (v5) | 89.2 | 91.1 | 97.0 | 53.9 | | ja_ginza (v4相当) | 89.0 | 91.0 | 95.1 | 53.1 | From 1b2e4da6909aab02e4439a1e77ffd2251f8b2204 Mon Sep 17 00:00:00 2001 From: Hiroshi Matsuda Date: Mon, 14 Sep 2026 13:08:56 +0900 Subject: [PATCH 18/24] improve init log --- ginza/command_line.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ginza/command_line.py b/ginza/command_line.py index e4da6ac..bacbf05 100644 --- a/ginza/command_line.py +++ b/ginza/command_line.py @@ -8,6 +8,7 @@ from typing import Generator, Iterable, Optional, List import plac +from thinc.api import get_current_ops from thinc.compat import has_cupy_gpu, has_torch_mps from . import default_model_name, GINZA_MODEL_PACKAGES from .analyzer import Analyzer @@ -114,6 +115,8 @@ def run( else: require_gpu = -1 print(f"GPU disabled because 'parallel_level' set to {parallel_level}", file=sys.stderr) + elif get_current_ops().name == "apple": + print(f"thinc-apple-ops enabled", file=sys.stderr) assert parallel_level == 1 or require_gpu == -1, "require_gpu not allowed for multi-processing. https://github.com/explosion/spaCy/issues/5507" From 5963fd0c8455176d93f4afde5312faa1ddd77922 Mon Sep 17 00:00:00 2001 From: Hiroshi Matsuda Date: Mon, 14 Sep 2026 13:09:19 +0900 Subject: [PATCH 19/24] update docs --- README.md | 9 ++++++--- docs/index.md | 23 +++++++++++++---------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index c7bfb91..65a27b7 100644 --- a/README.md +++ b/README.md @@ -76,12 +76,15 @@ The `-g` option of the `ginza` command allows you to specify the device number o ```console $ ginza -g 0 ``` -When running the `ginza` command without the `-g` option, the following log is output if GPU acceleration is enabled: +When running the `ginza` command without the `-g` option, the following init log is output if GPU acceleration is enabled: ```console $ ginza GPU #0 enabled ``` +> [!NOTE] +> When executing standard model in Mac OS running on Apple Silicon, the init log will be `thinc-apple-ops enabled`. + ### Execute ginza command Run `ginza` command from the console, then input some Japanese text. After pressing enter key, you will get the parsed results with [CoNLL-U Syntactic Annotation](https://universaldependencies.org/format.html#syntactic-annotation) format. @@ -278,7 +281,7 @@ The mC4 is published under the ODC Attribution License. ### version 5.x #### ginza-5.3.0 -- 2026-09-30 +- 2026-09-30, Grossular Garnet - Breaking Changes - We changed the supported Python version to 3.10 or later, and the supported spaCy version to 3.8.16 or later. - The recommended environment is Python 3.10 to 3.13. @@ -309,7 +312,7 @@ The mC4 is published under the ODC Attribution License. - [#266](https://github.com/megagonlabs/ginza/pull/266) - [#273](https://github.com/megagonlabs/ginza/pull/273) #### ginza-5.2.0 -- 2024-03-31 +- 2024-03-31, Fluorite - Require python>=3.8 - Migrate to spaCy v3.7 - New functionality diff --git a/docs/index.md b/docs/index.md index 3af17f3..7f3a42e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -12,15 +12,15 @@ GiNZAの解析モデルは、株式会社リクルートと国立国語研究所 ## What's new! -- `GiNZA v5.3.0`をリリースしました (2026.09.30) +- `GiNZA v5.3.0 - Grossular Garnet` をリリースしました (2026.09.30) - 動作環境を Python 3.10 以降に変更（Python 3.10 ~ 3.13 の使用を推奨） - `ja_ginza_bert_large` を正式にリリース - - Apple Silicon搭載のMac OS環境で自動的にGPUアクセラレーションを有効化するよう変更 + - Apple Siliconで動作するMac OS環境で自動的にGPUアクセラレーションを有効化するよう変更 - thinc-apple-ops を適用可能なPythonのバージョンは 3.10 〜 3.12 です - `GiNZA v5.2.1`をリリースしました (2026.09.01) - バグ改修を行いました - このリリースがPython 3.9以前で動作する最後のバージョンとなります -- `GiNZA v5.2.0`をリリースしました (2024.03.31) +- `GiNZA v5.2.0 - Fluorite`をリリースしました (2024.03.31) - 日本語の節認定のためのAPIを追加 (experimental) - [ginzaコマンドの解説ページ](https://megagonlabs.github.io/ginza/command_line_tool.html)の記述を拡充 - `ginza`コマンドで使用するGPUのgpu_idを`ginza -g 1`の形で指定可能に @@ -86,7 +86,7 @@ GiNZAをインストールする前に予めPython実行環境を構築してく > [!NOTE] > GiNZAによる依存構造解析処理は、GPUで大幅に高速化することができます。 -> Apple Silicon搭載のMac OS環境では、デフォルトでGPUアクセラレーションが有効化されます。 +> Apple Siliconで動作するMac OS環境では、デフォルトでGPUアクセラレーションが有効化されます。 > 詳細は [3. GPUの有効化](#3.-GPUの有効化) を参照してください。 #### 1. Transformersモデル @@ -119,7 +119,7 @@ $ pip install -U ja_ginza #### 3. GPUの有効化 -Apple Silicon搭載のMac OS環境では、次の条件を満たす場合に自動でGPUアクセラレーションが有効化されます。 +Apple Siliconで動作するMac OS環境では、次の条件を満たす場合に自動でGPUアクセラレーションが有効化されます。 - `ja_ginza` - Python 3.10 〜 3.12 (3.13以降は`thinc-apple-ops`が非対応) - `ja_ginza_electra` および `ja_ginza_large_bert` @@ -139,12 +139,15 @@ Linux OS環境でNVIDIA GPUによるアクセラレーションを有効化す ```console $ ginza -g 0 ``` -`ginza`コマンドを`-g`オプションなしで実行する際、GPUアクセラレーションが有効化された場合は次のログが出力されます。 +`ginza`コマンドを`-g`オプションなしで実行する際、GPUアクセラレーションが有効化された場合は次のログが最初に出力されます。 ```console $ ginza GPU #0 enabled ``` +> [!NOTE] +> Apple Siliconで動作するMac OS環境で従来型モデルを実行する場合、最初のログ出力は `thinc-apple-ops enabled` となります。 + ### ginzaコマンドによる解析処理の実行 `ginza`コマンドを実行して、日本語の文に続けてEnterを入力すると、[CoNLL-U Syntactic Annotation](https://universaldependencies.org/format.html#syntactic-annotation) 形式で解析結果が出力されます。 @@ -301,7 +304,7 @@ Contains information from mC4 which is made available under the ODC Attribution ### version 5.x #### ginza-5.3.0 -- 2026-09-30 +- 2026-09-30, Grossular Garnet - 重要な変更 - 本リリースからサポート対象のPythonバージョンが3.10以上に、spaCyのバージョンが3.8.16以上に変更されました。 - 推奨動作環境は Python 3.10 〜 3.13 です。 @@ -311,8 +314,8 @@ Contains information from mC4 which is made available under the ODC Attribution - `ginza`コマンドで一定の条件を満たす場合にデフォルトでGPUアクセラレーションが有効化されるようになりました。 - GPUアクセラレーションは `ginza -g -1` で無効化できます。 - 新機能 - - Apple Silicon搭載のMac OS環境で自動的にGPUアクセラレーションを有効化するよう変更しました。 - - Apple Silicon搭載のMac OSで thinc-apple-ops を適用可能なPythonのバージョンは 3.10 〜 3.12 です。 + - Apple Siliconで動作するMac OS環境で自動的にGPUアクセラレーションを有効化するよう変更しました。 + - Apple Siliconで動作するMac OSで thinc-apple-ops を適用可能なPythonのバージョンは 3.10 〜 3.12 です。 - `ja_ginza_bert_large` を正式にリリースしました。 - [`ginza-transformers`](https://github.com/megagonlabs/ginza-transformers) をv1.4.0にアップグレードしました。 - transformers componentでmodelとtokenizerの両方をHugging Face Hubから取得する形に変更しました。 @@ -333,7 +336,7 @@ Contains information from mC4 which is made available under the ODC Attribution - [#266](https://github.com/megagonlabs/ginza/pull/266) - [#273](https://github.com/megagonlabs/ginza/pull/273) #### ginza-5.2.0 -- 2024-03-31 +- 2024-03-31, Fluorite - Require python>=3.8 - Migrate to spaCy v3.7 - New functionality From a27967376d139a243580c746b14db64dd3a88324 Mon Sep 17 00:00:00 2001 From: Hiroshi Matsuda Date: Mon, 14 Sep 2026 13:22:41 +0900 Subject: [PATCH 20/24] improve docs --- README.md | 14 ++++++++------ docs/index.md | 18 ++++++++++-------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 65a27b7..455de0d 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Please also see the Development Environment section below. > [!NOTE] > The GiNZA dependency parsing can be significantly accelerated with a GPU. -> GPU acceleration is enabled by default in Mac OS environments running on Apple Silicon. +> The hardware acceleration is enabled by default in Mac OS environments running on Apple Silicon. > For details, see [3. Enabling the GPU](#3.-Enabling-the-GPU)。 #### 1. Install GiNZA NLP Library with Transformer-based Model @@ -56,7 +56,7 @@ $ pip install -U ja_ginza #### 3. Enabling the GPU -On Mac OS environments running on Apple Silicon, GPU acceleration is automatically enabled under the following conditions: +On Mac OS environments running on Apple Silicon, the hardware acceleration is automatically enabled under the following conditions: - `ja_ginza` - Python 3.10 to 3.12 (thinc-apple-ops is not supported in 3.13 and later) - `ja_ginza_electra` and `ja_ginza_large_bert` @@ -81,9 +81,11 @@ When running the `ginza` command without the `-g` option, the following init log $ ginza GPU #0 enabled ``` - -> [!NOTE] -> When executing standard model in Mac OS running on Apple Silicon, the init log will be `thinc-apple-ops enabled`. +When executing standard model in Mac OS running on Apple Silicon, if hardware acceleration is enabled, the init log will be: +```console +$ ginza +thinc-apple-ops enabled +``` ### Execute ginza command Run `ginza` command from the console, then input some Japanese text. @@ -290,7 +292,7 @@ The mC4 is published under the ODC Attribution License. - `ginza` command enables GPU acceleration by default under certain conditions. - GPU acceleration can be disabled with `ginza -g -1`. - New Features - - Enabling GPU acceleration by default in Mac OS environments with Apple Silicon + - Enabling hardware acceleration by default in Mac OS environments with Apple Silicon - The Python versions that can be used with `thinc-apple-ops` on Mac OS running Apple Silicon are 3.10 through 3.12. - Official release of `ja_ginza_bert_large` - [`ginza-transformers`](https://github.com/megagonlabs/ginza-transformers) was upgraded to v1.4.0. diff --git a/docs/index.md b/docs/index.md index 7f3a42e..ece46cc 100644 --- a/docs/index.md +++ b/docs/index.md @@ -15,7 +15,7 @@ GiNZAの解析モデルは、株式会社リクルートと国立国語研究所 - `GiNZA v5.3.0 - Grossular Garnet` をリリースしました (2026.09.30) - 動作環境を Python 3.10 以降に変更（Python 3.10 ~ 3.13 の使用を推奨） - `ja_ginza_bert_large` を正式にリリース - - Apple Siliconで動作するMac OS環境で自動的にGPUアクセラレーションを有効化するよう変更 + - Apple Siliconで動作するMac OS環境で自動的にハードウェアアクセラレーションを有効化するよう変更 - thinc-apple-ops を適用可能なPythonのバージョンは 3.10 〜 3.12 です - `GiNZA v5.2.1`をリリースしました (2026.09.01) - バグ改修を行いました @@ -86,7 +86,7 @@ GiNZAをインストールする前に予めPython実行環境を構築してく > [!NOTE] > GiNZAによる依存構造解析処理は、GPUで大幅に高速化することができます。 -> Apple Siliconで動作するMac OS環境では、デフォルトでGPUアクセラレーションが有効化されます。 +> Apple Siliconで動作するMac OS環境では、デフォルトでハードウェアアクセラレーションが有効化されます。 > 詳細は [3. GPUの有効化](#3.-GPUの有効化) を参照してください。 #### 1. Transformersモデル @@ -119,11 +119,11 @@ $ pip install -U ja_ginza #### 3. GPUの有効化 -Apple Siliconで動作するMac OS環境では、次の条件を満たす場合に自動でGPUアクセラレーションが有効化されます。 +Apple Siliconで動作するMac OS環境では、次の条件を満たす場合に自動でハードウェアアクセラレーションが有効化されます。 - `ja_ginza` - Python 3.10 〜 3.12 (3.13以降は`thinc-apple-ops`が非対応) - `ja_ginza_electra` および `ja_ginza_large_bert` - - Python 3.10 以降 (3.13以降は`thinc-apple-ops`が非対応のため`transformers`のみGPUが有効化される) + - Python 3.10 以降 (3.13以降は`thinc-apple-ops`が非対応のため`transformers`のみGPUアクセラレーションが有効化される) Linux OS環境でNVIDIA GPUによるアクセラレーションを有効化するには、LinuxにCUDAをインストールし、`export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH` のように環境変数 `LD_LIBRARY_PATH` にCUDAライブラリのパスを追加した上で、次のようにCUDAバージョンをextrasに指定して`ginza`パッケージのインストールを行います。 - CUDA 11.x @@ -144,9 +144,11 @@ $ ginza -g 0 $ ginza GPU #0 enabled ``` - -> [!NOTE] -> Apple Siliconで動作するMac OS環境で従来型モデルを実行する場合、最初のログ出力は `thinc-apple-ops enabled` となります。 +Apple Siliconで動作するMac OS環境で従来型モデルを実行する際、ハードウェアアクセラレーションが有効化された場合は次のログが最初に出力されます。 +```console +$ ginza +thinc-apple-ops enabled +``` ### ginzaコマンドによる解析処理の実行 @@ -314,7 +316,7 @@ Contains information from mC4 which is made available under the ODC Attribution - `ginza`コマンドで一定の条件を満たす場合にデフォルトでGPUアクセラレーションが有効化されるようになりました。 - GPUアクセラレーションは `ginza -g -1` で無効化できます。 - 新機能 - - Apple Siliconで動作するMac OS環境で自動的にGPUアクセラレーションを有効化するよう変更しました。 + - Apple Siliconで動作するMac OS環境で自動的にハードウェアアクセラレーションを有効化するよう変更しました。 - Apple Siliconで動作するMac OSで thinc-apple-ops を適用可能なPythonのバージョンは 3.10 〜 3.12 です。 - `ja_ginza_bert_large` を正式にリリースしました。 - [`ginza-transformers`](https://github.com/megagonlabs/ginza-transformers) をv1.4.0にアップグレードしました。 From e9710f21f31fd42644da15950cf87a5c31aa2dbc Mon Sep 17 00:00:00 2001 From: Hiroshi Matsuda Date: Fri, 25 Sep 2026 16:42:30 +0900 Subject: [PATCH 21/24] update reference URLs --- README.md | 3 +++ docs/index.md | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 455de0d..1e87623 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,9 @@ An Open Source Japanese NLP Library, based on Universal Dependencies The GiNZA parsing models are released as part of the results of a joint research project between Recruit Co., Ltd. and the National Institute for Japanese Language and Linguistics. +Please also read the English documentation. +- [GiNZA Version 4.0: Improving Syntactic Structure Analysis Through Japanese Bunsetsu-Phrase Extraction API Integration](https://megagon.ai/ginza-version-4-0-improving-syntactic-structure-analysis-through-japanese-bunsetsu-phrase-extraction-api-integration/) - Megagon Labs Blog (2021.03) + ## Runtime Environment From GiNZA v5.3.0, the runtime environment has changed to Python 3.10 or later. We recommend using Python 3.10–3.13 because installation errors may occur with some dependent libraries. diff --git a/docs/index.md b/docs/index.md index ece46cc..3d039ac 100644 --- a/docs/index.md +++ b/docs/index.md @@ -225,8 +225,8 @@ for sent in doc.sents: ### 解説記事 -- [GiNZA Version 4.0: Improving Syntactic Structure Analysis Through Japanese Bunsetsu-Phrase Extraction API Integration](https://megagon.ai/en/ginza-version-4-0/) - Megagon Labs Blog (2021.03) - [GiNZA version 4.0: 多言語依存構造解析技術への文節APIの統合](https://megagon.ai/jp/ginza-version-4-0/) - Megagon Labs Blog (2020.09) + - (英語版) [GiNZA Version 4.0: Improving Syntactic Structure Analysis Through Japanese Bunsetsu-Phrase Extraction API Integration](https://megagon.ai/ginza-version-4-0-improving-syntactic-structure-analysis-through-japanese-bunsetsu-phrase-extraction-api-integration/) - [GiNZA: 日本語自然言語処理オープンソースライブラリ](https://megagon.ai/jp/ginza/) - Megagon Labs (2019) ## ライセンス From 721bb513c28e4c110b8f12b6a54e137ef88b2ef1 Mon Sep 17 00:00:00 2001 From: Hiroshi Matsuda Date: Sat, 26 Sep 2026 01:26:21 +0900 Subject: [PATCH 22/24] update reference URLs --- docs/bunsetu_api.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/bunsetu_api.md b/docs/bunsetu_api.md index aba75bc..d58bf19 100644 --- a/docs/bunsetu_api.md +++ b/docs/bunsetu_api.md @@ -82,4 +82,5 @@ for frame, count in sorted(frames.items(), key=lambda t: -t[1]): 詳細な解説はこちらの記事をご覧ください。 - [GiNZA version 4.0: 多言語依存構造解析技術への文節APIの統合 - Megagon Labs Blog](https://megagon.ai/jp/ginza-version-4-0/) + - [English version](https://megagon.ai/ginza-version-4-0-improving-syntactic-structure-analysis-through-japanese-bunsetsu-phrase-extraction-api-integration/) - [GiNZA - Universal Dependenciesによる実用的日本語解析 - 自然言語処理 Volume 27 Number 3](https://www.jstage.jst.go.jp/article/jnlp/27/3/27_695/_article/-char/ja/) From 516d8821bbe95f2566ba1cc292e4593e3d1a4701 Mon Sep 17 00:00:00 2001 From: Hiroshi Matsuda Date: Sat, 26 Sep 2026 02:19:11 +0900 Subject: [PATCH 23/24] add performance notes --- README.md | 8 ++++++-- docs/index.md | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1e87623..0a01429 100644 --- a/README.md +++ b/README.md @@ -61,9 +61,13 @@ $ pip install -U ja_ginza On Mac OS environments running on Apple Silicon, the hardware acceleration is automatically enabled under the following conditions: - `ja_ginza` - - Python 3.10 to 3.12 (thinc-apple-ops is not supported in 3.13 and later) + - Acceleration by `thinc-apple-ops` + - Supports Python 3.10 to 3.12 (`thinc-apple-ops` is not supported in 3.13 and later) + - The throughput improvement effect from enabling `thinc-apple-ops` on MacBook Pro M4 Max 128GB is 2.6 times for Python 3.10-3.11 and 5.2 times for Python 3.12 in `ja_ginza`. - `ja_ginza_electra` and `ja_ginza_large_bert` - - Python 3.10 or later (GPU acceleration is only enabled for `transformers` in 3.13 and later) + - Acceleration by `torch.backends.mps` + - Supports Python 3.10 to 3.13 (some build errors occur when building dependent libraries in version 3.14) + - The throughput improvement effect from enabling `torch.backends.mps` on MacBook Pro M4 Max 128GB is 1.3 times for Python 3.10-13 in `ja_ginza_bert_large`. To enable NVIDIA GPU acceleration in a Linux OS environment, install CUDA on Linux, add the path to the CUDA libraries to the environment variable `LD_LIBRARY_PATH` like `export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH`, and then install the `ginza` package specifying the CUDA version in extras as follows: - CUDA 11.x diff --git a/docs/index.md b/docs/index.md index 3d039ac..7f5af25 100644 --- a/docs/index.md +++ b/docs/index.md @@ -121,9 +121,13 @@ $ pip install -U ja_ginza Apple Siliconで動作するMac OS環境では、次の条件を満たす場合に自動でハードウェアアクセラレーションが有効化されます。 - `ja_ginza` - - Python 3.10 〜 3.12 (3.13以降は`thinc-apple-ops`が非対応) + - `thinc-apple-ops`によるacceleration + - Python 3.10 〜 3.12 に対応 (3.13以降は`thinc-apple-ops`が非対応) + - `ja_ginza`の`thinc-apple-ops`によるスループット向上効果は、MacBook Pro M4 Max 128GBにおいて、Python 3.10-3.11で2.6倍、Python 3.12で5.2倍です。 - `ja_ginza_electra` および `ja_ginza_large_bert` - - Python 3.10 以降 (3.13以降は`thinc-apple-ops`が非対応のため`transformers`のみGPUアクセラレーションが有効化される) + - `torch.backends.mps`によるacceleration + - Python 3.10 〜 3.13 に対応 (3.14では依存ライブラリでビルドエラーが生じる) + - `ja_ginza_bert_large`の`torch.backends.mps`によるスループット向上効果は、MacBook Pro M4 Max 128GBにおいて、Python 3.10-3.13で1.3倍です。 Linux OS環境でNVIDIA GPUによるアクセラレーションを有効化するには、LinuxにCUDAをインストールし、`export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH` のように環境変数 `LD_LIBRARY_PATH` にCUDAライブラリのパスを追加した上で、次のようにCUDAバージョンをextrasに指定して`ginza`パッケージのインストールを行います。 - CUDA 11.x From ebfb8d64cfa2f934abda75b6572c3c8f2d1c1e27 Mon Sep 17 00:00:00 2001 From: Hiroshi Matsuda Date: Sat, 26 Sep 2026 10:29:00 +0900 Subject: [PATCH 24/24] python version descriptions --- README.md | 8 ++++---- docs/index.md | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 0a01429..e33806a 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Please also read the English documentation. ## Runtime Environment From GiNZA v5.3.0, the runtime environment has changed to Python 3.10 or later. -We recommend using Python 3.10–3.13 because installation errors may occur with some dependent libraries. +We recommend using Python 3.12 because of simpler dependency installation process and higher throughput. We do not recommend to use Anaconda environment because the pip install step may not work properly. Please also see the Development Environment section below. @@ -66,7 +66,7 @@ On Mac OS environments running on Apple Silicon, the hardware acceleration is au - The throughput improvement effect from enabling `thinc-apple-ops` on MacBook Pro M4 Max 128GB is 2.6 times for Python 3.10-3.11 and 5.2 times for Python 3.12 in `ja_ginza`. - `ja_ginza_electra` and `ja_ginza_large_bert` - Acceleration by `torch.backends.mps` - - Supports Python 3.10 to 3.13 (some build errors occur when building dependent libraries in version 3.14) + - Supports Python 3.10 to 3.13 (some compilation environments required to build dependent libraries in Python 3.14) - The throughput improvement effect from enabling `torch.backends.mps` on MacBook Pro M4 Max 128GB is 1.3 times for Python 3.10-13 in `ja_ginza_bert_large`. To enable NVIDIA GPU acceleration in a Linux OS environment, install CUDA on Linux, add the path to the CUDA libraries to the environment variable `LD_LIBRARY_PATH` like `export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH`, and then install the `ginza` package specifying the CUDA version in extras as follows: @@ -293,8 +293,8 @@ The mC4 is published under the ODC Attribution License. - 2026-09-30, Grossular Garnet - Breaking Changes - We changed the supported Python version to 3.10 or later, and the supported spaCy version to 3.8.16 or later. - - The recommended environment is Python 3.10 to 3.13. - - In a Python 3.14 environment, the Rust compiler is required to build some dependent libraries. + - We recommend using Python 3.12 because of simpler dependency installation process and higher throughput. + - Some compilation environments required to build dependent libraries in Python 3.14. - The model package loading priority was changed to `ja_ginza_bert_large`, `ja_ginza_electra`, and `ja_ginza`. - `ginza` command enables GPU acceleration by default under certain conditions. - GPU acceleration can be disabled with `ginza -g -1`. diff --git a/docs/index.md b/docs/index.md index 7f5af25..e8bfe46 100644 --- a/docs/index.md +++ b/docs/index.md @@ -13,7 +13,7 @@ GiNZAの解析モデルは、株式会社リクルートと国立国語研究所 ## What's new! - `GiNZA v5.3.0 - Grossular Garnet` をリリースしました (2026.09.30) - - 動作環境を Python 3.10 以降に変更（Python 3.10 ~ 3.13 の使用を推奨） + - 動作環境を Python 3.10 以降に変更（インストールの簡単さと性能が両立する Python 3.12 の使用を推奨） - `ja_ginza_bert_large` を正式にリリース - Apple Siliconで動作するMac OS環境で自動的にハードウェアアクセラレーションを有効化するよう変更 - thinc-apple-ops を適用可能なPythonのバージョンは 3.10 〜 3.12 です @@ -27,10 +27,10 @@ GiNZAの解析モデルは、株式会社リクルートと国立国語研究所 ## GiNZA v5.3 互換性情報 - Pythonの対応バージョンが3.10以上に変更されました - - 推奨動作環境は Python 3.10 ~ 3.13 です - - Python 3.14では一部の依存ライブラリのビルドにRustコンパイラが必要です + - インストールの簡単さと性能が両立する Python 3.12 の使用を推奨します + - Python 3.14では依存ライブラリのビルドのためにコンパイル環境を構築する必要があります - spaCyの対応バージョンがv3.8.16以上に変更されました -- GiNZA v5.3.0より前のモデルパケージはv5.3.0で使用できないため更新が必要です +- GiNZA v5.3.0より前のモデルパッケージはv5.3.0で使用できないため更新が必要です - モデルの読み込み優先度を `ja_ginza_bert_large`, `ja_ginza_electra`, `ja_ginza` の順に変更しました - `ginza`コマンドで一定の条件を満たす場合にデフォルトでGPUアクセラレーションが有効化されるようになりました - GPUアクセラレーションは `ginza -g -1` で無効化できます @@ -126,7 +126,7 @@ Apple Siliconで動作するMac OS環境では、次の条件を満たす場合 - `ja_ginza`の`thinc-apple-ops`によるスループット向上効果は、MacBook Pro M4 Max 128GBにおいて、Python 3.10-3.11で2.6倍、Python 3.12で5.2倍です。 - `ja_ginza_electra` および `ja_ginza_large_bert` - `torch.backends.mps`によるacceleration - - Python 3.10 〜 3.13 に対応 (3.14では依存ライブラリでビルドエラーが生じる) + - Python 3.10 〜 3.13 に対応 (3.14では依存ライブラリのビルドのためにコンパイル環境の構築が必要) - `ja_ginza_bert_large`の`torch.backends.mps`によるスループット向上効果は、MacBook Pro M4 Max 128GBにおいて、Python 3.10-3.13で1.3倍です。 Linux OS環境でNVIDIA GPUによるアクセラレーションを有効化するには、LinuxにCUDAをインストールし、`export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH` のように環境変数 `LD_LIBRARY_PATH` にCUDAライブラリのパスを追加した上で、次のようにCUDAバージョンをextrasに指定して`ginza`パッケージのインストールを行います。 @@ -313,9 +313,9 @@ Contains information from mC4 which is made available under the ODC Attribution - 2026-09-30, Grossular Garnet - 重要な変更 - 本リリースからサポート対象のPythonバージョンが3.10以上に、spaCyのバージョンが3.8.16以上に変更されました。 - - 推奨動作環境は Python 3.10 〜 3.13 です。 - - Python 3.14では一部の依存ライブラリのビルドにRustコンパイラが必要です。 - - GiNZA v5.3.0より前のモデルパケージはGiNZA v5.3.0以降で使用できません。 + - インストールの簡単さと性能が両立する Python 3.12 の使用を推奨します。 + - Python 3.14では依存ライブラリのビルドのためにコンパイル環境を構築する必要があります。 + - GiNZA v5.3.0より前のモデルパッケージはGiNZA v5.3.0以降で使用できません。 - モデルパッケージの読み込み優先度を `ja_ginza_bert_large`, `ja_ginza_electra`, `ja_ginza` の順に変更しました。 - `ginza`コマンドで一定の条件を満たす場合にデフォルトでGPUアクセラレーションが有効化されるようになりました。 - GPUアクセラレーションは `ginza -g -1` で無効化できます。