diff --git a/dingo/model/llm/text_quality/base_text_quality_v2.py b/dingo/model/llm/text_quality/base_text_quality_v2.py new file mode 100644 index 00000000..15c8c251 --- /dev/null +++ b/dingo/model/llm/text_quality/base_text_quality_v2.py @@ -0,0 +1,60 @@ +"""Shared response processing for multi-label text quality evaluators.""" + +import json + +from dingo.io.input import RequiredField +from dingo.io.output.eval_detail import EvalDetail +from dingo.model.llm.base_openai import BaseOpenAI +from dingo.model.response.response_class import ResponseScoreTypeNameReason + + +class BaseTextQualityV2(BaseOpenAI): + """Parse a JSON list of quality findings into one ``EvalDetail``.""" + + _required_fields = [RequiredField.CONTENT] + + @classmethod + def process_response(cls, response: str) -> EvalDetail: + response = response.strip() + if response.startswith("```json"): + response = response[7:] + elif response.startswith("```"): + response = response[3:] + if response.rstrip().endswith("```"): + response = response.rstrip()[:-3] + + response_json = json.loads(response.strip()) + if not isinstance(response_json, list) or not response_json: + raise ValueError("Text quality response must be a non-empty JSON list") + + findings = [ResponseScoreTypeNameReason(**item) for item in response_json] + good_findings = [item for item in findings if item.score == 1] + bad_findings = [item for item in findings if item.score == 0] + + if len(good_findings) == 1 and len(findings) == 1: + good = good_findings[0] + if good.type != "Good" or good.name != "None": + raise ValueError("A passing finding must use type 'Good' and name 'None'") + return EvalDetail( + metric=cls.__name__, + status=False, + score=1, + label=["QUALITY_GOOD"], + reason=[good.reason], + ) + + if good_findings: + raise ValueError("A Good finding cannot be mixed with defect findings") + if len(bad_findings) != len(findings): + raise ValueError("Each defect finding must have score 0") + label_keys = [(item.type, item.name) for item in bad_findings] + if len(set(label_keys)) != len(label_keys): + raise ValueError("Duplicate defect labels are not allowed") + + return EvalDetail( + metric=cls.__name__, + status=True, + score=0, + label=[f"{item.type}.{item.name}" for item in bad_findings], + reason=[item.reason for item in bad_findings], + ) diff --git a/dingo/model/llm/text_quality/llm_text_quality_v7.py b/dingo/model/llm/text_quality/llm_text_quality_v7.py new file mode 100644 index 00000000..1350a13a --- /dev/null +++ b/dingo/model/llm/text_quality/llm_text_quality_v7.py @@ -0,0 +1,39 @@ +from dingo.model import Model +from dingo.model.llm.text_quality.base_text_quality_v2 import BaseTextQualityV2 +from dingo.model.llm.text_quality.llm_text_quality_v6 import LLMTextQualityV6 + + +def _build_multi_label_prompt() -> str: + """Derive V7 from the V6 rubric while replacing its output contract.""" + prompt = LLMTextQualityV6.prompt + replacements = { + "5. Return only one label: the single defect with the greatest training impact. If no label is clearly supported, return Good.": + "5. Return every clearly supported label. Each label must describe a distinct material defect; do not emit duplicates or speculative secondary labels. If no label is clearly supported, return only Good.", + "4. **Identify Primary Cause**: If problematic, which single label best explains the dominant training harm?": + "4. **Identify Defects**: Collect every distinct label whose threshold is independently met.", + "6. **Assign Label**:\n - Score: 1 (suitable for training) or 0 (unsuitable)": + "6. **Assign Labels**:\n - Return one object per supported defect, each with score 0\n - If no defect is supported, return exactly one Good object with score 1", + 'Return JSON only: {"score": 0/1, "type": "", "name": "", "reason": ""}': + 'Return a non-empty JSON array only: [{"score": 0/1, "type": "", "name": "", "reason": ""}]\n\nFor defective text, include all independently supported labels. Do not include a Good object together with defect objects. Emit each label at most once.', + } + for old, new in replacements.items(): + if old not in prompt: + raise RuntimeError(f"V6 prompt fragment not found: {old}") + prompt = prompt.replace(old, new) + + # All V6 examples contain one object. V7 keeps them as one-item arrays. + prompt = prompt.replace("Output: {", "Output: [{").replace("}\n\n**Example", "}]\n\n**Example") + prompt = prompt.replace("}\n\n---\n\n# Input content", "}]\n\n**Example 5 (Bad - Multiple Labels)**:\nInput: \"Thequickbrownfox. Thequickbrownfox. Thequickbrownfox. Thequickbrownfox. Thequickbrownfox. Thequickbrownfox.\"\nOutput: [{\"score\": 0, \"type\": \"Effectiveness\", \"name\": \"Words_Stuck\", \"reason\": \"Word boundaries are missing in every repeated sentence\"}, {\"score\": 0, \"type\": \"Similarity\", \"name\": \"Duplication\", \"reason\": \"The same sentence repeats 6 times\"}]\n\n---\n\n# Input content") + return prompt + + +@Model.llm_register("LLMTextQualityV7") +class LLMTextQualityV7(BaseTextQualityV2): + """Multi-label variant of the V6 text quality evaluator.""" + + _metric_info = { + **LLMTextQualityV6._metric_info, + "metric_name": "LLMTextQualityV7", + "description": "Multi-label impact-driven text quality evaluation for LLM pretraining", + } + prompt = _build_multi_label_prompt() diff --git a/docs/metrics.md b/docs/metrics.md index 93161a5e..8a480465 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -28,6 +28,7 @@ This document provides comprehensive information about all quality metrics used | `LLMTextEquation` | LLMTextEquation | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [📊 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [📝 View Example](../examples/llm_and_rule/llm_local.py) | | `LLMTextQualityV4` | LLMTextQualityV4 | Enhanced text quality evaluation covering completeness (formulas, tables, code), effectiveness (garbled text, spacing... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [📊 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | N/A | | `LLMTextQualityV5` | LLMTextQualityV5 | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [📊 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [📝 View Example](../examples/llm_and_rule/llm_local.py) | +| `LLMTextQualityV7` | LLMTextQualityV7 | Multi-label impact-driven text quality evaluation for LLM pretraining; always returns a non-empty list of findings. | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [📊 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [📝 View Example](../examples/llm_and_rule/llm_local.py) | | `LLMTextTable` | LLMTextTable | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [📊 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [📝 View Example](../examples/llm_and_rule/llm_local.py) | ### National Standard LLM Assessment Metrics diff --git a/test/scripts/model/llm/test_text_quality_v7.py b/test/scripts/model/llm/test_text_quality_v7.py new file mode 100644 index 00000000..1ebd9245 --- /dev/null +++ b/test/scripts/model/llm/test_text_quality_v7.py @@ -0,0 +1,43 @@ +import json + +import pytest + +from dingo.model.llm.text_quality.llm_text_quality_v7 import LLMTextQualityV7 + + +def test_multiple_defects_are_aggregated(): + response = json.dumps([ + {"score": 0, "type": "Effectiveness", "name": "Words_Stuck", "reason": "Missing word boundaries"}, + {"score": 0, "type": "Similarity", "name": "Duplication", "reason": "Sentence repeats 6 times"}, + ]) + + result = LLMTextQualityV7.process_response(response) + + assert result.status is True + assert result.score == 0 + assert result.label == ["Effectiveness.Words_Stuck", "Similarity.Duplication"] + assert result.reason == ["Missing word boundaries", "Sentence repeats 6 times"] + + +def test_good_response_is_a_single_item_list(): + response = '```json\n[{"score": 1, "type": "Good", "name": "None", "reason": "Clear text"}]\n```' + + result = LLMTextQualityV7.process_response(response) + + assert result.status is False + assert result.score == 1 + assert result.label == ["QUALITY_GOOD"] + assert result.reason == ["Clear text"] + + +@pytest.mark.parametrize("response", [ + "[]", + '{"score": 1, "type": "Good", "name": "None", "reason": "Clear text"}', + '[{"score": 1, "type": "Good", "name": "None", "reason": "Clear"},' + ' {"score": 0, "type": "Similarity", "name": "Duplication", "reason": "Repeated"}]', + '[{"score": 0, "type": "Similarity", "name": "Duplication", "reason": "Repeated"},' + ' {"score": 0, "type": "Similarity", "name": "Duplication", "reason": "Repeated again"}]', +]) +def test_invalid_multi_label_shapes_are_rejected(response): + with pytest.raises(ValueError): + LLMTextQualityV7.process_response(response)