feat: LLMTextQualityV7

dingo/model/llm/text_quality/base_text_quality_v2.py

@@ -0,0 +1,60 @@
+"""Shared response processing for multi-label text quality evaluators."""
+
+import json
+
+from dingo.io.input import RequiredField
+from dingo.io.output.eval_detail import EvalDetail
+from dingo.model.llm.base_openai import BaseOpenAI
+from dingo.model.response.response_class import ResponseScoreTypeNameReason
+
+
+class BaseTextQualityV2(BaseOpenAI):
+    """Parse a JSON list of quality findings into one ``EvalDetail``."""
+
+    _required_fields = [RequiredField.CONTENT]
+
+    @classmethod
+    def process_response(cls, response: str) -> EvalDetail:
+        response = response.strip()
+        if response.startswith("```json"):
+            response = response[7:]
+        elif response.startswith("```"):
+            response = response[3:]
+        if response.rstrip().endswith("```"):
+            response = response.rstrip()[:-3]
+
+        response_json = json.loads(response.strip())
+        if not isinstance(response_json, list) or not response_json:
+            raise ValueError("Text quality response must be a non-empty JSON list")
+
+        findings = [ResponseScoreTypeNameReason(**item) for item in response_json]
+        good_findings = [item for item in findings if item.score == 1]
+        bad_findings = [item for item in findings if item.score == 0]
+
+        if len(good_findings) == 1 and len(findings) == 1:
+            good = good_findings[0]
+            if good.type != "Good" or good.name != "None":
+                raise ValueError("A passing finding must use type 'Good' and name 'None'")
+            return EvalDetail(
+                metric=cls.__name__,
+                status=False,
+                score=1,
+                label=["QUALITY_GOOD"],
+                reason=[good.reason],
+            )
+
+        if good_findings:
+            raise ValueError("A Good finding cannot be mixed with defect findings")
+        if len(bad_findings) != len(findings):
+            raise ValueError("Each defect finding must have score 0")
+        label_keys = [(item.type, item.name) for item in bad_findings]
+        if len(set(label_keys)) != len(label_keys):
+            raise ValueError("Duplicate defect labels are not allowed")
+
+        return EvalDetail(
+            metric=cls.__name__,
+            status=True,
+            score=0,
+            label=[f"{item.type}.{item.name}" for item in bad_findings],
+            reason=[item.reason for item in bad_findings],
+        )

dingo/model/llm/text_quality/llm_text_quality_v7.py

@@ -0,0 +1,39 @@
+from dingo.model import Model
+from dingo.model.llm.text_quality.base_text_quality_v2 import BaseTextQualityV2
+from dingo.model.llm.text_quality.llm_text_quality_v6 import LLMTextQualityV6
+
+
+def _build_multi_label_prompt() -> str:
+    """Derive V7 from the V6 rubric while replacing its output contract."""
+    prompt = LLMTextQualityV6.prompt
+    replacements = {
+        "5. Return only one label: the single defect with the greatest training impact. If no label is clearly supported, return Good.":
+            "5. Return every clearly supported label. Each label must describe a distinct material defect; do not emit duplicates or speculative secondary labels. If no label is clearly supported, return only Good.",
+        "4. **Identify Primary Cause**: If problematic, which single label best explains the dominant training harm?":
+            "4. **Identify Defects**: Collect every distinct label whose threshold is independently met.",
+        "6. **Assign Label**:\n   - Score: 1 (suitable for training) or 0 (unsuitable)":
+            "6. **Assign Labels**:\n   - Return one object per supported defect, each with score 0\n   - If no defect is supported, return exactly one Good object with score 1",
+        'Return JSON only: {"score": 0/1, "type": "", "name": "", "reason": ""}':
+            'Return a non-empty JSON array only: [{"score": 0/1, "type": "", "name": "", "reason": ""}]\n\nFor defective text, include all independently supported labels. Do not include a Good object together with defect objects. Emit each label at most once.',
+    }
+    for old, new in replacements.items():
+        if old not in prompt:
+            raise RuntimeError(f"V6 prompt fragment not found: {old}")
+        prompt = prompt.replace(old, new)
+
+    # All V6 examples contain one object. V7 keeps them as one-item arrays.
+    prompt = prompt.replace("Output: {", "Output: [{").replace("}\n\n**Example", "}]\n\n**Example")
+    prompt = prompt.replace("}\n\n---\n\n# Input content", "}]\n\n**Example 5 (Bad - Multiple Labels)**:\nInput: \"Thequickbrownfox. Thequickbrownfox. Thequickbrownfox. Thequickbrownfox. Thequickbrownfox. Thequickbrownfox.\"\nOutput: [{\"score\": 0, \"type\": \"Effectiveness\", \"name\": \"Words_Stuck\", \"reason\": \"Word boundaries are missing in every repeated sentence\"}, {\"score\": 0, \"type\": \"Similarity\", \"name\": \"Duplication\", \"reason\": \"The same sentence repeats 6 times\"}]\n\n---\n\n# Input content")
+    return prompt
+
+
+@Model.llm_register("LLMTextQualityV7")
+class LLMTextQualityV7(BaseTextQualityV2):
+    """Multi-label variant of the V6 text quality evaluator."""
+
+    _metric_info = {
+        **LLMTextQualityV6._metric_info,
+        "metric_name": "LLMTextQualityV7",
+        "description": "Multi-label impact-driven text quality evaluation for LLM pretraining",
+    }
+    prompt = _build_multi_label_prompt()

docs/metrics.md

@@ -28,6 +28,7 @@ This document provides comprehensive information about all quality metrics used
 | `LLMTextEquation` | LLMTextEquation | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [📊 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [📝 View Example](../examples/llm_and_rule/llm_local.py) |
 | `LLMTextQualityV4` | LLMTextQualityV4 | Enhanced text quality evaluation covering completeness (formulas, tables, code), effectiveness (garbled text, spacing... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [📊 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | N/A |
 | `LLMTextQualityV5` | LLMTextQualityV5 | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [📊 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [📝 View Example](../examples/llm_and_rule/llm_local.py) |
+| `LLMTextQualityV7` | LLMTextQualityV7 | Multi-label impact-driven text quality evaluation for LLM pretraining; always returns a non-empty list of findings. | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [📊 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [📝 View Example](../examples/llm_and_rule/llm_local.py) |
 | `LLMTextTable` | LLMTextTable | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [📊 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [📝 View Example](../examples/llm_and_rule/llm_local.py) |
 
 ### National Standard LLM Assessment Metrics

test/scripts/model/llm/test_text_quality_v7.py

@@ -0,0 +1,43 @@
+import json
+
+import pytest
+
+from dingo.model.llm.text_quality.llm_text_quality_v7 import LLMTextQualityV7
+
+
+def test_multiple_defects_are_aggregated():
+    response = json.dumps([
+        {"score": 0, "type": "Effectiveness", "name": "Words_Stuck", "reason": "Missing word boundaries"},
+        {"score": 0, "type": "Similarity", "name": "Duplication", "reason": "Sentence repeats 6 times"},
+    ])
+
+    result = LLMTextQualityV7.process_response(response)
+
+    assert result.status is True
+    assert result.score == 0
+    assert result.label == ["Effectiveness.Words_Stuck", "Similarity.Duplication"]
+    assert result.reason == ["Missing word boundaries", "Sentence repeats 6 times"]
+
+
+def test_good_response_is_a_single_item_list():
+    response = '```json\n[{"score": 1, "type": "Good", "name": "None", "reason": "Clear text"}]\n```'
+
+    result = LLMTextQualityV7.process_response(response)
+
+    assert result.status is False
+    assert result.score == 1
+    assert result.label == ["QUALITY_GOOD"]
+    assert result.reason == ["Clear text"]
+
+
+@pytest.mark.parametrize("response", [
+    "[]",
+    '{"score": 1, "type": "Good", "name": "None", "reason": "Clear text"}',
+    '[{"score": 1, "type": "Good", "name": "None", "reason": "Clear"},'
+    ' {"score": 0, "type": "Similarity", "name": "Duplication", "reason": "Repeated"}]',
+    '[{"score": 0, "type": "Similarity", "name": "Duplication", "reason": "Repeated"},'
+    ' {"score": 0, "type": "Similarity", "name": "Duplication", "reason": "Repeated again"}]',
+])
+def test_invalid_multi_label_shapes_are_rejected(response):
+    with pytest.raises(ValueError):
+        LLMTextQualityV7.process_response(response)