Report full formulas for unit metadata overrides

aidrin/structured_data_metrics/variable_unit_validation.py

@@ -3,6 +3,7 @@
 import hashlib
 import json
 import logging
+import math
 import re
 from functools import lru_cache
 from pathlib import Path
@@ -13,6 +14,9 @@
 import pyarrow.parquet as pq
 from celery import shared_task
 from pint import UnitRegistry, pint_eval
+from pint.errors import PintError
+from pint.facets.nonmultiplicative.definitions import OffsetConverter
+from pint.facets.plain import ScaleConverter
 from pint.util import string_preprocessor
 
 from aidrin.file_handling.value_iterators import iter_targets
@@ -506,6 +510,61 @@ def _display_unit(parsed: Dict[str, Any], fallback: str) -> str:
     return format(unit, "~P") if unit is not None else fallback
 
 
+def _has_affine_definition(unit: Any) -> bool:
+    """Only known scale/offset converters can establish an affine formula.
+
+    Inspect references too: a scale alias can depend on a logarithmic unit.
+    Pint's definition map is private; keep that dependency confined here.
+    """
+    pending = list(unit._units)
+    seen = set()
+    while pending:
+        # References may use symbols or prefixes absent from the definition map.
+        name = _UNIT_REGISTRY.get_name(pending.pop())
+        if not name or name in seen:
+            continue
+        seen.add(name)
+        definition = _UNIT_REGISTRY._units[name]
+        # LogarithmicConverter inherits ScaleConverter, so isinstance is unsafe.
+        if type(definition.converter) not in (ScaleConverter, OffsetConverter):
+            return False
+        if not definition.is_base:
+            pending.extend(definition.reference)
+    return True
+
+
+def _conversion_detail(observed: Any, chosen: Any, observed_unit: str, resolved_unit: str) -> tuple[str, Optional[float], str]:
+    """Describe a unit conversion, never converting dataset values."""
+    unavailable = ("conversion", None, "conversion formula unavailable for these units")
+    try:
+        if not (_has_affine_definition(observed) and _has_affine_definition(chosen)):
+            return unavailable
+        source_scale, source_root = _UNIT_REGISTRY.get_root_units(observed, check_nonmult=False)
+        target_scale, target_root = _UNIT_REGISTRY.get_root_units(chosen, check_nonmult=False)
+        # Definition scales avoid subtracting two almost equal converted values
+        # when an offset is large relative to the slope.
+        slope = float(source_scale / target_scale)
+        if source_root != target_root:
+            # Roots such as radian, count, and dimensionless still convert 1:1.
+            slope *= float(_UNIT_REGISTRY.Quantity(1, source_root).to(target_root).magnitude)
+        offset = float(_UNIT_REGISTRY.Quantity(0, observed).to(chosen).magnitude)
+    except (PintError, KeyError, ValueError, ArithmeticError):
+        return unavailable
+    if slope == 0 or not (math.isfinite(slope) and math.isfinite(offset)):
+        return unavailable
+    slope_text, offset_text = f"{slope:g}", f"{abs(offset):g}"
+    # Mark rounded coefficients as approximate, especially reverse conversions.
+    exact = math.isclose(slope, float(slope_text), rel_tol=1e-12, abs_tol=0) and math.isclose(
+        abs(offset), float(offset_text), rel_tol=1e-12, abs_tol=0,
+    )
+    relation = "=" if exact else "≈"
+    formula = f"{resolved_unit or 'dimensionless'} {relation} {observed_unit or 'dimensionless'} × {slope_text}"
+    if offset != 0:
+        formula += f" {'+' if offset > 0 else '-'} {offset_text}"
+        return "offset", None, f"offset conversion: {formula}"
+    return "scale", slope, f"{slope_text}× scale difference; {formula}"
+
+
 def _override_mismatches(
     parsed_candidates: List[Any],
     chosen: Dict[str, Any],
@@ -518,33 +577,35 @@ def _override_mismatches(
         if observed["status"] not in _READY_STATUSES or _units_equivalent(observed, chosen):
             continue
         observed_unit = _display_unit(observed, candidate["unit"])
+        observed_label = observed_unit or "dimensionless"
         if resolution["kind"] == "not_applicable":
             mismatches.append({
                 "kind": "classification",
                 "source": candidate["source"],
                 "observed_unit": observed_unit,
                 "resolved_unit": None,
                 "conversion_factor": None,
-                "message": f"Unit mismatch: detected {observed_unit}, overridden as not applicable.",
+                "message": f"Unit mismatch: detected {observed_label}, overridden as not applicable.",
             })
             continue
         resolved_unit = _display_unit(chosen, resolution.get("unit", "1"))
+        resolved_label = resolved_unit or "dimensionless"
         if observed.get("dimensionality") != chosen.get("dimensionality"):
             kind = "dimension"
             conversion_factor = None
             detail = "different dimensions"
         else:
-            kind = "scale"
-            conversion_factor = float((1 * observed["_parsed"]).to(chosen["_parsed"]).magnitude)
-            detail = f"{conversion_factor:g}× scale difference"
+            kind, conversion_factor, detail = _conversion_detail(
+                observed["_parsed"], chosen["_parsed"], observed_unit, resolved_unit,
+            )
         mismatches.append({
             "kind": kind,
             "source": candidate["source"],
             "observed_unit": observed_unit,
             "resolved_unit": resolved_unit,
             "conversion_factor": conversion_factor,
             "message": (
-                f"Unit mismatch: detected {observed_unit}, overridden with {resolved_unit} "
+                f"Unit mismatch: detected {observed_label}, overridden with {resolved_label} "
                 f"({detail}). Values were not converted."
             ),
         })

docs/source/_static/variable-unit-metadata.schema.json

@@ -219,18 +219,19 @@
       ],
       "properties": {
         "kind": {
-          "enum": ["scale", "dimension", "classification"]
+          "enum": ["scale", "offset", "conversion", "dimension", "classification"]
         },
         "source": {
           "type": "string",
           "minLength": 1
         },
         "observed_unit": {
           "type": "string",
-          "minLength": 1
+          "description": "Detected unit display; an empty string denotes a dimensionless unit."
         },
         "resolved_unit": {
-          "$ref": "#/$defs/nullableString"
+          "$ref": "#/$defs/nullableString",
+          "description": "Resolved unit display; an empty string denotes a dimensionless unit, and null denotes not applicable."
         },
         "conversion_factor": {
           "type": ["number", "null"]

docs/source/variable_unit_validation.rst

@@ -49,9 +49,10 @@ Every variable has one resolution:
 - unresolved when review remains necessary.
 
 A user resolution overrides detected metadata while preserving the observations.
-When the units differ, the result records whether their scale or physical
-dimension differs and states that values were not converted. The editor never
-embeds a correction in the source file.
+When the units differ, the result records whether their scale, offset, or physical
+dimension differs, or that no conversion formula is available, and states that
+values were not converted.
+The editor never embeds a correction in the source file.
 
 Canonical sidecar
 -----------------
@@ -171,7 +172,29 @@ The same ambiguity rule applies to a column ending in ``_g``.
 For an override, equivalent spellings remain neutral. Convertible units with a
 different scale and units with different dimensions are recorded separately.
 For example, changing detected ``kg/m³`` to ``g/m³`` reports a 1000-fold scale
-difference and does not convert dataset values.
+difference with the formula ``g/m³ = kg/m³ × 1000``. Scale conversions use mismatch
+kind ``scale`` and a numeric ``conversion_factor``.
+Convertible units with distinct dimensionless roots, such as ``count`` and
+``radian``, retain their scale factor, including 1.0 for a 1:1 conversion.
+Dimensionless units retain an empty string in the mismatch's ``observed_unit``
+or ``resolved_unit`` field; formulas label that unit as ``dimensionless``.
+
+Affine conversions with different zero points use mismatch kind ``offset`` and
+set ``conversion_factor`` to null. The message shows the full formula, including
+both scaling and offset: for example, ``°F = °C × 1.8 + 32`` or
+``°C = K × 1 - 273.15``. This applies to any supported Pint scale/offset definition,
+regardless of the variable's name or physical dimension. Formulas describe numeric
+values in the displayed units. Rounded coefficients use ``≈`` rather than ``=``;
+for example, the reverse conversion is ``°C ≈ °F × 0.555556 - 17.7778``.
+
+Logarithmic and other unrecognized converter types are not assumed to be affine.
+When a scale/offset formula cannot be established or Pint cannot perform the
+conversion, mismatch kind ``conversion`` reports that the formula is unavailable
+and sets ``conversion_factor`` to null. The audit never converts dataset values.
+
+The version-1 sidecar schema accepts the additional ``offset`` and ``conversion`` mismatch kinds;
+existing sidecars remain importable. Consumers validating against a cached schema
+should refresh it before validating results containing these mismatch kinds.
 
 Interfaces
 ----------

tests/integration/test_inspector_metrics.py

@@ -1,8 +1,11 @@
 """Tests for metric submission from the inspector."""
 
+import io
 import json
 from pathlib import Path
 
+import pytest
+
 import web.routes.metrics as metrics_routes
 
 
@@ -230,6 +233,44 @@ def test_understandability_variable_unit_validation_uses_request_local_sidecar(u
     assert result["summary"]["counts"]["not_applicable"] == 2
 
 
+@pytest.mark.parametrize(
+    ("observed", "resolved", "kind", "formula"),
+    [
+        ("degC", "kelvin", "offset", "K = °C × 1 + 273.15"),
+        ("degC", "degF", "offset", "°F = °C × 1.8 + 32"),
+        ("degF", "degC", "offset", "°C ≈ °F × 0.555556 - 17.7778"),
+        ("kPa", "Pa", "scale", "Pa = kPa × 1000"),
+        ("dBm", "watt", "conversion", "conversion formula unavailable for these units"),
+    ],
+)
+def test_variable_unit_web_override_preserves_conversion_reporting(client, observed, resolved, kind, formula):
+    response = client.post(
+        "/inspector",
+        data={
+            "file": (io.BytesIO(f"measurement ({observed})\n0\n25\n".encode()), "measurement.csv"),
+            "fileTypeSelector": ".csv",
+        },
+        content_type="multipart/form-data",
+    )
+    assert response.status_code == 302
+    metadata = client.post("/custom-outlier-targets").get_json()["unit_metadata"]
+    metadata["variables"][0]["resolution"] = {
+        "kind": "unit", "unit": resolved, "source": "user",
+    }
+
+    response = client.post(
+        "/variable-unit-validation?return_type=json",
+        data={"variable_unit_validation": "yes", "variable_unit_metadata": json.dumps(metadata)},
+    )
+    result = response.get_json()["Variable Unit Validation"]
+    mismatch = result["variables"][0]["finding"]["override_mismatches"][0]
+
+    assert mismatch["kind"] == kind
+    assert mismatch["conversion_factor"] == (1000.0 if kind == "scale" else None)
+    assert formula in mismatch["message"]
+    assert result["summary"]["all_variables_ready"] == 1
+
+
 def test_variable_unit_error_is_metric_scoped(uploaded_client):
     response = uploaded_client.post(
         "/variable-unit-validation?return_type=json",

tests/unit/test_variable_unit_interfaces.py

@@ -119,6 +119,42 @@ def test_cli_accepts_unit_metadata_json_and_file(tmp_path):
     assert json.loads(file_out)["summary"]["all_variables_ready"] is True
 
 
+@pytest.mark.parametrize(
+    ("observed", "resolved", "kind", "formula"),
+    [
+        ("degC", "kelvin", "offset", "K = °C × 1 + 273.15"),
+        ("degC", "degF", "offset", "°F = °C × 1.8 + 32"),
+        ("degF", "degC", "offset", "°C ≈ °F × 0.555556 - 17.7778"),
+        ("kPa", "Pa", "scale", "Pa = kPa × 1000"),
+        ("dBm", "watt", "conversion", "conversion formula unavailable for these units"),
+    ],
+)
+def test_cli_preserves_conversion_reporting(tmp_path, observed, resolved, kind, formula):
+    dataset = tmp_path / "temperature.csv"
+    pd.DataFrame({f"measurement ({observed})": [0.0, 25.0]}).to_csv(dataset, index=False)
+    original = dataset.read_bytes()
+    sidecar = calculate_variable_unit_validation((str(dataset), dataset.name, ".csv"))
+    sidecar["variables"][0]["resolution"] = {
+        "kind": "unit", "unit": resolved, "source": "user",
+    }
+    sidecar_path = tmp_path / "temperature.units.json"
+    sidecar_path.write_text(json.dumps(sidecar), encoding="utf-8")
+
+    output, error, code = _run_cli(
+        "run", "variable-unit-validation", str(dataset),
+        "--unit-metadata-file", str(sidecar_path),
+    )
+
+    assert code == 0, error
+    result = json.loads(output)
+    mismatch = result["variables"][0]["finding"]["override_mismatches"][0]
+    assert mismatch["kind"] == kind
+    assert mismatch["conversion_factor"] == (1000.0 if kind == "scale" else None)
+    assert formula in mismatch["message"]
+    assert result["summary"]["all_variables_ready"] is True
+    assert dataset.read_bytes() == original
+
+
 def test_cli_sidecar_options_are_mutually_exclusive(tmp_path):
     _out, error, code = _run_cli(
         "run",