diff --git a/nvflare/apis/analytix.py b/nvflare/apis/analytix.py index 586ba297f7..ff40b90dde 100644 --- a/nvflare/apis/analytix.py +++ b/nvflare/apis/analytix.py @@ -93,7 +93,7 @@ def __init__( sender (LogWriterName): Type of sender for syntax such as Tensorboard or MLflow kwargs (optional, dict): additional arguments to be passed. """ - self._validate_data_types(data_type, key, value, **kwargs) + value = self._validate_data_types(data_type, key, value, **kwargs) self.tag = key self.value = value self.data_type = data_type @@ -179,24 +179,67 @@ def _validate_data_types( if step < 0: raise ValueError(f"expect step to be non-negative int, but got {step}.") path = kwargs.get(TrackConst.PATH_KEY, None) - if path and not isinstance(path, str): + if path is not None and not isinstance(path, str): raise TypeError(f"expect path to be an instance of str, but got {type(path)}.") - if data_type in [AnalyticsDataType.SCALAR, AnalyticsDataType.METRIC] and not ( - isinstance(value, float) or isinstance(value, int) - ): - raise TypeError(f"expect '{key}' value to be an instance of float or int, but got '{type(value)}'.") + if data_type in [AnalyticsDataType.SCALAR, AnalyticsDataType.METRIC]: + is_numeric_scalar, normalized_value = self._normalize_numeric_scalar(value) + if not is_numeric_scalar: + raise TypeError( + f"expect '{key}' value to be a numeric scalar " + f"(float/int or scalar-like with item()), but got '{type(value)}'." + ) + value = normalized_value elif data_type in [ AnalyticsDataType.METRICS, AnalyticsDataType.PARAMETERS, AnalyticsDataType.SCALARS, - ] and not isinstance(value, dict): - raise TypeError(f"expect '{key}' value to be an instance of dict, but got '{type(value)}'.") + ]: + if not isinstance(value, dict): + raise TypeError(f"expect '{key}' value to be an instance of dict, but got '{type(value)}'.") + if data_type in [AnalyticsDataType.METRICS, AnalyticsDataType.SCALARS]: + normalized_dict = {} + for k, v in value.items(): + is_numeric_scalar, normalized_value = self._normalize_numeric_scalar(v) + if not is_numeric_scalar: + raise TypeError( + f"expect all values in '{key}' dict to be numeric scalars, " + f"but got '{type(v)}' for key '{k}'." + ) + normalized_dict[k] = normalized_value + value = normalized_dict elif data_type == AnalyticsDataType.TEXT and not isinstance(value, str): raise TypeError(f"expect '{key}' value to be an instance of str, but got '{type(value)}'.") elif data_type == AnalyticsDataType.TAGS and not isinstance(value, dict): raise TypeError( f"expect '{key}' data type expects value to be an instance of dict, but got '{type(value)}'" ) + return value + + @staticmethod + def _normalize_numeric_scalar(value): + if isinstance(value, (float, int)): + return True, value + + item = getattr(value, "item", None) + if not callable(item): + return False, value + + shape = getattr(value, "shape", None) + if shape is not None: + try: + if tuple(shape) != (): + return False, value + except TypeError: + return False, value + + try: + scalar = item() + except (TypeError, ValueError): + return False, value + + if isinstance(scalar, (float, int)): + return True, scalar + return False, value @classmethod def convert_data_type( diff --git a/tests/unit_test/apis/analytix_test.py b/tests/unit_test/apis/analytix_test.py index 8328e9e9d8..5e0fe0af7b 100644 --- a/tests/unit_test/apis/analytix_test.py +++ b/tests/unit_test/apis/analytix_test.py @@ -153,3 +153,81 @@ def test_from_dxo_torch_tb_to_wandb_preserves_data(self): assert result.tag == "loss" assert result.value == 0.5 assert result.data_type == AnalyticsDataType.METRIC + + def test_scalar_like_value_is_normalized(self): + class ScalarLike: + shape = () + + def item(self): + return 1.25 + + data = AnalyticsData(key="loss", value=ScalarLike(), data_type=AnalyticsDataType.SCALAR) + + assert data.value == 1.25 + assert isinstance(data.value, float) + + @pytest.mark.parametrize("path", [1, 0]) + def test_invalid_path_error_reports_path_type(self, path): + with pytest.raises(TypeError, match="expect path to be an instance of str, but got ."): + AnalyticsData(key="message", value="hello", data_type=AnalyticsDataType.TEXT, path=path) + + @pytest.mark.parametrize("data_type", [AnalyticsDataType.SCALARS, AnalyticsDataType.METRICS]) + def test_numeric_dict_values_are_normalized(self, data_type): + class ScalarLike: + shape = () + + def item(self): + return 1.25 + + data = AnalyticsData( + key="losses", + value={"train": ScalarLike(), "valid": 2}, + data_type=data_type, + ) + + assert data.value == {"train": 1.25, "valid": 2} + assert isinstance(data.value["train"], float) + + @pytest.mark.parametrize("data_type", [AnalyticsDataType.SCALARS, AnalyticsDataType.METRICS]) + def test_numeric_dict_values_reject_non_scalars(self, data_type): + with pytest.raises( + TypeError, + match="expect all values in 'losses' dict to be numeric scalars, " + "but got '' for key 'train'.", + ): + AnalyticsData( + key="losses", + value={"train": "bad_string"}, + data_type=data_type, + ) + + def test_numpy_numeric_values_are_normalized(self): + np = pytest.importorskip("numpy") + + data = AnalyticsData(key="loss", value=np.float32(1.25), data_type=AnalyticsDataType.SCALAR) + dxo = create_analytic_dxo( + tag="loss", value=np.asarray(1.25, dtype=np.float32), data_type=AnalyticsDataType.SCALAR + ) + scalars = AnalyticsData( + key="losses", + value={"train": np.float32(1.25), "valid": np.asarray(2, dtype=np.int32)}, + data_type=AnalyticsDataType.SCALARS, + ) + metrics = AnalyticsData( + key="metrics", + value={"train": np.float32(1.25), "valid": np.asarray(2, dtype=np.int32)}, + data_type=AnalyticsDataType.METRICS, + ) + + assert data.value == pytest.approx(1.25) + assert isinstance(data.value, float) + assert dxo.data[TrackConst.TRACK_VALUE] == pytest.approx(1.25) + assert isinstance(dxo.data[TrackConst.TRACK_VALUE], float) + assert scalars.value["train"] == pytest.approx(1.25) + assert scalars.value["valid"] == 2 + assert isinstance(scalars.value["train"], float) + assert isinstance(scalars.value["valid"], int) + assert metrics.value["train"] == pytest.approx(1.25) + assert metrics.value["valid"] == 2 + assert isinstance(metrics.value["train"], float) + assert isinstance(metrics.value["valid"], int) diff --git a/tests/unit_test/app_common/widgets/streaming_test.py b/tests/unit_test/app_common/widgets/streaming_test.py index 51a7965d3d..df9ec7dc5f 100644 --- a/tests/unit_test/app_common/widgets/streaming_test.py +++ b/tests/unit_test/app_common/widgets/streaming_test.py @@ -49,7 +49,8 @@ 2, AnalyticsDataType.SCALAR, TypeError, - f"expect 'tag' value to be an instance of float or int, but got '{type(list())}'", + r"expect 'tag' value to be a numeric scalar \(float/int or scalar-like with item\(\)\), " + f"but got '{type(list())}'", ), ( list(),