Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 39 additions & 5 deletions nvflare/apis/analytix.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ def __init__(
sender (LogWriterName): Type of sender for syntax such as Tensorboard or MLflow
kwargs (optional, dict): additional arguments to be passed.
"""
self._validate_data_types(data_type, key, value, **kwargs)
value = self._validate_data_types(data_type, key, value, **kwargs)
self.tag = key
self.value = value
self.data_type = data_type
Expand Down Expand Up @@ -183,22 +183,56 @@ def _validate_data_types(
path = kwargs.get(TrackConst.PATH_KEY, None)
if path and not isinstance(path, str):
raise TypeError("expect path to be an instance of str, but got {}.".format(type(step)))
Comment thread
YuanTingHsieh marked this conversation as resolved.
if data_type in [AnalyticsDataType.SCALAR, AnalyticsDataType.METRIC] and not (
isinstance(value, float) or isinstance(value, int)
):
raise TypeError(f"expect '{key}' value to be an instance of float or int, but got '{type(value)}'.")
if data_type in [AnalyticsDataType.SCALAR, AnalyticsDataType.METRIC]:
is_numeric_scalar, normalized_value = self._normalize_numeric_scalar(value)
if not is_numeric_scalar:
raise TypeError(f"expect '{key}' value to be an instance of float or int, but got '{type(value)}'.")
value = normalized_value
elif data_type in [
AnalyticsDataType.METRICS,
AnalyticsDataType.PARAMETERS,
AnalyticsDataType.SCALARS,
] and not isinstance(value, dict):
raise TypeError(f"expect '{key}' value to be an instance of dict, but got '{type(value)}'.")
elif data_type in [AnalyticsDataType.METRICS, AnalyticsDataType.SCALARS]:
normalized_dict = {}
for k, v in value.items():
is_numeric_scalar, normalized_value = self._normalize_numeric_scalar(v)
normalized_dict[k] = normalized_value if is_numeric_scalar else v
value = normalized_dict
Comment thread
YuanTingHsieh marked this conversation as resolved.
elif data_type == AnalyticsDataType.TEXT and not isinstance(value, str):
raise TypeError(f"expect '{key}' value to be an instance of str, but got '{type(value)}'.")
elif data_type == AnalyticsDataType.TAGS and not isinstance(value, dict):
raise TypeError(
f"expect '{key}' data type expects value to be an instance of dict, but got '{type(value)}'"
)
return value

@staticmethod
def _normalize_numeric_scalar(value):
if isinstance(value, (float, int)):
return True, value
Comment thread
YuanTingHsieh marked this conversation as resolved.

item = getattr(value, "item", None)
if not callable(item):
return False, value

shape = getattr(value, "shape", None)
if shape is not None:
try:
if tuple(shape) != ():
return False, value
except TypeError:
return False, value

try:
scalar = item()
except (TypeError, ValueError):
return False, value

if isinstance(scalar, (float, int)):
return True, scalar
return False, value

@classmethod
def convert_data_type(
Expand Down
50 changes: 50 additions & 0 deletions tests/unit_test/apis/analytix_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,3 +105,53 @@ def test_to_dxo(self, data: AnalyticsData):
def test_from_dxo_invalid(self, dxo, expected_error, expected_msg):
with pytest.raises(expected_error, match=expected_msg):
_ = AnalyticsData.from_dxo(dxo)

def test_scalar_like_value_is_normalized(self):
class ScalarLike:
shape = ()

def item(self):
return 1.25

data = AnalyticsData(key="loss", value=ScalarLike(), data_type=AnalyticsDataType.SCALAR)

assert data.value == 1.25
assert isinstance(data.value, float)

def test_numeric_dict_values_are_normalized(self):
class ScalarLike:
shape = ()

def item(self):
return 1.25

data = AnalyticsData(
key="losses",
value={"train": ScalarLike(), "valid": 2},
data_type=AnalyticsDataType.SCALARS,
)

assert data.value == {"train": 1.25, "valid": 2}
assert isinstance(data.value["train"], float)

def test_numpy_numeric_values_are_normalized(self):
np = pytest.importorskip("numpy")

data = AnalyticsData(key="loss", value=np.float32(1.25), data_type=AnalyticsDataType.SCALAR)
dxo = create_analytic_dxo(
tag="loss", value=np.asarray(1.25, dtype=np.float32), data_type=AnalyticsDataType.SCALAR
)
metrics = AnalyticsData(
key="losses",
value={"train": np.float32(1.25), "valid": np.asarray(2, dtype=np.int32)},
data_type=AnalyticsDataType.SCALARS,
)

assert data.value == pytest.approx(1.25)
assert isinstance(data.value, float)
assert dxo.data[TrackConst.TRACK_VALUE] == pytest.approx(1.25)
assert isinstance(dxo.data[TrackConst.TRACK_VALUE], float)
assert metrics.value["train"] == pytest.approx(1.25)
assert metrics.value["valid"] == 2
assert isinstance(metrics.value["train"], float)
assert isinstance(metrics.value["valid"], int)
Comment thread
YuanTingHsieh marked this conversation as resolved.
Loading