From 0b5336386dcc2929842eab5524244dc9ec21e458 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Sat, 13 Sep 2025 12:12:58 -0400 Subject: [PATCH 001/102] dev free --- nvflare/apis/fl_exception.py | 6 ++ nvflare/free/__init__.py | 0 nvflare/free/api/__init__.py | 0 nvflare/free/api/app.py | 85 +++++++++++++++++++ nvflare/free/api/backend.py | 14 ++++ nvflare/free/api/ctx.py | 8 ++ nvflare/free/api/group.py | 106 ++++++++++++++++++++++++ nvflare/free/api/proxy.py | 26 ++++++ nvflare/free/api/resp.py | 21 +++++ nvflare/free/api/runner.py | 74 +++++++++++++++++ nvflare/free/api/sim_backend.py | 77 +++++++++++++++++ nvflare/free/examples/__init__.py | 0 nvflare/free/examples/np_cyclic.py | 55 ++++++++++++ nvflare/free/examples/np_fed_avg.py | 70 ++++++++++++++++ nvflare/free/examples/np_fed_avg_grp.py | 82 ++++++++++++++++++ nvflare/free/examples/np_swarm.py | 88 ++++++++++++++++++++ 16 files changed, 712 insertions(+) create mode 100644 nvflare/free/__init__.py create mode 100644 nvflare/free/api/__init__.py create mode 100644 nvflare/free/api/app.py create mode 100644 nvflare/free/api/backend.py create mode 100644 nvflare/free/api/ctx.py create mode 100644 nvflare/free/api/group.py create mode 100644 nvflare/free/api/proxy.py create mode 100644 nvflare/free/api/resp.py create mode 100644 nvflare/free/api/runner.py create mode 100644 nvflare/free/api/sim_backend.py create mode 100644 nvflare/free/examples/__init__.py create mode 100644 nvflare/free/examples/np_cyclic.py create mode 100644 nvflare/free/examples/np_fed_avg.py create mode 100644 nvflare/free/examples/np_fed_avg_grp.py create mode 100644 nvflare/free/examples/np_swarm.py diff --git a/nvflare/apis/fl_exception.py b/nvflare/apis/fl_exception.py index 3bc5f13fbc..81d51d04ea 100644 --- a/nvflare/apis/fl_exception.py +++ b/nvflare/apis/fl_exception.py @@ -66,3 +66,9 @@ class NotReadyToEndRun(Exception): """Raised when a component is not ready to end run""" pass + + +class RunAborted(Exception): + """Raised when a run is aborted""" + + pass diff --git a/nvflare/free/__init__.py b/nvflare/free/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/nvflare/free/api/__init__.py b/nvflare/free/api/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/nvflare/free/api/app.py b/nvflare/free/api/app.py new file mode 100644 index 0000000000..552393c80f --- /dev/null +++ b/nvflare/free/api/app.py @@ -0,0 +1,85 @@ +from abc import abstractmethod, ABC +from typing import List +from .proxy import Proxy +from .group import Group + +SERVER_NAME = "server" + + +class App(ABC): + + def __init__(self): + self.name = None + self.server = None + self.clients = None + self._me = None + self._target_objs = {} + self._abort_signal = None + + def add_target_object(self, name: str, obj): + if name in ['name','server', 'clients', "add_target_object", "get_target_objects", "setup", "get_my_site"]: + raise ValueError(f"conflict with reserved name {name}") + + # TBD: more name validation needed + setattr(self, name, obj) + self._target_objs[name] = obj + + def get_target_objects(self): + return self._target_objs + + def setup(self, name: str, server: Proxy, clients: List[Proxy], abort_signal): + self.name = name + self.server = server + self._abort_signal = abort_signal + + self.clients = clients + self._me = None + if not name or name == "server": + self._me = server + else: + for c in clients: + if c.name == name: + self._me = c + break + + if not self._me: + raise ValueError(f"cannot find site for {name}") + + def get_my_site(self) -> Proxy: + return self._me + + def initialize(self, **kwargs): + pass + + def group( + self, + proxies: List[Proxy], + blocking: bool = True, + timeout: float = None, + min_resps: int = None, + wait_after_min_resps: float = None, + process_resp_cb=None, + **cb_kwargs, + ): + return Group( + self._abort_signal, + proxies, + blocking, + timeout, + min_resps, + wait_after_min_resps, + process_resp_cb, + **cb_kwargs + ) + + +class ServerApp(App): + + @abstractmethod + def run(self, **kwargs): + pass + + +class ClientApp(App): + pass + diff --git a/nvflare/free/api/backend.py b/nvflare/free/api/backend.py new file mode 100644 index 0000000000..785413a18b --- /dev/null +++ b/nvflare/free/api/backend.py @@ -0,0 +1,14 @@ +from abc import abstractmethod, ABC + +from .resp import Resp + + +class Backend(ABC): + + @abstractmethod + def call_target(self, target_name: str, func_name: str, *args, **kwargs): + pass + + @abstractmethod + def call_target_with_resp(self, resp: Resp, target_name: str, func_name: str, *args, **kwargs): + pass diff --git a/nvflare/free/api/ctx.py b/nvflare/free/api/ctx.py new file mode 100644 index 0000000000..24b34c3749 --- /dev/null +++ b/nvflare/free/api/ctx.py @@ -0,0 +1,8 @@ +class Context: + + def __init__(self, caller: str, callee: str): + self.caller = caller + self.callee = callee + + def set_prop(self, name: str, value): + setattr(self, name, value) diff --git a/nvflare/free/api/group.py b/nvflare/free/api/group.py new file mode 100644 index 0000000000..326fd8047d --- /dev/null +++ b/nvflare/free/api/group.py @@ -0,0 +1,106 @@ +import copy +import time +from typing import List + +from nvflare.apis.signal import Signal +from nvflare.apis.fl_exception import RunAborted + +from .proxy import Proxy +from .ctx import Context +from .resp import Resp + + +class Group: + + def __init__( + self, + abort_signal: Signal, + proxies: List[Proxy], + blocking: bool = True, + timeout: float = None, + min_resps: int = None, + wait_after_min_resps: float = None, + process_resp_cb=None, + **cb_kwargs, + ): + self._abort_signal = abort_signal + self._proxies = proxies + self._blocking = blocking + self._timeout = timeout + self._min_resps = min_resps + self._wait_after_min_resps = wait_after_min_resps + self._process_resp_cb = process_resp_cb + self._cb_kwargs = cb_kwargs + + if not min_resps: + self._min_resps = len(proxies) + + if not wait_after_min_resps: + self._wait_after_min_resps = 0 + + def __getattr__(self, func_name): + """ + This method is called when Python cannot find an invoked method func_name of this class. + """ + + def method(*args, **kwargs): + resps = {} + for p in self._proxies: + kwargs_copy = copy.copy(kwargs) + ctx = Context(p.caller_name, p.name) + kwargs_copy["context"] = ctx + + resp = Resp(self._process_resp_cb, self._cb_kwargs) + resps[p.name] = resp + kwargs_copy["abort_signal"] = self._abort_signal + p.backend.call_target_with_resp( + resp, p.name, func_name, *args, **kwargs_copy + ) + + # wait for responses + if not self._blocking: + return + + start_time = time.time() + min_received_time = None + while True: + if self._abort_signal.triggered: + raise RunAborted("run is aborted") + + # how many resps have been received? + resps_received = 0 + for name, resp in resps.items(): + if resp.resp_time: + resps_received += 1 + + if resps_received == len(self._proxies): + break + + now = time.time() + if resps_received >= self._min_resps: + if not min_received_time: + min_received_time = now + if now - min_received_time > self._wait_after_min_resps: + # waited long enough + break + elif self._timeout and now - start_time > self._timeout: + # timed out + break + else: + # still have not received min resps + time.sleep(0.1) + + # process results + results = {} + for name, resp in resps.items(): + if resp.resp_time: + if resp.exception: + result = resp.exception + else: + result = resp.result + else: + result = TimeoutError() + results[name] = result + return results + + return method diff --git a/nvflare/free/api/proxy.py b/nvflare/free/api/proxy.py new file mode 100644 index 0000000000..7c4bbd3069 --- /dev/null +++ b/nvflare/free/api/proxy.py @@ -0,0 +1,26 @@ +from .backend import Backend +from .ctx import Context + + +class Proxy: + + def __init__(self, target_name, backend: Backend, caller_name: str): + self.target_name = target_name + self.backend = backend + self.caller_name = caller_name + + @property + def name(self): + return self.target_name + + def __getattr__(self, func_name): + """ + This method is called when Python cannot find an invoked method func_name of this class. + """ + + def method(*args, **kwargs): + ctx = Context(self.caller_name, self.name) + kwargs["context"] = ctx + return self.backend.call_target(self.target_name, func_name, *args, **kwargs) + + return method diff --git a/nvflare/free/api/resp.py b/nvflare/free/api/resp.py new file mode 100644 index 0000000000..8250a9effa --- /dev/null +++ b/nvflare/free/api/resp.py @@ -0,0 +1,21 @@ +import time + + +class Resp: + + def __init__(self, process_cb, cb_kwargs): + self.result = None + self.exception = None + self.resp_time = None + self.process_cb = process_cb + self.cb_kwargs = cb_kwargs + + def set_result(self, result): + if self.process_cb: + result = self.process_cb(result, **self.cb_kwargs) + self.result = result + self.resp_time = time.time() + + def set_exception(self, ex): + self.exception = ex + self.resp_time = time.time() diff --git a/nvflare/free/api/runner.py b/nvflare/free/api/runner.py new file mode 100644 index 0000000000..f9d5918ac9 --- /dev/null +++ b/nvflare/free/api/runner.py @@ -0,0 +1,74 @@ +import copy +from concurrent.futures import ThreadPoolExecutor + +from nvflare.free.api.app import ServerApp, ClientApp, App, SERVER_NAME +from nvflare.free.api.proxy import Proxy +from nvflare.free.api.sim_backend import SimBackend +from nvflare.apis.signal import Signal + + +class AppRunner: + + def _prepare_app_backends(self, app: App): + bes = {"": SimBackend(app, self.abort_signal, self.thread_executor)} + targets = app.get_target_objects() + if targets: + for name, obj in targets.items(): + bes[name] = SimBackend(obj, self.abort_signal, self.thread_executor) + return bes + + def _prepare_app_proxy(self, app_name: str, app: App, caller_name: str, app_backends: dict): + app_proxy = Proxy(target_name=app_name, backend=app_backends[""], caller_name=caller_name) + cos = app.get_target_objects() + if cos: + for name, obj in cos.items(): + p = Proxy(target_name=name, backend=app_backends[name], caller_name=caller_name) + setattr(app_proxy, name, p) + return app_proxy + + def _prepare_proxies(self, server_app: App, client_apps: dict, caller_name, backends: dict): + server_proxy = self._prepare_app_proxy(SERVER_NAME, server_app, caller_name, backends[SERVER_NAME]) + client_proxies = [] + for name, app in client_apps.items(): + p = self._prepare_app_proxy(name, app, caller_name, backends[name]) + client_proxies.append(p) + return server_proxy, client_proxies + + def __init__( + self, + server_app: ServerApp, + client_app: ClientApp, + num_clients: int, + ): + self.abort_signal = Signal() + self.thread_executor = ThreadPoolExecutor(max_workers=100) + + client_apps = {} + for i in range(num_clients): + name = f"site-{i+1}" + app = copy.deepcopy(client_app) + app.name = name + client_apps[name] = app + + backends = { + SERVER_NAME: self._prepare_app_backends(server_app) + } + + for name, app in client_apps.items(): + backends[name] = self._prepare_app_backends(app) + + for name, app in client_apps.items(): + server_proxy, client_proxies = self._prepare_proxies(server_app, client_apps, name, backends) + app.setup(name, server_proxy, client_proxies, self.abort_signal) + app.initialize() + + # prepare server + server_proxy, client_proxies = self._prepare_proxies(server_app, client_apps, SERVER_NAME, backends) + server_app.setup(SERVER_NAME, server_proxy, client_proxies, self.abort_signal) + + self.server_app = server_app + + def run(self): + # run the server + self.server_app.run() + self.thread_executor.shutdown(wait=False, cancel_futures=True) diff --git a/nvflare/free/api/sim_backend.py b/nvflare/free/api/sim_backend.py new file mode 100644 index 0000000000..e9737a3dfb --- /dev/null +++ b/nvflare/free/api/sim_backend.py @@ -0,0 +1,77 @@ +import threading + +from .backend import Backend +from .resp import Resp + + +class _Waiter(threading.Event): + + def __init__(self): + super().__init__() + self.result = None + self.exception = None + + +class SimBackend(Backend): + + def __init__(self, callable_obj, abort_signal, thread_executor): + self.obj = callable_obj + self.executor = thread_executor + self.abort_signal = abort_signal + + def call_target(self, target_name: str, func_name: str, *args, **kwargs): + func = getattr(self.obj, func_name, None) + if not func: + raise AttributeError(f"{target_name} does not have {func_name}") + + if not callable(func): + raise AttributeError(f"the {func_name} of {target_name} is not callable") + + blocking = kwargs.pop("blocking", True) + timeout = kwargs.pop("timeout", None) + kwargs["abort_signal"] = self.abort_signal + + waiter = None + if blocking: + waiter = _Waiter() + + self.executor.submit(self._run_func, waiter, func, args, kwargs) + if waiter: + ok = waiter.wait(timeout) + if not ok: + # timed out + raise TimeoutError(f"function {func_name} timed out after {timeout} seconds") + if waiter.exception: + raise waiter.exception + return waiter.result + + @staticmethod + def _run_func(waiter: _Waiter, func, args, kwargs): + try: + result = func(*args, **kwargs) + if waiter: + waiter.result = result + except Exception as ex: + if waiter: + waiter.exception = ex + finally: + if waiter: + waiter.set() + + def call_target_with_resp(self, resp: Resp, target_name: str, func_name: str, *args, **kwargs): + func = getattr(self.obj, func_name, None) + if not func: + raise AttributeError(f"{target_name} does not have {func_name}") + + if not callable(func): + raise AttributeError(f"the {func_name} of {target_name} is not callable") + + self.executor.submit(self._run_func_with_resp, resp, func, args, kwargs) + + @staticmethod + def _run_func_with_resp(resp: Resp, func, args, kwargs): + try: + result = func(*args, **kwargs) + resp.set_result(result) + except Exception as ex: + resp.set_exception(ex) diff --git a/nvflare/free/examples/__init__.py b/nvflare/free/examples/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/nvflare/free/examples/np_cyclic.py b/nvflare/free/examples/np_cyclic.py new file mode 100644 index 0000000000..b814572d07 --- /dev/null +++ b/nvflare/free/examples/np_cyclic.py @@ -0,0 +1,55 @@ +import random + +import numpy as np + +from nvflare.free.api.app import ServerApp, ClientApp +from nvflare.free.api.runner import AppRunner + + +class NPCyclic(ServerApp): + + def __init__(self, num_rounds=10): + ServerApp.__init__(self) + self.num_rounds = num_rounds + self.initial_model = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32) + + def run(self): + current_model = self.initial_model + for _ in range(self.num_rounds): + current_model = self._do_one_round(current_model) + print(f"final result: {current_model}") + + def _do_one_round(self, current_model): + random.shuffle(self.clients) + for c in self.clients: + current_model = c.train(current_model) + print(f"result from {c.name}: {current_model}") + return current_model + + +class NPTrainer(ClientApp): + + def __init__(self, delta: float): + ClientApp.__init__(self) + self.delta = delta + + def train(self, weights, **kwargs): + return weights + self.delta + + def evaluate(self, model): + pass + + +def main(): + + runner = AppRunner( + server_app=NPCyclic(num_rounds=2), + client_app=NPTrainer(delta=1.0), + num_clients=2 + ) + + runner.run() + + +if __name__ == "__main__": + main() diff --git a/nvflare/free/examples/np_fed_avg.py b/nvflare/free/examples/np_fed_avg.py new file mode 100644 index 0000000000..ccbdda9afd --- /dev/null +++ b/nvflare/free/examples/np_fed_avg.py @@ -0,0 +1,70 @@ +import numpy as np + +from nvflare.free.api.app import ServerApp, ClientApp +from nvflare.free.api.runner import AppRunner +from nvflare.free.api.ctx import Context +from nvflare.apis.signal import Signal + + +class NPFedAvg(ServerApp): + + def __init__(self, num_rounds=10): + ServerApp.__init__(self) + self.num_rounds = num_rounds + self.initial_model = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32) + + def run(self, **kwargs): + print(f"[{self.name}] Start training for {self.num_rounds} rounds") + current_model = self.initial_model + for i in range(self.num_rounds): + current_model = self._do_one_round(i, current_model) + + def _do_one_round(self, r, current_model): + total = np.array([[0, 0, 0], [0, 0, 0], [0, 0, 0]], dtype=np.float32) + n = 0 + for c in self.clients: + result = c.train(r, current_model) + print(f"[{self.name}] round {r}: got result from client {c.name}: {result}") + total += result + n += 1 + return total / n + + +class MetricReceiver: + + def accept_metric(self, metrics: dict, context: Context, **kwargs): + print(f"[{context.callee}] received metric report from {context.caller}: {metrics}") + + +class NPTrainer(ClientApp): + + def __init__(self, delta: float): + ClientApp.__init__(self) + self.delta = delta + + def train(self, r, weights, abort_signal: Signal, context: Context, **kwargs): + if abort_signal.triggered: + print("training aborted") + return 0 + print(f"[{self.name}] called by {context.caller}: client {context.callee} trained round {r}") + + self.server.metric_receiver.accept_metric({"round": r, "y": 2}) + return weights + self.delta + + +def main(): + + server_app = NPFedAvg(num_rounds=2) + server_app.add_target_object("metric_receiver", MetricReceiver()) + + runner = AppRunner( + server_app=server_app, + client_app=NPTrainer(delta=1.0), + num_clients=2 + ) + + runner.run() + + +if __name__ == "__main__": + main() diff --git a/nvflare/free/examples/np_fed_avg_grp.py b/nvflare/free/examples/np_fed_avg_grp.py new file mode 100644 index 0000000000..48bee6d3a8 --- /dev/null +++ b/nvflare/free/examples/np_fed_avg_grp.py @@ -0,0 +1,82 @@ +import numpy as np +import random + +from nvflare.free.api.app import ServerApp, ClientApp +from nvflare.free.api.runner import AppRunner +from nvflare.free.api.ctx import Context +from nvflare.apis.signal import Signal + + +class NPFedAvg(ServerApp): + + def __init__(self, num_rounds=10): + ServerApp.__init__(self) + self.num_rounds = num_rounds + self.initial_model = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32) + + def run(self, **kwargs): + print(f"[{self.name}] Start training for {self.num_rounds} rounds") + current_model = self.initial_model + for i in range(self.num_rounds): + current_model = self._do_one_round(i, current_model) + score = self._do_eval(current_model) + print(f"[{self.name}]: eval score in round {i}: {score}") + + def _do_eval(self, model): + results = self.group(self.clients).evaluate(model) + total= 0.0 + for n, v in results.items(): + print(f"[{self.name}]: got eval result from client {n}: {v}") + total += v + return total / len(results) + + def _do_one_round(self, r, current_model): + total = np.array([[0, 0, 0], [0, 0, 0], [0, 0, 0]], dtype=np.float32) + results = self.group(self.clients).train(r, current_model) + for n, v in results.items(): + print(f"[{self.name}] round {r}: got group result from client {n}: {v}") + total += v + return total / len(results) + + +class MetricReceiver: + + def accept_metric(self, metrics: dict, context: Context, **kwargs): + print(f"[{context.callee}] received metric report from {context.caller}: {metrics}") + + +class NPTrainer(ClientApp): + + def __init__(self, delta: float): + ClientApp.__init__(self) + self.delta = delta + + def train(self, r, weights, abort_signal: Signal, context: Context, **kwargs): + if abort_signal.triggered: + print("training aborted") + return 0 + print(f"[{self.name}] called by {context.caller}: client {context.callee} trained round {r}") + self.server.metric_receiver.accept_metric({"round": r, "y": 2}) + return weights + self.delta + + def evaluate(self, model, context: Context, **kwargs): + print(f"[{self.name}] called by {context.caller}: client {context.callee} to evaluate") + return random.random() + + +def main(): + + server_app = NPFedAvg(num_rounds=2) + server_app.add_target_object("metric_receiver", MetricReceiver()) + + runner = AppRunner( + server_app=server_app, + client_app=NPTrainer(delta=1.0), + num_clients=2 + ) + + runner.run() + + +if __name__ == "__main__": + main() diff --git a/nvflare/free/examples/np_swarm.py b/nvflare/free/examples/np_swarm.py new file mode 100644 index 0000000000..d7a340692f --- /dev/null +++ b/nvflare/free/examples/np_swarm.py @@ -0,0 +1,88 @@ +import random +import threading + +import numpy as np + +from nvflare.free.api.app import ServerApp, ClientApp +from nvflare.free.api.ctx import Context +from nvflare.free.api.runner import AppRunner + + +class NPSwarmServer(ServerApp): + + def __init__(self, num_rounds=10): + ServerApp.__init__(self) + self.num_rounds = num_rounds + self.initial_model = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32) + self.waiter = threading.Event() + + def run(self, **kwargs): + # randomly pick a client to start + start_client_idx = random.randint(0, len(self.clients)-1) + start_client = self.clients[start_client_idx] + start_client.start(self.num_rounds, self.initial_model) + self.waiter.wait() + + def notify_done(self, context: Context, **kwargs): + print(f"[{context.callee}]: received DONE from client: {context.caller}") + self.waiter.set() + + +class NPSwarmClient(ClientApp): + + def __init__(self, delta: float): + super().__init__() + self.delta = delta + + def train(self, weights, **kwargs): + return weights + self.delta + + def sag(self, model): + results = self.group(self.clients).train(model, blocking=True) + results = list(results.values()) + total = results[0] + for i in range(1, len(results)): + total += results[i] + return total / len(results) + + def swarm_learn(self, num_rounds, model, current_round, context: Context, **kwargs): + print(f"[{context.callee}]: swarm learn asked by {context.caller}: {num_rounds=} {current_round=} {model=}") + new_model = self.sag(model) + + print(f"[{context.callee}]: trained model {new_model=}") + if current_round == num_rounds - 1: + # all done + self.group(self.clients).accept_final_model(new_model, blocking=False) + self.server.notify_done() + return + + # determine next client + next_round = current_round+1 + next_client_idx = random.randint(0, len(self.clients) - 1) + next_client = self.clients[next_client_idx] + next_client.swarm_learn(num_rounds, new_model, next_round, blocking=False) + + def start(self, num_rounds, initial_model, **kwargs): + self.swarm_learn(num_rounds, initial_model, 0, **kwargs) + + def accept_final_model(self, model, context: Context, **kwargs): + # accept the final model + # write model to disk + print(f"[{context.callee}]: received final model from {context.caller}: {model}") + + +def main(): + + server_app = NPSwarmServer(num_rounds=5) + + runner = AppRunner( + server_app=server_app, + client_app=NPSwarmClient(delta=1.0), + num_clients=3 + ) + + runner.run() + + +if __name__ == "__main__": + main() From dc31e2acf6fb8f8d3ff83c68ba8444f787d3c844 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Tue, 16 Sep 2025 20:58:56 -0400 Subject: [PATCH 002/102] continue dev --- nvflare/free/api/app.py | 8 +- nvflare/free/api/proxy.py | 9 ++ nvflare/free/api/runner.py | 50 +++++++++-- nvflare/free/examples/np/__init__.py | 0 nvflare/free/examples/np/client.py | 27 ++++++ nvflare/free/examples/np/cyclic.py | 18 ++++ nvflare/free/examples/np/fed_avg_para.py | 28 +++++++ nvflare/free/examples/np/fed_avg_seq.py | 28 +++++++ nvflare/free/examples/np/server.py | 81 ++++++++++++++++++ .../examples/{np_swarm.py => np/swarm.py} | 0 nvflare/free/examples/np_cyclic.py | 55 ------------- nvflare/free/examples/np_fed_avg.py | 70 ---------------- nvflare/free/examples/np_fed_avg_grp.py | 82 ------------------- 13 files changed, 239 insertions(+), 217 deletions(-) create mode 100644 nvflare/free/examples/np/__init__.py create mode 100644 nvflare/free/examples/np/client.py create mode 100644 nvflare/free/examples/np/cyclic.py create mode 100644 nvflare/free/examples/np/fed_avg_para.py create mode 100644 nvflare/free/examples/np/fed_avg_seq.py create mode 100644 nvflare/free/examples/np/server.py rename nvflare/free/examples/{np_swarm.py => np/swarm.py} (100%) delete mode 100644 nvflare/free/examples/np_cyclic.py delete mode 100644 nvflare/free/examples/np_fed_avg.py delete mode 100644 nvflare/free/examples/np_fed_avg_grp.py diff --git a/nvflare/free/api/app.py b/nvflare/free/api/app.py index 552393c80f..a6ef53e744 100644 --- a/nvflare/free/api/app.py +++ b/nvflare/free/api/app.py @@ -1,5 +1,5 @@ from abc import abstractmethod, ABC -from typing import List +from typing import List, Union from .proxy import Proxy from .group import Group @@ -83,3 +83,9 @@ def run(self, **kwargs): class ClientApp(App): pass + +class ClientAppFactory(ABC): + + @abstractmethod + def make_client_app(self, name: str) -> ClientApp: + pass diff --git a/nvflare/free/api/proxy.py b/nvflare/free/api/proxy.py index 7c4bbd3069..f946222613 100644 --- a/nvflare/free/api/proxy.py +++ b/nvflare/free/api/proxy.py @@ -13,6 +13,15 @@ def __init__(self, target_name, backend: Backend, caller_name: str): def name(self): return self.target_name + def get_target(self, name: str): + obj = getattr(self, name, None) + if not obj: + return None + if isinstance(obj, Proxy): + return obj + else: + return None + def __getattr__(self, func_name): """ This method is called when Python cannot find an invoked method func_name of this class. diff --git a/nvflare/free/api/runner.py b/nvflare/free/api/runner.py index f9d5918ac9..759a13bad5 100644 --- a/nvflare/free/api/runner.py +++ b/nvflare/free/api/runner.py @@ -1,7 +1,9 @@ import copy +import traceback from concurrent.futures import ThreadPoolExecutor +from typing import Union -from nvflare.free.api.app import ServerApp, ClientApp, App, SERVER_NAME +from nvflare.free.api.app import ServerApp, ClientApp, App, SERVER_NAME, ClientAppFactory from nvflare.free.api.proxy import Proxy from nvflare.free.api.sim_backend import SimBackend from nvflare.apis.signal import Signal @@ -37,16 +39,39 @@ def _prepare_proxies(self, server_app: App, client_apps: dict, caller_name, back def __init__( self, server_app: ServerApp, - client_app: ClientApp, - num_clients: int, + client_app: Union[ClientAppFactory, ClientApp], + max_workers: int = 100, + num_clients: int = 2, ): + if not isinstance(server_app, ServerApp): + raise ValueError(f"server_app must be ServerApp but got {type(server_app)}") + + if not isinstance(client_app, (ClientAppFactory, ClientApp)): + raise ValueError(f"client_app must be ClientApp or ClientAppFactory but got {type(client_app)}") + self.abort_signal = Signal() - self.thread_executor = ThreadPoolExecutor(max_workers=100) + self.thread_executor = ThreadPoolExecutor(max_workers=max_workers) + self.workflows = [(server_app, client_app)] + self.num_clients = num_clients + + def add_workflow(self, server_app: ServerApp, client_app: Union[ClientAppFactory, ClientApp]): + if not isinstance(server_app, ServerApp): + raise ValueError(f"server_app must be ServerApp but got {type(server_app)}") + if not isinstance(client_app, (ClientAppFactory, ClientApp)): + raise ValueError(f"client_app must be ClientApp or ClientAppFactory but got {type(client_app)}") + + self.workflows.append((server_app, client_app)) + + def _run_workflow(self, wf): + server_app, client_app = wf client_apps = {} - for i in range(num_clients): - name = f"site-{i+1}" - app = copy.deepcopy(client_app) + for i in range(self.num_clients): + name = f"site-{i + 1}" + if isinstance(client_app, ClientApp): + app = copy.deepcopy(client_app) + else: + app = client_app.make_client_app(name) app.name = name client_apps[name] = app @@ -66,9 +91,16 @@ def __init__( server_proxy, client_proxies = self._prepare_proxies(server_app, client_apps, SERVER_NAME, backends) server_app.setup(SERVER_NAME, server_proxy, client_proxies, self.abort_signal) - self.server_app = server_app + server_app.run() def run(self): # run the server - self.server_app.run() + for idx, wf in enumerate(self.workflows): + try: + print(f"Running Workflow #{idx+1}") + self._run_workflow(wf) + except: + traceback.print_exc() + break + self.thread_executor.shutdown(wait=False, cancel_futures=True) diff --git a/nvflare/free/examples/np/__init__.py b/nvflare/free/examples/np/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/nvflare/free/examples/np/client.py b/nvflare/free/examples/np/client.py new file mode 100644 index 0000000000..24ba2b23f3 --- /dev/null +++ b/nvflare/free/examples/np/client.py @@ -0,0 +1,27 @@ +import random + +from nvflare.apis.signal import Signal +from nvflare.free.api.app import ClientApp +from nvflare.free.api.ctx import Context + + +class NPTrainer(ClientApp): + + def __init__(self, delta: float): + ClientApp.__init__(self) + self.delta = delta + + def train(self, r, weights, abort_signal: Signal, context: Context, **kwargs): + if abort_signal.triggered: + print("training aborted") + return 0 + print(f"[{self.name}] called by {context.caller}: client {context.callee} trained round {r}") + + metric_receiver = self.server.get_target("metric_receiver") + if metric_receiver: + metric_receiver.accept_metric({"round": r, "y": 2}) + return weights + self.delta + + def evaluate(self, model, context: Context, **kwargs): + print(f"[{self.name}] called by {context.caller}: client {context.callee} to evaluate") + return random.random() diff --git a/nvflare/free/examples/np/cyclic.py b/nvflare/free/examples/np/cyclic.py new file mode 100644 index 0000000000..dc10390869 --- /dev/null +++ b/nvflare/free/examples/np/cyclic.py @@ -0,0 +1,18 @@ +from nvflare.free.examples.np.server import NPCyclic +from nvflare.free.examples.np.client import NPTrainer +from nvflare.free.api.runner import AppRunner + + +def main(): + + runner = AppRunner( + server_app=NPCyclic(num_rounds=2), + client_app=NPTrainer(delta=1.0), + num_clients=2 + ) + + runner.run() + + +if __name__ == "__main__": + main() diff --git a/nvflare/free/examples/np/fed_avg_para.py b/nvflare/free/examples/np/fed_avg_para.py new file mode 100644 index 0000000000..bff7371ab0 --- /dev/null +++ b/nvflare/free/examples/np/fed_avg_para.py @@ -0,0 +1,28 @@ +from nvflare.free.examples.np.server import NPFedAvgParallel +from nvflare.free.examples.np.client import NPTrainer +from nvflare.free.api.runner import AppRunner +from nvflare.free.api.ctx import Context + + +class MetricReceiver: + + def accept_metric(self, metrics: dict, context: Context, **kwargs): + print(f"[{context.callee}] received metric report from {context.caller}: {metrics}") + + +def main(): + + server_app = NPFedAvgParallel(num_rounds=2) + server_app.add_target_object("metric_receiver", MetricReceiver()) + + runner = AppRunner( + server_app=server_app, + client_app=NPTrainer(delta=1.0), + num_clients=10, + ) + + runner.run() + + +if __name__ == "__main__": + main() diff --git a/nvflare/free/examples/np/fed_avg_seq.py b/nvflare/free/examples/np/fed_avg_seq.py new file mode 100644 index 0000000000..72913f11d7 --- /dev/null +++ b/nvflare/free/examples/np/fed_avg_seq.py @@ -0,0 +1,28 @@ +from nvflare.free.examples.np.server import NPFedAvgSequential +from nvflare.free.examples.np.client import NPTrainer +from nvflare.free.api.runner import AppRunner +from nvflare.free.api.ctx import Context + + +class MetricReceiver: + + def accept_metric(self, metrics: dict, context: Context, **kwargs): + print(f"[{context.callee}] received metric report from {context.caller}: {metrics}") + + +def main(): + + server_app = NPFedAvgSequential(num_rounds=2) + server_app.add_target_object("metric_receiver", MetricReceiver()) + + runner = AppRunner( + server_app=server_app, + client_app=NPTrainer(delta=1.0), + num_clients=2, + ) + + runner.run() + + +if __name__ == "__main__": + main() diff --git a/nvflare/free/examples/np/server.py b/nvflare/free/examples/np/server.py new file mode 100644 index 0000000000..12783bfa73 --- /dev/null +++ b/nvflare/free/examples/np/server.py @@ -0,0 +1,81 @@ +import numpy as np +import random + +from nvflare.free.api.app import ServerApp + + +class NPFedAvgSequential(ServerApp): + + def __init__(self, num_rounds=10): + ServerApp.__init__(self) + self.num_rounds = num_rounds + self.initial_model = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32) + + def run(self, **kwargs): + print(f"[{self.name}] Start training for {self.num_rounds} rounds") + current_model = self.initial_model + for i in range(self.num_rounds): + current_model = self._do_one_round(i, current_model) + + def _do_one_round(self, r, current_model): + total = np.array([[0, 0, 0], [0, 0, 0], [0, 0, 0]], dtype=np.float32) + n = 0 + for c in self.clients: + result = c.train(r, current_model) + print(f"[{self.name}] round {r}: got result from client {c.name}: {result}") + total += result + n += 1 + return total / n + + +class NPFedAvgParallel(ServerApp): + + def __init__(self, num_rounds=10): + ServerApp.__init__(self) + self.num_rounds = num_rounds + self.initial_model = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32) + + def run(self, **kwargs): + print(f"[{self.name}] Start training for {self.num_rounds} rounds") + current_model = self.initial_model + for i in range(self.num_rounds): + current_model = self._do_one_round(i, current_model) + score = self._do_eval(current_model) + print(f"[{self.name}]: eval score in round {i}: {score}") + + def _do_eval(self, model): + results = self.group(self.clients).evaluate(model) + total= 0.0 + for n, v in results.items(): + print(f"[{self.name}]: got eval result from client {n}: {v}") + total += v + return total / len(results) + + def _do_one_round(self, r, current_model): + total = np.array([[0, 0, 0], [0, 0, 0], [0, 0, 0]], dtype=np.float32) + results = self.group(self.clients).train(r, current_model) + for n, v in results.items(): + print(f"[{self.name}] round {r}: got group result from client {n}: {v}") + total += v + return total / len(results) + + +class NPCyclic(ServerApp): + + def __init__(self, num_rounds=10): + ServerApp.__init__(self) + self.num_rounds = num_rounds + self.initial_model = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32) + + def run(self): + current_model = self.initial_model + for current_round in range(self.num_rounds): + current_model = self._do_one_round(current_round, current_model) + print(f"final result: {current_model}") + + def _do_one_round(self, current_round, current_model): + random.shuffle(self.clients) + for c in self.clients: + current_model = c.train(current_round, current_model) + print(f"result from {c.name}: {current_model}") + return current_model diff --git a/nvflare/free/examples/np_swarm.py b/nvflare/free/examples/np/swarm.py similarity index 100% rename from nvflare/free/examples/np_swarm.py rename to nvflare/free/examples/np/swarm.py diff --git a/nvflare/free/examples/np_cyclic.py b/nvflare/free/examples/np_cyclic.py deleted file mode 100644 index b814572d07..0000000000 --- a/nvflare/free/examples/np_cyclic.py +++ /dev/null @@ -1,55 +0,0 @@ -import random - -import numpy as np - -from nvflare.free.api.app import ServerApp, ClientApp -from nvflare.free.api.runner import AppRunner - - -class NPCyclic(ServerApp): - - def __init__(self, num_rounds=10): - ServerApp.__init__(self) - self.num_rounds = num_rounds - self.initial_model = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32) - - def run(self): - current_model = self.initial_model - for _ in range(self.num_rounds): - current_model = self._do_one_round(current_model) - print(f"final result: {current_model}") - - def _do_one_round(self, current_model): - random.shuffle(self.clients) - for c in self.clients: - current_model = c.train(current_model) - print(f"result from {c.name}: {current_model}") - return current_model - - -class NPTrainer(ClientApp): - - def __init__(self, delta: float): - ClientApp.__init__(self) - self.delta = delta - - def train(self, weights, **kwargs): - return weights + self.delta - - def evaluate(self, model): - pass - - -def main(): - - runner = AppRunner( - server_app=NPCyclic(num_rounds=2), - client_app=NPTrainer(delta=1.0), - num_clients=2 - ) - - runner.run() - - -if __name__ == "__main__": - main() diff --git a/nvflare/free/examples/np_fed_avg.py b/nvflare/free/examples/np_fed_avg.py deleted file mode 100644 index ccbdda9afd..0000000000 --- a/nvflare/free/examples/np_fed_avg.py +++ /dev/null @@ -1,70 +0,0 @@ -import numpy as np - -from nvflare.free.api.app import ServerApp, ClientApp -from nvflare.free.api.runner import AppRunner -from nvflare.free.api.ctx import Context -from nvflare.apis.signal import Signal - - -class NPFedAvg(ServerApp): - - def __init__(self, num_rounds=10): - ServerApp.__init__(self) - self.num_rounds = num_rounds - self.initial_model = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32) - - def run(self, **kwargs): - print(f"[{self.name}] Start training for {self.num_rounds} rounds") - current_model = self.initial_model - for i in range(self.num_rounds): - current_model = self._do_one_round(i, current_model) - - def _do_one_round(self, r, current_model): - total = np.array([[0, 0, 0], [0, 0, 0], [0, 0, 0]], dtype=np.float32) - n = 0 - for c in self.clients: - result = c.train(r, current_model) - print(f"[{self.name}] round {r}: got result from client {c.name}: {result}") - total += result - n += 1 - return total / n - - -class MetricReceiver: - - def accept_metric(self, metrics: dict, context: Context, **kwargs): - print(f"[{context.callee}] received metric report from {context.caller}: {metrics}") - - -class NPTrainer(ClientApp): - - def __init__(self, delta: float): - ClientApp.__init__(self) - self.delta = delta - - def train(self, r, weights, abort_signal: Signal, context: Context, **kwargs): - if abort_signal.triggered: - print("training aborted") - return 0 - print(f"[{self.name}] called by {context.caller}: client {context.callee} trained round {r}") - - self.server.metric_receiver.accept_metric({"round": r, "y": 2}) - return weights + self.delta - - -def main(): - - server_app = NPFedAvg(num_rounds=2) - server_app.add_target_object("metric_receiver", MetricReceiver()) - - runner = AppRunner( - server_app=server_app, - client_app=NPTrainer(delta=1.0), - num_clients=2 - ) - - runner.run() - - -if __name__ == "__main__": - main() diff --git a/nvflare/free/examples/np_fed_avg_grp.py b/nvflare/free/examples/np_fed_avg_grp.py deleted file mode 100644 index 48bee6d3a8..0000000000 --- a/nvflare/free/examples/np_fed_avg_grp.py +++ /dev/null @@ -1,82 +0,0 @@ -import numpy as np -import random - -from nvflare.free.api.app import ServerApp, ClientApp -from nvflare.free.api.runner import AppRunner -from nvflare.free.api.ctx import Context -from nvflare.apis.signal import Signal - - -class NPFedAvg(ServerApp): - - def __init__(self, num_rounds=10): - ServerApp.__init__(self) - self.num_rounds = num_rounds - self.initial_model = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32) - - def run(self, **kwargs): - print(f"[{self.name}] Start training for {self.num_rounds} rounds") - current_model = self.initial_model - for i in range(self.num_rounds): - current_model = self._do_one_round(i, current_model) - score = self._do_eval(current_model) - print(f"[{self.name}]: eval score in round {i}: {score}") - - def _do_eval(self, model): - results = self.group(self.clients).evaluate(model) - total= 0.0 - for n, v in results.items(): - print(f"[{self.name}]: got eval result from client {n}: {v}") - total += v - return total / len(results) - - def _do_one_round(self, r, current_model): - total = np.array([[0, 0, 0], [0, 0, 0], [0, 0, 0]], dtype=np.float32) - results = self.group(self.clients).train(r, current_model) - for n, v in results.items(): - print(f"[{self.name}] round {r}: got group result from client {n}: {v}") - total += v - return total / len(results) - - -class MetricReceiver: - - def accept_metric(self, metrics: dict, context: Context, **kwargs): - print(f"[{context.callee}] received metric report from {context.caller}: {metrics}") - - -class NPTrainer(ClientApp): - - def __init__(self, delta: float): - ClientApp.__init__(self) - self.delta = delta - - def train(self, r, weights, abort_signal: Signal, context: Context, **kwargs): - if abort_signal.triggered: - print("training aborted") - return 0 - print(f"[{self.name}] called by {context.caller}: client {context.callee} trained round {r}") - self.server.metric_receiver.accept_metric({"round": r, "y": 2}) - return weights + self.delta - - def evaluate(self, model, context: Context, **kwargs): - print(f"[{self.name}] called by {context.caller}: client {context.callee} to evaluate") - return random.random() - - -def main(): - - server_app = NPFedAvg(num_rounds=2) - server_app.add_target_object("metric_receiver", MetricReceiver()) - - runner = AppRunner( - server_app=server_app, - client_app=NPTrainer(delta=1.0), - num_clients=2 - ) - - runner.run() - - -if __name__ == "__main__": - main() From 0f097b0ee637d3620625726ed73ab230a484fc1a Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Thu, 18 Sep 2025 18:46:25 -0400 Subject: [PATCH 003/102] make collab methods optional args --- nvflare/free/api/constants.py | 9 +++++++++ nvflare/free/api/group.py | 6 ++++-- nvflare/free/api/proxy.py | 3 ++- nvflare/free/api/sim_backend.py | 11 ++++++++--- nvflare/free/api/strategy.py | 11 +++++++++++ nvflare/free/api/utils.py | 13 +++++++++++++ nvflare/free/examples/np/client.py | 4 ++-- nvflare/free/examples/np/fed_avg_para.py | 2 +- nvflare/free/examples/np/fed_avg_seq.py | 2 +- nvflare/free/examples/np/swarm.py | 12 ++++++------ 10 files changed, 57 insertions(+), 16 deletions(-) create mode 100644 nvflare/free/api/constants.py create mode 100644 nvflare/free/api/strategy.py create mode 100644 nvflare/free/api/utils.py diff --git a/nvflare/free/api/constants.py b/nvflare/free/api/constants.py new file mode 100644 index 0000000000..4852414007 --- /dev/null +++ b/nvflare/free/api/constants.py @@ -0,0 +1,9 @@ +class CollabMethodArgName: + # defines optional args that a target's collaboration method can have + ABORT_SIGNAL = "abort_signal" + CONTEXT = "context" + + +class CollabMethodOptionName: + BLOCKING = "blocking" + TIMEOUT = "timeout" diff --git a/nvflare/free/api/group.py b/nvflare/free/api/group.py index 326fd8047d..babccfe7ea 100644 --- a/nvflare/free/api/group.py +++ b/nvflare/free/api/group.py @@ -8,6 +8,7 @@ from .proxy import Proxy from .ctx import Context from .resp import Resp +from .constants import CollabMethodArgName class Group: @@ -47,12 +48,13 @@ def method(*args, **kwargs): resps = {} for p in self._proxies: kwargs_copy = copy.copy(kwargs) + ctx = Context(p.caller_name, p.name) - kwargs_copy["context"] = ctx + kwargs_copy[CollabMethodArgName.CONTEXT] = ctx resp = Resp(self._process_resp_cb, self._cb_kwargs) resps[p.name] = resp - kwargs_copy["abort_signal"] = self._abort_signal + kwargs_copy[CollabMethodArgName.ABORT_SIGNAL] = self._abort_signal p.backend.call_target_with_resp( resp, p.name, func_name, *args, **kwargs_copy ) diff --git a/nvflare/free/api/proxy.py b/nvflare/free/api/proxy.py index f946222613..74b02de9e0 100644 --- a/nvflare/free/api/proxy.py +++ b/nvflare/free/api/proxy.py @@ -1,5 +1,6 @@ from .backend import Backend from .ctx import Context +from .constants import CollabMethodArgName class Proxy: @@ -29,7 +30,7 @@ def __getattr__(self, func_name): def method(*args, **kwargs): ctx = Context(self.caller_name, self.name) - kwargs["context"] = ctx + kwargs[CollabMethodArgName.CONTEXT] = ctx return self.backend.call_target(self.target_name, func_name, *args, **kwargs) return method diff --git a/nvflare/free/api/sim_backend.py b/nvflare/free/api/sim_backend.py index e9737a3dfb..cac15436ef 100644 --- a/nvflare/free/api/sim_backend.py +++ b/nvflare/free/api/sim_backend.py @@ -2,6 +2,8 @@ from .backend import Backend from .resp import Resp +from .constants import CollabMethodOptionName, CollabMethodArgName +from .utils import check_optional_args class _Waiter(threading.Event): @@ -27,9 +29,10 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): if not callable(func): raise AttributeError(f"the {func_name} of {target_name} is not callable") - blocking = kwargs.pop("blocking", True) - timeout = kwargs.pop("timeout", None) - kwargs["abort_signal"] = self.abort_signal + kwargs[CollabMethodArgName.ABORT_SIGNAL] = self.abort_signal + + blocking = kwargs.pop(CollabMethodOptionName.BLOCKING, True) + timeout = kwargs.pop(CollabMethodOptionName.TIMEOUT, None) waiter = None if blocking: @@ -48,6 +51,7 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): @staticmethod def _run_func(waiter: _Waiter, func, args, kwargs): try: + check_optional_args(func, kwargs) result = func(*args, **kwargs) if waiter: waiter.result = result @@ -71,6 +75,7 @@ def call_target_with_resp(self, resp: Resp, target_name: str, func_name: str, *a @staticmethod def _run_func_with_resp(resp: Resp, func, args, kwargs): try: + check_optional_args(func, kwargs) result = func(*args, **kwargs) resp.set_result(result) except Exception as ex: diff --git a/nvflare/free/api/strategy.py b/nvflare/free/api/strategy.py new file mode 100644 index 0000000000..dbf603695f --- /dev/null +++ b/nvflare/free/api/strategy.py @@ -0,0 +1,11 @@ +from abc import ABC, abstractmethod +from typing import List + +from .proxy import Proxy + + +class Strategy(ABC): + + @abstractmethod + def run(self, clients: List[Proxy], **kwargs): + pass diff --git a/nvflare/free/api/utils.py b/nvflare/free/api/utils.py new file mode 100644 index 0000000000..24138aa474 --- /dev/null +++ b/nvflare/free/api/utils.py @@ -0,0 +1,13 @@ +import inspect + +from .constants import CollabMethodArgName + + +def check_optional_args(func, kwargs): + signature = inspect.signature(func) + parameter_names = signature.parameters.keys() + + # make sure to expose the optional args if the collab method supports them + for n in [CollabMethodArgName.ABORT_SIGNAL, CollabMethodArgName.CONTEXT]: + if n not in parameter_names: + kwargs.pop(n, None) diff --git a/nvflare/free/examples/np/client.py b/nvflare/free/examples/np/client.py index 24ba2b23f3..d80daf30bd 100644 --- a/nvflare/free/examples/np/client.py +++ b/nvflare/free/examples/np/client.py @@ -11,7 +11,7 @@ def __init__(self, delta: float): ClientApp.__init__(self) self.delta = delta - def train(self, r, weights, abort_signal: Signal, context: Context, **kwargs): + def train(self, r, weights, abort_signal: Signal, context: Context): if abort_signal.triggered: print("training aborted") return 0 @@ -22,6 +22,6 @@ def train(self, r, weights, abort_signal: Signal, context: Context, **kwargs): metric_receiver.accept_metric({"round": r, "y": 2}) return weights + self.delta - def evaluate(self, model, context: Context, **kwargs): + def evaluate(self, model, context: Context): print(f"[{self.name}] called by {context.caller}: client {context.callee} to evaluate") return random.random() diff --git a/nvflare/free/examples/np/fed_avg_para.py b/nvflare/free/examples/np/fed_avg_para.py index bff7371ab0..875b641497 100644 --- a/nvflare/free/examples/np/fed_avg_para.py +++ b/nvflare/free/examples/np/fed_avg_para.py @@ -6,7 +6,7 @@ class MetricReceiver: - def accept_metric(self, metrics: dict, context: Context, **kwargs): + def accept_metric(self, metrics: dict, context: Context): print(f"[{context.callee}] received metric report from {context.caller}: {metrics}") diff --git a/nvflare/free/examples/np/fed_avg_seq.py b/nvflare/free/examples/np/fed_avg_seq.py index 72913f11d7..ad84d2515f 100644 --- a/nvflare/free/examples/np/fed_avg_seq.py +++ b/nvflare/free/examples/np/fed_avg_seq.py @@ -6,7 +6,7 @@ class MetricReceiver: - def accept_metric(self, metrics: dict, context: Context, **kwargs): + def accept_metric(self, metrics: dict, context: Context): print(f"[{context.callee}] received metric report from {context.caller}: {metrics}") diff --git a/nvflare/free/examples/np/swarm.py b/nvflare/free/examples/np/swarm.py index d7a340692f..3fc82c28e3 100644 --- a/nvflare/free/examples/np/swarm.py +++ b/nvflare/free/examples/np/swarm.py @@ -34,18 +34,18 @@ def __init__(self, delta: float): super().__init__() self.delta = delta - def train(self, weights, **kwargs): + def train(self, weights): return weights + self.delta def sag(self, model): - results = self.group(self.clients).train(model, blocking=True) + results = self.group(self.clients, blocking=True).train(model) results = list(results.values()) total = results[0] for i in range(1, len(results)): total += results[i] return total / len(results) - def swarm_learn(self, num_rounds, model, current_round, context: Context, **kwargs): + def swarm_learn(self, num_rounds, model, current_round, context: Context): print(f"[{context.callee}]: swarm learn asked by {context.caller}: {num_rounds=} {current_round=} {model=}") new_model = self.sag(model) @@ -62,10 +62,10 @@ def swarm_learn(self, num_rounds, model, current_round, context: Context, **kwar next_client = self.clients[next_client_idx] next_client.swarm_learn(num_rounds, new_model, next_round, blocking=False) - def start(self, num_rounds, initial_model, **kwargs): - self.swarm_learn(num_rounds, initial_model, 0, **kwargs) + def start(self, num_rounds, initial_model, context: Context): + self.swarm_learn(num_rounds, initial_model, 0, context) - def accept_final_model(self, model, context: Context, **kwargs): + def accept_final_model(self, model, context: Context): # accept the final model # write model to disk print(f"[{context.callee}]: received final model from {context.caller}: {model}") From 66e2735444f467e58fc8b197eed769622e05ab1e Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Mon, 22 Sep 2025 16:32:27 -0400 Subject: [PATCH 004/102] dev --- nvflare/free/__init__.py | 13 ++ nvflare/free/api/__init__.py | 13 ++ nvflare/free/api/app.py | 68 +++++----- nvflare/free/api/backend.py | 20 ++- nvflare/free/api/constants.py | 14 +- nvflare/free/api/controller.py | 23 ++++ nvflare/free/api/ctx.py | 30 ++++- nvflare/free/api/group.py | 57 ++++++-- nvflare/free/api/proxy.py | 20 ++- nvflare/free/api/resp.py | 29 +++- nvflare/free/api/runner.py | 60 +++++---- nvflare/free/api/sim_backend.py | 66 +++++++-- nvflare/free/api/strategy.py | 11 -- nvflare/free/api/utils.py | 15 ++- nvflare/free/examples/__init__.py | 13 ++ nvflare/free/examples/np/__init__.py | 13 ++ nvflare/free/examples/np/client.py | 31 ++++- nvflare/free/examples/np/controllers.py | 149 +++++++++++++++++++++ nvflare/free/examples/np/cyclic.py | 28 +++- nvflare/free/examples/np/fed_avg_intime.py | 42 ++++++ nvflare/free/examples/np/fed_avg_para.py | 34 +++-- nvflare/free/examples/np/fed_avg_seq.py | 37 +++-- nvflare/free/examples/np/server.py | 81 ----------- nvflare/free/examples/np/swarm.py | 55 +++++--- nvflare/free/examples/np/widgets.py | 20 +++ 25 files changed, 710 insertions(+), 232 deletions(-) create mode 100644 nvflare/free/api/controller.py delete mode 100644 nvflare/free/api/strategy.py create mode 100644 nvflare/free/examples/np/controllers.py create mode 100644 nvflare/free/examples/np/fed_avg_intime.py delete mode 100644 nvflare/free/examples/np/server.py create mode 100644 nvflare/free/examples/np/widgets.py diff --git a/nvflare/free/__init__.py b/nvflare/free/__init__.py index e69de29bb2..341a77c5bc 100644 --- a/nvflare/free/__init__.py +++ b/nvflare/free/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nvflare/free/api/__init__.py b/nvflare/free/api/__init__.py index e69de29bb2..341a77c5bc 100644 --- a/nvflare/free/api/__init__.py +++ b/nvflare/free/api/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nvflare/free/api/app.py b/nvflare/free/api/app.py index a6ef53e744..9b7b3c132c 100644 --- a/nvflare/free/api/app.py +++ b/nvflare/free/api/app.py @@ -1,7 +1,21 @@ -from abc import abstractmethod, ABC -from typing import List, Union +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from abc import ABC, abstractmethod +from typing import List + +from .controller import Controller from .proxy import Proxy -from .group import Group SERVER_NAME = "server" @@ -13,16 +27,18 @@ def __init__(self): self.server = None self.clients = None self._me = None - self._target_objs = {} + self._target_objs = [] self._abort_signal = None + def get_default_target(self): + return None + def add_target_object(self, name: str, obj): - if name in ['name','server', 'clients', "add_target_object", "get_target_objects", "setup", "get_my_site"]: + if hasattr(obj, name): raise ValueError(f"conflict with reserved name {name}") - # TBD: more name validation needed setattr(self, name, obj) - self._target_objs[name] = obj + self._target_objs.append((name, obj)) def get_target_objects(self): return self._target_objs @@ -51,33 +67,23 @@ def get_my_site(self) -> Proxy: def initialize(self, **kwargs): pass - def group( - self, - proxies: List[Proxy], - blocking: bool = True, - timeout: float = None, - min_resps: int = None, - wait_after_min_resps: float = None, - process_resp_cb=None, - **cb_kwargs, - ): - return Group( - self._abort_signal, - proxies, - blocking, - timeout, - min_resps, - wait_after_min_resps, - process_resp_cb, - **cb_kwargs - ) - class ServerApp(App): - @abstractmethod - def run(self, **kwargs): - pass + def __init__(self, controller: Controller): + super().__init__() + if not isinstance(controller, Controller): + raise ValueError(f"controller must be Controller but got {type(controller)}") + self.controllers = [controller] + self.current_controller = None + + def add_controller(self, controller): + if not isinstance(controller, Controller): + raise ValueError(f"controller must be Controller but got {type(controller)}") + self.controllers.append(controller) + + def get_default_target(self): + return self.current_controller class ClientApp(App): diff --git a/nvflare/free/api/backend.py b/nvflare/free/api/backend.py index 785413a18b..ea67fb7a95 100644 --- a/nvflare/free/api/backend.py +++ b/nvflare/free/api/backend.py @@ -1,10 +1,28 @@ -from abc import abstractmethod, ABC +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from abc import ABC, abstractmethod + +from nvflare.apis.signal import Signal from .resp import Resp class Backend(ABC): + def __init__(self, abort_signal: Signal): + self.abort_signal = abort_signal + @abstractmethod def call_target(self, target_name: str, func_name: str, *args, **kwargs): pass diff --git a/nvflare/free/api/constants.py b/nvflare/free/api/constants.py index 4852414007..bc472f7454 100644 --- a/nvflare/free/api/constants.py +++ b/nvflare/free/api/constants.py @@ -1,6 +1,18 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. class CollabMethodArgName: # defines optional args that a target's collaboration method can have - ABORT_SIGNAL = "abort_signal" CONTEXT = "context" diff --git a/nvflare/free/api/controller.py b/nvflare/free/api/controller.py new file mode 100644 index 0000000000..3c8e688e5d --- /dev/null +++ b/nvflare/free/api/controller.py @@ -0,0 +1,23 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from abc import ABC, abstractmethod + +from .ctx import Context + + +class Controller(ABC): + + @abstractmethod + def run(self, context: Context): + pass diff --git a/nvflare/free/api/ctx.py b/nvflare/free/api/ctx.py index 24b34c3749..e652b3b3c2 100644 --- a/nvflare/free/api/ctx.py +++ b/nvflare/free/api/ctx.py @@ -1,8 +1,34 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nvflare.apis.signal import Signal + + class Context: - def __init__(self, caller: str, callee: str): + def __init__(self, caller: str, callee: str, abort_signal: Signal): self.caller = caller self.callee = callee + self.abort_signal = abort_signal + self.server = None + self.clients = None + self.props = {} def set_prop(self, name: str, value): - setattr(self, name, value) + self.props[name] = value + + def get_prop(self, name: str, default=None): + return self.props.get(name, default) + + def is_aborted(self): + return self.abort_signal and self.abort_signal.triggered diff --git a/nvflare/free/api/group.py b/nvflare/free/api/group.py index babccfe7ea..b675bab349 100644 --- a/nvflare/free/api/group.py +++ b/nvflare/free/api/group.py @@ -1,14 +1,27 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import copy import time from typing import List -from nvflare.apis.signal import Signal from nvflare.apis.fl_exception import RunAborted +from nvflare.apis.signal import Signal -from .proxy import Proxy +from .constants import CollabMethodArgName from .ctx import Context +from .proxy import Proxy from .resp import Resp -from .constants import CollabMethodArgName class Group: @@ -49,15 +62,12 @@ def method(*args, **kwargs): for p in self._proxies: kwargs_copy = copy.copy(kwargs) - ctx = Context(p.caller_name, p.name) + ctx = Context(p.caller_name, p.name, self._abort_signal) kwargs_copy[CollabMethodArgName.CONTEXT] = ctx - resp = Resp(self._process_resp_cb, self._cb_kwargs) + resp = Resp(self._process_resp_cb, self._cb_kwargs, ctx) resps[p.name] = resp - kwargs_copy[CollabMethodArgName.ABORT_SIGNAL] = self._abort_signal - p.backend.call_target_with_resp( - resp, p.name, func_name, *args, **kwargs_copy - ) + p.backend.call_target_with_resp(resp, p.name, func_name, *args, **kwargs_copy) # wait for responses if not self._blocking: @@ -106,3 +116,32 @@ def method(*args, **kwargs): return results return method + + +def group( + ctx: Context, + proxies: List[Proxy], + blocking: bool = True, + timeout: float = None, + min_resps: int = None, + wait_after_min_resps: float = None, + process_resp_cb=None, + **cb_kwargs, +): + return Group( + ctx.abort_signal, proxies, blocking, timeout, min_resps, wait_after_min_resps, process_resp_cb, **cb_kwargs + ) + + +def all_clients( + ctx: Context, + blocking: bool = True, + timeout: float = None, + min_resps: int = None, + wait_after_min_resps: float = None, + process_resp_cb=None, + **cb_kwargs, +): + return Group( + ctx.abort_signal, ctx.clients, blocking, timeout, min_resps, wait_after_min_resps, process_resp_cb, **cb_kwargs + ) diff --git a/nvflare/free/api/proxy.py b/nvflare/free/api/proxy.py index 74b02de9e0..da7d961e75 100644 --- a/nvflare/free/api/proxy.py +++ b/nvflare/free/api/proxy.py @@ -1,11 +1,25 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from .backend import Backend -from .ctx import Context from .constants import CollabMethodArgName +from .ctx import Context class Proxy: - def __init__(self, target_name, backend: Backend, caller_name: str): + def __init__(self, app, target_name, backend: Backend, caller_name: str): + self.app = app self.target_name = target_name self.backend = backend self.caller_name = caller_name @@ -29,7 +43,7 @@ def __getattr__(self, func_name): """ def method(*args, **kwargs): - ctx = Context(self.caller_name, self.name) + ctx = Context(self.caller_name, self.name, self.backend.abort_signal) kwargs[CollabMethodArgName.CONTEXT] = ctx return self.backend.call_target(self.target_name, func_name, *args, **kwargs) diff --git a/nvflare/free/api/resp.py b/nvflare/free/api/resp.py index 8250a9effa..67b6bcc609 100644 --- a/nvflare/free/api/resp.py +++ b/nvflare/free/api/resp.py @@ -1,17 +1,44 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import copy import time +from .constants import CollabMethodArgName +from .ctx import Context +from .utils import check_optional_args + class Resp: - def __init__(self, process_cb, cb_kwargs): + def __init__(self, process_cb, cb_kwargs, context: Context): self.result = None self.exception = None self.resp_time = None self.process_cb = process_cb self.cb_kwargs = cb_kwargs + self.context = context def set_result(self, result): if self.process_cb: + ctx = copy.copy(self.context) + + # swap caller/callee + original_caller = ctx.caller + ctx.caller = ctx.callee + ctx.callee = original_caller + self.cb_kwargs[CollabMethodArgName.CONTEXT] = ctx + check_optional_args(self.process_cb, self.cb_kwargs) result = self.process_cb(result, **self.cb_kwargs) self.result = result self.resp_time = time.time() diff --git a/nvflare/free/api/runner.py b/nvflare/free/api/runner.py index 759a13bad5..df2eca67ac 100644 --- a/nvflare/free/api/runner.py +++ b/nvflare/free/api/runner.py @@ -1,30 +1,44 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import copy import traceback from concurrent.futures import ThreadPoolExecutor from typing import Union -from nvflare.free.api.app import ServerApp, ClientApp, App, SERVER_NAME, ClientAppFactory +from nvflare.apis.signal import Signal +from nvflare.free.api.app import SERVER_NAME, App, ClientApp, ClientAppFactory, ServerApp +from nvflare.free.api.ctx import Context from nvflare.free.api.proxy import Proxy from nvflare.free.api.sim_backend import SimBackend -from nvflare.apis.signal import Signal class AppRunner: def _prepare_app_backends(self, app: App): - bes = {"": SimBackend(app, self.abort_signal, self.thread_executor)} + bes = {"": SimBackend(app, app, self.abort_signal, self.thread_executor)} targets = app.get_target_objects() if targets: - for name, obj in targets.items(): - bes[name] = SimBackend(obj, self.abort_signal, self.thread_executor) + for name, obj in targets: + bes[name] = SimBackend(app, obj, self.abort_signal, self.thread_executor) return bes def _prepare_app_proxy(self, app_name: str, app: App, caller_name: str, app_backends: dict): - app_proxy = Proxy(target_name=app_name, backend=app_backends[""], caller_name=caller_name) + app_proxy = Proxy(app=app, target_name=app_name, backend=app_backends[""], caller_name=caller_name) cos = app.get_target_objects() if cos: - for name, obj in cos.items(): - p = Proxy(target_name=name, backend=app_backends[name], caller_name=caller_name) + for name, obj in cos: + p = Proxy(app=app, target_name=name, backend=app_backends[name], caller_name=caller_name) setattr(app_proxy, name, p) return app_proxy @@ -50,21 +64,11 @@ def __init__( raise ValueError(f"client_app must be ClientApp or ClientAppFactory but got {type(client_app)}") self.abort_signal = Signal() + self.server_app = server_app + self.client_app = client_app self.thread_executor = ThreadPoolExecutor(max_workers=max_workers) - self.workflows = [(server_app, client_app)] self.num_clients = num_clients - def add_workflow(self, server_app: ServerApp, client_app: Union[ClientAppFactory, ClientApp]): - if not isinstance(server_app, ServerApp): - raise ValueError(f"server_app must be ServerApp but got {type(server_app)}") - - if not isinstance(client_app, (ClientAppFactory, ClientApp)): - raise ValueError(f"client_app must be ClientApp or ClientAppFactory but got {type(client_app)}") - - self.workflows.append((server_app, client_app)) - - def _run_workflow(self, wf): - server_app, client_app = wf client_apps = {} for i in range(self.num_clients): name = f"site-{i + 1}" @@ -75,9 +79,7 @@ def _run_workflow(self, wf): app.name = name client_apps[name] = app - backends = { - SERVER_NAME: self._prepare_app_backends(server_app) - } + backends = {SERVER_NAME: self._prepare_app_backends(server_app)} for name, app in client_apps.items(): backends[name] = self._prepare_app_backends(app) @@ -91,14 +93,16 @@ def _run_workflow(self, wf): server_proxy, client_proxies = self._prepare_proxies(server_app, client_apps, SERVER_NAME, backends) server_app.setup(SERVER_NAME, server_proxy, client_proxies, self.abort_signal) - server_app.run() - def run(self): # run the server - for idx, wf in enumerate(self.workflows): + for idx, controller in enumerate(self.server_app.controllers): try: - print(f"Running Workflow #{idx+1}") - self._run_workflow(wf) + print(f"Running Controller #{idx+1}") + self.server_app.current_controller = controller + ctx = Context(caller=self.server_app.name, callee=self.server_app.name, abort_signal=self.abort_signal) + ctx.server = self.server_app.server + ctx.clients = self.server_app.clients + controller.run(context=ctx) except: traceback.print_exc() break diff --git a/nvflare/free/api/sim_backend.py b/nvflare/free/api/sim_backend.py index cac15436ef..672cef849d 100644 --- a/nvflare/free/api/sim_backend.py +++ b/nvflare/free/api/sim_backend.py @@ -1,8 +1,22 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import threading +from .app import App from .backend import Backend +from .constants import CollabMethodArgName, CollabMethodOptionName from .resp import Resp -from .constants import CollabMethodOptionName, CollabMethodArgName from .utils import check_optional_args @@ -16,21 +30,40 @@ def __init__(self): class SimBackend(Backend): - def __init__(self, callable_obj, abort_signal, thread_executor): - self.obj = callable_obj + def __init__(self, target_app: App, target_obj, abort_signal, thread_executor): + Backend.__init__(self, abort_signal) + self.target_app = target_app + self.target_obj = target_obj self.executor = thread_executor - self.abort_signal = abort_signal + + def _get_func(self, func_name): + func = getattr(self.target_obj, func_name, None) + if func: + return func + + if isinstance(self.target_obj, App): + # see whether any targets have this method + default_target = self.target_obj.get_default_target() + if default_target: + func = getattr(default_target, func_name, None) + if func: + return func + + targets = self.target_obj.get_target_objects() + for _, obj in targets: + func = getattr(obj, func_name, None) + if func: + return func + return None def call_target(self, target_name: str, func_name: str, *args, **kwargs): - func = getattr(self.obj, func_name, None) + func = self._get_func(func_name) if not func: raise AttributeError(f"{target_name} does not have {func_name}") if not callable(func): raise AttributeError(f"the {func_name} of {target_name} is not callable") - kwargs[CollabMethodArgName.ABORT_SIGNAL] = self.abort_signal - blocking = kwargs.pop(CollabMethodOptionName.BLOCKING, True) timeout = kwargs.pop(CollabMethodOptionName.TIMEOUT, None) @@ -48,10 +81,16 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): raise waiter.exception return waiter.result - @staticmethod - def _run_func(waiter: _Waiter, func, args, kwargs): + def _augment_context(self, func, kwargs): + ctx = kwargs.get(CollabMethodArgName.CONTEXT) + if ctx: + ctx.server = self.target_app.server + ctx.clients = self.target_app.clients + check_optional_args(func, kwargs) + + def _run_func(self, waiter: _Waiter, func, args, kwargs): try: - check_optional_args(func, kwargs) + self._augment_context(func, kwargs) result = func(*args, **kwargs) if waiter: waiter.result = result @@ -63,7 +102,7 @@ def _run_func(waiter: _Waiter, func, args, kwargs): waiter.set() def call_target_with_resp(self, resp: Resp, target_name: str, func_name: str, *args, **kwargs): - func = getattr(self.obj, func_name, None) + func = self._get_func(func_name) if not func: raise AttributeError(f"{target_name} does not have {func_name}") @@ -72,10 +111,9 @@ def call_target_with_resp(self, resp: Resp, target_name: str, func_name: str, *a self.executor.submit(self._run_func_with_resp, resp, func, args, kwargs) - @staticmethod - def _run_func_with_resp(resp: Resp, func, args, kwargs): + def _run_func_with_resp(self, resp: Resp, func, args, kwargs): try: - check_optional_args(func, kwargs) + self._augment_context(func, kwargs) result = func(*args, **kwargs) resp.set_result(result) except Exception as ex: diff --git a/nvflare/free/api/strategy.py b/nvflare/free/api/strategy.py deleted file mode 100644 index dbf603695f..0000000000 --- a/nvflare/free/api/strategy.py +++ /dev/null @@ -1,11 +0,0 @@ -from abc import ABC, abstractmethod -from typing import List - -from .proxy import Proxy - - -class Strategy(ABC): - - @abstractmethod - def run(self, clients: List[Proxy], **kwargs): - pass diff --git a/nvflare/free/api/utils.py b/nvflare/free/api/utils.py index 24138aa474..32dded535d 100644 --- a/nvflare/free/api/utils.py +++ b/nvflare/free/api/utils.py @@ -1,3 +1,16 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import inspect from .constants import CollabMethodArgName @@ -8,6 +21,6 @@ def check_optional_args(func, kwargs): parameter_names = signature.parameters.keys() # make sure to expose the optional args if the collab method supports them - for n in [CollabMethodArgName.ABORT_SIGNAL, CollabMethodArgName.CONTEXT]: + for n in [CollabMethodArgName.CONTEXT]: if n not in parameter_names: kwargs.pop(n, None) diff --git a/nvflare/free/examples/__init__.py b/nvflare/free/examples/__init__.py index e69de29bb2..341a77c5bc 100644 --- a/nvflare/free/examples/__init__.py +++ b/nvflare/free/examples/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nvflare/free/examples/np/__init__.py b/nvflare/free/examples/np/__init__.py index e69de29bb2..341a77c5bc 100644 --- a/nvflare/free/examples/np/__init__.py +++ b/nvflare/free/examples/np/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nvflare/free/examples/np/client.py b/nvflare/free/examples/np/client.py index d80daf30bd..c70b8db096 100644 --- a/nvflare/free/examples/np/client.py +++ b/nvflare/free/examples/np/client.py @@ -1,7 +1,19 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import random -from nvflare.apis.signal import Signal -from nvflare.free.api.app import ClientApp +from nvflare.free.api.app import ClientApp, ClientAppFactory from nvflare.free.api.ctx import Context @@ -11,17 +23,26 @@ def __init__(self, delta: float): ClientApp.__init__(self) self.delta = delta - def train(self, r, weights, abort_signal: Signal, context: Context): - if abort_signal.triggered: + def train(self, r, weights, context: Context): + if context.is_aborted(): print("training aborted") return 0 print(f"[{self.name}] called by {context.caller}: client {context.callee} trained round {r}") metric_receiver = self.server.get_target("metric_receiver") if metric_receiver: - metric_receiver.accept_metric({"round": r, "y": 2}) + self.server.accept_metric({"round": r, "y": 2}) return weights + self.delta def evaluate(self, model, context: Context): print(f"[{self.name}] called by {context.caller}: client {context.callee} to evaluate") return random.random() + + +class TrainerFactory(ClientAppFactory): + + def __init__(self, delta): + self.delta = delta + + def make_client_app(self, name: str) -> ClientApp: + return NPTrainer(self.delta) diff --git a/nvflare/free/examples/np/controllers.py b/nvflare/free/examples/np/controllers.py new file mode 100644 index 0000000000..f363d9a01d --- /dev/null +++ b/nvflare/free/examples/np/controllers.py @@ -0,0 +1,149 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import random + +import numpy as np + +from nvflare.free.api.controller import Controller +from nvflare.free.api.ctx import Context +from nvflare.free.api.group import all_clients + + +class NPFedAvgSequential(Controller): + + def __init__(self, initial_model, num_rounds=10): + Controller.__init__(self) + self.name = "NPFedAvgSequential" + self.num_rounds = num_rounds + self.initial_model = initial_model + + def run(self, context: Context): + print(f"[{self.name}] Start training for {self.num_rounds} rounds") + current_model = self.initial_model + for i in range(self.num_rounds): + current_model = self._do_one_round(i, current_model, context) + + def _do_one_round(self, r, current_model, context: Context): + total = np.array([[0, 0, 0], [0, 0, 0], [0, 0, 0]], dtype=np.float32) + n = 0 + for c in context.clients: + result = c.train(r, current_model) + print(f"[{self.name}] round {r}: got result from client {c.name}: {result}") + total += result + n += 1 + return total / n + + +class NPFedAvgParallel(Controller): + + def __init__(self, initial_model, num_rounds=10): + self.num_rounds = num_rounds + self.initial_model = initial_model + self.name = "NPFedAvgParallel" + + def run(self, context: Context): + print(f"[{self.name}] Start training for {self.num_rounds} rounds") + current_model = self.initial_model + for i in range(self.num_rounds): + current_model = self._do_one_round(i, current_model, context) + score = self._do_eval(current_model, context) + print(f"[{self.name}]: eval score in round {i}: {score}") + + def _do_eval(self, model, ctx: Context): + results = all_clients(ctx).evaluate(model) + total = 0.0 + for n, v in results.items(): + print(f"[{self.name}]: got eval result from client {n}: {v}") + total += v + return total / len(results) + + def _do_one_round(self, r, current_model, ctx: Context): + total = np.array([[0, 0, 0], [0, 0, 0], [0, 0, 0]], dtype=np.float32) + results = all_clients(ctx).train(r, current_model) + for n, v in results.items(): + print(f"[{self.name}] round {r}: got group result from client {n}: {v}") + total += v + return total / len(results) + + +class _AggrResult: + + def __init__(self): + self.total = np.array([[0, 0, 0], [0, 0, 0], [0, 0, 0]], dtype=np.float32) + self.count = 0 + + +class NPFedAvgInTime(Controller): + + def __init__(self, initial_model, num_rounds=10, timeout=2.0): + self.num_rounds = num_rounds + self.initial_model = initial_model + self.timeout = timeout + self.name = "NPFedAvgInTime" + + def run(self, context: Context): + print(f"[{self.name}] Start training for {self.num_rounds} rounds") + current_model = self.initial_model + for i in range(self.num_rounds): + current_model = self._do_one_round(i, current_model, context) + score = self._do_eval(current_model, context) + print(f"[{self.name}]: eval score in round {i}: {score}") + + def _do_eval(self, model, ctx: Context): + results = all_clients(ctx).evaluate(model) + total = 0.0 + for n, v in results.items(): + print(f"[{self.name}]: got eval result from client {n}: {v}") + total += v + return total / len(results) + + def _do_one_round(self, r, current_model, ctx: Context): + aggr_result = _AggrResult() + all_clients( + ctx, + process_resp_cb=self._accept_train_result, + aggr_result=aggr_result, + ).train(r, current_model) + + print(f"[{self.name}] round {r}: aggr result from {aggr_result.count} clients: {aggr_result.total}") + if aggr_result.count == 0: + return None + else: + return aggr_result.total / aggr_result.count + + def _accept_train_result(self, result, aggr_result: _AggrResult, context: Context): + print(f"[{context.callee}] got train result from {context.caller} {result}") + aggr_result.total += result + aggr_result.count += 1 + return None + + +class NPCyclic(Controller): + + def __init__(self, initial_model, num_rounds=10): + self.num_rounds = num_rounds + self.initial_model = initial_model + + def run(self, context: Context): + current_model = self.initial_model + for current_round in range(self.num_rounds): + current_model = self._do_one_round(current_round, current_model, context) + print(f"final result: {current_model}") + + def _do_one_round(self, current_round, current_model, ctx: Context): + random.shuffle(ctx.clients) + for c in ctx.clients: + current_model = c.train(current_round, current_model) + print(f"result from {c.name}: {current_model}") + return current_model diff --git a/nvflare/free/examples/np/cyclic.py b/nvflare/free/examples/np/cyclic.py index dc10390869..be130595f4 100644 --- a/nvflare/free/examples/np/cyclic.py +++ b/nvflare/free/examples/np/cyclic.py @@ -1,14 +1,34 @@ -from nvflare.free.examples.np.server import NPCyclic -from nvflare.free.examples.np.client import NPTrainer +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import numpy as np + +from nvflare.free.api.app import ServerApp from nvflare.free.api.runner import AppRunner +from nvflare.free.examples.np.client import NPTrainer +from nvflare.free.examples.np.controllers import NPCyclic def main(): runner = AppRunner( - server_app=NPCyclic(num_rounds=2), + server_app=ServerApp( + controller=NPCyclic( + initial_model=np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32), num_rounds=2 + ) + ), client_app=NPTrainer(delta=1.0), - num_clients=2 + num_clients=2, ) runner.run() diff --git a/nvflare/free/examples/np/fed_avg_intime.py b/nvflare/free/examples/np/fed_avg_intime.py new file mode 100644 index 0000000000..ef240d32c6 --- /dev/null +++ b/nvflare/free/examples/np/fed_avg_intime.py @@ -0,0 +1,42 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import numpy as np + +from nvflare.free.api.app import ServerApp +from nvflare.free.api.runner import AppRunner +from nvflare.free.examples.np.client import NPTrainer +from nvflare.free.examples.np.controllers import NPFedAvgInTime +from nvflare.free.examples.np.widgets import MetricReceiver + + +def main(): + + server_app = ServerApp( + controller=NPFedAvgInTime( + initial_model=np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32), num_rounds=2 + ) + ) + server_app.add_target_object("metric_receiver", MetricReceiver()) + + runner = AppRunner( + server_app=server_app, + client_app=NPTrainer(delta=1.0), + num_clients=2, + ) + + runner.run() + + +if __name__ == "__main__": + main() diff --git a/nvflare/free/examples/np/fed_avg_para.py b/nvflare/free/examples/np/fed_avg_para.py index 875b641497..e1caaf603d 100644 --- a/nvflare/free/examples/np/fed_avg_para.py +++ b/nvflare/free/examples/np/fed_avg_para.py @@ -1,18 +1,32 @@ -from nvflare.free.examples.np.server import NPFedAvgParallel -from nvflare.free.examples.np.client import NPTrainer +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import numpy as np + +from nvflare.free.api.app import ServerApp from nvflare.free.api.runner import AppRunner -from nvflare.free.api.ctx import Context - - -class MetricReceiver: - - def accept_metric(self, metrics: dict, context: Context): - print(f"[{context.callee}] received metric report from {context.caller}: {metrics}") +from nvflare.free.examples.np.client import NPTrainer +from nvflare.free.examples.np.controllers import NPFedAvgParallel +from nvflare.free.examples.np.widgets import MetricReceiver def main(): - server_app = NPFedAvgParallel(num_rounds=2) + server_app = ServerApp( + controller=NPFedAvgParallel( + initial_model=np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32), num_rounds=2 + ) + ) server_app.add_target_object("metric_receiver", MetricReceiver()) runner = AppRunner( diff --git a/nvflare/free/examples/np/fed_avg_seq.py b/nvflare/free/examples/np/fed_avg_seq.py index ad84d2515f..b49415eb1b 100644 --- a/nvflare/free/examples/np/fed_avg_seq.py +++ b/nvflare/free/examples/np/fed_avg_seq.py @@ -1,23 +1,38 @@ -from nvflare.free.examples.np.server import NPFedAvgSequential -from nvflare.free.examples.np.client import NPTrainer +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import numpy as np + +from nvflare.free.api.app import ServerApp from nvflare.free.api.runner import AppRunner -from nvflare.free.api.ctx import Context - - -class MetricReceiver: - - def accept_metric(self, metrics: dict, context: Context): - print(f"[{context.callee}] received metric report from {context.caller}: {metrics}") +from nvflare.free.examples.np.client import TrainerFactory +from nvflare.free.examples.np.controllers import NPFedAvgSequential +from nvflare.free.examples.np.widgets import MetricReceiver def main(): - server_app = NPFedAvgSequential(num_rounds=2) + server_app = ServerApp( + controller=NPFedAvgSequential( + num_rounds=2, + initial_model=np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32), + ) + ) server_app.add_target_object("metric_receiver", MetricReceiver()) runner = AppRunner( server_app=server_app, - client_app=NPTrainer(delta=1.0), + client_app=TrainerFactory(delta=1.0), num_clients=2, ) diff --git a/nvflare/free/examples/np/server.py b/nvflare/free/examples/np/server.py deleted file mode 100644 index 12783bfa73..0000000000 --- a/nvflare/free/examples/np/server.py +++ /dev/null @@ -1,81 +0,0 @@ -import numpy as np -import random - -from nvflare.free.api.app import ServerApp - - -class NPFedAvgSequential(ServerApp): - - def __init__(self, num_rounds=10): - ServerApp.__init__(self) - self.num_rounds = num_rounds - self.initial_model = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32) - - def run(self, **kwargs): - print(f"[{self.name}] Start training for {self.num_rounds} rounds") - current_model = self.initial_model - for i in range(self.num_rounds): - current_model = self._do_one_round(i, current_model) - - def _do_one_round(self, r, current_model): - total = np.array([[0, 0, 0], [0, 0, 0], [0, 0, 0]], dtype=np.float32) - n = 0 - for c in self.clients: - result = c.train(r, current_model) - print(f"[{self.name}] round {r}: got result from client {c.name}: {result}") - total += result - n += 1 - return total / n - - -class NPFedAvgParallel(ServerApp): - - def __init__(self, num_rounds=10): - ServerApp.__init__(self) - self.num_rounds = num_rounds - self.initial_model = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32) - - def run(self, **kwargs): - print(f"[{self.name}] Start training for {self.num_rounds} rounds") - current_model = self.initial_model - for i in range(self.num_rounds): - current_model = self._do_one_round(i, current_model) - score = self._do_eval(current_model) - print(f"[{self.name}]: eval score in round {i}: {score}") - - def _do_eval(self, model): - results = self.group(self.clients).evaluate(model) - total= 0.0 - for n, v in results.items(): - print(f"[{self.name}]: got eval result from client {n}: {v}") - total += v - return total / len(results) - - def _do_one_round(self, r, current_model): - total = np.array([[0, 0, 0], [0, 0, 0], [0, 0, 0]], dtype=np.float32) - results = self.group(self.clients).train(r, current_model) - for n, v in results.items(): - print(f"[{self.name}] round {r}: got group result from client {n}: {v}") - total += v - return total / len(results) - - -class NPCyclic(ServerApp): - - def __init__(self, num_rounds=10): - ServerApp.__init__(self) - self.num_rounds = num_rounds - self.initial_model = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32) - - def run(self): - current_model = self.initial_model - for current_round in range(self.num_rounds): - current_model = self._do_one_round(current_round, current_model) - print(f"final result: {current_model}") - - def _do_one_round(self, current_round, current_model): - random.shuffle(self.clients) - for c in self.clients: - current_model = c.train(current_round, current_model) - print(f"result from {c.name}: {current_model}") - return current_model diff --git a/nvflare/free/examples/np/swarm.py b/nvflare/free/examples/np/swarm.py index 3fc82c28e3..695d6dbdc9 100644 --- a/nvflare/free/examples/np/swarm.py +++ b/nvflare/free/examples/np/swarm.py @@ -1,29 +1,43 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import random import threading import numpy as np -from nvflare.free.api.app import ServerApp, ClientApp +from nvflare.free.api.app import ClientApp, ServerApp +from nvflare.free.api.controller import Controller from nvflare.free.api.ctx import Context +from nvflare.free.api.group import all_clients from nvflare.free.api.runner import AppRunner -class NPSwarmServer(ServerApp): +class NPSwarm(Controller): - def __init__(self, num_rounds=10): - ServerApp.__init__(self) + def __init__(self, initial_model, num_rounds=10): self.num_rounds = num_rounds - self.initial_model = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32) + self.initial_model = initial_model self.waiter = threading.Event() - def run(self, **kwargs): + def run(self, context: Context): # randomly pick a client to start - start_client_idx = random.randint(0, len(self.clients)-1) - start_client = self.clients[start_client_idx] + start_client_idx = random.randint(0, len(context.clients) - 1) + start_client = context.clients[start_client_idx] start_client.start(self.num_rounds, self.initial_model) self.waiter.wait() - def notify_done(self, context: Context, **kwargs): + def notify_done(self, context: Context): print(f"[{context.callee}]: received DONE from client: {context.caller}") self.waiter.set() @@ -34,11 +48,12 @@ def __init__(self, delta: float): super().__init__() self.delta = delta - def train(self, weights): + def train(self, weights, current_round, context: Context): + print(f"[{context.callee}]: train asked by {context.caller}: {current_round=}") return weights + self.delta - def sag(self, model): - results = self.group(self.clients, blocking=True).train(model) + def sag(self, model, current_round, ctx: Context): + results = all_clients(ctx, blocking=True).train(model, current_round) results = list(results.values()) total = results[0] for i in range(1, len(results)): @@ -47,17 +62,17 @@ def sag(self, model): def swarm_learn(self, num_rounds, model, current_round, context: Context): print(f"[{context.callee}]: swarm learn asked by {context.caller}: {num_rounds=} {current_round=} {model=}") - new_model = self.sag(model) + new_model = self.sag(model, current_round, context) print(f"[{context.callee}]: trained model {new_model=}") if current_round == num_rounds - 1: # all done - self.group(self.clients).accept_final_model(new_model, blocking=False) + all_clients(context).accept_final_model(new_model, blocking=False) self.server.notify_done() return # determine next client - next_round = current_round+1 + next_round = current_round + 1 next_client_idx = random.randint(0, len(self.clients) - 1) next_client = self.clients[next_client_idx] next_client.swarm_learn(num_rounds, new_model, next_round, blocking=False) @@ -73,12 +88,14 @@ def accept_final_model(self, model, context: Context): def main(): - server_app = NPSwarmServer(num_rounds=5) - runner = AppRunner( - server_app=server_app, + server_app=ServerApp( + controller=NPSwarm( + initial_model=np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32), num_rounds=5 + ) + ), client_app=NPSwarmClient(delta=1.0), - num_clients=3 + num_clients=3, ) runner.run() diff --git a/nvflare/free/examples/np/widgets.py b/nvflare/free/examples/np/widgets.py new file mode 100644 index 0000000000..db8921074f --- /dev/null +++ b/nvflare/free/examples/np/widgets.py @@ -0,0 +1,20 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nvflare.free.api.ctx import Context + + +class MetricReceiver: + + def accept_metric(self, metrics: dict, context: Context): + print(f"[{context.callee}] received metric report from {context.caller}: {metrics}") From 56a3cfaf65851b2edc003dce1abeb52bc8ade5c0 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Tue, 23 Sep 2025 12:29:49 -0400 Subject: [PATCH 005/102] refactor --- nvflare/free/api/app.py | 8 +++++ nvflare/free/api/constants.py | 4 +++ nvflare/free/api/ctx.py | 5 +++- nvflare/free/api/group.py | 26 +++++++++++++---- nvflare/free/api/proxy.py | 3 +- nvflare/free/api/resp.py | 4 +-- nvflare/free/api/runner.py | 12 +++++--- nvflare/free/api/sim_backend.py | 4 +-- nvflare/free/api/utils.py | 9 ++++-- nvflare/free/examples/np/controllers.py | 13 ++++++--- nvflare/free/examples/np/cyclic_avg.py | 39 +++++++++++++++++++++++++ 11 files changed, 105 insertions(+), 22 deletions(-) create mode 100644 nvflare/free/examples/np/cyclic_avg.py diff --git a/nvflare/free/api/app.py b/nvflare/free/api/app.py index 9b7b3c132c..57344fc788 100644 --- a/nvflare/free/api/app.py +++ b/nvflare/free/api/app.py @@ -15,6 +15,7 @@ from typing import List from .controller import Controller +from .ctx import Context from .proxy import Proxy SERVER_NAME = "server" @@ -67,6 +68,13 @@ def get_my_site(self) -> Proxy: def initialize(self, **kwargs): pass + def new_context(self, caller: str, callee: str, props: dict = None): + ctx = Context(caller, callee, self._abort_signal, props) + ctx.app = self + ctx.server = self.server + ctx.clients = self.clients + return ctx + class ServerApp(App): diff --git a/nvflare/free/api/constants.py b/nvflare/free/api/constants.py index bc472f7454..bd40df886a 100644 --- a/nvflare/free/api/constants.py +++ b/nvflare/free/api/constants.py @@ -19,3 +19,7 @@ class CollabMethodArgName: class CollabMethodOptionName: BLOCKING = "blocking" TIMEOUT = "timeout" + + +class ContextKey: + INPUT = "input" diff --git a/nvflare/free/api/ctx.py b/nvflare/free/api/ctx.py index e652b3b3c2..1332177b37 100644 --- a/nvflare/free/api/ctx.py +++ b/nvflare/free/api/ctx.py @@ -16,13 +16,16 @@ class Context: - def __init__(self, caller: str, callee: str, abort_signal: Signal): + def __init__(self, caller: str, callee: str, abort_signal: Signal, props: dict = None): self.caller = caller self.callee = callee self.abort_signal = abort_signal self.server = None self.clients = None + self.app = None self.props = {} + if props: + self.props.update(props) def set_prop(self, name: str, value): self.props[name] = value diff --git a/nvflare/free/api/group.py b/nvflare/free/api/group.py index b675bab349..639e51fe6d 100644 --- a/nvflare/free/api/group.py +++ b/nvflare/free/api/group.py @@ -28,6 +28,7 @@ class Group: def __init__( self, + app, abort_signal: Signal, proxies: List[Proxy], blocking: bool = True, @@ -37,6 +38,7 @@ def __init__( process_resp_cb=None, **cb_kwargs, ): + self._app = app self._abort_signal = abort_signal self._proxies = proxies self._blocking = blocking @@ -61,10 +63,8 @@ def method(*args, **kwargs): resps = {} for p in self._proxies: kwargs_copy = copy.copy(kwargs) - - ctx = Context(p.caller_name, p.name, self._abort_signal) + ctx = self._app.new_context(p.caller_name, p.name) kwargs_copy[CollabMethodArgName.CONTEXT] = ctx - resp = Resp(self._process_resp_cb, self._cb_kwargs, ctx) resps[p.name] = resp p.backend.call_target_with_resp(resp, p.name, func_name, *args, **kwargs_copy) @@ -129,7 +129,15 @@ def group( **cb_kwargs, ): return Group( - ctx.abort_signal, proxies, blocking, timeout, min_resps, wait_after_min_resps, process_resp_cb, **cb_kwargs + ctx.app, + ctx.abort_signal, + proxies, + blocking, + timeout, + min_resps, + wait_after_min_resps, + process_resp_cb, + **cb_kwargs, ) @@ -143,5 +151,13 @@ def all_clients( **cb_kwargs, ): return Group( - ctx.abort_signal, ctx.clients, blocking, timeout, min_resps, wait_after_min_resps, process_resp_cb, **cb_kwargs + ctx.app, + ctx.abort_signal, + ctx.clients, + blocking, + timeout, + min_resps, + wait_after_min_resps, + process_resp_cb, + **cb_kwargs, ) diff --git a/nvflare/free/api/proxy.py b/nvflare/free/api/proxy.py index da7d961e75..c30fe2af2b 100644 --- a/nvflare/free/api/proxy.py +++ b/nvflare/free/api/proxy.py @@ -13,7 +13,6 @@ # limitations under the License. from .backend import Backend from .constants import CollabMethodArgName -from .ctx import Context class Proxy: @@ -43,7 +42,7 @@ def __getattr__(self, func_name): """ def method(*args, **kwargs): - ctx = Context(self.caller_name, self.name, self.backend.abort_signal) + ctx = self.app.new_context(self.caller_name, self.name) kwargs[CollabMethodArgName.CONTEXT] = ctx return self.backend.call_target(self.target_name, func_name, *args, **kwargs) diff --git a/nvflare/free/api/resp.py b/nvflare/free/api/resp.py index 67b6bcc609..044dc66bfe 100644 --- a/nvflare/free/api/resp.py +++ b/nvflare/free/api/resp.py @@ -16,7 +16,7 @@ from .constants import CollabMethodArgName from .ctx import Context -from .utils import check_optional_args +from .utils import check_context_support class Resp: @@ -38,7 +38,7 @@ def set_result(self, result): ctx.caller = ctx.callee ctx.callee = original_caller self.cb_kwargs[CollabMethodArgName.CONTEXT] = ctx - check_optional_args(self.process_cb, self.cb_kwargs) + check_context_support(self.process_cb, self.cb_kwargs) result = self.process_cb(result, **self.cb_kwargs) self.result = result self.resp_time = time.time() diff --git a/nvflare/free/api/runner.py b/nvflare/free/api/runner.py index df2eca67ac..76be6be2d8 100644 --- a/nvflare/free/api/runner.py +++ b/nvflare/free/api/runner.py @@ -18,6 +18,7 @@ from nvflare.apis.signal import Signal from nvflare.free.api.app import SERVER_NAME, App, ClientApp, ClientAppFactory, ServerApp +from nvflare.free.api.constants import ContextKey from nvflare.free.api.ctx import Context from nvflare.free.api.proxy import Proxy from nvflare.free.api.sim_backend import SimBackend @@ -95,16 +96,19 @@ def __init__( def run(self): # run the server + result = None + ctx = self.server_app.new_context(caller=self.server_app.name, callee=self.server_app.name) + ctx.server = self.server_app.server + ctx.clients = self.server_app.clients for idx, controller in enumerate(self.server_app.controllers): try: print(f"Running Controller #{idx+1}") self.server_app.current_controller = controller - ctx = Context(caller=self.server_app.name, callee=self.server_app.name, abort_signal=self.abort_signal) - ctx.server = self.server_app.server - ctx.clients = self.server_app.clients - controller.run(context=ctx) + result = controller.run(context=ctx) + ctx.set_prop(ContextKey.INPUT, result) except: traceback.print_exc() break self.thread_executor.shutdown(wait=False, cancel_futures=True) + return result diff --git a/nvflare/free/api/sim_backend.py b/nvflare/free/api/sim_backend.py index 672cef849d..ff898aeb6c 100644 --- a/nvflare/free/api/sim_backend.py +++ b/nvflare/free/api/sim_backend.py @@ -17,7 +17,7 @@ from .backend import Backend from .constants import CollabMethodArgName, CollabMethodOptionName from .resp import Resp -from .utils import check_optional_args +from .utils import check_context_support class _Waiter(threading.Event): @@ -86,7 +86,7 @@ def _augment_context(self, func, kwargs): if ctx: ctx.server = self.target_app.server ctx.clients = self.target_app.clients - check_optional_args(func, kwargs) + check_context_support(func, kwargs) def _run_func(self, waiter: _Waiter, func, args, kwargs): try: diff --git a/nvflare/free/api/utils.py b/nvflare/free/api/utils.py index 32dded535d..327da1fe08 100644 --- a/nvflare/free/api/utils.py +++ b/nvflare/free/api/utils.py @@ -12,15 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. import inspect +from typing import List from .constants import CollabMethodArgName -def check_optional_args(func, kwargs): +def check_optional_args(func, kwargs, arg_names: List[str]): signature = inspect.signature(func) parameter_names = signature.parameters.keys() # make sure to expose the optional args if the collab method supports them - for n in [CollabMethodArgName.CONTEXT]: + for n in arg_names: if n not in parameter_names: kwargs.pop(n, None) + + +def check_context_support(func, kwargs): + check_optional_args(func, kwargs, [CollabMethodArgName.CONTEXT]) diff --git a/nvflare/free/examples/np/controllers.py b/nvflare/free/examples/np/controllers.py index f363d9a01d..913cd5e22f 100644 --- a/nvflare/free/examples/np/controllers.py +++ b/nvflare/free/examples/np/controllers.py @@ -15,6 +15,7 @@ import numpy as np +from nvflare.free.api.constants import ContextKey from nvflare.free.api.controller import Controller from nvflare.free.api.ctx import Context from nvflare.free.api.group import all_clients @@ -30,9 +31,10 @@ def __init__(self, initial_model, num_rounds=10): def run(self, context: Context): print(f"[{self.name}] Start training for {self.num_rounds} rounds") - current_model = self.initial_model + current_model = context.get_prop(ContextKey.INPUT, self.initial_model) for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model, context) + return current_model def _do_one_round(self, r, current_model, context: Context): total = np.array([[0, 0, 0], [0, 0, 0], [0, 0, 0]], dtype=np.float32) @@ -54,11 +56,12 @@ def __init__(self, initial_model, num_rounds=10): def run(self, context: Context): print(f"[{self.name}] Start training for {self.num_rounds} rounds") - current_model = self.initial_model + current_model = context.get_prop(ContextKey.INPUT, self.initial_model) for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model, context) score = self._do_eval(current_model, context) print(f"[{self.name}]: eval score in round {i}: {score}") + return current_model def _do_eval(self, model, ctx: Context): results = all_clients(ctx).evaluate(model) @@ -94,11 +97,12 @@ def __init__(self, initial_model, num_rounds=10, timeout=2.0): def run(self, context: Context): print(f"[{self.name}] Start training for {self.num_rounds} rounds") - current_model = self.initial_model + current_model = context.get_prop(ContextKey.INPUT, self.initial_model) for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model, context) score = self._do_eval(current_model, context) print(f"[{self.name}]: eval score in round {i}: {score}") + return current_model def _do_eval(self, model, ctx: Context): results = all_clients(ctx).evaluate(model) @@ -136,10 +140,11 @@ def __init__(self, initial_model, num_rounds=10): self.initial_model = initial_model def run(self, context: Context): - current_model = self.initial_model + current_model = context.get_prop(ContextKey.INPUT, self.initial_model) for current_round in range(self.num_rounds): current_model = self._do_one_round(current_round, current_model, context) print(f"final result: {current_model}") + return current_model def _do_one_round(self, current_round, current_model, ctx: Context): random.shuffle(ctx.clients) diff --git a/nvflare/free/examples/np/cyclic_avg.py b/nvflare/free/examples/np/cyclic_avg.py new file mode 100644 index 0000000000..9b2ba87d8e --- /dev/null +++ b/nvflare/free/examples/np/cyclic_avg.py @@ -0,0 +1,39 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import numpy as np + +from nvflare.free.api.app import ServerApp +from nvflare.free.api.runner import AppRunner +from nvflare.free.examples.np.client import NPTrainer +from nvflare.free.examples.np.controllers import NPCyclic, NPFedAvgParallel + + +def main(): + server_app = ServerApp( + NPCyclic(initial_model=np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32), num_rounds=2) + ) + server_app.add_controller(NPFedAvgParallel(initial_model=None, num_rounds=2)) + + runner = AppRunner( + server_app=server_app, + client_app=NPTrainer(delta=1.0), + num_clients=2, + ) + + final_result = runner.run() + print(f"final model: {final_result}") + + +if __name__ == "__main__": + main() From 01ca6b2ddd633ce0022f9df65fd22fea8698fb79 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Tue, 23 Sep 2025 15:36:47 -0400 Subject: [PATCH 006/102] add event support --- nvflare/free/api/app.py | 41 ++++++++++++++++++++++++++++- nvflare/free/api/runner.py | 19 ++++++++----- nvflare/free/api/sim_backend.py | 4 +-- nvflare/free/examples/np/client.py | 2 ++ nvflare/free/examples/np/swarm.py | 7 ++--- nvflare/free/examples/np/widgets.py | 7 +++++ 6 files changed, 68 insertions(+), 12 deletions(-) diff --git a/nvflare/free/api/app.py b/nvflare/free/api/app.py index 57344fc788..e495a10286 100644 --- a/nvflare/free/api/app.py +++ b/nvflare/free/api/app.py @@ -11,12 +11,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import copy from abc import ABC, abstractmethod from typing import List +from .constants import CollabMethodArgName from .controller import Controller from .ctx import Context from .proxy import Proxy +from .utils import check_context_support SERVER_NAME = "server" @@ -30,6 +33,14 @@ def __init__(self): self._me = None self._target_objs = [] self._abort_signal = None + self._props = {} + self._event_handlers = {} # event type => list of (cb, kwargs) + + def set_prop(self, name: str, value): + self._props[name] = value + + def get_prop(self, name: str, default=None): + return self._props.get(name, default) def get_default_target(self): return None @@ -65,9 +76,21 @@ def setup(self, name: str, server: Proxy, clients: List[Proxy], abort_signal): def get_my_site(self) -> Proxy: return self._me - def initialize(self, **kwargs): + def initialize_app(self, context: Context): pass + def initialize(self, context: Context): + self.initialize_app(context) + + # initialize target objects + for name, obj in self._target_objs: + init_func = getattr(obj, "initialize", None) + if init_func and callable(init_func): + print(f"initializing target object {name}") + kwargs = {CollabMethodArgName.CONTEXT: context} + check_context_support(init_func, kwargs) + init_func(**kwargs) + def new_context(self, caller: str, callee: str, props: dict = None): ctx = Context(caller, callee, self._abort_signal, props) ctx.app = self @@ -75,6 +98,22 @@ def new_context(self, caller: str, callee: str, props: dict = None): ctx.clients = self.clients return ctx + def register_event_handler(self, event_type: str, handler, **handler_kwargs): + handlers = self._event_handlers.get(event_type) + if not handlers: + handlers = [] + self._event_handlers[event_type] = handlers + handlers.append((handler, handler_kwargs)) + + def fire_event(self, event_type: str, data, context: Context): + for e, handlers in self._event_handlers.items(): + if e == event_type: + for h, kwargs in handlers: + kwargs = copy.copy(kwargs) + kwargs.update({CollabMethodArgName.CONTEXT: context}) + check_context_support(h, kwargs) + h(event_type, data, **kwargs) + class ServerApp(App): diff --git a/nvflare/free/api/runner.py b/nvflare/free/api/runner.py index 76be6be2d8..af6355de31 100644 --- a/nvflare/free/api/runner.py +++ b/nvflare/free/api/runner.py @@ -88,24 +88,31 @@ def __init__( for name, app in client_apps.items(): server_proxy, client_proxies = self._prepare_proxies(server_app, client_apps, name, backends) app.setup(name, server_proxy, client_proxies, self.abort_signal) - app.initialize() # prepare server server_proxy, client_proxies = self._prepare_proxies(server_app, client_apps, SERVER_NAME, backends) server_app.setup(SERVER_NAME, server_proxy, client_proxies, self.abort_signal) + self.client_apps = client_apps + def run(self): + # initialize all apps + server_ctx = self.server_app.new_context(caller=self.server_app.name, callee=self.server_app.name) + print("initializing server app") + self.server_app.initialize(server_ctx) + + for n, app in self.client_apps.items(): + print(f"initializing client app for {n}") + app.initialize(app.new_context(n, n)) + # run the server result = None - ctx = self.server_app.new_context(caller=self.server_app.name, callee=self.server_app.name) - ctx.server = self.server_app.server - ctx.clients = self.server_app.clients for idx, controller in enumerate(self.server_app.controllers): try: print(f"Running Controller #{idx+1}") self.server_app.current_controller = controller - result = controller.run(context=ctx) - ctx.set_prop(ContextKey.INPUT, result) + result = controller.run(context=server_ctx) + server_ctx.set_prop(ContextKey.INPUT, result) except: traceback.print_exc() break diff --git a/nvflare/free/api/sim_backend.py b/nvflare/free/api/sim_backend.py index ff898aeb6c..6cc2f3602c 100644 --- a/nvflare/free/api/sim_backend.py +++ b/nvflare/free/api/sim_backend.py @@ -84,8 +84,8 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): def _augment_context(self, func, kwargs): ctx = kwargs.get(CollabMethodArgName.CONTEXT) if ctx: - ctx.server = self.target_app.server - ctx.clients = self.target_app.clients + target_ctx = self.target_app.new_context(ctx.caller, ctx.callee) + kwargs[CollabMethodArgName.CONTEXT] = target_ctx check_context_support(func, kwargs) def _run_func(self, waiter: _Waiter, func, args, kwargs): diff --git a/nvflare/free/examples/np/client.py b/nvflare/free/examples/np/client.py index c70b8db096..10a719f629 100644 --- a/nvflare/free/examples/np/client.py +++ b/nvflare/free/examples/np/client.py @@ -32,6 +32,8 @@ def train(self, r, weights, context: Context): metric_receiver = self.server.get_target("metric_receiver") if metric_receiver: self.server.accept_metric({"round": r, "y": 2}) + + self.server.fire_event("metrics", {"round": r, "y": 10}, blocking=False) return weights + self.delta def evaluate(self, model, context: Context): diff --git a/nvflare/free/examples/np/swarm.py b/nvflare/free/examples/np/swarm.py index 695d6dbdc9..491d99cf69 100644 --- a/nvflare/free/examples/np/swarm.py +++ b/nvflare/free/examples/np/swarm.py @@ -47,6 +47,7 @@ class NPSwarmClient(ClientApp): def __init__(self, delta: float): super().__init__() self.delta = delta + self.register_event_handler("final_model", self._accept_final_model) def train(self, weights, current_round, context: Context): print(f"[{context.callee}]: train asked by {context.caller}: {current_round=}") @@ -67,7 +68,7 @@ def swarm_learn(self, num_rounds, model, current_round, context: Context): print(f"[{context.callee}]: trained model {new_model=}") if current_round == num_rounds - 1: # all done - all_clients(context).accept_final_model(new_model, blocking=False) + all_clients(context, blocking=False).fire_event("final_model", new_model) self.server.notify_done() return @@ -80,10 +81,10 @@ def swarm_learn(self, num_rounds, model, current_round, context: Context): def start(self, num_rounds, initial_model, context: Context): self.swarm_learn(num_rounds, initial_model, 0, context) - def accept_final_model(self, model, context: Context): + def _accept_final_model(self, event_type: str, model, context: Context): # accept the final model # write model to disk - print(f"[{context.callee}]: received final model from {context.caller}: {model}") + print(f"[{context.callee}]: received event '{event_type}' from {context.caller}: {model}") def main(): diff --git a/nvflare/free/examples/np/widgets.py b/nvflare/free/examples/np/widgets.py index db8921074f..5110078eea 100644 --- a/nvflare/free/examples/np/widgets.py +++ b/nvflare/free/examples/np/widgets.py @@ -18,3 +18,10 @@ class MetricReceiver: def accept_metric(self, metrics: dict, context: Context): print(f"[{context.callee}] received metric report from {context.caller}: {metrics}") + + def initialize(self, context: Context): + context.app.register_event_handler("metrics", self._accept_metric) + print("MetricReceiver initialized!") + + def _accept_metric(self, event_type: str, data, context: Context): + print(f"[{context.callee}] received event '{event_type}' from {context.caller}: {data}") From 71609dc4adf3b6ad3c9a40e9fde8cec097a5b06e Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Wed, 24 Sep 2025 12:18:30 -0400 Subject: [PATCH 007/102] rename to focs --- nvflare/{free => focs}/__init__.py | 0 nvflare/{free => focs}/api/__init__.py | 0 nvflare/{free => focs}/api/app.py | 0 nvflare/{free => focs}/api/backend.py | 0 nvflare/{free => focs}/api/constants.py | 0 nvflare/{free => focs}/api/controller.py | 0 nvflare/{free => focs}/api/ctx.py | 0 nvflare/{free => focs}/api/group.py | 0 nvflare/{free => focs}/api/proxy.py | 0 nvflare/{free => focs}/api/resp.py | 0 nvflare/{free => focs}/api/utils.py | 0 nvflare/{free => focs}/examples/__init__.py | 0 nvflare/{free => focs}/examples/np/__init__.py | 0 nvflare/focs/examples/np/algos/__init__.py | 0 .../np => focs/examples/np/algos}/client.py | 4 ++-- .../np => focs/examples/np/algos}/controllers.py | 8 ++++---- .../np => focs/examples/np/algos}/widgets.py | 2 +- nvflare/{free => focs}/examples/np/cyclic.py | 8 ++++---- nvflare/{free => focs}/examples/np/cyclic_avg.py | 8 ++++---- .../{free => focs}/examples/np/fed_avg_intime.py | 10 +++++----- nvflare/{free => focs}/examples/np/fed_avg_para.py | 10 +++++----- nvflare/{free => focs}/examples/np/fed_avg_seq.py | 10 +++++----- nvflare/{free => focs}/examples/np/swarm.py | 10 +++++----- nvflare/focs/sim/__init__.py | 0 .../api/sim_backend.py => focs/sim/backend.py} | 13 +++++++------ nvflare/{free/api => focs/sim}/runner.py | 13 ++++++------- 26 files changed, 48 insertions(+), 48 deletions(-) rename nvflare/{free => focs}/__init__.py (100%) rename nvflare/{free => focs}/api/__init__.py (100%) rename nvflare/{free => focs}/api/app.py (100%) rename nvflare/{free => focs}/api/backend.py (100%) rename nvflare/{free => focs}/api/constants.py (100%) rename nvflare/{free => focs}/api/controller.py (100%) rename nvflare/{free => focs}/api/ctx.py (100%) rename nvflare/{free => focs}/api/group.py (100%) rename nvflare/{free => focs}/api/proxy.py (100%) rename nvflare/{free => focs}/api/resp.py (100%) rename nvflare/{free => focs}/api/utils.py (100%) rename nvflare/{free => focs}/examples/__init__.py (100%) rename nvflare/{free => focs}/examples/np/__init__.py (100%) create mode 100644 nvflare/focs/examples/np/algos/__init__.py rename nvflare/{free/examples/np => focs/examples/np/algos}/client.py (94%) rename nvflare/{free/examples/np => focs/examples/np/algos}/controllers.py (96%) rename nvflare/{free/examples/np => focs/examples/np/algos}/widgets.py (96%) rename nvflare/{free => focs}/examples/np/cyclic.py (82%) rename nvflare/{free => focs}/examples/np/cyclic_avg.py (82%) rename nvflare/{free => focs}/examples/np/fed_avg_intime.py (79%) rename nvflare/{free => focs}/examples/np/fed_avg_para.py (79%) rename nvflare/{free => focs}/examples/np/fed_avg_seq.py (78%) rename nvflare/{free => focs}/examples/np/swarm.py (93%) create mode 100644 nvflare/focs/sim/__init__.py rename nvflare/{free/api/sim_backend.py => focs/sim/backend.py} (90%) rename nvflare/{free/api => focs/sim}/runner.py (91%) diff --git a/nvflare/free/__init__.py b/nvflare/focs/__init__.py similarity index 100% rename from nvflare/free/__init__.py rename to nvflare/focs/__init__.py diff --git a/nvflare/free/api/__init__.py b/nvflare/focs/api/__init__.py similarity index 100% rename from nvflare/free/api/__init__.py rename to nvflare/focs/api/__init__.py diff --git a/nvflare/free/api/app.py b/nvflare/focs/api/app.py similarity index 100% rename from nvflare/free/api/app.py rename to nvflare/focs/api/app.py diff --git a/nvflare/free/api/backend.py b/nvflare/focs/api/backend.py similarity index 100% rename from nvflare/free/api/backend.py rename to nvflare/focs/api/backend.py diff --git a/nvflare/free/api/constants.py b/nvflare/focs/api/constants.py similarity index 100% rename from nvflare/free/api/constants.py rename to nvflare/focs/api/constants.py diff --git a/nvflare/free/api/controller.py b/nvflare/focs/api/controller.py similarity index 100% rename from nvflare/free/api/controller.py rename to nvflare/focs/api/controller.py diff --git a/nvflare/free/api/ctx.py b/nvflare/focs/api/ctx.py similarity index 100% rename from nvflare/free/api/ctx.py rename to nvflare/focs/api/ctx.py diff --git a/nvflare/free/api/group.py b/nvflare/focs/api/group.py similarity index 100% rename from nvflare/free/api/group.py rename to nvflare/focs/api/group.py diff --git a/nvflare/free/api/proxy.py b/nvflare/focs/api/proxy.py similarity index 100% rename from nvflare/free/api/proxy.py rename to nvflare/focs/api/proxy.py diff --git a/nvflare/free/api/resp.py b/nvflare/focs/api/resp.py similarity index 100% rename from nvflare/free/api/resp.py rename to nvflare/focs/api/resp.py diff --git a/nvflare/free/api/utils.py b/nvflare/focs/api/utils.py similarity index 100% rename from nvflare/free/api/utils.py rename to nvflare/focs/api/utils.py diff --git a/nvflare/free/examples/__init__.py b/nvflare/focs/examples/__init__.py similarity index 100% rename from nvflare/free/examples/__init__.py rename to nvflare/focs/examples/__init__.py diff --git a/nvflare/free/examples/np/__init__.py b/nvflare/focs/examples/np/__init__.py similarity index 100% rename from nvflare/free/examples/np/__init__.py rename to nvflare/focs/examples/np/__init__.py diff --git a/nvflare/focs/examples/np/algos/__init__.py b/nvflare/focs/examples/np/algos/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/nvflare/free/examples/np/client.py b/nvflare/focs/examples/np/algos/client.py similarity index 94% rename from nvflare/free/examples/np/client.py rename to nvflare/focs/examples/np/algos/client.py index 10a719f629..3b5f8703f7 100644 --- a/nvflare/free/examples/np/client.py +++ b/nvflare/focs/examples/np/algos/client.py @@ -13,8 +13,8 @@ # limitations under the License. import random -from nvflare.free.api.app import ClientApp, ClientAppFactory -from nvflare.free.api.ctx import Context +from nvflare.focs.api.app import ClientApp, ClientAppFactory +from nvflare.focs.api.ctx import Context class NPTrainer(ClientApp): diff --git a/nvflare/free/examples/np/controllers.py b/nvflare/focs/examples/np/algos/controllers.py similarity index 96% rename from nvflare/free/examples/np/controllers.py rename to nvflare/focs/examples/np/algos/controllers.py index 913cd5e22f..04ee45ef18 100644 --- a/nvflare/free/examples/np/controllers.py +++ b/nvflare/focs/examples/np/algos/controllers.py @@ -15,10 +15,10 @@ import numpy as np -from nvflare.free.api.constants import ContextKey -from nvflare.free.api.controller import Controller -from nvflare.free.api.ctx import Context -from nvflare.free.api.group import all_clients +from nvflare.focs.api.constants import ContextKey +from nvflare.focs.api.controller import Controller +from nvflare.focs.api.ctx import Context +from nvflare.focs.api.group import all_clients class NPFedAvgSequential(Controller): diff --git a/nvflare/free/examples/np/widgets.py b/nvflare/focs/examples/np/algos/widgets.py similarity index 96% rename from nvflare/free/examples/np/widgets.py rename to nvflare/focs/examples/np/algos/widgets.py index 5110078eea..1abe794326 100644 --- a/nvflare/free/examples/np/widgets.py +++ b/nvflare/focs/examples/np/algos/widgets.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from nvflare.free.api.ctx import Context +from nvflare.focs.api.ctx import Context class MetricReceiver: diff --git a/nvflare/free/examples/np/cyclic.py b/nvflare/focs/examples/np/cyclic.py similarity index 82% rename from nvflare/free/examples/np/cyclic.py rename to nvflare/focs/examples/np/cyclic.py index be130595f4..d640c269ee 100644 --- a/nvflare/free/examples/np/cyclic.py +++ b/nvflare/focs/examples/np/cyclic.py @@ -13,10 +13,10 @@ # limitations under the License. import numpy as np -from nvflare.free.api.app import ServerApp -from nvflare.free.api.runner import AppRunner -from nvflare.free.examples.np.client import NPTrainer -from nvflare.free.examples.np.controllers import NPCyclic +from nvflare.focs.api.app import ServerApp +from nvflare.focs.examples.np.algos.client import NPTrainer +from nvflare.focs.examples.np.algos.controllers import NPCyclic +from nvflare.focs.sim.runner import AppRunner def main(): diff --git a/nvflare/free/examples/np/cyclic_avg.py b/nvflare/focs/examples/np/cyclic_avg.py similarity index 82% rename from nvflare/free/examples/np/cyclic_avg.py rename to nvflare/focs/examples/np/cyclic_avg.py index 9b2ba87d8e..f6de8f2487 100644 --- a/nvflare/free/examples/np/cyclic_avg.py +++ b/nvflare/focs/examples/np/cyclic_avg.py @@ -13,10 +13,10 @@ # limitations under the License. import numpy as np -from nvflare.free.api.app import ServerApp -from nvflare.free.api.runner import AppRunner -from nvflare.free.examples.np.client import NPTrainer -from nvflare.free.examples.np.controllers import NPCyclic, NPFedAvgParallel +from nvflare.focs.api.app import ServerApp +from nvflare.focs.examples.np.algos.client import NPTrainer +from nvflare.focs.examples.np.algos.controllers import NPCyclic, NPFedAvgParallel +from nvflare.focs.sim.runner import AppRunner def main(): diff --git a/nvflare/free/examples/np/fed_avg_intime.py b/nvflare/focs/examples/np/fed_avg_intime.py similarity index 79% rename from nvflare/free/examples/np/fed_avg_intime.py rename to nvflare/focs/examples/np/fed_avg_intime.py index ef240d32c6..7c1c630e85 100644 --- a/nvflare/free/examples/np/fed_avg_intime.py +++ b/nvflare/focs/examples/np/fed_avg_intime.py @@ -13,11 +13,11 @@ # limitations under the License. import numpy as np -from nvflare.free.api.app import ServerApp -from nvflare.free.api.runner import AppRunner -from nvflare.free.examples.np.client import NPTrainer -from nvflare.free.examples.np.controllers import NPFedAvgInTime -from nvflare.free.examples.np.widgets import MetricReceiver +from nvflare.focs.api.app import ServerApp +from nvflare.focs.examples.np.algos.client import NPTrainer +from nvflare.focs.examples.np.algos.controllers import NPFedAvgInTime +from nvflare.focs.examples.np.algos.widgets import MetricReceiver +from nvflare.focs.sim.runner import AppRunner def main(): diff --git a/nvflare/free/examples/np/fed_avg_para.py b/nvflare/focs/examples/np/fed_avg_para.py similarity index 79% rename from nvflare/free/examples/np/fed_avg_para.py rename to nvflare/focs/examples/np/fed_avg_para.py index e1caaf603d..03b60fabbe 100644 --- a/nvflare/free/examples/np/fed_avg_para.py +++ b/nvflare/focs/examples/np/fed_avg_para.py @@ -13,11 +13,11 @@ # limitations under the License. import numpy as np -from nvflare.free.api.app import ServerApp -from nvflare.free.api.runner import AppRunner -from nvflare.free.examples.np.client import NPTrainer -from nvflare.free.examples.np.controllers import NPFedAvgParallel -from nvflare.free.examples.np.widgets import MetricReceiver +from nvflare.focs.api.app import ServerApp +from nvflare.focs.examples.np.algos.client import NPTrainer +from nvflare.focs.examples.np.algos.controllers import NPFedAvgParallel +from nvflare.focs.examples.np.algos.widgets import MetricReceiver +from nvflare.focs.sim.runner import AppRunner def main(): diff --git a/nvflare/free/examples/np/fed_avg_seq.py b/nvflare/focs/examples/np/fed_avg_seq.py similarity index 78% rename from nvflare/free/examples/np/fed_avg_seq.py rename to nvflare/focs/examples/np/fed_avg_seq.py index b49415eb1b..3e899269b8 100644 --- a/nvflare/free/examples/np/fed_avg_seq.py +++ b/nvflare/focs/examples/np/fed_avg_seq.py @@ -13,11 +13,11 @@ # limitations under the License. import numpy as np -from nvflare.free.api.app import ServerApp -from nvflare.free.api.runner import AppRunner -from nvflare.free.examples.np.client import TrainerFactory -from nvflare.free.examples.np.controllers import NPFedAvgSequential -from nvflare.free.examples.np.widgets import MetricReceiver +from nvflare.focs.api.app import ServerApp +from nvflare.focs.examples.np.algos.client import TrainerFactory +from nvflare.focs.examples.np.algos.controllers import NPFedAvgSequential +from nvflare.focs.examples.np.algos.widgets import MetricReceiver +from nvflare.focs.sim.runner import AppRunner def main(): diff --git a/nvflare/free/examples/np/swarm.py b/nvflare/focs/examples/np/swarm.py similarity index 93% rename from nvflare/free/examples/np/swarm.py rename to nvflare/focs/examples/np/swarm.py index 491d99cf69..3d0d00c96e 100644 --- a/nvflare/free/examples/np/swarm.py +++ b/nvflare/focs/examples/np/swarm.py @@ -16,11 +16,11 @@ import numpy as np -from nvflare.free.api.app import ClientApp, ServerApp -from nvflare.free.api.controller import Controller -from nvflare.free.api.ctx import Context -from nvflare.free.api.group import all_clients -from nvflare.free.api.runner import AppRunner +from nvflare.focs.api.app import ClientApp, ServerApp +from nvflare.focs.api.controller import Controller +from nvflare.focs.api.ctx import Context +from nvflare.focs.api.group import all_clients +from nvflare.focs.sim.runner import AppRunner class NPSwarm(Controller): diff --git a/nvflare/focs/sim/__init__.py b/nvflare/focs/sim/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/nvflare/free/api/sim_backend.py b/nvflare/focs/sim/backend.py similarity index 90% rename from nvflare/free/api/sim_backend.py rename to nvflare/focs/sim/backend.py index 6cc2f3602c..9c16347633 100644 --- a/nvflare/free/api/sim_backend.py +++ b/nvflare/focs/sim/backend.py @@ -13,11 +13,11 @@ # limitations under the License. import threading -from .app import App -from .backend import Backend -from .constants import CollabMethodArgName, CollabMethodOptionName -from .resp import Resp -from .utils import check_context_support +from nvflare.focs.api.app import App +from nvflare.focs.api.backend import Backend +from nvflare.focs.api.constants import CollabMethodArgName, CollabMethodOptionName +from nvflare.focs.api.resp import Resp +from nvflare.focs.api.utils import check_context_support class _Waiter(threading.Event): @@ -30,9 +30,10 @@ def __init__(self): class SimBackend(Backend): - def __init__(self, target_app: App, target_obj, abort_signal, thread_executor): + def __init__(self, target_app: App, target_name: str, target_obj, abort_signal, thread_executor): Backend.__init__(self, abort_signal) self.target_app = target_app + self.target_name = target_name self.target_obj = target_obj self.executor = thread_executor diff --git a/nvflare/free/api/runner.py b/nvflare/focs/sim/runner.py similarity index 91% rename from nvflare/free/api/runner.py rename to nvflare/focs/sim/runner.py index af6355de31..02213bd6c5 100644 --- a/nvflare/free/api/runner.py +++ b/nvflare/focs/sim/runner.py @@ -17,21 +17,20 @@ from typing import Union from nvflare.apis.signal import Signal -from nvflare.free.api.app import SERVER_NAME, App, ClientApp, ClientAppFactory, ServerApp -from nvflare.free.api.constants import ContextKey -from nvflare.free.api.ctx import Context -from nvflare.free.api.proxy import Proxy -from nvflare.free.api.sim_backend import SimBackend +from nvflare.focs.api.app import SERVER_NAME, App, ClientApp, ClientAppFactory, ServerApp +from nvflare.focs.api.constants import ContextKey +from nvflare.focs.api.proxy import Proxy +from nvflare.focs.sim.backend import SimBackend class AppRunner: def _prepare_app_backends(self, app: App): - bes = {"": SimBackend(app, app, self.abort_signal, self.thread_executor)} + bes = {"": SimBackend(app, "", app, self.abort_signal, self.thread_executor)} targets = app.get_target_objects() if targets: for name, obj in targets: - bes[name] = SimBackend(app, obj, self.abort_signal, self.thread_executor) + bes[name] = SimBackend(app, name, obj, self.abort_signal, self.thread_executor) return bes def _prepare_app_proxy(self, app_name: str, app: App, caller_name: str, app_backends: dict): From 6db8b8f67b3df135820148d7c2a216be15851306 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Wed, 24 Sep 2025 12:20:09 -0400 Subject: [PATCH 008/102] add license text --- nvflare/focs/examples/np/algos/__init__.py | 13 +++++++++++++ nvflare/focs/sim/__init__.py | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/nvflare/focs/examples/np/algos/__init__.py b/nvflare/focs/examples/np/algos/__init__.py index e69de29bb2..341a77c5bc 100644 --- a/nvflare/focs/examples/np/algos/__init__.py +++ b/nvflare/focs/examples/np/algos/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nvflare/focs/sim/__init__.py b/nvflare/focs/sim/__init__.py index e69de29bb2..341a77c5bc 100644 --- a/nvflare/focs/sim/__init__.py +++ b/nvflare/focs/sim/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. From c244740718580e722e2157e827fbec6dd39944f1 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Wed, 24 Sep 2025 12:34:48 -0400 Subject: [PATCH 009/102] rename controller to strategy --- nvflare/focs/api/app.py | 22 +++++++++---------- .../focs/api/{controller.py => strategy.py} | 4 ++-- .../algos/{controllers.py => strategies.py} | 20 ++++++++--------- nvflare/focs/examples/np/cyclic.py | 6 ++--- nvflare/focs/examples/np/cyclic_avg.py | 4 ++-- nvflare/focs/examples/np/fed_avg_intime.py | 4 ++-- nvflare/focs/examples/np/fed_avg_para.py | 4 ++-- nvflare/focs/examples/np/fed_avg_seq.py | 4 ++-- nvflare/focs/examples/np/swarm.py | 10 ++++----- nvflare/focs/sim/runner.py | 8 +++---- nvflare/focs/sys/__init__.py | 0 nvflare/focs/sys/controller.py | 0 12 files changed, 41 insertions(+), 45 deletions(-) rename nvflare/focs/api/{controller.py => strategy.py} (91%) rename nvflare/focs/examples/np/algos/{controllers.py => strategies.py} (93%) create mode 100644 nvflare/focs/sys/__init__.py create mode 100644 nvflare/focs/sys/controller.py diff --git a/nvflare/focs/api/app.py b/nvflare/focs/api/app.py index e495a10286..a8200dea86 100644 --- a/nvflare/focs/api/app.py +++ b/nvflare/focs/api/app.py @@ -16,9 +16,9 @@ from typing import List from .constants import CollabMethodArgName -from .controller import Controller from .ctx import Context from .proxy import Proxy +from .strategy import Strategy from .utils import check_context_support SERVER_NAME = "server" @@ -117,20 +117,20 @@ def fire_event(self, event_type: str, data, context: Context): class ServerApp(App): - def __init__(self, controller: Controller): + def __init__(self, strategy: Strategy): super().__init__() - if not isinstance(controller, Controller): - raise ValueError(f"controller must be Controller but got {type(controller)}") - self.controllers = [controller] - self.current_controller = None + if not isinstance(strategy, Strategy): + raise ValueError(f"strategy must be Strategy but got {type(strategy)}") + self.strategies = [strategy] + self.current_strategy = None - def add_controller(self, controller): - if not isinstance(controller, Controller): - raise ValueError(f"controller must be Controller but got {type(controller)}") - self.controllers.append(controller) + def add_strategy(self, strategy): + if not isinstance(strategy, Strategy): + raise ValueError(f"strategy must be Controller but got {type(strategy)}") + self.strategies.append(strategy) def get_default_target(self): - return self.current_controller + return self.current_strategy class ClientApp(App): diff --git a/nvflare/focs/api/controller.py b/nvflare/focs/api/strategy.py similarity index 91% rename from nvflare/focs/api/controller.py rename to nvflare/focs/api/strategy.py index 3c8e688e5d..c28f4b50ae 100644 --- a/nvflare/focs/api/controller.py +++ b/nvflare/focs/api/strategy.py @@ -16,8 +16,8 @@ from .ctx import Context -class Controller(ABC): +class Strategy(ABC): @abstractmethod - def run(self, context: Context): + def execute(self, context: Context): pass diff --git a/nvflare/focs/examples/np/algos/controllers.py b/nvflare/focs/examples/np/algos/strategies.py similarity index 93% rename from nvflare/focs/examples/np/algos/controllers.py rename to nvflare/focs/examples/np/algos/strategies.py index 04ee45ef18..e06dd5d27a 100644 --- a/nvflare/focs/examples/np/algos/controllers.py +++ b/nvflare/focs/examples/np/algos/strategies.py @@ -16,20 +16,20 @@ import numpy as np from nvflare.focs.api.constants import ContextKey -from nvflare.focs.api.controller import Controller from nvflare.focs.api.ctx import Context from nvflare.focs.api.group import all_clients +from nvflare.focs.api.strategy import Strategy -class NPFedAvgSequential(Controller): +class NPFedAvgSequential(Strategy): def __init__(self, initial_model, num_rounds=10): - Controller.__init__(self) + Strategy.__init__(self) self.name = "NPFedAvgSequential" self.num_rounds = num_rounds self.initial_model = initial_model - def run(self, context: Context): + def execute(self, context: Context): print(f"[{self.name}] Start training for {self.num_rounds} rounds") current_model = context.get_prop(ContextKey.INPUT, self.initial_model) for i in range(self.num_rounds): @@ -47,14 +47,14 @@ def _do_one_round(self, r, current_model, context: Context): return total / n -class NPFedAvgParallel(Controller): +class NPFedAvgParallel(Strategy): def __init__(self, initial_model, num_rounds=10): self.num_rounds = num_rounds self.initial_model = initial_model self.name = "NPFedAvgParallel" - def run(self, context: Context): + def execute(self, context: Context): print(f"[{self.name}] Start training for {self.num_rounds} rounds") current_model = context.get_prop(ContextKey.INPUT, self.initial_model) for i in range(self.num_rounds): @@ -87,7 +87,7 @@ def __init__(self): self.count = 0 -class NPFedAvgInTime(Controller): +class NPFedAvgInTime(Strategy): def __init__(self, initial_model, num_rounds=10, timeout=2.0): self.num_rounds = num_rounds @@ -95,7 +95,7 @@ def __init__(self, initial_model, num_rounds=10, timeout=2.0): self.timeout = timeout self.name = "NPFedAvgInTime" - def run(self, context: Context): + def execute(self, context: Context): print(f"[{self.name}] Start training for {self.num_rounds} rounds") current_model = context.get_prop(ContextKey.INPUT, self.initial_model) for i in range(self.num_rounds): @@ -133,13 +133,13 @@ def _accept_train_result(self, result, aggr_result: _AggrResult, context: Contex return None -class NPCyclic(Controller): +class NPCyclic(Strategy): def __init__(self, initial_model, num_rounds=10): self.num_rounds = num_rounds self.initial_model = initial_model - def run(self, context: Context): + def execute(self, context: Context): current_model = context.get_prop(ContextKey.INPUT, self.initial_model) for current_round in range(self.num_rounds): current_model = self._do_one_round(current_round, current_model, context) diff --git a/nvflare/focs/examples/np/cyclic.py b/nvflare/focs/examples/np/cyclic.py index d640c269ee..ee25625583 100644 --- a/nvflare/focs/examples/np/cyclic.py +++ b/nvflare/focs/examples/np/cyclic.py @@ -15,7 +15,7 @@ from nvflare.focs.api.app import ServerApp from nvflare.focs.examples.np.algos.client import NPTrainer -from nvflare.focs.examples.np.algos.controllers import NPCyclic +from nvflare.focs.examples.np.algos.strategies import NPCyclic from nvflare.focs.sim.runner import AppRunner @@ -23,9 +23,7 @@ def main(): runner = AppRunner( server_app=ServerApp( - controller=NPCyclic( - initial_model=np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32), num_rounds=2 - ) + strategy=NPCyclic(initial_model=np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32), num_rounds=2) ), client_app=NPTrainer(delta=1.0), num_clients=2, diff --git a/nvflare/focs/examples/np/cyclic_avg.py b/nvflare/focs/examples/np/cyclic_avg.py index f6de8f2487..d080ef2ec8 100644 --- a/nvflare/focs/examples/np/cyclic_avg.py +++ b/nvflare/focs/examples/np/cyclic_avg.py @@ -15,7 +15,7 @@ from nvflare.focs.api.app import ServerApp from nvflare.focs.examples.np.algos.client import NPTrainer -from nvflare.focs.examples.np.algos.controllers import NPCyclic, NPFedAvgParallel +from nvflare.focs.examples.np.algos.strategies import NPCyclic, NPFedAvgParallel from nvflare.focs.sim.runner import AppRunner @@ -23,7 +23,7 @@ def main(): server_app = ServerApp( NPCyclic(initial_model=np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32), num_rounds=2) ) - server_app.add_controller(NPFedAvgParallel(initial_model=None, num_rounds=2)) + server_app.add_strategy(NPFedAvgParallel(initial_model=None, num_rounds=2)) runner = AppRunner( server_app=server_app, diff --git a/nvflare/focs/examples/np/fed_avg_intime.py b/nvflare/focs/examples/np/fed_avg_intime.py index 7c1c630e85..a9022c707b 100644 --- a/nvflare/focs/examples/np/fed_avg_intime.py +++ b/nvflare/focs/examples/np/fed_avg_intime.py @@ -15,7 +15,7 @@ from nvflare.focs.api.app import ServerApp from nvflare.focs.examples.np.algos.client import NPTrainer -from nvflare.focs.examples.np.algos.controllers import NPFedAvgInTime +from nvflare.focs.examples.np.algos.strategies import NPFedAvgInTime from nvflare.focs.examples.np.algos.widgets import MetricReceiver from nvflare.focs.sim.runner import AppRunner @@ -23,7 +23,7 @@ def main(): server_app = ServerApp( - controller=NPFedAvgInTime( + strategy=NPFedAvgInTime( initial_model=np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32), num_rounds=2 ) ) diff --git a/nvflare/focs/examples/np/fed_avg_para.py b/nvflare/focs/examples/np/fed_avg_para.py index 03b60fabbe..fc5a8c09a5 100644 --- a/nvflare/focs/examples/np/fed_avg_para.py +++ b/nvflare/focs/examples/np/fed_avg_para.py @@ -15,7 +15,7 @@ from nvflare.focs.api.app import ServerApp from nvflare.focs.examples.np.algos.client import NPTrainer -from nvflare.focs.examples.np.algos.controllers import NPFedAvgParallel +from nvflare.focs.examples.np.algos.strategies import NPFedAvgParallel from nvflare.focs.examples.np.algos.widgets import MetricReceiver from nvflare.focs.sim.runner import AppRunner @@ -23,7 +23,7 @@ def main(): server_app = ServerApp( - controller=NPFedAvgParallel( + strategy=NPFedAvgParallel( initial_model=np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32), num_rounds=2 ) ) diff --git a/nvflare/focs/examples/np/fed_avg_seq.py b/nvflare/focs/examples/np/fed_avg_seq.py index 3e899269b8..8dfa74fd6a 100644 --- a/nvflare/focs/examples/np/fed_avg_seq.py +++ b/nvflare/focs/examples/np/fed_avg_seq.py @@ -15,7 +15,7 @@ from nvflare.focs.api.app import ServerApp from nvflare.focs.examples.np.algos.client import TrainerFactory -from nvflare.focs.examples.np.algos.controllers import NPFedAvgSequential +from nvflare.focs.examples.np.algos.strategies import NPFedAvgSequential from nvflare.focs.examples.np.algos.widgets import MetricReceiver from nvflare.focs.sim.runner import AppRunner @@ -23,7 +23,7 @@ def main(): server_app = ServerApp( - controller=NPFedAvgSequential( + strategy=NPFedAvgSequential( num_rounds=2, initial_model=np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32), ) diff --git a/nvflare/focs/examples/np/swarm.py b/nvflare/focs/examples/np/swarm.py index 3d0d00c96e..f63752e5aa 100644 --- a/nvflare/focs/examples/np/swarm.py +++ b/nvflare/focs/examples/np/swarm.py @@ -17,20 +17,20 @@ import numpy as np from nvflare.focs.api.app import ClientApp, ServerApp -from nvflare.focs.api.controller import Controller from nvflare.focs.api.ctx import Context from nvflare.focs.api.group import all_clients +from nvflare.focs.api.strategy import Strategy from nvflare.focs.sim.runner import AppRunner -class NPSwarm(Controller): +class NPSwarm(Strategy): def __init__(self, initial_model, num_rounds=10): self.num_rounds = num_rounds self.initial_model = initial_model self.waiter = threading.Event() - def run(self, context: Context): + def execute(self, context: Context): # randomly pick a client to start start_client_idx = random.randint(0, len(context.clients) - 1) start_client = context.clients[start_client_idx] @@ -91,9 +91,7 @@ def main(): runner = AppRunner( server_app=ServerApp( - controller=NPSwarm( - initial_model=np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32), num_rounds=5 - ) + strategy=NPSwarm(initial_model=np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32), num_rounds=5) ), client_app=NPSwarmClient(delta=1.0), num_clients=3, diff --git a/nvflare/focs/sim/runner.py b/nvflare/focs/sim/runner.py index 02213bd6c5..ed6f031989 100644 --- a/nvflare/focs/sim/runner.py +++ b/nvflare/focs/sim/runner.py @@ -106,11 +106,11 @@ def run(self): # run the server result = None - for idx, controller in enumerate(self.server_app.controllers): + for idx, strategy in enumerate(self.server_app.strategies): try: - print(f"Running Controller #{idx+1}") - self.server_app.current_controller = controller - result = controller.run(context=server_ctx) + print(f"Running Strategy #{idx+1}") + self.server_app.current_strategy = strategy + result = strategy.execute(context=server_ctx) server_ctx.set_prop(ContextKey.INPUT, result) except: traceback.print_exc() diff --git a/nvflare/focs/sys/__init__.py b/nvflare/focs/sys/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/nvflare/focs/sys/controller.py b/nvflare/focs/sys/controller.py new file mode 100644 index 0000000000..e69de29bb2 From 4a1b8c41ecc7caddba3f77b13aebb81ca04b0888 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Wed, 24 Sep 2025 16:38:00 -0400 Subject: [PATCH 010/102] dev controller --- nvflare/focs/api/app.py | 13 +-- nvflare/focs/api/proxy.py | 1 + nvflare/focs/sim/backend.py | 3 +- nvflare/focs/sim/runner.py | 45 +++++--- nvflare/focs/sys/backend.py | 28 +++++ nvflare/focs/sys/controller.py | 198 +++++++++++++++++++++++++++++++++ 6 files changed, 260 insertions(+), 28 deletions(-) create mode 100644 nvflare/focs/sys/backend.py diff --git a/nvflare/focs/api/app.py b/nvflare/focs/api/app.py index a8200dea86..22aca616ba 100644 --- a/nvflare/focs/api/app.py +++ b/nvflare/focs/api/app.py @@ -21,8 +21,6 @@ from .strategy import Strategy from .utils import check_context_support -SERVER_NAME = "server" - class App(ABC): @@ -55,23 +53,22 @@ def add_target_object(self, name: str, obj): def get_target_objects(self): return self._target_objs - def setup(self, name: str, server: Proxy, clients: List[Proxy], abort_signal): - self.name = name + def setup(self, server: Proxy, clients: List[Proxy], abort_signal): self.server = server self._abort_signal = abort_signal self.clients = clients self._me = None - if not name or name == "server": + if not self.name or self.name == "server": self._me = server else: for c in clients: - if c.name == name: + if c.name == self.name: self._me = c break if not self._me: - raise ValueError(f"cannot find site for {name}") + raise ValueError(f"cannot find site for {self.name}") def get_my_site(self) -> Proxy: return self._me @@ -117,7 +114,7 @@ def fire_event(self, event_type: str, data, context: Context): class ServerApp(App): - def __init__(self, strategy: Strategy): + def __init__(self, strategy: Strategy = None): super().__init__() if not isinstance(strategy, Strategy): raise ValueError(f"strategy must be Strategy but got {type(strategy)}") diff --git a/nvflare/focs/api/proxy.py b/nvflare/focs/api/proxy.py index c30fe2af2b..58c241df5a 100644 --- a/nvflare/focs/api/proxy.py +++ b/nvflare/focs/api/proxy.py @@ -18,6 +18,7 @@ class Proxy: def __init__(self, app, target_name, backend: Backend, caller_name: str): + """The Proxy represents a target in the App.""" self.app = app self.target_name = target_name self.backend = backend diff --git a/nvflare/focs/sim/backend.py b/nvflare/focs/sim/backend.py index 9c16347633..070b55797e 100644 --- a/nvflare/focs/sim/backend.py +++ b/nvflare/focs/sim/backend.py @@ -30,10 +30,9 @@ def __init__(self): class SimBackend(Backend): - def __init__(self, target_app: App, target_name: str, target_obj, abort_signal, thread_executor): + def __init__(self, target_app: App, target_obj, abort_signal, thread_executor): Backend.__init__(self, abort_signal) self.target_app = target_app - self.target_name = target_name self.target_obj = target_obj self.executor = thread_executor diff --git a/nvflare/focs/sim/runner.py b/nvflare/focs/sim/runner.py index ed6f031989..d2d8991894 100644 --- a/nvflare/focs/sim/runner.py +++ b/nvflare/focs/sim/runner.py @@ -17,7 +17,7 @@ from typing import Union from nvflare.apis.signal import Signal -from nvflare.focs.api.app import SERVER_NAME, App, ClientApp, ClientAppFactory, ServerApp +from nvflare.focs.api.app import App, ClientApp, ClientAppFactory, ServerApp from nvflare.focs.api.constants import ContextKey from nvflare.focs.api.proxy import Proxy from nvflare.focs.sim.backend import SimBackend @@ -26,27 +26,32 @@ class AppRunner: def _prepare_app_backends(self, app: App): - bes = {"": SimBackend(app, "", app, self.abort_signal, self.thread_executor)} + bes = {"": SimBackend(app, app, self.abort_signal, self.thread_executor)} targets = app.get_target_objects() if targets: for name, obj in targets: - bes[name] = SimBackend(app, name, obj, self.abort_signal, self.thread_executor) + bes[name] = SimBackend(app, obj, self.abort_signal, self.thread_executor) return bes - def _prepare_app_proxy(self, app_name: str, app: App, caller_name: str, app_backends: dict): - app_proxy = Proxy(app=app, target_name=app_name, backend=app_backends[""], caller_name=caller_name) - cos = app.get_target_objects() - if cos: - for name, obj in cos: - p = Proxy(app=app, target_name=name, backend=app_backends[name], caller_name=caller_name) + def _prepare_proxy(self, for_app: App, target_app: App, backends: dict): + app_proxy = Proxy(app=for_app, target_name=target_app.name, backend=backends[""], caller_name=for_app.name) + tos = target_app.get_target_objects() + if tos: + for name, obj in tos: + p = Proxy( + app=for_app, + target_name=f"{target_app.name}.{name}", + backend=backends[name], + caller_name=for_app.name, + ) setattr(app_proxy, name, p) return app_proxy - def _prepare_proxies(self, server_app: App, client_apps: dict, caller_name, backends: dict): - server_proxy = self._prepare_app_proxy(SERVER_NAME, server_app, caller_name, backends[SERVER_NAME]) + def _prepare_proxies(self, for_app: App, server_app: App, client_apps: dict, backends: dict): + server_proxy = self._prepare_proxy(for_app, server_app, backends[server_app.name]) client_proxies = [] for name, app in client_apps.items(): - p = self._prepare_app_proxy(name, app, caller_name, backends[name]) + p = self._prepare_proxy(for_app, app, backends[name]) client_proxies.append(p) return server_proxy, client_proxies @@ -64,6 +69,7 @@ def __init__( raise ValueError(f"client_app must be ClientApp or ClientAppFactory but got {type(client_app)}") self.abort_signal = Signal() + server_app.name = "server" self.server_app = server_app self.client_app = client_app self.thread_executor = ThreadPoolExecutor(max_workers=max_workers) @@ -79,18 +85,18 @@ def __init__( app.name = name client_apps[name] = app - backends = {SERVER_NAME: self._prepare_app_backends(server_app)} + backends = {server_app.name: self._prepare_app_backends(server_app)} for name, app in client_apps.items(): backends[name] = self._prepare_app_backends(app) for name, app in client_apps.items(): - server_proxy, client_proxies = self._prepare_proxies(server_app, client_apps, name, backends) - app.setup(name, server_proxy, client_proxies, self.abort_signal) + server_proxy, client_proxies = self._prepare_proxies(app, server_app, client_apps, backends) + app.setup(server_proxy, client_proxies, self.abort_signal) # prepare server - server_proxy, client_proxies = self._prepare_proxies(server_app, client_apps, SERVER_NAME, backends) - server_app.setup(SERVER_NAME, server_proxy, client_proxies, self.abort_signal) + server_proxy, client_proxies = self._prepare_proxies(server_app, server_app, client_apps, backends) + server_app.setup(server_proxy, client_proxies, self.abort_signal) self.client_apps = client_apps @@ -105,10 +111,13 @@ def run(self): app.initialize(app.new_context(n, n)) # run the server + if not self.server_app.strategies: + raise RuntimeError("server app does not have any strategies!") + result = None for idx, strategy in enumerate(self.server_app.strategies): try: - print(f"Running Strategy #{idx+1}") + print(f"Running Strategy #{idx+1} - {type(strategy).__name__}") self.server_app.current_strategy = strategy result = strategy.execute(context=server_ctx) server_ctx.set_prop(ContextKey.INPUT, result) diff --git a/nvflare/focs/sys/backend.py b/nvflare/focs/sys/backend.py new file mode 100644 index 0000000000..0d4f12b333 --- /dev/null +++ b/nvflare/focs/sys/backend.py @@ -0,0 +1,28 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nvflare.focs.api.backend import Backend +from nvflare.focs.api.resp import Resp + + +class SysBackend(Backend): + + def __init__(self, target_fqcn, abort_signal): + Backend.__init__(self, abort_signal) + self.target_fqcn = target_fqcn + + def call_target(self, target_name: str, func_name: str, *args, **kwargs): + pass + + def call_target_with_resp(self, resp: Resp, target_name: str, func_name: str, *args, **kwargs): + pass diff --git a/nvflare/focs/sys/controller.py b/nvflare/focs/sys/controller.py index e69de29bb2..24d883f644 100644 --- a/nvflare/focs/sys/controller.py +++ b/nvflare/focs/sys/controller.py @@ -0,0 +1,198 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import time +import traceback +from typing import Dict, List + +from nvflare.apis.client import Client as ClientSite +from nvflare.apis.controller_spec import Client, ClientTask, Task +from nvflare.apis.fl_context import FLContext +from nvflare.apis.impl.controller import Controller +from nvflare.apis.shareable import ReturnCode, Shareable +from nvflare.apis.signal import Signal +from nvflare.focs.api.app import ServerApp +from nvflare.focs.api.constants import ContextKey +from nvflare.focs.api.proxy import Proxy +from nvflare.focs.api.strategy import Strategy +from nvflare.focs.sys.backend import SysBackend + + +class FocsController(Controller): + + def __init__( + self, + server_app_id: str, + strategy_ids: List[str], + server_target_obj_ids: Dict[str, str] = None, + client_target_obj_names: List[str] = None, + sync_task_timeout=2, + ): + Controller.__init__(self) + self.server_app_id = server_app_id # component name + self.strategy_ids = strategy_ids # component names + self.server_target_obj_ids = server_target_obj_ids # component IDs + self.client_target_obj_names = client_target_obj_names # callable names + self.sync_task_timeout = sync_task_timeout + self.server_app = None + self.client_statuses = {} # client name => bool + self.ready = False + + if not strategy_ids: + raise ValueError(f"no strategies defined - there must be at least one strategy") + + def start_controller(self, fl_ctx: FLContext): + engine = fl_ctx.get_engine() + app = engine.get_component(self.server_app_id) + if not isinstance(app, ServerApp): + self.system_panic(f"component {self.server_app_id} must be ServerApp but got {type(app)}", fl_ctx) + return + + app.name = "server" + + for cid in self.strategy_ids: + strategy = engine.get_component(cid) + if not isinstance(strategy, Strategy): + self.system_panic(f"component {cid} must be Strategy but got {type(strategy)}", fl_ctx) + return + + app.add_strategy(strategy) + + if self.server_target_obj_ids: + for name, cid in self.server_target_obj_ids: + obj = engine.get_component(cid) + if not obj: + self.system_panic(f"component {cid} does not exist", fl_ctx) + return + + app.add_target_object(name, obj) + + self.server_app = app + + def _prepare_client_backend(self, client: ClientSite, abort_signal: Signal): + return SysBackend( + target_fqcn=client.get_fqcn(), + abort_signal=abort_signal, + ) + + def _prepare_server_backend(self, abort_signal: Signal): + return SysBackend( + target_fqcn="server", + abort_signal=abort_signal, + ) + + def _prepare_client_proxy(self, client: ClientSite, target_obj_names: List[str], abort_signal): + backend = self._prepare_client_backend(client, abort_signal) + proxy = Proxy(app=self.server_app, target_name=client.name, backend=backend, caller_name=self.server_app.name) + + for name in target_obj_names: + p = Proxy( + app=self.server_app, + target_name=f"{client.name}.{name}", + backend=backend, + caller_name=self.server_app.name, + ) + setattr(proxy, name, p) + return proxy + + def _prepare_server_proxy(self, abort_signal): + server_name = self.server_app.name + backend = self._prepare_server_backend(abort_signal) + proxy = Proxy(app=self.server_app, target_name=server_name, backend=backend, caller_name=server_name) + + tos = self.server_app.get_target_objects() + if tos: + for name in tos.keys(): + p = Proxy( + app=self.server_app, target_name=f"{server_name}.{name}", backend=backend, caller_name=server_name + ) + setattr(proxy, name, p) + return proxy + + def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): + # configure all sites + task = Task( + name="sync", + data=Shareable(), + timeout=self.sync_task_timeout, + result_received_cb=self._process_sync_reply, + ) + + engine = fl_ctx.get_engine() + all_clients = engine.get_clients() + num_clients = len(all_clients) + for c in all_clients: + assert isinstance(c, Client) + self.client_statuses[c.name] = False + + start_time = time.time() + self.broadcast_and_wait( + task=task, + min_responses=num_clients, + abort_signal=abort_signal, + fl_ctx=fl_ctx, + ) + time_taken = time.time() - start_time + self.log_info(fl_ctx, f"client sync took {time_taken} seconds") + + failed_clients = [] + for c, ok in self.client_statuses.items(): + if not ok: + failed_clients.append(c) + + if failed_clients: + self.system_panic( + f"failed to sync clients {failed_clients}", + fl_ctx, + ) + return + + self.log_info(fl_ctx, f"successfully configured clients {self.client_statuses.keys()}") + self.ready = True + + # prepare proxies and backends + server_proxy = self._prepare_server_proxy(abort_signal) + client_proxies = [] + for c in all_clients: + client_proxies.append(self._prepare_client_proxy(c, self.client_target_obj_names, abort_signal)) + + self.server_app.setup(server_proxy, client_proxies, abort_signal) + + server_ctx = self.server_app.new_context(caller=self.server_app.name, callee=self.server_app.name) + print("initializing server app") + self.server_app.initialize(server_ctx) + + for idx, strategy in enumerate(self.server_app.strategies): + if abort_signal.triggered: + break + + try: + print(f"Running Strategy #{idx + 1} - {type(strategy).__name__}") + self.server_app.current_strategy = strategy + result = strategy.execute(context=server_ctx) + server_ctx.set_prop(ContextKey.INPUT, result) + except: + traceback.print_exc() + break + + def _process_sync_reply(self, client_task: ClientTask, fl_ctx: FLContext): + result = client_task.result + client_name = client_task.client.name + + rc = result.get_return_code() + if rc == ReturnCode.OK: + self.log_info(fl_ctx, f"successfully synced client {client_name}") + self.client_statuses[client_name] = True + else: + self.log_error(fl_ctx, f"client {client_task.client.name} failed to sync: {rc}") + self.client_statuses[client_name] = False From 09374640192ad2460140da394b9885de31721a3b Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Thu, 25 Sep 2025 16:24:22 -0400 Subject: [PATCH 011/102] dev sys backend --- nvflare/focs/api/app.py | 20 +++ nvflare/focs/api/group.py | 6 +- nvflare/focs/examples/np/algos/strategies.py | 10 +- nvflare/focs/examples/np/algos/utils.py | 28 ++++ nvflare/focs/examples/np/cyclic.py | 6 +- nvflare/focs/examples/np/cyclic_avg.py | 6 +- nvflare/focs/examples/np/fed_avg_intime.py | 8 +- nvflare/focs/examples/np/fed_avg_para.py | 8 +- nvflare/focs/examples/np/fed_avg_seq.py | 4 +- nvflare/focs/examples/np/swarm.py | 9 +- nvflare/focs/sim/backend.py | 23 +-- nvflare/focs/sys/backend.py | 69 ++++++++- nvflare/focs/sys/constants.py | 35 +++++ nvflare/focs/sys/controller.py | 79 +++++++--- nvflare/focs/sys/executor.py | 144 +++++++++++++++++++ nvflare/focs/sys/utils.py | 88 ++++++++++++ 16 files changed, 466 insertions(+), 77 deletions(-) create mode 100644 nvflare/focs/examples/np/algos/utils.py create mode 100644 nvflare/focs/sys/constants.py create mode 100644 nvflare/focs/sys/executor.py create mode 100644 nvflare/focs/sys/utils.py diff --git a/nvflare/focs/api/app.py b/nvflare/focs/api/app.py index 22aca616ba..58bc5f928d 100644 --- a/nvflare/focs/api/app.py +++ b/nvflare/focs/api/app.py @@ -73,6 +73,26 @@ def setup(self, server: Proxy, clients: List[Proxy], abort_signal): def get_my_site(self) -> Proxy: return self._me + def find_method(self, target_obj, method_name): + m = getattr(target_obj, method_name, None) + if m: + return m + + if isinstance(target_obj, App): + # see whether any targets have this method + default_target = self.get_default_target() + if default_target: + m = getattr(default_target, method_name, None) + if m: + return m + + targets = self.get_target_objects() + for _, obj in targets: + m = getattr(obj, method_name, None) + if m: + return m + return None + def initialize_app(self, context: Context): pass diff --git a/nvflare/focs/api/group.py b/nvflare/focs/api/group.py index 639e51fe6d..711b05a1cf 100644 --- a/nvflare/focs/api/group.py +++ b/nvflare/focs/api/group.py @@ -18,7 +18,7 @@ from nvflare.apis.fl_exception import RunAborted from nvflare.apis.signal import Signal -from .constants import CollabMethodArgName +from .constants import CollabMethodArgName, CollabMethodOptionName from .ctx import Context from .proxy import Proxy from .resp import Resp @@ -65,6 +65,10 @@ def method(*args, **kwargs): kwargs_copy = copy.copy(kwargs) ctx = self._app.new_context(p.caller_name, p.name) kwargs_copy[CollabMethodArgName.CONTEXT] = ctx + + # set the optional args to help backend decide how to call + kwargs_copy[CollabMethodOptionName.TIMEOUT] = self._timeout + kwargs_copy[CollabMethodOptionName.BLOCKING] = self._blocking resp = Resp(self._process_resp_cb, self._cb_kwargs, ctx) resps[p.name] = resp p.backend.call_target_with_resp(resp, p.name, func_name, *args, **kwargs_copy) diff --git a/nvflare/focs/examples/np/algos/strategies.py b/nvflare/focs/examples/np/algos/strategies.py index e06dd5d27a..eda806f895 100644 --- a/nvflare/focs/examples/np/algos/strategies.py +++ b/nvflare/focs/examples/np/algos/strategies.py @@ -20,6 +20,8 @@ from nvflare.focs.api.group import all_clients from nvflare.focs.api.strategy import Strategy +from .utils import parse_array_def + class NPFedAvgSequential(Strategy): @@ -27,7 +29,7 @@ def __init__(self, initial_model, num_rounds=10): Strategy.__init__(self) self.name = "NPFedAvgSequential" self.num_rounds = num_rounds - self.initial_model = initial_model + self.initial_model = parse_array_def(initial_model) def execute(self, context: Context): print(f"[{self.name}] Start training for {self.num_rounds} rounds") @@ -51,7 +53,7 @@ class NPFedAvgParallel(Strategy): def __init__(self, initial_model, num_rounds=10): self.num_rounds = num_rounds - self.initial_model = initial_model + self.initial_model = parse_array_def(initial_model) self.name = "NPFedAvgParallel" def execute(self, context: Context): @@ -91,7 +93,7 @@ class NPFedAvgInTime(Strategy): def __init__(self, initial_model, num_rounds=10, timeout=2.0): self.num_rounds = num_rounds - self.initial_model = initial_model + self.initial_model = parse_array_def(initial_model) self.timeout = timeout self.name = "NPFedAvgInTime" @@ -137,7 +139,7 @@ class NPCyclic(Strategy): def __init__(self, initial_model, num_rounds=10): self.num_rounds = num_rounds - self.initial_model = initial_model + self.initial_model = parse_array_def(initial_model) def execute(self, context: Context): current_model = context.get_prop(ContextKey.INPUT, self.initial_model) diff --git a/nvflare/focs/examples/np/algos/utils.py b/nvflare/focs/examples/np/algos/utils.py new file mode 100644 index 0000000000..04f6b87424 --- /dev/null +++ b/nvflare/focs/examples/np/algos/utils.py @@ -0,0 +1,28 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np + + +def parse_array_def(array_def): + if array_def is None: + return array_def + + if isinstance(array_def, np.ndarray): + return array_def + + if isinstance(array_def, list): + return np.array(array_def, dtype=np.float32) + else: + raise ValueError(f"unsupported array def: {array_def}") diff --git a/nvflare/focs/examples/np/cyclic.py b/nvflare/focs/examples/np/cyclic.py index ee25625583..fa1d670b99 100644 --- a/nvflare/focs/examples/np/cyclic.py +++ b/nvflare/focs/examples/np/cyclic.py @@ -11,8 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import numpy as np - from nvflare.focs.api.app import ServerApp from nvflare.focs.examples.np.algos.client import NPTrainer from nvflare.focs.examples.np.algos.strategies import NPCyclic @@ -22,9 +20,7 @@ def main(): runner = AppRunner( - server_app=ServerApp( - strategy=NPCyclic(initial_model=np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32), num_rounds=2) - ), + server_app=ServerApp(strategy=NPCyclic(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2)), client_app=NPTrainer(delta=1.0), num_clients=2, ) diff --git a/nvflare/focs/examples/np/cyclic_avg.py b/nvflare/focs/examples/np/cyclic_avg.py index d080ef2ec8..a3d62e240e 100644 --- a/nvflare/focs/examples/np/cyclic_avg.py +++ b/nvflare/focs/examples/np/cyclic_avg.py @@ -11,8 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import numpy as np - from nvflare.focs.api.app import ServerApp from nvflare.focs.examples.np.algos.client import NPTrainer from nvflare.focs.examples.np.algos.strategies import NPCyclic, NPFedAvgParallel @@ -20,9 +18,7 @@ def main(): - server_app = ServerApp( - NPCyclic(initial_model=np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32), num_rounds=2) - ) + server_app = ServerApp(NPCyclic(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2)) server_app.add_strategy(NPFedAvgParallel(initial_model=None, num_rounds=2)) runner = AppRunner( diff --git a/nvflare/focs/examples/np/fed_avg_intime.py b/nvflare/focs/examples/np/fed_avg_intime.py index a9022c707b..32772330cd 100644 --- a/nvflare/focs/examples/np/fed_avg_intime.py +++ b/nvflare/focs/examples/np/fed_avg_intime.py @@ -11,8 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import numpy as np - from nvflare.focs.api.app import ServerApp from nvflare.focs.examples.np.algos.client import NPTrainer from nvflare.focs.examples.np.algos.strategies import NPFedAvgInTime @@ -22,11 +20,7 @@ def main(): - server_app = ServerApp( - strategy=NPFedAvgInTime( - initial_model=np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32), num_rounds=2 - ) - ) + server_app = ServerApp(strategy=NPFedAvgInTime(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2)) server_app.add_target_object("metric_receiver", MetricReceiver()) runner = AppRunner( diff --git a/nvflare/focs/examples/np/fed_avg_para.py b/nvflare/focs/examples/np/fed_avg_para.py index fc5a8c09a5..c43e208aeb 100644 --- a/nvflare/focs/examples/np/fed_avg_para.py +++ b/nvflare/focs/examples/np/fed_avg_para.py @@ -11,8 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import numpy as np - from nvflare.focs.api.app import ServerApp from nvflare.focs.examples.np.algos.client import NPTrainer from nvflare.focs.examples.np.algos.strategies import NPFedAvgParallel @@ -22,11 +20,7 @@ def main(): - server_app = ServerApp( - strategy=NPFedAvgParallel( - initial_model=np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32), num_rounds=2 - ) - ) + server_app = ServerApp(strategy=NPFedAvgParallel(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2)) server_app.add_target_object("metric_receiver", MetricReceiver()) runner = AppRunner( diff --git a/nvflare/focs/examples/np/fed_avg_seq.py b/nvflare/focs/examples/np/fed_avg_seq.py index 8dfa74fd6a..3d2498f182 100644 --- a/nvflare/focs/examples/np/fed_avg_seq.py +++ b/nvflare/focs/examples/np/fed_avg_seq.py @@ -11,8 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import numpy as np - from nvflare.focs.api.app import ServerApp from nvflare.focs.examples.np.algos.client import TrainerFactory from nvflare.focs.examples.np.algos.strategies import NPFedAvgSequential @@ -25,7 +23,7 @@ def main(): server_app = ServerApp( strategy=NPFedAvgSequential( num_rounds=2, - initial_model=np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32), + initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], ) ) server_app.add_target_object("metric_receiver", MetricReceiver()) diff --git a/nvflare/focs/examples/np/swarm.py b/nvflare/focs/examples/np/swarm.py index f63752e5aa..bf38ea7c40 100644 --- a/nvflare/focs/examples/np/swarm.py +++ b/nvflare/focs/examples/np/swarm.py @@ -14,12 +14,11 @@ import random import threading -import numpy as np - from nvflare.focs.api.app import ClientApp, ServerApp from nvflare.focs.api.ctx import Context from nvflare.focs.api.group import all_clients from nvflare.focs.api.strategy import Strategy +from nvflare.focs.examples.np.algos.utils import parse_array_def from nvflare.focs.sim.runner import AppRunner @@ -27,7 +26,7 @@ class NPSwarm(Strategy): def __init__(self, initial_model, num_rounds=10): self.num_rounds = num_rounds - self.initial_model = initial_model + self.initial_model = parse_array_def(initial_model) self.waiter = threading.Event() def execute(self, context: Context): @@ -90,9 +89,7 @@ def _accept_final_model(self, event_type: str, model, context: Context): def main(): runner = AppRunner( - server_app=ServerApp( - strategy=NPSwarm(initial_model=np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32), num_rounds=5) - ), + server_app=ServerApp(strategy=NPSwarm(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=5)), client_app=NPSwarmClient(delta=1.0), num_clients=3, ) diff --git a/nvflare/focs/sim/backend.py b/nvflare/focs/sim/backend.py index 070b55797e..9bb9612aba 100644 --- a/nvflare/focs/sim/backend.py +++ b/nvflare/focs/sim/backend.py @@ -37,24 +37,7 @@ def __init__(self, target_app: App, target_obj, abort_signal, thread_executor): self.executor = thread_executor def _get_func(self, func_name): - func = getattr(self.target_obj, func_name, None) - if func: - return func - - if isinstance(self.target_obj, App): - # see whether any targets have this method - default_target = self.target_obj.get_default_target() - if default_target: - func = getattr(default_target, func_name, None) - if func: - return func - - targets = self.target_obj.get_target_objects() - for _, obj in targets: - func = getattr(obj, func_name, None) - if func: - return func - return None + return self.target_app.find_method(self.target_obj, func_name) def call_target(self, target_name: str, func_name: str, *args, **kwargs): func = self._get_func(func_name) @@ -102,6 +85,10 @@ def _run_func(self, waiter: _Waiter, func, args, kwargs): waiter.set() def call_target_with_resp(self, resp: Resp, target_name: str, func_name: str, *args, **kwargs): + # do not use the optional args - they are managed by the group + kwargs.pop(CollabMethodOptionName.BLOCKING, None) + kwargs.pop(CollabMethodOptionName.TIMEOUT, None) + func = self._get_func(func_name) if not func: raise AttributeError(f"{target_name} does not have {func_name}") diff --git a/nvflare/focs/sys/backend.py b/nvflare/focs/sys/backend.py index 0d4f12b333..87630842d8 100644 --- a/nvflare/focs/sys/backend.py +++ b/nvflare/focs/sys/backend.py @@ -12,17 +12,80 @@ # See the License for the specific language governing permissions and # limitations under the License. from nvflare.focs.api.backend import Backend +from nvflare.focs.api.constants import CollabMethodOptionName from nvflare.focs.api.resp import Resp +from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey, ReturnCode +from nvflare.fuel.f3.cellnet.utils import new_cell_message +from nvflare.fuel.f3.message import Message + +from .constants import MSG_CHANNEL, CallReplyKey, ObjectCallKey class SysBackend(Backend): - def __init__(self, target_fqcn, abort_signal): + def __init__(self, caller, cell, target_fqcn, abort_signal, thread_executor): Backend.__init__(self, abort_signal) + self.caller = caller + self.cell = cell self.target_fqcn = target_fqcn + self.thread_executor = thread_executor def call_target(self, target_name: str, func_name: str, *args, **kwargs): - pass + blocking = kwargs.pop(CollabMethodOptionName.BLOCKING, True) + payload = { + ObjectCallKey.CALLER: self.caller, + ObjectCallKey.TARGET_NAME: target_name, + ObjectCallKey.METHOD_NAME: func_name, + ObjectCallKey.ARGS: args, + ObjectCallKey.KWARGS: kwargs, + } + request = new_cell_message({}, payload) + + if blocking: + timeout = kwargs.pop(CollabMethodOptionName.TIMEOUT, None) + + reply = self.cell.send_one_request( + channel=MSG_CHANNEL, + target=self.target_fqcn, + topic="call", + request=request, + timeout=timeout, + secure=False, + optional=False, + abort_signal=self.abort_signal, + ) + assert isinstance(reply, Message) + rc = reply.get_header(MessageHeaderKey.RETURN_CODE) + if rc == ReturnCode.TIMEOUT: + raise TimeoutError(f"function {func_name} timed out after {timeout} seconds") + elif rc != ReturnCode.OK: + raise RuntimeError(f"function {func_name} failed: {rc}") + + if not isinstance(reply.payload, dict): + raise RuntimeError(f"function {func_name} failed: reply must be dict but got {type(reply.payload)}") + + error = reply.payload.get(CallReplyKey.ERROR) + if error: + raise RuntimeError(f"function {func_name} failed: {error}") + + return reply.payload.get(CallReplyKey.RESULT) + else: + # fire and forget + self.cell.fire_and_forget( + channel=MSG_CHANNEL, + topic="call", + targets=self.target_fqcn, + message=request, + secure=False, + optional=False, + ) def call_target_with_resp(self, resp: Resp, target_name: str, func_name: str, *args, **kwargs): - pass + self.thread_executor.submit(self._run_func, resp, target_name, args, kwargs) + + def _run_func(self, resp: Resp, target_name: str, func_name: str, *args, **kwargs): + try: + result = self.call_target(target_name, func_name, *args, **kwargs) + resp.set_result(result) + except Exception as ex: + resp.set_exception(ex) diff --git a/nvflare/focs/sys/constants.py b/nvflare/focs/sys/constants.py new file mode 100644 index 0000000000..84ed2e0585 --- /dev/null +++ b/nvflare/focs/sys/constants.py @@ -0,0 +1,35 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +SYNC_TASK_NAME = "sync" + +MSG_CHANNEL = "focs" + + +class SyncKey: + TARGET_OBJ_NAMES = "target_obj_names" + + +class ObjectCallKey: + CALLER = "caller" + TARGET_NAME = "target_name" + METHOD_NAME = "method_name" + ARGS = "args" + KWARGS = "kwargs" + TIMEOUT = "timeout" + BLOCKING = "blocking" + + +class CallReplyKey: + ERROR = "error" + RESULT = "result" diff --git a/nvflare/focs/sys/controller.py b/nvflare/focs/sys/controller.py index 24d883f644..cf12103ce2 100644 --- a/nvflare/focs/sys/controller.py +++ b/nvflare/focs/sys/controller.py @@ -13,6 +13,7 @@ # limitations under the License. import time import traceback +from concurrent.futures import ThreadPoolExecutor from typing import Dict, List from nvflare.apis.client import Client as ClientSite @@ -25,7 +26,21 @@ from nvflare.focs.api.constants import ContextKey from nvflare.focs.api.proxy import Proxy from nvflare.focs.api.strategy import Strategy -from nvflare.focs.sys.backend import SysBackend + +from .backend import SysBackend +from .constants import SYNC_TASK_NAME, SyncKey +from .utils import prepare_for_remote_call + + +class _ClientInfo: + + def __init__(self, target_obj_names): + """Information about a client. Reported by the client in the sync response. + + Args: + target_obj_names: names of target objects on the client. + """ + self.target_obj_names = target_obj_names class FocsController(Controller): @@ -35,18 +50,18 @@ def __init__( server_app_id: str, strategy_ids: List[str], server_target_obj_ids: Dict[str, str] = None, - client_target_obj_names: List[str] = None, sync_task_timeout=2, + max_call_threads=100, ): Controller.__init__(self) self.server_app_id = server_app_id # component name self.strategy_ids = strategy_ids # component names self.server_target_obj_ids = server_target_obj_ids # component IDs - self.client_target_obj_names = client_target_obj_names # callable names self.sync_task_timeout = sync_task_timeout self.server_app = None - self.client_statuses = {} # client name => bool - self.ready = False + self.client_info = {} # client name => _ClientInfo + self.cell = None + self.thread_executor = ThreadPoolExecutor(max_workers=max_call_threads) if not strategy_ids: raise ValueError(f"no strategies defined - there must be at least one strategy") @@ -79,16 +94,26 @@ def start_controller(self, fl_ctx: FLContext): self.server_app = app + # register msg CB for processing object calls + self.cell = engine.get_cell() + prepare_for_remote_call(self.cell, self.server_app, self.logger) + def _prepare_client_backend(self, client: ClientSite, abort_signal: Signal): return SysBackend( + caller=self.server_app.name, + cell=self.cell, target_fqcn=client.get_fqcn(), abort_signal=abort_signal, + thread_executor=self.thread_executor, ) def _prepare_server_backend(self, abort_signal: Signal): return SysBackend( + caller=self.server_app.name, + cell=self.cell, target_fqcn="server", abort_signal=abort_signal, + thread_executor=self.thread_executor, ) def _prepare_client_proxy(self, client: ClientSite, target_obj_names: List[str], abort_signal): @@ -121,9 +146,15 @@ def _prepare_server_proxy(self, abort_signal): def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): # configure all sites + if self.server_target_obj_ids: + target_obj_names = list(self.server_target_obj_ids.keys()) + else: + target_obj_names = [] + + task_data = Shareable({SyncKey.TARGET_OBJ_NAMES: target_obj_names}) task = Task( - name="sync", - data=Shareable(), + name=SYNC_TASK_NAME, + data=task_data, timeout=self.sync_task_timeout, result_received_cb=self._process_sync_reply, ) @@ -132,8 +163,8 @@ def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): all_clients = engine.get_clients() num_clients = len(all_clients) for c in all_clients: - assert isinstance(c, Client) - self.client_statuses[c.name] = False + assert isinstance(c, ClientSite) + self.client_info[c.name] = None start_time = time.time() self.broadcast_and_wait( @@ -146,8 +177,8 @@ def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): self.log_info(fl_ctx, f"client sync took {time_taken} seconds") failed_clients = [] - for c, ok in self.client_statuses.items(): - if not ok: + for c, info in self.client_info.items(): + if not info: failed_clients.append(c) if failed_clients: @@ -157,19 +188,20 @@ def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): ) return - self.log_info(fl_ctx, f"successfully configured clients {self.client_statuses.keys()}") - self.ready = True + self.log_info(fl_ctx, f"successfully synced clients {self.client_info.keys()}") # prepare proxies and backends server_proxy = self._prepare_server_proxy(abort_signal) client_proxies = [] for c in all_clients: - client_proxies.append(self._prepare_client_proxy(c, self.client_target_obj_names, abort_signal)) + info = self.client_info[c.name] + assert isinstance(info, _ClientInfo) + client_proxies.append(self._prepare_client_proxy(c, info.target_obj_names, abort_signal)) self.server_app.setup(server_proxy, client_proxies, abort_signal) server_ctx = self.server_app.new_context(caller=self.server_app.name, callee=self.server_app.name) - print("initializing server app") + self.log_info(fl_ctx, "initializing server app") self.server_app.initialize(server_ctx) for idx, strategy in enumerate(self.server_app.strategies): @@ -177,7 +209,7 @@ def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): break try: - print(f"Running Strategy #{idx + 1} - {type(strategy).__name__}") + self.log_info(fl_ctx, f"Running Strategy #{idx + 1} - {type(strategy).__name__}") self.server_app.current_strategy = strategy result = strategy.execute(context=server_ctx) server_ctx.set_prop(ContextKey.INPUT, result) @@ -192,7 +224,18 @@ def _process_sync_reply(self, client_task: ClientTask, fl_ctx: FLContext): rc = result.get_return_code() if rc == ReturnCode.OK: self.log_info(fl_ctx, f"successfully synced client {client_name}") - self.client_statuses[client_name] = True + target_obj_names = result.get(SyncKey.TARGET_OBJ_NAMES) + if not target_obj_names: + target_obj_names = [] + self.client_info[client_name] = _ClientInfo(target_obj_names) else: self.log_error(fl_ctx, f"client {client_task.client.name} failed to sync: {rc}") - self.client_statuses[client_name] = False + self.client_info[client_name] = None + + def process_result_of_unknown_task( + self, client: Client, task_name: str, client_task_id: str, result: Shareable, fl_ctx: FLContext + ): + pass + + def stop_controller(self, fl_ctx: FLContext): + self.thread_executor.shutdown(wait=False, cancel_futures=True) diff --git a/nvflare/focs/sys/executor.py b/nvflare/focs/sys/executor.py new file mode 100644 index 0000000000..a19386109d --- /dev/null +++ b/nvflare/focs/sys/executor.py @@ -0,0 +1,144 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from concurrent.futures import ThreadPoolExecutor +from typing import Dict + +from nvflare.apis.client import Client, from_dict +from nvflare.apis.event_type import EventType +from nvflare.apis.executor import Executor +from nvflare.apis.fl_constant import FLContextKey, ReturnCode +from nvflare.apis.fl_context import FLContext +from nvflare.apis.job_def import JobMetaKey +from nvflare.apis.shareable import Shareable, make_reply +from nvflare.apis.signal import Signal +from nvflare.focs.api.app import ClientApp, ClientAppFactory +from nvflare.focs.api.proxy import Proxy + +from .backend import SysBackend +from .constants import SYNC_TASK_NAME, SyncKey +from .utils import prepare_for_remote_call + + +class FocsExecutor(Executor): + + def __init__(self, client_app_id: str, client_target_obj_ids: Dict[str, str] = None, max_call_threads=100): + Executor.__init__(self) + self.client_app_id = client_app_id + self.client_target_obj_ids = client_target_obj_ids + self.register_event_handler(EventType.START_RUN, self._handle_start_run) + self.client_app = None + self.thread_executor = ThreadPoolExecutor(max_workers=max_call_threads) + + def _handle_start_run(self, event_type: str, fl_ctx: FLContext): + engine = fl_ctx.get_engine() + client_app = engine.get_component(self.client_app_id) + if not isinstance(client_app, (ClientApp, ClientAppFactory)): + self.system_panic( + f"component {self.client_app_id} must be ClientApp or ClientAppFactory but got {type(client_app)}", + fl_ctx, + ) + return + + if isinstance(client_app, ClientApp): + self.client_app = client_app + else: + self.client_app = client_app.make_client_app(fl_ctx.get_identity_name()) + + if self.client_target_obj_ids: + for name, cid in self.client_target_obj_ids: + obj = engine.get_component(cid) + if not obj: + self.system_panic(f"component {cid} does not exist", fl_ctx) + return + + self.client_app.add_target_object(name, obj) + + prepare_for_remote_call( + cell=engine.get_cell(), + app=self.client_app, + logger=self.logger, + ) + + def _prepare_server_proxy(self, cell, server_target_obj_names, abort_signal): + my_name = self.client_app.name + server_name = "server" + backend = SysBackend( + caller=self.client_app.name, + cell=cell, + target_fqcn=server_name, + abort_signal=abort_signal, + thread_executor=self.thread_executor, + ) + proxy = Proxy(app=self.client_app, target_name=server_name, backend=backend, caller_name=my_name) + + for name in server_target_obj_names: + p = Proxy(app=self.client_app, target_name=f"{server_name}.{name}", backend=backend, caller_name=my_name) + setattr(proxy, name, p) + return proxy + + def _prepare_client_proxy(self, cell, client: Client, abort_signal): + my_name = self.client_app.name + backend = SysBackend( + caller=self.client_app.name, + cell=cell, + target_fqcn=client.get_fqcn(), + abort_signal=abort_signal, + thread_executor=self.thread_executor, + ) + proxy = Proxy(app=self.client_app, target_name=client.name, backend=backend, caller_name=my_name) + + if self.client_target_obj_ids: + for name in self.client_target_obj_ids.keys(): + p = Proxy( + app=self.client_app, + target_name=f"{client.name}.{name}", + backend=backend, + caller_name=my_name, + ) + setattr(proxy, name, p) + return proxy + + def execute(self, task_name: str, shareable: Shareable, fl_ctx: FLContext, abort_signal: Signal) -> Shareable: + if task_name != SYNC_TASK_NAME: + self.log_error(fl_ctx, f"received unsupported task {task_name}") + return make_reply(ReturnCode.TASK_UNKNOWN) + + server_target_obj_names = shareable.get(SyncKey.TARGET_OBJ_NAMES) + + engine = fl_ctx.get_engine() + cell = engine.get_cell() + + # build proxies + server_proxy = self._prepare_server_proxy(cell, server_target_obj_names, abort_signal) + + job_meta = fl_ctx.get_prop(FLContextKey.JOB_META) + job_clients = job_meta.get(JobMetaKey.JOB_CLIENTS) + all_clients = [from_dict(d) for d in job_clients] + client_proxies = [] + for c in all_clients: + p = self._prepare_client_proxy(cell, c, abort_signal) + client_proxies.append(p) + + self.client_app.setup(server_proxy, client_proxies, abort_signal) + + ctx = self.client_app.new_context(self.client_app.name, self.client_app.name) + self.client_app.initialize(ctx) + + reply = make_reply(ReturnCode.OK) + if self.client_target_obj_ids: + target_obj_names = list(self.client_target_obj_ids.keys()) + else: + target_obj_names = [] + reply[SyncKey.TARGET_OBJ_NAMES] = target_obj_names + return reply diff --git a/nvflare/focs/sys/utils.py b/nvflare/focs/sys/utils.py new file mode 100644 index 0000000000..f353535019 --- /dev/null +++ b/nvflare/focs/sys/utils.py @@ -0,0 +1,88 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nvflare.focs.api.app import App +from nvflare.focs.api.constants import CollabMethodArgName +from nvflare.focs.api.utils import check_context_support +from nvflare.fuel.f3.cellnet.utils import new_cell_message +from nvflare.fuel.f3.message import Message + +from .constants import MSG_CHANNEL, CallReplyKey, ObjectCallKey + + +def prepare_for_remote_call(cell, app, logger): + cell.register_request_cb(channel=MSG_CHANNEL, topic="*", cb=_call_app_method, app=app, logger=logger) + + +def _error_reply(error: str, logger) -> Message: + logger.error(error) + return new_cell_message(headers={}, payload={CallReplyKey.ERROR: error}) + + +def _call_app_method(request: Message, app: App, logger) -> Message: + logger.info(f"got call") + payload = request.payload + assert isinstance(payload, dict) + + caller = payload.get(ObjectCallKey.CALLER) + if not caller: + return _error_reply(f"missing '{ObjectCallKey.CALLER}' from call", logger) + + method_name = payload.get(ObjectCallKey.METHOD_NAME) + if not method_name: + return _error_reply(f"missing '{ObjectCallKey.METHOD_NAME}' from call", logger) + + target_name = payload.get(ObjectCallKey.TARGET_NAME) + if not isinstance(target_name, str): + return _error_reply( + f"bad '{ObjectCallKey.TARGET_NAME}' from call: expect str but got {type(target_name)}", + logger, + ) + + method_args = payload.get(ObjectCallKey.ARGS) + if not method_args: + method_args = [] + elif not isinstance(method_args, list): + return _error_reply(f"bad method args: should be list but got {type(method_args)}", logger) + + method_kwargs = payload.get(ObjectCallKey.KWARGS) + if not method_kwargs: + method_kwargs = {} + elif not isinstance(method_kwargs, dict): + return _error_reply(f"bad method kwargs: should be dict but got {type(method_kwargs)}", logger) + + parts = target_name.split(".") + obj_name = None + if len(parts) >= 2: + obj_name = parts[1] + if obj_name: + target_objs = app.get_target_objects() + target_obj = target_objs.get(obj_name) + else: + target_obj = app + if not target_obj: + return _error_reply(f"no object named '{target_name}'", logger) + + m = app.find_method(target_obj, method_name) + if not m: + return _error_reply(f"no method named '{method_name}'", logger) + + # invoke this method + try: + ctx = app.new_context(caller=caller, callee=app.name) + method_kwargs[CollabMethodArgName.CONTEXT] = ctx + check_context_support(m, method_kwargs) + result = m(*method_args, **method_kwargs) + return new_cell_message(headers={}, payload={CallReplyKey.RESULT: result}) + except Exception as ex: + return _error_reply(f"exception {type(ex)}", logger) From 73579ba11deee9f315ca4d2d15374ff9c7d7581f Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Fri, 26 Sep 2025 12:58:25 -0400 Subject: [PATCH 012/102] added collab decorator --- nvflare/focs/api/app.py | 8 +++++ nvflare/focs/api/dec.py | 44 +++++++++++++++++++++++ nvflare/focs/examples/np/algos/client.py | 3 ++ nvflare/focs/examples/np/algos/widgets.py | 2 ++ nvflare/focs/examples/np/swarm.py | 5 +++ nvflare/focs/sim/backend.py | 14 ++++---- nvflare/focs/sys/utils.py | 2 +- 7 files changed, 70 insertions(+), 8 deletions(-) create mode 100644 nvflare/focs/api/dec.py diff --git a/nvflare/focs/api/app.py b/nvflare/focs/api/app.py index 58bc5f928d..8714479b82 100644 --- a/nvflare/focs/api/app.py +++ b/nvflare/focs/api/app.py @@ -17,6 +17,7 @@ from .constants import CollabMethodArgName from .ctx import Context +from .dec import collab, is_collab from .proxy import Proxy from .strategy import Strategy from .utils import check_context_support @@ -93,6 +94,12 @@ def find_method(self, target_obj, method_name): return m return None + def find_collab_method(self, target_obj, method_name): + m = self.find_method(target_obj, method_name) + if m and is_collab(m): + return m + return None + def initialize_app(self, context: Context): pass @@ -122,6 +129,7 @@ def register_event_handler(self, event_type: str, handler, **handler_kwargs): self._event_handlers[event_type] = handlers handlers.append((handler, handler_kwargs)) + @collab def fire_event(self, event_type: str, data, context: Context): for e, handlers in self._event_handlers.items(): if e == event_type: diff --git a/nvflare/focs/api/dec.py b/nvflare/focs/api/dec.py new file mode 100644 index 0000000000..a8b1bafe29 --- /dev/null +++ b/nvflare/focs/api/dec.py @@ -0,0 +1,44 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import inspect + +from .constants import CollabMethodArgName + +_ATTR_COLLAB = "_is_fox_collab" +_ATTR_SUPPORT_CTX = "_supports_fox_ctx" + + +def collab(func): + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + signature = inspect.signature(func) + parameter_names = signature.parameters.keys() + if CollabMethodArgName.CONTEXT in parameter_names: + setattr(wrapper, _ATTR_SUPPORT_CTX, True) + setattr(wrapper, _ATTR_COLLAB, True) + return wrapper + + +def is_collab(func): + return hasattr(func, _ATTR_COLLAB) + + +def supports_context(func): + return hasattr(func, _ATTR_SUPPORT_CTX) + + +def adjust_kwargs(func, kwargs): + if not supports_context(func): + kwargs.pop(CollabMethodArgName.CONTEXT, None) diff --git a/nvflare/focs/examples/np/algos/client.py b/nvflare/focs/examples/np/algos/client.py index 3b5f8703f7..82c37eb7b6 100644 --- a/nvflare/focs/examples/np/algos/client.py +++ b/nvflare/focs/examples/np/algos/client.py @@ -15,6 +15,7 @@ from nvflare.focs.api.app import ClientApp, ClientAppFactory from nvflare.focs.api.ctx import Context +from nvflare.focs.api.dec import collab class NPTrainer(ClientApp): @@ -23,6 +24,7 @@ def __init__(self, delta: float): ClientApp.__init__(self) self.delta = delta + @collab def train(self, r, weights, context: Context): if context.is_aborted(): print("training aborted") @@ -36,6 +38,7 @@ def train(self, r, weights, context: Context): self.server.fire_event("metrics", {"round": r, "y": 10}, blocking=False) return weights + self.delta + @collab def evaluate(self, model, context: Context): print(f"[{self.name}] called by {context.caller}: client {context.callee} to evaluate") return random.random() diff --git a/nvflare/focs/examples/np/algos/widgets.py b/nvflare/focs/examples/np/algos/widgets.py index 1abe794326..3a2e7b0885 100644 --- a/nvflare/focs/examples/np/algos/widgets.py +++ b/nvflare/focs/examples/np/algos/widgets.py @@ -12,10 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. from nvflare.focs.api.ctx import Context +from nvflare.focs.api.dec import collab class MetricReceiver: + @collab def accept_metric(self, metrics: dict, context: Context): print(f"[{context.callee}] received metric report from {context.caller}: {metrics}") diff --git a/nvflare/focs/examples/np/swarm.py b/nvflare/focs/examples/np/swarm.py index bf38ea7c40..105b83f606 100644 --- a/nvflare/focs/examples/np/swarm.py +++ b/nvflare/focs/examples/np/swarm.py @@ -16,6 +16,7 @@ from nvflare.focs.api.app import ClientApp, ServerApp from nvflare.focs.api.ctx import Context +from nvflare.focs.api.dec import collab from nvflare.focs.api.group import all_clients from nvflare.focs.api.strategy import Strategy from nvflare.focs.examples.np.algos.utils import parse_array_def @@ -36,6 +37,7 @@ def execute(self, context: Context): start_client.start(self.num_rounds, self.initial_model) self.waiter.wait() + @collab def notify_done(self, context: Context): print(f"[{context.callee}]: received DONE from client: {context.caller}") self.waiter.set() @@ -48,6 +50,7 @@ def __init__(self, delta: float): self.delta = delta self.register_event_handler("final_model", self._accept_final_model) + @collab def train(self, weights, current_round, context: Context): print(f"[{context.callee}]: train asked by {context.caller}: {current_round=}") return weights + self.delta @@ -60,6 +63,7 @@ def sag(self, model, current_round, ctx: Context): total += results[i] return total / len(results) + @collab def swarm_learn(self, num_rounds, model, current_round, context: Context): print(f"[{context.callee}]: swarm learn asked by {context.caller}: {num_rounds=} {current_round=} {model=}") new_model = self.sag(model, current_round, context) @@ -77,6 +81,7 @@ def swarm_learn(self, num_rounds, model, current_round, context: Context): next_client = self.clients[next_client_idx] next_client.swarm_learn(num_rounds, new_model, next_round, blocking=False) + @collab def start(self, num_rounds, initial_model, context: Context): self.swarm_learn(num_rounds, initial_model, 0, context) diff --git a/nvflare/focs/sim/backend.py b/nvflare/focs/sim/backend.py index 9bb9612aba..088867fff1 100644 --- a/nvflare/focs/sim/backend.py +++ b/nvflare/focs/sim/backend.py @@ -16,8 +16,8 @@ from nvflare.focs.api.app import App from nvflare.focs.api.backend import Backend from nvflare.focs.api.constants import CollabMethodArgName, CollabMethodOptionName +from nvflare.focs.api.dec import adjust_kwargs from nvflare.focs.api.resp import Resp -from nvflare.focs.api.utils import check_context_support class _Waiter(threading.Event): @@ -37,15 +37,15 @@ def __init__(self, target_app: App, target_obj, abort_signal, thread_executor): self.executor = thread_executor def _get_func(self, func_name): - return self.target_app.find_method(self.target_obj, func_name) + return self.target_app.find_collab_method(self.target_obj, func_name) def call_target(self, target_name: str, func_name: str, *args, **kwargs): func = self._get_func(func_name) if not func: - raise AttributeError(f"{target_name} does not have {func_name}") + raise AttributeError(f"{target_name} does not have method '{func_name}' or it is not collab") if not callable(func): - raise AttributeError(f"the {func_name} of {target_name} is not callable") + raise AttributeError(f"the method '{func_name}' of {target_name} is not callable") blocking = kwargs.pop(CollabMethodOptionName.BLOCKING, True) timeout = kwargs.pop(CollabMethodOptionName.TIMEOUT, None) @@ -69,7 +69,7 @@ def _augment_context(self, func, kwargs): if ctx: target_ctx = self.target_app.new_context(ctx.caller, ctx.callee) kwargs[CollabMethodArgName.CONTEXT] = target_ctx - check_context_support(func, kwargs) + adjust_kwargs(func, kwargs) def _run_func(self, waiter: _Waiter, func, args, kwargs): try: @@ -91,10 +91,10 @@ def call_target_with_resp(self, resp: Resp, target_name: str, func_name: str, *a func = self._get_func(func_name) if not func: - raise AttributeError(f"{target_name} does not have {func_name}") + raise AttributeError(f"{target_name} does not have method '{func_name}' or it is not collab") if not callable(func): - raise AttributeError(f"the {func_name} of {target_name} is not callable") + raise AttributeError(f"the method '{func_name}' of {target_name} is not callable") self.executor.submit(self._run_func_with_resp, resp, func, args, kwargs) diff --git a/nvflare/focs/sys/utils.py b/nvflare/focs/sys/utils.py index f353535019..d299fc5071 100644 --- a/nvflare/focs/sys/utils.py +++ b/nvflare/focs/sys/utils.py @@ -73,7 +73,7 @@ def _call_app_method(request: Message, app: App, logger) -> Message: if not target_obj: return _error_reply(f"no object named '{target_name}'", logger) - m = app.find_method(target_obj, method_name) + m = app.find_collab_method(target_obj, method_name) if not m: return _error_reply(f"no method named '{method_name}'", logger) From b377266cded911d69f147b45f52b10194d367caa Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Sat, 27 Sep 2025 18:24:35 -0400 Subject: [PATCH 013/102] dev sys integration --- nvflare/focs/api/app.py | 10 +++-- nvflare/focs/examples/np/algos/client.py | 8 ++-- nvflare/focs/sys/backend.py | 23 ++++++---- nvflare/focs/sys/constants.py | 1 + nvflare/focs/sys/controller.py | 48 ++++++++++++--------- nvflare/focs/sys/executor.py | 31 +++++++------ nvflare/focs/sys/utils.py | 35 ++++++++++----- nvflare/private/fed/server/server_engine.py | 5 ++- 8 files changed, 102 insertions(+), 59 deletions(-) diff --git a/nvflare/focs/api/app.py b/nvflare/focs/api/app.py index 8714479b82..284fc95f95 100644 --- a/nvflare/focs/api/app.py +++ b/nvflare/focs/api/app.py @@ -23,7 +23,7 @@ from .utils import check_context_support -class App(ABC): +class App: def __init__(self): self.name = None @@ -144,9 +144,13 @@ class ServerApp(App): def __init__(self, strategy: Strategy = None): super().__init__() - if not isinstance(strategy, Strategy): + + if strategy and not isinstance(strategy, Strategy): raise ValueError(f"strategy must be Strategy but got {type(strategy)}") - self.strategies = [strategy] + + self.strategies = [] + if strategy: + self.strategies.append(strategy) self.current_strategy = None def add_strategy(self, strategy): diff --git a/nvflare/focs/examples/np/algos/client.py b/nvflare/focs/examples/np/algos/client.py index 82c37eb7b6..ba30ac24f8 100644 --- a/nvflare/focs/examples/np/algos/client.py +++ b/nvflare/focs/examples/np/algos/client.py @@ -31,10 +31,10 @@ def train(self, r, weights, context: Context): return 0 print(f"[{self.name}] called by {context.caller}: client {context.callee} trained round {r}") - metric_receiver = self.server.get_target("metric_receiver") - if metric_receiver: - self.server.accept_metric({"round": r, "y": 2}) - + # metric_receiver = self.server.get_target("metric_receiver") + # if metric_receiver: + # self.server.accept_metric({"round": r, "y": 2}) + # self.server.fire_event("metrics", {"round": r, "y": 10}, blocking=False) return weights + self.delta diff --git a/nvflare/focs/sys/backend.py b/nvflare/focs/sys/backend.py index 87630842d8..8f31710f0d 100644 --- a/nvflare/focs/sys/backend.py +++ b/nvflare/focs/sys/backend.py @@ -12,19 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. from nvflare.focs.api.backend import Backend -from nvflare.focs.api.constants import CollabMethodOptionName +from nvflare.focs.api.constants import CollabMethodArgName, CollabMethodOptionName from nvflare.focs.api.resp import Resp from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey, ReturnCode from nvflare.fuel.f3.cellnet.utils import new_cell_message from nvflare.fuel.f3.message import Message +from nvflare.fuel.utils.log_utils import get_obj_logger -from .constants import MSG_CHANNEL, CallReplyKey, ObjectCallKey +from .constants import MSG_CHANNEL, MSG_TOPIC, CallReplyKey, ObjectCallKey class SysBackend(Backend): def __init__(self, caller, cell, target_fqcn, abort_signal, thread_executor): Backend.__init__(self, abort_signal) + self.logger = get_obj_logger(self) self.caller = caller self.cell = cell self.target_fqcn = target_fqcn @@ -32,6 +34,9 @@ def __init__(self, caller, cell, target_fqcn, abort_signal, thread_executor): def call_target(self, target_name: str, func_name: str, *args, **kwargs): blocking = kwargs.pop(CollabMethodOptionName.BLOCKING, True) + timeout = kwargs.pop(CollabMethodOptionName.TIMEOUT, 10.0) + kwargs.pop(CollabMethodArgName.CONTEXT, None) + payload = { ObjectCallKey.CALLER: self.caller, ObjectCallKey.TARGET_NAME: target_name, @@ -42,12 +47,12 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): request = new_cell_message({}, payload) if blocking: - timeout = kwargs.pop(CollabMethodOptionName.TIMEOUT, None) + self.logger.info(f"request payload: {request.payload} from {self.cell.get_fqcn()} to {self.target_fqcn}") - reply = self.cell.send_one_request( + reply = self.cell.send_request( channel=MSG_CHANNEL, target=self.target_fqcn, - topic="call", + topic=MSG_TOPIC, request=request, timeout=timeout, secure=False, @@ -55,7 +60,7 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): abort_signal=self.abort_signal, ) assert isinstance(reply, Message) - rc = reply.get_header(MessageHeaderKey.RETURN_CODE) + rc = reply.get_header(MessageHeaderKey.RETURN_CODE, ReturnCode.OK) if rc == ReturnCode.TIMEOUT: raise TimeoutError(f"function {func_name} timed out after {timeout} seconds") elif rc != ReturnCode.OK: @@ -68,12 +73,14 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): if error: raise RuntimeError(f"function {func_name} failed: {error}") - return reply.payload.get(CallReplyKey.RESULT) + result = reply.payload.get(CallReplyKey.RESULT) + self.logger.info(f"got result from {self.target_fqcn}: {result}") + return result else: # fire and forget self.cell.fire_and_forget( channel=MSG_CHANNEL, - topic="call", + topic=MSG_TOPIC, targets=self.target_fqcn, message=request, secure=False, diff --git a/nvflare/focs/sys/constants.py b/nvflare/focs/sys/constants.py index 84ed2e0585..3f80c7edec 100644 --- a/nvflare/focs/sys/constants.py +++ b/nvflare/focs/sys/constants.py @@ -14,6 +14,7 @@ SYNC_TASK_NAME = "sync" MSG_CHANNEL = "focs" +MSG_TOPIC = "call" class SyncKey: diff --git a/nvflare/focs/sys/controller.py b/nvflare/focs/sys/controller.py index cf12103ce2..7644a5a698 100644 --- a/nvflare/focs/sys/controller.py +++ b/nvflare/focs/sys/controller.py @@ -22,10 +22,12 @@ from nvflare.apis.impl.controller import Controller from nvflare.apis.shareable import ReturnCode, Shareable from nvflare.apis.signal import Signal +from nvflare.app_common.decomposers.numpy_decomposers import register from nvflare.focs.api.app import ServerApp from nvflare.focs.api.constants import ContextKey from nvflare.focs.api.proxy import Proxy from nvflare.focs.api.strategy import Strategy +from nvflare.fuel.f3.cellnet.fqcn import FQCN from .backend import SysBackend from .constants import SYNC_TASK_NAME, SyncKey @@ -47,8 +49,8 @@ class FocsController(Controller): def __init__( self, - server_app_id: str, strategy_ids: List[str], + server_app_id: str = None, server_target_obj_ids: Dict[str, str] = None, sync_task_timeout=2, max_call_threads=100, @@ -67,14 +69,18 @@ def __init__( raise ValueError(f"no strategies defined - there must be at least one strategy") def start_controller(self, fl_ctx: FLContext): + register() engine = fl_ctx.get_engine() - app = engine.get_component(self.server_app_id) - if not isinstance(app, ServerApp): - self.system_panic(f"component {self.server_app_id} must be ServerApp but got {type(app)}", fl_ctx) - return - app.name = "server" + if self.server_app_id: + app = engine.get_component(self.server_app_id) + if not isinstance(app, ServerApp): + self.system_panic(f"component {self.server_app_id} must be ServerApp but got {type(app)}", fl_ctx) + return + else: + app = ServerApp() + app.name = "server" for cid in self.strategy_ids: strategy = engine.get_component(cid) if not isinstance(strategy, Strategy): @@ -94,30 +100,26 @@ def start_controller(self, fl_ctx: FLContext): self.server_app = app - # register msg CB for processing object calls - self.cell = engine.get_cell() - prepare_for_remote_call(self.cell, self.server_app, self.logger) - - def _prepare_client_backend(self, client: ClientSite, abort_signal: Signal): + def _prepare_client_backend(self, job_id, client: ClientSite, abort_signal: Signal): return SysBackend( caller=self.server_app.name, cell=self.cell, - target_fqcn=client.get_fqcn(), + target_fqcn=FQCN.join([client.get_fqcn(), job_id]), abort_signal=abort_signal, thread_executor=self.thread_executor, ) - def _prepare_server_backend(self, abort_signal: Signal): + def _prepare_server_backend(self, job_id: str, abort_signal: Signal): return SysBackend( caller=self.server_app.name, cell=self.cell, - target_fqcn="server", + target_fqcn=FQCN.join([FQCN.ROOT_SERVER, job_id]), abort_signal=abort_signal, thread_executor=self.thread_executor, ) - def _prepare_client_proxy(self, client: ClientSite, target_obj_names: List[str], abort_signal): - backend = self._prepare_client_backend(client, abort_signal) + def _prepare_client_proxy(self, job_id: str, client: ClientSite, target_obj_names: List[str], abort_signal): + backend = self._prepare_client_backend(job_id, client, abort_signal) proxy = Proxy(app=self.server_app, target_name=client.name, backend=backend, caller_name=self.server_app.name) for name in target_obj_names: @@ -130,9 +132,9 @@ def _prepare_client_proxy(self, client: ClientSite, target_obj_names: List[str], setattr(proxy, name, p) return proxy - def _prepare_server_proxy(self, abort_signal): + def _prepare_server_proxy(self, job_id, abort_signal): server_name = self.server_app.name - backend = self._prepare_server_backend(abort_signal) + backend = self._prepare_server_backend(job_id, abort_signal) proxy = Proxy(app=self.server_app, target_name=server_name, backend=backend, caller_name=server_name) tos = self.server_app.get_target_objects() @@ -160,6 +162,7 @@ def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): ) engine = fl_ctx.get_engine() + self.logger.info(f"server engine {type(engine)}") all_clients = engine.get_clients() num_clients = len(all_clients) for c in all_clients: @@ -190,13 +193,18 @@ def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): self.log_info(fl_ctx, f"successfully synced clients {self.client_info.keys()}") + # register msg CB for processing object calls + self.cell = engine.get_cell() + prepare_for_remote_call(self.cell, self.server_app, self.logger) + # prepare proxies and backends - server_proxy = self._prepare_server_proxy(abort_signal) + job_id = fl_ctx.get_job_id() + server_proxy = self._prepare_server_proxy(job_id, abort_signal) client_proxies = [] for c in all_clients: info = self.client_info[c.name] assert isinstance(info, _ClientInfo) - client_proxies.append(self._prepare_client_proxy(c, info.target_obj_names, abort_signal)) + client_proxies.append(self._prepare_client_proxy(job_id, c, info.target_obj_names, abort_signal)) self.server_app.setup(server_proxy, client_proxies, abort_signal) diff --git a/nvflare/focs/sys/executor.py b/nvflare/focs/sys/executor.py index a19386109d..197a159683 100644 --- a/nvflare/focs/sys/executor.py +++ b/nvflare/focs/sys/executor.py @@ -24,6 +24,7 @@ from nvflare.apis.signal import Signal from nvflare.focs.api.app import ClientApp, ClientAppFactory from nvflare.focs.api.proxy import Proxy +from nvflare.fuel.f3.cellnet.fqcn import FQCN from .backend import SysBackend from .constants import SYNC_TASK_NAME, SyncKey @@ -50,10 +51,13 @@ def _handle_start_run(self, event_type: str, fl_ctx: FLContext): ) return + client_name = fl_ctx.get_identity_name() if isinstance(client_app, ClientApp): self.client_app = client_app else: - self.client_app = client_app.make_client_app(fl_ctx.get_identity_name()) + self.client_app = client_app.make_client_app(client_name) + + self.client_app.name = client_name if self.client_target_obj_ids: for name, cid in self.client_target_obj_ids: @@ -64,19 +68,13 @@ def _handle_start_run(self, event_type: str, fl_ctx: FLContext): self.client_app.add_target_object(name, obj) - prepare_for_remote_call( - cell=engine.get_cell(), - app=self.client_app, - logger=self.logger, - ) - - def _prepare_server_proxy(self, cell, server_target_obj_names, abort_signal): + def _prepare_server_proxy(self, job_id, cell, server_target_obj_names, abort_signal): my_name = self.client_app.name server_name = "server" backend = SysBackend( caller=self.client_app.name, cell=cell, - target_fqcn=server_name, + target_fqcn=FQCN.join([FQCN.ROOT_SERVER, job_id]), abort_signal=abort_signal, thread_executor=self.thread_executor, ) @@ -87,12 +85,12 @@ def _prepare_server_proxy(self, cell, server_target_obj_names, abort_signal): setattr(proxy, name, p) return proxy - def _prepare_client_proxy(self, cell, client: Client, abort_signal): + def _prepare_client_proxy(self, job_id, cell, client: Client, abort_signal): my_name = self.client_app.name backend = SysBackend( caller=self.client_app.name, cell=cell, - target_fqcn=client.get_fqcn(), + target_fqcn=FQCN.join([client.get_fqcn(), job_id]), abort_signal=abort_signal, thread_executor=self.thread_executor, ) @@ -119,15 +117,22 @@ def execute(self, task_name: str, shareable: Shareable, fl_ctx: FLContext, abort engine = fl_ctx.get_engine() cell = engine.get_cell() + prepare_for_remote_call( + cell=cell, + app=self.client_app, + logger=self.logger, + ) + # build proxies - server_proxy = self._prepare_server_proxy(cell, server_target_obj_names, abort_signal) + job_id = fl_ctx.get_job_id() + server_proxy = self._prepare_server_proxy(job_id, cell, server_target_obj_names, abort_signal) job_meta = fl_ctx.get_prop(FLContextKey.JOB_META) job_clients = job_meta.get(JobMetaKey.JOB_CLIENTS) all_clients = [from_dict(d) for d in job_clients] client_proxies = [] for c in all_clients: - p = self._prepare_client_proxy(cell, c, abort_signal) + p = self._prepare_client_proxy(job_id, cell, c, abort_signal) client_proxies.append(p) self.client_app.setup(server_proxy, client_proxies, abort_signal) diff --git a/nvflare/focs/sys/utils.py b/nvflare/focs/sys/utils.py index d299fc5071..3a0edb836b 100644 --- a/nvflare/focs/sys/utils.py +++ b/nvflare/focs/sys/utils.py @@ -13,24 +13,29 @@ # limitations under the License. from nvflare.focs.api.app import App from nvflare.focs.api.constants import CollabMethodArgName -from nvflare.focs.api.utils import check_context_support +from nvflare.focs.api.dec import adjust_kwargs +from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey, ReturnCode from nvflare.fuel.f3.cellnet.utils import new_cell_message from nvflare.fuel.f3.message import Message -from .constants import MSG_CHANNEL, CallReplyKey, ObjectCallKey +from .constants import MSG_CHANNEL, MSG_TOPIC, CallReplyKey, ObjectCallKey def prepare_for_remote_call(cell, app, logger): - cell.register_request_cb(channel=MSG_CHANNEL, topic="*", cb=_call_app_method, app=app, logger=logger) + logger.info(f"register cb for cell {cell.get_fqcn()}: {type(cell)}") + cell.register_request_cb(channel=MSG_CHANNEL, topic=MSG_TOPIC, cb=_call_app_method, app=app, logger=logger) + logger.info(f"registered request CB for {MSG_CHANNEL}/{MSG_TOPIC}") def _error_reply(error: str, logger) -> Message: logger.error(error) - return new_cell_message(headers={}, payload={CallReplyKey.ERROR: error}) + return new_cell_message( + headers={MessageHeaderKey.RETURN_CODE: ReturnCode.PROCESS_EXCEPTION}, payload={CallReplyKey.ERROR: error} + ) def _call_app_method(request: Message, app: App, logger) -> Message: - logger.info(f"got call") + logger.info("got a remote call") payload = request.payload assert isinstance(payload, dict) @@ -52,8 +57,8 @@ def _call_app_method(request: Message, app: App, logger) -> Message: method_args = payload.get(ObjectCallKey.ARGS) if not method_args: method_args = [] - elif not isinstance(method_args, list): - return _error_reply(f"bad method args: should be list but got {type(method_args)}", logger) + elif not isinstance(method_args, (list, tuple)): + return _error_reply(f"bad method args: should be list/tuple but got {type(method_args)}", logger) method_kwargs = payload.get(ObjectCallKey.KWARGS) if not method_kwargs: @@ -68,21 +73,31 @@ def _call_app_method(request: Message, app: App, logger) -> Message: if obj_name: target_objs = app.get_target_objects() target_obj = target_objs.get(obj_name) + logger.info(f"calling target obj: {app.name}.{obj_name}") else: target_obj = app + logger.info(f"calling target app: {app.name}") + if not target_obj: return _error_reply(f"no object named '{target_name}'", logger) m = app.find_collab_method(target_obj, method_name) if not m: - return _error_reply(f"no method named '{method_name}'", logger) + return _error_reply(f"no method named '{method_name}' or it is not collab", logger) + else: + logger.info(f"found method for {method_name}") # invoke this method try: ctx = app.new_context(caller=caller, callee=app.name) method_kwargs[CollabMethodArgName.CONTEXT] = ctx - check_context_support(m, method_kwargs) + adjust_kwargs(m, method_kwargs) + + logger.info(f"calling method {method_name}: {caller=}: {method_args=} {method_kwargs=}") result = m(*method_args, **method_kwargs) - return new_cell_message(headers={}, payload={CallReplyKey.RESULT: result}) + logger.info(f"result from method {method_name}: {result}") + return new_cell_message( + headers={MessageHeaderKey.RETURN_CODE: ReturnCode.OK}, payload={CallReplyKey.RESULT: result} + ) except Exception as ex: return _error_reply(f"exception {type(ex)}", logger) diff --git a/nvflare/private/fed/server/server_engine.py b/nvflare/private/fed/server/server_engine.py index 8012228209..4d2534e12d 100644 --- a/nvflare/private/fed/server/server_engine.py +++ b/nvflare/private/fed/server/server_engine.py @@ -446,7 +446,10 @@ def set_run_manager(self, run_manager: RunManager): self.run_manager.add_handler(widget) def get_cell(self): - return self.cell + if self.cell: + return self.cell + elif self.run_manager and self.run_manager.cell: + return self.run_manager.cell def initialize_comm(self, cell: Cell): """This is called when the communication cell has been created. From 904aa816cd9a9b1cff83e5bdf091a8eceb535fe8 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Mon, 29 Sep 2025 11:10:15 -0400 Subject: [PATCH 014/102] rename to use collab --- nvflare/focs/api/app.py | 20 ++++++------ nvflare/focs/examples/np/fed_avg_intime.py | 2 +- nvflare/focs/examples/np/fed_avg_para.py | 2 +- nvflare/focs/examples/np/fed_avg_seq.py | 2 +- nvflare/focs/sim/runner.py | 4 +-- nvflare/focs/sys/backend.py | 3 +- nvflare/focs/sys/constants.py | 2 +- nvflare/focs/sys/controller.py | 38 +++++++++------------- nvflare/focs/sys/executor.py | 6 ++-- nvflare/focs/sys/utils.py | 2 +- 10 files changed, 38 insertions(+), 43 deletions(-) diff --git a/nvflare/focs/api/app.py b/nvflare/focs/api/app.py index 284fc95f95..d21e0b1c21 100644 --- a/nvflare/focs/api/app.py +++ b/nvflare/focs/api/app.py @@ -30,7 +30,7 @@ def __init__(self): self.server = None self.clients = None self._me = None - self._target_objs = [] + self._collab_objs = [] self._abort_signal = None self._props = {} self._event_handlers = {} # event type => list of (cb, kwargs) @@ -41,18 +41,18 @@ def set_prop(self, name: str, value): def get_prop(self, name: str, default=None): return self._props.get(name, default) - def get_default_target(self): + def get_default_collab_object(self): return None - def add_target_object(self, name: str, obj): + def add_collab_object(self, name: str, obj): if hasattr(obj, name): raise ValueError(f"conflict with reserved name {name}") setattr(self, name, obj) - self._target_objs.append((name, obj)) + self._collab_objs.append((name, obj)) - def get_target_objects(self): - return self._target_objs + def get_collab_objects(self): + return self._collab_objs def setup(self, server: Proxy, clients: List[Proxy], abort_signal): self.server = server @@ -81,13 +81,13 @@ def find_method(self, target_obj, method_name): if isinstance(target_obj, App): # see whether any targets have this method - default_target = self.get_default_target() + default_target = self.get_default_collab_object() if default_target: m = getattr(default_target, method_name, None) if m: return m - targets = self.get_target_objects() + targets = self.get_collab_objects() for _, obj in targets: m = getattr(obj, method_name, None) if m: @@ -107,7 +107,7 @@ def initialize(self, context: Context): self.initialize_app(context) # initialize target objects - for name, obj in self._target_objs: + for name, obj in self._collab_objs: init_func = getattr(obj, "initialize", None) if init_func and callable(init_func): print(f"initializing target object {name}") @@ -158,7 +158,7 @@ def add_strategy(self, strategy): raise ValueError(f"strategy must be Controller but got {type(strategy)}") self.strategies.append(strategy) - def get_default_target(self): + def get_default_collab_object(self): return self.current_strategy diff --git a/nvflare/focs/examples/np/fed_avg_intime.py b/nvflare/focs/examples/np/fed_avg_intime.py index 32772330cd..e2ef3c5937 100644 --- a/nvflare/focs/examples/np/fed_avg_intime.py +++ b/nvflare/focs/examples/np/fed_avg_intime.py @@ -21,7 +21,7 @@ def main(): server_app = ServerApp(strategy=NPFedAvgInTime(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2)) - server_app.add_target_object("metric_receiver", MetricReceiver()) + server_app.add_collab_object("metric_receiver", MetricReceiver()) runner = AppRunner( server_app=server_app, diff --git a/nvflare/focs/examples/np/fed_avg_para.py b/nvflare/focs/examples/np/fed_avg_para.py index c43e208aeb..4bee611a41 100644 --- a/nvflare/focs/examples/np/fed_avg_para.py +++ b/nvflare/focs/examples/np/fed_avg_para.py @@ -21,7 +21,7 @@ def main(): server_app = ServerApp(strategy=NPFedAvgParallel(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2)) - server_app.add_target_object("metric_receiver", MetricReceiver()) + server_app.add_collab_object("metric_receiver", MetricReceiver()) runner = AppRunner( server_app=server_app, diff --git a/nvflare/focs/examples/np/fed_avg_seq.py b/nvflare/focs/examples/np/fed_avg_seq.py index 3d2498f182..d5e6ccb620 100644 --- a/nvflare/focs/examples/np/fed_avg_seq.py +++ b/nvflare/focs/examples/np/fed_avg_seq.py @@ -26,7 +26,7 @@ def main(): initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], ) ) - server_app.add_target_object("metric_receiver", MetricReceiver()) + server_app.add_collab_object("metric_receiver", MetricReceiver()) runner = AppRunner( server_app=server_app, diff --git a/nvflare/focs/sim/runner.py b/nvflare/focs/sim/runner.py index d2d8991894..6ccc668c8e 100644 --- a/nvflare/focs/sim/runner.py +++ b/nvflare/focs/sim/runner.py @@ -27,7 +27,7 @@ class AppRunner: def _prepare_app_backends(self, app: App): bes = {"": SimBackend(app, app, self.abort_signal, self.thread_executor)} - targets = app.get_target_objects() + targets = app.get_collab_objects() if targets: for name, obj in targets: bes[name] = SimBackend(app, obj, self.abort_signal, self.thread_executor) @@ -35,7 +35,7 @@ def _prepare_app_backends(self, app: App): def _prepare_proxy(self, for_app: App, target_app: App, backends: dict): app_proxy = Proxy(app=for_app, target_name=target_app.name, backend=backends[""], caller_name=for_app.name) - tos = target_app.get_target_objects() + tos = target_app.get_collab_objects() if tos: for name, obj in tos: p = Proxy( diff --git a/nvflare/focs/sys/backend.py b/nvflare/focs/sys/backend.py index 8f31710f0d..e70553acba 100644 --- a/nvflare/focs/sys/backend.py +++ b/nvflare/focs/sys/backend.py @@ -47,7 +47,7 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): request = new_cell_message({}, payload) if blocking: - self.logger.info(f"request payload: {request.payload} from {self.cell.get_fqcn()} to {self.target_fqcn}") + self.logger.info(f"send_request from {self.cell.get_fqcn()} to {self.target_fqcn}") reply = self.cell.send_request( channel=MSG_CHANNEL, @@ -78,6 +78,7 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): return result else: # fire and forget + self.logger.info(f"fire_and_forget from {self.cell.get_fqcn()} to {self.target_fqcn}") self.cell.fire_and_forget( channel=MSG_CHANNEL, topic=MSG_TOPIC, diff --git a/nvflare/focs/sys/constants.py b/nvflare/focs/sys/constants.py index 3f80c7edec..ac25a92e92 100644 --- a/nvflare/focs/sys/constants.py +++ b/nvflare/focs/sys/constants.py @@ -18,7 +18,7 @@ class SyncKey: - TARGET_OBJ_NAMES = "target_obj_names" + COLLAB_OBJ_NAMES = "collab_obj_names" class ObjectCallKey: diff --git a/nvflare/focs/sys/controller.py b/nvflare/focs/sys/controller.py index 7644a5a698..a3087ac868 100644 --- a/nvflare/focs/sys/controller.py +++ b/nvflare/focs/sys/controller.py @@ -14,7 +14,7 @@ import time import traceback from concurrent.futures import ThreadPoolExecutor -from typing import Dict, List +from typing import List from nvflare.apis.client import Client as ClientSite from nvflare.apis.controller_spec import Client, ClientTask, Task @@ -22,7 +22,6 @@ from nvflare.apis.impl.controller import Controller from nvflare.apis.shareable import ReturnCode, Shareable from nvflare.apis.signal import Signal -from nvflare.app_common.decomposers.numpy_decomposers import register from nvflare.focs.api.app import ServerApp from nvflare.focs.api.constants import ContextKey from nvflare.focs.api.proxy import Proxy @@ -51,14 +50,16 @@ def __init__( self, strategy_ids: List[str], server_app_id: str = None, - server_target_obj_ids: Dict[str, str] = None, + server_collab_obj_ids: List[str] = None, sync_task_timeout=2, max_call_threads=100, ): Controller.__init__(self) + if not server_collab_obj_ids: + server_collab_obj_ids = [] self.server_app_id = server_app_id # component name self.strategy_ids = strategy_ids # component names - self.server_target_obj_ids = server_target_obj_ids # component IDs + self.server_collab_obj_ids = server_collab_obj_ids # component IDs self.sync_task_timeout = sync_task_timeout self.server_app = None self.client_info = {} # client name => _ClientInfo @@ -69,7 +70,6 @@ def __init__( raise ValueError(f"no strategies defined - there must be at least one strategy") def start_controller(self, fl_ctx: FLContext): - register() engine = fl_ctx.get_engine() if self.server_app_id: @@ -89,14 +89,14 @@ def start_controller(self, fl_ctx: FLContext): app.add_strategy(strategy) - if self.server_target_obj_ids: - for name, cid in self.server_target_obj_ids: + if self.server_collab_obj_ids: + for cid in self.server_collab_obj_ids: obj = engine.get_component(cid) if not obj: self.system_panic(f"component {cid} does not exist", fl_ctx) return - app.add_target_object(name, obj) + app.add_collab_object(cid, obj) self.server_app = app @@ -137,23 +137,17 @@ def _prepare_server_proxy(self, job_id, abort_signal): backend = self._prepare_server_backend(job_id, abort_signal) proxy = Proxy(app=self.server_app, target_name=server_name, backend=backend, caller_name=server_name) - tos = self.server_app.get_target_objects() - if tos: - for name in tos.keys(): - p = Proxy( - app=self.server_app, target_name=f"{server_name}.{name}", backend=backend, caller_name=server_name - ) - setattr(proxy, name, p) + for name in self.server_collab_obj_ids: + p = Proxy( + app=self.server_app, target_name=f"{server_name}.{name}", backend=backend, caller_name=server_name + ) + setattr(proxy, name, p) return proxy def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): # configure all sites - if self.server_target_obj_ids: - target_obj_names = list(self.server_target_obj_ids.keys()) - else: - target_obj_names = [] - - task_data = Shareable({SyncKey.TARGET_OBJ_NAMES: target_obj_names}) + collab_obj_names = self.server_collab_obj_ids + task_data = Shareable({SyncKey.COLLAB_OBJ_NAMES: collab_obj_names}) task = Task( name=SYNC_TASK_NAME, data=task_data, @@ -232,7 +226,7 @@ def _process_sync_reply(self, client_task: ClientTask, fl_ctx: FLContext): rc = result.get_return_code() if rc == ReturnCode.OK: self.log_info(fl_ctx, f"successfully synced client {client_name}") - target_obj_names = result.get(SyncKey.TARGET_OBJ_NAMES) + target_obj_names = result.get(SyncKey.COLLAB_OBJ_NAMES) if not target_obj_names: target_obj_names = [] self.client_info[client_name] = _ClientInfo(target_obj_names) diff --git a/nvflare/focs/sys/executor.py b/nvflare/focs/sys/executor.py index 197a159683..8fcfc2aa54 100644 --- a/nvflare/focs/sys/executor.py +++ b/nvflare/focs/sys/executor.py @@ -66,7 +66,7 @@ def _handle_start_run(self, event_type: str, fl_ctx: FLContext): self.system_panic(f"component {cid} does not exist", fl_ctx) return - self.client_app.add_target_object(name, obj) + self.client_app.add_collab_object(name, obj) def _prepare_server_proxy(self, job_id, cell, server_target_obj_names, abort_signal): my_name = self.client_app.name @@ -112,7 +112,7 @@ def execute(self, task_name: str, shareable: Shareable, fl_ctx: FLContext, abort self.log_error(fl_ctx, f"received unsupported task {task_name}") return make_reply(ReturnCode.TASK_UNKNOWN) - server_target_obj_names = shareable.get(SyncKey.TARGET_OBJ_NAMES) + server_target_obj_names = shareable.get(SyncKey.COLLAB_OBJ_NAMES) engine = fl_ctx.get_engine() cell = engine.get_cell() @@ -145,5 +145,5 @@ def execute(self, task_name: str, shareable: Shareable, fl_ctx: FLContext, abort target_obj_names = list(self.client_target_obj_ids.keys()) else: target_obj_names = [] - reply[SyncKey.TARGET_OBJ_NAMES] = target_obj_names + reply[SyncKey.COLLAB_OBJ_NAMES] = target_obj_names return reply diff --git a/nvflare/focs/sys/utils.py b/nvflare/focs/sys/utils.py index 3a0edb836b..67398148a1 100644 --- a/nvflare/focs/sys/utils.py +++ b/nvflare/focs/sys/utils.py @@ -71,7 +71,7 @@ def _call_app_method(request: Message, app: App, logger) -> Message: if len(parts) >= 2: obj_name = parts[1] if obj_name: - target_objs = app.get_target_objects() + target_objs = app.get_collab_objects() target_obj = target_objs.get(obj_name) logger.info(f"calling target obj: {app.name}.{obj_name}") else: From dea452140ba0a13cf02b8011a497edba12b4d1c8 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Mon, 29 Sep 2025 12:38:56 -0400 Subject: [PATCH 015/102] support collab signature --- nvflare/focs/api/app.py | 21 ++++++++++++++---- nvflare/focs/api/dec.py | 23 +++++++++++++++++--- nvflare/focs/api/proxy.py | 25 ++++++++++++++++++---- nvflare/focs/examples/np/cyclic.py | 4 +++- nvflare/focs/examples/np/cyclic_avg.py | 8 +++++-- nvflare/focs/examples/np/fed_avg_intime.py | 8 ++++++- nvflare/focs/examples/np/fed_avg_para.py | 5 ++++- nvflare/focs/examples/np/fed_avg_seq.py | 3 ++- nvflare/focs/examples/np/swarm.py | 14 +++++++++--- nvflare/focs/sim/runner.py | 10 +++++++-- nvflare/focs/sys/controller.py | 9 +++----- 11 files changed, 102 insertions(+), 28 deletions(-) diff --git a/nvflare/focs/api/app.py b/nvflare/focs/api/app.py index d21e0b1c21..a1ba95984f 100644 --- a/nvflare/focs/api/app.py +++ b/nvflare/focs/api/app.py @@ -17,7 +17,7 @@ from .constants import CollabMethodArgName from .ctx import Context -from .dec import collab, is_collab +from .dec import collab, get_object_collab_signature, is_collab from .proxy import Proxy from .strategy import Strategy from .utils import check_context_support @@ -45,6 +45,10 @@ def get_default_collab_object(self): return None def add_collab_object(self, name: str, obj): + for n, o in self._collab_objs: + if name == n: + raise ValueError(f"conflict with existing collab object '{name}' of {type(o)}") + if hasattr(obj, name): raise ValueError(f"conflict with reserved name {name}") @@ -129,6 +133,14 @@ def register_event_handler(self, event_type: str, handler, **handler_kwargs): self._event_handlers[event_type] = handlers handlers.append((handler, handler_kwargs)) + def get_collab_signature(self): + results = {"": get_object_collab_signature(self)} + + for name, obj in self._collab_objs: + results[name] = get_object_collab_signature(obj) + + print(f"get_collab_signature: {results}") + @collab def fire_event(self, event_type: str, data, context: Context): for e, handlers in self._event_handlers.items(): @@ -142,7 +154,7 @@ def fire_event(self, event_type: str, data, context: Context): class ServerApp(App): - def __init__(self, strategy: Strategy = None): + def __init__(self, strategy_name: str, strategy: Strategy = None): super().__init__() if strategy and not isinstance(strategy, Strategy): @@ -150,13 +162,14 @@ def __init__(self, strategy: Strategy = None): self.strategies = [] if strategy: - self.strategies.append(strategy) + self.add_strategy(strategy_name, strategy) self.current_strategy = None - def add_strategy(self, strategy): + def add_strategy(self, strategy_name: str, strategy): if not isinstance(strategy, Strategy): raise ValueError(f"strategy must be Controller but got {type(strategy)}") self.strategies.append(strategy) + self.add_collab_object(strategy_name, strategy) def get_default_collab_object(self): return self.current_strategy diff --git a/nvflare/focs/api/dec.py b/nvflare/focs/api/dec.py index a8b1bafe29..704b422d87 100644 --- a/nvflare/focs/api/dec.py +++ b/nvflare/focs/api/dec.py @@ -15,8 +15,9 @@ from .constants import CollabMethodArgName -_ATTR_COLLAB = "_is_fox_collab" -_ATTR_SUPPORT_CTX = "_supports_fox_ctx" +_ATTR_COLLAB = "_fox_is_collab" +_ATTR_SUPPORT_CTX = "_fox_supports_ctx" +_ATTR_PARAM_NAMES = "_fox_param_names" def collab(func): @@ -24,13 +25,20 @@ def wrapper(*args, **kwargs): return func(*args, **kwargs) signature = inspect.signature(func) - parameter_names = signature.parameters.keys() + parameter_names = list(signature.parameters.keys()) + if "self" in parameter_names: + parameter_names.remove("self") + setattr(wrapper, _ATTR_PARAM_NAMES, parameter_names) if CollabMethodArgName.CONTEXT in parameter_names: setattr(wrapper, _ATTR_SUPPORT_CTX, True) setattr(wrapper, _ATTR_COLLAB, True) return wrapper +def get_param_names(func): + return getattr(func, _ATTR_PARAM_NAMES, None) + + def is_collab(func): return hasattr(func, _ATTR_COLLAB) @@ -42,3 +50,12 @@ def supports_context(func): def adjust_kwargs(func, kwargs): if not supports_context(func): kwargs.pop(CollabMethodArgName.CONTEXT, None) + + +def get_object_collab_signature(obj): + result = {} + for name in dir(obj): + func = getattr(obj, name) + if callable(func) and is_collab(func): + result[name] = get_param_names(func) + return result diff --git a/nvflare/focs/api/proxy.py b/nvflare/focs/api/proxy.py index 58c241df5a..aaf12e2055 100644 --- a/nvflare/focs/api/proxy.py +++ b/nvflare/focs/api/proxy.py @@ -11,18 +11,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import copy + from .backend import Backend from .constants import CollabMethodArgName class Proxy: - def __init__(self, app, target_name, backend: Backend, caller_name: str): + def __init__(self, app, target_name, backend: Backend, target_signature=None): """The Proxy represents a target in the App.""" self.app = app self.target_name = target_name self.backend = backend - self.caller_name = caller_name + self.caller_name = app.name + self.target_signature = target_signature @property def name(self): @@ -43,8 +46,22 @@ def __getattr__(self, func_name): """ def method(*args, **kwargs): + call_args = args + call_kwargs = kwargs + + if self.target_signature: + arg_names = self.target_signature.get(func_name) + if arg_names: + # check args and turn them to kwargs + call_kwargs = copy.copy(kwargs) + call_args = [] + for i, arg_value in enumerate(args): + call_kwargs[arg_names[i]] = arg_value + ctx = self.app.new_context(self.caller_name, self.name) - kwargs[CollabMethodArgName.CONTEXT] = ctx - return self.backend.call_target(self.target_name, func_name, *args, **kwargs) + call_kwargs[CollabMethodArgName.CONTEXT] = ctx + + print(f"calling target {self.target_name} func {func_name}: {call_args=} {call_kwargs=}") + return self.backend.call_target(self.target_name, func_name, *call_args, **call_kwargs) return method diff --git a/nvflare/focs/examples/np/cyclic.py b/nvflare/focs/examples/np/cyclic.py index fa1d670b99..ae5da38852 100644 --- a/nvflare/focs/examples/np/cyclic.py +++ b/nvflare/focs/examples/np/cyclic.py @@ -20,7 +20,9 @@ def main(): runner = AppRunner( - server_app=ServerApp(strategy=NPCyclic(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2)), + server_app=ServerApp( + strategy_name="cyclic", strategy=NPCyclic(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2) + ), client_app=NPTrainer(delta=1.0), num_clients=2, ) diff --git a/nvflare/focs/examples/np/cyclic_avg.py b/nvflare/focs/examples/np/cyclic_avg.py index a3d62e240e..401fd04224 100644 --- a/nvflare/focs/examples/np/cyclic_avg.py +++ b/nvflare/focs/examples/np/cyclic_avg.py @@ -18,8 +18,12 @@ def main(): - server_app = ServerApp(NPCyclic(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2)) - server_app.add_strategy(NPFedAvgParallel(initial_model=None, num_rounds=2)) + server_app = ServerApp( + strategy_name="cyclic", strategy=NPCyclic(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2) + ) + server_app.add_strategy("fed_avg_parallel", NPFedAvgParallel(initial_model=None, num_rounds=2)) + + server_app.get_collab_signature() runner = AppRunner( server_app=server_app, diff --git a/nvflare/focs/examples/np/fed_avg_intime.py b/nvflare/focs/examples/np/fed_avg_intime.py index e2ef3c5937..8e9844e9d0 100644 --- a/nvflare/focs/examples/np/fed_avg_intime.py +++ b/nvflare/focs/examples/np/fed_avg_intime.py @@ -20,9 +20,15 @@ def main(): - server_app = ServerApp(strategy=NPFedAvgInTime(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2)) + server_app = ServerApp( + strategy_name="fed_avg_in_time", + strategy=NPFedAvgInTime(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), + ) + server_app.add_collab_object("metric_receiver", MetricReceiver()) + server_app.get_collab_signature() + runner = AppRunner( server_app=server_app, client_app=NPTrainer(delta=1.0), diff --git a/nvflare/focs/examples/np/fed_avg_para.py b/nvflare/focs/examples/np/fed_avg_para.py index 4bee611a41..7caf5d9b48 100644 --- a/nvflare/focs/examples/np/fed_avg_para.py +++ b/nvflare/focs/examples/np/fed_avg_para.py @@ -20,7 +20,10 @@ def main(): - server_app = ServerApp(strategy=NPFedAvgParallel(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2)) + server_app = ServerApp( + strategy_name="fed_avg", + strategy=NPFedAvgParallel(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), + ) server_app.add_collab_object("metric_receiver", MetricReceiver()) runner = AppRunner( diff --git a/nvflare/focs/examples/np/fed_avg_seq.py b/nvflare/focs/examples/np/fed_avg_seq.py index d5e6ccb620..0f8d463097 100644 --- a/nvflare/focs/examples/np/fed_avg_seq.py +++ b/nvflare/focs/examples/np/fed_avg_seq.py @@ -21,10 +21,11 @@ def main(): server_app = ServerApp( + strategy_name="fed_avg", strategy=NPFedAvgSequential( num_rounds=2, initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], - ) + ), ) server_app.add_collab_object("metric_receiver", MetricReceiver()) diff --git a/nvflare/focs/examples/np/swarm.py b/nvflare/focs/examples/np/swarm.py index 105b83f606..f2de8c3146 100644 --- a/nvflare/focs/examples/np/swarm.py +++ b/nvflare/focs/examples/np/swarm.py @@ -72,7 +72,7 @@ def swarm_learn(self, num_rounds, model, current_round, context: Context): if current_round == num_rounds - 1: # all done all_clients(context, blocking=False).fire_event("final_model", new_model) - self.server.notify_done() + self.server.swarm.notify_done() return # determine next client @@ -93,9 +93,17 @@ def _accept_final_model(self, event_type: str, model, context: Context): def main(): + server_app = ServerApp( + strategy_name="swarm", strategy=NPSwarm(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=5) + ) + server_app.get_collab_signature() + + client_app = NPSwarmClient(delta=1.0) + client_app.get_collab_signature() + runner = AppRunner( - server_app=ServerApp(strategy=NPSwarm(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=5)), - client_app=NPSwarmClient(delta=1.0), + server_app=server_app, + client_app=client_app, num_clients=3, ) diff --git a/nvflare/focs/sim/runner.py b/nvflare/focs/sim/runner.py index 6ccc668c8e..8ffa54e0f1 100644 --- a/nvflare/focs/sim/runner.py +++ b/nvflare/focs/sim/runner.py @@ -19,6 +19,7 @@ from nvflare.apis.signal import Signal from nvflare.focs.api.app import App, ClientApp, ClientAppFactory, ServerApp from nvflare.focs.api.constants import ContextKey +from nvflare.focs.api.dec import get_object_collab_signature from nvflare.focs.api.proxy import Proxy from nvflare.focs.sim.backend import SimBackend @@ -34,7 +35,12 @@ def _prepare_app_backends(self, app: App): return bes def _prepare_proxy(self, for_app: App, target_app: App, backends: dict): - app_proxy = Proxy(app=for_app, target_name=target_app.name, backend=backends[""], caller_name=for_app.name) + app_proxy = Proxy( + app=for_app, + target_name=target_app.name, + backend=backends[""], + target_signature=get_object_collab_signature(target_app), + ) tos = target_app.get_collab_objects() if tos: for name, obj in tos: @@ -42,7 +48,7 @@ def _prepare_proxy(self, for_app: App, target_app: App, backends: dict): app=for_app, target_name=f"{target_app.name}.{name}", backend=backends[name], - caller_name=for_app.name, + target_signature=get_object_collab_signature(obj), ) setattr(app_proxy, name, p) return app_proxy diff --git a/nvflare/focs/sys/controller.py b/nvflare/focs/sys/controller.py index a3087ac868..092871750f 100644 --- a/nvflare/focs/sys/controller.py +++ b/nvflare/focs/sys/controller.py @@ -120,14 +120,13 @@ def _prepare_server_backend(self, job_id: str, abort_signal: Signal): def _prepare_client_proxy(self, job_id: str, client: ClientSite, target_obj_names: List[str], abort_signal): backend = self._prepare_client_backend(job_id, client, abort_signal) - proxy = Proxy(app=self.server_app, target_name=client.name, backend=backend, caller_name=self.server_app.name) + proxy = Proxy(app=self.server_app, target_name=client.name, backend=backend) for name in target_obj_names: p = Proxy( app=self.server_app, target_name=f"{client.name}.{name}", backend=backend, - caller_name=self.server_app.name, ) setattr(proxy, name, p) return proxy @@ -135,12 +134,10 @@ def _prepare_client_proxy(self, job_id: str, client: ClientSite, target_obj_name def _prepare_server_proxy(self, job_id, abort_signal): server_name = self.server_app.name backend = self._prepare_server_backend(job_id, abort_signal) - proxy = Proxy(app=self.server_app, target_name=server_name, backend=backend, caller_name=server_name) + proxy = Proxy(app=self.server_app, target_name=server_name, backend=backend) for name in self.server_collab_obj_ids: - p = Proxy( - app=self.server_app, target_name=f"{server_name}.{name}", backend=backend, caller_name=server_name - ) + p = Proxy(app=self.server_app, target_name=f"{server_name}.{name}", backend=backend) setattr(proxy, name, p) return proxy From 0812ad04bd13a7c70d4e9af43e62417551e9eba2 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Mon, 29 Sep 2025 15:46:53 -0400 Subject: [PATCH 016/102] use collab signature --- nvflare/focs/api/app.py | 11 ++- nvflare/focs/api/group.py | 21 +++-- nvflare/focs/api/proxy.py | 30 +++++--- nvflare/focs/examples/np/algos/client.py | 6 +- nvflare/focs/examples/np/algos/swarm.py | 98 ++++++++++++++++++++++++ nvflare/focs/examples/np/swarm.py | 83 +------------------- nvflare/focs/sys/backend.py | 6 +- nvflare/focs/sys/constants.py | 2 +- nvflare/focs/sys/controller.py | 63 +++++++++------ nvflare/focs/sys/executor.py | 40 ++++++---- nvflare/focs/sys/utils.py | 6 +- 11 files changed, 217 insertions(+), 149 deletions(-) create mode 100644 nvflare/focs/examples/np/algos/swarm.py diff --git a/nvflare/focs/api/app.py b/nvflare/focs/api/app.py index a1ba95984f..057c6dfe15 100644 --- a/nvflare/focs/api/app.py +++ b/nvflare/focs/api/app.py @@ -134,12 +134,13 @@ def register_event_handler(self, event_type: str, handler, **handler_kwargs): handlers.append((handler, handler_kwargs)) def get_collab_signature(self): - results = {"": get_object_collab_signature(self)} + result = {"": get_object_collab_signature(self)} for name, obj in self._collab_objs: - results[name] = get_object_collab_signature(obj) + result[name] = get_object_collab_signature(obj) - print(f"get_collab_signature: {results}") + print(f"get_collab_signature: {result}") + return result @collab def fire_event(self, event_type: str, data, context: Context): @@ -154,7 +155,7 @@ def fire_event(self, event_type: str, data, context: Context): class ServerApp(App): - def __init__(self, strategy_name: str, strategy: Strategy = None): + def __init__(self, strategy_name: str = "strategy", strategy: Strategy = None): super().__init__() if strategy and not isinstance(strategy, Strategy): @@ -162,6 +163,8 @@ def __init__(self, strategy_name: str, strategy: Strategy = None): self.strategies = [] if strategy: + if not strategy_name: + raise ValueError("missing strategy name") self.add_strategy(strategy_name, strategy) self.current_strategy = None diff --git a/nvflare/focs/api/group.py b/nvflare/focs/api/group.py index 711b05a1cf..c2b1e99c01 100644 --- a/nvflare/focs/api/group.py +++ b/nvflare/focs/api/group.py @@ -32,12 +32,15 @@ def __init__( abort_signal: Signal, proxies: List[Proxy], blocking: bool = True, - timeout: float = None, + timeout: float = 5.0, min_resps: int = None, wait_after_min_resps: float = None, process_resp_cb=None, **cb_kwargs, ): + if not proxies: + raise ValueError("no proxies to group") + self._app = app self._abort_signal = abort_signal self._proxies = proxies @@ -61,17 +64,23 @@ def __getattr__(self, func_name): def method(*args, **kwargs): resps = {} + call_args, call_kwargs = self._proxies[0].adjust_func_args(func_name, args, kwargs) + for p in self._proxies: - kwargs_copy = copy.copy(kwargs) + kwargs_copy = copy.copy(call_kwargs) ctx = self._app.new_context(p.caller_name, p.name) kwargs_copy[CollabMethodArgName.CONTEXT] = ctx # set the optional args to help backend decide how to call - kwargs_copy[CollabMethodOptionName.TIMEOUT] = self._timeout + if self._timeout: + kwargs_copy[CollabMethodOptionName.TIMEOUT] = self._timeout + kwargs_copy[CollabMethodOptionName.BLOCKING] = self._blocking resp = Resp(self._process_resp_cb, self._cb_kwargs, ctx) resps[p.name] = resp - p.backend.call_target_with_resp(resp, p.name, func_name, *args, **kwargs_copy) + + print(f"group call: {func_name=} args={call_args} kwargs={kwargs_copy}") + p.backend.call_target_with_resp(resp, p.name, func_name, *call_args, **kwargs_copy) # wait for responses if not self._blocking: @@ -126,7 +135,7 @@ def group( ctx: Context, proxies: List[Proxy], blocking: bool = True, - timeout: float = None, + timeout: float = 5.0, min_resps: int = None, wait_after_min_resps: float = None, process_resp_cb=None, @@ -148,7 +157,7 @@ def group( def all_clients( ctx: Context, blocking: bool = True, - timeout: float = None, + timeout: float = 5.0, min_resps: int = None, wait_after_min_resps: float = None, process_resp_cb=None, diff --git a/nvflare/focs/api/proxy.py b/nvflare/focs/api/proxy.py index aaf12e2055..e811e62648 100644 --- a/nvflare/focs/api/proxy.py +++ b/nvflare/focs/api/proxy.py @@ -40,24 +40,30 @@ def get_target(self, name: str): else: return None + def adjust_func_args(self, func_name, args, kwargs): + call_args = args + call_kwargs = kwargs + + if self.target_signature: + arg_names = self.target_signature.get(func_name) + if arg_names: + # check args and turn them to kwargs + call_kwargs = copy.copy(kwargs) + call_args = [] + for i, arg_value in enumerate(args): + call_kwargs[arg_names[i]] = arg_value + + ctx = self.app.new_context(self.caller_name, self.name) + call_kwargs[CollabMethodArgName.CONTEXT] = ctx + return call_args, call_kwargs + def __getattr__(self, func_name): """ This method is called when Python cannot find an invoked method func_name of this class. """ def method(*args, **kwargs): - call_args = args - call_kwargs = kwargs - - if self.target_signature: - arg_names = self.target_signature.get(func_name) - if arg_names: - # check args and turn them to kwargs - call_kwargs = copy.copy(kwargs) - call_args = [] - for i, arg_value in enumerate(args): - call_kwargs[arg_names[i]] = arg_value - + call_args, call_kwargs = self.adjust_func_args(func_name, args, kwargs) ctx = self.app.new_context(self.caller_name, self.name) call_kwargs[CollabMethodArgName.CONTEXT] = ctx diff --git a/nvflare/focs/examples/np/algos/client.py b/nvflare/focs/examples/np/algos/client.py index ba30ac24f8..8c38d4f1af 100644 --- a/nvflare/focs/examples/np/algos/client.py +++ b/nvflare/focs/examples/np/algos/client.py @@ -25,17 +25,17 @@ def __init__(self, delta: float): self.delta = delta @collab - def train(self, r, weights, context: Context): + def train(self, current_round, weights, context: Context): if context.is_aborted(): print("training aborted") return 0 - print(f"[{self.name}] called by {context.caller}: client {context.callee} trained round {r}") + print(f"[{self.name}] called by {context.caller}: client {context.callee} trained round {current_round}") # metric_receiver = self.server.get_target("metric_receiver") # if metric_receiver: # self.server.accept_metric({"round": r, "y": 2}) # - self.server.fire_event("metrics", {"round": r, "y": 10}, blocking=False) + self.server.fire_event("metrics", {"round": current_round, "y": 10}, blocking=False) return weights + self.delta @collab diff --git a/nvflare/focs/examples/np/algos/swarm.py b/nvflare/focs/examples/np/algos/swarm.py new file mode 100644 index 0000000000..6182feb1df --- /dev/null +++ b/nvflare/focs/examples/np/algos/swarm.py @@ -0,0 +1,98 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import random +import threading + +from nvflare.focs.api.app import ClientApp, ServerApp +from nvflare.focs.api.ctx import Context +from nvflare.focs.api.dec import collab +from nvflare.focs.api.group import all_clients +from nvflare.focs.api.strategy import Strategy +from nvflare.focs.examples.np.algos.utils import parse_array_def +from nvflare.focs.sim.runner import AppRunner + + +class NPSwarm(Strategy): + + def __init__(self, initial_model, num_rounds=10): + self.num_rounds = num_rounds + self.initial_model = parse_array_def(initial_model) + self.waiter = threading.Event() + + def execute(self, context: Context): + context.app.register_event_handler("all_done", self._all_done) + + # randomly pick a client to start + start_client_idx = random.randint(0, len(context.clients) - 1) + start_client = context.clients[start_client_idx] + start_client.start(self.num_rounds, self.initial_model) + self.waiter.wait() + + def _all_done(self, event_type: str, data, context: Context): + print(f"[{context.callee}]: received {event_type} from client: {context.caller}: {data}") + self.all_done(data, context) + + @collab + def all_done(self, reason: str, context: Context): + print(f"[{context.callee}]: all done from client: {context.caller}: {reason}") + self.waiter.set() + + +class NPSwarmClient(ClientApp): + + def __init__(self, delta: float): + super().__init__() + self.delta = delta + self.register_event_handler("final_model", self._accept_final_model) + + @collab + def train(self, weights, current_round, context: Context): + print(f"[{context.callee}]: train asked by {context.caller}: {current_round=}") + return weights + self.delta + + def sag(self, model, current_round, ctx: Context): + results = all_clients(ctx, blocking=True).train(model, current_round) + results = list(results.values()) + total = results[0] + for i in range(1, len(results)): + total += results[i] + return total / len(results) + + @collab + def swarm_learn(self, num_rounds, model, current_round, context: Context): + print(f"[{context.callee}]: swarm learn asked by {context.caller}: {num_rounds=} {current_round=} {model=}") + new_model = self.sag(model, current_round, context) + + print(f"[{context.callee}]: trained model {new_model=}") + if current_round == num_rounds - 1: + # all done + all_clients(context, blocking=False).fire_event("final_model", new_model) + # self.server.fire_event("all_done", "OK", blocking=False) + self.server.strategy.all_done("OK", blocking=False) + return + + # determine next client + next_round = current_round + 1 + next_client_idx = random.randint(0, len(self.clients) - 1) + next_client = self.clients[next_client_idx] + next_client.swarm_learn(num_rounds, new_model, next_round, blocking=False) + + @collab + def start(self, num_rounds, initial_model, context: Context): + self.swarm_learn(num_rounds, initial_model, 0, context) + + def _accept_final_model(self, event_type: str, model, context: Context): + # accept the final model + # write model to disk + print(f"[{context.callee}]: received event '{event_type}' from {context.caller}: {model}") diff --git a/nvflare/focs/examples/np/swarm.py b/nvflare/focs/examples/np/swarm.py index f2de8c3146..d220cb5816 100644 --- a/nvflare/focs/examples/np/swarm.py +++ b/nvflare/focs/examples/np/swarm.py @@ -11,95 +11,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import random -import threading -from nvflare.focs.api.app import ClientApp, ServerApp -from nvflare.focs.api.ctx import Context -from nvflare.focs.api.dec import collab -from nvflare.focs.api.group import all_clients -from nvflare.focs.api.strategy import Strategy -from nvflare.focs.examples.np.algos.utils import parse_array_def +from nvflare.focs.api.app import ServerApp +from nvflare.focs.examples.np.algos.swarm import NPSwarm, NPSwarmClient from nvflare.focs.sim.runner import AppRunner -class NPSwarm(Strategy): - - def __init__(self, initial_model, num_rounds=10): - self.num_rounds = num_rounds - self.initial_model = parse_array_def(initial_model) - self.waiter = threading.Event() - - def execute(self, context: Context): - # randomly pick a client to start - start_client_idx = random.randint(0, len(context.clients) - 1) - start_client = context.clients[start_client_idx] - start_client.start(self.num_rounds, self.initial_model) - self.waiter.wait() - - @collab - def notify_done(self, context: Context): - print(f"[{context.callee}]: received DONE from client: {context.caller}") - self.waiter.set() - - -class NPSwarmClient(ClientApp): - - def __init__(self, delta: float): - super().__init__() - self.delta = delta - self.register_event_handler("final_model", self._accept_final_model) - - @collab - def train(self, weights, current_round, context: Context): - print(f"[{context.callee}]: train asked by {context.caller}: {current_round=}") - return weights + self.delta - - def sag(self, model, current_round, ctx: Context): - results = all_clients(ctx, blocking=True).train(model, current_round) - results = list(results.values()) - total = results[0] - for i in range(1, len(results)): - total += results[i] - return total / len(results) - - @collab - def swarm_learn(self, num_rounds, model, current_round, context: Context): - print(f"[{context.callee}]: swarm learn asked by {context.caller}: {num_rounds=} {current_round=} {model=}") - new_model = self.sag(model, current_round, context) - - print(f"[{context.callee}]: trained model {new_model=}") - if current_round == num_rounds - 1: - # all done - all_clients(context, blocking=False).fire_event("final_model", new_model) - self.server.swarm.notify_done() - return - - # determine next client - next_round = current_round + 1 - next_client_idx = random.randint(0, len(self.clients) - 1) - next_client = self.clients[next_client_idx] - next_client.swarm_learn(num_rounds, new_model, next_round, blocking=False) - - @collab - def start(self, num_rounds, initial_model, context: Context): - self.swarm_learn(num_rounds, initial_model, 0, context) - - def _accept_final_model(self, event_type: str, model, context: Context): - # accept the final model - # write model to disk - print(f"[{context.callee}]: received event '{event_type}' from {context.caller}: {model}") - - def main(): server_app = ServerApp( - strategy_name="swarm", strategy=NPSwarm(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=5) + strategy_name="strategy", strategy=NPSwarm(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=5) ) - server_app.get_collab_signature() - client_app = NPSwarmClient(delta=1.0) - client_app.get_collab_signature() runner = AppRunner( server_app=server_app, diff --git a/nvflare/focs/sys/backend.py b/nvflare/focs/sys/backend.py index e70553acba..b11d04f2b4 100644 --- a/nvflare/focs/sys/backend.py +++ b/nvflare/focs/sys/backend.py @@ -47,7 +47,7 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): request = new_cell_message({}, payload) if blocking: - self.logger.info(f"send_request from {self.cell.get_fqcn()} to {self.target_fqcn}") + self.logger.info(f"send_request from {self.cell.get_fqcn()} to {self.target_fqcn}: {payload=} {timeout=}") reply = self.cell.send_request( channel=MSG_CHANNEL, @@ -89,9 +89,9 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): ) def call_target_with_resp(self, resp: Resp, target_name: str, func_name: str, *args, **kwargs): - self.thread_executor.submit(self._run_func, resp, target_name, args, kwargs) + self.thread_executor.submit(self._run_func, resp, target_name, func_name, args, kwargs) - def _run_func(self, resp: Resp, target_name: str, func_name: str, *args, **kwargs): + def _run_func(self, resp: Resp, target_name: str, func_name: str, args, kwargs): try: result = self.call_target(target_name, func_name, *args, **kwargs) resp.set_result(result) diff --git a/nvflare/focs/sys/constants.py b/nvflare/focs/sys/constants.py index ac25a92e92..6490aa86ff 100644 --- a/nvflare/focs/sys/constants.py +++ b/nvflare/focs/sys/constants.py @@ -18,7 +18,7 @@ class SyncKey: - COLLAB_OBJ_NAMES = "collab_obj_names" + COLLAB_SIGNATURE = "collab_signature" class ObjectCallKey: diff --git a/nvflare/focs/sys/controller.py b/nvflare/focs/sys/controller.py index 092871750f..1597054682 100644 --- a/nvflare/focs/sys/controller.py +++ b/nvflare/focs/sys/controller.py @@ -35,13 +35,13 @@ class _ClientInfo: - def __init__(self, target_obj_names): + def __init__(self, collab_signature: dict): """Information about a client. Reported by the client in the sync response. Args: - target_obj_names: names of target objects on the client. + collab_signature: collab method signature of the client. """ - self.target_obj_names = target_obj_names + self.collab_signature = collab_signature class FocsController(Controller): @@ -87,7 +87,7 @@ def start_controller(self, fl_ctx: FLContext): self.system_panic(f"component {cid} must be Strategy but got {type(strategy)}", fl_ctx) return - app.add_strategy(strategy) + app.add_strategy(cid, strategy) if self.server_collab_obj_ids: for cid in self.server_collab_obj_ids: @@ -118,33 +118,52 @@ def _prepare_server_backend(self, job_id: str, abort_signal: Signal): thread_executor=self.thread_executor, ) - def _prepare_client_proxy(self, job_id: str, client: ClientSite, target_obj_names: List[str], abort_signal): + def _prepare_client_proxy( + self, + job_id: str, + client: ClientSite, + collab_signature: dict, + abort_signal, + ): backend = self._prepare_client_backend(job_id, client, abort_signal) - proxy = Proxy(app=self.server_app, target_name=client.name, backend=backend) + proxy = Proxy( + app=self.server_app, target_name=client.name, backend=backend, target_signature=collab_signature.get("") + ) - for name in target_obj_names: - p = Proxy( - app=self.server_app, - target_name=f"{client.name}.{name}", - backend=backend, - ) + for name, sig in collab_signature.items(): + if name == "": + continue + + p = Proxy(app=self.server_app, target_name=f"{client.name}.{name}", backend=backend, target_signature=sig) setattr(proxy, name, p) return proxy - def _prepare_server_proxy(self, job_id, abort_signal): + def _prepare_server_proxy( + self, + job_id, + abort_signal, + collab_signature: dict, + ): server_name = self.server_app.name backend = self._prepare_server_backend(job_id, abort_signal) - proxy = Proxy(app=self.server_app, target_name=server_name, backend=backend) + proxy = Proxy( + app=self.server_app, target_name=server_name, backend=backend, target_signature=collab_signature.get("") + ) for name in self.server_collab_obj_ids: - p = Proxy(app=self.server_app, target_name=f"{server_name}.{name}", backend=backend) + p = Proxy( + app=self.server_app, + target_name=f"{server_name}.{name}", + backend=backend, + target_signature=collab_signature.get(name), + ) setattr(proxy, name, p) return proxy def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): # configure all sites - collab_obj_names = self.server_collab_obj_ids - task_data = Shareable({SyncKey.COLLAB_OBJ_NAMES: collab_obj_names}) + server_collab_signature = self.server_app.get_collab_signature() + task_data = Shareable({SyncKey.COLLAB_SIGNATURE: server_collab_signature}) task = Task( name=SYNC_TASK_NAME, data=task_data, @@ -190,12 +209,12 @@ def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): # prepare proxies and backends job_id = fl_ctx.get_job_id() - server_proxy = self._prepare_server_proxy(job_id, abort_signal) + server_proxy = self._prepare_server_proxy(job_id, abort_signal, server_collab_signature) client_proxies = [] for c in all_clients: info = self.client_info[c.name] assert isinstance(info, _ClientInfo) - client_proxies.append(self._prepare_client_proxy(job_id, c, info.target_obj_names, abort_signal)) + client_proxies.append(self._prepare_client_proxy(job_id, c, info.collab_signature, abort_signal)) self.server_app.setup(server_proxy, client_proxies, abort_signal) @@ -223,10 +242,8 @@ def _process_sync_reply(self, client_task: ClientTask, fl_ctx: FLContext): rc = result.get_return_code() if rc == ReturnCode.OK: self.log_info(fl_ctx, f"successfully synced client {client_name}") - target_obj_names = result.get(SyncKey.COLLAB_OBJ_NAMES) - if not target_obj_names: - target_obj_names = [] - self.client_info[client_name] = _ClientInfo(target_obj_names) + collab_signature = result.get(SyncKey.COLLAB_SIGNATURE) + self.client_info[client_name] = _ClientInfo(collab_signature) else: self.log_error(fl_ctx, f"client {client_task.client.name} failed to sync: {rc}") self.client_info[client_name] = None diff --git a/nvflare/focs/sys/executor.py b/nvflare/focs/sys/executor.py index 8fcfc2aa54..6e62a246a6 100644 --- a/nvflare/focs/sys/executor.py +++ b/nvflare/focs/sys/executor.py @@ -38,6 +38,7 @@ def __init__(self, client_app_id: str, client_target_obj_ids: Dict[str, str] = N self.client_app_id = client_app_id self.client_target_obj_ids = client_target_obj_ids self.register_event_handler(EventType.START_RUN, self._handle_start_run) + self.register_event_handler(EventType.END_RUN, self._handle_end_run) self.client_app = None self.thread_executor = ThreadPoolExecutor(max_workers=max_call_threads) @@ -68,8 +69,10 @@ def _handle_start_run(self, event_type: str, fl_ctx: FLContext): self.client_app.add_collab_object(name, obj) - def _prepare_server_proxy(self, job_id, cell, server_target_obj_names, abort_signal): - my_name = self.client_app.name + def _handle_end_run(self, event_type: str, fl_ctx: FLContext): + self.thread_executor.shutdown(wait=False, cancel_futures=True) + + def _prepare_server_proxy(self, job_id, cell, collab_signature: dict, abort_signal): server_name = "server" backend = SysBackend( caller=self.client_app.name, @@ -78,14 +81,19 @@ def _prepare_server_proxy(self, job_id, cell, server_target_obj_names, abort_sig abort_signal=abort_signal, thread_executor=self.thread_executor, ) - proxy = Proxy(app=self.client_app, target_name=server_name, backend=backend, caller_name=my_name) + proxy = Proxy( + app=self.client_app, target_name=server_name, backend=backend, target_signature=collab_signature.get("") + ) - for name in server_target_obj_names: - p = Proxy(app=self.client_app, target_name=f"{server_name}.{name}", backend=backend, caller_name=my_name) + for name, sig in collab_signature.items(): + if name == "": + # this is the server app itself + continue + p = Proxy(app=self.client_app, target_name=f"{server_name}.{name}", backend=backend, target_signature=sig) setattr(proxy, name, p) return proxy - def _prepare_client_proxy(self, job_id, cell, client: Client, abort_signal): + def _prepare_client_proxy(self, job_id, cell, client: Client, abort_signal, collab_signature): my_name = self.client_app.name backend = SysBackend( caller=self.client_app.name, @@ -94,7 +102,9 @@ def _prepare_client_proxy(self, job_id, cell, client: Client, abort_signal): abort_signal=abort_signal, thread_executor=self.thread_executor, ) - proxy = Proxy(app=self.client_app, target_name=client.name, backend=backend, caller_name=my_name) + proxy = Proxy( + app=self.client_app, target_name=client.name, backend=backend, target_signature=collab_signature.get("") + ) if self.client_target_obj_ids: for name in self.client_target_obj_ids.keys(): @@ -102,7 +112,7 @@ def _prepare_client_proxy(self, job_id, cell, client: Client, abort_signal): app=self.client_app, target_name=f"{client.name}.{name}", backend=backend, - caller_name=my_name, + target_signature=collab_signature.get(name), ) setattr(proxy, name, p) return proxy @@ -112,7 +122,9 @@ def execute(self, task_name: str, shareable: Shareable, fl_ctx: FLContext, abort self.log_error(fl_ctx, f"received unsupported task {task_name}") return make_reply(ReturnCode.TASK_UNKNOWN) - server_target_obj_names = shareable.get(SyncKey.COLLAB_OBJ_NAMES) + server_collab_signature = shareable.get(SyncKey.COLLAB_SIGNATURE) + client_collab_signature = self.client_app.get_collab_signature() + self.log_info(fl_ctx, f"{client_collab_signature=} {server_collab_signature=}") engine = fl_ctx.get_engine() cell = engine.get_cell() @@ -125,14 +137,14 @@ def execute(self, task_name: str, shareable: Shareable, fl_ctx: FLContext, abort # build proxies job_id = fl_ctx.get_job_id() - server_proxy = self._prepare_server_proxy(job_id, cell, server_target_obj_names, abort_signal) + server_proxy = self._prepare_server_proxy(job_id, cell, server_collab_signature, abort_signal) job_meta = fl_ctx.get_prop(FLContextKey.JOB_META) job_clients = job_meta.get(JobMetaKey.JOB_CLIENTS) all_clients = [from_dict(d) for d in job_clients] client_proxies = [] for c in all_clients: - p = self._prepare_client_proxy(job_id, cell, c, abort_signal) + p = self._prepare_client_proxy(job_id, cell, c, abort_signal, client_collab_signature) client_proxies.append(p) self.client_app.setup(server_proxy, client_proxies, abort_signal) @@ -141,9 +153,5 @@ def execute(self, task_name: str, shareable: Shareable, fl_ctx: FLContext, abort self.client_app.initialize(ctx) reply = make_reply(ReturnCode.OK) - if self.client_target_obj_ids: - target_obj_names = list(self.client_target_obj_ids.keys()) - else: - target_obj_names = [] - reply[SyncKey.COLLAB_OBJ_NAMES] = target_obj_names + reply[SyncKey.COLLAB_SIGNATURE] = client_collab_signature return reply diff --git a/nvflare/focs/sys/utils.py b/nvflare/focs/sys/utils.py index 67398148a1..888c28a9b5 100644 --- a/nvflare/focs/sys/utils.py +++ b/nvflare/focs/sys/utils.py @@ -71,8 +71,12 @@ def _call_app_method(request: Message, app: App, logger) -> Message: if len(parts) >= 2: obj_name = parts[1] if obj_name: + target_obj = None target_objs = app.get_collab_objects() - target_obj = target_objs.get(obj_name) + for name, obj in target_objs: + if name == obj_name: + target_obj = obj + break logger.info(f"calling target obj: {app.name}.{obj_name}") else: target_obj = app From 2b264b2c36f227c1bc9a36d98923be2064c9eba7 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Tue, 30 Sep 2025 12:08:03 -0400 Subject: [PATCH 017/102] add proxy fallback --- nvflare/focs/api/group.py | 6 +-- nvflare/focs/api/proxy.py | 58 +++++++++++++++++++------ nvflare/focs/examples/np/algos/swarm.py | 2 +- nvflare/focs/sim/runner.py | 2 +- nvflare/focs/sys/controller.py | 2 +- nvflare/focs/sys/executor.py | 4 +- 6 files changed, 52 insertions(+), 22 deletions(-) diff --git a/nvflare/focs/api/group.py b/nvflare/focs/api/group.py index c2b1e99c01..96c2aaedf4 100644 --- a/nvflare/focs/api/group.py +++ b/nvflare/focs/api/group.py @@ -64,11 +64,11 @@ def __getattr__(self, func_name): def method(*args, **kwargs): resps = {} - call_args, call_kwargs = self._proxies[0].adjust_func_args(func_name, args, kwargs) for p in self._proxies: + the_proxy, call_args, call_kwargs = p.adjust_func_args(func_name, args, kwargs) kwargs_copy = copy.copy(call_kwargs) - ctx = self._app.new_context(p.caller_name, p.name) + ctx = self._app.new_context(the_proxy.caller_name, the_proxy.name) kwargs_copy[CollabMethodArgName.CONTEXT] = ctx # set the optional args to help backend decide how to call @@ -80,7 +80,7 @@ def method(*args, **kwargs): resps[p.name] = resp print(f"group call: {func_name=} args={call_args} kwargs={kwargs_copy}") - p.backend.call_target_with_resp(resp, p.name, func_name, *call_args, **kwargs_copy) + the_proxy.backend.call_target_with_resp(resp, the_proxy.name, func_name, *call_args, **kwargs_copy) # wait for responses if not self._blocking: diff --git a/nvflare/focs/api/proxy.py b/nvflare/focs/api/proxy.py index e811e62648..db4ba194bc 100644 --- a/nvflare/focs/api/proxy.py +++ b/nvflare/focs/api/proxy.py @@ -19,18 +19,23 @@ class Proxy: - def __init__(self, app, target_name, backend: Backend, target_signature=None): + def __init__(self, app, target_name, backend: Backend, target_signature): """The Proxy represents a target in the App.""" self.app = app self.target_name = target_name self.backend = backend self.caller_name = app.name self.target_signature = target_signature + self.children = {} # child proxies @property def name(self): return self.target_name + def add_child(self, name, p): + self.children[name] = p + setattr(self, name, p) + def get_target(self, name: str): obj = getattr(self, name, None) if not obj: @@ -40,22 +45,47 @@ def get_target(self, name: str): else: return None + def _find_signature(self, func_name): + args = self.target_signature.get(func_name) if self.target_signature else None + if args: + return self, args + + # try children + the_args = None + the_proxy = None + the_name = None + for n, c in self.children.items(): + args = c.target_signature.get(func_name) if c.target_signature else None + if not the_proxy: + the_name = n + the_proxy = c + the_args = args + else: + # already found a child proxy that has this func - ambiguity + raise RuntimeError( + f"multiple collab objects ({the_name} and {n}) have {func_name}: please use qualified call" + ) + return the_proxy, the_args + def adjust_func_args(self, func_name, args, kwargs): call_args = args call_kwargs = kwargs - if self.target_signature: - arg_names = self.target_signature.get(func_name) - if arg_names: - # check args and turn them to kwargs - call_kwargs = copy.copy(kwargs) - call_args = [] - for i, arg_value in enumerate(args): - call_kwargs[arg_names[i]] = arg_value + # find the proxy for the func + p, arg_names = self._find_signature(func_name) + if not p: + raise RuntimeError(f"target {self.target_name} does not have method '{func_name}'") + + if arg_names: + # check args and turn them to kwargs + call_kwargs = copy.copy(kwargs) + call_args = [] + for i, arg_value in enumerate(args): + call_kwargs[arg_names[i]] = arg_value ctx = self.app.new_context(self.caller_name, self.name) call_kwargs[CollabMethodArgName.CONTEXT] = ctx - return call_args, call_kwargs + return p, call_args, call_kwargs def __getattr__(self, func_name): """ @@ -63,11 +93,11 @@ def __getattr__(self, func_name): """ def method(*args, **kwargs): - call_args, call_kwargs = self.adjust_func_args(func_name, args, kwargs) - ctx = self.app.new_context(self.caller_name, self.name) + p, call_args, call_kwargs = self.adjust_func_args(func_name, args, kwargs) + ctx = p.app.new_context(self.caller_name, self.name) call_kwargs[CollabMethodArgName.CONTEXT] = ctx - print(f"calling target {self.target_name} func {func_name}: {call_args=} {call_kwargs=}") - return self.backend.call_target(self.target_name, func_name, *call_args, **call_kwargs) + print(f"calling target {p.target_name} func {func_name}: {call_args=} {call_kwargs=}") + return p.backend.call_target(self.target_name, func_name, *call_args, **call_kwargs) return method diff --git a/nvflare/focs/examples/np/algos/swarm.py b/nvflare/focs/examples/np/algos/swarm.py index 6182feb1df..575cd2b4bc 100644 --- a/nvflare/focs/examples/np/algos/swarm.py +++ b/nvflare/focs/examples/np/algos/swarm.py @@ -79,7 +79,7 @@ def swarm_learn(self, num_rounds, model, current_round, context: Context): # all done all_clients(context, blocking=False).fire_event("final_model", new_model) # self.server.fire_event("all_done", "OK", blocking=False) - self.server.strategy.all_done("OK", blocking=False) + self.server.all_done("OK", blocking=False) return # determine next client diff --git a/nvflare/focs/sim/runner.py b/nvflare/focs/sim/runner.py index 8ffa54e0f1..8bc8db5d37 100644 --- a/nvflare/focs/sim/runner.py +++ b/nvflare/focs/sim/runner.py @@ -50,7 +50,7 @@ def _prepare_proxy(self, for_app: App, target_app: App, backends: dict): backend=backends[name], target_signature=get_object_collab_signature(obj), ) - setattr(app_proxy, name, p) + app_proxy.add_child(name, p) return app_proxy def _prepare_proxies(self, for_app: App, server_app: App, client_apps: dict, backends: dict): diff --git a/nvflare/focs/sys/controller.py b/nvflare/focs/sys/controller.py index 1597054682..79218cf052 100644 --- a/nvflare/focs/sys/controller.py +++ b/nvflare/focs/sys/controller.py @@ -135,7 +135,7 @@ def _prepare_client_proxy( continue p = Proxy(app=self.server_app, target_name=f"{client.name}.{name}", backend=backend, target_signature=sig) - setattr(proxy, name, p) + proxy.add_child(name, p) return proxy def _prepare_server_proxy( diff --git a/nvflare/focs/sys/executor.py b/nvflare/focs/sys/executor.py index 6e62a246a6..ed7283f3ca 100644 --- a/nvflare/focs/sys/executor.py +++ b/nvflare/focs/sys/executor.py @@ -90,7 +90,7 @@ def _prepare_server_proxy(self, job_id, cell, collab_signature: dict, abort_sign # this is the server app itself continue p = Proxy(app=self.client_app, target_name=f"{server_name}.{name}", backend=backend, target_signature=sig) - setattr(proxy, name, p) + proxy.add_child(name, p) return proxy def _prepare_client_proxy(self, job_id, cell, client: Client, abort_signal, collab_signature): @@ -114,7 +114,7 @@ def _prepare_client_proxy(self, job_id, cell, client: Client, abort_signal, coll backend=backend, target_signature=collab_signature.get(name), ) - setattr(proxy, name, p) + proxy.add_child(name, p) return proxy def execute(self, task_name: str, shareable: Shareable, fl_ctx: FLContext, abort_signal: Signal) -> Shareable: From bfbfe6725c1df4f64ede8ab24f785248431eece4 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Tue, 30 Sep 2025 12:39:33 -0400 Subject: [PATCH 018/102] fix find sig --- nvflare/focs/api/proxy.py | 3 +++ nvflare/focs/examples/np/algos/swarm.py | 15 +++++++++++---- nvflare/focs/sys/executor.py | 1 - 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/nvflare/focs/api/proxy.py b/nvflare/focs/api/proxy.py index db4ba194bc..c69144c5c3 100644 --- a/nvflare/focs/api/proxy.py +++ b/nvflare/focs/api/proxy.py @@ -56,6 +56,9 @@ def _find_signature(self, func_name): the_name = None for n, c in self.children.items(): args = c.target_signature.get(func_name) if c.target_signature else None + if not args: + continue + if not the_proxy: the_name = n the_proxy = c diff --git a/nvflare/focs/examples/np/algos/swarm.py b/nvflare/focs/examples/np/algos/swarm.py index 575cd2b4bc..c8e4418c60 100644 --- a/nvflare/focs/examples/np/algos/swarm.py +++ b/nvflare/focs/examples/np/algos/swarm.py @@ -13,14 +13,14 @@ # limitations under the License. import random import threading +import traceback -from nvflare.focs.api.app import ClientApp, ServerApp +from nvflare.focs.api.app import ClientApp from nvflare.focs.api.ctx import Context from nvflare.focs.api.dec import collab from nvflare.focs.api.group import all_clients from nvflare.focs.api.strategy import Strategy from nvflare.focs.examples.np.algos.utils import parse_array_def -from nvflare.focs.sim.runner import AppRunner class NPSwarm(Strategy): @@ -37,7 +37,9 @@ def execute(self, context: Context): start_client_idx = random.randint(0, len(context.clients) - 1) start_client = context.clients[start_client_idx] start_client.start(self.num_rounds, self.initial_model) - self.waiter.wait() + while not context.is_aborted(): + if self.waiter.wait(timeout=0.5): + break def _all_done(self, event_type: str, data, context: Context): print(f"[{context.callee}]: received {event_type} from client: {context.caller}: {data}") @@ -79,7 +81,12 @@ def swarm_learn(self, num_rounds, model, current_round, context: Context): # all done all_clients(context, blocking=False).fire_event("final_model", new_model) # self.server.fire_event("all_done", "OK", blocking=False) - self.server.all_done("OK", blocking=False) + print("notify server all done!") + try: + self.server.all_done("OK", blocking=False) + except: + traceback.print_exc() + print("Swarm Training is DONE!") return # determine next client diff --git a/nvflare/focs/sys/executor.py b/nvflare/focs/sys/executor.py index ed7283f3ca..53c21a0d16 100644 --- a/nvflare/focs/sys/executor.py +++ b/nvflare/focs/sys/executor.py @@ -94,7 +94,6 @@ def _prepare_server_proxy(self, job_id, cell, collab_signature: dict, abort_sign return proxy def _prepare_client_proxy(self, job_id, cell, client: Client, abort_signal, collab_signature): - my_name = self.client_app.name backend = SysBackend( caller=self.client_app.name, cell=cell, From c7e9b96d9975444f7615529ece67c9656dc34ae5 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Tue, 30 Sep 2025 13:40:53 -0400 Subject: [PATCH 019/102] change signature to interface --- nvflare/focs/api/app.py | 26 ++++++++---------- nvflare/focs/api/dec.py | 2 +- nvflare/focs/api/proxy.py | 12 ++++---- nvflare/focs/examples/np/cyclic_avg.py | 2 +- nvflare/focs/examples/np/fed_avg_intime.py | 2 +- nvflare/focs/sim/runner.py | 28 +++++++++---------- nvflare/focs/sys/constants.py | 2 +- nvflare/focs/sys/controller.py | 32 +++++++++++----------- nvflare/focs/sys/executor.py | 26 +++++++++--------- nvflare/focs/sys/utils.py | 6 +--- 10 files changed, 64 insertions(+), 74 deletions(-) diff --git a/nvflare/focs/api/app.py b/nvflare/focs/api/app.py index 057c6dfe15..b80471387e 100644 --- a/nvflare/focs/api/app.py +++ b/nvflare/focs/api/app.py @@ -17,7 +17,7 @@ from .constants import CollabMethodArgName from .ctx import Context -from .dec import collab, get_object_collab_signature, is_collab +from .dec import collab, get_object_collab_interface, is_collab from .proxy import Proxy from .strategy import Strategy from .utils import check_context_support @@ -30,7 +30,7 @@ def __init__(self): self.server = None self.clients = None self._me = None - self._collab_objs = [] + self._collab_objs = {} self._abort_signal = None self._props = {} self._event_handlers = {} # event type => list of (cb, kwargs) @@ -45,15 +45,14 @@ def get_default_collab_object(self): return None def add_collab_object(self, name: str, obj): - for n, o in self._collab_objs: - if name == n: - raise ValueError(f"conflict with existing collab object '{name}' of {type(o)}") + if name in self._collab_objs: + raise ValueError(f"conflict with existing collab object '{name}' of {type(self._collab_objs[name])}") if hasattr(obj, name): raise ValueError(f"conflict with reserved name {name}") setattr(self, name, obj) - self._collab_objs.append((name, obj)) + self._collab_objs[name] = obj def get_collab_objects(self): return self._collab_objs @@ -92,7 +91,7 @@ def find_method(self, target_obj, method_name): return m targets = self.get_collab_objects() - for _, obj in targets: + for _, obj in targets.items(): m = getattr(obj, method_name, None) if m: return m @@ -111,7 +110,7 @@ def initialize(self, context: Context): self.initialize_app(context) # initialize target objects - for name, obj in self._collab_objs: + for name, obj in self._collab_objs.items(): init_func = getattr(obj, "initialize", None) if init_func and callable(init_func): print(f"initializing target object {name}") @@ -133,13 +132,10 @@ def register_event_handler(self, event_type: str, handler, **handler_kwargs): self._event_handlers[event_type] = handlers handlers.append((handler, handler_kwargs)) - def get_collab_signature(self): - result = {"": get_object_collab_signature(self)} - - for name, obj in self._collab_objs: - result[name] = get_object_collab_signature(obj) - - print(f"get_collab_signature: {result}") + def get_collab_interface(self): + result = {"": get_object_collab_interface(self)} + for name, obj in self._collab_objs.items(): + result[name] = get_object_collab_interface(obj) return result @collab diff --git a/nvflare/focs/api/dec.py b/nvflare/focs/api/dec.py index 704b422d87..584fafe836 100644 --- a/nvflare/focs/api/dec.py +++ b/nvflare/focs/api/dec.py @@ -52,7 +52,7 @@ def adjust_kwargs(func, kwargs): kwargs.pop(CollabMethodArgName.CONTEXT, None) -def get_object_collab_signature(obj): +def get_object_collab_interface(obj): result = {} for name in dir(obj): func = getattr(obj, name) diff --git a/nvflare/focs/api/proxy.py b/nvflare/focs/api/proxy.py index c69144c5c3..ce25332edc 100644 --- a/nvflare/focs/api/proxy.py +++ b/nvflare/focs/api/proxy.py @@ -19,13 +19,13 @@ class Proxy: - def __init__(self, app, target_name, backend: Backend, target_signature): + def __init__(self, app, target_name, backend: Backend, target_interface): """The Proxy represents a target in the App.""" self.app = app self.target_name = target_name self.backend = backend self.caller_name = app.name - self.target_signature = target_signature + self.target_interface = target_interface self.children = {} # child proxies @property @@ -45,8 +45,8 @@ def get_target(self, name: str): else: return None - def _find_signature(self, func_name): - args = self.target_signature.get(func_name) if self.target_signature else None + def _find_interface(self, func_name): + args = self.target_interface.get(func_name) if self.target_interface else None if args: return self, args @@ -55,7 +55,7 @@ def _find_signature(self, func_name): the_proxy = None the_name = None for n, c in self.children.items(): - args = c.target_signature.get(func_name) if c.target_signature else None + args = c.target_interface.get(func_name) if c.target_interface else None if not args: continue @@ -75,7 +75,7 @@ def adjust_func_args(self, func_name, args, kwargs): call_kwargs = kwargs # find the proxy for the func - p, arg_names = self._find_signature(func_name) + p, arg_names = self._find_interface(func_name) if not p: raise RuntimeError(f"target {self.target_name} does not have method '{func_name}'") diff --git a/nvflare/focs/examples/np/cyclic_avg.py b/nvflare/focs/examples/np/cyclic_avg.py index 401fd04224..b747f2e551 100644 --- a/nvflare/focs/examples/np/cyclic_avg.py +++ b/nvflare/focs/examples/np/cyclic_avg.py @@ -23,7 +23,7 @@ def main(): ) server_app.add_strategy("fed_avg_parallel", NPFedAvgParallel(initial_model=None, num_rounds=2)) - server_app.get_collab_signature() + server_app.get_collab_interface() runner = AppRunner( server_app=server_app, diff --git a/nvflare/focs/examples/np/fed_avg_intime.py b/nvflare/focs/examples/np/fed_avg_intime.py index 8e9844e9d0..962e80dd8b 100644 --- a/nvflare/focs/examples/np/fed_avg_intime.py +++ b/nvflare/focs/examples/np/fed_avg_intime.py @@ -27,7 +27,7 @@ def main(): server_app.add_collab_object("metric_receiver", MetricReceiver()) - server_app.get_collab_signature() + server_app.get_collab_interface() runner = AppRunner( server_app=server_app, diff --git a/nvflare/focs/sim/runner.py b/nvflare/focs/sim/runner.py index 8bc8db5d37..6a0bd69579 100644 --- a/nvflare/focs/sim/runner.py +++ b/nvflare/focs/sim/runner.py @@ -19,7 +19,7 @@ from nvflare.apis.signal import Signal from nvflare.focs.api.app import App, ClientApp, ClientAppFactory, ServerApp from nvflare.focs.api.constants import ContextKey -from nvflare.focs.api.dec import get_object_collab_signature +from nvflare.focs.api.dec import get_object_collab_interface from nvflare.focs.api.proxy import Proxy from nvflare.focs.sim.backend import SimBackend @@ -29,9 +29,8 @@ class AppRunner: def _prepare_app_backends(self, app: App): bes = {"": SimBackend(app, app, self.abort_signal, self.thread_executor)} targets = app.get_collab_objects() - if targets: - for name, obj in targets: - bes[name] = SimBackend(app, obj, self.abort_signal, self.thread_executor) + for name, obj in targets.items(): + bes[name] = SimBackend(app, obj, self.abort_signal, self.thread_executor) return bes def _prepare_proxy(self, for_app: App, target_app: App, backends: dict): @@ -39,18 +38,17 @@ def _prepare_proxy(self, for_app: App, target_app: App, backends: dict): app=for_app, target_name=target_app.name, backend=backends[""], - target_signature=get_object_collab_signature(target_app), + target_interface=get_object_collab_interface(target_app), ) - tos = target_app.get_collab_objects() - if tos: - for name, obj in tos: - p = Proxy( - app=for_app, - target_name=f"{target_app.name}.{name}", - backend=backends[name], - target_signature=get_object_collab_signature(obj), - ) - app_proxy.add_child(name, p) + collab_objs = target_app.get_collab_objects() + for name, obj in collab_objs.items(): + p = Proxy( + app=for_app, + target_name=f"{target_app.name}.{name}", + backend=backends[name], + target_interface=get_object_collab_interface(obj), + ) + app_proxy.add_child(name, p) return app_proxy def _prepare_proxies(self, for_app: App, server_app: App, client_apps: dict, backends: dict): diff --git a/nvflare/focs/sys/constants.py b/nvflare/focs/sys/constants.py index 6490aa86ff..b7f85b298d 100644 --- a/nvflare/focs/sys/constants.py +++ b/nvflare/focs/sys/constants.py @@ -18,7 +18,7 @@ class SyncKey: - COLLAB_SIGNATURE = "collab_signature" + COLLAB_INTERFACE = "collab_interface" class ObjectCallKey: diff --git a/nvflare/focs/sys/controller.py b/nvflare/focs/sys/controller.py index 79218cf052..362dc8ba2b 100644 --- a/nvflare/focs/sys/controller.py +++ b/nvflare/focs/sys/controller.py @@ -35,13 +35,13 @@ class _ClientInfo: - def __init__(self, collab_signature: dict): + def __init__(self, collab_interface: dict): """Information about a client. Reported by the client in the sync response. Args: - collab_signature: collab method signature of the client. + collab_interface: collab method interface of the client. """ - self.collab_signature = collab_signature + self.collab_interface = collab_interface class FocsController(Controller): @@ -122,19 +122,19 @@ def _prepare_client_proxy( self, job_id: str, client: ClientSite, - collab_signature: dict, + collab_interface: dict, abort_signal, ): backend = self._prepare_client_backend(job_id, client, abort_signal) proxy = Proxy( - app=self.server_app, target_name=client.name, backend=backend, target_signature=collab_signature.get("") + app=self.server_app, target_name=client.name, backend=backend, target_interface=collab_interface.get("") ) - for name, sig in collab_signature.items(): + for name, itf in collab_interface.items(): if name == "": continue - p = Proxy(app=self.server_app, target_name=f"{client.name}.{name}", backend=backend, target_signature=sig) + p = Proxy(app=self.server_app, target_name=f"{client.name}.{name}", backend=backend, target_interface=itf) proxy.add_child(name, p) return proxy @@ -142,12 +142,12 @@ def _prepare_server_proxy( self, job_id, abort_signal, - collab_signature: dict, + collab_interface: dict, ): server_name = self.server_app.name backend = self._prepare_server_backend(job_id, abort_signal) proxy = Proxy( - app=self.server_app, target_name=server_name, backend=backend, target_signature=collab_signature.get("") + app=self.server_app, target_name=server_name, backend=backend, target_interface=collab_interface.get("") ) for name in self.server_collab_obj_ids: @@ -155,15 +155,15 @@ def _prepare_server_proxy( app=self.server_app, target_name=f"{server_name}.{name}", backend=backend, - target_signature=collab_signature.get(name), + target_interface=collab_interface.get(name), ) setattr(proxy, name, p) return proxy def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): # configure all sites - server_collab_signature = self.server_app.get_collab_signature() - task_data = Shareable({SyncKey.COLLAB_SIGNATURE: server_collab_signature}) + server_collab_interface = self.server_app.get_collab_interface() + task_data = Shareable({SyncKey.COLLAB_INTERFACE: server_collab_interface}) task = Task( name=SYNC_TASK_NAME, data=task_data, @@ -209,12 +209,12 @@ def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): # prepare proxies and backends job_id = fl_ctx.get_job_id() - server_proxy = self._prepare_server_proxy(job_id, abort_signal, server_collab_signature) + server_proxy = self._prepare_server_proxy(job_id, abort_signal, server_collab_interface) client_proxies = [] for c in all_clients: info = self.client_info[c.name] assert isinstance(info, _ClientInfo) - client_proxies.append(self._prepare_client_proxy(job_id, c, info.collab_signature, abort_signal)) + client_proxies.append(self._prepare_client_proxy(job_id, c, info.collab_interface, abort_signal)) self.server_app.setup(server_proxy, client_proxies, abort_signal) @@ -242,8 +242,8 @@ def _process_sync_reply(self, client_task: ClientTask, fl_ctx: FLContext): rc = result.get_return_code() if rc == ReturnCode.OK: self.log_info(fl_ctx, f"successfully synced client {client_name}") - collab_signature = result.get(SyncKey.COLLAB_SIGNATURE) - self.client_info[client_name] = _ClientInfo(collab_signature) + collab_itf = result.get(SyncKey.COLLAB_INTERFACE) + self.client_info[client_name] = _ClientInfo(collab_itf) else: self.log_error(fl_ctx, f"client {client_task.client.name} failed to sync: {rc}") self.client_info[client_name] = None diff --git a/nvflare/focs/sys/executor.py b/nvflare/focs/sys/executor.py index 53c21a0d16..d1a138b3b0 100644 --- a/nvflare/focs/sys/executor.py +++ b/nvflare/focs/sys/executor.py @@ -72,7 +72,7 @@ def _handle_start_run(self, event_type: str, fl_ctx: FLContext): def _handle_end_run(self, event_type: str, fl_ctx: FLContext): self.thread_executor.shutdown(wait=False, cancel_futures=True) - def _prepare_server_proxy(self, job_id, cell, collab_signature: dict, abort_signal): + def _prepare_server_proxy(self, job_id, cell, collab_interface: dict, abort_signal): server_name = "server" backend = SysBackend( caller=self.client_app.name, @@ -82,18 +82,18 @@ def _prepare_server_proxy(self, job_id, cell, collab_signature: dict, abort_sign thread_executor=self.thread_executor, ) proxy = Proxy( - app=self.client_app, target_name=server_name, backend=backend, target_signature=collab_signature.get("") + app=self.client_app, target_name=server_name, backend=backend, target_interface=collab_interface.get("") ) - for name, sig in collab_signature.items(): + for name, itf in collab_interface.items(): if name == "": # this is the server app itself continue - p = Proxy(app=self.client_app, target_name=f"{server_name}.{name}", backend=backend, target_signature=sig) + p = Proxy(app=self.client_app, target_name=f"{server_name}.{name}", backend=backend, target_interface=itf) proxy.add_child(name, p) return proxy - def _prepare_client_proxy(self, job_id, cell, client: Client, abort_signal, collab_signature): + def _prepare_client_proxy(self, job_id, cell, client: Client, abort_signal, collab_interface): backend = SysBackend( caller=self.client_app.name, cell=cell, @@ -102,7 +102,7 @@ def _prepare_client_proxy(self, job_id, cell, client: Client, abort_signal, coll thread_executor=self.thread_executor, ) proxy = Proxy( - app=self.client_app, target_name=client.name, backend=backend, target_signature=collab_signature.get("") + app=self.client_app, target_name=client.name, backend=backend, target_interface=collab_interface.get("") ) if self.client_target_obj_ids: @@ -111,7 +111,7 @@ def _prepare_client_proxy(self, job_id, cell, client: Client, abort_signal, coll app=self.client_app, target_name=f"{client.name}.{name}", backend=backend, - target_signature=collab_signature.get(name), + target_interface=collab_interface.get(name), ) proxy.add_child(name, p) return proxy @@ -121,9 +121,9 @@ def execute(self, task_name: str, shareable: Shareable, fl_ctx: FLContext, abort self.log_error(fl_ctx, f"received unsupported task {task_name}") return make_reply(ReturnCode.TASK_UNKNOWN) - server_collab_signature = shareable.get(SyncKey.COLLAB_SIGNATURE) - client_collab_signature = self.client_app.get_collab_signature() - self.log_info(fl_ctx, f"{client_collab_signature=} {server_collab_signature=}") + server_collab_interface = shareable.get(SyncKey.COLLAB_INTERFACE) + client_collab_interface = self.client_app.get_collab_interface() + self.log_info(fl_ctx, f"{client_collab_interface=} {server_collab_interface=}") engine = fl_ctx.get_engine() cell = engine.get_cell() @@ -136,14 +136,14 @@ def execute(self, task_name: str, shareable: Shareable, fl_ctx: FLContext, abort # build proxies job_id = fl_ctx.get_job_id() - server_proxy = self._prepare_server_proxy(job_id, cell, server_collab_signature, abort_signal) + server_proxy = self._prepare_server_proxy(job_id, cell, server_collab_interface, abort_signal) job_meta = fl_ctx.get_prop(FLContextKey.JOB_META) job_clients = job_meta.get(JobMetaKey.JOB_CLIENTS) all_clients = [from_dict(d) for d in job_clients] client_proxies = [] for c in all_clients: - p = self._prepare_client_proxy(job_id, cell, c, abort_signal, client_collab_signature) + p = self._prepare_client_proxy(job_id, cell, c, abort_signal, client_collab_interface) client_proxies.append(p) self.client_app.setup(server_proxy, client_proxies, abort_signal) @@ -152,5 +152,5 @@ def execute(self, task_name: str, shareable: Shareable, fl_ctx: FLContext, abort self.client_app.initialize(ctx) reply = make_reply(ReturnCode.OK) - reply[SyncKey.COLLAB_SIGNATURE] = client_collab_signature + reply[SyncKey.COLLAB_INTERFACE] = client_collab_interface return reply diff --git a/nvflare/focs/sys/utils.py b/nvflare/focs/sys/utils.py index 888c28a9b5..67398148a1 100644 --- a/nvflare/focs/sys/utils.py +++ b/nvflare/focs/sys/utils.py @@ -71,12 +71,8 @@ def _call_app_method(request: Message, app: App, logger) -> Message: if len(parts) >= 2: obj_name = parts[1] if obj_name: - target_obj = None target_objs = app.get_collab_objects() - for name, obj in target_objs: - if name == obj_name: - target_obj = obj - break + target_obj = target_objs.get(obj_name) logger.info(f"calling target obj: {app.name}.{obj_name}") else: target_obj = app From b702fe620a0db0080123de2277d007062caa46f9 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Tue, 30 Sep 2025 14:31:48 -0400 Subject: [PATCH 020/102] rename app runner to simulator --- nvflare/focs/examples/np/cyclic.py | 6 +++--- nvflare/focs/examples/np/cyclic_avg.py | 6 +++--- nvflare/focs/examples/np/fed_avg_intime.py | 6 +++--- nvflare/focs/examples/np/fed_avg_para.py | 6 +++--- nvflare/focs/examples/np/fed_avg_seq.py | 6 +++--- nvflare/focs/examples/np/swarm.py | 6 +++--- nvflare/focs/sim/{runner.py => simulator.py} | 2 +- 7 files changed, 19 insertions(+), 19 deletions(-) rename nvflare/focs/sim/{runner.py => simulator.py} (99%) diff --git a/nvflare/focs/examples/np/cyclic.py b/nvflare/focs/examples/np/cyclic.py index ae5da38852..64062d6530 100644 --- a/nvflare/focs/examples/np/cyclic.py +++ b/nvflare/focs/examples/np/cyclic.py @@ -14,12 +14,12 @@ from nvflare.focs.api.app import ServerApp from nvflare.focs.examples.np.algos.client import NPTrainer from nvflare.focs.examples.np.algos.strategies import NPCyclic -from nvflare.focs.sim.runner import AppRunner +from nvflare.focs.sim.simulator import Simulator def main(): - runner = AppRunner( + simulator = Simulator( server_app=ServerApp( strategy_name="cyclic", strategy=NPCyclic(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2) ), @@ -27,7 +27,7 @@ def main(): num_clients=2, ) - runner.run() + simulator.run() if __name__ == "__main__": diff --git a/nvflare/focs/examples/np/cyclic_avg.py b/nvflare/focs/examples/np/cyclic_avg.py index b747f2e551..f270dd1a72 100644 --- a/nvflare/focs/examples/np/cyclic_avg.py +++ b/nvflare/focs/examples/np/cyclic_avg.py @@ -14,7 +14,7 @@ from nvflare.focs.api.app import ServerApp from nvflare.focs.examples.np.algos.client import NPTrainer from nvflare.focs.examples.np.algos.strategies import NPCyclic, NPFedAvgParallel -from nvflare.focs.sim.runner import AppRunner +from nvflare.focs.sim.simulator import Simulator def main(): @@ -25,13 +25,13 @@ def main(): server_app.get_collab_interface() - runner = AppRunner( + simulator = Simulator( server_app=server_app, client_app=NPTrainer(delta=1.0), num_clients=2, ) - final_result = runner.run() + final_result = simulator.run() print(f"final model: {final_result}") diff --git a/nvflare/focs/examples/np/fed_avg_intime.py b/nvflare/focs/examples/np/fed_avg_intime.py index 962e80dd8b..71d7b5c8df 100644 --- a/nvflare/focs/examples/np/fed_avg_intime.py +++ b/nvflare/focs/examples/np/fed_avg_intime.py @@ -15,7 +15,7 @@ from nvflare.focs.examples.np.algos.client import NPTrainer from nvflare.focs.examples.np.algos.strategies import NPFedAvgInTime from nvflare.focs.examples.np.algos.widgets import MetricReceiver -from nvflare.focs.sim.runner import AppRunner +from nvflare.focs.sim.simulator import Simulator def main(): @@ -29,13 +29,13 @@ def main(): server_app.get_collab_interface() - runner = AppRunner( + simulator = Simulator( server_app=server_app, client_app=NPTrainer(delta=1.0), num_clients=2, ) - runner.run() + simulator.run() if __name__ == "__main__": diff --git a/nvflare/focs/examples/np/fed_avg_para.py b/nvflare/focs/examples/np/fed_avg_para.py index 7caf5d9b48..c0dcaebada 100644 --- a/nvflare/focs/examples/np/fed_avg_para.py +++ b/nvflare/focs/examples/np/fed_avg_para.py @@ -15,7 +15,7 @@ from nvflare.focs.examples.np.algos.client import NPTrainer from nvflare.focs.examples.np.algos.strategies import NPFedAvgParallel from nvflare.focs.examples.np.algos.widgets import MetricReceiver -from nvflare.focs.sim.runner import AppRunner +from nvflare.focs.sim.simulator import Simulator def main(): @@ -26,13 +26,13 @@ def main(): ) server_app.add_collab_object("metric_receiver", MetricReceiver()) - runner = AppRunner( + simulator = Simulator( server_app=server_app, client_app=NPTrainer(delta=1.0), num_clients=10, ) - runner.run() + simulator.run() if __name__ == "__main__": diff --git a/nvflare/focs/examples/np/fed_avg_seq.py b/nvflare/focs/examples/np/fed_avg_seq.py index 0f8d463097..6322acf1b4 100644 --- a/nvflare/focs/examples/np/fed_avg_seq.py +++ b/nvflare/focs/examples/np/fed_avg_seq.py @@ -15,7 +15,7 @@ from nvflare.focs.examples.np.algos.client import TrainerFactory from nvflare.focs.examples.np.algos.strategies import NPFedAvgSequential from nvflare.focs.examples.np.algos.widgets import MetricReceiver -from nvflare.focs.sim.runner import AppRunner +from nvflare.focs.sim.simulator import Simulator def main(): @@ -29,13 +29,13 @@ def main(): ) server_app.add_collab_object("metric_receiver", MetricReceiver()) - runner = AppRunner( + simulator = Simulator( server_app=server_app, client_app=TrainerFactory(delta=1.0), num_clients=2, ) - runner.run() + simulator.run() if __name__ == "__main__": diff --git a/nvflare/focs/examples/np/swarm.py b/nvflare/focs/examples/np/swarm.py index d220cb5816..4b7494ae48 100644 --- a/nvflare/focs/examples/np/swarm.py +++ b/nvflare/focs/examples/np/swarm.py @@ -14,7 +14,7 @@ from nvflare.focs.api.app import ServerApp from nvflare.focs.examples.np.algos.swarm import NPSwarm, NPSwarmClient -from nvflare.focs.sim.runner import AppRunner +from nvflare.focs.sim.simulator import Simulator def main(): @@ -24,13 +24,13 @@ def main(): ) client_app = NPSwarmClient(delta=1.0) - runner = AppRunner( + simulator = Simulator( server_app=server_app, client_app=client_app, num_clients=3, ) - runner.run() + simulator.run() if __name__ == "__main__": diff --git a/nvflare/focs/sim/runner.py b/nvflare/focs/sim/simulator.py similarity index 99% rename from nvflare/focs/sim/runner.py rename to nvflare/focs/sim/simulator.py index 6a0bd69579..973a1865ff 100644 --- a/nvflare/focs/sim/runner.py +++ b/nvflare/focs/sim/simulator.py @@ -24,7 +24,7 @@ from nvflare.focs.sim.backend import SimBackend -class AppRunner: +class Simulator: def _prepare_app_backends(self, app: App): bes = {"": SimBackend(app, app, self.abort_signal, self.thread_executor)} From e79dd30a1a6c12578de66767585374ad61d4e36f Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Tue, 30 Sep 2025 16:13:25 -0400 Subject: [PATCH 021/102] handle abort --- nvflare/focs/api/group.py | 5 +---- nvflare/focs/api/proxy.py | 5 ++++- nvflare/focs/api/resp.py | 3 +-- nvflare/focs/sim/backend.py | 26 ++++++++++++++++++-------- nvflare/focs/sys/backend.py | 8 ++++---- 5 files changed, 28 insertions(+), 19 deletions(-) diff --git a/nvflare/focs/api/group.py b/nvflare/focs/api/group.py index 96c2aaedf4..03f8eb3105 100644 --- a/nvflare/focs/api/group.py +++ b/nvflare/focs/api/group.py @@ -119,10 +119,7 @@ def method(*args, **kwargs): results = {} for name, resp in resps.items(): if resp.resp_time: - if resp.exception: - result = resp.exception - else: - result = resp.result + result = resp.result else: result = TimeoutError() results[name] = result diff --git a/nvflare/focs/api/proxy.py b/nvflare/focs/api/proxy.py index ce25332edc..cd23eb5361 100644 --- a/nvflare/focs/api/proxy.py +++ b/nvflare/focs/api/proxy.py @@ -101,6 +101,9 @@ def method(*args, **kwargs): call_kwargs[CollabMethodArgName.CONTEXT] = ctx print(f"calling target {p.target_name} func {func_name}: {call_args=} {call_kwargs=}") - return p.backend.call_target(self.target_name, func_name, *call_args, **call_kwargs) + result = p.backend.call_target(self.target_name, func_name, *call_args, **call_kwargs) + if isinstance(result, Exception): + raise result + return result return method diff --git a/nvflare/focs/api/resp.py b/nvflare/focs/api/resp.py index 044dc66bfe..89b464bd09 100644 --- a/nvflare/focs/api/resp.py +++ b/nvflare/focs/api/resp.py @@ -23,7 +23,6 @@ class Resp: def __init__(self, process_cb, cb_kwargs, context: Context): self.result = None - self.exception = None self.resp_time = None self.process_cb = process_cb self.cb_kwargs = cb_kwargs @@ -44,5 +43,5 @@ def set_result(self, result): self.resp_time = time.time() def set_exception(self, ex): - self.exception = ex + self.result = ex self.resp_time = time.time() diff --git a/nvflare/focs/sim/backend.py b/nvflare/focs/sim/backend.py index 088867fff1..599dea87c3 100644 --- a/nvflare/focs/sim/backend.py +++ b/nvflare/focs/sim/backend.py @@ -12,7 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. import threading +import time +from nvflare.apis.fl_exception import RunAborted from nvflare.focs.api.app import App from nvflare.focs.api.backend import Backend from nvflare.focs.api.constants import CollabMethodArgName, CollabMethodOptionName @@ -25,7 +27,6 @@ class _Waiter(threading.Event): def __init__(self): super().__init__() self.result = None - self.exception = None class SimBackend(Backend): @@ -56,12 +57,21 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): self.executor.submit(self._run_func, waiter, func, args, kwargs) if waiter: - ok = waiter.wait(timeout) - if not ok: - # timed out - raise TimeoutError(f"function {func_name} timed out after {timeout} seconds") - if waiter.exception: - raise waiter.exception + start_time = time.time() + while True: + if self.abort_signal.triggered: + waiter.result = RunAborted("job is aborted") + + ok = waiter.wait(0.1) + if ok: + break + + waited = time.time() - start_time + if waited > timeout: + # timed out + waiter.result = TimeoutError(f"function {func_name} timed out after {waited} seconds") + break + return waiter.result def _augment_context(self, func, kwargs): @@ -79,7 +89,7 @@ def _run_func(self, waiter: _Waiter, func, args, kwargs): waiter.result = result except Exception as ex: if waiter: - waiter.exception = ex + waiter.result = ex finally: if waiter: waiter.set() diff --git a/nvflare/focs/sys/backend.py b/nvflare/focs/sys/backend.py index b11d04f2b4..caf7f56fff 100644 --- a/nvflare/focs/sys/backend.py +++ b/nvflare/focs/sys/backend.py @@ -62,16 +62,16 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): assert isinstance(reply, Message) rc = reply.get_header(MessageHeaderKey.RETURN_CODE, ReturnCode.OK) if rc == ReturnCode.TIMEOUT: - raise TimeoutError(f"function {func_name} timed out after {timeout} seconds") + return TimeoutError(f"function {func_name} timed out after {timeout} seconds") elif rc != ReturnCode.OK: - raise RuntimeError(f"function {func_name} failed: {rc}") + return RuntimeError(f"function {func_name} failed: {rc}") if not isinstance(reply.payload, dict): - raise RuntimeError(f"function {func_name} failed: reply must be dict but got {type(reply.payload)}") + return RuntimeError(f"function {func_name} failed: reply must be dict but got {type(reply.payload)}") error = reply.payload.get(CallReplyKey.ERROR) if error: - raise RuntimeError(f"function {func_name} failed: {error}") + return RuntimeError(f"function {func_name} failed: {error}") result = reply.payload.get(CallReplyKey.RESULT) self.logger.info(f"got result from {self.target_fqcn}: {result}") From 50cb1f575de662843d173f94fe9566660e1c5b06 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Tue, 30 Sep 2025 16:34:06 -0400 Subject: [PATCH 022/102] catch ctl-c and abort --- nvflare/focs/sim/simulator.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/nvflare/focs/sim/simulator.py b/nvflare/focs/sim/simulator.py index 973a1865ff..c16d585b93 100644 --- a/nvflare/focs/sim/simulator.py +++ b/nvflare/focs/sim/simulator.py @@ -105,6 +105,13 @@ def __init__( self.client_apps = client_apps def run(self): + try: + self._try_run() + except KeyboardInterrupt: + print("execution is aborted by user") + self.abort_signal.trigger(True) + + def _try_run(self): # initialize all apps server_ctx = self.server_app.new_context(caller=self.server_app.name, callee=self.server_app.name) print("initializing server app") @@ -120,14 +127,10 @@ def run(self): result = None for idx, strategy in enumerate(self.server_app.strategies): - try: - print(f"Running Strategy #{idx+1} - {type(strategy).__name__}") - self.server_app.current_strategy = strategy - result = strategy.execute(context=server_ctx) - server_ctx.set_prop(ContextKey.INPUT, result) - except: - traceback.print_exc() - break + print(f"Running Strategy #{idx+1} - {type(strategy).__name__}") + self.server_app.current_strategy = strategy + result = strategy.execute(context=server_ctx) + server_ctx.set_prop(ContextKey.INPUT, result) self.thread_executor.shutdown(wait=False, cancel_futures=True) return result From afc0d75011de44f0ec57bd1685c2c231572ba208 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Wed, 1 Oct 2025 11:47:24 -0400 Subject: [PATCH 023/102] rename to fox --- nvflare/{focs => fox}/__init__.py | 0 nvflare/{focs => fox}/api/__init__.py | 0 nvflare/{focs => fox}/api/app.py | 3 ++- nvflare/{focs => fox}/api/backend.py | 0 nvflare/{focs => fox}/api/constants.py | 11 +++++++++-- nvflare/{focs => fox}/api/ctx.py | 3 ++- nvflare/{focs => fox}/api/dec.py | 0 nvflare/{focs => fox}/api/group.py | 18 ++++++++++++++++++ nvflare/{focs => fox}/api/proxy.py | 0 nvflare/{focs => fox}/api/resp.py | 0 nvflare/{focs => fox}/api/strategy.py | 0 nvflare/{focs => fox}/api/utils.py | 0 nvflare/{focs => fox}/examples/__init__.py | 0 nvflare/{focs => fox}/examples/np/__init__.py | 0 .../examples/np/algos/__init__.py | 0 .../{focs => fox}/examples/np/algos/client.py | 8 ++++---- .../examples/np/algos/strategies.py | 8 ++++---- .../{focs => fox}/examples/np/algos/swarm.py | 16 ++++++++-------- .../{focs => fox}/examples/np/algos/utils.py | 0 .../{focs => fox}/examples/np/algos/widgets.py | 4 ++-- nvflare/{focs => fox}/examples/np/cyclic.py | 8 ++++---- .../{focs => fox}/examples/np/cyclic_avg.py | 8 ++++---- .../examples/np/fed_avg_intime.py | 10 +++++----- .../{focs => fox}/examples/np/fed_avg_para.py | 10 +++++----- .../{focs => fox}/examples/np/fed_avg_seq.py | 10 +++++----- nvflare/{focs => fox}/examples/np/swarm.py | 6 +++--- nvflare/{focs => fox}/sim/__init__.py | 0 nvflare/{focs => fox}/sim/backend.py | 16 +++++++++++----- nvflare/{focs => fox}/sim/simulator.py | 13 +++++++------ nvflare/{focs => fox}/sys/__init__.py | 0 nvflare/{focs => fox}/sys/backend.py | 17 ++++++++++------- nvflare/{focs => fox}/sys/constants.py | 0 nvflare/{focs => fox}/sys/controller.py | 12 +++++++----- nvflare/{focs => fox}/sys/executor.py | 8 +++++--- nvflare/{focs => fox}/sys/utils.py | 6 +++--- 35 files changed, 118 insertions(+), 77 deletions(-) rename nvflare/{focs => fox}/__init__.py (100%) rename nvflare/{focs => fox}/api/__init__.py (100%) rename nvflare/{focs => fox}/api/app.py (98%) rename nvflare/{focs => fox}/api/backend.py (100%) rename nvflare/{focs => fox}/api/constants.py (82%) rename nvflare/{focs => fox}/api/ctx.py (89%) rename nvflare/{focs => fox}/api/dec.py (100%) rename nvflare/{focs => fox}/api/group.py (90%) rename nvflare/{focs => fox}/api/proxy.py (100%) rename nvflare/{focs => fox}/api/resp.py (100%) rename nvflare/{focs => fox}/api/strategy.py (100%) rename nvflare/{focs => fox}/api/utils.py (100%) rename nvflare/{focs => fox}/examples/__init__.py (100%) rename nvflare/{focs => fox}/examples/np/__init__.py (100%) rename nvflare/{focs => fox}/examples/np/algos/__init__.py (100%) rename nvflare/{focs => fox}/examples/np/algos/client.py (90%) rename nvflare/{focs => fox}/examples/np/algos/strategies.py (96%) rename nvflare/{focs => fox}/examples/np/algos/swarm.py (91%) rename nvflare/{focs => fox}/examples/np/algos/utils.py (100%) rename nvflare/{focs => fox}/examples/np/algos/widgets.py (93%) rename nvflare/{focs => fox}/examples/np/cyclic.py (81%) rename nvflare/{focs => fox}/examples/np/cyclic_avg.py (83%) rename nvflare/{focs => fox}/examples/np/fed_avg_intime.py (79%) rename nvflare/{focs => fox}/examples/np/fed_avg_para.py (78%) rename nvflare/{focs => fox}/examples/np/fed_avg_seq.py (78%) rename nvflare/{focs => fox}/examples/np/swarm.py (85%) rename nvflare/{focs => fox}/sim/__init__.py (100%) rename nvflare/{focs => fox}/sim/backend.py (88%) rename nvflare/{focs => fox}/sim/simulator.py (93%) rename nvflare/{focs => fox}/sys/__init__.py (100%) rename nvflare/{focs => fox}/sys/backend.py (90%) rename nvflare/{focs => fox}/sys/constants.py (100%) rename nvflare/{focs => fox}/sys/controller.py (97%) rename nvflare/{focs => fox}/sys/executor.py (96%) rename nvflare/{focs => fox}/sys/utils.py (96%) diff --git a/nvflare/focs/__init__.py b/nvflare/fox/__init__.py similarity index 100% rename from nvflare/focs/__init__.py rename to nvflare/fox/__init__.py diff --git a/nvflare/focs/api/__init__.py b/nvflare/fox/api/__init__.py similarity index 100% rename from nvflare/focs/api/__init__.py rename to nvflare/fox/api/__init__.py diff --git a/nvflare/focs/api/app.py b/nvflare/fox/api/app.py similarity index 98% rename from nvflare/focs/api/app.py rename to nvflare/fox/api/app.py index b80471387e..222bf923fb 100644 --- a/nvflare/focs/api/app.py +++ b/nvflare/fox/api/app.py @@ -29,6 +29,7 @@ def __init__(self): self.name = None self.server = None self.clients = None + self.env_type = None self._me = None self._collab_objs = {} self._abort_signal = None @@ -119,7 +120,7 @@ def initialize(self, context: Context): init_func(**kwargs) def new_context(self, caller: str, callee: str, props: dict = None): - ctx = Context(caller, callee, self._abort_signal, props) + ctx = Context(self.env_type, caller, callee, self._abort_signal, props) ctx.app = self ctx.server = self.server ctx.clients = self.clients diff --git a/nvflare/focs/api/backend.py b/nvflare/fox/api/backend.py similarity index 100% rename from nvflare/focs/api/backend.py rename to nvflare/fox/api/backend.py diff --git a/nvflare/focs/api/constants.py b/nvflare/fox/api/constants.py similarity index 82% rename from nvflare/focs/api/constants.py rename to nvflare/fox/api/constants.py index bd40df886a..d28aaf3633 100644 --- a/nvflare/focs/api/constants.py +++ b/nvflare/fox/api/constants.py @@ -17,9 +17,16 @@ class CollabMethodArgName: class CollabMethodOptionName: - BLOCKING = "blocking" - TIMEOUT = "timeout" + BLOCKING = "_blocking" + TIMEOUT = "_timeout" + OPTIONAL = "_optional" + SECURE = "_secure" class ContextKey: INPUT = "input" + + +class EnvType: + SIMULATION = "simulation" + SYSTEM = "system" diff --git a/nvflare/focs/api/ctx.py b/nvflare/fox/api/ctx.py similarity index 89% rename from nvflare/focs/api/ctx.py rename to nvflare/fox/api/ctx.py index 1332177b37..c45c62cdb7 100644 --- a/nvflare/focs/api/ctx.py +++ b/nvflare/fox/api/ctx.py @@ -16,7 +16,8 @@ class Context: - def __init__(self, caller: str, callee: str, abort_signal: Signal, props: dict = None): + def __init__(self, env_type: str, caller: str, callee: str, abort_signal: Signal, props: dict = None): + self.env_type = env_type self.caller = caller self.callee = callee self.abort_signal = abort_signal diff --git a/nvflare/focs/api/dec.py b/nvflare/fox/api/dec.py similarity index 100% rename from nvflare/focs/api/dec.py rename to nvflare/fox/api/dec.py diff --git a/nvflare/focs/api/group.py b/nvflare/fox/api/group.py similarity index 90% rename from nvflare/focs/api/group.py rename to nvflare/fox/api/group.py index 03f8eb3105..5116dc5628 100644 --- a/nvflare/focs/api/group.py +++ b/nvflare/fox/api/group.py @@ -33,6 +33,8 @@ def __init__( proxies: List[Proxy], blocking: bool = True, timeout: float = 5.0, + optional: bool = False, + secure: bool = False, min_resps: int = None, wait_after_min_resps: float = None, process_resp_cb=None, @@ -46,6 +48,8 @@ def __init__( self._proxies = proxies self._blocking = blocking self._timeout = timeout + self._optional = optional + self._secure = secure self._min_resps = min_resps self._wait_after_min_resps = wait_after_min_resps self._process_resp_cb = process_resp_cb @@ -57,6 +61,8 @@ def __init__( if not wait_after_min_resps: self._wait_after_min_resps = 0 + print(f"min resps: {self._min_resps}") + def __getattr__(self, func_name): """ This method is called when Python cannot find an invoked method func_name of this class. @@ -76,6 +82,9 @@ def method(*args, **kwargs): kwargs_copy[CollabMethodOptionName.TIMEOUT] = self._timeout kwargs_copy[CollabMethodOptionName.BLOCKING] = self._blocking + kwargs_copy[CollabMethodOptionName.SECURE] = self._secure + kwargs_copy[CollabMethodOptionName.OPTIONAL] = self._optional + resp = Resp(self._process_resp_cb, self._cb_kwargs, ctx) resps[p.name] = resp @@ -102,6 +111,7 @@ def method(*args, **kwargs): break now = time.time() + print(f"what is min_resps: {self=} {self._min_resps}") if resps_received >= self._min_resps: if not min_received_time: min_received_time = now @@ -133,6 +143,8 @@ def group( proxies: List[Proxy], blocking: bool = True, timeout: float = 5.0, + optional: bool = False, + secure: bool = False, min_resps: int = None, wait_after_min_resps: float = None, process_resp_cb=None, @@ -144,6 +156,8 @@ def group( proxies, blocking, timeout, + optional, + secure, min_resps, wait_after_min_resps, process_resp_cb, @@ -155,6 +169,8 @@ def all_clients( ctx: Context, blocking: bool = True, timeout: float = 5.0, + optional: bool = False, + secure: bool = False, min_resps: int = None, wait_after_min_resps: float = None, process_resp_cb=None, @@ -166,6 +182,8 @@ def all_clients( ctx.clients, blocking, timeout, + optional, + secure, min_resps, wait_after_min_resps, process_resp_cb, diff --git a/nvflare/focs/api/proxy.py b/nvflare/fox/api/proxy.py similarity index 100% rename from nvflare/focs/api/proxy.py rename to nvflare/fox/api/proxy.py diff --git a/nvflare/focs/api/resp.py b/nvflare/fox/api/resp.py similarity index 100% rename from nvflare/focs/api/resp.py rename to nvflare/fox/api/resp.py diff --git a/nvflare/focs/api/strategy.py b/nvflare/fox/api/strategy.py similarity index 100% rename from nvflare/focs/api/strategy.py rename to nvflare/fox/api/strategy.py diff --git a/nvflare/focs/api/utils.py b/nvflare/fox/api/utils.py similarity index 100% rename from nvflare/focs/api/utils.py rename to nvflare/fox/api/utils.py diff --git a/nvflare/focs/examples/__init__.py b/nvflare/fox/examples/__init__.py similarity index 100% rename from nvflare/focs/examples/__init__.py rename to nvflare/fox/examples/__init__.py diff --git a/nvflare/focs/examples/np/__init__.py b/nvflare/fox/examples/np/__init__.py similarity index 100% rename from nvflare/focs/examples/np/__init__.py rename to nvflare/fox/examples/np/__init__.py diff --git a/nvflare/focs/examples/np/algos/__init__.py b/nvflare/fox/examples/np/algos/__init__.py similarity index 100% rename from nvflare/focs/examples/np/algos/__init__.py rename to nvflare/fox/examples/np/algos/__init__.py diff --git a/nvflare/focs/examples/np/algos/client.py b/nvflare/fox/examples/np/algos/client.py similarity index 90% rename from nvflare/focs/examples/np/algos/client.py rename to nvflare/fox/examples/np/algos/client.py index 8c38d4f1af..aea127a775 100644 --- a/nvflare/focs/examples/np/algos/client.py +++ b/nvflare/fox/examples/np/algos/client.py @@ -13,9 +13,9 @@ # limitations under the License. import random -from nvflare.focs.api.app import ClientApp, ClientAppFactory -from nvflare.focs.api.ctx import Context -from nvflare.focs.api.dec import collab +from nvflare.fox.api.app import ClientApp, ClientAppFactory +from nvflare.fox.api.ctx import Context +from nvflare.fox.api.dec import collab class NPTrainer(ClientApp): @@ -35,7 +35,7 @@ def train(self, current_round, weights, context: Context): # if metric_receiver: # self.server.accept_metric({"round": r, "y": 2}) # - self.server.fire_event("metrics", {"round": current_round, "y": 10}, blocking=False) + self.server.fire_event("metrics", {"round": current_round, "y": 10}, _blocking=False) return weights + self.delta @collab diff --git a/nvflare/focs/examples/np/algos/strategies.py b/nvflare/fox/examples/np/algos/strategies.py similarity index 96% rename from nvflare/focs/examples/np/algos/strategies.py rename to nvflare/fox/examples/np/algos/strategies.py index eda806f895..bea1767714 100644 --- a/nvflare/focs/examples/np/algos/strategies.py +++ b/nvflare/fox/examples/np/algos/strategies.py @@ -15,10 +15,10 @@ import numpy as np -from nvflare.focs.api.constants import ContextKey -from nvflare.focs.api.ctx import Context -from nvflare.focs.api.group import all_clients -from nvflare.focs.api.strategy import Strategy +from nvflare.fox.api.constants import ContextKey +from nvflare.fox.api.ctx import Context +from nvflare.fox.api.group import all_clients +from nvflare.fox.api.strategy import Strategy from .utils import parse_array_def diff --git a/nvflare/focs/examples/np/algos/swarm.py b/nvflare/fox/examples/np/algos/swarm.py similarity index 91% rename from nvflare/focs/examples/np/algos/swarm.py rename to nvflare/fox/examples/np/algos/swarm.py index c8e4418c60..60d6a996b5 100644 --- a/nvflare/focs/examples/np/algos/swarm.py +++ b/nvflare/fox/examples/np/algos/swarm.py @@ -15,12 +15,12 @@ import threading import traceback -from nvflare.focs.api.app import ClientApp -from nvflare.focs.api.ctx import Context -from nvflare.focs.api.dec import collab -from nvflare.focs.api.group import all_clients -from nvflare.focs.api.strategy import Strategy -from nvflare.focs.examples.np.algos.utils import parse_array_def +from nvflare.fox.api.app import ClientApp +from nvflare.fox.api.ctx import Context +from nvflare.fox.api.dec import collab +from nvflare.fox.api.group import all_clients +from nvflare.fox.api.strategy import Strategy +from nvflare.fox.examples.np.algos.utils import parse_array_def class NPSwarm(Strategy): @@ -83,7 +83,7 @@ def swarm_learn(self, num_rounds, model, current_round, context: Context): # self.server.fire_event("all_done", "OK", blocking=False) print("notify server all done!") try: - self.server.all_done("OK", blocking=False) + self.server.all_done("OK", _blocking=False) except: traceback.print_exc() print("Swarm Training is DONE!") @@ -93,7 +93,7 @@ def swarm_learn(self, num_rounds, model, current_round, context: Context): next_round = current_round + 1 next_client_idx = random.randint(0, len(self.clients) - 1) next_client = self.clients[next_client_idx] - next_client.swarm_learn(num_rounds, new_model, next_round, blocking=False) + next_client.swarm_learn(num_rounds, new_model, next_round, _blocking=False) @collab def start(self, num_rounds, initial_model, context: Context): diff --git a/nvflare/focs/examples/np/algos/utils.py b/nvflare/fox/examples/np/algos/utils.py similarity index 100% rename from nvflare/focs/examples/np/algos/utils.py rename to nvflare/fox/examples/np/algos/utils.py diff --git a/nvflare/focs/examples/np/algos/widgets.py b/nvflare/fox/examples/np/algos/widgets.py similarity index 93% rename from nvflare/focs/examples/np/algos/widgets.py rename to nvflare/fox/examples/np/algos/widgets.py index 3a2e7b0885..dd1abec680 100644 --- a/nvflare/focs/examples/np/algos/widgets.py +++ b/nvflare/fox/examples/np/algos/widgets.py @@ -11,8 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from nvflare.focs.api.ctx import Context -from nvflare.focs.api.dec import collab +from nvflare.fox.api.ctx import Context +from nvflare.fox.api.dec import collab class MetricReceiver: diff --git a/nvflare/focs/examples/np/cyclic.py b/nvflare/fox/examples/np/cyclic.py similarity index 81% rename from nvflare/focs/examples/np/cyclic.py rename to nvflare/fox/examples/np/cyclic.py index 64062d6530..e4ecaf69e2 100644 --- a/nvflare/focs/examples/np/cyclic.py +++ b/nvflare/fox/examples/np/cyclic.py @@ -11,10 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from nvflare.focs.api.app import ServerApp -from nvflare.focs.examples.np.algos.client import NPTrainer -from nvflare.focs.examples.np.algos.strategies import NPCyclic -from nvflare.focs.sim.simulator import Simulator +from nvflare.fox.api.app import ServerApp +from nvflare.fox.examples.np.algos.client import NPTrainer +from nvflare.fox.examples.np.algos.strategies import NPCyclic +from nvflare.fox.sim.simulator import Simulator def main(): diff --git a/nvflare/focs/examples/np/cyclic_avg.py b/nvflare/fox/examples/np/cyclic_avg.py similarity index 83% rename from nvflare/focs/examples/np/cyclic_avg.py rename to nvflare/fox/examples/np/cyclic_avg.py index f270dd1a72..041d2e3d85 100644 --- a/nvflare/focs/examples/np/cyclic_avg.py +++ b/nvflare/fox/examples/np/cyclic_avg.py @@ -11,10 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from nvflare.focs.api.app import ServerApp -from nvflare.focs.examples.np.algos.client import NPTrainer -from nvflare.focs.examples.np.algos.strategies import NPCyclic, NPFedAvgParallel -from nvflare.focs.sim.simulator import Simulator +from nvflare.fox.api.app import ServerApp +from nvflare.fox.examples.np.algos.client import NPTrainer +from nvflare.fox.examples.np.algos.strategies import NPCyclic, NPFedAvgParallel +from nvflare.fox.sim.simulator import Simulator def main(): diff --git a/nvflare/focs/examples/np/fed_avg_intime.py b/nvflare/fox/examples/np/fed_avg_intime.py similarity index 79% rename from nvflare/focs/examples/np/fed_avg_intime.py rename to nvflare/fox/examples/np/fed_avg_intime.py index 71d7b5c8df..1adba64a4e 100644 --- a/nvflare/focs/examples/np/fed_avg_intime.py +++ b/nvflare/fox/examples/np/fed_avg_intime.py @@ -11,11 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from nvflare.focs.api.app import ServerApp -from nvflare.focs.examples.np.algos.client import NPTrainer -from nvflare.focs.examples.np.algos.strategies import NPFedAvgInTime -from nvflare.focs.examples.np.algos.widgets import MetricReceiver -from nvflare.focs.sim.simulator import Simulator +from nvflare.fox.api.app import ServerApp +from nvflare.fox.examples.np.algos.client import NPTrainer +from nvflare.fox.examples.np.algos.strategies import NPFedAvgInTime +from nvflare.fox.examples.np.algos.widgets import MetricReceiver +from nvflare.fox.sim.simulator import Simulator def main(): diff --git a/nvflare/focs/examples/np/fed_avg_para.py b/nvflare/fox/examples/np/fed_avg_para.py similarity index 78% rename from nvflare/focs/examples/np/fed_avg_para.py rename to nvflare/fox/examples/np/fed_avg_para.py index c0dcaebada..997a04d64c 100644 --- a/nvflare/focs/examples/np/fed_avg_para.py +++ b/nvflare/fox/examples/np/fed_avg_para.py @@ -11,11 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from nvflare.focs.api.app import ServerApp -from nvflare.focs.examples.np.algos.client import NPTrainer -from nvflare.focs.examples.np.algos.strategies import NPFedAvgParallel -from nvflare.focs.examples.np.algos.widgets import MetricReceiver -from nvflare.focs.sim.simulator import Simulator +from nvflare.fox.api.app import ServerApp +from nvflare.fox.examples.np.algos.client import NPTrainer +from nvflare.fox.examples.np.algos.strategies import NPFedAvgParallel +from nvflare.fox.examples.np.algos.widgets import MetricReceiver +from nvflare.fox.sim.simulator import Simulator def main(): diff --git a/nvflare/focs/examples/np/fed_avg_seq.py b/nvflare/fox/examples/np/fed_avg_seq.py similarity index 78% rename from nvflare/focs/examples/np/fed_avg_seq.py rename to nvflare/fox/examples/np/fed_avg_seq.py index 6322acf1b4..cb9c86fe79 100644 --- a/nvflare/focs/examples/np/fed_avg_seq.py +++ b/nvflare/fox/examples/np/fed_avg_seq.py @@ -11,11 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from nvflare.focs.api.app import ServerApp -from nvflare.focs.examples.np.algos.client import TrainerFactory -from nvflare.focs.examples.np.algos.strategies import NPFedAvgSequential -from nvflare.focs.examples.np.algos.widgets import MetricReceiver -from nvflare.focs.sim.simulator import Simulator +from nvflare.fox.api.app import ServerApp +from nvflare.fox.examples.np.algos.client import TrainerFactory +from nvflare.fox.examples.np.algos.strategies import NPFedAvgSequential +from nvflare.fox.examples.np.algos.widgets import MetricReceiver +from nvflare.fox.sim.simulator import Simulator def main(): diff --git a/nvflare/focs/examples/np/swarm.py b/nvflare/fox/examples/np/swarm.py similarity index 85% rename from nvflare/focs/examples/np/swarm.py rename to nvflare/fox/examples/np/swarm.py index 4b7494ae48..b233f25876 100644 --- a/nvflare/focs/examples/np/swarm.py +++ b/nvflare/fox/examples/np/swarm.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from nvflare.focs.api.app import ServerApp -from nvflare.focs.examples.np.algos.swarm import NPSwarm, NPSwarmClient -from nvflare.focs.sim.simulator import Simulator +from nvflare.fox.api.app import ServerApp +from nvflare.fox.examples.np.algos.swarm import NPSwarm, NPSwarmClient +from nvflare.fox.sim.simulator import Simulator def main(): diff --git a/nvflare/focs/sim/__init__.py b/nvflare/fox/sim/__init__.py similarity index 100% rename from nvflare/focs/sim/__init__.py rename to nvflare/fox/sim/__init__.py diff --git a/nvflare/focs/sim/backend.py b/nvflare/fox/sim/backend.py similarity index 88% rename from nvflare/focs/sim/backend.py rename to nvflare/fox/sim/backend.py index 599dea87c3..511ff57023 100644 --- a/nvflare/focs/sim/backend.py +++ b/nvflare/fox/sim/backend.py @@ -15,11 +15,11 @@ import time from nvflare.apis.fl_exception import RunAborted -from nvflare.focs.api.app import App -from nvflare.focs.api.backend import Backend -from nvflare.focs.api.constants import CollabMethodArgName, CollabMethodOptionName -from nvflare.focs.api.dec import adjust_kwargs -from nvflare.focs.api.resp import Resp +from nvflare.fox.api.app import App +from nvflare.fox.api.backend import Backend +from nvflare.fox.api.constants import CollabMethodArgName, CollabMethodOptionName +from nvflare.fox.api.dec import adjust_kwargs +from nvflare.fox.api.resp import Resp class _Waiter(threading.Event): @@ -51,6 +51,10 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): blocking = kwargs.pop(CollabMethodOptionName.BLOCKING, True) timeout = kwargs.pop(CollabMethodOptionName.TIMEOUT, None) + # these options don't apply to simulation + kwargs.pop(CollabMethodOptionName.OPTIONAL, None) + kwargs.pop(CollabMethodOptionName.SECURE, None) + waiter = None if blocking: waiter = _Waiter() @@ -98,6 +102,8 @@ def call_target_with_resp(self, resp: Resp, target_name: str, func_name: str, *a # do not use the optional args - they are managed by the group kwargs.pop(CollabMethodOptionName.BLOCKING, None) kwargs.pop(CollabMethodOptionName.TIMEOUT, None) + kwargs.pop(CollabMethodOptionName.OPTIONAL, None) + kwargs.pop(CollabMethodOptionName.SECURE, None) func = self._get_func(func_name) if not func: diff --git a/nvflare/focs/sim/simulator.py b/nvflare/fox/sim/simulator.py similarity index 93% rename from nvflare/focs/sim/simulator.py rename to nvflare/fox/sim/simulator.py index c16d585b93..a5288d1e32 100644 --- a/nvflare/focs/sim/simulator.py +++ b/nvflare/fox/sim/simulator.py @@ -12,16 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. import copy -import traceback from concurrent.futures import ThreadPoolExecutor from typing import Union from nvflare.apis.signal import Signal -from nvflare.focs.api.app import App, ClientApp, ClientAppFactory, ServerApp -from nvflare.focs.api.constants import ContextKey -from nvflare.focs.api.dec import get_object_collab_interface -from nvflare.focs.api.proxy import Proxy -from nvflare.focs.sim.backend import SimBackend +from nvflare.fox.api.app import App, ClientApp, ClientAppFactory, ServerApp +from nvflare.fox.api.constants import ContextKey, EnvType +from nvflare.fox.api.dec import get_object_collab_interface +from nvflare.fox.api.proxy import Proxy +from nvflare.fox.sim.backend import SimBackend class Simulator: @@ -74,6 +73,7 @@ def __init__( self.abort_signal = Signal() server_app.name = "server" + server_app.env_type = EnvType.SIMULATION self.server_app = server_app self.client_app = client_app self.thread_executor = ThreadPoolExecutor(max_workers=max_workers) @@ -87,6 +87,7 @@ def __init__( else: app = client_app.make_client_app(name) app.name = name + app.env_type = EnvType.SIMULATION client_apps[name] = app backends = {server_app.name: self._prepare_app_backends(server_app)} diff --git a/nvflare/focs/sys/__init__.py b/nvflare/fox/sys/__init__.py similarity index 100% rename from nvflare/focs/sys/__init__.py rename to nvflare/fox/sys/__init__.py diff --git a/nvflare/focs/sys/backend.py b/nvflare/fox/sys/backend.py similarity index 90% rename from nvflare/focs/sys/backend.py rename to nvflare/fox/sys/backend.py index caf7f56fff..3ea00352fa 100644 --- a/nvflare/focs/sys/backend.py +++ b/nvflare/fox/sys/backend.py @@ -11,9 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from nvflare.focs.api.backend import Backend -from nvflare.focs.api.constants import CollabMethodArgName, CollabMethodOptionName -from nvflare.focs.api.resp import Resp +from nvflare.fox.api.backend import Backend +from nvflare.fox.api.constants import CollabMethodArgName, CollabMethodOptionName +from nvflare.fox.api.resp import Resp from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey, ReturnCode from nvflare.fuel.f3.cellnet.utils import new_cell_message from nvflare.fuel.f3.message import Message @@ -35,6 +35,9 @@ def __init__(self, caller, cell, target_fqcn, abort_signal, thread_executor): def call_target(self, target_name: str, func_name: str, *args, **kwargs): blocking = kwargs.pop(CollabMethodOptionName.BLOCKING, True) timeout = kwargs.pop(CollabMethodOptionName.TIMEOUT, 10.0) + optional = kwargs.pop(CollabMethodOptionName.OPTIONAL, False) + secure = kwargs.pop(CollabMethodOptionName.SECURE, False) + kwargs.pop(CollabMethodArgName.CONTEXT, None) payload = { @@ -55,8 +58,8 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): topic=MSG_TOPIC, request=request, timeout=timeout, - secure=False, - optional=False, + secure=secure, + optional=optional, abort_signal=self.abort_signal, ) assert isinstance(reply, Message) @@ -84,8 +87,8 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): topic=MSG_TOPIC, targets=self.target_fqcn, message=request, - secure=False, - optional=False, + secure=secure, + optional=optional, ) def call_target_with_resp(self, resp: Resp, target_name: str, func_name: str, *args, **kwargs): diff --git a/nvflare/focs/sys/constants.py b/nvflare/fox/sys/constants.py similarity index 100% rename from nvflare/focs/sys/constants.py rename to nvflare/fox/sys/constants.py diff --git a/nvflare/focs/sys/controller.py b/nvflare/fox/sys/controller.py similarity index 97% rename from nvflare/focs/sys/controller.py rename to nvflare/fox/sys/controller.py index 362dc8ba2b..3292bf72a7 100644 --- a/nvflare/focs/sys/controller.py +++ b/nvflare/fox/sys/controller.py @@ -22,10 +22,10 @@ from nvflare.apis.impl.controller import Controller from nvflare.apis.shareable import ReturnCode, Shareable from nvflare.apis.signal import Signal -from nvflare.focs.api.app import ServerApp -from nvflare.focs.api.constants import ContextKey -from nvflare.focs.api.proxy import Proxy -from nvflare.focs.api.strategy import Strategy +from nvflare.fox.api.app import ServerApp +from nvflare.fox.api.constants import ContextKey, EnvType +from nvflare.fox.api.proxy import Proxy +from nvflare.fox.api.strategy import Strategy from nvflare.fuel.f3.cellnet.fqcn import FQCN from .backend import SysBackend @@ -44,7 +44,7 @@ def __init__(self, collab_interface: dict): self.collab_interface = collab_interface -class FocsController(Controller): +class FoxController(Controller): def __init__( self, @@ -81,6 +81,8 @@ def start_controller(self, fl_ctx: FLContext): app = ServerApp() app.name = "server" + app.env_type = EnvType.SYSTEM + for cid in self.strategy_ids: strategy = engine.get_component(cid) if not isinstance(strategy, Strategy): diff --git a/nvflare/focs/sys/executor.py b/nvflare/fox/sys/executor.py similarity index 96% rename from nvflare/focs/sys/executor.py rename to nvflare/fox/sys/executor.py index d1a138b3b0..bc8f2f3c2d 100644 --- a/nvflare/focs/sys/executor.py +++ b/nvflare/fox/sys/executor.py @@ -22,8 +22,9 @@ from nvflare.apis.job_def import JobMetaKey from nvflare.apis.shareable import Shareable, make_reply from nvflare.apis.signal import Signal -from nvflare.focs.api.app import ClientApp, ClientAppFactory -from nvflare.focs.api.proxy import Proxy +from nvflare.fox.api.app import ClientApp, ClientAppFactory +from nvflare.fox.api.constants import EnvType +from nvflare.fox.api.proxy import Proxy from nvflare.fuel.f3.cellnet.fqcn import FQCN from .backend import SysBackend @@ -31,7 +32,7 @@ from .utils import prepare_for_remote_call -class FocsExecutor(Executor): +class FoxExecutor(Executor): def __init__(self, client_app_id: str, client_target_obj_ids: Dict[str, str] = None, max_call_threads=100): Executor.__init__(self) @@ -59,6 +60,7 @@ def _handle_start_run(self, event_type: str, fl_ctx: FLContext): self.client_app = client_app.make_client_app(client_name) self.client_app.name = client_name + self.client_app.env_type = EnvType.SYSTEM if self.client_target_obj_ids: for name, cid in self.client_target_obj_ids: diff --git a/nvflare/focs/sys/utils.py b/nvflare/fox/sys/utils.py similarity index 96% rename from nvflare/focs/sys/utils.py rename to nvflare/fox/sys/utils.py index 67398148a1..00425e9e7f 100644 --- a/nvflare/focs/sys/utils.py +++ b/nvflare/fox/sys/utils.py @@ -11,9 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from nvflare.focs.api.app import App -from nvflare.focs.api.constants import CollabMethodArgName -from nvflare.focs.api.dec import adjust_kwargs +from nvflare.fox.api.app import App +from nvflare.fox.api.constants import CollabMethodArgName +from nvflare.fox.api.dec import adjust_kwargs from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey, ReturnCode from nvflare.fuel.f3.cellnet.utils import new_cell_message from nvflare.fuel.f3.message import Message From ed751eb68dab7ccef5923a9080dd57bc65454682 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Thu, 2 Oct 2025 16:22:56 -0400 Subject: [PATCH 024/102] add filter support --- nvflare/fox/api/app.py | 94 ++++++++++++++++++++++- nvflare/fox/api/constants.py | 15 ++++ nvflare/fox/api/filter.py | 79 +++++++++++++++++++ nvflare/fox/api/group.py | 27 ++++--- nvflare/fox/api/proxy.py | 46 ++++++++--- nvflare/fox/api/utils.py | 17 ++++ nvflare/fox/examples/np/algos/client.py | 1 + nvflare/fox/examples/np/algos/filters.py | 39 ++++++++++ nvflare/fox/examples/np/fed_avg_intime.py | 4 +- nvflare/fox/sim/backend.py | 2 +- nvflare/fox/sys/controller.py | 46 ++++++++++- nvflare/fox/sys/executor.py | 12 +-- 12 files changed, 348 insertions(+), 34 deletions(-) create mode 100644 nvflare/fox/api/filter.py create mode 100644 nvflare/fox/examples/np/algos/filters.py diff --git a/nvflare/fox/api/app.py b/nvflare/fox/api/app.py index 222bf923fb..8fbd43bd60 100644 --- a/nvflare/fox/api/app.py +++ b/nvflare/fox/api/app.py @@ -12,15 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. import copy +import fnmatch from abc import ABC, abstractmethod from typing import List -from .constants import CollabMethodArgName +from .constants import CollabMethodArgName, ContextKey, FilterDirection from .ctx import Context from .dec import collab, get_object_collab_interface, is_collab +from .filter import CallFilter, FilterChain, ResultFilter from .proxy import Proxy from .strategy import Strategy -from .utils import check_context_support +from .utils import check_context_support, get_collab_object_name class App: @@ -35,6 +37,94 @@ def __init__(self): self._abort_signal = None self._props = {} self._event_handlers = {} # event type => list of (cb, kwargs) + self._incoming_call_filter_chains = [] + self._outgoing_call_filter_chains = [] + self._incoming_result_filter_chains = [] + self._outgoing_result_filter_chains = [] + + @staticmethod + def _add_filters(pattern: str, filters, to_list: list, filter_type): + if not filters: + return + + if not isinstance(filters, list): + raise ValueError(f"filters must be a list but got {type(filters)}") + + for i, f in enumerate(filters): + if not isinstance(f, filter_type): + raise ValueError(f"filter {i} must be {filter_type} but got {type(f)}") + + chain = FilterChain(pattern, filter_type) + chain.add_filters(filters) + to_list.append(chain) + + def add_incoming_call_filters(self, pattern: str, filters: List[CallFilter]): + self._add_filters(pattern, filters, self._incoming_call_filter_chains, CallFilter) + + def add_outgoing_call_filters(self, pattern: str, filters: List[CallFilter]): + self._add_filters(pattern, filters, self._outgoing_call_filter_chains, CallFilter) + + def add_incoming_result_filters(self, pattern: str, filters: List[ResultFilter]): + self._add_filters(pattern, filters, self._incoming_result_filter_chains, ResultFilter) + + def add_outgoing_result_filters(self, pattern: str, filters: List[ResultFilter]): + self._add_filters(pattern, filters, self._outgoing_result_filter_chains, ResultFilter) + + @staticmethod + def _find_filter_chain(chains: List[FilterChain], target_name: str, func_name: str, ctx: Context): + """ + + Args: + chains: + target_name: + func_name: + + Returns: + + """ + if not chains: + return None + + collab_obj_name = get_collab_object_name(target_name) + qualified_func_name = f"{collab_obj_name}.{func_name}" + ctx.set_prop(ContextKey.QUALIFIED_FUNC_NAME, qualified_func_name) + + for c in chains: + if fnmatch.fnmatch(qualified_func_name, c.pattern): + return c + return None + + def apply_incoming_call_filters(self, target_name: str, func_name: str, func_kwargs, context: Context): + filter_chain = self._find_filter_chain(self._incoming_call_filter_chains, target_name, func_name, context) + if filter_chain: + context.set_prop(ContextKey.DIRECTION, FilterDirection.INCOMING) + return filter_chain.apply_filters(func_kwargs, context) + else: + return func_kwargs + + def apply_outgoing_call_filters(self, target_name: str, func_name: str, func_kwargs, context: Context): + filter_chain = self._find_filter_chain(self._outgoing_call_filter_chains, target_name, func_name, context) + if filter_chain: + context.set_prop(ContextKey.DIRECTION, FilterDirection.OUTGOING) + return filter_chain.apply_filters(func_kwargs, context) + else: + return func_kwargs + + def apply_incoming_result_filters(self, target_name: str, func_name: str, result, context: Context): + filter_chain = self._find_filter_chain(self._incoming_result_filter_chains, target_name, func_name, context) + if filter_chain: + context.set_prop(ContextKey.DIRECTION, FilterDirection.INCOMING) + return filter_chain.apply_filters(result, context) + else: + return result + + def apply_outgoing_result_filters(self, target_name: str, func_name: str, result, context: Context): + filter_chain = self._find_filter_chain(self._outgoing_result_filter_chains, target_name, func_name, context) + if filter_chain: + context.set_prop(ContextKey.DIRECTION, FilterDirection.OUTGOING) + return filter_chain.apply_filters(result, context) + else: + return result def set_prop(self, name: str, value): self._props[name] = value diff --git a/nvflare/fox/api/constants.py b/nvflare/fox/api/constants.py index d28aaf3633..a7db0e808a 100644 --- a/nvflare/fox/api/constants.py +++ b/nvflare/fox/api/constants.py @@ -23,8 +23,23 @@ class CollabMethodOptionName: SECURE = "_secure" +OPTION_ARGS = [ + CollabMethodOptionName.BLOCKING, + CollabMethodOptionName.TIMEOUT, + CollabMethodOptionName.OPTIONAL, + CollabMethodOptionName.SECURE, +] + + class ContextKey: INPUT = "input" + QUALIFIED_FUNC_NAME = "qualified_func_name" + DIRECTION = "direction" + + +class FilterDirection: + INCOMING = "incoming" + OUTGOING = "outgoing" class EnvType: diff --git a/nvflare/fox/api/filter.py b/nvflare/fox/api/filter.py new file mode 100644 index 0000000000..790175e8a8 --- /dev/null +++ b/nvflare/fox/api/filter.py @@ -0,0 +1,79 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Any + +from .ctx import Context + + +class CallFilter: + + def filter_call(self, func_kwargs: dict, context: Context): + """Filter kwargs of function call. + + Args: + func_kwargs: kwargs to be filtered + context: call context + + Returns: filtered kwargs that will be passed to a collab func. + + """ + return func_kwargs + + +class ResultFilter: + + def filter_result(self, result: Any, context: Context): + """Filter result produced by a collab func. + + Args: + result: data to be filtered + context: call context + + Returns: filtered result + + """ + return result + + +class FilterChain: + + def __init__(self, pattern, filter_type): + if filter_type not in [ResultFilter, CallFilter]: + raise ValueError( + f"filter_type must be type of {ResultFilter.__name__} or {CallFilter.__name__} but got {filter_type}" + ) + self.pattern = pattern + self.filter_type = filter_type + self.filters = [] + + def add_filters(self, filters): + if not filters: + return + + if isinstance(filters, list): + if not all(isinstance(item, self.filter_type) for item in filters): + raise ValueError(f"some items in filters are not {self.filter_type}") + self.filters.extend(filters) + else: + if not isinstance(filters, self.filter_type): + raise ValueError(f"filter item must be {self.filter_type} but got {type(filters)}") + self.filters.append(filters) + + def apply_filters(self, data, context: Context): + for f in self.filters: + if isinstance(f, ResultFilter): + data = f.filter_result(data, context) + else: + data = f.filter_call(data, context) + return data diff --git a/nvflare/fox/api/group.py b/nvflare/fox/api/group.py index 5116dc5628..4563c1de81 100644 --- a/nvflare/fox/api/group.py +++ b/nvflare/fox/api/group.py @@ -71,25 +71,34 @@ def __getattr__(self, func_name): def method(*args, **kwargs): resps = {} + # filter once for all targets + p = self._proxies[0] + the_proxy, func_itf, adj_args, adj_kwargs = p.adjust_func_args(func_name, args, kwargs) + ctx = the_proxy.app.new_context(self.caller_name, self.name) + + # apply outgoing call filters + adj_kwargs = self._app.apply_outgoing_call_filters(p.target_name, func_name, adj_kwargs, ctx) + the_proxy.check_call_args(func_name, func_itf, adj_args, adj_kwargs) + for p in self._proxies: - the_proxy, call_args, call_kwargs = p.adjust_func_args(func_name, args, kwargs) - kwargs_copy = copy.copy(call_kwargs) + the_proxy, func_itf, call_args, call_kwargs = p.adjust_func_args(func_name, adj_args, adj_kwargs) + call_kwargs = copy.copy(call_kwargs) ctx = self._app.new_context(the_proxy.caller_name, the_proxy.name) - kwargs_copy[CollabMethodArgName.CONTEXT] = ctx + call_kwargs[CollabMethodArgName.CONTEXT] = ctx # set the optional args to help backend decide how to call if self._timeout: - kwargs_copy[CollabMethodOptionName.TIMEOUT] = self._timeout + call_kwargs[CollabMethodOptionName.TIMEOUT] = self._timeout - kwargs_copy[CollabMethodOptionName.BLOCKING] = self._blocking - kwargs_copy[CollabMethodOptionName.SECURE] = self._secure - kwargs_copy[CollabMethodOptionName.OPTIONAL] = self._optional + call_kwargs[CollabMethodOptionName.BLOCKING] = self._blocking + call_kwargs[CollabMethodOptionName.SECURE] = self._secure + call_kwargs[CollabMethodOptionName.OPTIONAL] = self._optional resp = Resp(self._process_resp_cb, self._cb_kwargs, ctx) resps[p.name] = resp - print(f"group call: {func_name=} args={call_args} kwargs={kwargs_copy}") - the_proxy.backend.call_target_with_resp(resp, the_proxy.name, func_name, *call_args, **kwargs_copy) + print(f"group call: {func_name=} args={call_args} kwargs={call_kwargs}") + the_proxy.backend.call_target_with_resp(resp, the_proxy.name, func_name, *call_args, **call_kwargs) # wait for responses if not self._blocking: diff --git a/nvflare/fox/api/proxy.py b/nvflare/fox/api/proxy.py index cd23eb5361..7a87a1e953 100644 --- a/nvflare/fox/api/proxy.py +++ b/nvflare/fox/api/proxy.py @@ -14,7 +14,7 @@ import copy from .backend import Backend -from .constants import CollabMethodArgName +from .constants import OPTION_ARGS, CollabMethodArgName class Proxy: @@ -75,20 +75,38 @@ def adjust_func_args(self, func_name, args, kwargs): call_kwargs = kwargs # find the proxy for the func - p, arg_names = self._find_interface(func_name) + p, func_itf = self._find_interface(func_name) if not p: raise RuntimeError(f"target {self.target_name} does not have method '{func_name}'") - if arg_names: + if func_itf: # check args and turn them to kwargs + num_call_args = len(args) + len(kwargs) + if num_call_args > len(func_itf): + raise RuntimeError( + f"there are {num_call_args} call args ({args=} {kwargs=}), " + f"but function '{func_name}' only supports {len(func_itf)} args ({func_itf})" + ) call_kwargs = copy.copy(kwargs) call_args = [] for i, arg_value in enumerate(args): - call_kwargs[arg_names[i]] = arg_value + call_kwargs[func_itf[i]] = arg_value + + return p, func_itf, call_args, call_kwargs - ctx = self.app.new_context(self.caller_name, self.name) - call_kwargs[CollabMethodArgName.CONTEXT] = ctx - return p, call_args, call_kwargs + @staticmethod + def check_call_args(func_name, func_itf, call_args, call_kwargs: dict): + num_call_args = len(call_args) + len(call_kwargs) + if num_call_args > len(func_itf): + raise RuntimeError( + f"there are {num_call_args} call args ({call_args=} {call_kwargs=}), " + f"but function '{func_name}' only supports {len(func_itf)} args ({func_itf})" + ) + + # make sure every arg in kwargs is valid + for arg_name in call_kwargs.keys(): + if (arg_name not in OPTION_ARGS) and (arg_name not in func_itf): + raise RuntimeError(f"call arg {arg_name} is not supported by func '{func_name}'") def __getattr__(self, func_name): """ @@ -96,14 +114,22 @@ def __getattr__(self, func_name): """ def method(*args, **kwargs): - p, call_args, call_kwargs = self.adjust_func_args(func_name, args, kwargs) + p, func_itf, call_args, call_kwargs = self.adjust_func_args(func_name, args, kwargs) ctx = p.app.new_context(self.caller_name, self.name) - call_kwargs[CollabMethodArgName.CONTEXT] = ctx print(f"calling target {p.target_name} func {func_name}: {call_args=} {call_kwargs=}") - result = p.backend.call_target(self.target_name, func_name, *call_args, **call_kwargs) + + # apply outgoing call filters + call_kwargs = self.app.apply_outgoing_call_filters(p.target_name, func_name, call_kwargs, ctx) + self.check_call_args(func_name, func_itf, call_args, call_kwargs) + + call_kwargs[CollabMethodArgName.CONTEXT] = ctx + result = p.backend.call_target(p.target_name, func_name, *call_args, **call_kwargs) if isinstance(result, Exception): raise result + + # filter incoming result filters + result = self.app.apply_incoming_result_filters(p.target_name, func_name, result, ctx) return result return method diff --git a/nvflare/fox/api/utils.py b/nvflare/fox/api/utils.py index 327da1fe08..189c9f383e 100644 --- a/nvflare/fox/api/utils.py +++ b/nvflare/fox/api/utils.py @@ -29,3 +29,20 @@ def check_optional_args(func, kwargs, arg_names: List[str]): def check_context_support(func, kwargs): check_optional_args(func, kwargs, [CollabMethodArgName.CONTEXT]) + + +def get_collab_object_name(target_name: str): + """The target_name is either the site name or .. + This function gets the collab object name. + + Args: + target_name: + + Returns: + + """ + parts = target_name.split(".") + if len(parts) == 1: + return "app" + else: + return parts[1] diff --git a/nvflare/fox/examples/np/algos/client.py b/nvflare/fox/examples/np/algos/client.py index aea127a775..698681ace1 100644 --- a/nvflare/fox/examples/np/algos/client.py +++ b/nvflare/fox/examples/np/algos/client.py @@ -34,6 +34,7 @@ def train(self, current_round, weights, context: Context): # metric_receiver = self.server.get_target("metric_receiver") # if metric_receiver: # self.server.accept_metric({"round": r, "y": 2}) + # self.server.metric_receiver.accept_metric({"round": r, "y": 2}) # self.server.fire_event("metrics", {"round": current_round, "y": 10}, _blocking=False) return weights + self.delta diff --git a/nvflare/fox/examples/np/algos/filters.py b/nvflare/fox/examples/np/algos/filters.py new file mode 100644 index 0000000000..418561bf90 --- /dev/null +++ b/nvflare/fox/examples/np/algos/filters.py @@ -0,0 +1,39 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import random + +from nvflare.fox.api.constants import ContextKey +from nvflare.fox.api.ctx import Context +from nvflare.fox.api.filter import CallFilter + + +class AddNoiseToModel(CallFilter): + + def filter_call(self, func_kwargs: dict, context: Context): + direction = context.get_prop(ContextKey.DIRECTION) + qual_func_name = context.get_prop(ContextKey.QUALIFIED_FUNC_NAME) + print(f"filtering {func_kwargs} {direction=} {qual_func_name=}") + weights_key = "weights" + weights = func_kwargs.get(weights_key) + if weights is None: + # nothing to filter + print(f"nothing to filter in {func_kwargs}") + return func_kwargs + + # add some noise to weights + noise = random.random() + print(f"adding noise {noise}") + weights += noise + func_kwargs[weights_key] = weights + return func_kwargs diff --git a/nvflare/fox/examples/np/fed_avg_intime.py b/nvflare/fox/examples/np/fed_avg_intime.py index 1adba64a4e..0b8af8447a 100644 --- a/nvflare/fox/examples/np/fed_avg_intime.py +++ b/nvflare/fox/examples/np/fed_avg_intime.py @@ -13,6 +13,7 @@ # limitations under the License. from nvflare.fox.api.app import ServerApp from nvflare.fox.examples.np.algos.client import NPTrainer +from nvflare.fox.examples.np.algos.filters import AddNoiseToModel from nvflare.fox.examples.np.algos.strategies import NPFedAvgInTime from nvflare.fox.examples.np.algos.widgets import MetricReceiver from nvflare.fox.sim.simulator import Simulator @@ -26,8 +27,7 @@ def main(): ) server_app.add_collab_object("metric_receiver", MetricReceiver()) - - server_app.get_collab_interface() + server_app.add_outgoing_call_filters("*.train", [AddNoiseToModel()]) simulator = Simulator( server_app=server_app, diff --git a/nvflare/fox/sim/backend.py b/nvflare/fox/sim/backend.py index 511ff57023..7c1fd209c9 100644 --- a/nvflare/fox/sim/backend.py +++ b/nvflare/fox/sim/backend.py @@ -49,7 +49,7 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): raise AttributeError(f"the method '{func_name}' of {target_name} is not callable") blocking = kwargs.pop(CollabMethodOptionName.BLOCKING, True) - timeout = kwargs.pop(CollabMethodOptionName.TIMEOUT, None) + timeout = kwargs.pop(CollabMethodOptionName.TIMEOUT, 5.0) # these options don't apply to simulation kwargs.pop(CollabMethodOptionName.OPTIONAL, None) diff --git a/nvflare/fox/sys/controller.py b/nvflare/fox/sys/controller.py index 3292bf72a7..338b94875a 100644 --- a/nvflare/fox/sys/controller.py +++ b/nvflare/fox/sys/controller.py @@ -24,6 +24,7 @@ from nvflare.apis.signal import Signal from nvflare.fox.api.app import ServerApp from nvflare.fox.api.constants import ContextKey, EnvType +from nvflare.fox.api.filter import CallFilter, FilterChain from nvflare.fox.api.proxy import Proxy from nvflare.fox.api.strategy import Strategy from nvflare.fuel.f3.cellnet.fqcn import FQCN @@ -50,16 +51,18 @@ def __init__( self, strategy_ids: List[str], server_app_id: str = None, - server_collab_obj_ids: List[str] = None, + collab_obj_ids: List[str] = None, + outgoing_call_filters=None, sync_task_timeout=2, max_call_threads=100, ): Controller.__init__(self) - if not server_collab_obj_ids: - server_collab_obj_ids = [] + if not collab_obj_ids: + collab_obj_ids = [] self.server_app_id = server_app_id # component name + self.outgoing_call_filters = outgoing_call_filters self.strategy_ids = strategy_ids # component names - self.server_collab_obj_ids = server_collab_obj_ids # component IDs + self.server_collab_obj_ids = collab_obj_ids # component IDs self.sync_task_timeout = sync_task_timeout self.server_app = None self.client_info = {} # client name => _ClientInfo @@ -100,6 +103,41 @@ def start_controller(self, fl_ctx: FLContext): app.add_collab_object(cid, obj) + if self.outgoing_call_filters: + if not isinstance(self.outgoing_call_filters, list): + self.system_panic( + f"outgoing_call_filters must be a list but got {type(self.outgoing_call_filters)}", fl_ctx + ) + return + + for chain_dict in self.outgoing_call_filters: + if not isinstance(chain_dict, dict): + self.system_panic( + f"element in outgoing_call_filters must be dict but got {type(chain_dict)}", fl_ctx + ) + return + + pattern = chain_dict.get("pattern") + if not pattern: + self.system_panic("missing 'pattern' in outgoing_call_filters chain", fl_ctx) + return + + filter_ids = chain_dict.get("filters") + if not filter_ids: + self.system_panic("missing 'filters' in outgoing_call_filters chain", fl_ctx) + return + + filters = [] + for fid in filter_ids: + f = engine.get_component(fid) + if not f: + self.system_panic(f"component {fid} does not exist", fl_ctx) + return + if not isinstance(f, CallFilter): + self.system_panic(f"component {fid} should be a CallFilter but got {type(f)}", fl_ctx) + filters.append(f) + app.add_outgoing_call_filters(pattern, filters) + self.server_app = app def _prepare_client_backend(self, job_id, client: ClientSite, abort_signal: Signal): diff --git a/nvflare/fox/sys/executor.py b/nvflare/fox/sys/executor.py index bc8f2f3c2d..66b0d0b25a 100644 --- a/nvflare/fox/sys/executor.py +++ b/nvflare/fox/sys/executor.py @@ -34,10 +34,10 @@ class FoxExecutor(Executor): - def __init__(self, client_app_id: str, client_target_obj_ids: Dict[str, str] = None, max_call_threads=100): + def __init__(self, client_app_id: str, collab_obj_ids: Dict[str, str] = None, max_call_threads=100): Executor.__init__(self) self.client_app_id = client_app_id - self.client_target_obj_ids = client_target_obj_ids + self.collab_obj_ids = collab_obj_ids self.register_event_handler(EventType.START_RUN, self._handle_start_run) self.register_event_handler(EventType.END_RUN, self._handle_end_run) self.client_app = None @@ -62,8 +62,8 @@ def _handle_start_run(self, event_type: str, fl_ctx: FLContext): self.client_app.name = client_name self.client_app.env_type = EnvType.SYSTEM - if self.client_target_obj_ids: - for name, cid in self.client_target_obj_ids: + if self.collab_obj_ids: + for name, cid in self.collab_obj_ids: obj = engine.get_component(cid) if not obj: self.system_panic(f"component {cid} does not exist", fl_ctx) @@ -107,8 +107,8 @@ def _prepare_client_proxy(self, job_id, cell, client: Client, abort_signal, coll app=self.client_app, target_name=client.name, backend=backend, target_interface=collab_interface.get("") ) - if self.client_target_obj_ids: - for name in self.client_target_obj_ids.keys(): + if self.collab_obj_ids: + for name in self.collab_obj_ids.keys(): p = Proxy( app=self.client_app, target_name=f"{client.name}.{name}", From 0671623cceed373b06c64446b6f07868200f29f2 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Fri, 3 Oct 2025 12:58:48 -0400 Subject: [PATCH 025/102] add func itf check --- nvflare/fox/api/app.py | 13 +++++--- nvflare/fox/api/group.py | 3 +- nvflare/fox/api/proxy.py | 28 ++++++++--------- nvflare/fox/api/utils.py | 25 +++++++++++++++ nvflare/fox/sim/backend.py | 61 ++++++++++++++++++++++++------------ nvflare/fox/sim/simulator.py | 4 +-- nvflare/fox/sys/utils.py | 28 ++++++++++++++--- 7 files changed, 116 insertions(+), 46 deletions(-) diff --git a/nvflare/fox/api/app.py b/nvflare/fox/api/app.py index 8fbd43bd60..ac1ab5b058 100644 --- a/nvflare/fox/api/app.py +++ b/nvflare/fox/api/app.py @@ -41,6 +41,7 @@ def __init__(self): self._outgoing_call_filter_chains = [] self._incoming_result_filter_chains = [] self._outgoing_result_filter_chains = [] + self._collab_interface = {"": get_object_collab_interface(self)} @staticmethod def _add_filters(pattern: str, filters, to_list: list, filter_type): @@ -144,6 +145,7 @@ def add_collab_object(self, name: str, obj): setattr(self, name, obj) self._collab_objs[name] = obj + self._collab_interface[name] = get_object_collab_interface(obj) def get_collab_objects(self): return self._collab_objs @@ -224,10 +226,13 @@ def register_event_handler(self, event_type: str, handler, **handler_kwargs): handlers.append((handler, handler_kwargs)) def get_collab_interface(self): - result = {"": get_object_collab_interface(self)} - for name, obj in self._collab_objs.items(): - result[name] = get_object_collab_interface(obj) - return result + return self._collab_interface + + def get_target_object_collab_interface(self, target_name: str): + if not target_name or target_name.lower() == "app": + return self._collab_interface.get("") + else: + return self._collab_interface.get(target_name) @collab def fire_event(self, event_type: str, data, context: Context): diff --git a/nvflare/fox/api/group.py b/nvflare/fox/api/group.py index 4563c1de81..8e6beda516 100644 --- a/nvflare/fox/api/group.py +++ b/nvflare/fox/api/group.py @@ -22,6 +22,7 @@ from .ctx import Context from .proxy import Proxy from .resp import Resp +from .utils import check_call_args class Group: @@ -78,7 +79,7 @@ def method(*args, **kwargs): # apply outgoing call filters adj_kwargs = self._app.apply_outgoing_call_filters(p.target_name, func_name, adj_kwargs, ctx) - the_proxy.check_call_args(func_name, func_itf, adj_args, adj_kwargs) + check_call_args(func_name, func_itf, adj_args, adj_kwargs) for p in self._proxies: the_proxy, func_itf, call_args, call_kwargs = p.adjust_func_args(func_name, adj_args, adj_kwargs) diff --git a/nvflare/fox/api/proxy.py b/nvflare/fox/api/proxy.py index 7a87a1e953..ab1e800530 100644 --- a/nvflare/fox/api/proxy.py +++ b/nvflare/fox/api/proxy.py @@ -15,6 +15,7 @@ from .backend import Backend from .constants import OPTION_ARGS, CollabMethodArgName +from .utils import check_call_args class Proxy: @@ -94,26 +95,18 @@ def adjust_func_args(self, func_name, args, kwargs): return p, func_itf, call_args, call_kwargs - @staticmethod - def check_call_args(func_name, func_itf, call_args, call_kwargs: dict): - num_call_args = len(call_args) + len(call_kwargs) - if num_call_args > len(func_itf): - raise RuntimeError( - f"there are {num_call_args} call args ({call_args=} {call_kwargs=}), " - f"but function '{func_name}' only supports {len(func_itf)} args ({func_itf})" - ) - - # make sure every arg in kwargs is valid - for arg_name in call_kwargs.keys(): - if (arg_name not in OPTION_ARGS) and (arg_name not in func_itf): - raise RuntimeError(f"call arg {arg_name} is not supported by func '{func_name}'") - def __getattr__(self, func_name): """ This method is called when Python cannot find an invoked method func_name of this class. """ def method(*args, **kwargs): + # remove option args + option_args = {} + for k in OPTION_ARGS: + if k in kwargs: + option_args[k] = kwargs.pop(k) + p, func_itf, call_args, call_kwargs = self.adjust_func_args(func_name, args, kwargs) ctx = p.app.new_context(self.caller_name, self.name) @@ -121,9 +114,14 @@ def method(*args, **kwargs): # apply outgoing call filters call_kwargs = self.app.apply_outgoing_call_filters(p.target_name, func_name, call_kwargs, ctx) - self.check_call_args(func_name, func_itf, call_args, call_kwargs) + check_call_args(func_name, func_itf, call_args, call_kwargs) call_kwargs[CollabMethodArgName.CONTEXT] = ctx + + # restore option args + for k, v in option_args.items(): + call_kwargs[k] = v + result = p.backend.call_target(p.target_name, func_name, *call_args, **call_kwargs) if isinstance(result, Exception): raise result diff --git a/nvflare/fox/api/utils.py b/nvflare/fox/api/utils.py index 189c9f383e..40ac5988c7 100644 --- a/nvflare/fox/api/utils.py +++ b/nvflare/fox/api/utils.py @@ -46,3 +46,28 @@ def get_collab_object_name(target_name: str): return "app" else: return parts[1] + + +def check_call_args(func_name, func_itf, call_args, call_kwargs: dict): + """Check call args against the function's interface. + + Args: + func_name: + func_itf: + call_args: + call_kwargs: + + Returns: + + """ + num_call_args = len(call_args) + len(call_kwargs) + if num_call_args > len(func_itf): + raise RuntimeError( + f"there are {num_call_args} call args ({call_args=} {call_kwargs=}), " + f"but function '{func_name}' only supports {len(func_itf)} args ({func_itf})" + ) + + # make sure every arg in kwargs is valid + for arg_name in call_kwargs.keys(): + if arg_name not in func_itf: + raise RuntimeError(f"call arg {arg_name} is not supported by func '{func_name}'") diff --git a/nvflare/fox/sim/backend.py b/nvflare/fox/sim/backend.py index 7c1fd209c9..8ef051888a 100644 --- a/nvflare/fox/sim/backend.py +++ b/nvflare/fox/sim/backend.py @@ -17,9 +17,10 @@ from nvflare.apis.fl_exception import RunAborted from nvflare.fox.api.app import App from nvflare.fox.api.backend import Backend -from nvflare.fox.api.constants import CollabMethodArgName, CollabMethodOptionName +from nvflare.fox.api.constants import OPTION_ARGS, CollabMethodArgName, CollabMethodOptionName from nvflare.fox.api.dec import adjust_kwargs from nvflare.fox.api.resp import Resp +from nvflare.fox.api.utils import check_call_args class _Waiter(threading.Event): @@ -31,8 +32,9 @@ def __init__(self): class SimBackend(Backend): - def __init__(self, target_app: App, target_obj, abort_signal, thread_executor): + def __init__(self, target_obj_name: str, target_app: App, target_obj, abort_signal, thread_executor): Backend.__init__(self, abort_signal) + self.target_obj_name = target_obj_name self.target_app = target_app self.target_obj = target_obj self.executor = thread_executor @@ -51,15 +53,15 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): blocking = kwargs.pop(CollabMethodOptionName.BLOCKING, True) timeout = kwargs.pop(CollabMethodOptionName.TIMEOUT, 5.0) - # these options don't apply to simulation - kwargs.pop(CollabMethodOptionName.OPTIONAL, None) - kwargs.pop(CollabMethodOptionName.SECURE, None) + # other options don't apply to simulation + for k in OPTION_ARGS: + kwargs.pop(k, None) waiter = None if blocking: waiter = _Waiter() - self.executor.submit(self._run_func, waiter, func, args, kwargs) + self.executor.submit(self._run_func, waiter, target_name, func_name, func, args, kwargs) if waiter: start_time = time.time() while True: @@ -78,17 +80,35 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): return waiter.result - def _augment_context(self, func, kwargs): - ctx = kwargs.get(CollabMethodArgName.CONTEXT) - if ctx: - target_ctx = self.target_app.new_context(ctx.caller, ctx.callee) - kwargs[CollabMethodArgName.CONTEXT] = target_ctx + def _preprocess(self, target_name, func_name, func, kwargs): + caller_ctx = kwargs.pop(CollabMethodArgName.CONTEXT) + my_ctx = self.target_app.new_context(caller_ctx.caller, caller_ctx.caller) + kwargs = self.target_app.apply_incoming_call_filters(target_name, func_name, kwargs, my_ctx) + + # make sure the final kwargs conforms to func interface + obj_itf = self.target_app.get_target_object_collab_interface(self.target_obj_name) + if not obj_itf: + raise RuntimeError(f"cannot find collab interface for object {self.target_obj_name}") + + func_itf = obj_itf.get(func_name) + if not func_itf: + raise RuntimeError(f"cannot find interface for func '{func_name}' of object {self.target_obj_name}") + + check_call_args(func_name, func_itf, [], kwargs) + print(f"received kwargs is good: {kwargs}") + + kwargs[CollabMethodArgName.CONTEXT] = my_ctx adjust_kwargs(func, kwargs) + return my_ctx, kwargs - def _run_func(self, waiter: _Waiter, func, args, kwargs): + def _run_func(self, waiter: _Waiter, target_name, func_name, func, args, kwargs): try: - self._augment_context(func, kwargs) + ctx, kwargs = self._preprocess(target_name, func_name, func, kwargs) result = func(*args, **kwargs) + + # apply result filter + result = self.target_app.apply_outgoing_result_filters(target_name, func_name, result, ctx) + if waiter: waiter.result = result except Exception as ex: @@ -100,10 +120,8 @@ def _run_func(self, waiter: _Waiter, func, args, kwargs): def call_target_with_resp(self, resp: Resp, target_name: str, func_name: str, *args, **kwargs): # do not use the optional args - they are managed by the group - kwargs.pop(CollabMethodOptionName.BLOCKING, None) - kwargs.pop(CollabMethodOptionName.TIMEOUT, None) - kwargs.pop(CollabMethodOptionName.OPTIONAL, None) - kwargs.pop(CollabMethodOptionName.SECURE, None) + for k in OPTION_ARGS: + kwargs.pop(k, None) func = self._get_func(func_name) if not func: @@ -112,12 +130,15 @@ def call_target_with_resp(self, resp: Resp, target_name: str, func_name: str, *a if not callable(func): raise AttributeError(f"the method '{func_name}' of {target_name} is not callable") - self.executor.submit(self._run_func_with_resp, resp, func, args, kwargs) + self.executor.submit(self._run_func_with_resp, resp, target_name, func_name, func, args, kwargs) - def _run_func_with_resp(self, resp: Resp, func, args, kwargs): + def _run_func_with_resp(self, resp: Resp, target_name, func_name, func, args, kwargs): try: - self._augment_context(func, kwargs) + ctx, kwargs = self._preprocess(target_name, func_name, func, kwargs) result = func(*args, **kwargs) + + # apply result filter + result = self.target_app.apply_outgoing_result_filters(target_name, func_name, result, ctx) resp.set_result(result) except Exception as ex: resp.set_exception(ex) diff --git a/nvflare/fox/sim/simulator.py b/nvflare/fox/sim/simulator.py index a5288d1e32..1309c8b4c5 100644 --- a/nvflare/fox/sim/simulator.py +++ b/nvflare/fox/sim/simulator.py @@ -26,10 +26,10 @@ class Simulator: def _prepare_app_backends(self, app: App): - bes = {"": SimBackend(app, app, self.abort_signal, self.thread_executor)} + bes = {"": SimBackend("", app, app, self.abort_signal, self.thread_executor)} targets = app.get_collab_objects() for name, obj in targets.items(): - bes[name] = SimBackend(app, obj, self.abort_signal, self.thread_executor) + bes[name] = SimBackend(name, app, obj, self.abort_signal, self.thread_executor) return bes def _prepare_proxy(self, for_app: App, target_app: App, backends: dict): diff --git a/nvflare/fox/sys/utils.py b/nvflare/fox/sys/utils.py index 00425e9e7f..26ef735055 100644 --- a/nvflare/fox/sys/utils.py +++ b/nvflare/fox/sys/utils.py @@ -14,6 +14,7 @@ from nvflare.fox.api.app import App from nvflare.fox.api.constants import CollabMethodArgName from nvflare.fox.api.dec import adjust_kwargs +from nvflare.fox.api.utils import check_call_args from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey, ReturnCode from nvflare.fuel.f3.cellnet.utils import new_cell_message from nvflare.fuel.f3.message import Message @@ -34,6 +35,27 @@ def _error_reply(error: str, logger) -> Message: ) +def _preprocess(app: App, caller, target_obj_name, target_name, func_name, func, args, kwargs): + ctx = app.new_context(caller=caller, callee=app.name) + kwargs = app.apply_incoming_call_filters(target_name, func_name, kwargs, ctx) + + # make sure the final kwargs conforms to func interface + obj_itf = app.get_target_object_collab_interface(target_obj_name) + if not obj_itf: + raise RuntimeError(f"cannot find collab interface for object {target_obj_name}") + + func_itf = obj_itf.get(func_name) + if not func_itf: + raise RuntimeError(f"cannot find interface for func '{func_name}' of object {target_obj_name}") + + check_call_args(func_name, func_itf, args, kwargs) + print(f"received kwargs is good: {kwargs}") + + kwargs[CollabMethodArgName.CONTEXT] = ctx + adjust_kwargs(func, kwargs) + return ctx, kwargs + + def _call_app_method(request: Message, app: App, logger) -> Message: logger.info("got a remote call") payload = request.payload @@ -67,7 +89,7 @@ def _call_app_method(request: Message, app: App, logger) -> Message: return _error_reply(f"bad method kwargs: should be dict but got {type(method_kwargs)}", logger) parts = target_name.split(".") - obj_name = None + obj_name = "" if len(parts) >= 2: obj_name = parts[1] if obj_name: @@ -89,9 +111,7 @@ def _call_app_method(request: Message, app: App, logger) -> Message: # invoke this method try: - ctx = app.new_context(caller=caller, callee=app.name) - method_kwargs[CollabMethodArgName.CONTEXT] = ctx - adjust_kwargs(m, method_kwargs) + ctx, method_kwargs = _preprocess(app, caller, obj_name, target_name, method_name, m, method_args, method_kwargs) logger.info(f"calling method {method_name}: {caller=}: {method_args=} {method_kwargs=}") result = m(*method_args, **method_kwargs) From 68d55bab6b08e0109229adddcb839ab9e847db54 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Fri, 3 Oct 2025 14:19:20 -0400 Subject: [PATCH 026/102] added outgoing result filtering --- nvflare/fox/sys/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nvflare/fox/sys/utils.py b/nvflare/fox/sys/utils.py index 26ef735055..62a2000695 100644 --- a/nvflare/fox/sys/utils.py +++ b/nvflare/fox/sys/utils.py @@ -112,10 +112,14 @@ def _call_app_method(request: Message, app: App, logger) -> Message: # invoke this method try: ctx, method_kwargs = _preprocess(app, caller, obj_name, target_name, method_name, m, method_args, method_kwargs) - logger.info(f"calling method {method_name}: {caller=}: {method_args=} {method_kwargs=}") result = m(*method_args, **method_kwargs) logger.info(f"result from method {method_name}: {result}") + + # apply result filters + result = app.apply_outgoing_result_filters(target_name, method_name, result, ctx) + logger.info(f"result after filtering: {result}") + return new_cell_message( headers={MessageHeaderKey.RETURN_CODE: ReturnCode.OK}, payload={CallReplyKey.RESULT: result} ) From 1453c0700f91ba16ab5b3a3faecc2832c799236e Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Fri, 3 Oct 2025 15:53:30 -0400 Subject: [PATCH 027/102] fix context --- nvflare/fox/api/ctx.py | 12 ++++++++++ nvflare/fox/api/group.py | 16 ++++++++----- nvflare/fox/api/proxy.py | 2 +- nvflare/fox/api/resp.py | 21 +++++++++++------ nvflare/fox/examples/np/algos/client.py | 4 ++-- nvflare/fox/examples/np/algos/filters.py | 24 ++++++++++++++++--- nvflare/fox/examples/np/algos/strategies.py | 26 ++++++++++----------- nvflare/fox/examples/np/algos/swarm.py | 12 +++++----- nvflare/fox/examples/np/fed_avg_intime.py | 9 +++++-- nvflare/fox/sim/backend.py | 4 ++-- 10 files changed, 88 insertions(+), 42 deletions(-) diff --git a/nvflare/fox/api/ctx.py b/nvflare/fox/api/ctx.py index c45c62cdb7..76e03231cd 100644 --- a/nvflare/fox/api/ctx.py +++ b/nvflare/fox/api/ctx.py @@ -17,6 +17,15 @@ class Context: def __init__(self, env_type: str, caller: str, callee: str, abort_signal: Signal, props: dict = None): + if not isinstance(env_type, str): + raise ValueError(f"env_type must be str but got {type(env_type)}") + + if not isinstance(caller, str): + raise ValueError(f"caller must be str but got {type(caller)}") + + if not isinstance(callee, str): + raise ValueError(f"callee must be str but got {type(callee)}") + self.env_type = env_type self.caller = caller self.callee = callee @@ -36,3 +45,6 @@ def get_prop(self, name: str, default=None): def is_aborted(self): return self.abort_signal and self.abort_signal.triggered + + def header_str(self): + return f"{self.app.name}:{self.caller}=>{self.callee}" diff --git a/nvflare/fox/api/group.py b/nvflare/fox/api/group.py index 8e6beda516..19aedb22f2 100644 --- a/nvflare/fox/api/group.py +++ b/nvflare/fox/api/group.py @@ -62,8 +62,6 @@ def __init__( if not wait_after_min_resps: self._wait_after_min_resps = 0 - print(f"min resps: {self._min_resps}") - def __getattr__(self, func_name): """ This method is called when Python cannot find an invoked method func_name of this class. @@ -75,7 +73,7 @@ def method(*args, **kwargs): # filter once for all targets p = self._proxies[0] the_proxy, func_itf, adj_args, adj_kwargs = p.adjust_func_args(func_name, args, kwargs) - ctx = the_proxy.app.new_context(self.caller_name, self.name) + ctx = the_proxy.app.new_context(the_proxy.app.name, the_proxy.name) # apply outgoing call filters adj_kwargs = self._app.apply_outgoing_call_filters(p.target_name, func_name, adj_kwargs, ctx) @@ -95,10 +93,17 @@ def method(*args, **kwargs): call_kwargs[CollabMethodOptionName.SECURE] = self._secure call_kwargs[CollabMethodOptionName.OPTIONAL] = self._optional - resp = Resp(self._process_resp_cb, self._cb_kwargs, ctx) + resp = Resp( + self._app, + p.target_name, + func_name, + self._process_resp_cb, + self._cb_kwargs, + ctx, + ) resps[p.name] = resp - print(f"group call: {func_name=} args={call_args} kwargs={call_kwargs}") + print(f"[{ctx.header_str()}] group call: {func_name=} args={call_args} kwargs={call_kwargs}") the_proxy.backend.call_target_with_resp(resp, the_proxy.name, func_name, *call_args, **call_kwargs) # wait for responses @@ -121,7 +126,6 @@ def method(*args, **kwargs): break now = time.time() - print(f"what is min_resps: {self=} {self._min_resps}") if resps_received >= self._min_resps: if not min_received_time: min_received_time = now diff --git a/nvflare/fox/api/proxy.py b/nvflare/fox/api/proxy.py index ab1e800530..53ad562f61 100644 --- a/nvflare/fox/api/proxy.py +++ b/nvflare/fox/api/proxy.py @@ -110,7 +110,7 @@ def method(*args, **kwargs): p, func_itf, call_args, call_kwargs = self.adjust_func_args(func_name, args, kwargs) ctx = p.app.new_context(self.caller_name, self.name) - print(f"calling target {p.target_name} func {func_name}: {call_args=} {call_kwargs=}") + print(f"[{ctx.header_str()}] calling target {p.target_name} func {func_name}: {call_args=} {call_kwargs=}") # apply outgoing call filters call_kwargs = self.app.apply_outgoing_call_filters(p.target_name, func_name, call_kwargs, ctx) diff --git a/nvflare/fox/api/resp.py b/nvflare/fox/api/resp.py index 89b464bd09..9dfeef0709 100644 --- a/nvflare/fox/api/resp.py +++ b/nvflare/fox/api/resp.py @@ -21,7 +21,10 @@ class Resp: - def __init__(self, process_cb, cb_kwargs, context: Context): + def __init__(self, app, target_name, func_name, process_cb, cb_kwargs, context: Context): + self.app = app + self.target_name = target_name + self.func_name = func_name self.result = None self.resp_time = None self.process_cb = process_cb @@ -29,13 +32,17 @@ def __init__(self, process_cb, cb_kwargs, context: Context): self.context = context def set_result(self, result): - if self.process_cb: - ctx = copy.copy(self.context) + # filter incoming result + ctx = copy.copy(self.context) + # swap caller/callee + original_caller = ctx.caller + ctx.caller = ctx.callee + ctx.callee = original_caller + + if not isinstance(result, Exception): + result = self.app.apply_incoming_result_filters(self.target_name, self.func_name, result, ctx) - # swap caller/callee - original_caller = ctx.caller - ctx.caller = ctx.callee - ctx.callee = original_caller + if self.process_cb: self.cb_kwargs[CollabMethodArgName.CONTEXT] = ctx check_context_support(self.process_cb, self.cb_kwargs) result = self.process_cb(result, **self.cb_kwargs) diff --git a/nvflare/fox/examples/np/algos/client.py b/nvflare/fox/examples/np/algos/client.py index 698681ace1..cbbf7d9377 100644 --- a/nvflare/fox/examples/np/algos/client.py +++ b/nvflare/fox/examples/np/algos/client.py @@ -29,7 +29,7 @@ def train(self, current_round, weights, context: Context): if context.is_aborted(): print("training aborted") return 0 - print(f"[{self.name}] called by {context.caller}: client {context.callee} trained round {current_round}") + print(f"[{context.header_str()}] trained round {current_round}") # metric_receiver = self.server.get_target("metric_receiver") # if metric_receiver: @@ -41,7 +41,7 @@ def train(self, current_round, weights, context: Context): @collab def evaluate(self, model, context: Context): - print(f"[{self.name}] called by {context.caller}: client {context.callee} to evaluate") + print(f"[{context.header_str()}] evaluate") return random.random() diff --git a/nvflare/fox/examples/np/algos/filters.py b/nvflare/fox/examples/np/algos/filters.py index 418561bf90..cff2ebcf4f 100644 --- a/nvflare/fox/examples/np/algos/filters.py +++ b/nvflare/fox/examples/np/algos/filters.py @@ -15,7 +15,7 @@ from nvflare.fox.api.constants import ContextKey from nvflare.fox.api.ctx import Context -from nvflare.fox.api.filter import CallFilter +from nvflare.fox.api.filter import CallFilter, ResultFilter class AddNoiseToModel(CallFilter): @@ -23,7 +23,7 @@ class AddNoiseToModel(CallFilter): def filter_call(self, func_kwargs: dict, context: Context): direction = context.get_prop(ContextKey.DIRECTION) qual_func_name = context.get_prop(ContextKey.QUALIFIED_FUNC_NAME) - print(f"filtering {func_kwargs} {direction=} {qual_func_name=}") + print(f"[{context.header_str()}] filtering call: {func_kwargs=} {direction=} {qual_func_name=}") weights_key = "weights" weights = func_kwargs.get(weights_key) if weights is None: @@ -33,7 +33,25 @@ def filter_call(self, func_kwargs: dict, context: Context): # add some noise to weights noise = random.random() - print(f"adding noise {noise}") + print(f"[{context.header_str()}] adding noise {noise}") weights += noise func_kwargs[weights_key] = weights return func_kwargs + + +class PrintCall(CallFilter): + + def filter_call(self, func_kwargs: dict, context: Context): + direction = context.get_prop(ContextKey.DIRECTION) + qual_func_name = context.get_prop(ContextKey.QUALIFIED_FUNC_NAME) + print(f"[{context.header_str()}] printing call: {func_kwargs=} {direction=} {qual_func_name=}") + return func_kwargs + + +class PrintResult(ResultFilter): + + def filter_result(self, result, context: Context): + direction = context.get_prop(ContextKey.DIRECTION) + qual_func_name = context.get_prop(ContextKey.QUALIFIED_FUNC_NAME) + print(f"[{context.header_str()}] printing result: {result=} {direction=} {qual_func_name=}") + return result diff --git a/nvflare/fox/examples/np/algos/strategies.py b/nvflare/fox/examples/np/algos/strategies.py index bea1767714..7e2fac9d9a 100644 --- a/nvflare/fox/examples/np/algos/strategies.py +++ b/nvflare/fox/examples/np/algos/strategies.py @@ -32,7 +32,7 @@ def __init__(self, initial_model, num_rounds=10): self.initial_model = parse_array_def(initial_model) def execute(self, context: Context): - print(f"[{self.name}] Start training for {self.num_rounds} rounds") + print(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") current_model = context.get_prop(ContextKey.INPUT, self.initial_model) for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model, context) @@ -43,7 +43,7 @@ def _do_one_round(self, r, current_model, context: Context): n = 0 for c in context.clients: result = c.train(r, current_model) - print(f"[{self.name}] round {r}: got result from client {c.name}: {result}") + print(f"[{context.header_str()}] round {r}: got result from client {c.name}: {result}") total += result n += 1 return total / n @@ -57,19 +57,19 @@ def __init__(self, initial_model, num_rounds=10): self.name = "NPFedAvgParallel" def execute(self, context: Context): - print(f"[{self.name}] Start training for {self.num_rounds} rounds") + print(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") current_model = context.get_prop(ContextKey.INPUT, self.initial_model) for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model, context) score = self._do_eval(current_model, context) - print(f"[{self.name}]: eval score in round {i}: {score}") + print(f"[{context.header_str()}]: eval score in round {i}: {score}") return current_model def _do_eval(self, model, ctx: Context): results = all_clients(ctx).evaluate(model) total = 0.0 for n, v in results.items(): - print(f"[{self.name}]: got eval result from client {n}: {v}") + print(f"[{ctx.header_str()}]: got eval result from client {n}: {v}") total += v return total / len(results) @@ -77,7 +77,7 @@ def _do_one_round(self, r, current_model, ctx: Context): total = np.array([[0, 0, 0], [0, 0, 0], [0, 0, 0]], dtype=np.float32) results = all_clients(ctx).train(r, current_model) for n, v in results.items(): - print(f"[{self.name}] round {r}: got group result from client {n}: {v}") + print(f"[{ctx.header_str()}] round {r}: got group result from client {n}: {v}") total += v return total / len(results) @@ -98,19 +98,19 @@ def __init__(self, initial_model, num_rounds=10, timeout=2.0): self.name = "NPFedAvgInTime" def execute(self, context: Context): - print(f"[{self.name}] Start training for {self.num_rounds} rounds") + print(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") current_model = context.get_prop(ContextKey.INPUT, self.initial_model) for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model, context) score = self._do_eval(current_model, context) - print(f"[{self.name}]: eval score in round {i}: {score}") + print(f"[{context.header_str()}]: eval score in round {i}: {score}") return current_model def _do_eval(self, model, ctx: Context): results = all_clients(ctx).evaluate(model) total = 0.0 for n, v in results.items(): - print(f"[{self.name}]: got eval result from client {n}: {v}") + print(f"[{ctx.header_str()}]: got eval result from client {n}: {v}") total += v return total / len(results) @@ -122,14 +122,14 @@ def _do_one_round(self, r, current_model, ctx: Context): aggr_result=aggr_result, ).train(r, current_model) - print(f"[{self.name}] round {r}: aggr result from {aggr_result.count} clients: {aggr_result.total}") + print(f"[{ctx.header_str()}] round {r}: aggr result from {aggr_result.count} clients: {aggr_result.total}") if aggr_result.count == 0: return None else: return aggr_result.total / aggr_result.count def _accept_train_result(self, result, aggr_result: _AggrResult, context: Context): - print(f"[{context.callee}] got train result from {context.caller} {result}") + print(f"[{context.header_str()}] got train result from {context.caller} {result}") aggr_result.total += result aggr_result.count += 1 return None @@ -145,12 +145,12 @@ def execute(self, context: Context): current_model = context.get_prop(ContextKey.INPUT, self.initial_model) for current_round in range(self.num_rounds): current_model = self._do_one_round(current_round, current_model, context) - print(f"final result: {current_model}") + print(f"[{context.header_str()}] final result: {current_model}") return current_model def _do_one_round(self, current_round, current_model, ctx: Context): random.shuffle(ctx.clients) for c in ctx.clients: current_model = c.train(current_round, current_model) - print(f"result from {c.name}: {current_model}") + print(f"[{ctx.header_str()}] result from {c.name}: {current_model}") return current_model diff --git a/nvflare/fox/examples/np/algos/swarm.py b/nvflare/fox/examples/np/algos/swarm.py index 60d6a996b5..079db90cfd 100644 --- a/nvflare/fox/examples/np/algos/swarm.py +++ b/nvflare/fox/examples/np/algos/swarm.py @@ -42,12 +42,12 @@ def execute(self, context: Context): break def _all_done(self, event_type: str, data, context: Context): - print(f"[{context.callee}]: received {event_type} from client: {context.caller}: {data}") + print(f"[{context.header_str()}]: received {event_type} from client: {context.caller}: {data}") self.all_done(data, context) @collab def all_done(self, reason: str, context: Context): - print(f"[{context.callee}]: all done from client: {context.caller}: {reason}") + print(f"[{context.header_str()}]: all done from client: {context.caller}: {reason}") self.waiter.set() @@ -60,7 +60,7 @@ def __init__(self, delta: float): @collab def train(self, weights, current_round, context: Context): - print(f"[{context.callee}]: train asked by {context.caller}: {current_round=}") + print(f"[{context.header_str()}]: train asked by {context.caller}: {current_round=}") return weights + self.delta def sag(self, model, current_round, ctx: Context): @@ -73,10 +73,10 @@ def sag(self, model, current_round, ctx: Context): @collab def swarm_learn(self, num_rounds, model, current_round, context: Context): - print(f"[{context.callee}]: swarm learn asked by {context.caller}: {num_rounds=} {current_round=} {model=}") + print(f"[{context.header_str()}]: swarm learn asked: {num_rounds=} {current_round=} {model=}") new_model = self.sag(model, current_round, context) - print(f"[{context.callee}]: trained model {new_model=}") + print(f"[{context.header_str()}]: trained model {new_model=}") if current_round == num_rounds - 1: # all done all_clients(context, blocking=False).fire_event("final_model", new_model) @@ -102,4 +102,4 @@ def start(self, num_rounds, initial_model, context: Context): def _accept_final_model(self, event_type: str, model, context: Context): # accept the final model # write model to disk - print(f"[{context.callee}]: received event '{event_type}' from {context.caller}: {model}") + print(f"[{context.header_str()}]: received event '{event_type}' from {context.caller}: {model}") diff --git a/nvflare/fox/examples/np/fed_avg_intime.py b/nvflare/fox/examples/np/fed_avg_intime.py index 0b8af8447a..f8e2c11355 100644 --- a/nvflare/fox/examples/np/fed_avg_intime.py +++ b/nvflare/fox/examples/np/fed_avg_intime.py @@ -13,7 +13,7 @@ # limitations under the License. from nvflare.fox.api.app import ServerApp from nvflare.fox.examples.np.algos.client import NPTrainer -from nvflare.fox.examples.np.algos.filters import AddNoiseToModel +from nvflare.fox.examples.np.algos.filters import AddNoiseToModel, PrintCall, PrintResult from nvflare.fox.examples.np.algos.strategies import NPFedAvgInTime from nvflare.fox.examples.np.algos.widgets import MetricReceiver from nvflare.fox.sim.simulator import Simulator @@ -28,10 +28,15 @@ def main(): server_app.add_collab_object("metric_receiver", MetricReceiver()) server_app.add_outgoing_call_filters("*.train", [AddNoiseToModel()]) + server_app.add_incoming_result_filters("*.train", [PrintResult()]) + + client_app = NPTrainer(delta=1.0) + client_app.add_incoming_call_filters("*.train", [PrintCall()]) + client_app.add_outgoing_result_filters("*.train", [PrintResult()]) simulator = Simulator( server_app=server_app, - client_app=NPTrainer(delta=1.0), + client_app=client_app, num_clients=2, ) diff --git a/nvflare/fox/sim/backend.py b/nvflare/fox/sim/backend.py index 8ef051888a..4a78b8b738 100644 --- a/nvflare/fox/sim/backend.py +++ b/nvflare/fox/sim/backend.py @@ -82,7 +82,7 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): def _preprocess(self, target_name, func_name, func, kwargs): caller_ctx = kwargs.pop(CollabMethodArgName.CONTEXT) - my_ctx = self.target_app.new_context(caller_ctx.caller, caller_ctx.caller) + my_ctx = self.target_app.new_context(caller_ctx.caller, caller_ctx.callee) kwargs = self.target_app.apply_incoming_call_filters(target_name, func_name, kwargs, my_ctx) # make sure the final kwargs conforms to func interface @@ -95,7 +95,7 @@ def _preprocess(self, target_name, func_name, func, kwargs): raise RuntimeError(f"cannot find interface for func '{func_name}' of object {self.target_obj_name}") check_call_args(func_name, func_itf, [], kwargs) - print(f"received kwargs is good: {kwargs}") + print(f"[{my_ctx.header_str()}] received kwargs is good: {kwargs}") kwargs[CollabMethodArgName.CONTEXT] = my_ctx adjust_kwargs(func, kwargs) From 8f64e093857d123b8e7416fb3a5f016b6c770322 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Mon, 6 Oct 2025 11:45:24 -0400 Subject: [PATCH 028/102] use logger --- nvflare/fox/api/app.py | 5 +++- nvflare/fox/api/group.py | 6 +++- nvflare/fox/api/proxy.py | 7 ++++- nvflare/fox/api/utils.py | 5 ++++ nvflare/fox/examples/np/algos/client.py | 6 ++-- nvflare/fox/examples/np/algos/filters.py | 18 ++++++++--- nvflare/fox/examples/np/algos/strategies.py | 33 +++++++++++++-------- nvflare/fox/examples/np/algos/swarm.py | 18 ++++++----- nvflare/fox/examples/np/algos/widgets.py | 10 +++++-- nvflare/fox/examples/np/cyclic.py | 4 +++ nvflare/fox/examples/np/cyclic_avg.py | 5 ++++ nvflare/fox/examples/np/fed_avg_intime.py | 4 +++ nvflare/fox/examples/np/fed_avg_para.py | 4 +++ nvflare/fox/examples/np/fed_avg_seq.py | 4 +++ nvflare/fox/examples/np/swarm.py | 3 ++ nvflare/fox/sim/backend.py | 4 ++- nvflare/fox/sim/simulator.py | 14 +++++---- 17 files changed, 109 insertions(+), 41 deletions(-) diff --git a/nvflare/fox/api/app.py b/nvflare/fox/api/app.py index ac1ab5b058..fe8777154e 100644 --- a/nvflare/fox/api/app.py +++ b/nvflare/fox/api/app.py @@ -16,6 +16,8 @@ from abc import ABC, abstractmethod from typing import List +from nvflare.fuel.utils.log_utils import get_obj_logger + from .constants import CollabMethodArgName, ContextKey, FilterDirection from .ctx import Context from .dec import collab, get_object_collab_interface, is_collab @@ -42,6 +44,7 @@ def __init__(self): self._incoming_result_filter_chains = [] self._outgoing_result_filter_chains = [] self._collab_interface = {"": get_object_collab_interface(self)} + self.logger = get_obj_logger(self) @staticmethod def _add_filters(pattern: str, filters, to_list: list, filter_type): @@ -206,7 +209,7 @@ def initialize(self, context: Context): for name, obj in self._collab_objs.items(): init_func = getattr(obj, "initialize", None) if init_func and callable(init_func): - print(f"initializing target object {name}") + self.logger.info(f"initializing target object {name}") kwargs = {CollabMethodArgName.CONTEXT: context} check_context_support(init_func, kwargs) init_func(**kwargs) diff --git a/nvflare/fox/api/group.py b/nvflare/fox/api/group.py index 19aedb22f2..1ffebcbbd2 100644 --- a/nvflare/fox/api/group.py +++ b/nvflare/fox/api/group.py @@ -17,6 +17,7 @@ from nvflare.apis.fl_exception import RunAborted from nvflare.apis.signal import Signal +from nvflare.fuel.utils.log_utils import get_obj_logger from .constants import CollabMethodArgName, CollabMethodOptionName from .ctx import Context @@ -55,6 +56,7 @@ def __init__( self._wait_after_min_resps = wait_after_min_resps self._process_resp_cb = process_resp_cb self._cb_kwargs = cb_kwargs + self._logger = get_obj_logger(self) if not min_resps: self._min_resps = len(proxies) @@ -103,7 +105,9 @@ def method(*args, **kwargs): ) resps[p.name] = resp - print(f"[{ctx.header_str()}] group call: {func_name=} args={call_args} kwargs={call_kwargs}") + self._logger.debug( + f"[{ctx.header_str()}] group call: {func_name=} args={call_args} kwargs={call_kwargs}" + ) the_proxy.backend.call_target_with_resp(resp, the_proxy.name, func_name, *call_args, **call_kwargs) # wait for responses diff --git a/nvflare/fox/api/proxy.py b/nvflare/fox/api/proxy.py index 53ad562f61..d32339f84c 100644 --- a/nvflare/fox/api/proxy.py +++ b/nvflare/fox/api/proxy.py @@ -13,6 +13,8 @@ # limitations under the License. import copy +from nvflare.fuel.utils.log_utils import get_obj_logger + from .backend import Backend from .constants import OPTION_ARGS, CollabMethodArgName from .utils import check_call_args @@ -28,6 +30,7 @@ def __init__(self, app, target_name, backend: Backend, target_interface): self.caller_name = app.name self.target_interface = target_interface self.children = {} # child proxies + self.logger = get_obj_logger(self) @property def name(self): @@ -110,7 +113,9 @@ def method(*args, **kwargs): p, func_itf, call_args, call_kwargs = self.adjust_func_args(func_name, args, kwargs) ctx = p.app.new_context(self.caller_name, self.name) - print(f"[{ctx.header_str()}] calling target {p.target_name} func {func_name}: {call_args=} {call_kwargs=}") + self.logger.debug( + f"[{ctx.header_str()}] calling target {p.target_name} func {func_name}: {call_args=} {call_kwargs=}" + ) # apply outgoing call filters call_kwargs = self.app.apply_outgoing_call_filters(p.target_name, func_name, call_kwargs, ctx) diff --git a/nvflare/fox/api/utils.py b/nvflare/fox/api/utils.py index 40ac5988c7..cef7613bf8 100644 --- a/nvflare/fox/api/utils.py +++ b/nvflare/fox/api/utils.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import inspect +import logging from typing import List from .constants import CollabMethodArgName @@ -71,3 +72,7 @@ def check_call_args(func_name, func_itf, call_args, call_kwargs: dict): for arg_name in call_kwargs.keys(): if arg_name not in func_itf: raise RuntimeError(f"call arg {arg_name} is not supported by func '{func_name}'") + + +def simple_logging(level=logging.INFO): + logging.basicConfig(level=level, format="%(asctime)s - %(levelname)s - %(message)s") diff --git a/nvflare/fox/examples/np/algos/client.py b/nvflare/fox/examples/np/algos/client.py index cbbf7d9377..9f4c226374 100644 --- a/nvflare/fox/examples/np/algos/client.py +++ b/nvflare/fox/examples/np/algos/client.py @@ -27,9 +27,9 @@ def __init__(self, delta: float): @collab def train(self, current_round, weights, context: Context): if context.is_aborted(): - print("training aborted") + self.logger.debug("training aborted") return 0 - print(f"[{context.header_str()}] trained round {current_round}") + self.logger.debug(f"[{context.header_str()}] trained round {current_round}") # metric_receiver = self.server.get_target("metric_receiver") # if metric_receiver: @@ -41,7 +41,7 @@ def train(self, current_round, weights, context: Context): @collab def evaluate(self, model, context: Context): - print(f"[{context.header_str()}] evaluate") + self.logger.debug(f"[{context.header_str()}] evaluate") return random.random() diff --git a/nvflare/fox/examples/np/algos/filters.py b/nvflare/fox/examples/np/algos/filters.py index cff2ebcf4f..f3e857df5f 100644 --- a/nvflare/fox/examples/np/algos/filters.py +++ b/nvflare/fox/examples/np/algos/filters.py @@ -16,14 +16,18 @@ from nvflare.fox.api.constants import ContextKey from nvflare.fox.api.ctx import Context from nvflare.fox.api.filter import CallFilter, ResultFilter +from nvflare.fuel.utils.log_utils import get_obj_logger class AddNoiseToModel(CallFilter): + def __init__(self): + self.logger = get_obj_logger(self) + def filter_call(self, func_kwargs: dict, context: Context): direction = context.get_prop(ContextKey.DIRECTION) qual_func_name = context.get_prop(ContextKey.QUALIFIED_FUNC_NAME) - print(f"[{context.header_str()}] filtering call: {func_kwargs=} {direction=} {qual_func_name=}") + self.logger.debug(f"[{context.header_str()}] filtering call: {func_kwargs=} {direction=} {qual_func_name=}") weights_key = "weights" weights = func_kwargs.get(weights_key) if weights is None: @@ -33,7 +37,7 @@ def filter_call(self, func_kwargs: dict, context: Context): # add some noise to weights noise = random.random() - print(f"[{context.header_str()}] adding noise {noise}") + self.logger.debug(f"[{context.header_str()}] adding noise {noise}") weights += noise func_kwargs[weights_key] = weights return func_kwargs @@ -41,17 +45,23 @@ def filter_call(self, func_kwargs: dict, context: Context): class PrintCall(CallFilter): + def __init__(self): + self.logger = get_obj_logger(self) + def filter_call(self, func_kwargs: dict, context: Context): direction = context.get_prop(ContextKey.DIRECTION) qual_func_name = context.get_prop(ContextKey.QUALIFIED_FUNC_NAME) - print(f"[{context.header_str()}] printing call: {func_kwargs=} {direction=} {qual_func_name=}") + self.logger.debug(f"[{context.header_str()}] printing call: {func_kwargs=} {direction=} {qual_func_name=}") return func_kwargs class PrintResult(ResultFilter): + def __init__(self): + self.logger = get_obj_logger(self) + def filter_result(self, result, context: Context): direction = context.get_prop(ContextKey.DIRECTION) qual_func_name = context.get_prop(ContextKey.QUALIFIED_FUNC_NAME) - print(f"[{context.header_str()}] printing result: {result=} {direction=} {qual_func_name=}") + self.logger.debug(f"[{context.header_str()}] printing result: {result=} {direction=} {qual_func_name=}") return result diff --git a/nvflare/fox/examples/np/algos/strategies.py b/nvflare/fox/examples/np/algos/strategies.py index 7e2fac9d9a..d4babbf98b 100644 --- a/nvflare/fox/examples/np/algos/strategies.py +++ b/nvflare/fox/examples/np/algos/strategies.py @@ -19,6 +19,7 @@ from nvflare.fox.api.ctx import Context from nvflare.fox.api.group import all_clients from nvflare.fox.api.strategy import Strategy +from nvflare.fuel.utils.log_utils import get_obj_logger from .utils import parse_array_def @@ -30,9 +31,10 @@ def __init__(self, initial_model, num_rounds=10): self.name = "NPFedAvgSequential" self.num_rounds = num_rounds self.initial_model = parse_array_def(initial_model) + self.logger = get_obj_logger(self) def execute(self, context: Context): - print(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") + self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") current_model = context.get_prop(ContextKey.INPUT, self.initial_model) for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model, context) @@ -43,7 +45,7 @@ def _do_one_round(self, r, current_model, context: Context): n = 0 for c in context.clients: result = c.train(r, current_model) - print(f"[{context.header_str()}] round {r}: got result from client {c.name}: {result}") + self.logger.info(f"[{context.header_str()}] round {r}: got result from client {c.name}: {result}") total += result n += 1 return total / n @@ -55,21 +57,22 @@ def __init__(self, initial_model, num_rounds=10): self.num_rounds = num_rounds self.initial_model = parse_array_def(initial_model) self.name = "NPFedAvgParallel" + self.logger = get_obj_logger(self) def execute(self, context: Context): - print(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") + self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") current_model = context.get_prop(ContextKey.INPUT, self.initial_model) for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model, context) score = self._do_eval(current_model, context) - print(f"[{context.header_str()}]: eval score in round {i}: {score}") + self.logger.info(f"[{context.header_str()}]: eval score in round {i}: {score}") return current_model def _do_eval(self, model, ctx: Context): results = all_clients(ctx).evaluate(model) total = 0.0 for n, v in results.items(): - print(f"[{ctx.header_str()}]: got eval result from client {n}: {v}") + self.logger.info(f"[{ctx.header_str()}]: got eval result from client {n}: {v}") total += v return total / len(results) @@ -77,7 +80,7 @@ def _do_one_round(self, r, current_model, ctx: Context): total = np.array([[0, 0, 0], [0, 0, 0], [0, 0, 0]], dtype=np.float32) results = all_clients(ctx).train(r, current_model) for n, v in results.items(): - print(f"[{ctx.header_str()}] round {r}: got group result from client {n}: {v}") + self.logger.info(f"[{ctx.header_str()}] round {r}: got group result from client {n}: {v}") total += v return total / len(results) @@ -96,21 +99,22 @@ def __init__(self, initial_model, num_rounds=10, timeout=2.0): self.initial_model = parse_array_def(initial_model) self.timeout = timeout self.name = "NPFedAvgInTime" + self.logger = get_obj_logger(self) def execute(self, context: Context): - print(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") + self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") current_model = context.get_prop(ContextKey.INPUT, self.initial_model) for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model, context) score = self._do_eval(current_model, context) - print(f"[{context.header_str()}]: eval score in round {i}: {score}") + self.logger.info(f"[{context.header_str()}]: eval score in round {i}: {score}") return current_model def _do_eval(self, model, ctx: Context): results = all_clients(ctx).evaluate(model) total = 0.0 for n, v in results.items(): - print(f"[{ctx.header_str()}]: got eval result from client {n}: {v}") + self.logger.info(f"[{ctx.header_str()}]: got eval result from client {n}: {v}") total += v return total / len(results) @@ -122,14 +126,16 @@ def _do_one_round(self, r, current_model, ctx: Context): aggr_result=aggr_result, ).train(r, current_model) - print(f"[{ctx.header_str()}] round {r}: aggr result from {aggr_result.count} clients: {aggr_result.total}") + self.logger.info( + f"[{ctx.header_str()}] round {r}: aggr result from {aggr_result.count} clients: {aggr_result.total}" + ) if aggr_result.count == 0: return None else: return aggr_result.total / aggr_result.count def _accept_train_result(self, result, aggr_result: _AggrResult, context: Context): - print(f"[{context.header_str()}] got train result from {context.caller} {result}") + self.logger.info(f"[{context.header_str()}] got train result from {context.caller} {result}") aggr_result.total += result aggr_result.count += 1 return None @@ -140,17 +146,18 @@ class NPCyclic(Strategy): def __init__(self, initial_model, num_rounds=10): self.num_rounds = num_rounds self.initial_model = parse_array_def(initial_model) + self.logger = get_obj_logger(self) def execute(self, context: Context): current_model = context.get_prop(ContextKey.INPUT, self.initial_model) for current_round in range(self.num_rounds): current_model = self._do_one_round(current_round, current_model, context) - print(f"[{context.header_str()}] final result: {current_model}") + self.logger.info(f"[{context.header_str()}] final result: {current_model}") return current_model def _do_one_round(self, current_round, current_model, ctx: Context): random.shuffle(ctx.clients) for c in ctx.clients: current_model = c.train(current_round, current_model) - print(f"[{ctx.header_str()}] result from {c.name}: {current_model}") + self.logger.info(f"[{ctx.header_str()}] result from {c.name}: {current_model}") return current_model diff --git a/nvflare/fox/examples/np/algos/swarm.py b/nvflare/fox/examples/np/algos/swarm.py index 079db90cfd..a9b6f7836a 100644 --- a/nvflare/fox/examples/np/algos/swarm.py +++ b/nvflare/fox/examples/np/algos/swarm.py @@ -21,6 +21,7 @@ from nvflare.fox.api.group import all_clients from nvflare.fox.api.strategy import Strategy from nvflare.fox.examples.np.algos.utils import parse_array_def +from nvflare.fuel.utils.log_utils import get_obj_logger class NPSwarm(Strategy): @@ -29,6 +30,7 @@ def __init__(self, initial_model, num_rounds=10): self.num_rounds = num_rounds self.initial_model = parse_array_def(initial_model) self.waiter = threading.Event() + self.logger = get_obj_logger(self) def execute(self, context: Context): context.app.register_event_handler("all_done", self._all_done) @@ -42,12 +44,12 @@ def execute(self, context: Context): break def _all_done(self, event_type: str, data, context: Context): - print(f"[{context.header_str()}]: received {event_type} from client: {context.caller}: {data}") + self.logger.info(f"[{context.header_str()}]: received {event_type} from client: {context.caller}: {data}") self.all_done(data, context) @collab def all_done(self, reason: str, context: Context): - print(f"[{context.header_str()}]: all done from client: {context.caller}: {reason}") + self.logger.info(f"[{context.header_str()}]: all done from client: {context.caller}: {reason}") self.waiter.set() @@ -60,7 +62,7 @@ def __init__(self, delta: float): @collab def train(self, weights, current_round, context: Context): - print(f"[{context.header_str()}]: train asked by {context.caller}: {current_round=}") + self.logger.info(f"[{context.header_str()}]: train asked by {context.caller}: {current_round=}") return weights + self.delta def sag(self, model, current_round, ctx: Context): @@ -73,20 +75,20 @@ def sag(self, model, current_round, ctx: Context): @collab def swarm_learn(self, num_rounds, model, current_round, context: Context): - print(f"[{context.header_str()}]: swarm learn asked: {num_rounds=} {current_round=} {model=}") + self.logger.info(f"[{context.header_str()}]: swarm learn asked: {num_rounds=} {current_round=} {model=}") new_model = self.sag(model, current_round, context) - print(f"[{context.header_str()}]: trained model {new_model=}") + self.logger.info(f"[{context.header_str()}]: trained model {new_model=}") if current_round == num_rounds - 1: # all done all_clients(context, blocking=False).fire_event("final_model", new_model) # self.server.fire_event("all_done", "OK", blocking=False) - print("notify server all done!") + self.logger.info("notify server all done!") try: self.server.all_done("OK", _blocking=False) except: traceback.print_exc() - print("Swarm Training is DONE!") + self.logger.info("Swarm Training is DONE!") return # determine next client @@ -102,4 +104,4 @@ def start(self, num_rounds, initial_model, context: Context): def _accept_final_model(self, event_type: str, model, context: Context): # accept the final model # write model to disk - print(f"[{context.header_str()}]: received event '{event_type}' from {context.caller}: {model}") + self.logger.info(f"[{context.header_str()}]: received event '{event_type}' from {context.caller}: {model}") diff --git a/nvflare/fox/examples/np/algos/widgets.py b/nvflare/fox/examples/np/algos/widgets.py index dd1abec680..aa27e286cf 100644 --- a/nvflare/fox/examples/np/algos/widgets.py +++ b/nvflare/fox/examples/np/algos/widgets.py @@ -13,17 +13,21 @@ # limitations under the License. from nvflare.fox.api.ctx import Context from nvflare.fox.api.dec import collab +from nvflare.fuel.utils.log_utils import get_obj_logger class MetricReceiver: + def __init__(self): + self.logger = get_obj_logger(self) + @collab def accept_metric(self, metrics: dict, context: Context): - print(f"[{context.callee}] received metric report from {context.caller}: {metrics}") + self.logger.info(f"[{context.callee}] received metric report from {context.caller}: {metrics}") def initialize(self, context: Context): context.app.register_event_handler("metrics", self._accept_metric) - print("MetricReceiver initialized!") + self.logger.info("MetricReceiver initialized!") def _accept_metric(self, event_type: str, data, context: Context): - print(f"[{context.callee}] received event '{event_type}' from {context.caller}: {data}") + self.logger.info(f"[{context.callee}] received event '{event_type}' from {context.caller}: {data}") diff --git a/nvflare/fox/examples/np/cyclic.py b/nvflare/fox/examples/np/cyclic.py index e4ecaf69e2..182f1345b8 100644 --- a/nvflare/fox/examples/np/cyclic.py +++ b/nvflare/fox/examples/np/cyclic.py @@ -11,13 +11,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import logging + from nvflare.fox.api.app import ServerApp +from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.strategies import NPCyclic from nvflare.fox.sim.simulator import Simulator def main(): + simple_logging(logging.DEBUG) simulator = Simulator( server_app=ServerApp( diff --git a/nvflare/fox/examples/np/cyclic_avg.py b/nvflare/fox/examples/np/cyclic_avg.py index 041d2e3d85..6440ab9b8e 100644 --- a/nvflare/fox/examples/np/cyclic_avg.py +++ b/nvflare/fox/examples/np/cyclic_avg.py @@ -11,13 +11,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import logging + from nvflare.fox.api.app import ServerApp +from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.strategies import NPCyclic, NPFedAvgParallel from nvflare.fox.sim.simulator import Simulator def main(): + simple_logging(logging.DEBUG) + server_app = ServerApp( strategy_name="cyclic", strategy=NPCyclic(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2) ) diff --git a/nvflare/fox/examples/np/fed_avg_intime.py b/nvflare/fox/examples/np/fed_avg_intime.py index f8e2c11355..cfa769a444 100644 --- a/nvflare/fox/examples/np/fed_avg_intime.py +++ b/nvflare/fox/examples/np/fed_avg_intime.py @@ -11,7 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import logging + from nvflare.fox.api.app import ServerApp +from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.filters import AddNoiseToModel, PrintCall, PrintResult from nvflare.fox.examples.np.algos.strategies import NPFedAvgInTime @@ -20,6 +23,7 @@ def main(): + simple_logging(logging.DEBUG) server_app = ServerApp( strategy_name="fed_avg_in_time", diff --git a/nvflare/fox/examples/np/fed_avg_para.py b/nvflare/fox/examples/np/fed_avg_para.py index 997a04d64c..79304a3d9a 100644 --- a/nvflare/fox/examples/np/fed_avg_para.py +++ b/nvflare/fox/examples/np/fed_avg_para.py @@ -11,7 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import logging + from nvflare.fox.api.app import ServerApp +from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.strategies import NPFedAvgParallel from nvflare.fox.examples.np.algos.widgets import MetricReceiver @@ -19,6 +22,7 @@ def main(): + simple_logging(logging.DEBUG) server_app = ServerApp( strategy_name="fed_avg", diff --git a/nvflare/fox/examples/np/fed_avg_seq.py b/nvflare/fox/examples/np/fed_avg_seq.py index cb9c86fe79..2430096851 100644 --- a/nvflare/fox/examples/np/fed_avg_seq.py +++ b/nvflare/fox/examples/np/fed_avg_seq.py @@ -11,7 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import logging + from nvflare.fox.api.app import ServerApp +from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.client import TrainerFactory from nvflare.fox.examples.np.algos.strategies import NPFedAvgSequential from nvflare.fox.examples.np.algos.widgets import MetricReceiver @@ -19,6 +22,7 @@ def main(): + simple_logging(logging.DEBUG) server_app = ServerApp( strategy_name="fed_avg", diff --git a/nvflare/fox/examples/np/swarm.py b/nvflare/fox/examples/np/swarm.py index b233f25876..effc563c8f 100644 --- a/nvflare/fox/examples/np/swarm.py +++ b/nvflare/fox/examples/np/swarm.py @@ -11,13 +11,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import logging from nvflare.fox.api.app import ServerApp +from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.swarm import NPSwarm, NPSwarmClient from nvflare.fox.sim.simulator import Simulator def main(): + simple_logging(logging.DEBUG) server_app = ServerApp( strategy_name="strategy", strategy=NPSwarm(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=5) diff --git a/nvflare/fox/sim/backend.py b/nvflare/fox/sim/backend.py index 4a78b8b738..f5f41b20be 100644 --- a/nvflare/fox/sim/backend.py +++ b/nvflare/fox/sim/backend.py @@ -21,6 +21,7 @@ from nvflare.fox.api.dec import adjust_kwargs from nvflare.fox.api.resp import Resp from nvflare.fox.api.utils import check_call_args +from nvflare.fuel.utils.log_utils import get_obj_logger class _Waiter(threading.Event): @@ -38,6 +39,7 @@ def __init__(self, target_obj_name: str, target_app: App, target_obj, abort_sign self.target_app = target_app self.target_obj = target_obj self.executor = thread_executor + self.logger = get_obj_logger(self) def _get_func(self, func_name): return self.target_app.find_collab_method(self.target_obj, func_name) @@ -95,7 +97,7 @@ def _preprocess(self, target_name, func_name, func, kwargs): raise RuntimeError(f"cannot find interface for func '{func_name}' of object {self.target_obj_name}") check_call_args(func_name, func_itf, [], kwargs) - print(f"[{my_ctx.header_str()}] received kwargs is good: {kwargs}") + self.logger.debug(f"[{my_ctx.header_str()}] received kwargs is good: {kwargs}") kwargs[CollabMethodArgName.CONTEXT] = my_ctx adjust_kwargs(func, kwargs) diff --git a/nvflare/fox/sim/simulator.py b/nvflare/fox/sim/simulator.py index 1309c8b4c5..1b1e6f23c0 100644 --- a/nvflare/fox/sim/simulator.py +++ b/nvflare/fox/sim/simulator.py @@ -21,6 +21,7 @@ from nvflare.fox.api.dec import get_object_collab_interface from nvflare.fox.api.proxy import Proxy from nvflare.fox.sim.backend import SimBackend +from nvflare.fuel.utils.log_utils import get_obj_logger class Simulator: @@ -71,6 +72,7 @@ def __init__( if not isinstance(client_app, (ClientAppFactory, ClientApp)): raise ValueError(f"client_app must be ClientApp or ClientAppFactory but got {type(client_app)}") + self.logger = get_obj_logger(self) self.abort_signal = Signal() server_app.name = "server" server_app.env_type = EnvType.SIMULATION @@ -109,17 +111,19 @@ def run(self): try: self._try_run() except KeyboardInterrupt: - print("execution is aborted by user") + self.logger.info("execution is aborted by user") self.abort_signal.trigger(True) + finally: + self.thread_executor.shutdown(wait=False, cancel_futures=True) def _try_run(self): # initialize all apps server_ctx = self.server_app.new_context(caller=self.server_app.name, callee=self.server_app.name) - print("initializing server app") + self.logger.info("initializing server app") self.server_app.initialize(server_ctx) for n, app in self.client_apps.items(): - print(f"initializing client app for {n}") + self.logger.info(f"initializing client app for {n}") app.initialize(app.new_context(n, n)) # run the server @@ -128,10 +132,8 @@ def _try_run(self): result = None for idx, strategy in enumerate(self.server_app.strategies): - print(f"Running Strategy #{idx+1} - {type(strategy).__name__}") + self.logger.info(f"Running Strategy #{idx+1} - {type(strategy).__name__}") self.server_app.current_strategy = strategy result = strategy.execute(context=server_ctx) server_ctx.set_prop(ContextKey.INPUT, result) - - self.thread_executor.shutdown(wait=False, cancel_futures=True) return result From 7a661772df2c44148889c06f4e702b1e2ed3608d Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Mon, 6 Oct 2025 16:33:42 -0400 Subject: [PATCH 029/102] support hierarchical --- nvflare/fox/api/app.py | 41 +++++++++- nvflare/fox/api/group.py | 62 +++++++++++++++ nvflare/fox/api/proxy.py | 3 +- nvflare/fox/examples/np/algos/client.py | 46 +++++++++++ nvflare/fox/examples/np/algos/strategies.py | 39 +++++++++- nvflare/fox/examples/np/cyclic_avg.py | 2 - nvflare/fox/examples/np/fed_avg_h.py | 39 ++++++++++ nvflare/fox/sim/simulator.py | 86 +++++++++++++++++---- nvflare/fuel/utils/tree_utils.py | 5 ++ 9 files changed, 304 insertions(+), 19 deletions(-) create mode 100644 nvflare/fox/examples/np/fed_avg_h.py diff --git a/nvflare/fox/api/app.py b/nvflare/fox/api/app.py index fe8777154e..105f67efc5 100644 --- a/nvflare/fox/api/app.py +++ b/nvflare/fox/api/app.py @@ -17,6 +17,7 @@ from typing import List from nvflare.fuel.utils.log_utils import get_obj_logger +from nvflare.fuel.utils.tree_utils import Forest, Node from .constants import CollabMethodArgName, ContextKey, FilterDirection from .ctx import Context @@ -31,8 +32,10 @@ class App: def __init__(self): self.name = None + self.fqn = None self.server = None self.clients = None + self.client_hierarchy = None self.env_type = None self._me = None self._collab_objs = {} @@ -247,6 +250,17 @@ def fire_event(self, event_type: str, data, context: Context): check_context_support(h, kwargs) h(event_type, data, **kwargs) + def get_children(self): + return [] + + def has_children(self): + return True + + def get_leaf_clients(self): + assert isinstance(self.client_hierarchy, Forest) + leaf_nodes = [self.client_hierarchy.nodes[n] for n in self.client_hierarchy.leaves] + return [node.obj for node in leaf_nodes] + class ServerApp(App): @@ -272,9 +286,34 @@ def add_strategy(self, strategy_name: str, strategy): def get_default_collab_object(self): return self.current_strategy + def get_children(self): + assert isinstance(self.client_hierarchy, Forest) + root_nodes = [self.client_hierarchy.nodes[n] for n in self.client_hierarchy.roots] + return [node.obj for node in root_nodes] + + def has_children(self): + return True + class ClientApp(App): - pass + + def get_children(self): + assert isinstance(self.client_hierarchy, Forest) + my_node = self.client_hierarchy.nodes[self.name] + assert isinstance(my_node, Node) + if my_node.children: + return [node.obj for node in my_node.children] + else: + return None + + def has_children(self): + assert isinstance(self.client_hierarchy, Forest) + my_node = self.client_hierarchy.nodes[self.name] + assert isinstance(my_node, Node) + if my_node.children: + return True + else: + return False class ClientAppFactory(ABC): diff --git a/nvflare/fox/api/group.py b/nvflare/fox/api/group.py index 1ffebcbbd2..57bedd6d5e 100644 --- a/nvflare/fox/api/group.py +++ b/nvflare/fox/api/group.py @@ -77,6 +77,8 @@ def method(*args, **kwargs): the_proxy, func_itf, adj_args, adj_kwargs = p.adjust_func_args(func_name, args, kwargs) ctx = the_proxy.app.new_context(the_proxy.app.name, the_proxy.name) + self._logger.debug(f"[{ctx.header_str()}] calling {func_name} of group {[p.name for p in self._proxies]}") + # apply outgoing call filters adj_kwargs = self._app.apply_outgoing_call_filters(p.target_name, func_name, adj_kwargs, ctx) check_call_args(func_name, func_itf, adj_args, adj_kwargs) @@ -207,3 +209,63 @@ def all_clients( process_resp_cb, **cb_kwargs, ) + + +def all_children( + ctx: Context, + blocking: bool = True, + timeout: float = 5.0, + optional: bool = False, + secure: bool = False, + min_resps: int = None, + wait_after_min_resps: float = None, + process_resp_cb=None, + **cb_kwargs, +): + clients = ctx.app.get_children() + if not clients: + raise RuntimeError(f"app {ctx.app.name} has no child clients") + + return Group( + ctx.app, + ctx.abort_signal, + clients, + blocking, + timeout, + optional, + secure, + min_resps, + wait_after_min_resps, + process_resp_cb, + **cb_kwargs, + ) + + +def all_leaf_clients( + ctx: Context, + blocking: bool = True, + timeout: float = 5.0, + optional: bool = False, + secure: bool = False, + min_resps: int = None, + wait_after_min_resps: float = None, + process_resp_cb=None, + **cb_kwargs, +): + clients = ctx.app.get_leaf_clients() + if not clients: + raise RuntimeError(f"app {ctx.app.name} has no leaf clients") + + return Group( + ctx.app, + ctx.abort_signal, + clients, + blocking, + timeout, + optional, + secure, + min_resps, + wait_after_min_resps, + process_resp_cb, + **cb_kwargs, + ) diff --git a/nvflare/fox/api/proxy.py b/nvflare/fox/api/proxy.py index d32339f84c..29c03cef64 100644 --- a/nvflare/fox/api/proxy.py +++ b/nvflare/fox/api/proxy.py @@ -22,10 +22,11 @@ class Proxy: - def __init__(self, app, target_name, backend: Backend, target_interface): + def __init__(self, app, target_name, target_fqn: str, backend: Backend, target_interface): """The Proxy represents a target in the App.""" self.app = app self.target_name = target_name + self.fqn = target_fqn # fully qualified name of the target in hierarchy self.backend = backend self.caller_name = app.name self.target_interface = target_interface diff --git a/nvflare/fox/examples/np/algos/client.py b/nvflare/fox/examples/np/algos/client.py index 9f4c226374..82be10cdf7 100644 --- a/nvflare/fox/examples/np/algos/client.py +++ b/nvflare/fox/examples/np/algos/client.py @@ -16,6 +16,7 @@ from nvflare.fox.api.app import ClientApp, ClientAppFactory from nvflare.fox.api.ctx import Context from nvflare.fox.api.dec import collab +from nvflare.fox.api.group import all_children class NPTrainer(ClientApp): @@ -45,6 +46,51 @@ def evaluate(self, model, context: Context): return random.random() +class NPHierarchicalTrainer(ClientApp): + + def __init__(self, delta: float): + ClientApp.__init__(self) + self.delta = delta + + @collab + def train(self, current_round, weights, context: Context): + if context.is_aborted(): + self.logger.debug("training aborted") + return 0 + + self.logger.debug(f"[{context.header_str()}] training round {current_round}") + if context.app.has_children(): + total = 0 + results = all_children(context).train(current_round, weights) + for n, v in results.items(): + total += v + result = total / len(results) + self.logger.debug(f"[{context.header_str()}]: aggr result from children of round {current_round}: {result}") + else: + result = self._local_train(current_round, weights, context) + self.logger.debug(f"[{context.header_str()}]: local train result of round {current_round}: {result}") + return result + + def _local_train(self, current_round, weights, context: Context): + if context.is_aborted(): + self.logger.debug("training aborted") + return 0 + self.logger.debug(f"[{context.header_str()}] trained round {current_round}") + + # metric_receiver = self.server.get_target("metric_receiver") + # if metric_receiver: + # self.server.accept_metric({"round": r, "y": 2}) + # self.server.metric_receiver.accept_metric({"round": r, "y": 2}) + # + self.server.fire_event("metrics", {"round": current_round, "y": 10}, _blocking=False) + return weights + self.delta + + @collab + def evaluate(self, model, context: Context): + self.logger.debug(f"[{context.header_str()}] evaluate") + return random.random() + + class TrainerFactory(ClientAppFactory): def __init__(self, delta): diff --git a/nvflare/fox/examples/np/algos/strategies.py b/nvflare/fox/examples/np/algos/strategies.py index d4babbf98b..73bb3c180b 100644 --- a/nvflare/fox/examples/np/algos/strategies.py +++ b/nvflare/fox/examples/np/algos/strategies.py @@ -17,7 +17,7 @@ from nvflare.fox.api.constants import ContextKey from nvflare.fox.api.ctx import Context -from nvflare.fox.api.group import all_clients +from nvflare.fox.api.group import all_children, all_clients, all_leaf_clients from nvflare.fox.api.strategy import Strategy from nvflare.fuel.utils.log_utils import get_obj_logger @@ -85,6 +85,41 @@ def _do_one_round(self, r, current_model, ctx: Context): return total / len(results) +class NPHierarchicalFedAvg(Strategy): + + def __init__(self, initial_model, num_rounds=10): + self.num_rounds = num_rounds + self.initial_model = parse_array_def(initial_model) + self.name = self.__class__.__name__ + self.logger = get_obj_logger(self) + + def execute(self, context: Context): + self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") + current_model = context.get_prop(ContextKey.INPUT, self.initial_model) + for i in range(self.num_rounds): + current_model = self._do_one_round(i, current_model, context) + score = self._do_eval(current_model, context) + self.logger.info(f"[{context.header_str()}]: eval score in round {i}: {score}") + self.logger.info(f"FINAL MODEL: {current_model}") + return current_model + + def _do_eval(self, model, ctx: Context): + results = all_leaf_clients(ctx).evaluate(model) + total = 0.0 + for n, v in results.items(): + self.logger.info(f"[{ctx.header_str()}]: got eval result from client {n}: {v}") + total += v + return total / len(results) + + def _do_one_round(self, r, current_model, ctx: Context): + total = np.array([[0, 0, 0], [0, 0, 0], [0, 0, 0]], dtype=np.float32) + results = all_children(ctx).train(r, current_model) + for n, v in results.items(): + self.logger.info(f"[{ctx.header_str()}] round {r}: got group result from client {n}: {v}") + total += v + return total / len(results) + + class _AggrResult: def __init__(self): @@ -143,7 +178,7 @@ def _accept_train_result(self, result, aggr_result: _AggrResult, context: Contex class NPCyclic(Strategy): - def __init__(self, initial_model, num_rounds=10): + def __init__(self, initial_model, num_rounds=(2, 3)): self.num_rounds = num_rounds self.initial_model = parse_array_def(initial_model) self.logger = get_obj_logger(self) diff --git a/nvflare/fox/examples/np/cyclic_avg.py b/nvflare/fox/examples/np/cyclic_avg.py index 6440ab9b8e..ae572ec512 100644 --- a/nvflare/fox/examples/np/cyclic_avg.py +++ b/nvflare/fox/examples/np/cyclic_avg.py @@ -28,8 +28,6 @@ def main(): ) server_app.add_strategy("fed_avg_parallel", NPFedAvgParallel(initial_model=None, num_rounds=2)) - server_app.get_collab_interface() - simulator = Simulator( server_app=server_app, client_app=NPTrainer(delta=1.0), diff --git a/nvflare/fox/examples/np/fed_avg_h.py b/nvflare/fox/examples/np/fed_avg_h.py new file mode 100644 index 0000000000..be12c9619e --- /dev/null +++ b/nvflare/fox/examples/np/fed_avg_h.py @@ -0,0 +1,39 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from nvflare.fox.api.app import ServerApp +from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples.np.algos.client import NPHierarchicalTrainer +from nvflare.fox.examples.np.algos.strategies import NPHierarchicalFedAvg +from nvflare.fox.examples.np.algos.widgets import MetricReceiver +from nvflare.fox.sim.simulator import Simulator + + +def main(): + simple_logging(logging.DEBUG) + + server_app = ServerApp( + strategy_name="fed_avg", + strategy=NPHierarchicalFedAvg(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=3), + ) + server_app.add_collab_object("metric_receiver", MetricReceiver()) + + simulator = Simulator(server_app=server_app, client_app=NPHierarchicalTrainer(delta=1.0), num_clients=(3, 2)) + + simulator.run() + + +if __name__ == "__main__": + main() diff --git a/nvflare/fox/sim/simulator.py b/nvflare/fox/sim/simulator.py index 1b1e6f23c0..0f2cefa306 100644 --- a/nvflare/fox/sim/simulator.py +++ b/nvflare/fox/sim/simulator.py @@ -13,7 +13,7 @@ # limitations under the License. import copy from concurrent.futures import ThreadPoolExecutor -from typing import Union +from typing import Tuple, Union from nvflare.apis.signal import Signal from nvflare.fox.api.app import App, ClientApp, ClientAppFactory, ServerApp @@ -22,6 +22,7 @@ from nvflare.fox.api.proxy import Proxy from nvflare.fox.sim.backend import SimBackend from nvflare.fuel.utils.log_utils import get_obj_logger +from nvflare.fuel.utils.tree_utils import build_forest class Simulator: @@ -37,6 +38,7 @@ def _prepare_proxy(self, for_app: App, target_app: App, backends: dict): app_proxy = Proxy( app=for_app, target_name=target_app.name, + target_fqn=target_app.fqn, backend=backends[""], target_interface=get_object_collab_interface(target_app), ) @@ -45,18 +47,36 @@ def _prepare_proxy(self, for_app: App, target_app: App, backends: dict): p = Proxy( app=for_app, target_name=f"{target_app.name}.{name}", + target_fqn="", backend=backends[name], target_interface=get_object_collab_interface(obj), ) app_proxy.add_child(name, p) return app_proxy + def _make_app(self, name, fqn): + if isinstance(self.client_app, ClientApp): + app = copy.deepcopy(self.client_app) + else: + app = self.client_app.make_client_app(name) + app.name = name + app.fqn = fqn + app.env_type = EnvType.SIMULATION + return app + def _prepare_proxies(self, for_app: App, server_app: App, client_apps: dict, backends: dict): server_proxy = self._prepare_proxy(for_app, server_app, backends[server_app.name]) client_proxies = [] for name, app in client_apps.items(): p = self._prepare_proxy(for_app, app, backends[name]) client_proxies.append(p) + + # compute client proxy hierarchy + forest = build_forest(objs=client_proxies, get_fqn_f=lambda c: c.fqn, get_name_f=lambda c: c.name) + # d = forest_to_dict(forest, lambda c: c.name) + # print(f"client proxy forest: {d}") + for_app.client_hierarchy = forest + return server_proxy, client_proxies def __init__( @@ -64,7 +84,7 @@ def __init__( server_app: ServerApp, client_app: Union[ClientAppFactory, ClientApp], max_workers: int = 100, - num_clients: int = 2, + num_clients: Union[int, Tuple[int, int]] = 2, ): if not isinstance(server_app, ServerApp): raise ValueError(f"server_app must be ServerApp but got {type(server_app)}") @@ -75,22 +95,37 @@ def __init__( self.logger = get_obj_logger(self) self.abort_signal = Signal() server_app.name = "server" + server_app.fqn = server_app.name server_app.env_type = EnvType.SIMULATION self.server_app = server_app self.client_app = client_app self.thread_executor = ThreadPoolExecutor(max_workers=max_workers) - self.num_clients = num_clients - client_apps = {} - for i in range(self.num_clients): - name = f"site-{i + 1}" - if isinstance(client_app, ClientApp): - app = copy.deepcopy(client_app) - else: - app = client_app.make_client_app(name) - app.name = name - app.env_type = EnvType.SIMULATION - client_apps[name] = app + if isinstance(num_clients, int): + if num_clients <= 0: + raise ValueError(f"num_clients must > 0 but got {num_clients}") + client_apps = {} + for i in range(num_clients): + name = f"site-{i + 1}" + client_apps[name] = self._make_app(name, name) + elif isinstance(num_clients, tuple): + if len(num_clients) != 2: + raise ValueError(f"num_clients must be an int or tuple(int, int) but got {num_clients}") + + # tuple of (height x width) + height, num_children_per_parent = num_clients + if not isinstance(height, int) or not isinstance(num_children_per_parent, int): + raise ValueError(f"num_clients must be an int or tuple(int, int) but got {num_clients}") + + if height <= 0 or num_children_per_parent <= 0: + raise ValueError(f"num_clients must contain positive ints but got {num_clients}") + + print(f"creating clients {height} x {num_children_per_parent}") + client_apps = self._build_hierarchical_clients(height, num_children_per_parent) + else: + raise ValueError(f"num_clients must be an int or tuple(int, int) but got {type(num_clients)}") + + self.logger.info(f"created client apps: {client_apps.keys()}") backends = {server_app.name: self._prepare_app_backends(server_app)} @@ -107,6 +142,31 @@ def __init__( self.client_apps = client_apps + def _build_hierarchical_clients(self, height: int, num_children_per_parent: int): + client_apps = {} + last_client_fqns = {} + current_client_fqns = {} + for i in range(height): + if not last_client_fqns: + for j in range(num_children_per_parent): + name = f"site-{j + 1}" + fqn = name + app = self._make_app(name, fqn) + client_apps[name] = app + current_client_fqns[fqn] = app + else: + for fqn, parent_app in last_client_fqns.items(): + # create w clients for each parent + for k in range(num_children_per_parent): + child_name = f"{parent_app.name}-{k + 1}" + child_fqn = f"{fqn}.{child_name}" + app = self._make_app(child_name, child_fqn) + client_apps[child_name] = app + current_client_fqns[child_fqn] = app + last_client_fqns = current_client_fqns + current_client_fqns = {} + return client_apps + def run(self): try: self._try_run() diff --git a/nvflare/fuel/utils/tree_utils.py b/nvflare/fuel/utils/tree_utils.py index ebd92b8850..bda0c84a06 100644 --- a/nvflare/fuel/utils/tree_utils.py +++ b/nvflare/fuel/utils/tree_utils.py @@ -85,6 +85,7 @@ def __init__(self): """Constructor of Forest""" self.roots = [] # one or more names of the root nodes self.nodes = {} # name => Node + self.leaves = [] # names of leaf nodes def build_forest(objs: List[Any], get_name_f, get_fqn_f, **kwargs) -> Forest: @@ -140,6 +141,10 @@ def build_forest(objs: List[Any], get_name_f, get_fqn_f, **kwargs) -> Forest: # this node has no parent - it's a root forest.roots.append(name) + for name, node in forest.nodes.items(): + if not node.children: + forest.leaves.append(name) + return forest From bc6d121034ea66bd403e00cfdc7a3cfee18ca591 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Tue, 7 Oct 2025 12:22:31 -0400 Subject: [PATCH 030/102] support hierarchical training --- nvflare/apis/fl_constant.py | 1 + nvflare/fox/api/app.py | 13 +++++++++++- nvflare/fox/api/ctx.py | 2 -- nvflare/fox/api/group.py | 2 +- nvflare/fox/sim/simulator.py | 7 ------- nvflare/fox/sys/controller.py | 21 +++++++++++++++++--- nvflare/fox/sys/executor.py | 22 ++++++++++++++++++--- nvflare/private/fed/client/client_runner.py | 5 +++++ nvflare/private/fed/client/communicator.py | 2 +- nvflare/private/fed/client/utils.py | 7 +++++++ 10 files changed, 64 insertions(+), 18 deletions(-) diff --git a/nvflare/apis/fl_constant.py b/nvflare/apis/fl_constant.py index 908a4d1bfb..b2e79ffba9 100644 --- a/nvflare/apis/fl_constant.py +++ b/nvflare/apis/fl_constant.py @@ -195,6 +195,7 @@ class FLContextKey(object): EVENT_PROCESSED = "__event_processed__" CELL_MESSAGE = "__cell_message__" CLIENT_HIERARCHY = "__client_hierarchy__" + FOX_MODE = "__fox_mode__" class ProcessType: diff --git a/nvflare/fox/api/app.py b/nvflare/fox/api/app.py index 105f67efc5..50a32d2c46 100644 --- a/nvflare/fox/api/app.py +++ b/nvflare/fox/api/app.py @@ -17,7 +17,7 @@ from typing import List from nvflare.fuel.utils.log_utils import get_obj_logger -from nvflare.fuel.utils.tree_utils import Forest, Node +from nvflare.fuel.utils.tree_utils import Forest, Node, build_forest from .constants import CollabMethodArgName, ContextKey, FilterDirection from .ctx import Context @@ -49,6 +49,12 @@ def __init__(self): self._collab_interface = {"": get_object_collab_interface(self)} self.logger = get_obj_logger(self) + def get_server_proxy(self): + return self.server + + def get_client_proxies(self): + return copy.copy(self.clients) + @staticmethod def _add_filters(pattern: str, filters, to_list: list, filter_type): if not filters: @@ -173,6 +179,11 @@ def setup(self, server: Proxy, clients: List[Proxy], abort_signal): if not self._me: raise ValueError(f"cannot find site for {self.name}") + forest = build_forest(objs=clients, get_fqn_f=lambda c: c.fqn, get_name_f=lambda c: c.name) + # d = forest_to_dict(forest, lambda c: c.name) + # print(f"client proxy forest: {d}") + self.client_hierarchy = forest + def get_my_site(self) -> Proxy: return self._me diff --git a/nvflare/fox/api/ctx.py b/nvflare/fox/api/ctx.py index 76e03231cd..a96c941dd9 100644 --- a/nvflare/fox/api/ctx.py +++ b/nvflare/fox/api/ctx.py @@ -30,8 +30,6 @@ def __init__(self, env_type: str, caller: str, callee: str, abort_signal: Signal self.caller = caller self.callee = callee self.abort_signal = abort_signal - self.server = None - self.clients = None self.app = None self.props = {} if props: diff --git a/nvflare/fox/api/group.py b/nvflare/fox/api/group.py index 57bedd6d5e..66b8c1b932 100644 --- a/nvflare/fox/api/group.py +++ b/nvflare/fox/api/group.py @@ -199,7 +199,7 @@ def all_clients( return Group( ctx.app, ctx.abort_signal, - ctx.clients, + ctx.app.get_client_proxies(), blocking, timeout, optional, diff --git a/nvflare/fox/sim/simulator.py b/nvflare/fox/sim/simulator.py index 0f2cefa306..72593bd2d5 100644 --- a/nvflare/fox/sim/simulator.py +++ b/nvflare/fox/sim/simulator.py @@ -22,7 +22,6 @@ from nvflare.fox.api.proxy import Proxy from nvflare.fox.sim.backend import SimBackend from nvflare.fuel.utils.log_utils import get_obj_logger -from nvflare.fuel.utils.tree_utils import build_forest class Simulator: @@ -71,12 +70,6 @@ def _prepare_proxies(self, for_app: App, server_app: App, client_apps: dict, bac p = self._prepare_proxy(for_app, app, backends[name]) client_proxies.append(p) - # compute client proxy hierarchy - forest = build_forest(objs=client_proxies, get_fqn_f=lambda c: c.fqn, get_name_f=lambda c: c.name) - # d = forest_to_dict(forest, lambda c: c.name) - # print(f"client proxy forest: {d}") - for_app.client_hierarchy = forest - return server_proxy, client_proxies def __init__( diff --git a/nvflare/fox/sys/controller.py b/nvflare/fox/sys/controller.py index 338b94875a..542c25f694 100644 --- a/nvflare/fox/sys/controller.py +++ b/nvflare/fox/sys/controller.py @@ -167,14 +167,24 @@ def _prepare_client_proxy( ): backend = self._prepare_client_backend(job_id, client, abort_signal) proxy = Proxy( - app=self.server_app, target_name=client.name, backend=backend, target_interface=collab_interface.get("") + app=self.server_app, + target_name=client.name, + target_fqn=client.get_fqsn(), + backend=backend, + target_interface=collab_interface.get(""), ) for name, itf in collab_interface.items(): if name == "": continue - p = Proxy(app=self.server_app, target_name=f"{client.name}.{name}", backend=backend, target_interface=itf) + p = Proxy( + app=self.server_app, + target_name=f"{client.name}.{name}", + target_fqn="", + backend=backend, + target_interface=itf, + ) proxy.add_child(name, p) return proxy @@ -187,13 +197,18 @@ def _prepare_server_proxy( server_name = self.server_app.name backend = self._prepare_server_backend(job_id, abort_signal) proxy = Proxy( - app=self.server_app, target_name=server_name, backend=backend, target_interface=collab_interface.get("") + app=self.server_app, + target_name=server_name, + target_fqn=server_name, + backend=backend, + target_interface=collab_interface.get(""), ) for name in self.server_collab_obj_ids: p = Proxy( app=self.server_app, target_name=f"{server_name}.{name}", + target_fqn="", backend=backend, target_interface=collab_interface.get(name), ) diff --git a/nvflare/fox/sys/executor.py b/nvflare/fox/sys/executor.py index 66b0d0b25a..29bc9ddce9 100644 --- a/nvflare/fox/sys/executor.py +++ b/nvflare/fox/sys/executor.py @@ -44,6 +44,7 @@ def __init__(self, client_app_id: str, collab_obj_ids: Dict[str, str] = None, ma self.thread_executor = ThreadPoolExecutor(max_workers=max_call_threads) def _handle_start_run(self, event_type: str, fl_ctx: FLContext): + fl_ctx.set_prop(FLContextKey.FOX_MODE, True, private=True, sticky=True) engine = fl_ctx.get_engine() client_app = engine.get_component(self.client_app_id) if not isinstance(client_app, (ClientApp, ClientAppFactory)): @@ -84,14 +85,24 @@ def _prepare_server_proxy(self, job_id, cell, collab_interface: dict, abort_sign thread_executor=self.thread_executor, ) proxy = Proxy( - app=self.client_app, target_name=server_name, backend=backend, target_interface=collab_interface.get("") + app=self.client_app, + target_name=server_name, + target_fqn=server_name, + backend=backend, + target_interface=collab_interface.get(""), ) for name, itf in collab_interface.items(): if name == "": # this is the server app itself continue - p = Proxy(app=self.client_app, target_name=f"{server_name}.{name}", backend=backend, target_interface=itf) + p = Proxy( + app=self.client_app, + target_name=f"{server_name}.{name}", + target_fqn="", + backend=backend, + target_interface=itf, + ) proxy.add_child(name, p) return proxy @@ -104,7 +115,11 @@ def _prepare_client_proxy(self, job_id, cell, client: Client, abort_signal, coll thread_executor=self.thread_executor, ) proxy = Proxy( - app=self.client_app, target_name=client.name, backend=backend, target_interface=collab_interface.get("") + app=self.client_app, + target_name=client.name, + target_fqn=client.get_fqsn(), + backend=backend, + target_interface=collab_interface.get(""), ) if self.collab_obj_ids: @@ -112,6 +127,7 @@ def _prepare_client_proxy(self, job_id, cell, client: Client, abort_signal, coll p = Proxy( app=self.client_app, target_name=f"{client.name}.{name}", + target_fqn="", backend=backend, target_interface=collab_interface.get(name), ) diff --git a/nvflare/private/fed/client/client_runner.py b/nvflare/private/fed/client/client_runner.py index b8221dedb8..142385c789 100644 --- a/nvflare/private/fed/client/client_runner.py +++ b/nvflare/private/fed/client/client_runner.py @@ -732,6 +732,11 @@ def init_run(self, app_root, args): self.log_debug(fl_ctx, "firing event EventType.START_RUN") self.fire_event(EventType.START_RUN, fl_ctx) self.log_info(fl_ctx, "client runner started") + fox_mode = fl_ctx.get_prop(FLContextKey.FOX_MODE, False) + if fox_mode: + # in fox mode, all tasks go to the server + self.logger.debug(f"changed parent target from {self.parent_target} to {FQCN.ROOT_SERVER}") + self.parent_target = FQCN.ROOT_SERVER def _handle_sync_runner(self, topic: str, request: Shareable, fl_ctx: FLContext) -> Shareable: # simply ack diff --git a/nvflare/private/fed/client/communicator.py b/nvflare/private/fed/client/communicator.py index 650081627f..57faee59f4 100644 --- a/nvflare/private/fed/client/communicator.py +++ b/nvflare/private/fed/client/communicator.py @@ -456,7 +456,7 @@ def submit_update( timeout = self.timeout parent_fqcn = determine_parent_fqcn(self.client_config, fl_ctx) - self.logger.debug(f"submitting update to parent FQCN: {parent_fqcn}") + self.logger.info(f"submitting update to parent FQCN: {parent_fqcn}") fqcn = FQCN.join([parent_fqcn, job_id]) result = self.cell.send_request( diff --git a/nvflare/private/fed/client/utils.py b/nvflare/private/fed/client/utils.py index f18690d8cc..6270d83e2d 100644 --- a/nvflare/private/fed/client/utils.py +++ b/nvflare/private/fed/client/utils.py @@ -14,6 +14,7 @@ from typing import Optional from nvflare.apis.client import Client +from nvflare.apis.fl_constant import FLContextKey from nvflare.apis.fl_context import FLContext from nvflare.fuel.f3.cellnet.fqcn import FQCN from nvflare.private.fed.utils.identity_utils import get_parent_site_name @@ -49,6 +50,12 @@ def determine_parent_fqcn(client_config: dict, fl_ctx: FLContext) -> str: Returns: the FQCN of the parent cell """ + fox_mode = fl_ctx.get_prop(FLContextKey.FOX_MODE, False) + my_identity = fl_ctx.get_identity_name() + print(f"FOX MODE FOR {my_identity}: {fox_mode}") + if fox_mode: + return FQCN.ROOT_SERVER + parent_client_name = determine_parent_name(client_config) if parent_client_name: engine = fl_ctx.get_engine() From 6b8328e9517863856afa564616873cca214fa258 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Tue, 7 Oct 2025 15:43:42 -0400 Subject: [PATCH 031/102] use fox adaptor --- nvflare/fox/examples/np/algos/filters.py | 4 +- nvflare/fox/examples/np/algos/strategies.py | 8 +- nvflare/fox/sys/adaptor.py | 104 ++++++++++++++++++++ nvflare/fox/sys/controller.py | 73 ++++---------- nvflare/fox/sys/executor.py | 34 ++++--- 5 files changed, 153 insertions(+), 70 deletions(-) create mode 100644 nvflare/fox/sys/adaptor.py diff --git a/nvflare/fox/examples/np/algos/filters.py b/nvflare/fox/examples/np/algos/filters.py index f3e857df5f..341c87f708 100644 --- a/nvflare/fox/examples/np/algos/filters.py +++ b/nvflare/fox/examples/np/algos/filters.py @@ -51,7 +51,7 @@ def __init__(self): def filter_call(self, func_kwargs: dict, context: Context): direction = context.get_prop(ContextKey.DIRECTION) qual_func_name = context.get_prop(ContextKey.QUALIFIED_FUNC_NAME) - self.logger.debug(f"[{context.header_str()}] printing call: {func_kwargs=} {direction=} {qual_func_name=}") + self.logger.info(f"[{context.header_str()}] printing call: {func_kwargs=} {direction=} {qual_func_name=}") return func_kwargs @@ -63,5 +63,5 @@ def __init__(self): def filter_result(self, result, context: Context): direction = context.get_prop(ContextKey.DIRECTION) qual_func_name = context.get_prop(ContextKey.QUALIFIED_FUNC_NAME) - self.logger.debug(f"[{context.header_str()}] printing result: {result=} {direction=} {qual_func_name=}") + self.logger.info(f"[{context.header_str()}] printing result: {result=} {direction=} {qual_func_name=}") return result diff --git a/nvflare/fox/examples/np/algos/strategies.py b/nvflare/fox/examples/np/algos/strategies.py index 73bb3c180b..068849cb7b 100644 --- a/nvflare/fox/examples/np/algos/strategies.py +++ b/nvflare/fox/examples/np/algos/strategies.py @@ -143,6 +143,7 @@ def execute(self, context: Context): current_model = self._do_one_round(i, current_model, context) score = self._do_eval(current_model, context) self.logger.info(f"[{context.header_str()}]: eval score in round {i}: {score}") + self.logger.info(f"FINAL MODEL: {current_model}") return current_model def _do_eval(self, model, ctx: Context): @@ -161,13 +162,12 @@ def _do_one_round(self, r, current_model, ctx: Context): aggr_result=aggr_result, ).train(r, current_model) - self.logger.info( - f"[{ctx.header_str()}] round {r}: aggr result from {aggr_result.count} clients: {aggr_result.total}" - ) if aggr_result.count == 0: return None else: - return aggr_result.total / aggr_result.count + result = aggr_result.total / aggr_result.count + self.logger.info(f"[{ctx.header_str()}] round {r}: aggr result from {aggr_result.count} clients: {result}") + return result def _accept_train_result(self, result, aggr_result: _AggrResult, context: Context): self.logger.info(f"[{context.header_str()}] got train result from {context.caller} {result}") diff --git a/nvflare/fox/sys/adaptor.py b/nvflare/fox/sys/adaptor.py new file mode 100644 index 0000000000..00076acf1f --- /dev/null +++ b/nvflare/fox/sys/adaptor.py @@ -0,0 +1,104 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Dict + +from nvflare.apis.fl_context import FLContext +from nvflare.fox.api.app import App +from nvflare.fox.api.filter import CallFilter, ResultFilter + + +class FoxAdaptor: + + def __init__( + self, + collab_obj_ids: Dict[str, str] = None, + incoming_call_filters=None, + outgoing_call_filters=None, + incoming_result_filters=None, + outgoing_result_filters=None, + ): + if not collab_obj_ids: + collab_obj_ids = [] + self.collab_obj_ids = collab_obj_ids + self.incoming_call_filters = incoming_call_filters + self.outgoing_call_filters = outgoing_call_filters + self.incoming_result_filters = incoming_result_filters + self.outgoing_result_filters = outgoing_result_filters + + def process_config(self, app: App, fl_ctx: FLContext): + engine = fl_ctx.get_engine() + if self.collab_obj_ids: + for cid in self.collab_obj_ids: + obj = engine.get_component(cid) + if not obj: + return f"component {cid} does not exist" + + app.add_collab_object(cid, obj) + + err = self._parse_filters("incoming_call_filters", CallFilter, app.add_incoming_call_filters, fl_ctx) + if err: + return err + + err = self._parse_filters("outgoing_call_filters", CallFilter, app.add_outgoing_call_filters, fl_ctx) + if err: + return err + + err = self._parse_filters("incoming_result_filters", ResultFilter, app.add_incoming_result_filters, fl_ctx) + if err: + return err + + err = self._parse_filters("outgoing_result_filters", ResultFilter, app.add_outgoing_result_filters, fl_ctx) + if err: + return err + + return None + + def _parse_filters(self, name, filter_type, add_f, fl_ctx): + filters = getattr(self, name) + if not filters: + return None + + if not isinstance(filters, list): + return f"{name} must be a list but got {type(filters)}" + + for chain_dict in filters: + pattern, filters, err = self._parse_filter_chain(name, chain_dict, filter_type, fl_ctx) + if err: + return err + add_f(pattern, filters) + return None + + @staticmethod + def _parse_filter_chain(chain_name, chain_dict: dict, filter_type, fl_ctx): + if not isinstance(chain_dict, dict): + return None, None, f"element in {chain_name} must be dict but got {type(chain_dict)}" + + pattern = chain_dict.get("pattern") + if not pattern: + return None, None, f"missing 'pattern' in {chain_name}" + + filter_ids = chain_dict.get("filters") + if not filter_ids: + return None, None, f"missing 'filters' in {chain_name}" + + engine = fl_ctx.get_engine() + filters = [] + for fid in filter_ids: + f = engine.get_component(fid) + if not f: + return None, None, f"component {fid} does not exist" + if not isinstance(f, filter_type): + return None, None, f"component {fid} should be a {filter_type} but got {type(f)}" + filters.append(f) + return pattern, filters, None diff --git a/nvflare/fox/sys/controller.py b/nvflare/fox/sys/controller.py index 542c25f694..ac7df5bfac 100644 --- a/nvflare/fox/sys/controller.py +++ b/nvflare/fox/sys/controller.py @@ -24,11 +24,11 @@ from nvflare.apis.signal import Signal from nvflare.fox.api.app import ServerApp from nvflare.fox.api.constants import ContextKey, EnvType -from nvflare.fox.api.filter import CallFilter, FilterChain from nvflare.fox.api.proxy import Proxy from nvflare.fox.api.strategy import Strategy from nvflare.fuel.f3.cellnet.fqcn import FQCN +from .adaptor import FoxAdaptor from .backend import SysBackend from .constants import SYNC_TASK_NAME, SyncKey from .utils import prepare_for_remote_call @@ -45,24 +45,31 @@ def __init__(self, collab_interface: dict): self.collab_interface = collab_interface -class FoxController(Controller): +class FoxController(Controller, FoxAdaptor): def __init__( self, strategy_ids: List[str], server_app_id: str = None, collab_obj_ids: List[str] = None, + incoming_call_filters=None, outgoing_call_filters=None, - sync_task_timeout=2, + incoming_result_filters=None, + outgoing_result_filters=None, + sync_task_timeout=5, max_call_threads=100, ): Controller.__init__(self) - if not collab_obj_ids: - collab_obj_ids = [] + FoxAdaptor.__init__( + self, + collab_obj_ids=collab_obj_ids, + incoming_call_filters=incoming_call_filters, + outgoing_call_filters=outgoing_call_filters, + incoming_result_filters=incoming_result_filters, + outgoing_result_filters=outgoing_result_filters, + ) self.server_app_id = server_app_id # component name - self.outgoing_call_filters = outgoing_call_filters self.strategy_ids = strategy_ids # component names - self.server_collab_obj_ids = collab_obj_ids # component IDs self.sync_task_timeout = sync_task_timeout self.server_app = None self.client_info = {} # client name => _ClientInfo @@ -94,50 +101,10 @@ def start_controller(self, fl_ctx: FLContext): app.add_strategy(cid, strategy) - if self.server_collab_obj_ids: - for cid in self.server_collab_obj_ids: - obj = engine.get_component(cid) - if not obj: - self.system_panic(f"component {cid} does not exist", fl_ctx) - return - - app.add_collab_object(cid, obj) - - if self.outgoing_call_filters: - if not isinstance(self.outgoing_call_filters, list): - self.system_panic( - f"outgoing_call_filters must be a list but got {type(self.outgoing_call_filters)}", fl_ctx - ) - return - - for chain_dict in self.outgoing_call_filters: - if not isinstance(chain_dict, dict): - self.system_panic( - f"element in outgoing_call_filters must be dict but got {type(chain_dict)}", fl_ctx - ) - return - - pattern = chain_dict.get("pattern") - if not pattern: - self.system_panic("missing 'pattern' in outgoing_call_filters chain", fl_ctx) - return - - filter_ids = chain_dict.get("filters") - if not filter_ids: - self.system_panic("missing 'filters' in outgoing_call_filters chain", fl_ctx) - return - - filters = [] - for fid in filter_ids: - f = engine.get_component(fid) - if not f: - self.system_panic(f"component {fid} does not exist", fl_ctx) - return - if not isinstance(f, CallFilter): - self.system_panic(f"component {fid} should be a CallFilter but got {type(f)}", fl_ctx) - filters.append(f) - app.add_outgoing_call_filters(pattern, filters) - + err = self.process_config(app, fl_ctx) + if err: + self.system_panic(err, fl_ctx) + return self.server_app = app def _prepare_client_backend(self, job_id, client: ClientSite, abort_signal: Signal): @@ -204,7 +171,7 @@ def _prepare_server_proxy( target_interface=collab_interface.get(""), ) - for name in self.server_collab_obj_ids: + for name in self.collab_obj_ids: p = Proxy( app=self.server_app, target_name=f"{server_name}.{name}", @@ -222,7 +189,7 @@ def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): task = Task( name=SYNC_TASK_NAME, data=task_data, - timeout=self.sync_task_timeout, + timeout=int(self.sync_task_timeout), result_received_cb=self._process_sync_reply, ) diff --git a/nvflare/fox/sys/executor.py b/nvflare/fox/sys/executor.py index 29bc9ddce9..8cf63e4932 100644 --- a/nvflare/fox/sys/executor.py +++ b/nvflare/fox/sys/executor.py @@ -27,17 +27,34 @@ from nvflare.fox.api.proxy import Proxy from nvflare.fuel.f3.cellnet.fqcn import FQCN +from .adaptor import FoxAdaptor from .backend import SysBackend from .constants import SYNC_TASK_NAME, SyncKey from .utils import prepare_for_remote_call -class FoxExecutor(Executor): +class FoxExecutor(Executor, FoxAdaptor): - def __init__(self, client_app_id: str, collab_obj_ids: Dict[str, str] = None, max_call_threads=100): + def __init__( + self, + client_app_id: str, + collab_obj_ids: Dict[str, str] = None, + incoming_call_filters=None, + outgoing_call_filters=None, + incoming_result_filters=None, + outgoing_result_filters=None, + max_call_threads=100, + ): Executor.__init__(self) + FoxAdaptor.__init__( + self, + collab_obj_ids=collab_obj_ids, + incoming_call_filters=incoming_call_filters, + outgoing_call_filters=outgoing_call_filters, + incoming_result_filters=incoming_result_filters, + outgoing_result_filters=outgoing_result_filters, + ) self.client_app_id = client_app_id - self.collab_obj_ids = collab_obj_ids self.register_event_handler(EventType.START_RUN, self._handle_start_run) self.register_event_handler(EventType.END_RUN, self._handle_end_run) self.client_app = None @@ -63,14 +80,9 @@ def _handle_start_run(self, event_type: str, fl_ctx: FLContext): self.client_app.name = client_name self.client_app.env_type = EnvType.SYSTEM - if self.collab_obj_ids: - for name, cid in self.collab_obj_ids: - obj = engine.get_component(cid) - if not obj: - self.system_panic(f"component {cid} does not exist", fl_ctx) - return - - self.client_app.add_collab_object(name, obj) + err = self.process_config(self.client_app, fl_ctx) + if err: + self.system_panic(err, fl_ctx) def _handle_end_run(self, event_type: str, fl_ctx: FLContext): self.thread_executor.shutdown(wait=False, cancel_futures=True) From 804e1d89b5d6877524a0b6be611722f53d352dfa Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Tue, 7 Oct 2025 16:25:11 -0400 Subject: [PATCH 032/102] add all_other_clients --- nvflare/fox/api/app.py | 6 +---- nvflare/fox/api/ctx.py | 24 ++++++++++++++----- nvflare/fox/api/group.py | 33 +++++++++++++++++++++++++- nvflare/fox/examples/np/algos/swarm.py | 2 ++ nvflare/fox/sim/backend.py | 2 +- nvflare/fox/sys/utils.py | 1 - 6 files changed, 54 insertions(+), 14 deletions(-) diff --git a/nvflare/fox/api/app.py b/nvflare/fox/api/app.py index 50a32d2c46..be643b9d7b 100644 --- a/nvflare/fox/api/app.py +++ b/nvflare/fox/api/app.py @@ -229,11 +229,7 @@ def initialize(self, context: Context): init_func(**kwargs) def new_context(self, caller: str, callee: str, props: dict = None): - ctx = Context(self.env_type, caller, callee, self._abort_signal, props) - ctx.app = self - ctx.server = self.server - ctx.clients = self.clients - return ctx + return Context(self, caller, callee, self._abort_signal, props) def register_event_handler(self, event_type: str, handler, **handler_kwargs): handlers = self._event_handlers.get(event_type) diff --git a/nvflare/fox/api/ctx.py b/nvflare/fox/api/ctx.py index a96c941dd9..9ba8b56d01 100644 --- a/nvflare/fox/api/ctx.py +++ b/nvflare/fox/api/ctx.py @@ -16,25 +16,37 @@ class Context: - def __init__(self, env_type: str, caller: str, callee: str, abort_signal: Signal, props: dict = None): - if not isinstance(env_type, str): - raise ValueError(f"env_type must be str but got {type(env_type)}") - + def __init__(self, app, caller: str, callee: str, abort_signal: Signal, props: dict = None): if not isinstance(caller, str): raise ValueError(f"caller must be str but got {type(caller)}") if not isinstance(callee, str): raise ValueError(f"callee must be str but got {type(callee)}") - self.env_type = env_type self.caller = caller self.callee = callee self.abort_signal = abort_signal - self.app = None + self.app = app self.props = {} if props: self.props.update(props) + @property + def env_type(self): + return self.app.env_type + + @property + def clients(self): + return self.app.get_client_proxies() + + @property + def server(self): + return self.app.get_server_proxy() + + @property + def client_hierarchy(self): + return self.app.client_hierarchy + def set_prop(self, name: str, value): self.props[name] = value diff --git a/nvflare/fox/api/group.py b/nvflare/fox/api/group.py index 66b8c1b932..b7ec79d865 100644 --- a/nvflare/fox/api/group.py +++ b/nvflare/fox/api/group.py @@ -199,7 +199,38 @@ def all_clients( return Group( ctx.app, ctx.abort_signal, - ctx.app.get_client_proxies(), + ctx.clients, + blocking, + timeout, + optional, + secure, + min_resps, + wait_after_min_resps, + process_resp_cb, + **cb_kwargs, + ) + + +def all_other_clients( + ctx: Context, + blocking: bool = True, + timeout: float = 5.0, + optional: bool = False, + secure: bool = False, + min_resps: int = None, + wait_after_min_resps: float = None, + process_resp_cb=None, + **cb_kwargs, +): + candidates = ctx.clients + me = ctx.app.get_my_site() + if me in candidates: + candidates.remove(me) + + return Group( + ctx.app, + ctx.abort_signal, + candidates, blocking, timeout, optional, diff --git a/nvflare/fox/examples/np/algos/swarm.py b/nvflare/fox/examples/np/algos/swarm.py index a9b6f7836a..2e617ba117 100644 --- a/nvflare/fox/examples/np/algos/swarm.py +++ b/nvflare/fox/examples/np/algos/swarm.py @@ -67,6 +67,7 @@ def train(self, weights, current_round, context: Context): def sag(self, model, current_round, ctx: Context): results = all_clients(ctx, blocking=True).train(model, current_round) + # results = all_other_clients(ctx, blocking=True).train(model, current_round) results = list(results.values()) total = results[0] for i in range(1, len(results)): @@ -94,6 +95,7 @@ def swarm_learn(self, num_rounds, model, current_round, context: Context): # determine next client next_round = current_round + 1 next_client_idx = random.randint(0, len(self.clients) - 1) + self.logger.debug(f"chose aggr client for round {next_round}: {next_client_idx}") next_client = self.clients[next_client_idx] next_client.swarm_learn(num_rounds, new_model, next_round, _blocking=False) diff --git a/nvflare/fox/sim/backend.py b/nvflare/fox/sim/backend.py index f5f41b20be..452a1945bb 100644 --- a/nvflare/fox/sim/backend.py +++ b/nvflare/fox/sim/backend.py @@ -97,7 +97,7 @@ def _preprocess(self, target_name, func_name, func, kwargs): raise RuntimeError(f"cannot find interface for func '{func_name}' of object {self.target_obj_name}") check_call_args(func_name, func_itf, [], kwargs) - self.logger.debug(f"[{my_ctx.header_str()}] received kwargs is good: {kwargs}") + self.logger.debug(f"[{my_ctx.header_str()}] received kwargs is good for '{func_name}': {kwargs}") kwargs[CollabMethodArgName.CONTEXT] = my_ctx adjust_kwargs(func, kwargs) diff --git a/nvflare/fox/sys/utils.py b/nvflare/fox/sys/utils.py index 62a2000695..be42cccdc7 100644 --- a/nvflare/fox/sys/utils.py +++ b/nvflare/fox/sys/utils.py @@ -49,7 +49,6 @@ def _preprocess(app: App, caller, target_obj_name, target_name, func_name, func, raise RuntimeError(f"cannot find interface for func '{func_name}' of object {target_obj_name}") check_call_args(func_name, func_itf, args, kwargs) - print(f"received kwargs is good: {kwargs}") kwargs[CollabMethodArgName.CONTEXT] = ctx adjust_kwargs(func, kwargs) From 761161f9c1106eaa8fb57272fef5165e259de6b1 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Thu, 9 Oct 2025 11:42:47 -0400 Subject: [PATCH 033/102] support workspace --- nvflare/fox/api/app.py | 8 +++- nvflare/fox/api/ctx.py | 4 ++ nvflare/fox/api/run_server.py | 41 ++++++++++++++++++++ nvflare/fox/api/workspace.py | 36 ++++++++++++++++++ nvflare/fox/examples/np/algos/strategies.py | 7 +++- nvflare/fox/examples/np/algos/utils.py | 4 ++ nvflare/fox/examples/np/cyclic.py | 2 + nvflare/fox/examples/np/cyclic_avg.py | 5 ++- nvflare/fox/examples/np/fed_avg_h.py | 8 +++- nvflare/fox/examples/np/fed_avg_intime.py | 2 + nvflare/fox/examples/np/fed_avg_para.py | 2 + nvflare/fox/examples/np/fed_avg_seq.py | 2 + nvflare/fox/examples/np/swarm.py | 2 + nvflare/fox/sim/simulator.py | 35 ++++++++--------- nvflare/fox/sim/ws.py | 42 +++++++++++++++++++++ nvflare/fox/sys/controller.py | 26 +++---------- nvflare/fox/sys/executor.py | 4 +- nvflare/fox/sys/ws.py | 34 +++++++++++++++++ 18 files changed, 220 insertions(+), 44 deletions(-) create mode 100644 nvflare/fox/api/run_server.py create mode 100644 nvflare/fox/api/workspace.py create mode 100644 nvflare/fox/sim/ws.py create mode 100644 nvflare/fox/sys/ws.py diff --git a/nvflare/fox/api/app.py b/nvflare/fox/api/app.py index be643b9d7b..29631d5bf6 100644 --- a/nvflare/fox/api/app.py +++ b/nvflare/fox/api/app.py @@ -26,6 +26,7 @@ from .proxy import Proxy from .strategy import Strategy from .utils import check_context_support, get_collab_object_name +from .workspace import Workspace class App: @@ -47,8 +48,12 @@ def __init__(self): self._incoming_result_filter_chains = [] self._outgoing_result_filter_chains = [] self._collab_interface = {"": get_object_collab_interface(self)} + self._workspace = None self.logger = get_obj_logger(self) + def get_workspace(self): + return self._workspace + def get_server_proxy(self): return self.server @@ -162,7 +167,8 @@ def add_collab_object(self, name: str, obj): def get_collab_objects(self): return self._collab_objs - def setup(self, server: Proxy, clients: List[Proxy], abort_signal): + def setup(self, workspace: Workspace, server: Proxy, clients: List[Proxy], abort_signal): + self._workspace = workspace self.server = server self._abort_signal = abort_signal diff --git a/nvflare/fox/api/ctx.py b/nvflare/fox/api/ctx.py index 9ba8b56d01..77ed9b0728 100644 --- a/nvflare/fox/api/ctx.py +++ b/nvflare/fox/api/ctx.py @@ -47,6 +47,10 @@ def server(self): def client_hierarchy(self): return self.app.client_hierarchy + @property + def workspace(self): + return self.app.get_workspace() + def set_prop(self, name: str, value): self.props[name] = value diff --git a/nvflare/fox/api/run_server.py b/nvflare/fox/api/run_server.py new file mode 100644 index 0000000000..937f10dfd8 --- /dev/null +++ b/nvflare/fox/api/run_server.py @@ -0,0 +1,41 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import traceback + +from .app import ServerApp +from .constants import ContextKey + + +def run_server(server_app: ServerApp, logger): + server_ctx = server_app.new_context(caller=server_app.name, callee=server_app.name) + logger.info("initializing server app") + server_app.initialize(server_ctx) + + if not server_app.strategies: + raise RuntimeError("server app does not have any strategies!") + + result = None + for idx, strategy in enumerate(server_app.strategies): + if server_ctx.is_aborted(): + break + + try: + logger.info(f"Running Strategy #{idx + 1} - {type(strategy).__name__}") + server_app.current_strategy = strategy + result = strategy.execute(context=server_ctx) + server_ctx.set_prop(ContextKey.INPUT, result) + except: + traceback.print_exc() + break + return result diff --git a/nvflare/fox/api/workspace.py b/nvflare/fox/api/workspace.py new file mode 100644 index 0000000000..caad378850 --- /dev/null +++ b/nvflare/fox/api/workspace.py @@ -0,0 +1,36 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from abc import ABC, abstractmethod + + +class Workspace(ABC): + + @abstractmethod + def get_root_dir(self) -> str: + pass + + @abstractmethod + def get_work_dir(self) -> str: + pass + + def get_subdir(self, name: str, create: bool = True) -> str: + p = f"{self.get_work_dir()}/{name}" + if not os.path.exists(p) and create: + os.makedirs(p, exist_ok=True) + return p + + @abstractmethod + def get_experiment_dir(self) -> str: + pass diff --git a/nvflare/fox/examples/np/algos/strategies.py b/nvflare/fox/examples/np/algos/strategies.py index 068849cb7b..8a65ef2892 100644 --- a/nvflare/fox/examples/np/algos/strategies.py +++ b/nvflare/fox/examples/np/algos/strategies.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os import random import numpy as np @@ -21,7 +22,7 @@ from nvflare.fox.api.strategy import Strategy from nvflare.fuel.utils.log_utils import get_obj_logger -from .utils import parse_array_def +from .utils import parse_array_def, save_np_model class NPFedAvgSequential(Strategy): @@ -38,6 +39,10 @@ def execute(self, context: Context): current_model = context.get_prop(ContextKey.INPUT, self.initial_model) for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model, context) + + # save model to work dir + file_name = os.path.join(context.workspace.get_work_dir(), "model.npy") + save_np_model(current_model, file_name) return current_model def _do_one_round(self, r, current_model, context: Context): diff --git a/nvflare/fox/examples/np/algos/utils.py b/nvflare/fox/examples/np/algos/utils.py index 04f6b87424..1a164c8530 100644 --- a/nvflare/fox/examples/np/algos/utils.py +++ b/nvflare/fox/examples/np/algos/utils.py @@ -26,3 +26,7 @@ def parse_array_def(array_def): return np.array(array_def, dtype=np.float32) else: raise ValueError(f"unsupported array def: {array_def}") + + +def save_np_model(model: np.ndarray, file_name: str): + np.save(file_name, model) diff --git a/nvflare/fox/examples/np/cyclic.py b/nvflare/fox/examples/np/cyclic.py index 182f1345b8..f02c5d47ff 100644 --- a/nvflare/fox/examples/np/cyclic.py +++ b/nvflare/fox/examples/np/cyclic.py @@ -24,6 +24,8 @@ def main(): simple_logging(logging.DEBUG) simulator = Simulator( + root_dir="/tmp/fox", + experiment_name="cyclic", server_app=ServerApp( strategy_name="cyclic", strategy=NPCyclic(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2) ), diff --git a/nvflare/fox/examples/np/cyclic_avg.py b/nvflare/fox/examples/np/cyclic_avg.py index ae572ec512..bb3e97c7d6 100644 --- a/nvflare/fox/examples/np/cyclic_avg.py +++ b/nvflare/fox/examples/np/cyclic_avg.py @@ -22,13 +22,16 @@ def main(): simple_logging(logging.DEBUG) + exp_name = "cyclic_avg" server_app = ServerApp( - strategy_name="cyclic", strategy=NPCyclic(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2) + strategy_name=exp_name, strategy=NPCyclic(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2) ) server_app.add_strategy("fed_avg_parallel", NPFedAvgParallel(initial_model=None, num_rounds=2)) simulator = Simulator( + root_dir="/tmp/fox", + experiment_name=exp_name, server_app=server_app, client_app=NPTrainer(delta=1.0), num_clients=2, diff --git a/nvflare/fox/examples/np/fed_avg_h.py b/nvflare/fox/examples/np/fed_avg_h.py index be12c9619e..234e9c1c9d 100644 --- a/nvflare/fox/examples/np/fed_avg_h.py +++ b/nvflare/fox/examples/np/fed_avg_h.py @@ -30,7 +30,13 @@ def main(): ) server_app.add_collab_object("metric_receiver", MetricReceiver()) - simulator = Simulator(server_app=server_app, client_app=NPHierarchicalTrainer(delta=1.0), num_clients=(3, 2)) + simulator = Simulator( + root_dir="/tmp/fox", + experiment_name="fedavg_h", + server_app=server_app, + client_app=NPHierarchicalTrainer(delta=1.0), + num_clients=(3, 2), + ) simulator.run() diff --git a/nvflare/fox/examples/np/fed_avg_intime.py b/nvflare/fox/examples/np/fed_avg_intime.py index cfa769a444..c8d5fed884 100644 --- a/nvflare/fox/examples/np/fed_avg_intime.py +++ b/nvflare/fox/examples/np/fed_avg_intime.py @@ -39,6 +39,8 @@ def main(): client_app.add_outgoing_result_filters("*.train", [PrintResult()]) simulator = Simulator( + root_dir="/tmp/fox", + experiment_name="fedavg_intime", server_app=server_app, client_app=client_app, num_clients=2, diff --git a/nvflare/fox/examples/np/fed_avg_para.py b/nvflare/fox/examples/np/fed_avg_para.py index 79304a3d9a..2ae7413c6a 100644 --- a/nvflare/fox/examples/np/fed_avg_para.py +++ b/nvflare/fox/examples/np/fed_avg_para.py @@ -31,6 +31,8 @@ def main(): server_app.add_collab_object("metric_receiver", MetricReceiver()) simulator = Simulator( + root_dir="/tmp/fox", + experiment_name="fedavg_para", server_app=server_app, client_app=NPTrainer(delta=1.0), num_clients=10, diff --git a/nvflare/fox/examples/np/fed_avg_seq.py b/nvflare/fox/examples/np/fed_avg_seq.py index 2430096851..4d384e3a34 100644 --- a/nvflare/fox/examples/np/fed_avg_seq.py +++ b/nvflare/fox/examples/np/fed_avg_seq.py @@ -34,6 +34,8 @@ def main(): server_app.add_collab_object("metric_receiver", MetricReceiver()) simulator = Simulator( + root_dir="/tmp/fox", + experiment_name="fedavg_seq", server_app=server_app, client_app=TrainerFactory(delta=1.0), num_clients=2, diff --git a/nvflare/fox/examples/np/swarm.py b/nvflare/fox/examples/np/swarm.py index effc563c8f..bfa1f28b43 100644 --- a/nvflare/fox/examples/np/swarm.py +++ b/nvflare/fox/examples/np/swarm.py @@ -28,6 +28,8 @@ def main(): client_app = NPSwarmClient(delta=1.0) simulator = Simulator( + root_dir="/tmp/fox", + experiment_name="swarm", server_app=server_app, client_app=client_app, num_clients=3, diff --git a/nvflare/fox/sim/simulator.py b/nvflare/fox/sim/simulator.py index 72593bd2d5..5e8a436374 100644 --- a/nvflare/fox/sim/simulator.py +++ b/nvflare/fox/sim/simulator.py @@ -12,15 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. import copy +import uuid from concurrent.futures import ThreadPoolExecutor from typing import Tuple, Union from nvflare.apis.signal import Signal from nvflare.fox.api.app import App, ClientApp, ClientAppFactory, ServerApp -from nvflare.fox.api.constants import ContextKey, EnvType +from nvflare.fox.api.constants import EnvType from nvflare.fox.api.dec import get_object_collab_interface from nvflare.fox.api.proxy import Proxy +from nvflare.fox.api.run_server import run_server from nvflare.fox.sim.backend import SimBackend +from nvflare.fox.sim.ws import SimWorkspace from nvflare.fuel.utils.log_utils import get_obj_logger @@ -74,6 +77,8 @@ def _prepare_proxies(self, for_app: App, server_app: App, client_apps: dict, bac def __init__( self, + root_dir: str, + experiment_name: str, server_app: ServerApp, client_app: Union[ClientAppFactory, ClientApp], max_workers: int = 100, @@ -113,7 +118,7 @@ def __init__( if height <= 0 or num_children_per_parent <= 0: raise ValueError(f"num_clients must contain positive ints but got {num_clients}") - print(f"creating clients {height} x {num_children_per_parent}") + self.logger.info(f"creating clients {height} x {num_children_per_parent}") client_apps = self._build_hierarchical_clients(height, num_children_per_parent) else: raise ValueError(f"num_clients must be an int or tuple(int, int) but got {type(num_clients)}") @@ -125,15 +130,19 @@ def __init__( for name, app in client_apps.items(): backends[name] = self._prepare_app_backends(app) + exp_id = str(uuid.uuid4()) + for name, app in client_apps.items(): server_proxy, client_proxies = self._prepare_proxies(app, server_app, client_apps, backends) - app.setup(server_proxy, client_proxies, self.abort_signal) + ws = SimWorkspace(root_dir=root_dir, experiment_name=experiment_name, site_name=name, exp_id=exp_id) + app.setup(ws, server_proxy, client_proxies, self.abort_signal) # prepare server server_proxy, client_proxies = self._prepare_proxies(server_app, server_app, client_apps, backends) - server_app.setup(server_proxy, client_proxies, self.abort_signal) - + ws = SimWorkspace(root_dir=root_dir, experiment_name=experiment_name, site_name=server_app.name, exp_id=exp_id) + server_app.setup(ws, server_proxy, client_proxies, self.abort_signal) self.client_apps = client_apps + self.exp_dir = ws.get_experiment_dir() def _build_hierarchical_clients(self, height: int, num_children_per_parent: int): client_apps = {} @@ -168,25 +177,13 @@ def run(self): self.abort_signal.trigger(True) finally: self.thread_executor.shutdown(wait=False, cancel_futures=True) + self.logger.info(f"Experiment results are in {self.exp_dir}") def _try_run(self): # initialize all apps - server_ctx = self.server_app.new_context(caller=self.server_app.name, callee=self.server_app.name) - self.logger.info("initializing server app") - self.server_app.initialize(server_ctx) - for n, app in self.client_apps.items(): self.logger.info(f"initializing client app for {n}") app.initialize(app.new_context(n, n)) # run the server - if not self.server_app.strategies: - raise RuntimeError("server app does not have any strategies!") - - result = None - for idx, strategy in enumerate(self.server_app.strategies): - self.logger.info(f"Running Strategy #{idx+1} - {type(strategy).__name__}") - self.server_app.current_strategy = strategy - result = strategy.execute(context=server_ctx) - server_ctx.set_prop(ContextKey.INPUT, result) - return result + return run_server(self.server_app, self.logger) diff --git a/nvflare/fox/sim/ws.py b/nvflare/fox/sim/ws.py new file mode 100644 index 0000000000..6a7bedf41a --- /dev/null +++ b/nvflare/fox/sim/ws.py @@ -0,0 +1,42 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os.path + +from nvflare.fox.api.workspace import Workspace + + +class SimWorkspace(Workspace): + + def __init__(self, root_dir: str, experiment_name: str, exp_id: str, site_name: str): + if not isinstance(root_dir, str): + raise ValueError(f"root_dir must be str but got {type(root_dir)}") + + if not isinstance(experiment_name, str): + raise ValueError(f"experiment_name must be str but got {type(experiment_name)}") + + self.root_dir = root_dir + self.site_name = site_name + self.exp_name = experiment_name + self.exp_dir = os.path.join(root_dir, experiment_name, exp_id) + self.work_dir = os.path.join(self.exp_dir, site_name) + os.makedirs(self.work_dir, exist_ok=True) + + def get_root_dir(self) -> str: + return self.root_dir + + def get_work_dir(self) -> str: + return self.work_dir + + def get_experiment_dir(self) -> str: + return self.exp_dir diff --git a/nvflare/fox/sys/controller.py b/nvflare/fox/sys/controller.py index ac7df5bfac..ba504c2e12 100644 --- a/nvflare/fox/sys/controller.py +++ b/nvflare/fox/sys/controller.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. import time -import traceback from concurrent.futures import ThreadPoolExecutor from typing import List @@ -23,8 +22,9 @@ from nvflare.apis.shareable import ReturnCode, Shareable from nvflare.apis.signal import Signal from nvflare.fox.api.app import ServerApp -from nvflare.fox.api.constants import ContextKey, EnvType +from nvflare.fox.api.constants import EnvType from nvflare.fox.api.proxy import Proxy +from nvflare.fox.api.run_server import run_server from nvflare.fox.api.strategy import Strategy from nvflare.fuel.f3.cellnet.fqcn import FQCN @@ -32,6 +32,7 @@ from .backend import SysBackend from .constants import SYNC_TASK_NAME, SyncKey from .utils import prepare_for_remote_call +from .ws import SysWorkspace class _ClientInfo: @@ -238,24 +239,9 @@ def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): assert isinstance(info, _ClientInfo) client_proxies.append(self._prepare_client_proxy(job_id, c, info.collab_interface, abort_signal)) - self.server_app.setup(server_proxy, client_proxies, abort_signal) - - server_ctx = self.server_app.new_context(caller=self.server_app.name, callee=self.server_app.name) - self.log_info(fl_ctx, "initializing server app") - self.server_app.initialize(server_ctx) - - for idx, strategy in enumerate(self.server_app.strategies): - if abort_signal.triggered: - break - - try: - self.log_info(fl_ctx, f"Running Strategy #{idx + 1} - {type(strategy).__name__}") - self.server_app.current_strategy = strategy - result = strategy.execute(context=server_ctx) - server_ctx.set_prop(ContextKey.INPUT, result) - except: - traceback.print_exc() - break + ws = SysWorkspace(fl_ctx) + self.server_app.setup(ws, server_proxy, client_proxies, abort_signal) + run_server(self.server_app, self.logger) def _process_sync_reply(self, client_task: ClientTask, fl_ctx: FLContext): result = client_task.result diff --git a/nvflare/fox/sys/executor.py b/nvflare/fox/sys/executor.py index 8cf63e4932..f77b78e6a5 100644 --- a/nvflare/fox/sys/executor.py +++ b/nvflare/fox/sys/executor.py @@ -31,6 +31,7 @@ from .backend import SysBackend from .constants import SYNC_TASK_NAME, SyncKey from .utils import prepare_for_remote_call +from .ws import SysWorkspace class FoxExecutor(Executor, FoxAdaptor): @@ -176,7 +177,8 @@ def execute(self, task_name: str, shareable: Shareable, fl_ctx: FLContext, abort p = self._prepare_client_proxy(job_id, cell, c, abort_signal, client_collab_interface) client_proxies.append(p) - self.client_app.setup(server_proxy, client_proxies, abort_signal) + ws = SysWorkspace(fl_ctx) + self.client_app.setup(ws, server_proxy, client_proxies, abort_signal) ctx = self.client_app.new_context(self.client_app.name, self.client_app.name) self.client_app.initialize(ctx) diff --git a/nvflare/fox/sys/ws.py b/nvflare/fox/sys/ws.py new file mode 100644 index 0000000000..f5363d0fdf --- /dev/null +++ b/nvflare/fox/sys/ws.py @@ -0,0 +1,34 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nvflare.apis.fl_context import FLContext +from nvflare.apis.workspace import Workspace as FlareWorkspace +from nvflare.fox.api.workspace import Workspace + + +class SysWorkspace(Workspace): + + def __init__(self, fl_ctx: FLContext): + ws_obj = fl_ctx.get_workspace() + assert isinstance(ws_obj, FlareWorkspace) + self.flare_ws = ws_obj + self.job_id = fl_ctx.get_job_id() + + def get_root_dir(self) -> str: + return self.flare_ws.get_root_dir() + + def get_work_dir(self) -> str: + return self.flare_ws.get_app_dir(self.job_id) + + def get_experiment_dir(self) -> str: + return self.get_work_dir() From b07ca343395367a4edfed077e52f3edb1b80c08a Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Thu, 9 Oct 2025 15:28:52 -0400 Subject: [PATCH 034/102] support recipe --- nvflare/fox/api/app.py | 29 +++- nvflare/fox/api/run_server.py | 4 +- nvflare/fox/examples/np/algos/client.py | 9 +- nvflare/fox/examples/np/algos/filters.py | 1 + nvflare/fox/examples/np/algos/strategies.py | 23 ++- nvflare/fox/examples/np/algos/swarm.py | 5 +- nvflare/fox/examples/np/fed_avg_intime.py | 2 + nvflare/fox/examples/np/fed_avg_seq.py | 4 +- .../fox/examples/np/recipe_fed_avg_intime.py | 54 ++++++ nvflare/fox/examples/np/recipe_swarm.py | 41 +++++ nvflare/fox/examples/np/swarm.py | 2 +- nvflare/fox/sim/simulator.py | 16 +- nvflare/fox/sys/adaptor.py | 6 +- nvflare/fox/sys/controller.py | 2 + nvflare/fox/sys/executor.py | 20 ++- nvflare/fox/sys/recipe.py | 164 ++++++++++++++++++ 16 files changed, 337 insertions(+), 45 deletions(-) create mode 100644 nvflare/fox/examples/np/recipe_fed_avg_intime.py create mode 100644 nvflare/fox/examples/np/recipe_swarm.py create mode 100644 nvflare/fox/sys/recipe.py diff --git a/nvflare/fox/api/app.py b/nvflare/fox/api/app.py index 29631d5bf6..79c0201b98 100644 --- a/nvflare/fox/api/app.py +++ b/nvflare/fox/api/app.py @@ -13,7 +13,6 @@ # limitations under the License. import copy import fnmatch -from abc import ABC, abstractmethod from typing import List from nvflare.fuel.utils.log_utils import get_obj_logger @@ -79,15 +78,27 @@ def _add_filters(pattern: str, filters, to_list: list, filter_type): def add_incoming_call_filters(self, pattern: str, filters: List[CallFilter]): self._add_filters(pattern, filters, self._incoming_call_filter_chains, CallFilter) + def get_incoming_call_filters(self): + return self._incoming_call_filter_chains + def add_outgoing_call_filters(self, pattern: str, filters: List[CallFilter]): self._add_filters(pattern, filters, self._outgoing_call_filter_chains, CallFilter) + def get_outgoing_call_filters(self): + return self._outgoing_call_filter_chains + def add_incoming_result_filters(self, pattern: str, filters: List[ResultFilter]): self._add_filters(pattern, filters, self._incoming_result_filter_chains, ResultFilter) + def get_incoming_result_filters(self): + return self._incoming_result_filter_chains + def add_outgoing_result_filters(self, pattern: str, filters: List[ResultFilter]): self._add_filters(pattern, filters, self._outgoing_result_filter_chains, ResultFilter) + def get_outgoing_result_filters(self): + return self._outgoing_result_filter_chains + @staticmethod def _find_filter_chain(chains: List[FilterChain], target_name: str, func_name: str, ctx: Context): """ @@ -150,6 +161,13 @@ def set_prop(self, name: str, value): def get_prop(self, name: str, default=None): return self._props.get(name, default) + def get_props(self): + return self._props + + def update_props(self, props: dict): + if isinstance(props, dict): + self._props.update(props) + def get_default_collab_object(self): return None @@ -293,7 +311,7 @@ def __init__(self, strategy_name: str = "strategy", strategy: Strategy = None): def add_strategy(self, strategy_name: str, strategy): if not isinstance(strategy, Strategy): raise ValueError(f"strategy must be Controller but got {type(strategy)}") - self.strategies.append(strategy) + self.strategies.append((strategy_name, strategy)) self.add_collab_object(strategy_name, strategy) def get_default_collab_object(self): @@ -327,10 +345,3 @@ def has_children(self): return True else: return False - - -class ClientAppFactory(ABC): - - @abstractmethod - def make_client_app(self, name: str) -> ClientApp: - pass diff --git a/nvflare/fox/api/run_server.py b/nvflare/fox/api/run_server.py index 937f10dfd8..04b0593dd3 100644 --- a/nvflare/fox/api/run_server.py +++ b/nvflare/fox/api/run_server.py @@ -26,12 +26,12 @@ def run_server(server_app: ServerApp, logger): raise RuntimeError("server app does not have any strategies!") result = None - for idx, strategy in enumerate(server_app.strategies): + for name, strategy in server_app.strategies: if server_ctx.is_aborted(): break try: - logger.info(f"Running Strategy #{idx + 1} - {type(strategy).__name__}") + logger.info(f"Running Strategy {name} - {type(strategy).__name__}") server_app.current_strategy = strategy result = strategy.execute(context=server_ctx) server_ctx.set_prop(ContextKey.INPUT, result) diff --git a/nvflare/fox/examples/np/algos/client.py b/nvflare/fox/examples/np/algos/client.py index 82be10cdf7..1ae02ceb26 100644 --- a/nvflare/fox/examples/np/algos/client.py +++ b/nvflare/fox/examples/np/algos/client.py @@ -13,7 +13,7 @@ # limitations under the License. import random -from nvflare.fox.api.app import ClientApp, ClientAppFactory +from nvflare.fox.api.app import ClientApp from nvflare.fox.api.ctx import Context from nvflare.fox.api.dec import collab from nvflare.fox.api.group import all_children @@ -91,10 +91,13 @@ def evaluate(self, model, context: Context): return random.random() -class TrainerFactory(ClientAppFactory): +class NPTrainerMaker(ClientApp): def __init__(self, delta): + ClientApp.__init__(self) self.delta = delta def make_client_app(self, name: str) -> ClientApp: - return NPTrainer(self.delta) + app = NPTrainer(self.delta) + app.name = name + return app diff --git a/nvflare/fox/examples/np/algos/filters.py b/nvflare/fox/examples/np/algos/filters.py index 341c87f708..e0e5b2c9a3 100644 --- a/nvflare/fox/examples/np/algos/filters.py +++ b/nvflare/fox/examples/np/algos/filters.py @@ -40,6 +40,7 @@ def filter_call(self, func_kwargs: dict, context: Context): self.logger.debug(f"[{context.header_str()}] adding noise {noise}") weights += noise func_kwargs[weights_key] = weights + self.logger.info(f"[{context.header_str()}] weights after adding noise {noise}: {weights}") return func_kwargs diff --git a/nvflare/fox/examples/np/algos/strategies.py b/nvflare/fox/examples/np/algos/strategies.py index 8a65ef2892..464f6a26e0 100644 --- a/nvflare/fox/examples/np/algos/strategies.py +++ b/nvflare/fox/examples/np/algos/strategies.py @@ -31,12 +31,13 @@ def __init__(self, initial_model, num_rounds=10): Strategy.__init__(self) self.name = "NPFedAvgSequential" self.num_rounds = num_rounds - self.initial_model = parse_array_def(initial_model) + self.initial_model = initial_model # need to remember init for job API to work! + self._initial_model = parse_array_def(initial_model) self.logger = get_obj_logger(self) def execute(self, context: Context): self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") - current_model = context.get_prop(ContextKey.INPUT, self.initial_model) + current_model = context.get_prop(ContextKey.INPUT, self._initial_model) for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model, context) @@ -60,13 +61,14 @@ class NPFedAvgParallel(Strategy): def __init__(self, initial_model, num_rounds=10): self.num_rounds = num_rounds - self.initial_model = parse_array_def(initial_model) + self.initial_model = initial_model + self._initial_model = parse_array_def(initial_model) self.name = "NPFedAvgParallel" self.logger = get_obj_logger(self) def execute(self, context: Context): self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") - current_model = context.get_prop(ContextKey.INPUT, self.initial_model) + current_model = context.get_prop(ContextKey.INPUT, self._initial_model) for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model, context) score = self._do_eval(current_model, context) @@ -94,7 +96,8 @@ class NPHierarchicalFedAvg(Strategy): def __init__(self, initial_model, num_rounds=10): self.num_rounds = num_rounds - self.initial_model = parse_array_def(initial_model) + self.initial_model = initial_model + self._initial_model = parse_array_def(initial_model) self.name = self.__class__.__name__ self.logger = get_obj_logger(self) @@ -136,14 +139,15 @@ class NPFedAvgInTime(Strategy): def __init__(self, initial_model, num_rounds=10, timeout=2.0): self.num_rounds = num_rounds - self.initial_model = parse_array_def(initial_model) + self.initial_model = initial_model self.timeout = timeout self.name = "NPFedAvgInTime" self.logger = get_obj_logger(self) + self._init_model = parse_array_def(initial_model) def execute(self, context: Context): self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") - current_model = context.get_prop(ContextKey.INPUT, self.initial_model) + current_model = context.get_prop(ContextKey.INPUT, self._init_model) for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model, context) score = self._do_eval(current_model, context) @@ -185,11 +189,12 @@ class NPCyclic(Strategy): def __init__(self, initial_model, num_rounds=(2, 3)): self.num_rounds = num_rounds - self.initial_model = parse_array_def(initial_model) + self.initial_model = initial_model + self._initial_model = parse_array_def(initial_model) self.logger = get_obj_logger(self) def execute(self, context: Context): - current_model = context.get_prop(ContextKey.INPUT, self.initial_model) + current_model = context.get_prop(ContextKey.INPUT, self._initial_model) for current_round in range(self.num_rounds): current_model = self._do_one_round(current_round, current_model, context) self.logger.info(f"[{context.header_str()}] final result: {current_model}") diff --git a/nvflare/fox/examples/np/algos/swarm.py b/nvflare/fox/examples/np/algos/swarm.py index 2e617ba117..5e376655ea 100644 --- a/nvflare/fox/examples/np/algos/swarm.py +++ b/nvflare/fox/examples/np/algos/swarm.py @@ -28,7 +28,8 @@ class NPSwarm(Strategy): def __init__(self, initial_model, num_rounds=10): self.num_rounds = num_rounds - self.initial_model = parse_array_def(initial_model) + self.initial_model = initial_model + self._initial_model = parse_array_def(initial_model) self.waiter = threading.Event() self.logger = get_obj_logger(self) @@ -38,7 +39,7 @@ def execute(self, context: Context): # randomly pick a client to start start_client_idx = random.randint(0, len(context.clients) - 1) start_client = context.clients[start_client_idx] - start_client.start(self.num_rounds, self.initial_model) + start_client.start(self.num_rounds, self._initial_model) while not context.is_aborted(): if self.waiter.wait(timeout=0.5): break diff --git a/nvflare/fox/examples/np/fed_avg_intime.py b/nvflare/fox/examples/np/fed_avg_intime.py index c8d5fed884..89e22067d6 100644 --- a/nvflare/fox/examples/np/fed_avg_intime.py +++ b/nvflare/fox/examples/np/fed_avg_intime.py @@ -33,10 +33,12 @@ def main(): server_app.add_collab_object("metric_receiver", MetricReceiver()) server_app.add_outgoing_call_filters("*.train", [AddNoiseToModel()]) server_app.add_incoming_result_filters("*.train", [PrintResult()]) + server_app.set_prop("default_timeout", 5.0) client_app = NPTrainer(delta=1.0) client_app.add_incoming_call_filters("*.train", [PrintCall()]) client_app.add_outgoing_result_filters("*.train", [PrintResult()]) + server_app.set_prop("default_timeout", 8.0) simulator = Simulator( root_dir="/tmp/fox", diff --git a/nvflare/fox/examples/np/fed_avg_seq.py b/nvflare/fox/examples/np/fed_avg_seq.py index 4d384e3a34..63745a6c51 100644 --- a/nvflare/fox/examples/np/fed_avg_seq.py +++ b/nvflare/fox/examples/np/fed_avg_seq.py @@ -15,7 +15,7 @@ from nvflare.fox.api.app import ServerApp from nvflare.fox.api.utils import simple_logging -from nvflare.fox.examples.np.algos.client import TrainerFactory +from nvflare.fox.examples.np.algos.client import NPTrainerMaker from nvflare.fox.examples.np.algos.strategies import NPFedAvgSequential from nvflare.fox.examples.np.algos.widgets import MetricReceiver from nvflare.fox.sim.simulator import Simulator @@ -37,7 +37,7 @@ def main(): root_dir="/tmp/fox", experiment_name="fedavg_seq", server_app=server_app, - client_app=TrainerFactory(delta=1.0), + client_app=NPTrainerMaker(delta=1.0), num_clients=2, ) diff --git a/nvflare/fox/examples/np/recipe_fed_avg_intime.py b/nvflare/fox/examples/np/recipe_fed_avg_intime.py new file mode 100644 index 0000000000..12fdf3591e --- /dev/null +++ b/nvflare/fox/examples/np/recipe_fed_avg_intime.py @@ -0,0 +1,54 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from nvflare.fox.api.app import ServerApp +from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples.np.algos.client import NPTrainer +from nvflare.fox.examples.np.algos.filters import AddNoiseToModel, PrintCall, PrintResult +from nvflare.fox.examples.np.algos.strategies import NPFedAvgInTime +from nvflare.fox.examples.np.algos.widgets import MetricReceiver +from nvflare.fox.sys.recipe import FoxRecipe + +JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" + + +def main(): + simple_logging(logging.DEBUG) + + server_app = ServerApp( + strategy_name="fed_avg_in_time", + strategy=NPFedAvgInTime(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), + ) + + server_app.add_collab_object("metric_receiver", MetricReceiver()) + server_app.add_outgoing_call_filters("*.train", [AddNoiseToModel()]) + server_app.add_incoming_result_filters("*.train", [PrintResult()]) + server_app.set_prop("default_timeout", 5.0) + + client_app = NPTrainer(delta=1.0) + client_app.add_incoming_call_filters("*.train", [PrintCall()]) + client_app.add_outgoing_result_filters("*.train", [PrintResult()]) + client_app.set_prop("default_timeout", 8.0) + + recipe = FoxRecipe( + job_name="fedavg_intime", + server_app=server_app, + client_app=client_app, + ) + recipe.export(JOB_ROOT_DIR) + + +if __name__ == "__main__": + main() diff --git a/nvflare/fox/examples/np/recipe_swarm.py b/nvflare/fox/examples/np/recipe_swarm.py new file mode 100644 index 0000000000..eae4b2fccb --- /dev/null +++ b/nvflare/fox/examples/np/recipe_swarm.py @@ -0,0 +1,41 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from nvflare.fox.api.app import ServerApp +from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples.np.algos.swarm import NPSwarm, NPSwarmClient +from nvflare.fox.sys.recipe import FoxRecipe + +JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" + + +def main(): + simple_logging(logging.DEBUG) + + server_app = ServerApp( + strategy_name="swarm", strategy=NPSwarm(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=5) + ) + client_app = NPSwarmClient(delta=1.0) + + recipe = FoxRecipe( + job_name="recipe_swarm", + server_app=server_app, + client_app=client_app, + ) + recipe.export(JOB_ROOT_DIR) + + +if __name__ == "__main__": + main() diff --git a/nvflare/fox/examples/np/swarm.py b/nvflare/fox/examples/np/swarm.py index bfa1f28b43..363d3f38df 100644 --- a/nvflare/fox/examples/np/swarm.py +++ b/nvflare/fox/examples/np/swarm.py @@ -23,7 +23,7 @@ def main(): simple_logging(logging.DEBUG) server_app = ServerApp( - strategy_name="strategy", strategy=NPSwarm(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=5) + strategy_name="swarm", strategy=NPSwarm(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=5) ) client_app = NPSwarmClient(delta=1.0) diff --git a/nvflare/fox/sim/simulator.py b/nvflare/fox/sim/simulator.py index 5e8a436374..422820bd83 100644 --- a/nvflare/fox/sim/simulator.py +++ b/nvflare/fox/sim/simulator.py @@ -17,7 +17,7 @@ from typing import Tuple, Union from nvflare.apis.signal import Signal -from nvflare.fox.api.app import App, ClientApp, ClientAppFactory, ServerApp +from nvflare.fox.api.app import App, ClientApp, ServerApp from nvflare.fox.api.constants import EnvType from nvflare.fox.api.dec import get_object_collab_interface from nvflare.fox.api.proxy import Proxy @@ -57,10 +57,12 @@ def _prepare_proxy(self, for_app: App, target_app: App, backends: dict): return app_proxy def _make_app(self, name, fqn): - if isinstance(self.client_app, ClientApp): - app = copy.deepcopy(self.client_app) + make_client_app_f = getattr(self.client_app, "make_client_app", None) + if make_client_app_f and callable(make_client_app_f): + app = make_client_app_f(name) else: - app = self.client_app.make_client_app(name) + app = copy.deepcopy(self.client_app) + app.name = name app.fqn = fqn app.env_type = EnvType.SIMULATION @@ -80,15 +82,15 @@ def __init__( root_dir: str, experiment_name: str, server_app: ServerApp, - client_app: Union[ClientAppFactory, ClientApp], + client_app: ClientApp, max_workers: int = 100, num_clients: Union[int, Tuple[int, int]] = 2, ): if not isinstance(server_app, ServerApp): raise ValueError(f"server_app must be ServerApp but got {type(server_app)}") - if not isinstance(client_app, (ClientAppFactory, ClientApp)): - raise ValueError(f"client_app must be ClientApp or ClientAppFactory but got {type(client_app)}") + if not isinstance(client_app, ClientApp): + raise ValueError(f"client_app must be ClientApp but got {type(client_app)}") self.logger = get_obj_logger(self) self.abort_signal = Signal() diff --git a/nvflare/fox/sys/adaptor.py b/nvflare/fox/sys/adaptor.py index 00076acf1f..c035089a36 100644 --- a/nvflare/fox/sys/adaptor.py +++ b/nvflare/fox/sys/adaptor.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Dict +from typing import Any, Dict, List from nvflare.apis.fl_context import FLContext from nvflare.fox.api.app import App @@ -22,7 +22,8 @@ class FoxAdaptor: def __init__( self, - collab_obj_ids: Dict[str, str] = None, + collab_obj_ids: List[str] = None, + props: Dict[str, Any] = None, incoming_call_filters=None, outgoing_call_filters=None, incoming_result_filters=None, @@ -30,6 +31,7 @@ def __init__( ): if not collab_obj_ids: collab_obj_ids = [] + self.props = props self.collab_obj_ids = collab_obj_ids self.incoming_call_filters = incoming_call_filters self.outgoing_call_filters = outgoing_call_filters diff --git a/nvflare/fox/sys/controller.py b/nvflare/fox/sys/controller.py index ba504c2e12..23c5258d7f 100644 --- a/nvflare/fox/sys/controller.py +++ b/nvflare/fox/sys/controller.py @@ -57,12 +57,14 @@ def __init__( outgoing_call_filters=None, incoming_result_filters=None, outgoing_result_filters=None, + props=None, sync_task_timeout=5, max_call_threads=100, ): Controller.__init__(self) FoxAdaptor.__init__( self, + props=props, collab_obj_ids=collab_obj_ids, incoming_call_filters=incoming_call_filters, outgoing_call_filters=outgoing_call_filters, diff --git a/nvflare/fox/sys/executor.py b/nvflare/fox/sys/executor.py index f77b78e6a5..baac4d08ec 100644 --- a/nvflare/fox/sys/executor.py +++ b/nvflare/fox/sys/executor.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from concurrent.futures import ThreadPoolExecutor -from typing import Dict +from typing import Any, Dict, List from nvflare.apis.client import Client, from_dict from nvflare.apis.event_type import EventType @@ -22,7 +22,7 @@ from nvflare.apis.job_def import JobMetaKey from nvflare.apis.shareable import Shareable, make_reply from nvflare.apis.signal import Signal -from nvflare.fox.api.app import ClientApp, ClientAppFactory +from nvflare.fox.api.app import ClientApp from nvflare.fox.api.constants import EnvType from nvflare.fox.api.proxy import Proxy from nvflare.fuel.f3.cellnet.fqcn import FQCN @@ -39,17 +39,19 @@ class FoxExecutor(Executor, FoxAdaptor): def __init__( self, client_app_id: str, - collab_obj_ids: Dict[str, str] = None, + collab_obj_ids: List[str] = None, incoming_call_filters=None, outgoing_call_filters=None, incoming_result_filters=None, outgoing_result_filters=None, + props: Dict[str, Any] = None, max_call_threads=100, ): Executor.__init__(self) FoxAdaptor.__init__( self, collab_obj_ids=collab_obj_ids, + props=props, incoming_call_filters=incoming_call_filters, outgoing_call_filters=outgoing_call_filters, incoming_result_filters=incoming_result_filters, @@ -65,18 +67,20 @@ def _handle_start_run(self, event_type: str, fl_ctx: FLContext): fl_ctx.set_prop(FLContextKey.FOX_MODE, True, private=True, sticky=True) engine = fl_ctx.get_engine() client_app = engine.get_component(self.client_app_id) - if not isinstance(client_app, (ClientApp, ClientAppFactory)): + if not isinstance(client_app, ClientApp): self.system_panic( - f"component {self.client_app_id} must be ClientApp or ClientAppFactory but got {type(client_app)}", + f"component {self.client_app_id} must be ClientApp but got {type(client_app)}", fl_ctx, ) return client_name = fl_ctx.get_identity_name() - if isinstance(client_app, ClientApp): - self.client_app = client_app + + make_client_app_f = getattr(self.client_app, "make_client_app", None) + if make_client_app_f and callable(make_client_app_f): + self.client_app = make_client_app_f(client_name) else: - self.client_app = client_app.make_client_app(client_name) + self.client_app = client_app self.client_app.name = client_name self.client_app.env_type = EnvType.SYSTEM diff --git a/nvflare/fox/sys/recipe.py b/nvflare/fox/sys/recipe.py new file mode 100644 index 0000000000..bc2853c6fd --- /dev/null +++ b/nvflare/fox/sys/recipe.py @@ -0,0 +1,164 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nvflare.fox.api.app import App, ClientApp, ServerApp +from nvflare.fox.api.filter import FilterChain +from nvflare.fox.api.strategy import Strategy +from nvflare.fuel.utils.validation_utils import check_object_type, check_positive_int, check_positive_number, check_str +from nvflare.job_config.api import FedJob +from nvflare.recipe.spec import Recipe + +from .controller import FoxController +from .executor import FoxExecutor + + +class FoxRecipe(Recipe): + + def __init__( + self, + job_name: str, + server_app: ServerApp, + client_app: ClientApp, + sync_task_timeout=5, + max_call_threads_for_server=100, + max_call_threads_for_client=100, + min_clients: int = 1, + ): + check_str("job_name", job_name) + check_object_type("server_app", server_app, ServerApp) + check_positive_number("sync_task_timeout", sync_task_timeout) + check_positive_int("max_call_threads_for_server", max_call_threads_for_server) + check_positive_int("max_call_threads_for_client", max_call_threads_for_client) + check_positive_int("min_clients", min_clients) + + if not isinstance(client_app, ClientApp): + raise ValueError(f"client_app must be ClientApp but got {type(client_app)}") + + # make sure server app has strategy + if not server_app.strategies: + raise ValueError(f"server_app has no strategies") + + self.job_name = job_name + self.server_app = server_app + self.client_app = client_app + self.sync_task_timeout = sync_task_timeout + self.max_call_threads_for_server = max_call_threads_for_server + self.max_call_threads_for_client = max_call_threads_for_client + self.min_clients = min_clients + + job = self._create_job() + Recipe.__init__(self, job) + + def _create_job(self) -> FedJob: + job = FedJob(name=self.job_name, min_clients=self.min_clients) + + server_app_id = job.to_server(self.server_app, "_app") + + # get all strategies + strategy_ids = [] + for name, strategy in self.server_app.strategies: + comp_id = job.to_server(strategy, id=name) + strategy_ids.append(comp_id) + + collab_obj_ids, in_cf_arg, out_cf_arg, in_rf_arg, out_rf_arg = self._create_app_args( + self.server_app, job.to_server + ) + + controller = FoxController( + strategy_ids=strategy_ids, + server_app_id=server_app_id, + collab_obj_ids=collab_obj_ids, + incoming_call_filters=in_cf_arg, + outgoing_call_filters=out_cf_arg, + incoming_result_filters=in_rf_arg, + outgoing_result_filters=out_rf_arg, + sync_task_timeout=self.sync_task_timeout, + max_call_threads=self.max_call_threads_for_server, + props=self.server_app.get_props(), + ) + + job.to_server(controller, id="controller") + + # add client config + client_app_id = job.to_clients(self.client_app, "_app") + c_collab_obj_ids, c_in_cf_arg, c_out_cf_arg, c_in_rf_arg, c_out_rf_arg = self._create_app_args( + self.client_app, job.to_clients + ) + executor = FoxExecutor( + client_app_id=client_app_id, + collab_obj_ids=c_collab_obj_ids, + incoming_call_filters=c_in_cf_arg, + outgoing_call_filters=c_out_cf_arg, + incoming_result_filters=c_in_rf_arg, + outgoing_result_filters=c_out_rf_arg, + max_call_threads=self.max_call_threads_for_client, + props=self.client_app.get_props(), + ) + job.to_clients(executor, id="executor", tasks=["*"]) + return job + + def _create_app_args(self, app: App, to_f): + # collab objs + collab_obj_ids = [] + collab_objs = app.get_collab_objects() + for name, obj in collab_objs.items(): + if isinstance(obj, Strategy): + # do not include strategy in collab objs since it's done separately. + continue + comp_id = to_f(obj, id=name) + collab_obj_ids.append(comp_id) + + # build filter components + # since a filter object could be used multiple times, we must make sure that only one component is created + # for the same object! + filter_comp_table = {} + incoming_call_filters = app.get_incoming_call_filters() + outgoing_call_filters = app.get_outgoing_call_filters() + incoming_result_filters = app.get_incoming_result_filters() + outgoing_result_filters = app.get_outgoing_result_filters() + + self._create_filter_components(to_f, incoming_call_filters, filter_comp_table) + self._create_filter_components(to_f, outgoing_call_filters, filter_comp_table) + self._create_filter_components(to_f, incoming_result_filters, filter_comp_table) + self._create_filter_components(to_f, outgoing_result_filters, filter_comp_table) + + # filters + in_cf_arg = self._create_filer_chain_arg(incoming_call_filters, filter_comp_table) + out_cf_arg = self._create_filer_chain_arg(outgoing_call_filters, filter_comp_table) + in_rf_arg = self._create_filer_chain_arg(incoming_result_filters, filter_comp_table) + out_rf_arg = self._create_filer_chain_arg(outgoing_result_filters, filter_comp_table) + return collab_obj_ids, in_cf_arg, out_cf_arg, in_rf_arg, out_rf_arg + + @staticmethod + def _create_filer_chain_arg(filter_chains: list, comp_table: dict): + result = [] + for chain in filter_chains: + assert isinstance(chain, FilterChain) + filter_ids = [] + for f in chain.filters: + comp_id = comp_table[id(f)] + filter_ids.append(comp_id) + d = {"pattern": chain.pattern, "filters": filter_ids} + result.append(d) + return result + + @staticmethod + def _create_filter_components(to_f, filter_chains: list, comp_table: dict): + for chain in filter_chains: + assert isinstance(chain, FilterChain) + for f in chain.filters: + fid = id(f) + comp_id = comp_table.get(fid) + if not comp_id: + comp_id = to_f(f, id="_filter") + comp_table[fid] = comp_id From 38801a83e6db69d32041541acb57be00207c4abc Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Sat, 11 Oct 2025 16:50:02 -0400 Subject: [PATCH 035/102] reorg --- nvflare/fox/api/app.py | 3 + nvflare/fox/api/ctx.py | 4 + nvflare/fox/examples/np/algos/avg_stream.py | 151 +++++++++++++ nvflare/fox/examples/np/algos/client.py | 9 +- nvflare/fox/examples/np/algos/strategies.py | 208 ------------------ .../examples/np/algos/strategies/__init__.py | 13 ++ .../fox/examples/np/algos/strategies/avg_h.py | 57 +++++ .../np/algos/strategies/avg_intime.py | 78 +++++++ .../examples/np/algos/strategies/avg_para.py | 56 +++++ .../examples/np/algos/strategies/avg_seq.py | 54 +++++ .../examples/np/algos/strategies/cyclic.py | 43 ++++ nvflare/fox/examples/np/algos/swarm.py | 4 +- nvflare/fox/examples/np/algos/utils.py | 4 + nvflare/fox/examples/np/cyclic.py | 2 +- nvflare/fox/examples/np/cyclic_avg.py | 3 +- nvflare/fox/examples/np/fed_avg_h.py | 2 +- nvflare/fox/examples/np/fed_avg_intime.py | 2 +- nvflare/fox/examples/np/fed_avg_para.py | 4 +- nvflare/fox/examples/np/fed_avg_seq.py | 2 +- nvflare/fox/examples/np/fed_avg_stream.py | 44 ++++ .../fox/examples/np/recipe_fed_avg_intime.py | 2 +- .../fox/examples/np/recipe_fed_avg_stream.py | 43 ++++ nvflare/fox/sys/executor.py | 2 +- nvflare/fox/sys/file_downloader.py | 57 +++++ nvflare/fox/sys/utils.py | 3 + 25 files changed, 623 insertions(+), 227 deletions(-) create mode 100644 nvflare/fox/examples/np/algos/avg_stream.py delete mode 100644 nvflare/fox/examples/np/algos/strategies.py create mode 100644 nvflare/fox/examples/np/algos/strategies/__init__.py create mode 100644 nvflare/fox/examples/np/algos/strategies/avg_h.py create mode 100644 nvflare/fox/examples/np/algos/strategies/avg_intime.py create mode 100644 nvflare/fox/examples/np/algos/strategies/avg_para.py create mode 100644 nvflare/fox/examples/np/algos/strategies/avg_seq.py create mode 100644 nvflare/fox/examples/np/algos/strategies/cyclic.py create mode 100644 nvflare/fox/examples/np/fed_avg_stream.py create mode 100644 nvflare/fox/examples/np/recipe_fed_avg_stream.py create mode 100644 nvflare/fox/sys/file_downloader.py diff --git a/nvflare/fox/api/app.py b/nvflare/fox/api/app.py index 79c0201b98..79b042753a 100644 --- a/nvflare/fox/api/app.py +++ b/nvflare/fox/api/app.py @@ -50,6 +50,9 @@ def __init__(self): self._workspace = None self.logger = get_obj_logger(self) + def get_backend(self): + return self._me.backend + def get_workspace(self): return self._workspace diff --git a/nvflare/fox/api/ctx.py b/nvflare/fox/api/ctx.py index 77ed9b0728..708cec702f 100644 --- a/nvflare/fox/api/ctx.py +++ b/nvflare/fox/api/ctx.py @@ -31,6 +31,10 @@ def __init__(self, app, caller: str, callee: str, abort_signal: Signal, props: d if props: self.props.update(props) + @property + def backend(self): + return self.app.get_backend() + @property def env_type(self): return self.app.env_type diff --git a/nvflare/fox/examples/np/algos/avg_stream.py b/nvflare/fox/examples/np/algos/avg_stream.py new file mode 100644 index 0000000000..5b5fcc867c --- /dev/null +++ b/nvflare/fox/examples/np/algos/avg_stream.py @@ -0,0 +1,151 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os.path +import uuid + +from nvflare.fox.api.app import ClientApp +from nvflare.fox.api.constants import ContextKey, EnvType +from nvflare.fox.api.ctx import Context +from nvflare.fox.api.dec import collab +from nvflare.fox.api.group import all_clients +from nvflare.fox.api.strategy import Strategy +from nvflare.fox.examples.np.algos.utils import load_np_model, parse_array_def, save_np_model +from nvflare.fox.sys.file_downloader import download_file, prepare_file_for_download +from nvflare.fuel.f3.streaming.obj_downloader import DownloadStatus +from nvflare.fuel.utils.log_utils import get_obj_logger + + +class _AggrResult: + + def __init__(self): + self.total = 0 + self.count = 0 + + +class NPFedAvgStream(Strategy): + + def __init__(self, initial_model, num_rounds=10, timeout=2.0): + self.num_rounds = num_rounds + self.initial_model = initial_model + self.timeout = timeout + self.name = "NPFedAvgStream" + self.logger = get_obj_logger(self) + self._init_model = parse_array_def(initial_model) + + def execute(self, context: Context): + self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") + current_model = context.get_prop(ContextKey.INPUT, self._init_model) + for i in range(self.num_rounds): + current_model = self._do_one_round(i, current_model, context) + self.logger.info(f"FINAL MODEL: {current_model}") + return current_model + + def _do_one_round(self, r, current_model, ctx: Context): + aggr_result = _AggrResult() + + # pretend the model is big + file_name = None + if ctx.env_type == EnvType.SYSTEM: + file_name = f"/tmp/np_{str(uuid.uuid4())}.npy" + save_np_model(current_model, file_name) + model = prepare_file_for_download( + file_name=file_name, + ctx=ctx, + timeout=5.0, + file_downloaded_cb=self._model_downloaded, + ) + model_type = "ref" + self.logger.info(f"prepared model as ref: {model}") + else: + model = current_model + model_type = "model" + + all_clients( + ctx, + process_resp_cb=self._accept_train_result, + aggr_result=aggr_result, + ).train(r, model, model_type) + + if file_name: + os.remove(file_name) + + if aggr_result.count == 0: + return None + else: + result = aggr_result.total / aggr_result.count + self.logger.info(f"[{ctx.header_str()}] round {r}: aggr result from {aggr_result.count} clients: {result}") + return result + + def _accept_train_result(self, result, aggr_result: _AggrResult, context: Context): + self.logger.info(f"[{context.header_str()}] got train result from {context.caller}: {result}") + + model, model_type = result + if model_type == "ref": + err, file_path = download_file(ref=model, per_request_timeout=5.0, ctx=context) + if err: + raise RuntimeError(f"failed to download model file {model}: {err}") + self.logger.info(f"downloaded model file to {file_path}") + model = load_np_model(file_path) + os.remove(file_path) + + aggr_result.total += model + aggr_result.count += 1 + return None + + def _model_downloaded(self, ref_id: str, to_site: str, status: str, file_name): + self.logger.info(f"model file {file_name} downloaded by {to_site}: {ref_id=} {status=}") + + +class NPTrainer(ClientApp): + + def __init__(self, delta: float): + ClientApp.__init__(self) + self.delta = delta + + @collab + def train(self, current_round, weights, model_type: str, context: Context): + if context.is_aborted(): + self.logger.debug("training aborted") + return 0 + self.logger.debug(f"[{context.header_str()}] training round {current_round}: {model_type=} {weights=}") + if model_type == "ref": + err, file_path = download_file(ref=weights, per_request_timeout=5.0, ctx=context) + if err: + raise RuntimeError(f"failed to download model file {weights}: {err}") + self.logger.info(f"downloaded model file to {file_path}") + weights = load_np_model(file_path) + self.logger.info(f"loaded model from file: {weights}") + os.remove(file_path) + + result = weights + self.delta + if model_type == "ref": + # stream it + file_name = f"/tmp/np_{str(uuid.uuid4())}.npy" + save_np_model(result, file_name) + model = prepare_file_for_download( + file_name=file_name, + ctx=context, + timeout=5.0, + file_downloaded_cb=self._result_downloaded, + ) + model_type = "ref" + self.logger.info(f"prepared result as ref: {model}") + else: + model = result + model_type = "model" + return model, model_type + + def _result_downloaded(self, ref_id: str, to_site: str, status: str, file_name): + self.logger.info(f"file {file_name} downloaded to {to_site}: {ref_id=} {status=}") + os.remove(file_name) diff --git a/nvflare/fox/examples/np/algos/client.py b/nvflare/fox/examples/np/algos/client.py index 1ae02ceb26..646d148441 100644 --- a/nvflare/fox/examples/np/algos/client.py +++ b/nvflare/fox/examples/np/algos/client.py @@ -75,14 +75,7 @@ def _local_train(self, current_round, weights, context: Context): if context.is_aborted(): self.logger.debug("training aborted") return 0 - self.logger.debug(f"[{context.header_str()}] trained round {current_round}") - - # metric_receiver = self.server.get_target("metric_receiver") - # if metric_receiver: - # self.server.accept_metric({"round": r, "y": 2}) - # self.server.metric_receiver.accept_metric({"round": r, "y": 2}) - # - self.server.fire_event("metrics", {"round": current_round, "y": 10}, _blocking=False) + self.logger.info(f"[{context.header_str()}] local trained round {current_round} {weights} {type(weights)}") return weights + self.delta @collab diff --git a/nvflare/fox/examples/np/algos/strategies.py b/nvflare/fox/examples/np/algos/strategies.py deleted file mode 100644 index 464f6a26e0..0000000000 --- a/nvflare/fox/examples/np/algos/strategies.py +++ /dev/null @@ -1,208 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import random - -import numpy as np - -from nvflare.fox.api.constants import ContextKey -from nvflare.fox.api.ctx import Context -from nvflare.fox.api.group import all_children, all_clients, all_leaf_clients -from nvflare.fox.api.strategy import Strategy -from nvflare.fuel.utils.log_utils import get_obj_logger - -from .utils import parse_array_def, save_np_model - - -class NPFedAvgSequential(Strategy): - - def __init__(self, initial_model, num_rounds=10): - Strategy.__init__(self) - self.name = "NPFedAvgSequential" - self.num_rounds = num_rounds - self.initial_model = initial_model # need to remember init for job API to work! - self._initial_model = parse_array_def(initial_model) - self.logger = get_obj_logger(self) - - def execute(self, context: Context): - self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") - current_model = context.get_prop(ContextKey.INPUT, self._initial_model) - for i in range(self.num_rounds): - current_model = self._do_one_round(i, current_model, context) - - # save model to work dir - file_name = os.path.join(context.workspace.get_work_dir(), "model.npy") - save_np_model(current_model, file_name) - return current_model - - def _do_one_round(self, r, current_model, context: Context): - total = np.array([[0, 0, 0], [0, 0, 0], [0, 0, 0]], dtype=np.float32) - n = 0 - for c in context.clients: - result = c.train(r, current_model) - self.logger.info(f"[{context.header_str()}] round {r}: got result from client {c.name}: {result}") - total += result - n += 1 - return total / n - - -class NPFedAvgParallel(Strategy): - - def __init__(self, initial_model, num_rounds=10): - self.num_rounds = num_rounds - self.initial_model = initial_model - self._initial_model = parse_array_def(initial_model) - self.name = "NPFedAvgParallel" - self.logger = get_obj_logger(self) - - def execute(self, context: Context): - self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") - current_model = context.get_prop(ContextKey.INPUT, self._initial_model) - for i in range(self.num_rounds): - current_model = self._do_one_round(i, current_model, context) - score = self._do_eval(current_model, context) - self.logger.info(f"[{context.header_str()}]: eval score in round {i}: {score}") - return current_model - - def _do_eval(self, model, ctx: Context): - results = all_clients(ctx).evaluate(model) - total = 0.0 - for n, v in results.items(): - self.logger.info(f"[{ctx.header_str()}]: got eval result from client {n}: {v}") - total += v - return total / len(results) - - def _do_one_round(self, r, current_model, ctx: Context): - total = np.array([[0, 0, 0], [0, 0, 0], [0, 0, 0]], dtype=np.float32) - results = all_clients(ctx).train(r, current_model) - for n, v in results.items(): - self.logger.info(f"[{ctx.header_str()}] round {r}: got group result from client {n}: {v}") - total += v - return total / len(results) - - -class NPHierarchicalFedAvg(Strategy): - - def __init__(self, initial_model, num_rounds=10): - self.num_rounds = num_rounds - self.initial_model = initial_model - self._initial_model = parse_array_def(initial_model) - self.name = self.__class__.__name__ - self.logger = get_obj_logger(self) - - def execute(self, context: Context): - self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") - current_model = context.get_prop(ContextKey.INPUT, self.initial_model) - for i in range(self.num_rounds): - current_model = self._do_one_round(i, current_model, context) - score = self._do_eval(current_model, context) - self.logger.info(f"[{context.header_str()}]: eval score in round {i}: {score}") - self.logger.info(f"FINAL MODEL: {current_model}") - return current_model - - def _do_eval(self, model, ctx: Context): - results = all_leaf_clients(ctx).evaluate(model) - total = 0.0 - for n, v in results.items(): - self.logger.info(f"[{ctx.header_str()}]: got eval result from client {n}: {v}") - total += v - return total / len(results) - - def _do_one_round(self, r, current_model, ctx: Context): - total = np.array([[0, 0, 0], [0, 0, 0], [0, 0, 0]], dtype=np.float32) - results = all_children(ctx).train(r, current_model) - for n, v in results.items(): - self.logger.info(f"[{ctx.header_str()}] round {r}: got group result from client {n}: {v}") - total += v - return total / len(results) - - -class _AggrResult: - - def __init__(self): - self.total = np.array([[0, 0, 0], [0, 0, 0], [0, 0, 0]], dtype=np.float32) - self.count = 0 - - -class NPFedAvgInTime(Strategy): - - def __init__(self, initial_model, num_rounds=10, timeout=2.0): - self.num_rounds = num_rounds - self.initial_model = initial_model - self.timeout = timeout - self.name = "NPFedAvgInTime" - self.logger = get_obj_logger(self) - self._init_model = parse_array_def(initial_model) - - def execute(self, context: Context): - self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") - current_model = context.get_prop(ContextKey.INPUT, self._init_model) - for i in range(self.num_rounds): - current_model = self._do_one_round(i, current_model, context) - score = self._do_eval(current_model, context) - self.logger.info(f"[{context.header_str()}]: eval score in round {i}: {score}") - self.logger.info(f"FINAL MODEL: {current_model}") - return current_model - - def _do_eval(self, model, ctx: Context): - results = all_clients(ctx).evaluate(model) - total = 0.0 - for n, v in results.items(): - self.logger.info(f"[{ctx.header_str()}]: got eval result from client {n}: {v}") - total += v - return total / len(results) - - def _do_one_round(self, r, current_model, ctx: Context): - aggr_result = _AggrResult() - all_clients( - ctx, - process_resp_cb=self._accept_train_result, - aggr_result=aggr_result, - ).train(r, current_model) - - if aggr_result.count == 0: - return None - else: - result = aggr_result.total / aggr_result.count - self.logger.info(f"[{ctx.header_str()}] round {r}: aggr result from {aggr_result.count} clients: {result}") - return result - - def _accept_train_result(self, result, aggr_result: _AggrResult, context: Context): - self.logger.info(f"[{context.header_str()}] got train result from {context.caller} {result}") - aggr_result.total += result - aggr_result.count += 1 - return None - - -class NPCyclic(Strategy): - - def __init__(self, initial_model, num_rounds=(2, 3)): - self.num_rounds = num_rounds - self.initial_model = initial_model - self._initial_model = parse_array_def(initial_model) - self.logger = get_obj_logger(self) - - def execute(self, context: Context): - current_model = context.get_prop(ContextKey.INPUT, self._initial_model) - for current_round in range(self.num_rounds): - current_model = self._do_one_round(current_round, current_model, context) - self.logger.info(f"[{context.header_str()}] final result: {current_model}") - return current_model - - def _do_one_round(self, current_round, current_model, ctx: Context): - random.shuffle(ctx.clients) - for c in ctx.clients: - current_model = c.train(current_round, current_model) - self.logger.info(f"[{ctx.header_str()}] result from {c.name}: {current_model}") - return current_model diff --git a/nvflare/fox/examples/np/algos/strategies/__init__.py b/nvflare/fox/examples/np/algos/strategies/__init__.py new file mode 100644 index 0000000000..341a77c5bc --- /dev/null +++ b/nvflare/fox/examples/np/algos/strategies/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nvflare/fox/examples/np/algos/strategies/avg_h.py b/nvflare/fox/examples/np/algos/strategies/avg_h.py new file mode 100644 index 0000000000..151eb79ded --- /dev/null +++ b/nvflare/fox/examples/np/algos/strategies/avg_h.py @@ -0,0 +1,57 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import numpy as np + +from nvflare.fox.api.constants import ContextKey +from nvflare.fox.api.ctx import Context +from nvflare.fox.api.group import all_children, all_leaf_clients +from nvflare.fox.api.strategy import Strategy +from nvflare.fox.examples.np.algos.utils import parse_array_def +from nvflare.fuel.utils.log_utils import get_obj_logger + + +class NPHierarchicalFedAvg(Strategy): + + def __init__(self, initial_model, num_rounds=10): + self.num_rounds = num_rounds + self.initial_model = initial_model + self._initial_model = parse_array_def(initial_model) + self.name = self.__class__.__name__ + self.logger = get_obj_logger(self) + + def execute(self, context: Context): + self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") + current_model = context.get_prop(ContextKey.INPUT, self._initial_model) + for i in range(self.num_rounds): + current_model = self._do_one_round(i, current_model, context) + score = self._do_eval(current_model, context) + self.logger.info(f"[{context.header_str()}]: eval score in round {i}: {score}") + self.logger.info(f"FINAL MODEL: {current_model}") + return current_model + + def _do_eval(self, model, ctx: Context): + results = all_leaf_clients(ctx).evaluate(model) + total = 0.0 + for n, v in results.items(): + self.logger.info(f"[{ctx.header_str()}]: got eval result from client {n}: {v}") + total += v + return total / len(results) + + def _do_one_round(self, r, current_model, ctx: Context): + total = 0 + results = all_children(ctx).train(r, current_model) + for n, v in results.items(): + self.logger.info(f"[{ctx.header_str()}] round {r}: got group result from client {n}: {v}") + total += v + return total / len(results) diff --git a/nvflare/fox/examples/np/algos/strategies/avg_intime.py b/nvflare/fox/examples/np/algos/strategies/avg_intime.py new file mode 100644 index 0000000000..22e8077a70 --- /dev/null +++ b/nvflare/fox/examples/np/algos/strategies/avg_intime.py @@ -0,0 +1,78 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import numpy as np + +from nvflare.fox.api.constants import ContextKey +from nvflare.fox.api.ctx import Context +from nvflare.fox.api.group import all_clients +from nvflare.fox.api.strategy import Strategy +from nvflare.fox.examples.np.algos.utils import parse_array_def +from nvflare.fuel.utils.log_utils import get_obj_logger + + +class _AggrResult: + + def __init__(self): + self.total = 0 + self.count = 0 + + +class NPFedAvgInTime(Strategy): + + def __init__(self, initial_model, num_rounds=10, timeout=2.0): + self.num_rounds = num_rounds + self.initial_model = initial_model + self.timeout = timeout + self.name = "NPFedAvgInTime" + self.logger = get_obj_logger(self) + self._init_model = parse_array_def(initial_model) + + def execute(self, context: Context): + self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") + current_model = context.get_prop(ContextKey.INPUT, self._init_model) + for i in range(self.num_rounds): + current_model = self._do_one_round(i, current_model, context) + score = self._do_eval(current_model, context) + self.logger.info(f"[{context.header_str()}]: eval score in round {i}: {score}") + self.logger.info(f"FINAL MODEL: {current_model}") + return current_model + + def _do_eval(self, model, ctx: Context): + results = all_clients(ctx).evaluate(model) + total = 0.0 + for n, v in results.items(): + self.logger.info(f"[{ctx.header_str()}]: got eval result from client {n}: {v}") + total += v + return total / len(results) + + def _do_one_round(self, r, current_model, ctx: Context): + aggr_result = _AggrResult() + all_clients( + ctx, + process_resp_cb=self._accept_train_result, + aggr_result=aggr_result, + ).train(r, current_model) + + if aggr_result.count == 0: + return None + else: + result = aggr_result.total / aggr_result.count + self.logger.info(f"[{ctx.header_str()}] round {r}: aggr result from {aggr_result.count} clients: {result}") + return result + + def _accept_train_result(self, result, aggr_result: _AggrResult, context: Context): + self.logger.info(f"[{context.header_str()}] got train result from {context.caller} {result}") + aggr_result.total += result + aggr_result.count += 1 + return None diff --git a/nvflare/fox/examples/np/algos/strategies/avg_para.py b/nvflare/fox/examples/np/algos/strategies/avg_para.py new file mode 100644 index 0000000000..d8bcf334ef --- /dev/null +++ b/nvflare/fox/examples/np/algos/strategies/avg_para.py @@ -0,0 +1,56 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import numpy as np + +from nvflare.fox.api.constants import ContextKey +from nvflare.fox.api.ctx import Context +from nvflare.fox.api.group import all_clients +from nvflare.fox.api.strategy import Strategy +from nvflare.fox.examples.np.algos.utils import parse_array_def +from nvflare.fuel.utils.log_utils import get_obj_logger + + +class NPFedAvgParallel(Strategy): + + def __init__(self, initial_model, num_rounds=10): + self.num_rounds = num_rounds + self.initial_model = initial_model + self._initial_model = parse_array_def(initial_model) + self.name = "NPFedAvgParallel" + self.logger = get_obj_logger(self) + + def execute(self, context: Context): + self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") + current_model = context.get_prop(ContextKey.INPUT, self._initial_model) + for i in range(self.num_rounds): + current_model = self._do_one_round(i, current_model, context) + score = self._do_eval(current_model, context) + self.logger.info(f"[{context.header_str()}]: eval score in round {i}: {score}") + return current_model + + def _do_eval(self, model, ctx: Context): + results = all_clients(ctx).evaluate(model) + total = 0.0 + for n, v in results.items(): + self.logger.info(f"[{ctx.header_str()}]: got eval result from client {n}: {v}") + total += v + return total / len(results) + + def _do_one_round(self, r, current_model, ctx: Context): + total = 0 + results = all_clients(ctx, timeout=4, min_resps=2, wait_after_min_resps=1).train(r, current_model) + for n, v in results.items(): + self.logger.info(f"[{ctx.header_str()}] round {r}: got group result from client {n}: {v}") + total += v + return total / len(results) diff --git a/nvflare/fox/examples/np/algos/strategies/avg_seq.py b/nvflare/fox/examples/np/algos/strategies/avg_seq.py new file mode 100644 index 0000000000..8432c907b1 --- /dev/null +++ b/nvflare/fox/examples/np/algos/strategies/avg_seq.py @@ -0,0 +1,54 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import numpy as np + +from nvflare.fox.api.constants import ContextKey +from nvflare.fox.api.ctx import Context +from nvflare.fox.api.strategy import Strategy +from nvflare.fox.examples.np.algos.utils import parse_array_def, save_np_model +from nvflare.fuel.utils.log_utils import get_obj_logger + + +class NPFedAvgSequential(Strategy): + + def __init__(self, initial_model, num_rounds=10): + Strategy.__init__(self) + self.name = "NPFedAvgSequential" + self.num_rounds = num_rounds + self.initial_model = initial_model # need to remember init for job API to work! + self._initial_model = parse_array_def(initial_model) + self.logger = get_obj_logger(self) + + def execute(self, context: Context): + self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") + current_model = context.get_prop(ContextKey.INPUT, self._initial_model) + for i in range(self.num_rounds): + current_model = self._do_one_round(i, current_model, context) + + # save model to work dir + file_name = os.path.join(context.workspace.get_work_dir(), "model.npy") + save_np_model(current_model, file_name) + return current_model + + def _do_one_round(self, r, current_model, context: Context): + total = 0 + n = 0 + for c in context.clients: + result = c.train(r, current_model, _blocking=True, _timeout=2.0, _optional=True, _secure=True) + self.logger.info(f"[{context.header_str()}] round {r}: got result from client {c.name}: {result}") + total += result + n += 1 + return total / n diff --git a/nvflare/fox/examples/np/algos/strategies/cyclic.py b/nvflare/fox/examples/np/algos/strategies/cyclic.py new file mode 100644 index 0000000000..84386e6fe5 --- /dev/null +++ b/nvflare/fox/examples/np/algos/strategies/cyclic.py @@ -0,0 +1,43 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import random + +from nvflare.fox.api.constants import ContextKey +from nvflare.fox.api.ctx import Context +from nvflare.fox.api.strategy import Strategy +from nvflare.fox.examples.np.algos.utils import parse_array_def +from nvflare.fuel.utils.log_utils import get_obj_logger + + +class NPCyclic(Strategy): + + def __init__(self, initial_model, num_rounds=2): + self.num_rounds = num_rounds + self.initial_model = initial_model + self._initial_model = parse_array_def(initial_model) + self.logger = get_obj_logger(self) + + def execute(self, context: Context): + current_model = context.get_prop(ContextKey.INPUT, self._initial_model) + for current_round in range(self.num_rounds): + current_model = self._do_one_round(current_round, current_model, context) + self.logger.info(f"[{context.header_str()}] final result: {current_model}") + return current_model + + def _do_one_round(self, current_round, current_model, ctx: Context): + random.shuffle(ctx.clients) + for c in ctx.clients: + current_model = c.train(current_round, current_model) + self.logger.info(f"[{ctx.header_str()}] result from {c.name}: {current_model}") + return current_model diff --git a/nvflare/fox/examples/np/algos/swarm.py b/nvflare/fox/examples/np/algos/swarm.py index 5e376655ea..ed2de25bab 100644 --- a/nvflare/fox/examples/np/algos/swarm.py +++ b/nvflare/fox/examples/np/algos/swarm.py @@ -70,8 +70,8 @@ def sag(self, model, current_round, ctx: Context): results = all_clients(ctx, blocking=True).train(model, current_round) # results = all_other_clients(ctx, blocking=True).train(model, current_round) results = list(results.values()) - total = results[0] - for i in range(1, len(results)): + total = 0 + for i in range(len(results)): total += results[i] return total / len(results) diff --git a/nvflare/fox/examples/np/algos/utils.py b/nvflare/fox/examples/np/algos/utils.py index 1a164c8530..13db56f999 100644 --- a/nvflare/fox/examples/np/algos/utils.py +++ b/nvflare/fox/examples/np/algos/utils.py @@ -30,3 +30,7 @@ def parse_array_def(array_def): def save_np_model(model: np.ndarray, file_name: str): np.save(file_name, model) + + +def load_np_model(file_name: str): + return np.load(file_name) diff --git a/nvflare/fox/examples/np/cyclic.py b/nvflare/fox/examples/np/cyclic.py index f02c5d47ff..e0194ab42e 100644 --- a/nvflare/fox/examples/np/cyclic.py +++ b/nvflare/fox/examples/np/cyclic.py @@ -16,7 +16,7 @@ from nvflare.fox.api.app import ServerApp from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.client import NPTrainer -from nvflare.fox.examples.np.algos.strategies import NPCyclic +from nvflare.fox.examples.np.algos.strategies.cyclic import NPCyclic from nvflare.fox.sim.simulator import Simulator diff --git a/nvflare/fox/examples/np/cyclic_avg.py b/nvflare/fox/examples/np/cyclic_avg.py index bb3e97c7d6..dbcdb2c89d 100644 --- a/nvflare/fox/examples/np/cyclic_avg.py +++ b/nvflare/fox/examples/np/cyclic_avg.py @@ -16,7 +16,8 @@ from nvflare.fox.api.app import ServerApp from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.client import NPTrainer -from nvflare.fox.examples.np.algos.strategies import NPCyclic, NPFedAvgParallel +from nvflare.fox.examples.np.algos.strategies.avg_para import NPFedAvgParallel +from nvflare.fox.examples.np.algos.strategies.cyclic import NPCyclic from nvflare.fox.sim.simulator import Simulator diff --git a/nvflare/fox/examples/np/fed_avg_h.py b/nvflare/fox/examples/np/fed_avg_h.py index 234e9c1c9d..55dee43e1b 100644 --- a/nvflare/fox/examples/np/fed_avg_h.py +++ b/nvflare/fox/examples/np/fed_avg_h.py @@ -16,7 +16,7 @@ from nvflare.fox.api.app import ServerApp from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.client import NPHierarchicalTrainer -from nvflare.fox.examples.np.algos.strategies import NPHierarchicalFedAvg +from nvflare.fox.examples.np.algos.strategies.avg_h import NPHierarchicalFedAvg from nvflare.fox.examples.np.algos.widgets import MetricReceiver from nvflare.fox.sim.simulator import Simulator diff --git a/nvflare/fox/examples/np/fed_avg_intime.py b/nvflare/fox/examples/np/fed_avg_intime.py index 89e22067d6..56f1efd286 100644 --- a/nvflare/fox/examples/np/fed_avg_intime.py +++ b/nvflare/fox/examples/np/fed_avg_intime.py @@ -17,7 +17,7 @@ from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.filters import AddNoiseToModel, PrintCall, PrintResult -from nvflare.fox.examples.np.algos.strategies import NPFedAvgInTime +from nvflare.fox.examples.np.algos.strategies.avg_intime import NPFedAvgInTime from nvflare.fox.examples.np.algos.widgets import MetricReceiver from nvflare.fox.sim.simulator import Simulator diff --git a/nvflare/fox/examples/np/fed_avg_para.py b/nvflare/fox/examples/np/fed_avg_para.py index 2ae7413c6a..057e38edad 100644 --- a/nvflare/fox/examples/np/fed_avg_para.py +++ b/nvflare/fox/examples/np/fed_avg_para.py @@ -16,7 +16,7 @@ from nvflare.fox.api.app import ServerApp from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.client import NPTrainer -from nvflare.fox.examples.np.algos.strategies import NPFedAvgParallel +from nvflare.fox.examples.np.algos.strategies.avg_para import NPFedAvgParallel from nvflare.fox.examples.np.algos.widgets import MetricReceiver from nvflare.fox.sim.simulator import Simulator @@ -26,7 +26,7 @@ def main(): server_app = ServerApp( strategy_name="fed_avg", - strategy=NPFedAvgParallel(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), + strategy=NPFedAvgParallel(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]], num_rounds=2), ) server_app.add_collab_object("metric_receiver", MetricReceiver()) diff --git a/nvflare/fox/examples/np/fed_avg_seq.py b/nvflare/fox/examples/np/fed_avg_seq.py index 63745a6c51..8c0d6addaf 100644 --- a/nvflare/fox/examples/np/fed_avg_seq.py +++ b/nvflare/fox/examples/np/fed_avg_seq.py @@ -16,7 +16,7 @@ from nvflare.fox.api.app import ServerApp from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.client import NPTrainerMaker -from nvflare.fox.examples.np.algos.strategies import NPFedAvgSequential +from nvflare.fox.examples.np.algos.strategies.avg_seq import NPFedAvgSequential from nvflare.fox.examples.np.algos.widgets import MetricReceiver from nvflare.fox.sim.simulator import Simulator diff --git a/nvflare/fox/examples/np/fed_avg_stream.py b/nvflare/fox/examples/np/fed_avg_stream.py new file mode 100644 index 0000000000..60b19ed93c --- /dev/null +++ b/nvflare/fox/examples/np/fed_avg_stream.py @@ -0,0 +1,44 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from nvflare.fox.api.app import ServerApp +from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples.np.algos.avg_stream import NPFedAvgStream, NPTrainer +from nvflare.fox.sim.simulator import Simulator + + +def main(): + simple_logging(logging.DEBUG) + + server_app = ServerApp( + strategy_name="fed_avg_in_time", + strategy=NPFedAvgStream(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=4), + ) + + client_app = NPTrainer(delta=1.0) + + simulator = Simulator( + root_dir="/tmp/fox", + experiment_name="fedavg_intime", + server_app=server_app, + client_app=client_app, + num_clients=2, + ) + + simulator.run() + + +if __name__ == "__main__": + main() diff --git a/nvflare/fox/examples/np/recipe_fed_avg_intime.py b/nvflare/fox/examples/np/recipe_fed_avg_intime.py index 12fdf3591e..f889a2b2e2 100644 --- a/nvflare/fox/examples/np/recipe_fed_avg_intime.py +++ b/nvflare/fox/examples/np/recipe_fed_avg_intime.py @@ -17,7 +17,7 @@ from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.filters import AddNoiseToModel, PrintCall, PrintResult -from nvflare.fox.examples.np.algos.strategies import NPFedAvgInTime +from nvflare.fox.examples.np.algos.strategies.avg_intime import NPFedAvgInTime from nvflare.fox.examples.np.algos.widgets import MetricReceiver from nvflare.fox.sys.recipe import FoxRecipe diff --git a/nvflare/fox/examples/np/recipe_fed_avg_stream.py b/nvflare/fox/examples/np/recipe_fed_avg_stream.py new file mode 100644 index 0000000000..28572db7aa --- /dev/null +++ b/nvflare/fox/examples/np/recipe_fed_avg_stream.py @@ -0,0 +1,43 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from nvflare.fox.api.app import ServerApp +from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples.np.algos.avg_stream import NPFedAvgStream, NPTrainer +from nvflare.fox.sys.recipe import FoxRecipe + +JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" + + +def main(): + simple_logging(logging.DEBUG) + + server_app = ServerApp( + strategy_name="fedavg_stream", + strategy=NPFedAvgStream(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), + ) + + client_app = NPTrainer(delta=1.0) + + recipe = FoxRecipe( + job_name="fedavg_stream", + server_app=server_app, + client_app=client_app, + ) + recipe.export(JOB_ROOT_DIR) + + +if __name__ == "__main__": + main() diff --git a/nvflare/fox/sys/executor.py b/nvflare/fox/sys/executor.py index baac4d08ec..af699b44bb 100644 --- a/nvflare/fox/sys/executor.py +++ b/nvflare/fox/sys/executor.py @@ -140,7 +140,7 @@ def _prepare_client_proxy(self, job_id, cell, client: Client, abort_signal, coll ) if self.collab_obj_ids: - for name in self.collab_obj_ids.keys(): + for name in self.collab_obj_ids: p = Proxy( app=self.client_app, target_name=f"{client.name}.{name}", diff --git a/nvflare/fox/sys/file_downloader.py b/nvflare/fox/sys/file_downloader.py new file mode 100644 index 0000000000..f69c3a8494 --- /dev/null +++ b/nvflare/fox/sys/file_downloader.py @@ -0,0 +1,57 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nvflare.fox.api.ctx import Context +from nvflare.fox.sys.backend import SysBackend +from nvflare.fuel.f3.streaming.file_downloader import FileDownloader + + +def prepare_file_for_download( + file_name: str, + timeout: float, + ctx: Context, + file_downloaded_cb=None, + **cb_kwargs, +): + backend = ctx.backend + if not isinstance(backend, SysBackend): + raise ValueError(f"backend must be SysBackend but got {type(backend)}") + + cell = backend.cell + tx_id = FileDownloader.new_transaction( + cell=cell, + timeout=timeout, + timeout_cb=None, + ) + rid = FileDownloader.add_file( + tx_id, + file_name, + file_downloaded_cb=file_downloaded_cb, + **cb_kwargs, + ) + return {"source": cell.get_fqcn(), "rid": rid} + + +def download_file(ref: dict, per_request_timeout: float, ctx: Context): + backend = ctx.backend + if not isinstance(backend, SysBackend): + raise ValueError(f"backend must be SysBackend but got {type(backend)}") + + err, file_path = FileDownloader.download_file( + from_fqcn=ref.get("source"), + ref_id=ref.get("rid"), + per_request_timeout=per_request_timeout, + cell=backend.cell, + abort_signal=ctx.abort_signal, + ) + return err, file_path diff --git a/nvflare/fox/sys/utils.py b/nvflare/fox/sys/utils.py index be42cccdc7..d4013f96a4 100644 --- a/nvflare/fox/sys/utils.py +++ b/nvflare/fox/sys/utils.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import traceback + from nvflare.fox.api.app import App from nvflare.fox.api.constants import CollabMethodArgName from nvflare.fox.api.dec import adjust_kwargs @@ -123,4 +125,5 @@ def _call_app_method(request: Message, app: App, logger) -> Message: headers={MessageHeaderKey.RETURN_CODE: ReturnCode.OK}, payload={CallReplyKey.RESULT: result} ) except Exception as ex: + traceback.print_exc() return _error_reply(f"exception {type(ex)}", logger) From 093710ec282bd2ee0f8182e8c94b4c013e9252c3 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Mon, 13 Oct 2025 17:49:57 -0400 Subject: [PATCH 036/102] added pt downloader and example --- nvflare/app_opt/pt/tensor_downloader.py | 234 ++++++++++++++++++ nvflare/fox/examples/np/algos/avg_stream.py | 1 - nvflare/fox/examples/pt/__init__.py | 0 nvflare/fox/examples/pt/pt_avg_stream.py | 190 ++++++++++++++ .../fox/examples/pt/recipe_pt_avg_stream.py | 50 ++++ nvflare/fox/examples/pt/utils.py | 34 +++ nvflare/fox/sys/tensor_downloader.py | 63 +++++ 7 files changed, 571 insertions(+), 1 deletion(-) create mode 100644 nvflare/app_opt/pt/tensor_downloader.py create mode 100644 nvflare/fox/examples/pt/__init__.py create mode 100644 nvflare/fox/examples/pt/pt_avg_stream.py create mode 100644 nvflare/fox/examples/pt/recipe_pt_avg_stream.py create mode 100644 nvflare/fox/examples/pt/utils.py create mode 100644 nvflare/fox/sys/tensor_downloader.py diff --git a/nvflare/app_opt/pt/tensor_downloader.py b/nvflare/app_opt/pt/tensor_downloader.py new file mode 100644 index 0000000000..5752983771 --- /dev/null +++ b/nvflare/app_opt/pt/tensor_downloader.py @@ -0,0 +1,234 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Any, List, Optional + +import torch +from safetensors.torch import load as load_tensors +from safetensors.torch import save as save_tensors + +from nvflare.fuel.f3.cellnet.cell import Cell +from nvflare.fuel.f3.streaming.obj_downloader import Consumer, ObjDownloader, Producer, ProduceRC, download_object + + +class _StateKey: + NUM_RECEIVED_TENSORS = "num_received_tensors" + NUM_TENSORS_INCLUDED = "num_tensors_included" + + +class _SourceTensor: + + def __init__(self, state_dict: dict[str, torch.Tensor]): + self.state_dict = state_dict + self.size = len(state_dict) + self.keys = list(state_dict.keys()) + + +class _ChunkProducer(Producer): + + def __init__(self, num_tensors_per_chunk=1): + Producer.__init__(self) + self.num_tensors_per_chunk = num_tensors_per_chunk + + def produce(self, ref_id: str, obj: _SourceTensor, state: dict, requester: str) -> (str, Any, dict): + assert isinstance(obj, _SourceTensor) + num_received_tensors = 0 + if state: + num_received_tensors = state.get(_StateKey.NUM_RECEIVED_TENSORS, 0) + + if not isinstance(num_received_tensors, int) or num_received_tensors < 0: + self.logger.error(f"bad {_StateKey.NUM_RECEIVED_TENSORS} {num_received_tensors} from {requester}") + return ProduceRC.ERROR, None, None + + if num_received_tensors >= obj.size: + # already done + return ProduceRC.EOF, None, None + + start = num_received_tensors + end = min(num_received_tensors + self.num_tensors_per_chunk, obj.size) + tensor_to_send = {} + for i in range(start, end): + next_key = obj.keys[i] + tensor_to_send[next_key] = obj.state_dict[next_key] + + chunk = save_tensors(tensor_to_send) + self.logger.debug(f"{num_received_tensors=}; sending {len(chunk)} bytes") + return ProduceRC.OK, chunk, {_StateKey.NUM_TENSORS_INCLUDED: len(tensor_to_send)} + + +class _ChunkConsumer(Consumer): + + def __init__(self, tensors_received_cb, cb_kwargs): + Consumer.__init__(self) + self.num_tensors_received = 0 + self.result = {} + self.error = None + self.tensors_received_cb = tensors_received_cb + self.cb_kwargs = cb_kwargs + if tensors_received_cb is not None and not callable(tensors_received_cb): + raise ValueError("tensors_received_cb must be callable") + + def consume(self, ref_id, state: dict, data: Any) -> dict: + assert isinstance(data, bytes) + td = load_tensors(data) + if not isinstance(td, dict): + raise ValueError("cannot load received bytes to tensors") + + num_tensors_included = state.get(_StateKey.NUM_TENSORS_INCLUDED, 0) + if num_tensors_included != len(td): + raise ValueError(f"tensor count mismatch: {num_tensors_included=}, received={len(td)}") + + if self.tensors_received_cb: + result = self.tensors_received_cb(td, **self.cb_kwargs) + if isinstance(result, dict): + self.result.update(result) + else: + self.result.update(td) + self.num_tensors_received += num_tensors_included + self.logger.debug(f"received {len(td)} tensor(s)") + return {_StateKey.NUM_RECEIVED_TENSORS: self.num_tensors_received} + + def download_failed(self, ref_id, reason: str): + self.logger.error(f"failed to download state dict with ref {ref_id}: {reason}") + self.error = reason + self.result = None + + def download_completed(self, ref_id: str): + self.logger.debug(f"received state dict with {self.num_tensors_received} tensors") + + +class TensorDownloader(ObjDownloader): + + @classmethod + def new_transaction( + cls, + cell: Cell, + num_tensors_per_chunk: int = 1, + timeout: float = 5.0, + timeout_cb=None, + **cb_kwargs, + ): + """Create a new tensor download transaction. + + Args: + cell: the cell for communication with recipients + num_tensors_per_chunk: number of tensors to send for each chunk + timeout: timeout for the transaction + timeout_cb: CB to be called when the transaction is timed out + **cb_kwargs: args to be passed to the CB + + Returns: transaction id + + The timeout_cb must follow this signature: + + cb(tx_id, state_dicts: List[dict[str, torch.Tensor], **cb_args) + + """ + return ObjDownloader.new_transaction( + cell=cell, + producer=_ChunkProducer(num_tensors_per_chunk=num_tensors_per_chunk), + timeout=timeout, + timeout_cb=cls._tx_timeout, + app_timeout_cb=timeout_cb, + **cb_kwargs, + ) + + @classmethod + def _tx_timeout(cls, tx_id: str, objs: List[Any], app_timeout_cb, **cb_kwargs): + if app_timeout_cb: + sds = [obj.state_dict for obj in objs] + app_timeout_cb(tx_id, sds, **cb_kwargs) + + @classmethod + def add_state_dict( + cls, + transaction_id: str, + state_dict: dict[str, torch.Tensor], + ref_id=None, + state_dict_downloaded_cb=None, + **cb_kwargs, + ) -> str: + """Add a file to be downloaded to the specified transaction. + + Args: + transaction_id: ID of the transaction + state_dict: state dict to be downloaded + ref_id: ref id to be used, if provided + state_dict_downloaded_cb: CB to be called when the state dict is done downloading + **cb_kwargs: args to be passed to the CB + + Returns: reference id for the file. + + The state_dict_downloaded_cb must follow this signature: + + cb(ref_id: str, to_site: str, status: str, state_dict: dict[str, torch.Tensor], **cb_kwargs) + + """ + obj = _SourceTensor(state_dict) + return ObjDownloader.add_download_object( + transaction_id=transaction_id, + obj=obj, + ref_id=ref_id, + obj_downloaded_cb=cls._source_tensor_downloaded, + app_downloaded_cb=state_dict_downloaded_cb, + **cb_kwargs, + ) + + @classmethod + def _source_tensor_downloaded( + cls, ref_id: str, to_site: str, status: str, obj: _SourceTensor, app_downloaded_cb, **cb_kwargs + ): + if app_downloaded_cb: + app_downloaded_cb(ref_id, to_site, status, obj.state_dict, **cb_kwargs) + + @classmethod + def download_state_dict( + cls, + from_fqcn: str, + ref_id: str, + per_request_timeout: float, + cell: Cell, + secure=False, + optional=False, + abort_signal=None, + tensors_received_cb=None, + **cb_kwargs, + ) -> (str, Optional[dict[str, torch.Tensor]]): + """Download the referenced state dict from the source. + + Args: + from_fqcn: FQCN of the data source. + ref_id: reference ID of the state dict to be downloaded. + per_request_timeout: timeout for requests sent to the data source. + cell: cell to be used for communicating to the data source. + secure: P2P private mode for communication + optional: supress log messages of communication + abort_signal: signal for aborting download. + tensors_received_cb: the callback to be called when one set of tensors are received + + Returns: tuple of (error message if any, downloaded state dict). + + """ + consumer = _ChunkConsumer(tensors_received_cb, cb_kwargs) + download_object( + from_fqcn=from_fqcn, + ref_id=ref_id, + consumer=consumer, + per_request_timeout=per_request_timeout, + cell=cell, + secure=secure, + optional=optional, + abort_signal=abort_signal, + ) + + return consumer.error, consumer.result diff --git a/nvflare/fox/examples/np/algos/avg_stream.py b/nvflare/fox/examples/np/algos/avg_stream.py index 5b5fcc867c..4d33df1ab9 100644 --- a/nvflare/fox/examples/np/algos/avg_stream.py +++ b/nvflare/fox/examples/np/algos/avg_stream.py @@ -22,7 +22,6 @@ from nvflare.fox.api.strategy import Strategy from nvflare.fox.examples.np.algos.utils import load_np_model, parse_array_def, save_np_model from nvflare.fox.sys.file_downloader import download_file, prepare_file_for_download -from nvflare.fuel.f3.streaming.obj_downloader import DownloadStatus from nvflare.fuel.utils.log_utils import get_obj_logger diff --git a/nvflare/fox/examples/pt/__init__.py b/nvflare/fox/examples/pt/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/nvflare/fox/examples/pt/pt_avg_stream.py b/nvflare/fox/examples/pt/pt_avg_stream.py new file mode 100644 index 0000000000..ce047c6f05 --- /dev/null +++ b/nvflare/fox/examples/pt/pt_avg_stream.py @@ -0,0 +1,190 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +import torch + +from nvflare.fox.api.app import ClientApp, ServerApp +from nvflare.fox.api.constants import ContextKey, EnvType +from nvflare.fox.api.ctx import Context +from nvflare.fox.api.dec import collab +from nvflare.fox.api.group import all_clients +from nvflare.fox.api.strategy import Strategy +from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples.pt.utils import parse_state_dict +from nvflare.fox.sim.simulator import Simulator +from nvflare.fox.sys.tensor_downloader import download_state_dict, prepare_for_download +from nvflare.fuel.utils.log_utils import get_obj_logger + + +class _AggrResult: + + def __init__(self): + self.total = {} + self.count = 0 + + +class PTFedAvgStream(Strategy): + + def __init__(self, initial_model, num_rounds=10, timeout=2.0): + self.num_rounds = num_rounds + self.initial_model = initial_model + self.timeout = timeout + self.name = "PTFedAvgStream" + self.logger = get_obj_logger(self) + self._init_model = parse_state_dict(initial_model) + + def execute(self, context: Context): + self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") + current_model = context.get_prop(ContextKey.INPUT, self._init_model) + for i in range(self.num_rounds): + current_model = self._do_one_round(i, current_model, context) + self.logger.info(f"FINAL MODEL: {current_model}") + return current_model + + def _do_one_round(self, r, current_model, ctx: Context): + aggr_result = _AggrResult() + + # pretend the model is big + if ctx.env_type == EnvType.SYSTEM: + model = prepare_for_download( + state_dict=current_model, + ctx=ctx, + timeout=5.0, + num_tensors_per_chunk=2, + ) + model_type = "ref" + self.logger.info(f"prepared model as ref: {model}") + else: + model = current_model + model_type = "model" + + all_clients( + ctx, + process_resp_cb=self._accept_train_result, + aggr_result=aggr_result, + ).train(r, model, model_type) + + if aggr_result.count == 0: + return None + else: + result = {} + for k, v in aggr_result.total.items(): + result[k] = torch.div(v, aggr_result.count) + self.logger.info(f"[{ctx.header_str()}] round {r}: aggr result from {aggr_result.count} clients: {result}") + return result + + def _accept_train_result(self, result, aggr_result: _AggrResult, context: Context): + self.logger.info(f"[{context.header_str()}] got train result from {context.caller}: {result}") + + model, model_type = result + if model_type == "ref": + err, model = download_state_dict( + ref=model, + per_request_timeout=5.0, + ctx=context, + tensors_received_cb=self._aggregate_tensors, + aggr_result=aggr_result, + context=context, + ) + if err: + raise RuntimeError(f"failed to download model {model}: {err}") + else: + for k, v in model.items(): + if k not in aggr_result.total: + aggr_result.total[k] = v + else: + aggr_result.total[k] += v + + aggr_result.count += 1 + return None + + def _aggregate_tensors(self, td: dict[str, torch.Tensor], aggr_result: _AggrResult, context: Context): + self.logger.info(f"[{context.header_str()}] aggregating received tensor: {td}") + for k, v in td.items(): + if k not in aggr_result.total: + aggr_result.total[k] = v + else: + aggr_result.total[k] += v + + +class PTTrainer(ClientApp): + + def __init__(self, delta: float): + ClientApp.__init__(self) + self.delta = delta + + @collab + def train(self, current_round, weights, model_type: str, context: Context): + if context.is_aborted(): + self.logger.debug("training aborted") + return 0 + self.logger.debug(f"[{context.header_str()}] training round {current_round}: {model_type=} {weights=}") + if model_type == "ref": + err, model = download_state_dict(ref=weights, per_request_timeout=5.0, ctx=context) + if err: + raise RuntimeError(f"failed to download model {weights}: {err}") + self.logger.info(f"downloaded model {model}") + weights = model + + result = {} + for k, v in weights.items(): + result[k] = v + self.delta + + if model_type == "ref": + # stream it + model = prepare_for_download( + state_dict=result, + ctx=context, + timeout=5.0, + num_tensors_per_chunk=2, + ) + model_type = "ref" + self.logger.info(f"prepared result as ref: {model}") + else: + model = result + model_type = "model" + return model, model_type + + +def main(): + simple_logging(logging.DEBUG) + + server_app = ServerApp( + strategy_name="fed_avg_in_time", + strategy=PTFedAvgStream( + initial_model={ + "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "y": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "z": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + }, + num_rounds=4, + ), + ) + + client_app = PTTrainer(delta=1.0) + + simulator = Simulator( + root_dir="/tmp/fox", + experiment_name="pt_fedavg_intime", + server_app=server_app, + client_app=client_app, + num_clients=2, + ) + + simulator.run() + + +if __name__ == "__main__": + main() diff --git a/nvflare/fox/examples/pt/recipe_pt_avg_stream.py b/nvflare/fox/examples/pt/recipe_pt_avg_stream.py new file mode 100644 index 0000000000..8159e2f202 --- /dev/null +++ b/nvflare/fox/examples/pt/recipe_pt_avg_stream.py @@ -0,0 +1,50 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from nvflare.fox.api.app import ServerApp +from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples.pt.pt_avg_stream import PTFedAvgStream, PTTrainer +from nvflare.fox.sys.recipe import FoxRecipe + +JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" + + +def main(): + simple_logging(logging.DEBUG) + + server_app = ServerApp( + strategy_name="fedavg_stream", + strategy=PTFedAvgStream( + initial_model={ + "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "y": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "z": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + }, + num_rounds=2, + ), + ) + + client_app = PTTrainer(delta=1.0) + + recipe = FoxRecipe( + job_name="pt_fedavg_stream", + server_app=server_app, + client_app=client_app, + ) + recipe.export(JOB_ROOT_DIR) + + +if __name__ == "__main__": + main() diff --git a/nvflare/fox/examples/pt/utils.py b/nvflare/fox/examples/pt/utils.py new file mode 100644 index 0000000000..405ba5f338 --- /dev/null +++ b/nvflare/fox/examples/pt/utils.py @@ -0,0 +1,34 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch + + +def parse_array_def(array_def): + if array_def is None: + return array_def + + if isinstance(array_def, torch.Tensor): + return array_def + + if isinstance(array_def, list): + return torch.Tensor(array_def) + else: + raise ValueError(f"unsupported array def: {array_def}") + + +def parse_state_dict(d: dict[str, list]): + result = {} + for k, v in d.items(): + result[k] = parse_array_def(v) + return result diff --git a/nvflare/fox/sys/tensor_downloader.py b/nvflare/fox/sys/tensor_downloader.py new file mode 100644 index 0000000000..0fe96908c2 --- /dev/null +++ b/nvflare/fox/sys/tensor_downloader.py @@ -0,0 +1,63 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch + +from nvflare.app_opt.pt.tensor_downloader import TensorDownloader +from nvflare.fox.api.ctx import Context +from nvflare.fox.sys.backend import SysBackend + + +def prepare_for_download( + state_dict: dict[str, torch.Tensor], + timeout: float, + ctx: Context, + num_tensors_per_chunk: int = 1, + state_dict_downloaded_cb=None, + **cb_kwargs, +): + backend = ctx.backend + if not isinstance(backend, SysBackend): + raise ValueError(f"backend must be SysBackend but got {type(backend)}") + + cell = backend.cell + tx_id = TensorDownloader.new_transaction( + cell=cell, + num_tensors_per_chunk=num_tensors_per_chunk, + timeout=timeout, + timeout_cb=None, + ) + rid = TensorDownloader.add_state_dict( + transaction_id=tx_id, + state_dict=state_dict, + state_dict_downloaded_cb=state_dict_downloaded_cb, + **cb_kwargs, + ) + return {"source": cell.get_fqcn(), "rid": rid} + + +def download_state_dict(ref: dict, per_request_timeout: float, ctx: Context, tensors_received_cb=None, **cb_kwargs): + backend = ctx.backend + if not isinstance(backend, SysBackend): + raise ValueError(f"backend must be SysBackend but got {type(backend)}") + + err, result = TensorDownloader.download_state_dict( + from_fqcn=ref.get("source"), + ref_id=ref.get("rid"), + per_request_timeout=per_request_timeout, + cell=backend.cell, + abort_signal=ctx.abort_signal, + tensors_received_cb=tensors_received_cb, + **cb_kwargs, + ) + return err, result From 84e5b7437b64012f0961944c94fb28ec659bfdad Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Tue, 14 Oct 2025 12:32:48 -0400 Subject: [PATCH 037/102] add comment --- nvflare/fox/examples/pt/pt_avg_stream.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nvflare/fox/examples/pt/pt_avg_stream.py b/nvflare/fox/examples/pt/pt_avg_stream.py index ce047c6f05..013fdf9511 100644 --- a/nvflare/fox/examples/pt/pt_avg_stream.py +++ b/nvflare/fox/examples/pt/pt_avg_stream.py @@ -30,6 +30,8 @@ class _AggrResult: + # Used for aggregation + def __init__(self): self.total = {} self.count = 0 From d746655597b2deb5fabea2e5afc8c5c993430635 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Tue, 14 Oct 2025 14:59:18 -0400 Subject: [PATCH 038/102] support multiple tensor chunks --- nvflare/app_opt/pt/tensor_downloader.py | 37 ++++++++++++------- .../fox/examples/np/recipe_fed_avg_intime.py | 2 +- nvflare/fox/examples/pt/pt_avg_stream.py | 15 ++++---- .../fox/examples/pt/recipe_pt_avg_stream.py | 2 +- 4 files changed, 33 insertions(+), 23 deletions(-) diff --git a/nvflare/app_opt/pt/tensor_downloader.py b/nvflare/app_opt/pt/tensor_downloader.py index 5752983771..be683fa511 100644 --- a/nvflare/app_opt/pt/tensor_downloader.py +++ b/nvflare/app_opt/pt/tensor_downloader.py @@ -56,14 +56,20 @@ def produce(self, ref_id: str, obj: _SourceTensor, state: dict, requester: str) start = num_received_tensors end = min(num_received_tensors + self.num_tensors_per_chunk, obj.size) - tensor_to_send = {} + chunks = [] + total_len = 0 + + # NOTE: we call save_tensors for one tensor at a time. This is because save_tensors won't work for + # multiple tensors if there are shared memory among them! for i in range(start, end): next_key = obj.keys[i] - tensor_to_send[next_key] = obj.state_dict[next_key] + tensor_to_send = {next_key: obj.state_dict[next_key]} + chunk = save_tensors(tensor_to_send) + chunks.append(chunk) + total_len += len(chunk) - chunk = save_tensors(tensor_to_send) - self.logger.debug(f"{num_received_tensors=}; sending {len(chunk)} bytes") - return ProduceRC.OK, chunk, {_StateKey.NUM_TENSORS_INCLUDED: len(tensor_to_send)} + self.logger.debug(f"{num_received_tensors=}; sending {total_len} bytes") + return ProduceRC.OK, chunks, {_StateKey.NUM_TENSORS_INCLUDED: end - start} class _ChunkConsumer(Consumer): @@ -79,23 +85,26 @@ def __init__(self, tensors_received_cb, cb_kwargs): raise ValueError("tensors_received_cb must be callable") def consume(self, ref_id, state: dict, data: Any) -> dict: - assert isinstance(data, bytes) - td = load_tensors(data) - if not isinstance(td, dict): - raise ValueError("cannot load received bytes to tensors") + assert isinstance(data, list) + tensors = {} + for chunk in data: + td = load_tensors(chunk) + if not isinstance(td, dict): + raise ValueError("cannot load received bytes to tensors") + tensors.update(td) num_tensors_included = state.get(_StateKey.NUM_TENSORS_INCLUDED, 0) - if num_tensors_included != len(td): - raise ValueError(f"tensor count mismatch: {num_tensors_included=}, received={len(td)}") + if num_tensors_included != len(tensors): + raise ValueError(f"tensor count mismatch: {num_tensors_included=}, received={len(tensors)}") if self.tensors_received_cb: - result = self.tensors_received_cb(td, **self.cb_kwargs) + result = self.tensors_received_cb(tensors, **self.cb_kwargs) if isinstance(result, dict): self.result.update(result) else: - self.result.update(td) + self.result.update(tensors) self.num_tensors_received += num_tensors_included - self.logger.debug(f"received {len(td)} tensor(s)") + self.logger.debug(f"received {len(tensors)} tensor(s)") return {_StateKey.NUM_RECEIVED_TENSORS: self.num_tensors_received} def download_failed(self, ref_id, reason: str): diff --git a/nvflare/fox/examples/np/recipe_fed_avg_intime.py b/nvflare/fox/examples/np/recipe_fed_avg_intime.py index f889a2b2e2..b03f9d834e 100644 --- a/nvflare/fox/examples/np/recipe_fed_avg_intime.py +++ b/nvflare/fox/examples/np/recipe_fed_avg_intime.py @@ -21,7 +21,7 @@ from nvflare.fox.examples.np.algos.widgets import MetricReceiver from nvflare.fox.sys.recipe import FoxRecipe -JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" +JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/fox/prod_00/admin@nvidia.com/transfer" def main(): diff --git a/nvflare/fox/examples/pt/pt_avg_stream.py b/nvflare/fox/examples/pt/pt_avg_stream.py index 013fdf9511..26defa5e7a 100644 --- a/nvflare/fox/examples/pt/pt_avg_stream.py +++ b/nvflare/fox/examples/pt/pt_avg_stream.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging +import threading import torch @@ -30,11 +31,10 @@ class _AggrResult: - # Used for aggregation - def __init__(self): self.total = {} self.count = 0 + self.lock = threading.Lock() # ensure update integrity class PTFedAvgStream(Strategy): @@ -114,11 +114,12 @@ def _accept_train_result(self, result, aggr_result: _AggrResult, context: Contex def _aggregate_tensors(self, td: dict[str, torch.Tensor], aggr_result: _AggrResult, context: Context): self.logger.info(f"[{context.header_str()}] aggregating received tensor: {td}") - for k, v in td.items(): - if k not in aggr_result.total: - aggr_result.total[k] = v - else: - aggr_result.total[k] += v + with aggr_result.lock: + for k, v in td.items(): + if k not in aggr_result.total: + aggr_result.total[k] = v + else: + aggr_result.total[k] += v class PTTrainer(ClientApp): diff --git a/nvflare/fox/examples/pt/recipe_pt_avg_stream.py b/nvflare/fox/examples/pt/recipe_pt_avg_stream.py index 8159e2f202..5479f8bba4 100644 --- a/nvflare/fox/examples/pt/recipe_pt_avg_stream.py +++ b/nvflare/fox/examples/pt/recipe_pt_avg_stream.py @@ -18,7 +18,7 @@ from nvflare.fox.examples.pt.pt_avg_stream import PTFedAvgStream, PTTrainer from nvflare.fox.sys.recipe import FoxRecipe -JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" +JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/fox/prod_00/admin@nvidia.com/transfer" def main(): From 9ceb73e44408fbc3655304d0ae4aae5c2be4178d Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Fri, 17 Oct 2025 08:32:24 +0800 Subject: [PATCH 039/102] pt example --- nvflare/fox/api/workspace.py | 14 +- nvflare/fox/examples/pt/filters.py | 79 +++++++++++ nvflare/fox/examples/pt/pt_avg_filter.py | 134 ++++++++++++++++++ nvflare/fox/examples/pt/pt_avg_stream.py | 17 ++- .../fox/examples/pt/recipe_pt_avg_filter.py | 67 +++++++++ nvflare/fox/sim/ws.py | 1 + ...downloader.py => state_dict_downloader.py} | 64 +++++---- nvflare/fox/sys/ws.py | 1 + 8 files changed, 349 insertions(+), 28 deletions(-) create mode 100644 nvflare/fox/examples/pt/filters.py create mode 100644 nvflare/fox/examples/pt/pt_avg_filter.py create mode 100644 nvflare/fox/examples/pt/recipe_pt_avg_filter.py rename nvflare/fox/sys/{tensor_downloader.py => state_dict_downloader.py} (51%) diff --git a/nvflare/fox/api/workspace.py b/nvflare/fox/api/workspace.py index caad378850..21973d4ef6 100644 --- a/nvflare/fox/api/workspace.py +++ b/nvflare/fox/api/workspace.py @@ -17,6 +17,14 @@ class Workspace(ABC): + def __init__(self): + self.resource_dirs = {} + + def add_resource_dir(self, name, resource_dir): + if not os.path.isdir(resource_dir): + raise ValueError(f"Resource dir {resource_dir} does not exist") + self.resource_dirs[name] = resource_dir + @abstractmethod def get_root_dir(self) -> str: pass @@ -26,7 +34,11 @@ def get_work_dir(self) -> str: pass def get_subdir(self, name: str, create: bool = True) -> str: - p = f"{self.get_work_dir()}/{name}" + resource_dir = self.resource_dirs.get(name) + if resource_dir: + return resource_dir + + p = os.path.join(self.get_work_dir(), name) if not os.path.exists(p) and create: os.makedirs(p, exist_ok=True) return p diff --git a/nvflare/fox/examples/pt/filters.py b/nvflare/fox/examples/pt/filters.py new file mode 100644 index 0000000000..4cb0a08959 --- /dev/null +++ b/nvflare/fox/examples/pt/filters.py @@ -0,0 +1,79 @@ +from typing import Any + +from nvflare.fox.api.ctx import Context +from nvflare.fox.api.filter import CallFilter, ResultFilter +from nvflare.fox.sys.state_dict_downloader import StateDictDownloader, download_state_dict +from nvflare.fuel.utils.log_utils import get_obj_logger + + +class OutgoingModelCallFilter(CallFilter): + + def __init__(self, model_arg_name: str): + super().__init__() + self.model_arg_name = model_arg_name + + def filter_call(self, func_kwargs: dict, context: Context): + arg_value = func_kwargs.get(self.model_arg_name) + if not arg_value: + return func_kwargs + + downloader = StateDictDownloader( + state_dict=arg_value, + ctx=context, + timeout=5.0, + num_tensors_per_chunk=2, + ) + model = downloader.get_ref() + func_kwargs[self.model_arg_name] = model + return func_kwargs + + +class IncomingModelCallFilter(CallFilter): + + def __init__(self, model_arg_name: str): + super().__init__() + self.model_arg_name = model_arg_name + self.logger = get_obj_logger(self) + + def filter_call(self, func_kwargs: dict, context: Context): + arg_value = func_kwargs.get(self.model_arg_name) + if not arg_value: + return func_kwargs + + err, model = download_state_dict(ref=arg_value, ctx=context, per_request_timeout=5.0) + if err: + self.logger.error(f"error filtering call arg {arg_value}: {err}") + else: + func_kwargs[self.model_arg_name] = model + return func_kwargs + return func_kwargs + + +class OutgoingModelResultFilter(ResultFilter): + + def filter_result(self, result: Any, context: Context): + if not isinstance(result, dict): + return result + + downloader = StateDictDownloader( + state_dict=result, + ctx=context, + timeout=5.0, + num_tensors_per_chunk=2, + ) + return downloader.get_ref() + + +class IncomingModelResultFilter(ResultFilter): + + def __init__(self): + super().__init__() + self.logger = get_obj_logger(self) + + def filter_result(self, result: Any, context: Context): + err, model = download_state_dict(ref=result, ctx=context, per_request_timeout=5.0) + if err: + self.logger.error(f"error filtering result {result}: {err}") + return result + else: + return model diff --git a/nvflare/fox/examples/pt/pt_avg_filter.py b/nvflare/fox/examples/pt/pt_avg_filter.py new file mode 100644 index 0000000000..c019701a66 --- /dev/null +++ b/nvflare/fox/examples/pt/pt_avg_filter.py @@ -0,0 +1,134 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import threading + +import torch + +from nvflare.fox.api.app import ClientApp, ServerApp +from nvflare.fox.api.constants import ContextKey, EnvType +from nvflare.fox.api.ctx import Context +from nvflare.fox.api.dec import collab +from nvflare.fox.api.group import all_clients +from nvflare.fox.api.strategy import Strategy +from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples.pt.utils import parse_state_dict +from nvflare.fox.sim.simulator import Simulator +from nvflare.fuel.utils.log_utils import get_obj_logger + + +class _AggrResult: + + def __init__(self): + self.total = {} + self.count = 0 + self.lock = threading.Lock() # ensure update integrity + + +class PTFedAvg(Strategy): + + def __init__(self, initial_model, num_rounds=10, timeout=2.0): + self.num_rounds = num_rounds + self.initial_model = initial_model + self.timeout = timeout + self.name = "PTFedAvgStream" + self.logger = get_obj_logger(self) + self._init_model = parse_state_dict(initial_model) + + def execute(self, context: Context): + self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") + current_model = context.get_prop(ContextKey.INPUT, self._init_model) + for i in range(self.num_rounds): + current_model = self._do_one_round(i, current_model, context) + self.logger.info(f"FINAL MODEL: {current_model}") + return current_model + + def _do_one_round(self, r, current_model, ctx: Context): + aggr_result = _AggrResult() + + all_clients( + ctx, + process_resp_cb=self._accept_train_result, + aggr_result=aggr_result, + ).train(r, current_model) + + if aggr_result.count == 0: + return None + else: + result = {} + for k, v in aggr_result.total.items(): + result[k] = torch.div(v, aggr_result.count) + self.logger.info(f"[{ctx.header_str()}] round {r}: aggr result from {aggr_result.count} clients: {result}") + return result + + def _accept_train_result(self, result, aggr_result: _AggrResult, context: Context): + self.logger.info(f"[{context.header_str()}] got train result from {context.caller}: {result}") + + for k, v in result.items(): + if k not in aggr_result.total: + aggr_result.total[k] = v + else: + aggr_result.total[k] += v + + aggr_result.count += 1 + return None + + +class PTTrainer(ClientApp): + + def __init__(self, delta: float): + ClientApp.__init__(self) + self.delta = delta + + @collab + def train(self, current_round, weights, context: Context): + if context.is_aborted(): + self.logger.debug("training aborted") + return 0 + self.logger.debug(f"[{context.header_str()}] training round {current_round}: {weights=}") + + result = {} + for k, v in weights.items(): + result[k] = v + self.delta + return result + + +def main(): + simple_logging(logging.DEBUG) + + server_app = ServerApp( + strategy_name="fed_avg_in_time", + strategy=PTFedAvg( + initial_model={ + "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "y": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "z": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + }, + num_rounds=4, + ), + ) + + simulator = Simulator( + root_dir="/tmp/fox", + experiment_name="pt_fedavg_intime", + server_app=server_app, + client_app=client_app, + num_clients=2, + ) + + simulator.run() + + +if __name__ == "__main__": + main() diff --git a/nvflare/fox/examples/pt/pt_avg_stream.py b/nvflare/fox/examples/pt/pt_avg_stream.py index 26defa5e7a..a06112de64 100644 --- a/nvflare/fox/examples/pt/pt_avg_stream.py +++ b/nvflare/fox/examples/pt/pt_avg_stream.py @@ -25,7 +25,7 @@ from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.pt.utils import parse_state_dict from nvflare.fox.sim.simulator import Simulator -from nvflare.fox.sys.tensor_downloader import download_state_dict, prepare_for_download +from nvflare.fox.sys.state_dict_downloader import StateDictDownloader, download_state_dict from nvflare.fuel.utils.log_utils import get_obj_logger @@ -59,14 +59,16 @@ def _do_one_round(self, r, current_model, ctx: Context): aggr_result = _AggrResult() # pretend the model is big + downloader = None if ctx.env_type == EnvType.SYSTEM: - model = prepare_for_download( + downloader = StateDictDownloader( state_dict=current_model, ctx=ctx, timeout=5.0, num_tensors_per_chunk=2, ) model_type = "ref" + model = downloader.get_ref() self.logger.info(f"prepared model as ref: {model}") else: model = current_model @@ -78,6 +80,9 @@ def _do_one_round(self, r, current_model, ctx: Context): aggr_result=aggr_result, ).train(r, model, model_type) + if downloader: + downloader.clear() + if aggr_result.count == 0: return None else: @@ -147,19 +152,25 @@ def train(self, current_round, weights, model_type: str, context: Context): if model_type == "ref": # stream it - model = prepare_for_download( + downloader = StateDictDownloader( state_dict=result, ctx=context, timeout=5.0, num_tensors_per_chunk=2, + state_dict_downloaded_cb=self._download_done, ) model_type = "ref" + model = downloader.get_ref() self.logger.info(f"prepared result as ref: {model}") else: model = result model_type = "model" return model, model_type + def _download_done(self, downloader: StateDictDownloader, to_site: str, status: str, ctx: Context): + self.logger.info(f"[{ctx.header_str()}] finished downloading model to {to_site}: {status}") + downloader.clear() + def main(): simple_logging(logging.DEBUG) diff --git a/nvflare/fox/examples/pt/recipe_pt_avg_filter.py b/nvflare/fox/examples/pt/recipe_pt_avg_filter.py new file mode 100644 index 0000000000..acf69e9029 --- /dev/null +++ b/nvflare/fox/examples/pt/recipe_pt_avg_filter.py @@ -0,0 +1,67 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from nvflare.fox.api.app import ServerApp +from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples.pt.filters import ( + IncomingModelCallFilter, + IncomingModelResultFilter, + OutgoingModelCallFilter, + OutgoingModelResultFilter, +) +from nvflare.fox.examples.pt.pt_avg_filter import PTFedAvg, PTTrainer +from nvflare.fox.sys.recipe import FoxRecipe + +JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/fox/prod_00/admin@nvidia.com/transfer" + + +def main(): + simple_logging(logging.DEBUG) + + server_app = ServerApp( + strategy_name="fedavg_stream", + strategy=PTFedAvg( + initial_model={ + "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "y": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "z": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + }, + num_rounds=2, + ), + ) + + server_app.add_outgoing_call_filters( + pattern="*.train", + filters=[OutgoingModelCallFilter("weights")], + ) + server_app.add_incoming_result_filters(pattern="*.train", filters=[IncomingModelResultFilter()]) + + client_app = PTTrainer(delta=1.0) + client_app.add_incoming_call_filters( + pattern="*.train", + filters=[IncomingModelCallFilter("weights")], + ) + client_app.add_outgoing_result_filters(pattern="*.train", filters=[OutgoingModelResultFilter()]) + + recipe = FoxRecipe( + job_name="pt_fedavg_filter", + server_app=server_app, + client_app=client_app, + ) + recipe.export(JOB_ROOT_DIR) + + +if __name__ == "__main__": + main() diff --git a/nvflare/fox/sim/ws.py b/nvflare/fox/sim/ws.py index 6a7bedf41a..2a57cfbfdb 100644 --- a/nvflare/fox/sim/ws.py +++ b/nvflare/fox/sim/ws.py @@ -19,6 +19,7 @@ class SimWorkspace(Workspace): def __init__(self, root_dir: str, experiment_name: str, exp_id: str, site_name: str): + super().__init__() if not isinstance(root_dir, str): raise ValueError(f"root_dir must be str but got {type(root_dir)}") diff --git a/nvflare/fox/sys/tensor_downloader.py b/nvflare/fox/sys/state_dict_downloader.py similarity index 51% rename from nvflare/fox/sys/tensor_downloader.py rename to nvflare/fox/sys/state_dict_downloader.py index 0fe96908c2..e524ee3ceb 100644 --- a/nvflare/fox/sys/tensor_downloader.py +++ b/nvflare/fox/sys/state_dict_downloader.py @@ -18,32 +18,48 @@ from nvflare.fox.sys.backend import SysBackend -def prepare_for_download( - state_dict: dict[str, torch.Tensor], - timeout: float, - ctx: Context, - num_tensors_per_chunk: int = 1, - state_dict_downloaded_cb=None, - **cb_kwargs, -): - backend = ctx.backend - if not isinstance(backend, SysBackend): - raise ValueError(f"backend must be SysBackend but got {type(backend)}") +class StateDictDownloader: - cell = backend.cell - tx_id = TensorDownloader.new_transaction( - cell=cell, - num_tensors_per_chunk=num_tensors_per_chunk, - timeout=timeout, - timeout_cb=None, - ) - rid = TensorDownloader.add_state_dict( - transaction_id=tx_id, - state_dict=state_dict, - state_dict_downloaded_cb=state_dict_downloaded_cb, + def __init__( + self, + timeout: float, + ctx: Context, + state_dict: dict[str, torch.Tensor], + num_tensors_per_chunk: int = 1, + state_dict_downloaded_cb=None, **cb_kwargs, - ) - return {"source": cell.get_fqcn(), "rid": rid} + ): + self.state_dict_downloaded_cb = state_dict_downloaded_cb + self.cb_kwargs = cb_kwargs + self.ctx = ctx + + backend = ctx.backend + if not isinstance(backend, SysBackend): + raise ValueError(f"backend must be SysBackend but got {type(backend)}") + + self.cell = backend.cell + self.tx_id = TensorDownloader.new_transaction( + cell=self.cell, + num_tensors_per_chunk=num_tensors_per_chunk, + timeout=timeout, + timeout_cb=None, + ) + + self.rid = TensorDownloader.add_state_dict( + transaction_id=self.tx_id, + state_dict=state_dict, + state_dict_downloaded_cb=self._state_dict_downloaded_cb, + ) + + def _state_dict_downloaded_cb(self, rid, to_site, status, obj): + if self.state_dict_downloaded_cb: + self.state_dict_downloaded_cb(self, to_site, status, self.ctx, **self.cb_kwargs) + + def get_ref(self): + return {"source": self.cell.get_fqcn(), "rid": self.rid} + + def clear(self): + TensorDownloader.delete_transaction(self.tx_id) def download_state_dict(ref: dict, per_request_timeout: float, ctx: Context, tensors_received_cb=None, **cb_kwargs): diff --git a/nvflare/fox/sys/ws.py b/nvflare/fox/sys/ws.py index f5363d0fdf..cea2ffd89e 100644 --- a/nvflare/fox/sys/ws.py +++ b/nvflare/fox/sys/ws.py @@ -19,6 +19,7 @@ class SysWorkspace(Workspace): def __init__(self, fl_ctx: FLContext): + super().__init__() ws_obj = fl_ctx.get_workspace() assert isinstance(ws_obj, FlareWorkspace) self.flare_ws = ws_obj From 2b3c122f9714ab2770c6dca88943ff7b3367164c Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Fri, 17 Oct 2025 09:31:50 +0800 Subject: [PATCH 040/102] rename to fox_init --- nvflare/fox/api/app.py | 29 ++++++++++++++---------- nvflare/fox/api/group.py | 2 ++ nvflare/fox/examples/np/algos/widgets.py | 2 +- nvflare/fox/examples/pt/pt_avg_filter.py | 2 ++ 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/nvflare/fox/api/app.py b/nvflare/fox/api/app.py index 79b042753a..3dc08a5330 100644 --- a/nvflare/fox/api/app.py +++ b/nvflare/fox/api/app.py @@ -48,8 +48,12 @@ def __init__(self): self._outgoing_result_filter_chains = [] self._collab_interface = {"": get_object_collab_interface(self)} self._workspace = None + self._managed_objects = {} # id => obj self.logger = get_obj_logger(self) + def _add_managed_object(self, obj): + self._managed_objects[id(obj)] = obj + def get_backend(self): return self._me.backend @@ -62,8 +66,7 @@ def get_server_proxy(self): def get_client_proxies(self): return copy.copy(self.clients) - @staticmethod - def _add_filters(pattern: str, filters, to_list: list, filter_type): + def _add_filters(self, pattern: str, filters, to_list: list, filter_type): if not filters: return @@ -73,6 +76,7 @@ def _add_filters(pattern: str, filters, to_list: list, filter_type): for i, f in enumerate(filters): if not isinstance(f, filter_type): raise ValueError(f"filter {i} must be {filter_type} but got {type(f)}") + self._add_managed_object(f) chain = FilterChain(pattern, filter_type) chain.add_filters(filters) @@ -184,6 +188,7 @@ def add_collab_object(self, name: str, obj): setattr(self, name, obj) self._collab_objs[name] = obj self._collab_interface[name] = get_object_collab_interface(obj) + self._add_managed_object(obj) def get_collab_objects(self): return self._collab_objs @@ -240,20 +245,20 @@ def find_collab_method(self, target_obj, method_name): return m return None - def initialize_app(self, context: Context): - pass + def _fox_init(self, obj, ctx: Context): + init_func = getattr(obj, "fox_init", None) + if init_func and callable(init_func): + self.logger.info(f"fox_init object {obj.__class__.__name__}") + kwargs = {CollabMethodArgName.CONTEXT: ctx} + check_context_support(init_func, kwargs) + init_func(**kwargs) def initialize(self, context: Context): - self.initialize_app(context) + self._fox_init(self, context) # initialize target objects - for name, obj in self._collab_objs.items(): - init_func = getattr(obj, "initialize", None) - if init_func and callable(init_func): - self.logger.info(f"initializing target object {name}") - kwargs = {CollabMethodArgName.CONTEXT: context} - check_context_support(init_func, kwargs) - init_func(**kwargs) + for obj in self._managed_objects.values(): + self._fox_init(obj, context) def new_context(self, caller: str, callee: str, props: dict = None): return Context(self, caller, callee, self._abort_signal, props) diff --git a/nvflare/fox/api/group.py b/nvflare/fox/api/group.py index b7ec79d865..0c2c02c477 100644 --- a/nvflare/fox/api/group.py +++ b/nvflare/fox/api/group.py @@ -19,6 +19,7 @@ from nvflare.apis.signal import Signal from nvflare.fuel.utils.log_utils import get_obj_logger +from .app import App from .constants import CollabMethodArgName, CollabMethodOptionName from .ctx import Context from .proxy import Proxy @@ -80,6 +81,7 @@ def method(*args, **kwargs): self._logger.debug(f"[{ctx.header_str()}] calling {func_name} of group {[p.name for p in self._proxies]}") # apply outgoing call filters + assert isinstance(self._app, App) adj_kwargs = self._app.apply_outgoing_call_filters(p.target_name, func_name, adj_kwargs, ctx) check_call_args(func_name, func_itf, adj_args, adj_kwargs) diff --git a/nvflare/fox/examples/np/algos/widgets.py b/nvflare/fox/examples/np/algos/widgets.py index aa27e286cf..283c4f7d49 100644 --- a/nvflare/fox/examples/np/algos/widgets.py +++ b/nvflare/fox/examples/np/algos/widgets.py @@ -25,7 +25,7 @@ def __init__(self): def accept_metric(self, metrics: dict, context: Context): self.logger.info(f"[{context.callee}] received metric report from {context.caller}: {metrics}") - def initialize(self, context: Context): + def fox_init(self, context: Context): context.app.register_event_handler("metrics", self._accept_metric) self.logger.info("MetricReceiver initialized!") diff --git a/nvflare/fox/examples/pt/pt_avg_filter.py b/nvflare/fox/examples/pt/pt_avg_filter.py index c019701a66..aa0f647902 100644 --- a/nvflare/fox/examples/pt/pt_avg_filter.py +++ b/nvflare/fox/examples/pt/pt_avg_filter.py @@ -119,6 +119,8 @@ def main(): ), ) + client_app = PTTrainer(delta=1.0) + simulator = Simulator( root_dir="/tmp/fox", experiment_name="pt_fedavg_intime", From 6550f66df591f5f3820a6ce4e02a0276568d85e9 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Mon, 20 Oct 2025 14:51:36 +0800 Subject: [PATCH 041/102] add cacheed downloader --- nvflare/app_opt/pt/tensor_downloader.py | 169 ++++--------- nvflare/fox/api/app.py | 4 +- nvflare/fox/api/ctx.py | 10 +- nvflare/fox/api/group.py | 14 +- nvflare/fox/examples/pt/filters.py | 24 +- nvflare/fox/examples/pt/pt_avg_stream.py | 44 ++-- ...dict_downloader.py => model_downloader.py} | 45 ++-- .../f3/streaming/cached_obj_downloader.py | 234 ++++++++++++++++++ nvflare/fuel/f3/streaming/obj_downloader.py | 110 +++++++- nvflare/fuel/f3/streaming/smod.py | 105 ++++++++ 10 files changed, 559 insertions(+), 200 deletions(-) rename nvflare/fox/sys/{state_dict_downloader.py => model_downloader.py} (58%) create mode 100644 nvflare/fuel/f3/streaming/cached_obj_downloader.py create mode 100644 nvflare/fuel/f3/streaming/smod.py diff --git a/nvflare/app_opt/pt/tensor_downloader.py b/nvflare/app_opt/pt/tensor_downloader.py index be683fa511..d88651c97b 100644 --- a/nvflare/app_opt/pt/tensor_downloader.py +++ b/nvflare/app_opt/pt/tensor_downloader.py @@ -11,118 +11,69 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, List, Optional +from typing import Any, List, Optional, Tuple import torch from safetensors.torch import load as load_tensors from safetensors.torch import save as save_tensors from nvflare.fuel.f3.cellnet.cell import Cell -from nvflare.fuel.f3.streaming.obj_downloader import Consumer, ObjDownloader, Producer, ProduceRC, download_object +from nvflare.fuel.f3.streaming.cached_obj_downloader import CachedObjDownloader, CountableObject, ItemConsumer -class _StateKey: - NUM_RECEIVED_TENSORS = "num_received_tensors" - NUM_TENSORS_INCLUDED = "num_tensors_included" +class _TensorSource(CountableObject): + def __init__(self, tensors: dict[str, torch.Tensor]): + self.tensors = tensors + self.size = len(tensors) + self.keys = list(tensors.keys()) -class _SourceTensor: + def get_item_count(self) -> int: + return self.size - def __init__(self, state_dict: dict[str, torch.Tensor]): - self.state_dict = state_dict - self.size = len(state_dict) - self.keys = list(state_dict.keys()) + def produce_item(self, index: int) -> Any: + key = self.keys[index] + tensor_to_send = {key: self.tensors[key]} + return save_tensors(tensor_to_send) -class _ChunkProducer(Producer): - - def __init__(self, num_tensors_per_chunk=1): - Producer.__init__(self) - self.num_tensors_per_chunk = num_tensors_per_chunk - - def produce(self, ref_id: str, obj: _SourceTensor, state: dict, requester: str) -> (str, Any, dict): - assert isinstance(obj, _SourceTensor) - num_received_tensors = 0 - if state: - num_received_tensors = state.get(_StateKey.NUM_RECEIVED_TENSORS, 0) - - if not isinstance(num_received_tensors, int) or num_received_tensors < 0: - self.logger.error(f"bad {_StateKey.NUM_RECEIVED_TENSORS} {num_received_tensors} from {requester}") - return ProduceRC.ERROR, None, None - - if num_received_tensors >= obj.size: - # already done - return ProduceRC.EOF, None, None - - start = num_received_tensors - end = min(num_received_tensors + self.num_tensors_per_chunk, obj.size) - chunks = [] - total_len = 0 - - # NOTE: we call save_tensors for one tensor at a time. This is because save_tensors won't work for - # multiple tensors if there are shared memory among them! - for i in range(start, end): - next_key = obj.keys[i] - tensor_to_send = {next_key: obj.state_dict[next_key]} - chunk = save_tensors(tensor_to_send) - chunks.append(chunk) - total_len += len(chunk) - - self.logger.debug(f"{num_received_tensors=}; sending {total_len} bytes") - return ProduceRC.OK, chunks, {_StateKey.NUM_TENSORS_INCLUDED: end - start} - - -class _ChunkConsumer(Consumer): +class _ChunkConsumer(ItemConsumer): def __init__(self, tensors_received_cb, cb_kwargs): - Consumer.__init__(self) - self.num_tensors_received = 0 - self.result = {} - self.error = None + ItemConsumer.__init__(self) self.tensors_received_cb = tensors_received_cb self.cb_kwargs = cb_kwargs if tensors_received_cb is not None and not callable(tensors_received_cb): raise ValueError("tensors_received_cb must be callable") - def consume(self, ref_id, state: dict, data: Any) -> dict: - assert isinstance(data, list) + def consume_items(self, items: List[Any], result: Any) -> Any: + assert isinstance(items, list) + if result is None: + result = {} + tensors = {} - for chunk in data: - td = load_tensors(chunk) + for item in items: + td = load_tensors(item) if not isinstance(td, dict): raise ValueError("cannot load received bytes to tensors") tensors.update(td) - num_tensors_included = state.get(_StateKey.NUM_TENSORS_INCLUDED, 0) - if num_tensors_included != len(tensors): - raise ValueError(f"tensor count mismatch: {num_tensors_included=}, received={len(tensors)}") - if self.tensors_received_cb: - result = self.tensors_received_cb(tensors, **self.cb_kwargs) - if isinstance(result, dict): - self.result.update(result) + cb_result = self.tensors_received_cb(tensors, **self.cb_kwargs) + if isinstance(cb_result, dict): + result.update(cb_result) else: - self.result.update(tensors) - self.num_tensors_received += num_tensors_included - self.logger.debug(f"received {len(tensors)} tensor(s)") - return {_StateKey.NUM_RECEIVED_TENSORS: self.num_tensors_received} - - def download_failed(self, ref_id, reason: str): - self.logger.error(f"failed to download state dict with ref {ref_id}: {reason}") - self.error = reason - self.result = None + result.update(tensors) + return result - def download_completed(self, ref_id: str): - self.logger.debug(f"received state dict with {self.num_tensors_received} tensors") - -class TensorDownloader(ObjDownloader): +class TensorDownloader: @classmethod def new_transaction( cls, cell: Cell, - num_tensors_per_chunk: int = 1, + num_receivers: int, timeout: float = 5.0, timeout_cb=None, **cb_kwargs, @@ -131,7 +82,7 @@ def new_transaction( Args: cell: the cell for communication with recipients - num_tensors_per_chunk: number of tensors to send for each chunk + num_receivers: number of receivers timeout: timeout for the transaction timeout_cb: CB to be called when the transaction is timed out **cb_kwargs: args to be passed to the CB @@ -140,68 +91,50 @@ def new_transaction( The timeout_cb must follow this signature: - cb(tx_id, state_dicts: List[dict[str, torch.Tensor], **cb_args) + cb(tx_id, tensors: List[dict[str, torch.Tensor], **cb_args) """ - return ObjDownloader.new_transaction( + return CachedObjDownloader.new_transaction( cell=cell, - producer=_ChunkProducer(num_tensors_per_chunk=num_tensors_per_chunk), + num_receivers=num_receivers, timeout=timeout, timeout_cb=cls._tx_timeout, - app_timeout_cb=timeout_cb, - **cb_kwargs, + cb_info=(timeout_cb, cb_kwargs), ) @classmethod - def _tx_timeout(cls, tx_id: str, objs: List[Any], app_timeout_cb, **cb_kwargs): + def _tx_timeout(cls, tx_id: str, objs: List[Any], cb_info: tuple): + app_timeout_cb, cb_kwargs = cb_info if app_timeout_cb: - sds = [obj.state_dict for obj in objs] + sds = [obj.tensors for obj in objs] app_timeout_cb(tx_id, sds, **cb_kwargs) @classmethod - def add_state_dict( + def add_tensors( cls, transaction_id: str, - state_dict: dict[str, torch.Tensor], - ref_id=None, - state_dict_downloaded_cb=None, - **cb_kwargs, + tensors: dict[str, torch.Tensor], + num_tensors_per_chunk: int = 1, ) -> str: """Add a file to be downloaded to the specified transaction. Args: transaction_id: ID of the transaction - state_dict: state dict to be downloaded - ref_id: ref id to be used, if provided - state_dict_downloaded_cb: CB to be called when the state dict is done downloading - **cb_kwargs: args to be passed to the CB - - Returns: reference id for the file. + tensors: state dict to be downloaded + num_tensors_per_chunk: number of tensors per chunk - The state_dict_downloaded_cb must follow this signature: - - cb(ref_id: str, to_site: str, status: str, state_dict: dict[str, torch.Tensor], **cb_kwargs) + Returns: reference id for the state dict. """ - obj = _SourceTensor(state_dict) - return ObjDownloader.add_download_object( + obj = _TensorSource(tensors) + return CachedObjDownloader.add_object( transaction_id=transaction_id, obj=obj, - ref_id=ref_id, - obj_downloaded_cb=cls._source_tensor_downloaded, - app_downloaded_cb=state_dict_downloaded_cb, - **cb_kwargs, + num_items_per_chunk=num_tensors_per_chunk, ) @classmethod - def _source_tensor_downloaded( - cls, ref_id: str, to_site: str, status: str, obj: _SourceTensor, app_downloaded_cb, **cb_kwargs - ): - if app_downloaded_cb: - app_downloaded_cb(ref_id, to_site, status, obj.state_dict, **cb_kwargs) - - @classmethod - def download_state_dict( + def download_tensors( cls, from_fqcn: str, ref_id: str, @@ -212,7 +145,7 @@ def download_state_dict( abort_signal=None, tensors_received_cb=None, **cb_kwargs, - ) -> (str, Optional[dict[str, torch.Tensor]]): + ) -> Tuple[str, Optional[dict[str, torch.Tensor]]]: """Download the referenced state dict from the source. Args: @@ -229,15 +162,13 @@ def download_state_dict( """ consumer = _ChunkConsumer(tensors_received_cb, cb_kwargs) - download_object( + return CachedObjDownloader.download_object( from_fqcn=from_fqcn, ref_id=ref_id, - consumer=consumer, + item_consumer=consumer, per_request_timeout=per_request_timeout, cell=cell, secure=secure, optional=optional, abort_signal=abort_signal, ) - - return consumer.error, consumer.result diff --git a/nvflare/fox/api/app.py b/nvflare/fox/api/app.py index 3dc08a5330..5ebacbae35 100644 --- a/nvflare/fox/api/app.py +++ b/nvflare/fox/api/app.py @@ -260,8 +260,8 @@ def initialize(self, context: Context): for obj in self._managed_objects.values(): self._fox_init(obj, context) - def new_context(self, caller: str, callee: str, props: dict = None): - return Context(self, caller, callee, self._abort_signal, props) + def new_context(self, caller: str, callee: str, props: dict = None, target_group=None): + return Context(self, caller, callee, self._abort_signal, props, target_group=target_group) def register_event_handler(self, event_type: str, handler, **handler_kwargs): handlers = self._event_handlers.get(event_type) diff --git a/nvflare/fox/api/ctx.py b/nvflare/fox/api/ctx.py index 708cec702f..e6aecded13 100644 --- a/nvflare/fox/api/ctx.py +++ b/nvflare/fox/api/ctx.py @@ -16,7 +16,7 @@ class Context: - def __init__(self, app, caller: str, callee: str, abort_signal: Signal, props: dict = None): + def __init__(self, app, caller: str, callee: str, abort_signal: Signal, props: dict = None, target_group=None): if not isinstance(caller, str): raise ValueError(f"caller must be str but got {type(caller)}") @@ -25,6 +25,7 @@ def __init__(self, app, caller: str, callee: str, abort_signal: Signal, props: d self.caller = caller self.callee = callee + self.target_group = target_group self.abort_signal = abort_signal self.app = app self.props = {} @@ -55,6 +56,13 @@ def client_hierarchy(self): def workspace(self): return self.app.get_workspace() + @property + def target_group_size(self): + if self.target_group: + return self.target_group.size + else: + return 1 + def set_prop(self, name: str, value): self.props[name] = value diff --git a/nvflare/fox/api/group.py b/nvflare/fox/api/group.py index 0c2c02c477..a1822b0134 100644 --- a/nvflare/fox/api/group.py +++ b/nvflare/fox/api/group.py @@ -65,6 +65,14 @@ def __init__( if not wait_after_min_resps: self._wait_after_min_resps = 0 + @property + def size(self): + return len(self._proxies) + + @property + def members(self): + return self._proxies + def __getattr__(self, func_name): """ This method is called when Python cannot find an invoked method func_name of this class. @@ -76,7 +84,7 @@ def method(*args, **kwargs): # filter once for all targets p = self._proxies[0] the_proxy, func_itf, adj_args, adj_kwargs = p.adjust_func_args(func_name, args, kwargs) - ctx = the_proxy.app.new_context(the_proxy.app.name, the_proxy.name) + ctx = the_proxy.app.new_context(the_proxy.app.name, the_proxy.name, target_group=self) self._logger.debug(f"[{ctx.header_str()}] calling {func_name} of group {[p.name for p in self._proxies]}") @@ -88,7 +96,7 @@ def method(*args, **kwargs): for p in self._proxies: the_proxy, func_itf, call_args, call_kwargs = p.adjust_func_args(func_name, adj_args, adj_kwargs) call_kwargs = copy.copy(call_kwargs) - ctx = self._app.new_context(the_proxy.caller_name, the_proxy.name) + ctx = self._app.new_context(the_proxy.caller_name, the_proxy.name, target_group=self) call_kwargs[CollabMethodArgName.CONTEXT] = ctx # set the optional args to help backend decide how to call @@ -116,7 +124,7 @@ def method(*args, **kwargs): # wait for responses if not self._blocking: - return + return None start_time = time.time() min_received_time = None diff --git a/nvflare/fox/examples/pt/filters.py b/nvflare/fox/examples/pt/filters.py index 4cb0a08959..4321f6d1c7 100644 --- a/nvflare/fox/examples/pt/filters.py +++ b/nvflare/fox/examples/pt/filters.py @@ -2,7 +2,7 @@ from nvflare.fox.api.ctx import Context from nvflare.fox.api.filter import CallFilter, ResultFilter -from nvflare.fox.sys.state_dict_downloader import StateDictDownloader, download_state_dict +from nvflare.fox.sys.model_downloader import ModelDownloader, download_model from nvflare.fuel.utils.log_utils import get_obj_logger @@ -11,19 +11,22 @@ class OutgoingModelCallFilter(CallFilter): def __init__(self, model_arg_name: str): super().__init__() self.model_arg_name = model_arg_name + self.logger = get_obj_logger(self) def filter_call(self, func_kwargs: dict, context: Context): arg_value = func_kwargs.get(self.model_arg_name) if not arg_value: return func_kwargs - downloader = StateDictDownloader( - state_dict=arg_value, + num_receivers = context.target_group_size + self.logger.info(f"target group size={num_receivers}") + + downloader = ModelDownloader( + num_receivers=context.target_group_size, ctx=context, timeout=5.0, - num_tensors_per_chunk=2, ) - model = downloader.get_ref() + model = downloader.add_model(arg_value, 2) func_kwargs[self.model_arg_name] = model return func_kwargs @@ -40,7 +43,7 @@ def filter_call(self, func_kwargs: dict, context: Context): if not arg_value: return func_kwargs - err, model = download_state_dict(ref=arg_value, ctx=context, per_request_timeout=5.0) + err, model = download_model(ref=arg_value, ctx=context, per_request_timeout=5.0) if err: self.logger.error(f"error filtering call arg {arg_value}: {err}") else: @@ -55,13 +58,12 @@ def filter_result(self, result: Any, context: Context): if not isinstance(result, dict): return result - downloader = StateDictDownloader( - state_dict=result, + downloader = ModelDownloader( + num_receivers=1, ctx=context, timeout=5.0, - num_tensors_per_chunk=2, ) - return downloader.get_ref() + return downloader.add_model(result, 2) class IncomingModelResultFilter(ResultFilter): @@ -71,7 +73,7 @@ def __init__(self): self.logger = get_obj_logger(self) def filter_result(self, result: Any, context: Context): - err, model = download_state_dict(ref=result, ctx=context, per_request_timeout=5.0) + err, model = download_model(ref=result, ctx=context, per_request_timeout=5.0) if err: self.logger.error(f"error filtering result {result}: {err}") return result diff --git a/nvflare/fox/examples/pt/pt_avg_stream.py b/nvflare/fox/examples/pt/pt_avg_stream.py index a06112de64..cd5960b33d 100644 --- a/nvflare/fox/examples/pt/pt_avg_stream.py +++ b/nvflare/fox/examples/pt/pt_avg_stream.py @@ -25,7 +25,7 @@ from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.pt.utils import parse_state_dict from nvflare.fox.sim.simulator import Simulator -from nvflare.fox.sys.state_dict_downloader import StateDictDownloader, download_state_dict +from nvflare.fox.sys.model_downloader import ModelDownloader, download_model from nvflare.fuel.utils.log_utils import get_obj_logger @@ -58,30 +58,26 @@ def execute(self, context: Context): def _do_one_round(self, r, current_model, ctx: Context): aggr_result = _AggrResult() - # pretend the model is big - downloader = None + grp = all_clients( + ctx, + process_resp_cb=self._accept_train_result, + aggr_result=aggr_result, + ) + if ctx.env_type == EnvType.SYSTEM: - downloader = StateDictDownloader( - state_dict=current_model, + downloader = ModelDownloader( + num_receivers=grp.size, ctx=ctx, timeout=5.0, - num_tensors_per_chunk=2, ) model_type = "ref" - model = downloader.get_ref() + model = downloader.add_model(current_model, 2) self.logger.info(f"prepared model as ref: {model}") else: model = current_model model_type = "model" - all_clients( - ctx, - process_resp_cb=self._accept_train_result, - aggr_result=aggr_result, - ).train(r, model, model_type) - - if downloader: - downloader.clear() + grp.train(r, model, model_type) if aggr_result.count == 0: return None @@ -97,11 +93,11 @@ def _accept_train_result(self, result, aggr_result: _AggrResult, context: Contex model, model_type = result if model_type == "ref": - err, model = download_state_dict( + err, model = download_model( ref=model, per_request_timeout=5.0, ctx=context, - tensors_received_cb=self._aggregate_tensors, + model_received_cb=self._aggregate_tensors, aggr_result=aggr_result, context=context, ) @@ -140,7 +136,7 @@ def train(self, current_round, weights, model_type: str, context: Context): return 0 self.logger.debug(f"[{context.header_str()}] training round {current_round}: {model_type=} {weights=}") if model_type == "ref": - err, model = download_state_dict(ref=weights, per_request_timeout=5.0, ctx=context) + err, model = download_model(ref=weights, per_request_timeout=5.0, ctx=context) if err: raise RuntimeError(f"failed to download model {weights}: {err}") self.logger.info(f"downloaded model {model}") @@ -152,25 +148,19 @@ def train(self, current_round, weights, model_type: str, context: Context): if model_type == "ref": # stream it - downloader = StateDictDownloader( - state_dict=result, + downloader = ModelDownloader( + num_receivers=1, ctx=context, timeout=5.0, - num_tensors_per_chunk=2, - state_dict_downloaded_cb=self._download_done, ) model_type = "ref" - model = downloader.get_ref() + model = downloader.add_model(result, 2) self.logger.info(f"prepared result as ref: {model}") else: model = result model_type = "model" return model, model_type - def _download_done(self, downloader: StateDictDownloader, to_site: str, status: str, ctx: Context): - self.logger.info(f"[{ctx.header_str()}] finished downloading model to {to_site}: {status}") - downloader.clear() - def main(): simple_logging(logging.DEBUG) diff --git a/nvflare/fox/sys/state_dict_downloader.py b/nvflare/fox/sys/model_downloader.py similarity index 58% rename from nvflare/fox/sys/state_dict_downloader.py rename to nvflare/fox/sys/model_downloader.py index e524ee3ceb..189f7dd948 100644 --- a/nvflare/fox/sys/state_dict_downloader.py +++ b/nvflare/fox/sys/model_downloader.py @@ -17,22 +17,18 @@ from nvflare.fox.api.ctx import Context from nvflare.fox.sys.backend import SysBackend +SOURCE = "source" +REF_ID = "ref_id" -class StateDictDownloader: + +class ModelDownloader: def __init__( self, + num_receivers: int, timeout: float, ctx: Context, - state_dict: dict[str, torch.Tensor], - num_tensors_per_chunk: int = 1, - state_dict_downloaded_cb=None, - **cb_kwargs, ): - self.state_dict_downloaded_cb = state_dict_downloaded_cb - self.cb_kwargs = cb_kwargs - self.ctx = ctx - backend = ctx.backend if not isinstance(backend, SysBackend): raise ValueError(f"backend must be SysBackend but got {type(backend)}") @@ -40,40 +36,31 @@ def __init__( self.cell = backend.cell self.tx_id = TensorDownloader.new_transaction( cell=self.cell, - num_tensors_per_chunk=num_tensors_per_chunk, timeout=timeout, timeout_cb=None, + num_receivers=num_receivers, ) - self.rid = TensorDownloader.add_state_dict( + def add_model(self, model: dict[str, torch.Tensor], num_tensors_per_chunk: int = 1): + rid = TensorDownloader.add_tensors( transaction_id=self.tx_id, - state_dict=state_dict, - state_dict_downloaded_cb=self._state_dict_downloaded_cb, + tensors=model, + num_tensors_per_chunk=num_tensors_per_chunk, ) - - def _state_dict_downloaded_cb(self, rid, to_site, status, obj): - if self.state_dict_downloaded_cb: - self.state_dict_downloaded_cb(self, to_site, status, self.ctx, **self.cb_kwargs) - - def get_ref(self): - return {"source": self.cell.get_fqcn(), "rid": self.rid} - - def clear(self): - TensorDownloader.delete_transaction(self.tx_id) + return {SOURCE: self.cell.get_fqcn(), REF_ID: rid} -def download_state_dict(ref: dict, per_request_timeout: float, ctx: Context, tensors_received_cb=None, **cb_kwargs): +def download_model(ref: dict, per_request_timeout: float, ctx: Context, model_received_cb=None, **cb_kwargs): backend = ctx.backend if not isinstance(backend, SysBackend): raise ValueError(f"backend must be SysBackend but got {type(backend)}") - err, result = TensorDownloader.download_state_dict( - from_fqcn=ref.get("source"), - ref_id=ref.get("rid"), + return TensorDownloader.download_tensors( + from_fqcn=ref.get(SOURCE), + ref_id=ref.get(REF_ID), per_request_timeout=per_request_timeout, cell=backend.cell, abort_signal=ctx.abort_signal, - tensors_received_cb=tensors_received_cb, + tensors_received_cb=model_received_cb, **cb_kwargs, ) - return err, result diff --git a/nvflare/fuel/f3/streaming/cached_obj_downloader.py b/nvflare/fuel/f3/streaming/cached_obj_downloader.py new file mode 100644 index 0000000000..6e69a13781 --- /dev/null +++ b/nvflare/fuel/f3/streaming/cached_obj_downloader.py @@ -0,0 +1,234 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import threading +from abc import ABC, abstractmethod +from typing import Any, List, Tuple + +from nvflare.fuel.f3.cellnet.cell import Cell +from nvflare.fuel.f3.streaming.obj_downloader import Consumer, ObjDownloader, ProduceRC, download_object +from nvflare.fuel.f3.streaming.smod import SelfManagedObject, SelfManagedObjectDownloader +from nvflare.fuel.utils.log_utils import get_obj_logger + + +class _StateKey: + START = "start" + COUNT = "count" + + +class CountableObject(ABC): + + @abstractmethod + def get_item_count(self) -> int: + pass + + @abstractmethod + def produce_item(self, index: int) -> Any: + pass + + +class ItemConsumer(ABC): + + @abstractmethod + def consume_items(self, items: List[Any], result: Any) -> Any: + """Process items and return updated result.""" + pass + + +class _CacheableObject(SelfManagedObject): + + def __init__(self, obj: CountableObject, num_items_per_chunk: int): + super().__init__() + self.obj = obj + self.num_items_per_chunk = num_items_per_chunk + self.size = obj.get_item_count() + self.cache = [(None, 0)] * self.size + self.lock = threading.Lock() + self.num_receivers = 0 + self.logger = get_obj_logger(self) + + def set_transaction(self, tx_id, ref_id): + tx_info = ObjDownloader.get_transaction_info(tx_id) + self.num_receivers = tx_info.num_receivers + self.logger.info(f"set transaction info: {tx_id=}, {ref_id=} {self.num_receivers=}") + + def downloaded_to_all(self): + self.logger.info(f"object has been downloaded to all {self.num_receivers=} sites - clear cache") + self.clear_cache() + + def clear_cache(self): + with self.lock: + self.cache = None + + def _get_item(self, index: int) -> bytes: + with self.lock: + data, _ = self.cache[index] + if data is None: + data = self.obj.produce_item(index) + self.cache[index] = (data, 0) + else: + self.logger.info(f"got item {index} from cache") + return data + + def _adjust_cache(self, start: int, count: int): + with self.lock: + for i in range(start, start + count): + data, num_received = self.cache[i] + num_received += 1 + if num_received >= self.num_receivers: + self.logger.info(f"item {i} was received by {num_received} sites - clear cache") + self.cache[i] = (None, num_received) + else: + self.cache[i] = (data, num_received) + + def produce(self, state: dict, requester: str) -> Tuple[str, Any, dict]: + if not state: + # first request + start = 0 + else: + received_start = state.get(_StateKey.START, 0) + received_count = state.get(_StateKey.COUNT, 0) + if received_count > 0: + self._adjust_cache(received_start, received_count) + + start = received_start + received_count + + end = min(start + self.num_items_per_chunk, self.size) + count = end - start + + if count <= 0: + # already done + return ProduceRC.EOF, None, {} + + result = [] + for i in range(start, end): + item = self._get_item(i) + result.append(item) + return ProduceRC.OK, result, {_StateKey.START: start, _StateKey.COUNT: count} + + +class _ConsumerWrapper(Consumer): + + def __init__(self, c: ItemConsumer): + super().__init__() + self.consumer = c + self.error = None + self.result = None + + def consume(self, ref_id: str, state: dict, data: Any) -> dict: + assert isinstance(data, list) + self.result = self.consumer.consume_items(data, self.result) + return state + + def download_failed(self, ref_id, reason: str): + self.logger.error(f"failed to download object with ref {ref_id}: {reason}") + self.error = reason + self.result = None + + def download_completed(self, ref_id: str): + self.logger.debug(f"received object with ref {ref_id}") + + +class CachedObjDownloader: + + @classmethod + def new_transaction( + cls, + cell: Cell, + timeout: float, + num_receivers: int, + timeout_cb=None, + **cb_kwargs, + ): + return SelfManagedObjectDownloader.new_transaction( + cell, + timeout=timeout, + num_receivers=num_receivers, + timeout_cb=cls._handle_timeout, + cb_info=(timeout_cb, cb_kwargs), + ) + + @classmethod + def _handle_timeout(cls, tx_id: str, objs: list, cb_info): + app_timeout_cb, cb_kwargs = cb_info + if app_timeout_cb: + original_objs = [obj.obj for obj in objs] + app_timeout_cb(tx_id, original_objs, **cb_kwargs) + + for obj in objs: + assert isinstance(obj, _CacheableObject) + obj.clear_cache() + + @classmethod + def add_object( + cls, + transaction_id: str, + obj: CountableObject, + num_items_per_chunk: int = 1, + ) -> str: + """Add an object to be downloaded to the specified transaction. + + Args: + transaction_id: ID of the transaction + obj: object to be downloaded + num_items_per_chunk: number of items per chunk + + Returns: reference id for the object. + + """ + obj = _CacheableObject(obj, num_items_per_chunk) + return SelfManagedObjectDownloader.add_download_object( + transaction_id=transaction_id, + obj=obj, + ) + + @classmethod + def download_object( + cls, + from_fqcn: str, + ref_id: str, + item_consumer: ItemConsumer, + per_request_timeout: float, + cell: Cell, + secure=False, + optional=False, + abort_signal=None, + ) -> Tuple[str, Any]: + """Download the referenced object from the source. + + Args: + from_fqcn: FQCN of the data source. + ref_id: reference ID of the state dict to be downloaded. + item_consumer: item consumer. + per_request_timeout: timeout for requests sent to the data source. + cell: cell to be used for communicating to the data source. + secure: P2P private mode for communication + optional: supress log messages of communication + abort_signal: signal for aborting download. + + Returns: tuple of (error message if any, object). + + """ + consumer = _ConsumerWrapper(item_consumer) + download_object( + from_fqcn=from_fqcn, + ref_id=ref_id, + consumer=consumer, + per_request_timeout=per_request_timeout, + cell=cell, + secure=secure, + optional=optional, + abort_signal=abort_signal, + ) + + return consumer.error, consumer.result diff --git a/nvflare/fuel/f3/streaming/obj_downloader.py b/nvflare/fuel/f3/streaming/obj_downloader.py index d3bb7337f2..c327c0e5b5 100644 --- a/nvflare/fuel/f3/streaming/obj_downloader.py +++ b/nvflare/fuel/f3/streaming/obj_downloader.py @@ -15,7 +15,7 @@ import time import uuid from abc import ABC, abstractmethod -from typing import Any +from typing import Any, List, Optional, Tuple from nvflare.apis.signal import Signal from nvflare.fuel.f3.cellnet.cell import Cell @@ -117,6 +117,7 @@ def __init__( self.rid = "R" + str(uuid.uuid4()) self.tx = tx self.obj = obj + self.num_sites_done = 0 self.obj_downloaded_cb = obj_downloaded_cb self.cb_kwargs = cb_kwargs @@ -124,9 +125,18 @@ def mark_active(self): self.tx.mark_active() def obj_downloaded(self, to_site: str, status: str): + self.num_sites_done += 1 if self.obj_downloaded_cb: self.obj_downloaded_cb(self.rid, to_site, status, self.obj, **self.cb_kwargs) + assert isinstance(self.tx.producer, Producer) + self.tx.producer.object_downloaded(self.rid, self.obj, to_site, status) + + assert isinstance(self.tx, _Transaction) + if 0 < self.tx.num_receivers <= self.num_sites_done: + # this object is done for all sites + self.tx.producer.object_done(self.rid, self.obj) + class ProduceRC: """Defines return code for the Producer's produce method.""" @@ -141,13 +151,19 @@ class DownloadStatus: FAILED = "failed" +class TransactionDoneStatus: + FINISHED = "finished" + TIMEOUT = "timeout" + DELETED = "deleted" + + class Producer(ABC): def __init__(self): self.logger = get_obj_logger(self) @abstractmethod - def produce(self, ref_id: str, obj: Any, state: dict, requester: str) -> (str, Any, dict): + def produce(self, ref_id: str, obj: Any, state: dict, requester: str) -> Tuple[str, Any, dict]: """Produce a small object to be sent (on object sender side). Args: @@ -161,6 +177,18 @@ def produce(self, ref_id: str, obj: Any, state: dict, requester: str) -> (str, A """ pass + def object_downloaded(self, ref_id: str, obj: Any, to_site: str, status: str): + """Called when an object is downloaded to a site.""" + pass + + def object_done(self, ref_id: str, obj: Any): + """Called when the object is fully downloaded to all sites.""" + pass + + def transaction_done(self, transaction_id: str, objs: List[Any], status: str): + """Called when the transaction is finished.""" + pass + class _Transaction: @@ -168,6 +196,7 @@ def __init__( self, producer: Producer, timeout: float, + num_receivers: int, tx_id=None, timeout_cb=None, **cb_kwargs, @@ -177,6 +206,7 @@ def __init__( Args: producer: the Producer object to produce small objects. timeout: amount of time since last activity + num_receivers: number of receivers. 0 means unlimited. tx_id: if provided, use it; otherwise create one timeout_cb: the CB to be called when the transaction timed out **cb_kwargs: args to be passed to the timeout CB @@ -189,6 +219,7 @@ def __init__( self.tid = "T" + str(uuid.uuid4()) self.producer = producer self.timeout = timeout + self.num_receivers = num_receivers self.timeout_cb = timeout_cb self.cb_kwargs = cb_kwargs self.last_active_time = time.time() @@ -247,6 +278,30 @@ def timed_out(self): if self.timeout_cb: self.timeout_cb(self.tid, [r.obj for r in self.refs], **self.cb_kwargs) + def is_finished(self): + """Check whether the transaction is finished (all objects are downloaded).""" + if self.num_receivers <= 0: + return False + + for ref in self.refs: + assert isinstance(ref, _Ref) + if ref.num_sites_done < self.num_receivers: + return False + return True + + def transaction_done(self, status: str): + """Called when the transaction is finished.""" + self.producer.transaction_done(self.tid, [r.obj for r in self.refs], status) + + +class TransactionInfo: + + def __init__(self, tx: _Transaction): + self.producer = tx.producer + self.timeout = tx.timeout + self.num_receivers = tx.num_receivers + self.objects = [r.obj for r in tx.refs] + class ObjDownloader: @@ -279,9 +334,18 @@ def _initialize(cls, cell: Cell): cls._initialized_cells[id(cell)] = True @classmethod - def new_transaction(cls, cell: Cell, producer: Producer, timeout: float, tx_id=None, timeout_cb=None, **cb_kwargs): + def new_transaction( + cls, + cell: Cell, + producer: Producer, + timeout: float, + num_receivers: int = 0, + tx_id=None, + timeout_cb=None, + **cb_kwargs, + ): cls._initialize(cell) - tx = _Transaction(producer, timeout, tx_id, timeout_cb, **cb_kwargs) + tx = _Transaction(producer, timeout, num_receivers, tx_id, timeout_cb, **cb_kwargs) with cls._tx_lock: cls._tx_table[tx.tid] = tx return tx.tid @@ -314,6 +378,7 @@ def delete_transaction(cls, transaction_id: str, call_cb=False): tx = cls._tx_table.get(transaction_id) if tx: cls._delete_tx(tx, call_cb) + tx.transaction_done(TransactionDoneStatus.DELETED) @classmethod def shutdown(cls): @@ -327,6 +392,7 @@ def shutdown(cls): if tx_list: for tx in tx_list: cls._delete_tx(tx, True) + tx.transaction_done(TransactionDoneStatus.DELETED) @classmethod def _delete_tx(cls, tx: _Transaction, call_cb=False): @@ -342,6 +408,23 @@ def _delete_tx(cls, tx: _Transaction, call_cb=False): for r in tx.refs: cls._ref_table.pop(r.rid, None) + @classmethod + def get_transaction_info(cls, transaction_id: str) -> Optional[TransactionInfo]: + tx = cls._tx_table.get(transaction_id) + if not tx: + return None + else: + return TransactionInfo(tx) + + @classmethod + def get_transaction_id(cls, ref_id: str) -> Optional[str]: + ref = cls._ref_table.get(ref_id) + if not ref: + return None + else: + assert isinstance(ref, _Ref) + return ref.tx.tid + @classmethod def _handle_download(cls, request: Message) -> Message: requester = request.get_header(MessageHeaderKey.ORIGIN) @@ -392,15 +475,26 @@ def _monitor_tx(cls): while True: now = time.time() expired_tx = [] + finished_tx = [] with cls._tx_lock: for tid, tx in cls._tx_table.items(): assert isinstance(tx, _Transaction) - if now - tx.last_active_time > tx.timeout: + + # check whether all refs are done + if tx.is_finished(): + finished_tx.append(tx) + elif now - tx.last_active_time > tx.timeout: expired_tx.append(tx) - if expired_tx: - for tx in expired_tx: - cls._delete_tx(tx, True) + for tx in expired_tx: + assert isinstance(tx, _Transaction) + tx.transaction_done(TransactionDoneStatus.TIMEOUT) + cls._delete_tx(tx, True) + + for tx in finished_tx: + tx.transaction_done(TransactionDoneStatus.FINISHED) + cls._delete_tx(tx, False) + time.sleep(5.0) diff --git a/nvflare/fuel/f3/streaming/smod.py b/nvflare/fuel/f3/streaming/smod.py new file mode 100644 index 0000000000..d16d7941eb --- /dev/null +++ b/nvflare/fuel/f3/streaming/smod.py @@ -0,0 +1,105 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from abc import ABC, abstractmethod +from typing import Any, List, Tuple + +from nvflare.fuel.f3.cellnet.cell import Cell +from nvflare.fuel.f3.streaming.obj_downloader import ObjDownloader, Producer +from nvflare.fuel.utils.log_utils import get_obj_logger + + +class SelfManagedObject(ABC): + + def set_transaction(self, tx_id, ref_id): + pass + + @abstractmethod + def produce(self, state: dict, requester: str) -> Tuple[str, Any, dict]: + """Produce a small object to be sent (on object sender side). + + Args: + state: current state of downloading, received from the downloading site + requester: the FQCN of the site that is downloading + + Returns: a tuple of (return code, a small object to be sent, new state to be sent). + + """ + pass + + def downloaded_to_one(self, to_site: str, status: str): + """Called when an object is downloaded to a site.""" + pass + + def downloaded_to_all(self): + """Called when the object is fully downloaded to all sites.""" + pass + + def transaction_done(self, transaction_id: str, status: str): + """Called when the transaction is finished.""" + pass + + +class _SMODProducer(Producer): + + def __init__(self): + super().__init__() + + def produce(self, ref_id: str, obj: SelfManagedObject, state: dict, requester: str) -> Tuple[str, Any, dict]: + return obj.produce(state, requester) + + def object_downloaded(self, ref_id: str, obj: SelfManagedObject, to_site: str, status: str): + obj.downloaded_to_one(to_site, status) + + def object_done(self, ref_id: str, obj: SelfManagedObject): + """Called when the object is fully downloaded to all sites.""" + obj.downloaded_to_all() + + def transaction_done(self, transaction_id: str, objs: List[SelfManagedObject], status: str): + for obj in objs: + obj.transaction_done(transaction_id, status) + + +class SelfManagedObjectDownloader: + + @classmethod + def new_transaction( + cls, cell: Cell, timeout: float, num_receivers: int = 0, tx_id=None, timeout_cb=None, **cb_kwargs + ): + return ObjDownloader.new_transaction( + cell=cell, + producer=_SMODProducer(), + timeout=timeout, + num_receivers=num_receivers, + tx_id=tx_id, + timeout_cb=timeout_cb, + **cb_kwargs, + ) + + @classmethod + def add_download_object( + cls, + transaction_id: str, + obj: SelfManagedObject, + ref_id=None, + ) -> str: + if not issubclass(type(obj), SelfManagedObject): + raise TypeError(f"obj must be an instance of SelfManagedObject but got {type(obj)}") + + rid = ObjDownloader.add_download_object( + transaction_id=transaction_id, + obj=obj, + ref_id=ref_id, + ) + obj.set_transaction(transaction_id, rid) + return rid From 9f391ee15180b34f75e52b153e8143f7518e9f27 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Mon, 20 Oct 2025 15:13:51 +0800 Subject: [PATCH 042/102] download 2 models --- nvflare/fox/examples/pt/pt_avg_stream.py | 26 +++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/nvflare/fox/examples/pt/pt_avg_stream.py b/nvflare/fox/examples/pt/pt_avg_stream.py index cd5960b33d..ead90d7036 100644 --- a/nvflare/fox/examples/pt/pt_avg_stream.py +++ b/nvflare/fox/examples/pt/pt_avg_stream.py @@ -57,6 +57,9 @@ def execute(self, context: Context): def _do_one_round(self, r, current_model, ctx: Context): aggr_result = _AggrResult() + model2 = {} + for k, v in current_model.items(): + model2[k] = v + 2.0 grp = all_clients( ctx, @@ -72,12 +75,13 @@ def _do_one_round(self, r, current_model, ctx: Context): ) model_type = "ref" model = downloader.add_model(current_model, 2) + model2 = downloader.add_model(model2, 2) self.logger.info(f"prepared model as ref: {model}") else: model = current_model model_type = "model" - grp.train(r, model, model_type) + grp.train(r, model, model2, model_type) if aggr_result.count == 0: return None @@ -130,17 +134,25 @@ def __init__(self, delta: float): self.delta = delta @collab - def train(self, current_round, weights, model_type: str, context: Context): + def train(self, current_round, model1, model2, model_type: str, context: Context): if context.is_aborted(): self.logger.debug("training aborted") return 0 - self.logger.debug(f"[{context.header_str()}] training round {current_round}: {model_type=} {weights=}") + self.logger.debug(f"[{context.header_str()}] training round {current_round}: {model_type=} {model1=} {model2=}") if model_type == "ref": - err, model = download_model(ref=weights, per_request_timeout=5.0, ctx=context) + err, model1 = download_model(ref=model1, per_request_timeout=5.0, ctx=context) if err: - raise RuntimeError(f"failed to download model {weights}: {err}") - self.logger.info(f"downloaded model {model}") - weights = model + raise RuntimeError(f"failed to download model1 {model1}: {err}") + self.logger.info(f"downloaded model1 {model1}") + + err, model2 = download_model(ref=model2, per_request_timeout=5.0, ctx=context) + if err: + raise RuntimeError(f"failed to download model2 {model2}: {err}") + self.logger.info(f"downloaded model2 {model2}") + + weights = {} + for k, v in model1.items(): + weights[k] = v + model2[k] result = {} for k, v in weights.items(): From 4a28b6b333d400e4f121edcb77ee3011dbc62049 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Tue, 21 Oct 2025 10:52:02 +0800 Subject: [PATCH 043/102] enhance obj downloader system --- nvflare/app_opt/pt/tensor_downloader.py | 44 ++--- nvflare/fox/examples/np/algos/avg_stream.py | 26 ++- .../fox/examples/np/recipe_fed_avg_stream.py | 2 +- nvflare/fox/sys/file_downloader.py | 3 +- nvflare/fox/sys/model_downloader.py | 1 - ...{cached_obj_downloader.py => cacheable.py} | 152 +++------------ nvflare/fuel/f3/streaming/file_downloader.py | 104 ++++++----- nvflare/fuel/f3/streaming/obj_downloader.py | 174 +++++++----------- nvflare/fuel/f3/streaming/smod.py | 105 ----------- nvflare/fuel/hci/server/binary_transfer.py | 2 +- .../fuel/utils/fobs/decomposers/via_file.py | 4 +- nvflare/private/fed/client/client_runner.py | 4 +- nvflare/private/fed/server/server_runner.py | 4 +- 13 files changed, 200 insertions(+), 425 deletions(-) rename nvflare/fuel/f3/streaming/{cached_obj_downloader.py => cacheable.py} (52%) delete mode 100644 nvflare/fuel/f3/streaming/smod.py diff --git a/nvflare/app_opt/pt/tensor_downloader.py b/nvflare/app_opt/pt/tensor_downloader.py index d88651c97b..422cc9109b 100644 --- a/nvflare/app_opt/pt/tensor_downloader.py +++ b/nvflare/app_opt/pt/tensor_downloader.py @@ -18,15 +18,17 @@ from safetensors.torch import save as save_tensors from nvflare.fuel.f3.cellnet.cell import Cell -from nvflare.fuel.f3.streaming.cached_obj_downloader import CachedObjDownloader, CountableObject, ItemConsumer +from nvflare.fuel.f3.streaming.cacheable import CacheableObject, ItemConsumer +from nvflare.fuel.f3.streaming.obj_downloader import ObjDownloader, download_object -class _TensorSource(CountableObject): +class TensorDownloadable(CacheableObject): - def __init__(self, tensors: dict[str, torch.Tensor]): + def __init__(self, tensors: dict[str, torch.Tensor], num_items_per_chunk: int): self.tensors = tensors self.size = len(tensors) self.keys = list(tensors.keys()) + super().__init__(num_items_per_chunk) def get_item_count(self) -> int: return self.size @@ -37,7 +39,7 @@ def produce_item(self, index: int) -> Any: return save_tensors(tensor_to_send) -class _ChunkConsumer(ItemConsumer): +class TensorConsumer(ItemConsumer): def __init__(self, tensors_received_cb, cb_kwargs): ItemConsumer.__init__(self) @@ -75,7 +77,7 @@ def new_transaction( cell: Cell, num_receivers: int, timeout: float = 5.0, - timeout_cb=None, + transaction_done_cb=None, **cb_kwargs, ): """Create a new tensor download transaction. @@ -84,30 +86,30 @@ def new_transaction( cell: the cell for communication with recipients num_receivers: number of receivers timeout: timeout for the transaction - timeout_cb: CB to be called when the transaction is timed out + transaction_done_cb: CB to be called when the transaction is done **cb_kwargs: args to be passed to the CB Returns: transaction id - The timeout_cb must follow this signature: + The transaction_done_cb must follow this signature: cb(tx_id, tensors: List[dict[str, torch.Tensor], **cb_args) """ - return CachedObjDownloader.new_transaction( + return ObjDownloader.new_transaction( cell=cell, num_receivers=num_receivers, timeout=timeout, - timeout_cb=cls._tx_timeout, - cb_info=(timeout_cb, cb_kwargs), + transaction_done_cb=cls._tx_done, + cb_info=(transaction_done_cb, cb_kwargs), ) @classmethod - def _tx_timeout(cls, tx_id: str, objs: List[Any], cb_info: tuple): - app_timeout_cb, cb_kwargs = cb_info - if app_timeout_cb: - sds = [obj.tensors for obj in objs] - app_timeout_cb(tx_id, sds, **cb_kwargs) + def _tx_done(cls, tx_id: str, status: str, objs, cb_info): + transaction_done_cb, cb_kwargs = cb_info + if transaction_done_cb: + tensors = [obj.tensors for obj in objs] + transaction_done_cb(tx_id, tensors, **cb_kwargs) @classmethod def add_tensors( @@ -126,11 +128,10 @@ def add_tensors( Returns: reference id for the state dict. """ - obj = _TensorSource(tensors) - return CachedObjDownloader.add_object( + obj = TensorDownloadable(tensors, num_tensors_per_chunk) + return ObjDownloader.add_object( transaction_id=transaction_id, obj=obj, - num_items_per_chunk=num_tensors_per_chunk, ) @classmethod @@ -161,14 +162,15 @@ def download_tensors( Returns: tuple of (error message if any, downloaded state dict). """ - consumer = _ChunkConsumer(tensors_received_cb, cb_kwargs) - return CachedObjDownloader.download_object( + consumer = TensorConsumer(tensors_received_cb, cb_kwargs) + download_object( from_fqcn=from_fqcn, ref_id=ref_id, - item_consumer=consumer, + consumer=consumer, per_request_timeout=per_request_timeout, cell=cell, secure=secure, optional=optional, abort_signal=abort_signal, ) + return consumer.error, consumer.result diff --git a/nvflare/fox/examples/np/algos/avg_stream.py b/nvflare/fox/examples/np/algos/avg_stream.py index 4d33df1ab9..e7e97ba402 100644 --- a/nvflare/fox/examples/np/algos/avg_stream.py +++ b/nvflare/fox/examples/np/algos/avg_stream.py @@ -52,6 +52,11 @@ def execute(self, context: Context): def _do_one_round(self, r, current_model, ctx: Context): aggr_result = _AggrResult() + grp = all_clients( + ctx, + process_resp_cb=self._accept_train_result, + aggr_result=aggr_result, + ) # pretend the model is big file_name = None @@ -59,6 +64,7 @@ def _do_one_round(self, r, current_model, ctx: Context): file_name = f"/tmp/np_{str(uuid.uuid4())}.npy" save_np_model(current_model, file_name) model = prepare_file_for_download( + num_receivers=grp.size, file_name=file_name, ctx=ctx, timeout=5.0, @@ -70,11 +76,7 @@ def _do_one_round(self, r, current_model, ctx: Context): model = current_model model_type = "model" - all_clients( - ctx, - process_resp_cb=self._accept_train_result, - aggr_result=aggr_result, - ).train(r, model, model_type) + grp.train(r, model, model_type) if file_name: os.remove(file_name) @@ -102,8 +104,8 @@ def _accept_train_result(self, result, aggr_result: _AggrResult, context: Contex aggr_result.count += 1 return None - def _model_downloaded(self, ref_id: str, to_site: str, status: str, file_name): - self.logger.info(f"model file {file_name} downloaded by {to_site}: {ref_id=} {status=}") + def _model_downloaded(self, to_site: str, status: str, file_name): + self.logger.info(f"model file {file_name} downloaded by {to_site}: {status=}") class NPTrainer(ClientApp): @@ -133,6 +135,7 @@ def train(self, current_round, weights, model_type: str, context: Context): file_name = f"/tmp/np_{str(uuid.uuid4())}.npy" save_np_model(result, file_name) model = prepare_file_for_download( + num_receivers=1, file_name=file_name, ctx=context, timeout=5.0, @@ -145,6 +148,9 @@ def train(self, current_round, weights, model_type: str, context: Context): model_type = "model" return model, model_type - def _result_downloaded(self, ref_id: str, to_site: str, status: str, file_name): - self.logger.info(f"file {file_name} downloaded to {to_site}: {ref_id=} {status=}") - os.remove(file_name) + def _result_downloaded(self, to_site: str, status: str, file_name): + self.logger.info(f"model file {file_name} downloaded to {to_site}: {status=}") + if not to_site: + # downloaded to all sites + os.remove(file_name) + self.logger.info(f"model file {file_name} removed") diff --git a/nvflare/fox/examples/np/recipe_fed_avg_stream.py b/nvflare/fox/examples/np/recipe_fed_avg_stream.py index 28572db7aa..ed8ff49df6 100644 --- a/nvflare/fox/examples/np/recipe_fed_avg_stream.py +++ b/nvflare/fox/examples/np/recipe_fed_avg_stream.py @@ -18,7 +18,7 @@ from nvflare.fox.examples.np.algos.avg_stream import NPFedAvgStream, NPTrainer from nvflare.fox.sys.recipe import FoxRecipe -JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" +JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/fox/prod_00/admin@nvidia.com/transfer" def main(): diff --git a/nvflare/fox/sys/file_downloader.py b/nvflare/fox/sys/file_downloader.py index f69c3a8494..4d2d45d47b 100644 --- a/nvflare/fox/sys/file_downloader.py +++ b/nvflare/fox/sys/file_downloader.py @@ -19,6 +19,7 @@ def prepare_file_for_download( file_name: str, timeout: float, + num_receivers: int, ctx: Context, file_downloaded_cb=None, **cb_kwargs, @@ -31,7 +32,7 @@ def prepare_file_for_download( tx_id = FileDownloader.new_transaction( cell=cell, timeout=timeout, - timeout_cb=None, + num_receivers=num_receivers, ) rid = FileDownloader.add_file( tx_id, diff --git a/nvflare/fox/sys/model_downloader.py b/nvflare/fox/sys/model_downloader.py index 189f7dd948..01fa7111d1 100644 --- a/nvflare/fox/sys/model_downloader.py +++ b/nvflare/fox/sys/model_downloader.py @@ -37,7 +37,6 @@ def __init__( self.tx_id = TensorDownloader.new_transaction( cell=self.cell, timeout=timeout, - timeout_cb=None, num_receivers=num_receivers, ) diff --git a/nvflare/fuel/f3/streaming/cached_obj_downloader.py b/nvflare/fuel/f3/streaming/cacheable.py similarity index 52% rename from nvflare/fuel/f3/streaming/cached_obj_downloader.py rename to nvflare/fuel/f3/streaming/cacheable.py index 6e69a13781..6b093e95e1 100644 --- a/nvflare/fuel/f3/streaming/cached_obj_downloader.py +++ b/nvflare/fuel/f3/streaming/cacheable.py @@ -12,12 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import threading -from abc import ABC, abstractmethod +from abc import abstractmethod from typing import Any, List, Tuple -from nvflare.fuel.f3.cellnet.cell import Cell -from nvflare.fuel.f3.streaming.obj_downloader import Consumer, ObjDownloader, ProduceRC, download_object -from nvflare.fuel.f3.streaming.smod import SelfManagedObject, SelfManagedObjectDownloader +from nvflare.fuel.f3.streaming.obj_downloader import Consumer, Downloadable, ObjDownloader, ProduceRC from nvflare.fuel.utils.log_utils import get_obj_logger @@ -26,7 +24,16 @@ class _StateKey: COUNT = "count" -class CountableObject(ABC): +class CacheableObject(Downloadable): + + def __init__(self, num_items_per_chunk: int): + super().__init__() + self.num_items_per_chunk = num_items_per_chunk + self.size = self.get_item_count() + self.cache = [(None, 0)] * self.size + self.lock = threading.Lock() + self.num_receivers = 0 + self.logger = get_obj_logger(self) @abstractmethod def get_item_count(self) -> int: @@ -36,34 +43,16 @@ def get_item_count(self) -> int: def produce_item(self, index: int) -> Any: pass - -class ItemConsumer(ABC): - - @abstractmethod - def consume_items(self, items: List[Any], result: Any) -> Any: - """Process items and return updated result.""" - pass - - -class _CacheableObject(SelfManagedObject): - - def __init__(self, obj: CountableObject, num_items_per_chunk: int): - super().__init__() - self.obj = obj - self.num_items_per_chunk = num_items_per_chunk - self.size = obj.get_item_count() - self.cache = [(None, 0)] * self.size - self.lock = threading.Lock() - self.num_receivers = 0 - self.logger = get_obj_logger(self) - def set_transaction(self, tx_id, ref_id): tx_info = ObjDownloader.get_transaction_info(tx_id) self.num_receivers = tx_info.num_receivers self.logger.info(f"set transaction info: {tx_id=}, {ref_id=} {self.num_receivers=}") def downloaded_to_all(self): - self.logger.info(f"object has been downloaded to all {self.num_receivers=} sites - clear cache") + self.logger.info(f"object has been downloaded to all {self.num_receivers} sites - clear cache") + self.clear_cache() + + def transaction_done(self, transaction_id: str, status: str): self.clear_cache() def clear_cache(self): @@ -74,7 +63,7 @@ def _get_item(self, index: int) -> bytes: with self.lock: data, _ = self.cache[index] if data is None: - data = self.obj.produce_item(index) + data = self.produce_item(index) self.cache[index] = (data, 0) else: self.logger.info(f"got item {index} from cache") @@ -117,17 +106,21 @@ def produce(self, state: dict, requester: str) -> Tuple[str, Any, dict]: return ProduceRC.OK, result, {_StateKey.START: start, _StateKey.COUNT: count} -class _ConsumerWrapper(Consumer): +class ItemConsumer(Consumer): - def __init__(self, c: ItemConsumer): + def __init__(self): super().__init__() - self.consumer = c self.error = None self.result = None + @abstractmethod + def consume_items(self, items: List[Any], result: Any) -> Any: + """Process items and return updated result.""" + pass + def consume(self, ref_id: str, state: dict, data: Any) -> dict: assert isinstance(data, list) - self.result = self.consumer.consume_items(data, self.result) + self.result = self.consume_items(data, self.result) return state def download_failed(self, ref_id, reason: str): @@ -137,98 +130,3 @@ def download_failed(self, ref_id, reason: str): def download_completed(self, ref_id: str): self.logger.debug(f"received object with ref {ref_id}") - - -class CachedObjDownloader: - - @classmethod - def new_transaction( - cls, - cell: Cell, - timeout: float, - num_receivers: int, - timeout_cb=None, - **cb_kwargs, - ): - return SelfManagedObjectDownloader.new_transaction( - cell, - timeout=timeout, - num_receivers=num_receivers, - timeout_cb=cls._handle_timeout, - cb_info=(timeout_cb, cb_kwargs), - ) - - @classmethod - def _handle_timeout(cls, tx_id: str, objs: list, cb_info): - app_timeout_cb, cb_kwargs = cb_info - if app_timeout_cb: - original_objs = [obj.obj for obj in objs] - app_timeout_cb(tx_id, original_objs, **cb_kwargs) - - for obj in objs: - assert isinstance(obj, _CacheableObject) - obj.clear_cache() - - @classmethod - def add_object( - cls, - transaction_id: str, - obj: CountableObject, - num_items_per_chunk: int = 1, - ) -> str: - """Add an object to be downloaded to the specified transaction. - - Args: - transaction_id: ID of the transaction - obj: object to be downloaded - num_items_per_chunk: number of items per chunk - - Returns: reference id for the object. - - """ - obj = _CacheableObject(obj, num_items_per_chunk) - return SelfManagedObjectDownloader.add_download_object( - transaction_id=transaction_id, - obj=obj, - ) - - @classmethod - def download_object( - cls, - from_fqcn: str, - ref_id: str, - item_consumer: ItemConsumer, - per_request_timeout: float, - cell: Cell, - secure=False, - optional=False, - abort_signal=None, - ) -> Tuple[str, Any]: - """Download the referenced object from the source. - - Args: - from_fqcn: FQCN of the data source. - ref_id: reference ID of the state dict to be downloaded. - item_consumer: item consumer. - per_request_timeout: timeout for requests sent to the data source. - cell: cell to be used for communicating to the data source. - secure: P2P private mode for communication - optional: supress log messages of communication - abort_signal: signal for aborting download. - - Returns: tuple of (error message if any, object). - - """ - consumer = _ConsumerWrapper(item_consumer) - download_object( - from_fqcn=from_fqcn, - ref_id=ref_id, - consumer=consumer, - per_request_timeout=per_request_timeout, - cell=cell, - secure=secure, - optional=optional, - abort_signal=abort_signal, - ) - - return consumer.error, consumer.result diff --git a/nvflare/fuel/f3/streaming/file_downloader.py b/nvflare/fuel/f3/streaming/file_downloader.py index 0d17997458..38cfee5cee 100644 --- a/nvflare/fuel/f3/streaming/file_downloader.py +++ b/nvflare/fuel/f3/streaming/file_downloader.py @@ -14,11 +14,12 @@ import os.path import tempfile import uuid -from typing import Any, List, Optional +from typing import Any, Optional, Tuple from nvflare.fuel.f3.cellnet.cell import Cell -from nvflare.fuel.f3.streaming.obj_downloader import Consumer, ObjDownloader, Producer, ProduceRC, download_object -from nvflare.fuel.utils.validation_utils import check_positive_int +from nvflare.fuel.f3.streaming.obj_downloader import Consumer, Downloadable, ObjDownloader, ProduceRC, download_object +from nvflare.fuel.utils.log_utils import get_obj_logger +from nvflare.fuel.utils.validation_utils import check_callable, check_positive_int DEFAULT_CHUNK_SIZE = 5 * 1024 * 1024 @@ -32,9 +33,15 @@ class _StateKey: RECEIVED_BYTES = "received_bytes" -class _File: +class FileDownloadable(Downloadable): - def __init__(self, file_name): + def __init__( + self, + file_name: str, + chunk_size=None, + file_downloaded_cb=None, + **cb_kwargs, + ): """This is the "object" to be downloaded. Args: @@ -43,39 +50,46 @@ def __init__(self, file_name): self.name = file_name self.size = os.path.getsize(file_name) - -class _ChunkProducer(Producer): - - def __init__(self, chunk_size=None): - Producer.__init__(self) if not chunk_size: chunk_size = DEFAULT_CHUNK_SIZE check_positive_int("chunk_size", chunk_size) + if file_downloaded_cb: + check_callable("file_downloaded_cb", file_downloaded_cb) self.chunk_size = chunk_size + self.file_downloaded_cb = file_downloaded_cb + self.cb_kwargs = cb_kwargs + self.logger = get_obj_logger(self) - def produce(self, ref_id: str, obj, state: dict, requester: str) -> (str, Any, dict): - assert isinstance(obj, _File) + def produce(self, state: dict, requester: str) -> Tuple[str, Any, dict]: received_bytes = 0 if state: received_bytes = state.get(_StateKey.RECEIVED_BYTES, 0) if not isinstance(received_bytes, int) or received_bytes < 0: self.logger.error(f"bad {_StateKey.RECEIVED_BYTES} {received_bytes} from {requester}") - return ProduceRC.ERROR, None, None + return ProduceRC.ERROR, None, {} - if received_bytes >= obj.size: + if received_bytes >= self.size: # already done - return ProduceRC.EOF, None, None + return ProduceRC.EOF, None, {} - num_bytes_to_send = min(self.chunk_size, obj.size - received_bytes) - with open(obj.name, "rb") as f: + num_bytes_to_send = min(self.chunk_size, self.size - received_bytes) + with open(self.name, "rb") as f: f.seek(received_bytes) chunk = f.read(num_bytes_to_send) self.logger.debug(f"{received_bytes=}; sending {len(chunk)} bytes") return ProduceRC.OK, chunk, {_StateKey.RECEIVED_BYTES: received_bytes + len(chunk)} + def downloaded_to_one(self, to_site: str, status: str): + if self.file_downloaded_cb: + self.file_downloaded_cb(to_site, status, self.name, **self.cb_kwargs) + + def downloaded_to_all(self): + if self.file_downloaded_cb: + self.file_downloaded_cb("", "", self.name, **self.cb_kwargs) + class FileDownloader(ObjDownloader): @@ -84,7 +98,8 @@ def new_transaction( cls, cell: Cell, timeout: float, - timeout_cb, + num_receivers: int = 0, + transaction_done_cb=None, **cb_kwargs, ): """Create a new file download transaction. @@ -92,36 +107,31 @@ def new_transaction( Args: cell: the cell for communication with recipients timeout: timeout for the transaction - timeout_cb: CB to be called when the transaction is timed out - **cb_kwargs: args to be passed to the CB + num_receivers: number of receivers. 0 means unknown. + transaction_done_cb: callback function that will be called when the transaction is done. Returns: transaction id - - The timeout_cb must follow this signature: - - cb(tx_id, file_names: List[str], **cb_args) - """ return ObjDownloader.new_transaction( cell=cell, - producer=_ChunkProducer(), timeout=timeout, - timeout_cb=cls._tx_timeout, - app_timeout_cb=timeout_cb, - **cb_kwargs, + num_receivers=num_receivers, + transaction_done_cb=cls._transaction_done, + cb_info=(transaction_done_cb, cb_kwargs), ) @classmethod - def _tx_timeout(cls, tx_id: str, objs: List[Any], app_timeout_cb, **cb_kwargs): - if app_timeout_cb: - file_names = [obj.name for obj in objs] - app_timeout_cb(tx_id, file_names, **cb_kwargs) + def _transaction_done(cls, tx_id: str, status: str, objs, cb_info): + transaction_done_cb, cb_kwargs = cb_info + if transaction_done_cb: + transaction_done_cb(tx_id, [obj.name for obj in objs], **cb_kwargs) @classmethod def add_file( cls, transaction_id: str, file_name: str, + chunk_size=None, ref_id=None, file_downloaded_cb=None, **cb_kwargs, @@ -131,6 +141,7 @@ def add_file( Args: transaction_id: ID of the transaction file_name: name of the file to be downloaded + chunk_size: chunk size in bytes ref_id: ref id to be used, if provided file_downloaded_cb: CB to be called when the file is done downloading **cb_kwargs: args to be passed to the CB @@ -139,24 +150,16 @@ def add_file( The file_downloaded_cb must follow this signature: - cb(ref_id: str, to_site: str, status: str, file_name: str, **cb_kwargs) + cb(to_site: str, status: str, file_name: str, **cb_kwargs) """ - obj = _File(file_name) - return ObjDownloader.add_download_object( + obj = FileDownloadable(file_name, chunk_size=chunk_size, file_downloaded_cb=file_downloaded_cb, **cb_kwargs) + return ObjDownloader.add_object( transaction_id=transaction_id, obj=obj, ref_id=ref_id, - obj_downloaded_cb=cls._file_downloaded, - app_downloaded_cb=file_downloaded_cb, - **cb_kwargs, ) - @classmethod - def _file_downloaded(cls, ref_id: str, to_site: str, status: str, obj: _File, app_downloaded_cb, **cb_kwargs): - if app_downloaded_cb: - app_downloaded_cb(ref_id, to_site, status, obj.name, **cb_kwargs) - @classmethod def download_file( cls, @@ -168,7 +171,7 @@ def download_file( secure=False, optional=False, abort_signal=None, - ) -> (str, Optional[str]): + ) -> Tuple[str, Optional[str]]: """Download the referenced file from the file owner. Args: @@ -245,7 +248,14 @@ def download_file( secure=False, optional=False, abort_signal=None, -) -> (str, Optional[str]): +) -> Tuple[str, Optional[str]]: return FileDownloader.download_file( - from_fqcn, ref_id, per_request_timeout, cell, location, secure, optional, abort_signal + from_fqcn=from_fqcn, + ref_id=ref_id, + per_request_timeout=per_request_timeout, + cell=cell, + location=location, + secure=secure, + optional=optional, + abort_signal=abort_signal, ) diff --git a/nvflare/fuel/f3/streaming/obj_downloader.py b/nvflare/fuel/f3/streaming/obj_downloader.py index c327c0e5b5..f27ad76bd0 100644 --- a/nvflare/fuel/f3/streaming/obj_downloader.py +++ b/nvflare/fuel/f3/streaming/obj_downloader.py @@ -15,7 +15,7 @@ import time import uuid from abc import ABC, abstractmethod -from typing import Any, List, Optional, Tuple +from typing import Any, Optional, Tuple from nvflare.apis.signal import Signal from nvflare.fuel.f3.cellnet.cell import Cell @@ -23,7 +23,6 @@ from nvflare.fuel.f3.cellnet.utils import make_reply, new_cell_message from nvflare.fuel.f3.message import Message from nvflare.fuel.utils.log_utils import get_obj_logger -from nvflare.fuel.utils.validation_utils import check_callable, check_object_type from nvflare.security.logging import secure_format_exception OBJ_DOWNLOADER_CHANNEL = "obj_downloader__" @@ -93,6 +92,37 @@ """ +class Downloadable(ABC): + + def set_transaction(self, tx_id, ref_id): + pass + + @abstractmethod + def produce(self, state: dict, requester: str) -> Tuple[str, Any, dict]: + """Produce a small object to be sent (on object sender side). + + Args: + state: current state of downloading, received from the downloading site + requester: the FQCN of the site that is downloading + + Returns: a tuple of (return code, a small object to be sent, new state to be sent). + + """ + pass + + def downloaded_to_one(self, to_site: str, status: str): + """Called when an object is downloaded to a site.""" + pass + + def downloaded_to_all(self): + """Called when the object is fully downloaded to all sites.""" + pass + + def transaction_done(self, transaction_id: str, status: str): + """Called when the transaction is finished.""" + pass + + class _PropKey: REF_ID = "ref_id" STATE = "state" @@ -105,10 +135,8 @@ class _Ref: def __init__( self, tx, - obj: Any, + obj: Downloadable, ref_id=None, - obj_downloaded_cb=None, - **cb_kwargs, ): if ref_id: # use provided ref_id @@ -118,24 +146,20 @@ def __init__( self.tx = tx self.obj = obj self.num_sites_done = 0 - self.obj_downloaded_cb = obj_downloaded_cb - self.cb_kwargs = cb_kwargs def mark_active(self): self.tx.mark_active() def obj_downloaded(self, to_site: str, status: str): self.num_sites_done += 1 - if self.obj_downloaded_cb: - self.obj_downloaded_cb(self.rid, to_site, status, self.obj, **self.cb_kwargs) - assert isinstance(self.tx.producer, Producer) - self.tx.producer.object_downloaded(self.rid, self.obj, to_site, status) + assert isinstance(self.obj, Downloadable) + self.obj.downloaded_to_one(to_site, status) assert isinstance(self.tx, _Transaction) if 0 < self.tx.num_receivers <= self.num_sites_done: # this object is done for all sites - self.tx.producer.object_done(self.rid, self.obj) + self.obj.downloaded_to_all() class ProduceRC: @@ -157,88 +181,34 @@ class TransactionDoneStatus: DELETED = "deleted" -class Producer(ABC): - - def __init__(self): - self.logger = get_obj_logger(self) - - @abstractmethod - def produce(self, ref_id: str, obj: Any, state: dict, requester: str) -> Tuple[str, Any, dict]: - """Produce a small object to be sent (on object sender side). - - Args: - ref_id: the ref id of the object being downloaded - obj: the large object - state: current state of downloading, received from the downloading site - requester: the FQCN of the site that is downloading - - Returns: a tuple of (return code, a small object to be sent, new state to be sent). - - """ - pass - - def object_downloaded(self, ref_id: str, obj: Any, to_site: str, status: str): - """Called when an object is downloaded to a site.""" - pass - - def object_done(self, ref_id: str, obj: Any): - """Called when the object is fully downloaded to all sites.""" - pass - - def transaction_done(self, transaction_id: str, objs: List[Any], status: str): - """Called when the transaction is finished.""" - pass - - class _Transaction: def __init__( self, - producer: Producer, timeout: float, num_receivers: int, tx_id=None, - timeout_cb=None, - **cb_kwargs, + transaction_done_cb=None, + cb_kwargs=None, ): """Constructor of the transaction object. Args: - producer: the Producer object to produce small objects. timeout: amount of time since last activity num_receivers: number of receivers. 0 means unlimited. tx_id: if provided, use it; otherwise create one - timeout_cb: the CB to be called when the transaction timed out - **cb_kwargs: args to be passed to the timeout CB """ - check_callable("timeout_cb", timeout_cb) - check_object_type("producer", producer, Producer) if tx_id: self.tid = tx_id else: self.tid = "T" + str(uuid.uuid4()) - self.producer = producer self.timeout = timeout self.num_receivers = num_receivers - self.timeout_cb = timeout_cb - self.cb_kwargs = cb_kwargs self.last_active_time = time.time() + self.transaction_done_cb = transaction_done_cb + self.cb_kwargs = cb_kwargs self.refs = [] - def produce(self, ref_id: str, obj: Any, state: dict, requester: str): - """Called to produce the next small object to be sent. - - Args: - ref_id: ref id of the object being downloaded - obj: the large object being downloaded - state: current state received from the downloading site (requester). - requester: FQCN of the requester - - Returns: - - """ - return self.producer.produce(ref_id, obj, state, requester) - def mark_active(self): """Called to update the last active time of the transaction. @@ -247,26 +217,23 @@ def mark_active(self): """ self.last_active_time = time.time() - def add_download_object( + def add_object( self, - obj: Any, + obj: Downloadable, ref_id=None, - obj_downloaded_cb=None, - **cb_kwargs, ): """Add a large object (to be downloaded) to the transaction. Args: obj: the large object to be downloaded ref_id: the ref id to be used, if specified - obj_downloaded_cb: the CB to be called when the object is fully downloaded - **cb_kwargs: args to be passed to the CB. Returns: """ - r = _Ref(self, obj, ref_id, obj_downloaded_cb, **cb_kwargs) + r = _Ref(self, obj, ref_id) self.refs.append(r) + obj.set_transaction(self.tid, r.rid) return r def timed_out(self): @@ -275,8 +242,7 @@ def timed_out(self): Returns: """ - if self.timeout_cb: - self.timeout_cb(self.tid, [r.obj for r in self.refs], **self.cb_kwargs) + self.transaction_done(TransactionDoneStatus.TIMEOUT) def is_finished(self): """Check whether the transaction is finished (all objects are downloaded).""" @@ -291,13 +257,18 @@ def is_finished(self): def transaction_done(self, status: str): """Called when the transaction is finished.""" - self.producer.transaction_done(self.tid, [r.obj for r in self.refs], status) + for ref in self.refs: + obj = ref.obj + assert isinstance(obj, Downloadable) + obj.transaction_done(self.tid, status) + + if self.transaction_done_cb: + self.transaction_done_cb(self.tid, status, [ref.obj for ref in self.refs], **self.cb_kwargs) class TransactionInfo: def __init__(self, tx: _Transaction): - self.producer = tx.producer self.timeout = tx.timeout self.num_receivers = tx.num_receivers self.objects = [r.obj for r in tx.refs] @@ -337,47 +308,44 @@ def _initialize(cls, cell: Cell): def new_transaction( cls, cell: Cell, - producer: Producer, timeout: float, num_receivers: int = 0, tx_id=None, - timeout_cb=None, + transaction_done_cb=None, **cb_kwargs, ): cls._initialize(cell) - tx = _Transaction(producer, timeout, num_receivers, tx_id, timeout_cb, **cb_kwargs) + tx = _Transaction(timeout, num_receivers, tx_id, transaction_done_cb, cb_kwargs) with cls._tx_lock: cls._tx_table[tx.tid] = tx return tx.tid @classmethod - def add_download_object( + def add_object( cls, transaction_id: str, - obj: Any, + obj: Downloadable, ref_id=None, - obj_downloaded_cb=None, - **cb_kwargs, ) -> str: - if obj_downloaded_cb is not None: - check_callable("obj_downloaded_cb", obj_downloaded_cb) + if not issubclass(type(obj), Downloadable): + raise ValueError(f"obj must be of type {Downloadable} but got {type(obj)}") tx = cls._tx_table.get(transaction_id) if not tx: raise ValueError(f"no such transaction {transaction_id}") assert isinstance(tx, _Transaction) - ref = tx.add_download_object(obj, ref_id, obj_downloaded_cb, **cb_kwargs) + ref = tx.add_object(obj, ref_id) with cls._tx_lock: cls._ref_table[ref.rid] = ref return ref.rid @classmethod - def delete_transaction(cls, transaction_id: str, call_cb=False): + def delete_transaction(cls, transaction_id: str): with cls._tx_lock: tx = cls._tx_table.get(transaction_id) if tx: - cls._delete_tx(tx, call_cb) + cls._delete_tx(tx) tx.transaction_done(TransactionDoneStatus.DELETED) @classmethod @@ -391,17 +359,11 @@ def shutdown(cls): tx_list = list(cls._tx_table.values()) if tx_list: for tx in tx_list: - cls._delete_tx(tx, True) + cls._delete_tx(tx) tx.transaction_done(TransactionDoneStatus.DELETED) @classmethod - def _delete_tx(cls, tx: _Transaction, call_cb=False): - if call_cb: - try: - tx.timed_out() - except Exception as ex: - cls._logger.error(f"exception from timeout_cb: {secure_format_exception(ex)}") - + def _delete_tx(cls, tx: _Transaction): cls._tx_table.pop(tx.tid, None) # remove all refs @@ -448,9 +410,11 @@ def _handle_download(cls, request: Message) -> Message: assert isinstance(tx, _Transaction) try: - rc, data, new_state = tx.produce(rid, ref.obj, current_state, requester) + rc, data, new_state = ref.obj.produce(current_state, requester) except Exception as ex: - cls._logger.error(f"Producer {type(tx.producer)} encountered exception: {secure_format_exception(ex)}") + cls._logger.error( + f"Object {type(ref.obj)} encountered exception when produce: {secure_format_exception(ex)}" + ) return make_reply(ReturnCode.PROCESS_EXCEPTION) if rc != ProduceRC.OK: @@ -489,11 +453,11 @@ def _monitor_tx(cls): for tx in expired_tx: assert isinstance(tx, _Transaction) tx.transaction_done(TransactionDoneStatus.TIMEOUT) - cls._delete_tx(tx, True) + cls._delete_tx(tx) for tx in finished_tx: tx.transaction_done(TransactionDoneStatus.FINISHED) - cls._delete_tx(tx, False) + cls._delete_tx(tx) time.sleep(5.0) diff --git a/nvflare/fuel/f3/streaming/smod.py b/nvflare/fuel/f3/streaming/smod.py deleted file mode 100644 index d16d7941eb..0000000000 --- a/nvflare/fuel/f3/streaming/smod.py +++ /dev/null @@ -1,105 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from abc import ABC, abstractmethod -from typing import Any, List, Tuple - -from nvflare.fuel.f3.cellnet.cell import Cell -from nvflare.fuel.f3.streaming.obj_downloader import ObjDownloader, Producer -from nvflare.fuel.utils.log_utils import get_obj_logger - - -class SelfManagedObject(ABC): - - def set_transaction(self, tx_id, ref_id): - pass - - @abstractmethod - def produce(self, state: dict, requester: str) -> Tuple[str, Any, dict]: - """Produce a small object to be sent (on object sender side). - - Args: - state: current state of downloading, received from the downloading site - requester: the FQCN of the site that is downloading - - Returns: a tuple of (return code, a small object to be sent, new state to be sent). - - """ - pass - - def downloaded_to_one(self, to_site: str, status: str): - """Called when an object is downloaded to a site.""" - pass - - def downloaded_to_all(self): - """Called when the object is fully downloaded to all sites.""" - pass - - def transaction_done(self, transaction_id: str, status: str): - """Called when the transaction is finished.""" - pass - - -class _SMODProducer(Producer): - - def __init__(self): - super().__init__() - - def produce(self, ref_id: str, obj: SelfManagedObject, state: dict, requester: str) -> Tuple[str, Any, dict]: - return obj.produce(state, requester) - - def object_downloaded(self, ref_id: str, obj: SelfManagedObject, to_site: str, status: str): - obj.downloaded_to_one(to_site, status) - - def object_done(self, ref_id: str, obj: SelfManagedObject): - """Called when the object is fully downloaded to all sites.""" - obj.downloaded_to_all() - - def transaction_done(self, transaction_id: str, objs: List[SelfManagedObject], status: str): - for obj in objs: - obj.transaction_done(transaction_id, status) - - -class SelfManagedObjectDownloader: - - @classmethod - def new_transaction( - cls, cell: Cell, timeout: float, num_receivers: int = 0, tx_id=None, timeout_cb=None, **cb_kwargs - ): - return ObjDownloader.new_transaction( - cell=cell, - producer=_SMODProducer(), - timeout=timeout, - num_receivers=num_receivers, - tx_id=tx_id, - timeout_cb=timeout_cb, - **cb_kwargs, - ) - - @classmethod - def add_download_object( - cls, - transaction_id: str, - obj: SelfManagedObject, - ref_id=None, - ) -> str: - if not issubclass(type(obj), SelfManagedObject): - raise TypeError(f"obj must be an instance of SelfManagedObject but got {type(obj)}") - - rid = ObjDownloader.add_download_object( - transaction_id=transaction_id, - obj=obj, - ref_id=ref_id, - ) - obj.set_transaction(transaction_id, rid) - return rid diff --git a/nvflare/fuel/hci/server/binary_transfer.py b/nvflare/fuel/hci/server/binary_transfer.py index 3b4b75f8f9..bdb030b76b 100644 --- a/nvflare/fuel/hci/server/binary_transfer.py +++ b/nvflare/fuel/hci/server/binary_transfer.py @@ -44,7 +44,7 @@ def download_folder(self, conn: Connection, tx_id: str, folder_name: str): download_tid = FileDownloader.new_transaction( cell=engine.get_cell(), timeout=5, - timeout_cb=self._cleanup_tx, + transaction_done_cb=self._cleanup_tx, tx_path=tx_path, ) diff --git a/nvflare/fuel/utils/fobs/decomposers/via_file.py b/nvflare/fuel/utils/fobs/decomposers/via_file.py index cb029f5f8a..67baac0fca 100644 --- a/nvflare/fuel/utils/fobs/decomposers/via_file.py +++ b/nvflare/fuel/utils/fobs/decomposers/via_file.py @@ -272,7 +272,7 @@ def _create_download_tx(self, fobs_ctx: dict): tx_id = self.file_downloader_class.new_transaction( cell=cell, timeout=timeout, - timeout_cb=self._delete_download_tx, + transaction_done_cb=self._delete_download_tx, ) if msg_root_id: @@ -385,7 +385,7 @@ def _finalize_download_tx(self, mgr: DatumManager): def _delete_download_tx_on_msg_root(self, msg_root_id: str, download_tx_id: str): # this CB is triggered when msg root is deleted. self.logger.debug(f"deleting download_tx_id {download_tx_id} associated with {msg_root_id=}") - self.file_downloader_class.delete_transaction(download_tx_id, call_cb=True) + self.file_downloader_class.delete_transaction(download_tx_id) def _delete_download_tx(self, tx_id, file_names): # this CB is triggered when download tx times out or is deleted diff --git a/nvflare/private/fed/client/client_runner.py b/nvflare/private/fed/client/client_runner.py index 12ef61f4fb..466c1ae272 100644 --- a/nvflare/private/fed/client/client_runner.py +++ b/nvflare/private/fed/client/client_runner.py @@ -29,7 +29,7 @@ from nvflare.apis.utils.reliable_message import ReliableMessage from nvflare.apis.utils.task_utils import apply_filters from nvflare.fuel.f3.cellnet.fqcn import FQCN -from nvflare.fuel.f3.streaming.file_downloader import FileDownloader +from nvflare.fuel.f3.streaming.obj_downloader import ObjDownloader from nvflare.fuel.utils.msg_root_utils import delete_msg_root from nvflare.private.defs import SpecialTaskName, TaskConstant from nvflare.private.fed.client.client_engine_executor_spec import ClientEngineExecutorSpec, TaskAssignment @@ -655,7 +655,7 @@ def run(self, app_root, args): self.end_run_events_sequence() ReliableMessage.shutdown() self.engine.shutdown_streamer() - FileDownloader.shutdown() + ObjDownloader.shutdown() with self.task_lock: self.running_tasks = {} diff --git a/nvflare/private/fed/server/server_runner.py b/nvflare/private/fed/server/server_runner.py index 2e0532f663..39080a4f7a 100644 --- a/nvflare/private/fed/server/server_runner.py +++ b/nvflare/private/fed/server/server_runner.py @@ -26,7 +26,7 @@ from nvflare.apis.utils.fl_context_utils import add_job_audit_event from nvflare.apis.utils.reliable_message import ReliableMessage from nvflare.apis.utils.task_utils import apply_filters -from nvflare.fuel.f3.streaming.file_downloader import FileDownloader +from nvflare.fuel.f3.streaming.obj_downloader import ObjDownloader from nvflare.fuel.utils.job_utils import build_client_hierarchy from nvflare.private.defs import SpecialTaskName, TaskConstant from nvflare.private.fed.tbi import TBI @@ -230,7 +230,7 @@ def run(self): ReliableMessage.shutdown() self.engine.shutdown_streamer() - FileDownloader.shutdown() + ObjDownloader.shutdown() self.log_info(fl_ctx, "Server runner finished.") def handle_event(self, event_type: str, fl_ctx: FLContext): From eec0235228a32a9dcaafa20e6181dd77b0b6be03 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Tue, 21 Oct 2025 14:44:43 +0800 Subject: [PATCH 044/102] add mixed example --- nvflare/fox/examples/np/algos/avg_stream.py | 12 +- nvflare/fox/examples/pt/pt_avg_mixed.py | 248 ++++++++++++++++++ .../fox/examples/pt/recipe_pt_avg_mixed.py | 51 ++++ nvflare/fox/sys/constants.py | 5 + nvflare/fox/sys/file_downloader.py | 7 +- nvflare/fox/sys/general_downloader.py | 45 ++++ nvflare/fox/sys/model_downloader.py | 9 +- nvflare/fuel/f3/streaming/obj_downloader.py | 2 +- 8 files changed, 363 insertions(+), 16 deletions(-) create mode 100644 nvflare/fox/examples/pt/pt_avg_mixed.py create mode 100644 nvflare/fox/examples/pt/recipe_pt_avg_mixed.py create mode 100644 nvflare/fox/sys/general_downloader.py diff --git a/nvflare/fox/examples/np/algos/avg_stream.py b/nvflare/fox/examples/np/algos/avg_stream.py index e7e97ba402..ea72952067 100644 --- a/nvflare/fox/examples/np/algos/avg_stream.py +++ b/nvflare/fox/examples/np/algos/avg_stream.py @@ -130,23 +130,21 @@ def train(self, current_round, weights, model_type: str, context: Context): os.remove(file_path) result = weights + self.delta + if model_type == "ref": # stream it file_name = f"/tmp/np_{str(uuid.uuid4())}.npy" save_np_model(result, file_name) - model = prepare_file_for_download( + result = prepare_file_for_download( num_receivers=1, file_name=file_name, ctx=context, timeout=5.0, file_downloaded_cb=self._result_downloaded, ) - model_type = "ref" - self.logger.info(f"prepared result as ref: {model}") - else: - model = result - model_type = "model" - return model, model_type + self.logger.info(f"prepared result as ref: {result}") + + return result, model_type def _result_downloaded(self, to_site: str, status: str, file_name): self.logger.info(f"model file {file_name} downloaded to {to_site}: {status=}") diff --git a/nvflare/fox/examples/pt/pt_avg_mixed.py b/nvflare/fox/examples/pt/pt_avg_mixed.py new file mode 100644 index 0000000000..1f33297e22 --- /dev/null +++ b/nvflare/fox/examples/pt/pt_avg_mixed.py @@ -0,0 +1,248 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import os +import threading +import uuid + +import torch + +from nvflare.app_opt.pt.tensor_downloader import TensorDownloadable +from nvflare.fox.api.app import ClientApp, ServerApp +from nvflare.fox.api.constants import EnvType +from nvflare.fox.api.ctx import Context +from nvflare.fox.api.dec import collab +from nvflare.fox.api.group import all_clients +from nvflare.fox.api.strategy import Strategy +from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples.np.algos.utils import load_np_model, parse_array_def, save_np_model +from nvflare.fox.examples.pt.utils import parse_state_dict +from nvflare.fox.sim.simulator import Simulator +from nvflare.fox.sys.file_downloader import download_file +from nvflare.fox.sys.general_downloader import GeneralDownloader +from nvflare.fox.sys.model_downloader import download_model +from nvflare.fuel.f3.streaming.file_downloader import FileDownloadable +from nvflare.fuel.utils.log_utils import get_obj_logger + + +class _AggrResult: + + def __init__(self): + self.pt_total = {} + self.np_total = 0 + self.count = 0 + self.lock = threading.Lock() # ensure update integrity + + +class PTFedAvgMixed(Strategy): + + def __init__(self, pt_model, np_model, num_rounds=10, timeout=2.0): + self.num_rounds = num_rounds + self.pt_model = pt_model + self.np_model = np_model + self.timeout = timeout + self.name = "PTFedAvgMixed" + self.logger = get_obj_logger(self) + self._pt_model = parse_state_dict(pt_model) + self._np_model = parse_array_def(np_model) + + def execute(self, context: Context): + self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") + pt_model, np_model = self._pt_model, self._np_model + for i in range(self.num_rounds): + pt_model, np_model = self._do_one_round(i, pt_model, np_model, context) + self.logger.info(f"FINAL MODEL: {pt_model=} {np_model=}") + return pt_model, np_model + + def _do_one_round(self, r, pt_model, np_model, ctx: Context): + aggr_result = _AggrResult() + + grp = all_clients( + ctx, + process_resp_cb=self._accept_train_result, + aggr_result=aggr_result, + ) + + file_name = None + if ctx.env_type == EnvType.SYSTEM: + file_name = f"/tmp/np_{str(uuid.uuid4())}.npy" + save_np_model(np_model, file_name) + + downloader = GeneralDownloader( + num_receivers=grp.size, + ctx=ctx, + timeout=5.0, + ) + model_type = "ref" + pt_model = downloader.add_object(TensorDownloadable(pt_model, 2)) + np_model = downloader.add_object(FileDownloadable(file_name)) + self.logger.info(f"prepared model as ref: {pt_model=} {np_model=}") + else: + model_type = "model" + + grp.train(r, pt_model, np_model, model_type) + + if file_name: + os.remove(file_name) + + if aggr_result.count == 0: + return None + else: + pt_result = {} + for k, v in aggr_result.pt_total.items(): + pt_result[k] = torch.div(v, aggr_result.count) + + self.logger.info( + f"[{ctx.header_str()}] round {r}: aggr PT result from {aggr_result.count} clients: {pt_result}" + ) + + np_result = aggr_result.np_total / aggr_result.count + self.logger.info( + f"[{ctx.header_str()}] round {r}: aggr NP result from {aggr_result.count} clients: {np_result}" + ) + return pt_result, np_result + + def _accept_train_result(self, result, aggr_result: _AggrResult, context: Context): + self.logger.info(f"[{context.header_str()}] got train result from {context.caller}: {result}") + + pt_result, np_result, model_type = result + if model_type == "ref": + err, pt_result = download_model( + ref=pt_result, + per_request_timeout=5.0, + ctx=context, + model_received_cb=self._aggregate_tensors, + aggr_result=aggr_result, + context=context, + ) + if err: + raise RuntimeError(f"failed to download model {pt_result}: {err}") + + err, file_path = download_file(ref=np_result, per_request_timeout=5.0, ctx=context) + if err: + raise RuntimeError(f"failed to download NP model file {np_result}: {err}") + self.logger.info(f"downloaded model file to {file_path}") + np_result = load_np_model(file_path) + os.remove(file_path) + else: + for k, v in pt_result.items(): + if k not in aggr_result.pt_total: + aggr_result.pt_total[k] = v + else: + aggr_result.pt_total[k] += v + + aggr_result.np_total += np_result + aggr_result.count += 1 + return None + + def _aggregate_tensors(self, td: dict[str, torch.Tensor], aggr_result: _AggrResult, context: Context): + self.logger.info(f"[{context.header_str()}] aggregating received tensor: {td}") + with aggr_result.lock: + for k, v in td.items(): + if k not in aggr_result.pt_total: + aggr_result.pt_total[k] = v + else: + aggr_result.pt_total[k] += v + + +class PTTrainer(ClientApp): + + def __init__(self, delta: float): + ClientApp.__init__(self) + self.delta = delta + + @collab + def train(self, current_round, pt_model, np_model, model_type: str, context: Context): + if context.is_aborted(): + self.logger.debug("training aborted") + return 0 + self.logger.debug( + f"[{context.header_str()}] training round {current_round}: {model_type=} {pt_model=} {np_model=}" + ) + if model_type == "ref": + err, pt_model = download_model(ref=pt_model, per_request_timeout=5.0, ctx=context) + if err: + raise RuntimeError(f"failed to download PT model {pt_model}: {err}") + self.logger.info(f"downloaded PT model {pt_model}") + + err, file_path = download_file(ref=np_model, per_request_timeout=5.0, ctx=context) + if err: + raise RuntimeError(f"failed to download NP model file {np_model}: {err}") + self.logger.info(f"downloaded model file to {file_path}") + np_model = load_np_model(file_path) + self.logger.info(f"loaded NP model from file: {np_model}") + os.remove(file_path) + + pt_result = {} + for k, v in pt_model.items(): + pt_result[k] = v + self.delta + + np_result = np_model + self.delta + + if model_type == "ref": + # stream it + downloader = GeneralDownloader( + num_receivers=1, + ctx=context, + timeout=5.0, + ) + pt_result = downloader.add_object(TensorDownloadable(pt_result, 2)) + self.logger.info(f"prepared PT result as ref: {pt_result}") + + file_name = f"/tmp/np_{str(uuid.uuid4())}.npy" + save_np_model(np_result, file_name) + np_result = downloader.add_object(FileDownloadable(file_name, file_downloaded_cb=self._result_downloaded)) + self.logger.info(f"prepared NP result as ref: {np_result}") + + return pt_result, np_result, model_type + + def _result_downloaded(self, to_site: str, status: str, file_name): + self.logger.info(f"NP model file {file_name} downloaded to {to_site}: {status=}") + if not to_site: + # downloaded to all sites + os.remove(file_name) + self.logger.info(f"NP model file {file_name} removed") + + +def main(): + simple_logging(logging.DEBUG) + + server_app = ServerApp( + strategy_name="fed_avg_mixed", + strategy=PTFedAvgMixed( + pt_model={ + "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "y": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "z": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + }, + np_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], + num_rounds=4, + ), + ) + + client_app = PTTrainer(delta=1.0) + + simulator = Simulator( + root_dir="/tmp/fox", + experiment_name="fedavg_mixed", + server_app=server_app, + client_app=client_app, + num_clients=2, + ) + + simulator.run() + + +if __name__ == "__main__": + main() diff --git a/nvflare/fox/examples/pt/recipe_pt_avg_mixed.py b/nvflare/fox/examples/pt/recipe_pt_avg_mixed.py new file mode 100644 index 0000000000..c0c14655e4 --- /dev/null +++ b/nvflare/fox/examples/pt/recipe_pt_avg_mixed.py @@ -0,0 +1,51 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from nvflare.fox.api.app import ServerApp +from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples.pt.pt_avg_mixed import PTFedAvgMixed, PTTrainer +from nvflare.fox.sys.recipe import FoxRecipe + +JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/fox/prod_00/admin@nvidia.com/transfer" + + +def main(): + simple_logging(logging.DEBUG) + + server_app = ServerApp( + strategy_name="fedavg_mixed", + strategy=PTFedAvgMixed( + pt_model={ + "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "y": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "z": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + }, + np_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], + num_rounds=2, + ), + ) + + client_app = PTTrainer(delta=1.0) + + recipe = FoxRecipe( + job_name="pt_fedavg_mixed", + server_app=server_app, + client_app=client_app, + ) + recipe.export(JOB_ROOT_DIR) + + +if __name__ == "__main__": + main() diff --git a/nvflare/fox/sys/constants.py b/nvflare/fox/sys/constants.py index b7f85b298d..569a2f87cf 100644 --- a/nvflare/fox/sys/constants.py +++ b/nvflare/fox/sys/constants.py @@ -34,3 +34,8 @@ class ObjectCallKey: class CallReplyKey: ERROR = "error" RESULT = "result" + + +class DownloaderKey: + SOURCE = "source" + REF_ID = "ref_id" diff --git a/nvflare/fox/sys/file_downloader.py b/nvflare/fox/sys/file_downloader.py index 4d2d45d47b..3a4b40b6b2 100644 --- a/nvflare/fox/sys/file_downloader.py +++ b/nvflare/fox/sys/file_downloader.py @@ -13,6 +13,7 @@ # limitations under the License. from nvflare.fox.api.ctx import Context from nvflare.fox.sys.backend import SysBackend +from nvflare.fox.sys.constants import DownloaderKey from nvflare.fuel.f3.streaming.file_downloader import FileDownloader @@ -40,7 +41,7 @@ def prepare_file_for_download( file_downloaded_cb=file_downloaded_cb, **cb_kwargs, ) - return {"source": cell.get_fqcn(), "rid": rid} + return {DownloaderKey.SOURCE: cell.get_fqcn(), DownloaderKey.REF_ID: rid} def download_file(ref: dict, per_request_timeout: float, ctx: Context): @@ -49,8 +50,8 @@ def download_file(ref: dict, per_request_timeout: float, ctx: Context): raise ValueError(f"backend must be SysBackend but got {type(backend)}") err, file_path = FileDownloader.download_file( - from_fqcn=ref.get("source"), - ref_id=ref.get("rid"), + from_fqcn=ref.get(DownloaderKey.SOURCE), + ref_id=ref.get(DownloaderKey.REF_ID), per_request_timeout=per_request_timeout, cell=backend.cell, abort_signal=ctx.abort_signal, diff --git a/nvflare/fox/sys/general_downloader.py b/nvflare/fox/sys/general_downloader.py new file mode 100644 index 0000000000..1b0762ab83 --- /dev/null +++ b/nvflare/fox/sys/general_downloader.py @@ -0,0 +1,45 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nvflare.fox.api.ctx import Context +from nvflare.fox.sys.backend import SysBackend +from nvflare.fuel.f3.streaming.obj_downloader import Downloadable, ObjDownloader + +from .constants import DownloaderKey + + +class GeneralDownloader: + + def __init__( + self, + num_receivers: int, + timeout: float, + ctx: Context, + ): + backend = ctx.backend + if not isinstance(backend, SysBackend): + raise ValueError(f"backend must be SysBackend but got {type(backend)}") + + self.cell = backend.cell + self.tx_id = ObjDownloader.new_transaction( + cell=self.cell, + timeout=timeout, + num_receivers=num_receivers, + ) + + def add_object(self, obj: Downloadable): + rid = ObjDownloader.add_object( + transaction_id=self.tx_id, + obj=obj, + ) + return {DownloaderKey.SOURCE: self.cell.get_fqcn(), DownloaderKey.REF_ID: rid} diff --git a/nvflare/fox/sys/model_downloader.py b/nvflare/fox/sys/model_downloader.py index 01fa7111d1..d2257056c6 100644 --- a/nvflare/fox/sys/model_downloader.py +++ b/nvflare/fox/sys/model_downloader.py @@ -17,8 +17,7 @@ from nvflare.fox.api.ctx import Context from nvflare.fox.sys.backend import SysBackend -SOURCE = "source" -REF_ID = "ref_id" +from .constants import DownloaderKey class ModelDownloader: @@ -46,7 +45,7 @@ def add_model(self, model: dict[str, torch.Tensor], num_tensors_per_chunk: int = tensors=model, num_tensors_per_chunk=num_tensors_per_chunk, ) - return {SOURCE: self.cell.get_fqcn(), REF_ID: rid} + return {DownloaderKey.SOURCE: self.cell.get_fqcn(), DownloaderKey.REF_ID: rid} def download_model(ref: dict, per_request_timeout: float, ctx: Context, model_received_cb=None, **cb_kwargs): @@ -55,8 +54,8 @@ def download_model(ref: dict, per_request_timeout: float, ctx: Context, model_re raise ValueError(f"backend must be SysBackend but got {type(backend)}") return TensorDownloader.download_tensors( - from_fqcn=ref.get(SOURCE), - ref_id=ref.get(REF_ID), + from_fqcn=ref.get(DownloaderKey.SOURCE), + ref_id=ref.get(DownloaderKey.REF_ID), per_request_timeout=per_request_timeout, cell=backend.cell, abort_signal=ctx.abort_signal, diff --git a/nvflare/fuel/f3/streaming/obj_downloader.py b/nvflare/fuel/f3/streaming/obj_downloader.py index f27ad76bd0..c654663059 100644 --- a/nvflare/fuel/f3/streaming/obj_downloader.py +++ b/nvflare/fuel/f3/streaming/obj_downloader.py @@ -394,7 +394,7 @@ def _handle_download(cls, request: Message) -> Message: assert isinstance(payload, dict) rid = payload.get(_PropKey.REF_ID) if not rid: - cls._logger.erro(f"missing {_PropKey.REF_ID} in request from {requester}") + cls._logger.error(f"missing {_PropKey.REF_ID} in request from {requester}") return make_reply(ReturnCode.INVALID_REQUEST) current_state = payload.get(_PropKey.STATE) From 2de9be0bb112661448ebaf83b2fa036834fc7796 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Wed, 22 Oct 2025 14:55:39 +0800 Subject: [PATCH 045/102] adjust downloader system --- nvflare/app_opt/pt/tensor_downloader.py | 168 ++--- nvflare/fox/examples/np/algos/avg_stream.py | 12 +- nvflare/fox/examples/np/fed_avg_para.py | 3 +- nvflare/fox/examples/np/fed_avg_seq.py | 3 +- nvflare/fox/examples/np/fed_avg_stream.py | 3 +- nvflare/fox/examples/np/recipe_swarm.py | 4 +- nvflare/fox/examples/np/swarm.py | 3 +- nvflare/fox/examples/pt/filters.py | 14 +- nvflare/fox/examples/pt/pt_avg_filter.py | 5 +- nvflare/fox/examples/pt/pt_avg_mixed.py | 27 +- nvflare/fox/examples/pt/pt_avg_stream.py | 25 +- nvflare/fox/sim/simulator.py | 4 +- nvflare/fox/sys/constants.py | 5 - nvflare/fox/sys/downloader.py | 112 ++++ nvflare/fox/sys/file_downloader.py | 59 -- nvflare/fox/sys/general_downloader.py | 45 -- nvflare/fox/sys/model_downloader.py | 64 -- nvflare/fuel/f3/streaming/cacheable.py | 5 +- nvflare/fuel/f3/streaming/download_service.py | 600 ++++++++++++++++++ nvflare/fuel/f3/streaming/file_downloader.py | 211 +++--- nvflare/fuel/f3/streaming/obj_downloader.py | 588 +---------------- nvflare/fuel/hci/server/binary_transfer.py | 16 +- .../fuel/utils/fobs/decomposers/via_file.py | 48 +- nvflare/private/fed/client/client_runner.py | 4 +- nvflare/private/fed/server/server_runner.py | 4 +- 25 files changed, 958 insertions(+), 1074 deletions(-) create mode 100644 nvflare/fox/sys/downloader.py delete mode 100644 nvflare/fox/sys/file_downloader.py delete mode 100644 nvflare/fox/sys/general_downloader.py delete mode 100644 nvflare/fox/sys/model_downloader.py create mode 100644 nvflare/fuel/f3/streaming/download_service.py diff --git a/nvflare/app_opt/pt/tensor_downloader.py b/nvflare/app_opt/pt/tensor_downloader.py index 422cc9109b..8e4be46ae8 100644 --- a/nvflare/app_opt/pt/tensor_downloader.py +++ b/nvflare/app_opt/pt/tensor_downloader.py @@ -19,7 +19,8 @@ from nvflare.fuel.f3.cellnet.cell import Cell from nvflare.fuel.f3.streaming.cacheable import CacheableObject, ItemConsumer -from nvflare.fuel.f3.streaming.obj_downloader import ObjDownloader, download_object +from nvflare.fuel.f3.streaming.download_service import download_object +from nvflare.fuel.f3.streaming.obj_downloader import ObjectDownloader class TensorDownloadable(CacheableObject): @@ -30,6 +31,9 @@ def __init__(self, tensors: dict[str, torch.Tensor], num_items_per_chunk: int): self.keys = list(tensors.keys()) super().__init__(num_items_per_chunk) + def get_base_object(self): + return self.tensors + def get_item_count(self) -> int: return self.size @@ -69,108 +73,60 @@ def consume_items(self, items: List[Any], result: Any) -> Any: return result -class TensorDownloader: - - @classmethod - def new_transaction( - cls, - cell: Cell, - num_receivers: int, - timeout: float = 5.0, - transaction_done_cb=None, - **cb_kwargs, - ): - """Create a new tensor download transaction. - - Args: - cell: the cell for communication with recipients - num_receivers: number of receivers - timeout: timeout for the transaction - transaction_done_cb: CB to be called when the transaction is done - **cb_kwargs: args to be passed to the CB - - Returns: transaction id - - The transaction_done_cb must follow this signature: - - cb(tx_id, tensors: List[dict[str, torch.Tensor], **cb_args) - - """ - return ObjDownloader.new_transaction( - cell=cell, - num_receivers=num_receivers, - timeout=timeout, - transaction_done_cb=cls._tx_done, - cb_info=(transaction_done_cb, cb_kwargs), - ) - - @classmethod - def _tx_done(cls, tx_id: str, status: str, objs, cb_info): - transaction_done_cb, cb_kwargs = cb_info - if transaction_done_cb: - tensors = [obj.tensors for obj in objs] - transaction_done_cb(tx_id, tensors, **cb_kwargs) - - @classmethod - def add_tensors( - cls, - transaction_id: str, - tensors: dict[str, torch.Tensor], - num_tensors_per_chunk: int = 1, - ) -> str: - """Add a file to be downloaded to the specified transaction. - - Args: - transaction_id: ID of the transaction - tensors: state dict to be downloaded - num_tensors_per_chunk: number of tensors per chunk - - Returns: reference id for the state dict. - - """ - obj = TensorDownloadable(tensors, num_tensors_per_chunk) - return ObjDownloader.add_object( - transaction_id=transaction_id, - obj=obj, - ) - - @classmethod - def download_tensors( - cls, - from_fqcn: str, - ref_id: str, - per_request_timeout: float, - cell: Cell, - secure=False, - optional=False, - abort_signal=None, - tensors_received_cb=None, - **cb_kwargs, - ) -> Tuple[str, Optional[dict[str, torch.Tensor]]]: - """Download the referenced state dict from the source. - - Args: - from_fqcn: FQCN of the data source. - ref_id: reference ID of the state dict to be downloaded. - per_request_timeout: timeout for requests sent to the data source. - cell: cell to be used for communicating to the data source. - secure: P2P private mode for communication - optional: supress log messages of communication - abort_signal: signal for aborting download. - tensors_received_cb: the callback to be called when one set of tensors are received - - Returns: tuple of (error message if any, downloaded state dict). - - """ - consumer = TensorConsumer(tensors_received_cb, cb_kwargs) - download_object( - from_fqcn=from_fqcn, - ref_id=ref_id, - consumer=consumer, - per_request_timeout=per_request_timeout, - cell=cell, - secure=secure, - optional=optional, - abort_signal=abort_signal, - ) - return consumer.error, consumer.result +def add_tensors( + downloader: ObjectDownloader, + tensors: dict[str, torch.Tensor], + num_tensors_per_chunk: int = 1, +) -> str: + """Add a file to be downloaded to the specified downloader. + + Args: + downloader: the downloader to add tensors to. + tensors: state dict to be downloaded + num_tensors_per_chunk: number of tensors per chunk + + Returns: reference id for the state dict. + + """ + obj = TensorDownloadable(tensors, num_tensors_per_chunk) + return downloader.add_object(obj) + + +def download_tensors( + from_fqcn: str, + ref_id: str, + per_request_timeout: float, + cell: Cell, + secure=False, + optional=False, + abort_signal=None, + tensors_received_cb=None, + **cb_kwargs, +) -> Tuple[str, Optional[dict[str, torch.Tensor]]]: + """Download the referenced state dict from the source. + + Args: + from_fqcn: FQCN of the data source. + ref_id: reference ID of the state dict to be downloaded. + per_request_timeout: timeout for requests sent to the data source. + cell: cell to be used for communicating to the data source. + secure: P2P private mode for communication + optional: supress log messages of communication + abort_signal: signal for aborting download. + tensors_received_cb: the callback to be called when one set of tensors are received + + Returns: tuple of (error message if any, downloaded state dict). + + """ + consumer = TensorConsumer(tensors_received_cb, cb_kwargs) + download_object( + from_fqcn=from_fqcn, + ref_id=ref_id, + consumer=consumer, + per_request_timeout=per_request_timeout, + cell=cell, + secure=secure, + optional=optional, + abort_signal=abort_signal, + ) + return consumer.error, consumer.result diff --git a/nvflare/fox/examples/np/algos/avg_stream.py b/nvflare/fox/examples/np/algos/avg_stream.py index ea72952067..6560f1ffb3 100644 --- a/nvflare/fox/examples/np/algos/avg_stream.py +++ b/nvflare/fox/examples/np/algos/avg_stream.py @@ -21,7 +21,7 @@ from nvflare.fox.api.group import all_clients from nvflare.fox.api.strategy import Strategy from nvflare.fox.examples.np.algos.utils import load_np_model, parse_array_def, save_np_model -from nvflare.fox.sys.file_downloader import download_file, prepare_file_for_download +from nvflare.fox.sys.downloader import Downloader, download_file from nvflare.fuel.utils.log_utils import get_obj_logger @@ -63,13 +63,12 @@ def _do_one_round(self, r, current_model, ctx: Context): if ctx.env_type == EnvType.SYSTEM: file_name = f"/tmp/np_{str(uuid.uuid4())}.npy" save_np_model(current_model, file_name) - model = prepare_file_for_download( + downloader = Downloader( num_receivers=grp.size, - file_name=file_name, ctx=ctx, timeout=5.0, - file_downloaded_cb=self._model_downloaded, ) + model = downloader.add_file(file_name=file_name, file_downloaded_cb=self._model_downloaded) model_type = "ref" self.logger.info(f"prepared model as ref: {model}") else: @@ -135,13 +134,12 @@ def train(self, current_round, weights, model_type: str, context: Context): # stream it file_name = f"/tmp/np_{str(uuid.uuid4())}.npy" save_np_model(result, file_name) - result = prepare_file_for_download( + downloader = Downloader( num_receivers=1, - file_name=file_name, ctx=context, timeout=5.0, - file_downloaded_cb=self._result_downloaded, ) + result = downloader.add_file(file_name=file_name, file_downloaded_cb=self._result_downloaded) self.logger.info(f"prepared result as ref: {result}") return result, model_type diff --git a/nvflare/fox/examples/np/fed_avg_para.py b/nvflare/fox/examples/np/fed_avg_para.py index 057e38edad..42de4dc91f 100644 --- a/nvflare/fox/examples/np/fed_avg_para.py +++ b/nvflare/fox/examples/np/fed_avg_para.py @@ -38,7 +38,8 @@ def main(): num_clients=10, ) - simulator.run() + result = simulator.run() + print(f"Final result: {result}") if __name__ == "__main__": diff --git a/nvflare/fox/examples/np/fed_avg_seq.py b/nvflare/fox/examples/np/fed_avg_seq.py index 8c0d6addaf..c934c72076 100644 --- a/nvflare/fox/examples/np/fed_avg_seq.py +++ b/nvflare/fox/examples/np/fed_avg_seq.py @@ -41,7 +41,8 @@ def main(): num_clients=2, ) - simulator.run() + result = simulator.run() + print(f"Final result: {result}") if __name__ == "__main__": diff --git a/nvflare/fox/examples/np/fed_avg_stream.py b/nvflare/fox/examples/np/fed_avg_stream.py index 60b19ed93c..8575976e26 100644 --- a/nvflare/fox/examples/np/fed_avg_stream.py +++ b/nvflare/fox/examples/np/fed_avg_stream.py @@ -37,7 +37,8 @@ def main(): num_clients=2, ) - simulator.run() + result = simulator.run() + print(f"Final result: {result}") if __name__ == "__main__": diff --git a/nvflare/fox/examples/np/recipe_swarm.py b/nvflare/fox/examples/np/recipe_swarm.py index eae4b2fccb..3e126db8f2 100644 --- a/nvflare/fox/examples/np/recipe_swarm.py +++ b/nvflare/fox/examples/np/recipe_swarm.py @@ -18,7 +18,7 @@ from nvflare.fox.examples.np.algos.swarm import NPSwarm, NPSwarmClient from nvflare.fox.sys.recipe import FoxRecipe -JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" +JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/fox/prod_00/admin@nvidia.com/transfer" def main(): @@ -30,7 +30,7 @@ def main(): client_app = NPSwarmClient(delta=1.0) recipe = FoxRecipe( - job_name="recipe_swarm", + job_name="swarm", server_app=server_app, client_app=client_app, ) diff --git a/nvflare/fox/examples/np/swarm.py b/nvflare/fox/examples/np/swarm.py index 363d3f38df..b461ac31e2 100644 --- a/nvflare/fox/examples/np/swarm.py +++ b/nvflare/fox/examples/np/swarm.py @@ -35,7 +35,8 @@ def main(): num_clients=3, ) - simulator.run() + result = simulator.run() + print(f"Final result: {result}") if __name__ == "__main__": diff --git a/nvflare/fox/examples/pt/filters.py b/nvflare/fox/examples/pt/filters.py index 4321f6d1c7..905ff093f8 100644 --- a/nvflare/fox/examples/pt/filters.py +++ b/nvflare/fox/examples/pt/filters.py @@ -2,7 +2,7 @@ from nvflare.fox.api.ctx import Context from nvflare.fox.api.filter import CallFilter, ResultFilter -from nvflare.fox.sys.model_downloader import ModelDownloader, download_model +from nvflare.fox.sys.downloader import Downloader, download_tensors from nvflare.fuel.utils.log_utils import get_obj_logger @@ -21,12 +21,12 @@ def filter_call(self, func_kwargs: dict, context: Context): num_receivers = context.target_group_size self.logger.info(f"target group size={num_receivers}") - downloader = ModelDownloader( + downloader = Downloader( num_receivers=context.target_group_size, ctx=context, timeout=5.0, ) - model = downloader.add_model(arg_value, 2) + model = downloader.add_tensors(arg_value, 2) func_kwargs[self.model_arg_name] = model return func_kwargs @@ -43,7 +43,7 @@ def filter_call(self, func_kwargs: dict, context: Context): if not arg_value: return func_kwargs - err, model = download_model(ref=arg_value, ctx=context, per_request_timeout=5.0) + err, model = download_tensors(ref=arg_value, ctx=context, per_request_timeout=5.0) if err: self.logger.error(f"error filtering call arg {arg_value}: {err}") else: @@ -58,12 +58,12 @@ def filter_result(self, result: Any, context: Context): if not isinstance(result, dict): return result - downloader = ModelDownloader( + downloader = Downloader( num_receivers=1, ctx=context, timeout=5.0, ) - return downloader.add_model(result, 2) + return downloader.add_tensors(result, 2) class IncomingModelResultFilter(ResultFilter): @@ -73,7 +73,7 @@ def __init__(self): self.logger = get_obj_logger(self) def filter_result(self, result: Any, context: Context): - err, model = download_model(ref=result, ctx=context, per_request_timeout=5.0) + err, model = download_tensors(ref=result, ctx=context, per_request_timeout=5.0) if err: self.logger.error(f"error filtering result {result}: {err}") return result diff --git a/nvflare/fox/examples/pt/pt_avg_filter.py b/nvflare/fox/examples/pt/pt_avg_filter.py index aa0f647902..35779df6a5 100644 --- a/nvflare/fox/examples/pt/pt_avg_filter.py +++ b/nvflare/fox/examples/pt/pt_avg_filter.py @@ -17,7 +17,7 @@ import torch from nvflare.fox.api.app import ClientApp, ServerApp -from nvflare.fox.api.constants import ContextKey, EnvType +from nvflare.fox.api.constants import ContextKey from nvflare.fox.api.ctx import Context from nvflare.fox.api.dec import collab from nvflare.fox.api.group import all_clients @@ -129,7 +129,8 @@ def main(): num_clients=2, ) - simulator.run() + result = simulator.run() + print(f"final result: {result}") if __name__ == "__main__": diff --git a/nvflare/fox/examples/pt/pt_avg_mixed.py b/nvflare/fox/examples/pt/pt_avg_mixed.py index 1f33297e22..a7b1b7a36e 100644 --- a/nvflare/fox/examples/pt/pt_avg_mixed.py +++ b/nvflare/fox/examples/pt/pt_avg_mixed.py @@ -18,7 +18,6 @@ import torch -from nvflare.app_opt.pt.tensor_downloader import TensorDownloadable from nvflare.fox.api.app import ClientApp, ServerApp from nvflare.fox.api.constants import EnvType from nvflare.fox.api.ctx import Context @@ -29,10 +28,7 @@ from nvflare.fox.examples.np.algos.utils import load_np_model, parse_array_def, save_np_model from nvflare.fox.examples.pt.utils import parse_state_dict from nvflare.fox.sim.simulator import Simulator -from nvflare.fox.sys.file_downloader import download_file -from nvflare.fox.sys.general_downloader import GeneralDownloader -from nvflare.fox.sys.model_downloader import download_model -from nvflare.fuel.f3.streaming.file_downloader import FileDownloadable +from nvflare.fox.sys.downloader import Downloader, download_file, download_tensors from nvflare.fuel.utils.log_utils import get_obj_logger @@ -79,14 +75,14 @@ def _do_one_round(self, r, pt_model, np_model, ctx: Context): file_name = f"/tmp/np_{str(uuid.uuid4())}.npy" save_np_model(np_model, file_name) - downloader = GeneralDownloader( + downloader = Downloader( num_receivers=grp.size, ctx=ctx, timeout=5.0, ) model_type = "ref" - pt_model = downloader.add_object(TensorDownloadable(pt_model, 2)) - np_model = downloader.add_object(FileDownloadable(file_name)) + pt_model = downloader.add_tensors(pt_model, 2) + np_model = downloader.add_file(file_name) self.logger.info(f"prepared model as ref: {pt_model=} {np_model=}") else: model_type = "model" @@ -118,11 +114,11 @@ def _accept_train_result(self, result, aggr_result: _AggrResult, context: Contex pt_result, np_result, model_type = result if model_type == "ref": - err, pt_result = download_model( + err, pt_result = download_tensors( ref=pt_result, per_request_timeout=5.0, ctx=context, - model_received_cb=self._aggregate_tensors, + tensors_received_cb=self._aggregate_tensors, aggr_result=aggr_result, context=context, ) @@ -171,7 +167,7 @@ def train(self, current_round, pt_model, np_model, model_type: str, context: Con f"[{context.header_str()}] training round {current_round}: {model_type=} {pt_model=} {np_model=}" ) if model_type == "ref": - err, pt_model = download_model(ref=pt_model, per_request_timeout=5.0, ctx=context) + err, pt_model = download_tensors(ref=pt_model, per_request_timeout=5.0, ctx=context) if err: raise RuntimeError(f"failed to download PT model {pt_model}: {err}") self.logger.info(f"downloaded PT model {pt_model}") @@ -192,17 +188,17 @@ def train(self, current_round, pt_model, np_model, model_type: str, context: Con if model_type == "ref": # stream it - downloader = GeneralDownloader( + downloader = Downloader( num_receivers=1, ctx=context, timeout=5.0, ) - pt_result = downloader.add_object(TensorDownloadable(pt_result, 2)) + pt_result = downloader.add_tensors(pt_result, 2) self.logger.info(f"prepared PT result as ref: {pt_result}") file_name = f"/tmp/np_{str(uuid.uuid4())}.npy" save_np_model(np_result, file_name) - np_result = downloader.add_object(FileDownloadable(file_name, file_downloaded_cb=self._result_downloaded)) + np_result = downloader.add_file(file_name, file_downloaded_cb=self._result_downloaded) self.logger.info(f"prepared NP result as ref: {np_result}") return pt_result, np_result, model_type @@ -241,7 +237,8 @@ def main(): num_clients=2, ) - simulator.run() + result = simulator.run() + print(f"Final result: {result}") if __name__ == "__main__": diff --git a/nvflare/fox/examples/pt/pt_avg_stream.py b/nvflare/fox/examples/pt/pt_avg_stream.py index ead90d7036..75f986f1b9 100644 --- a/nvflare/fox/examples/pt/pt_avg_stream.py +++ b/nvflare/fox/examples/pt/pt_avg_stream.py @@ -25,7 +25,7 @@ from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.pt.utils import parse_state_dict from nvflare.fox.sim.simulator import Simulator -from nvflare.fox.sys.model_downloader import ModelDownloader, download_model +from nvflare.fox.sys.downloader import Downloader, download_tensors from nvflare.fuel.utils.log_utils import get_obj_logger @@ -68,14 +68,14 @@ def _do_one_round(self, r, current_model, ctx: Context): ) if ctx.env_type == EnvType.SYSTEM: - downloader = ModelDownloader( + downloader = Downloader( num_receivers=grp.size, ctx=ctx, timeout=5.0, ) model_type = "ref" - model = downloader.add_model(current_model, 2) - model2 = downloader.add_model(model2, 2) + model = downloader.add_tensors(current_model, 2) + model2 = downloader.add_tensors(model2, 2) self.logger.info(f"prepared model as ref: {model}") else: model = current_model @@ -97,11 +97,11 @@ def _accept_train_result(self, result, aggr_result: _AggrResult, context: Contex model, model_type = result if model_type == "ref": - err, model = download_model( + err, model = download_tensors( ref=model, per_request_timeout=5.0, ctx=context, - model_received_cb=self._aggregate_tensors, + tensors_received_cb=self._aggregate_tensors, aggr_result=aggr_result, context=context, ) @@ -140,12 +140,12 @@ def train(self, current_round, model1, model2, model_type: str, context: Context return 0 self.logger.debug(f"[{context.header_str()}] training round {current_round}: {model_type=} {model1=} {model2=}") if model_type == "ref": - err, model1 = download_model(ref=model1, per_request_timeout=5.0, ctx=context) + err, model1 = download_tensors(ref=model1, per_request_timeout=5.0, ctx=context) if err: raise RuntimeError(f"failed to download model1 {model1}: {err}") self.logger.info(f"downloaded model1 {model1}") - err, model2 = download_model(ref=model2, per_request_timeout=5.0, ctx=context) + err, model2 = download_tensors(ref=model2, per_request_timeout=5.0, ctx=context) if err: raise RuntimeError(f"failed to download model2 {model2}: {err}") self.logger.info(f"downloaded model2 {model2}") @@ -160,13 +160,13 @@ def train(self, current_round, model1, model2, model_type: str, context: Context if model_type == "ref": # stream it - downloader = ModelDownloader( + downloader = Downloader( num_receivers=1, ctx=context, timeout=5.0, ) model_type = "ref" - model = downloader.add_model(result, 2) + model = downloader.add_tensors(result, 2) self.logger.info(f"prepared result as ref: {model}") else: model = result @@ -193,13 +193,14 @@ def main(): simulator = Simulator( root_dir="/tmp/fox", - experiment_name="pt_fedavg_intime", + experiment_name="pt_fedavg_stream", server_app=server_app, client_app=client_app, num_clients=2, ) - simulator.run() + result = simulator.run() + print(f"final result: {result}") if __name__ == "__main__": diff --git a/nvflare/fox/sim/simulator.py b/nvflare/fox/sim/simulator.py index 422820bd83..0c9386674d 100644 --- a/nvflare/fox/sim/simulator.py +++ b/nvflare/fox/sim/simulator.py @@ -173,13 +173,15 @@ def _build_hierarchical_clients(self, height: int, num_children_per_parent: int) def run(self): try: - self._try_run() + result = self._try_run() except KeyboardInterrupt: self.logger.info("execution is aborted by user") self.abort_signal.trigger(True) + result = None finally: self.thread_executor.shutdown(wait=False, cancel_futures=True) self.logger.info(f"Experiment results are in {self.exp_dir}") + return result def _try_run(self): # initialize all apps diff --git a/nvflare/fox/sys/constants.py b/nvflare/fox/sys/constants.py index 569a2f87cf..b7f85b298d 100644 --- a/nvflare/fox/sys/constants.py +++ b/nvflare/fox/sys/constants.py @@ -34,8 +34,3 @@ class ObjectCallKey: class CallReplyKey: ERROR = "error" RESULT = "result" - - -class DownloaderKey: - SOURCE = "source" - REF_ID = "ref_id" diff --git a/nvflare/fox/sys/downloader.py b/nvflare/fox/sys/downloader.py new file mode 100644 index 0000000000..3b327b268f --- /dev/null +++ b/nvflare/fox/sys/downloader.py @@ -0,0 +1,112 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch + +from nvflare.fox.api.ctx import Context +from nvflare.fox.sys.backend import SysBackend +from nvflare.fuel.f3.streaming.file_downloader import add_file +from nvflare.fuel.f3.streaming.file_downloader import download_file as pull_file +from nvflare.fuel.f3.streaming.obj_downloader import ObjectDownloader + +from ...app_opt.pt.tensor_downloader import add_tensors +from ...app_opt.pt.tensor_downloader import download_tensors as pull_tensors + + +class DownloadRefKey: + SOURCE = "source" + REF_ID = "ref_id" + OBJECT_TYPE = "object_type" + + +class ObjectType: + FILE = "file" + TENSORS = "tensors" + + +class Downloader(ObjectDownloader): + + def __init__( + self, + num_receivers: int, + timeout: float, + ctx: Context, + ): + backend = ctx.backend + if not isinstance(backend, SysBackend): + raise ValueError(f"backend must be SysBackend but got {type(backend)}") + + super().__init__( + cell=backend.cell, + timeout=timeout, + num_receivers=num_receivers, + ) + + def _to_ref(self, obj_type, ref_id): + return { + DownloadRefKey.OBJECT_TYPE: obj_type, + DownloadRefKey.REF_ID: ref_id, + DownloadRefKey.SOURCE: self.cell.get_fqcn(), + } + + def add_file( + self, + file_name: str, + chunk_size=None, + file_downloaded_cb=None, + **cb_kwargs, + ): + rid = add_file(self, file_name, chunk_size=chunk_size, file_downloaded_cb=file_downloaded_cb, **cb_kwargs) + return self._to_ref(ObjectType.FILE, rid) + + def add_tensors(self, tensors: dict[str, torch.Tensor], num_tensors_per_chunk: int = 1): + rid = add_tensors(self, tensors, num_tensors_per_chunk=num_tensors_per_chunk) + return self._to_ref(ObjectType.TENSORS, rid) + + +def download_file(ref: dict, per_request_timeout: float, ctx: Context): + backend = ctx.backend + if not isinstance(backend, SysBackend): + raise ValueError(f"backend must be SysBackend but got {type(backend)}") + + obj_type = ref.get(DownloadRefKey.OBJECT_TYPE) + if obj_type != ObjectType.FILE: + raise ValueError(f"obj_type must be {ObjectType.FILE} but got {obj_type}") + + return pull_file( + from_fqcn=ref.get(DownloadRefKey.SOURCE), + ref_id=ref.get(DownloadRefKey.REF_ID), + per_request_timeout=per_request_timeout, + cell=backend.cell, + abort_signal=ctx.abort_signal, + ) + + +def download_tensors(ref: dict, per_request_timeout: float, ctx: Context, tensors_received_cb=None, **cb_kwargs): + backend = ctx.backend + if not isinstance(backend, SysBackend): + raise ValueError(f"backend must be SysBackend but got {type(backend)}") + + obj_type = ref.get(DownloadRefKey.OBJECT_TYPE) + if obj_type != ObjectType.TENSORS: + raise ValueError(f"obj_type must be {ObjectType.TENSORS} but got {obj_type}") + + return pull_tensors( + from_fqcn=ref.get(DownloadRefKey.SOURCE), + ref_id=ref.get(DownloadRefKey.REF_ID), + per_request_timeout=per_request_timeout, + cell=backend.cell, + abort_signal=ctx.abort_signal, + tensors_received_cb=tensors_received_cb, + **cb_kwargs, + ) diff --git a/nvflare/fox/sys/file_downloader.py b/nvflare/fox/sys/file_downloader.py deleted file mode 100644 index 3a4b40b6b2..0000000000 --- a/nvflare/fox/sys/file_downloader.py +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from nvflare.fox.api.ctx import Context -from nvflare.fox.sys.backend import SysBackend -from nvflare.fox.sys.constants import DownloaderKey -from nvflare.fuel.f3.streaming.file_downloader import FileDownloader - - -def prepare_file_for_download( - file_name: str, - timeout: float, - num_receivers: int, - ctx: Context, - file_downloaded_cb=None, - **cb_kwargs, -): - backend = ctx.backend - if not isinstance(backend, SysBackend): - raise ValueError(f"backend must be SysBackend but got {type(backend)}") - - cell = backend.cell - tx_id = FileDownloader.new_transaction( - cell=cell, - timeout=timeout, - num_receivers=num_receivers, - ) - rid = FileDownloader.add_file( - tx_id, - file_name, - file_downloaded_cb=file_downloaded_cb, - **cb_kwargs, - ) - return {DownloaderKey.SOURCE: cell.get_fqcn(), DownloaderKey.REF_ID: rid} - - -def download_file(ref: dict, per_request_timeout: float, ctx: Context): - backend = ctx.backend - if not isinstance(backend, SysBackend): - raise ValueError(f"backend must be SysBackend but got {type(backend)}") - - err, file_path = FileDownloader.download_file( - from_fqcn=ref.get(DownloaderKey.SOURCE), - ref_id=ref.get(DownloaderKey.REF_ID), - per_request_timeout=per_request_timeout, - cell=backend.cell, - abort_signal=ctx.abort_signal, - ) - return err, file_path diff --git a/nvflare/fox/sys/general_downloader.py b/nvflare/fox/sys/general_downloader.py deleted file mode 100644 index 1b0762ab83..0000000000 --- a/nvflare/fox/sys/general_downloader.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from nvflare.fox.api.ctx import Context -from nvflare.fox.sys.backend import SysBackend -from nvflare.fuel.f3.streaming.obj_downloader import Downloadable, ObjDownloader - -from .constants import DownloaderKey - - -class GeneralDownloader: - - def __init__( - self, - num_receivers: int, - timeout: float, - ctx: Context, - ): - backend = ctx.backend - if not isinstance(backend, SysBackend): - raise ValueError(f"backend must be SysBackend but got {type(backend)}") - - self.cell = backend.cell - self.tx_id = ObjDownloader.new_transaction( - cell=self.cell, - timeout=timeout, - num_receivers=num_receivers, - ) - - def add_object(self, obj: Downloadable): - rid = ObjDownloader.add_object( - transaction_id=self.tx_id, - obj=obj, - ) - return {DownloaderKey.SOURCE: self.cell.get_fqcn(), DownloaderKey.REF_ID: rid} diff --git a/nvflare/fox/sys/model_downloader.py b/nvflare/fox/sys/model_downloader.py deleted file mode 100644 index d2257056c6..0000000000 --- a/nvflare/fox/sys/model_downloader.py +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import torch - -from nvflare.app_opt.pt.tensor_downloader import TensorDownloader -from nvflare.fox.api.ctx import Context -from nvflare.fox.sys.backend import SysBackend - -from .constants import DownloaderKey - - -class ModelDownloader: - - def __init__( - self, - num_receivers: int, - timeout: float, - ctx: Context, - ): - backend = ctx.backend - if not isinstance(backend, SysBackend): - raise ValueError(f"backend must be SysBackend but got {type(backend)}") - - self.cell = backend.cell - self.tx_id = TensorDownloader.new_transaction( - cell=self.cell, - timeout=timeout, - num_receivers=num_receivers, - ) - - def add_model(self, model: dict[str, torch.Tensor], num_tensors_per_chunk: int = 1): - rid = TensorDownloader.add_tensors( - transaction_id=self.tx_id, - tensors=model, - num_tensors_per_chunk=num_tensors_per_chunk, - ) - return {DownloaderKey.SOURCE: self.cell.get_fqcn(), DownloaderKey.REF_ID: rid} - - -def download_model(ref: dict, per_request_timeout: float, ctx: Context, model_received_cb=None, **cb_kwargs): - backend = ctx.backend - if not isinstance(backend, SysBackend): - raise ValueError(f"backend must be SysBackend but got {type(backend)}") - - return TensorDownloader.download_tensors( - from_fqcn=ref.get(DownloaderKey.SOURCE), - ref_id=ref.get(DownloaderKey.REF_ID), - per_request_timeout=per_request_timeout, - cell=backend.cell, - abort_signal=ctx.abort_signal, - tensors_received_cb=model_received_cb, - **cb_kwargs, - ) diff --git a/nvflare/fuel/f3/streaming/cacheable.py b/nvflare/fuel/f3/streaming/cacheable.py index 6b093e95e1..2ac39a11db 100644 --- a/nvflare/fuel/f3/streaming/cacheable.py +++ b/nvflare/fuel/f3/streaming/cacheable.py @@ -15,7 +15,7 @@ from abc import abstractmethod from typing import Any, List, Tuple -from nvflare.fuel.f3.streaming.obj_downloader import Consumer, Downloadable, ObjDownloader, ProduceRC +from nvflare.fuel.f3.streaming.download_service import Consumer, Downloadable, DownloadService, ProduceRC from nvflare.fuel.utils.log_utils import get_obj_logger @@ -44,7 +44,7 @@ def produce_item(self, index: int) -> Any: pass def set_transaction(self, tx_id, ref_id): - tx_info = ObjDownloader.get_transaction_info(tx_id) + tx_info = DownloadService.get_transaction_info(tx_id) self.num_receivers = tx_info.num_receivers self.logger.info(f"set transaction info: {tx_id=}, {ref_id=} {self.num_receivers=}") @@ -113,7 +113,6 @@ def __init__(self): self.error = None self.result = None - @abstractmethod def consume_items(self, items: List[Any], result: Any) -> Any: """Process items and return updated result.""" pass diff --git a/nvflare/fuel/f3/streaming/download_service.py b/nvflare/fuel/f3/streaming/download_service.py new file mode 100644 index 0000000000..e145b051cf --- /dev/null +++ b/nvflare/fuel/f3/streaming/download_service.py @@ -0,0 +1,600 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import threading +import time +import uuid +from abc import ABC, abstractmethod +from typing import Any, Optional, Tuple + +from nvflare.apis.signal import Signal +from nvflare.fuel.f3.cellnet.cell import Cell +from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey, ReturnCode +from nvflare.fuel.f3.cellnet.utils import make_reply, new_cell_message +from nvflare.fuel.f3.message import Message +from nvflare.fuel.utils.log_utils import get_obj_logger +from nvflare.security.logging import secure_format_exception + +OBJ_DOWNLOADER_CHANNEL = "obj_downloader__" +OBJ_DOWNLOADER_TOPIC = "obj_downloader__download" + +""" +This package provides a framework for building object downloading capability (e.g. file download). + +A large object takes a lot of memory space. Sending a large object in one message needs even more memory space since +the object needs to be serialized into large number of bytes. Additional memory space may still be needed for the +transport layer to send the message. If the message is to be sent to multiple endpoints, even more memory is needed. + +Object Downloading can drastically reduce memory consumption: +- Instead of sending the large object in one message, it is divided into many smaller objects; +- Instead of pushing the message to the endpoints, each endpoint will come to request. This makes it more reliable when +different endpoints have different speed. + +Object Downloading works as follows: +- The sender prepares the object(s) for downloading. It first creates a transaction to get a tx_id. It then adds each +object to be downloaded to the transaction, and get a reference id (ref_id). +- The sender sends the ref_id(s) to all recipients through a separate message. +- Each recipient then calls the download_object function to download each referenced large object. + +To develop the downloading capability for a type of object (e.g. a file, a large dict, etc.), you need to provide +the implementation of a Producer and a Consumer. +- On the sending side, the Producer is responsible for producing the next small object to be sent (a chunk of bytes; +a small subset of the large dict; etc.). +- On the receiving side, the Consumer is responsible for processing the received small objects (writing the received +bytes to a temp file; putting the received small dict to the end result; etc.). + +One issue with object downloading is object life cycle management. Since the large objects to be downloaded are usually +temporary, you need to remove them when they are downloaded by all sites. But the problem is that you don't know how +quickly each site can finish downloading these large objects. When a transaction contains multiple objects to be +downloaded, it's even harder to know it. + +There are two ways to handle this issue: object downloaded callback, and transaction timeout. + +You can register an object_downloaded CB when adding an object to transaction. When the object is fully downloaded +to a site, this CB will be called. The obj_downloaded CB must follow this signature: + + downloaded_cb(ref_id: str, to_site: str, status: str, obj: Any, **cb_kwargs) + +where ref_id is the reference id of the object; +to_site is the FQCN of the site that has just finished downloading; +status is the status of downloading, as defined in DownloadStatus class; +obj is the large object that was just downloaded; +cb_kwargs are the kw args registered with the CB. + +Transaction timeout is the amount of time after the last downloading activity on any object in the +transaction from any site. For example, suppose you want to send 2 large files to 3 sites, each time a download +request is received on any file from any of the 3 sites, the last activity time of the transaction is updated to now. +If no downloading activity is received from any site on any objects in the transaction for the specified timeout, +the transaction is considered "timed out", and the timeout callback registered with the transaction is called. +The transaction timeout CB must follow this signature: + + timeout_cb(tx_id: str, objs: List[Any], **cb_kwargs) + +where tx_id is the ID of the transaction; +objs is the list of large objects registered with the transaction; +cb_kwargs are the kw args registered with the CB. + +You may need to use both mechanisms to fully take care of object life cycles. The object downloaded CB may never be +called since the site somehow didn't finish the downloading. In reality the timeout mechanism may be sufficient. + +Unlike with Object Streamer that the object owner pushes small objects to the recipients; with Object Downloader, +each recipient pulls the data from the object owner. +""" + + +class Downloadable(ABC): + + def set_transaction(self, tx_id, ref_id): + pass + + @abstractmethod + def get_base_object(self): + pass + + @abstractmethod + def produce(self, state: dict, requester: str) -> Tuple[str, Any, dict]: + """Produce a small object to be sent (on object sender side). + + Args: + state: current state of downloading, received from the downloading site + requester: the FQCN of the site that is downloading + + Returns: a tuple of (return code, a small object to be sent, new state to be sent). + + """ + pass + + def downloaded_to_one(self, to_site: str, status: str): + """Called when an object is downloaded to a site.""" + pass + + def downloaded_to_all(self): + """Called when the object is fully downloaded to all sites.""" + pass + + def transaction_done(self, transaction_id: str, status: str): + """Called when the transaction is finished.""" + pass + + +class _PropKey: + REF_ID = "ref_id" + STATE = "state" + DATA = "data" + STATUS = "status" + + +class _Ref: + + def __init__( + self, + tx, + obj: Downloadable, + ref_id=None, + ): + if ref_id: + # use provided ref_id + self.rid = ref_id + else: + self.rid = "R" + str(uuid.uuid4()) + self.tx = tx + self.obj = obj + self.num_sites_done = 0 + + def mark_active(self): + self.tx.mark_active() + + def obj_downloaded(self, to_site: str, status: str): + self.num_sites_done += 1 + + assert isinstance(self.obj, Downloadable) + self.obj.downloaded_to_one(to_site, status) + + assert isinstance(self.tx, _Transaction) + if 0 < self.tx.num_receivers <= self.num_sites_done: + # this object is done for all sites + self.obj.downloaded_to_all() + + +class ProduceRC: + """Defines return code for the Producer's produce method.""" + + OK = "ok" + ERROR = "error" + EOF = "eof" + + +class DownloadStatus: + SUCCESS = "success" + FAILED = "failed" + + +class TransactionDoneStatus: + FINISHED = "finished" + TIMEOUT = "timeout" + DELETED = "deleted" + + +class _Transaction: + + def __init__( + self, + timeout: float, + num_receivers: int, + tx_id=None, + transaction_done_cb=None, + cb_kwargs=None, + ): + """Constructor of the transaction object. + + Args: + timeout: amount of time since last activity + num_receivers: number of receivers. 0 means unlimited. + tx_id: if provided, use it; otherwise create one + """ + if tx_id: + self.tid = tx_id + else: + self.tid = "T" + str(uuid.uuid4()) + self.timeout = timeout + self.num_receivers = num_receivers + self.last_active_time = time.time() + self.transaction_done_cb = transaction_done_cb + self.cb_kwargs = cb_kwargs + self.refs = [] + + def mark_active(self): + """Called to update the last active time of the transaction. + + Returns: + + """ + self.last_active_time = time.time() + + def add_object( + self, + obj: Downloadable, + ref_id=None, + ): + """Add a large object (to be downloaded) to the transaction. + + Args: + obj: the large object to be downloaded + ref_id: the ref id to be used, if specified + + Returns: + + """ + r = _Ref(self, obj, ref_id) + self.refs.append(r) + obj.set_transaction(self.tid, r.rid) + return r + + def timed_out(self): + """Called when the transaction is timed out. + + Returns: + + """ + self.transaction_done(TransactionDoneStatus.TIMEOUT) + + def is_finished(self): + """Check whether the transaction is finished (all objects are downloaded).""" + if self.num_receivers <= 0: + return False + + for ref in self.refs: + assert isinstance(ref, _Ref) + if ref.num_sites_done < self.num_receivers: + return False + return True + + def transaction_done(self, status: str): + """Called when the transaction is finished.""" + for ref in self.refs: + obj = ref.obj + assert isinstance(obj, Downloadable) + obj.transaction_done(self.tid, status) + + if self.transaction_done_cb: + self.transaction_done_cb( + self.tid, status, [ref.obj.get_base_object() for ref in self.refs], **self.cb_kwargs + ) + + +class TransactionInfo: + + def __init__(self, tx: _Transaction): + self.timeout = tx.timeout + self.num_receivers = tx.num_receivers + self.objects = [r.obj for r in tx.refs] + + +class DownloadService: + + _init_lock = threading.Lock() + _tx_table = {} + _ref_table = {} + _logger = None + _tx_monitor = None + _tx_lock = threading.Lock() + _initialized_cells = {} + + @classmethod + def _initialize(cls, cell: Cell): + with cls._init_lock: + if not cls._logger: + cls._logger = get_obj_logger(cls) + + if not cls._tx_monitor: + cls._tx_monitor = threading.Thread(target=cls._monitor_tx, daemon=True) + cls._tx_monitor.start() + + initialized = cls._initialized_cells.get(id(cell)) + if not initialized: + # register CBs + cell.register_request_cb( + channel=OBJ_DOWNLOADER_CHANNEL, + topic=OBJ_DOWNLOADER_TOPIC, + cb=cls._handle_download, + ) + cls._initialized_cells[id(cell)] = True + + @classmethod + def new_transaction( + cls, + cell: Cell, + timeout: float, + num_receivers: int = 0, + tx_id=None, + transaction_done_cb=None, + **cb_kwargs, + ): + cls._initialize(cell) + tx = _Transaction(timeout, num_receivers, tx_id, transaction_done_cb, cb_kwargs) + with cls._tx_lock: + cls._tx_table[tx.tid] = tx + return tx.tid + + @classmethod + def add_object( + cls, + transaction_id: str, + obj: Downloadable, + ref_id=None, + ) -> str: + if not issubclass(type(obj), Downloadable): + raise ValueError(f"obj must be of type {Downloadable} but got {type(obj)}") + + tx = cls._tx_table.get(transaction_id) + if not tx: + raise ValueError(f"no such transaction {transaction_id}") + + assert isinstance(tx, _Transaction) + ref = tx.add_object(obj, ref_id) + with cls._tx_lock: + cls._ref_table[ref.rid] = ref + return ref.rid + + @classmethod + def delete_transaction(cls, transaction_id: str): + with cls._tx_lock: + tx = cls._tx_table.get(transaction_id) + if tx: + cls._delete_tx(tx) + tx.transaction_done(TransactionDoneStatus.DELETED) + + @classmethod + def shutdown(cls): + """Shutdown and clean up resources. + + Returns: None + + """ + with cls._tx_lock: + tx_list = list(cls._tx_table.values()) + if tx_list: + for tx in tx_list: + cls._delete_tx(tx) + tx.transaction_done(TransactionDoneStatus.DELETED) + + @classmethod + def _delete_tx(cls, tx: _Transaction): + cls._tx_table.pop(tx.tid, None) + + # remove all refs + for r in tx.refs: + cls._ref_table.pop(r.rid, None) + + @classmethod + def get_transaction_info(cls, transaction_id: str) -> Optional[TransactionInfo]: + tx = cls._tx_table.get(transaction_id) + if not tx: + return None + else: + return TransactionInfo(tx) + + @classmethod + def get_transaction_id(cls, ref_id: str) -> Optional[str]: + ref = cls._ref_table.get(ref_id) + if not ref: + return None + else: + assert isinstance(ref, _Ref) + return ref.tx.tid + + @classmethod + def _handle_download(cls, request: Message) -> Message: + requester = request.get_header(MessageHeaderKey.ORIGIN) + payload = request.payload + assert isinstance(payload, dict) + rid = payload.get(_PropKey.REF_ID) + if not rid: + cls._logger.error(f"missing {_PropKey.REF_ID} in request from {requester}") + return make_reply(ReturnCode.INVALID_REQUEST) + + current_state = payload.get(_PropKey.STATE) + with cls._tx_lock: + ref = cls._ref_table.get(rid) + if not ref: + cls._logger.error(f"no ref found for {rid} from {requester}") + return make_reply(ReturnCode.INVALID_REQUEST) + + assert isinstance(ref, _Ref) + ref.mark_active() + tx = ref.tx + assert isinstance(tx, _Transaction) + + try: + rc, data, new_state = ref.obj.produce(current_state, requester) + except Exception as ex: + cls._logger.error( + f"Object {type(ref.obj)} encountered exception when produce: {secure_format_exception(ex)}" + ) + return make_reply(ReturnCode.PROCESS_EXCEPTION) + + if rc != ProduceRC.OK: + # already done + ref.obj_downloaded( + requester, status=DownloadStatus.SUCCESS if rc == ProduceRC.EOF else DownloadStatus.FAILED + ) + return make_reply(ReturnCode.OK, body={_PropKey.STATUS: rc}) + else: + # continue + return make_reply( + ReturnCode.OK, + body={ + _PropKey.STATUS: rc, + _PropKey.STATE: new_state, + _PropKey.DATA: data, + }, + ) + + @classmethod + def _monitor_tx(cls): + while True: + now = time.time() + expired_tx = [] + finished_tx = [] + with cls._tx_lock: + for tid, tx in cls._tx_table.items(): + assert isinstance(tx, _Transaction) + + # check whether all refs are done + if tx.is_finished(): + finished_tx.append(tx) + elif now - tx.last_active_time > tx.timeout: + expired_tx.append(tx) + + for tx in expired_tx: + assert isinstance(tx, _Transaction) + tx.transaction_done(TransactionDoneStatus.TIMEOUT) + cls._delete_tx(tx) + + for tx in finished_tx: + tx.transaction_done(TransactionDoneStatus.FINISHED) + cls._delete_tx(tx) + + time.sleep(5.0) + + +class Consumer(ABC): + + def __init__(self): + self.logger = get_obj_logger(self) + + @abstractmethod + def consume(self, ref_id: str, state: dict, data: Any) -> dict: + """Called to process the received data. + + Args: + ref_id: ref id of the object being downloaded + state: current state of downloading + data: data to be processed + + Returns: new state to be sent back to the data owner. + + """ + pass + + @abstractmethod + def download_completed(self, ref_id: str): + """Called when the downloading is finished successfully. + + Args: + ref_id: ref id of the object being downloaded + + Returns: None + + """ + pass + + @abstractmethod + def download_failed(self, ref_id: str, reason: str): + """Called when the downloading is finished unsuccessfully. + + Args: + ref_id: ref id of the object being downloaded + reason: explain the reason of failure + + Returns: None + + """ + pass + + +def download_object( + from_fqcn: str, + ref_id: str, + per_request_timeout: float, + cell: Cell, + consumer: Consumer, + secure=False, + optional=False, + abort_signal: Signal = None, +): + """Download a large object from the object owner. + + Args: + from_fqcn: the FQCN of the object owner + ref_id: reference id of the object to be downloaded + per_request_timeout: timeout for each request to the object owner. + cell: the cell to be used for communication withe object owner. + consumer: the Consumer object used for processing received data + secure: use P2P private communication with the data owner + optional: supress log messages + abort_signal: for signaling abort + + Returns: None + + """ + request = new_cell_message( + headers={}, + payload={ + _PropKey.REF_ID: ref_id, + }, + ) + + while True: + start_time = time.time() + reply = cell.send_request( + channel=OBJ_DOWNLOADER_CHANNEL, + target=from_fqcn, + topic=OBJ_DOWNLOADER_TOPIC, + request=request, + timeout=per_request_timeout, + secure=secure, + optional=optional, + abort_signal=abort_signal, + ) + duration = time.time() - start_time + + if abort_signal and abort_signal.triggered: + consumer.download_failed(ref_id, f"download aborted after {duration} secs") + return + + assert isinstance(reply, Message) + rc = reply.get_header(MessageHeaderKey.RETURN_CODE) + if rc != ReturnCode.OK: + consumer.download_failed(ref_id, f"error requesting data from {from_fqcn} after {duration} secs: {rc}") + return + + payload = reply.payload + assert isinstance(payload, dict) + status = payload.get(_PropKey.STATUS) + if status == ProduceRC.EOF: + consumer.download_completed(ref_id) + return + elif status == ProduceRC.ERROR: + consumer.download_failed(ref_id, f"producer error after {duration} secs") + return + + # continue + data = payload.get(_PropKey.DATA) + state = payload.get(_PropKey.STATE) + try: + new_state = consumer.consume(ref_id, state, data) + except Exception as ex: + consumer.download_failed(ref_id, f"exception when consuming data: {secure_format_exception(ex)}") + return + + if not isinstance(new_state, dict): + consumer.download_failed(ref_id, f"consumer error: new_state should be dict but got {type(new_state)}") + return + + if abort_signal and abort_signal.triggered: + consumer.download_failed(ref_id, "download aborted") + return + + # ask for more + request = new_cell_message(headers={}, payload={_PropKey.REF_ID: ref_id, _PropKey.STATE: new_state}) diff --git a/nvflare/fuel/f3/streaming/file_downloader.py b/nvflare/fuel/f3/streaming/file_downloader.py index 38cfee5cee..fa829694bf 100644 --- a/nvflare/fuel/f3/streaming/file_downloader.py +++ b/nvflare/fuel/f3/streaming/file_downloader.py @@ -17,14 +17,15 @@ from typing import Any, Optional, Tuple from nvflare.fuel.f3.cellnet.cell import Cell -from nvflare.fuel.f3.streaming.obj_downloader import Consumer, Downloadable, ObjDownloader, ProduceRC, download_object +from nvflare.fuel.f3.streaming.download_service import Consumer, Downloadable, ProduceRC, download_object +from nvflare.fuel.f3.streaming.obj_downloader import ObjectDownloader from nvflare.fuel.utils.log_utils import get_obj_logger from nvflare.fuel.utils.validation_utils import check_callable, check_positive_int DEFAULT_CHUNK_SIZE = 5 * 1024 * 1024 """ -This package implements file downloading capability based on the ObjDownloader framework. +This package implements file downloading capability based on the ObjectDownloader framework. It provides implementation of the Producer and Consumer objects, required by ObjDownloader. """ @@ -61,6 +62,9 @@ def __init__( self.cb_kwargs = cb_kwargs self.logger = get_obj_logger(self) + def get_base_object(self): + return self.name + def produce(self, state: dict, requester: str) -> Tuple[str, Any, dict]: received_bytes = 0 if state: @@ -91,124 +95,85 @@ def downloaded_to_all(self): self.file_downloaded_cb("", "", self.name, **self.cb_kwargs) -class FileDownloader(ObjDownloader): - - @classmethod - def new_transaction( - cls, - cell: Cell, - timeout: float, - num_receivers: int = 0, - transaction_done_cb=None, - **cb_kwargs, - ): - """Create a new file download transaction. - - Args: - cell: the cell for communication with recipients - timeout: timeout for the transaction - num_receivers: number of receivers. 0 means unknown. - transaction_done_cb: callback function that will be called when the transaction is done. - - Returns: transaction id - """ - return ObjDownloader.new_transaction( - cell=cell, - timeout=timeout, - num_receivers=num_receivers, - transaction_done_cb=cls._transaction_done, - cb_info=(transaction_done_cb, cb_kwargs), - ) - - @classmethod - def _transaction_done(cls, tx_id: str, status: str, objs, cb_info): - transaction_done_cb, cb_kwargs = cb_info - if transaction_done_cb: - transaction_done_cb(tx_id, [obj.name for obj in objs], **cb_kwargs) - - @classmethod - def add_file( - cls, - transaction_id: str, - file_name: str, - chunk_size=None, - ref_id=None, - file_downloaded_cb=None, - **cb_kwargs, - ) -> str: - """Add a file to be downloaded to the specified transaction. +def add_file( + downloader: ObjectDownloader, + file_name: str, + chunk_size=None, + ref_id=None, + file_downloaded_cb=None, + **cb_kwargs, +) -> str: + """Add a file to be downloaded to the specified downloader. - Args: - transaction_id: ID of the transaction - file_name: name of the file to be downloaded - chunk_size: chunk size in bytes - ref_id: ref id to be used, if provided - file_downloaded_cb: CB to be called when the file is done downloading - **cb_kwargs: args to be passed to the CB + Args: + downloader: the downloader to add to. + file_name: name of the file to be downloaded + chunk_size: chunk size in bytes + ref_id: ref id to be used, if provided + file_downloaded_cb: CB to be called when the file is done downloading + **cb_kwargs: args to be passed to the CB - Returns: reference id for the file. + Returns: reference id for the file. - The file_downloaded_cb must follow this signature: + The file_downloaded_cb must follow this signature: - cb(to_site: str, status: str, file_name: str, **cb_kwargs) + cb(to_site: str, status: str, file_name: str, **cb_kwargs) - """ - obj = FileDownloadable(file_name, chunk_size=chunk_size, file_downloaded_cb=file_downloaded_cb, **cb_kwargs) - return ObjDownloader.add_object( - transaction_id=transaction_id, - obj=obj, - ref_id=ref_id, - ) - - @classmethod - def download_file( - cls, - from_fqcn: str, - ref_id: str, - per_request_timeout: float, - cell: Cell, - location: str = None, - secure=False, - optional=False, - abort_signal=None, - ) -> Tuple[str, Optional[str]]: - """Download the referenced file from the file owner. + """ + obj = FileDownloadable(file_name, chunk_size=chunk_size, file_downloaded_cb=file_downloaded_cb, **cb_kwargs) + return downloader.add_object( + obj=obj, + ref_id=ref_id, + ) - Args: - from_fqcn: FQCN of the file owner. - ref_id: reference ID of the file to be downloaded. - per_request_timeout: timeout for requests sent to the file owner. - cell: cell to be used for communicating to the file owner. - location: dir for keeping the received file. If not specified, will use temp dir. - secure: P2P private mode for communication - optional: supress log messages of communication - abort_signal: signal for aborting download. - Returns: tuple of (error message if any, full path of the downloaded file). +def download_file( + from_fqcn: str, + ref_id: str, + per_request_timeout: float, + cell: Cell, + location: str = None, + secure=False, + optional=False, + abort_signal=None, +) -> Tuple[str, Optional[str]]: + """Download the referenced file from the file owner. + + Args: + from_fqcn: FQCN of the file owner. + ref_id: reference ID of the file to be downloaded. + per_request_timeout: timeout for requests sent to the file owner. + cell: cell to be used for communicating to the file owner. + location: dir for keeping the received file. If not specified, will use temp dir. + secure: P2P private mode for communication + optional: supress log messages of communication + abort_signal: signal for aborting download. + + Returns: tuple of (error message if any, full path of the downloaded file). + + """ + if location is not None: + if not os.path.exists(location): + raise ValueError(f"location '{location}' does not exist") + + if not os.path.isdir(location): + raise ValueError(f"location '{location}' is not a valid dir") + else: + location = tempfile.gettempdir() + + consumer = _ChunkConsumer(location) + download_object( + from_fqcn=from_fqcn, + ref_id=ref_id, + consumer=consumer, + per_request_timeout=per_request_timeout, + cell=cell, + secure=secure, + optional=optional, + abort_signal=abort_signal, + ) - """ - if location is not None: - if not os.path.exists(location): - raise ValueError(f"location '{location}' does not exist") - - if not os.path.isdir(location): - raise ValueError(f"location '{location}' is not a valid dir") - else: - location = tempfile.gettempdir() - - consumer = _ChunkConsumer(location) - download_object( - from_fqcn=from_fqcn, - ref_id=ref_id, - consumer=consumer, - per_request_timeout=per_request_timeout, - cell=cell, - secure=secure, - optional=optional, - abort_signal=abort_signal, - ) - - return consumer.error, consumer.file_path + return consumer.error, consumer.file_path class _ChunkConsumer(Consumer): @@ -237,25 +202,3 @@ def download_failed(self, ref_id, reason: str): def download_completed(self, ref_id: str): self.file.close() self.logger.debug(f"closed file {self.file_path}") - - -def download_file( - from_fqcn: str, - ref_id: str, - per_request_timeout: float, - cell: Cell, - location: str = None, - secure=False, - optional=False, - abort_signal=None, -) -> Tuple[str, Optional[str]]: - return FileDownloader.download_file( - from_fqcn=from_fqcn, - ref_id=ref_id, - per_request_timeout=per_request_timeout, - cell=cell, - location=location, - secure=secure, - optional=optional, - abort_signal=abort_signal, - ) diff --git a/nvflare/fuel/f3/streaming/obj_downloader.py b/nvflare/fuel/f3/streaming/obj_downloader.py index c654663059..7815908e78 100644 --- a/nvflare/fuel/f3/streaming/obj_downloader.py +++ b/nvflare/fuel/f3/streaming/obj_downloader.py @@ -11,584 +11,40 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import threading -import time -import uuid -from abc import ABC, abstractmethod -from typing import Any, Optional, Tuple - -from nvflare.apis.signal import Signal +from nvflare.fox.api.ctx import Context +from nvflare.fox.sys.backend import SysBackend from nvflare.fuel.f3.cellnet.cell import Cell -from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey, ReturnCode -from nvflare.fuel.f3.cellnet.utils import make_reply, new_cell_message -from nvflare.fuel.f3.message import Message -from nvflare.fuel.utils.log_utils import get_obj_logger -from nvflare.security.logging import secure_format_exception - -OBJ_DOWNLOADER_CHANNEL = "obj_downloader__" -OBJ_DOWNLOADER_TOPIC = "obj_downloader__download" - -""" -This package provides a framework for building object downloading capability (e.g. file download). - -A large object takes a lot of memory space. Sending a large object in one message needs even more memory space since -the object needs to be serialized into large number of bytes. Additional memory space may still be needed for the -transport layer to send the message. If the message is to be sent to multiple endpoints, even more memory is needed. - -Object Downloading can drastically reduce memory consumption: -- Instead of sending the large object in one message, it is divided into many smaller objects; -- Instead of pushing the message to the endpoints, each endpoint will come to request. This makes it more reliable when -different endpoints have different speed. - -Object Downloading works as follows: -- The sender prepares the object(s) for downloading. It first creates a transaction to get a tx_id. It then adds each -object to be downloaded to the transaction, and get a reference id (ref_id). -- The sender sends the ref_id(s) to all recipients through a separate message. -- Each recipient then calls the download_object function to download each referenced large object. - -To develop the downloading capability for a type of object (e.g. a file, a large dict, etc.), you need to provide -the implementation of a Producer and a Consumer. -- On the sending side, the Producer is responsible for producing the next small object to be sent (a chunk of bytes; -a small subset of the large dict; etc.). -- On the receiving side, the Consumer is responsible for processing the received small objects (writing the received -bytes to a temp file; putting the received small dict to the end result; etc.). - -One issue with object downloading is object life cycle management. Since the large objects to be downloaded are usually -temporary, you need to remove them when they are downloaded by all sites. But the problem is that you don't know how -quickly each site can finish downloading these large objects. When a transaction contains multiple objects to be -downloaded, it's even harder to know it. - -There are two ways to handle this issue: object downloaded callback, and transaction timeout. - -You can register an object_downloaded CB when adding an object to transaction. When the object is fully downloaded -to a site, this CB will be called. The obj_downloaded CB must follow this signature: - - downloaded_cb(ref_id: str, to_site: str, status: str, obj: Any, **cb_kwargs) - -where ref_id is the reference id of the object; -to_site is the FQCN of the site that has just finished downloading; -status is the status of downloading, as defined in DownloadStatus class; -obj is the large object that was just downloaded; -cb_kwargs are the kw args registered with the CB. - -Transaction timeout is the amount of time after the last downloading activity on any object in the -transaction from any site. For example, suppose you want to send 2 large files to 3 sites, each time a download -request is received on any file from any of the 3 sites, the last activity time of the transaction is updated to now. -If no downloading activity is received from any site on any objects in the transaction for the specified timeout, -the transaction is considered "timed out", and the timeout callback registered with the transaction is called. -The transaction timeout CB must follow this signature: - - timeout_cb(tx_id: str, objs: List[Any], **cb_kwargs) - -where tx_id is the ID of the transaction; -objs is the list of large objects registered with the transaction; -cb_kwargs are the kw args registered with the CB. - -You may need to use both mechanisms to fully take care of object life cycles. The object downloaded CB may never be -called since the site somehow didn't finish the downloading. In reality the timeout mechanism may be sufficient. - -Unlike with Object Streamer that the object owner pushes small objects to the recipients; with Object Downloader, -each recipient pulls the data from the object owner. -""" - - -class Downloadable(ABC): - - def set_transaction(self, tx_id, ref_id): - pass - - @abstractmethod - def produce(self, state: dict, requester: str) -> Tuple[str, Any, dict]: - """Produce a small object to be sent (on object sender side). - - Args: - state: current state of downloading, received from the downloading site - requester: the FQCN of the site that is downloading - - Returns: a tuple of (return code, a small object to be sent, new state to be sent). - - """ - pass - - def downloaded_to_one(self, to_site: str, status: str): - """Called when an object is downloaded to a site.""" - pass - - def downloaded_to_all(self): - """Called when the object is fully downloaded to all sites.""" - pass - - def transaction_done(self, transaction_id: str, status: str): - """Called when the transaction is finished.""" - pass - - -class _PropKey: - REF_ID = "ref_id" - STATE = "state" - DATA = "data" - STATUS = "status" - - -class _Ref: - - def __init__( - self, - tx, - obj: Downloadable, - ref_id=None, - ): - if ref_id: - # use provided ref_id - self.rid = ref_id - else: - self.rid = "R" + str(uuid.uuid4()) - self.tx = tx - self.obj = obj - self.num_sites_done = 0 - - def mark_active(self): - self.tx.mark_active() - - def obj_downloaded(self, to_site: str, status: str): - self.num_sites_done += 1 - - assert isinstance(self.obj, Downloadable) - self.obj.downloaded_to_one(to_site, status) - - assert isinstance(self.tx, _Transaction) - if 0 < self.tx.num_receivers <= self.num_sites_done: - # this object is done for all sites - self.obj.downloaded_to_all() - +from nvflare.fuel.f3.streaming.download_service import Downloadable, DownloadService -class ProduceRC: - """Defines return code for the Producer's produce method.""" - OK = "ok" - ERROR = "error" - EOF = "eof" - - -class DownloadStatus: - SUCCESS = "success" - FAILED = "failed" - - -class TransactionDoneStatus: - FINISHED = "finished" - TIMEOUT = "timeout" - DELETED = "deleted" - - -class _Transaction: +class ObjectDownloader: def __init__( self, - timeout: float, - num_receivers: int, - tx_id=None, - transaction_done_cb=None, - cb_kwargs=None, - ): - """Constructor of the transaction object. - - Args: - timeout: amount of time since last activity - num_receivers: number of receivers. 0 means unlimited. - tx_id: if provided, use it; otherwise create one - """ - if tx_id: - self.tid = tx_id - else: - self.tid = "T" + str(uuid.uuid4()) - self.timeout = timeout - self.num_receivers = num_receivers - self.last_active_time = time.time() - self.transaction_done_cb = transaction_done_cb - self.cb_kwargs = cb_kwargs - self.refs = [] - - def mark_active(self): - """Called to update the last active time of the transaction. - - Returns: - - """ - self.last_active_time = time.time() - - def add_object( - self, - obj: Downloadable, - ref_id=None, - ): - """Add a large object (to be downloaded) to the transaction. - - Args: - obj: the large object to be downloaded - ref_id: the ref id to be used, if specified - - Returns: - - """ - r = _Ref(self, obj, ref_id) - self.refs.append(r) - obj.set_transaction(self.tid, r.rid) - return r - - def timed_out(self): - """Called when the transaction is timed out. - - Returns: - - """ - self.transaction_done(TransactionDoneStatus.TIMEOUT) - - def is_finished(self): - """Check whether the transaction is finished (all objects are downloaded).""" - if self.num_receivers <= 0: - return False - - for ref in self.refs: - assert isinstance(ref, _Ref) - if ref.num_sites_done < self.num_receivers: - return False - return True - - def transaction_done(self, status: str): - """Called when the transaction is finished.""" - for ref in self.refs: - obj = ref.obj - assert isinstance(obj, Downloadable) - obj.transaction_done(self.tid, status) - - if self.transaction_done_cb: - self.transaction_done_cb(self.tid, status, [ref.obj for ref in self.refs], **self.cb_kwargs) - - -class TransactionInfo: - - def __init__(self, tx: _Transaction): - self.timeout = tx.timeout - self.num_receivers = tx.num_receivers - self.objects = [r.obj for r in tx.refs] - - -class ObjDownloader: - - _init_lock = threading.Lock() - _tx_table = {} - _ref_table = {} - _logger = None - _tx_monitor = None - _tx_lock = threading.Lock() - _initialized_cells = {} - - @classmethod - def _initialize(cls, cell: Cell): - with cls._init_lock: - if not cls._logger: - cls._logger = get_obj_logger(cls) - - if not cls._tx_monitor: - cls._tx_monitor = threading.Thread(target=cls._monitor_tx, daemon=True) - cls._tx_monitor.start() - - initialized = cls._initialized_cells.get(id(cell)) - if not initialized: - # register CBs - cell.register_request_cb( - channel=OBJ_DOWNLOADER_CHANNEL, - topic=OBJ_DOWNLOADER_TOPIC, - cb=cls._handle_download, - ) - cls._initialized_cells[id(cell)] = True - - @classmethod - def new_transaction( - cls, cell: Cell, timeout: float, - num_receivers: int = 0, + num_receivers: int, tx_id=None, transaction_done_cb=None, **cb_kwargs, ): - cls._initialize(cell) - tx = _Transaction(timeout, num_receivers, tx_id, transaction_done_cb, cb_kwargs) - with cls._tx_lock: - cls._tx_table[tx.tid] = tx - return tx.tid - - @classmethod - def add_object( - cls, - transaction_id: str, - obj: Downloadable, - ref_id=None, - ) -> str: - if not issubclass(type(obj), Downloadable): - raise ValueError(f"obj must be of type {Downloadable} but got {type(obj)}") - - tx = cls._tx_table.get(transaction_id) - if not tx: - raise ValueError(f"no such transaction {transaction_id}") - - assert isinstance(tx, _Transaction) - ref = tx.add_object(obj, ref_id) - with cls._tx_lock: - cls._ref_table[ref.rid] = ref - return ref.rid - - @classmethod - def delete_transaction(cls, transaction_id: str): - with cls._tx_lock: - tx = cls._tx_table.get(transaction_id) - if tx: - cls._delete_tx(tx) - tx.transaction_done(TransactionDoneStatus.DELETED) - - @classmethod - def shutdown(cls): - """Shutdown and clean up resources. - - Returns: None - - """ - with cls._tx_lock: - tx_list = list(cls._tx_table.values()) - if tx_list: - for tx in tx_list: - cls._delete_tx(tx) - tx.transaction_done(TransactionDoneStatus.DELETED) - - @classmethod - def _delete_tx(cls, tx: _Transaction): - cls._tx_table.pop(tx.tid, None) - - # remove all refs - for r in tx.refs: - cls._ref_table.pop(r.rid, None) - - @classmethod - def get_transaction_info(cls, transaction_id: str) -> Optional[TransactionInfo]: - tx = cls._tx_table.get(transaction_id) - if not tx: - return None - else: - return TransactionInfo(tx) - - @classmethod - def get_transaction_id(cls, ref_id: str) -> Optional[str]: - ref = cls._ref_table.get(ref_id) - if not ref: - return None - else: - assert isinstance(ref, _Ref) - return ref.tx.tid - - @classmethod - def _handle_download(cls, request: Message) -> Message: - requester = request.get_header(MessageHeaderKey.ORIGIN) - payload = request.payload - assert isinstance(payload, dict) - rid = payload.get(_PropKey.REF_ID) - if not rid: - cls._logger.error(f"missing {_PropKey.REF_ID} in request from {requester}") - return make_reply(ReturnCode.INVALID_REQUEST) - - current_state = payload.get(_PropKey.STATE) - with cls._tx_lock: - ref = cls._ref_table.get(rid) - if not ref: - cls._logger.error(f"no ref found for {rid} from {requester}") - return make_reply(ReturnCode.INVALID_REQUEST) - - assert isinstance(ref, _Ref) - ref.mark_active() - tx = ref.tx - assert isinstance(tx, _Transaction) - - try: - rc, data, new_state = ref.obj.produce(current_state, requester) - except Exception as ex: - cls._logger.error( - f"Object {type(ref.obj)} encountered exception when produce: {secure_format_exception(ex)}" - ) - return make_reply(ReturnCode.PROCESS_EXCEPTION) - - if rc != ProduceRC.OK: - # already done - ref.obj_downloaded( - requester, status=DownloadStatus.SUCCESS if rc == ProduceRC.EOF else DownloadStatus.FAILED - ) - return make_reply(ReturnCode.OK, body={_PropKey.STATUS: rc}) - else: - # continue - return make_reply( - ReturnCode.OK, - body={ - _PropKey.STATUS: rc, - _PropKey.STATE: new_state, - _PropKey.DATA: data, - }, - ) - - @classmethod - def _monitor_tx(cls): - while True: - now = time.time() - expired_tx = [] - finished_tx = [] - with cls._tx_lock: - for tid, tx in cls._tx_table.items(): - assert isinstance(tx, _Transaction) - - # check whether all refs are done - if tx.is_finished(): - finished_tx.append(tx) - elif now - tx.last_active_time > tx.timeout: - expired_tx.append(tx) - - for tx in expired_tx: - assert isinstance(tx, _Transaction) - tx.transaction_done(TransactionDoneStatus.TIMEOUT) - cls._delete_tx(tx) - - for tx in finished_tx: - tx.transaction_done(TransactionDoneStatus.FINISHED) - cls._delete_tx(tx) - - time.sleep(5.0) - - -class Consumer(ABC): - - def __init__(self): - self.logger = get_obj_logger(self) - - @abstractmethod - def consume(self, ref_id: str, state: dict, data: Any) -> dict: - """Called to process the received data. - - Args: - ref_id: ref id of the object being downloaded - state: current state of downloading - data: data to be processed - - Returns: new state to be sent back to the data owner. - - """ - pass - - @abstractmethod - def download_completed(self, ref_id: str): - """Called when the downloading is finished successfully. - - Args: - ref_id: ref id of the object being downloaded - - Returns: None - - """ - pass - - @abstractmethod - def download_failed(self, ref_id: str, reason: str): - """Called when the downloading is finished unsuccessfully. - - Args: - ref_id: ref id of the object being downloaded - reason: explain the reason of failure - - Returns: None - - """ - pass - - -def download_object( - from_fqcn: str, - ref_id: str, - per_request_timeout: float, - cell: Cell, - consumer: Consumer, - secure=False, - optional=False, - abort_signal: Signal = None, -): - """Download a large object from the object owner. - - Args: - from_fqcn: the FQCN of the object owner - ref_id: reference id of the object to be downloaded - per_request_timeout: timeout for each request to the object owner. - cell: the cell to be used for communication withe object owner. - consumer: the Consumer object used for processing received data - secure: use P2P private communication with the data owner - optional: supress log messages - abort_signal: for signaling abort - - Returns: None - - """ - request = new_cell_message( - headers={}, - payload={ - _PropKey.REF_ID: ref_id, - }, - ) - - while True: - start_time = time.time() - reply = cell.send_request( - channel=OBJ_DOWNLOADER_CHANNEL, - target=from_fqcn, - topic=OBJ_DOWNLOADER_TOPIC, - request=request, - timeout=per_request_timeout, - secure=secure, - optional=optional, - abort_signal=abort_signal, + self.cell = cell + self.tx_id = DownloadService.new_transaction( + cell=self.cell, + timeout=timeout, + num_receivers=num_receivers, + tx_id=tx_id, + transaction_done_cb=transaction_done_cb, + **cb_kwargs, ) - duration = time.time() - start_time - - if abort_signal and abort_signal.triggered: - consumer.download_failed(ref_id, f"download aborted after {duration} secs") - return - - assert isinstance(reply, Message) - rc = reply.get_header(MessageHeaderKey.RETURN_CODE) - if rc != ReturnCode.OK: - consumer.download_failed(ref_id, f"error requesting data from {from_fqcn} after {duration} secs: {rc}") - return - - payload = reply.payload - assert isinstance(payload, dict) - status = payload.get(_PropKey.STATUS) - if status == ProduceRC.EOF: - consumer.download_completed(ref_id) - return - elif status == ProduceRC.ERROR: - consumer.download_failed(ref_id, f"producer error after {duration} secs") - return - # continue - data = payload.get(_PropKey.DATA) - state = payload.get(_PropKey.STATE) - try: - new_state = consumer.consume(ref_id, state, data) - except Exception as ex: - consumer.download_failed(ref_id, f"exception when consuming data: {secure_format_exception(ex)}") - return - - if not isinstance(new_state, dict): - consumer.download_failed(ref_id, f"consumer error: new_state should be dict but got {type(new_state)}") - return - - if abort_signal and abort_signal.triggered: - consumer.download_failed(ref_id, "download aborted") - return + def add_object(self, obj: Downloadable, ref_id=None) -> str: + rid = DownloadService.add_object( + transaction_id=self.tx_id, + obj=obj, + ref_id=ref_id, + ) + return rid - # ask for more - request = new_cell_message(headers={}, payload={_PropKey.REF_ID: ref_id, _PropKey.STATE: new_state}) + def delete_transaction(self): + DownloadService.delete_transaction(self.tx_id) diff --git a/nvflare/fuel/hci/server/binary_transfer.py b/nvflare/fuel/hci/server/binary_transfer.py index bdb030b76b..e23a3ebaba 100644 --- a/nvflare/fuel/hci/server/binary_transfer.py +++ b/nvflare/fuel/hci/server/binary_transfer.py @@ -15,7 +15,7 @@ import os import shutil -from nvflare.fuel.f3.streaming.file_downloader import FileDownloader +from nvflare.fuel.f3.streaming.file_downloader import ObjectDownloader, add_file from nvflare.fuel.hci.conn import Connection from nvflare.fuel.hci.proto import MetaKey, MetaStatusValue, make_meta from nvflare.fuel.hci.server.constants import ConnProps @@ -41,7 +41,8 @@ def download_folder(self, conn: Connection, tx_id: str, folder_name: str): engine = conn.get_prop(ConnProps.ENGINE) cell = engine.get_cell() source_fqcn = cell.get_fqcn() - download_tid = FileDownloader.new_transaction( + downloader = ObjectDownloader( + num_receivers=1, cell=engine.get_cell(), timeout=5, transaction_done_cb=self._cleanup_tx, @@ -53,12 +54,7 @@ def download_folder(self, conn: Connection, tx_id: str, folder_name: str): for dir_path, dir_names, file_names in os.walk(tx_path): for f in file_names: full_path = os.path.join(dir_path, f) - - ref_id = FileDownloader.add_file( - transaction_id=download_tid, - file_name=full_path, - ) - + ref_id = add_file(downloader, file_name=full_path) p = os.path.relpath(full_path, tx_path) files.append([p, ref_id]) @@ -85,9 +81,9 @@ def download_folder(self, conn: Connection, tx_id: str, folder_name: str): ), ) - def _cleanup_tx(self, tx_id: str, files, tx_path): + def _cleanup_tx(self, tx_id: str, status, files, tx_path): """ Remove the job download folder """ shutil.rmtree(tx_path, ignore_errors=True) - self.logger.debug(f"deleted download path: {tx_id=} {tx_path=} {files=}") + self.logger.debug(f"deleted download path: {tx_id=} {status=} {tx_path=} {files=}") diff --git a/nvflare/fuel/utils/fobs/decomposers/via_file.py b/nvflare/fuel/utils/fobs/decomposers/via_file.py index 67baac0fca..b78b78e35d 100644 --- a/nvflare/fuel/utils/fobs/decomposers/via_file.py +++ b/nvflare/fuel/utils/fobs/decomposers/via_file.py @@ -17,12 +17,12 @@ import time import uuid from abc import ABC, abstractmethod -from typing import Any, Optional +from typing import Any, Optional, Tuple import nvflare.fuel.utils.app_config_utils as acu from nvflare.apis.fl_constant import ConfigVarName from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey -from nvflare.fuel.f3.streaming.file_downloader import FileDownloader +from nvflare.fuel.f3.streaming.file_downloader import ObjectDownloader, add_file, download_file from nvflare.fuel.utils import fobs from nvflare.fuel.utils.fobs.datum import Datum, DatumManager, DatumType from nvflare.fuel.utils.fobs.lobs import get_datum_dir @@ -98,20 +98,11 @@ def __init__(self, min_size_for_file, config_var_prefix): self.prefix = self.__class__.__name__ self.decompose_ctx_key = f"{self.prefix}_dc" # kept in fobs_ctx: each target type has its own DecomposeCtx self.items_key = f"{self.prefix}_items" # in fobs_ctx: each target type has its own set of items - self.file_downloader_class = FileDownloader self.min_size_for_file = min_size_for_file self.config_var_prefix = config_var_prefix - def set_file_downloader_class(self, file_downloader_class): - # used only for offline testing! - self.file_downloader_class = file_downloader_class - - def set_min_size_for_file(self, size: int): - # used only for testing! - self.min_size_for_file = size - @abstractmethod - def dump_to_file(self, items: dict, path: str, fobs_ctx: dict) -> (Optional[str], Optional[dict]): + def dump_to_file(self, items: dict, path: str, fobs_ctx: dict) -> Tuple[Optional[str], Optional[dict]]: """Dump the items to the file with the specified path Args: @@ -191,7 +182,7 @@ def _create_ref(self, target: Any, manager: DatumManager, fobs_ctx: dict): manager.register_post_cb(self._process_items_to_datum) return item_id, target_id - def _create_file(self, fobs_ctx: dict) -> (str, int, dict): + def _create_file(self, fobs_ctx: dict) -> Tuple[str, int, dict]: dc = fobs_ctx.get(self.decompose_ctx_key) assert isinstance(dc, _DecomposeCtx) items = dc.target_items @@ -253,7 +244,7 @@ def decompose(self, target: Any, manager: DatumManager = None) -> Any: self.logger.debug(f"ViaFile: created ref for target {target_id}: {item_id}") return {EncKey.TYPE: EncType.REF, EncKey.DATA: item_id} - def _create_download_tx(self, fobs_ctx: dict): + def _create_downloader(self, fobs_ctx: dict): msg_root_id, msg_root_ttl = self._determine_msg_root(fobs_ctx) if msg_root_ttl: @@ -266,10 +257,11 @@ def _create_download_tx(self, fobs_ctx: dict): self.logger.debug(f"determined: {msg_root_id=} {timeout=}") - tx_id = None + downloader = None cell = fobs_ctx.get(fobs.FOBSContextKey.CELL) if cell: - tx_id = self.file_downloader_class.new_transaction( + downloader = ObjectDownloader( + num_receivers=1, cell=cell, timeout=timeout, transaction_done_cb=self._delete_download_tx, @@ -279,10 +271,10 @@ def _create_download_tx(self, fobs_ctx: dict): subscribe_to_msg_root( msg_root_id=msg_root_id, cb=self._delete_download_tx_on_msg_root, - download_tx_id=tx_id, + downloader=downloader, ) - return tx_id + return downloader def _process_items_to_datum(self, mgr: DatumManager): """This method is called during serialization after all target items are serialized. @@ -373,23 +365,23 @@ def _finalize_download_tx(self, mgr: DatumManager): files = fobs_ctx.get(_CtxKey.FILES) if files: - download_tx_id = self._create_download_tx(fobs_ctx) + downloader = self._create_downloader(fobs_ctx) for file_ref_id, file_name in files: - self.logger.debug(f"ViaFile: adding file to downloader: {download_tx_id=} {file_name=}") - self.file_downloader_class.add_file( - transaction_id=download_tx_id, + self.logger.debug(f"ViaFile: adding file to downloader: {file_name=}") + add_file( + downloader=downloader, file_name=file_name, ref_id=file_ref_id, ) - def _delete_download_tx_on_msg_root(self, msg_root_id: str, download_tx_id: str): + def _delete_download_tx_on_msg_root(self, msg_root_id: str, downloader: ObjectDownloader): # this CB is triggered when msg root is deleted. - self.logger.debug(f"deleting download_tx_id {download_tx_id} associated with {msg_root_id=}") - self.file_downloader_class.delete_transaction(download_tx_id) + self.logger.debug(f"deleting download transaction associated with {msg_root_id=}") + downloader.delete_transaction() - def _delete_download_tx(self, tx_id, file_names): + def _delete_download_tx(self, tx_id, status, file_names): # this CB is triggered when download tx times out or is deleted - self.logger.debug(f"ViaFile: deleting download tx: {tx_id}") + self.logger.debug(f"ViaFile: deleting download tx: {tx_id} {status=} {file_names=}") # delete all files in the tx for f in file_names: @@ -524,7 +516,7 @@ def _download_from_remote_cell(self, fobs_ctx: dict, file_ref: dict): abort_signal = fobs_ctx.get(fobs.FOBSContextKey.ABORT_SIGNAL) self.logger.debug(f"trying to download file: {file_ref_id=} {fqcn=}") - err, file_path = self.file_downloader_class.download_file( + err, file_path = download_file( from_fqcn=fqcn, ref_id=file_ref_id, location=get_datum_dir(), diff --git a/nvflare/private/fed/client/client_runner.py b/nvflare/private/fed/client/client_runner.py index 466c1ae272..4872def017 100644 --- a/nvflare/private/fed/client/client_runner.py +++ b/nvflare/private/fed/client/client_runner.py @@ -29,7 +29,7 @@ from nvflare.apis.utils.reliable_message import ReliableMessage from nvflare.apis.utils.task_utils import apply_filters from nvflare.fuel.f3.cellnet.fqcn import FQCN -from nvflare.fuel.f3.streaming.obj_downloader import ObjDownloader +from nvflare.fuel.f3.streaming.download_service import DownloadService from nvflare.fuel.utils.msg_root_utils import delete_msg_root from nvflare.private.defs import SpecialTaskName, TaskConstant from nvflare.private.fed.client.client_engine_executor_spec import ClientEngineExecutorSpec, TaskAssignment @@ -655,7 +655,7 @@ def run(self, app_root, args): self.end_run_events_sequence() ReliableMessage.shutdown() self.engine.shutdown_streamer() - ObjDownloader.shutdown() + DownloadService.shutdown() with self.task_lock: self.running_tasks = {} diff --git a/nvflare/private/fed/server/server_runner.py b/nvflare/private/fed/server/server_runner.py index 39080a4f7a..5647ccaeff 100644 --- a/nvflare/private/fed/server/server_runner.py +++ b/nvflare/private/fed/server/server_runner.py @@ -26,7 +26,7 @@ from nvflare.apis.utils.fl_context_utils import add_job_audit_event from nvflare.apis.utils.reliable_message import ReliableMessage from nvflare.apis.utils.task_utils import apply_filters -from nvflare.fuel.f3.streaming.obj_downloader import ObjDownloader +from nvflare.fuel.f3.streaming.download_service import DownloadService from nvflare.fuel.utils.job_utils import build_client_hierarchy from nvflare.private.defs import SpecialTaskName, TaskConstant from nvflare.private.fed.tbi import TBI @@ -230,7 +230,7 @@ def run(self): ReliableMessage.shutdown() self.engine.shutdown_streamer() - ObjDownloader.shutdown() + DownloadService.shutdown() self.log_info(fl_ctx, "Server runner finished.") def handle_event(self, event_type: str, fl_ctx: FLContext): From 3944c184cec5dd77c481cafdd919d29bcd9bf57b Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Thu, 23 Oct 2025 13:39:55 +0800 Subject: [PATCH 046/102] add np downloader --- nvflare/app_common/np/np_downloader.py | 142 ++++++++++++++++++ nvflare/app_common/workflows/lr/fedavg.py | 4 +- nvflare/app_opt/pt/tensor_downloader.py | 10 +- nvflare/fox/examples/np/algos/utils.py | 27 ++++ nvflare/fox/examples/pt/filters.py | 4 +- nvflare/fox/examples/pt/pt_avg_mixed.py | 113 +++++++------- nvflare/fox/examples/pt/pt_avg_stream.py | 6 +- .../fox/examples/pt/recipe_pt_avg_mixed.py | 14 +- nvflare/fox/examples/pt/utils.py | 20 +++ nvflare/fox/sys/constants.py | 2 +- nvflare/fox/sys/downloader.py | 32 +++- nvflare/fuel/f3/streaming/cacheable.py | 32 ++-- nvflare/fuel/f3/streaming/obj_downloader.py | 2 - 13 files changed, 311 insertions(+), 97 deletions(-) create mode 100644 nvflare/app_common/np/np_downloader.py diff --git a/nvflare/app_common/np/np_downloader.py b/nvflare/app_common/np/np_downloader.py new file mode 100644 index 0000000000..cb2204cdba --- /dev/null +++ b/nvflare/app_common/np/np_downloader.py @@ -0,0 +1,142 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from io import BytesIO +from typing import Any, List, Optional, Tuple + +import numpy as np + +from nvflare.fuel.f3.cellnet.cell import Cell +from nvflare.fuel.f3.streaming.cacheable import CacheableObject, ItemConsumer +from nvflare.fuel.f3.streaming.download_service import download_object +from nvflare.fuel.f3.streaming.obj_downloader import ObjectDownloader + + +class ArrayDownloadable(CacheableObject): + + def __init__(self, arrays: dict[str, np.ndarray], max_chunk_size: int): + self.arrays = arrays + self.size = len(arrays) + self.keys = list(arrays.keys()) + super().__init__(max_chunk_size) + + def get_base_object(self): + return self.arrays + + def get_item_count(self) -> int: + return self.size + + def produce_item(self, index: int) -> Any: + key = self.keys[index] + arrays_to_send = {key: self.arrays[key]} + stream = BytesIO() + np.savez(allow_pickle=False, file=stream, **arrays_to_send) + return stream.getvalue() + + +class ArrayConsumer(ItemConsumer): + + def __init__(self, arrays_received_cb, cb_kwargs): + ItemConsumer.__init__(self) + self.arrays_received_cb = arrays_received_cb + self.cb_kwargs = cb_kwargs + if arrays_received_cb is not None and not callable(arrays_received_cb): + raise ValueError("arrays_received_cb must be callable") + + @staticmethod + def _to_dict(item: bytes) -> dict: + result = {} + stream = BytesIO(item) + with np.load(stream, allow_pickle=False) as npz_obj: + for k in npz_obj.files: + result[k] = npz_obj[k] + return result + + def consume_items(self, items: List[Any], result: Any) -> Any: + assert isinstance(items, list) + if result is None: + result = {} + + arrays = {} + for item in items: + td = self._to_dict(item) + if not isinstance(td, dict): + raise ValueError("cannot load received bytes to arrays") + arrays.update(td) + + if self.arrays_received_cb is not None: + cb_result = self.arrays_received_cb(arrays, **self.cb_kwargs) + if isinstance(cb_result, dict): + result.update(cb_result) + else: + result.update(arrays) + return result + + +def add_arrays( + downloader: ObjectDownloader, + arrays: dict[str, np.ndarray], + max_chunk_size: int = 1, +) -> str: + """Add arrays to be downloaded to the specified downloader. + + Args: + downloader: the downloader to add tensors to. + arrays: arrays to be downloaded + max_chunk_size: max chunk size + + Returns: reference id for the state dict. + + """ + obj = ArrayDownloadable(arrays, max_chunk_size) + return downloader.add_object(obj) + + +def download_arrays( + from_fqcn: str, + ref_id: str, + per_request_timeout: float, + cell: Cell, + secure=False, + optional=False, + abort_signal=None, + arrays_received_cb=None, + **cb_kwargs, +) -> Tuple[str, Optional[dict[str, np.ndarray]]]: + """Download the referenced arrays from the source. + + Args: + from_fqcn: FQCN of the data source. + ref_id: reference ID of the arrays to be downloaded. + per_request_timeout: timeout for requests sent to the data source. + cell: cell to be used for communicating to the data source. + secure: P2P private mode for communication + optional: supress log messages of communication + abort_signal: signal for aborting download. + arrays_received_cb: the callback to be called when one set of arrays are received + + Returns: tuple of (error message if any, downloaded state dict). + + """ + consumer = ArrayConsumer(arrays_received_cb, cb_kwargs) + download_object( + from_fqcn=from_fqcn, + ref_id=ref_id, + consumer=consumer, + per_request_timeout=per_request_timeout, + cell=cell, + secure=secure, + optional=optional, + abort_signal=abort_signal, + ) + return consumer.error, consumer.result diff --git a/nvflare/app_common/workflows/lr/fedavg.py b/nvflare/app_common/workflows/lr/fedavg.py index f807ae7aa0..7904aa1b84 100644 --- a/nvflare/app_common/workflows/lr/fedavg.py +++ b/nvflare/app_common/workflows/lr/fedavg.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. """ - Federated Averaging for Logistic Regression with Newton-Raphson method - using Numpy +Federated Averaging for Logistic Regression with Newton-Raphson method +using Numpy """ from typing import List, Optional diff --git a/nvflare/app_opt/pt/tensor_downloader.py b/nvflare/app_opt/pt/tensor_downloader.py index 8e4be46ae8..f48099f13b 100644 --- a/nvflare/app_opt/pt/tensor_downloader.py +++ b/nvflare/app_opt/pt/tensor_downloader.py @@ -25,11 +25,11 @@ class TensorDownloadable(CacheableObject): - def __init__(self, tensors: dict[str, torch.Tensor], num_items_per_chunk: int): + def __init__(self, tensors: dict[str, torch.Tensor], max_chunk_size: int): self.tensors = tensors self.size = len(tensors) self.keys = list(tensors.keys()) - super().__init__(num_items_per_chunk) + super().__init__(max_chunk_size) def get_base_object(self): return self.tensors @@ -76,19 +76,19 @@ def consume_items(self, items: List[Any], result: Any) -> Any: def add_tensors( downloader: ObjectDownloader, tensors: dict[str, torch.Tensor], - num_tensors_per_chunk: int = 1, + max_chunk_size: int = 1, ) -> str: """Add a file to be downloaded to the specified downloader. Args: downloader: the downloader to add tensors to. tensors: state dict to be downloaded - num_tensors_per_chunk: number of tensors per chunk + max_chunk_size: max chunk size Returns: reference id for the state dict. """ - obj = TensorDownloadable(tensors, num_tensors_per_chunk) + obj = TensorDownloadable(tensors, max_chunk_size) return downloader.add_object(obj) diff --git a/nvflare/fox/examples/np/algos/utils.py b/nvflare/fox/examples/np/algos/utils.py index 13db56f999..10719506cf 100644 --- a/nvflare/fox/examples/np/algos/utils.py +++ b/nvflare/fox/examples/np/algos/utils.py @@ -28,9 +28,36 @@ def parse_array_def(array_def): raise ValueError(f"unsupported array def: {array_def}") +def parse_state_dict(d: dict[str, list]): + result = {} + for k, v in d.items(): + result[k] = parse_array_def(v) + return result + + +def parse_model_def(model_def): + if isinstance(model_def, dict): + return parse_state_dict(model_def) + else: + return parse_array_def(model_def) + + def save_np_model(model: np.ndarray, file_name: str): np.save(file_name, model) def load_np_model(file_name: str): return np.load(file_name) + + +def add(model: dict, to_model: dict): + for k, v in model.items(): + if k not in to_model: + to_model[k] = v + else: + to_model[k] += v + + +def div(model: dict, value): + for k, v in model.items(): + model[k] = v / value diff --git a/nvflare/fox/examples/pt/filters.py b/nvflare/fox/examples/pt/filters.py index 905ff093f8..348b2fc3a6 100644 --- a/nvflare/fox/examples/pt/filters.py +++ b/nvflare/fox/examples/pt/filters.py @@ -26,7 +26,7 @@ def filter_call(self, func_kwargs: dict, context: Context): ctx=context, timeout=5.0, ) - model = downloader.add_tensors(arg_value, 2) + model = downloader.add_tensors(arg_value, 0) func_kwargs[self.model_arg_name] = model return func_kwargs @@ -63,7 +63,7 @@ def filter_result(self, result: Any, context: Context): ctx=context, timeout=5.0, ) - return downloader.add_tensors(result, 2) + return downloader.add_tensors(result, 0) class IncomingModelResultFilter(ResultFilter): diff --git a/nvflare/fox/examples/pt/pt_avg_mixed.py b/nvflare/fox/examples/pt/pt_avg_mixed.py index a7b1b7a36e..3babce1ec2 100644 --- a/nvflare/fox/examples/pt/pt_avg_mixed.py +++ b/nvflare/fox/examples/pt/pt_avg_mixed.py @@ -12,10 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -import os import threading -import uuid +import numpy as np import torch from nvflare.fox.api.app import ClientApp, ServerApp @@ -25,10 +24,14 @@ from nvflare.fox.api.group import all_clients from nvflare.fox.api.strategy import Strategy from nvflare.fox.api.utils import simple_logging -from nvflare.fox.examples.np.algos.utils import load_np_model, parse_array_def, save_np_model -from nvflare.fox.examples.pt.utils import parse_state_dict +from nvflare.fox.examples.np.algos.utils import add as add_np +from nvflare.fox.examples.np.algos.utils import div as div_np +from nvflare.fox.examples.np.algos.utils import parse_state_dict as parse_np +from nvflare.fox.examples.pt.utils import add as add_pt +from nvflare.fox.examples.pt.utils import div as div_pt +from nvflare.fox.examples.pt.utils import parse_state_dict as parse_pt from nvflare.fox.sim.simulator import Simulator -from nvflare.fox.sys.downloader import Downloader, download_file, download_tensors +from nvflare.fox.sys.downloader import Downloader, download_arrays, download_tensors from nvflare.fuel.utils.log_utils import get_obj_logger @@ -36,7 +39,7 @@ class _AggrResult: def __init__(self): self.pt_total = {} - self.np_total = 0 + self.np_total = {} self.count = 0 self.lock = threading.Lock() # ensure update integrity @@ -50,8 +53,8 @@ def __init__(self, pt_model, np_model, num_rounds=10, timeout=2.0): self.timeout = timeout self.name = "PTFedAvgMixed" self.logger = get_obj_logger(self) - self._pt_model = parse_state_dict(pt_model) - self._np_model = parse_array_def(np_model) + self._pt_model = parse_pt(pt_model) + self._np_model = parse_np(np_model) def execute(self, context: Context): self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") @@ -70,40 +73,32 @@ def _do_one_round(self, r, pt_model, np_model, ctx: Context): aggr_result=aggr_result, ) - file_name = None if ctx.env_type == EnvType.SYSTEM: - file_name = f"/tmp/np_{str(uuid.uuid4())}.npy" - save_np_model(np_model, file_name) - downloader = Downloader( num_receivers=grp.size, ctx=ctx, timeout=5.0, ) model_type = "ref" - pt_model = downloader.add_tensors(pt_model, 2) - np_model = downloader.add_file(file_name) + pt_model = downloader.add_tensors(pt_model, 0) + np_model = downloader.add_arrays(np_model, 0) self.logger.info(f"prepared model as ref: {pt_model=} {np_model=}") else: model_type = "model" grp.train(r, pt_model, np_model, model_type) - if file_name: - os.remove(file_name) - if aggr_result.count == 0: return None else: - pt_result = {} - for k, v in aggr_result.pt_total.items(): - pt_result[k] = torch.div(v, aggr_result.count) - + pt_result = aggr_result.pt_total + div_pt(pt_result, aggr_result.count) self.logger.info( f"[{ctx.header_str()}] round {r}: aggr PT result from {aggr_result.count} clients: {pt_result}" ) - np_result = aggr_result.np_total / aggr_result.count + np_result = aggr_result.np_total + div_np(np_result, aggr_result.count) self.logger.info( f"[{ctx.header_str()}] round {r}: aggr NP result from {aggr_result.count} clients: {np_result}" ) @@ -125,31 +120,32 @@ def _accept_train_result(self, result, aggr_result: _AggrResult, context: Contex if err: raise RuntimeError(f"failed to download model {pt_result}: {err}") - err, file_path = download_file(ref=np_result, per_request_timeout=5.0, ctx=context) + err, np_result = download_arrays( + ref=np_result, + per_request_timeout=5.0, + ctx=context, + arrays_received_cb=self._aggregate_arrays, + aggr_result=aggr_result, + context=context, + ) if err: raise RuntimeError(f"failed to download NP model file {np_result}: {err}") - self.logger.info(f"downloaded model file to {file_path}") - np_result = load_np_model(file_path) - os.remove(file_path) else: - for k, v in pt_result.items(): - if k not in aggr_result.pt_total: - aggr_result.pt_total[k] = v - else: - aggr_result.pt_total[k] += v + add_pt(pt_result, aggr_result.pt_total) + add_np(np_result, aggr_result.np_total) - aggr_result.np_total += np_result aggr_result.count += 1 return None def _aggregate_tensors(self, td: dict[str, torch.Tensor], aggr_result: _AggrResult, context: Context): self.logger.info(f"[{context.header_str()}] aggregating received tensor: {td}") with aggr_result.lock: - for k, v in td.items(): - if k not in aggr_result.pt_total: - aggr_result.pt_total[k] = v - else: - aggr_result.pt_total[k] += v + add_pt(td, aggr_result.pt_total) + + def _aggregate_arrays(self, td: dict[str, np.ndarray], aggr_result: _AggrResult, context: Context): + self.logger.info(f"[{context.header_str()}] aggregating received array: {td}") + with aggr_result.lock: + add_np(td, aggr_result.np_total) class PTTrainer(ClientApp): @@ -163,28 +159,29 @@ def train(self, current_round, pt_model, np_model, model_type: str, context: Con if context.is_aborted(): self.logger.debug("training aborted") return 0 + self.logger.debug( f"[{context.header_str()}] training round {current_round}: {model_type=} {pt_model=} {np_model=}" ) + if model_type == "ref": err, pt_model = download_tensors(ref=pt_model, per_request_timeout=5.0, ctx=context) if err: raise RuntimeError(f"failed to download PT model {pt_model}: {err}") self.logger.info(f"downloaded PT model {pt_model}") - err, file_path = download_file(ref=np_model, per_request_timeout=5.0, ctx=context) + err, np_model = download_arrays(ref=np_model, per_request_timeout=5.0, ctx=context) if err: - raise RuntimeError(f"failed to download NP model file {np_model}: {err}") - self.logger.info(f"downloaded model file to {file_path}") - np_model = load_np_model(file_path) - self.logger.info(f"loaded NP model from file: {np_model}") - os.remove(file_path) + raise RuntimeError(f"failed to download NP model {np_model}: {err}") + self.logger.info(f"downloaded NP model {np_model}") pt_result = {} for k, v in pt_model.items(): pt_result[k] = v + self.delta - np_result = np_model + self.delta + np_result = {} + for k, v in np_model.items(): + np_result[k] = v + self.delta if model_type == "ref": # stream it @@ -193,36 +190,28 @@ def train(self, current_round, pt_model, np_model, model_type: str, context: Con ctx=context, timeout=5.0, ) - pt_result = downloader.add_tensors(pt_result, 2) + pt_result = downloader.add_tensors(pt_result, 0) self.logger.info(f"prepared PT result as ref: {pt_result}") - file_name = f"/tmp/np_{str(uuid.uuid4())}.npy" - save_np_model(np_result, file_name) - np_result = downloader.add_file(file_name, file_downloaded_cb=self._result_downloaded) + np_result = downloader.add_arrays(np_result, 0) self.logger.info(f"prepared NP result as ref: {np_result}") - return pt_result, np_result, model_type - def _result_downloaded(self, to_site: str, status: str, file_name): - self.logger.info(f"NP model file {file_name} downloaded to {to_site}: {status=}") - if not to_site: - # downloaded to all sites - os.remove(file_name) - self.logger.info(f"NP model file {file_name} removed") - def main(): simple_logging(logging.DEBUG) + init_model = { + "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "y": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "z": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + } + server_app = ServerApp( strategy_name="fed_avg_mixed", strategy=PTFedAvgMixed( - pt_model={ - "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], - "y": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], - "z": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], - }, - np_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], + pt_model=init_model, + np_model=init_model, num_rounds=4, ), ) diff --git a/nvflare/fox/examples/pt/pt_avg_stream.py b/nvflare/fox/examples/pt/pt_avg_stream.py index 75f986f1b9..f0bd1e0823 100644 --- a/nvflare/fox/examples/pt/pt_avg_stream.py +++ b/nvflare/fox/examples/pt/pt_avg_stream.py @@ -74,8 +74,8 @@ def _do_one_round(self, r, current_model, ctx: Context): timeout=5.0, ) model_type = "ref" - model = downloader.add_tensors(current_model, 2) - model2 = downloader.add_tensors(model2, 2) + model = downloader.add_tensors(current_model, 0) + model2 = downloader.add_tensors(model2, 0) self.logger.info(f"prepared model as ref: {model}") else: model = current_model @@ -166,7 +166,7 @@ def train(self, current_round, model1, model2, model_type: str, context: Context timeout=5.0, ) model_type = "ref" - model = downloader.add_tensors(result, 2) + model = downloader.add_tensors(result, 0) self.logger.info(f"prepared result as ref: {model}") else: model = result diff --git a/nvflare/fox/examples/pt/recipe_pt_avg_mixed.py b/nvflare/fox/examples/pt/recipe_pt_avg_mixed.py index c0c14655e4..f9cd0bfe18 100644 --- a/nvflare/fox/examples/pt/recipe_pt_avg_mixed.py +++ b/nvflare/fox/examples/pt/recipe_pt_avg_mixed.py @@ -24,15 +24,17 @@ def main(): simple_logging(logging.DEBUG) + init_model = { + "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "y": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "z": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + } + server_app = ServerApp( strategy_name="fedavg_mixed", strategy=PTFedAvgMixed( - pt_model={ - "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], - "y": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], - "z": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], - }, - np_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], + pt_model=init_model, + np_model=init_model, num_rounds=2, ), ) diff --git a/nvflare/fox/examples/pt/utils.py b/nvflare/fox/examples/pt/utils.py index 405ba5f338..4d580668f6 100644 --- a/nvflare/fox/examples/pt/utils.py +++ b/nvflare/fox/examples/pt/utils.py @@ -32,3 +32,23 @@ def parse_state_dict(d: dict[str, list]): for k, v in d.items(): result[k] = parse_array_def(v) return result + + +def parse_model_def(model_def): + if isinstance(model_def, dict): + return parse_state_dict(model_def) + else: + return parse_array_def(model_def) + + +def add(value: dict, to_model: dict): + for k, v in value.items(): + if k not in to_model: + to_model[k] = v + else: + to_model[k] += v + + +def div(model: dict, value): + for k, v in model.items(): + model[k] = torch.div(v, value) diff --git a/nvflare/fox/sys/constants.py b/nvflare/fox/sys/constants.py index b7f85b298d..afbb3cef79 100644 --- a/nvflare/fox/sys/constants.py +++ b/nvflare/fox/sys/constants.py @@ -13,7 +13,7 @@ # limitations under the License. SYNC_TASK_NAME = "sync" -MSG_CHANNEL = "focs" +MSG_CHANNEL = "fox" MSG_TOPIC = "call" diff --git a/nvflare/fox/sys/downloader.py b/nvflare/fox/sys/downloader.py index 3b327b268f..9f46ce400b 100644 --- a/nvflare/fox/sys/downloader.py +++ b/nvflare/fox/sys/downloader.py @@ -11,8 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import numpy as np import torch +from nvflare.app_common.np.np_downloader import add_arrays +from nvflare.app_common.np.np_downloader import download_arrays as pull_arrays from nvflare.fox.api.ctx import Context from nvflare.fox.sys.backend import SysBackend from nvflare.fuel.f3.streaming.file_downloader import add_file @@ -32,6 +35,7 @@ class DownloadRefKey: class ObjectType: FILE = "file" TENSORS = "tensors" + ARRAYS = "arrays" class Downloader(ObjectDownloader): @@ -69,10 +73,14 @@ def add_file( rid = add_file(self, file_name, chunk_size=chunk_size, file_downloaded_cb=file_downloaded_cb, **cb_kwargs) return self._to_ref(ObjectType.FILE, rid) - def add_tensors(self, tensors: dict[str, torch.Tensor], num_tensors_per_chunk: int = 1): - rid = add_tensors(self, tensors, num_tensors_per_chunk=num_tensors_per_chunk) + def add_tensors(self, tensors: dict[str, torch.Tensor], max_chunk_size: int = 0): + rid = add_tensors(self, tensors, max_chunk_size=max_chunk_size) return self._to_ref(ObjectType.TENSORS, rid) + def add_arrays(self, arrays: dict[str, np.ndarray], max_chunk_size: int = 0): + rid = add_arrays(self, arrays, max_chunk_size=max_chunk_size) + return self._to_ref(ObjectType.ARRAYS, rid) + def download_file(ref: dict, per_request_timeout: float, ctx: Context): backend = ctx.backend @@ -110,3 +118,23 @@ def download_tensors(ref: dict, per_request_timeout: float, ctx: Context, tensor tensors_received_cb=tensors_received_cb, **cb_kwargs, ) + + +def download_arrays(ref: dict, per_request_timeout: float, ctx: Context, arrays_received_cb=None, **cb_kwargs): + backend = ctx.backend + if not isinstance(backend, SysBackend): + raise ValueError(f"backend must be SysBackend but got {type(backend)}") + + obj_type = ref.get(DownloadRefKey.OBJECT_TYPE) + if obj_type != ObjectType.ARRAYS: + raise ValueError(f"obj_type must be {ObjectType.ARRAYS} but got {obj_type}") + + return pull_arrays( + from_fqcn=ref.get(DownloadRefKey.SOURCE), + ref_id=ref.get(DownloadRefKey.REF_ID), + per_request_timeout=per_request_timeout, + cell=backend.cell, + abort_signal=ctx.abort_signal, + arrays_received_cb=arrays_received_cb, + **cb_kwargs, + ) diff --git a/nvflare/fuel/f3/streaming/cacheable.py b/nvflare/fuel/f3/streaming/cacheable.py index 2ac39a11db..50ae3867f6 100644 --- a/nvflare/fuel/f3/streaming/cacheable.py +++ b/nvflare/fuel/f3/streaming/cacheable.py @@ -17,6 +17,7 @@ from nvflare.fuel.f3.streaming.download_service import Consumer, Downloadable, DownloadService, ProduceRC from nvflare.fuel.utils.log_utils import get_obj_logger +from nvflare.fuel.utils.validation_utils import check_non_negative_int class _StateKey: @@ -26,9 +27,10 @@ class _StateKey: class CacheableObject(Downloadable): - def __init__(self, num_items_per_chunk: int): + def __init__(self, max_chunk_size: int): super().__init__() - self.num_items_per_chunk = num_items_per_chunk + check_non_negative_int("max_chunk_size", max_chunk_size) + self.max_chunk_size = max_chunk_size self.size = self.get_item_count() self.cache = [(None, 0)] * self.size self.lock = threading.Lock() @@ -59,14 +61,15 @@ def clear_cache(self): with self.lock: self.cache = None - def _get_item(self, index: int) -> bytes: + def _get_item(self, index: int, requester: str) -> bytes: with self.lock: data, _ = self.cache[index] if data is None: data = self.produce_item(index) self.cache[index] = (data, 0) + self.logger.info(f"created and cached item {index} for {requester}") else: - self.logger.info(f"got item {index} from cache") + self.logger.info(f"got item {index} from cache for {requester}") return data def _adjust_cache(self, start: int, count: int): @@ -92,18 +95,23 @@ def produce(self, state: dict, requester: str) -> Tuple[str, Any, dict]: start = received_start + received_count - end = min(start + self.num_items_per_chunk, self.size) - count = end - start - - if count <= 0: + if start >= self.size: # already done return ProduceRC.EOF, None, {} result = [] - for i in range(start, end): - item = self._get_item(i) - result.append(item) - return ProduceRC.OK, result, {_StateKey.START: start, _StateKey.COUNT: count} + total_size = 0 + + for i in range(start, self.size): + item = self._get_item(i, requester) + item_size = len(item) + if not result or total_size + item_size < self.max_chunk_size: + result.append(item) + total_size += item_size + else: + break + + return ProduceRC.OK, result, {_StateKey.START: start, _StateKey.COUNT: len(result)} class ItemConsumer(Consumer): diff --git a/nvflare/fuel/f3/streaming/obj_downloader.py b/nvflare/fuel/f3/streaming/obj_downloader.py index 7815908e78..b953565b2a 100644 --- a/nvflare/fuel/f3/streaming/obj_downloader.py +++ b/nvflare/fuel/f3/streaming/obj_downloader.py @@ -11,8 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from nvflare.fox.api.ctx import Context -from nvflare.fox.sys.backend import SysBackend from nvflare.fuel.f3.cellnet.cell import Cell from nvflare.fuel.f3.streaming.download_service import Downloadable, DownloadService From 702bb6af1b1d21ca0535912d0341509dd36309f8 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Fri, 24 Oct 2025 11:29:27 +0800 Subject: [PATCH 047/102] add config example --- nvflare/app_common/np/np_downloader.py | 8 +-- nvflare/app_opt/pt/tensor_downloader.py | 8 +-- nvflare/fox/api/app.py | 4 +- nvflare/fox/api/ctx.py | 4 +- nvflare/fox/examples/np/algos/client.py | 6 ++ .../examples/np/algos/strategies/avg_seq.py | 24 ++++++-- nvflare/fox/examples/np/fed_avg_seq.py | 17 +++++- nvflare/fox/examples/np/recipe_fed_avg_seq.py | 55 +++++++++++++++++++ nvflare/fox/sys/adaptor.py | 2 + nvflare/fuel/f3/streaming/cacheable.py | 4 +- nvflare/fuel/f3/streaming/download_service.py | 11 ++-- nvflare/fuel/f3/streaming/file_downloader.py | 6 +- 12 files changed, 114 insertions(+), 35 deletions(-) create mode 100644 nvflare/fox/examples/np/recipe_fed_avg_seq.py diff --git a/nvflare/app_common/np/np_downloader.py b/nvflare/app_common/np/np_downloader.py index cb2204cdba..8ebe5a0b5b 100644 --- a/nvflare/app_common/np/np_downloader.py +++ b/nvflare/app_common/np/np_downloader.py @@ -25,20 +25,16 @@ class ArrayDownloadable(CacheableObject): def __init__(self, arrays: dict[str, np.ndarray], max_chunk_size: int): - self.arrays = arrays self.size = len(arrays) self.keys = list(arrays.keys()) - super().__init__(max_chunk_size) - - def get_base_object(self): - return self.arrays + super().__init__(arrays, max_chunk_size) def get_item_count(self) -> int: return self.size def produce_item(self, index: int) -> Any: key = self.keys[index] - arrays_to_send = {key: self.arrays[key]} + arrays_to_send = {key: self.base_obj[key]} stream = BytesIO() np.savez(allow_pickle=False, file=stream, **arrays_to_send) return stream.getvalue() diff --git a/nvflare/app_opt/pt/tensor_downloader.py b/nvflare/app_opt/pt/tensor_downloader.py index f48099f13b..289ad6beb4 100644 --- a/nvflare/app_opt/pt/tensor_downloader.py +++ b/nvflare/app_opt/pt/tensor_downloader.py @@ -26,20 +26,16 @@ class TensorDownloadable(CacheableObject): def __init__(self, tensors: dict[str, torch.Tensor], max_chunk_size: int): - self.tensors = tensors self.size = len(tensors) self.keys = list(tensors.keys()) - super().__init__(max_chunk_size) - - def get_base_object(self): - return self.tensors + super().__init__(tensors, max_chunk_size) def get_item_count(self) -> int: return self.size def produce_item(self, index: int) -> Any: key = self.keys[index] - tensor_to_send = {key: self.tensors[key]} + tensor_to_send = {key: self.base_obj[key]} return save_tensors(tensor_to_send) diff --git a/nvflare/fox/api/app.py b/nvflare/fox/api/app.py index 5ebacbae35..1a05279dee 100644 --- a/nvflare/fox/api/app.py +++ b/nvflare/fox/api/app.py @@ -260,8 +260,8 @@ def initialize(self, context: Context): for obj in self._managed_objects.values(): self._fox_init(obj, context) - def new_context(self, caller: str, callee: str, props: dict = None, target_group=None): - return Context(self, caller, callee, self._abort_signal, props, target_group=target_group) + def new_context(self, caller: str, callee: str, target_group=None): + return Context(self, caller, callee, self._abort_signal, target_group=target_group) def register_event_handler(self, event_type: str, handler, **handler_kwargs): handlers = self._event_handlers.get(event_type) diff --git a/nvflare/fox/api/ctx.py b/nvflare/fox/api/ctx.py index e6aecded13..6aa2df6707 100644 --- a/nvflare/fox/api/ctx.py +++ b/nvflare/fox/api/ctx.py @@ -16,7 +16,7 @@ class Context: - def __init__(self, app, caller: str, callee: str, abort_signal: Signal, props: dict = None, target_group=None): + def __init__(self, app, caller: str, callee: str, abort_signal: Signal, target_group=None): if not isinstance(caller, str): raise ValueError(f"caller must be str but got {type(caller)}") @@ -29,8 +29,6 @@ def __init__(self, app, caller: str, callee: str, abort_signal: Signal, props: d self.abort_signal = abort_signal self.app = app self.props = {} - if props: - self.props.update(props) @property def backend(self): diff --git a/nvflare/fox/examples/np/algos/client.py b/nvflare/fox/examples/np/algos/client.py index 646d148441..5cb65d788a 100644 --- a/nvflare/fox/examples/np/algos/client.py +++ b/nvflare/fox/examples/np/algos/client.py @@ -25,6 +25,11 @@ def __init__(self, delta: float): ClientApp.__init__(self) self.delta = delta + def fox_init(self, context: Context): + delta_config = context.app.get_prop("client_delta", {}) + self.delta = delta_config.get(self.name, self.delta) + self.logger.info(f"client {self.name}: delta={self.delta}") + @collab def train(self, current_round, weights, context: Context): if context.is_aborted(): @@ -92,5 +97,6 @@ def __init__(self, delta): def make_client_app(self, name: str) -> ClientApp: app = NPTrainer(self.delta) + app.update_props(self.get_props()) app.name = name return app diff --git a/nvflare/fox/examples/np/algos/strategies/avg_seq.py b/nvflare/fox/examples/np/algos/strategies/avg_seq.py index 8432c907b1..cb54b56050 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_seq.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_seq.py @@ -31,6 +31,23 @@ def __init__(self, initial_model, num_rounds=10): self.initial_model = initial_model # need to remember init for job API to work! self._initial_model = parse_array_def(initial_model) self.logger = get_obj_logger(self) + self.client_weights = None + + def fox_init(self, context: Context): + weight_config = context.app.get_prop("client_weight_config", {}) + client_weights = {} + total = 0 + for c in context.clients: + w = weight_config.get(c.name, 100) + client_weights[c.name] = w + total += w + + # normalize weights + for c in context.clients: + client_weights[c.name] = client_weights[c.name] / total + + self.client_weights = client_weights + self.logger.info("client_weights: {}".format(client_weights)) def execute(self, context: Context): self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") @@ -47,8 +64,7 @@ def _do_one_round(self, r, current_model, context: Context): total = 0 n = 0 for c in context.clients: - result = c.train(r, current_model, _blocking=True, _timeout=2.0, _optional=True, _secure=True) + result = c.train(r, current_model, _blocking=True, _timeout=2.0, _optional=True, _secure=False) self.logger.info(f"[{context.header_str()}] round {r}: got result from client {c.name}: {result}") - total += result - n += 1 - return total / n + total += result * self.client_weights[c.name] + return total diff --git a/nvflare/fox/examples/np/fed_avg_seq.py b/nvflare/fox/examples/np/fed_avg_seq.py index c934c72076..2309b18d5a 100644 --- a/nvflare/fox/examples/np/fed_avg_seq.py +++ b/nvflare/fox/examples/np/fed_avg_seq.py @@ -32,12 +32,27 @@ def main(): ), ) server_app.add_collab_object("metric_receiver", MetricReceiver()) + server_app.set_prop( + "client_weight_config", + { + "site-1": 70, + "site-2": 100, + }, + ) + client_app = NPTrainerMaker(delta=1.0) + client_app.set_prop( + "client_delta", + { + "site-1": 1.0, + "site-2": 2.0, + }, + ) simulator = Simulator( root_dir="/tmp/fox", experiment_name="fedavg_seq", server_app=server_app, - client_app=NPTrainerMaker(delta=1.0), + client_app=client_app, num_clients=2, ) diff --git a/nvflare/fox/examples/np/recipe_fed_avg_seq.py b/nvflare/fox/examples/np/recipe_fed_avg_seq.py new file mode 100644 index 0000000000..8de14777d4 --- /dev/null +++ b/nvflare/fox/examples/np/recipe_fed_avg_seq.py @@ -0,0 +1,55 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from nvflare.fox.api.app import ServerApp +from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples.np.algos.client import NPTrainer +from nvflare.fox.examples.np.algos.strategies.avg_seq import NPFedAvgSequential +from nvflare.fox.examples.np.algos.widgets import MetricReceiver +from nvflare.fox.sys.recipe import FoxRecipe + +JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/fox/prod_00/admin@nvidia.com/transfer" + + +def main(): + simple_logging(logging.DEBUG) + + server_app = ServerApp( + strategy_name="fedavg", + strategy=NPFedAvgSequential(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), + ) + server_app.add_collab_object("metric_receiver", MetricReceiver()) + server_app.set_prop("client_weight_config", {"red": 70, "blue": 100, "silver": 50}) + + client_app = NPTrainer(delta=1.0) + client_app.set_prop( + "client_delta", + { + "red": 1.0, + "blue": 2.0, + "silver": 3.0, + }, + ) + + recipe = FoxRecipe( + job_name="fedavg_seq", + server_app=server_app, + client_app=client_app, + ) + recipe.export(JOB_ROOT_DIR) + + +if __name__ == "__main__": + main() diff --git a/nvflare/fox/sys/adaptor.py b/nvflare/fox/sys/adaptor.py index c035089a36..96ad8c4006 100644 --- a/nvflare/fox/sys/adaptor.py +++ b/nvflare/fox/sys/adaptor.py @@ -39,6 +39,8 @@ def __init__( self.outgoing_result_filters = outgoing_result_filters def process_config(self, app: App, fl_ctx: FLContext): + app.update_props(self.props) + engine = fl_ctx.get_engine() if self.collab_obj_ids: for cid in self.collab_obj_ids: diff --git a/nvflare/fuel/f3/streaming/cacheable.py b/nvflare/fuel/f3/streaming/cacheable.py index 50ae3867f6..63b250788b 100644 --- a/nvflare/fuel/f3/streaming/cacheable.py +++ b/nvflare/fuel/f3/streaming/cacheable.py @@ -27,8 +27,8 @@ class _StateKey: class CacheableObject(Downloadable): - def __init__(self, max_chunk_size: int): - super().__init__() + def __init__(self, obj: Any, max_chunk_size: int): + super().__init__(obj) check_non_negative_int("max_chunk_size", max_chunk_size) self.max_chunk_size = max_chunk_size self.size = self.get_item_count() diff --git a/nvflare/fuel/f3/streaming/download_service.py b/nvflare/fuel/f3/streaming/download_service.py index e145b051cf..e34c778af9 100644 --- a/nvflare/fuel/f3/streaming/download_service.py +++ b/nvflare/fuel/f3/streaming/download_service.py @@ -94,11 +94,10 @@ class Downloadable(ABC): - def set_transaction(self, tx_id, ref_id): - pass + def __init__(self, obj: Any): + self.base_obj = obj - @abstractmethod - def get_base_object(self): + def set_transaction(self, tx_id, ref_id): pass @abstractmethod @@ -267,9 +266,7 @@ def transaction_done(self, status: str): obj.transaction_done(self.tid, status) if self.transaction_done_cb: - self.transaction_done_cb( - self.tid, status, [ref.obj.get_base_object() for ref in self.refs], **self.cb_kwargs - ) + self.transaction_done_cb(self.tid, status, [ref.obj.base_obj for ref in self.refs], **self.cb_kwargs) class TransactionInfo: diff --git a/nvflare/fuel/f3/streaming/file_downloader.py b/nvflare/fuel/f3/streaming/file_downloader.py index fa829694bf..356a10099c 100644 --- a/nvflare/fuel/f3/streaming/file_downloader.py +++ b/nvflare/fuel/f3/streaming/file_downloader.py @@ -26,7 +26,7 @@ """ This package implements file downloading capability based on the ObjectDownloader framework. -It provides implementation of the Producer and Consumer objects, required by ObjDownloader. +It provides implementation of the Downloadable and Consumer objects, required by ObjDownloader. """ @@ -48,6 +48,7 @@ def __init__( Args: file_name: name of the file. """ + super().__init__(file_name) self.name = file_name self.size = os.path.getsize(file_name) @@ -62,9 +63,6 @@ def __init__( self.cb_kwargs = cb_kwargs self.logger = get_obj_logger(self) - def get_base_object(self): - return self.name - def produce(self, state: dict, requester: str) -> Tuple[str, Any, dict]: received_bytes = 0 if state: From 56f9a725b97aa00eeb3a9cad9f803db977a41f8a Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Mon, 27 Oct 2025 17:38:17 +0800 Subject: [PATCH 048/102] use downloader for decomposers --- nvflare/apis/fl_constant.py | 3 + .../decomposers/numpy_decomposers.py | 62 +-- nvflare/app_opt/pt/decomposers.py | 113 ++--- nvflare/fox/api/app.py | 17 +- nvflare/fox/examples/pt/recipe_pt_avg.py | 52 +++ nvflare/fox/sys/adaptor.py | 3 + nvflare/fox/sys/controller.py | 2 + nvflare/fox/sys/executor.py | 2 + nvflare/fox/sys/recipe.py | 2 + nvflare/fuel/f3/streaming/cacheable.py | 3 +- .../utils/fobs/decomposers/via_downloader.py | 416 ++++++++++++++++++ nvflare/fuel/utils/fobs/dots.py | 6 +- 12 files changed, 566 insertions(+), 115 deletions(-) create mode 100644 nvflare/fox/examples/pt/recipe_pt_avg.py create mode 100644 nvflare/fuel/utils/fobs/decomposers/via_downloader.py diff --git a/nvflare/apis/fl_constant.py b/nvflare/apis/fl_constant.py index b2e79ffba9..9eb01bcb6d 100644 --- a/nvflare/apis/fl_constant.py +++ b/nvflare/apis/fl_constant.py @@ -551,6 +551,9 @@ class ConfigVarName: # SJ and CJ: per-msg timeout for streaming STREAMING_PER_REQUEST_TIMEOUT = "streaming_per_request_timeout" + # SJ and CJ: chunk size for downloading + DOWNLOAD_CHUNK_SIZE = "download_chunk_size" + # SJ and CJ: min file size for streaming. If file size is less than this, it will be attached to msg directly. MIN_FILE_SIZE_FOR_STREAMING = "min_file_size_for_streaming" diff --git a/nvflare/app_common/decomposers/numpy_decomposers.py b/nvflare/app_common/decomposers/numpy_decomposers.py index 10a1528ce4..8d4e7b85c4 100644 --- a/nvflare/app_common/decomposers/numpy_decomposers.py +++ b/nvflare/app_common/decomposers/numpy_decomposers.py @@ -15,16 +15,17 @@ import os from abc import ABC from io import BytesIO -from typing import Any +from typing import Any, Tuple import numpy as np import nvflare.fuel.utils.fobs.dots as dots +from nvflare.app_common.np.np_downloader import ArrayDownloadable, download_arrays +from nvflare.fuel.f3.cellnet.cell import Cell +from nvflare.fuel.f3.streaming.download_service import Downloadable from nvflare.fuel.utils import fobs from nvflare.fuel.utils.fobs.datum import DatumManager -from nvflare.fuel.utils.fobs.decomposers.via_file import ViaFileDecomposer - -_NPZ_EXTENSION = ".npz" +from nvflare.fuel.utils.fobs.decomposers.via_downloader import ViaDownloaderDecomposer class NumpyScalarDecomposer(fobs.Decomposer, ABC): @@ -57,39 +58,40 @@ def supported_type(self): return np.int32 -class NumpyArrayDecomposer(ViaFileDecomposer): +class NumpyArrayDecomposer(ViaDownloaderDecomposer): def __init__(self): # by default do not use file downloading. - ViaFileDecomposer.__init__(self, 0, "np_") + ViaDownloaderDecomposer.__init__(self, 1, "np_") def supported_type(self): return np.ndarray - def get_bytes_dot(self) -> int: - return dots.NUMPY_BYTES - - def get_file_dot(self) -> int: - return dots.NUMPY_FILE - - def dump_to_file(self, items: dict, path: str, fobs_ctx: dict): - if not path.endswith(_NPZ_EXTENSION): - path += _NPZ_EXTENSION - self.logger.info(f"NP: dumping {len(items)} arrays to file {path}") - try: - np.savez(allow_pickle=False, file=path, **items) - return path, None - except Exception as e: - self.logger.error(f"exception dumping NP to file: {e}") - raise e - - def load_from_file(self, path: str, fobs_ctx: dict, meta: dict = None) -> Any: - result = {} - with np.load(path, allow_pickle=False) as npz_obj: - for k in npz_obj.files: - result[k] = npz_obj[k] - self.logger.info(f"loaded {len(result)} array(s) from file {path}") - return result + def get_download_dot(self) -> int: + return dots.NUMPY_DOWNLOAD + + def to_downloadable(self, items: dict, max_chunk_size: int, fobs_ctx: dict) -> Downloadable: + return ArrayDownloadable(items, max_chunk_size) + + def download( + self, + from_fqcn: str, + ref_id: str, + per_request_timeout: float, + cell: Cell, + secure=False, + optional=False, + abort_signal=None, + ) -> Tuple[str, dict]: + return download_arrays( + from_fqcn, + ref_id, + per_request_timeout, + cell, + secure, + optional, + abort_signal, + ) def native_decompose(self, target: np.ndarray, manager: DatumManager = None) -> bytes: stream = BytesIO() diff --git a/nvflare/app_opt/pt/decomposers.py b/nvflare/app_opt/pt/decomposers.py index a800d2c00d..4ea6b57b46 100644 --- a/nvflare/app_opt/pt/decomposers.py +++ b/nvflare/app_opt/pt/decomposers.py @@ -12,16 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Optional +from typing import Any, Optional, Tuple import torch -from safetensors.torch import _remove_duplicate_names, load, load_file, save, save_file +from safetensors.torch import load, save import nvflare.fuel.utils.fobs.dots as dots +from nvflare.fuel.f3.streaming.download_service import Downloadable from nvflare.fuel.utils.fobs.datum import DatumManager -from nvflare.fuel.utils.fobs.decomposers.via_file import ViaFileDecomposer +from nvflare.fuel.utils.fobs.decomposers.via_downloader import ViaDownloaderDecomposer -MIN_SIZE_FOR_FILE = 0 # The default value +from ...fuel.f3.cellnet.cell import Cell +from .tensor_downloader import TensorDownloadable, download_tensors class SerializationModule(torch.nn.Module): @@ -30,88 +32,39 @@ def __init__(self, tensor): self.register_buffer("saved_tensor", tensor) -def _safe_save(state_dict, filename: str) -> Optional[dict]: - """Save model weights with the safetensors format. - The model weights may contain tensors with shared memory. In this case, save_file won't work. - We first try to find and remove such tensors, and then save the remaining tensors with save_file. - We then return the information about the removed tensors as a dict. - The key of the dict is the name of the tensor kept in the weights. - The value is a list of tensor names that are to be substituted by the kept tensor. - - For example, the state_dict contains multiple tensors: - - { - "t1": t1, - "t2": t2, - "t3": t3, - "t4": t4 - } - - Suppose tensors t1, t2 and t3 are shared, the state_dict after removing shared tensors will look like this: - - { - "t1": t1, - "t4": t4 - } - - And the removed tensors dict looks like this: - { - "t1": ["t2", "t3"] - } - - Args: - state_dict: the model weights to be saved - filename: name of the file - - Returns: a dict that contains removed tensor info - - """ - to_removes = _remove_duplicate_names(state_dict) - for kept_name, to_remove_group in to_removes.items(): - for to_remove in to_remove_group: - del state_dict[to_remove] - state_dict = {k: v.contiguous() for k, v in state_dict.items()} - save_file(state_dict, filename) - if to_removes: - # to_removes is dict-like but not a simple dict - return {k: v for k, v in to_removes.items()} - else: - return None - - -class TensorDecomposer(ViaFileDecomposer): +class TensorDecomposer(ViaDownloaderDecomposer): def __init__(self): - ViaFileDecomposer.__init__(self, MIN_SIZE_FOR_FILE, "tensor_") + ViaDownloaderDecomposer.__init__(self, 1024 * 1024 * 2, "tensor_") def supported_type(self): return torch.Tensor - def dump_to_file(self, items: dict, path: str, fobs_ctx: dict): - try: - meta = _safe_save(items, path) - removed = len(meta) if meta else 0 - self.logger.info(f"Saving {len(items)} tensors to file {path}: Number of duplicate removed: {removed}") - return path, meta - except Exception as e: - self.logger.error(f"exception saving tensors to file: {e}") - raise e - - def load_from_file(self, path: str, fobs_ctx: dict, meta: dict = None) -> Any: - items = load_file(path) - self.logger.info(f"loaded {len(items)} tensor(s) from file {path}") - if meta: - # the meta keeps names of removed tensors and the name of the tensor for them - for kept, removed_group in meta.items(): - for r in removed_group: - items[r] = items[kept] - return items - - def get_bytes_dot(self) -> int: - return dots.TENSOR_BYTES - - def get_file_dot(self) -> int: - return dots.TENSOR_FILE + def get_download_dot(self) -> int: + return dots.TENSOR_DOWNLOAD + + def to_downloadable(self, items: dict, max_chunk_size: int, fobs_ctx: dict) -> Downloadable: + return TensorDownloadable(items, max_chunk_size) + + def download( + self, + from_fqcn: str, + ref_id: str, + per_request_timeout: float, + cell: Cell, + secure=False, + optional=False, + abort_signal=None, + ) -> Tuple[str, dict]: + return download_tensors( + from_fqcn, + ref_id, + per_request_timeout, + cell, + secure, + optional, + abort_signal, + ) def native_decompose(self, target: torch.Tensor, manager: DatumManager = None) -> bytes: # save the tensor to bytes using safetensors diff --git a/nvflare/fox/api/app.py b/nvflare/fox/api/app.py index 1a05279dee..dc1d7052aa 100644 --- a/nvflare/fox/api/app.py +++ b/nvflare/fox/api/app.py @@ -13,6 +13,7 @@ # limitations under the License. import copy import fnmatch +import os from typing import List from nvflare.fuel.utils.log_utils import get_obj_logger @@ -48,9 +49,21 @@ def __init__(self): self._outgoing_result_filter_chains = [] self._collab_interface = {"": get_object_collab_interface(self)} self._workspace = None + self._resource_dirs = {} self._managed_objects = {} # id => obj self.logger = get_obj_logger(self) + def set_resource_dirs(self, resource_dirs: dict[str, str]): + if not isinstance(resource_dirs, dict): + raise TypeError(f"resource_dirs must be a dict but got {type(resource_dirs)}") + for name, resource_dir in resource_dirs.items(): + if not os.path.isdir(resource_dir): + raise ValueError(f"Resource dir {resource_dir} does not exist for {name}") + self._resource_dirs = resource_dirs + + def get_resource_dirs(self): + return self._resource_dirs + def _add_managed_object(self, obj): self._managed_objects[id(obj)] = obj @@ -195,6 +208,8 @@ def get_collab_objects(self): def setup(self, workspace: Workspace, server: Proxy, clients: List[Proxy], abort_signal): self._workspace = workspace + workspace.resource_dirs = self._resource_dirs + self.server = server self._abort_signal = abort_signal @@ -212,8 +227,6 @@ def setup(self, workspace: Workspace, server: Proxy, clients: List[Proxy], abort raise ValueError(f"cannot find site for {self.name}") forest = build_forest(objs=clients, get_fqn_f=lambda c: c.fqn, get_name_f=lambda c: c.name) - # d = forest_to_dict(forest, lambda c: c.name) - # print(f"client proxy forest: {d}") self.client_hierarchy = forest def get_my_site(self) -> Proxy: diff --git a/nvflare/fox/examples/pt/recipe_pt_avg.py b/nvflare/fox/examples/pt/recipe_pt_avg.py new file mode 100644 index 0000000000..798ffd21c7 --- /dev/null +++ b/nvflare/fox/examples/pt/recipe_pt_avg.py @@ -0,0 +1,52 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from nvflare.app_opt.pt.decomposers import TensorDecomposer +from nvflare.fox.api.app import ServerApp +from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples.pt.pt_avg_filter import PTFedAvg, PTTrainer +from nvflare.fox.sys.recipe import FoxRecipe + +JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/fox/prod_00/admin@nvidia.com/transfer" + + +def main(): + simple_logging(logging.DEBUG) + + server_app = ServerApp( + strategy_name="fedavg", + strategy=PTFedAvg( + initial_model={ + "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "y": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "z": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + }, + num_rounds=2, + ), + ) + + client_app = PTTrainer(delta=1.0) + + recipe = FoxRecipe( + job_name="pt_fedavg", + server_app=server_app, + client_app=client_app, + ) + recipe.add_decomposers([TensorDecomposer()]) + recipe.export(JOB_ROOT_DIR) + + +if __name__ == "__main__": + main() diff --git a/nvflare/fox/sys/adaptor.py b/nvflare/fox/sys/adaptor.py index 96ad8c4006..2a7d684b46 100644 --- a/nvflare/fox/sys/adaptor.py +++ b/nvflare/fox/sys/adaptor.py @@ -24,6 +24,7 @@ def __init__( self, collab_obj_ids: List[str] = None, props: Dict[str, Any] = None, + resource_dirs: Dict[str, str] = None, incoming_call_filters=None, outgoing_call_filters=None, incoming_result_filters=None, @@ -32,6 +33,7 @@ def __init__( if not collab_obj_ids: collab_obj_ids = [] self.props = props + self.resource_dirs = resource_dirs self.collab_obj_ids = collab_obj_ids self.incoming_call_filters = incoming_call_filters self.outgoing_call_filters = outgoing_call_filters @@ -40,6 +42,7 @@ def __init__( def process_config(self, app: App, fl_ctx: FLContext): app.update_props(self.props) + app.set_resource_dirs(self.resource_dirs) engine = fl_ctx.get_engine() if self.collab_obj_ids: diff --git a/nvflare/fox/sys/controller.py b/nvflare/fox/sys/controller.py index 23c5258d7f..5b670e160c 100644 --- a/nvflare/fox/sys/controller.py +++ b/nvflare/fox/sys/controller.py @@ -58,6 +58,7 @@ def __init__( incoming_result_filters=None, outgoing_result_filters=None, props=None, + resource_dirs=None, sync_task_timeout=5, max_call_threads=100, ): @@ -65,6 +66,7 @@ def __init__( FoxAdaptor.__init__( self, props=props, + resource_dirs=resource_dirs, collab_obj_ids=collab_obj_ids, incoming_call_filters=incoming_call_filters, outgoing_call_filters=outgoing_call_filters, diff --git a/nvflare/fox/sys/executor.py b/nvflare/fox/sys/executor.py index af699b44bb..38cf1ad286 100644 --- a/nvflare/fox/sys/executor.py +++ b/nvflare/fox/sys/executor.py @@ -45,6 +45,7 @@ def __init__( incoming_result_filters=None, outgoing_result_filters=None, props: Dict[str, Any] = None, + resource_dirs: Dict[str, str] = None, max_call_threads=100, ): Executor.__init__(self) @@ -52,6 +53,7 @@ def __init__( self, collab_obj_ids=collab_obj_ids, props=props, + resource_dirs=resource_dirs, incoming_call_filters=incoming_call_filters, outgoing_call_filters=outgoing_call_filters, incoming_result_filters=incoming_result_filters, diff --git a/nvflare/fox/sys/recipe.py b/nvflare/fox/sys/recipe.py index bc2853c6fd..fa2fc88ca8 100644 --- a/nvflare/fox/sys/recipe.py +++ b/nvflare/fox/sys/recipe.py @@ -85,6 +85,7 @@ def _create_job(self) -> FedJob: sync_task_timeout=self.sync_task_timeout, max_call_threads=self.max_call_threads_for_server, props=self.server_app.get_props(), + resource_dirs=self.server_app.get_resource_dirs(), ) job.to_server(controller, id="controller") @@ -103,6 +104,7 @@ def _create_job(self) -> FedJob: outgoing_result_filters=c_out_rf_arg, max_call_threads=self.max_call_threads_for_client, props=self.client_app.get_props(), + resource_dirs=self.client_app.get_resource_dirs(), ) job.to_clients(executor, id="executor", tasks=["*"]) return job diff --git a/nvflare/fuel/f3/streaming/cacheable.py b/nvflare/fuel/f3/streaming/cacheable.py index 63b250788b..3cf007046f 100644 --- a/nvflare/fuel/f3/streaming/cacheable.py +++ b/nvflare/fuel/f3/streaming/cacheable.py @@ -67,7 +67,7 @@ def _get_item(self, index: int, requester: str) -> bytes: if data is None: data = self.produce_item(index) self.cache[index] = (data, 0) - self.logger.info(f"created and cached item {index} for {requester}") + self.logger.info(f"created and cached item {index} for {requester}: {len(data)} bytes") else: self.logger.info(f"got item {index} from cache for {requester}") return data @@ -111,6 +111,7 @@ def produce(self, state: dict, requester: str) -> Tuple[str, Any, dict]: else: break + self.logger.info(f"produced {len(result)} items for {requester}: {total_size} bytes") return ProduceRC.OK, result, {_StateKey.START: start, _StateKey.COUNT: len(result)} diff --git a/nvflare/fuel/utils/fobs/decomposers/via_downloader.py b/nvflare/fuel/utils/fobs/decomposers/via_downloader.py new file mode 100644 index 0000000000..97588bba6d --- /dev/null +++ b/nvflare/fuel/utils/fobs/decomposers/via_downloader.py @@ -0,0 +1,416 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +import threading +import uuid +from abc import ABC, abstractmethod +from typing import Any, Tuple + +import nvflare.fuel.utils.app_config_utils as acu +from nvflare.apis.fl_constant import ConfigVarName +from nvflare.fuel.f3.cellnet.cell import Cell +from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey +from nvflare.fuel.f3.streaming.download_service import Downloadable +from nvflare.fuel.f3.streaming.file_downloader import ObjectDownloader +from nvflare.fuel.utils import fobs +from nvflare.fuel.utils.fobs.datum import Datum, DatumManager, DatumType +from nvflare.fuel.utils.log_utils import get_obj_logger +from nvflare.fuel.utils.msg_root_utils import subscribe_to_msg_root + +_MIN_DOWNLOAD_TIMEOUT = 60 # allow at least 1 minute gap between download activities + + +class EncKey: + TYPE = "type" + DATA = "data" + + +class EncType: + NATIVE = "native" + REF = "ref" + + +class _RefKey: + FILE_REF_ID = "file_ref_id" + FQCN = "fqcn" + + +class _CtxKey: + MSG_ROOT_ID = "msg_root_id" + MSG_ROOT_TTL = "msg_root_ttl" + OBJECTS = "objects" # objects to be downloaded + FINAL_CB_REGISTERED = "final_cb_registered" + + +class _DecomposeCtx: + + def __init__(self): + self.target_to_item = {} # target_id => item_id + self.target_items = {} # item_id => item value + self.last_item_id = 0 + + def add_item(self, item: Any): + target_id = id(item) + item_id = self.target_to_item.get(target_id) + if not item_id: + item_id = f"T{self.last_item_id}" + self.last_item_id += 1 + self.target_items[item_id] = item + self.target_to_item[target_id] = item_id + return item_id, target_id + + def get_item_count(self): + return len(self.target_items) + + +class ViaDownloaderDecomposer(fobs.Decomposer, ABC): + + def __init__(self, max_chunk_size: int, config_var_prefix): + self.logger = get_obj_logger(self) + self.prefix = self.__class__.__name__ + self.decompose_ctx_key = f"{self.prefix}_dc" # kept in fobs_ctx: each target type has its own DecomposeCtx + self.items_key = f"{self.prefix}_items" # in fobs_ctx: each target type has its own set of items + self.config_var_prefix = config_var_prefix + self.max_chunk_size = max_chunk_size + + @abstractmethod + def to_downloadable(self, items: dict, max_chunk_size: int, fobs_ctx: dict) -> Downloadable: + """Dump the items to the file with the specified path + + Args: + items: a dict of items of target object type to be dumped to file + max_chunk_size: max size of one chunk. + fobs_ctx: FOBS Context + + Returns: a Downloadable object + + The "items" is a dict of target objects. The dict contains all objects of the target type in one payload. + + """ + pass + + @abstractmethod + def download( + self, + from_fqcn: str, + ref_id: str, + per_request_timeout: float, + cell: Cell, + secure=False, + optional=False, + abort_signal=None, + ) -> Tuple[str, dict]: + pass + + def supported_dots(self): + return [self.get_download_dot()] + + @abstractmethod + def get_download_dot(self) -> int: + """Get the Datum Object Type to be used for download ref datum + + Returns: the DOT for download ref datum + + """ + pass + + @abstractmethod + def native_decompose(self, target: Any, manager: DatumManager = None) -> bytes: + pass + + @abstractmethod + def native_recompose(self, data: bytes, manager: DatumManager = None) -> Any: + pass + + def _create_ref(self, target: Any, manager: DatumManager, fobs_ctx: dict): + # create a reference item for the target object. The ref item represents the target object in + # the serialized payload. + dc = fobs_ctx.get(self.decompose_ctx_key) + item_id, target_id = dc.add_item(target) + if dc.get_item_count() == 1: + # register the post_process callback to further process these items. + # only register cb once! + manager.register_post_cb(self._process_items_to_datum) + return item_id, target_id + + def _create_downloadable(self, fobs_ctx: dict) -> Downloadable: + dc = fobs_ctx.get(self.decompose_ctx_key) + assert isinstance(dc, _DecomposeCtx) + items = dc.target_items + max_chunk_size = acu.get_int_var( + self._config_var_name(ConfigVarName.DOWNLOAD_CHUNK_SIZE), + self.max_chunk_size, + ) + try: + return self.to_downloadable(items, max_chunk_size, fobs_ctx) + except Exception as e: + self.logger.error(f"Error converting {len(items)} items to Downloadable: {e}") + raise e + + @staticmethod + def _determine_msg_root(fobs_ctx: dict): + msg_root_id = fobs_ctx.get(_CtxKey.MSG_ROOT_ID) + msg_root_ttl = fobs_ctx.get(_CtxKey.MSG_ROOT_TTL) + + if not msg_root_id: + # try to get from msg + msg = fobs_ctx.get(fobs.FOBSContextKey.MESSAGE) + if msg: + msg_root_id = msg.get_header(MessageHeaderKey.MSG_ROOT_ID) + msg_root_ttl = msg.get_header(MessageHeaderKey.MSG_ROOT_TTL) + return msg_root_id, msg_root_ttl + + def decompose(self, target: Any, manager: DatumManager = None) -> Any: + if not manager: + # this should never happen + raise RuntimeError("FOBS System Error: missing DatumManager") + + max_chunk_size = acu.get_int_var( + self._config_var_name(ConfigVarName.DOWNLOAD_CHUNK_SIZE), + self.max_chunk_size, + ) + if max_chunk_size <= 0: + # use native decompose + self.logger.info("using native_decompose") + data = self.native_decompose(target, manager) + return {EncKey.TYPE: EncType.NATIVE, EncKey.DATA: data} + else: + self.logger.info(f"using download decompose: {max_chunk_size=}") + + fobs_ctx = manager.fobs_ctx + + # Create a DecomposeCtx for this target type. + # Note: there could be multiple target types - each target type has its own DecomposeCtx! + dc = fobs_ctx.get(self.decompose_ctx_key) + if not dc: + dc = _DecomposeCtx() + fobs_ctx[self.decompose_ctx_key] = dc + + item_id, target_id = self._create_ref(target, manager, fobs_ctx) + self.logger.info(f"ViaDownloader: created ref for target {target_id}: {item_id}") + return {EncKey.TYPE: EncType.REF, EncKey.DATA: item_id} + + def _create_downloader(self, fobs_ctx: dict): + msg_root_id, msg_root_ttl = self._determine_msg_root(fobs_ctx) + + if msg_root_ttl: + timeout = msg_root_ttl + else: + timeout = _MIN_DOWNLOAD_TIMEOUT + + if timeout < _MIN_DOWNLOAD_TIMEOUT: + timeout = _MIN_DOWNLOAD_TIMEOUT + + self.logger.debug(f"ViaDownloader: {msg_root_id=} {timeout=}") + + downloader = None + cell = fobs_ctx.get(fobs.FOBSContextKey.CELL) + if cell: + downloader = ObjectDownloader( + num_receivers=1, + cell=cell, + timeout=timeout, + ) + + if msg_root_id: + subscribe_to_msg_root( + msg_root_id=msg_root_id, + cb=self._delete_download_tx_on_msg_root, + downloader=downloader, + ) + + return downloader + + def _process_items_to_datum(self, mgr: DatumManager): + """This method is called during serialization after all target items are serialized. + For primary msg, we turn the collected items into a file, and add file info as a Datum to the datum manager. + + Args: + mgr: + + Returns: + + """ + fobs_ctx = mgr.fobs_ctx + dc = fobs_ctx.get(self.decompose_ctx_key) + assert isinstance(dc, _DecomposeCtx) + + # create datum for the collected target items + # This is called once for each target object type! + + # register the final CB to be called after the post_process. + # Note that the post_process (this CB) only generates files but does not create download transaction. + # For large object, file generation could take long time. If we create the download transaction, it may + # become expired even before file generation is done! + # This is why we do the file generation in this CB, and then create the transaction in the final_cb! + final_cb_registered = fobs_ctx.get(_CtxKey.FINAL_CB_REGISTERED) + if not final_cb_registered: + # register final_cb + mgr.register_post_cb(self._finalize_download_tx) + fobs_ctx[_CtxKey.FINAL_CB_REGISTERED] = True + + try: + if not mgr.get_error(): + datum = self._create_datum(fobs_ctx) + mgr.add_datum(datum) + except Exception as ex: + self.logger.error(f"exception creating datum: {ex}") + mgr.set_error(f"exception creating datum in {type(self)}") + + def _config_var_name(self, base_name: str): + return f"{self.config_var_prefix}{base_name}" + + def _create_datum(self, fobs_ctx: dict): + downloadable = self._create_downloadable(fobs_ctx) + cell = fobs_ctx.get(fobs.FOBSContextKey.CELL) + + # use download DOT + # keep files in fobs_ctx + downloadable_objs = fobs_ctx.get(_CtxKey.OBJECTS) + if not downloadable_objs: + downloadable_objs = [] + fobs_ctx[_CtxKey.OBJECTS] = downloadable_objs + + # create a new ref id + ref_id = str(uuid.uuid4()) + downloadable_objs.append((ref_id, downloadable)) + + ref = { + _RefKey.FQCN: cell.get_fqcn(), + _RefKey.FILE_REF_ID: ref_id, + } + self.logger.info(f"ViaDownloader: created download ref for target type {self.__class__.__name__}: {ref=}") + datum = Datum(datum_type=DatumType.TEXT, value=json.dumps(ref), dot=self.get_download_dot()) + return datum + + def _finalize_download_tx(self, mgr: DatumManager): + self.logger.info("ViaDownloader: finalizing download tx") + fobs_ctx = mgr.fobs_ctx + downloadable_objs = fobs_ctx.get(_CtxKey.OBJECTS) + + if downloadable_objs: + downloader = self._create_downloader(fobs_ctx) + for ref_id, obj in downloadable_objs: + self.logger.debug("ViaDownloader: adding object to downloader: {ref_id=}") + downloader.add_object(obj, ref_id=ref_id) + + def _delete_download_tx_on_msg_root(self, msg_root_id: str, downloader: ObjectDownloader): + # this CB is triggered when msg root is deleted. + self.logger.info(f"ViaDownloader: deleting download transaction associated with {msg_root_id=}") + downloader.delete_transaction() + + def process_datum(self, datum: Datum, manager: DatumManager): + """This is called by the manager to process a datum that has a DOT. + This happens before the recompose processing. + + The datum contains information about where the data is: + For bytes DOT, the data is included in the datum directly. + For file DOT, the data is in a file, and the location of the file is further specified: + - If the location is local, then the file is on local file system; + - If the location is remote_cell, then the file is on a remote cell, and needs to be downloaded. + + Args: + datum: datum to be processed. + manager: the datum manager. + + Returns: None + + """ + self.logger.info(f"ViaDownloader: pre-processing datum {datum.dot=} before recompose") + fobs_ctx = manager.fobs_ctx + + # data is to be downloaded + ref = json.loads(datum.value) + items = self._download_from_remote_cell(manager.fobs_ctx, ref) + fobs_ctx[self.items_key] = items + + def recompose(self, data: Any, manager: DatumManager = None) -> Any: + if not manager: + # should never happen! + raise RuntimeError("missing DatumManager") + + if not isinstance(data, dict): + self.logger.error(f"data to be recomposed should be dict but got {type(data)}") + raise RuntimeError("FOBS protocol error") + + enc_type = data.get(EncKey.TYPE) + data = data.get(EncKey.DATA) + if not data: + self.logger.error("missing 'data' property from the recompose data") + raise RuntimeError("FOBS protocol error") + + if enc_type == EncType.NATIVE: + self.logger.info("using native_recompose") + return self.native_recompose(data, manager) + elif enc_type != EncType.REF: + self.logger.error(f"invalid enc_type {enc_type} in recompose data") + raise RuntimeError("FOBS protocol error") + + if not isinstance(data, str): + self.logger.error(f"ref data must be str but got {type(data)}") + raise RuntimeError("FOBS protocol error") + + # data is the item id + tid = threading.get_ident() + self.logger.info(f"ViaDownloader: {tid=} recomposing data item {data}") + item_id = data + fobs_ctx = manager.fobs_ctx + items = fobs_ctx.get(self.items_key) + self.logger.info(f"trying to get item for {item_id=} from {type(items)=}") + item = items.get(item_id) + self.logger.info(f"{tid=} found item {item_id}: {type(item)}") + if item is None: + self.logger.error(f"cannot find item {item_id} from loaded data") + return item + + def _download_from_remote_cell(self, fobs_ctx: dict, ref: dict): + self.logger.info(f"trying to download from remote cell for {ref=}") + cell = fobs_ctx.get(fobs.FOBSContextKey.CELL) + if not cell: + self.logger.error("cannot download from remote cell since cell not available in fobs context") + raise RuntimeError("FOBS Protocol Error") + + ref_id = ref.get(_RefKey.FILE_REF_ID) + if not ref_id: + self.logger.error(f"missing {_RefKey.FILE_REF_ID} from {ref}") + raise RuntimeError("FOBS Protocol Error") + + fqcn = ref.get(_RefKey.FQCN) + if not fqcn: + self.logger.error(f"missing {_RefKey.FQCN} from {ref}") + raise RuntimeError("FOBS Protocol Error") + + req_timeout = fobs_ctx.get(fobs.FOBSContextKey.DOWNLOAD_REQ_TIMEOUT, None) + if not req_timeout: + req_timeout = acu.get_positive_float_var( + self._config_var_name(ConfigVarName.STREAMING_PER_REQUEST_TIMEOUT), 10.0 + ) + self.logger.info(f"DOWNLOAD_REQ_TIMEOUT={req_timeout}") + + abort_signal = fobs_ctx.get(fobs.FOBSContextKey.ABORT_SIGNAL) + + self.logger.info(f"trying to download: {ref_id=} {fqcn=}") + err, items = self.download( + from_fqcn=fqcn, + ref_id=ref_id, + per_request_timeout=req_timeout, + cell=cell, + abort_signal=abort_signal, + ) + if err: + self.logger.error(f"failed to download from {fqcn} for source {ref}: {err}") + raise RuntimeError(f"failed to download from {fqcn}") + else: + self.logger.info(f"downloaded {len(items)} items successfully") + return items diff --git a/nvflare/fuel/utils/fobs/dots.py b/nvflare/fuel/utils/fobs/dots.py index 8dde5f0c66..bdf5308c2a 100644 --- a/nvflare/fuel/utils/fobs/dots.py +++ b/nvflare/fuel/utils/fobs/dots.py @@ -18,5 +18,7 @@ """ NUMPY_BYTES = 1 NUMPY_FILE = 2 -TENSOR_BYTES = 3 -TENSOR_FILE = 4 +NUMPY_DOWNLOAD = 3 +TENSOR_BYTES = 4 +TENSOR_FILE = 5 +TENSOR_DOWNLOAD = 6 From 1365ff74fc4819ea815448745f8620013cb1bd96 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Wed, 29 Oct 2025 11:10:03 +0800 Subject: [PATCH 049/102] adjust naming --- .../decomposers/numpy_decomposers.py | 2 +- nvflare/fox/api/backend.py | 33 +++- nvflare/fox/api/{resp.py => gcc.py} | 31 +++- nvflare/fox/api/group.py | 127 ++++++++++++-- nvflare/fox/examples/pt/pt_np.py | 157 ++++++++++++++++++ nvflare/fox/examples/pt/recipe_pt_np.py | 56 +++++++ nvflare/fox/sim/backend.py | 14 +- nvflare/fox/sim/sim2.py | 53 ++++++ nvflare/fox/sim/simulator.py | 3 +- nvflare/fox/sys/backend.py | 7 +- 10 files changed, 459 insertions(+), 24 deletions(-) rename nvflare/fox/api/{resp.py => gcc.py} (63%) create mode 100644 nvflare/fox/examples/pt/pt_np.py create mode 100644 nvflare/fox/examples/pt/recipe_pt_np.py create mode 100644 nvflare/fox/sim/sim2.py diff --git a/nvflare/app_common/decomposers/numpy_decomposers.py b/nvflare/app_common/decomposers/numpy_decomposers.py index 8d4e7b85c4..0af2129fbd 100644 --- a/nvflare/app_common/decomposers/numpy_decomposers.py +++ b/nvflare/app_common/decomposers/numpy_decomposers.py @@ -62,7 +62,7 @@ class NumpyArrayDecomposer(ViaDownloaderDecomposer): def __init__(self): # by default do not use file downloading. - ViaDownloaderDecomposer.__init__(self, 1, "np_") + ViaDownloaderDecomposer.__init__(self, 1024 * 1024 * 2, "np_") def supported_type(self): return np.ndarray diff --git a/nvflare/fox/api/backend.py b/nvflare/fox/api/backend.py index ea67fb7a95..4de58b7f29 100644 --- a/nvflare/fox/api/backend.py +++ b/nvflare/fox/api/backend.py @@ -15,18 +15,47 @@ from nvflare.apis.signal import Signal -from .resp import Resp +from .gcc import GroupCallContext class Backend(ABC): + """A FOX Backend implements remote object calls. This interface defines the required methods that a Backend + must implement. + """ def __init__(self, abort_signal: Signal): self.abort_signal = abort_signal @abstractmethod def call_target(self, target_name: str, func_name: str, *args, **kwargs): + """ + Call a target function with arguments and return a result. + + Args: + target_name: the fully qualified name of the target object to be called in the remote app. + func_name: name of the function to be called in the remote app. + *args: args to pass to the target function. + **kwargs: kwargs to pass to the target function. + + Notes: the target name is fully qualified: . + + Returns: + + """ pass @abstractmethod - def call_target_with_resp(self, resp: Resp, target_name: str, func_name: str, *args, **kwargs): + def call_target_in_group(self, gcc: GroupCallContext, target_name: str, func_name: str, *args, **kwargs): + """Call a remote object as part of a group. + + Args: + gcc: contextual information about group call. + target_name: fully qualified name of the target object to be called in the remote app. + func_name: mame of the function to be called in the remote app. + *args: args to pass to the target function. + **kwargs: kwargs to pass to the target function. + + Returns: + + """ pass diff --git a/nvflare/fox/api/resp.py b/nvflare/fox/api/gcc.py similarity index 63% rename from nvflare/fox/api/resp.py rename to nvflare/fox/api/gcc.py index 9dfeef0709..2810f8269f 100644 --- a/nvflare/fox/api/resp.py +++ b/nvflare/fox/api/gcc.py @@ -19,9 +19,19 @@ from .utils import check_context_support -class Resp: +class GroupCallContext: def __init__(self, app, target_name, func_name, process_cb, cb_kwargs, context: Context): + """GroupCallContext contains contextual information about a group call to a target.. + + Args: + app: the calling app. + target_name: name of the target to be called in the remote app. + func_name: name of the function to be called in the remote app. + process_cb: the callback function to be called to process response from the remote app. + cb_kwargs: kwargs passed to the callback function. + context: call context. + """ self.app = app self.target_name = target_name self.func_name = func_name @@ -32,8 +42,18 @@ def __init__(self, app, target_name, func_name, process_cb, cb_kwargs, context: self.context = context def set_result(self, result): + """This is called by the backend to set the result received from the remote app. + If process_cb is available, it will be called with the result from the remote app. + + Args: + result: the result received from the remote app. + + Returns: None + + """ # filter incoming result ctx = copy.copy(self.context) + # swap caller/callee original_caller = ctx.caller ctx.caller = ctx.callee @@ -50,5 +70,14 @@ def set_result(self, result): self.resp_time = time.time() def set_exception(self, ex): + """This is called by the backend to set the exception received from the remote app. + The process_cb will NOT be called. + + Args: + ex: the exception received from the remote app. + + Returns: + + """ self.result = ex self.resp_time = time.time() diff --git a/nvflare/fox/api/group.py b/nvflare/fox/api/group.py index a1822b0134..2a27830099 100644 --- a/nvflare/fox/api/group.py +++ b/nvflare/fox/api/group.py @@ -22,8 +22,8 @@ from .app import App from .constants import CollabMethodArgName, CollabMethodOptionName from .ctx import Context +from .gcc import GroupCallContext from .proxy import Proxy -from .resp import Resp from .utils import check_call_args @@ -43,6 +43,21 @@ def __init__( process_resp_cb=None, **cb_kwargs, ): + """A Group is a group of remote apps to be called. + + Args: + app: the calling app. + abort_signal: signal to abort execution. + proxies: proxies of the remote apps to be called. + blocking: whether to block until responses are received from all remote apps. + timeout: how long to wait for responses. + optional: whether the call is optional or not. + secure: whether the call is secure or not. + min_resps: minimum number of responses expected. + wait_after_min_resps: how much longer to wait after min_resps are received. + process_resp_cb: callback function to be called to process responses from remote apps. + **cb_kwargs: kwargs passed to process_resp_cb. + """ if not proxies: raise ValueError("no proxies to group") @@ -67,10 +82,21 @@ def __init__( @property def size(self): + """Size of the group, which is the number of remote apps to be called. + + Returns: size of the group. + + """ return len(self._proxies) @property def members(self): + """ + Returns the members of the group, which is the list of all remote apps to be called. + + Returns: the members of the group + + """ return self._proxies def __getattr__(self, func_name): @@ -79,7 +105,7 @@ def __getattr__(self, func_name): """ def method(*args, **kwargs): - resps = {} + gccs = {} # filter once for all targets p = self._proxies[0] @@ -107,7 +133,7 @@ def method(*args, **kwargs): call_kwargs[CollabMethodOptionName.SECURE] = self._secure call_kwargs[CollabMethodOptionName.OPTIONAL] = self._optional - resp = Resp( + gcc = GroupCallContext( self._app, p.target_name, func_name, @@ -115,12 +141,12 @@ def method(*args, **kwargs): self._cb_kwargs, ctx, ) - resps[p.name] = resp + gccs[p.name] = gcc self._logger.debug( f"[{ctx.header_str()}] group call: {func_name=} args={call_args} kwargs={call_kwargs}" ) - the_proxy.backend.call_target_with_resp(resp, the_proxy.name, func_name, *call_args, **call_kwargs) + the_proxy.backend.call_target_in_group(gcc, the_proxy.name, func_name, *call_args, **call_kwargs) # wait for responses if not self._blocking: @@ -134,8 +160,8 @@ def method(*args, **kwargs): # how many resps have been received? resps_received = 0 - for name, resp in resps.items(): - if resp.resp_time: + for name, gcc in gccs.items(): + if gcc.resp_time: resps_received += 1 if resps_received == len(self._proxies): @@ -157,9 +183,9 @@ def method(*args, **kwargs): # process results results = {} - for name, resp in resps.items(): - if resp.resp_time: - result = resp.result + for name, gcc in gccs.items(): + if gcc.resp_time: + result = gcc.result else: result = TimeoutError() results[name] = result @@ -180,6 +206,23 @@ def group( process_resp_cb=None, **cb_kwargs, ): + """This is a convenience method for creating a group. + + Args: + ctx: + proxies: + blocking: + timeout: + optional: + secure: + min_resps: + wait_after_min_resps: + process_resp_cb: + **cb_kwargs: + + Returns: + + """ return Group( ctx.app, ctx.abort_signal, @@ -206,6 +249,22 @@ def all_clients( process_resp_cb=None, **cb_kwargs, ): + """This is a convenience method for creating a group with all clients. + + Args: + ctx: + blocking: + timeout: + optional: + secure: + min_resps: + wait_after_min_resps: + process_resp_cb: + **cb_kwargs: + + Returns: + + """ return Group( ctx.app, ctx.abort_signal, @@ -232,6 +291,22 @@ def all_other_clients( process_resp_cb=None, **cb_kwargs, ): + """This is a convenience method for creating a group with all other clients (excluding myself). + + Args: + ctx: + blocking: + timeout: + optional: + secure: + min_resps: + wait_after_min_resps: + process_resp_cb: + **cb_kwargs: + + Returns: + + """ candidates = ctx.clients me = ctx.app.get_my_site() if me in candidates: @@ -263,6 +338,22 @@ def all_children( process_resp_cb=None, **cb_kwargs, ): + """This is a convenience method for creating a group with all my child clients. + + Args: + ctx: + blocking: + timeout: + optional: + secure: + min_resps: + wait_after_min_resps: + process_resp_cb: + **cb_kwargs: + + Returns: + + """ clients = ctx.app.get_children() if not clients: raise RuntimeError(f"app {ctx.app.name} has no child clients") @@ -293,6 +384,22 @@ def all_leaf_clients( process_resp_cb=None, **cb_kwargs, ): + """This is a convenience method for creating a group with all leaf clients. + + Args: + ctx: + blocking: + timeout: + optional: + secure: + min_resps: + wait_after_min_resps: + process_resp_cb: + **cb_kwargs: + + Returns: + + """ clients = ctx.app.get_leaf_clients() if not clients: raise RuntimeError(f"app {ctx.app.name} has no leaf clients") diff --git a/nvflare/fox/examples/pt/pt_np.py b/nvflare/fox/examples/pt/pt_np.py new file mode 100644 index 0000000000..05845844da --- /dev/null +++ b/nvflare/fox/examples/pt/pt_np.py @@ -0,0 +1,157 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import threading + +from nvflare.fox.api.app import ClientApp, ServerApp +from nvflare.fox.api.ctx import Context +from nvflare.fox.api.dec import collab +from nvflare.fox.api.group import all_clients +from nvflare.fox.api.strategy import Strategy +from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples.np.algos.utils import add as add_np +from nvflare.fox.examples.np.algos.utils import div as div_np +from nvflare.fox.examples.np.algos.utils import parse_state_dict as parse_np +from nvflare.fox.examples.pt.utils import add as add_pt +from nvflare.fox.examples.pt.utils import div as div_pt +from nvflare.fox.examples.pt.utils import parse_state_dict as parse_pt +from nvflare.fox.sim.simulator import Simulator +from nvflare.fuel.utils.log_utils import get_obj_logger + + +class _AggrResult: + + def __init__(self): + self.pt_total = {} + self.np_total = {} + self.count = 0 + self.lock = threading.Lock() # ensure update integrity + + +class PTFedAvgMixed(Strategy): + + def __init__(self, pt_model, np_model, num_rounds=10, timeout=2.0): + self.num_rounds = num_rounds + self.pt_model = pt_model + self.np_model = np_model + self.timeout = timeout + self.name = "PTFedAvg" + self.logger = get_obj_logger(self) + self._pt_model = parse_pt(pt_model) + self._np_model = parse_np(np_model) + + def execute(self, context: Context): + self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") + pt_model, np_model = self._pt_model, self._np_model + for i in range(self.num_rounds): + pt_model, np_model = self._do_one_round(i, pt_model, np_model, context) + self.logger.info(f"FINAL MODEL: {pt_model=} {np_model=}") + return pt_model, np_model + + def _do_one_round(self, r, pt_model, np_model, ctx: Context): + aggr_result = _AggrResult() + + grp = all_clients( + ctx, + process_resp_cb=self._accept_train_result, + aggr_result=aggr_result, + ) + + grp.train(r, pt_model, np_model) + + if aggr_result.count == 0: + return None + else: + pt_result = aggr_result.pt_total + div_pt(pt_result, aggr_result.count) + self.logger.info( + f"[{ctx.header_str()}] round {r}: aggr PT result from {aggr_result.count} clients: {pt_result}" + ) + + np_result = aggr_result.np_total + div_np(np_result, aggr_result.count) + self.logger.info( + f"[{ctx.header_str()}] round {r}: aggr NP result from {aggr_result.count} clients: {np_result}" + ) + return pt_result, np_result + + def _accept_train_result(self, result, aggr_result: _AggrResult, context: Context): + self.logger.info(f"[{context.header_str()}] got train result from {context.caller}: {result}") + + pt_result, np_result = result + add_pt(pt_result, aggr_result.pt_total) + add_np(np_result, aggr_result.np_total) + aggr_result.count += 1 + return None + + +class PTTrainer(ClientApp): + + def __init__(self, delta: float): + ClientApp.__init__(self) + self.delta = delta + + @collab + def train(self, current_round, pt_model, np_model, context: Context): + if context.is_aborted(): + self.logger.debug("training aborted") + return 0 + + self.logger.debug(f"[{context.header_str()}] training round {current_round}: {pt_model=} {np_model=}") + + pt_result = {} + for k, v in pt_model.items(): + pt_result[k] = v + self.delta + + np_result = {} + for k, v in np_model.items(): + np_result[k] = v + self.delta + + return pt_result, np_result + + +def main(): + simple_logging(logging.DEBUG) + + init_model = { + "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "y": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "z": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + } + + server_app = ServerApp( + strategy_name="fed_avg", + strategy=PTFedAvgMixed( + pt_model=init_model, + np_model=init_model, + num_rounds=4, + ), + ) + + client_app = PTTrainer(delta=1.0) + + simulator = Simulator( + root_dir="/tmp/fox", + experiment_name="pt_np", + server_app=server_app, + client_app=client_app, + num_clients=2, + ) + + result = simulator.run() + print(f"Final result: {result}") + + +if __name__ == "__main__": + main() diff --git a/nvflare/fox/examples/pt/recipe_pt_np.py b/nvflare/fox/examples/pt/recipe_pt_np.py new file mode 100644 index 0000000000..32c3de7a0b --- /dev/null +++ b/nvflare/fox/examples/pt/recipe_pt_np.py @@ -0,0 +1,56 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from nvflare.app_common.decomposers.numpy_decomposers import NumpyArrayDecomposer +from nvflare.app_opt.pt.decomposers import TensorDecomposer +from nvflare.fox.api.app import ServerApp +from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples.pt.pt_np import PTFedAvgMixed, PTTrainer +from nvflare.fox.sys.recipe import FoxRecipe + +JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/fox/prod_00/admin@nvidia.com/transfer" + + +def main(): + simple_logging(logging.DEBUG) + + init_model = { + "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "y": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "z": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + } + + server_app = ServerApp( + strategy_name="fedavg", + strategy=PTFedAvgMixed( + pt_model=init_model, + np_model=init_model, + num_rounds=2, + ), + ) + + client_app = PTTrainer(delta=1.0) + + recipe = FoxRecipe( + job_name="pt_np", + server_app=server_app, + client_app=client_app, + ) + recipe.add_decomposers([TensorDecomposer(), NumpyArrayDecomposer]) + recipe.export(JOB_ROOT_DIR) + + +if __name__ == "__main__": + main() diff --git a/nvflare/fox/sim/backend.py b/nvflare/fox/sim/backend.py index 452a1945bb..7ae976435d 100644 --- a/nvflare/fox/sim/backend.py +++ b/nvflare/fox/sim/backend.py @@ -19,7 +19,7 @@ from nvflare.fox.api.backend import Backend from nvflare.fox.api.constants import OPTION_ARGS, CollabMethodArgName, CollabMethodOptionName from nvflare.fox.api.dec import adjust_kwargs -from nvflare.fox.api.resp import Resp +from nvflare.fox.api.gcc import GroupCallContext from nvflare.fox.api.utils import check_call_args from nvflare.fuel.utils.log_utils import get_obj_logger @@ -81,6 +81,8 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): break return waiter.result + else: + return None def _preprocess(self, target_name, func_name, func, kwargs): caller_ctx = kwargs.pop(CollabMethodArgName.CONTEXT) @@ -120,7 +122,7 @@ def _run_func(self, waiter: _Waiter, target_name, func_name, func, args, kwargs) if waiter: waiter.set() - def call_target_with_resp(self, resp: Resp, target_name: str, func_name: str, *args, **kwargs): + def call_target_in_group(self, gcc: GroupCallContext, target_name: str, func_name: str, *args, **kwargs): # do not use the optional args - they are managed by the group for k in OPTION_ARGS: kwargs.pop(k, None) @@ -132,15 +134,15 @@ def call_target_with_resp(self, resp: Resp, target_name: str, func_name: str, *a if not callable(func): raise AttributeError(f"the method '{func_name}' of {target_name} is not callable") - self.executor.submit(self._run_func_with_resp, resp, target_name, func_name, func, args, kwargs) + self.executor.submit(self._run_func_with_resp, gcc, target_name, func_name, func, args, kwargs) - def _run_func_with_resp(self, resp: Resp, target_name, func_name, func, args, kwargs): + def _run_func_with_resp(self, gcc: GroupCallContext, target_name, func_name, func, args, kwargs): try: ctx, kwargs = self._preprocess(target_name, func_name, func, kwargs) result = func(*args, **kwargs) # apply result filter result = self.target_app.apply_outgoing_result_filters(target_name, func_name, result, ctx) - resp.set_result(result) + gcc.set_result(result) except Exception as ex: - resp.set_exception(ex) + gcc.set_exception(ex) diff --git a/nvflare/fox/sim/sim2.py b/nvflare/fox/sim/sim2.py new file mode 100644 index 0000000000..4f1a4c69df --- /dev/null +++ b/nvflare/fox/sim/sim2.py @@ -0,0 +1,53 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Tuple, Union + +from nvflare.fox.api.app import ClientApp, ServerApp +from nvflare.fox.api.strategy import Strategy + +from .simulator import Simulator + + +class Simulator2: + + def __init__( + self, + root_dir: str, + experiment_name: str, + strategy: Strategy, + server_objects: dict[str, object], + client_objects: dict[str, object], + max_workers: int = 100, + num_clients: Union[int, Tuple[int, int]] = 2, + ): + server_app: ServerApp = ServerApp(strategy=strategy) + client_app: ClientApp = ClientApp() + + for name, obj in server_objects.items(): + server_app.add_collab_object(name, obj) + + for name, obj in client_objects.items(): + client_app.add_collab_object(name, obj) + + self.simulator = Simulator( + root_dir=root_dir, + experiment_name=experiment_name, + server_app=server_app, + client_app=client_app, + max_workers=max_workers, + num_clients=num_clients, + ) + + def run(self): + return self.simulator.run() diff --git a/nvflare/fox/sim/simulator.py b/nvflare/fox/sim/simulator.py index 0c9386674d..64a5e253ef 100644 --- a/nvflare/fox/sim/simulator.py +++ b/nvflare/fox/sim/simulator.py @@ -36,7 +36,8 @@ def _prepare_app_backends(self, app: App): bes[name] = SimBackend(name, app, obj, self.abort_signal, self.thread_executor) return bes - def _prepare_proxy(self, for_app: App, target_app: App, backends: dict): + @staticmethod + def _prepare_proxy(for_app: App, target_app: App, backends: dict): app_proxy = Proxy( app=for_app, target_name=target_app.name, diff --git a/nvflare/fox/sys/backend.py b/nvflare/fox/sys/backend.py index 3ea00352fa..66545eaefa 100644 --- a/nvflare/fox/sys/backend.py +++ b/nvflare/fox/sys/backend.py @@ -13,7 +13,7 @@ # limitations under the License. from nvflare.fox.api.backend import Backend from nvflare.fox.api.constants import CollabMethodArgName, CollabMethodOptionName -from nvflare.fox.api.resp import Resp +from nvflare.fox.api.gcc import GroupCallContext from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey, ReturnCode from nvflare.fuel.f3.cellnet.utils import new_cell_message from nvflare.fuel.f3.message import Message @@ -90,11 +90,12 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): secure=secure, optional=optional, ) + return None - def call_target_with_resp(self, resp: Resp, target_name: str, func_name: str, *args, **kwargs): + def call_target_in_group(self, resp: GroupCallContext, target_name: str, func_name: str, *args, **kwargs): self.thread_executor.submit(self._run_func, resp, target_name, func_name, args, kwargs) - def _run_func(self, resp: Resp, target_name: str, func_name: str, args, kwargs): + def _run_func(self, resp: GroupCallContext, target_name: str, func_name: str, args, kwargs): try: result = self.call_target(target_name, func_name, *args, **kwargs) resp.set_result(result) From 3aab25bc4b85bb04cccab4b047cea3c9eb57604a Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Fri, 31 Oct 2025 15:52:38 +0800 Subject: [PATCH 050/102] fix typo --- nvflare/fox/sys/backend.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nvflare/fox/sys/backend.py b/nvflare/fox/sys/backend.py index 66545eaefa..0be56b3c84 100644 --- a/nvflare/fox/sys/backend.py +++ b/nvflare/fox/sys/backend.py @@ -92,12 +92,12 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): ) return None - def call_target_in_group(self, resp: GroupCallContext, target_name: str, func_name: str, *args, **kwargs): - self.thread_executor.submit(self._run_func, resp, target_name, func_name, args, kwargs) + def call_target_in_group(self, gcc: GroupCallContext, target_name: str, func_name: str, *args, **kwargs): + self.thread_executor.submit(self._run_func, gcc, target_name, func_name, args, kwargs) - def _run_func(self, resp: GroupCallContext, target_name: str, func_name: str, args, kwargs): + def _run_func(self, gcc: GroupCallContext, target_name: str, func_name: str, args, kwargs): try: result = self.call_target(target_name, func_name, *args, **kwargs) - resp.set_result(result) + gcc.set_result(result) except Exception as ex: - resp.set_exception(ex) + gcc.set_exception(ex) From 16eeb0ed9c99f21af4fd34db13ee14c516111317 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Wed, 12 Nov 2025 10:13:11 -0500 Subject: [PATCH 051/102] add docstr --- nvflare/fuel/f3/streaming/download_service.py | 87 ++++++++++++------- nvflare/fuel/f3/streaming/obj_downloader.py | 40 +++++++++ 2 files changed, 98 insertions(+), 29 deletions(-) diff --git a/nvflare/fuel/f3/streaming/download_service.py b/nvflare/fuel/f3/streaming/download_service.py index e34c778af9..b76089d9f4 100644 --- a/nvflare/fuel/f3/streaming/download_service.py +++ b/nvflare/fuel/f3/streaming/download_service.py @@ -29,7 +29,7 @@ OBJ_DOWNLOADER_TOPIC = "obj_downloader__download" """ -This package provides a framework for building object downloading capability (e.g. file download). +This package provides a framework for building object downloading capability (file download, tensor download, etc.). A large object takes a lot of memory space. Sending a large object in one message needs even more memory space since the object needs to be serialized into large number of bytes. Additional memory space may still be needed for the @@ -38,17 +38,20 @@ Object Downloading can drastically reduce memory consumption: - Instead of sending the large object in one message, it is divided into many smaller objects; - Instead of pushing the message to the endpoints, each endpoint will come to request. This makes it more reliable when -different endpoints have different speed. +different endpoints have different speed. Object Downloading works as follows: - The sender prepares the object(s) for downloading. It first creates a transaction to get a tx_id. It then adds each -object to be downloaded to the transaction, and get a reference id (ref_id). +object (called Downloadable) to be downloaded to the transaction, and get a reference id (ref_id). - The sender sends the ref_id(s) to all recipients through a separate message. - Each recipient then calls the download_object function to download each referenced large object. -To develop the downloading capability for a type of object (e.g. a file, a large dict, etc.), you need to provide -the implementation of a Producer and a Consumer. -- On the sending side, the Producer is responsible for producing the next small object to be sent (a chunk of bytes; +Note that the endpoint that received object refs may forward the refs to another endpoint, which then downloads the +referenced object(s). + +To develop the downloading capability for a type of object (e.g. a file, a tensor state dict, etc.), you need to provide +the implementation of a Downloadable and a Consumer. +- On the sending side, the Downloadable is responsible for producing the next small object to be sent (a chunk of bytes; a small subset of the large dict; etc.). - On the receiving side, the Consumer is responsible for processing the received small objects (writing the received bytes to a temp file; putting the received small dict to the end result; etc.). @@ -60,32 +63,23 @@ There are two ways to handle this issue: object downloaded callback, and transaction timeout. -You can register an object_downloaded CB when adding an object to transaction. When the object is fully downloaded -to a site, this CB will be called. The obj_downloaded CB must follow this signature: +You can implement the downloaded_to_one method for the Downloadable object. This method is called when the object is +downloaded to one site. + +You can also implement the downloaded_to_all method for the Downloadable object. This method is called when the object +is downloaded to all sites. - downloaded_cb(ref_id: str, to_site: str, status: str, obj: Any, **cb_kwargs) +Note that the downloaded_to_all method only works if you know how many sites the object will be downloaded to! -where ref_id is the reference id of the object; -to_site is the FQCN of the site that has just finished downloading; -status is the status of downloading, as defined in DownloadStatus class; -obj is the large object that was just downloaded; -cb_kwargs are the kw args registered with the CB. +You can always implement the transaction_done method for the Downloadable object. This method is called when the +transaction is done for some reason (normal completion or timeout). Transaction timeout is the amount of time after the last downloading activity on any object in the transaction from any site. For example, suppose you want to send 2 large files to 3 sites, each time a download request is received on any file from any of the 3 sites, the last activity time of the transaction is updated to now. If no downloading activity is received from any site on any objects in the transaction for the specified timeout, -the transaction is considered "timed out", and the timeout callback registered with the transaction is called. -The transaction timeout CB must follow this signature: - - timeout_cb(tx_id: str, objs: List[Any], **cb_kwargs) - -where tx_id is the ID of the transaction; -objs is the list of large objects registered with the transaction; -cb_kwargs are the kw args registered with the CB. - -You may need to use both mechanisms to fully take care of object life cycles. The object downloaded CB may never be -called since the site somehow didn't finish the downloading. In reality the timeout mechanism may be sufficient. +the transaction is considered "timed out", and the transaction_done method is called for each Downloadable object +added to the transaction. Unlike with Object Streamer that the object owner pushes small objects to the recipients; with Object Downloader, each recipient pulls the data from the object owner. @@ -97,7 +91,17 @@ class Downloadable(ABC): def __init__(self, obj: Any): self.base_obj = obj - def set_transaction(self, tx_id, ref_id): + def set_transaction(self, tx_id: str, ref_id: str): + """This method is called when the object is added to a transaction. + You can use this method to keep transaction ID and/or ref ID for your own purpose. + + Args: + tx_id: the ID of the transaction that the object has been added to. + ref_id: ref ID generated for the object. + + Returns: None + + """ pass @abstractmethod @@ -114,7 +118,15 @@ def produce(self, state: dict, requester: str) -> Tuple[str, Any, dict]: pass def downloaded_to_one(self, to_site: str, status: str): - """Called when an object is downloaded to a site.""" + """Called when an object is downloaded to a site. + + Args: + to_site: name of the site that the object has been completely downloaded to. + status: the download status: DownloadStatus.SUCCESS or DownloadStatus.FAILED. + + Returns: None + + """ pass def downloaded_to_all(self): @@ -122,7 +134,15 @@ def downloaded_to_all(self): pass def transaction_done(self, transaction_id: str, status: str): - """Called when the transaction is finished.""" + """Called when the transaction is finished. + + Args: + transaction_id: ID of the transaction. + status: completion status, a value defined in TransactionDoneStatus. + + Returns: None + + """ pass @@ -166,7 +186,7 @@ def obj_downloaded(self, to_site: str, status: str): class ProduceRC: - """Defines return code for the Producer's produce method.""" + """Defines return code for the Downloadable object's 'produce' method.""" OK = "ok" ERROR = "error" @@ -174,11 +194,15 @@ class ProduceRC: class DownloadStatus: + """Constants for object download status. + """ SUCCESS = "success" FAILED = "failed" class TransactionDoneStatus: + """Constants for transaction completion status. + """ FINISHED = "finished" TIMEOUT = "timeout" DELETED = "deleted" @@ -270,6 +294,11 @@ def transaction_done(self, status: str): class TransactionInfo: + """This structure contains public info of a transaction: + timeout value of the transaction; + number of sites that objects in the transaction will be downloaded to. 0 means unknown. + objects that are added to the transaction. + """ def __init__(self, tx: _Transaction): self.timeout = tx.timeout diff --git a/nvflare/fuel/f3/streaming/obj_downloader.py b/nvflare/fuel/f3/streaming/obj_downloader.py index b953565b2a..d86149ba68 100644 --- a/nvflare/fuel/f3/streaming/obj_downloader.py +++ b/nvflare/fuel/f3/streaming/obj_downloader.py @@ -16,6 +16,9 @@ class ObjectDownloader: + """Defines a universal object downloader that can be used to download any Downloadable objects. + + """ def __init__( self, @@ -26,6 +29,28 @@ def __init__( transaction_done_cb=None, **cb_kwargs, ): + """Constructor of ObjectDownloader. + + Args: + cell: the communication cell. + timeout: timeout of the transaction + num_receivers: number of sites to download the objects to. 0 means unknown. + tx_id: if specified, this will be used as the ID of the new transaction. If not specified, dynamically + generates the transaction id. + transaction_done_cb: the callback to be called when the transaction is done. + **cb_kwargs: kwargs to be passed to transaction_done_cb. + + Notes: the CB signature is: + + transaction_done_cb(tx_id, status, objects: list, **cb_kwargs) + + where tx_id is the ID of the transaction; status is a value as defined in TransactionDoneStatus; + objects is a list of base objects to be downloaded. Note that the base object is not the Downloadable object! + For example, in case of file downloading, the Downloadable object is FileDownloadable, whereas the base object + is the name of the file. + + Downloadable object is only needed to work with the Downloader, and not useful for apps. + """ self.cell = cell self.tx_id = DownloadService.new_transaction( cell=self.cell, @@ -37,6 +62,15 @@ def __init__( ) def add_object(self, obj: Downloadable, ref_id=None) -> str: + """Add a Downloadable object to the downloader. + + Args: + obj: the Downloadable object to be added. + ref_id: if specified, use it as the generated ref. If not specified, dynamically generates a ref ID. + + Returns: the ref ID for the object. + + """ rid = DownloadService.add_object( transaction_id=self.tx_id, obj=obj, @@ -45,4 +79,10 @@ def add_object(self, obj: Downloadable, ref_id=None) -> str: return rid def delete_transaction(self): + """Delete the download transaction forcefully. + You call this method only if you want to stop the downloading process prematurely. + + Returns: None. + + """ DownloadService.delete_transaction(self.tx_id) From e38fa8a0447e5634a3ad079c96f6c0480e51209c Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Thu, 13 Nov 2025 14:39:51 -0500 Subject: [PATCH 052/102] use thread local for ctx --- nvflare/fox/api/app.py | 6 +- nvflare/fox/api/ctx.py | 5 ++ nvflare/fox/api/ez.py | 70 +++++++++++++++++++ nvflare/fox/api/proxy.py | 49 ++++++++++++- nvflare/fox/api/proxy_list.py | 47 +++++++++++++ nvflare/fox/examples/np/algos/client.py | 8 ++- .../examples/np/algos/strategies/avg_para.py | 11 ++- .../examples/np/algos/strategies/avg_seq.py | 13 ++-- nvflare/fox/sim/backend.py | 1 + 9 files changed, 191 insertions(+), 19 deletions(-) create mode 100644 nvflare/fox/api/ez.py create mode 100644 nvflare/fox/api/proxy_list.py diff --git a/nvflare/fox/api/app.py b/nvflare/fox/api/app.py index dc1d7052aa..16ac40ab2b 100644 --- a/nvflare/fox/api/app.py +++ b/nvflare/fox/api/app.py @@ -20,7 +20,7 @@ from nvflare.fuel.utils.tree_utils import Forest, Node, build_forest from .constants import CollabMethodArgName, ContextKey, FilterDirection -from .ctx import Context +from .ctx import Context, fox_context from .dec import collab, get_object_collab_interface, is_collab from .filter import CallFilter, FilterChain, ResultFilter from .proxy import Proxy @@ -274,7 +274,9 @@ def initialize(self, context: Context): self._fox_init(obj, context) def new_context(self, caller: str, callee: str, target_group=None): - return Context(self, caller, callee, self._abort_signal, target_group=target_group) + ctx = Context(self, caller, callee, self._abort_signal, target_group=target_group) + fox_context.call_ctx = ctx + return ctx def register_event_handler(self, event_type: str, handler, **handler_kwargs): handlers = self._event_handlers.get(event_type) diff --git a/nvflare/fox/api/ctx.py b/nvflare/fox/api/ctx.py index 6aa2df6707..15c5999e84 100644 --- a/nvflare/fox/api/ctx.py +++ b/nvflare/fox/api/ctx.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import threading + from nvflare.apis.signal import Signal @@ -72,3 +74,6 @@ def is_aborted(self): def header_str(self): return f"{self.app.name}:{self.caller}=>{self.callee}" + + +fox_context = threading.local() diff --git a/nvflare/fox/api/ez.py b/nvflare/fox/api/ez.py new file mode 100644 index 0000000000..509649e0ef --- /dev/null +++ b/nvflare/fox/api/ez.py @@ -0,0 +1,70 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from .ctx import fox_context +from .proxy_list import ProxyList + + +class classproperty: + def __init__(self, fget): + self.fget = fget + + def __get__(self, owner_instance, owner_class): + return self.fget(owner_class) + + +class EZ: + + @classproperty + def context(cls): + return fox_context.call_ctx + + @classproperty + def clients(cls): + ctx = fox_context.call_ctx + return ProxyList(ctx.clients) + + @classproperty + def other_clients(cls): + ctx = fox_context.call_ctx + candidates = ctx.clients + me = ctx.app.get_my_site() + if me in candidates: + candidates.remove(me) + return ProxyList(candidates) + + @classproperty + def child_clients(cls): + ctx = fox_context.call_ctx + candidates = ctx.app.get_children() + if not candidates: + raise RuntimeError(f"app {ctx.app.name} has no child clients") + return ProxyList(candidates) + + @classproperty + def leaf_clients(cls): + ctx = fox_context.call_ctx + candidates = ctx.app.get_leaf_clients() + if not candidates: + raise RuntimeError(f"app {ctx.app.name} has no leaf clients") + return ProxyList(candidates) + + @classproperty + def env_type(cls): + ctx = fox_context.call_ctx + return ctx.env_type + + @classproperty + def is_aborted(cls): + ctx = fox_context.call_ctx + return ctx.is_aborted() diff --git a/nvflare/fox/api/proxy.py b/nvflare/fox/api/proxy.py index 29c03cef64..776bebb6a7 100644 --- a/nvflare/fox/api/proxy.py +++ b/nvflare/fox/api/proxy.py @@ -16,10 +16,41 @@ from nvflare.fuel.utils.log_utils import get_obj_logger from .backend import Backend -from .constants import OPTION_ARGS, CollabMethodArgName +from .constants import OPTION_ARGS, CollabMethodArgName, CollabMethodOptionName from .utils import check_call_args +class _ProxyCall: + + def __init__( + self, + proxy, + blocking: bool = True, + timeout: float = 5.0, + optional: bool = False, + secure: bool = False, + ): + self.proxy = proxy + self.blocking = blocking + self.timeout = timeout + self.optional = optional + self.secure = secure + + def __getattr__(self, func_name): + def method(*args, **kwargs): + kwargs.update( + { + CollabMethodOptionName.OPTIONAL: self.optional, + CollabMethodOptionName.SECURE: self.secure, + CollabMethodOptionName.BLOCKING: self.blocking, + CollabMethodOptionName.TIMEOUT: self.timeout, + } + ) + return getattr(self.proxy, func_name)(*args, **kwargs) + + return method + + class Proxy: def __init__(self, app, target_name, target_fqn: str, backend: Backend, target_interface): @@ -33,6 +64,22 @@ def __init__(self, app, target_name, target_fqn: str, backend: Backend, target_i self.children = {} # child proxies self.logger = get_obj_logger(self) + def __call__( + self, + blocking: bool = True, + timeout: float = 5.0, + optional: bool = False, + secure: bool = False, + ): + print(f"creating _ProxyCall: {blocking=} {timeout=} {optional=} {secure=}") + return _ProxyCall( + proxy=self, + blocking=blocking, + timeout=timeout, + optional=optional, + secure=secure, + ) + @property def name(self): return self.target_name diff --git a/nvflare/fox/api/proxy_list.py b/nvflare/fox/api/proxy_list.py new file mode 100644 index 0000000000..3929ed23c9 --- /dev/null +++ b/nvflare/fox/api/proxy_list.py @@ -0,0 +1,47 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from .ctx import fox_context +from .group import group + + +class ProxyList(list): + + def __init__(self, proxies: list): + super().__init__() + self.extend(proxies) + + def __call__( + self, + blocking: bool = True, + timeout: float = 5.0, + optional: bool = False, + secure: bool = False, + min_resps: int = None, + wait_after_min_resps: float = None, + process_resp_cb=None, + **cb_kwargs, + ): + print(f"creating group: {blocking=} {timeout=} {optional=} {secure=} {min_resps=} {wait_after_min_resps=}") + return group( + ctx=fox_context.call_ctx, + proxies=self, + blocking=blocking, + timeout=timeout, + optional=optional, + secure=secure, + min_resps=min_resps, + wait_after_min_resps=wait_after_min_resps, + process_resp_cb=process_resp_cb, + **cb_kwargs, + ) diff --git a/nvflare/fox/examples/np/algos/client.py b/nvflare/fox/examples/np/algos/client.py index 5cb65d788a..6e3035d0fd 100644 --- a/nvflare/fox/examples/np/algos/client.py +++ b/nvflare/fox/examples/np/algos/client.py @@ -16,6 +16,7 @@ from nvflare.fox.api.app import ClientApp from nvflare.fox.api.ctx import Context from nvflare.fox.api.dec import collab +from nvflare.fox.api.ez import EZ from nvflare.fox.api.group import all_children @@ -31,11 +32,12 @@ def fox_init(self, context: Context): self.logger.info(f"client {self.name}: delta={self.delta}") @collab - def train(self, current_round, weights, context: Context): - if context.is_aborted(): + def train(self, current_round, weights): + context = EZ.context + if EZ.is_aborted: self.logger.debug("training aborted") return 0 - self.logger.debug(f"[{context.header_str()}] trained round {current_round}") + self.logger.debug(f"[{context.header_str()}] EZ trained round {current_round}") # metric_receiver = self.server.get_target("metric_receiver") # if metric_receiver: diff --git a/nvflare/fox/examples/np/algos/strategies/avg_para.py b/nvflare/fox/examples/np/algos/strategies/avg_para.py index d8bcf334ef..b5107a10f6 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_para.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_para.py @@ -11,10 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import numpy as np - from nvflare.fox.api.constants import ContextKey from nvflare.fox.api.ctx import Context +from nvflare.fox.api.ez import EZ from nvflare.fox.api.group import all_clients from nvflare.fox.api.strategy import Strategy from nvflare.fox.examples.np.algos.utils import parse_array_def @@ -34,7 +33,7 @@ def execute(self, context: Context): self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") current_model = context.get_prop(ContextKey.INPUT, self._initial_model) for i in range(self.num_rounds): - current_model = self._do_one_round(i, current_model, context) + current_model = self._do_one_round(i, current_model) score = self._do_eval(current_model, context) self.logger.info(f"[{context.header_str()}]: eval score in round {i}: {score}") return current_model @@ -47,10 +46,10 @@ def _do_eval(self, model, ctx: Context): total += v return total / len(results) - def _do_one_round(self, r, current_model, ctx: Context): + def _do_one_round(self, r, current_model): total = 0 - results = all_clients(ctx, timeout=4, min_resps=2, wait_after_min_resps=1).train(r, current_model) + results = EZ.clients(timeout=4, min_resps=2, wait_after_min_resps=1).train(r, current_model) for n, v in results.items(): - self.logger.info(f"[{ctx.header_str()}] round {r}: got group result from client {n}: {v}") + self.logger.info(f"[{EZ.context.header_str()}] round {r}: got group result from client {n}: {v}") total += v return total / len(results) diff --git a/nvflare/fox/examples/np/algos/strategies/avg_seq.py b/nvflare/fox/examples/np/algos/strategies/avg_seq.py index cb54b56050..f70982a556 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_seq.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_seq.py @@ -13,10 +13,9 @@ # limitations under the License. import os -import numpy as np - from nvflare.fox.api.constants import ContextKey from nvflare.fox.api.ctx import Context +from nvflare.fox.api.ez import EZ from nvflare.fox.api.strategy import Strategy from nvflare.fox.examples.np.algos.utils import parse_array_def, save_np_model from nvflare.fuel.utils.log_utils import get_obj_logger @@ -53,18 +52,18 @@ def execute(self, context: Context): self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") current_model = context.get_prop(ContextKey.INPUT, self._initial_model) for i in range(self.num_rounds): - current_model = self._do_one_round(i, current_model, context) + current_model = self._do_one_round(i, current_model) # save model to work dir file_name = os.path.join(context.workspace.get_work_dir(), "model.npy") save_np_model(current_model, file_name) return current_model - def _do_one_round(self, r, current_model, context: Context): + def _do_one_round(self, r, current_model): total = 0 n = 0 - for c in context.clients: - result = c.train(r, current_model, _blocking=True, _timeout=2.0, _optional=True, _secure=False) - self.logger.info(f"[{context.header_str()}] round {r}: got result from client {c.name}: {result}") + for c in EZ.clients: + result = c(blocking=True, timeout=2.0, optional=True, secure=False).train(r, current_model) + self.logger.info(f"[{EZ.context.header_str()}] round {r}: got result from client {c.name}: {result}") total += result * self.client_weights[c.name] return total diff --git a/nvflare/fox/sim/backend.py b/nvflare/fox/sim/backend.py index 7ae976435d..d387d9fcf7 100644 --- a/nvflare/fox/sim/backend.py +++ b/nvflare/fox/sim/backend.py @@ -18,6 +18,7 @@ from nvflare.fox.api.app import App from nvflare.fox.api.backend import Backend from nvflare.fox.api.constants import OPTION_ARGS, CollabMethodArgName, CollabMethodOptionName +from nvflare.fox.api.ctx import fox_context from nvflare.fox.api.dec import adjust_kwargs from nvflare.fox.api.gcc import GroupCallContext from nvflare.fox.api.utils import check_call_args From 336c5526bcb8720f84063c455fc0ce386208179a Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Thu, 13 Nov 2025 15:51:36 -0500 Subject: [PATCH 053/102] use EZ in examples --- nvflare/fox/api/app.py | 4 +- nvflare/fox/api/ctx.py | 9 ++++- nvflare/fox/api/ez.py | 31 ++++++++++---- nvflare/fox/api/proxy_list.py | 14 ++++++- .../fox/examples/np/algos/strategies/avg_h.py | 22 +++++----- .../np/algos/strategies/avg_intime.py | 21 +++++----- nvflare/fox/examples/np/algos/swarm.py | 40 +++++++++---------- 7 files changed, 86 insertions(+), 55 deletions(-) diff --git a/nvflare/fox/api/app.py b/nvflare/fox/api/app.py index 16ac40ab2b..d6ddac4225 100644 --- a/nvflare/fox/api/app.py +++ b/nvflare/fox/api/app.py @@ -20,7 +20,7 @@ from nvflare.fuel.utils.tree_utils import Forest, Node, build_forest from .constants import CollabMethodArgName, ContextKey, FilterDirection -from .ctx import Context, fox_context +from .ctx import Context, set_call_context from .dec import collab, get_object_collab_interface, is_collab from .filter import CallFilter, FilterChain, ResultFilter from .proxy import Proxy @@ -275,7 +275,7 @@ def initialize(self, context: Context): def new_context(self, caller: str, callee: str, target_group=None): ctx = Context(self, caller, callee, self._abort_signal, target_group=target_group) - fox_context.call_ctx = ctx + set_call_context(ctx) return ctx def register_event_handler(self, event_type: str, handler, **handler_kwargs): diff --git a/nvflare/fox/api/ctx.py b/nvflare/fox/api/ctx.py index 15c5999e84..0d95821ca7 100644 --- a/nvflare/fox/api/ctx.py +++ b/nvflare/fox/api/ctx.py @@ -15,6 +15,8 @@ from nvflare.apis.signal import Signal +fox_context = threading.local() + class Context: @@ -76,4 +78,9 @@ def header_str(self): return f"{self.app.name}:{self.caller}=>{self.callee}" -fox_context = threading.local() +def get_call_context(): + return fox_context.call_ctx + + +def set_call_context(ctx): + fox_context.call_ctx = ctx diff --git a/nvflare/fox/api/ez.py b/nvflare/fox/api/ez.py index 509649e0ef..f7cb6a5335 100644 --- a/nvflare/fox/api/ez.py +++ b/nvflare/fox/api/ez.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from .ctx import fox_context +from .ctx import get_call_context from .proxy_list import ProxyList @@ -27,16 +27,31 @@ class EZ: @classproperty def context(cls): - return fox_context.call_ctx + return get_call_context() + + @classproperty + def caller(cls): + ctx = get_call_context() + return ctx.caller + + @classproperty + def callee(cls): + ctx = get_call_context() + return ctx.callee + + @classproperty + def call_info(cls): + ctx = get_call_context() + return ctx.header_str() @classproperty def clients(cls): - ctx = fox_context.call_ctx + ctx = get_call_context() return ProxyList(ctx.clients) @classproperty def other_clients(cls): - ctx = fox_context.call_ctx + ctx = get_call_context() candidates = ctx.clients me = ctx.app.get_my_site() if me in candidates: @@ -45,7 +60,7 @@ def other_clients(cls): @classproperty def child_clients(cls): - ctx = fox_context.call_ctx + ctx = get_call_context() candidates = ctx.app.get_children() if not candidates: raise RuntimeError(f"app {ctx.app.name} has no child clients") @@ -53,7 +68,7 @@ def child_clients(cls): @classproperty def leaf_clients(cls): - ctx = fox_context.call_ctx + ctx = get_call_context() candidates = ctx.app.get_leaf_clients() if not candidates: raise RuntimeError(f"app {ctx.app.name} has no leaf clients") @@ -61,10 +76,10 @@ def leaf_clients(cls): @classproperty def env_type(cls): - ctx = fox_context.call_ctx + ctx = get_call_context() return ctx.env_type @classproperty def is_aborted(cls): - ctx = fox_context.call_ctx + ctx = get_call_context() return ctx.is_aborted() diff --git a/nvflare/fox/api/proxy_list.py b/nvflare/fox/api/proxy_list.py index 3929ed23c9..cc47acf80d 100644 --- a/nvflare/fox/api/proxy_list.py +++ b/nvflare/fox/api/proxy_list.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from .ctx import fox_context +from .ctx import get_call_context from .group import group @@ -21,6 +21,16 @@ def __init__(self, proxies: list): super().__init__() self.extend(proxies) + def __getattr__(self, func_name): + def method(*args, **kwargs): + grp = group( + ctx=get_call_context(), + proxies=self, + ) + return getattr(grp, func_name)(*args, **kwargs) + + return method + def __call__( self, blocking: bool = True, @@ -34,7 +44,7 @@ def __call__( ): print(f"creating group: {blocking=} {timeout=} {optional=} {secure=} {min_resps=} {wait_after_min_resps=}") return group( - ctx=fox_context.call_ctx, + ctx=get_call_context(), proxies=self, blocking=blocking, timeout=timeout, diff --git a/nvflare/fox/examples/np/algos/strategies/avg_h.py b/nvflare/fox/examples/np/algos/strategies/avg_h.py index 151eb79ded..890745749e 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_h.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_h.py @@ -11,11 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import numpy as np - from nvflare.fox.api.constants import ContextKey from nvflare.fox.api.ctx import Context -from nvflare.fox.api.group import all_children, all_leaf_clients +from nvflare.fox.api.ez import EZ from nvflare.fox.api.strategy import Strategy from nvflare.fox.examples.np.algos.utils import parse_array_def from nvflare.fuel.utils.log_utils import get_obj_logger @@ -31,27 +29,27 @@ def __init__(self, initial_model, num_rounds=10): self.logger = get_obj_logger(self) def execute(self, context: Context): - self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") + self.logger.info(f"[{EZ.call_info}] Start training for {self.num_rounds} rounds") current_model = context.get_prop(ContextKey.INPUT, self._initial_model) for i in range(self.num_rounds): - current_model = self._do_one_round(i, current_model, context) - score = self._do_eval(current_model, context) + current_model = self._do_one_round(i, current_model) + score = self._do_eval(current_model) self.logger.info(f"[{context.header_str()}]: eval score in round {i}: {score}") self.logger.info(f"FINAL MODEL: {current_model}") return current_model - def _do_eval(self, model, ctx: Context): - results = all_leaf_clients(ctx).evaluate(model) + def _do_eval(self, model): + results = EZ.leaf_clients.evaluate(model) total = 0.0 for n, v in results.items(): - self.logger.info(f"[{ctx.header_str()}]: got eval result from client {n}: {v}") + self.logger.info(f"[{EZ.call_info}]: got eval result from client {n}: {v}") total += v return total / len(results) - def _do_one_round(self, r, current_model, ctx: Context): + def _do_one_round(self, r, current_model): total = 0 - results = all_children(ctx).train(r, current_model) + results = EZ.child_clients.train(r, current_model) for n, v in results.items(): - self.logger.info(f"[{ctx.header_str()}] round {r}: got group result from client {n}: {v}") + self.logger.info(f"[{EZ.call_info}] round {r}: got group result from client {n}: {v}") total += v return total / len(results) diff --git a/nvflare/fox/examples/np/algos/strategies/avg_intime.py b/nvflare/fox/examples/np/algos/strategies/avg_intime.py index 22e8077a70..ad9098fa7d 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_intime.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_intime.py @@ -15,7 +15,7 @@ from nvflare.fox.api.constants import ContextKey from nvflare.fox.api.ctx import Context -from nvflare.fox.api.group import all_clients +from nvflare.fox.api.ez import EZ from nvflare.fox.api.strategy import Strategy from nvflare.fox.examples.np.algos.utils import parse_array_def from nvflare.fuel.utils.log_utils import get_obj_logger @@ -42,24 +42,23 @@ def execute(self, context: Context): self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") current_model = context.get_prop(ContextKey.INPUT, self._init_model) for i in range(self.num_rounds): - current_model = self._do_one_round(i, current_model, context) - score = self._do_eval(current_model, context) + current_model = self._do_one_round(i, current_model) + score = self._do_eval(current_model) self.logger.info(f"[{context.header_str()}]: eval score in round {i}: {score}") self.logger.info(f"FINAL MODEL: {current_model}") return current_model - def _do_eval(self, model, ctx: Context): - results = all_clients(ctx).evaluate(model) + def _do_eval(self, model): + results = EZ.clients.evaluate(model) total = 0.0 for n, v in results.items(): - self.logger.info(f"[{ctx.header_str()}]: got eval result from client {n}: {v}") + self.logger.info(f"[{EZ.context.header_str()}]: got eval result from client {n}: {v}") total += v return total / len(results) - def _do_one_round(self, r, current_model, ctx: Context): + def _do_one_round(self, r, current_model): aggr_result = _AggrResult() - all_clients( - ctx, + EZ.clients( process_resp_cb=self._accept_train_result, aggr_result=aggr_result, ).train(r, current_model) @@ -68,7 +67,9 @@ def _do_one_round(self, r, current_model, ctx: Context): return None else: result = aggr_result.total / aggr_result.count - self.logger.info(f"[{ctx.header_str()}] round {r}: aggr result from {aggr_result.count} clients: {result}") + self.logger.info( + f"[{EZ.context.header_str()}] round {r}: aggr result from {aggr_result.count} clients: {result}" + ) return result def _accept_train_result(self, result, aggr_result: _AggrResult, context: Context): diff --git a/nvflare/fox/examples/np/algos/swarm.py b/nvflare/fox/examples/np/algos/swarm.py index ed2de25bab..f32690804a 100644 --- a/nvflare/fox/examples/np/algos/swarm.py +++ b/nvflare/fox/examples/np/algos/swarm.py @@ -18,7 +18,7 @@ from nvflare.fox.api.app import ClientApp from nvflare.fox.api.ctx import Context from nvflare.fox.api.dec import collab -from nvflare.fox.api.group import all_clients +from nvflare.fox.api.ez import EZ from nvflare.fox.api.strategy import Strategy from nvflare.fox.examples.np.algos.utils import parse_array_def from nvflare.fuel.utils.log_utils import get_obj_logger @@ -44,13 +44,13 @@ def execute(self, context: Context): if self.waiter.wait(timeout=0.5): break - def _all_done(self, event_type: str, data, context: Context): - self.logger.info(f"[{context.header_str()}]: received {event_type} from client: {context.caller}: {data}") - self.all_done(data, context) + def _all_done(self, event_type: str, data): + self.logger.info(f"[{EZ.call_info}]: received {event_type} from client: {EZ.caller}: {data}") + self.all_done(data) @collab - def all_done(self, reason: str, context: Context): - self.logger.info(f"[{context.header_str()}]: all done from client: {context.caller}: {reason}") + def all_done(self, reason: str): + self.logger.info(f"[{EZ.call_info}]: all done from client: {EZ.caller}: {reason}") self.waiter.set() @@ -62,13 +62,13 @@ def __init__(self, delta: float): self.register_event_handler("final_model", self._accept_final_model) @collab - def train(self, weights, current_round, context: Context): - self.logger.info(f"[{context.header_str()}]: train asked by {context.caller}: {current_round=}") + def train(self, weights, current_round): + self.logger.info(f"[{EZ.call_info}]: train asked by {EZ.caller}: {current_round=}") return weights + self.delta - def sag(self, model, current_round, ctx: Context): - results = all_clients(ctx, blocking=True).train(model, current_round) - # results = all_other_clients(ctx, blocking=True).train(model, current_round) + def sag(self, model, current_round): + # results = EZ.clients.train(model, current_round) + results = EZ.other_clients.train(model, current_round) results = list(results.values()) total = 0 for i in range(len(results)): @@ -76,14 +76,14 @@ def sag(self, model, current_round, ctx: Context): return total / len(results) @collab - def swarm_learn(self, num_rounds, model, current_round, context: Context): - self.logger.info(f"[{context.header_str()}]: swarm learn asked: {num_rounds=} {current_round=} {model=}") - new_model = self.sag(model, current_round, context) + def swarm_learn(self, num_rounds, model, current_round): + self.logger.info(f"[{EZ.call_info}]: swarm learn asked: {num_rounds=} {current_round=} {model=}") + new_model = self.sag(model, current_round) - self.logger.info(f"[{context.header_str()}]: trained model {new_model=}") + self.logger.info(f"[{EZ.call_info}]: trained model {new_model=}") if current_round == num_rounds - 1: # all done - all_clients(context, blocking=False).fire_event("final_model", new_model) + EZ.clients(blocking=False).fire_event("final_model", new_model) # self.server.fire_event("all_done", "OK", blocking=False) self.logger.info("notify server all done!") try: @@ -101,10 +101,10 @@ def swarm_learn(self, num_rounds, model, current_round, context: Context): next_client.swarm_learn(num_rounds, new_model, next_round, _blocking=False) @collab - def start(self, num_rounds, initial_model, context: Context): - self.swarm_learn(num_rounds, initial_model, 0, context) + def start(self, num_rounds, initial_model): + self.swarm_learn(num_rounds, initial_model, 0) - def _accept_final_model(self, event_type: str, model, context: Context): + def _accept_final_model(self, event_type: str, model): # accept the final model # write model to disk - self.logger.info(f"[{context.header_str()}]: received event '{event_type}' from {context.caller}: {model}") + self.logger.info(f"[{EZ.call_info}]: received event '{event_type}' from {EZ.caller}: {model}") From f7ee545bc9871c64aba7dec0f47305ce565fbbc6 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Thu, 13 Nov 2025 16:03:40 -0500 Subject: [PATCH 054/102] add license text --- nvflare/fox/api/ez.py | 5 +++++ nvflare/fox/sys/__init__.py | 13 +++++++++++++ 2 files changed, 18 insertions(+) diff --git a/nvflare/fox/api/ez.py b/nvflare/fox/api/ez.py index f7cb6a5335..57e731032c 100644 --- a/nvflare/fox/api/ez.py +++ b/nvflare/fox/api/ez.py @@ -83,3 +83,8 @@ def env_type(cls): def is_aborted(cls): ctx = get_call_context() return ctx.is_aborted() + + @classmethod + def fire_event(cls, event_type: str, data): + ctx = get_call_context() + return ctx.app.fire_event(event_type, data, ctx) diff --git a/nvflare/fox/sys/__init__.py b/nvflare/fox/sys/__init__.py index e69de29bb2..341a77c5bc 100644 --- a/nvflare/fox/sys/__init__.py +++ b/nvflare/fox/sys/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. From b11f3388cb31ad61996e611e037af4bd18bcc893 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Thu, 13 Nov 2025 17:00:37 -0500 Subject: [PATCH 055/102] rename ez to fox --- nvflare/fox/api/{ez.py => fox.py} | 7 ++- nvflare/fox/examples/np/algos/client.py | 20 ++++---- .../fox/examples/np/algos/strategies/avg_h.py | 12 ++--- .../np/algos/strategies/avg_intime.py | 16 +++---- .../examples/np/algos/strategies/avg_para.py | 14 +++--- .../examples/np/algos/strategies/avg_seq.py | 7 ++- nvflare/fox/examples/np/algos/swarm.py | 27 ++++++----- nvflare/fox/examples/pt/pt_avg_mixed.py | 46 +++++++++---------- 8 files changed, 72 insertions(+), 77 deletions(-) rename nvflare/fox/api/{ez.py => fox.py} (95%) diff --git a/nvflare/fox/api/ez.py b/nvflare/fox/api/fox.py similarity index 95% rename from nvflare/fox/api/ez.py rename to nvflare/fox/api/fox.py index 57e731032c..fe6af12b4b 100644 --- a/nvflare/fox/api/ez.py +++ b/nvflare/fox/api/fox.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from .ctx import get_call_context +from .dec import collab as dec_collab from .proxy_list import ProxyList @@ -23,12 +24,16 @@ def __get__(self, owner_instance, owner_class): return self.fget(owner_class) -class EZ: +class fox: @classproperty def context(cls): return get_call_context() + @classproperty + def collab(cls): + return dec_collab + @classproperty def caller(cls): ctx = get_call_context() diff --git a/nvflare/fox/examples/np/algos/client.py b/nvflare/fox/examples/np/algos/client.py index 6e3035d0fd..40ed4ae6b4 100644 --- a/nvflare/fox/examples/np/algos/client.py +++ b/nvflare/fox/examples/np/algos/client.py @@ -15,8 +15,7 @@ from nvflare.fox.api.app import ClientApp from nvflare.fox.api.ctx import Context -from nvflare.fox.api.dec import collab -from nvflare.fox.api.ez import EZ +from nvflare.fox.api.fox import fox from nvflare.fox.api.group import all_children @@ -31,13 +30,12 @@ def fox_init(self, context: Context): self.delta = delta_config.get(self.name, self.delta) self.logger.info(f"client {self.name}: delta={self.delta}") - @collab + @fox.collab def train(self, current_round, weights): - context = EZ.context - if EZ.is_aborted: + if fox.is_aborted: self.logger.debug("training aborted") return 0 - self.logger.debug(f"[{context.header_str()}] EZ trained round {current_round}") + self.logger.debug(f"[{fox.call_info}] EZ trained round {current_round}") # metric_receiver = self.server.get_target("metric_receiver") # if metric_receiver: @@ -47,9 +45,9 @@ def train(self, current_round, weights): self.server.fire_event("metrics", {"round": current_round, "y": 10}, _blocking=False) return weights + self.delta - @collab - def evaluate(self, model, context: Context): - self.logger.debug(f"[{context.header_str()}] evaluate") + @fox.collab + def evaluate(self, model): + self.logger.debug(f"[{fox.call_info}] evaluate") return random.random() @@ -59,7 +57,7 @@ def __init__(self, delta: float): ClientApp.__init__(self) self.delta = delta - @collab + @fox.collab def train(self, current_round, weights, context: Context): if context.is_aborted(): self.logger.debug("training aborted") @@ -85,7 +83,7 @@ def _local_train(self, current_round, weights, context: Context): self.logger.info(f"[{context.header_str()}] local trained round {current_round} {weights} {type(weights)}") return weights + self.delta - @collab + @fox.collab def evaluate(self, model, context: Context): self.logger.debug(f"[{context.header_str()}] evaluate") return random.random() diff --git a/nvflare/fox/examples/np/algos/strategies/avg_h.py b/nvflare/fox/examples/np/algos/strategies/avg_h.py index 890745749e..d5b84c8971 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_h.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_h.py @@ -13,7 +13,7 @@ # limitations under the License. from nvflare.fox.api.constants import ContextKey from nvflare.fox.api.ctx import Context -from nvflare.fox.api.ez import EZ +from nvflare.fox.api.fox import fox from nvflare.fox.api.strategy import Strategy from nvflare.fox.examples.np.algos.utils import parse_array_def from nvflare.fuel.utils.log_utils import get_obj_logger @@ -29,7 +29,7 @@ def __init__(self, initial_model, num_rounds=10): self.logger = get_obj_logger(self) def execute(self, context: Context): - self.logger.info(f"[{EZ.call_info}] Start training for {self.num_rounds} rounds") + self.logger.info(f"[{fox.call_info}] Start training for {self.num_rounds} rounds") current_model = context.get_prop(ContextKey.INPUT, self._initial_model) for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model) @@ -39,17 +39,17 @@ def execute(self, context: Context): return current_model def _do_eval(self, model): - results = EZ.leaf_clients.evaluate(model) + results = fox.leaf_clients.evaluate(model) total = 0.0 for n, v in results.items(): - self.logger.info(f"[{EZ.call_info}]: got eval result from client {n}: {v}") + self.logger.info(f"[{fox.call_info}]: got eval result from client {n}: {v}") total += v return total / len(results) def _do_one_round(self, r, current_model): total = 0 - results = EZ.child_clients.train(r, current_model) + results = fox.child_clients.train(r, current_model) for n, v in results.items(): - self.logger.info(f"[{EZ.call_info}] round {r}: got group result from client {n}: {v}") + self.logger.info(f"[{fox.call_info}] round {r}: got group result from client {n}: {v}") total += v return total / len(results) diff --git a/nvflare/fox/examples/np/algos/strategies/avg_intime.py b/nvflare/fox/examples/np/algos/strategies/avg_intime.py index ad9098fa7d..0f42927ff1 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_intime.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_intime.py @@ -11,11 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import numpy as np - from nvflare.fox.api.constants import ContextKey from nvflare.fox.api.ctx import Context -from nvflare.fox.api.ez import EZ +from nvflare.fox.api.fox import fox from nvflare.fox.api.strategy import Strategy from nvflare.fox.examples.np.algos.utils import parse_array_def from nvflare.fuel.utils.log_utils import get_obj_logger @@ -49,16 +47,16 @@ def execute(self, context: Context): return current_model def _do_eval(self, model): - results = EZ.clients.evaluate(model) + results = fox.clients.evaluate(model) total = 0.0 for n, v in results.items(): - self.logger.info(f"[{EZ.context.header_str()}]: got eval result from client {n}: {v}") + self.logger.info(f"[{fox.context.header_str()}]: got eval result from client {n}: {v}") total += v return total / len(results) def _do_one_round(self, r, current_model): aggr_result = _AggrResult() - EZ.clients( + fox.clients( process_resp_cb=self._accept_train_result, aggr_result=aggr_result, ).train(r, current_model) @@ -68,12 +66,12 @@ def _do_one_round(self, r, current_model): else: result = aggr_result.total / aggr_result.count self.logger.info( - f"[{EZ.context.header_str()}] round {r}: aggr result from {aggr_result.count} clients: {result}" + f"[{fox.context.header_str()}] round {r}: aggr result from {aggr_result.count} clients: {result}" ) return result - def _accept_train_result(self, result, aggr_result: _AggrResult, context: Context): - self.logger.info(f"[{context.header_str()}] got train result from {context.caller} {result}") + def _accept_train_result(self, result, aggr_result: _AggrResult): + self.logger.info(f"[{fox.call_info}] got train result from {fox.caller} {result}") aggr_result.total += result aggr_result.count += 1 return None diff --git a/nvflare/fox/examples/np/algos/strategies/avg_para.py b/nvflare/fox/examples/np/algos/strategies/avg_para.py index b5107a10f6..4ffee01f29 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_para.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_para.py @@ -13,7 +13,7 @@ # limitations under the License. from nvflare.fox.api.constants import ContextKey from nvflare.fox.api.ctx import Context -from nvflare.fox.api.ez import EZ +from nvflare.fox.api.fox import fox from nvflare.fox.api.group import all_clients from nvflare.fox.api.strategy import Strategy from nvflare.fox.examples.np.algos.utils import parse_array_def @@ -34,22 +34,22 @@ def execute(self, context: Context): current_model = context.get_prop(ContextKey.INPUT, self._initial_model) for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model) - score = self._do_eval(current_model, context) + score = self._do_eval(current_model) self.logger.info(f"[{context.header_str()}]: eval score in round {i}: {score}") return current_model - def _do_eval(self, model, ctx: Context): - results = all_clients(ctx).evaluate(model) + def _do_eval(self, model): + results = fox.clients.evaluate(model) total = 0.0 for n, v in results.items(): - self.logger.info(f"[{ctx.header_str()}]: got eval result from client {n}: {v}") + self.logger.info(f"[{fox.call_info}]: got eval result from client {n}: {v}") total += v return total / len(results) def _do_one_round(self, r, current_model): total = 0 - results = EZ.clients(timeout=4, min_resps=2, wait_after_min_resps=1).train(r, current_model) + results = fox.clients(timeout=4, min_resps=2, wait_after_min_resps=1).train(r, current_model) for n, v in results.items(): - self.logger.info(f"[{EZ.context.header_str()}] round {r}: got group result from client {n}: {v}") + self.logger.info(f"[{fox.context.header_str()}] round {r}: got group result from client {n}: {v}") total += v return total / len(results) diff --git a/nvflare/fox/examples/np/algos/strategies/avg_seq.py b/nvflare/fox/examples/np/algos/strategies/avg_seq.py index f70982a556..4f00643501 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_seq.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_seq.py @@ -15,7 +15,7 @@ from nvflare.fox.api.constants import ContextKey from nvflare.fox.api.ctx import Context -from nvflare.fox.api.ez import EZ +from nvflare.fox.api.fox import fox from nvflare.fox.api.strategy import Strategy from nvflare.fox.examples.np.algos.utils import parse_array_def, save_np_model from nvflare.fuel.utils.log_utils import get_obj_logger @@ -61,9 +61,8 @@ def execute(self, context: Context): def _do_one_round(self, r, current_model): total = 0 - n = 0 - for c in EZ.clients: + for c in fox.clients: result = c(blocking=True, timeout=2.0, optional=True, secure=False).train(r, current_model) - self.logger.info(f"[{EZ.context.header_str()}] round {r}: got result from client {c.name}: {result}") + self.logger.info(f"[{fox.context.header_str()}] round {r}: got result from client {c.name}: {result}") total += result * self.client_weights[c.name] return total diff --git a/nvflare/fox/examples/np/algos/swarm.py b/nvflare/fox/examples/np/algos/swarm.py index f32690804a..1a38859550 100644 --- a/nvflare/fox/examples/np/algos/swarm.py +++ b/nvflare/fox/examples/np/algos/swarm.py @@ -17,8 +17,7 @@ from nvflare.fox.api.app import ClientApp from nvflare.fox.api.ctx import Context -from nvflare.fox.api.dec import collab -from nvflare.fox.api.ez import EZ +from nvflare.fox.api.fox import fox from nvflare.fox.api.strategy import Strategy from nvflare.fox.examples.np.algos.utils import parse_array_def from nvflare.fuel.utils.log_utils import get_obj_logger @@ -45,12 +44,12 @@ def execute(self, context: Context): break def _all_done(self, event_type: str, data): - self.logger.info(f"[{EZ.call_info}]: received {event_type} from client: {EZ.caller}: {data}") + self.logger.info(f"[{fox.call_info}]: received {event_type} from client: {fox.caller}: {data}") self.all_done(data) - @collab + @fox.collab def all_done(self, reason: str): - self.logger.info(f"[{EZ.call_info}]: all done from client: {EZ.caller}: {reason}") + self.logger.info(f"[{fox.call_info}]: all done from client: {fox.caller}: {reason}") self.waiter.set() @@ -61,29 +60,29 @@ def __init__(self, delta: float): self.delta = delta self.register_event_handler("final_model", self._accept_final_model) - @collab + @fox.collab def train(self, weights, current_round): - self.logger.info(f"[{EZ.call_info}]: train asked by {EZ.caller}: {current_round=}") + self.logger.info(f"[{fox.call_info}]: train asked by {fox.caller}: {current_round=}") return weights + self.delta def sag(self, model, current_round): # results = EZ.clients.train(model, current_round) - results = EZ.other_clients.train(model, current_round) + results = fox.other_clients.train(model, current_round) results = list(results.values()) total = 0 for i in range(len(results)): total += results[i] return total / len(results) - @collab + @fox.collab def swarm_learn(self, num_rounds, model, current_round): - self.logger.info(f"[{EZ.call_info}]: swarm learn asked: {num_rounds=} {current_round=} {model=}") + self.logger.info(f"[{fox.call_info}]: swarm learn asked: {num_rounds=} {current_round=} {model=}") new_model = self.sag(model, current_round) - self.logger.info(f"[{EZ.call_info}]: trained model {new_model=}") + self.logger.info(f"[{fox.call_info}]: trained model {new_model=}") if current_round == num_rounds - 1: # all done - EZ.clients(blocking=False).fire_event("final_model", new_model) + fox.clients(blocking=False).fire_event("final_model", new_model) # self.server.fire_event("all_done", "OK", blocking=False) self.logger.info("notify server all done!") try: @@ -100,11 +99,11 @@ def swarm_learn(self, num_rounds, model, current_round): next_client = self.clients[next_client_idx] next_client.swarm_learn(num_rounds, new_model, next_round, _blocking=False) - @collab + @fox.collab def start(self, num_rounds, initial_model): self.swarm_learn(num_rounds, initial_model, 0) def _accept_final_model(self, event_type: str, model): # accept the final model # write model to disk - self.logger.info(f"[{EZ.call_info}]: received event '{event_type}' from {EZ.caller}: {model}") + self.logger.info(f"[{fox.call_info}]: received event '{event_type}' from {fox.caller}: {model}") diff --git a/nvflare/fox/examples/pt/pt_avg_mixed.py b/nvflare/fox/examples/pt/pt_avg_mixed.py index 3babce1ec2..6d37bcc996 100644 --- a/nvflare/fox/examples/pt/pt_avg_mixed.py +++ b/nvflare/fox/examples/pt/pt_avg_mixed.py @@ -20,8 +20,7 @@ from nvflare.fox.api.app import ClientApp, ServerApp from nvflare.fox.api.constants import EnvType from nvflare.fox.api.ctx import Context -from nvflare.fox.api.dec import collab -from nvflare.fox.api.group import all_clients +from nvflare.fox.api.fox import fox from nvflare.fox.api.strategy import Strategy from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.utils import add as add_np @@ -60,23 +59,22 @@ def execute(self, context: Context): self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") pt_model, np_model = self._pt_model, self._np_model for i in range(self.num_rounds): - pt_model, np_model = self._do_one_round(i, pt_model, np_model, context) + pt_model, np_model = self._do_one_round(i, pt_model, np_model) self.logger.info(f"FINAL MODEL: {pt_model=} {np_model=}") return pt_model, np_model - def _do_one_round(self, r, pt_model, np_model, ctx: Context): + def _do_one_round(self, r, pt_model, np_model): aggr_result = _AggrResult() - grp = all_clients( - ctx, + grp = fox.clients( process_resp_cb=self._accept_train_result, aggr_result=aggr_result, ) - if ctx.env_type == EnvType.SYSTEM: + if fox.env_type == EnvType.SYSTEM: downloader = Downloader( num_receivers=grp.size, - ctx=ctx, + ctx=fox.context, timeout=5.0, ) model_type = "ref" @@ -94,28 +92,28 @@ def _do_one_round(self, r, pt_model, np_model, ctx: Context): pt_result = aggr_result.pt_total div_pt(pt_result, aggr_result.count) self.logger.info( - f"[{ctx.header_str()}] round {r}: aggr PT result from {aggr_result.count} clients: {pt_result}" + f"[{fox.call_info}] round {r}: aggr PT result from {aggr_result.count} clients: {pt_result}" ) np_result = aggr_result.np_total div_np(np_result, aggr_result.count) self.logger.info( - f"[{ctx.header_str()}] round {r}: aggr NP result from {aggr_result.count} clients: {np_result}" + f"[{fox.call_info}] round {r}: aggr NP result from {aggr_result.count} clients: {np_result}" ) return pt_result, np_result - def _accept_train_result(self, result, aggr_result: _AggrResult, context: Context): - self.logger.info(f"[{context.header_str()}] got train result from {context.caller}: {result}") + def _accept_train_result(self, result, aggr_result: _AggrResult): + self.logger.info(f"[{fox.call_info}] got train result from {fox.caller}: {result}") pt_result, np_result, model_type = result if model_type == "ref": err, pt_result = download_tensors( ref=pt_result, per_request_timeout=5.0, - ctx=context, + ctx=fox.context, tensors_received_cb=self._aggregate_tensors, aggr_result=aggr_result, - context=context, + context=fox.context, ) if err: raise RuntimeError(f"failed to download model {pt_result}: {err}") @@ -123,10 +121,10 @@ def _accept_train_result(self, result, aggr_result: _AggrResult, context: Contex err, np_result = download_arrays( ref=np_result, per_request_timeout=5.0, - ctx=context, + ctx=fox.context, arrays_received_cb=self._aggregate_arrays, aggr_result=aggr_result, - context=context, + context=fox.context, ) if err: raise RuntimeError(f"failed to download NP model file {np_result}: {err}") @@ -154,23 +152,21 @@ def __init__(self, delta: float): ClientApp.__init__(self) self.delta = delta - @collab - def train(self, current_round, pt_model, np_model, model_type: str, context: Context): - if context.is_aborted(): + @fox.collab + def train(self, current_round, pt_model, np_model, model_type: str): + if fox.is_aborted: self.logger.debug("training aborted") return 0 - self.logger.debug( - f"[{context.header_str()}] training round {current_round}: {model_type=} {pt_model=} {np_model=}" - ) + self.logger.debug(f"[{fox.call_info}] training round {current_round}: {model_type=} {pt_model=} {np_model=}") if model_type == "ref": - err, pt_model = download_tensors(ref=pt_model, per_request_timeout=5.0, ctx=context) + err, pt_model = download_tensors(ref=pt_model, per_request_timeout=5.0, ctx=fox.context) if err: raise RuntimeError(f"failed to download PT model {pt_model}: {err}") self.logger.info(f"downloaded PT model {pt_model}") - err, np_model = download_arrays(ref=np_model, per_request_timeout=5.0, ctx=context) + err, np_model = download_arrays(ref=np_model, per_request_timeout=5.0, ctx=fox.context) if err: raise RuntimeError(f"failed to download NP model {np_model}: {err}") self.logger.info(f"downloaded NP model {np_model}") @@ -187,7 +183,7 @@ def train(self, current_round, pt_model, np_model, model_type: str, context: Con # stream it downloader = Downloader( num_receivers=1, - ctx=context, + ctx=fox.context, timeout=5.0, ) pt_result = downloader.add_tensors(pt_result, 0) From ed360369f096eea0a2eb4595ffb9f72a5197d310 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Fri, 14 Nov 2025 11:02:20 -0500 Subject: [PATCH 056/102] refactor facade --- nvflare/fox/__init__.py | 1 + nvflare/fox/api/dec.py | 8 +++++++ nvflare/fox/api/{fox.py => facade.py} | 24 +++++++++---------- nvflare/fox/examples/np/algos/client.py | 2 +- .../fox/examples/np/algos/strategies/avg_h.py | 2 +- .../np/algos/strategies/avg_intime.py | 2 +- .../examples/np/algos/strategies/avg_para.py | 3 +-- .../examples/np/algos/strategies/avg_seq.py | 2 +- nvflare/fox/examples/np/algos/swarm.py | 2 +- nvflare/fox/examples/np/algos/widgets.py | 17 +++++++------ nvflare/fox/examples/pt/pt_avg_mixed.py | 2 +- 11 files changed, 36 insertions(+), 29 deletions(-) rename nvflare/fox/api/{fox.py => facade.py} (87%) diff --git a/nvflare/fox/__init__.py b/nvflare/fox/__init__.py index 341a77c5bc..8afd3df8ce 100644 --- a/nvflare/fox/__init__.py +++ b/nvflare/fox/__init__.py @@ -11,3 +11,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from .api.facade import facade as fox diff --git a/nvflare/fox/api/dec.py b/nvflare/fox/api/dec.py index 584fafe836..5d71b2a370 100644 --- a/nvflare/fox/api/dec.py +++ b/nvflare/fox/api/dec.py @@ -20,6 +20,14 @@ _ATTR_PARAM_NAMES = "_fox_param_names" +class classproperty: + def __init__(self, fget): + self.fget = fget + + def __get__(self, owner_instance, owner_class): + return self.fget(owner_class) + + def collab(func): def wrapper(*args, **kwargs): return func(*args, **kwargs) diff --git a/nvflare/fox/api/fox.py b/nvflare/fox/api/facade.py similarity index 87% rename from nvflare/fox/api/fox.py rename to nvflare/fox/api/facade.py index fe6af12b4b..84db83596d 100644 --- a/nvflare/fox/api/fox.py +++ b/nvflare/fox/api/facade.py @@ -12,28 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. from .ctx import get_call_context +from .dec import classproperty from .dec import collab as dec_collab from .proxy_list import ProxyList -class classproperty: - def __init__(self, fget): - self.fget = fget +class facade: - def __get__(self, owner_instance, owner_class): - return self.fget(owner_class) - - -class fox: + collab = dec_collab @classproperty def context(cls): return get_call_context() - @classproperty - def collab(cls): - return dec_collab - @classproperty def caller(cls): ctx = get_call_context() @@ -51,6 +42,10 @@ def call_info(cls): @classproperty def clients(cls): + return cls.get_clients() + + @staticmethod + def get_clients(): ctx = get_call_context() return ProxyList(ctx.clients) @@ -93,3 +88,8 @@ def is_aborted(cls): def fire_event(cls, event_type: str, data): ctx = get_call_context() return ctx.app.fire_event(event_type, data, ctx) + + @classmethod + def register_event_handler(cls, event_type: str, handler, **handler_kwargs): + ctx = get_call_context() + ctx.app.register_event_handler(event_type, handler, **handler_kwargs) diff --git a/nvflare/fox/examples/np/algos/client.py b/nvflare/fox/examples/np/algos/client.py index 40ed4ae6b4..02ba57a131 100644 --- a/nvflare/fox/examples/np/algos/client.py +++ b/nvflare/fox/examples/np/algos/client.py @@ -13,9 +13,9 @@ # limitations under the License. import random +from nvflare.fox import fox from nvflare.fox.api.app import ClientApp from nvflare.fox.api.ctx import Context -from nvflare.fox.api.fox import fox from nvflare.fox.api.group import all_children diff --git a/nvflare/fox/examples/np/algos/strategies/avg_h.py b/nvflare/fox/examples/np/algos/strategies/avg_h.py index d5b84c8971..6397851f9f 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_h.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_h.py @@ -11,9 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from nvflare.fox import fox from nvflare.fox.api.constants import ContextKey from nvflare.fox.api.ctx import Context -from nvflare.fox.api.fox import fox from nvflare.fox.api.strategy import Strategy from nvflare.fox.examples.np.algos.utils import parse_array_def from nvflare.fuel.utils.log_utils import get_obj_logger diff --git a/nvflare/fox/examples/np/algos/strategies/avg_intime.py b/nvflare/fox/examples/np/algos/strategies/avg_intime.py index 0f42927ff1..968028334a 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_intime.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_intime.py @@ -11,9 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from nvflare.fox import fox from nvflare.fox.api.constants import ContextKey from nvflare.fox.api.ctx import Context -from nvflare.fox.api.fox import fox from nvflare.fox.api.strategy import Strategy from nvflare.fox.examples.np.algos.utils import parse_array_def from nvflare.fuel.utils.log_utils import get_obj_logger diff --git a/nvflare/fox/examples/np/algos/strategies/avg_para.py b/nvflare/fox/examples/np/algos/strategies/avg_para.py index 4ffee01f29..bfbc558bbf 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_para.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_para.py @@ -11,10 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from nvflare.fox import fox from nvflare.fox.api.constants import ContextKey from nvflare.fox.api.ctx import Context -from nvflare.fox.api.fox import fox -from nvflare.fox.api.group import all_clients from nvflare.fox.api.strategy import Strategy from nvflare.fox.examples.np.algos.utils import parse_array_def from nvflare.fuel.utils.log_utils import get_obj_logger diff --git a/nvflare/fox/examples/np/algos/strategies/avg_seq.py b/nvflare/fox/examples/np/algos/strategies/avg_seq.py index 4f00643501..57ff01dcb8 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_seq.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_seq.py @@ -13,9 +13,9 @@ # limitations under the License. import os +from nvflare.fox import fox from nvflare.fox.api.constants import ContextKey from nvflare.fox.api.ctx import Context -from nvflare.fox.api.fox import fox from nvflare.fox.api.strategy import Strategy from nvflare.fox.examples.np.algos.utils import parse_array_def, save_np_model from nvflare.fuel.utils.log_utils import get_obj_logger diff --git a/nvflare/fox/examples/np/algos/swarm.py b/nvflare/fox/examples/np/algos/swarm.py index 1a38859550..5c022750fe 100644 --- a/nvflare/fox/examples/np/algos/swarm.py +++ b/nvflare/fox/examples/np/algos/swarm.py @@ -15,9 +15,9 @@ import threading import traceback +from nvflare.fox import fox from nvflare.fox.api.app import ClientApp from nvflare.fox.api.ctx import Context -from nvflare.fox.api.fox import fox from nvflare.fox.api.strategy import Strategy from nvflare.fox.examples.np.algos.utils import parse_array_def from nvflare.fuel.utils.log_utils import get_obj_logger diff --git a/nvflare/fox/examples/np/algos/widgets.py b/nvflare/fox/examples/np/algos/widgets.py index 283c4f7d49..b393f96c52 100644 --- a/nvflare/fox/examples/np/algos/widgets.py +++ b/nvflare/fox/examples/np/algos/widgets.py @@ -11,8 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from nvflare.fox.api.ctx import Context -from nvflare.fox.api.dec import collab +from nvflare.fox import fox from nvflare.fuel.utils.log_utils import get_obj_logger @@ -21,13 +20,13 @@ class MetricReceiver: def __init__(self): self.logger = get_obj_logger(self) - @collab - def accept_metric(self, metrics: dict, context: Context): - self.logger.info(f"[{context.callee}] received metric report from {context.caller}: {metrics}") + @fox.collab + def accept_metric(self, metrics: dict): + self.logger.info(f"[{fox.callee}] received metric report from {fox.caller}: {metrics}") - def fox_init(self, context: Context): - context.app.register_event_handler("metrics", self._accept_metric) + def fox_init(self): + fox.register_event_handler("metrics", self._accept_metric) self.logger.info("MetricReceiver initialized!") - def _accept_metric(self, event_type: str, data, context: Context): - self.logger.info(f"[{context.callee}] received event '{event_type}' from {context.caller}: {data}") + def _accept_metric(self, event_type: str, data): + self.logger.info(f"[{fox.callee}] received event '{event_type}' from {fox.caller}: {data}") diff --git a/nvflare/fox/examples/pt/pt_avg_mixed.py b/nvflare/fox/examples/pt/pt_avg_mixed.py index 6d37bcc996..215e7d71ec 100644 --- a/nvflare/fox/examples/pt/pt_avg_mixed.py +++ b/nvflare/fox/examples/pt/pt_avg_mixed.py @@ -17,10 +17,10 @@ import numpy as np import torch +from nvflare.fox import fox from nvflare.fox.api.app import ClientApp, ServerApp from nvflare.fox.api.constants import EnvType from nvflare.fox.api.ctx import Context -from nvflare.fox.api.fox import fox from nvflare.fox.api.strategy import Strategy from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.utils import add as add_np From a8e67dab2eb736322b470ee97989b41aec4cef72 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Fri, 14 Nov 2025 11:52:37 -0500 Subject: [PATCH 057/102] support fox.init dec --- nvflare/fox/api/app.py | 11 ++- nvflare/fox/api/dec.py | 76 ++++++++++++++++--- nvflare/fox/api/facade.py | 20 ++++- nvflare/fox/examples/np/algos/client.py | 11 ++- .../examples/np/algos/strategies/avg_seq.py | 10 ++- nvflare/fox/examples/np/algos/widgets.py | 3 +- 6 files changed, 103 insertions(+), 28 deletions(-) diff --git a/nvflare/fox/api/app.py b/nvflare/fox/api/app.py index d6ddac4225..ff293a00dd 100644 --- a/nvflare/fox/api/app.py +++ b/nvflare/fox/api/app.py @@ -21,7 +21,7 @@ from .constants import CollabMethodArgName, ContextKey, FilterDirection from .ctx import Context, set_call_context -from .dec import collab, get_object_collab_interface, is_collab +from .dec import collab, get_object_collab_interface, get_object_init_funcs, is_collab from .filter import CallFilter, FilterChain, ResultFilter from .proxy import Proxy from .strategy import Strategy @@ -259,12 +259,11 @@ def find_collab_method(self, target_obj, method_name): return None def _fox_init(self, obj, ctx: Context): - init_func = getattr(obj, "fox_init", None) - if init_func and callable(init_func): - self.logger.info(f"fox_init object {obj.__class__.__name__}") + init_funcs = get_object_init_funcs(obj) + for f in init_funcs: kwargs = {CollabMethodArgName.CONTEXT: ctx} - check_context_support(init_func, kwargs) - init_func(**kwargs) + check_context_support(f, kwargs) + f(**kwargs) def initialize(self, context: Context): self._fox_init(self, context) diff --git a/nvflare/fox/api/dec.py b/nvflare/fox/api/dec.py index 5d71b2a370..3b75651d96 100644 --- a/nvflare/fox/api/dec.py +++ b/nvflare/fox/api/dec.py @@ -15,8 +15,10 @@ from .constants import CollabMethodArgName -_ATTR_COLLAB = "_fox_is_collab" -_ATTR_SUPPORT_CTX = "_fox_supports_ctx" +_FLAG_COLLAB = "_fox_is_collab" +_FLAG_INIT = "_fox_is_init" +_FLAG_ALGO = "_fox_is_algo" +_FLAG_SUPPORT_CTX = "_fox_supports_ctx" _ATTR_PARAM_NAMES = "_fox_param_names" @@ -28,18 +30,40 @@ def __get__(self, owner_instance, owner_class): return self.fget(owner_class) -def collab(func): - def wrapper(*args, **kwargs): - return func(*args, **kwargs) - +def _set_attrs(func, wrapper): signature = inspect.signature(func) parameter_names = list(signature.parameters.keys()) if "self" in parameter_names: parameter_names.remove("self") setattr(wrapper, _ATTR_PARAM_NAMES, parameter_names) if CollabMethodArgName.CONTEXT in parameter_names: - setattr(wrapper, _ATTR_SUPPORT_CTX, True) - setattr(wrapper, _ATTR_COLLAB, True) + setattr(wrapper, _FLAG_SUPPORT_CTX, True) + + +def collab(func): + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + _set_attrs(func, wrapper) + setattr(wrapper, _FLAG_COLLAB, True) + return wrapper + + +def init(func): + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + _set_attrs(func, wrapper) + setattr(wrapper, _FLAG_INIT, True) + return wrapper + + +def algo(func): + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + _set_attrs(func, wrapper) + setattr(wrapper, _FLAG_ALGO, True) return wrapper @@ -47,12 +71,25 @@ def get_param_names(func): return getattr(func, _ATTR_PARAM_NAMES, None) +def _has_flag(func, flag: str) -> bool: + v = getattr(func, flag, None) + return v is True + + def is_collab(func): - return hasattr(func, _ATTR_COLLAB) + return _has_flag(func, _FLAG_COLLAB) + + +def is_init(func): + return _has_flag(func, _FLAG_INIT) + + +def is_algo(func): + return _has_flag(func, _FLAG_ALGO) def supports_context(func): - return hasattr(func, _ATTR_SUPPORT_CTX) + return _has_flag(func, _FLAG_SUPPORT_CTX) def adjust_kwargs(func, kwargs): @@ -67,3 +104,22 @@ def get_object_collab_interface(obj): if callable(func) and is_collab(func): result[name] = get_param_names(func) return result + + +def get_object_init_funcs(obj): + result = [] + for name in dir(obj): + func = getattr(obj, name) + if callable(func) and is_init(func): + print(f"found init func of object {obj.__class__.__name__}.{name}") + result.append(func) + return result + + +def get_object_algo_funcs(obj): + result = [] + for name in dir(obj): + func = getattr(obj, name) + if callable(func) and is_algo(func): + result.append(func) + return result diff --git a/nvflare/fox/api/facade.py b/nvflare/fox/api/facade.py index 84db83596d..30925e82d4 100644 --- a/nvflare/fox/api/facade.py +++ b/nvflare/fox/api/facade.py @@ -14,12 +14,14 @@ from .ctx import get_call_context from .dec import classproperty from .dec import collab as dec_collab +from .dec import init as dec_init from .proxy_list import ProxyList class facade: collab = dec_collab + init = dec_init @classproperty def context(cls): @@ -84,12 +86,22 @@ def is_aborted(cls): ctx = get_call_context() return ctx.is_aborted() - @classmethod - def fire_event(cls, event_type: str, data): + @staticmethod + def fire_event(event_type: str, data): ctx = get_call_context() return ctx.app.fire_event(event_type, data, ctx) - @classmethod - def register_event_handler(cls, event_type: str, handler, **handler_kwargs): + @staticmethod + def register_event_handler(event_type: str, handler, **handler_kwargs): ctx = get_call_context() ctx.app.register_event_handler(event_type, handler, **handler_kwargs) + + @staticmethod + def get_prop(name: str, default=None): + ctx = get_call_context() + return ctx.app.get_prop(name, default) + + @staticmethod + def set_prop(name: str, value): + ctx = get_call_context() + ctx.app.set_prop(name, value) diff --git a/nvflare/fox/examples/np/algos/client.py b/nvflare/fox/examples/np/algos/client.py index 02ba57a131..fa5f29af6b 100644 --- a/nvflare/fox/examples/np/algos/client.py +++ b/nvflare/fox/examples/np/algos/client.py @@ -25,10 +25,15 @@ def __init__(self, delta: float): ClientApp.__init__(self) self.delta = delta - def fox_init(self, context: Context): - delta_config = context.app.get_prop("client_delta", {}) + @fox.init + def init_trainer(self): + delta_config = fox.get_prop("client_delta", {}) self.delta = delta_config.get(self.name, self.delta) - self.logger.info(f"client {self.name}: delta={self.delta}") + self.logger.info(f"init_trainer: client {self.name}: delta={self.delta}") + + @fox.init + def init_trainer2(self): + self.logger.info(f"init_trainer2: client {self.name}: init again") @fox.collab def train(self, current_round, weights): diff --git a/nvflare/fox/examples/np/algos/strategies/avg_seq.py b/nvflare/fox/examples/np/algos/strategies/avg_seq.py index 57ff01dcb8..643297e561 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_seq.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_seq.py @@ -32,17 +32,19 @@ def __init__(self, initial_model, num_rounds=10): self.logger = get_obj_logger(self) self.client_weights = None - def fox_init(self, context: Context): - weight_config = context.app.get_prop("client_weight_config", {}) + @fox.init + def init(self): + self.logger.info("fox init NPFedAvgSequential") + weight_config = fox.get_prop("client_weight_config", {}) client_weights = {} total = 0 - for c in context.clients: + for c in fox.clients: w = weight_config.get(c.name, 100) client_weights[c.name] = w total += w # normalize weights - for c in context.clients: + for c in fox.clients: client_weights[c.name] = client_weights[c.name] / total self.client_weights = client_weights diff --git a/nvflare/fox/examples/np/algos/widgets.py b/nvflare/fox/examples/np/algos/widgets.py index b393f96c52..6a3977923d 100644 --- a/nvflare/fox/examples/np/algos/widgets.py +++ b/nvflare/fox/examples/np/algos/widgets.py @@ -24,7 +24,8 @@ def __init__(self): def accept_metric(self, metrics: dict): self.logger.info(f"[{fox.callee}] received metric report from {fox.caller}: {metrics}") - def fox_init(self): + @fox.init + def init(self): fox.register_event_handler("metrics", self._accept_metric) self.logger.info("MetricReceiver initialized!") From 20e331aab76ff70bf9b969cb967d46fcc6e08464 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Fri, 14 Nov 2025 16:09:33 -0500 Subject: [PATCH 058/102] fix examples --- nvflare/fox/api/app.py | 52 +++++------- nvflare/fox/api/dec.py | 2 +- nvflare/fox/api/facade.py | 27 +++++++ nvflare/fox/api/gcc.py | 7 +- nvflare/fox/api/run_server.py | 17 ++-- nvflare/fox/api/strategy.py | 23 ------ nvflare/fox/examples/np/algos/avg_stream.py | 49 ++++++------ nvflare/fox/examples/np/algos/client.py | 60 ++++++-------- .../fox/examples/np/algos/strategies/avg_h.py | 12 ++- .../np/algos/strategies/avg_intime.py | 22 ++--- .../examples/np/algos/strategies/avg_para.py | 16 ++-- .../examples/np/algos/strategies/avg_seq.py | 15 ++-- .../examples/np/algos/strategies/cyclic.py | 23 +++--- nvflare/fox/examples/np/algos/swarm.py | 34 ++++---- nvflare/fox/examples/np/algos/widgets.py | 2 +- nvflare/fox/examples/np/cyclic.py | 11 ++- nvflare/fox/examples/np/cyclic2.py | 38 +++++++++ nvflare/fox/examples/np/cyclic_avg.py | 35 ++++++-- nvflare/fox/examples/np/fed_avg_h.py | 7 +- nvflare/fox/examples/np/fed_avg_h2.py | 40 ++++++++++ nvflare/fox/examples/np/fed_avg_intime.py | 12 +-- nvflare/fox/examples/np/fed_avg_intime2.py | 49 ++++++++++++ nvflare/fox/examples/np/fed_avg_para.py | 7 +- nvflare/fox/examples/np/fed_avg_seq.py | 9 +-- nvflare/fox/examples/np/fed_avg_stream.py | 7 +- nvflare/fox/examples/np/recipe_swarm.py | 8 +- nvflare/fox/examples/np/swarm.py | 8 +- nvflare/fox/examples/pt/pt_avg_filter.py | 66 +++++++-------- nvflare/fox/examples/pt/pt_avg_mixed.py | 32 ++++---- nvflare/fox/examples/pt/pt_avg_stream.py | 80 +++++++++---------- nvflare/fox/examples/pt/pt_np.py | 63 ++++++--------- nvflare/fox/sim/sim2.py | 64 +++++++++++---- 32 files changed, 510 insertions(+), 387 deletions(-) delete mode 100644 nvflare/fox/api/strategy.py create mode 100644 nvflare/fox/examples/np/cyclic2.py create mode 100644 nvflare/fox/examples/np/fed_avg_h2.py create mode 100644 nvflare/fox/examples/np/fed_avg_intime2.py diff --git a/nvflare/fox/api/app.py b/nvflare/fox/api/app.py index ff293a00dd..5196c418fa 100644 --- a/nvflare/fox/api/app.py +++ b/nvflare/fox/api/app.py @@ -21,17 +21,17 @@ from .constants import CollabMethodArgName, ContextKey, FilterDirection from .ctx import Context, set_call_context -from .dec import collab, get_object_collab_interface, get_object_init_funcs, is_collab +from .dec import collab, get_object_algo_funcs, get_object_collab_interface, get_object_init_funcs, is_collab from .filter import CallFilter, FilterChain, ResultFilter from .proxy import Proxy -from .strategy import Strategy from .utils import check_context_support, get_collab_object_name from .workspace import Workspace class App: - def __init__(self): + def __init__(self, obj, name: str): + self.obj = obj self.name = None self.fqn = None self.server = None @@ -52,6 +52,7 @@ def __init__(self): self._resource_dirs = {} self._managed_objects = {} # id => obj self.logger = get_obj_logger(self) + self.add_collab_object(name, obj) def set_resource_dirs(self, resource_dirs: dict[str, str]): if not isinstance(resource_dirs, dict): @@ -188,9 +189,6 @@ def update_props(self, props: dict): if isinstance(props, dict): self._props.update(props) - def get_default_collab_object(self): - return None - def add_collab_object(self, name: str, obj): if name in self._collab_objs: raise ValueError(f"conflict with existing collab object '{name}' of {type(self._collab_objs[name])}") @@ -239,11 +237,10 @@ def find_method(self, target_obj, method_name): if isinstance(target_obj, App): # see whether any targets have this method - default_target = self.get_default_collab_object() - if default_target: - m = getattr(default_target, method_name, None) - if m: - return m + default_target = self.obj + m = getattr(default_target, method_name, None) + if m: + return m targets = self.get_collab_objects() for _, obj in targets.items(): @@ -317,27 +314,13 @@ def get_leaf_clients(self): class ServerApp(App): - def __init__(self, strategy_name: str = "strategy", strategy: Strategy = None): - super().__init__() - - if strategy and not isinstance(strategy, Strategy): - raise ValueError(f"strategy must be Strategy but got {type(strategy)}") - - self.strategies = [] - if strategy: - if not strategy_name: - raise ValueError("missing strategy name") - self.add_strategy(strategy_name, strategy) - self.current_strategy = None - - def add_strategy(self, strategy_name: str, strategy): - if not isinstance(strategy, Strategy): - raise ValueError(f"strategy must be Controller but got {type(strategy)}") - self.strategies.append((strategy_name, strategy)) - self.add_collab_object(strategy_name, strategy) - - def get_default_collab_object(self): - return self.current_strategy + def __init__(self, obj, name: str = "server"): + if not obj: + raise ValueError("server object must be specified") + super().__init__(obj, name) + self.algos = get_object_algo_funcs(obj) + if not self.algos: + raise ValueError("server object must have at least one algo") def get_children(self): assert isinstance(self.client_hierarchy, Forest) @@ -350,6 +333,11 @@ def has_children(self): class ClientApp(App): + def __init__(self, obj, name: str = "client"): + if not obj: + raise ValueError("client object must be specified") + super().__init__(obj, name) + def get_children(self): assert isinstance(self.client_hierarchy, Forest) my_node = self.client_hierarchy.nodes[self.name] diff --git a/nvflare/fox/api/dec.py b/nvflare/fox/api/dec.py index 3b75651d96..7f2648327a 100644 --- a/nvflare/fox/api/dec.py +++ b/nvflare/fox/api/dec.py @@ -121,5 +121,5 @@ def get_object_algo_funcs(obj): for name in dir(obj): func = getattr(obj, name) if callable(func) and is_algo(func): - result.append(func) + result.append((name, func)) return result diff --git a/nvflare/fox/api/facade.py b/nvflare/fox/api/facade.py index 30925e82d4..2618eb1b2a 100644 --- a/nvflare/fox/api/facade.py +++ b/nvflare/fox/api/facade.py @@ -11,7 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from .constants import ContextKey from .ctx import get_call_context +from .dec import algo as dec_algo from .dec import classproperty from .dec import collab as dec_collab from .dec import init as dec_init @@ -22,6 +24,7 @@ class facade: collab = dec_collab init = dec_init + algo = dec_algo @classproperty def context(cls): @@ -42,6 +45,16 @@ def call_info(cls): ctx = get_call_context() return ctx.header_str() + @classproperty + def site_name(cls): + ctx = get_call_context() + return ctx.app.name + + @classproperty + def server(cls): + ctx = get_call_context() + return ctx.app.server + @classproperty def clients(cls): return cls.get_clients() @@ -68,6 +81,11 @@ def child_clients(cls): raise RuntimeError(f"app {ctx.app.name} has no child clients") return ProxyList(candidates) + @classproperty + def has_children(cls): + ctx = get_call_context() + return ctx.app.has_children() + @classproperty def leaf_clients(cls): ctx = get_call_context() @@ -86,6 +104,11 @@ def is_aborted(cls): ctx = get_call_context() return ctx.is_aborted() + @classproperty + def workspace(cls): + ctx = get_call_context() + return ctx.workspace + @staticmethod def fire_event(event_type: str, data): ctx = get_call_context() @@ -101,6 +124,10 @@ def get_prop(name: str, default=None): ctx = get_call_context() return ctx.app.get_prop(name, default) + @staticmethod + def get_input(default=None): + return facade.get_prop(ContextKey.INPUT, default) + @staticmethod def set_prop(name: str, value): ctx = get_call_context() diff --git a/nvflare/fox/api/gcc.py b/nvflare/fox/api/gcc.py index 2810f8269f..4813e6031d 100644 --- a/nvflare/fox/api/gcc.py +++ b/nvflare/fox/api/gcc.py @@ -15,7 +15,7 @@ import time from .constants import CollabMethodArgName -from .ctx import Context +from .ctx import Context, set_call_context from .utils import check_context_support @@ -63,9 +63,14 @@ def set_result(self, result): result = self.app.apply_incoming_result_filters(self.target_name, self.func_name, result, ctx) if self.process_cb: + # set the context for the process_cb only + set_call_context(ctx) self.cb_kwargs[CollabMethodArgName.CONTEXT] = ctx check_context_support(self.process_cb, self.cb_kwargs) result = self.process_cb(result, **self.cb_kwargs) + + # set back to original context + set_call_context(self.context) self.result = result self.resp_time = time.time() diff --git a/nvflare/fox/api/run_server.py b/nvflare/fox/api/run_server.py index 04b0593dd3..7b980299b6 100644 --- a/nvflare/fox/api/run_server.py +++ b/nvflare/fox/api/run_server.py @@ -14,7 +14,8 @@ import traceback from .app import ServerApp -from .constants import ContextKey +from .constants import CollabMethodArgName, ContextKey +from .dec import supports_context def run_server(server_app: ServerApp, logger): @@ -22,18 +23,20 @@ def run_server(server_app: ServerApp, logger): logger.info("initializing server app") server_app.initialize(server_ctx) - if not server_app.strategies: - raise RuntimeError("server app does not have any strategies!") + if not server_app.algos: + raise RuntimeError("server app does not have any algos!") result = None - for name, strategy in server_app.strategies: + for name, f in server_app.algos: if server_ctx.is_aborted(): break try: - logger.info(f"Running Strategy {name} - {type(strategy).__name__}") - server_app.current_strategy = strategy - result = strategy.execute(context=server_ctx) + logger.info(f"Running algo {name}") + kwargs = {CollabMethodArgName.CONTEXT: server_ctx} + if not supports_context(f): + kwargs = {} + result = f(**kwargs) server_ctx.set_prop(ContextKey.INPUT, result) except: traceback.print_exc() diff --git a/nvflare/fox/api/strategy.py b/nvflare/fox/api/strategy.py deleted file mode 100644 index c28f4b50ae..0000000000 --- a/nvflare/fox/api/strategy.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from abc import ABC, abstractmethod - -from .ctx import Context - - -class Strategy(ABC): - - @abstractmethod - def execute(self, context: Context): - pass diff --git a/nvflare/fox/examples/np/algos/avg_stream.py b/nvflare/fox/examples/np/algos/avg_stream.py index 6560f1ffb3..fe67f54836 100644 --- a/nvflare/fox/examples/np/algos/avg_stream.py +++ b/nvflare/fox/examples/np/algos/avg_stream.py @@ -14,12 +14,9 @@ import os.path import uuid -from nvflare.fox.api.app import ClientApp -from nvflare.fox.api.constants import ContextKey, EnvType -from nvflare.fox.api.ctx import Context +from nvflare.fox import fox +from nvflare.fox.api.constants import EnvType from nvflare.fox.api.dec import collab -from nvflare.fox.api.group import all_clients -from nvflare.fox.api.strategy import Strategy from nvflare.fox.examples.np.algos.utils import load_np_model, parse_array_def, save_np_model from nvflare.fox.sys.downloader import Downloader, download_file from nvflare.fuel.utils.log_utils import get_obj_logger @@ -32,7 +29,7 @@ def __init__(self): self.count = 0 -class NPFedAvgStream(Strategy): +class NPFedAvgStream: def __init__(self, initial_model, num_rounds=10, timeout=2.0): self.num_rounds = num_rounds @@ -42,30 +39,30 @@ def __init__(self, initial_model, num_rounds=10, timeout=2.0): self.logger = get_obj_logger(self) self._init_model = parse_array_def(initial_model) - def execute(self, context: Context): - self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") - current_model = context.get_prop(ContextKey.INPUT, self._init_model) + @fox.algo + def execute(self): + self.logger.info(f"[{fox.call_info}] Start training for {self.num_rounds} rounds") + current_model = fox.get_input(self._init_model) for i in range(self.num_rounds): - current_model = self._do_one_round(i, current_model, context) + current_model = self._do_one_round(i, current_model) self.logger.info(f"FINAL MODEL: {current_model}") return current_model - def _do_one_round(self, r, current_model, ctx: Context): + def _do_one_round(self, r, current_model): aggr_result = _AggrResult() - grp = all_clients( - ctx, + grp = fox.clients( process_resp_cb=self._accept_train_result, aggr_result=aggr_result, ) # pretend the model is big file_name = None - if ctx.env_type == EnvType.SYSTEM: + if fox.env_type == EnvType.SYSTEM: file_name = f"/tmp/np_{str(uuid.uuid4())}.npy" save_np_model(current_model, file_name) downloader = Downloader( num_receivers=grp.size, - ctx=ctx, + ctx=fox.context, timeout=5.0, ) model = downloader.add_file(file_name=file_name, file_downloaded_cb=self._model_downloaded) @@ -84,15 +81,15 @@ def _do_one_round(self, r, current_model, ctx: Context): return None else: result = aggr_result.total / aggr_result.count - self.logger.info(f"[{ctx.header_str()}] round {r}: aggr result from {aggr_result.count} clients: {result}") + self.logger.info(f"[{fox.call_info}] round {r}: aggr result from {aggr_result.count} clients: {result}") return result - def _accept_train_result(self, result, aggr_result: _AggrResult, context: Context): - self.logger.info(f"[{context.header_str()}] got train result from {context.caller}: {result}") + def _accept_train_result(self, result, aggr_result: _AggrResult): + self.logger.info(f"[{fox.call_info}] got train result from {fox.caller}: {result}") model, model_type = result if model_type == "ref": - err, file_path = download_file(ref=model, per_request_timeout=5.0, ctx=context) + err, file_path = download_file(ref=model, per_request_timeout=5.0, ctx=fox.context) if err: raise RuntimeError(f"failed to download model file {model}: {err}") self.logger.info(f"downloaded model file to {file_path}") @@ -107,20 +104,20 @@ def _model_downloaded(self, to_site: str, status: str, file_name): self.logger.info(f"model file {file_name} downloaded by {to_site}: {status=}") -class NPTrainer(ClientApp): +class NPTrainer: def __init__(self, delta: float): - ClientApp.__init__(self) self.delta = delta + self.logger = get_obj_logger(self) @collab - def train(self, current_round, weights, model_type: str, context: Context): - if context.is_aborted(): + def train(self, current_round, weights, model_type: str): + if fox.is_aborted: self.logger.debug("training aborted") return 0 - self.logger.debug(f"[{context.header_str()}] training round {current_round}: {model_type=} {weights=}") + self.logger.debug(f"[{fox.call_info}] training round {current_round}: {model_type=} {weights=}") if model_type == "ref": - err, file_path = download_file(ref=weights, per_request_timeout=5.0, ctx=context) + err, file_path = download_file(ref=weights, per_request_timeout=5.0, ctx=fox.context) if err: raise RuntimeError(f"failed to download model file {weights}: {err}") self.logger.info(f"downloaded model file to {file_path}") @@ -136,7 +133,7 @@ def train(self, current_round, weights, model_type: str, context: Context): save_np_model(result, file_name) downloader = Downloader( num_receivers=1, - ctx=context, + ctx=fox.context, timeout=5.0, ) result = downloader.add_file(file_name=file_name, file_downloaded_cb=self._result_downloaded) diff --git a/nvflare/fox/examples/np/algos/client.py b/nvflare/fox/examples/np/algos/client.py index fa5f29af6b..834ffadd6c 100644 --- a/nvflare/fox/examples/np/algos/client.py +++ b/nvflare/fox/examples/np/algos/client.py @@ -14,26 +14,24 @@ import random from nvflare.fox import fox -from nvflare.fox.api.app import ClientApp -from nvflare.fox.api.ctx import Context -from nvflare.fox.api.group import all_children +from nvflare.fuel.utils.log_utils import get_obj_logger -class NPTrainer(ClientApp): +class NPTrainer: def __init__(self, delta: float): - ClientApp.__init__(self) self.delta = delta + self.logger = get_obj_logger(self) @fox.init def init_trainer(self): delta_config = fox.get_prop("client_delta", {}) - self.delta = delta_config.get(self.name, self.delta) - self.logger.info(f"init_trainer: client {self.name}: delta={self.delta}") + self.delta = delta_config.get(fox.site_name, self.delta) + self.logger.info(f"init_trainer: client {fox.site_name}: delta={self.delta}") @fox.init def init_trainer2(self): - self.logger.info(f"init_trainer2: client {self.name}: init again") + self.logger.info(f"init_trainer2: client {fox.site_name}: init again") @fox.collab def train(self, current_round, weights): @@ -47,7 +45,7 @@ def train(self, current_round, weights): # self.server.accept_metric({"round": r, "y": 2}) # self.server.metric_receiver.accept_metric({"round": r, "y": 2}) # - self.server.fire_event("metrics", {"round": current_round, "y": 10}, _blocking=False) + fox.server.fire_event("metrics", {"round": current_round, "y": 10}, _blocking=False) return weights + self.delta @fox.collab @@ -56,52 +54,40 @@ def evaluate(self, model): return random.random() -class NPHierarchicalTrainer(ClientApp): +class NPHierarchicalTrainer: def __init__(self, delta: float): - ClientApp.__init__(self) self.delta = delta + self.logger = get_obj_logger(self) @fox.collab - def train(self, current_round, weights, context: Context): - if context.is_aborted(): + def train(self, current_round, weights): + if fox.is_aborted: self.logger.debug("training aborted") return 0 - self.logger.debug(f"[{context.header_str()}] training round {current_round}") - if context.app.has_children(): + self.logger.debug(f"[{fox.call_info}] training round {current_round}") + if fox.has_children: total = 0 - results = all_children(context).train(current_round, weights) + results = fox.child_clients.train(current_round, weights) for n, v in results.items(): total += v result = total / len(results) - self.logger.debug(f"[{context.header_str()}]: aggr result from children of round {current_round}: {result}") + self.logger.debug(f"[{fox.call_info}]: aggr result from children of round {current_round}: {result}") else: - result = self._local_train(current_round, weights, context) - self.logger.debug(f"[{context.header_str()}]: local train result of round {current_round}: {result}") + result = self._local_train(current_round, weights) + self.logger.debug(f"[{fox.call_info}]: local train result of round {current_round}: {result}") + fox.server.fire_event("metrics", {"round": current_round, "y": 10}, _blocking=False) return result - def _local_train(self, current_round, weights, context: Context): - if context.is_aborted(): + def _local_train(self, current_round, weights): + if fox.is_aborted: self.logger.debug("training aborted") return 0 - self.logger.info(f"[{context.header_str()}] local trained round {current_round} {weights} {type(weights)}") + self.logger.info(f"[{fox.call_info}] local trained round {current_round} {weights} {type(weights)}") return weights + self.delta @fox.collab - def evaluate(self, model, context: Context): - self.logger.debug(f"[{context.header_str()}] evaluate") + def evaluate(self, model): + self.logger.debug(f"[{fox.call_info}] evaluate") return random.random() - - -class NPTrainerMaker(ClientApp): - - def __init__(self, delta): - ClientApp.__init__(self) - self.delta = delta - - def make_client_app(self, name: str) -> ClientApp: - app = NPTrainer(self.delta) - app.update_props(self.get_props()) - app.name = name - return app diff --git a/nvflare/fox/examples/np/algos/strategies/avg_h.py b/nvflare/fox/examples/np/algos/strategies/avg_h.py index 6397851f9f..ab603d7d86 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_h.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_h.py @@ -12,14 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. from nvflare.fox import fox -from nvflare.fox.api.constants import ContextKey -from nvflare.fox.api.ctx import Context -from nvflare.fox.api.strategy import Strategy from nvflare.fox.examples.np.algos.utils import parse_array_def from nvflare.fuel.utils.log_utils import get_obj_logger -class NPHierarchicalFedAvg(Strategy): +class NPHierarchicalFedAvg: def __init__(self, initial_model, num_rounds=10): self.num_rounds = num_rounds @@ -28,13 +25,14 @@ def __init__(self, initial_model, num_rounds=10): self.name = self.__class__.__name__ self.logger = get_obj_logger(self) - def execute(self, context: Context): + @fox.algo + def execute(self): self.logger.info(f"[{fox.call_info}] Start training for {self.num_rounds} rounds") - current_model = context.get_prop(ContextKey.INPUT, self._initial_model) + current_model = fox.get_input(self._initial_model) for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model) score = self._do_eval(current_model) - self.logger.info(f"[{context.header_str()}]: eval score in round {i}: {score}") + self.logger.info(f"[{fox.call_info}]: eval score in round {i}: {score}") self.logger.info(f"FINAL MODEL: {current_model}") return current_model diff --git a/nvflare/fox/examples/np/algos/strategies/avg_intime.py b/nvflare/fox/examples/np/algos/strategies/avg_intime.py index 968028334a..ec9d69d13d 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_intime.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_intime.py @@ -13,8 +13,6 @@ # limitations under the License. from nvflare.fox import fox from nvflare.fox.api.constants import ContextKey -from nvflare.fox.api.ctx import Context -from nvflare.fox.api.strategy import Strategy from nvflare.fox.examples.np.algos.utils import parse_array_def from nvflare.fuel.utils.log_utils import get_obj_logger @@ -26,7 +24,7 @@ def __init__(self): self.count = 0 -class NPFedAvgInTime(Strategy): +class NPFedAvgInTime: def __init__(self, initial_model, num_rounds=10, timeout=2.0): self.num_rounds = num_rounds @@ -36,13 +34,14 @@ def __init__(self, initial_model, num_rounds=10, timeout=2.0): self.logger = get_obj_logger(self) self._init_model = parse_array_def(initial_model) - def execute(self, context: Context): - self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") - current_model = context.get_prop(ContextKey.INPUT, self._init_model) + @fox.algo + def execute(self): + self.logger.info(f"[{fox.call_info}] Start training for {self.num_rounds} rounds") + current_model = fox.get_prop(ContextKey.INPUT, self._init_model) for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model) score = self._do_eval(current_model) - self.logger.info(f"[{context.header_str()}]: eval score in round {i}: {score}") + self.logger.info(f"[{fox.call_info}]: eval score in round {i}: {score}") self.logger.info(f"FINAL MODEL: {current_model}") return current_model @@ -50,13 +49,16 @@ def _do_eval(self, model): results = fox.clients.evaluate(model) total = 0.0 for n, v in results.items(): - self.logger.info(f"[{fox.context.header_str()}]: got eval result from client {n}: {v}") + self.logger.info(f"[{fox.call_info}]: got eval result from client {n}: {v}") total += v return total / len(results) def _do_one_round(self, r, current_model): aggr_result = _AggrResult() + timeout = fox.get_prop("default_timeout", 10) + self.logger.info(f"got timeout: {timeout}") fox.clients( + timeout=timeout, process_resp_cb=self._accept_train_result, aggr_result=aggr_result, ).train(r, current_model) @@ -65,9 +67,7 @@ def _do_one_round(self, r, current_model): return None else: result = aggr_result.total / aggr_result.count - self.logger.info( - f"[{fox.context.header_str()}] round {r}: aggr result from {aggr_result.count} clients: {result}" - ) + self.logger.info(f"[{fox.call_info}] round {r}: aggr result from {aggr_result.count} clients: {result}") return result def _accept_train_result(self, result, aggr_result: _AggrResult): diff --git a/nvflare/fox/examples/np/algos/strategies/avg_para.py b/nvflare/fox/examples/np/algos/strategies/avg_para.py index bfbc558bbf..1346789e61 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_para.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_para.py @@ -12,14 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. from nvflare.fox import fox -from nvflare.fox.api.constants import ContextKey -from nvflare.fox.api.ctx import Context -from nvflare.fox.api.strategy import Strategy from nvflare.fox.examples.np.algos.utils import parse_array_def from nvflare.fuel.utils.log_utils import get_obj_logger -class NPFedAvgParallel(Strategy): +class NPFedAvgParallel: def __init__(self, initial_model, num_rounds=10): self.num_rounds = num_rounds @@ -28,13 +25,14 @@ def __init__(self, initial_model, num_rounds=10): self.name = "NPFedAvgParallel" self.logger = get_obj_logger(self) - def execute(self, context: Context): - self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") - current_model = context.get_prop(ContextKey.INPUT, self._initial_model) + @fox.algo + def execute(self): + self.logger.info(f"[{fox.call_info}] Start training for {self.num_rounds} rounds") + current_model = fox.get_input(self._initial_model) for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model) score = self._do_eval(current_model) - self.logger.info(f"[{context.header_str()}]: eval score in round {i}: {score}") + self.logger.info(f"[{fox.call_info}]: eval score in round {i}: {score}") return current_model def _do_eval(self, model): @@ -49,6 +47,6 @@ def _do_one_round(self, r, current_model): total = 0 results = fox.clients(timeout=4, min_resps=2, wait_after_min_resps=1).train(r, current_model) for n, v in results.items(): - self.logger.info(f"[{fox.context.header_str()}] round {r}: got group result from client {n}: {v}") + self.logger.info(f"[{fox.call_info}] round {r}: got group result from client {n}: {v}") total += v return total / len(results) diff --git a/nvflare/fox/examples/np/algos/strategies/avg_seq.py b/nvflare/fox/examples/np/algos/strategies/avg_seq.py index 643297e561..aa71047959 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_seq.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_seq.py @@ -14,17 +14,13 @@ import os from nvflare.fox import fox -from nvflare.fox.api.constants import ContextKey -from nvflare.fox.api.ctx import Context -from nvflare.fox.api.strategy import Strategy from nvflare.fox.examples.np.algos.utils import parse_array_def, save_np_model from nvflare.fuel.utils.log_utils import get_obj_logger -class NPFedAvgSequential(Strategy): +class NPFedAvgSequential: def __init__(self, initial_model, num_rounds=10): - Strategy.__init__(self) self.name = "NPFedAvgSequential" self.num_rounds = num_rounds self.initial_model = initial_model # need to remember init for job API to work! @@ -50,14 +46,15 @@ def init(self): self.client_weights = client_weights self.logger.info("client_weights: {}".format(client_weights)) - def execute(self, context: Context): - self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") - current_model = context.get_prop(ContextKey.INPUT, self._initial_model) + @fox.algo + def execute(self): + self.logger.info(f"[{fox.call_info}] Start training for {self.num_rounds} rounds") + current_model = fox.get_input(self._initial_model) for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model) # save model to work dir - file_name = os.path.join(context.workspace.get_work_dir(), "model.npy") + file_name = os.path.join(fox.workspace.get_work_dir(), "model.npy") save_np_model(current_model, file_name) return current_model diff --git a/nvflare/fox/examples/np/algos/strategies/cyclic.py b/nvflare/fox/examples/np/algos/strategies/cyclic.py index 84386e6fe5..87810b5fc3 100644 --- a/nvflare/fox/examples/np/algos/strategies/cyclic.py +++ b/nvflare/fox/examples/np/algos/strategies/cyclic.py @@ -13,14 +13,12 @@ # limitations under the License. import random -from nvflare.fox.api.constants import ContextKey -from nvflare.fox.api.ctx import Context -from nvflare.fox.api.strategy import Strategy +from nvflare.fox import fox from nvflare.fox.examples.np.algos.utils import parse_array_def from nvflare.fuel.utils.log_utils import get_obj_logger -class NPCyclic(Strategy): +class NPCyclic: def __init__(self, initial_model, num_rounds=2): self.num_rounds = num_rounds @@ -28,16 +26,17 @@ def __init__(self, initial_model, num_rounds=2): self._initial_model = parse_array_def(initial_model) self.logger = get_obj_logger(self) - def execute(self, context: Context): - current_model = context.get_prop(ContextKey.INPUT, self._initial_model) + @fox.algo + def execute(self): + current_model = fox.get_input(self._initial_model) for current_round in range(self.num_rounds): - current_model = self._do_one_round(current_round, current_model, context) - self.logger.info(f"[{context.header_str()}] final result: {current_model}") + current_model = self._do_one_round(current_round, current_model) + self.logger.info(f"[{fox.call_info}] final result: {current_model}") return current_model - def _do_one_round(self, current_round, current_model, ctx: Context): - random.shuffle(ctx.clients) - for c in ctx.clients: + def _do_one_round(self, current_round, current_model): + random.shuffle(fox.clients) + for c in fox.clients: current_model = c.train(current_round, current_model) - self.logger.info(f"[{ctx.header_str()}] result from {c.name}: {current_model}") + self.logger.info(f"[{fox.call_info}] result from {c.name}: {current_model}") return current_model diff --git a/nvflare/fox/examples/np/algos/swarm.py b/nvflare/fox/examples/np/algos/swarm.py index 5c022750fe..7d2797c215 100644 --- a/nvflare/fox/examples/np/algos/swarm.py +++ b/nvflare/fox/examples/np/algos/swarm.py @@ -16,14 +16,11 @@ import traceback from nvflare.fox import fox -from nvflare.fox.api.app import ClientApp -from nvflare.fox.api.ctx import Context -from nvflare.fox.api.strategy import Strategy from nvflare.fox.examples.np.algos.utils import parse_array_def from nvflare.fuel.utils.log_utils import get_obj_logger -class NPSwarm(Strategy): +class NPSwarm: def __init__(self, initial_model, num_rounds=10): self.num_rounds = num_rounds @@ -32,14 +29,15 @@ def __init__(self, initial_model, num_rounds=10): self.waiter = threading.Event() self.logger = get_obj_logger(self) - def execute(self, context: Context): - context.app.register_event_handler("all_done", self._all_done) + @fox.algo + def execute(self): + fox.register_event_handler("all_done", self._all_done) # randomly pick a client to start - start_client_idx = random.randint(0, len(context.clients) - 1) - start_client = context.clients[start_client_idx] + start_client_idx = random.randint(0, len(fox.clients) - 1) + start_client = fox.clients[start_client_idx] start_client.start(self.num_rounds, self._initial_model) - while not context.is_aborted(): + while not fox.is_aborted: if self.waiter.wait(timeout=0.5): break @@ -53,12 +51,15 @@ def all_done(self, reason: str): self.waiter.set() -class NPSwarmClient(ClientApp): +class NPSwarmClient: def __init__(self, delta: float): - super().__init__() self.delta = delta - self.register_event_handler("final_model", self._accept_final_model) + self.logger = get_obj_logger(self) + + @fox.init + def init(self): + fox.register_event_handler("final_model", self._accept_final_model) @fox.collab def train(self, weights, current_round): @@ -66,8 +67,7 @@ def train(self, weights, current_round): return weights + self.delta def sag(self, model, current_round): - # results = EZ.clients.train(model, current_round) - results = fox.other_clients.train(model, current_round) + results = fox.clients.train(model, current_round) results = list(results.values()) total = 0 for i in range(len(results)): @@ -86,7 +86,7 @@ def swarm_learn(self, num_rounds, model, current_round): # self.server.fire_event("all_done", "OK", blocking=False) self.logger.info("notify server all done!") try: - self.server.all_done("OK", _blocking=False) + fox.server.all_done("OK", _blocking=False) except: traceback.print_exc() self.logger.info("Swarm Training is DONE!") @@ -94,9 +94,9 @@ def swarm_learn(self, num_rounds, model, current_round): # determine next client next_round = current_round + 1 - next_client_idx = random.randint(0, len(self.clients) - 1) + next_client_idx = random.randint(0, len(fox.clients) - 1) self.logger.debug(f"chose aggr client for round {next_round}: {next_client_idx}") - next_client = self.clients[next_client_idx] + next_client = fox.clients[next_client_idx] next_client.swarm_learn(num_rounds, new_model, next_round, _blocking=False) @fox.collab diff --git a/nvflare/fox/examples/np/algos/widgets.py b/nvflare/fox/examples/np/algos/widgets.py index 6a3977923d..79abe591dc 100644 --- a/nvflare/fox/examples/np/algos/widgets.py +++ b/nvflare/fox/examples/np/algos/widgets.py @@ -30,4 +30,4 @@ def init(self): self.logger.info("MetricReceiver initialized!") def _accept_metric(self, event_type: str, data): - self.logger.info(f"[{fox.callee}] received event '{event_type}' from {fox.caller}: {data}") + self.logger.info(f"[{fox.callee}] received metrics event '{event_type}' from {fox.caller}: {data}") diff --git a/nvflare/fox/examples/np/cyclic.py b/nvflare/fox/examples/np/cyclic.py index e0194ab42e..0714c0bdbf 100644 --- a/nvflare/fox/examples/np/cyclic.py +++ b/nvflare/fox/examples/np/cyclic.py @@ -13,7 +13,7 @@ # limitations under the License. import logging -from nvflare.fox.api.app import ServerApp +from nvflare.fox.api.app import ClientApp, ServerApp from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.strategies.cyclic import NPCyclic @@ -26,14 +26,13 @@ def main(): simulator = Simulator( root_dir="/tmp/fox", experiment_name="cyclic", - server_app=ServerApp( - strategy_name="cyclic", strategy=NPCyclic(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2) - ), - client_app=NPTrainer(delta=1.0), + server_app=ServerApp(NPCyclic(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2)), + client_app=ClientApp(NPTrainer(delta=1.0)), num_clients=2, ) - simulator.run() + final_result = simulator.run() + print(f"final model: {final_result}") if __name__ == "__main__": diff --git a/nvflare/fox/examples/np/cyclic2.py b/nvflare/fox/examples/np/cyclic2.py new file mode 100644 index 0000000000..7dac7ac24b --- /dev/null +++ b/nvflare/fox/examples/np/cyclic2.py @@ -0,0 +1,38 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples.np.algos.client import NPTrainer +from nvflare.fox.examples.np.algos.strategies.cyclic import NPCyclic +from nvflare.fox.sim.sim2 import Simulator + + +def main(): + simple_logging(logging.DEBUG) + + simulator = Simulator( + root_dir="/tmp/fox", + experiment_name="cyclic", + server=NPCyclic(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), + client=NPTrainer(delta=1.0), + num_clients=2, + ) + + final_result = simulator.run() + print(f"final model: {final_result}") + + +if __name__ == "__main__": + main() diff --git a/nvflare/fox/examples/np/cyclic_avg.py b/nvflare/fox/examples/np/cyclic_avg.py index dbcdb2c89d..3b795b31f0 100644 --- a/nvflare/fox/examples/np/cyclic_avg.py +++ b/nvflare/fox/examples/np/cyclic_avg.py @@ -13,28 +13,51 @@ # limitations under the License. import logging -from nvflare.fox.api.app import ServerApp +from nvflare.fox import fox +from nvflare.fox.api.app import ClientApp, ServerApp from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.strategies.avg_para import NPFedAvgParallel from nvflare.fox.examples.np.algos.strategies.cyclic import NPCyclic from nvflare.fox.sim.simulator import Simulator +from nvflare.fuel.utils.log_utils import get_obj_logger + + +class Controller: + + def __init__( + self, + initial_model, + cyclic_rounds, + avg_rounds, + ): + self.initial_model = initial_model + self.cyclic_rounds = cyclic_rounds + self.avg_rounds = avg_rounds + self.logger = get_obj_logger(self) + + @fox.algo + def run(self): + self.logger.info("running cyclic ...") + ctl = NPCyclic(self.initial_model, num_rounds=self.cyclic_rounds) + result = ctl.execute() + + self.logger.info("running fed-avg ...") + ctl = NPFedAvgParallel(initial_model=result, num_rounds=self.avg_rounds) + return ctl.execute() def main(): simple_logging(logging.DEBUG) exp_name = "cyclic_avg" - server_app = ServerApp( - strategy_name=exp_name, strategy=NPCyclic(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2) - ) - server_app.add_strategy("fed_avg_parallel", NPFedAvgParallel(initial_model=None, num_rounds=2)) + server_app = ServerApp(Controller(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], cyclic_rounds=2, avg_rounds=3)) simulator = Simulator( root_dir="/tmp/fox", experiment_name=exp_name, server_app=server_app, - client_app=NPTrainer(delta=1.0), + client_app=ClientApp(NPTrainer(delta=1.0)), num_clients=2, ) diff --git a/nvflare/fox/examples/np/fed_avg_h.py b/nvflare/fox/examples/np/fed_avg_h.py index 55dee43e1b..42501a6f50 100644 --- a/nvflare/fox/examples/np/fed_avg_h.py +++ b/nvflare/fox/examples/np/fed_avg_h.py @@ -13,7 +13,7 @@ # limitations under the License. import logging -from nvflare.fox.api.app import ServerApp +from nvflare.fox.api.app import ClientApp, ServerApp from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.client import NPHierarchicalTrainer from nvflare.fox.examples.np.algos.strategies.avg_h import NPHierarchicalFedAvg @@ -25,8 +25,7 @@ def main(): simple_logging(logging.DEBUG) server_app = ServerApp( - strategy_name="fed_avg", - strategy=NPHierarchicalFedAvg(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=3), + obj=NPHierarchicalFedAvg(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=3), ) server_app.add_collab_object("metric_receiver", MetricReceiver()) @@ -34,7 +33,7 @@ def main(): root_dir="/tmp/fox", experiment_name="fedavg_h", server_app=server_app, - client_app=NPHierarchicalTrainer(delta=1.0), + client_app=ClientApp(NPHierarchicalTrainer(delta=1.0)), num_clients=(3, 2), ) diff --git a/nvflare/fox/examples/np/fed_avg_h2.py b/nvflare/fox/examples/np/fed_avg_h2.py new file mode 100644 index 0000000000..6fbcc990c1 --- /dev/null +++ b/nvflare/fox/examples/np/fed_avg_h2.py @@ -0,0 +1,40 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples.np.algos.client import NPHierarchicalTrainer +from nvflare.fox.examples.np.algos.strategies.avg_h import NPHierarchicalFedAvg +from nvflare.fox.examples.np.algos.widgets import MetricReceiver +from nvflare.fox.sim.sim2 import Simulator + + +def main(): + simple_logging(logging.DEBUG) + + simulator = Simulator( + root_dir="/tmp/fox", + experiment_name="fedavg_h", + server=NPHierarchicalFedAvg(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=3), + client=NPHierarchicalTrainer(delta=1.0), + server_objects={"metric_receiver": MetricReceiver()}, + num_clients=(3, 2), + ) + + result = simulator.run() + print(f"Final Result: {result}") + + +if __name__ == "__main__": + main() diff --git a/nvflare/fox/examples/np/fed_avg_intime.py b/nvflare/fox/examples/np/fed_avg_intime.py index 56f1efd286..785fcd87ff 100644 --- a/nvflare/fox/examples/np/fed_avg_intime.py +++ b/nvflare/fox/examples/np/fed_avg_intime.py @@ -13,7 +13,7 @@ # limitations under the License. import logging -from nvflare.fox.api.app import ServerApp +from nvflare.fox.api.app import ClientApp, ServerApp from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.filters import AddNoiseToModel, PrintCall, PrintResult @@ -26,8 +26,7 @@ def main(): simple_logging(logging.DEBUG) server_app = ServerApp( - strategy_name="fed_avg_in_time", - strategy=NPFedAvgInTime(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), + NPFedAvgInTime(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), ) server_app.add_collab_object("metric_receiver", MetricReceiver()) @@ -35,10 +34,10 @@ def main(): server_app.add_incoming_result_filters("*.train", [PrintResult()]) server_app.set_prop("default_timeout", 5.0) - client_app = NPTrainer(delta=1.0) + client_app = ClientApp(NPTrainer(delta=1.0)) client_app.add_incoming_call_filters("*.train", [PrintCall()]) client_app.add_outgoing_result_filters("*.train", [PrintResult()]) - server_app.set_prop("default_timeout", 8.0) + client_app.set_prop("default_timeout", 8.0) simulator = Simulator( root_dir="/tmp/fox", @@ -48,7 +47,8 @@ def main(): num_clients=2, ) - simulator.run() + result = simulator.run() + print(f"final model: {result}") if __name__ == "__main__": diff --git a/nvflare/fox/examples/np/fed_avg_intime2.py b/nvflare/fox/examples/np/fed_avg_intime2.py new file mode 100644 index 0000000000..cc7a16c308 --- /dev/null +++ b/nvflare/fox/examples/np/fed_avg_intime2.py @@ -0,0 +1,49 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples.np.algos.client import NPTrainer +from nvflare.fox.examples.np.algos.filters import AddNoiseToModel, PrintCall, PrintResult +from nvflare.fox.examples.np.algos.strategies.avg_intime import NPFedAvgInTime +from nvflare.fox.examples.np.algos.widgets import MetricReceiver +from nvflare.fox.sim.sim2 import Simulator + + +def main(): + simple_logging(logging.DEBUG) + + simulator = Simulator( + root_dir="/tmp/fox", + experiment_name="fedavg_intime", + server=NPFedAvgInTime(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), + client=NPTrainer(delta=1.0), + server_objects={"metric_receiver": MetricReceiver()}, + num_clients=2, + ) + + simulator.add_server_outgoing_call_filters("*.train", [AddNoiseToModel()]) + simulator.add_server_incoming_result_filters("*.train", [PrintResult()]) + simulator.set_server_prop("default_timeout", 5.0) + + simulator.add_client_incoming_call_filters("*.train", [PrintCall()]) + simulator.add_client_outgoing_result_filters("*.train", [PrintResult()]) + simulator.set_client_prop("default_timeout", 8.0) + + result = simulator.run() + print(f"final model: {result}") + + +if __name__ == "__main__": + main() diff --git a/nvflare/fox/examples/np/fed_avg_para.py b/nvflare/fox/examples/np/fed_avg_para.py index 42de4dc91f..3aab541e7d 100644 --- a/nvflare/fox/examples/np/fed_avg_para.py +++ b/nvflare/fox/examples/np/fed_avg_para.py @@ -13,7 +13,7 @@ # limitations under the License. import logging -from nvflare.fox.api.app import ServerApp +from nvflare.fox.api.app import ClientApp, ServerApp from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.strategies.avg_para import NPFedAvgParallel @@ -25,8 +25,7 @@ def main(): simple_logging(logging.DEBUG) server_app = ServerApp( - strategy_name="fed_avg", - strategy=NPFedAvgParallel(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]], num_rounds=2), + NPFedAvgParallel(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]], num_rounds=2), ) server_app.add_collab_object("metric_receiver", MetricReceiver()) @@ -34,7 +33,7 @@ def main(): root_dir="/tmp/fox", experiment_name="fedavg_para", server_app=server_app, - client_app=NPTrainer(delta=1.0), + client_app=ClientApp(NPTrainer(delta=1.0)), num_clients=10, ) diff --git a/nvflare/fox/examples/np/fed_avg_seq.py b/nvflare/fox/examples/np/fed_avg_seq.py index 2309b18d5a..4aed9248fb 100644 --- a/nvflare/fox/examples/np/fed_avg_seq.py +++ b/nvflare/fox/examples/np/fed_avg_seq.py @@ -13,9 +13,9 @@ # limitations under the License. import logging -from nvflare.fox.api.app import ServerApp +from nvflare.fox.api.app import ClientApp, ServerApp from nvflare.fox.api.utils import simple_logging -from nvflare.fox.examples.np.algos.client import NPTrainerMaker +from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.strategies.avg_seq import NPFedAvgSequential from nvflare.fox.examples.np.algos.widgets import MetricReceiver from nvflare.fox.sim.simulator import Simulator @@ -25,8 +25,7 @@ def main(): simple_logging(logging.DEBUG) server_app = ServerApp( - strategy_name="fed_avg", - strategy=NPFedAvgSequential( + NPFedAvgSequential( num_rounds=2, initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], ), @@ -40,7 +39,7 @@ def main(): }, ) - client_app = NPTrainerMaker(delta=1.0) + client_app = ClientApp(NPTrainer(delta=1.0)) client_app.set_prop( "client_delta", { diff --git a/nvflare/fox/examples/np/fed_avg_stream.py b/nvflare/fox/examples/np/fed_avg_stream.py index 8575976e26..f138cc1bc5 100644 --- a/nvflare/fox/examples/np/fed_avg_stream.py +++ b/nvflare/fox/examples/np/fed_avg_stream.py @@ -13,7 +13,7 @@ # limitations under the License. import logging -from nvflare.fox.api.app import ServerApp +from nvflare.fox.api.app import ClientApp, ServerApp from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.avg_stream import NPFedAvgStream, NPTrainer from nvflare.fox.sim.simulator import Simulator @@ -23,11 +23,10 @@ def main(): simple_logging(logging.DEBUG) server_app = ServerApp( - strategy_name="fed_avg_in_time", - strategy=NPFedAvgStream(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=4), + NPFedAvgStream(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=4), ) - client_app = NPTrainer(delta=1.0) + client_app = ClientApp(NPTrainer(delta=1.0)) simulator = Simulator( root_dir="/tmp/fox", diff --git a/nvflare/fox/examples/np/recipe_swarm.py b/nvflare/fox/examples/np/recipe_swarm.py index 3e126db8f2..0650926da1 100644 --- a/nvflare/fox/examples/np/recipe_swarm.py +++ b/nvflare/fox/examples/np/recipe_swarm.py @@ -13,7 +13,7 @@ # limitations under the License. import logging -from nvflare.fox.api.app import ServerApp +from nvflare.fox.api.app import ClientApp, ServerApp from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.swarm import NPSwarm, NPSwarmClient from nvflare.fox.sys.recipe import FoxRecipe @@ -24,10 +24,8 @@ def main(): simple_logging(logging.DEBUG) - server_app = ServerApp( - strategy_name="swarm", strategy=NPSwarm(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=5) - ) - client_app = NPSwarmClient(delta=1.0) + server_app = ServerApp(NPSwarm(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=5)) + client_app = ClientApp(NPSwarmClient(delta=1.0)) recipe = FoxRecipe( job_name="swarm", diff --git a/nvflare/fox/examples/np/swarm.py b/nvflare/fox/examples/np/swarm.py index b461ac31e2..b24bec451d 100644 --- a/nvflare/fox/examples/np/swarm.py +++ b/nvflare/fox/examples/np/swarm.py @@ -13,7 +13,7 @@ # limitations under the License. import logging -from nvflare.fox.api.app import ServerApp +from nvflare.fox.api.app import ClientApp, ServerApp from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.swarm import NPSwarm, NPSwarmClient from nvflare.fox.sim.simulator import Simulator @@ -22,10 +22,8 @@ def main(): simple_logging(logging.DEBUG) - server_app = ServerApp( - strategy_name="swarm", strategy=NPSwarm(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=5) - ) - client_app = NPSwarmClient(delta=1.0) + server_app = ServerApp(NPSwarm(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=5)) + client_app = ClientApp(NPSwarmClient(delta=1.0)) simulator = Simulator( root_dir="/tmp/fox", diff --git a/nvflare/fox/examples/pt/pt_avg_filter.py b/nvflare/fox/examples/pt/pt_avg_filter.py index 35779df6a5..ca1f6e5be8 100644 --- a/nvflare/fox/examples/pt/pt_avg_filter.py +++ b/nvflare/fox/examples/pt/pt_avg_filter.py @@ -16,15 +16,10 @@ import torch -from nvflare.fox.api.app import ClientApp, ServerApp -from nvflare.fox.api.constants import ContextKey -from nvflare.fox.api.ctx import Context -from nvflare.fox.api.dec import collab -from nvflare.fox.api.group import all_clients -from nvflare.fox.api.strategy import Strategy +from nvflare.fox import fox from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.pt.utils import parse_state_dict -from nvflare.fox.sim.simulator import Simulator +from nvflare.fox.sim.sim2 import Simulator from nvflare.fuel.utils.log_utils import get_obj_logger @@ -36,7 +31,7 @@ def __init__(self): self.lock = threading.Lock() # ensure update integrity -class PTFedAvg(Strategy): +class PTFedAvg: def __init__(self, initial_model, num_rounds=10, timeout=2.0): self.num_rounds = num_rounds @@ -46,19 +41,19 @@ def __init__(self, initial_model, num_rounds=10, timeout=2.0): self.logger = get_obj_logger(self) self._init_model = parse_state_dict(initial_model) - def execute(self, context: Context): - self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") - current_model = context.get_prop(ContextKey.INPUT, self._init_model) + @fox.algo + def execute(self): + self.logger.info(f"[{fox.call_info}] Start training for {self.num_rounds} rounds") + current_model = fox.get_input(self._init_model) for i in range(self.num_rounds): - current_model = self._do_one_round(i, current_model, context) + current_model = self._do_one_round(i, current_model) self.logger.info(f"FINAL MODEL: {current_model}") return current_model - def _do_one_round(self, r, current_model, ctx: Context): + def _do_one_round(self, r, current_model): aggr_result = _AggrResult() - all_clients( - ctx, + fox.clients( process_resp_cb=self._accept_train_result, aggr_result=aggr_result, ).train(r, current_model) @@ -69,11 +64,11 @@ def _do_one_round(self, r, current_model, ctx: Context): result = {} for k, v in aggr_result.total.items(): result[k] = torch.div(v, aggr_result.count) - self.logger.info(f"[{ctx.header_str()}] round {r}: aggr result from {aggr_result.count} clients: {result}") + self.logger.info(f"[{fox.call_info}] round {r}: aggr result from {aggr_result.count} clients: {result}") return result - def _accept_train_result(self, result, aggr_result: _AggrResult, context: Context): - self.logger.info(f"[{context.header_str()}] got train result from {context.caller}: {result}") + def _accept_train_result(self, result, aggr_result: _AggrResult): + self.logger.info(f"[{fox.call_info}] got train result from {fox.caller}: {result}") for k, v in result.items(): if k not in aggr_result.total: @@ -85,18 +80,18 @@ def _accept_train_result(self, result, aggr_result: _AggrResult, context: Contex return None -class PTTrainer(ClientApp): +class PTTrainer: def __init__(self, delta: float): - ClientApp.__init__(self) self.delta = delta + self.logger = get_obj_logger(self) - @collab - def train(self, current_round, weights, context: Context): - if context.is_aborted(): + @fox.collab + def train(self, current_round, weights): + if fox.is_aborted: self.logger.debug("training aborted") return 0 - self.logger.debug(f"[{context.header_str()}] training round {current_round}: {weights=}") + self.logger.debug(f"[{fox.call_info}] training round {current_round}: {weights=}") result = {} for k, v in weights.items(): @@ -107,25 +102,22 @@ def train(self, current_round, weights, context: Context): def main(): simple_logging(logging.DEBUG) - server_app = ServerApp( - strategy_name="fed_avg_in_time", - strategy=PTFedAvg( - initial_model={ - "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], - "y": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], - "z": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], - }, - num_rounds=4, - ), + server = PTFedAvg( + initial_model={ + "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "y": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "z": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + }, + num_rounds=4, ) - client_app = PTTrainer(delta=1.0) + client = PTTrainer(delta=1.0) simulator = Simulator( root_dir="/tmp/fox", experiment_name="pt_fedavg_intime", - server_app=server_app, - client_app=client_app, + server=server, + client=client, num_clients=2, ) diff --git a/nvflare/fox/examples/pt/pt_avg_mixed.py b/nvflare/fox/examples/pt/pt_avg_mixed.py index 215e7d71ec..4a2d3e25a8 100644 --- a/nvflare/fox/examples/pt/pt_avg_mixed.py +++ b/nvflare/fox/examples/pt/pt_avg_mixed.py @@ -18,10 +18,8 @@ import torch from nvflare.fox import fox -from nvflare.fox.api.app import ClientApp, ServerApp from nvflare.fox.api.constants import EnvType from nvflare.fox.api.ctx import Context -from nvflare.fox.api.strategy import Strategy from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.utils import add as add_np from nvflare.fox.examples.np.algos.utils import div as div_np @@ -29,7 +27,7 @@ from nvflare.fox.examples.pt.utils import add as add_pt from nvflare.fox.examples.pt.utils import div as div_pt from nvflare.fox.examples.pt.utils import parse_state_dict as parse_pt -from nvflare.fox.sim.simulator import Simulator +from nvflare.fox.sim.sim2 import Simulator from nvflare.fox.sys.downloader import Downloader, download_arrays, download_tensors from nvflare.fuel.utils.log_utils import get_obj_logger @@ -43,7 +41,7 @@ def __init__(self): self.lock = threading.Lock() # ensure update integrity -class PTFedAvgMixed(Strategy): +class PTFedAvgMixed: def __init__(self, pt_model, np_model, num_rounds=10, timeout=2.0): self.num_rounds = num_rounds @@ -55,8 +53,9 @@ def __init__(self, pt_model, np_model, num_rounds=10, timeout=2.0): self._pt_model = parse_pt(pt_model) self._np_model = parse_np(np_model) - def execute(self, context: Context): - self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") + @fox.algo + def execute(self): + self.logger.info(f"[{fox.call_info}] Start training for {self.num_rounds} rounds") pt_model, np_model = self._pt_model, self._np_model for i in range(self.num_rounds): pt_model, np_model = self._do_one_round(i, pt_model, np_model) @@ -146,11 +145,11 @@ def _aggregate_arrays(self, td: dict[str, np.ndarray], aggr_result: _AggrResult, add_np(td, aggr_result.np_total) -class PTTrainer(ClientApp): +class PTTrainer: def __init__(self, delta: float): - ClientApp.__init__(self) self.delta = delta + self.logger = get_obj_logger(self) @fox.collab def train(self, current_round, pt_model, np_model, model_type: str): @@ -203,22 +202,19 @@ def main(): "z": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], } - server_app = ServerApp( - strategy_name="fed_avg_mixed", - strategy=PTFedAvgMixed( - pt_model=init_model, - np_model=init_model, - num_rounds=4, - ), + server = PTFedAvgMixed( + pt_model=init_model, + np_model=init_model, + num_rounds=4, ) - client_app = PTTrainer(delta=1.0) + client = PTTrainer(delta=1.0) simulator = Simulator( root_dir="/tmp/fox", experiment_name="fedavg_mixed", - server_app=server_app, - client_app=client_app, + server=server, + client=client, num_clients=2, ) diff --git a/nvflare/fox/examples/pt/pt_avg_stream.py b/nvflare/fox/examples/pt/pt_avg_stream.py index f0bd1e0823..3599eed7c0 100644 --- a/nvflare/fox/examples/pt/pt_avg_stream.py +++ b/nvflare/fox/examples/pt/pt_avg_stream.py @@ -16,15 +16,12 @@ import torch -from nvflare.fox.api.app import ClientApp, ServerApp -from nvflare.fox.api.constants import ContextKey, EnvType +from nvflare.fox import fox +from nvflare.fox.api.constants import EnvType from nvflare.fox.api.ctx import Context -from nvflare.fox.api.dec import collab -from nvflare.fox.api.group import all_clients -from nvflare.fox.api.strategy import Strategy from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.pt.utils import parse_state_dict -from nvflare.fox.sim.simulator import Simulator +from nvflare.fox.sim.sim2 import Simulator from nvflare.fox.sys.downloader import Downloader, download_tensors from nvflare.fuel.utils.log_utils import get_obj_logger @@ -37,7 +34,7 @@ def __init__(self): self.lock = threading.Lock() # ensure update integrity -class PTFedAvgStream(Strategy): +class PTFedAvgStream: def __init__(self, initial_model, num_rounds=10, timeout=2.0): self.num_rounds = num_rounds @@ -47,30 +44,30 @@ def __init__(self, initial_model, num_rounds=10, timeout=2.0): self.logger = get_obj_logger(self) self._init_model = parse_state_dict(initial_model) - def execute(self, context: Context): - self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") - current_model = context.get_prop(ContextKey.INPUT, self._init_model) + @fox.algo + def execute(self): + self.logger.info(f"[{fox.call_info}] Start training for {self.num_rounds} rounds") + current_model = fox.get_input(self._init_model) for i in range(self.num_rounds): - current_model = self._do_one_round(i, current_model, context) + current_model = self._do_one_round(i, current_model) self.logger.info(f"FINAL MODEL: {current_model}") return current_model - def _do_one_round(self, r, current_model, ctx: Context): + def _do_one_round(self, r, current_model): aggr_result = _AggrResult() model2 = {} for k, v in current_model.items(): model2[k] = v + 2.0 - grp = all_clients( - ctx, + grp = fox.clients( process_resp_cb=self._accept_train_result, aggr_result=aggr_result, ) - if ctx.env_type == EnvType.SYSTEM: + if fox.env_type == EnvType.SYSTEM: downloader = Downloader( num_receivers=grp.size, - ctx=ctx, + ctx=fox.context, timeout=5.0, ) model_type = "ref" @@ -89,21 +86,21 @@ def _do_one_round(self, r, current_model, ctx: Context): result = {} for k, v in aggr_result.total.items(): result[k] = torch.div(v, aggr_result.count) - self.logger.info(f"[{ctx.header_str()}] round {r}: aggr result from {aggr_result.count} clients: {result}") + self.logger.info(f"[{fox.call_info}] round {r}: aggr result from {aggr_result.count} clients: {result}") return result - def _accept_train_result(self, result, aggr_result: _AggrResult, context: Context): - self.logger.info(f"[{context.header_str()}] got train result from {context.caller}: {result}") + def _accept_train_result(self, result, aggr_result: _AggrResult): + self.logger.info(f"[{fox.call_info}] got train result from {fox.caller}: {result}") model, model_type = result if model_type == "ref": err, model = download_tensors( ref=model, per_request_timeout=5.0, - ctx=context, + ctx=fox.context, tensors_received_cb=self._aggregate_tensors, aggr_result=aggr_result, - context=context, + context=fox.context, ) if err: raise RuntimeError(f"failed to download model {model}: {err}") @@ -127,25 +124,25 @@ def _aggregate_tensors(self, td: dict[str, torch.Tensor], aggr_result: _AggrResu aggr_result.total[k] += v -class PTTrainer(ClientApp): +class PTTrainer: def __init__(self, delta: float): - ClientApp.__init__(self) self.delta = delta + self.logger = get_obj_logger(self) - @collab - def train(self, current_round, model1, model2, model_type: str, context: Context): - if context.is_aborted(): + @fox.collab + def train(self, current_round, model1, model2, model_type: str): + if fox.is_aborted: self.logger.debug("training aborted") return 0 - self.logger.debug(f"[{context.header_str()}] training round {current_round}: {model_type=} {model1=} {model2=}") + self.logger.debug(f"[{fox.call_info}] training round {current_round}: {model_type=} {model1=} {model2=}") if model_type == "ref": - err, model1 = download_tensors(ref=model1, per_request_timeout=5.0, ctx=context) + err, model1 = download_tensors(ref=model1, per_request_timeout=5.0, ctx=fox.context) if err: raise RuntimeError(f"failed to download model1 {model1}: {err}") self.logger.info(f"downloaded model1 {model1}") - err, model2 = download_tensors(ref=model2, per_request_timeout=5.0, ctx=context) + err, model2 = download_tensors(ref=model2, per_request_timeout=5.0, ctx=fox.context) if err: raise RuntimeError(f"failed to download model2 {model2}: {err}") self.logger.info(f"downloaded model2 {model2}") @@ -162,7 +159,7 @@ def train(self, current_round, model1, model2, model_type: str, context: Context # stream it downloader = Downloader( num_receivers=1, - ctx=context, + ctx=fox.context, timeout=5.0, ) model_type = "ref" @@ -177,25 +174,22 @@ def train(self, current_round, model1, model2, model_type: str, context: Context def main(): simple_logging(logging.DEBUG) - server_app = ServerApp( - strategy_name="fed_avg_in_time", - strategy=PTFedAvgStream( - initial_model={ - "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], - "y": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], - "z": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], - }, - num_rounds=4, - ), + server = PTFedAvgStream( + initial_model={ + "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "y": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "z": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + }, + num_rounds=4, ) - client_app = PTTrainer(delta=1.0) + client = PTTrainer(delta=1.0) simulator = Simulator( root_dir="/tmp/fox", experiment_name="pt_fedavg_stream", - server_app=server_app, - client_app=client_app, + server=server, + client=client, num_clients=2, ) diff --git a/nvflare/fox/examples/pt/pt_np.py b/nvflare/fox/examples/pt/pt_np.py index 05845844da..1a12a4fbcc 100644 --- a/nvflare/fox/examples/pt/pt_np.py +++ b/nvflare/fox/examples/pt/pt_np.py @@ -14,11 +14,7 @@ import logging import threading -from nvflare.fox.api.app import ClientApp, ServerApp -from nvflare.fox.api.ctx import Context -from nvflare.fox.api.dec import collab -from nvflare.fox.api.group import all_clients -from nvflare.fox.api.strategy import Strategy +from nvflare.fox import fox from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.utils import add as add_np from nvflare.fox.examples.np.algos.utils import div as div_np @@ -26,7 +22,7 @@ from nvflare.fox.examples.pt.utils import add as add_pt from nvflare.fox.examples.pt.utils import div as div_pt from nvflare.fox.examples.pt.utils import parse_state_dict as parse_pt -from nvflare.fox.sim.simulator import Simulator +from nvflare.fox.sim.sim2 import Simulator from nvflare.fuel.utils.log_utils import get_obj_logger @@ -39,7 +35,7 @@ def __init__(self): self.lock = threading.Lock() # ensure update integrity -class PTFedAvgMixed(Strategy): +class PTFedAvgMixed: def __init__(self, pt_model, np_model, num_rounds=10, timeout=2.0): self.num_rounds = num_rounds @@ -51,24 +47,22 @@ def __init__(self, pt_model, np_model, num_rounds=10, timeout=2.0): self._pt_model = parse_pt(pt_model) self._np_model = parse_np(np_model) - def execute(self, context: Context): - self.logger.info(f"[{context.header_str()}] Start training for {self.num_rounds} rounds") + @fox.algo + def execute(self): + self.logger.info(f"[{fox.call_info}] Start training for {self.num_rounds} rounds") pt_model, np_model = self._pt_model, self._np_model for i in range(self.num_rounds): - pt_model, np_model = self._do_one_round(i, pt_model, np_model, context) + pt_model, np_model = self._do_one_round(i, pt_model, np_model) self.logger.info(f"FINAL MODEL: {pt_model=} {np_model=}") return pt_model, np_model - def _do_one_round(self, r, pt_model, np_model, ctx: Context): + def _do_one_round(self, r, pt_model, np_model): aggr_result = _AggrResult() - grp = all_clients( - ctx, + fox.clients( process_resp_cb=self._accept_train_result, aggr_result=aggr_result, - ) - - grp.train(r, pt_model, np_model) + ).train(r, pt_model, np_model) if aggr_result.count == 0: return None @@ -76,18 +70,18 @@ def _do_one_round(self, r, pt_model, np_model, ctx: Context): pt_result = aggr_result.pt_total div_pt(pt_result, aggr_result.count) self.logger.info( - f"[{ctx.header_str()}] round {r}: aggr PT result from {aggr_result.count} clients: {pt_result}" + f"[{fox.call_info}] round {r}: aggr PT result from {aggr_result.count} clients: {pt_result}" ) np_result = aggr_result.np_total div_np(np_result, aggr_result.count) self.logger.info( - f"[{ctx.header_str()}] round {r}: aggr NP result from {aggr_result.count} clients: {np_result}" + f"[{fox.call_info}] round {r}: aggr NP result from {aggr_result.count} clients: {np_result}" ) return pt_result, np_result - def _accept_train_result(self, result, aggr_result: _AggrResult, context: Context): - self.logger.info(f"[{context.header_str()}] got train result from {context.caller}: {result}") + def _accept_train_result(self, result, aggr_result: _AggrResult): + self.logger.info(f"[{fox.call_info}] got train result from {fox.caller}: {result}") pt_result, np_result = result add_pt(pt_result, aggr_result.pt_total) @@ -96,19 +90,19 @@ def _accept_train_result(self, result, aggr_result: _AggrResult, context: Contex return None -class PTTrainer(ClientApp): +class PTTrainer: def __init__(self, delta: float): - ClientApp.__init__(self) self.delta = delta + self.logger = get_obj_logger(self) - @collab - def train(self, current_round, pt_model, np_model, context: Context): - if context.is_aborted(): + @fox.collab + def train(self, current_round, pt_model, np_model): + if fox.is_aborted: self.logger.debug("training aborted") return 0 - self.logger.debug(f"[{context.header_str()}] training round {current_round}: {pt_model=} {np_model=}") + self.logger.debug(f"[{fox.call_info}] training round {current_round}: {pt_model=} {np_model=}") pt_result = {} for k, v in pt_model.items(): @@ -130,22 +124,17 @@ def main(): "z": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], } - server_app = ServerApp( - strategy_name="fed_avg", - strategy=PTFedAvgMixed( - pt_model=init_model, - np_model=init_model, - num_rounds=4, - ), + server = PTFedAvgMixed( + pt_model=init_model, + np_model=init_model, + num_rounds=4, ) - client_app = PTTrainer(delta=1.0) - simulator = Simulator( root_dir="/tmp/fox", experiment_name="pt_np", - server_app=server_app, - client_app=client_app, + server=server, + client=PTTrainer(delta=1.0), num_clients=2, ) diff --git a/nvflare/fox/sim/sim2.py b/nvflare/fox/sim/sim2.py index 4f1a4c69df..4fbb04c737 100644 --- a/nvflare/fox/sim/sim2.py +++ b/nvflare/fox/sim/sim2.py @@ -11,36 +11,39 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Tuple, Union +from typing import List, Tuple, Union from nvflare.fox.api.app import ClientApp, ServerApp -from nvflare.fox.api.strategy import Strategy +from nvflare.fox.api.filter import CallFilter, ResultFilter -from .simulator import Simulator +from .simulator import Simulator as Sim1 -class Simulator2: +class Simulator: def __init__( self, root_dir: str, experiment_name: str, - strategy: Strategy, - server_objects: dict[str, object], - client_objects: dict[str, object], + server, + client, + server_objects: dict[str, object] = None, + client_objects: dict[str, object] = None, max_workers: int = 100, num_clients: Union[int, Tuple[int, int]] = 2, ): - server_app: ServerApp = ServerApp(strategy=strategy) - client_app: ClientApp = ClientApp() + server_app: ServerApp = ServerApp(server) + client_app: ClientApp = ClientApp(client) - for name, obj in server_objects.items(): - server_app.add_collab_object(name, obj) + if server_objects: + for name, obj in server_objects.items(): + server_app.add_collab_object(name, obj) - for name, obj in client_objects.items(): - client_app.add_collab_object(name, obj) + if client_objects: + for name, obj in client_objects.items(): + client_app.add_collab_object(name, obj) - self.simulator = Simulator( + self.simulator = Sim1( root_dir=root_dir, experiment_name=experiment_name, server_app=server_app, @@ -49,5 +52,38 @@ def __init__( num_clients=num_clients, ) + self.server_app = server_app + self.client_app = client_app + + def add_server_outgoing_call_filters(self, pattern: str, filters: List[CallFilter]): + self.server_app.add_outgoing_call_filters(pattern, filters) + + def add_server_incoming_call_filters(self, pattern: str, filters: List[CallFilter]): + self.server_app.add_incoming_call_filters(pattern, filters) + + def add_server_outgoing_result_filters(self, pattern: str, filters: List[ResultFilter]): + self.server_app.add_outgoing_result_filters(pattern, filters) + + def add_server_incoming_result_filters(self, pattern: str, filters: List[ResultFilter]): + self.server_app.add_incoming_result_filters(pattern, filters) + + def add_client_outgoing_call_filters(self, pattern: str, filters: List[CallFilter]): + self.client_app.add_outgoing_call_filters(pattern, filters) + + def add_client_incoming_call_filters(self, pattern: str, filters: List[CallFilter]): + self.client_app.add_incoming_call_filters(pattern, filters) + + def add_client_outgoing_result_filters(self, pattern: str, filters: List[ResultFilter]): + self.client_app.add_outgoing_result_filters(pattern, filters) + + def add_client_incoming_result_filters(self, pattern: str, filters: List[ResultFilter]): + self.client_app.add_incoming_result_filters(pattern, filters) + + def set_server_prop(self, name: str, value): + self.server_app.set_prop(name, value) + + def set_client_prop(self, name: str, value): + self.client_app.set_prop(name, value) + def run(self): return self.simulator.run() From 5756adbb44e4fadfeeb21a31b0b3ccfdc2d9e49d Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Mon, 17 Nov 2025 11:42:57 -0500 Subject: [PATCH 059/102] use dec for filters --- nvflare/fox/api/app.py | 37 +++++--- nvflare/fox/api/dec.py | 87 +++++++++++-------- nvflare/fox/api/facade.py | 26 ++++-- nvflare/fox/api/filter.py | 56 +++++++++++- nvflare/fox/examples/np/algos/client.py | 2 +- nvflare/fox/examples/np/algos/filters.py | 46 +++++----- .../np/algos/strategies/avg_intime.py | 2 +- .../examples/np/algos/strategies/avg_seq.py | 2 +- nvflare/fox/examples/np/fed_avg_intime.py | 9 +- nvflare/fox/examples/np/fed_avg_intime2.py | 8 +- nvflare/fox/sim/sim2.py | 17 ++-- 11 files changed, 194 insertions(+), 98 deletions(-) diff --git a/nvflare/fox/api/app.py b/nvflare/fox/api/app.py index 5196c418fa..e62d69118d 100644 --- a/nvflare/fox/api/app.py +++ b/nvflare/fox/api/app.py @@ -21,7 +21,14 @@ from .constants import CollabMethodArgName, ContextKey, FilterDirection from .ctx import Context, set_call_context -from .dec import collab, get_object_algo_funcs, get_object_collab_interface, get_object_init_funcs, is_collab +from .dec import ( + collab, + get_object_algo_funcs, + get_object_collab_interface, + get_object_init_funcs, + is_collab, + supports_context, +) from .filter import CallFilter, FilterChain, ResultFilter from .proxy import Proxy from .utils import check_context_support, get_collab_object_name @@ -87,34 +94,39 @@ def _add_filters(self, pattern: str, filters, to_list: list, filter_type): if not isinstance(filters, list): raise ValueError(f"filters must be a list but got {type(filters)}") - for i, f in enumerate(filters): + filter_objs = [] + for f in filters: if not isinstance(f, filter_type): - raise ValueError(f"filter {i} must be {filter_type} but got {type(f)}") + # convert to proper filter type + filter_obj = filter_type(f) + else: + filter_obj = f self._add_managed_object(f) + filter_objs.append(filter_obj) chain = FilterChain(pattern, filter_type) - chain.add_filters(filters) + chain.add_filters(filter_objs) to_list.append(chain) - def add_incoming_call_filters(self, pattern: str, filters: List[CallFilter]): + def add_incoming_call_filters(self, pattern: str, filters: List[object]): self._add_filters(pattern, filters, self._incoming_call_filter_chains, CallFilter) def get_incoming_call_filters(self): return self._incoming_call_filter_chains - def add_outgoing_call_filters(self, pattern: str, filters: List[CallFilter]): + def add_outgoing_call_filters(self, pattern: str, filters: List[object]): self._add_filters(pattern, filters, self._outgoing_call_filter_chains, CallFilter) def get_outgoing_call_filters(self): return self._outgoing_call_filter_chains - def add_incoming_result_filters(self, pattern: str, filters: List[ResultFilter]): + def add_incoming_result_filters(self, pattern: str, filters: List[object]): self._add_filters(pattern, filters, self._incoming_result_filter_chains, ResultFilter) def get_incoming_result_filters(self): return self._incoming_result_filter_chains - def add_outgoing_result_filters(self, pattern: str, filters: List[ResultFilter]): + def add_outgoing_result_filters(self, pattern: str, filters: List[object]): self._add_filters(pattern, filters, self._outgoing_result_filter_chains, ResultFilter) def get_outgoing_result_filters(self): @@ -257,9 +269,12 @@ def find_collab_method(self, target_obj, method_name): def _fox_init(self, obj, ctx: Context): init_funcs = get_object_init_funcs(obj) - for f in init_funcs: - kwargs = {CollabMethodArgName.CONTEXT: ctx} - check_context_support(f, kwargs) + for name, f in init_funcs: + self.logger.info(f"calling init func {name} ...") + if supports_context(f): + kwargs = {CollabMethodArgName.CONTEXT: ctx} + else: + kwargs = {} f(**kwargs) def initialize(self, context: Context): diff --git a/nvflare/fox/api/dec.py b/nvflare/fox/api/dec.py index 7f2648327a..ff9bdcec00 100644 --- a/nvflare/fox/api/dec.py +++ b/nvflare/fox/api/dec.py @@ -18,6 +18,8 @@ _FLAG_COLLAB = "_fox_is_collab" _FLAG_INIT = "_fox_is_init" _FLAG_ALGO = "_fox_is_algo" +_FLAG_CALL_FILTER = "_fox_is_call_filter" +_FLAG_RESULT_FILTER = "_fox_is_result_filter" _FLAG_SUPPORT_CTX = "_fox_supports_ctx" _ATTR_PARAM_NAMES = "_fox_param_names" @@ -49,6 +51,19 @@ def wrapper(*args, **kwargs): return wrapper +def is_collab(func): + return _has_flag(func, _FLAG_COLLAB) + + +def get_object_collab_interface(obj): + result = {} + for name in dir(obj): + func = getattr(obj, name) + if callable(func) and is_collab(func): + result[name] = get_param_names(func) + return result + + def init(func): def wrapper(*args, **kwargs): return func(*args, **kwargs) @@ -58,6 +73,10 @@ def wrapper(*args, **kwargs): return wrapper +def get_object_init_funcs(obj): + return _get_object_funcs(obj, _FLAG_INIT, "init") + + def algo(func): def wrapper(*args, **kwargs): return func(*args, **kwargs) @@ -67,59 +86,59 @@ def wrapper(*args, **kwargs): return wrapper -def get_param_names(func): - return getattr(func, _ATTR_PARAM_NAMES, None) +def get_object_algo_funcs(obj): + return _get_object_funcs(obj, _FLAG_ALGO, "algo") -def _has_flag(func, flag: str) -> bool: - v = getattr(func, flag, None) - return v is True +def call_filter(func): + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + _set_attrs(func, wrapper) + setattr(wrapper, _FLAG_CALL_FILTER, True) + return wrapper -def is_collab(func): - return _has_flag(func, _FLAG_COLLAB) +def get_object_call_filter_funcs(obj): + return _get_object_funcs(obj, _FLAG_CALL_FILTER, "call_filter") -def is_init(func): - return _has_flag(func, _FLAG_INIT) +def result_filter(func): + def wrapper(*args, **kwargs): + return func(*args, **kwargs) -def is_algo(func): - return _has_flag(func, _FLAG_ALGO) + _set_attrs(func, wrapper) + setattr(wrapper, _FLAG_RESULT_FILTER, True) + return wrapper -def supports_context(func): - return _has_flag(func, _FLAG_SUPPORT_CTX) +def get_object_result_filter_funcs(obj): + return _get_object_funcs(obj, _FLAG_RESULT_FILTER, "result_filter") -def adjust_kwargs(func, kwargs): - if not supports_context(func): - kwargs.pop(CollabMethodArgName.CONTEXT, None) +def get_param_names(func): + return getattr(func, _ATTR_PARAM_NAMES, None) -def get_object_collab_interface(obj): - result = {} - for name in dir(obj): - func = getattr(obj, name) - if callable(func) and is_collab(func): - result[name] = get_param_names(func) - return result +def _has_flag(func, flag: str) -> bool: + v = getattr(func, flag, None) + return v is True -def get_object_init_funcs(obj): +def _get_object_funcs(obj, flag, func_type): result = [] for name in dir(obj): func = getattr(obj, name) - if callable(func) and is_init(func): - print(f"found init func of object {obj.__class__.__name__}.{name}") - result.append(func) + if callable(func) and _has_flag(func, flag): + print(f"found {func_type} func of object {obj.__class__.__name__}.{name}") + result.append((name, func)) return result -def get_object_algo_funcs(obj): - result = [] - for name in dir(obj): - func = getattr(obj, name) - if callable(func) and is_algo(func): - result.append((name, func)) - return result +def supports_context(func): + return _has_flag(func, _FLAG_SUPPORT_CTX) + + +def adjust_kwargs(func, kwargs): + if not supports_context(func): + kwargs.pop(CollabMethodArgName.CONTEXT, None) diff --git a/nvflare/fox/api/facade.py b/nvflare/fox/api/facade.py index 2618eb1b2a..17206e9dbc 100644 --- a/nvflare/fox/api/facade.py +++ b/nvflare/fox/api/facade.py @@ -14,9 +14,11 @@ from .constants import ContextKey from .ctx import get_call_context from .dec import algo as dec_algo +from .dec import call_filter as dec_call_filter from .dec import classproperty from .dec import collab as dec_collab from .dec import init as dec_init +from .dec import result_filter as dec_result_filter from .proxy_list import ProxyList @@ -25,6 +27,8 @@ class facade: collab = dec_collab init = dec_init algo = dec_algo + call_filter = dec_call_filter + result_filter = dec_result_filter @classproperty def context(cls): @@ -109,6 +113,16 @@ def workspace(cls): ctx = get_call_context() return ctx.workspace + @classproperty + def filter_direction(cls): + ctx = get_call_context() + return ctx.get_prop(ContextKey.DIRECTION) + + @classproperty + def qual_func_name(cls): + ctx = get_call_context() + return ctx.get_prop(ContextKey.QUALIFIED_FUNC_NAME) + @staticmethod def fire_event(event_type: str, data): ctx = get_call_context() @@ -120,15 +134,15 @@ def register_event_handler(event_type: str, handler, **handler_kwargs): ctx.app.register_event_handler(event_type, handler, **handler_kwargs) @staticmethod - def get_prop(name: str, default=None): + def get_app_prop(name: str, default=None): ctx = get_call_context() return ctx.app.get_prop(name, default) @staticmethod - def get_input(default=None): - return facade.get_prop(ContextKey.INPUT, default) + def get_prop(name: str, default=None): + ctx = get_call_context() + return ctx.get_prop(name, default) @staticmethod - def set_prop(name: str, value): - ctx = get_call_context() - ctx.app.set_prop(name, value) + def get_input(default=None): + return facade.get_prop(ContextKey.INPUT, default) diff --git a/nvflare/fox/api/filter.py b/nvflare/fox/api/filter.py index 790175e8a8..bf33171005 100644 --- a/nvflare/fox/api/filter.py +++ b/nvflare/fox/api/filter.py @@ -13,11 +13,30 @@ # limitations under the License. from typing import Any +from nvflare.fuel.utils.log_utils import get_obj_logger + +from .constants import CollabMethodArgName from .ctx import Context +from .dec import get_object_call_filter_funcs, get_object_result_filter_funcs, supports_context class CallFilter: + def __init__(self, impl: object = None): + self.logger = get_obj_logger(self) + if impl: + funcs = get_object_call_filter_funcs(impl) + if not funcs: + raise ValueError(f"filter impl object {impl.__class__.__name__} has no call_filter func") + + if len(funcs) > 1: + raise ValueError( + f"filter object {impl.__class__.__name__} must have one call_filter func but got {len(funcs)}" + ) + self.impl_func = funcs[0] + else: + self.impl_func = None + def filter_call(self, func_kwargs: dict, context: Context): """Filter kwargs of function call. @@ -28,11 +47,35 @@ def filter_call(self, func_kwargs: dict, context: Context): Returns: filtered kwargs that will be passed to a collab func. """ - return func_kwargs + if self.impl_func is not None: + name, f = self.impl_func + self.logger.info(f"calling call filter: {name} ...") + if supports_context(f): + kwargs = {CollabMethodArgName.CONTEXT: context} + else: + kwargs = {} + return f(func_kwargs, **kwargs) + else: + return func_kwargs class ResultFilter: + def __init__(self, impl: object = None): + self.logger = get_obj_logger(self) + if impl: + funcs = get_object_result_filter_funcs(impl) + if not funcs: + raise ValueError(f"filter object {impl.__class__.__name__} has no result_filter func") + + if len(funcs) > 1: + raise ValueError( + f"filter object {impl.__class__.__name__} must have one result_filter func but got {len(funcs)}" + ) + self.impl_func = funcs[0] + else: + self.impl_func = None + def filter_result(self, result: Any, context: Context): """Filter result produced by a collab func. @@ -43,7 +86,16 @@ def filter_result(self, result: Any, context: Context): Returns: filtered result """ - return result + if self.impl_func is not None: + name, f = self.impl_func + self.logger.info(f"calling result filter: {name} ...") + if supports_context(f): + kwargs = {CollabMethodArgName.CONTEXT: context} + else: + kwargs = {} + return f(result, **kwargs) + else: + return result class FilterChain: diff --git a/nvflare/fox/examples/np/algos/client.py b/nvflare/fox/examples/np/algos/client.py index 834ffadd6c..a5f9bc716e 100644 --- a/nvflare/fox/examples/np/algos/client.py +++ b/nvflare/fox/examples/np/algos/client.py @@ -25,7 +25,7 @@ def __init__(self, delta: float): @fox.init def init_trainer(self): - delta_config = fox.get_prop("client_delta", {}) + delta_config = fox.get_app_prop("client_delta", {}) self.delta = delta_config.get(fox.site_name, self.delta) self.logger.info(f"init_trainer: client {fox.site_name}: delta={self.delta}") diff --git a/nvflare/fox/examples/np/algos/filters.py b/nvflare/fox/examples/np/algos/filters.py index e0e5b2c9a3..03f14564bd 100644 --- a/nvflare/fox/examples/np/algos/filters.py +++ b/nvflare/fox/examples/np/algos/filters.py @@ -13,21 +13,20 @@ # limitations under the License. import random -from nvflare.fox.api.constants import ContextKey -from nvflare.fox.api.ctx import Context -from nvflare.fox.api.filter import CallFilter, ResultFilter +from nvflare.fox import fox from nvflare.fuel.utils.log_utils import get_obj_logger -class AddNoiseToModel(CallFilter): +class AddNoiseToModel: def __init__(self): self.logger = get_obj_logger(self) - def filter_call(self, func_kwargs: dict, context: Context): - direction = context.get_prop(ContextKey.DIRECTION) - qual_func_name = context.get_prop(ContextKey.QUALIFIED_FUNC_NAME) - self.logger.debug(f"[{context.header_str()}] filtering call: {func_kwargs=} {direction=} {qual_func_name=}") + @fox.call_filter + def add_noise(self, func_kwargs: dict): + direction = fox.filter_direction + qual_func_name = fox.qual_func_name + self.logger.debug(f"[{fox.call_info}] filtering call: {func_kwargs=} {direction=} {qual_func_name=}") weights_key = "weights" weights = func_kwargs.get(weights_key) if weights is None: @@ -37,32 +36,29 @@ def filter_call(self, func_kwargs: dict, context: Context): # add some noise to weights noise = random.random() - self.logger.debug(f"[{context.header_str()}] adding noise {noise}") + self.logger.debug(f"[{fox.call_info}] adding noise {noise}") weights += noise func_kwargs[weights_key] = weights - self.logger.info(f"[{context.header_str()}] weights after adding noise {noise}: {weights}") + self.logger.info(f"[{fox.call_info}] weights after adding noise {noise}: {weights}") return func_kwargs -class PrintCall(CallFilter): +class Print: def __init__(self): + super().__init__() self.logger = get_obj_logger(self) - def filter_call(self, func_kwargs: dict, context: Context): - direction = context.get_prop(ContextKey.DIRECTION) - qual_func_name = context.get_prop(ContextKey.QUALIFIED_FUNC_NAME) - self.logger.info(f"[{context.header_str()}] printing call: {func_kwargs=} {direction=} {qual_func_name=}") + @fox.call_filter + def print_call(self, func_kwargs: dict): + direction = fox.filter_direction + qual_func_name = fox.qual_func_name + self.logger.info(f"[{fox.call_info}] printing call: {func_kwargs=} {direction=} {qual_func_name=}") return func_kwargs - -class PrintResult(ResultFilter): - - def __init__(self): - self.logger = get_obj_logger(self) - - def filter_result(self, result, context: Context): - direction = context.get_prop(ContextKey.DIRECTION) - qual_func_name = context.get_prop(ContextKey.QUALIFIED_FUNC_NAME) - self.logger.info(f"[{context.header_str()}] printing result: {result=} {direction=} {qual_func_name=}") + @fox.result_filter + def print_result(self, result): + direction = fox.filter_direction + qual_func_name = fox.qual_func_name + self.logger.info(f"[{fox.call_info}] printing result: {result=} {direction=} {qual_func_name=}") return result diff --git a/nvflare/fox/examples/np/algos/strategies/avg_intime.py b/nvflare/fox/examples/np/algos/strategies/avg_intime.py index ec9d69d13d..dbc05b8b3f 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_intime.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_intime.py @@ -55,7 +55,7 @@ def _do_eval(self, model): def _do_one_round(self, r, current_model): aggr_result = _AggrResult() - timeout = fox.get_prop("default_timeout", 10) + timeout = fox.get_app_prop("default_timeout", 10) self.logger.info(f"got timeout: {timeout}") fox.clients( timeout=timeout, diff --git a/nvflare/fox/examples/np/algos/strategies/avg_seq.py b/nvflare/fox/examples/np/algos/strategies/avg_seq.py index aa71047959..5bb8e8aa1a 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_seq.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_seq.py @@ -31,7 +31,7 @@ def __init__(self, initial_model, num_rounds=10): @fox.init def init(self): self.logger.info("fox init NPFedAvgSequential") - weight_config = fox.get_prop("client_weight_config", {}) + weight_config = fox.get_app_prop("client_weight_config", {}) client_weights = {} total = 0 for c in fox.clients: diff --git a/nvflare/fox/examples/np/fed_avg_intime.py b/nvflare/fox/examples/np/fed_avg_intime.py index 785fcd87ff..4bfb2010d8 100644 --- a/nvflare/fox/examples/np/fed_avg_intime.py +++ b/nvflare/fox/examples/np/fed_avg_intime.py @@ -16,7 +16,7 @@ from nvflare.fox.api.app import ClientApp, ServerApp from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.client import NPTrainer -from nvflare.fox.examples.np.algos.filters import AddNoiseToModel, PrintCall, PrintResult +from nvflare.fox.examples.np.algos.filters import AddNoiseToModel, Print from nvflare.fox.examples.np.algos.strategies.avg_intime import NPFedAvgInTime from nvflare.fox.examples.np.algos.widgets import MetricReceiver from nvflare.fox.sim.simulator import Simulator @@ -29,14 +29,15 @@ def main(): NPFedAvgInTime(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), ) + print_filter = Print() server_app.add_collab_object("metric_receiver", MetricReceiver()) server_app.add_outgoing_call_filters("*.train", [AddNoiseToModel()]) - server_app.add_incoming_result_filters("*.train", [PrintResult()]) + server_app.add_incoming_result_filters("*.train", [print_filter]) server_app.set_prop("default_timeout", 5.0) client_app = ClientApp(NPTrainer(delta=1.0)) - client_app.add_incoming_call_filters("*.train", [PrintCall()]) - client_app.add_outgoing_result_filters("*.train", [PrintResult()]) + client_app.add_incoming_call_filters("*.train", [print_filter]) + client_app.add_outgoing_result_filters("*.train", [print_filter]) client_app.set_prop("default_timeout", 8.0) simulator = Simulator( diff --git a/nvflare/fox/examples/np/fed_avg_intime2.py b/nvflare/fox/examples/np/fed_avg_intime2.py index cc7a16c308..d9545d77ad 100644 --- a/nvflare/fox/examples/np/fed_avg_intime2.py +++ b/nvflare/fox/examples/np/fed_avg_intime2.py @@ -15,7 +15,7 @@ from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.client import NPTrainer -from nvflare.fox.examples.np.algos.filters import AddNoiseToModel, PrintCall, PrintResult +from nvflare.fox.examples.np.algos.filters import AddNoiseToModel, Print from nvflare.fox.examples.np.algos.strategies.avg_intime import NPFedAvgInTime from nvflare.fox.examples.np.algos.widgets import MetricReceiver from nvflare.fox.sim.sim2 import Simulator @@ -34,11 +34,11 @@ def main(): ) simulator.add_server_outgoing_call_filters("*.train", [AddNoiseToModel()]) - simulator.add_server_incoming_result_filters("*.train", [PrintResult()]) + simulator.add_server_incoming_result_filters("*.train", [Print()]) simulator.set_server_prop("default_timeout", 5.0) - simulator.add_client_incoming_call_filters("*.train", [PrintCall()]) - simulator.add_client_outgoing_result_filters("*.train", [PrintResult()]) + simulator.add_client_incoming_call_filters("*.train", [Print()]) + simulator.add_client_outgoing_result_filters("*.train", [Print()]) simulator.set_client_prop("default_timeout", 8.0) result = simulator.run() diff --git a/nvflare/fox/sim/sim2.py b/nvflare/fox/sim/sim2.py index 4fbb04c737..cce752b4f2 100644 --- a/nvflare/fox/sim/sim2.py +++ b/nvflare/fox/sim/sim2.py @@ -14,7 +14,6 @@ from typing import List, Tuple, Union from nvflare.fox.api.app import ClientApp, ServerApp -from nvflare.fox.api.filter import CallFilter, ResultFilter from .simulator import Simulator as Sim1 @@ -55,28 +54,28 @@ def __init__( self.server_app = server_app self.client_app = client_app - def add_server_outgoing_call_filters(self, pattern: str, filters: List[CallFilter]): + def add_server_outgoing_call_filters(self, pattern: str, filters: List[object]): self.server_app.add_outgoing_call_filters(pattern, filters) - def add_server_incoming_call_filters(self, pattern: str, filters: List[CallFilter]): + def add_server_incoming_call_filters(self, pattern: str, filters: List[object]): self.server_app.add_incoming_call_filters(pattern, filters) - def add_server_outgoing_result_filters(self, pattern: str, filters: List[ResultFilter]): + def add_server_outgoing_result_filters(self, pattern: str, filters: List[object]): self.server_app.add_outgoing_result_filters(pattern, filters) - def add_server_incoming_result_filters(self, pattern: str, filters: List[ResultFilter]): + def add_server_incoming_result_filters(self, pattern: str, filters: List[object]): self.server_app.add_incoming_result_filters(pattern, filters) - def add_client_outgoing_call_filters(self, pattern: str, filters: List[CallFilter]): + def add_client_outgoing_call_filters(self, pattern: str, filters: List[object]): self.client_app.add_outgoing_call_filters(pattern, filters) - def add_client_incoming_call_filters(self, pattern: str, filters: List[CallFilter]): + def add_client_incoming_call_filters(self, pattern: str, filters: List[object]): self.client_app.add_incoming_call_filters(pattern, filters) - def add_client_outgoing_result_filters(self, pattern: str, filters: List[ResultFilter]): + def add_client_outgoing_result_filters(self, pattern: str, filters: List[object]): self.client_app.add_outgoing_result_filters(pattern, filters) - def add_client_incoming_result_filters(self, pattern: str, filters: List[ResultFilter]): + def add_client_incoming_result_filters(self, pattern: str, filters: List[object]): self.client_app.add_incoming_result_filters(pattern, filters) def set_server_prop(self, name: str, value): From 8b0298c069b8b89ce99c39aaf9b3cbcfe0970bf5 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Tue, 18 Nov 2025 15:41:37 -0500 Subject: [PATCH 060/102] fix recipes --- nvflare/fox/api/backend.py | 3 + nvflare/fox/api/ctx.py | 2 + nvflare/fox/api/filter.py | 20 ++++- nvflare/fox/api/gcc.py | 12 +++ nvflare/fox/api/proxy.py | 6 ++ nvflare/fox/api/run_server.py | 4 +- nvflare/fox/examples/np/algos/client.py | 2 +- nvflare/fox/examples/np/algos/filters.py | 3 +- nvflare/fox/examples/np/algos/swarm.py | 4 +- .../fox/examples/np/recipe_fed_avg_intime.py | 35 ++++---- nvflare/fox/examples/np/recipe_swarm.py | 10 +-- nvflare/fox/sys/adaptor.py | 17 ++-- nvflare/fox/sys/backend.py | 12 ++- nvflare/fox/sys/controller.py | 65 ++++++-------- nvflare/fox/sys/executor.py | 42 ++++----- nvflare/fox/sys/recipe.py | 90 +++++++++++++------ nvflare/fox/sys/utils.py | 1 + 17 files changed, 191 insertions(+), 137 deletions(-) diff --git a/nvflare/fox/api/backend.py b/nvflare/fox/api/backend.py index 4de58b7f29..43c381d51a 100644 --- a/nvflare/fox/api/backend.py +++ b/nvflare/fox/api/backend.py @@ -59,3 +59,6 @@ def call_target_in_group(self, gcc: GroupCallContext, target_name: str, func_nam """ pass + + def handle_exception(self, exception: Exception): + pass diff --git a/nvflare/fox/api/ctx.py b/nvflare/fox/api/ctx.py index 0d95821ca7..697271b28c 100644 --- a/nvflare/fox/api/ctx.py +++ b/nvflare/fox/api/ctx.py @@ -79,6 +79,8 @@ def header_str(self): def get_call_context(): + if not fox_context.call_ctx: + print("NO CALL_CTX in FOX Context!!!") return fox_context.call_ctx diff --git a/nvflare/fox/api/filter.py b/nvflare/fox/api/filter.py index bf33171005..461ac46ba4 100644 --- a/nvflare/fox/api/filter.py +++ b/nvflare/fox/api/filter.py @@ -20,10 +20,23 @@ from .dec import get_object_call_filter_funcs, get_object_result_filter_funcs, supports_context -class CallFilter: +class _Filter: def __init__(self, impl: object = None): + self.impl = impl self.logger = get_obj_logger(self) + + def get_impl_object(self): + if self.impl: + return self.impl + else: + return self + + +class CallFilter(_Filter): + + def __init__(self, impl: object = None): + super().__init__(impl) if impl: funcs = get_object_call_filter_funcs(impl) if not funcs: @@ -59,10 +72,10 @@ def filter_call(self, func_kwargs: dict, context: Context): return func_kwargs -class ResultFilter: +class ResultFilter(_Filter): def __init__(self, impl: object = None): - self.logger = get_obj_logger(self) + super().__init__(impl) if impl: funcs = get_object_result_filter_funcs(impl) if not funcs: @@ -88,7 +101,6 @@ def filter_result(self, result: Any, context: Context): """ if self.impl_func is not None: name, f = self.impl_func - self.logger.info(f"calling result filter: {name} ...") if supports_context(f): kwargs = {CollabMethodArgName.CONTEXT: context} else: diff --git a/nvflare/fox/api/gcc.py b/nvflare/fox/api/gcc.py index 4813e6031d..8e177b7de0 100644 --- a/nvflare/fox/api/gcc.py +++ b/nvflare/fox/api/gcc.py @@ -14,6 +14,8 @@ import copy import time +from nvflare.fuel.utils.log_utils import get_obj_logger + from .constants import CollabMethodArgName from .ctx import Context, set_call_context from .utils import check_context_support @@ -40,6 +42,7 @@ def __init__(self, app, target_name, func_name, process_cb, cb_kwargs, context: self.process_cb = process_cb self.cb_kwargs = cb_kwargs self.context = context + self.logger = get_obj_logger(self) def set_result(self, result): """This is called by the backend to set the result received from the remote app. @@ -59,10 +62,16 @@ def set_result(self, result): ctx.caller = ctx.callee ctx.callee = original_caller + self.logger.info(f"set result {result}") + if not isinstance(result, Exception): + self.logger.info(f"{self.func_name}: result before filtering: {result}") result = self.app.apply_incoming_result_filters(self.target_name, self.func_name, result, ctx) + self.logger.info(f"{self.func_name}: result after filtering: {result}") if self.process_cb: + self.logger.info(f"calling process_cb for {self.func_name} with result: {result}") + # set the context for the process_cb only set_call_context(ctx) self.cb_kwargs[CollabMethodArgName.CONTEXT] = ctx @@ -71,6 +80,9 @@ def set_result(self, result): # set back to original context set_call_context(self.context) + else: + self.logger.info(f"{self.func_name} does not have process_cb!") + self.result = result self.resp_time = time.time() diff --git a/nvflare/fox/api/proxy.py b/nvflare/fox/api/proxy.py index 776bebb6a7..fc231260bb 100644 --- a/nvflare/fox/api/proxy.py +++ b/nvflare/fox/api/proxy.py @@ -98,19 +98,25 @@ def get_target(self, name: str): return None def _find_interface(self, func_name): + self.logger.info(f"trying to find interface for {func_name}") args = self.target_interface.get(func_name) if self.target_interface else None if args: return self, args # try children + self.logger.info(f"not defined for {func_name}: trying children") the_args = None the_proxy = None the_name = None for n, c in self.children.items(): + self.logger.info(f"trying child {n} ...") args = c.target_interface.get(func_name) if c.target_interface else None if not args: + self.logger.info(f"child {n} has no {func_name}") continue + self.logger.info(f"found interface for func {func_name}: defined in child {n}") + if not the_proxy: the_name = n the_proxy = c diff --git a/nvflare/fox/api/run_server.py b/nvflare/fox/api/run_server.py index 7b980299b6..8b9748de03 100644 --- a/nvflare/fox/api/run_server.py +++ b/nvflare/fox/api/run_server.py @@ -38,7 +38,9 @@ def run_server(server_app: ServerApp, logger): kwargs = {} result = f(**kwargs) server_ctx.set_prop(ContextKey.INPUT, result) - except: + except Exception as ex: traceback.print_exc() + backend = server_app.get_backend() + backend.handle_exception(ex) break return result diff --git a/nvflare/fox/examples/np/algos/client.py b/nvflare/fox/examples/np/algos/client.py index a5f9bc716e..098e9a11c4 100644 --- a/nvflare/fox/examples/np/algos/client.py +++ b/nvflare/fox/examples/np/algos/client.py @@ -38,7 +38,7 @@ def train(self, current_round, weights): if fox.is_aborted: self.logger.debug("training aborted") return 0 - self.logger.debug(f"[{fox.call_info}] EZ trained round {current_round}") + self.logger.info(f"[{fox.call_info}] EZ trained round {current_round=} {weights=}") # metric_receiver = self.server.get_target("metric_receiver") # if metric_receiver: diff --git a/nvflare/fox/examples/np/algos/filters.py b/nvflare/fox/examples/np/algos/filters.py index 03f14564bd..1508e658b3 100644 --- a/nvflare/fox/examples/np/algos/filters.py +++ b/nvflare/fox/examples/np/algos/filters.py @@ -19,8 +19,9 @@ class AddNoiseToModel: - def __init__(self): + def __init__(self, x=0.1): self.logger = get_obj_logger(self) + self.x = x @fox.call_filter def add_noise(self, func_kwargs: dict): diff --git a/nvflare/fox/examples/np/algos/swarm.py b/nvflare/fox/examples/np/algos/swarm.py index 7d2797c215..9a0bb916a6 100644 --- a/nvflare/fox/examples/np/algos/swarm.py +++ b/nvflare/fox/examples/np/algos/swarm.py @@ -67,7 +67,7 @@ def train(self, weights, current_round): return weights + self.delta def sag(self, model, current_round): - results = fox.clients.train(model, current_round) + results = fox.clients.train2(model, current_round) results = list(results.values()) total = 0 for i in range(len(results)): @@ -86,7 +86,7 @@ def swarm_learn(self, num_rounds, model, current_round): # self.server.fire_event("all_done", "OK", blocking=False) self.logger.info("notify server all done!") try: - fox.server.all_done("OK", _blocking=False) + fox.server(blocking=False).all_done("OK") except: traceback.print_exc() self.logger.info("Swarm Training is DONE!") diff --git a/nvflare/fox/examples/np/recipe_fed_avg_intime.py b/nvflare/fox/examples/np/recipe_fed_avg_intime.py index b03f9d834e..3088a57aef 100644 --- a/nvflare/fox/examples/np/recipe_fed_avg_intime.py +++ b/nvflare/fox/examples/np/recipe_fed_avg_intime.py @@ -13,40 +13,37 @@ # limitations under the License. import logging -from nvflare.fox.api.app import ServerApp from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.client import NPTrainer -from nvflare.fox.examples.np.algos.filters import AddNoiseToModel, PrintCall, PrintResult +from nvflare.fox.examples.np.algos.filters import AddNoiseToModel, Print from nvflare.fox.examples.np.algos.strategies.avg_intime import NPFedAvgInTime from nvflare.fox.examples.np.algos.widgets import MetricReceiver from nvflare.fox.sys.recipe import FoxRecipe -JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/fox/prod_00/admin@nvidia.com/transfer" +JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" def main(): simple_logging(logging.DEBUG) - server_app = ServerApp( - strategy_name="fed_avg_in_time", - strategy=NPFedAvgInTime(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), + recipe = FoxRecipe( + job_name="fedavg_intime", + server=NPFedAvgInTime(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), + server_objects={ + "metric_receiver": MetricReceiver(), + }, + client=NPTrainer(delta=1.0), ) - server_app.add_collab_object("metric_receiver", MetricReceiver()) - server_app.add_outgoing_call_filters("*.train", [AddNoiseToModel()]) - server_app.add_incoming_result_filters("*.train", [PrintResult()]) - server_app.set_prop("default_timeout", 5.0) + print_filter = Print() + recipe.add_server_outgoing_call_filters("*.train", [AddNoiseToModel(0.3)]) + recipe.add_server_incoming_result_filters("*.train", [print_filter]) + recipe.set_server_prop("default_timeout", 5.0) - client_app = NPTrainer(delta=1.0) - client_app.add_incoming_call_filters("*.train", [PrintCall()]) - client_app.add_outgoing_result_filters("*.train", [PrintResult()]) - client_app.set_prop("default_timeout", 8.0) + recipe.add_client_incoming_call_filters("*.train", [print_filter]) + recipe.add_client_outgoing_result_filters("*.train", [print_filter]) + recipe.set_client_prop("default_timeout", 8.0) - recipe = FoxRecipe( - job_name="fedavg_intime", - server_app=server_app, - client_app=client_app, - ) recipe.export(JOB_ROOT_DIR) diff --git a/nvflare/fox/examples/np/recipe_swarm.py b/nvflare/fox/examples/np/recipe_swarm.py index 0650926da1..fa62ef612d 100644 --- a/nvflare/fox/examples/np/recipe_swarm.py +++ b/nvflare/fox/examples/np/recipe_swarm.py @@ -13,24 +13,20 @@ # limitations under the License. import logging -from nvflare.fox.api.app import ClientApp, ServerApp from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.swarm import NPSwarm, NPSwarmClient from nvflare.fox.sys.recipe import FoxRecipe -JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/fox/prod_00/admin@nvidia.com/transfer" +JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" def main(): simple_logging(logging.DEBUG) - server_app = ServerApp(NPSwarm(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=5)) - client_app = ClientApp(NPSwarmClient(delta=1.0)) - recipe = FoxRecipe( job_name="swarm", - server_app=server_app, - client_app=client_app, + server=NPSwarm(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=5), + client=NPSwarmClient(delta=1.0), ) recipe.export(JOB_ROOT_DIR) diff --git a/nvflare/fox/sys/adaptor.py b/nvflare/fox/sys/adaptor.py index 2a7d684b46..3aafce2f1f 100644 --- a/nvflare/fox/sys/adaptor.py +++ b/nvflare/fox/sys/adaptor.py @@ -15,7 +15,6 @@ from nvflare.apis.fl_context import FLContext from nvflare.fox.api.app import App -from nvflare.fox.api.filter import CallFilter, ResultFilter class FoxAdaptor: @@ -53,25 +52,25 @@ def process_config(self, app: App, fl_ctx: FLContext): app.add_collab_object(cid, obj) - err = self._parse_filters("incoming_call_filters", CallFilter, app.add_incoming_call_filters, fl_ctx) + err = self._parse_filters("incoming_call_filters", app.add_incoming_call_filters, fl_ctx) if err: return err - err = self._parse_filters("outgoing_call_filters", CallFilter, app.add_outgoing_call_filters, fl_ctx) + err = self._parse_filters("outgoing_call_filters", app.add_outgoing_call_filters, fl_ctx) if err: return err - err = self._parse_filters("incoming_result_filters", ResultFilter, app.add_incoming_result_filters, fl_ctx) + err = self._parse_filters("incoming_result_filters", app.add_incoming_result_filters, fl_ctx) if err: return err - err = self._parse_filters("outgoing_result_filters", ResultFilter, app.add_outgoing_result_filters, fl_ctx) + err = self._parse_filters("outgoing_result_filters", app.add_outgoing_result_filters, fl_ctx) if err: return err return None - def _parse_filters(self, name, filter_type, add_f, fl_ctx): + def _parse_filters(self, name, add_f, fl_ctx): filters = getattr(self, name) if not filters: return None @@ -80,14 +79,14 @@ def _parse_filters(self, name, filter_type, add_f, fl_ctx): return f"{name} must be a list but got {type(filters)}" for chain_dict in filters: - pattern, filters, err = self._parse_filter_chain(name, chain_dict, filter_type, fl_ctx) + pattern, filters, err = self._parse_filter_chain(name, chain_dict, fl_ctx) if err: return err add_f(pattern, filters) return None @staticmethod - def _parse_filter_chain(chain_name, chain_dict: dict, filter_type, fl_ctx): + def _parse_filter_chain(chain_name, chain_dict: dict, fl_ctx): if not isinstance(chain_dict, dict): return None, None, f"element in {chain_name} must be dict but got {type(chain_dict)}" @@ -105,7 +104,5 @@ def _parse_filter_chain(chain_name, chain_dict: dict, filter_type, fl_ctx): f = engine.get_component(fid) if not f: return None, None, f"component {fid} does not exist" - if not isinstance(f, filter_type): - return None, None, f"component {fid} should be a {filter_type} but got {type(f)}" filters.append(f) return pattern, filters, None diff --git a/nvflare/fox/sys/backend.py b/nvflare/fox/sys/backend.py index 0be56b3c84..e7720fe136 100644 --- a/nvflare/fox/sys/backend.py +++ b/nvflare/fox/sys/backend.py @@ -13,6 +13,7 @@ # limitations under the License. from nvflare.fox.api.backend import Backend from nvflare.fox.api.constants import CollabMethodArgName, CollabMethodOptionName +from nvflare.fox.api.ctx import set_call_context from nvflare.fox.api.gcc import GroupCallContext from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey, ReturnCode from nvflare.fuel.f3.cellnet.utils import new_cell_message @@ -24,8 +25,10 @@ class SysBackend(Backend): - def __init__(self, caller, cell, target_fqcn, abort_signal, thread_executor): + def __init__(self, manager, engine, caller, cell, target_fqcn, abort_signal, thread_executor): Backend.__init__(self, abort_signal) + self.manager = manager + self.engine = engine self.logger = get_obj_logger(self) self.caller = caller self.cell = cell @@ -38,7 +41,8 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): optional = kwargs.pop(CollabMethodOptionName.OPTIONAL, False) secure = kwargs.pop(CollabMethodOptionName.SECURE, False) - kwargs.pop(CollabMethodArgName.CONTEXT, None) + ctx = kwargs.pop(CollabMethodArgName.CONTEXT) + set_call_context(ctx) payload = { ObjectCallKey.CALLER: self.caller, @@ -101,3 +105,7 @@ def _run_func(self, gcc: GroupCallContext, target_name: str, func_name: str, arg gcc.set_result(result) except Exception as ex: gcc.set_exception(ex) + + def handle_exception(self, exception: Exception): + fl_ctx = self.engine.new_context() + self.manager.system_panic(f"exception occurred: {exception}", fl_ctx) diff --git a/nvflare/fox/sys/controller.py b/nvflare/fox/sys/controller.py index 5b670e160c..3d64d51660 100644 --- a/nvflare/fox/sys/controller.py +++ b/nvflare/fox/sys/controller.py @@ -25,7 +25,6 @@ from nvflare.fox.api.constants import EnvType from nvflare.fox.api.proxy import Proxy from nvflare.fox.api.run_server import run_server -from nvflare.fox.api.strategy import Strategy from nvflare.fuel.f3.cellnet.fqcn import FQCN from .adaptor import FoxAdaptor @@ -50,8 +49,7 @@ class FoxController(Controller, FoxAdaptor): def __init__( self, - strategy_ids: List[str], - server_app_id: str = None, + server_obj_id: str = None, collab_obj_ids: List[str] = None, incoming_call_filters=None, outgoing_call_filters=None, @@ -73,47 +71,32 @@ def __init__( incoming_result_filters=incoming_result_filters, outgoing_result_filters=outgoing_result_filters, ) - self.server_app_id = server_app_id # component name - self.strategy_ids = strategy_ids # component names + self.server_obj_id = server_obj_id # component name self.sync_task_timeout = sync_task_timeout self.server_app = None self.client_info = {} # client name => _ClientInfo self.cell = None self.thread_executor = ThreadPoolExecutor(max_workers=max_call_threads) - if not strategy_ids: - raise ValueError(f"no strategies defined - there must be at least one strategy") - def start_controller(self, fl_ctx: FLContext): engine = fl_ctx.get_engine() - if self.server_app_id: - app = engine.get_component(self.server_app_id) - if not isinstance(app, ServerApp): - self.system_panic(f"component {self.server_app_id} must be ServerApp but got {type(app)}", fl_ctx) - return - else: - app = ServerApp() + server_obj = engine.get_component(self.server_obj_id) + app = ServerApp(server_obj) app.name = "server" app.env_type = EnvType.SYSTEM - for cid in self.strategy_ids: - strategy = engine.get_component(cid) - if not isinstance(strategy, Strategy): - self.system_panic(f"component {cid} must be Strategy but got {type(strategy)}", fl_ctx) - return - - app.add_strategy(cid, strategy) - err = self.process_config(app, fl_ctx) if err: self.system_panic(err, fl_ctx) return self.server_app = app - def _prepare_client_backend(self, job_id, client: ClientSite, abort_signal: Signal): + def _prepare_client_backend(self, job_id, client: ClientSite, abort_signal: Signal, fl_ctx: FLContext): return SysBackend( + manager=self, + engine=fl_ctx.get_engine(), caller=self.server_app.name, cell=self.cell, target_fqcn=FQCN.join([client.get_fqcn(), job_id]), @@ -121,8 +104,10 @@ def _prepare_client_backend(self, job_id, client: ClientSite, abort_signal: Sign thread_executor=self.thread_executor, ) - def _prepare_server_backend(self, job_id: str, abort_signal: Signal): + def _prepare_server_backend(self, job_id: str, abort_signal: Signal, fl_ctx: FLContext): return SysBackend( + manager=self, + engine=fl_ctx.get_engine(), caller=self.server_app.name, cell=self.cell, target_fqcn=FQCN.join([FQCN.ROOT_SERVER, job_id]), @@ -136,8 +121,9 @@ def _prepare_client_proxy( client: ClientSite, collab_interface: dict, abort_signal, + fl_ctx: FLContext, ): - backend = self._prepare_client_backend(job_id, client, abort_signal) + backend = self._prepare_client_backend(job_id, client, abort_signal, fl_ctx) proxy = Proxy( app=self.server_app, target_name=client.name, @@ -165,9 +151,10 @@ def _prepare_server_proxy( job_id, abort_signal, collab_interface: dict, + fl_ctx: FLContext, ): server_name = self.server_app.name - backend = self._prepare_server_backend(job_id, abort_signal) + backend = self._prepare_server_backend(job_id, abort_signal, fl_ctx) proxy = Proxy( app=self.server_app, target_name=server_name, @@ -176,15 +163,17 @@ def _prepare_server_proxy( target_interface=collab_interface.get(""), ) - for name in self.collab_obj_ids: - p = Proxy( - app=self.server_app, - target_name=f"{server_name}.{name}", - target_fqn="", - backend=backend, - target_interface=collab_interface.get(name), - ) - setattr(proxy, name, p) + collab_objs = self.server_app.get_collab_objects() + if collab_objs: + for name in collab_objs.keys(): + p = Proxy( + app=self.server_app, + target_name=f"{server_name}.{name}", + target_fqn="", + backend=backend, + target_interface=collab_interface.get(name), + ) + proxy.add_child(name, p) return proxy def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): @@ -236,12 +225,12 @@ def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): # prepare proxies and backends job_id = fl_ctx.get_job_id() - server_proxy = self._prepare_server_proxy(job_id, abort_signal, server_collab_interface) + server_proxy = self._prepare_server_proxy(job_id, abort_signal, server_collab_interface, fl_ctx) client_proxies = [] for c in all_clients: info = self.client_info[c.name] assert isinstance(info, _ClientInfo) - client_proxies.append(self._prepare_client_proxy(job_id, c, info.collab_interface, abort_signal)) + client_proxies.append(self._prepare_client_proxy(job_id, c, info.collab_interface, abort_signal, fl_ctx)) ws = SysWorkspace(fl_ctx) self.server_app.setup(ws, server_proxy, client_proxies, abort_signal) diff --git a/nvflare/fox/sys/executor.py b/nvflare/fox/sys/executor.py index 38cf1ad286..fef4e70b9c 100644 --- a/nvflare/fox/sys/executor.py +++ b/nvflare/fox/sys/executor.py @@ -38,7 +38,7 @@ class FoxExecutor(Executor, FoxAdaptor): def __init__( self, - client_app_id: str, + client_obj_id: str, collab_obj_ids: List[str] = None, incoming_call_filters=None, outgoing_call_filters=None, @@ -59,7 +59,7 @@ def __init__( incoming_result_filters=incoming_result_filters, outgoing_result_filters=outgoing_result_filters, ) - self.client_app_id = client_app_id + self.client_obj_id = client_obj_id self.register_event_handler(EventType.START_RUN, self._handle_start_run) self.register_event_handler(EventType.END_RUN, self._handle_end_run) self.client_app = None @@ -68,24 +68,13 @@ def __init__( def _handle_start_run(self, event_type: str, fl_ctx: FLContext): fl_ctx.set_prop(FLContextKey.FOX_MODE, True, private=True, sticky=True) engine = fl_ctx.get_engine() - client_app = engine.get_component(self.client_app_id) - if not isinstance(client_app, ClientApp): - self.system_panic( - f"component {self.client_app_id} must be ClientApp but got {type(client_app)}", - fl_ctx, - ) - return - + client_obj = engine.get_component(self.client_obj_id) client_name = fl_ctx.get_identity_name() - make_client_app_f = getattr(self.client_app, "make_client_app", None) - if make_client_app_f and callable(make_client_app_f): - self.client_app = make_client_app_f(client_name) - else: - self.client_app = client_app - - self.client_app.name = client_name - self.client_app.env_type = EnvType.SYSTEM + app = ClientApp(client_obj) + app.name = client_name + app.env_type = EnvType.SYSTEM + self.client_app = app err = self.process_config(self.client_app, fl_ctx) if err: @@ -94,9 +83,11 @@ def _handle_start_run(self, event_type: str, fl_ctx: FLContext): def _handle_end_run(self, event_type: str, fl_ctx: FLContext): self.thread_executor.shutdown(wait=False, cancel_futures=True) - def _prepare_server_proxy(self, job_id, cell, collab_interface: dict, abort_signal): + def _prepare_server_proxy(self, job_id, cell, collab_interface: dict, abort_signal, fl_ctx: FLContext): server_name = "server" backend = SysBackend( + manager=self, + engine=fl_ctx.get_engine(), caller=self.client_app.name, cell=cell, target_fqcn=FQCN.join([FQCN.ROOT_SERVER, job_id]), @@ -125,8 +116,10 @@ def _prepare_server_proxy(self, job_id, cell, collab_interface: dict, abort_sign proxy.add_child(name, p) return proxy - def _prepare_client_proxy(self, job_id, cell, client: Client, abort_signal, collab_interface): + def _prepare_client_proxy(self, job_id, cell, client: Client, abort_signal, collab_interface, fl_ctx: FLContext): backend = SysBackend( + manager=self, + engine=fl_ctx.get_engine(), caller=self.client_app.name, cell=cell, target_fqcn=FQCN.join([client.get_fqcn(), job_id]), @@ -141,8 +134,9 @@ def _prepare_client_proxy(self, job_id, cell, client: Client, abort_signal, coll target_interface=collab_interface.get(""), ) - if self.collab_obj_ids: - for name in self.collab_obj_ids: + collab_objs = self.client_app.get_collab_objects() + if collab_objs: + for name in collab_objs.keys(): p = Proxy( app=self.client_app, target_name=f"{client.name}.{name}", @@ -173,14 +167,14 @@ def execute(self, task_name: str, shareable: Shareable, fl_ctx: FLContext, abort # build proxies job_id = fl_ctx.get_job_id() - server_proxy = self._prepare_server_proxy(job_id, cell, server_collab_interface, abort_signal) + server_proxy = self._prepare_server_proxy(job_id, cell, server_collab_interface, abort_signal, fl_ctx) job_meta = fl_ctx.get_prop(FLContextKey.JOB_META) job_clients = job_meta.get(JobMetaKey.JOB_CLIENTS) all_clients = [from_dict(d) for d in job_clients] client_proxies = [] for c in all_clients: - p = self._prepare_client_proxy(job_id, cell, c, abort_signal, client_collab_interface) + p = self._prepare_client_proxy(job_id, cell, c, abort_signal, client_collab_interface, fl_ctx) client_proxies.append(p) ws = SysWorkspace(fl_ctx) diff --git a/nvflare/fox/sys/recipe.py b/nvflare/fox/sys/recipe.py index fa2fc88ca8..1e088f7768 100644 --- a/nvflare/fox/sys/recipe.py +++ b/nvflare/fox/sys/recipe.py @@ -11,10 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from typing import List + from nvflare.fox.api.app import App, ClientApp, ServerApp from nvflare.fox.api.filter import FilterChain -from nvflare.fox.api.strategy import Strategy -from nvflare.fuel.utils.validation_utils import check_object_type, check_positive_int, check_positive_number, check_str +from nvflare.fuel.utils.validation_utils import check_positive_int, check_positive_number, check_str from nvflare.job_config.api import FedJob from nvflare.recipe.spec import Recipe @@ -27,56 +28,87 @@ class FoxRecipe(Recipe): def __init__( self, job_name: str, - server_app: ServerApp, - client_app: ClientApp, + server: object, + client: object, + server_objects: dict[str, object] = None, + client_objects: dict[str, object] = None, sync_task_timeout=5, max_call_threads_for_server=100, max_call_threads_for_client=100, min_clients: int = 1, ): check_str("job_name", job_name) - check_object_type("server_app", server_app, ServerApp) check_positive_number("sync_task_timeout", sync_task_timeout) check_positive_int("max_call_threads_for_server", max_call_threads_for_server) check_positive_int("max_call_threads_for_client", max_call_threads_for_client) check_positive_int("min_clients", min_clients) - if not isinstance(client_app, ClientApp): - raise ValueError(f"client_app must be ClientApp but got {type(client_app)}") + self.job_name = job_name + self.server_app = ServerApp(server) + self.client_app = ClientApp(client) + + if server_objects: + for name, obj in server_objects.items(): + self.server_app.add_collab_object(name, obj) - # make sure server app has strategy - if not server_app.strategies: - raise ValueError(f"server_app has no strategies") + if client_objects: + for name, obj in client_objects.items(): + self.client_app.add_collab_object(name, obj) - self.job_name = job_name - self.server_app = server_app - self.client_app = client_app self.sync_task_timeout = sync_task_timeout self.max_call_threads_for_server = max_call_threads_for_server self.max_call_threads_for_client = max_call_threads_for_client self.min_clients = min_clients - job = self._create_job() + job = FedJob(name=self.job_name, min_clients=self.min_clients) Recipe.__init__(self, job) - def _create_job(self) -> FedJob: - job = FedJob(name=self.job_name, min_clients=self.min_clients) + def set_server_prop(self, name: str, value): + self.server_app.set_prop(name, value) + + def set_server_resource_dirs(self, resource_dirs): + self.server_app.set_resource_dirs(resource_dirs) + + def add_server_outgoing_call_filters(self, pattern: str, filters: List[object]): + self.server_app.add_outgoing_call_filters(pattern, filters) + + def add_server_incoming_call_filters(self, pattern: str, filters: List[object]): + self.server_app.add_incoming_call_filters(pattern, filters) + + def add_server_outgoing_result_filters(self, pattern: str, filters: List[object]): + self.server_app.add_outgoing_result_filters(pattern, filters) + + def add_server_incoming_result_filters(self, pattern: str, filters: List[object]): + self.server_app.add_incoming_result_filters(pattern, filters) + + def add_client_outgoing_call_filters(self, pattern: str, filters: List[object]): + self.client_app.add_outgoing_call_filters(pattern, filters) + + def add_client_incoming_call_filters(self, pattern: str, filters: List[object]): + self.client_app.add_incoming_call_filters(pattern, filters) + + def add_client_outgoing_result_filters(self, pattern: str, filters: List[object]): + self.client_app.add_outgoing_result_filters(pattern, filters) + + def add_client_incoming_result_filters(self, pattern: str, filters: List[object]): + self.client_app.add_incoming_result_filters(pattern, filters) + + def set_client_prop(self, name: str, value): + self.client_app.set_prop(name, value) - server_app_id = job.to_server(self.server_app, "_app") + def set_client_resource_dirs(self, resource_dirs): + self.client_app.set_resource_dirs(resource_dirs) - # get all strategies - strategy_ids = [] - for name, strategy in self.server_app.strategies: - comp_id = job.to_server(strategy, id=name) - strategy_ids.append(comp_id) + def finalize(self) -> FedJob: + server_obj_id = self.job.to_server(self.server_app.obj, "_server") + job = self.job collab_obj_ids, in_cf_arg, out_cf_arg, in_rf_arg, out_rf_arg = self._create_app_args( self.server_app, job.to_server ) controller = FoxController( - strategy_ids=strategy_ids, - server_app_id=server_app_id, + server_obj_id=server_obj_id, collab_obj_ids=collab_obj_ids, incoming_call_filters=in_cf_arg, outgoing_call_filters=out_cf_arg, @@ -91,12 +123,12 @@ def _create_job(self) -> FedJob: job.to_server(controller, id="controller") # add client config - client_app_id = job.to_clients(self.client_app, "_app") + client_obj_id = job.to_clients(self.client_app.obj, "_client") c_collab_obj_ids, c_in_cf_arg, c_out_cf_arg, c_in_rf_arg, c_out_rf_arg = self._create_app_args( self.client_app, job.to_clients ) executor = FoxExecutor( - client_app_id=client_app_id, + client_obj_id=client_obj_id, collab_obj_ids=c_collab_obj_ids, incoming_call_filters=c_in_cf_arg, outgoing_call_filters=c_out_cf_arg, @@ -114,8 +146,8 @@ def _create_app_args(self, app: App, to_f): collab_obj_ids = [] collab_objs = app.get_collab_objects() for name, obj in collab_objs.items(): - if isinstance(obj, Strategy): - # do not include strategy in collab objs since it's done separately. + if obj == app.obj: + # do not include in collab objs since it's done separately. continue comp_id = to_f(obj, id=name) collab_obj_ids.append(comp_id) @@ -148,6 +180,7 @@ def _create_filer_chain_arg(filter_chains: list, comp_table: dict): assert isinstance(chain, FilterChain) filter_ids = [] for f in chain.filters: + f = f.get_impl_object() comp_id = comp_table[id(f)] filter_ids.append(comp_id) d = {"pattern": chain.pattern, "filters": filter_ids} @@ -159,6 +192,7 @@ def _create_filter_components(to_f, filter_chains: list, comp_table: dict): for chain in filter_chains: assert isinstance(chain, FilterChain) for f in chain.filters: + f = f.get_impl_object() fid = id(f) comp_id = comp_table.get(fid) if not comp_id: diff --git a/nvflare/fox/sys/utils.py b/nvflare/fox/sys/utils.py index d4013f96a4..b992691c48 100644 --- a/nvflare/fox/sys/utils.py +++ b/nvflare/fox/sys/utils.py @@ -84,6 +84,7 @@ def _call_app_method(request: Message, app: App, logger) -> Message: return _error_reply(f"bad method args: should be list/tuple but got {type(method_args)}", logger) method_kwargs = payload.get(ObjectCallKey.KWARGS) + logger.info(f"received kwargs for method {method_name}: {method_kwargs}") if not method_kwargs: method_kwargs = {} elif not isinstance(method_kwargs, dict): From 24c5ab47e4798f11b925518ef8d0d889e2f62f7f Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Tue, 18 Nov 2025 15:43:14 -0500 Subject: [PATCH 061/102] added finalize --- nvflare/recipe/spec.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/nvflare/recipe/spec.py b/nvflare/recipe/spec.py index 53307ba8a3..cf8435e666 100644 --- a/nvflare/recipe/spec.py +++ b/nvflare/recipe/spec.py @@ -109,6 +109,14 @@ def __init__(self, job: FedJob): def process_env(self, env: ExecEnv): pass + def finalize(self): + """Called to finalize the setup of the recipe. + + Returns: + + """ + pass + def add_client_input_filter( self, filter: Filter, tasks: Optional[List[str]] = None, clients: Optional[List[str]] = None ): @@ -225,6 +233,8 @@ def export( Returns: None """ + self.finalize() + if server_exec_params: self.job.to_server(server_exec_params) @@ -247,6 +257,8 @@ def execute(self, env: ExecEnv, server_exec_params: dict = None, client_exec_par Returns: Run to get job ID and execution results """ + self.finalize() + if server_exec_params: self.job.to_server(server_exec_params) From c9f8e8070e45c04e01ad7f369ae0759d3bcf32fa Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Wed, 19 Nov 2025 12:00:29 -0500 Subject: [PATCH 062/102] fix recipes --- nvflare/fox/api/backend.py | 6 +- nvflare/fox/api/group.py | 162 +++++++++--------- nvflare/fox/api/proxy.py | 53 +++--- nvflare/fox/examples/np/recipe_fed_avg_seq.py | 27 +-- .../fox/examples/np/recipe_fed_avg_stream.py | 14 +- nvflare/fox/examples/pt/pt_avg_stream.py | 2 +- nvflare/fox/examples/pt/recipe_pt_avg.py | 18 +- .../fox/examples/pt/recipe_pt_avg_filter.py | 25 +-- .../fox/examples/pt/recipe_pt_avg_mixed.py | 18 +- .../fox/examples/pt/recipe_pt_avg_stream.py | 18 +- nvflare/fox/examples/pt/recipe_pt_np.py | 18 +- nvflare/fox/sim/backend.py | 1 - nvflare/fox/sys/backend.py | 4 +- 13 files changed, 159 insertions(+), 207 deletions(-) diff --git a/nvflare/fox/api/backend.py b/nvflare/fox/api/backend.py index 43c381d51a..6fa2ec3a6a 100644 --- a/nvflare/fox/api/backend.py +++ b/nvflare/fox/api/backend.py @@ -14,6 +14,8 @@ from abc import ABC, abstractmethod from nvflare.apis.signal import Signal +from nvflare.fuel.utils.log_utils import get_obj_logger +from nvflare.security.logging import secure_format_traceback from .gcc import GroupCallContext @@ -25,6 +27,7 @@ class Backend(ABC): def __init__(self, abort_signal: Signal): self.abort_signal = abort_signal + self.logger = get_obj_logger(self) @abstractmethod def call_target(self, target_name: str, func_name: str, *args, **kwargs): @@ -61,4 +64,5 @@ def call_target_in_group(self, gcc: GroupCallContext, target_name: str, func_nam pass def handle_exception(self, exception: Exception): - pass + self.logger.error(f"exception occurred: {secure_format_traceback()}") + raise exception diff --git a/nvflare/fox/api/group.py b/nvflare/fox/api/group.py index 2a27830099..14b8e3681e 100644 --- a/nvflare/fox/api/group.py +++ b/nvflare/fox/api/group.py @@ -105,91 +105,99 @@ def __getattr__(self, func_name): """ def method(*args, **kwargs): - gccs = {} - - # filter once for all targets - p = self._proxies[0] - the_proxy, func_itf, adj_args, adj_kwargs = p.adjust_func_args(func_name, args, kwargs) - ctx = the_proxy.app.new_context(the_proxy.app.name, the_proxy.name, target_group=self) - - self._logger.debug(f"[{ctx.header_str()}] calling {func_name} of group {[p.name for p in self._proxies]}") - - # apply outgoing call filters - assert isinstance(self._app, App) - adj_kwargs = self._app.apply_outgoing_call_filters(p.target_name, func_name, adj_kwargs, ctx) - check_call_args(func_name, func_itf, adj_args, adj_kwargs) - - for p in self._proxies: - the_proxy, func_itf, call_args, call_kwargs = p.adjust_func_args(func_name, adj_args, adj_kwargs) - call_kwargs = copy.copy(call_kwargs) - ctx = self._app.new_context(the_proxy.caller_name, the_proxy.name, target_group=self) - call_kwargs[CollabMethodArgName.CONTEXT] = ctx - - # set the optional args to help backend decide how to call - if self._timeout: - call_kwargs[CollabMethodOptionName.TIMEOUT] = self._timeout - - call_kwargs[CollabMethodOptionName.BLOCKING] = self._blocking - call_kwargs[CollabMethodOptionName.SECURE] = self._secure - call_kwargs[CollabMethodOptionName.OPTIONAL] = self._optional - - gcc = GroupCallContext( - self._app, - p.target_name, - func_name, - self._process_resp_cb, - self._cb_kwargs, - ctx, - ) - gccs[p.name] = gcc + the_backend = None + try: + gccs = {} + + # filter once for all targets + p = self._proxies[0] + the_proxy, func_itf, adj_args, adj_kwargs = p.adjust_func_args(func_name, args, kwargs) + the_backend = the_proxy.backend + ctx = the_proxy.app.new_context(the_proxy.app.name, the_proxy.name, target_group=self) self._logger.debug( - f"[{ctx.header_str()}] group call: {func_name=} args={call_args} kwargs={call_kwargs}" + f"[{ctx.header_str()}] calling {func_name} of group {[p.name for p in self._proxies]}" ) - the_proxy.backend.call_target_in_group(gcc, the_proxy.name, func_name, *call_args, **call_kwargs) - # wait for responses - if not self._blocking: - return None + # apply outgoing call filters + assert isinstance(self._app, App) + adj_kwargs = self._app.apply_outgoing_call_filters(p.target_name, func_name, adj_kwargs, ctx) + check_call_args(func_name, func_itf, adj_args, adj_kwargs) + + for p in self._proxies: + the_proxy, func_itf, call_args, call_kwargs = p.adjust_func_args(func_name, adj_args, adj_kwargs) + call_kwargs = copy.copy(call_kwargs) + ctx = self._app.new_context(the_proxy.caller_name, the_proxy.name, target_group=self) + call_kwargs[CollabMethodArgName.CONTEXT] = ctx + + # set the optional args to help backend decide how to call + if self._timeout: + call_kwargs[CollabMethodOptionName.TIMEOUT] = self._timeout + + call_kwargs[CollabMethodOptionName.BLOCKING] = self._blocking + call_kwargs[CollabMethodOptionName.SECURE] = self._secure + call_kwargs[CollabMethodOptionName.OPTIONAL] = self._optional + + gcc = GroupCallContext( + self._app, + p.target_name, + func_name, + self._process_resp_cb, + self._cb_kwargs, + ctx, + ) + gccs[p.name] = gcc + + self._logger.debug( + f"[{ctx.header_str()}] group call: {func_name=} args={call_args} kwargs={call_kwargs}" + ) + the_proxy.backend.call_target_in_group(gcc, the_proxy.name, func_name, *call_args, **call_kwargs) + + # wait for responses + if not self._blocking: + return None + + start_time = time.time() + min_received_time = None + while True: + if self._abort_signal.triggered: + raise RunAborted("run is aborted") + + # how many resps have been received? + resps_received = 0 + for name, gcc in gccs.items(): + if gcc.resp_time: + resps_received += 1 + + if resps_received == len(self._proxies): + break - start_time = time.time() - min_received_time = None - while True: - if self._abort_signal.triggered: - raise RunAborted("run is aborted") + now = time.time() + if resps_received >= self._min_resps: + if not min_received_time: + min_received_time = now + if now - min_received_time > self._wait_after_min_resps: + # waited long enough + break + elif self._timeout and now - start_time > self._timeout: + # timed out + break + else: + # still have not received min resps + time.sleep(0.1) - # how many resps have been received? - resps_received = 0 + # process results + results = {} for name, gcc in gccs.items(): if gcc.resp_time: - resps_received += 1 - - if resps_received == len(self._proxies): - break - - now = time.time() - if resps_received >= self._min_resps: - if not min_received_time: - min_received_time = now - if now - min_received_time > self._wait_after_min_resps: - # waited long enough - break - elif self._timeout and now - start_time > self._timeout: - # timed out - break - else: - # still have not received min resps - time.sleep(0.1) - - # process results - results = {} - for name, gcc in gccs.items(): - if gcc.resp_time: - result = gcc.result - else: - result = TimeoutError() - results[name] = result - return results + result = gcc.result + else: + result = TimeoutError() + results[name] = result + return results + except Exception as ex: + if the_backend: + the_backend.handle_exception(ex) return method diff --git a/nvflare/fox/api/proxy.py b/nvflare/fox/api/proxy.py index fc231260bb..69aea1733c 100644 --- a/nvflare/fox/api/proxy.py +++ b/nvflare/fox/api/proxy.py @@ -158,35 +158,38 @@ def __getattr__(self, func_name): """ def method(*args, **kwargs): - # remove option args - option_args = {} - for k in OPTION_ARGS: - if k in kwargs: - option_args[k] = kwargs.pop(k) - - p, func_itf, call_args, call_kwargs = self.adjust_func_args(func_name, args, kwargs) - ctx = p.app.new_context(self.caller_name, self.name) - - self.logger.debug( - f"[{ctx.header_str()}] calling target {p.target_name} func {func_name}: {call_args=} {call_kwargs=}" - ) + try: + # remove option args + option_args = {} + for k in OPTION_ARGS: + if k in kwargs: + option_args[k] = kwargs.pop(k) + + p, func_itf, call_args, call_kwargs = self.adjust_func_args(func_name, args, kwargs) + ctx = p.app.new_context(self.caller_name, self.name) + + self.logger.debug( + f"[{ctx.header_str()}] calling target {p.target_name} func {func_name}: {call_args=} {call_kwargs=}" + ) - # apply outgoing call filters - call_kwargs = self.app.apply_outgoing_call_filters(p.target_name, func_name, call_kwargs, ctx) - check_call_args(func_name, func_itf, call_args, call_kwargs) + # apply outgoing call filters + call_kwargs = self.app.apply_outgoing_call_filters(p.target_name, func_name, call_kwargs, ctx) + check_call_args(func_name, func_itf, call_args, call_kwargs) - call_kwargs[CollabMethodArgName.CONTEXT] = ctx + call_kwargs[CollabMethodArgName.CONTEXT] = ctx - # restore option args - for k, v in option_args.items(): - call_kwargs[k] = v + # restore option args + for k, v in option_args.items(): + call_kwargs[k] = v - result = p.backend.call_target(p.target_name, func_name, *call_args, **call_kwargs) - if isinstance(result, Exception): - raise result + result = p.backend.call_target(p.target_name, func_name, *call_args, **call_kwargs) + if isinstance(result, Exception): + raise result - # filter incoming result filters - result = self.app.apply_incoming_result_filters(p.target_name, func_name, result, ctx) - return result + # filter incoming result filters + result = self.app.apply_incoming_result_filters(p.target_name, func_name, result, ctx) + return result + except Exception as ex: + self.backend.handle_exception(ex) return method diff --git a/nvflare/fox/examples/np/recipe_fed_avg_seq.py b/nvflare/fox/examples/np/recipe_fed_avg_seq.py index 8de14777d4..2ea1ed9b44 100644 --- a/nvflare/fox/examples/np/recipe_fed_avg_seq.py +++ b/nvflare/fox/examples/np/recipe_fed_avg_seq.py @@ -13,41 +13,26 @@ # limitations under the License. import logging -from nvflare.fox.api.app import ServerApp from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.strategies.avg_seq import NPFedAvgSequential from nvflare.fox.examples.np.algos.widgets import MetricReceiver from nvflare.fox.sys.recipe import FoxRecipe -JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/fox/prod_00/admin@nvidia.com/transfer" +JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" def main(): simple_logging(logging.DEBUG) - server_app = ServerApp( - strategy_name="fedavg", - strategy=NPFedAvgSequential(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), - ) - server_app.add_collab_object("metric_receiver", MetricReceiver()) - server_app.set_prop("client_weight_config", {"red": 70, "blue": 100, "silver": 50}) - - client_app = NPTrainer(delta=1.0) - client_app.set_prop( - "client_delta", - { - "red": 1.0, - "blue": 2.0, - "silver": 3.0, - }, - ) - recipe = FoxRecipe( job_name="fedavg_seq", - server_app=server_app, - client_app=client_app, + server=NPFedAvgSequential(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), + client=NPTrainer(delta=1.0), + server_objects={"metric_receiver": MetricReceiver()}, ) + recipe.set_server_prop("client_weight_config", {"red": 70, "blue": 100, "silver": 50}) + recipe.set_client_prop("client_delta", {"red": 1.0, "blue": 2.0, "silver": 3.0}) recipe.export(JOB_ROOT_DIR) diff --git a/nvflare/fox/examples/np/recipe_fed_avg_stream.py b/nvflare/fox/examples/np/recipe_fed_avg_stream.py index ed8ff49df6..f6b04380eb 100644 --- a/nvflare/fox/examples/np/recipe_fed_avg_stream.py +++ b/nvflare/fox/examples/np/recipe_fed_avg_stream.py @@ -13,28 +13,20 @@ # limitations under the License. import logging -from nvflare.fox.api.app import ServerApp from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.avg_stream import NPFedAvgStream, NPTrainer from nvflare.fox.sys.recipe import FoxRecipe -JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/fox/prod_00/admin@nvidia.com/transfer" +JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" def main(): simple_logging(logging.DEBUG) - server_app = ServerApp( - strategy_name="fedavg_stream", - strategy=NPFedAvgStream(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), - ) - - client_app = NPTrainer(delta=1.0) - recipe = FoxRecipe( job_name="fedavg_stream", - server_app=server_app, - client_app=client_app, + server=NPFedAvgStream(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), + client=NPTrainer(delta=1.0), ) recipe.export(JOB_ROOT_DIR) diff --git a/nvflare/fox/examples/pt/pt_avg_stream.py b/nvflare/fox/examples/pt/pt_avg_stream.py index 3599eed7c0..29cd5dc10a 100644 --- a/nvflare/fox/examples/pt/pt_avg_stream.py +++ b/nvflare/fox/examples/pt/pt_avg_stream.py @@ -180,7 +180,7 @@ def main(): "y": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], "z": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], }, - num_rounds=4, + num_rounds=2, ) client = PTTrainer(delta=1.0) diff --git a/nvflare/fox/examples/pt/recipe_pt_avg.py b/nvflare/fox/examples/pt/recipe_pt_avg.py index 798ffd21c7..1fcd83c377 100644 --- a/nvflare/fox/examples/pt/recipe_pt_avg.py +++ b/nvflare/fox/examples/pt/recipe_pt_avg.py @@ -14,20 +14,19 @@ import logging from nvflare.app_opt.pt.decomposers import TensorDecomposer -from nvflare.fox.api.app import ServerApp from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.pt.pt_avg_filter import PTFedAvg, PTTrainer from nvflare.fox.sys.recipe import FoxRecipe -JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/fox/prod_00/admin@nvidia.com/transfer" +JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" def main(): simple_logging(logging.DEBUG) - server_app = ServerApp( - strategy_name="fedavg", - strategy=PTFedAvg( + recipe = FoxRecipe( + job_name="pt_fedavg", + server=PTFedAvg( initial_model={ "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], "y": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], @@ -35,14 +34,7 @@ def main(): }, num_rounds=2, ), - ) - - client_app = PTTrainer(delta=1.0) - - recipe = FoxRecipe( - job_name="pt_fedavg", - server_app=server_app, - client_app=client_app, + client=PTTrainer(delta=1.0), ) recipe.add_decomposers([TensorDecomposer()]) recipe.export(JOB_ROOT_DIR) diff --git a/nvflare/fox/examples/pt/recipe_pt_avg_filter.py b/nvflare/fox/examples/pt/recipe_pt_avg_filter.py index acf69e9029..c1866698c8 100644 --- a/nvflare/fox/examples/pt/recipe_pt_avg_filter.py +++ b/nvflare/fox/examples/pt/recipe_pt_avg_filter.py @@ -13,7 +13,6 @@ # limitations under the License. import logging -from nvflare.fox.api.app import ServerApp from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.pt.filters import ( IncomingModelCallFilter, @@ -24,15 +23,15 @@ from nvflare.fox.examples.pt.pt_avg_filter import PTFedAvg, PTTrainer from nvflare.fox.sys.recipe import FoxRecipe -JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/fox/prod_00/admin@nvidia.com/transfer" +JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" def main(): simple_logging(logging.DEBUG) - server_app = ServerApp( - strategy_name="fedavg_stream", - strategy=PTFedAvg( + recipe = FoxRecipe( + job_name="pt_fedavg_filter", + server=PTFedAvg( initial_model={ "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], "y": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], @@ -40,26 +39,20 @@ def main(): }, num_rounds=2, ), + client=PTTrainer(delta=1.0), ) - - server_app.add_outgoing_call_filters( + recipe.add_server_outgoing_call_filters( pattern="*.train", filters=[OutgoingModelCallFilter("weights")], ) - server_app.add_incoming_result_filters(pattern="*.train", filters=[IncomingModelResultFilter()]) + recipe.add_server_incoming_result_filters(pattern="*.train", filters=[IncomingModelResultFilter()]) - client_app = PTTrainer(delta=1.0) - client_app.add_incoming_call_filters( + recipe.add_client_incoming_call_filters( pattern="*.train", filters=[IncomingModelCallFilter("weights")], ) - client_app.add_outgoing_result_filters(pattern="*.train", filters=[OutgoingModelResultFilter()]) + recipe.add_client_outgoing_result_filters(pattern="*.train", filters=[OutgoingModelResultFilter()]) - recipe = FoxRecipe( - job_name="pt_fedavg_filter", - server_app=server_app, - client_app=client_app, - ) recipe.export(JOB_ROOT_DIR) diff --git a/nvflare/fox/examples/pt/recipe_pt_avg_mixed.py b/nvflare/fox/examples/pt/recipe_pt_avg_mixed.py index f9cd0bfe18..f34a9225ae 100644 --- a/nvflare/fox/examples/pt/recipe_pt_avg_mixed.py +++ b/nvflare/fox/examples/pt/recipe_pt_avg_mixed.py @@ -13,12 +13,11 @@ # limitations under the License. import logging -from nvflare.fox.api.app import ServerApp from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.pt.pt_avg_mixed import PTFedAvgMixed, PTTrainer from nvflare.fox.sys.recipe import FoxRecipe -JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/fox/prod_00/admin@nvidia.com/transfer" +JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" def main(): @@ -30,21 +29,14 @@ def main(): "z": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], } - server_app = ServerApp( - strategy_name="fedavg_mixed", - strategy=PTFedAvgMixed( + recipe = FoxRecipe( + job_name="pt_fedavg_mixed", + server=PTFedAvgMixed( pt_model=init_model, np_model=init_model, num_rounds=2, ), - ) - - client_app = PTTrainer(delta=1.0) - - recipe = FoxRecipe( - job_name="pt_fedavg_mixed", - server_app=server_app, - client_app=client_app, + client=PTTrainer(delta=1.0), ) recipe.export(JOB_ROOT_DIR) diff --git a/nvflare/fox/examples/pt/recipe_pt_avg_stream.py b/nvflare/fox/examples/pt/recipe_pt_avg_stream.py index 5479f8bba4..cd9567962b 100644 --- a/nvflare/fox/examples/pt/recipe_pt_avg_stream.py +++ b/nvflare/fox/examples/pt/recipe_pt_avg_stream.py @@ -13,20 +13,19 @@ # limitations under the License. import logging -from nvflare.fox.api.app import ServerApp from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.pt.pt_avg_stream import PTFedAvgStream, PTTrainer from nvflare.fox.sys.recipe import FoxRecipe -JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/fox/prod_00/admin@nvidia.com/transfer" +JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" def main(): simple_logging(logging.DEBUG) - server_app = ServerApp( - strategy_name="fedavg_stream", - strategy=PTFedAvgStream( + recipe = FoxRecipe( + job_name="pt_fedavg_stream", + server=PTFedAvgStream( initial_model={ "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], "y": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], @@ -34,14 +33,7 @@ def main(): }, num_rounds=2, ), - ) - - client_app = PTTrainer(delta=1.0) - - recipe = FoxRecipe( - job_name="pt_fedavg_stream", - server_app=server_app, - client_app=client_app, + client=PTTrainer(delta=1.0), ) recipe.export(JOB_ROOT_DIR) diff --git a/nvflare/fox/examples/pt/recipe_pt_np.py b/nvflare/fox/examples/pt/recipe_pt_np.py index 32c3de7a0b..60fe71a23e 100644 --- a/nvflare/fox/examples/pt/recipe_pt_np.py +++ b/nvflare/fox/examples/pt/recipe_pt_np.py @@ -15,12 +15,11 @@ from nvflare.app_common.decomposers.numpy_decomposers import NumpyArrayDecomposer from nvflare.app_opt.pt.decomposers import TensorDecomposer -from nvflare.fox.api.app import ServerApp from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.pt.pt_np import PTFedAvgMixed, PTTrainer from nvflare.fox.sys.recipe import FoxRecipe -JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/fox/prod_00/admin@nvidia.com/transfer" +JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" def main(): @@ -32,21 +31,14 @@ def main(): "z": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], } - server_app = ServerApp( - strategy_name="fedavg", - strategy=PTFedAvgMixed( + recipe = FoxRecipe( + job_name="pt_np", + server=PTFedAvgMixed( pt_model=init_model, np_model=init_model, num_rounds=2, ), - ) - - client_app = PTTrainer(delta=1.0) - - recipe = FoxRecipe( - job_name="pt_np", - server_app=server_app, - client_app=client_app, + client=PTTrainer(delta=1.0), ) recipe.add_decomposers([TensorDecomposer(), NumpyArrayDecomposer]) recipe.export(JOB_ROOT_DIR) diff --git a/nvflare/fox/sim/backend.py b/nvflare/fox/sim/backend.py index d387d9fcf7..e65f50b424 100644 --- a/nvflare/fox/sim/backend.py +++ b/nvflare/fox/sim/backend.py @@ -40,7 +40,6 @@ def __init__(self, target_obj_name: str, target_app: App, target_obj, abort_sign self.target_app = target_app self.target_obj = target_obj self.executor = thread_executor - self.logger = get_obj_logger(self) def _get_func(self, func_name): return self.target_app.find_collab_method(self.target_obj, func_name) diff --git a/nvflare/fox/sys/backend.py b/nvflare/fox/sys/backend.py index e7720fe136..6dbb8de9ec 100644 --- a/nvflare/fox/sys/backend.py +++ b/nvflare/fox/sys/backend.py @@ -18,7 +18,7 @@ from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey, ReturnCode from nvflare.fuel.f3.cellnet.utils import new_cell_message from nvflare.fuel.f3.message import Message -from nvflare.fuel.utils.log_utils import get_obj_logger +from nvflare.security.logging import secure_log_traceback from .constants import MSG_CHANNEL, MSG_TOPIC, CallReplyKey, ObjectCallKey @@ -29,7 +29,6 @@ def __init__(self, manager, engine, caller, cell, target_fqcn, abort_signal, thr Backend.__init__(self, abort_signal) self.manager = manager self.engine = engine - self.logger = get_obj_logger(self) self.caller = caller self.cell = cell self.target_fqcn = target_fqcn @@ -108,4 +107,5 @@ def _run_func(self, gcc: GroupCallContext, target_name: str, func_name: str, arg def handle_exception(self, exception: Exception): fl_ctx = self.engine.new_context() + secure_log_traceback(self.logger) self.manager.system_panic(f"exception occurred: {exception}", fl_ctx) From 2661bc9bfe51254f3720f5f78f805156a27f75f5 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Wed, 19 Nov 2025 12:46:51 -0500 Subject: [PATCH 063/102] support final dec --- nvflare/fox/api/app.py | 20 ++++++++++- nvflare/fox/api/dec.py | 16 ++++++++- nvflare/fox/api/facade.py | 2 ++ nvflare/fox/api/proxy.py | 8 ++--- nvflare/fox/api/run_server.py | 3 ++ .../examples/np/algos/strategies/cyclic.py | 6 ++++ nvflare/fox/examples/np/algos/swarm.py | 2 +- nvflare/fox/examples/np/recipe_cyclic.py | 36 +++++++++++++++++++ .../fox/examples/np/recipe_fed_avg_intime.py | 2 +- nvflare/fox/examples/np/recipe_fed_avg_seq.py | 2 +- .../fox/examples/np/recipe_fed_avg_stream.py | 2 +- nvflare/fox/examples/np/recipe_swarm.py | 2 +- nvflare/fox/examples/pt/recipe_pt_avg.py | 2 +- .../fox/examples/pt/recipe_pt_avg_filter.py | 2 +- .../fox/examples/pt/recipe_pt_avg_mixed.py | 2 +- .../fox/examples/pt/recipe_pt_avg_stream.py | 2 +- nvflare/fox/examples/pt/recipe_pt_np.py | 2 +- nvflare/fox/sim/simulator.py | 15 ++++++-- nvflare/fox/sys/executor.py | 9 +++-- 19 files changed, 112 insertions(+), 23 deletions(-) create mode 100644 nvflare/fox/examples/np/recipe_cyclic.py diff --git a/nvflare/fox/api/app.py b/nvflare/fox/api/app.py index e62d69118d..d0efbd0c6b 100644 --- a/nvflare/fox/api/app.py +++ b/nvflare/fox/api/app.py @@ -25,6 +25,7 @@ collab, get_object_algo_funcs, get_object_collab_interface, + get_object_final_funcs, get_object_init_funcs, is_collab, supports_context, @@ -270,7 +271,7 @@ def find_collab_method(self, target_obj, method_name): def _fox_init(self, obj, ctx: Context): init_funcs = get_object_init_funcs(obj) for name, f in init_funcs: - self.logger.info(f"calling init func {name} ...") + self.logger.debug(f"calling init func {name} ...") if supports_context(f): kwargs = {CollabMethodArgName.CONTEXT: ctx} else: @@ -284,6 +285,23 @@ def initialize(self, context: Context): for obj in self._managed_objects.values(): self._fox_init(obj, context) + def _fox_finalize(self, obj, ctx: Context): + funcs = get_object_final_funcs(obj) + for name, f in funcs: + self.logger.debug(f"calling final func {name} ...") + if supports_context(f): + kwargs = {CollabMethodArgName.CONTEXT: ctx} + else: + kwargs = {} + f(**kwargs) + + def finalize(self, context: Context): + self._fox_finalize(self, context) + + # initialize target objects + for obj in self._managed_objects.values(): + self._fox_finalize(obj, context) + def new_context(self, caller: str, callee: str, target_group=None): ctx = Context(self, caller, callee, self._abort_signal, target_group=target_group) set_call_context(ctx) diff --git a/nvflare/fox/api/dec.py b/nvflare/fox/api/dec.py index ff9bdcec00..4ff5ecdb95 100644 --- a/nvflare/fox/api/dec.py +++ b/nvflare/fox/api/dec.py @@ -17,6 +17,7 @@ _FLAG_COLLAB = "_fox_is_collab" _FLAG_INIT = "_fox_is_init" +_FLAG_FINAL = "_fox_is_final" _FLAG_ALGO = "_fox_is_algo" _FLAG_CALL_FILTER = "_fox_is_call_filter" _FLAG_RESULT_FILTER = "_fox_is_result_filter" @@ -77,6 +78,19 @@ def get_object_init_funcs(obj): return _get_object_funcs(obj, _FLAG_INIT, "init") +def final(func): + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + _set_attrs(func, wrapper) + setattr(wrapper, _FLAG_FINAL, True) + return wrapper + + +def get_object_final_funcs(obj): + return _get_object_funcs(obj, _FLAG_FINAL, "final") + + def algo(func): def wrapper(*args, **kwargs): return func(*args, **kwargs) @@ -130,7 +144,7 @@ def _get_object_funcs(obj, flag, func_type): for name in dir(obj): func = getattr(obj, name) if callable(func) and _has_flag(func, flag): - print(f"found {func_type} func of object {obj.__class__.__name__}.{name}") + # print(f"found {func_type} func of object {obj.__class__.__name__}.{name}") result.append((name, func)) return result diff --git a/nvflare/fox/api/facade.py b/nvflare/fox/api/facade.py index 17206e9dbc..bc2acbd4b1 100644 --- a/nvflare/fox/api/facade.py +++ b/nvflare/fox/api/facade.py @@ -17,6 +17,7 @@ from .dec import call_filter as dec_call_filter from .dec import classproperty from .dec import collab as dec_collab +from .dec import final as dec_final from .dec import init as dec_init from .dec import result_filter as dec_result_filter from .proxy_list import ProxyList @@ -26,6 +27,7 @@ class facade: collab = dec_collab init = dec_init + final = dec_final algo = dec_algo call_filter = dec_call_filter result_filter = dec_result_filter diff --git a/nvflare/fox/api/proxy.py b/nvflare/fox/api/proxy.py index 69aea1733c..02da833310 100644 --- a/nvflare/fox/api/proxy.py +++ b/nvflare/fox/api/proxy.py @@ -71,7 +71,6 @@ def __call__( optional: bool = False, secure: bool = False, ): - print(f"creating _ProxyCall: {blocking=} {timeout=} {optional=} {secure=}") return _ProxyCall( proxy=self, blocking=blocking, @@ -98,24 +97,21 @@ def get_target(self, name: str): return None def _find_interface(self, func_name): - self.logger.info(f"trying to find interface for {func_name}") + self.logger.debug(f"trying to find interface for {func_name}") args = self.target_interface.get(func_name) if self.target_interface else None if args: return self, args # try children - self.logger.info(f"not defined for {func_name}: trying children") the_args = None the_proxy = None the_name = None for n, c in self.children.items(): - self.logger.info(f"trying child {n} ...") args = c.target_interface.get(func_name) if c.target_interface else None if not args: - self.logger.info(f"child {n} has no {func_name}") continue - self.logger.info(f"found interface for func {func_name}: defined in child {n}") + self.logger.debug(f"found interface for func {func_name}: defined in child {n}") if not the_proxy: the_name = n diff --git a/nvflare/fox/api/run_server.py b/nvflare/fox/api/run_server.py index 8b9748de03..2d12878f7a 100644 --- a/nvflare/fox/api/run_server.py +++ b/nvflare/fox/api/run_server.py @@ -43,4 +43,7 @@ def run_server(server_app: ServerApp, logger): backend = server_app.get_backend() backend.handle_exception(ex) break + + logger.info("finalizing server app") + server_app.finalize(server_ctx) return result diff --git a/nvflare/fox/examples/np/algos/strategies/cyclic.py b/nvflare/fox/examples/np/algos/strategies/cyclic.py index 87810b5fc3..b11da72c4d 100644 --- a/nvflare/fox/examples/np/algos/strategies/cyclic.py +++ b/nvflare/fox/examples/np/algos/strategies/cyclic.py @@ -24,6 +24,7 @@ def __init__(self, initial_model, num_rounds=2): self.num_rounds = num_rounds self.initial_model = initial_model self._initial_model = parse_array_def(initial_model) + self.final_model = None self.logger = get_obj_logger(self) @fox.algo @@ -32,8 +33,13 @@ def execute(self): for current_round in range(self.num_rounds): current_model = self._do_one_round(current_round, current_model) self.logger.info(f"[{fox.call_info}] final result: {current_model}") + self.final_model = current_model return current_model + @fox.final + def done(self): + self.logger.info(f"Cyclic is done: final model: {self.final_model}") + def _do_one_round(self, current_round, current_model): random.shuffle(fox.clients) for c in fox.clients: diff --git a/nvflare/fox/examples/np/algos/swarm.py b/nvflare/fox/examples/np/algos/swarm.py index 9a0bb916a6..86e92abe35 100644 --- a/nvflare/fox/examples/np/algos/swarm.py +++ b/nvflare/fox/examples/np/algos/swarm.py @@ -67,7 +67,7 @@ def train(self, weights, current_round): return weights + self.delta def sag(self, model, current_round): - results = fox.clients.train2(model, current_round) + results = fox.clients.train(model, current_round) results = list(results.values()) total = 0 for i in range(len(results)): diff --git a/nvflare/fox/examples/np/recipe_cyclic.py b/nvflare/fox/examples/np/recipe_cyclic.py new file mode 100644 index 0000000000..22db6cceca --- /dev/null +++ b/nvflare/fox/examples/np/recipe_cyclic.py @@ -0,0 +1,36 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples.np.algos.client import NPTrainer +from nvflare.fox.examples.np.algos.strategies.cyclic import NPCyclic +from nvflare.fox.sys.recipe import FoxRecipe + +JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" + + +def main(): + simple_logging(logging.DEBUG) + + recipe = FoxRecipe( + job_name="fox_cyclic", + server=NPCyclic(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), + client=NPTrainer(delta=1.0), + ) + recipe.export(JOB_ROOT_DIR) + + +if __name__ == "__main__": + main() diff --git a/nvflare/fox/examples/np/recipe_fed_avg_intime.py b/nvflare/fox/examples/np/recipe_fed_avg_intime.py index 3088a57aef..c0a8cf5f0d 100644 --- a/nvflare/fox/examples/np/recipe_fed_avg_intime.py +++ b/nvflare/fox/examples/np/recipe_fed_avg_intime.py @@ -27,7 +27,7 @@ def main(): simple_logging(logging.DEBUG) recipe = FoxRecipe( - job_name="fedavg_intime", + job_name="fox_fedavg_intime", server=NPFedAvgInTime(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), server_objects={ "metric_receiver": MetricReceiver(), diff --git a/nvflare/fox/examples/np/recipe_fed_avg_seq.py b/nvflare/fox/examples/np/recipe_fed_avg_seq.py index 2ea1ed9b44..9856925423 100644 --- a/nvflare/fox/examples/np/recipe_fed_avg_seq.py +++ b/nvflare/fox/examples/np/recipe_fed_avg_seq.py @@ -26,7 +26,7 @@ def main(): simple_logging(logging.DEBUG) recipe = FoxRecipe( - job_name="fedavg_seq", + job_name="fox_fedavg_seq", server=NPFedAvgSequential(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), client=NPTrainer(delta=1.0), server_objects={"metric_receiver": MetricReceiver()}, diff --git a/nvflare/fox/examples/np/recipe_fed_avg_stream.py b/nvflare/fox/examples/np/recipe_fed_avg_stream.py index f6b04380eb..4cba5663c0 100644 --- a/nvflare/fox/examples/np/recipe_fed_avg_stream.py +++ b/nvflare/fox/examples/np/recipe_fed_avg_stream.py @@ -24,7 +24,7 @@ def main(): simple_logging(logging.DEBUG) recipe = FoxRecipe( - job_name="fedavg_stream", + job_name="fox_fedavg_stream", server=NPFedAvgStream(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), client=NPTrainer(delta=1.0), ) diff --git a/nvflare/fox/examples/np/recipe_swarm.py b/nvflare/fox/examples/np/recipe_swarm.py index fa62ef612d..253e1677da 100644 --- a/nvflare/fox/examples/np/recipe_swarm.py +++ b/nvflare/fox/examples/np/recipe_swarm.py @@ -24,7 +24,7 @@ def main(): simple_logging(logging.DEBUG) recipe = FoxRecipe( - job_name="swarm", + job_name="fox_swarm", server=NPSwarm(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=5), client=NPSwarmClient(delta=1.0), ) diff --git a/nvflare/fox/examples/pt/recipe_pt_avg.py b/nvflare/fox/examples/pt/recipe_pt_avg.py index 1fcd83c377..b93c166200 100644 --- a/nvflare/fox/examples/pt/recipe_pt_avg.py +++ b/nvflare/fox/examples/pt/recipe_pt_avg.py @@ -25,7 +25,7 @@ def main(): simple_logging(logging.DEBUG) recipe = FoxRecipe( - job_name="pt_fedavg", + job_name="fox_pt_fedavg", server=PTFedAvg( initial_model={ "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], diff --git a/nvflare/fox/examples/pt/recipe_pt_avg_filter.py b/nvflare/fox/examples/pt/recipe_pt_avg_filter.py index c1866698c8..0e9f1c68b6 100644 --- a/nvflare/fox/examples/pt/recipe_pt_avg_filter.py +++ b/nvflare/fox/examples/pt/recipe_pt_avg_filter.py @@ -30,7 +30,7 @@ def main(): simple_logging(logging.DEBUG) recipe = FoxRecipe( - job_name="pt_fedavg_filter", + job_name="fox_pt_fedavg_filter", server=PTFedAvg( initial_model={ "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], diff --git a/nvflare/fox/examples/pt/recipe_pt_avg_mixed.py b/nvflare/fox/examples/pt/recipe_pt_avg_mixed.py index f34a9225ae..ad1317ec79 100644 --- a/nvflare/fox/examples/pt/recipe_pt_avg_mixed.py +++ b/nvflare/fox/examples/pt/recipe_pt_avg_mixed.py @@ -30,7 +30,7 @@ def main(): } recipe = FoxRecipe( - job_name="pt_fedavg_mixed", + job_name="fox_pt_fedavg_mixed", server=PTFedAvgMixed( pt_model=init_model, np_model=init_model, diff --git a/nvflare/fox/examples/pt/recipe_pt_avg_stream.py b/nvflare/fox/examples/pt/recipe_pt_avg_stream.py index cd9567962b..f42171d6f5 100644 --- a/nvflare/fox/examples/pt/recipe_pt_avg_stream.py +++ b/nvflare/fox/examples/pt/recipe_pt_avg_stream.py @@ -24,7 +24,7 @@ def main(): simple_logging(logging.DEBUG) recipe = FoxRecipe( - job_name="pt_fedavg_stream", + job_name="fox_pt_fedavg_stream", server=PTFedAvgStream( initial_model={ "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], diff --git a/nvflare/fox/examples/pt/recipe_pt_np.py b/nvflare/fox/examples/pt/recipe_pt_np.py index 60fe71a23e..7aa4f1b5b8 100644 --- a/nvflare/fox/examples/pt/recipe_pt_np.py +++ b/nvflare/fox/examples/pt/recipe_pt_np.py @@ -32,7 +32,7 @@ def main(): } recipe = FoxRecipe( - job_name="pt_np", + job_name="fox_pt_np", server=PTFedAvgMixed( pt_model=init_model, np_model=init_model, diff --git a/nvflare/fox/sim/simulator.py b/nvflare/fox/sim/simulator.py index 64a5e253ef..314e1f5089 100644 --- a/nvflare/fox/sim/simulator.py +++ b/nvflare/fox/sim/simulator.py @@ -186,9 +186,18 @@ def run(self): def _try_run(self): # initialize all apps + client_ctx = {} for n, app in self.client_apps.items(): - self.logger.info(f"initializing client app for {n}") - app.initialize(app.new_context(n, n)) + ctx = app.new_context(n, n) + client_ctx[n] = ctx + self.logger.info(f"initializing client app {n}") + app.initialize(ctx) # run the server - return run_server(self.server_app, self.logger) + result = run_server(self.server_app, self.logger) + for n, app in self.client_apps.items(): + ctx = client_ctx[n] + self.logger.info(f"finalizing client app {n}") + app.finalize(ctx) + + return result diff --git a/nvflare/fox/sys/executor.py b/nvflare/fox/sys/executor.py index fef4e70b9c..3b7ed8382b 100644 --- a/nvflare/fox/sys/executor.py +++ b/nvflare/fox/sys/executor.py @@ -63,6 +63,7 @@ def __init__( self.register_event_handler(EventType.START_RUN, self._handle_start_run) self.register_event_handler(EventType.END_RUN, self._handle_end_run) self.client_app = None + self.client_ctx = None self.thread_executor = ThreadPoolExecutor(max_workers=max_call_threads) def _handle_start_run(self, event_type: str, fl_ctx: FLContext): @@ -81,6 +82,9 @@ def _handle_start_run(self, event_type: str, fl_ctx: FLContext): self.system_panic(err, fl_ctx) def _handle_end_run(self, event_type: str, fl_ctx: FLContext): + if self.client_ctx: + self.logger.info(f"finalizing client app {self.client_app.name}") + self.client_app.finalize(self.client_ctx) self.thread_executor.shutdown(wait=False, cancel_futures=True) def _prepare_server_proxy(self, job_id, cell, collab_interface: dict, abort_signal, fl_ctx: FLContext): @@ -180,8 +184,9 @@ def execute(self, task_name: str, shareable: Shareable, fl_ctx: FLContext, abort ws = SysWorkspace(fl_ctx) self.client_app.setup(ws, server_proxy, client_proxies, abort_signal) - ctx = self.client_app.new_context(self.client_app.name, self.client_app.name) - self.client_app.initialize(ctx) + self.client_ctx = self.client_app.new_context(self.client_app.name, self.client_app.name) + self.logger.info(f"initializing client app {self.client_app.name}") + self.client_app.initialize(self.client_ctx) reply = make_reply(ReturnCode.OK) reply[SyncKey.COLLAB_INTERFACE] = client_collab_interface From 676f520a70cd7646862d101c886651274510181c Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Wed, 19 Nov 2025 15:23:11 -0500 Subject: [PATCH 064/102] add more filter decs --- nvflare/fox/api/app.py | 12 +- nvflare/fox/api/dec.py | 56 ++++++++ nvflare/fox/api/facade.py | 8 ++ nvflare/fox/api/filter.py | 133 +++++++++++------- nvflare/fox/api/gcc.py | 6 - nvflare/fox/examples/pt/filters.py | 13 ++ nvflare/fox/examples/pt/filters2.py | 79 +++++++++++ .../fox/examples/pt/recipe_pt_avg_filter2.py | 56 ++++++++ nvflare/fox/sys/utils.py | 6 +- 9 files changed, 307 insertions(+), 62 deletions(-) create mode 100644 nvflare/fox/examples/pt/filters2.py create mode 100644 nvflare/fox/examples/pt/recipe_pt_avg_filter2.py diff --git a/nvflare/fox/api/app.py b/nvflare/fox/api/app.py index d0efbd0c6b..8399d76251 100644 --- a/nvflare/fox/api/app.py +++ b/nvflare/fox/api/app.py @@ -88,7 +88,7 @@ def get_server_proxy(self): def get_client_proxies(self): return copy.copy(self.clients) - def _add_filters(self, pattern: str, filters, to_list: list, filter_type): + def _add_filters(self, pattern: str, filters, to_list: list, filter_type, incoming): if not filters: return @@ -99,7 +99,7 @@ def _add_filters(self, pattern: str, filters, to_list: list, filter_type): for f in filters: if not isinstance(f, filter_type): # convert to proper filter type - filter_obj = filter_type(f) + filter_obj = filter_type(f, incoming) else: filter_obj = f self._add_managed_object(f) @@ -110,25 +110,25 @@ def _add_filters(self, pattern: str, filters, to_list: list, filter_type): to_list.append(chain) def add_incoming_call_filters(self, pattern: str, filters: List[object]): - self._add_filters(pattern, filters, self._incoming_call_filter_chains, CallFilter) + self._add_filters(pattern, filters, self._incoming_call_filter_chains, CallFilter, True) def get_incoming_call_filters(self): return self._incoming_call_filter_chains def add_outgoing_call_filters(self, pattern: str, filters: List[object]): - self._add_filters(pattern, filters, self._outgoing_call_filter_chains, CallFilter) + self._add_filters(pattern, filters, self._outgoing_call_filter_chains, CallFilter, False) def get_outgoing_call_filters(self): return self._outgoing_call_filter_chains def add_incoming_result_filters(self, pattern: str, filters: List[object]): - self._add_filters(pattern, filters, self._incoming_result_filter_chains, ResultFilter) + self._add_filters(pattern, filters, self._incoming_result_filter_chains, ResultFilter, True) def get_incoming_result_filters(self): return self._incoming_result_filter_chains def add_outgoing_result_filters(self, pattern: str, filters: List[object]): - self._add_filters(pattern, filters, self._outgoing_result_filter_chains, ResultFilter) + self._add_filters(pattern, filters, self._outgoing_result_filter_chains, ResultFilter, False) def get_outgoing_result_filters(self): return self._outgoing_result_filter_chains diff --git a/nvflare/fox/api/dec.py b/nvflare/fox/api/dec.py index 4ff5ecdb95..bdc9a8e9fa 100644 --- a/nvflare/fox/api/dec.py +++ b/nvflare/fox/api/dec.py @@ -20,7 +20,11 @@ _FLAG_FINAL = "_fox_is_final" _FLAG_ALGO = "_fox_is_algo" _FLAG_CALL_FILTER = "_fox_is_call_filter" +_FLAG_IN_CALL_FILTER = "_fox_is_in_call_filter" +_FLAG_OUT_CALL_FILTER = "_fox_is_out_call_filter" _FLAG_RESULT_FILTER = "_fox_is_result_filter" +_FLAG_IN_RESULT_FILTER = "_fox_is_in_result_filter" +_FLAG_OUT_RESULT_FILTER = "_fox_is_out_result_filter" _FLAG_SUPPORT_CTX = "_fox_supports_ctx" _ATTR_PARAM_NAMES = "_fox_param_names" @@ -117,6 +121,32 @@ def get_object_call_filter_funcs(obj): return _get_object_funcs(obj, _FLAG_CALL_FILTER, "call_filter") +def in_call_filter(func): + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + _set_attrs(func, wrapper) + setattr(wrapper, _FLAG_IN_CALL_FILTER, True) + return wrapper + + +def get_object_in_call_filter_funcs(obj): + return _get_object_funcs(obj, _FLAG_IN_CALL_FILTER, "in_call_filter") + + +def out_call_filter(func): + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + _set_attrs(func, wrapper) + setattr(wrapper, _FLAG_OUT_CALL_FILTER, True) + return wrapper + + +def get_object_out_call_filter_funcs(obj): + return _get_object_funcs(obj, _FLAG_OUT_CALL_FILTER, "out_call_filter") + + def result_filter(func): def wrapper(*args, **kwargs): return func(*args, **kwargs) @@ -130,6 +160,32 @@ def get_object_result_filter_funcs(obj): return _get_object_funcs(obj, _FLAG_RESULT_FILTER, "result_filter") +def in_result_filter(func): + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + _set_attrs(func, wrapper) + setattr(wrapper, _FLAG_IN_RESULT_FILTER, True) + return wrapper + + +def get_object_in_result_filter_funcs(obj): + return _get_object_funcs(obj, _FLAG_IN_RESULT_FILTER, "in_result_filter") + + +def out_result_filter(func): + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + _set_attrs(func, wrapper) + setattr(wrapper, _FLAG_OUT_RESULT_FILTER, True) + return wrapper + + +def get_object_out_result_filter_funcs(obj): + return _get_object_funcs(obj, _FLAG_OUT_RESULT_FILTER, "out_result_filter") + + def get_param_names(func): return getattr(func, _ATTR_PARAM_NAMES, None) diff --git a/nvflare/fox/api/facade.py b/nvflare/fox/api/facade.py index bc2acbd4b1..25881e296a 100644 --- a/nvflare/fox/api/facade.py +++ b/nvflare/fox/api/facade.py @@ -18,7 +18,11 @@ from .dec import classproperty from .dec import collab as dec_collab from .dec import final as dec_final +from .dec import in_call_filter as dec_in_call_filter +from .dec import in_result_filter as dec_in_result_filter from .dec import init as dec_init +from .dec import out_call_filter as dec_out_call_filter +from .dec import out_result_filter as dec_out_result_filter from .dec import result_filter as dec_result_filter from .proxy_list import ProxyList @@ -30,7 +34,11 @@ class facade: final = dec_final algo = dec_algo call_filter = dec_call_filter + in_call_filter = dec_in_call_filter + out_call_filter = dec_out_call_filter result_filter = dec_result_filter + in_result_filter = dec_in_result_filter + out_result_filter = dec_out_result_filter @classproperty def context(cls): diff --git a/nvflare/fox/api/filter.py b/nvflare/fox/api/filter.py index 461ac46ba4..7fc4755f26 100644 --- a/nvflare/fox/api/filter.py +++ b/nvflare/fox/api/filter.py @@ -17,13 +17,24 @@ from .constants import CollabMethodArgName from .ctx import Context -from .dec import get_object_call_filter_funcs, get_object_result_filter_funcs, supports_context +from .dec import ( + get_object_call_filter_funcs, + get_object_in_call_filter_funcs, + get_object_in_result_filter_funcs, + get_object_out_call_filter_funcs, + get_object_out_result_filter_funcs, + get_object_result_filter_funcs, + supports_context, +) class _Filter: - def __init__(self, impl: object = None): + def __init__(self, filter_type: str, impl: object = None, incoming=True): + self.filter_type = filter_type self.impl = impl + self.incoming = incoming + self.impl_func = None self.logger = get_obj_logger(self) def get_impl_object(self): @@ -32,23 +43,71 @@ def get_impl_object(self): else: return self + def filter_data(self, data, context: Context): + if self.impl_func is not None: + if self.incoming: + d = "incoming" + else: + d = "outgoing" + + name, f = self.impl_func + self.logger.info(f"calling {d} {self.filter_type}: {name} ...") + if supports_context(f): + kwargs = {CollabMethodArgName.CONTEXT: context} + else: + kwargs = {} + return f(data, **kwargs) + else: + return data + + +def _determine_filter_impl_func( + obj, + filter_type: str, + incoming: bool, + get_filter_f, + get_in_filter_f, + get_out_filter_f, +): + if incoming: + funcs = get_in_filter_f(obj) + d = "in" + else: + funcs = get_out_filter_f(obj) + d = "out" + + if len(funcs) > 1: + raise ValueError( + f"filter object {obj.__class__.__name__} must have one {d}_{filter_type} func but got {len(funcs)}" + ) + + if len(funcs) == 1: + return funcs[0] + + funcs = get_filter_f(obj) + if not funcs: + raise ValueError(f"filter impl object {obj.__class__.__name__} has no {filter_type} func") + + if len(funcs) > 1: + raise ValueError( + f"filter object {obj.__class__.__name__} must have one {filter_type} func but got {len(funcs)}" + ) + return funcs[0] + class CallFilter(_Filter): - def __init__(self, impl: object = None): - super().__init__(impl) + def __init__(self, impl: object = None, incoming=True): + super().__init__("call filter", impl, incoming) if impl: - funcs = get_object_call_filter_funcs(impl) - if not funcs: - raise ValueError(f"filter impl object {impl.__class__.__name__} has no call_filter func") - - if len(funcs) > 1: - raise ValueError( - f"filter object {impl.__class__.__name__} must have one call_filter func but got {len(funcs)}" - ) - self.impl_func = funcs[0] - else: - self.impl_func = None + self.impl_func = _determine_filter_impl_func( + obj=impl, + incoming=incoming, + filter_type="call_filter", + get_filter_f=get_object_call_filter_funcs, + get_in_filter_f=get_object_in_call_filter_funcs, + get_out_filter_f=get_object_out_call_filter_funcs, + ) def filter_call(self, func_kwargs: dict, context: Context): """Filter kwargs of function call. @@ -60,34 +119,22 @@ def filter_call(self, func_kwargs: dict, context: Context): Returns: filtered kwargs that will be passed to a collab func. """ - if self.impl_func is not None: - name, f = self.impl_func - self.logger.info(f"calling call filter: {name} ...") - if supports_context(f): - kwargs = {CollabMethodArgName.CONTEXT: context} - else: - kwargs = {} - return f(func_kwargs, **kwargs) - else: - return func_kwargs + return self.filter_data(func_kwargs, context) class ResultFilter(_Filter): - def __init__(self, impl: object = None): - super().__init__(impl) + def __init__(self, impl: object = None, incoming=True): + super().__init__("result filter", impl, incoming) if impl: - funcs = get_object_result_filter_funcs(impl) - if not funcs: - raise ValueError(f"filter object {impl.__class__.__name__} has no result_filter func") - - if len(funcs) > 1: - raise ValueError( - f"filter object {impl.__class__.__name__} must have one result_filter func but got {len(funcs)}" - ) - self.impl_func = funcs[0] - else: - self.impl_func = None + self.impl_func = _determine_filter_impl_func( + obj=impl, + filter_type="result_filter", + incoming=incoming, + get_filter_f=get_object_result_filter_funcs, + get_in_filter_f=get_object_in_result_filter_funcs, + get_out_filter_f=get_object_out_result_filter_funcs, + ) def filter_result(self, result: Any, context: Context): """Filter result produced by a collab func. @@ -99,15 +146,7 @@ def filter_result(self, result: Any, context: Context): Returns: filtered result """ - if self.impl_func is not None: - name, f = self.impl_func - if supports_context(f): - kwargs = {CollabMethodArgName.CONTEXT: context} - else: - kwargs = {} - return f(result, **kwargs) - else: - return result + return self.filter_data(result, context) class FilterChain: diff --git a/nvflare/fox/api/gcc.py b/nvflare/fox/api/gcc.py index 8e177b7de0..e37e03a77e 100644 --- a/nvflare/fox/api/gcc.py +++ b/nvflare/fox/api/gcc.py @@ -62,16 +62,10 @@ def set_result(self, result): ctx.caller = ctx.callee ctx.callee = original_caller - self.logger.info(f"set result {result}") - if not isinstance(result, Exception): - self.logger.info(f"{self.func_name}: result before filtering: {result}") result = self.app.apply_incoming_result_filters(self.target_name, self.func_name, result, ctx) - self.logger.info(f"{self.func_name}: result after filtering: {result}") if self.process_cb: - self.logger.info(f"calling process_cb for {self.func_name} with result: {result}") - # set the context for the process_cb only set_call_context(ctx) self.cb_kwargs[CollabMethodArgName.CONTEXT] = ctx diff --git a/nvflare/fox/examples/pt/filters.py b/nvflare/fox/examples/pt/filters.py index 348b2fc3a6..ac3f51f85d 100644 --- a/nvflare/fox/examples/pt/filters.py +++ b/nvflare/fox/examples/pt/filters.py @@ -1,3 +1,16 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from typing import Any from nvflare.fox.api.ctx import Context diff --git a/nvflare/fox/examples/pt/filters2.py b/nvflare/fox/examples/pt/filters2.py new file mode 100644 index 0000000000..3d7a7d4d46 --- /dev/null +++ b/nvflare/fox/examples/pt/filters2.py @@ -0,0 +1,79 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Any + +from nvflare.fox import fox +from nvflare.fox.sys.downloader import Downloader, download_tensors +from nvflare.fuel.utils.log_utils import get_obj_logger + + +class ModelFilter: + + def __init__(self, model_arg_name: str): + super().__init__() + self.model_arg_name = model_arg_name + self.logger = get_obj_logger(self) + + @fox.out_call_filter + def prepare_weights_for_download(self, func_kwargs: dict): + arg_value = func_kwargs.get(self.model_arg_name) + if not arg_value: + return func_kwargs + + num_receivers = fox.context.target_group_size + self.logger.info(f"target group size={num_receivers}") + + downloader = Downloader( + num_receivers=num_receivers, + ctx=fox.context, + timeout=5.0, + ) + model = downloader.add_tensors(arg_value, 0) + func_kwargs[self.model_arg_name] = model + return func_kwargs + + @fox.in_call_filter + def download_weights(self, func_kwargs: dict): + arg_value = func_kwargs.get(self.model_arg_name) + if not arg_value: + return func_kwargs + + err, model = download_tensors(ref=arg_value, ctx=fox.context, per_request_timeout=5.0) + if err: + self.logger.error(f"error filtering call arg {arg_value}: {err}") + else: + func_kwargs[self.model_arg_name] = model + return func_kwargs + return func_kwargs + + @fox.out_result_filter + def prepare_result_for_download(self, result: Any): + if not isinstance(result, dict): + return result + + downloader = Downloader( + num_receivers=1, + ctx=fox.context, + timeout=5.0, + ) + return downloader.add_tensors(result, 0) + + @fox.in_result_filter + def download_result(self, result: Any): + err, model = download_tensors(ref=result, ctx=fox.context, per_request_timeout=5.0) + if err: + self.logger.error(f"error filtering result {result}: {err}") + return result + else: + return model diff --git a/nvflare/fox/examples/pt/recipe_pt_avg_filter2.py b/nvflare/fox/examples/pt/recipe_pt_avg_filter2.py new file mode 100644 index 0000000000..0fb2e8a3f9 --- /dev/null +++ b/nvflare/fox/examples/pt/recipe_pt_avg_filter2.py @@ -0,0 +1,56 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples.pt.filters2 import ModelFilter +from nvflare.fox.examples.pt.pt_avg_filter import PTFedAvg, PTTrainer +from nvflare.fox.sys.recipe import FoxRecipe + +JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" + + +def main(): + simple_logging(logging.DEBUG) + + recipe = FoxRecipe( + job_name="fox_pt_fedavg_filter2", + server=PTFedAvg( + initial_model={ + "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "y": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "z": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + }, + num_rounds=2, + ), + client=PTTrainer(delta=1.0), + ) + model_filter = ModelFilter("weights") + recipe.add_server_outgoing_call_filters( + pattern="*.train", + filters=[model_filter], + ) + recipe.add_server_incoming_result_filters(pattern="*.train", filters=[model_filter]) + + recipe.add_client_incoming_call_filters( + pattern="*.train", + filters=[model_filter], + ) + recipe.add_client_outgoing_result_filters(pattern="*.train", filters=[model_filter]) + + recipe.export(JOB_ROOT_DIR) + + +if __name__ == "__main__": + main() diff --git a/nvflare/fox/sys/utils.py b/nvflare/fox/sys/utils.py index b992691c48..926a7b201d 100644 --- a/nvflare/fox/sys/utils.py +++ b/nvflare/fox/sys/utils.py @@ -114,13 +114,13 @@ def _call_app_method(request: Message, app: App, logger) -> Message: # invoke this method try: ctx, method_kwargs = _preprocess(app, caller, obj_name, target_name, method_name, m, method_args, method_kwargs) - logger.info(f"calling method {method_name}: {caller=}: {method_args=} {method_kwargs=}") + # logger.info(f"calling method {method_name}: {caller=}: {method_args=} {method_kwargs=}") result = m(*method_args, **method_kwargs) - logger.info(f"result from method {method_name}: {result}") + # logger.info(f"result from method {method_name}: {result}") # apply result filters result = app.apply_outgoing_result_filters(target_name, method_name, result, ctx) - logger.info(f"result after filtering: {result}") + # logger.info(f"result after filtering: {result}") return new_cell_message( headers={MessageHeaderKey.RETURN_CODE: ReturnCode.OK}, payload={CallReplyKey.RESULT: result} From 8dac4da2cc707ca99141873f20ab26ea32e99f0c Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Thu, 20 Nov 2025 15:46:16 -0500 Subject: [PATCH 065/102] refactor group call --- nvflare/fox/api/backend.py | 3 +- nvflare/fox/api/gcc.py | 46 ++++++++++++++++++++++++------ nvflare/fox/api/group.py | 58 ++++++++++++++++++-------------------- nvflare/fox/sim/backend.py | 9 ++++-- nvflare/fox/sys/backend.py | 8 +++--- 5 files changed, 77 insertions(+), 47 deletions(-) diff --git a/nvflare/fox/api/backend.py b/nvflare/fox/api/backend.py index 6fa2ec3a6a..1625030b9d 100644 --- a/nvflare/fox/api/backend.py +++ b/nvflare/fox/api/backend.py @@ -48,12 +48,11 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): pass @abstractmethod - def call_target_in_group(self, gcc: GroupCallContext, target_name: str, func_name: str, *args, **kwargs): + def call_target_in_group(self, gcc: GroupCallContext, func_name: str, *args, **kwargs): """Call a remote object as part of a group. Args: gcc: contextual information about group call. - target_name: fully qualified name of the target object to be called in the remote app. func_name: mame of the function to be called in the remote app. *args: args to pass to the target function. **kwargs: kwargs to pass to the target function. diff --git a/nvflare/fox/api/gcc.py b/nvflare/fox/api/gcc.py index e37e03a77e..5e19a61c90 100644 --- a/nvflare/fox/api/gcc.py +++ b/nvflare/fox/api/gcc.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import copy +import threading import time from nvflare.fuel.utils.log_utils import get_obj_logger @@ -21,10 +22,41 @@ from .utils import check_context_support +class ResultWaiter(threading.Event): + + def __init__(self, sites: list[str]): + super().__init__() + self.num_results_expected = len(sites) + self.sites = sites + self.results = {} + self.lock = threading.Lock() + + def set_result(self, target_name: str, result): + # target_name is either or . + parts = target_name.split(".") + site_name = parts[0] + with self.lock: + print(f"set result for {target_name} on site {site_name}") + self.results[site_name] = result + if self.num_results_received() >= self.num_results_expected: + # print(f"received results from all {self.num_results_expected} sites!") + self.set() + + def finalize_results(self): + with self.lock: + for s in self.sites: + if s not in self.results: + self.results[s] = TimeoutError + return self.results + + def num_results_received(self): + return len(self.results) + + class GroupCallContext: - def __init__(self, app, target_name, func_name, process_cb, cb_kwargs, context: Context): - """GroupCallContext contains contextual information about a group call to a target.. + def __init__(self, app, target_name, func_name, process_cb, cb_kwargs, context: Context, waiter: ResultWaiter): + """GroupCallContext contains contextual information about a group call to a target. Args: app: the calling app. @@ -33,15 +65,15 @@ def __init__(self, app, target_name, func_name, process_cb, cb_kwargs, context: process_cb: the callback function to be called to process response from the remote app. cb_kwargs: kwargs passed to the callback function. context: call context. + waiter: the waiter to wait for result """ self.app = app self.target_name = target_name self.func_name = func_name - self.result = None - self.resp_time = None self.process_cb = process_cb self.cb_kwargs = cb_kwargs self.context = context + self.waiter = waiter self.logger = get_obj_logger(self) def set_result(self, result): @@ -77,8 +109,7 @@ def set_result(self, result): else: self.logger.info(f"{self.func_name} does not have process_cb!") - self.result = result - self.resp_time = time.time() + self.waiter.set_result(self.target_name, result) def set_exception(self, ex): """This is called by the backend to set the exception received from the remote app. @@ -90,5 +121,4 @@ def set_exception(self, ex): Returns: """ - self.result = ex - self.resp_time = time.time() + self.waiter.set_result(self.target_name, ex) diff --git a/nvflare/fox/api/group.py b/nvflare/fox/api/group.py index 14b8e3681e..56e5254f24 100644 --- a/nvflare/fox/api/group.py +++ b/nvflare/fox/api/group.py @@ -22,7 +22,7 @@ from .app import App from .constants import CollabMethodArgName, CollabMethodOptionName from .ctx import Context -from .gcc import GroupCallContext +from .gcc import GroupCallContext, ResultWaiter from .proxy import Proxy from .utils import check_call_args @@ -111,9 +111,12 @@ def method(*args, **kwargs): # filter once for all targets p = self._proxies[0] - the_proxy, func_itf, adj_args, adj_kwargs = p.adjust_func_args(func_name, args, kwargs) - the_backend = the_proxy.backend - ctx = the_proxy.app.new_context(the_proxy.app.name, the_proxy.name, target_group=self) + + # func_proxy is the proxy that actually has the func. + # the func_proxy is either "p" or a child of "p". + func_proxy, func_itf, adj_args, adj_kwargs = p.adjust_func_args(func_name, args, kwargs) + the_backend = p.backend + ctx = func_proxy.app.new_context(func_proxy.caller_name, func_proxy.name, target_group=self) self._logger.debug( f"[{ctx.header_str()}] calling {func_name} of group {[p.name for p in self._proxies]}" @@ -121,13 +124,14 @@ def method(*args, **kwargs): # apply outgoing call filters assert isinstance(self._app, App) - adj_kwargs = self._app.apply_outgoing_call_filters(p.target_name, func_name, adj_kwargs, ctx) + adj_kwargs = self._app.apply_outgoing_call_filters(func_proxy.target_name, func_name, adj_kwargs, ctx) check_call_args(func_name, func_itf, adj_args, adj_kwargs) + waiter = ResultWaiter([p.name for p in self._proxies]) for p in self._proxies: - the_proxy, func_itf, call_args, call_kwargs = p.adjust_func_args(func_name, adj_args, adj_kwargs) + func_proxy, func_itf, call_args, call_kwargs = p.adjust_func_args(func_name, adj_args, adj_kwargs) call_kwargs = copy.copy(call_kwargs) - ctx = self._app.new_context(the_proxy.caller_name, the_proxy.name, target_group=self) + ctx = self._app.new_context(func_proxy.caller_name, func_proxy.name, target_group=self) call_kwargs[CollabMethodArgName.CONTEXT] = ctx # set the optional args to help backend decide how to call @@ -140,40 +144,43 @@ def method(*args, **kwargs): gcc = GroupCallContext( self._app, - p.target_name, + func_proxy.target_name, func_name, self._process_resp_cb, self._cb_kwargs, ctx, + waiter, ) gccs[p.name] = gcc self._logger.debug( f"[{ctx.header_str()}] group call: {func_name=} args={call_args} kwargs={call_kwargs}" ) - the_proxy.backend.call_target_in_group(gcc, the_proxy.name, func_name, *call_args, **call_kwargs) + func_proxy.backend.call_target_in_group(gcc, func_name, *call_args, **call_kwargs) - # wait for responses if not self._blocking: + # do not wait for responses return None + # wait for responses start_time = time.time() min_received_time = None while True: if self._abort_signal.triggered: raise RunAborted("run is aborted") - # how many resps have been received? - resps_received = 0 - for name, gcc in gccs.items(): - if gcc.resp_time: - resps_received += 1 + # wait for a short time, so we can check other conditions + done = waiter.wait(0.1) + if done: + self._logger.info(f"all received from {waiter.results} for func {func_name}") + break - if resps_received == len(self._proxies): + results_received = waiter.num_results_received() + if results_received >= self.size: break now = time.time() - if resps_received >= self._min_resps: + if results_received >= self._min_resps: if not min_received_time: min_received_time = now if now - min_received_time > self._wait_after_min_resps: @@ -181,20 +188,11 @@ def method(*args, **kwargs): break elif self._timeout and now - start_time > self._timeout: # timed out + self._logger.error(f"not all received from {waiter.results.keys()} for func {func_name}") break - else: - # still have not received min resps - time.sleep(0.1) - - # process results - results = {} - for name, gcc in gccs.items(): - if gcc.resp_time: - result = gcc.result - else: - result = TimeoutError() - results[name] = result - return results + + # process received results: fill in TimeoutError for missing result + return waiter.finalize_results() except Exception as ex: if the_backend: the_backend.handle_exception(ex) diff --git a/nvflare/fox/sim/backend.py b/nvflare/fox/sim/backend.py index e65f50b424..88e57ea0e7 100644 --- a/nvflare/fox/sim/backend.py +++ b/nvflare/fox/sim/backend.py @@ -122,11 +122,13 @@ def _run_func(self, waiter: _Waiter, target_name, func_name, func, args, kwargs) if waiter: waiter.set() - def call_target_in_group(self, gcc: GroupCallContext, target_name: str, func_name: str, *args, **kwargs): + def call_target_in_group(self, gcc: GroupCallContext, func_name: str, *args, **kwargs): # do not use the optional args - they are managed by the group + self.logger.info(f"call_target_in_group: {gcc.target_name=}") for k in OPTION_ARGS: kwargs.pop(k, None) + target_name = gcc.target_name func = self._get_func(func_name) if not func: raise AttributeError(f"{target_name} does not have method '{func_name}' or it is not collab") @@ -134,10 +136,11 @@ def call_target_in_group(self, gcc: GroupCallContext, target_name: str, func_nam if not callable(func): raise AttributeError(f"the method '{func_name}' of {target_name} is not callable") - self.executor.submit(self._run_func_with_resp, gcc, target_name, func_name, func, args, kwargs) + self.executor.submit(self._run_func_with_resp, gcc, func_name, func, args, kwargs) - def _run_func_with_resp(self, gcc: GroupCallContext, target_name, func_name, func, args, kwargs): + def _run_func_with_resp(self, gcc: GroupCallContext, func_name, func, args, kwargs): try: + target_name = gcc.target_name ctx, kwargs = self._preprocess(target_name, func_name, func, kwargs) result = func(*args, **kwargs) diff --git a/nvflare/fox/sys/backend.py b/nvflare/fox/sys/backend.py index 6dbb8de9ec..65249b7b0e 100644 --- a/nvflare/fox/sys/backend.py +++ b/nvflare/fox/sys/backend.py @@ -95,12 +95,12 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): ) return None - def call_target_in_group(self, gcc: GroupCallContext, target_name: str, func_name: str, *args, **kwargs): - self.thread_executor.submit(self._run_func, gcc, target_name, func_name, args, kwargs) + def call_target_in_group(self, gcc: GroupCallContext, func_name: str, *args, **kwargs): + self.thread_executor.submit(self._run_func, gcc, func_name, args, kwargs) - def _run_func(self, gcc: GroupCallContext, target_name: str, func_name: str, args, kwargs): + def _run_func(self, gcc: GroupCallContext, func_name: str, args, kwargs): try: - result = self.call_target(target_name, func_name, *args, **kwargs) + result = self.call_target(gcc.target_name, func_name, *args, **kwargs) gcc.set_result(result) except Exception as ex: gcc.set_exception(ex) From 74f6aecda3de5fc1ccda818b33785c727aa140e3 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Thu, 20 Nov 2025 19:05:53 -0500 Subject: [PATCH 066/102] refactor group call --- nvflare/fox/api/app.py | 6 ++++++ nvflare/fox/api/proxy.py | 12 +++++++++--- nvflare/fox/api/utils.py | 2 +- nvflare/fox/examples/np/algos/client.py | 2 +- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/nvflare/fox/api/app.py b/nvflare/fox/api/app.py index 8399d76251..5e0d64105b 100644 --- a/nvflare/fox/api/app.py +++ b/nvflare/fox/api/app.py @@ -14,6 +14,7 @@ import copy import fnmatch import os +import re from typing import List from nvflare.fuel.utils.log_utils import get_obj_logger @@ -203,6 +204,11 @@ def update_props(self, props: dict): self._props.update(props) def add_collab_object(self, name: str, obj): + # name must be acceptable str + pattern = r"^[A-Za-z][A-Za-z0-9_]+$" + if not re.match(pattern, name): + raise ValueError(f"invalid name {name} for collab object - must be simple name starting with a letter") + if name in self._collab_objs: raise ValueError(f"conflict with existing collab object '{name}' of {type(self._collab_objs[name])}") diff --git a/nvflare/fox/api/proxy.py b/nvflare/fox/api/proxy.py index 02da833310..0466d98b47 100644 --- a/nvflare/fox/api/proxy.py +++ b/nvflare/fox/api/proxy.py @@ -165,7 +165,7 @@ def method(*args, **kwargs): ctx = p.app.new_context(self.caller_name, self.name) self.logger.debug( - f"[{ctx.header_str()}] calling target {p.target_name} func {func_name}: {call_args=} {call_kwargs=}" + f"[{ctx.header_str()}] apply_outgoing_call_filters on {p.target_name} func {func_name}" ) # apply outgoing call filters @@ -178,12 +178,18 @@ def method(*args, **kwargs): for k, v in option_args.items(): call_kwargs[k] = v + self.logger.debug( + f"[{ctx.header_str()}] calling target {p.target_name} func {func_name}: {option_args=}" + ) + result = p.backend.call_target(p.target_name, func_name, *call_args, **call_kwargs) if isinstance(result, Exception): raise result - # filter incoming result filters - result = self.app.apply_incoming_result_filters(p.target_name, func_name, result, ctx) + if result: + # filter incoming result filters + result = self.app.apply_incoming_result_filters(p.target_name, func_name, result, ctx) + return result except Exception as ex: self.backend.handle_exception(ex) diff --git a/nvflare/fox/api/utils.py b/nvflare/fox/api/utils.py index cef7613bf8..6dbb6860be 100644 --- a/nvflare/fox/api/utils.py +++ b/nvflare/fox/api/utils.py @@ -44,7 +44,7 @@ def get_collab_object_name(target_name: str): """ parts = target_name.split(".") if len(parts) == 1: - return "app" + return "_app_" else: return parts[1] diff --git a/nvflare/fox/examples/np/algos/client.py b/nvflare/fox/examples/np/algos/client.py index 098e9a11c4..fac04cd326 100644 --- a/nvflare/fox/examples/np/algos/client.py +++ b/nvflare/fox/examples/np/algos/client.py @@ -45,7 +45,7 @@ def train(self, current_round, weights): # self.server.accept_metric({"round": r, "y": 2}) # self.server.metric_receiver.accept_metric({"round": r, "y": 2}) # - fox.server.fire_event("metrics", {"round": current_round, "y": 10}, _blocking=False) + fox.server(blocking=False).fire_event("metrics", {"round": current_round, "y": 10}) return weights + self.delta @fox.collab From f348bb84457977c6efca4e79e76a901083531853 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Fri, 21 Nov 2025 16:54:45 -0500 Subject: [PATCH 067/102] fix context handling --- nvflare/fox/api/app.py | 40 ++++--- nvflare/fox/api/constants.py | 2 + nvflare/fox/api/filter.py | 2 +- nvflare/fox/api/gcc.py | 100 +++++++++++------- nvflare/fox/api/group.py | 98 ++++++----------- nvflare/fox/api/proxy.py | 7 +- nvflare/fox/api/proxy_list.py | 8 +- nvflare/fox/examples/np/algos/client.py | 2 + nvflare/fox/examples/np/algos/filters.py | 12 ++- .../np/algos/strategies/avg_intime.py | 21 +++- .../examples/np/algos/strategies/avg_para.py | 6 +- nvflare/fox/examples/np/fed_avg_intime.py | 2 +- nvflare/fox/sim/backend.py | 29 ++++- nvflare/fox/sys/backend.py | 5 +- 14 files changed, 194 insertions(+), 140 deletions(-) diff --git a/nvflare/fox/api/app.py b/nvflare/fox/api/app.py index 5e0d64105b..4bf8ee4032 100644 --- a/nvflare/fox/api/app.py +++ b/nvflare/fox/api/app.py @@ -134,8 +134,7 @@ def add_outgoing_result_filters(self, pattern: str, filters: List[object]): def get_outgoing_result_filters(self): return self._outgoing_result_filter_chains - @staticmethod - def _find_filter_chain(chains: List[FilterChain], target_name: str, func_name: str, ctx: Context): + def _find_filter_chain(self, direction, chains: List[FilterChain], target_name: str, func_name: str, ctx: Context): """ Args: @@ -146,12 +145,14 @@ def _find_filter_chain(chains: List[FilterChain], target_name: str, func_name: s Returns: """ - if not chains: - return None - collab_obj_name = get_collab_object_name(target_name) qualified_func_name = f"{collab_obj_name}.{func_name}" ctx.set_prop(ContextKey.QUALIFIED_FUNC_NAME, qualified_func_name) + ctx.set_prop(ContextKey.DIRECTION, direction) + self.logger.info(f"set filter context {id(ctx)}: {direction} {qualified_func_name}") + + if not chains: + return None for c in chains: if fnmatch.fnmatch(qualified_func_name, c.pattern): @@ -159,33 +160,41 @@ def _find_filter_chain(chains: List[FilterChain], target_name: str, func_name: s return None def apply_incoming_call_filters(self, target_name: str, func_name: str, func_kwargs, context: Context): - filter_chain = self._find_filter_chain(self._incoming_call_filter_chains, target_name, func_name, context) + self.logger.info(f"apply_incoming_call_filters on ctx {id(context)}") + filter_chain = self._find_filter_chain( + FilterDirection.INCOMING, self._incoming_call_filter_chains, target_name, func_name, context + ) if filter_chain: - context.set_prop(ContextKey.DIRECTION, FilterDirection.INCOMING) return filter_chain.apply_filters(func_kwargs, context) else: return func_kwargs def apply_outgoing_call_filters(self, target_name: str, func_name: str, func_kwargs, context: Context): - filter_chain = self._find_filter_chain(self._outgoing_call_filter_chains, target_name, func_name, context) + self.logger.info(f"apply_outgoing_call_filters on ctx {id(context)}") + filter_chain = self._find_filter_chain( + FilterDirection.OUTGOING, self._outgoing_call_filter_chains, target_name, func_name, context + ) if filter_chain: - context.set_prop(ContextKey.DIRECTION, FilterDirection.OUTGOING) return filter_chain.apply_filters(func_kwargs, context) else: return func_kwargs def apply_incoming_result_filters(self, target_name: str, func_name: str, result, context: Context): - filter_chain = self._find_filter_chain(self._incoming_result_filter_chains, target_name, func_name, context) + self.logger.info(f"apply_incoming_result_filters on ctx {id(context)}") + filter_chain = self._find_filter_chain( + FilterDirection.INCOMING, self._incoming_result_filter_chains, target_name, func_name, context + ) if filter_chain: - context.set_prop(ContextKey.DIRECTION, FilterDirection.INCOMING) return filter_chain.apply_filters(result, context) else: return result def apply_outgoing_result_filters(self, target_name: str, func_name: str, result, context: Context): - filter_chain = self._find_filter_chain(self._outgoing_result_filter_chains, target_name, func_name, context) + self.logger.info(f"apply_outgoing_result_filters on ctx {id(context)}") + filter_chain = self._find_filter_chain( + FilterDirection.OUTGOING, self._outgoing_result_filter_chains, target_name, func_name, context + ) if filter_chain: - context.set_prop(ContextKey.DIRECTION, FilterDirection.OUTGOING) return filter_chain.apply_filters(result, context) else: return result @@ -308,9 +317,10 @@ def finalize(self, context: Context): for obj in self._managed_objects.values(): self._fox_finalize(obj, context) - def new_context(self, caller: str, callee: str, target_group=None): + def new_context(self, caller: str, callee: str, target_group=None, set_call_ctx=True): ctx = Context(self, caller, callee, self._abort_signal, target_group=target_group) - set_call_context(ctx) + if set_call_ctx: + set_call_context(ctx) return ctx def register_event_handler(self, event_type: str, handler, **handler_kwargs): diff --git a/nvflare/fox/api/constants.py b/nvflare/fox/api/constants.py index a7db0e808a..90aee53526 100644 --- a/nvflare/fox/api/constants.py +++ b/nvflare/fox/api/constants.py @@ -21,6 +21,7 @@ class CollabMethodOptionName: TIMEOUT = "_timeout" OPTIONAL = "_optional" SECURE = "_secure" + EXPECT_RESULT = "_expect_result" OPTION_ARGS = [ @@ -28,6 +29,7 @@ class CollabMethodOptionName: CollabMethodOptionName.TIMEOUT, CollabMethodOptionName.OPTIONAL, CollabMethodOptionName.SECURE, + CollabMethodOptionName.EXPECT_RESULT, ] diff --git a/nvflare/fox/api/filter.py b/nvflare/fox/api/filter.py index 7fc4755f26..6584819f24 100644 --- a/nvflare/fox/api/filter.py +++ b/nvflare/fox/api/filter.py @@ -51,7 +51,7 @@ def filter_data(self, data, context: Context): d = "outgoing" name, f = self.impl_func - self.logger.info(f"calling {d} {self.filter_type}: {name} ...") + self.logger.info(f"calling {d} {self.filter_type}: {name} on ctx {id(context)}") if supports_context(f): kwargs = {CollabMethodArgName.CONTEXT: context} else: diff --git a/nvflare/fox/api/gcc.py b/nvflare/fox/api/gcc.py index 5e19a61c90..4ad6ca59a2 100644 --- a/nvflare/fox/api/gcc.py +++ b/nvflare/fox/api/gcc.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import copy +import queue import threading -import time from nvflare.fuel.utils.log_utils import get_obj_logger @@ -22,13 +22,43 @@ from .utils import check_context_support +class ResultQueue: + def __init__(self, limit: int): + if limit <= 0: + raise ValueError(f"bad queue limit {limit}: must be > 0") + self.limit = limit + self.q = queue.Queue() + self.consumed = 0 + self.num_items_received = 0 + + def append(self, i): + if self.num_items_received == self.limit: + raise RuntimeError(f"queue is full: {self.limit} items are already appended") + self.num_items_received += 1 + self.q.put_nowait(i) + return self.num_items_received == self.limit + + def __iter__(self): + return self + + def __next__(self): + if self.consumed == self.limit: + raise StopIteration() + else: + i = self.q.get(block=True) + self.consumed += 1 + return i + + def __len__(self): + return self.num_items_received + + class ResultWaiter(threading.Event): def __init__(self, sites: list[str]): super().__init__() - self.num_results_expected = len(sites) self.sites = sites - self.results = {} + self.results = ResultQueue(len(sites)) self.lock = threading.Lock() def set_result(self, target_name: str, result): @@ -37,21 +67,10 @@ def set_result(self, target_name: str, result): site_name = parts[0] with self.lock: print(f"set result for {target_name} on site {site_name}") - self.results[site_name] = result - if self.num_results_received() >= self.num_results_expected: - # print(f"received results from all {self.num_results_expected} sites!") + all_received = self.results.append((site_name, result)) + if all_received: self.set() - def finalize_results(self): - with self.lock: - for s in self.sites: - if s not in self.results: - self.results[s] = TimeoutError - return self.results - - def num_results_received(self): - return len(self.results) - class GroupCallContext: @@ -86,30 +105,31 @@ def set_result(self, result): Returns: None """ - # filter incoming result - ctx = copy.copy(self.context) - - # swap caller/callee - original_caller = ctx.caller - ctx.caller = ctx.callee - ctx.callee = original_caller - - if not isinstance(result, Exception): - result = self.app.apply_incoming_result_filters(self.target_name, self.func_name, result, ctx) - - if self.process_cb: - # set the context for the process_cb only - set_call_context(ctx) - self.cb_kwargs[CollabMethodArgName.CONTEXT] = ctx - check_context_support(self.process_cb, self.cb_kwargs) - result = self.process_cb(result, **self.cb_kwargs) - - # set back to original context - set_call_context(self.context) - else: - self.logger.info(f"{self.func_name} does not have process_cb!") - - self.waiter.set_result(self.target_name, result) + try: + # filter incoming result + ctx = copy.copy(self.context) + + # swap caller/callee + original_caller = ctx.caller + ctx.caller = ctx.callee + ctx.callee = original_caller + + if not isinstance(result, Exception): + set_call_context(ctx) + result = self.app.apply_incoming_result_filters(self.target_name, self.func_name, result, ctx) + if self.process_cb: + self.cb_kwargs[CollabMethodArgName.CONTEXT] = ctx + check_context_support(self.process_cb, self.cb_kwargs) + result = self.process_cb(result, **self.cb_kwargs) + else: + self.logger.info(f"{self.func_name} does not have process_cb!") + + # set back to original context + set_call_context(self.context) + except Exception as ex: + result = ex + finally: + self.waiter.set_result(self.target_name, result) def set_exception(self, ex): """This is called by the backend to set the exception received from the remote app. diff --git a/nvflare/fox/api/group.py b/nvflare/fox/api/group.py index 56e5254f24..1fc6a8ecc4 100644 --- a/nvflare/fox/api/group.py +++ b/nvflare/fox/api/group.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. import copy -import time from typing import List from nvflare.apis.fl_exception import RunAborted @@ -21,7 +20,7 @@ from .app import App from .constants import CollabMethodArgName, CollabMethodOptionName -from .ctx import Context +from .ctx import Context, get_call_context, set_call_context from .gcc import GroupCallContext, ResultWaiter from .proxy import Proxy from .utils import check_call_args @@ -35,11 +34,10 @@ def __init__( abort_signal: Signal, proxies: List[Proxy], blocking: bool = True, + expect_result: bool = True, timeout: float = 5.0, optional: bool = False, secure: bool = False, - min_resps: int = None, - wait_after_min_resps: float = None, process_resp_cb=None, **cb_kwargs, ): @@ -50,6 +48,7 @@ def __init__( abort_signal: signal to abort execution. proxies: proxies of the remote apps to be called. blocking: whether to block until responses are received from all remote apps. + expect_result: whether to expect result from remote sites. timeout: how long to wait for responses. optional: whether the call is optional or not. secure: whether the call is secure or not. @@ -65,21 +64,14 @@ def __init__( self._abort_signal = abort_signal self._proxies = proxies self._blocking = blocking + self._expect_result = expect_result self._timeout = timeout self._optional = optional self._secure = secure - self._min_resps = min_resps - self._wait_after_min_resps = wait_after_min_resps self._process_resp_cb = process_resp_cb self._cb_kwargs = cb_kwargs self._logger = get_obj_logger(self) - if not min_resps: - self._min_resps = len(proxies) - - if not wait_after_min_resps: - self._wait_after_min_resps = 0 - @property def size(self): """Size of the group, which is the number of remote apps to be called. @@ -106,6 +98,7 @@ def __getattr__(self, func_name): def method(*args, **kwargs): the_backend = None + orig_ctx = get_call_context() try: gccs = {} @@ -131,7 +124,10 @@ def method(*args, **kwargs): for p in self._proxies: func_proxy, func_itf, call_args, call_kwargs = p.adjust_func_args(func_name, adj_args, adj_kwargs) call_kwargs = copy.copy(call_kwargs) - ctx = self._app.new_context(func_proxy.caller_name, func_proxy.name, target_group=self) + ctx = self._app.new_context( + func_proxy.caller_name, func_proxy.name, target_group=self, set_call_ctx=False + ) + call_kwargs[CollabMethodArgName.CONTEXT] = ctx # set the optional args to help backend decide how to call @@ -139,6 +135,7 @@ def method(*args, **kwargs): call_kwargs[CollabMethodOptionName.TIMEOUT] = self._timeout call_kwargs[CollabMethodOptionName.BLOCKING] = self._blocking + call_kwargs[CollabMethodOptionName.EXPECT_RESULT] = self._expect_result call_kwargs[CollabMethodOptionName.SECURE] = self._secure call_kwargs[CollabMethodOptionName.OPTIONAL] = self._optional @@ -158,13 +155,15 @@ def method(*args, **kwargs): ) func_proxy.backend.call_target_in_group(gcc, func_name, *call_args, **call_kwargs) - if not self._blocking: + if not self._expect_result: # do not wait for responses return None + if not self._blocking: + self._logger.debug(f"not blocking {func_name}") + return waiter.results + # wait for responses - start_time = time.time() - min_received_time = None while True: if self._abort_signal.triggered: raise RunAborted("run is aborted") @@ -175,27 +174,13 @@ def method(*args, **kwargs): self._logger.info(f"all received from {waiter.results} for func {func_name}") break - results_received = waiter.num_results_received() - if results_received >= self.size: - break - - now = time.time() - if results_received >= self._min_resps: - if not min_received_time: - min_received_time = now - if now - min_received_time > self._wait_after_min_resps: - # waited long enough - break - elif self._timeout and now - start_time > self._timeout: - # timed out - self._logger.error(f"not all received from {waiter.results.keys()} for func {func_name}") - break - - # process received results: fill in TimeoutError for missing result - return waiter.finalize_results() + return waiter.results except Exception as ex: if the_backend: the_backend.handle_exception(ex) + finally: + if orig_ctx: + set_call_context(orig_ctx) return method @@ -204,11 +189,10 @@ def group( ctx: Context, proxies: List[Proxy], blocking: bool = True, + expect_result: bool = True, timeout: float = 5.0, optional: bool = False, secure: bool = False, - min_resps: int = None, - wait_after_min_resps: float = None, process_resp_cb=None, **cb_kwargs, ): @@ -218,11 +202,10 @@ def group( ctx: proxies: blocking: + expect_result: timeout: optional: secure: - min_resps: - wait_after_min_resps: process_resp_cb: **cb_kwargs: @@ -234,11 +217,10 @@ def group( ctx.abort_signal, proxies, blocking, + expect_result, timeout, optional, secure, - min_resps, - wait_after_min_resps, process_resp_cb, **cb_kwargs, ) @@ -247,11 +229,10 @@ def group( def all_clients( ctx: Context, blocking: bool = True, + expect_result: bool = True, timeout: float = 5.0, optional: bool = False, secure: bool = False, - min_resps: int = None, - wait_after_min_resps: float = None, process_resp_cb=None, **cb_kwargs, ): @@ -260,11 +241,10 @@ def all_clients( Args: ctx: blocking: + expect_result: timeout: optional: secure: - min_resps: - wait_after_min_resps: process_resp_cb: **cb_kwargs: @@ -276,11 +256,10 @@ def all_clients( ctx.abort_signal, ctx.clients, blocking, + expect_result, timeout, optional, secure, - min_resps, - wait_after_min_resps, process_resp_cb, **cb_kwargs, ) @@ -289,11 +268,10 @@ def all_clients( def all_other_clients( ctx: Context, blocking: bool = True, + expect_result: bool = True, timeout: float = 5.0, optional: bool = False, secure: bool = False, - min_resps: int = None, - wait_after_min_resps: float = None, process_resp_cb=None, **cb_kwargs, ): @@ -302,11 +280,10 @@ def all_other_clients( Args: ctx: blocking: + expect_result: timeout: optional: secure: - min_resps: - wait_after_min_resps: process_resp_cb: **cb_kwargs: @@ -323,11 +300,10 @@ def all_other_clients( ctx.abort_signal, candidates, blocking, + expect_result, timeout, optional, secure, - min_resps, - wait_after_min_resps, process_resp_cb, **cb_kwargs, ) @@ -336,11 +312,10 @@ def all_other_clients( def all_children( ctx: Context, blocking: bool = True, + expect_result: bool = True, timeout: float = 5.0, optional: bool = False, secure: bool = False, - min_resps: int = None, - wait_after_min_resps: float = None, process_resp_cb=None, **cb_kwargs, ): @@ -349,11 +324,10 @@ def all_children( Args: ctx: blocking: + expect_result: timeout: optional: secure: - min_resps: - wait_after_min_resps: process_resp_cb: **cb_kwargs: @@ -369,11 +343,10 @@ def all_children( ctx.abort_signal, clients, blocking, + expect_result, timeout, optional, secure, - min_resps, - wait_after_min_resps, process_resp_cb, **cb_kwargs, ) @@ -382,11 +355,10 @@ def all_children( def all_leaf_clients( ctx: Context, blocking: bool = True, + expect_result: bool = True, timeout: float = 5.0, optional: bool = False, secure: bool = False, - min_resps: int = None, - wait_after_min_resps: float = None, process_resp_cb=None, **cb_kwargs, ): @@ -395,11 +367,10 @@ def all_leaf_clients( Args: ctx: blocking: + expect_result: timeout: optional: secure: - min_resps: - wait_after_min_resps: process_resp_cb: **cb_kwargs: @@ -415,11 +386,10 @@ def all_leaf_clients( ctx.abort_signal, clients, blocking, + expect_result, timeout, optional, secure, - min_resps, - wait_after_min_resps, process_resp_cb, **cb_kwargs, ) diff --git a/nvflare/fox/api/proxy.py b/nvflare/fox/api/proxy.py index 0466d98b47..6483a7974e 100644 --- a/nvflare/fox/api/proxy.py +++ b/nvflare/fox/api/proxy.py @@ -17,6 +17,7 @@ from .backend import Backend from .constants import OPTION_ARGS, CollabMethodArgName, CollabMethodOptionName +from .ctx import get_call_context, set_call_context from .utils import check_call_args @@ -154,6 +155,7 @@ def __getattr__(self, func_name): """ def method(*args, **kwargs): + orig_ctx = get_call_context() try: # remove option args option_args = {} @@ -186,12 +188,15 @@ def method(*args, **kwargs): if isinstance(result, Exception): raise result - if result: + if result is not None: # filter incoming result filters result = self.app.apply_incoming_result_filters(p.target_name, func_name, result, ctx) return result except Exception as ex: self.backend.handle_exception(ex) + finally: + if orig_ctx: + set_call_context(orig_ctx) return method diff --git a/nvflare/fox/api/proxy_list.py b/nvflare/fox/api/proxy_list.py index cc47acf80d..f6011dcb38 100644 --- a/nvflare/fox/api/proxy_list.py +++ b/nvflare/fox/api/proxy_list.py @@ -34,24 +34,22 @@ def method(*args, **kwargs): def __call__( self, blocking: bool = True, + expect_result: bool = True, timeout: float = 5.0, optional: bool = False, secure: bool = False, - min_resps: int = None, - wait_after_min_resps: float = None, process_resp_cb=None, **cb_kwargs, ): - print(f"creating group: {blocking=} {timeout=} {optional=} {secure=} {min_resps=} {wait_after_min_resps=}") + print(f"creating group: {blocking=} {timeout=} {optional=} {secure=}") return group( ctx=get_call_context(), proxies=self, blocking=blocking, + expect_result=expect_result, timeout=timeout, optional=optional, secure=secure, - min_resps=min_resps, - wait_after_min_resps=wait_after_min_resps, process_resp_cb=process_resp_cb, **cb_kwargs, ) diff --git a/nvflare/fox/examples/np/algos/client.py b/nvflare/fox/examples/np/algos/client.py index fac04cd326..ab4feec7de 100644 --- a/nvflare/fox/examples/np/algos/client.py +++ b/nvflare/fox/examples/np/algos/client.py @@ -45,7 +45,9 @@ def train(self, current_round, weights): # self.server.accept_metric({"round": r, "y": 2}) # self.server.metric_receiver.accept_metric({"round": r, "y": 2}) # + self.logger.info(f"before fire_event: fox ctx={id(fox.context)}") fox.server(blocking=False).fire_event("metrics", {"round": current_round, "y": 10}) + self.logger.info(f"after fire_event: fox ctx={id(fox.context)}") return weights + self.delta @fox.collab diff --git a/nvflare/fox/examples/np/algos/filters.py b/nvflare/fox/examples/np/algos/filters.py index 1508e658b3..3c68a05383 100644 --- a/nvflare/fox/examples/np/algos/filters.py +++ b/nvflare/fox/examples/np/algos/filters.py @@ -52,14 +52,20 @@ def __init__(self): @fox.call_filter def print_call(self, func_kwargs: dict): + self.logger.info(f"[{fox.call_info}] print_call on fox ctx {id(fox.context)}") direction = fox.filter_direction qual_func_name = fox.qual_func_name - self.logger.info(f"[{fox.call_info}] printing call: {func_kwargs=} {direction=} {qual_func_name=}") + self.logger.info( + f"[{fox.call_info}] printing call ctx {id(fox.context)}: {func_kwargs=} {direction=} {qual_func_name=}" + ) return func_kwargs @fox.result_filter - def print_result(self, result): + def print_result(self, result, context): + self.logger.info(f"[{fox.call_info}] print_result on {id(context)} fox ctx {id(fox.context)}") direction = fox.filter_direction qual_func_name = fox.qual_func_name - self.logger.info(f"[{fox.call_info}] printing result: {result=} {direction=} {qual_func_name=}") + self.logger.info( + f"[{fox.call_info}] printing result ctx {id(fox.context)}: {result=} {direction=} {qual_func_name=}" + ) return result diff --git a/nvflare/fox/examples/np/algos/strategies/avg_intime.py b/nvflare/fox/examples/np/algos/strategies/avg_intime.py index dbc05b8b3f..d9c6017c73 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_intime.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_intime.py @@ -39,7 +39,8 @@ def execute(self): self.logger.info(f"[{fox.call_info}] Start training for {self.num_rounds} rounds") current_model = fox.get_prop(ContextKey.INPUT, self._init_model) for i in range(self.num_rounds): - current_model = self._do_one_round(i, current_model) + # current_model = self._do_one_round(i, current_model) + current_model = self._do_one_round_non_blocking(i, current_model) score = self._do_eval(current_model) self.logger.info(f"[{fox.call_info}]: eval score in round {i}: {score}") self.logger.info(f"FINAL MODEL: {current_model}") @@ -48,7 +49,7 @@ def execute(self): def _do_eval(self, model): results = fox.clients.evaluate(model) total = 0.0 - for n, v in results.items(): + for n, v in results: self.logger.info(f"[{fox.call_info}]: got eval result from client {n}: {v}") total += v return total / len(results) @@ -70,6 +71,22 @@ def _do_one_round(self, r, current_model): self.logger.info(f"[{fox.call_info}] round {r}: aggr result from {aggr_result.count} clients: {result}") return result + def _do_one_round_non_blocking(self, r, current_model): + timeout = fox.get_app_prop("default_timeout", 10) + self.logger.info(f"got timeout: {timeout}") + results = fox.clients( + timeout=timeout, + blocking=False, + ).train(r, current_model) + + total = 0 + for n, v in results: + self.logger.info(f"[{fox.call_info}] round {r}: got group result from client {n}: {v}") + total += v + result = total / len(results) + self.logger.info(f"[{fox.call_info}] round {r}: aggr result from {len(results)} clients: {result}") + return result + def _accept_train_result(self, result, aggr_result: _AggrResult): self.logger.info(f"[{fox.call_info}] got train result from {fox.caller} {result}") aggr_result.total += result diff --git a/nvflare/fox/examples/np/algos/strategies/avg_para.py b/nvflare/fox/examples/np/algos/strategies/avg_para.py index 1346789e61..c273991df3 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_para.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_para.py @@ -38,15 +38,15 @@ def execute(self): def _do_eval(self, model): results = fox.clients.evaluate(model) total = 0.0 - for n, v in results.items(): + for n, v in results: self.logger.info(f"[{fox.call_info}]: got eval result from client {n}: {v}") total += v return total / len(results) def _do_one_round(self, r, current_model): total = 0 - results = fox.clients(timeout=4, min_resps=2, wait_after_min_resps=1).train(r, current_model) - for n, v in results.items(): + results = fox.clients(timeout=4, blocking=False).train(r, current_model) + for n, v in results: self.logger.info(f"[{fox.call_info}] round {r}: got group result from client {n}: {v}") total += v return total / len(results) diff --git a/nvflare/fox/examples/np/fed_avg_intime.py b/nvflare/fox/examples/np/fed_avg_intime.py index 4bfb2010d8..b5da1d9289 100644 --- a/nvflare/fox/examples/np/fed_avg_intime.py +++ b/nvflare/fox/examples/np/fed_avg_intime.py @@ -45,7 +45,7 @@ def main(): experiment_name="fedavg_intime", server_app=server_app, client_app=client_app, - num_clients=2, + num_clients=1, ) result = simulator.run() diff --git a/nvflare/fox/sim/backend.py b/nvflare/fox/sim/backend.py index 88e57ea0e7..a82f5d0425 100644 --- a/nvflare/fox/sim/backend.py +++ b/nvflare/fox/sim/backend.py @@ -15,14 +15,13 @@ import time from nvflare.apis.fl_exception import RunAborted +from nvflare.fox import fox from nvflare.fox.api.app import App from nvflare.fox.api.backend import Backend -from nvflare.fox.api.constants import OPTION_ARGS, CollabMethodArgName, CollabMethodOptionName -from nvflare.fox.api.ctx import fox_context +from nvflare.fox.api.constants import OPTION_ARGS, CollabMethodArgName, CollabMethodOptionName, ContextKey from nvflare.fox.api.dec import adjust_kwargs from nvflare.fox.api.gcc import GroupCallContext from nvflare.fox.api.utils import check_call_args -from nvflare.fuel.utils.log_utils import get_obj_logger class _Waiter(threading.Event): @@ -87,7 +86,15 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): def _preprocess(self, target_name, func_name, func, kwargs): caller_ctx = kwargs.pop(CollabMethodArgName.CONTEXT) my_ctx = self.target_app.new_context(caller_ctx.caller, caller_ctx.callee) + + self.logger.info( + f"established call_ctx {id(my_ctx)} for {target_name=} {func_name=}: fox_ctx={id(fox.context)}" + ) + kwargs = self.target_app.apply_incoming_call_filters(target_name, func_name, kwargs, my_ctx) + self.logger.info( + f"_preprocess: {id(my_ctx)} apply_incoming_call_filters: {my_ctx.get_prop(ContextKey.DIRECTION)}" + ) # make sure the final kwargs conforms to func interface obj_itf = self.target_app.get_target_object_collab_interface(self.target_obj_name) @@ -109,9 +116,13 @@ def _run_func(self, waiter: _Waiter, target_name, func_name, func, args, kwargs) try: ctx, kwargs = self._preprocess(target_name, func_name, func, kwargs) result = func(*args, **kwargs) + # set_call_context(ctx) # apply result filter result = self.target_app.apply_outgoing_result_filters(target_name, func_name, result, ctx) + self.logger.info( + f"_run_func: {id(ctx)} apply_outgoing_result_filters: {ctx.get_prop(ContextKey.DIRECTION)}" + ) if waiter: waiter.result = result @@ -142,10 +153,22 @@ def _run_func_with_resp(self, gcc: GroupCallContext, func_name, func, args, kwar try: target_name = gcc.target_name ctx, kwargs = self._preprocess(target_name, func_name, func, kwargs) + self.logger.info(f"after _preprocess: ctx={id(ctx)} fox_ctx={id(fox.context)}") result = func(*args, **kwargs) + # set_call_context(ctx) + self.logger.info(f"after calling {func_name}: ctx={id(ctx)} fox_ctx={id(fox.context)}") # apply result filter result = self.target_app.apply_outgoing_result_filters(target_name, func_name, result, ctx) + + self.logger.info( + f"after apply_outgoing_result_filters {func_name}: ctx={id(ctx)} fox_ctx={id(fox.context)}" + ) + + self.logger.info( + f"_run_func_with_resp: {id(ctx)} apply_outgoing_result_filters: {ctx.get_prop(ContextKey.DIRECTION)}" + f" {ctx.get_prop(ContextKey.QUALIFIED_FUNC_NAME)}" + ) gcc.set_result(result) except Exception as ex: gcc.set_exception(ex) diff --git a/nvflare/fox/sys/backend.py b/nvflare/fox/sys/backend.py index 65249b7b0e..ced3729585 100644 --- a/nvflare/fox/sys/backend.py +++ b/nvflare/fox/sys/backend.py @@ -35,7 +35,8 @@ def __init__(self, manager, engine, caller, cell, target_fqcn, abort_signal, thr self.thread_executor = thread_executor def call_target(self, target_name: str, func_name: str, *args, **kwargs): - blocking = kwargs.pop(CollabMethodOptionName.BLOCKING, True) + kwargs.pop(CollabMethodOptionName.BLOCKING, True) + expect_result = kwargs.pop(CollabMethodOptionName.EXPECT_RESULT, True) timeout = kwargs.pop(CollabMethodOptionName.TIMEOUT, 10.0) optional = kwargs.pop(CollabMethodOptionName.OPTIONAL, False) secure = kwargs.pop(CollabMethodOptionName.SECURE, False) @@ -52,7 +53,7 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): } request = new_cell_message({}, payload) - if blocking: + if expect_result: self.logger.info(f"send_request from {self.cell.get_fqcn()} to {self.target_fqcn}: {payload=} {timeout=}") reply = self.cell.send_request( From c6353b7e319cb72262420d1d9b6c793e6734f9f1 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Mon, 24 Nov 2025 14:08:21 -0500 Subject: [PATCH 068/102] support expect_result --- nvflare/fox/api/app.py | 9 +- nvflare/fox/api/constants.py | 10 +- nvflare/fox/api/ctx.py | 17 ++- nvflare/fox/api/facade.py | 2 +- nvflare/fox/api/gcc.py | 3 +- nvflare/fox/api/group.py | 127 ++++++++---------- nvflare/fox/api/proxy.py | 53 +++----- nvflare/fox/api/proxy_list.py | 1 - nvflare/fox/examples/np/algos/client.py | 15 +-- .../fox/examples/np/algos/strategies/avg_h.py | 4 +- .../examples/np/algos/strategies/avg_seq.py | 5 +- nvflare/fox/examples/np/algos/swarm.py | 12 +- nvflare/fox/examples/pt/pt_avg_mixed.py | 4 +- nvflare/fox/examples/pt/pt_avg_stream.py | 5 +- nvflare/fox/sim/backend.py | 31 ++--- 15 files changed, 134 insertions(+), 164 deletions(-) diff --git a/nvflare/fox/api/app.py b/nvflare/fox/api/app.py index 4bf8ee4032..1cdaf63774 100644 --- a/nvflare/fox/api/app.py +++ b/nvflare/fox/api/app.py @@ -149,7 +149,6 @@ def _find_filter_chain(self, direction, chains: List[FilterChain], target_name: qualified_func_name = f"{collab_obj_name}.{func_name}" ctx.set_prop(ContextKey.QUALIFIED_FUNC_NAME, qualified_func_name) ctx.set_prop(ContextKey.DIRECTION, direction) - self.logger.info(f"set filter context {id(ctx)}: {direction} {qualified_func_name}") if not chains: return None @@ -160,7 +159,7 @@ def _find_filter_chain(self, direction, chains: List[FilterChain], target_name: return None def apply_incoming_call_filters(self, target_name: str, func_name: str, func_kwargs, context: Context): - self.logger.info(f"apply_incoming_call_filters on ctx {id(context)}") + self.logger.debug(f"apply_incoming_call_filters on ctx {id(context)}") filter_chain = self._find_filter_chain( FilterDirection.INCOMING, self._incoming_call_filter_chains, target_name, func_name, context ) @@ -170,7 +169,7 @@ def apply_incoming_call_filters(self, target_name: str, func_name: str, func_kwa return func_kwargs def apply_outgoing_call_filters(self, target_name: str, func_name: str, func_kwargs, context: Context): - self.logger.info(f"apply_outgoing_call_filters on ctx {id(context)}") + self.logger.debug(f"apply_outgoing_call_filters on ctx {id(context)}") filter_chain = self._find_filter_chain( FilterDirection.OUTGOING, self._outgoing_call_filter_chains, target_name, func_name, context ) @@ -180,7 +179,7 @@ def apply_outgoing_call_filters(self, target_name: str, func_name: str, func_kwa return func_kwargs def apply_incoming_result_filters(self, target_name: str, func_name: str, result, context: Context): - self.logger.info(f"apply_incoming_result_filters on ctx {id(context)}") + self.logger.debug(f"apply_incoming_result_filters on ctx {id(context)}") filter_chain = self._find_filter_chain( FilterDirection.INCOMING, self._incoming_result_filter_chains, target_name, func_name, context ) @@ -190,7 +189,7 @@ def apply_incoming_result_filters(self, target_name: str, func_name: str, result return result def apply_outgoing_result_filters(self, target_name: str, func_name: str, result, context: Context): - self.logger.info(f"apply_outgoing_result_filters on ctx {id(context)}") + self.logger.debug(f"apply_outgoing_result_filters on ctx {id(context)}") filter_chain = self._find_filter_chain( FilterDirection.OUTGOING, self._outgoing_result_filter_chains, target_name, func_name, context ) diff --git a/nvflare/fox/api/constants.py b/nvflare/fox/api/constants.py index 90aee53526..a27f3c9e81 100644 --- a/nvflare/fox/api/constants.py +++ b/nvflare/fox/api/constants.py @@ -17,11 +17,11 @@ class CollabMethodArgName: class CollabMethodOptionName: - BLOCKING = "_blocking" - TIMEOUT = "_timeout" - OPTIONAL = "_optional" - SECURE = "_secure" - EXPECT_RESULT = "_expect_result" + BLOCKING = "__blocking__" + TIMEOUT = "__timeout__" + OPTIONAL = "__optional__" + SECURE = "__secure__" + EXPECT_RESULT = "__expect_result__" OPTION_ARGS = [ diff --git a/nvflare/fox/api/ctx.py b/nvflare/fox/api/ctx.py index 697271b28c..c6c7c08642 100644 --- a/nvflare/fox/api/ctx.py +++ b/nvflare/fox/api/ctx.py @@ -33,6 +33,7 @@ def __init__(self, app, caller: str, callee: str, abort_signal: Signal, target_g self.abort_signal = abort_signal self.app = app self.props = {} + self.parent_ctx = get_call_context() @property def backend(self): @@ -74,14 +75,22 @@ def get_prop(self, name: str, default=None): def is_aborted(self): return self.abort_signal and self.abort_signal.triggered - def header_str(self): + def __str__(self): return f"{self.app.name}:{self.caller}=>{self.callee}" + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.parent_ctx: + set_call_context(self.parent_ctx) + def get_call_context(): - if not fox_context.call_ctx: - print("NO CALL_CTX in FOX Context!!!") - return fox_context.call_ctx + if hasattr(fox_context, "call_ctx"): + return fox_context.call_ctx + else: + return None def set_call_context(ctx): diff --git a/nvflare/fox/api/facade.py b/nvflare/fox/api/facade.py index 25881e296a..41a38009c6 100644 --- a/nvflare/fox/api/facade.py +++ b/nvflare/fox/api/facade.py @@ -57,7 +57,7 @@ def callee(cls): @classproperty def call_info(cls): ctx = get_call_context() - return ctx.header_str() + return str(ctx) @classproperty def site_name(cls): diff --git a/nvflare/fox/api/gcc.py b/nvflare/fox/api/gcc.py index 4ad6ca59a2..2a51f9df1a 100644 --- a/nvflare/fox/api/gcc.py +++ b/nvflare/fox/api/gcc.py @@ -66,7 +66,6 @@ def set_result(self, target_name: str, result): parts = target_name.split(".") site_name = parts[0] with self.lock: - print(f"set result for {target_name} on site {site_name}") all_received = self.results.append((site_name, result)) if all_received: self.set() @@ -93,6 +92,8 @@ def __init__(self, app, target_name, func_name, process_cb, cb_kwargs, context: self.cb_kwargs = cb_kwargs self.context = context self.waiter = waiter + self.expect_result = True + self.timeout = None self.logger = get_obj_logger(self) def set_result(self, result): diff --git a/nvflare/fox/api/group.py b/nvflare/fox/api/group.py index 1fc6a8ecc4..4ccbe28fd7 100644 --- a/nvflare/fox/api/group.py +++ b/nvflare/fox/api/group.py @@ -20,7 +20,7 @@ from .app import App from .constants import CollabMethodArgName, CollabMethodOptionName -from .ctx import Context, get_call_context, set_call_context +from .ctx import Context from .gcc import GroupCallContext, ResultWaiter from .proxy import Proxy from .utils import check_call_args @@ -52,8 +52,6 @@ def __init__( timeout: how long to wait for responses. optional: whether the call is optional or not. secure: whether the call is secure or not. - min_resps: minimum number of responses expected. - wait_after_min_resps: how much longer to wait after min_resps are received. process_resp_cb: callback function to be called to process responses from remote apps. **cb_kwargs: kwargs passed to process_resp_cb. """ @@ -98,10 +96,7 @@ def __getattr__(self, func_name): def method(*args, **kwargs): the_backend = None - orig_ctx = get_call_context() try: - gccs = {} - # filter once for all targets p = self._proxies[0] @@ -109,78 +104,74 @@ def method(*args, **kwargs): # the func_proxy is either "p" or a child of "p". func_proxy, func_itf, adj_args, adj_kwargs = p.adjust_func_args(func_name, args, kwargs) the_backend = p.backend - ctx = func_proxy.app.new_context(func_proxy.caller_name, func_proxy.name, target_group=self) - - self._logger.debug( - f"[{ctx.header_str()}] calling {func_name} of group {[p.name for p in self._proxies]}" - ) - - # apply outgoing call filters - assert isinstance(self._app, App) - adj_kwargs = self._app.apply_outgoing_call_filters(func_proxy.target_name, func_name, adj_kwargs, ctx) - check_call_args(func_name, func_itf, adj_args, adj_kwargs) - - waiter = ResultWaiter([p.name for p in self._proxies]) - for p in self._proxies: - func_proxy, func_itf, call_args, call_kwargs = p.adjust_func_args(func_name, adj_args, adj_kwargs) - call_kwargs = copy.copy(call_kwargs) - ctx = self._app.new_context( - func_proxy.caller_name, func_proxy.name, target_group=self, set_call_ctx=False - ) - - call_kwargs[CollabMethodArgName.CONTEXT] = ctx - - # set the optional args to help backend decide how to call - if self._timeout: - call_kwargs[CollabMethodOptionName.TIMEOUT] = self._timeout - call_kwargs[CollabMethodOptionName.BLOCKING] = self._blocking - call_kwargs[CollabMethodOptionName.EXPECT_RESULT] = self._expect_result - call_kwargs[CollabMethodOptionName.SECURE] = self._secure - call_kwargs[CollabMethodOptionName.OPTIONAL] = self._optional - - gcc = GroupCallContext( - self._app, - func_proxy.target_name, - func_name, - self._process_resp_cb, - self._cb_kwargs, - ctx, - waiter, - ) - gccs[p.name] = gcc + with func_proxy.app.new_context(func_proxy.caller_name, func_proxy.name, target_group=self) as ctx: + self._logger.debug(f"[{ctx}] calling {func_name} of group {[p.name for p in self._proxies]}") - self._logger.debug( - f"[{ctx.header_str()}] group call: {func_name=} args={call_args} kwargs={call_kwargs}" + # apply outgoing call filters + assert isinstance(self._app, App) + adj_kwargs = self._app.apply_outgoing_call_filters( + func_proxy.target_name, func_name, adj_kwargs, ctx ) - func_proxy.backend.call_target_in_group(gcc, func_name, *call_args, **call_kwargs) - - if not self._expect_result: - # do not wait for responses - return None + check_call_args(func_name, func_itf, adj_args, adj_kwargs) - if not self._blocking: - self._logger.debug(f"not blocking {func_name}") - return waiter.results + waiter = ResultWaiter([p.name for p in self._proxies]) + for p in self._proxies: + func_proxy, func_itf, call_args, call_kwargs = p.adjust_func_args( + func_name, adj_args, adj_kwargs + ) + call_kwargs = copy.copy(call_kwargs) + ctx = self._app.new_context( + func_proxy.caller_name, func_proxy.name, target_group=self, set_call_ctx=False + ) - # wait for responses - while True: - if self._abort_signal.triggered: - raise RunAborted("run is aborted") + call_kwargs[CollabMethodArgName.CONTEXT] = ctx - # wait for a short time, so we can check other conditions - done = waiter.wait(0.1) - if done: - self._logger.info(f"all received from {waiter.results} for func {func_name}") - break + # set the optional args to help backend decide how to call + call_kwargs[CollabMethodOptionName.TIMEOUT] = self._timeout + call_kwargs[CollabMethodOptionName.BLOCKING] = self._blocking + call_kwargs[CollabMethodOptionName.EXPECT_RESULT] = self._expect_result + call_kwargs[CollabMethodOptionName.SECURE] = self._secure + call_kwargs[CollabMethodOptionName.OPTIONAL] = self._optional + + gcc = GroupCallContext( + self._app, + func_proxy.target_name, + func_name, + self._process_resp_cb, + self._cb_kwargs, + ctx, + waiter, + ) + gcc.expect_result = self._expect_result + gcc.timeout = self._timeout + + self._logger.debug(f"[{ctx}] group call: {func_name=} args={call_args} kwargs={call_kwargs}") + func_proxy.backend.call_target_in_group(gcc, func_name, *call_args, **call_kwargs) + + if not self._expect_result: + # do not wait for responses + return None + + if not self._blocking: + self._logger.debug(f"not blocking {func_name}") + return waiter.results + + # wait for responses + while True: + if self._abort_signal.triggered: + raise RunAborted("run is aborted") + + # wait for a short time, so we can check other conditions + done = waiter.wait(0.1) + if done: + self._logger.info(f"all received from {waiter.results} for func {func_name}") + break - return waiter.results + return waiter.results except Exception as ex: if the_backend: the_backend.handle_exception(ex) - finally: - if orig_ctx: - set_call_context(orig_ctx) return method diff --git a/nvflare/fox/api/proxy.py b/nvflare/fox/api/proxy.py index 6483a7974e..b45b6a140a 100644 --- a/nvflare/fox/api/proxy.py +++ b/nvflare/fox/api/proxy.py @@ -17,7 +17,6 @@ from .backend import Backend from .constants import OPTION_ARGS, CollabMethodArgName, CollabMethodOptionName -from .ctx import get_call_context, set_call_context from .utils import check_call_args @@ -26,13 +25,13 @@ class _ProxyCall: def __init__( self, proxy, - blocking: bool = True, + expect_result: bool = True, timeout: float = 5.0, optional: bool = False, secure: bool = False, ): self.proxy = proxy - self.blocking = blocking + self.expect_result = expect_result self.timeout = timeout self.optional = optional self.secure = secure @@ -43,7 +42,7 @@ def method(*args, **kwargs): { CollabMethodOptionName.OPTIONAL: self.optional, CollabMethodOptionName.SECURE: self.secure, - CollabMethodOptionName.BLOCKING: self.blocking, + CollabMethodOptionName.EXPECT_RESULT: self.expect_result, CollabMethodOptionName.TIMEOUT: self.timeout, } ) @@ -67,14 +66,14 @@ def __init__(self, app, target_name, target_fqn: str, backend: Backend, target_i def __call__( self, - blocking: bool = True, + expect_result: bool = True, timeout: float = 5.0, optional: bool = False, secure: bool = False, ): return _ProxyCall( proxy=self, - blocking=blocking, + expect_result=expect_result, timeout=timeout, optional=optional, secure=secure, @@ -155,7 +154,6 @@ def __getattr__(self, func_name): """ def method(*args, **kwargs): - orig_ctx = get_call_context() try: # remove option args option_args = {} @@ -164,39 +162,32 @@ def method(*args, **kwargs): option_args[k] = kwargs.pop(k) p, func_itf, call_args, call_kwargs = self.adjust_func_args(func_name, args, kwargs) - ctx = p.app.new_context(self.caller_name, self.name) - self.logger.debug( - f"[{ctx.header_str()}] apply_outgoing_call_filters on {p.target_name} func {func_name}" - ) + with p.app.new_context(self.caller_name, self.name) as ctx: + self.logger.debug(f"[{ctx}] apply_outgoing_call_filters on {p.target_name} func {func_name}") - # apply outgoing call filters - call_kwargs = self.app.apply_outgoing_call_filters(p.target_name, func_name, call_kwargs, ctx) - check_call_args(func_name, func_itf, call_args, call_kwargs) + # apply outgoing call filters + call_kwargs = self.app.apply_outgoing_call_filters(p.target_name, func_name, call_kwargs, ctx) + check_call_args(func_name, func_itf, call_args, call_kwargs) - call_kwargs[CollabMethodArgName.CONTEXT] = ctx + call_kwargs[CollabMethodArgName.CONTEXT] = ctx - # restore option args - for k, v in option_args.items(): - call_kwargs[k] = v + # restore option args + for k, v in option_args.items(): + call_kwargs[k] = v - self.logger.debug( - f"[{ctx.header_str()}] calling target {p.target_name} func {func_name}: {option_args=}" - ) + self.logger.debug(f"[{ctx}] calling target {p.target_name} func {func_name}: {option_args=}") - result = p.backend.call_target(p.target_name, func_name, *call_args, **call_kwargs) - if isinstance(result, Exception): - raise result + result = p.backend.call_target(p.target_name, func_name, *call_args, **call_kwargs) + if isinstance(result, Exception): + raise result - if result is not None: - # filter incoming result filters - result = self.app.apply_incoming_result_filters(p.target_name, func_name, result, ctx) + if result is not None: + # filter incoming result filters + result = self.app.apply_incoming_result_filters(p.target_name, func_name, result, ctx) - return result + return result except Exception as ex: self.backend.handle_exception(ex) - finally: - if orig_ctx: - set_call_context(orig_ctx) return method diff --git a/nvflare/fox/api/proxy_list.py b/nvflare/fox/api/proxy_list.py index f6011dcb38..63deb808bd 100644 --- a/nvflare/fox/api/proxy_list.py +++ b/nvflare/fox/api/proxy_list.py @@ -41,7 +41,6 @@ def __call__( process_resp_cb=None, **cb_kwargs, ): - print(f"creating group: {blocking=} {timeout=} {optional=} {secure=}") return group( ctx=get_call_context(), proxies=self, diff --git a/nvflare/fox/examples/np/algos/client.py b/nvflare/fox/examples/np/algos/client.py index ab4feec7de..975350bbd4 100644 --- a/nvflare/fox/examples/np/algos/client.py +++ b/nvflare/fox/examples/np/algos/client.py @@ -39,15 +39,10 @@ def train(self, current_round, weights): self.logger.debug("training aborted") return 0 self.logger.info(f"[{fox.call_info}] EZ trained round {current_round=} {weights=}") + fox.server(expect_result=False).fire_event("metrics", {"round": current_round, "y": 10}) - # metric_receiver = self.server.get_target("metric_receiver") - # if metric_receiver: - # self.server.accept_metric({"round": r, "y": 2}) - # self.server.metric_receiver.accept_metric({"round": r, "y": 2}) - # - self.logger.info(f"before fire_event: fox ctx={id(fox.context)}") - fox.server(blocking=False).fire_event("metrics", {"round": current_round, "y": 10}) - self.logger.info(f"after fire_event: fox ctx={id(fox.context)}") + # force timeout to test timeout handling + # time.sleep(10) return weights + self.delta @fox.collab @@ -72,14 +67,14 @@ def train(self, current_round, weights): if fox.has_children: total = 0 results = fox.child_clients.train(current_round, weights) - for n, v in results.items(): + for n, v in results: total += v result = total / len(results) self.logger.debug(f"[{fox.call_info}]: aggr result from children of round {current_round}: {result}") else: result = self._local_train(current_round, weights) self.logger.debug(f"[{fox.call_info}]: local train result of round {current_round}: {result}") - fox.server.fire_event("metrics", {"round": current_round, "y": 10}, _blocking=False) + fox.server(expect_result=False).fire_event("metrics", {"round": current_round, "y": 10}) return result def _local_train(self, current_round, weights): diff --git a/nvflare/fox/examples/np/algos/strategies/avg_h.py b/nvflare/fox/examples/np/algos/strategies/avg_h.py index ab603d7d86..e28d8ddf49 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_h.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_h.py @@ -39,7 +39,7 @@ def execute(self): def _do_eval(self, model): results = fox.leaf_clients.evaluate(model) total = 0.0 - for n, v in results.items(): + for n, v in results: self.logger.info(f"[{fox.call_info}]: got eval result from client {n}: {v}") total += v return total / len(results) @@ -47,7 +47,7 @@ def _do_eval(self, model): def _do_one_round(self, r, current_model): total = 0 results = fox.child_clients.train(r, current_model) - for n, v in results.items(): + for n, v in results: self.logger.info(f"[{fox.call_info}] round {r}: got group result from client {n}: {v}") total += v return total / len(results) diff --git a/nvflare/fox/examples/np/algos/strategies/avg_seq.py b/nvflare/fox/examples/np/algos/strategies/avg_seq.py index 5bb8e8aa1a..ede5e8654a 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_seq.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_seq.py @@ -56,12 +56,13 @@ def execute(self): # save model to work dir file_name = os.path.join(fox.workspace.get_work_dir(), "model.npy") save_np_model(current_model, file_name) + self.logger.info(f"FINAL RESULT: {current_model}") return current_model def _do_one_round(self, r, current_model): total = 0 for c in fox.clients: - result = c(blocking=True, timeout=2.0, optional=True, secure=False).train(r, current_model) - self.logger.info(f"[{fox.context.header_str()}] round {r}: got result from client {c.name}: {result}") + result = c(expect_result=True, timeout=2.0, optional=True, secure=False).train(r, current_model) + self.logger.info(f"[{fox.call_info}] round {r}: got result from client {c.name}: {result}") total += result * self.client_weights[c.name] return total diff --git a/nvflare/fox/examples/np/algos/swarm.py b/nvflare/fox/examples/np/algos/swarm.py index 86e92abe35..1a692b9a7c 100644 --- a/nvflare/fox/examples/np/algos/swarm.py +++ b/nvflare/fox/examples/np/algos/swarm.py @@ -68,10 +68,9 @@ def train(self, weights, current_round): def sag(self, model, current_round): results = fox.clients.train(model, current_round) - results = list(results.values()) total = 0 - for i in range(len(results)): - total += results[i] + for n, v in results: + total += v return total / len(results) @fox.collab @@ -82,11 +81,10 @@ def swarm_learn(self, num_rounds, model, current_round): self.logger.info(f"[{fox.call_info}]: trained model {new_model=}") if current_round == num_rounds - 1: # all done - fox.clients(blocking=False).fire_event("final_model", new_model) - # self.server.fire_event("all_done", "OK", blocking=False) + fox.clients(expect_result=False).fire_event("final_model", new_model) self.logger.info("notify server all done!") try: - fox.server(blocking=False).all_done("OK") + fox.server(expect_result=False).all_done("OK") except: traceback.print_exc() self.logger.info("Swarm Training is DONE!") @@ -97,7 +95,7 @@ def swarm_learn(self, num_rounds, model, current_round): next_client_idx = random.randint(0, len(fox.clients) - 1) self.logger.debug(f"chose aggr client for round {next_round}: {next_client_idx}") next_client = fox.clients[next_client_idx] - next_client.swarm_learn(num_rounds, new_model, next_round, _blocking=False) + next_client(expect_result=False).swarm_learn(num_rounds, new_model, next_round) @fox.collab def start(self, num_rounds, initial_model): diff --git a/nvflare/fox/examples/pt/pt_avg_mixed.py b/nvflare/fox/examples/pt/pt_avg_mixed.py index 4a2d3e25a8..5673248b4d 100644 --- a/nvflare/fox/examples/pt/pt_avg_mixed.py +++ b/nvflare/fox/examples/pt/pt_avg_mixed.py @@ -135,12 +135,12 @@ def _accept_train_result(self, result, aggr_result: _AggrResult): return None def _aggregate_tensors(self, td: dict[str, torch.Tensor], aggr_result: _AggrResult, context: Context): - self.logger.info(f"[{context.header_str()}] aggregating received tensor: {td}") + self.logger.info(f"[{context}] aggregating received tensor: {td}") with aggr_result.lock: add_pt(td, aggr_result.pt_total) def _aggregate_arrays(self, td: dict[str, np.ndarray], aggr_result: _AggrResult, context: Context): - self.logger.info(f"[{context.header_str()}] aggregating received array: {td}") + self.logger.info(f"[{context}] aggregating received array: {td}") with aggr_result.lock: add_np(td, aggr_result.np_total) diff --git a/nvflare/fox/examples/pt/pt_avg_stream.py b/nvflare/fox/examples/pt/pt_avg_stream.py index 29cd5dc10a..ae58efffb3 100644 --- a/nvflare/fox/examples/pt/pt_avg_stream.py +++ b/nvflare/fox/examples/pt/pt_avg_stream.py @@ -100,7 +100,6 @@ def _accept_train_result(self, result, aggr_result: _AggrResult): ctx=fox.context, tensors_received_cb=self._aggregate_tensors, aggr_result=aggr_result, - context=fox.context, ) if err: raise RuntimeError(f"failed to download model {model}: {err}") @@ -114,8 +113,8 @@ def _accept_train_result(self, result, aggr_result: _AggrResult): aggr_result.count += 1 return None - def _aggregate_tensors(self, td: dict[str, torch.Tensor], aggr_result: _AggrResult, context: Context): - self.logger.info(f"[{context.header_str()}] aggregating received tensor: {td}") + def _aggregate_tensors(self, td: dict[str, torch.Tensor], aggr_result: _AggrResult): + self.logger.info(f"[{fox.call_info}] aggregating received tensor: {td}") with aggr_result.lock: for k, v in td.items(): if k not in aggr_result.total: diff --git a/nvflare/fox/sim/backend.py b/nvflare/fox/sim/backend.py index a82f5d0425..e365b4b43f 100644 --- a/nvflare/fox/sim/backend.py +++ b/nvflare/fox/sim/backend.py @@ -51,7 +51,7 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): if not callable(func): raise AttributeError(f"the method '{func_name}' of {target_name} is not callable") - blocking = kwargs.pop(CollabMethodOptionName.BLOCKING, True) + expect_result = kwargs.pop(CollabMethodOptionName.EXPECT_RESULT, True) timeout = kwargs.pop(CollabMethodOptionName.TIMEOUT, 5.0) # other options don't apply to simulation @@ -59,7 +59,7 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): kwargs.pop(k, None) waiter = None - if blocking: + if expect_result: waiter = _Waiter() self.executor.submit(self._run_func, waiter, target_name, func_name, func, args, kwargs) @@ -106,7 +106,7 @@ def _preprocess(self, target_name, func_name, func, kwargs): raise RuntimeError(f"cannot find interface for func '{func_name}' of object {self.target_obj_name}") check_call_args(func_name, func_itf, [], kwargs) - self.logger.debug(f"[{my_ctx.header_str()}] received kwargs is good for '{func_name}': {kwargs}") + self.logger.debug(f"[{my_ctx}] received kwargs is good for '{func_name}': {kwargs}") kwargs[CollabMethodArgName.CONTEXT] = my_ctx adjust_kwargs(func, kwargs) @@ -136,6 +136,7 @@ def _run_func(self, waiter: _Waiter, target_name, func_name, func, args, kwargs) def call_target_in_group(self, gcc: GroupCallContext, func_name: str, *args, **kwargs): # do not use the optional args - they are managed by the group self.logger.info(f"call_target_in_group: {gcc.target_name=}") + for k in OPTION_ARGS: kwargs.pop(k, None) @@ -147,28 +148,14 @@ def call_target_in_group(self, gcc: GroupCallContext, func_name: str, *args, **k if not callable(func): raise AttributeError(f"the method '{func_name}' of {target_name} is not callable") - self.executor.submit(self._run_func_with_resp, gcc, func_name, func, args, kwargs) + self.executor.submit(self._run_func_in_group, gcc, func_name, args, kwargs) - def _run_func_with_resp(self, gcc: GroupCallContext, func_name, func, args, kwargs): + def _run_func_in_group(self, gcc: GroupCallContext, func_name, args, kwargs): try: target_name = gcc.target_name - ctx, kwargs = self._preprocess(target_name, func_name, func, kwargs) - self.logger.info(f"after _preprocess: ctx={id(ctx)} fox_ctx={id(fox.context)}") - result = func(*args, **kwargs) - # set_call_context(ctx) - self.logger.info(f"after calling {func_name}: ctx={id(ctx)} fox_ctx={id(fox.context)}") - - # apply result filter - result = self.target_app.apply_outgoing_result_filters(target_name, func_name, result, ctx) - - self.logger.info( - f"after apply_outgoing_result_filters {func_name}: ctx={id(ctx)} fox_ctx={id(fox.context)}" - ) - - self.logger.info( - f"_run_func_with_resp: {id(ctx)} apply_outgoing_result_filters: {ctx.get_prop(ContextKey.DIRECTION)}" - f" {ctx.get_prop(ContextKey.QUALIFIED_FUNC_NAME)}" - ) + kwargs[CollabMethodOptionName.EXPECT_RESULT] = gcc.expect_result + kwargs[CollabMethodOptionName.TIMEOUT] = gcc.timeout + result = self.call_target(target_name, func_name, *args, **kwargs) gcc.set_result(result) except Exception as ex: gcc.set_exception(ex) From 6235b3634dc73c4d62fa43fce4284cb76bcdb537 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Mon, 24 Nov 2025 16:25:12 -0500 Subject: [PATCH 069/102] refactor downloader --- nvflare/fox/api/app.py | 2 +- nvflare/fox/api/backend.py | 4 +- nvflare/fox/api/call_opt.py | 34 +++++ nvflare/fox/api/constants.py | 19 +-- nvflare/fox/api/ctx.py | 4 +- nvflare/fox/api/facade.py | 4 +- nvflare/fox/api/gcc.py | 17 ++- nvflare/fox/api/group.py | 50 ++++--- nvflare/fox/api/proxy.py | 138 ++++++++++++-------- nvflare/fox/examples/np/algos/avg_stream.py | 10 +- nvflare/fox/examples/pt/filters.py | 6 +- nvflare/fox/examples/pt/filters2.py | 6 +- nvflare/fox/examples/pt/pt_avg_mixed.py | 23 ++-- nvflare/fox/examples/pt/pt_avg_stream.py | 12 +- nvflare/fox/sim/backend.py | 20 +-- nvflare/fox/sim/simulator.py | 6 +- nvflare/fox/sys/backend.py | 24 ++-- nvflare/fox/sys/controller.py | 4 +- nvflare/fox/sys/downloader.py | 13 +- nvflare/fox/sys/executor.py | 4 +- 20 files changed, 217 insertions(+), 183 deletions(-) create mode 100644 nvflare/fox/api/call_opt.py diff --git a/nvflare/fox/api/app.py b/nvflare/fox/api/app.py index 1cdaf63774..e6b42f9dbb 100644 --- a/nvflare/fox/api/app.py +++ b/nvflare/fox/api/app.py @@ -46,7 +46,7 @@ def __init__(self, obj, name: str): self.server = None self.clients = None self.client_hierarchy = None - self.env_type = None + self.backend_type = None self._me = None self._collab_objs = {} self._abort_signal = None diff --git a/nvflare/fox/api/backend.py b/nvflare/fox/api/backend.py index 1625030b9d..8e279d95f3 100644 --- a/nvflare/fox/api/backend.py +++ b/nvflare/fox/api/backend.py @@ -17,6 +17,7 @@ from nvflare.fuel.utils.log_utils import get_obj_logger from nvflare.security.logging import secure_format_traceback +from .call_opt import CallOpt from .gcc import GroupCallContext @@ -30,12 +31,13 @@ def __init__(self, abort_signal: Signal): self.logger = get_obj_logger(self) @abstractmethod - def call_target(self, target_name: str, func_name: str, *args, **kwargs): + def call_target(self, target_name: str, call_opt: CallOpt, func_name: str, *args, **kwargs): """ Call a target function with arguments and return a result. Args: target_name: the fully qualified name of the target object to be called in the remote app. + call_opt: call options. func_name: name of the function to be called in the remote app. *args: args to pass to the target function. **kwargs: kwargs to pass to the target function. diff --git a/nvflare/fox/api/call_opt.py b/nvflare/fox/api/call_opt.py new file mode 100644 index 0000000000..f7e93d93c9 --- /dev/null +++ b/nvflare/fox/api/call_opt.py @@ -0,0 +1,34 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +class CallOpt: + + def __init__( + self, + expect_result: bool = True, + blocking: bool = True, + timeout: float = 5.0, + secure: bool = False, + optional: bool = False, + ): + self.expect_result = expect_result + self.blocking = blocking + self.timeout = timeout + self.secure = secure + self.optional = optional + + def __str__(self): + return ( + f"expect_result={self.expect_result} blocking={self.blocking} timeout={self.timeout} " + f"secure={self.secure} optional={self.optional}" + ) diff --git a/nvflare/fox/api/constants.py b/nvflare/fox/api/constants.py index a27f3c9e81..0d8ef7b617 100644 --- a/nvflare/fox/api/constants.py +++ b/nvflare/fox/api/constants.py @@ -16,23 +16,6 @@ class CollabMethodArgName: CONTEXT = "context" -class CollabMethodOptionName: - BLOCKING = "__blocking__" - TIMEOUT = "__timeout__" - OPTIONAL = "__optional__" - SECURE = "__secure__" - EXPECT_RESULT = "__expect_result__" - - -OPTION_ARGS = [ - CollabMethodOptionName.BLOCKING, - CollabMethodOptionName.TIMEOUT, - CollabMethodOptionName.OPTIONAL, - CollabMethodOptionName.SECURE, - CollabMethodOptionName.EXPECT_RESULT, -] - - class ContextKey: INPUT = "input" QUALIFIED_FUNC_NAME = "qualified_func_name" @@ -44,6 +27,6 @@ class FilterDirection: OUTGOING = "outgoing" -class EnvType: +class BackendType: SIMULATION = "simulation" SYSTEM = "system" diff --git a/nvflare/fox/api/ctx.py b/nvflare/fox/api/ctx.py index c6c7c08642..b03cb53dcb 100644 --- a/nvflare/fox/api/ctx.py +++ b/nvflare/fox/api/ctx.py @@ -40,8 +40,8 @@ def backend(self): return self.app.get_backend() @property - def env_type(self): - return self.app.env_type + def backend_type(self): + return self.app.backend_type @property def clients(self): diff --git a/nvflare/fox/api/facade.py b/nvflare/fox/api/facade.py index 41a38009c6..936cac0547 100644 --- a/nvflare/fox/api/facade.py +++ b/nvflare/fox/api/facade.py @@ -109,9 +109,9 @@ def leaf_clients(cls): return ProxyList(candidates) @classproperty - def env_type(cls): + def backend_type(cls): ctx = get_call_context() - return ctx.env_type + return ctx.backend_type @classproperty def is_aborted(cls): diff --git a/nvflare/fox/api/gcc.py b/nvflare/fox/api/gcc.py index 2a51f9df1a..4490e63886 100644 --- a/nvflare/fox/api/gcc.py +++ b/nvflare/fox/api/gcc.py @@ -17,6 +17,7 @@ from nvflare.fuel.utils.log_utils import get_obj_logger +from .call_opt import CallOpt from .constants import CollabMethodArgName from .ctx import Context, set_call_context from .utils import check_context_support @@ -73,12 +74,23 @@ def set_result(self, target_name: str, result): class GroupCallContext: - def __init__(self, app, target_name, func_name, process_cb, cb_kwargs, context: Context, waiter: ResultWaiter): + def __init__( + self, + app, + target_name: str, + call_opt: CallOpt, + func_name: str, + process_cb, + cb_kwargs, + context: Context, + waiter: ResultWaiter, + ): """GroupCallContext contains contextual information about a group call to a target. Args: app: the calling app. target_name: name of the target to be called in the remote app. + call_opt: call options. func_name: name of the function to be called in the remote app. process_cb: the callback function to be called to process response from the remote app. cb_kwargs: kwargs passed to the callback function. @@ -86,14 +98,13 @@ def __init__(self, app, target_name, func_name, process_cb, cb_kwargs, context: waiter: the waiter to wait for result """ self.app = app + self.call_opt = call_opt self.target_name = target_name self.func_name = func_name self.process_cb = process_cb self.cb_kwargs = cb_kwargs self.context = context self.waiter = waiter - self.expect_result = True - self.timeout = None self.logger = get_obj_logger(self) def set_result(self, result): diff --git a/nvflare/fox/api/group.py b/nvflare/fox/api/group.py index 4ccbe28fd7..0d7d65cea3 100644 --- a/nvflare/fox/api/group.py +++ b/nvflare/fox/api/group.py @@ -19,7 +19,8 @@ from nvflare.fuel.utils.log_utils import get_obj_logger from .app import App -from .constants import CollabMethodArgName, CollabMethodOptionName +from .call_opt import CallOpt +from .constants import CollabMethodArgName from .ctx import Context from .gcc import GroupCallContext, ResultWaiter from .proxy import Proxy @@ -61,11 +62,13 @@ def __init__( self._app = app self._abort_signal = abort_signal self._proxies = proxies - self._blocking = blocking - self._expect_result = expect_result - self._timeout = timeout - self._optional = optional - self._secure = secure + self._call_opt = CallOpt( + blocking=blocking, + expect_result=expect_result, + timeout=timeout, + optional=optional, + secure=secure, + ) self._process_resp_cb = process_resp_cb self._cb_kwargs = cb_kwargs self._logger = get_obj_logger(self) @@ -106,7 +109,9 @@ def method(*args, **kwargs): the_backend = p.backend with func_proxy.app.new_context(func_proxy.caller_name, func_proxy.name, target_group=self) as ctx: - self._logger.debug(f"[{ctx}] calling {func_name} of group {[p.name for p in self._proxies]}") + self._logger.debug( + f"[{ctx}] calling {func_name} {self._call_opt} of group {[p.name for p in self._proxies]}" + ) # apply outgoing call filters assert isinstance(self._app, App) @@ -127,33 +132,23 @@ def method(*args, **kwargs): call_kwargs[CollabMethodArgName.CONTEXT] = ctx - # set the optional args to help backend decide how to call - call_kwargs[CollabMethodOptionName.TIMEOUT] = self._timeout - call_kwargs[CollabMethodOptionName.BLOCKING] = self._blocking - call_kwargs[CollabMethodOptionName.EXPECT_RESULT] = self._expect_result - call_kwargs[CollabMethodOptionName.SECURE] = self._secure - call_kwargs[CollabMethodOptionName.OPTIONAL] = self._optional - gcc = GroupCallContext( - self._app, - func_proxy.target_name, - func_name, - self._process_resp_cb, - self._cb_kwargs, - ctx, - waiter, + app=self._app, + target_name=func_proxy.target_name, + call_opt=self._call_opt, + func_name=func_name, + process_cb=self._process_resp_cb, + cb_kwargs=self._cb_kwargs, + context=ctx, + waiter=waiter, ) - gcc.expect_result = self._expect_result - gcc.timeout = self._timeout - - self._logger.debug(f"[{ctx}] group call: {func_name=} args={call_args} kwargs={call_kwargs}") func_proxy.backend.call_target_in_group(gcc, func_name, *call_args, **call_kwargs) - if not self._expect_result: + if not self._call_opt.expect_result: # do not wait for responses return None - if not self._blocking: + if not self._call_opt.blocking: self._logger.debug(f"not blocking {func_name}") return waiter.results @@ -165,7 +160,6 @@ def method(*args, **kwargs): # wait for a short time, so we can check other conditions done = waiter.wait(0.1) if done: - self._logger.info(f"all received from {waiter.results} for func {func_name}") break return waiter.results diff --git a/nvflare/fox/api/proxy.py b/nvflare/fox/api/proxy.py index b45b6a140a..46d17123d5 100644 --- a/nvflare/fox/api/proxy.py +++ b/nvflare/fox/api/proxy.py @@ -16,7 +16,8 @@ from nvflare.fuel.utils.log_utils import get_obj_logger from .backend import Backend -from .constants import OPTION_ARGS, CollabMethodArgName, CollabMethodOptionName +from .call_opt import CallOpt +from .constants import CollabMethodArgName from .utils import check_call_args @@ -31,22 +32,17 @@ def __init__( secure: bool = False, ): self.proxy = proxy - self.expect_result = expect_result - self.timeout = timeout - self.optional = optional - self.secure = secure + self.call_opt = CallOpt( + expect_result=expect_result, + blocking=expect_result, + timeout=timeout, + optional=optional, + secure=secure, + ) def __getattr__(self, func_name): def method(*args, **kwargs): - kwargs.update( - { - CollabMethodOptionName.OPTIONAL: self.optional, - CollabMethodOptionName.SECURE: self.secure, - CollabMethodOptionName.EXPECT_RESULT: self.expect_result, - CollabMethodOptionName.TIMEOUT: self.timeout, - } - ) - return getattr(self.proxy, func_name)(*args, **kwargs) + return self.proxy.call_func(self.call_opt, func_name, args, kwargs) return method @@ -71,6 +67,17 @@ def __call__( optional: bool = False, secure: bool = False, ): + """This is called when the proxy is used with call options. + + Args: + expect_result: + timeout: + optional: + secure: + + Returns: + + """ return _ProxyCall( proxy=self, expect_result=expect_result, @@ -97,6 +104,19 @@ def get_target(self, name: str): return None def _find_interface(self, func_name): + """Find interface for specified func name. + + Args: + func_name: name of the func. + + Returns: the proxy that the func belongs to, the func interface. + + Notes: the proxy represents a remote object. The remote object could have sub-objects. In this case, + the proxy will have child proxies, each representing a sub-object. + + We first try to find the interface from the proxy itself. If not found, we try to find it from child proxies. + + """ self.logger.debug(f"trying to find interface for {func_name}") args = self.target_interface.get(func_name) if self.target_interface else None if args: @@ -125,10 +145,23 @@ def _find_interface(self, func_name): return the_proxy, the_args def adjust_func_args(self, func_name, args, kwargs): + """Based on specified args and kwargs, this method finds corresponding keywords for all positional + args based on the interface of the func, and then moves the positional args into kwargs. + + Once done, all args will have keywords, which makes it easy for call filters to identify the args to process. + + Args: + func_name: name of the func. + args: positional arg values. + kwargs: keyword arg values + + Returns: the proxy that the func belongs to, interface of the func, empty args, and new kwargs + + """ call_args = args call_kwargs = kwargs - # find the proxy for the func + # find the proxy and interface for the func p, func_itf = self._find_interface(func_name) if not p: raise RuntimeError(f"target {self.target_name} does not have method '{func_name}'") @@ -148,46 +181,49 @@ def adjust_func_args(self, func_name, args, kwargs): return p, func_itf, call_args, call_kwargs - def __getattr__(self, func_name): - """ - This method is called when Python cannot find an invoked method func_name of this class. - """ - - def method(*args, **kwargs): - try: - # remove option args - option_args = {} - for k in OPTION_ARGS: - if k in kwargs: - option_args[k] = kwargs.pop(k) - - p, func_itf, call_args, call_kwargs = self.adjust_func_args(func_name, args, kwargs) + def call_func(self, call_opt: CallOpt, func_name, args, kwargs): + """Call the specified function with call options. - with p.app.new_context(self.caller_name, self.name) as ctx: - self.logger.debug(f"[{ctx}] apply_outgoing_call_filters on {p.target_name} func {func_name}") + Args: + call_opt: call option that controls the call behavior. + func_name: name of func to be called. + args: args to be passed to the func. + kwargs: kwargs to be passed to the func. - # apply outgoing call filters - call_kwargs = self.app.apply_outgoing_call_filters(p.target_name, func_name, call_kwargs, ctx) - check_call_args(func_name, func_itf, call_args, call_kwargs) + Returns: result of the function, or exception. - call_kwargs[CollabMethodArgName.CONTEXT] = ctx - - # restore option args - for k, v in option_args.items(): - call_kwargs[k] = v - - self.logger.debug(f"[{ctx}] calling target {p.target_name} func {func_name}: {option_args=}") - - result = p.backend.call_target(p.target_name, func_name, *call_args, **call_kwargs) - if isinstance(result, Exception): - raise result + """ + try: + p, func_itf, call_args, call_kwargs = self.adjust_func_args(func_name, args, kwargs) + + with p.app.new_context(self.caller_name, self.name) as ctx: + self.logger.debug(f"[{ctx}] apply_outgoing_call_filters on {p.target_name} func {func_name}") + + # apply outgoing call filters + call_kwargs = self.app.apply_outgoing_call_filters(p.target_name, func_name, call_kwargs, ctx) + check_call_args(func_name, func_itf, call_args, call_kwargs) + + self.logger.debug(f"[{ctx}] calling target {p.target_name} func {func_name}") + call_kwargs[CollabMethodArgName.CONTEXT] = ctx + result = p.backend.call_target(p.target_name, call_opt, func_name, *call_args, **call_kwargs) + if isinstance(result, Exception): + raise result + + if result is not None: + # filter incoming result filters + result = self.app.apply_incoming_result_filters(p.target_name, func_name, result, ctx) + return result + except Exception as ex: + self.backend.handle_exception(ex) + return ex - if result is not None: - # filter incoming result filters - result = self.app.apply_incoming_result_filters(p.target_name, func_name, result, ctx) + def __getattr__(self, func_name): + """ + This method is called when the proxy is invoked to perform the specified func without any call options. + In this case, we use a CallOpt with default values. + """ - return result - except Exception as ex: - self.backend.handle_exception(ex) + def method(*args, **kwargs): + return self.call_func(CallOpt(), func_name, args, kwargs) return method diff --git a/nvflare/fox/examples/np/algos/avg_stream.py b/nvflare/fox/examples/np/algos/avg_stream.py index fe67f54836..988d0e0867 100644 --- a/nvflare/fox/examples/np/algos/avg_stream.py +++ b/nvflare/fox/examples/np/algos/avg_stream.py @@ -15,7 +15,7 @@ import uuid from nvflare.fox import fox -from nvflare.fox.api.constants import EnvType +from nvflare.fox.api.constants import BackendType from nvflare.fox.api.dec import collab from nvflare.fox.examples.np.algos.utils import load_np_model, parse_array_def, save_np_model from nvflare.fox.sys.downloader import Downloader, download_file @@ -57,12 +57,11 @@ def _do_one_round(self, r, current_model): # pretend the model is big file_name = None - if fox.env_type == EnvType.SYSTEM: + if fox.backend_type == BackendType.SYSTEM: file_name = f"/tmp/np_{str(uuid.uuid4())}.npy" save_np_model(current_model, file_name) downloader = Downloader( num_receivers=grp.size, - ctx=fox.context, timeout=5.0, ) model = downloader.add_file(file_name=file_name, file_downloaded_cb=self._model_downloaded) @@ -89,7 +88,7 @@ def _accept_train_result(self, result, aggr_result: _AggrResult): model, model_type = result if model_type == "ref": - err, file_path = download_file(ref=model, per_request_timeout=5.0, ctx=fox.context) + err, file_path = download_file(ref=model, per_request_timeout=5.0) if err: raise RuntimeError(f"failed to download model file {model}: {err}") self.logger.info(f"downloaded model file to {file_path}") @@ -117,7 +116,7 @@ def train(self, current_round, weights, model_type: str): return 0 self.logger.debug(f"[{fox.call_info}] training round {current_round}: {model_type=} {weights=}") if model_type == "ref": - err, file_path = download_file(ref=weights, per_request_timeout=5.0, ctx=fox.context) + err, file_path = download_file(ref=weights, per_request_timeout=5.0) if err: raise RuntimeError(f"failed to download model file {weights}: {err}") self.logger.info(f"downloaded model file to {file_path}") @@ -133,7 +132,6 @@ def train(self, current_round, weights, model_type: str): save_np_model(result, file_name) downloader = Downloader( num_receivers=1, - ctx=fox.context, timeout=5.0, ) result = downloader.add_file(file_name=file_name, file_downloaded_cb=self._result_downloaded) diff --git a/nvflare/fox/examples/pt/filters.py b/nvflare/fox/examples/pt/filters.py index ac3f51f85d..77eaf3d6e9 100644 --- a/nvflare/fox/examples/pt/filters.py +++ b/nvflare/fox/examples/pt/filters.py @@ -36,7 +36,6 @@ def filter_call(self, func_kwargs: dict, context: Context): downloader = Downloader( num_receivers=context.target_group_size, - ctx=context, timeout=5.0, ) model = downloader.add_tensors(arg_value, 0) @@ -56,7 +55,7 @@ def filter_call(self, func_kwargs: dict, context: Context): if not arg_value: return func_kwargs - err, model = download_tensors(ref=arg_value, ctx=context, per_request_timeout=5.0) + err, model = download_tensors(ref=arg_value, per_request_timeout=5.0) if err: self.logger.error(f"error filtering call arg {arg_value}: {err}") else: @@ -73,7 +72,6 @@ def filter_result(self, result: Any, context: Context): downloader = Downloader( num_receivers=1, - ctx=context, timeout=5.0, ) return downloader.add_tensors(result, 0) @@ -86,7 +84,7 @@ def __init__(self): self.logger = get_obj_logger(self) def filter_result(self, result: Any, context: Context): - err, model = download_tensors(ref=result, ctx=context, per_request_timeout=5.0) + err, model = download_tensors(ref=result, per_request_timeout=5.0) if err: self.logger.error(f"error filtering result {result}: {err}") return result diff --git a/nvflare/fox/examples/pt/filters2.py b/nvflare/fox/examples/pt/filters2.py index 3d7a7d4d46..2fbd2ed66e 100644 --- a/nvflare/fox/examples/pt/filters2.py +++ b/nvflare/fox/examples/pt/filters2.py @@ -36,7 +36,6 @@ def prepare_weights_for_download(self, func_kwargs: dict): downloader = Downloader( num_receivers=num_receivers, - ctx=fox.context, timeout=5.0, ) model = downloader.add_tensors(arg_value, 0) @@ -49,7 +48,7 @@ def download_weights(self, func_kwargs: dict): if not arg_value: return func_kwargs - err, model = download_tensors(ref=arg_value, ctx=fox.context, per_request_timeout=5.0) + err, model = download_tensors(ref=arg_value, per_request_timeout=5.0) if err: self.logger.error(f"error filtering call arg {arg_value}: {err}") else: @@ -64,14 +63,13 @@ def prepare_result_for_download(self, result: Any): downloader = Downloader( num_receivers=1, - ctx=fox.context, timeout=5.0, ) return downloader.add_tensors(result, 0) @fox.in_result_filter def download_result(self, result: Any): - err, model = download_tensors(ref=result, ctx=fox.context, per_request_timeout=5.0) + err, model = download_tensors(ref=result, per_request_timeout=5.0) if err: self.logger.error(f"error filtering result {result}: {err}") return result diff --git a/nvflare/fox/examples/pt/pt_avg_mixed.py b/nvflare/fox/examples/pt/pt_avg_mixed.py index 5673248b4d..5e2ca84947 100644 --- a/nvflare/fox/examples/pt/pt_avg_mixed.py +++ b/nvflare/fox/examples/pt/pt_avg_mixed.py @@ -18,8 +18,7 @@ import torch from nvflare.fox import fox -from nvflare.fox.api.constants import EnvType -from nvflare.fox.api.ctx import Context +from nvflare.fox.api.constants import BackendType from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.utils import add as add_np from nvflare.fox.examples.np.algos.utils import div as div_np @@ -70,10 +69,9 @@ def _do_one_round(self, r, pt_model, np_model): aggr_result=aggr_result, ) - if fox.env_type == EnvType.SYSTEM: + if fox.backend_type == BackendType.SYSTEM: downloader = Downloader( num_receivers=grp.size, - ctx=fox.context, timeout=5.0, ) model_type = "ref" @@ -109,10 +107,8 @@ def _accept_train_result(self, result, aggr_result: _AggrResult): err, pt_result = download_tensors( ref=pt_result, per_request_timeout=5.0, - ctx=fox.context, tensors_received_cb=self._aggregate_tensors, aggr_result=aggr_result, - context=fox.context, ) if err: raise RuntimeError(f"failed to download model {pt_result}: {err}") @@ -120,10 +116,8 @@ def _accept_train_result(self, result, aggr_result: _AggrResult): err, np_result = download_arrays( ref=np_result, per_request_timeout=5.0, - ctx=fox.context, arrays_received_cb=self._aggregate_arrays, aggr_result=aggr_result, - context=fox.context, ) if err: raise RuntimeError(f"failed to download NP model file {np_result}: {err}") @@ -134,13 +128,13 @@ def _accept_train_result(self, result, aggr_result: _AggrResult): aggr_result.count += 1 return None - def _aggregate_tensors(self, td: dict[str, torch.Tensor], aggr_result: _AggrResult, context: Context): - self.logger.info(f"[{context}] aggregating received tensor: {td}") + def _aggregate_tensors(self, td: dict[str, torch.Tensor], aggr_result: _AggrResult): + self.logger.info(f"[{fox.call_info}] aggregating received tensor: {td}") with aggr_result.lock: add_pt(td, aggr_result.pt_total) - def _aggregate_arrays(self, td: dict[str, np.ndarray], aggr_result: _AggrResult, context: Context): - self.logger.info(f"[{context}] aggregating received array: {td}") + def _aggregate_arrays(self, td: dict[str, np.ndarray], aggr_result: _AggrResult): + self.logger.info(f"[{fox.call_info}] aggregating received array: {td}") with aggr_result.lock: add_np(td, aggr_result.np_total) @@ -160,12 +154,12 @@ def train(self, current_round, pt_model, np_model, model_type: str): self.logger.debug(f"[{fox.call_info}] training round {current_round}: {model_type=} {pt_model=} {np_model=}") if model_type == "ref": - err, pt_model = download_tensors(ref=pt_model, per_request_timeout=5.0, ctx=fox.context) + err, pt_model = download_tensors(ref=pt_model, per_request_timeout=5.0) if err: raise RuntimeError(f"failed to download PT model {pt_model}: {err}") self.logger.info(f"downloaded PT model {pt_model}") - err, np_model = download_arrays(ref=np_model, per_request_timeout=5.0, ctx=fox.context) + err, np_model = download_arrays(ref=np_model, per_request_timeout=5.0) if err: raise RuntimeError(f"failed to download NP model {np_model}: {err}") self.logger.info(f"downloaded NP model {np_model}") @@ -182,7 +176,6 @@ def train(self, current_round, pt_model, np_model, model_type: str): # stream it downloader = Downloader( num_receivers=1, - ctx=fox.context, timeout=5.0, ) pt_result = downloader.add_tensors(pt_result, 0) diff --git a/nvflare/fox/examples/pt/pt_avg_stream.py b/nvflare/fox/examples/pt/pt_avg_stream.py index ae58efffb3..89d46d6021 100644 --- a/nvflare/fox/examples/pt/pt_avg_stream.py +++ b/nvflare/fox/examples/pt/pt_avg_stream.py @@ -17,8 +17,7 @@ import torch from nvflare.fox import fox -from nvflare.fox.api.constants import EnvType -from nvflare.fox.api.ctx import Context +from nvflare.fox.api.constants import BackendType from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.pt.utils import parse_state_dict from nvflare.fox.sim.sim2 import Simulator @@ -64,10 +63,9 @@ def _do_one_round(self, r, current_model): aggr_result=aggr_result, ) - if fox.env_type == EnvType.SYSTEM: + if fox.backend_type == BackendType.SYSTEM: downloader = Downloader( num_receivers=grp.size, - ctx=fox.context, timeout=5.0, ) model_type = "ref" @@ -97,7 +95,6 @@ def _accept_train_result(self, result, aggr_result: _AggrResult): err, model = download_tensors( ref=model, per_request_timeout=5.0, - ctx=fox.context, tensors_received_cb=self._aggregate_tensors, aggr_result=aggr_result, ) @@ -136,12 +133,12 @@ def train(self, current_round, model1, model2, model_type: str): return 0 self.logger.debug(f"[{fox.call_info}] training round {current_round}: {model_type=} {model1=} {model2=}") if model_type == "ref": - err, model1 = download_tensors(ref=model1, per_request_timeout=5.0, ctx=fox.context) + err, model1 = download_tensors(ref=model1, per_request_timeout=5.0) if err: raise RuntimeError(f"failed to download model1 {model1}: {err}") self.logger.info(f"downloaded model1 {model1}") - err, model2 = download_tensors(ref=model2, per_request_timeout=5.0, ctx=fox.context) + err, model2 = download_tensors(ref=model2, per_request_timeout=5.0) if err: raise RuntimeError(f"failed to download model2 {model2}: {err}") self.logger.info(f"downloaded model2 {model2}") @@ -158,7 +155,6 @@ def train(self, current_round, model1, model2, model_type: str): # stream it downloader = Downloader( num_receivers=1, - ctx=fox.context, timeout=5.0, ) model_type = "ref" diff --git a/nvflare/fox/sim/backend.py b/nvflare/fox/sim/backend.py index e365b4b43f..53a5af565f 100644 --- a/nvflare/fox/sim/backend.py +++ b/nvflare/fox/sim/backend.py @@ -18,7 +18,8 @@ from nvflare.fox import fox from nvflare.fox.api.app import App from nvflare.fox.api.backend import Backend -from nvflare.fox.api.constants import OPTION_ARGS, CollabMethodArgName, CollabMethodOptionName, ContextKey +from nvflare.fox.api.call_opt import CallOpt +from nvflare.fox.api.constants import CollabMethodArgName, ContextKey from nvflare.fox.api.dec import adjust_kwargs from nvflare.fox.api.gcc import GroupCallContext from nvflare.fox.api.utils import check_call_args @@ -43,7 +44,7 @@ def __init__(self, target_obj_name: str, target_app: App, target_obj, abort_sign def _get_func(self, func_name): return self.target_app.find_collab_method(self.target_obj, func_name) - def call_target(self, target_name: str, func_name: str, *args, **kwargs): + def call_target(self, target_name: str, call_opt: CallOpt, func_name: str, *args, **kwargs): func = self._get_func(func_name) if not func: raise AttributeError(f"{target_name} does not have method '{func_name}' or it is not collab") @@ -51,12 +52,8 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): if not callable(func): raise AttributeError(f"the method '{func_name}' of {target_name} is not callable") - expect_result = kwargs.pop(CollabMethodOptionName.EXPECT_RESULT, True) - timeout = kwargs.pop(CollabMethodOptionName.TIMEOUT, 5.0) - - # other options don't apply to simulation - for k in OPTION_ARGS: - kwargs.pop(k, None) + expect_result = call_opt.expect_result + timeout = call_opt.timeout waiter = None if expect_result: @@ -137,9 +134,6 @@ def call_target_in_group(self, gcc: GroupCallContext, func_name: str, *args, **k # do not use the optional args - they are managed by the group self.logger.info(f"call_target_in_group: {gcc.target_name=}") - for k in OPTION_ARGS: - kwargs.pop(k, None) - target_name = gcc.target_name func = self._get_func(func_name) if not func: @@ -153,9 +147,7 @@ def call_target_in_group(self, gcc: GroupCallContext, func_name: str, *args, **k def _run_func_in_group(self, gcc: GroupCallContext, func_name, args, kwargs): try: target_name = gcc.target_name - kwargs[CollabMethodOptionName.EXPECT_RESULT] = gcc.expect_result - kwargs[CollabMethodOptionName.TIMEOUT] = gcc.timeout - result = self.call_target(target_name, func_name, *args, **kwargs) + result = self.call_target(target_name, gcc.call_opt, func_name, *args, **kwargs) gcc.set_result(result) except Exception as ex: gcc.set_exception(ex) diff --git a/nvflare/fox/sim/simulator.py b/nvflare/fox/sim/simulator.py index 314e1f5089..0e93e30b7c 100644 --- a/nvflare/fox/sim/simulator.py +++ b/nvflare/fox/sim/simulator.py @@ -18,7 +18,7 @@ from nvflare.apis.signal import Signal from nvflare.fox.api.app import App, ClientApp, ServerApp -from nvflare.fox.api.constants import EnvType +from nvflare.fox.api.constants import BackendType from nvflare.fox.api.dec import get_object_collab_interface from nvflare.fox.api.proxy import Proxy from nvflare.fox.api.run_server import run_server @@ -66,7 +66,7 @@ def _make_app(self, name, fqn): app.name = name app.fqn = fqn - app.env_type = EnvType.SIMULATION + app.backend_type = BackendType.SIMULATION return app def _prepare_proxies(self, for_app: App, server_app: App, client_apps: dict, backends: dict): @@ -97,7 +97,7 @@ def __init__( self.abort_signal = Signal() server_app.name = "server" server_app.fqn = server_app.name - server_app.env_type = EnvType.SIMULATION + server_app.backend_type = BackendType.SIMULATION self.server_app = server_app self.client_app = client_app self.thread_executor = ThreadPoolExecutor(max_workers=max_workers) diff --git a/nvflare/fox/sys/backend.py b/nvflare/fox/sys/backend.py index ced3729585..9aa92bbc7d 100644 --- a/nvflare/fox/sys/backend.py +++ b/nvflare/fox/sys/backend.py @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. from nvflare.fox.api.backend import Backend -from nvflare.fox.api.constants import CollabMethodArgName, CollabMethodOptionName +from nvflare.fox.api.call_opt import CallOpt +from nvflare.fox.api.constants import CollabMethodArgName from nvflare.fox.api.ctx import set_call_context from nvflare.fox.api.gcc import GroupCallContext from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey, ReturnCode @@ -34,13 +35,7 @@ def __init__(self, manager, engine, caller, cell, target_fqcn, abort_signal, thr self.target_fqcn = target_fqcn self.thread_executor = thread_executor - def call_target(self, target_name: str, func_name: str, *args, **kwargs): - kwargs.pop(CollabMethodOptionName.BLOCKING, True) - expect_result = kwargs.pop(CollabMethodOptionName.EXPECT_RESULT, True) - timeout = kwargs.pop(CollabMethodOptionName.TIMEOUT, 10.0) - optional = kwargs.pop(CollabMethodOptionName.OPTIONAL, False) - secure = kwargs.pop(CollabMethodOptionName.SECURE, False) - + def call_target(self, target_name: str, call_opt: CallOpt, func_name: str, *args, **kwargs): ctx = kwargs.pop(CollabMethodArgName.CONTEXT) set_call_context(ctx) @@ -53,7 +48,8 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): } request = new_cell_message({}, payload) - if expect_result: + timeout = call_opt.timeout + if call_opt.expect_result: self.logger.info(f"send_request from {self.cell.get_fqcn()} to {self.target_fqcn}: {payload=} {timeout=}") reply = self.cell.send_request( @@ -62,8 +58,8 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): topic=MSG_TOPIC, request=request, timeout=timeout, - secure=secure, - optional=optional, + secure=call_opt.secure, + optional=call_opt.optional, abort_signal=self.abort_signal, ) assert isinstance(reply, Message) @@ -91,8 +87,8 @@ def call_target(self, target_name: str, func_name: str, *args, **kwargs): topic=MSG_TOPIC, targets=self.target_fqcn, message=request, - secure=secure, - optional=optional, + secure=call_opt.secure, + optional=call_opt.optional, ) return None @@ -101,7 +97,7 @@ def call_target_in_group(self, gcc: GroupCallContext, func_name: str, *args, **k def _run_func(self, gcc: GroupCallContext, func_name: str, args, kwargs): try: - result = self.call_target(gcc.target_name, func_name, *args, **kwargs) + result = self.call_target(gcc.target_name, gcc.call_opt, func_name, *args, **kwargs) gcc.set_result(result) except Exception as ex: gcc.set_exception(ex) diff --git a/nvflare/fox/sys/controller.py b/nvflare/fox/sys/controller.py index 3d64d51660..009f15b5b6 100644 --- a/nvflare/fox/sys/controller.py +++ b/nvflare/fox/sys/controller.py @@ -22,7 +22,7 @@ from nvflare.apis.shareable import ReturnCode, Shareable from nvflare.apis.signal import Signal from nvflare.fox.api.app import ServerApp -from nvflare.fox.api.constants import EnvType +from nvflare.fox.api.constants import BackendType from nvflare.fox.api.proxy import Proxy from nvflare.fox.api.run_server import run_server from nvflare.fuel.f3.cellnet.fqcn import FQCN @@ -85,7 +85,7 @@ def start_controller(self, fl_ctx: FLContext): app = ServerApp(server_obj) app.name = "server" - app.env_type = EnvType.SYSTEM + app.backend_type = BackendType.SYSTEM err = self.process_config(app, fl_ctx) if err: diff --git a/nvflare/fox/sys/downloader.py b/nvflare/fox/sys/downloader.py index 9f46ce400b..bd7cfb4ee6 100644 --- a/nvflare/fox/sys/downloader.py +++ b/nvflare/fox/sys/downloader.py @@ -16,7 +16,7 @@ from nvflare.app_common.np.np_downloader import add_arrays from nvflare.app_common.np.np_downloader import download_arrays as pull_arrays -from nvflare.fox.api.ctx import Context +from nvflare.fox import fox from nvflare.fox.sys.backend import SysBackend from nvflare.fuel.f3.streaming.file_downloader import add_file from nvflare.fuel.f3.streaming.file_downloader import download_file as pull_file @@ -44,8 +44,8 @@ def __init__( self, num_receivers: int, timeout: float, - ctx: Context, ): + ctx = fox.context backend = ctx.backend if not isinstance(backend, SysBackend): raise ValueError(f"backend must be SysBackend but got {type(backend)}") @@ -82,7 +82,8 @@ def add_arrays(self, arrays: dict[str, np.ndarray], max_chunk_size: int = 0): return self._to_ref(ObjectType.ARRAYS, rid) -def download_file(ref: dict, per_request_timeout: float, ctx: Context): +def download_file(ref: dict, per_request_timeout: float): + ctx = fox.context backend = ctx.backend if not isinstance(backend, SysBackend): raise ValueError(f"backend must be SysBackend but got {type(backend)}") @@ -100,7 +101,8 @@ def download_file(ref: dict, per_request_timeout: float, ctx: Context): ) -def download_tensors(ref: dict, per_request_timeout: float, ctx: Context, tensors_received_cb=None, **cb_kwargs): +def download_tensors(ref: dict, per_request_timeout: float, tensors_received_cb=None, **cb_kwargs): + ctx = fox.context backend = ctx.backend if not isinstance(backend, SysBackend): raise ValueError(f"backend must be SysBackend but got {type(backend)}") @@ -120,7 +122,8 @@ def download_tensors(ref: dict, per_request_timeout: float, ctx: Context, tensor ) -def download_arrays(ref: dict, per_request_timeout: float, ctx: Context, arrays_received_cb=None, **cb_kwargs): +def download_arrays(ref: dict, per_request_timeout: float, arrays_received_cb=None, **cb_kwargs): + ctx = fox.context backend = ctx.backend if not isinstance(backend, SysBackend): raise ValueError(f"backend must be SysBackend but got {type(backend)}") diff --git a/nvflare/fox/sys/executor.py b/nvflare/fox/sys/executor.py index 3b7ed8382b..c5a329ec83 100644 --- a/nvflare/fox/sys/executor.py +++ b/nvflare/fox/sys/executor.py @@ -23,7 +23,7 @@ from nvflare.apis.shareable import Shareable, make_reply from nvflare.apis.signal import Signal from nvflare.fox.api.app import ClientApp -from nvflare.fox.api.constants import EnvType +from nvflare.fox.api.constants import BackendType from nvflare.fox.api.proxy import Proxy from nvflare.fuel.f3.cellnet.fqcn import FQCN @@ -74,7 +74,7 @@ def _handle_start_run(self, event_type: str, fl_ctx: FLContext): app = ClientApp(client_obj) app.name = client_name - app.env_type = EnvType.SYSTEM + app.backend_type = BackendType.SYSTEM self.client_app = app err = self.process_config(self.client_app, fl_ctx) From 7ac733e48a43030061c1bc03ed04d98c70789ab3 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Mon, 24 Nov 2025 17:09:08 -0500 Subject: [PATCH 070/102] refactor --- nvflare/fox/api/constants.py | 2 +- nvflare/fox/examples/np/algos/avg_stream.py | 5 ++--- nvflare/fox/examples/pt/pt_avg_mixed.py | 2 +- nvflare/fox/examples/pt/pt_avg_stream.py | 2 +- nvflare/fox/sys/controller.py | 2 +- nvflare/fox/sys/executor.py | 2 +- 6 files changed, 7 insertions(+), 8 deletions(-) diff --git a/nvflare/fox/api/constants.py b/nvflare/fox/api/constants.py index 0d8ef7b617..9d6c6d283a 100644 --- a/nvflare/fox/api/constants.py +++ b/nvflare/fox/api/constants.py @@ -29,4 +29,4 @@ class FilterDirection: class BackendType: SIMULATION = "simulation" - SYSTEM = "system" + FLARE = "flare" diff --git a/nvflare/fox/examples/np/algos/avg_stream.py b/nvflare/fox/examples/np/algos/avg_stream.py index 988d0e0867..dd0c942f3e 100644 --- a/nvflare/fox/examples/np/algos/avg_stream.py +++ b/nvflare/fox/examples/np/algos/avg_stream.py @@ -16,7 +16,6 @@ from nvflare.fox import fox from nvflare.fox.api.constants import BackendType -from nvflare.fox.api.dec import collab from nvflare.fox.examples.np.algos.utils import load_np_model, parse_array_def, save_np_model from nvflare.fox.sys.downloader import Downloader, download_file from nvflare.fuel.utils.log_utils import get_obj_logger @@ -57,7 +56,7 @@ def _do_one_round(self, r, current_model): # pretend the model is big file_name = None - if fox.backend_type == BackendType.SYSTEM: + if fox.backend_type == BackendType.FLARE: file_name = f"/tmp/np_{str(uuid.uuid4())}.npy" save_np_model(current_model, file_name) downloader = Downloader( @@ -109,7 +108,7 @@ def __init__(self, delta: float): self.delta = delta self.logger = get_obj_logger(self) - @collab + @fox.collab def train(self, current_round, weights, model_type: str): if fox.is_aborted: self.logger.debug("training aborted") diff --git a/nvflare/fox/examples/pt/pt_avg_mixed.py b/nvflare/fox/examples/pt/pt_avg_mixed.py index 5e2ca84947..95ca111334 100644 --- a/nvflare/fox/examples/pt/pt_avg_mixed.py +++ b/nvflare/fox/examples/pt/pt_avg_mixed.py @@ -69,7 +69,7 @@ def _do_one_round(self, r, pt_model, np_model): aggr_result=aggr_result, ) - if fox.backend_type == BackendType.SYSTEM: + if fox.backend_type == BackendType.FLARE: downloader = Downloader( num_receivers=grp.size, timeout=5.0, diff --git a/nvflare/fox/examples/pt/pt_avg_stream.py b/nvflare/fox/examples/pt/pt_avg_stream.py index 89d46d6021..560192ab7e 100644 --- a/nvflare/fox/examples/pt/pt_avg_stream.py +++ b/nvflare/fox/examples/pt/pt_avg_stream.py @@ -63,7 +63,7 @@ def _do_one_round(self, r, current_model): aggr_result=aggr_result, ) - if fox.backend_type == BackendType.SYSTEM: + if fox.backend_type == BackendType.FLARE: downloader = Downloader( num_receivers=grp.size, timeout=5.0, diff --git a/nvflare/fox/sys/controller.py b/nvflare/fox/sys/controller.py index 009f15b5b6..e62c25314b 100644 --- a/nvflare/fox/sys/controller.py +++ b/nvflare/fox/sys/controller.py @@ -85,7 +85,7 @@ def start_controller(self, fl_ctx: FLContext): app = ServerApp(server_obj) app.name = "server" - app.backend_type = BackendType.SYSTEM + app.backend_type = BackendType.FLARE err = self.process_config(app, fl_ctx) if err: diff --git a/nvflare/fox/sys/executor.py b/nvflare/fox/sys/executor.py index c5a329ec83..7ceb8a6bd5 100644 --- a/nvflare/fox/sys/executor.py +++ b/nvflare/fox/sys/executor.py @@ -74,7 +74,7 @@ def _handle_start_run(self, event_type: str, fl_ctx: FLContext): app = ClientApp(client_obj) app.name = client_name - app.backend_type = BackendType.SYSTEM + app.backend_type = BackendType.FLARE self.client_app = app err = self.process_config(self.client_app, fl_ctx) From 06165a008364e488b53a9a69b02eb3951914c509 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Tue, 25 Nov 2025 12:22:04 -0500 Subject: [PATCH 071/102] support results from event handlers --- nvflare/fox/api/app.py | 9 +- nvflare/fox/api/backend.py | 4 +- nvflare/fox/api/call_opt.py | 11 +- nvflare/fox/api/facade.py | 4 - nvflare/fox/api/gcc.py | 6 +- nvflare/fox/api/group.py | 237 +++-------------------- nvflare/fox/api/proxy.py | 15 +- nvflare/fox/api/proxy_list.py | 37 +++- nvflare/fox/examples/np/algos/client.py | 5 +- nvflare/fox/examples/np/algos/swarm.py | 17 +- nvflare/fox/examples/np/algos/widgets.py | 1 + nvflare/fox/sim/backend.py | 21 +- nvflare/fox/sys/backend.py | 4 +- 13 files changed, 107 insertions(+), 264 deletions(-) diff --git a/nvflare/fox/api/app.py b/nvflare/fox/api/app.py index e6b42f9dbb..619989febd 100644 --- a/nvflare/fox/api/app.py +++ b/nvflare/fox/api/app.py @@ -159,7 +159,6 @@ def _find_filter_chain(self, direction, chains: List[FilterChain], target_name: return None def apply_incoming_call_filters(self, target_name: str, func_name: str, func_kwargs, context: Context): - self.logger.debug(f"apply_incoming_call_filters on ctx {id(context)}") filter_chain = self._find_filter_chain( FilterDirection.INCOMING, self._incoming_call_filter_chains, target_name, func_name, context ) @@ -169,7 +168,6 @@ def apply_incoming_call_filters(self, target_name: str, func_name: str, func_kwa return func_kwargs def apply_outgoing_call_filters(self, target_name: str, func_name: str, func_kwargs, context: Context): - self.logger.debug(f"apply_outgoing_call_filters on ctx {id(context)}") filter_chain = self._find_filter_chain( FilterDirection.OUTGOING, self._outgoing_call_filter_chains, target_name, func_name, context ) @@ -179,7 +177,6 @@ def apply_outgoing_call_filters(self, target_name: str, func_name: str, func_kwa return func_kwargs def apply_incoming_result_filters(self, target_name: str, func_name: str, result, context: Context): - self.logger.debug(f"apply_incoming_result_filters on ctx {id(context)}") filter_chain = self._find_filter_chain( FilterDirection.INCOMING, self._incoming_result_filter_chains, target_name, func_name, context ) @@ -189,7 +186,6 @@ def apply_incoming_result_filters(self, target_name: str, func_name: str, result return result def apply_outgoing_result_filters(self, target_name: str, func_name: str, result, context: Context): - self.logger.debug(f"apply_outgoing_result_filters on ctx {id(context)}") filter_chain = self._find_filter_chain( FilterDirection.OUTGOING, self._outgoing_result_filter_chains, target_name, func_name, context ) @@ -328,6 +324,7 @@ def register_event_handler(self, event_type: str, handler, **handler_kwargs): handlers = [] self._event_handlers[event_type] = handlers handlers.append((handler, handler_kwargs)) + self.logger.debug(f"registered event handler {handler.__qualname__} for {event_type=}") def get_collab_interface(self): return self._collab_interface @@ -340,13 +337,15 @@ def get_target_object_collab_interface(self, target_name: str): @collab def fire_event(self, event_type: str, data, context: Context): + result = {} for e, handlers in self._event_handlers.items(): if e == event_type: for h, kwargs in handlers: kwargs = copy.copy(kwargs) kwargs.update({CollabMethodArgName.CONTEXT: context}) check_context_support(h, kwargs) - h(event_type, data, **kwargs) + result[h.__qualname__] = h(event_type, data, **kwargs) + return result def get_children(self): return [] diff --git a/nvflare/fox/api/backend.py b/nvflare/fox/api/backend.py index 8e279d95f3..40e64a2161 100644 --- a/nvflare/fox/api/backend.py +++ b/nvflare/fox/api/backend.py @@ -17,7 +17,7 @@ from nvflare.fuel.utils.log_utils import get_obj_logger from nvflare.security.logging import secure_format_traceback -from .call_opt import CallOpt +from .call_opt import CallOption from .gcc import GroupCallContext @@ -31,7 +31,7 @@ def __init__(self, abort_signal: Signal): self.logger = get_obj_logger(self) @abstractmethod - def call_target(self, target_name: str, call_opt: CallOpt, func_name: str, *args, **kwargs): + def call_target(self, target_name: str, call_opt: CallOption, func_name: str, *args, **kwargs): """ Call a target function with arguments and return a result. diff --git a/nvflare/fox/api/call_opt.py b/nvflare/fox/api/call_opt.py index f7e93d93c9..b19ee26cfc 100644 --- a/nvflare/fox/api/call_opt.py +++ b/nvflare/fox/api/call_opt.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -class CallOpt: +class CallOption: def __init__( self, @@ -21,6 +21,15 @@ def __init__( secure: bool = False, optional: bool = False, ): + """CallOption defines behavior of a collab call. + + Args: + expect_result: whether result is expected from the remote object. + blocking: whether rhe call is blocking. Only for group calls. + timeout: when expecting result, the max number of secs to wait for result. + secure: whether to use P2P secure messaging. + optional: whether the call is optional. + """ self.expect_result = expect_result self.blocking = blocking self.timeout = timeout diff --git a/nvflare/fox/api/facade.py b/nvflare/fox/api/facade.py index 936cac0547..d0a95066bc 100644 --- a/nvflare/fox/api/facade.py +++ b/nvflare/fox/api/facade.py @@ -71,10 +71,6 @@ def server(cls): @classproperty def clients(cls): - return cls.get_clients() - - @staticmethod - def get_clients(): ctx = get_call_context() return ProxyList(ctx.clients) diff --git a/nvflare/fox/api/gcc.py b/nvflare/fox/api/gcc.py index 4490e63886..f8d1b1555e 100644 --- a/nvflare/fox/api/gcc.py +++ b/nvflare/fox/api/gcc.py @@ -17,7 +17,7 @@ from nvflare.fuel.utils.log_utils import get_obj_logger -from .call_opt import CallOpt +from .call_opt import CallOption from .constants import CollabMethodArgName from .ctx import Context, set_call_context from .utils import check_context_support @@ -78,7 +78,7 @@ def __init__( self, app, target_name: str, - call_opt: CallOpt, + call_opt: CallOption, func_name: str, process_cb, cb_kwargs, @@ -133,8 +133,6 @@ def set_result(self, result): self.cb_kwargs[CollabMethodArgName.CONTEXT] = ctx check_context_support(self.process_cb, self.cb_kwargs) result = self.process_cb(result, **self.cb_kwargs) - else: - self.logger.info(f"{self.func_name} does not have process_cb!") # set back to original context set_call_context(self.context) diff --git a/nvflare/fox/api/group.py b/nvflare/fox/api/group.py index 0d7d65cea3..a00406f934 100644 --- a/nvflare/fox/api/group.py +++ b/nvflare/fox/api/group.py @@ -19,7 +19,7 @@ from nvflare.fuel.utils.log_utils import get_obj_logger from .app import App -from .call_opt import CallOpt +from .call_opt import CallOption from .constants import CollabMethodArgName from .ctx import Context from .gcc import GroupCallContext, ResultWaiter @@ -34,11 +34,7 @@ def __init__( app, abort_signal: Signal, proxies: List[Proxy], - blocking: bool = True, - expect_result: bool = True, - timeout: float = 5.0, - optional: bool = False, - secure: bool = False, + call_opt: CallOption = None, process_resp_cb=None, **cb_kwargs, ): @@ -48,11 +44,7 @@ def __init__( app: the calling app. abort_signal: signal to abort execution. proxies: proxies of the remote apps to be called. - blocking: whether to block until responses are received from all remote apps. - expect_result: whether to expect result from remote sites. - timeout: how long to wait for responses. - optional: whether the call is optional or not. - secure: whether the call is secure or not. + call_opt: call option that specifies call behavior process_resp_cb: callback function to be called to process responses from remote apps. **cb_kwargs: kwargs passed to process_resp_cb. """ @@ -62,13 +54,9 @@ def __init__( self._app = app self._abort_signal = abort_signal self._proxies = proxies - self._call_opt = CallOpt( - blocking=blocking, - expect_result=expect_result, - timeout=timeout, - optional=optional, - secure=secure, - ) + if not call_opt: + call_opt = CallOption() + self._call_opt = call_opt self._process_resp_cb = process_resp_cb self._cb_kwargs = cb_kwargs self._logger = get_obj_logger(self) @@ -94,7 +82,19 @@ def members(self): def __getattr__(self, func_name): """ - This method is called when Python cannot find an invoked method func_name of this class. + This method is called to invoke the specified collab function. + + If expect_result is False, then the call immediately returns None. + + If expect_result is True, a ResultQueue object is returned. Results from each site will be appended to + the queue when they become available. If a site does not return result before timeout, the site's result + is TimeoutError exception. Each item in the queue is a tuple of (site_name, result). + + The blocking flag is only meaningful when expect_result is True. If blocking is True, the call does not + return until results are received from all sites (or timed out). If blocking is False, the call immediately + returns. In both cases, the ResultQueue object is returned, and the application should iterate through it + to process site results. + """ def method(*args, **kwargs): @@ -173,208 +173,27 @@ def method(*args, **kwargs): def group( ctx: Context, proxies: List[Proxy], - blocking: bool = True, - expect_result: bool = True, - timeout: float = 5.0, - optional: bool = False, - secure: bool = False, + call_opt: CallOption = None, process_resp_cb=None, **cb_kwargs, ): """This is a convenience method for creating a group. Args: - ctx: - proxies: - blocking: - expect_result: - timeout: - optional: - secure: - process_resp_cb: - **cb_kwargs: - - Returns: - - """ - return Group( - ctx.app, - ctx.abort_signal, - proxies, - blocking, - expect_result, - timeout, - optional, - secure, - process_resp_cb, - **cb_kwargs, - ) - - -def all_clients( - ctx: Context, - blocking: bool = True, - expect_result: bool = True, - timeout: float = 5.0, - optional: bool = False, - secure: bool = False, - process_resp_cb=None, - **cb_kwargs, -): - """This is a convenience method for creating a group with all clients. - - Args: - ctx: - blocking: - expect_result: - timeout: - optional: - secure: - process_resp_cb: - **cb_kwargs: - - Returns: - - """ - return Group( - ctx.app, - ctx.abort_signal, - ctx.clients, - blocking, - expect_result, - timeout, - optional, - secure, - process_resp_cb, - **cb_kwargs, - ) - - -def all_other_clients( - ctx: Context, - blocking: bool = True, - expect_result: bool = True, - timeout: float = 5.0, - optional: bool = False, - secure: bool = False, - process_resp_cb=None, - **cb_kwargs, -): - """This is a convenience method for creating a group with all other clients (excluding myself). - - Args: - ctx: - blocking: - expect_result: - timeout: - optional: - secure: - process_resp_cb: - **cb_kwargs: + ctx: call context. + proxies: list of proxies. + call_opt: call option that defines call behavior. + process_resp_cb: callback to be called to process response from remote site. + **cb_kwargs: kwargs to be passed to the CB. - Returns: + Returns: a Group object. """ - candidates = ctx.clients - me = ctx.app.get_my_site() - if me in candidates: - candidates.remove(me) - - return Group( - ctx.app, - ctx.abort_signal, - candidates, - blocking, - expect_result, - timeout, - optional, - secure, - process_resp_cb, - **cb_kwargs, - ) - - -def all_children( - ctx: Context, - blocking: bool = True, - expect_result: bool = True, - timeout: float = 5.0, - optional: bool = False, - secure: bool = False, - process_resp_cb=None, - **cb_kwargs, -): - """This is a convenience method for creating a group with all my child clients. - - Args: - ctx: - blocking: - expect_result: - timeout: - optional: - secure: - process_resp_cb: - **cb_kwargs: - - Returns: - - """ - clients = ctx.app.get_children() - if not clients: - raise RuntimeError(f"app {ctx.app.name} has no child clients") - return Group( ctx.app, ctx.abort_signal, - clients, - blocking, - expect_result, - timeout, - optional, - secure, - process_resp_cb, - **cb_kwargs, - ) - - -def all_leaf_clients( - ctx: Context, - blocking: bool = True, - expect_result: bool = True, - timeout: float = 5.0, - optional: bool = False, - secure: bool = False, - process_resp_cb=None, - **cb_kwargs, -): - """This is a convenience method for creating a group with all leaf clients. - - Args: - ctx: - blocking: - expect_result: - timeout: - optional: - secure: - process_resp_cb: - **cb_kwargs: - - Returns: - - """ - clients = ctx.app.get_leaf_clients() - if not clients: - raise RuntimeError(f"app {ctx.app.name} has no leaf clients") - - return Group( - ctx.app, - ctx.abort_signal, - clients, - blocking, - expect_result, - timeout, - optional, - secure, + proxies, + call_opt, process_resp_cb, **cb_kwargs, ) diff --git a/nvflare/fox/api/proxy.py b/nvflare/fox/api/proxy.py index 46d17123d5..67c97393ad 100644 --- a/nvflare/fox/api/proxy.py +++ b/nvflare/fox/api/proxy.py @@ -16,7 +16,7 @@ from nvflare.fuel.utils.log_utils import get_obj_logger from .backend import Backend -from .call_opt import CallOpt +from .call_opt import CallOption from .constants import CollabMethodArgName from .utils import check_call_args @@ -32,7 +32,7 @@ def __init__( secure: bool = False, ): self.proxy = proxy - self.call_opt = CallOpt( + self.call_opt = CallOption( expect_result=expect_result, blocking=expect_result, timeout=timeout, @@ -117,7 +117,7 @@ def _find_interface(self, func_name): We first try to find the interface from the proxy itself. If not found, we try to find it from child proxies. """ - self.logger.debug(f"trying to find interface for {func_name}") + # self.logger.debug(f"trying to find interface for {func_name}") args = self.target_interface.get(func_name) if self.target_interface else None if args: return self, args @@ -131,7 +131,7 @@ def _find_interface(self, func_name): if not args: continue - self.logger.debug(f"found interface for func {func_name}: defined in child {n}") + # self.logger.debug(f"found interface for func {func_name}: defined in child {n}") if not the_proxy: the_name = n @@ -181,7 +181,7 @@ def adjust_func_args(self, func_name, args, kwargs): return p, func_itf, call_args, call_kwargs - def call_func(self, call_opt: CallOpt, func_name, args, kwargs): + def call_func(self, call_opt: CallOption, func_name, args, kwargs): """Call the specified function with call options. Args: @@ -197,13 +197,10 @@ def call_func(self, call_opt: CallOpt, func_name, args, kwargs): p, func_itf, call_args, call_kwargs = self.adjust_func_args(func_name, args, kwargs) with p.app.new_context(self.caller_name, self.name) as ctx: - self.logger.debug(f"[{ctx}] apply_outgoing_call_filters on {p.target_name} func {func_name}") - # apply outgoing call filters call_kwargs = self.app.apply_outgoing_call_filters(p.target_name, func_name, call_kwargs, ctx) check_call_args(func_name, func_itf, call_args, call_kwargs) - self.logger.debug(f"[{ctx}] calling target {p.target_name} func {func_name}") call_kwargs[CollabMethodArgName.CONTEXT] = ctx result = p.backend.call_target(p.target_name, call_opt, func_name, *call_args, **call_kwargs) if isinstance(result, Exception): @@ -224,6 +221,6 @@ def __getattr__(self, func_name): """ def method(*args, **kwargs): - return self.call_func(CallOpt(), func_name, args, kwargs) + return self.call_func(CallOption(), func_name, args, kwargs) return method diff --git a/nvflare/fox/api/proxy_list.py b/nvflare/fox/api/proxy_list.py index 63deb808bd..529192903e 100644 --- a/nvflare/fox/api/proxy_list.py +++ b/nvflare/fox/api/proxy_list.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from .call_opt import CallOption from .ctx import get_call_context from .group import group @@ -22,6 +23,16 @@ def __init__(self, proxies: list): self.extend(proxies) def __getattr__(self, func_name): + """This is called to invoke the specified func without specifying call option. + In this case, default call option will be used. + + Args: + func_name: + + Returns: + + """ + def method(*args, **kwargs): grp = group( ctx=get_call_context(), @@ -41,14 +52,30 @@ def __call__( process_resp_cb=None, **cb_kwargs, ): + """This is called to define the behavior (Call Option) of the group call. + + Args: + blocking: + expect_result: + timeout: + optional: + secure: + process_resp_cb: + **cb_kwargs: + + Returns: + + """ return group( ctx=get_call_context(), proxies=self, - blocking=blocking, - expect_result=expect_result, - timeout=timeout, - optional=optional, - secure=secure, + call_opt=CallOption( + blocking=blocking, + expect_result=expect_result, + timeout=timeout, + optional=optional, + secure=secure, + ), process_resp_cb=process_resp_cb, **cb_kwargs, ) diff --git a/nvflare/fox/examples/np/algos/client.py b/nvflare/fox/examples/np/algos/client.py index 975350bbd4..96eb9ae5d2 100644 --- a/nvflare/fox/examples/np/algos/client.py +++ b/nvflare/fox/examples/np/algos/client.py @@ -38,8 +38,9 @@ def train(self, current_round, weights): if fox.is_aborted: self.logger.debug("training aborted") return 0 - self.logger.info(f"[{fox.call_info}] EZ trained round {current_round=} {weights=}") - fox.server(expect_result=False).fire_event("metrics", {"round": current_round, "y": 10}) + self.logger.info(f"[{fox.call_info}] training round {current_round=} {weights=}") + result = fox.server(expect_result=True).fire_event("metrics", {"round": current_round, "y": 10}) + self.logger.info(f"[{fox.call_info}] got event result: {result}") # force timeout to test timeout handling # time.sleep(10) diff --git a/nvflare/fox/examples/np/algos/swarm.py b/nvflare/fox/examples/np/algos/swarm.py index 1a692b9a7c..04916db4de 100644 --- a/nvflare/fox/examples/np/algos/swarm.py +++ b/nvflare/fox/examples/np/algos/swarm.py @@ -11,12 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os import random import threading import traceback from nvflare.fox import fox -from nvflare.fox.examples.np.algos.utils import parse_array_def +from nvflare.fox.examples.np.algos.utils import parse_array_def, save_np_model from nvflare.fuel.utils.log_utils import get_obj_logger @@ -60,6 +61,7 @@ def __init__(self, delta: float): @fox.init def init(self): fox.register_event_handler("final_model", self._accept_final_model) + fox.register_event_handler("final_model", self._save_final_model) @fox.collab def train(self, weights, current_round): @@ -81,7 +83,9 @@ def swarm_learn(self, num_rounds, model, current_round): self.logger.info(f"[{fox.call_info}]: trained model {new_model=}") if current_round == num_rounds - 1: # all done - fox.clients(expect_result=False).fire_event("final_model", new_model) + result = fox.clients(expect_result=True).fire_event("final_model", new_model) + for n, v in result: + self.logger.info(f"[{fox.call_info}] final_model reply from {n}: {v}") self.logger.info("notify server all done!") try: fox.server(expect_result=False).all_done("OK") @@ -105,3 +109,12 @@ def _accept_final_model(self, event_type: str, model): # accept the final model # write model to disk self.logger.info(f"[{fox.call_info}]: received event '{event_type}' from {fox.caller}: {model}") + return "received" + + def _save_final_model(self, event_type: str, model): + # accept the final model + # write model to disk + file_name = os.path.join(fox.workspace.get_work_dir(), "final_model.npy") + save_np_model(model, file_name) + self.logger.info(f"[{fox.call_info}]: saved model {model} to {file_name}") + return "saved" diff --git a/nvflare/fox/examples/np/algos/widgets.py b/nvflare/fox/examples/np/algos/widgets.py index 79abe591dc..a6f19611f3 100644 --- a/nvflare/fox/examples/np/algos/widgets.py +++ b/nvflare/fox/examples/np/algos/widgets.py @@ -31,3 +31,4 @@ def init(self): def _accept_metric(self, event_type: str, data): self.logger.info(f"[{fox.callee}] received metrics event '{event_type}' from {fox.caller}: {data}") + return "OK" diff --git a/nvflare/fox/sim/backend.py b/nvflare/fox/sim/backend.py index 53a5af565f..4e3ec1110d 100644 --- a/nvflare/fox/sim/backend.py +++ b/nvflare/fox/sim/backend.py @@ -18,7 +18,7 @@ from nvflare.fox import fox from nvflare.fox.api.app import App from nvflare.fox.api.backend import Backend -from nvflare.fox.api.call_opt import CallOpt +from nvflare.fox.api.call_opt import CallOption from nvflare.fox.api.constants import CollabMethodArgName, ContextKey from nvflare.fox.api.dec import adjust_kwargs from nvflare.fox.api.gcc import GroupCallContext @@ -44,7 +44,7 @@ def __init__(self, target_obj_name: str, target_app: App, target_obj, abort_sign def _get_func(self, func_name): return self.target_app.find_collab_method(self.target_obj, func_name) - def call_target(self, target_name: str, call_opt: CallOpt, func_name: str, *args, **kwargs): + def call_target(self, target_name: str, call_opt: CallOption, func_name: str, *args, **kwargs): func = self._get_func(func_name) if not func: raise AttributeError(f"{target_name} does not have method '{func_name}' or it is not collab") @@ -83,15 +83,7 @@ def call_target(self, target_name: str, call_opt: CallOpt, func_name: str, *args def _preprocess(self, target_name, func_name, func, kwargs): caller_ctx = kwargs.pop(CollabMethodArgName.CONTEXT) my_ctx = self.target_app.new_context(caller_ctx.caller, caller_ctx.callee) - - self.logger.info( - f"established call_ctx {id(my_ctx)} for {target_name=} {func_name=}: fox_ctx={id(fox.context)}" - ) - kwargs = self.target_app.apply_incoming_call_filters(target_name, func_name, kwargs, my_ctx) - self.logger.info( - f"_preprocess: {id(my_ctx)} apply_incoming_call_filters: {my_ctx.get_prop(ContextKey.DIRECTION)}" - ) # make sure the final kwargs conforms to func interface obj_itf = self.target_app.get_target_object_collab_interface(self.target_obj_name) @@ -103,8 +95,6 @@ def _preprocess(self, target_name, func_name, func, kwargs): raise RuntimeError(f"cannot find interface for func '{func_name}' of object {self.target_obj_name}") check_call_args(func_name, func_itf, [], kwargs) - self.logger.debug(f"[{my_ctx}] received kwargs is good for '{func_name}': {kwargs}") - kwargs[CollabMethodArgName.CONTEXT] = my_ctx adjust_kwargs(func, kwargs) return my_ctx, kwargs @@ -117,10 +107,6 @@ def _run_func(self, waiter: _Waiter, target_name, func_name, func, args, kwargs) # apply result filter result = self.target_app.apply_outgoing_result_filters(target_name, func_name, result, ctx) - self.logger.info( - f"_run_func: {id(ctx)} apply_outgoing_result_filters: {ctx.get_prop(ContextKey.DIRECTION)}" - ) - if waiter: waiter.result = result except Exception as ex: @@ -131,9 +117,6 @@ def _run_func(self, waiter: _Waiter, target_name, func_name, func, args, kwargs) waiter.set() def call_target_in_group(self, gcc: GroupCallContext, func_name: str, *args, **kwargs): - # do not use the optional args - they are managed by the group - self.logger.info(f"call_target_in_group: {gcc.target_name=}") - target_name = gcc.target_name func = self._get_func(func_name) if not func: diff --git a/nvflare/fox/sys/backend.py b/nvflare/fox/sys/backend.py index 9aa92bbc7d..9946e9545f 100644 --- a/nvflare/fox/sys/backend.py +++ b/nvflare/fox/sys/backend.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from nvflare.fox.api.backend import Backend -from nvflare.fox.api.call_opt import CallOpt +from nvflare.fox.api.call_opt import CallOption from nvflare.fox.api.constants import CollabMethodArgName from nvflare.fox.api.ctx import set_call_context from nvflare.fox.api.gcc import GroupCallContext @@ -35,7 +35,7 @@ def __init__(self, manager, engine, caller, cell, target_fqcn, abort_signal, thr self.target_fqcn = target_fqcn self.thread_executor = thread_executor - def call_target(self, target_name: str, call_opt: CallOpt, func_name: str, *args, **kwargs): + def call_target(self, target_name: str, call_opt: CallOption, func_name: str, *args, **kwargs): ctx = kwargs.pop(CollabMethodArgName.CONTEXT) set_call_context(ctx) From 3b30b41639704a3590eeaed169f9be1068a8f66d Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Tue, 25 Nov 2025 14:48:51 -0500 Subject: [PATCH 072/102] add cyclic-avg recipe --- nvflare/fox/examples/np/cyclic_avg.py | 7 +- nvflare/fox/examples/np/recipe_cyclic.py | 5 -- nvflare/fox/examples/np/recipe_cyclic_avg.py | 76 +++++++++++++++++++ .../fox/examples/np/recipe_fed_avg_intime.py | 5 -- nvflare/fox/examples/np/recipe_fed_avg_seq.py | 5 -- .../fox/examples/np/recipe_fed_avg_stream.py | 5 -- nvflare/fox/examples/np/recipe_swarm.py | 5 -- nvflare/fox/examples/pt/recipe_pt_avg.py | 5 -- .../fox/examples/pt/recipe_pt_avg_filter.py | 5 -- .../fox/examples/pt/recipe_pt_avg_filter2.py | 4 - .../fox/examples/pt/recipe_pt_avg_mixed.py | 5 -- .../fox/examples/pt/recipe_pt_avg_stream.py | 5 -- nvflare/fox/examples/pt/recipe_pt_np.py | 5 -- nvflare/fox/sys/ws.py | 2 +- 14 files changed, 82 insertions(+), 57 deletions(-) create mode 100644 nvflare/fox/examples/np/recipe_cyclic_avg.py diff --git a/nvflare/fox/examples/np/cyclic_avg.py b/nvflare/fox/examples/np/cyclic_avg.py index 3b795b31f0..2adef9e32a 100644 --- a/nvflare/fox/examples/np/cyclic_avg.py +++ b/nvflare/fox/examples/np/cyclic_avg.py @@ -41,10 +41,13 @@ def run(self): self.logger.info("running cyclic ...") ctl = NPCyclic(self.initial_model, num_rounds=self.cyclic_rounds) result = ctl.execute() + self.logger.info(f"final cyclic model: {result}") self.logger.info("running fed-avg ...") ctl = NPFedAvgParallel(initial_model=result, num_rounds=self.avg_rounds) - return ctl.execute() + result = ctl.execute() + self.logger.info(f"final model: {result}") + return result def main(): @@ -58,7 +61,7 @@ def main(): experiment_name=exp_name, server_app=server_app, client_app=ClientApp(NPTrainer(delta=1.0)), - num_clients=2, + num_clients=3, ) final_result = simulator.run() diff --git a/nvflare/fox/examples/np/recipe_cyclic.py b/nvflare/fox/examples/np/recipe_cyclic.py index 22db6cceca..e6f91ce01f 100644 --- a/nvflare/fox/examples/np/recipe_cyclic.py +++ b/nvflare/fox/examples/np/recipe_cyclic.py @@ -11,9 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import logging - -from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.strategies.cyclic import NPCyclic from nvflare.fox.sys.recipe import FoxRecipe @@ -22,8 +19,6 @@ def main(): - simple_logging(logging.DEBUG) - recipe = FoxRecipe( job_name="fox_cyclic", server=NPCyclic(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), diff --git a/nvflare/fox/examples/np/recipe_cyclic_avg.py b/nvflare/fox/examples/np/recipe_cyclic_avg.py new file mode 100644 index 0000000000..4eb8b65658 --- /dev/null +++ b/nvflare/fox/examples/np/recipe_cyclic_avg.py @@ -0,0 +1,76 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +from nvflare.fox import fox +from nvflare.fox.examples.np.algos.client import NPTrainer +from nvflare.fox.examples.np.algos.strategies.avg_para import NPFedAvgParallel +from nvflare.fox.examples.np.algos.strategies.cyclic import NPCyclic +from nvflare.fox.examples.np.algos.utils import save_np_model +from nvflare.fox.sys.recipe import FoxRecipe +from nvflare.fuel.utils.log_utils import get_obj_logger + +JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" + + +class Controller: + + def __init__( + self, + initial_model, + cyclic_rounds, + avg_rounds, + ): + self.initial_model = initial_model + self.cyclic_rounds = cyclic_rounds + self.avg_rounds = avg_rounds + self.logger = get_obj_logger(self) + + @fox.algo + def run(self): + self.logger.info("running cyclic ...") + ctl = NPCyclic(self.initial_model, num_rounds=self.cyclic_rounds) + result = ctl.execute() + + file_name = os.path.join(fox.workspace.get_work_dir(), "cyclic_model.npy") + save_np_model(result, file_name) + self.logger.info(f"[{fox.call_info}]: saved cyclic model {result} to {file_name}") + + self.logger.info("running fed-avg ...") + ctl = NPFedAvgParallel(initial_model=result, num_rounds=self.avg_rounds) + return ctl.execute() + + @fox.final + def save_result(self): + final_result = fox.get_input() + file_name = os.path.join(fox.workspace.get_work_dir(), "final_model.npy") + save_np_model(final_result, file_name) + self.logger.info(f"[{fox.call_info}]: saved final model {final_result} to {file_name}") + + +def main(): + recipe = FoxRecipe( + job_name="fox_cyclic_avg", + server=Controller( + initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], + cyclic_rounds=2, + avg_rounds=3, + ), + client=NPTrainer(delta=1.0), + ) + recipe.export(JOB_ROOT_DIR) + + +if __name__ == "__main__": + main() diff --git a/nvflare/fox/examples/np/recipe_fed_avg_intime.py b/nvflare/fox/examples/np/recipe_fed_avg_intime.py index c0a8cf5f0d..c617de7fba 100644 --- a/nvflare/fox/examples/np/recipe_fed_avg_intime.py +++ b/nvflare/fox/examples/np/recipe_fed_avg_intime.py @@ -11,9 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import logging - -from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.filters import AddNoiseToModel, Print from nvflare.fox.examples.np.algos.strategies.avg_intime import NPFedAvgInTime @@ -24,8 +21,6 @@ def main(): - simple_logging(logging.DEBUG) - recipe = FoxRecipe( job_name="fox_fedavg_intime", server=NPFedAvgInTime(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), diff --git a/nvflare/fox/examples/np/recipe_fed_avg_seq.py b/nvflare/fox/examples/np/recipe_fed_avg_seq.py index 9856925423..a1a360fb24 100644 --- a/nvflare/fox/examples/np/recipe_fed_avg_seq.py +++ b/nvflare/fox/examples/np/recipe_fed_avg_seq.py @@ -11,9 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import logging - -from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.strategies.avg_seq import NPFedAvgSequential from nvflare.fox.examples.np.algos.widgets import MetricReceiver @@ -23,8 +20,6 @@ def main(): - simple_logging(logging.DEBUG) - recipe = FoxRecipe( job_name="fox_fedavg_seq", server=NPFedAvgSequential(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), diff --git a/nvflare/fox/examples/np/recipe_fed_avg_stream.py b/nvflare/fox/examples/np/recipe_fed_avg_stream.py index 4cba5663c0..ee557541d6 100644 --- a/nvflare/fox/examples/np/recipe_fed_avg_stream.py +++ b/nvflare/fox/examples/np/recipe_fed_avg_stream.py @@ -11,9 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import logging - -from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.avg_stream import NPFedAvgStream, NPTrainer from nvflare.fox.sys.recipe import FoxRecipe @@ -21,8 +18,6 @@ def main(): - simple_logging(logging.DEBUG) - recipe = FoxRecipe( job_name="fox_fedavg_stream", server=NPFedAvgStream(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), diff --git a/nvflare/fox/examples/np/recipe_swarm.py b/nvflare/fox/examples/np/recipe_swarm.py index 253e1677da..3bcd59034a 100644 --- a/nvflare/fox/examples/np/recipe_swarm.py +++ b/nvflare/fox/examples/np/recipe_swarm.py @@ -11,9 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import logging - -from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.swarm import NPSwarm, NPSwarmClient from nvflare.fox.sys.recipe import FoxRecipe @@ -21,8 +18,6 @@ def main(): - simple_logging(logging.DEBUG) - recipe = FoxRecipe( job_name="fox_swarm", server=NPSwarm(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=5), diff --git a/nvflare/fox/examples/pt/recipe_pt_avg.py b/nvflare/fox/examples/pt/recipe_pt_avg.py index b93c166200..d64292ef52 100644 --- a/nvflare/fox/examples/pt/recipe_pt_avg.py +++ b/nvflare/fox/examples/pt/recipe_pt_avg.py @@ -11,10 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import logging - from nvflare.app_opt.pt.decomposers import TensorDecomposer -from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.pt.pt_avg_filter import PTFedAvg, PTTrainer from nvflare.fox.sys.recipe import FoxRecipe @@ -22,8 +19,6 @@ def main(): - simple_logging(logging.DEBUG) - recipe = FoxRecipe( job_name="fox_pt_fedavg", server=PTFedAvg( diff --git a/nvflare/fox/examples/pt/recipe_pt_avg_filter.py b/nvflare/fox/examples/pt/recipe_pt_avg_filter.py index 0e9f1c68b6..c52bacb59e 100644 --- a/nvflare/fox/examples/pt/recipe_pt_avg_filter.py +++ b/nvflare/fox/examples/pt/recipe_pt_avg_filter.py @@ -11,9 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import logging - -from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.pt.filters import ( IncomingModelCallFilter, IncomingModelResultFilter, @@ -27,8 +24,6 @@ def main(): - simple_logging(logging.DEBUG) - recipe = FoxRecipe( job_name="fox_pt_fedavg_filter", server=PTFedAvg( diff --git a/nvflare/fox/examples/pt/recipe_pt_avg_filter2.py b/nvflare/fox/examples/pt/recipe_pt_avg_filter2.py index 0fb2e8a3f9..2764409d24 100644 --- a/nvflare/fox/examples/pt/recipe_pt_avg_filter2.py +++ b/nvflare/fox/examples/pt/recipe_pt_avg_filter2.py @@ -11,9 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import logging - -from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.pt.filters2 import ModelFilter from nvflare.fox.examples.pt.pt_avg_filter import PTFedAvg, PTTrainer from nvflare.fox.sys.recipe import FoxRecipe @@ -22,7 +19,6 @@ def main(): - simple_logging(logging.DEBUG) recipe = FoxRecipe( job_name="fox_pt_fedavg_filter2", diff --git a/nvflare/fox/examples/pt/recipe_pt_avg_mixed.py b/nvflare/fox/examples/pt/recipe_pt_avg_mixed.py index ad1317ec79..147224373c 100644 --- a/nvflare/fox/examples/pt/recipe_pt_avg_mixed.py +++ b/nvflare/fox/examples/pt/recipe_pt_avg_mixed.py @@ -11,9 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import logging - -from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.pt.pt_avg_mixed import PTFedAvgMixed, PTTrainer from nvflare.fox.sys.recipe import FoxRecipe @@ -21,8 +18,6 @@ def main(): - simple_logging(logging.DEBUG) - init_model = { "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], "y": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], diff --git a/nvflare/fox/examples/pt/recipe_pt_avg_stream.py b/nvflare/fox/examples/pt/recipe_pt_avg_stream.py index f42171d6f5..9699cf04c3 100644 --- a/nvflare/fox/examples/pt/recipe_pt_avg_stream.py +++ b/nvflare/fox/examples/pt/recipe_pt_avg_stream.py @@ -11,9 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import logging - -from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.pt.pt_avg_stream import PTFedAvgStream, PTTrainer from nvflare.fox.sys.recipe import FoxRecipe @@ -21,8 +18,6 @@ def main(): - simple_logging(logging.DEBUG) - recipe = FoxRecipe( job_name="fox_pt_fedavg_stream", server=PTFedAvgStream( diff --git a/nvflare/fox/examples/pt/recipe_pt_np.py b/nvflare/fox/examples/pt/recipe_pt_np.py index 7aa4f1b5b8..40ad249872 100644 --- a/nvflare/fox/examples/pt/recipe_pt_np.py +++ b/nvflare/fox/examples/pt/recipe_pt_np.py @@ -11,11 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import logging - from nvflare.app_common.decomposers.numpy_decomposers import NumpyArrayDecomposer from nvflare.app_opt.pt.decomposers import TensorDecomposer -from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.pt.pt_np import PTFedAvgMixed, PTTrainer from nvflare.fox.sys.recipe import FoxRecipe @@ -23,8 +20,6 @@ def main(): - simple_logging(logging.DEBUG) - init_model = { "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], "y": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], diff --git a/nvflare/fox/sys/ws.py b/nvflare/fox/sys/ws.py index cea2ffd89e..6c94ac1c90 100644 --- a/nvflare/fox/sys/ws.py +++ b/nvflare/fox/sys/ws.py @@ -29,7 +29,7 @@ def get_root_dir(self) -> str: return self.flare_ws.get_root_dir() def get_work_dir(self) -> str: - return self.flare_ws.get_app_dir(self.job_id) + return self.flare_ws.get_run_dir(self.job_id) def get_experiment_dir(self) -> str: return self.get_work_dir() From 435ec03644c04361efd20c9ac9c21c9019caca4f Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Tue, 25 Nov 2025 15:05:38 -0500 Subject: [PATCH 073/102] clean up --- nvflare/fox/sys/backend.py | 6 +++--- nvflare/fox/sys/controller.py | 10 +++++----- nvflare/fox/sys/downloader.py | 10 +++++----- nvflare/fox/sys/executor.py | 10 +++++----- nvflare/fox/sys/utils.py | 12 ++++-------- nvflare/fox/sys/ws.py | 6 +++--- 6 files changed, 25 insertions(+), 29 deletions(-) diff --git a/nvflare/fox/sys/backend.py b/nvflare/fox/sys/backend.py index 9946e9545f..01092e21da 100644 --- a/nvflare/fox/sys/backend.py +++ b/nvflare/fox/sys/backend.py @@ -24,7 +24,7 @@ from .constants import MSG_CHANNEL, MSG_TOPIC, CallReplyKey, ObjectCallKey -class SysBackend(Backend): +class FlareBackend(Backend): def __init__(self, manager, engine, caller, cell, target_fqcn, abort_signal, thread_executor): Backend.__init__(self, abort_signal) @@ -50,7 +50,7 @@ def call_target(self, target_name: str, call_opt: CallOption, func_name: str, *a timeout = call_opt.timeout if call_opt.expect_result: - self.logger.info(f"send_request from {self.cell.get_fqcn()} to {self.target_fqcn}: {payload=} {timeout=}") + self.logger.info(f"send_request from {self.cell.get_fqcn()} to {self.target_fqcn}: {func_name=} {call_opt}") reply = self.cell.send_request( channel=MSG_CHANNEL, @@ -77,7 +77,7 @@ def call_target(self, target_name: str, call_opt: CallOption, func_name: str, *a return RuntimeError(f"function {func_name} failed: {error}") result = reply.payload.get(CallReplyKey.RESULT) - self.logger.info(f"got result from {self.target_fqcn}: {result}") + self.logger.info(f"got result from {self.target_fqcn} {func_name=}") return result else: # fire and forget diff --git a/nvflare/fox/sys/controller.py b/nvflare/fox/sys/controller.py index e62c25314b..11c49dcd5a 100644 --- a/nvflare/fox/sys/controller.py +++ b/nvflare/fox/sys/controller.py @@ -28,10 +28,10 @@ from nvflare.fuel.f3.cellnet.fqcn import FQCN from .adaptor import FoxAdaptor -from .backend import SysBackend +from .backend import FlareBackend from .constants import SYNC_TASK_NAME, SyncKey from .utils import prepare_for_remote_call -from .ws import SysWorkspace +from .ws import FlareWorkspace class _ClientInfo: @@ -94,7 +94,7 @@ def start_controller(self, fl_ctx: FLContext): self.server_app = app def _prepare_client_backend(self, job_id, client: ClientSite, abort_signal: Signal, fl_ctx: FLContext): - return SysBackend( + return FlareBackend( manager=self, engine=fl_ctx.get_engine(), caller=self.server_app.name, @@ -105,7 +105,7 @@ def _prepare_client_backend(self, job_id, client: ClientSite, abort_signal: Sign ) def _prepare_server_backend(self, job_id: str, abort_signal: Signal, fl_ctx: FLContext): - return SysBackend( + return FlareBackend( manager=self, engine=fl_ctx.get_engine(), caller=self.server_app.name, @@ -232,7 +232,7 @@ def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): assert isinstance(info, _ClientInfo) client_proxies.append(self._prepare_client_proxy(job_id, c, info.collab_interface, abort_signal, fl_ctx)) - ws = SysWorkspace(fl_ctx) + ws = FlareWorkspace(fl_ctx) self.server_app.setup(ws, server_proxy, client_proxies, abort_signal) run_server(self.server_app, self.logger) diff --git a/nvflare/fox/sys/downloader.py b/nvflare/fox/sys/downloader.py index bd7cfb4ee6..a1125ebdbb 100644 --- a/nvflare/fox/sys/downloader.py +++ b/nvflare/fox/sys/downloader.py @@ -17,7 +17,7 @@ from nvflare.app_common.np.np_downloader import add_arrays from nvflare.app_common.np.np_downloader import download_arrays as pull_arrays from nvflare.fox import fox -from nvflare.fox.sys.backend import SysBackend +from nvflare.fox.sys.backend import FlareBackend from nvflare.fuel.f3.streaming.file_downloader import add_file from nvflare.fuel.f3.streaming.file_downloader import download_file as pull_file from nvflare.fuel.f3.streaming.obj_downloader import ObjectDownloader @@ -47,7 +47,7 @@ def __init__( ): ctx = fox.context backend = ctx.backend - if not isinstance(backend, SysBackend): + if not isinstance(backend, FlareBackend): raise ValueError(f"backend must be SysBackend but got {type(backend)}") super().__init__( @@ -85,7 +85,7 @@ def add_arrays(self, arrays: dict[str, np.ndarray], max_chunk_size: int = 0): def download_file(ref: dict, per_request_timeout: float): ctx = fox.context backend = ctx.backend - if not isinstance(backend, SysBackend): + if not isinstance(backend, FlareBackend): raise ValueError(f"backend must be SysBackend but got {type(backend)}") obj_type = ref.get(DownloadRefKey.OBJECT_TYPE) @@ -104,7 +104,7 @@ def download_file(ref: dict, per_request_timeout: float): def download_tensors(ref: dict, per_request_timeout: float, tensors_received_cb=None, **cb_kwargs): ctx = fox.context backend = ctx.backend - if not isinstance(backend, SysBackend): + if not isinstance(backend, FlareBackend): raise ValueError(f"backend must be SysBackend but got {type(backend)}") obj_type = ref.get(DownloadRefKey.OBJECT_TYPE) @@ -125,7 +125,7 @@ def download_tensors(ref: dict, per_request_timeout: float, tensors_received_cb= def download_arrays(ref: dict, per_request_timeout: float, arrays_received_cb=None, **cb_kwargs): ctx = fox.context backend = ctx.backend - if not isinstance(backend, SysBackend): + if not isinstance(backend, FlareBackend): raise ValueError(f"backend must be SysBackend but got {type(backend)}") obj_type = ref.get(DownloadRefKey.OBJECT_TYPE) diff --git a/nvflare/fox/sys/executor.py b/nvflare/fox/sys/executor.py index 7ceb8a6bd5..36f8c0b2f5 100644 --- a/nvflare/fox/sys/executor.py +++ b/nvflare/fox/sys/executor.py @@ -28,10 +28,10 @@ from nvflare.fuel.f3.cellnet.fqcn import FQCN from .adaptor import FoxAdaptor -from .backend import SysBackend +from .backend import FlareBackend from .constants import SYNC_TASK_NAME, SyncKey from .utils import prepare_for_remote_call -from .ws import SysWorkspace +from .ws import FlareWorkspace class FoxExecutor(Executor, FoxAdaptor): @@ -89,7 +89,7 @@ def _handle_end_run(self, event_type: str, fl_ctx: FLContext): def _prepare_server_proxy(self, job_id, cell, collab_interface: dict, abort_signal, fl_ctx: FLContext): server_name = "server" - backend = SysBackend( + backend = FlareBackend( manager=self, engine=fl_ctx.get_engine(), caller=self.client_app.name, @@ -121,7 +121,7 @@ def _prepare_server_proxy(self, job_id, cell, collab_interface: dict, abort_sign return proxy def _prepare_client_proxy(self, job_id, cell, client: Client, abort_signal, collab_interface, fl_ctx: FLContext): - backend = SysBackend( + backend = FlareBackend( manager=self, engine=fl_ctx.get_engine(), caller=self.client_app.name, @@ -181,7 +181,7 @@ def execute(self, task_name: str, shareable: Shareable, fl_ctx: FLContext, abort p = self._prepare_client_proxy(job_id, cell, c, abort_signal, client_collab_interface, fl_ctx) client_proxies.append(p) - ws = SysWorkspace(fl_ctx) + ws = FlareWorkspace(fl_ctx) self.client_app.setup(ws, server_proxy, client_proxies, abort_signal) self.client_ctx = self.client_app.new_context(self.client_app.name, self.client_app.name) diff --git a/nvflare/fox/sys/utils.py b/nvflare/fox/sys/utils.py index 926a7b201d..ecd06fe20e 100644 --- a/nvflare/fox/sys/utils.py +++ b/nvflare/fox/sys/utils.py @@ -58,7 +58,7 @@ def _preprocess(app: App, caller, target_obj_name, target_name, func_name, func, def _call_app_method(request: Message, app: App, logger) -> Message: - logger.info("got a remote call") + logger.debug("got a remote call") payload = request.payload assert isinstance(payload, dict) @@ -84,7 +84,6 @@ def _call_app_method(request: Message, app: App, logger) -> Message: return _error_reply(f"bad method args: should be list/tuple but got {type(method_args)}", logger) method_kwargs = payload.get(ObjectCallKey.KWARGS) - logger.info(f"received kwargs for method {method_name}: {method_kwargs}") if not method_kwargs: method_kwargs = {} elif not isinstance(method_kwargs, dict): @@ -97,10 +96,10 @@ def _call_app_method(request: Message, app: App, logger) -> Message: if obj_name: target_objs = app.get_collab_objects() target_obj = target_objs.get(obj_name) - logger.info(f"calling target obj: {app.name}.{obj_name}") + logger.debug(f"calling target obj: {app.name}.{obj_name}") else: target_obj = app - logger.info(f"calling target app: {app.name}") + logger.debug(f"calling target app: {app.name}") if not target_obj: return _error_reply(f"no object named '{target_name}'", logger) @@ -109,18 +108,15 @@ def _call_app_method(request: Message, app: App, logger) -> Message: if not m: return _error_reply(f"no method named '{method_name}' or it is not collab", logger) else: - logger.info(f"found method for {method_name}") + logger.debug(f"found method for {method_name}") # invoke this method try: ctx, method_kwargs = _preprocess(app, caller, obj_name, target_name, method_name, m, method_args, method_kwargs) - # logger.info(f"calling method {method_name}: {caller=}: {method_args=} {method_kwargs=}") result = m(*method_args, **method_kwargs) - # logger.info(f"result from method {method_name}: {result}") # apply result filters result = app.apply_outgoing_result_filters(target_name, method_name, result, ctx) - # logger.info(f"result after filtering: {result}") return new_cell_message( headers={MessageHeaderKey.RETURN_CODE: ReturnCode.OK}, payload={CallReplyKey.RESULT: result} diff --git a/nvflare/fox/sys/ws.py b/nvflare/fox/sys/ws.py index 6c94ac1c90..4a7f0c2499 100644 --- a/nvflare/fox/sys/ws.py +++ b/nvflare/fox/sys/ws.py @@ -12,16 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. from nvflare.apis.fl_context import FLContext -from nvflare.apis.workspace import Workspace as FlareWorkspace +from nvflare.apis.workspace import Workspace as NVFWorkspace from nvflare.fox.api.workspace import Workspace -class SysWorkspace(Workspace): +class FlareWorkspace(Workspace): def __init__(self, fl_ctx: FLContext): super().__init__() ws_obj = fl_ctx.get_workspace() - assert isinstance(ws_obj, FlareWorkspace) + assert isinstance(ws_obj, NVFWorkspace) self.flare_ws = ws_obj self.job_id = fl_ctx.get_job_id() From dd34313aaf4fbbebc9550965a033a60ac360c581 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Wed, 26 Nov 2025 12:23:42 -0500 Subject: [PATCH 074/102] refactor --- nvflare/fox/api/facade.py | 19 ++++ nvflare/fox/api/workspace.py | 2 +- .../examples/np/algos/strategies/cyclic.py | 20 ++++- nvflare/fox/examples/np/algos/utils.py | 4 + nvflare/fox/examples/np/cyclic.py | 5 +- nvflare/fox/examples/np/cyclic_avg.py | 7 +- .../np/{cyclic2.py => cyclic_file.py} | 6 +- .../np/{fed_avg_h2.py => cyclic_runner.py} | 23 +++-- nvflare/fox/examples/np/fed_avg_h.py | 14 ++- nvflare/fox/examples/np/fed_avg_intime.py | 31 +++---- nvflare/fox/examples/np/fed_avg_intime2.py | 49 ----------- nvflare/fox/examples/np/fed_avg_para.py | 11 +-- nvflare/fox/examples/np/fed_avg_seq.py | 32 ++----- nvflare/fox/examples/np/fed_avg_stream.py | 11 +-- nvflare/fox/examples/np/swarm.py | 8 +- nvflare/fox/examples/pt/pt_avg_filter.py | 2 +- nvflare/fox/examples/pt/pt_avg_mixed.py | 2 +- nvflare/fox/examples/pt/pt_avg_stream.py | 2 +- nvflare/fox/examples/pt/pt_np.py | 2 +- nvflare/fox/sim/sim2.py | 88 ------------------- nvflare/fox/sim/simulator.py | 84 +++++++++++++++++- 21 files changed, 180 insertions(+), 242 deletions(-) rename nvflare/fox/examples/np/{cyclic2.py => cyclic_file.py} (86%) rename nvflare/fox/examples/np/{fed_avg_h2.py => cyclic_runner.py} (56%) delete mode 100644 nvflare/fox/examples/np/fed_avg_intime2.py delete mode 100644 nvflare/fox/sim/sim2.py diff --git a/nvflare/fox/api/facade.py b/nvflare/fox/api/facade.py index d0a95066bc..1dea858a2b 100644 --- a/nvflare/fox/api/facade.py +++ b/nvflare/fox/api/facade.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import copy + from .constants import ContextKey from .ctx import get_call_context from .dec import algo as dec_algo @@ -104,6 +106,23 @@ def leaf_clients(cls): raise RuntimeError(f"app {ctx.app.name} has no leaf clients") return ProxyList(candidates) + @classmethod + def get_clients(cls, names: list[str]): + ctx = get_call_context() + candidates = ctx.clients + result = [] + for n in names: + p = None + for c in candidates: + if c.name == n: + p = c + break + if not p: + # no proxy for this name + raise RuntimeError(f"app {ctx.app.name} has no client '{n}'") + result.append(p) + return ProxyList(result) + @classproperty def backend_type(cls): ctx = get_call_context() diff --git a/nvflare/fox/api/workspace.py b/nvflare/fox/api/workspace.py index 21973d4ef6..405bd20fc0 100644 --- a/nvflare/fox/api/workspace.py +++ b/nvflare/fox/api/workspace.py @@ -33,7 +33,7 @@ def get_root_dir(self) -> str: def get_work_dir(self) -> str: pass - def get_subdir(self, name: str, create: bool = True) -> str: + def get_resource_dir(self, name: str, create: bool = True) -> str: resource_dir = self.resource_dirs.get(name) if resource_dir: return resource_dir diff --git a/nvflare/fox/examples/np/algos/strategies/cyclic.py b/nvflare/fox/examples/np/algos/strategies/cyclic.py index b11da72c4d..7f7458eabc 100644 --- a/nvflare/fox/examples/np/algos/strategies/cyclic.py +++ b/nvflare/fox/examples/np/algos/strategies/cyclic.py @@ -11,10 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os import random from nvflare.fox import fox -from nvflare.fox.examples.np.algos.utils import parse_array_def +from nvflare.fox.examples.np.algos.utils import load_np_model, parse_array_def, save_np_model from nvflare.fuel.utils.log_utils import get_obj_logger @@ -27,6 +28,16 @@ def __init__(self, initial_model, num_rounds=2): self.final_model = None self.logger = get_obj_logger(self) + @fox.init + def check_initial_model(self): + if isinstance(self._initial_model, str): + # this is name of the file that contains model data + # load the model. + resource_dir = fox.workspace.get_resource_dir("data") + file_name = os.path.join(resource_dir, self._initial_model) + self._initial_model = load_np_model(file_name) + self.logger.info(f"loaded initial model from {file_name}: {self._initial_model}") + @fox.algo def execute(self): current_model = fox.get_input(self._initial_model) @@ -37,8 +48,11 @@ def execute(self): return current_model @fox.final - def done(self): - self.logger.info(f"Cyclic is done: final model: {self.final_model}") + def save_result(self): + final_result = fox.get_input() + file_name = os.path.join(fox.workspace.get_work_dir(), "final_model.npy") + save_np_model(final_result, file_name) + self.logger.info(f"[{fox.call_info}]: saved final model {final_result} to {file_name}") def _do_one_round(self, current_round, current_model): random.shuffle(fox.clients) diff --git a/nvflare/fox/examples/np/algos/utils.py b/nvflare/fox/examples/np/algos/utils.py index 10719506cf..f4b056ff3e 100644 --- a/nvflare/fox/examples/np/algos/utils.py +++ b/nvflare/fox/examples/np/algos/utils.py @@ -22,6 +22,10 @@ def parse_array_def(array_def): if isinstance(array_def, np.ndarray): return array_def + if isinstance(array_def, str): + # this is base name of the file that contains NP array + return array_def + if isinstance(array_def, list): return np.array(array_def, dtype=np.float32) else: diff --git a/nvflare/fox/examples/np/cyclic.py b/nvflare/fox/examples/np/cyclic.py index 0714c0bdbf..dfb1d8f33e 100644 --- a/nvflare/fox/examples/np/cyclic.py +++ b/nvflare/fox/examples/np/cyclic.py @@ -13,7 +13,6 @@ # limitations under the License. import logging -from nvflare.fox.api.app import ClientApp, ServerApp from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.strategies.cyclic import NPCyclic @@ -26,8 +25,8 @@ def main(): simulator = Simulator( root_dir="/tmp/fox", experiment_name="cyclic", - server_app=ServerApp(NPCyclic(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2)), - client_app=ClientApp(NPTrainer(delta=1.0)), + server=NPCyclic(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), + client=NPTrainer(delta=1.0), num_clients=2, ) diff --git a/nvflare/fox/examples/np/cyclic_avg.py b/nvflare/fox/examples/np/cyclic_avg.py index 2adef9e32a..79320411af 100644 --- a/nvflare/fox/examples/np/cyclic_avg.py +++ b/nvflare/fox/examples/np/cyclic_avg.py @@ -14,7 +14,6 @@ import logging from nvflare.fox import fox -from nvflare.fox.api.app import ClientApp, ServerApp from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.strategies.avg_para import NPFedAvgParallel @@ -54,13 +53,13 @@ def main(): simple_logging(logging.DEBUG) exp_name = "cyclic_avg" - server_app = ServerApp(Controller(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], cyclic_rounds=2, avg_rounds=3)) + server = Controller(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], cyclic_rounds=2, avg_rounds=3) simulator = Simulator( root_dir="/tmp/fox", experiment_name=exp_name, - server_app=server_app, - client_app=ClientApp(NPTrainer(delta=1.0)), + server=server, + client=NPTrainer(delta=1.0), num_clients=3, ) diff --git a/nvflare/fox/examples/np/cyclic2.py b/nvflare/fox/examples/np/cyclic_file.py similarity index 86% rename from nvflare/fox/examples/np/cyclic2.py rename to nvflare/fox/examples/np/cyclic_file.py index 7dac7ac24b..a5f1ff011b 100644 --- a/nvflare/fox/examples/np/cyclic2.py +++ b/nvflare/fox/examples/np/cyclic_file.py @@ -16,7 +16,7 @@ from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.strategies.cyclic import NPCyclic -from nvflare.fox.sim.sim2 import Simulator +from nvflare.fox.sim.simulator import Simulator def main(): @@ -25,11 +25,13 @@ def main(): simulator = Simulator( root_dir="/tmp/fox", experiment_name="cyclic", - server=NPCyclic(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), + server=NPCyclic(initial_model="model.npy", num_rounds=2), client=NPTrainer(delta=1.0), num_clients=2, ) + simulator.set_server_resource_dirs({"data": "/tmp/fox/data"}) + final_result = simulator.run() print(f"final model: {final_result}") diff --git a/nvflare/fox/examples/np/fed_avg_h2.py b/nvflare/fox/examples/np/cyclic_runner.py similarity index 56% rename from nvflare/fox/examples/np/fed_avg_h2.py rename to nvflare/fox/examples/np/cyclic_runner.py index 6fbcc990c1..3eb73b4830 100644 --- a/nvflare/fox/examples/np/fed_avg_h2.py +++ b/nvflare/fox/examples/np/cyclic_runner.py @@ -13,27 +13,26 @@ # limitations under the License. import logging +from nvflare.fox.api.app import ClientApp, ServerApp from nvflare.fox.api.utils import simple_logging -from nvflare.fox.examples.np.algos.client import NPHierarchicalTrainer -from nvflare.fox.examples.np.algos.strategies.avg_h import NPHierarchicalFedAvg -from nvflare.fox.examples.np.algos.widgets import MetricReceiver -from nvflare.fox.sim.sim2 import Simulator +from nvflare.fox.examples.np.algos.client import NPTrainer +from nvflare.fox.examples.np.algos.strategies.cyclic import NPCyclic +from nvflare.fox.sim.simulator import AppRunner def main(): simple_logging(logging.DEBUG) - simulator = Simulator( + runner = AppRunner( root_dir="/tmp/fox", - experiment_name="fedavg_h", - server=NPHierarchicalFedAvg(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=3), - client=NPHierarchicalTrainer(delta=1.0), - server_objects={"metric_receiver": MetricReceiver()}, - num_clients=(3, 2), + experiment_name="cyclic", + server_app=ServerApp(NPCyclic(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2)), + client_app=ClientApp(NPTrainer(delta=1.0)), + num_clients=2, ) - result = simulator.run() - print(f"Final Result: {result}") + final_result = runner.run() + print(f"final model: {final_result}") if __name__ == "__main__": diff --git a/nvflare/fox/examples/np/fed_avg_h.py b/nvflare/fox/examples/np/fed_avg_h.py index 42501a6f50..89f84bf584 100644 --- a/nvflare/fox/examples/np/fed_avg_h.py +++ b/nvflare/fox/examples/np/fed_avg_h.py @@ -13,7 +13,6 @@ # limitations under the License. import logging -from nvflare.fox.api.app import ClientApp, ServerApp from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.client import NPHierarchicalTrainer from nvflare.fox.examples.np.algos.strategies.avg_h import NPHierarchicalFedAvg @@ -24,20 +23,17 @@ def main(): simple_logging(logging.DEBUG) - server_app = ServerApp( - obj=NPHierarchicalFedAvg(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=3), - ) - server_app.add_collab_object("metric_receiver", MetricReceiver()) - simulator = Simulator( root_dir="/tmp/fox", experiment_name="fedavg_h", - server_app=server_app, - client_app=ClientApp(NPHierarchicalTrainer(delta=1.0)), + server=NPHierarchicalFedAvg(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=3), + client=NPHierarchicalTrainer(delta=1.0), + server_objects={"metric_receiver": MetricReceiver()}, num_clients=(3, 2), ) - simulator.run() + result = simulator.run() + print(f"Final Result: {result}") if __name__ == "__main__": diff --git a/nvflare/fox/examples/np/fed_avg_intime.py b/nvflare/fox/examples/np/fed_avg_intime.py index b5da1d9289..f2746bcc6e 100644 --- a/nvflare/fox/examples/np/fed_avg_intime.py +++ b/nvflare/fox/examples/np/fed_avg_intime.py @@ -13,7 +13,6 @@ # limitations under the License. import logging -from nvflare.fox.api.app import ClientApp, ServerApp from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.filters import AddNoiseToModel, Print @@ -25,29 +24,23 @@ def main(): simple_logging(logging.DEBUG) - server_app = ServerApp( - NPFedAvgInTime(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), - ) - - print_filter = Print() - server_app.add_collab_object("metric_receiver", MetricReceiver()) - server_app.add_outgoing_call_filters("*.train", [AddNoiseToModel()]) - server_app.add_incoming_result_filters("*.train", [print_filter]) - server_app.set_prop("default_timeout", 5.0) - - client_app = ClientApp(NPTrainer(delta=1.0)) - client_app.add_incoming_call_filters("*.train", [print_filter]) - client_app.add_outgoing_result_filters("*.train", [print_filter]) - client_app.set_prop("default_timeout", 8.0) - simulator = Simulator( root_dir="/tmp/fox", experiment_name="fedavg_intime", - server_app=server_app, - client_app=client_app, - num_clients=1, + server=NPFedAvgInTime(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), + client=NPTrainer(delta=1.0), + server_objects={"metric_receiver": MetricReceiver()}, + num_clients=2, ) + simulator.add_server_outgoing_call_filters("*.train", [AddNoiseToModel()]) + simulator.add_server_incoming_result_filters("*.train", [Print()]) + simulator.set_server_prop("default_timeout", 5.0) + + simulator.add_client_incoming_call_filters("*.train", [Print()]) + simulator.add_client_outgoing_result_filters("*.train", [Print()]) + simulator.set_client_prop("default_timeout", 8.0) + result = simulator.run() print(f"final model: {result}") diff --git a/nvflare/fox/examples/np/fed_avg_intime2.py b/nvflare/fox/examples/np/fed_avg_intime2.py deleted file mode 100644 index d9545d77ad..0000000000 --- a/nvflare/fox/examples/np/fed_avg_intime2.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging - -from nvflare.fox.api.utils import simple_logging -from nvflare.fox.examples.np.algos.client import NPTrainer -from nvflare.fox.examples.np.algos.filters import AddNoiseToModel, Print -from nvflare.fox.examples.np.algos.strategies.avg_intime import NPFedAvgInTime -from nvflare.fox.examples.np.algos.widgets import MetricReceiver -from nvflare.fox.sim.sim2 import Simulator - - -def main(): - simple_logging(logging.DEBUG) - - simulator = Simulator( - root_dir="/tmp/fox", - experiment_name="fedavg_intime", - server=NPFedAvgInTime(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), - client=NPTrainer(delta=1.0), - server_objects={"metric_receiver": MetricReceiver()}, - num_clients=2, - ) - - simulator.add_server_outgoing_call_filters("*.train", [AddNoiseToModel()]) - simulator.add_server_incoming_result_filters("*.train", [Print()]) - simulator.set_server_prop("default_timeout", 5.0) - - simulator.add_client_incoming_call_filters("*.train", [Print()]) - simulator.add_client_outgoing_result_filters("*.train", [Print()]) - simulator.set_client_prop("default_timeout", 8.0) - - result = simulator.run() - print(f"final model: {result}") - - -if __name__ == "__main__": - main() diff --git a/nvflare/fox/examples/np/fed_avg_para.py b/nvflare/fox/examples/np/fed_avg_para.py index 3aab541e7d..22a9008723 100644 --- a/nvflare/fox/examples/np/fed_avg_para.py +++ b/nvflare/fox/examples/np/fed_avg_para.py @@ -13,7 +13,6 @@ # limitations under the License. import logging -from nvflare.fox.api.app import ClientApp, ServerApp from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.strategies.avg_para import NPFedAvgParallel @@ -24,16 +23,14 @@ def main(): simple_logging(logging.DEBUG) - server_app = ServerApp( - NPFedAvgParallel(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]], num_rounds=2), - ) - server_app.add_collab_object("metric_receiver", MetricReceiver()) + server = NPFedAvgParallel(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]], num_rounds=2) simulator = Simulator( root_dir="/tmp/fox", experiment_name="fedavg_para", - server_app=server_app, - client_app=ClientApp(NPTrainer(delta=1.0)), + server=server, + client=NPTrainer(delta=1.0), + server_objects={"metric_receiver": MetricReceiver()}, num_clients=10, ) diff --git a/nvflare/fox/examples/np/fed_avg_seq.py b/nvflare/fox/examples/np/fed_avg_seq.py index 4aed9248fb..66f4386e90 100644 --- a/nvflare/fox/examples/np/fed_avg_seq.py +++ b/nvflare/fox/examples/np/fed_avg_seq.py @@ -13,7 +13,6 @@ # limitations under the License. import logging -from nvflare.fox.api.app import ClientApp, ServerApp from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.strategies.avg_seq import NPFedAvgSequential @@ -24,36 +23,21 @@ def main(): simple_logging(logging.DEBUG) - server_app = ServerApp( - NPFedAvgSequential( - num_rounds=2, - initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], - ), - ) - server_app.add_collab_object("metric_receiver", MetricReceiver()) - server_app.set_prop( - "client_weight_config", - { - "site-1": 70, - "site-2": 100, - }, + server = NPFedAvgSequential( + num_rounds=2, + initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], ) - client_app = ClientApp(NPTrainer(delta=1.0)) - client_app.set_prop( - "client_delta", - { - "site-1": 1.0, - "site-2": 2.0, - }, - ) simulator = Simulator( root_dir="/tmp/fox", experiment_name="fedavg_seq", - server_app=server_app, - client_app=client_app, + server=server, + client=NPTrainer(delta=1.0), + server_objects={"metric_receiver": MetricReceiver()}, num_clients=2, ) + simulator.set_server_prop("client_weight_config", {"site-1": 70, "site-2": 100}) + simulator.set_client_prop("client_delta", {"site-1": 1.0, "site-2": 2.0}) result = simulator.run() print(f"Final result: {result}") diff --git a/nvflare/fox/examples/np/fed_avg_stream.py b/nvflare/fox/examples/np/fed_avg_stream.py index f138cc1bc5..5961da7931 100644 --- a/nvflare/fox/examples/np/fed_avg_stream.py +++ b/nvflare/fox/examples/np/fed_avg_stream.py @@ -13,7 +13,6 @@ # limitations under the License. import logging -from nvflare.fox.api.app import ClientApp, ServerApp from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.avg_stream import NPFedAvgStream, NPTrainer from nvflare.fox.sim.simulator import Simulator @@ -22,17 +21,11 @@ def main(): simple_logging(logging.DEBUG) - server_app = ServerApp( - NPFedAvgStream(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=4), - ) - - client_app = ClientApp(NPTrainer(delta=1.0)) - simulator = Simulator( root_dir="/tmp/fox", experiment_name="fedavg_intime", - server_app=server_app, - client_app=client_app, + server=NPFedAvgStream(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=4), + client=NPTrainer(delta=1.0), num_clients=2, ) diff --git a/nvflare/fox/examples/np/swarm.py b/nvflare/fox/examples/np/swarm.py index b24bec451d..8bf7a8f9c2 100644 --- a/nvflare/fox/examples/np/swarm.py +++ b/nvflare/fox/examples/np/swarm.py @@ -13,7 +13,6 @@ # limitations under the License. import logging -from nvflare.fox.api.app import ClientApp, ServerApp from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.np.algos.swarm import NPSwarm, NPSwarmClient from nvflare.fox.sim.simulator import Simulator @@ -22,14 +21,11 @@ def main(): simple_logging(logging.DEBUG) - server_app = ServerApp(NPSwarm(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=5)) - client_app = ClientApp(NPSwarmClient(delta=1.0)) - simulator = Simulator( root_dir="/tmp/fox", experiment_name="swarm", - server_app=server_app, - client_app=client_app, + server=NPSwarm(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=5), + client=NPSwarmClient(delta=1.0), num_clients=3, ) diff --git a/nvflare/fox/examples/pt/pt_avg_filter.py b/nvflare/fox/examples/pt/pt_avg_filter.py index ca1f6e5be8..5072e86241 100644 --- a/nvflare/fox/examples/pt/pt_avg_filter.py +++ b/nvflare/fox/examples/pt/pt_avg_filter.py @@ -19,7 +19,7 @@ from nvflare.fox import fox from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.pt.utils import parse_state_dict -from nvflare.fox.sim.sim2 import Simulator +from nvflare.fox.sim.simulator import Simulator from nvflare.fuel.utils.log_utils import get_obj_logger diff --git a/nvflare/fox/examples/pt/pt_avg_mixed.py b/nvflare/fox/examples/pt/pt_avg_mixed.py index 95ca111334..695416dedd 100644 --- a/nvflare/fox/examples/pt/pt_avg_mixed.py +++ b/nvflare/fox/examples/pt/pt_avg_mixed.py @@ -26,7 +26,7 @@ from nvflare.fox.examples.pt.utils import add as add_pt from nvflare.fox.examples.pt.utils import div as div_pt from nvflare.fox.examples.pt.utils import parse_state_dict as parse_pt -from nvflare.fox.sim.sim2 import Simulator +from nvflare.fox.sim.simulator import Simulator from nvflare.fox.sys.downloader import Downloader, download_arrays, download_tensors from nvflare.fuel.utils.log_utils import get_obj_logger diff --git a/nvflare/fox/examples/pt/pt_avg_stream.py b/nvflare/fox/examples/pt/pt_avg_stream.py index 560192ab7e..d0cc96dc25 100644 --- a/nvflare/fox/examples/pt/pt_avg_stream.py +++ b/nvflare/fox/examples/pt/pt_avg_stream.py @@ -20,7 +20,7 @@ from nvflare.fox.api.constants import BackendType from nvflare.fox.api.utils import simple_logging from nvflare.fox.examples.pt.utils import parse_state_dict -from nvflare.fox.sim.sim2 import Simulator +from nvflare.fox.sim.simulator import Simulator from nvflare.fox.sys.downloader import Downloader, download_tensors from nvflare.fuel.utils.log_utils import get_obj_logger diff --git a/nvflare/fox/examples/pt/pt_np.py b/nvflare/fox/examples/pt/pt_np.py index 1a12a4fbcc..3e92b6aab7 100644 --- a/nvflare/fox/examples/pt/pt_np.py +++ b/nvflare/fox/examples/pt/pt_np.py @@ -22,7 +22,7 @@ from nvflare.fox.examples.pt.utils import add as add_pt from nvflare.fox.examples.pt.utils import div as div_pt from nvflare.fox.examples.pt.utils import parse_state_dict as parse_pt -from nvflare.fox.sim.sim2 import Simulator +from nvflare.fox.sim.simulator import Simulator from nvflare.fuel.utils.log_utils import get_obj_logger diff --git a/nvflare/fox/sim/sim2.py b/nvflare/fox/sim/sim2.py deleted file mode 100644 index cce752b4f2..0000000000 --- a/nvflare/fox/sim/sim2.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from typing import List, Tuple, Union - -from nvflare.fox.api.app import ClientApp, ServerApp - -from .simulator import Simulator as Sim1 - - -class Simulator: - - def __init__( - self, - root_dir: str, - experiment_name: str, - server, - client, - server_objects: dict[str, object] = None, - client_objects: dict[str, object] = None, - max_workers: int = 100, - num_clients: Union[int, Tuple[int, int]] = 2, - ): - server_app: ServerApp = ServerApp(server) - client_app: ClientApp = ClientApp(client) - - if server_objects: - for name, obj in server_objects.items(): - server_app.add_collab_object(name, obj) - - if client_objects: - for name, obj in client_objects.items(): - client_app.add_collab_object(name, obj) - - self.simulator = Sim1( - root_dir=root_dir, - experiment_name=experiment_name, - server_app=server_app, - client_app=client_app, - max_workers=max_workers, - num_clients=num_clients, - ) - - self.server_app = server_app - self.client_app = client_app - - def add_server_outgoing_call_filters(self, pattern: str, filters: List[object]): - self.server_app.add_outgoing_call_filters(pattern, filters) - - def add_server_incoming_call_filters(self, pattern: str, filters: List[object]): - self.server_app.add_incoming_call_filters(pattern, filters) - - def add_server_outgoing_result_filters(self, pattern: str, filters: List[object]): - self.server_app.add_outgoing_result_filters(pattern, filters) - - def add_server_incoming_result_filters(self, pattern: str, filters: List[object]): - self.server_app.add_incoming_result_filters(pattern, filters) - - def add_client_outgoing_call_filters(self, pattern: str, filters: List[object]): - self.client_app.add_outgoing_call_filters(pattern, filters) - - def add_client_incoming_call_filters(self, pattern: str, filters: List[object]): - self.client_app.add_incoming_call_filters(pattern, filters) - - def add_client_outgoing_result_filters(self, pattern: str, filters: List[object]): - self.client_app.add_outgoing_result_filters(pattern, filters) - - def add_client_incoming_result_filters(self, pattern: str, filters: List[object]): - self.client_app.add_incoming_result_filters(pattern, filters) - - def set_server_prop(self, name: str, value): - self.server_app.set_prop(name, value) - - def set_client_prop(self, name: str, value): - self.client_app.set_prop(name, value) - - def run(self): - return self.simulator.run() diff --git a/nvflare/fox/sim/simulator.py b/nvflare/fox/sim/simulator.py index 0e93e30b7c..074d7fb6be 100644 --- a/nvflare/fox/sim/simulator.py +++ b/nvflare/fox/sim/simulator.py @@ -14,7 +14,7 @@ import copy import uuid from concurrent.futures import ThreadPoolExecutor -from typing import Tuple, Union +from typing import List, Tuple, Union from nvflare.apis.signal import Signal from nvflare.fox.api.app import App, ClientApp, ServerApp @@ -27,7 +27,7 @@ from nvflare.fuel.utils.log_utils import get_obj_logger -class Simulator: +class AppRunner: def _prepare_app_backends(self, app: App): bes = {"": SimBackend("", app, app, self.abort_signal, self.thread_executor)} @@ -201,3 +201,83 @@ def _try_run(self): app.finalize(ctx) return result + + +class Simulator: + + def __init__( + self, + root_dir: str, + experiment_name: str, + server, + client, + server_objects: dict[str, object] = None, + client_objects: dict[str, object] = None, + max_workers: int = 100, + num_clients: Union[int, Tuple[int, int]] = 2, + ): + server_app: ServerApp = ServerApp(server) + client_app: ClientApp = ClientApp(client) + + self.root_dir = root_dir + self.experiment_name = experiment_name + self.max_workers = max_workers + self.num_clients = num_clients + + if server_objects: + for name, obj in server_objects.items(): + server_app.add_collab_object(name, obj) + + if client_objects: + for name, obj in client_objects.items(): + client_app.add_collab_object(name, obj) + + self.server_app = server_app + self.client_app = client_app + + def add_server_outgoing_call_filters(self, pattern: str, filters: List[object]): + self.server_app.add_outgoing_call_filters(pattern, filters) + + def add_server_incoming_call_filters(self, pattern: str, filters: List[object]): + self.server_app.add_incoming_call_filters(pattern, filters) + + def add_server_outgoing_result_filters(self, pattern: str, filters: List[object]): + self.server_app.add_outgoing_result_filters(pattern, filters) + + def add_server_incoming_result_filters(self, pattern: str, filters: List[object]): + self.server_app.add_incoming_result_filters(pattern, filters) + + def add_client_outgoing_call_filters(self, pattern: str, filters: List[object]): + self.client_app.add_outgoing_call_filters(pattern, filters) + + def add_client_incoming_call_filters(self, pattern: str, filters: List[object]): + self.client_app.add_incoming_call_filters(pattern, filters) + + def add_client_outgoing_result_filters(self, pattern: str, filters: List[object]): + self.client_app.add_outgoing_result_filters(pattern, filters) + + def add_client_incoming_result_filters(self, pattern: str, filters: List[object]): + self.client_app.add_incoming_result_filters(pattern, filters) + + def set_server_prop(self, name: str, value): + self.server_app.set_prop(name, value) + + def set_client_prop(self, name: str, value): + self.client_app.set_prop(name, value) + + def set_server_resource_dirs(self, resource_dirs): + self.server_app.set_resource_dirs(resource_dirs) + + def set_client_resource_dirs(self, resource_dirs): + self.client_app.set_resource_dirs(resource_dirs) + + def run(self): + runner = AppRunner( + root_dir=self.root_dir, + experiment_name=self.experiment_name, + server_app=self.server_app, + client_app=self.client_app, + max_workers=self.max_workers, + num_clients=self.num_clients, + ) + return runner.run() From 340986472272671ac3642f62def365f89ed6b390 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Wed, 26 Nov 2025 15:36:47 -0500 Subject: [PATCH 075/102] fix for example --- nvflare/fox/examples/np/algos/strategies/avg_intime.py | 4 ++-- nvflare/fox/examples/np/algos/strategies/cyclic.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/nvflare/fox/examples/np/algos/strategies/avg_intime.py b/nvflare/fox/examples/np/algos/strategies/avg_intime.py index d9c6017c73..b3c9770191 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_intime.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_intime.py @@ -39,8 +39,8 @@ def execute(self): self.logger.info(f"[{fox.call_info}] Start training for {self.num_rounds} rounds") current_model = fox.get_prop(ContextKey.INPUT, self._init_model) for i in range(self.num_rounds): - # current_model = self._do_one_round(i, current_model) - current_model = self._do_one_round_non_blocking(i, current_model) + current_model = self._do_one_round(i, current_model) + # current_model = self._do_one_round_non_blocking(i, current_model) score = self._do_eval(current_model) self.logger.info(f"[{fox.call_info}]: eval score in round {i}: {score}") self.logger.info(f"FINAL MODEL: {current_model}") diff --git a/nvflare/fox/examples/np/algos/strategies/cyclic.py b/nvflare/fox/examples/np/algos/strategies/cyclic.py index 7f7458eabc..3de8473e61 100644 --- a/nvflare/fox/examples/np/algos/strategies/cyclic.py +++ b/nvflare/fox/examples/np/algos/strategies/cyclic.py @@ -55,8 +55,10 @@ def save_result(self): self.logger.info(f"[{fox.call_info}]: saved final model {final_result} to {file_name}") def _do_one_round(self, current_round, current_model): - random.shuffle(fox.clients) - for c in fox.clients: + # Note: fox.clients always returns a new copy of all clients! + clients = fox.clients + random.shuffle(clients) + for c in clients: current_model = c.train(current_round, current_model) self.logger.info(f"[{fox.call_info}] result from {c.name}: {current_model}") return current_model From 9bdd49acd938055791a0eda16f400e7b063ebd3c Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Mon, 1 Dec 2025 16:28:16 -0500 Subject: [PATCH 076/102] refactor --- nvflare/fox/api/constants.py | 2 +- nvflare/fox/api/facade.py | 4 +-- nvflare/fox/api/gcc.py | 26 +++++++++++---- nvflare/fox/api/run_server.py | 2 +- nvflare/fox/examples/np/algos/avg_stream.py | 4 +-- .../fox/examples/np/algos/strategies/avg_h.py | 2 +- .../np/algos/strategies/avg_intime.py | 4 +-- .../examples/np/algos/strategies/avg_para.py | 2 +- .../examples/np/algos/strategies/avg_seq.py | 2 +- .../examples/np/algos/strategies/cyclic.py | 4 +-- nvflare/fox/examples/np/cyclic_file.py | 4 +-- nvflare/fox/examples/np/recipe_cyclic_avg.py | 2 +- nvflare/fox/examples/np/recipes/__init__.py | 0 .../fox/examples/np/recipes/cyclic_recipe.py | 32 +++++++++++++++++++ nvflare/fox/examples/pt/pt_avg_filter.py | 4 +-- nvflare/fox/examples/pt/pt_avg_mixed.py | 2 +- nvflare/fox/examples/pt/pt_avg_stream.py | 4 +-- nvflare/fox/examples/pt/pt_np.py | 2 +- 18 files changed, 74 insertions(+), 28 deletions(-) create mode 100644 nvflare/fox/examples/np/recipes/__init__.py create mode 100644 nvflare/fox/examples/np/recipes/cyclic_recipe.py diff --git a/nvflare/fox/api/constants.py b/nvflare/fox/api/constants.py index 9d6c6d283a..437df6fd1e 100644 --- a/nvflare/fox/api/constants.py +++ b/nvflare/fox/api/constants.py @@ -17,7 +17,7 @@ class CollabMethodArgName: class ContextKey: - INPUT = "input" + RESULT = "result" QUALIFIED_FUNC_NAME = "qualified_func_name" DIRECTION = "direction" diff --git a/nvflare/fox/api/facade.py b/nvflare/fox/api/facade.py index 1dea858a2b..79fdcda692 100644 --- a/nvflare/fox/api/facade.py +++ b/nvflare/fox/api/facade.py @@ -169,5 +169,5 @@ def get_prop(name: str, default=None): return ctx.get_prop(name, default) @staticmethod - def get_input(default=None): - return facade.get_prop(ContextKey.INPUT, default) + def get_result(default=None): + return facade.get_prop(ContextKey.RESULT, default) diff --git a/nvflare/fox/api/gcc.py b/nvflare/fox/api/gcc.py index f8d1b1555e..bab0d7ecab 100644 --- a/nvflare/fox/api/gcc.py +++ b/nvflare/fox/api/gcc.py @@ -32,11 +32,13 @@ def __init__(self, limit: int): self.consumed = 0 self.num_items_received = 0 - def append(self, i): + def append(self, item, complete=True): if self.num_items_received == self.limit: raise RuntimeError(f"queue is full: {self.limit} items are already appended") - self.num_items_received += 1 - self.q.put_nowait(i) + self.q.put_nowait(item) + + if complete: + self.num_items_received += 1 return self.num_items_received == self.limit def __iter__(self): @@ -62,15 +64,24 @@ def __init__(self, sites: list[str]): self.results = ResultQueue(len(sites)) self.lock = threading.Lock() - def set_result(self, target_name: str, result): + @staticmethod + def _get_site_name(target_name: str): # target_name is either or . parts = target_name.split(".") - site_name = parts[0] + return parts[0] + + def set_result(self, target_name: str, result): + site_name = self._get_site_name(target_name) with self.lock: all_received = self.results.append((site_name, result)) if all_received: self.set() + def add_partial_result(self, target_name: str, partial_result): + site_name = self._get_site_name(target_name) + with self.lock: + self.results.append((site_name, partial_result), complete=False) + class GroupCallContext: @@ -132,7 +143,7 @@ def set_result(self, result): if self.process_cb: self.cb_kwargs[CollabMethodArgName.CONTEXT] = ctx check_context_support(self.process_cb, self.cb_kwargs) - result = self.process_cb(result, **self.cb_kwargs) + result = self.process_cb(self, result, **self.cb_kwargs) # set back to original context set_call_context(self.context) @@ -152,3 +163,6 @@ def set_exception(self, ex): """ self.waiter.set_result(self.target_name, ex) + + def add_partial_result(self, partial_result): + self.waiter.add_partial_result(self.target_name, partial_result) diff --git a/nvflare/fox/api/run_server.py b/nvflare/fox/api/run_server.py index 2d12878f7a..8c8eab3234 100644 --- a/nvflare/fox/api/run_server.py +++ b/nvflare/fox/api/run_server.py @@ -37,7 +37,7 @@ def run_server(server_app: ServerApp, logger): if not supports_context(f): kwargs = {} result = f(**kwargs) - server_ctx.set_prop(ContextKey.INPUT, result) + server_ctx.set_prop(ContextKey.RESULT, result) except Exception as ex: traceback.print_exc() backend = server_app.get_backend() diff --git a/nvflare/fox/examples/np/algos/avg_stream.py b/nvflare/fox/examples/np/algos/avg_stream.py index dd0c942f3e..d0411cfffe 100644 --- a/nvflare/fox/examples/np/algos/avg_stream.py +++ b/nvflare/fox/examples/np/algos/avg_stream.py @@ -41,7 +41,7 @@ def __init__(self, initial_model, num_rounds=10, timeout=2.0): @fox.algo def execute(self): self.logger.info(f"[{fox.call_info}] Start training for {self.num_rounds} rounds") - current_model = fox.get_input(self._init_model) + current_model = self._init_model for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model) self.logger.info(f"FINAL MODEL: {current_model}") @@ -82,7 +82,7 @@ def _do_one_round(self, r, current_model): self.logger.info(f"[{fox.call_info}] round {r}: aggr result from {aggr_result.count} clients: {result}") return result - def _accept_train_result(self, result, aggr_result: _AggrResult): + def _accept_train_result(self, gcc, result, aggr_result: _AggrResult): self.logger.info(f"[{fox.call_info}] got train result from {fox.caller}: {result}") model, model_type = result diff --git a/nvflare/fox/examples/np/algos/strategies/avg_h.py b/nvflare/fox/examples/np/algos/strategies/avg_h.py index e28d8ddf49..c0416782dd 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_h.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_h.py @@ -28,7 +28,7 @@ def __init__(self, initial_model, num_rounds=10): @fox.algo def execute(self): self.logger.info(f"[{fox.call_info}] Start training for {self.num_rounds} rounds") - current_model = fox.get_input(self._initial_model) + current_model = self._initial_model for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model) score = self._do_eval(current_model) diff --git a/nvflare/fox/examples/np/algos/strategies/avg_intime.py b/nvflare/fox/examples/np/algos/strategies/avg_intime.py index b3c9770191..2466e68dbf 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_intime.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_intime.py @@ -37,7 +37,7 @@ def __init__(self, initial_model, num_rounds=10, timeout=2.0): @fox.algo def execute(self): self.logger.info(f"[{fox.call_info}] Start training for {self.num_rounds} rounds") - current_model = fox.get_prop(ContextKey.INPUT, self._init_model) + current_model = fox.get_prop(ContextKey.RESULT, self._init_model) for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model) # current_model = self._do_one_round_non_blocking(i, current_model) @@ -87,7 +87,7 @@ def _do_one_round_non_blocking(self, r, current_model): self.logger.info(f"[{fox.call_info}] round {r}: aggr result from {len(results)} clients: {result}") return result - def _accept_train_result(self, result, aggr_result: _AggrResult): + def _accept_train_result(self, gcc, result, aggr_result: _AggrResult): self.logger.info(f"[{fox.call_info}] got train result from {fox.caller} {result}") aggr_result.total += result aggr_result.count += 1 diff --git a/nvflare/fox/examples/np/algos/strategies/avg_para.py b/nvflare/fox/examples/np/algos/strategies/avg_para.py index c273991df3..0ccb27e982 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_para.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_para.py @@ -28,7 +28,7 @@ def __init__(self, initial_model, num_rounds=10): @fox.algo def execute(self): self.logger.info(f"[{fox.call_info}] Start training for {self.num_rounds} rounds") - current_model = fox.get_input(self._initial_model) + current_model = self._initial_model for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model) score = self._do_eval(current_model) diff --git a/nvflare/fox/examples/np/algos/strategies/avg_seq.py b/nvflare/fox/examples/np/algos/strategies/avg_seq.py index ede5e8654a..f5db9a4fa2 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_seq.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_seq.py @@ -49,7 +49,7 @@ def init(self): @fox.algo def execute(self): self.logger.info(f"[{fox.call_info}] Start training for {self.num_rounds} rounds") - current_model = fox.get_input(self._initial_model) + current_model = self._initial_model for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model) diff --git a/nvflare/fox/examples/np/algos/strategies/cyclic.py b/nvflare/fox/examples/np/algos/strategies/cyclic.py index 3de8473e61..e2338b0f4f 100644 --- a/nvflare/fox/examples/np/algos/strategies/cyclic.py +++ b/nvflare/fox/examples/np/algos/strategies/cyclic.py @@ -40,7 +40,7 @@ def check_initial_model(self): @fox.algo def execute(self): - current_model = fox.get_input(self._initial_model) + current_model = self._initial_model for current_round in range(self.num_rounds): current_model = self._do_one_round(current_round, current_model) self.logger.info(f"[{fox.call_info}] final result: {current_model}") @@ -49,7 +49,7 @@ def execute(self): @fox.final def save_result(self): - final_result = fox.get_input() + final_result = fox.get_result() file_name = os.path.join(fox.workspace.get_work_dir(), "final_model.npy") save_np_model(final_result, file_name) self.logger.info(f"[{fox.call_info}]: saved final model {final_result} to {file_name}") diff --git a/nvflare/fox/examples/np/cyclic_file.py b/nvflare/fox/examples/np/cyclic_file.py index a5f1ff011b..796ded9bec 100644 --- a/nvflare/fox/examples/np/cyclic_file.py +++ b/nvflare/fox/examples/np/cyclic_file.py @@ -25,12 +25,12 @@ def main(): simulator = Simulator( root_dir="/tmp/fox", experiment_name="cyclic", - server=NPCyclic(initial_model="model.npy", num_rounds=2), + server=NPCyclic(initial_model="initial_model.npy", num_rounds=2), client=NPTrainer(delta=1.0), num_clients=2, ) - simulator.set_server_resource_dirs({"data": "/tmp/fox/data"}) + simulator.set_server_resource_dirs({"data": "/Users/yanc/NVFlare/sandbox/data"}) final_result = simulator.run() print(f"final model: {final_result}") diff --git a/nvflare/fox/examples/np/recipe_cyclic_avg.py b/nvflare/fox/examples/np/recipe_cyclic_avg.py index 4eb8b65658..c36de69c69 100644 --- a/nvflare/fox/examples/np/recipe_cyclic_avg.py +++ b/nvflare/fox/examples/np/recipe_cyclic_avg.py @@ -53,7 +53,7 @@ def run(self): @fox.final def save_result(self): - final_result = fox.get_input() + final_result = fox.get_result() file_name = os.path.join(fox.workspace.get_work_dir(), "final_model.npy") save_np_model(final_result, file_name) self.logger.info(f"[{fox.call_info}]: saved final model {final_result} to {file_name}") diff --git a/nvflare/fox/examples/np/recipes/__init__.py b/nvflare/fox/examples/np/recipes/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/nvflare/fox/examples/np/recipes/cyclic_recipe.py b/nvflare/fox/examples/np/recipes/cyclic_recipe.py new file mode 100644 index 0000000000..3ca9cef037 --- /dev/null +++ b/nvflare/fox/examples/np/recipes/cyclic_recipe.py @@ -0,0 +1,32 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nvflare.fox.examples.np.algos.strategies.cyclic import NPCyclic +from nvflare.fox.sys.recipe import FoxRecipe + + +class CyclicRecipe(FoxRecipe): + + def __init__( + self, + job_name, + initial_model, + num_rounds, + client, + ): + FoxRecipe.__init__( + self, + job_name, + server=NPCyclic(initial_model, num_rounds), + client=client, + ) diff --git a/nvflare/fox/examples/pt/pt_avg_filter.py b/nvflare/fox/examples/pt/pt_avg_filter.py index 5072e86241..168d3acb63 100644 --- a/nvflare/fox/examples/pt/pt_avg_filter.py +++ b/nvflare/fox/examples/pt/pt_avg_filter.py @@ -44,7 +44,7 @@ def __init__(self, initial_model, num_rounds=10, timeout=2.0): @fox.algo def execute(self): self.logger.info(f"[{fox.call_info}] Start training for {self.num_rounds} rounds") - current_model = fox.get_input(self._init_model) + current_model = self._init_model for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model) self.logger.info(f"FINAL MODEL: {current_model}") @@ -67,7 +67,7 @@ def _do_one_round(self, r, current_model): self.logger.info(f"[{fox.call_info}] round {r}: aggr result from {aggr_result.count} clients: {result}") return result - def _accept_train_result(self, result, aggr_result: _AggrResult): + def _accept_train_result(self, gcc, result, aggr_result: _AggrResult): self.logger.info(f"[{fox.call_info}] got train result from {fox.caller}: {result}") for k, v in result.items(): diff --git a/nvflare/fox/examples/pt/pt_avg_mixed.py b/nvflare/fox/examples/pt/pt_avg_mixed.py index 695416dedd..f9d5a814fe 100644 --- a/nvflare/fox/examples/pt/pt_avg_mixed.py +++ b/nvflare/fox/examples/pt/pt_avg_mixed.py @@ -99,7 +99,7 @@ def _do_one_round(self, r, pt_model, np_model): ) return pt_result, np_result - def _accept_train_result(self, result, aggr_result: _AggrResult): + def _accept_train_result(self, gcc, result, aggr_result: _AggrResult): self.logger.info(f"[{fox.call_info}] got train result from {fox.caller}: {result}") pt_result, np_result, model_type = result diff --git a/nvflare/fox/examples/pt/pt_avg_stream.py b/nvflare/fox/examples/pt/pt_avg_stream.py index d0cc96dc25..129e6baf8d 100644 --- a/nvflare/fox/examples/pt/pt_avg_stream.py +++ b/nvflare/fox/examples/pt/pt_avg_stream.py @@ -46,7 +46,7 @@ def __init__(self, initial_model, num_rounds=10, timeout=2.0): @fox.algo def execute(self): self.logger.info(f"[{fox.call_info}] Start training for {self.num_rounds} rounds") - current_model = fox.get_input(self._init_model) + current_model = self._init_model for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model) self.logger.info(f"FINAL MODEL: {current_model}") @@ -87,7 +87,7 @@ def _do_one_round(self, r, current_model): self.logger.info(f"[{fox.call_info}] round {r}: aggr result from {aggr_result.count} clients: {result}") return result - def _accept_train_result(self, result, aggr_result: _AggrResult): + def _accept_train_result(self, gcc, result, aggr_result: _AggrResult): self.logger.info(f"[{fox.call_info}] got train result from {fox.caller}: {result}") model, model_type = result diff --git a/nvflare/fox/examples/pt/pt_np.py b/nvflare/fox/examples/pt/pt_np.py index 3e92b6aab7..62c49381b3 100644 --- a/nvflare/fox/examples/pt/pt_np.py +++ b/nvflare/fox/examples/pt/pt_np.py @@ -80,7 +80,7 @@ def _do_one_round(self, r, pt_model, np_model): ) return pt_result, np_result - def _accept_train_result(self, result, aggr_result: _AggrResult): + def _accept_train_result(self, gcc, result, aggr_result: _AggrResult): self.logger.info(f"[{fox.call_info}] got train result from {fox.caller}: {result}") pt_result, np_result = result From a9c167554492ad7a714aaa8249d60309b7e1df22 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Tue, 2 Dec 2025 13:06:08 -0500 Subject: [PATCH 077/102] added TensorReceiver --- nvflare/fox/api/gcc.py | 33 +++- nvflare/fox/examples/pt/pt_avg_stream2.py | 157 ++++++++++++++++++ .../fox/examples/pt/recipe_pt_avg_stream2.py | 37 +++++ nvflare/fox/utils/__init__.py | 13 ++ nvflare/fox/utils/tensor_receiver.py | 55 ++++++ 5 files changed, 287 insertions(+), 8 deletions(-) create mode 100644 nvflare/fox/examples/pt/pt_avg_stream2.py create mode 100644 nvflare/fox/examples/pt/recipe_pt_avg_stream2.py create mode 100644 nvflare/fox/utils/__init__.py create mode 100644 nvflare/fox/utils/tensor_receiver.py diff --git a/nvflare/fox/api/gcc.py b/nvflare/fox/api/gcc.py index bab0d7ecab..5973ba60f6 100644 --- a/nvflare/fox/api/gcc.py +++ b/nvflare/fox/api/gcc.py @@ -30,14 +30,27 @@ def __init__(self, limit: int): self.limit = limit self.q = queue.Queue() self.consumed = 0 + + # num_items_received is the number of WHOLE items received. + # the queue could contain partial results. self.num_items_received = 0 - def append(self, item, complete=True): + def append(self, item, is_whole=True): + """Append an item to the result queue. + + Args: + item: the item to be appended. + is_whole: whether the item is whole. + + Returns: + + """ if self.num_items_received == self.limit: raise RuntimeError(f"queue is full: {self.limit} items are already appended") self.q.put_nowait(item) - if complete: + if is_whole: + # increment num_items_received only if the item is whole! self.num_items_received += 1 return self.num_items_received == self.limit @@ -45,12 +58,16 @@ def __iter__(self): return self def __next__(self): - if self.consumed == self.limit: - raise StopIteration() + if not self.q.empty(): + return self.q.get() + + # queue is empty: do we expect more? + if self.num_items_received < self.limit: + # there will be more items - wait until more item is received + return self.q.get(block=True) else: - i = self.q.get(block=True) - self.consumed += 1 - return i + # no more items + raise StopIteration() def __len__(self): return self.num_items_received @@ -80,7 +97,7 @@ def set_result(self, target_name: str, result): def add_partial_result(self, target_name: str, partial_result): site_name = self._get_site_name(target_name) with self.lock: - self.results.append((site_name, partial_result), complete=False) + self.results.append((site_name, partial_result), is_whole=False) class GroupCallContext: diff --git a/nvflare/fox/examples/pt/pt_avg_stream2.py b/nvflare/fox/examples/pt/pt_avg_stream2.py new file mode 100644 index 0000000000..d5c07b4e80 --- /dev/null +++ b/nvflare/fox/examples/pt/pt_avg_stream2.py @@ -0,0 +1,157 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +import torch + +from nvflare.fox import fox +from nvflare.fox.api.constants import BackendType +from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples.pt.utils import parse_state_dict +from nvflare.fox.sim.simulator import Simulator +from nvflare.fox.sys.downloader import Downloader, download_tensors +from nvflare.fox.utils.tensor_receiver import TensorReceiver +from nvflare.fuel.utils.log_utils import get_obj_logger + + +class PTFedAvgStream: + + def __init__(self, initial_model, num_rounds=10, timeout=2.0): + self.num_rounds = num_rounds + self.initial_model = initial_model + self.timeout = timeout + self.name = "PTFedAvgStream" + self.logger = get_obj_logger(self) + self._init_model = parse_state_dict(initial_model) + + @fox.algo + def execute(self): + self.logger.info(f"[{fox.call_info}] Start training for {self.num_rounds} rounds") + current_model = self._init_model + for i in range(self.num_rounds): + current_model = self._do_one_round(i, current_model) + self.logger.info(f"FINAL MODEL: {current_model}") + return current_model + + def _do_one_round(self, r, current_model): + grp = fox.clients( + blocking=False, + process_resp_cb=TensorReceiver(), + ) + + if fox.backend_type == BackendType.FLARE: + downloader = Downloader( + num_receivers=grp.size, + timeout=5.0, + ) + model_type = "ref" + model = downloader.add_tensors(current_model, 0) + self.logger.info(f"prepared model as ref: {model}") + else: + model = current_model + model_type = "model" + + aggr_result = {} + aggr_count = {} + results = grp.train(r, model, model_type) + + # results is a queue that contains chunks of tensors that are downloaded from clients. + # we aggregate them while they are being downloaded in parallel. + # Note that the chunks from different sites may arrive in any order. + for n, tensors in results: + self.logger.info(f"got tensors from {n}: {tensors}") + if not tensors: + # we use None to indicate the end of all chunks from a site. + continue + + for k, v in tensors.items(): + if k not in aggr_result: + aggr_result[k] = v + aggr_count[k] = 1 + else: + aggr_result[k] += v + aggr_count[k] += 1 + + final_result = {} + for k, v in aggr_result.items(): + final_result[k] = torch.div(v, aggr_count[k]) + self.logger.info(f"[{fox.call_info}] round {r}: aggr result: {final_result}") + return final_result + + +class PTTrainer: + + def __init__(self, delta: float): + self.delta = delta + self.logger = get_obj_logger(self) + + @fox.collab + def train(self, current_round, model, model_type: str): + if fox.is_aborted: + self.logger.debug("training aborted") + return 0 + self.logger.debug(f"[{fox.call_info}] training round {current_round}: {model_type=} {model=}") + if model_type == "ref": + err, model = download_tensors(ref=model, per_request_timeout=5.0) + if err: + raise RuntimeError(f"failed to download model {model}: {err}") + self.logger.info(f"downloaded model1 {model}") + + result = {} + for k, v in model.items(): + result[k] = v + self.delta + + if model_type == "ref": + # stream it + downloader = Downloader( + num_receivers=1, + timeout=5.0, + ) + model_type = "ref" + model = downloader.add_tensors(result, 0) + self.logger.info(f"prepared result as ref: {model}") + else: + model = result + model_type = "model" + return model, model_type + + +def main(): + simple_logging(logging.DEBUG) + + server = PTFedAvgStream( + initial_model={ + "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "y": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "z": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + }, + num_rounds=2, + ) + + client = PTTrainer(delta=1.0) + + simulator = Simulator( + root_dir="/tmp/fox", + experiment_name="pt_fedavg_stream2", + server=server, + client=client, + num_clients=2, + ) + + result = simulator.run() + print(f"final result: {result}") + + +if __name__ == "__main__": + main() diff --git a/nvflare/fox/examples/pt/recipe_pt_avg_stream2.py b/nvflare/fox/examples/pt/recipe_pt_avg_stream2.py new file mode 100644 index 0000000000..a7a516e2a0 --- /dev/null +++ b/nvflare/fox/examples/pt/recipe_pt_avg_stream2.py @@ -0,0 +1,37 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nvflare.fox.examples.pt.pt_avg_stream2 import PTFedAvgStream, PTTrainer +from nvflare.fox.sys.recipe import FoxRecipe + +JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" + + +def main(): + recipe = FoxRecipe( + job_name="fox_pt_fedavg_stream2", + server=PTFedAvgStream( + initial_model={ + "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "y": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "z": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + }, + num_rounds=2, + ), + client=PTTrainer(delta=1.0), + ) + recipe.export(JOB_ROOT_DIR) + + +if __name__ == "__main__": + main() diff --git a/nvflare/fox/utils/__init__.py b/nvflare/fox/utils/__init__.py new file mode 100644 index 0000000000..341a77c5bc --- /dev/null +++ b/nvflare/fox/utils/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nvflare/fox/utils/tensor_receiver.py b/nvflare/fox/utils/tensor_receiver.py new file mode 100644 index 0000000000..6e7a4f1e0f --- /dev/null +++ b/nvflare/fox/utils/tensor_receiver.py @@ -0,0 +1,55 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nvflare.fox import fox +from nvflare.fox.api.gcc import GroupCallContext +from nvflare.fox.sys.downloader import download_tensors +from nvflare.fuel.utils.log_utils import get_obj_logger + + +class TensorReceiver: + """This class implements the callback function to add partially received tensors to the result queue. + This will enable the semi-in-time tensor aggregation by iterating through result queue. + To use, the application simply sets the process_resp_cb to an instance of this class when making group call. + + Example: + fox.clients( + blocking=False, + process_resp_cb=TensorReceiver(), + ).train(...) + """ + + def __init__(self): + self.logger = get_obj_logger(self) + + def __call__(self, gcc: GroupCallContext, result): + self.logger.info(f"[{fox.call_info}] got train result from {fox.caller}: {result}") + model, model_type = result + if model_type == "ref": + err, model = download_tensors( + ref=model, + per_request_timeout=5.0, + tensors_received_cb=self._receive_tensors, + gcc=gcc, + ) + if err: + return RuntimeError(f"failed to download model {model}: {err}") + else: + return None + else: + return model + + def _receive_tensors(self, tensors, gcc: GroupCallContext): + self.logger.info(f"adding partial result: {tensors}") + gcc.add_partial_result(tensors) + return None From d75e3270a01a69103ad437fc7c6d6a28bc291b23 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Tue, 2 Dec 2025 16:01:37 -0500 Subject: [PATCH 078/102] support explicit target in group calls --- nvflare/fox/api/call_opt.py | 5 ++- nvflare/fox/api/gcc.py | 26 +++++++++------ nvflare/fox/api/group.py | 16 ++++++++- nvflare/fox/api/proxy.py | 18 +++++----- nvflare/fox/api/proxy_list.py | 3 ++ .../examples/np/algos/strategies/avg_para.py | 2 +- nvflare/fox/examples/np/recipe_cyclic_file.py | 33 +++++++++++++++++++ 7 files changed, 82 insertions(+), 21 deletions(-) create mode 100644 nvflare/fox/examples/np/recipe_cyclic_file.py diff --git a/nvflare/fox/api/call_opt.py b/nvflare/fox/api/call_opt.py index b19ee26cfc..a19af734e2 100644 --- a/nvflare/fox/api/call_opt.py +++ b/nvflare/fox/api/call_opt.py @@ -20,6 +20,7 @@ def __init__( timeout: float = 5.0, secure: bool = False, optional: bool = False, + collab_obj_name=None, ): """CallOption defines behavior of a collab call. @@ -29,15 +30,17 @@ def __init__( timeout: when expecting result, the max number of secs to wait for result. secure: whether to use P2P secure messaging. optional: whether the call is optional. + collab_obj_name: name of the collab object to be called. """ self.expect_result = expect_result self.blocking = blocking self.timeout = timeout self.secure = secure self.optional = optional + self.collab_obj_name = collab_obj_name def __str__(self): return ( f"expect_result={self.expect_result} blocking={self.blocking} timeout={self.timeout} " - f"secure={self.secure} optional={self.optional}" + f"secure={self.secure} optional={self.optional} collab_obj={self.collab_obj_name}" ) diff --git a/nvflare/fox/api/gcc.py b/nvflare/fox/api/gcc.py index 5973ba60f6..77b1ebe57f 100644 --- a/nvflare/fox/api/gcc.py +++ b/nvflare/fox/api/gcc.py @@ -27,13 +27,13 @@ class ResultQueue: def __init__(self, limit: int): if limit <= 0: raise ValueError(f"bad queue limit {limit}: must be > 0") + self.limit = limit self.q = queue.Queue() - self.consumed = 0 - # num_items_received is the number of WHOLE items received. + # num_whole_items_received is the number of WHOLE items received. # the queue could contain partial results. - self.num_items_received = 0 + self.num_whole_items_received = 0 def append(self, item, is_whole=True): """Append an item to the result queue. @@ -42,17 +42,17 @@ def append(self, item, is_whole=True): item: the item to be appended. is_whole: whether the item is whole. - Returns: + Returns: whether the queue has received all whole items. """ - if self.num_items_received == self.limit: + if self.num_whole_items_received == self.limit: raise RuntimeError(f"queue is full: {self.limit} items are already appended") self.q.put_nowait(item) if is_whole: - # increment num_items_received only if the item is whole! - self.num_items_received += 1 - return self.num_items_received == self.limit + # increment num_whole_items_received only if the item is whole! + self.num_whole_items_received += 1 + return self.num_whole_items_received == self.limit def __iter__(self): return self @@ -62,7 +62,7 @@ def __next__(self): return self.q.get() # queue is empty: do we expect more? - if self.num_items_received < self.limit: + if self.num_whole_items_received < self.limit: # there will be more items - wait until more item is received return self.q.get(block=True) else: @@ -70,7 +70,13 @@ def __next__(self): raise StopIteration() def __len__(self): - return self.num_items_received + """Return the number of whole items that have been received. + Note that this is NOT the current number of items in the queue! + + Returns: the number of whole items that have been received + + """ + return self.num_whole_items_received class ResultWaiter(threading.Event): diff --git a/nvflare/fox/api/group.py b/nvflare/fox/api/group.py index a00406f934..f6f775a577 100644 --- a/nvflare/fox/api/group.py +++ b/nvflare/fox/api/group.py @@ -80,6 +80,18 @@ def members(self): """ return self._proxies + def _get_work_proxy(self, p): + if self._call_opt.collab_obj_name: + child = p.get_child(self._call_opt.collab_obj_name) + if not child: + raise RuntimeError( + f"site {p.name} does not have collab object named '{self._call_opt.collab_obj_name}': " + f"make sure to use correct collab_obj_name in the group call." + ) + return child + else: + return p + def __getattr__(self, func_name): """ This method is called to invoke the specified collab function. @@ -101,7 +113,7 @@ def method(*args, **kwargs): the_backend = None try: # filter once for all targets - p = self._proxies[0] + p = self._get_work_proxy(self._proxies[0]) # func_proxy is the proxy that actually has the func. # the func_proxy is either "p" or a child of "p". @@ -122,6 +134,7 @@ def method(*args, **kwargs): waiter = ResultWaiter([p.name for p in self._proxies]) for p in self._proxies: + p = self._get_work_proxy(p) func_proxy, func_itf, call_args, call_kwargs = p.adjust_func_args( func_name, adj_args, adj_kwargs ) @@ -164,6 +177,7 @@ def method(*args, **kwargs): return waiter.results except Exception as ex: + self._logger.error(f"exception {type(ex)} occurred: {ex}") if the_backend: the_backend.handle_exception(ex) diff --git a/nvflare/fox/api/proxy.py b/nvflare/fox/api/proxy.py index 67c97393ad..bf6638c2d1 100644 --- a/nvflare/fox/api/proxy.py +++ b/nvflare/fox/api/proxy.py @@ -94,14 +94,16 @@ def add_child(self, name, p): self.children[name] = p setattr(self, name, p) - def get_target(self, name: str): - obj = getattr(self, name, None) - if not obj: - return None - if isinstance(obj, Proxy): - return obj - else: - return None + def get_child(self, name): + """Get the specified child proxy. + + Args: + name: name of the child proxy. + + Returns: the child proxy if defined. + + """ + return self.children.get(name) def _find_interface(self, func_name): """Find interface for specified func name. diff --git a/nvflare/fox/api/proxy_list.py b/nvflare/fox/api/proxy_list.py index 529192903e..3472c68c97 100644 --- a/nvflare/fox/api/proxy_list.py +++ b/nvflare/fox/api/proxy_list.py @@ -49,6 +49,7 @@ def __call__( timeout: float = 5.0, optional: bool = False, secure: bool = False, + collab_obj_name=None, process_resp_cb=None, **cb_kwargs, ): @@ -60,6 +61,7 @@ def __call__( timeout: optional: secure: + collab_obj_name: process_resp_cb: **cb_kwargs: @@ -75,6 +77,7 @@ def __call__( timeout=timeout, optional=optional, secure=secure, + collab_obj_name=collab_obj_name, ), process_resp_cb=process_resp_cb, **cb_kwargs, diff --git a/nvflare/fox/examples/np/algos/strategies/avg_para.py b/nvflare/fox/examples/np/algos/strategies/avg_para.py index 0ccb27e982..2a7a7ea23c 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_para.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_para.py @@ -45,7 +45,7 @@ def _do_eval(self, model): def _do_one_round(self, r, current_model): total = 0 - results = fox.clients(timeout=4, blocking=False).train(r, current_model) + results = fox.clients(timeout=4, blocking=False, collab_obj_name="client").train(r, current_model) for n, v in results: self.logger.info(f"[{fox.call_info}] round {r}: got group result from client {n}: {v}") total += v diff --git a/nvflare/fox/examples/np/recipe_cyclic_file.py b/nvflare/fox/examples/np/recipe_cyclic_file.py new file mode 100644 index 0000000000..58d7221922 --- /dev/null +++ b/nvflare/fox/examples/np/recipe_cyclic_file.py @@ -0,0 +1,33 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nvflare.fox.examples.np.algos.client import NPTrainer +from nvflare.fox.examples.np.algos.strategies.cyclic import NPCyclic +from nvflare.fox.sys.recipe import FoxRecipe + +JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" + + +def main(): + recipe = FoxRecipe( + job_name="fox_cyclic_file", + server=NPCyclic(initial_model="initial_model.npy", num_rounds=2), + client=NPTrainer(delta=1.0), + ) + recipe.set_server_resource_dirs({"data": "/Users/yanc/NVFlare/sandbox/data"}) + + recipe.export(JOB_ROOT_DIR) + + +if __name__ == "__main__": + main() From 88cb0cf7d537b12ac954e0cd0dc2554784e3469d Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Wed, 3 Dec 2025 14:41:28 -0500 Subject: [PATCH 079/102] support target call opt for single call --- nvflare/fox/api/call_opt.py | 8 ++++---- nvflare/fox/api/group.py | 14 +++++++------- nvflare/fox/api/proxy.py | 17 ++++++++++++++++- nvflare/fox/api/proxy_list.py | 6 +++--- .../examples/np/algos/strategies/avg_para.py | 2 +- nvflare/fox/examples/np/algos/swarm.py | 2 +- 6 files changed, 32 insertions(+), 17 deletions(-) diff --git a/nvflare/fox/api/call_opt.py b/nvflare/fox/api/call_opt.py index a19af734e2..094ba2279d 100644 --- a/nvflare/fox/api/call_opt.py +++ b/nvflare/fox/api/call_opt.py @@ -20,7 +20,7 @@ def __init__( timeout: float = 5.0, secure: bool = False, optional: bool = False, - collab_obj_name=None, + target=None, ): """CallOption defines behavior of a collab call. @@ -30,17 +30,17 @@ def __init__( timeout: when expecting result, the max number of secs to wait for result. secure: whether to use P2P secure messaging. optional: whether the call is optional. - collab_obj_name: name of the collab object to be called. + target: name of the collab object to be called. """ self.expect_result = expect_result self.blocking = blocking self.timeout = timeout self.secure = secure self.optional = optional - self.collab_obj_name = collab_obj_name + self.target = target def __str__(self): return ( f"expect_result={self.expect_result} blocking={self.blocking} timeout={self.timeout} " - f"secure={self.secure} optional={self.optional} collab_obj={self.collab_obj_name}" + f"secure={self.secure} optional={self.optional} target={self.target}" ) diff --git a/nvflare/fox/api/group.py b/nvflare/fox/api/group.py index f6f775a577..ac830772b6 100644 --- a/nvflare/fox/api/group.py +++ b/nvflare/fox/api/group.py @@ -80,13 +80,13 @@ def members(self): """ return self._proxies - def _get_work_proxy(self, p): - if self._call_opt.collab_obj_name: - child = p.get_child(self._call_opt.collab_obj_name) + def _get_work_proxy(self, p, func_name): + if self._call_opt.target: + child = p.get_child(self._call_opt.target) if not child: raise RuntimeError( - f"site {p.name} does not have collab object named '{self._call_opt.collab_obj_name}': " - f"make sure to use correct collab_obj_name in the group call." + f"site {p.name} does not have collab target named '{self._call_opt.target}': " + f"make sure to use correct target in the group call of '{func_name}'." ) return child else: @@ -113,7 +113,7 @@ def method(*args, **kwargs): the_backend = None try: # filter once for all targets - p = self._get_work_proxy(self._proxies[0]) + p = self._get_work_proxy(self._proxies[0], func_name) # func_proxy is the proxy that actually has the func. # the func_proxy is either "p" or a child of "p". @@ -134,7 +134,7 @@ def method(*args, **kwargs): waiter = ResultWaiter([p.name for p in self._proxies]) for p in self._proxies: - p = self._get_work_proxy(p) + p = self._get_work_proxy(p, func_name) func_proxy, func_itf, call_args, call_kwargs = p.adjust_func_args( func_name, adj_args, adj_kwargs ) diff --git a/nvflare/fox/api/proxy.py b/nvflare/fox/api/proxy.py index bf6638c2d1..2d81f041a8 100644 --- a/nvflare/fox/api/proxy.py +++ b/nvflare/fox/api/proxy.py @@ -30,6 +30,7 @@ def __init__( timeout: float = 5.0, optional: bool = False, secure: bool = False, + target: str = None, ): self.proxy = proxy self.call_opt = CallOption( @@ -38,6 +39,7 @@ def __init__( timeout=timeout, optional=optional, secure=secure, + target=target, ) def __getattr__(self, func_name): @@ -66,6 +68,7 @@ def __call__( timeout: float = 5.0, optional: bool = False, secure: bool = False, + target: str = None, ): """This is called when the proxy is used with call options. @@ -74,6 +77,7 @@ def __call__( timeout: optional: secure: + target: Returns: @@ -84,6 +88,7 @@ def __call__( timeout=timeout, optional=optional, secure=secure, + target=target, ) @property @@ -196,7 +201,17 @@ def call_func(self, call_opt: CallOption, func_name, args, kwargs): """ try: - p, func_itf, call_args, call_kwargs = self.adjust_func_args(func_name, args, kwargs) + if call_opt.target: + p = self.get_child(call_opt.target) + if not p: + raise RuntimeError( + f"site {self.name} does not have collab target named '{call_opt.target}': " + f"make sure to use correct target when calling '{func_name}'." + ) + else: + p = self + + p, func_itf, call_args, call_kwargs = p.adjust_func_args(func_name, args, kwargs) with p.app.new_context(self.caller_name, self.name) as ctx: # apply outgoing call filters diff --git a/nvflare/fox/api/proxy_list.py b/nvflare/fox/api/proxy_list.py index 3472c68c97..ad69d94e1d 100644 --- a/nvflare/fox/api/proxy_list.py +++ b/nvflare/fox/api/proxy_list.py @@ -49,7 +49,7 @@ def __call__( timeout: float = 5.0, optional: bool = False, secure: bool = False, - collab_obj_name=None, + target=None, process_resp_cb=None, **cb_kwargs, ): @@ -61,7 +61,7 @@ def __call__( timeout: optional: secure: - collab_obj_name: + target: process_resp_cb: **cb_kwargs: @@ -77,7 +77,7 @@ def __call__( timeout=timeout, optional=optional, secure=secure, - collab_obj_name=collab_obj_name, + target=target, ), process_resp_cb=process_resp_cb, **cb_kwargs, diff --git a/nvflare/fox/examples/np/algos/strategies/avg_para.py b/nvflare/fox/examples/np/algos/strategies/avg_para.py index 2a7a7ea23c..a466f9bcb6 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_para.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_para.py @@ -45,7 +45,7 @@ def _do_eval(self, model): def _do_one_round(self, r, current_model): total = 0 - results = fox.clients(timeout=4, blocking=False, collab_obj_name="client").train(r, current_model) + results = fox.clients(timeout=4, blocking=False, target="client").train(r, current_model) for n, v in results: self.logger.info(f"[{fox.call_info}] round {r}: got group result from client {n}: {v}") total += v diff --git a/nvflare/fox/examples/np/algos/swarm.py b/nvflare/fox/examples/np/algos/swarm.py index 04916db4de..cf73268335 100644 --- a/nvflare/fox/examples/np/algos/swarm.py +++ b/nvflare/fox/examples/np/algos/swarm.py @@ -37,7 +37,7 @@ def execute(self): # randomly pick a client to start start_client_idx = random.randint(0, len(fox.clients) - 1) start_client = fox.clients[start_client_idx] - start_client.start(self.num_rounds, self._initial_model) + start_client(target="client").start(self.num_rounds, self._initial_model) while not fox.is_aborted: if self.waiter.wait(timeout=0.5): break From 912c510b3341c3d0e94840fcfa9511a52867e47b Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Fri, 5 Dec 2025 14:45:53 -0500 Subject: [PATCH 080/102] added traffic control --- nvflare/fox/api/call_opt.py | 9 +++- nvflare/fox/api/gcc.py | 15 ++++++ nvflare/fox/api/group.py | 22 +++++++- nvflare/fox/api/proxy_list.py | 3 ++ nvflare/fox/examples/np/algos/client.py | 12 +++-- .../examples/np/algos/strategies/avg_para.py | 6 +++ .../np/algos/strategies/avg_para_tc.py | 51 +++++++++++++++++++ nvflare/fox/examples/np/fed_avg_para_tc.py | 46 +++++++++++++++++ .../examples/np/recipes/recipe_fed_avg_tc.py | 37 ++++++++++++++ nvflare/fox/sim/backend.py | 2 + nvflare/fox/sys/backend.py | 25 ++++++++- nvflare/fuel/f3/cellnet/cell.py | 12 ++++- 12 files changed, 231 insertions(+), 9 deletions(-) create mode 100644 nvflare/fox/examples/np/algos/strategies/avg_para_tc.py create mode 100644 nvflare/fox/examples/np/fed_avg_para_tc.py create mode 100644 nvflare/fox/examples/np/recipes/recipe_fed_avg_tc.py diff --git a/nvflare/fox/api/call_opt.py b/nvflare/fox/api/call_opt.py index 094ba2279d..a38aa5c304 100644 --- a/nvflare/fox/api/call_opt.py +++ b/nvflare/fox/api/call_opt.py @@ -21,6 +21,7 @@ def __init__( secure: bool = False, optional: bool = False, target=None, + parallel=0, ): """CallOption defines behavior of a collab call. @@ -31,6 +32,7 @@ def __init__( secure: whether to use P2P secure messaging. optional: whether the call is optional. target: name of the collab object to be called. + parallel: number of parallel outgoing messages. """ self.expect_result = expect_result self.blocking = blocking @@ -38,9 +40,14 @@ def __init__( self.secure = secure self.optional = optional self.target = target + self.parallel = parallel + + if not self.expect_result: + # fire and forget - no need to control parallel + self.parallel = 0 def __str__(self): return ( f"expect_result={self.expect_result} blocking={self.blocking} timeout={self.timeout} " - f"secure={self.secure} optional={self.optional} target={self.target}" + f"secure={self.secure} optional={self.optional} target={self.target} parallel={self.parallel}" ) diff --git a/nvflare/fox/api/gcc.py b/nvflare/fox/api/gcc.py index 77b1ebe57f..40c6babae1 100644 --- a/nvflare/fox/api/gcc.py +++ b/nvflare/fox/api/gcc.py @@ -85,8 +85,17 @@ def __init__(self, sites: list[str]): super().__init__() self.sites = sites self.results = ResultQueue(len(sites)) + self.in_sending_count = 0 self.lock = threading.Lock() + def inc_sending(self): + with self.lock: + self.in_sending_count += 1 + + def dec_sending(self): + with self.lock: + self.in_sending_count -= 1 + @staticmethod def _get_site_name(target_name: str): # target_name is either or . @@ -139,8 +148,14 @@ def __init__( self.cb_kwargs = cb_kwargs self.context = context self.waiter = waiter + self.send_complete_cb = None + self.cb_kwargs = {} self.logger = get_obj_logger(self) + def set_send_complete_cb(self, cb, **cb_kwargs): + self.send_complete_cb = cb + self.cb_kwargs = cb_kwargs + def set_result(self, result): """This is called by the backend to set the result received from the remote app. If process_cb is available, it will be called with the result from the remote app. diff --git a/nvflare/fox/api/group.py b/nvflare/fox/api/group.py index ac830772b6..8efcd825c7 100644 --- a/nvflare/fox/api/group.py +++ b/nvflare/fox/api/group.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import copy +import time from typing import List from nvflare.apis.fl_exception import RunAborted @@ -133,6 +134,10 @@ def method(*args, **kwargs): check_call_args(func_name, func_itf, adj_args, adj_kwargs) waiter = ResultWaiter([p.name for p in self._proxies]) + max_parallel = self._call_opt.parallel + if max_parallel <= 0: + max_parallel = len(self._proxies) + for p in self._proxies: p = self._get_work_proxy(p, func_name) func_proxy, func_itf, call_args, call_kwargs = p.adjust_func_args( @@ -155,7 +160,18 @@ def method(*args, **kwargs): context=ctx, waiter=waiter, ) - func_proxy.backend.call_target_in_group(gcc, func_name, *call_args, **call_kwargs) + + gcc.set_send_complete_cb(self._request_sent, waiter=waiter) + + while True: + in_sending = waiter.in_sending_count + if in_sending < max_parallel: + waiter.inc_sending() + func_proxy.backend.call_target_in_group(gcc, func_name, *call_args, **call_kwargs) + break + else: + # self._logger.debug(f"requests being sent: {in_sending} - wait for runway") + time.sleep(0.1) if not self._call_opt.expect_result: # do not wait for responses @@ -183,6 +199,10 @@ def method(*args, **kwargs): return method + def _request_sent(self, waiter: ResultWaiter): + self._logger.info("received _request_sent ...") + waiter.dec_sending() + def group( ctx: Context, diff --git a/nvflare/fox/api/proxy_list.py b/nvflare/fox/api/proxy_list.py index ad69d94e1d..b41474c5e8 100644 --- a/nvflare/fox/api/proxy_list.py +++ b/nvflare/fox/api/proxy_list.py @@ -50,6 +50,7 @@ def __call__( optional: bool = False, secure: bool = False, target=None, + parallel=0, process_resp_cb=None, **cb_kwargs, ): @@ -62,6 +63,7 @@ def __call__( optional: secure: target: + parallel: process_resp_cb: **cb_kwargs: @@ -78,6 +80,7 @@ def __call__( optional=optional, secure=secure, target=target, + parallel=parallel, ), process_resp_cb=process_resp_cb, **cb_kwargs, diff --git a/nvflare/fox/examples/np/algos/client.py b/nvflare/fox/examples/np/algos/client.py index 96eb9ae5d2..154ab9a6c3 100644 --- a/nvflare/fox/examples/np/algos/client.py +++ b/nvflare/fox/examples/np/algos/client.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import random +import time from nvflare.fox import fox from nvflare.fuel.utils.log_utils import get_obj_logger @@ -19,8 +20,9 @@ class NPTrainer: - def __init__(self, delta: float): + def __init__(self, delta: float, delay=0): self.delta = delta + self.delay = delay self.logger = get_obj_logger(self) @fox.init @@ -39,11 +41,11 @@ def train(self, current_round, weights): self.logger.debug("training aborted") return 0 self.logger.info(f"[{fox.call_info}] training round {current_round=} {weights=}") - result = fox.server(expect_result=True).fire_event("metrics", {"round": current_round, "y": 10}) - self.logger.info(f"[{fox.call_info}] got event result: {result}") + # result = fox.server(expect_result=True).fire_event("metrics", {"round": current_round, "y": 10}) + # self.logger.info(f"[{fox.call_info}] got event result: {result}") - # force timeout to test timeout handling - # time.sleep(10) + if self.delay > 0: + time.sleep(self.delay) return weights + self.delta @fox.collab diff --git a/nvflare/fox/examples/np/algos/strategies/avg_para.py b/nvflare/fox/examples/np/algos/strategies/avg_para.py index a466f9bcb6..eaa7ab790a 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_para.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_para.py @@ -47,6 +47,12 @@ def _do_one_round(self, r, current_model): total = 0 results = fox.clients(timeout=4, blocking=False, target="client").train(r, current_model) for n, v in results: + # the value 'v' could be an exception! + if isinstance(v, Exception): + # this site encountered problem + self.logger.error(f"[{fox.call_info}] round {r}: got exception from client {n}: {v}") + raise v + self.logger.info(f"[{fox.call_info}] round {r}: got group result from client {n}: {v}") total += v return total / len(results) diff --git a/nvflare/fox/examples/np/algos/strategies/avg_para_tc.py b/nvflare/fox/examples/np/algos/strategies/avg_para_tc.py new file mode 100644 index 0000000000..dec55989a2 --- /dev/null +++ b/nvflare/fox/examples/np/algos/strategies/avg_para_tc.py @@ -0,0 +1,51 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nvflare.fox import fox +from nvflare.fox.examples.np.algos.utils import parse_array_def +from nvflare.fuel.utils.log_utils import get_obj_logger + + +class NPFedAvgParallelWithTrafficControl: + + def __init__(self, initial_model, num_rounds=10, parallel=2): + self.num_rounds = num_rounds + self.initial_model = initial_model + self._initial_model = parse_array_def(initial_model) + self.parallel = parallel + self.logger = get_obj_logger(self) + + @fox.algo + def execute(self): + self.logger.info(f"[{fox.call_info}] Start training for {self.num_rounds} rounds") + current_model = self._initial_model + for i in range(self.num_rounds): + current_model = self._do_one_round(i, current_model) + return current_model + + def _do_one_round(self, r, current_model): + total = 0 + results = fox.clients(timeout=4, blocking=False, target="client", parallel=self.parallel).train( + r, current_model + ) + + for n, v in results: + # the value 'v' could be an exception! + if isinstance(v, Exception): + # this site encountered problem + self.logger.error(f"[{fox.call_info}] round {r}: got exception from client {n}: {v}") + raise v + + self.logger.info(f"[{fox.call_info}] round {r}: got group result from client {n}: {v}") + total += v + return total / len(results) diff --git a/nvflare/fox/examples/np/fed_avg_para_tc.py b/nvflare/fox/examples/np/fed_avg_para_tc.py new file mode 100644 index 0000000000..a96f080daa --- /dev/null +++ b/nvflare/fox/examples/np/fed_avg_para_tc.py @@ -0,0 +1,46 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples.np.algos.client import NPTrainer +from nvflare.fox.examples.np.algos.strategies.avg_para_tc import NPFedAvgParallelWithTrafficControl +from nvflare.fox.examples.np.algos.widgets import MetricReceiver +from nvflare.fox.sim.simulator import Simulator + + +def main(): + simple_logging(logging.DEBUG) + + server = NPFedAvgParallelWithTrafficControl( + initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]], + num_rounds=2, + parallel=3, + ) + + simulator = Simulator( + root_dir="/tmp/fox", + experiment_name="fedavg_para", + server=server, + client=NPTrainer(delta=1.0, delay=1.5), + server_objects={"metric_receiver": MetricReceiver()}, + num_clients=10, + ) + + result = simulator.run() + print(f"Final result: {result}") + + +if __name__ == "__main__": + main() diff --git a/nvflare/fox/examples/np/recipes/recipe_fed_avg_tc.py b/nvflare/fox/examples/np/recipes/recipe_fed_avg_tc.py new file mode 100644 index 0000000000..64417de596 --- /dev/null +++ b/nvflare/fox/examples/np/recipes/recipe_fed_avg_tc.py @@ -0,0 +1,37 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nvflare.fox.examples.np.algos.client import NPTrainer +from nvflare.fox.examples.np.algos.strategies.avg_para_tc import NPFedAvgParallelWithTrafficControl +from nvflare.fox.examples.np.algos.widgets import MetricReceiver +from nvflare.fox.sys.recipe import FoxRecipe + +JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" + + +def main(): + recipe = FoxRecipe( + job_name="fox_fedavg_tc", + server=NPFedAvgParallelWithTrafficControl( + initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], + num_rounds=2, + parallel=2, + ), + client=NPTrainer(delta=1.0, delay=2.0), + server_objects={"metric_receiver": MetricReceiver()}, + ) + recipe.export(JOB_ROOT_DIR) + + +if __name__ == "__main__": + main() diff --git a/nvflare/fox/sim/backend.py b/nvflare/fox/sim/backend.py index 4e3ec1110d..c6038e41d4 100644 --- a/nvflare/fox/sim/backend.py +++ b/nvflare/fox/sim/backend.py @@ -131,6 +131,8 @@ def _run_func_in_group(self, gcc: GroupCallContext, func_name, args, kwargs): try: target_name = gcc.target_name result = self.call_target(target_name, gcc.call_opt, func_name, *args, **kwargs) + if gcc.send_complete_cb: + gcc.send_complete_cb(**gcc.cb_kwargs) gcc.set_result(result) except Exception as ex: gcc.set_exception(ex) diff --git a/nvflare/fox/sys/backend.py b/nvflare/fox/sys/backend.py index 01092e21da..14f00c2ac3 100644 --- a/nvflare/fox/sys/backend.py +++ b/nvflare/fox/sys/backend.py @@ -36,6 +36,19 @@ def __init__(self, manager, engine, caller, cell, target_fqcn, abort_signal, thr self.thread_executor = thread_executor def call_target(self, target_name: str, call_opt: CallOption, func_name: str, *args, **kwargs): + return self._call_target( + target_name=target_name, + call_opt=call_opt, + send_complete_cb=None, + cb_kwargs={}, + func_name=func_name, + *args, + **kwargs, + ) + + def _call_target( + self, target_name: str, call_opt: CallOption, send_complete_cb, cb_kwargs, func_name: str, *args, **kwargs + ): ctx = kwargs.pop(CollabMethodArgName.CONTEXT) set_call_context(ctx) @@ -61,6 +74,8 @@ def call_target(self, target_name: str, call_opt: CallOption, func_name: str, *a secure=call_opt.secure, optional=call_opt.optional, abort_signal=self.abort_signal, + send_complete_cb=send_complete_cb, + **cb_kwargs, ) assert isinstance(reply, Message) rc = reply.get_header(MessageHeaderKey.RETURN_CODE, ReturnCode.OK) @@ -97,7 +112,15 @@ def call_target_in_group(self, gcc: GroupCallContext, func_name: str, *args, **k def _run_func(self, gcc: GroupCallContext, func_name: str, args, kwargs): try: - result = self.call_target(gcc.target_name, gcc.call_opt, func_name, *args, **kwargs) + result = self._call_target( + target_name=gcc.target_name, + call_opt=gcc.call_opt, + func_name=func_name, + send_complete_cb=gcc.send_complete_cb, + cb_kwargs=gcc.cb_kwargs, + *args, + **kwargs, + ) gcc.set_result(result) except Exception as ex: gcc.set_exception(ex) diff --git a/nvflare/fuel/f3/cellnet/cell.py b/nvflare/fuel/f3/cellnet/cell.py index 837548bf89..d6f0c52e1e 100644 --- a/nvflare/fuel/f3/cellnet/cell.py +++ b/nvflare/fuel/f3/cellnet/cell.py @@ -318,6 +318,8 @@ def _send_request( secure=False, optional=False, abort_signal: Signal = None, + send_complete_cb=None, + **cb_kwargs, ): """Stream one request to the target @@ -335,7 +337,9 @@ def _send_request( """ self._encode_message(request, abort_signal) - return self._send_one_request(channel, target, topic, request, timeout, secure, optional, abort_signal) + return self._send_one_request( + channel, target, topic, request, timeout, secure, optional, abort_signal, send_complete_cb, **cb_kwargs + ) def _send_one_request( self, @@ -347,6 +351,8 @@ def _send_one_request( secure=False, optional=False, abort_signal=None, + send_complete_cb=None, + **cb_kwargs, ): req_id = str(uuid.uuid4()) request.add_headers({StreamHeaderKey.STREAM_REQ_ID: req_id}) @@ -368,6 +374,10 @@ def _send_one_request( # sending with progress timeout self.logger.debug(f"{req_id=}: entering sending wait {timeout=}") sending_complete = self._future_wait(future, timeout, abort_signal) + + if send_complete_cb: + send_complete_cb(**cb_kwargs) + if not sending_complete: self.logger.debug(f"{req_id=}: sending timeout {timeout=}") return self._get_result(req_id) From fe4273d15cef53310f658e54fb5ee9e07715d8af Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Fri, 5 Dec 2025 17:22:26 -0500 Subject: [PATCH 081/102] add licese text --- nvflare/fox/examples/np/recipes/__init__.py | 13 ++++++++ .../examples/np/recipes/recipe_fed_avg_h.py | 33 +++++++++++++++++++ nvflare/fox/examples/pt/__init__.py | 13 ++++++++ 3 files changed, 59 insertions(+) create mode 100644 nvflare/fox/examples/np/recipes/recipe_fed_avg_h.py diff --git a/nvflare/fox/examples/np/recipes/__init__.py b/nvflare/fox/examples/np/recipes/__init__.py index e69de29bb2..341a77c5bc 100644 --- a/nvflare/fox/examples/np/recipes/__init__.py +++ b/nvflare/fox/examples/np/recipes/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nvflare/fox/examples/np/recipes/recipe_fed_avg_h.py b/nvflare/fox/examples/np/recipes/recipe_fed_avg_h.py new file mode 100644 index 0000000000..9a37ada0c1 --- /dev/null +++ b/nvflare/fox/examples/np/recipes/recipe_fed_avg_h.py @@ -0,0 +1,33 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nvflare.fox.examples.np.algos.client import NPHierarchicalTrainer +from nvflare.fox.examples.np.algos.strategies.avg_h import NPHierarchicalFedAvg +from nvflare.fox.examples.np.algos.widgets import MetricReceiver +from nvflare.fox.sys.recipe import FoxRecipe + +JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" + + +def main(): + recipe = FoxRecipe( + job_name="fox_fedavg_h", + server=NPHierarchicalFedAvg(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), + client=NPHierarchicalTrainer(delta=1.0), + server_objects={"metric_receiver": MetricReceiver()}, + ) + recipe.export(JOB_ROOT_DIR) + + +if __name__ == "__main__": + main() diff --git a/nvflare/fox/examples/pt/__init__.py b/nvflare/fox/examples/pt/__init__.py index e69de29bb2..341a77c5bc 100644 --- a/nvflare/fox/examples/pt/__init__.py +++ b/nvflare/fox/examples/pt/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. From 5cc360b2087c266d981b4cafb897523833d8267e Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Fri, 5 Dec 2025 17:25:40 -0500 Subject: [PATCH 082/102] refactor --- nvflare/fox/examples/np/{recipes => }/recipe_fed_avg_h.py | 2 +- nvflare/fox/examples/np/{recipes => }/recipe_fed_avg_tc.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename nvflare/fox/examples/np/{recipes => }/recipe_fed_avg_h.py (93%) rename nvflare/fox/examples/np/{recipes => }/recipe_fed_avg_tc.py (94%) diff --git a/nvflare/fox/examples/np/recipes/recipe_fed_avg_h.py b/nvflare/fox/examples/np/recipe_fed_avg_h.py similarity index 93% rename from nvflare/fox/examples/np/recipes/recipe_fed_avg_h.py rename to nvflare/fox/examples/np/recipe_fed_avg_h.py index 9a37ada0c1..e15a861080 100644 --- a/nvflare/fox/examples/np/recipes/recipe_fed_avg_h.py +++ b/nvflare/fox/examples/np/recipe_fed_avg_h.py @@ -16,7 +16,7 @@ from nvflare.fox.examples.np.algos.widgets import MetricReceiver from nvflare.fox.sys.recipe import FoxRecipe -JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" +JOB_ROOT_DIR = "/sandbox/v27/prod_00/admin@nvidia.com/transfer" def main(): diff --git a/nvflare/fox/examples/np/recipes/recipe_fed_avg_tc.py b/nvflare/fox/examples/np/recipe_fed_avg_tc.py similarity index 94% rename from nvflare/fox/examples/np/recipes/recipe_fed_avg_tc.py rename to nvflare/fox/examples/np/recipe_fed_avg_tc.py index 64417de596..ecc8d67334 100644 --- a/nvflare/fox/examples/np/recipes/recipe_fed_avg_tc.py +++ b/nvflare/fox/examples/np/recipe_fed_avg_tc.py @@ -16,7 +16,7 @@ from nvflare.fox.examples.np.algos.widgets import MetricReceiver from nvflare.fox.sys.recipe import FoxRecipe -JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" +JOB_ROOT_DIR = "/sandbox/v27/prod_00/admin@nvidia.com/transfer" def main(): From 4d6c691efdf464b30c7a132b695fdc2204a7500f Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Thu, 11 Dec 2025 18:05:00 -0500 Subject: [PATCH 083/102] removed unused import --- nvflare/fox/api/facade.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nvflare/fox/api/facade.py b/nvflare/fox/api/facade.py index 79fdcda692..12f8625414 100644 --- a/nvflare/fox/api/facade.py +++ b/nvflare/fox/api/facade.py @@ -11,8 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import copy - from .constants import ContextKey from .ctx import get_call_context from .dec import algo as dec_algo From 0df0850774070ee4c5f355a8f65a33d95b08d911 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Thu, 11 Dec 2025 18:23:35 -0500 Subject: [PATCH 084/102] sync with new downloader --- .../decomposers/numpy_decomposers.py | 3 +- nvflare/app_common/np/np_downloader.py | 12 +- nvflare/app_opt/pt/decomposers.py | 2 +- nvflare/app_opt/pt/tensor_downloader.py | 8 +- nvflare/fuel/f3/streaming/cacheable.py | 45 +- nvflare/fuel/f3/streaming/download_service.py | 18 +- nvflare/fuel/f3/streaming/file_downloader.py | 23 +- nvflare/fuel/f3/streaming/obj_downloader.py | 4 +- nvflare/fuel/hci/server/binary_transfer.py | 5 +- .../utils/fobs/decomposers/via_downloader.py | 18 +- .../fuel/utils/fobs/decomposers/via_file.py | 532 ------------------ 11 files changed, 101 insertions(+), 569 deletions(-) delete mode 100644 nvflare/fuel/utils/fobs/decomposers/via_file.py diff --git a/nvflare/app_common/decomposers/numpy_decomposers.py b/nvflare/app_common/decomposers/numpy_decomposers.py index 0af2129fbd..5268fca22a 100644 --- a/nvflare/app_common/decomposers/numpy_decomposers.py +++ b/nvflare/app_common/decomposers/numpy_decomposers.py @@ -27,6 +27,8 @@ from nvflare.fuel.utils.fobs.datum import DatumManager from nvflare.fuel.utils.fobs.decomposers.via_downloader import ViaDownloaderDecomposer +_NPZ_EXTENSION = ".npz" + class NumpyScalarDecomposer(fobs.Decomposer, ABC): """Decomposer base class for all numpy types with item method.""" @@ -61,7 +63,6 @@ def supported_type(self): class NumpyArrayDecomposer(ViaDownloaderDecomposer): def __init__(self): - # by default do not use file downloading. ViaDownloaderDecomposer.__init__(self, 1024 * 1024 * 2, "np_") def supported_type(self): diff --git a/nvflare/app_common/np/np_downloader.py b/nvflare/app_common/np/np_downloader.py index 8ebe5a0b5b..401dbcc068 100644 --- a/nvflare/app_common/np/np_downloader.py +++ b/nvflare/app_common/np/np_downloader.py @@ -21,6 +21,8 @@ from nvflare.fuel.f3.streaming.download_service import download_object from nvflare.fuel.f3.streaming.obj_downloader import ObjectDownloader +_TWO_MB = 2 * 1024 * 1024 + class ArrayDownloadable(CacheableObject): @@ -32,7 +34,7 @@ def __init__(self, arrays: dict[str, np.ndarray], max_chunk_size: int): def get_item_count(self) -> int: return self.size - def produce_item(self, index: int) -> Any: + def produce_item(self, index: int) -> bytes: key = self.keys[index] arrays_to_send = {key: self.base_obj[key]} stream = BytesIO() @@ -82,16 +84,16 @@ def consume_items(self, items: List[Any], result: Any) -> Any: def add_arrays( downloader: ObjectDownloader, arrays: dict[str, np.ndarray], - max_chunk_size: int = 1, + max_chunk_size: int = _TWO_MB, ) -> str: """Add arrays to be downloaded to the specified downloader. Args: - downloader: the downloader to add tensors to. + downloader: the downloader to add arrays to. arrays: arrays to be downloaded max_chunk_size: max chunk size - Returns: reference id for the state dict. + Returns: reference id for the arrays. """ obj = ArrayDownloadable(arrays, max_chunk_size) @@ -117,7 +119,7 @@ def download_arrays( per_request_timeout: timeout for requests sent to the data source. cell: cell to be used for communicating to the data source. secure: P2P private mode for communication - optional: supress log messages of communication + optional: suppress log messages of communication abort_signal: signal for aborting download. arrays_received_cb: the callback to be called when one set of arrays are received diff --git a/nvflare/app_opt/pt/decomposers.py b/nvflare/app_opt/pt/decomposers.py index 4ea6b57b46..981c4e96ba 100644 --- a/nvflare/app_opt/pt/decomposers.py +++ b/nvflare/app_opt/pt/decomposers.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Optional, Tuple +from typing import Tuple import torch from safetensors.torch import load, save diff --git a/nvflare/app_opt/pt/tensor_downloader.py b/nvflare/app_opt/pt/tensor_downloader.py index 289ad6beb4..02bbf402d6 100644 --- a/nvflare/app_opt/pt/tensor_downloader.py +++ b/nvflare/app_opt/pt/tensor_downloader.py @@ -22,6 +22,8 @@ from nvflare.fuel.f3.streaming.download_service import download_object from nvflare.fuel.f3.streaming.obj_downloader import ObjectDownloader +_TWO_MB = 2 * 1024 * 1024 + class TensorDownloadable(CacheableObject): @@ -33,7 +35,7 @@ def __init__(self, tensors: dict[str, torch.Tensor], max_chunk_size: int): def get_item_count(self) -> int: return self.size - def produce_item(self, index: int) -> Any: + def produce_item(self, index: int) -> bytes: key = self.keys[index] tensor_to_send = {key: self.base_obj[key]} return save_tensors(tensor_to_send) @@ -72,9 +74,9 @@ def consume_items(self, items: List[Any], result: Any) -> Any: def add_tensors( downloader: ObjectDownloader, tensors: dict[str, torch.Tensor], - max_chunk_size: int = 1, + max_chunk_size: int = _TWO_MB, ) -> str: - """Add a file to be downloaded to the specified downloader. + """Add tensors to be downloaded to the specified downloader. Args: downloader: the downloader to add tensors to. diff --git a/nvflare/fuel/f3/streaming/cacheable.py b/nvflare/fuel/f3/streaming/cacheable.py index 3cf007046f..aa96ef822f 100644 --- a/nvflare/fuel/f3/streaming/cacheable.py +++ b/nvflare/fuel/f3/streaming/cacheable.py @@ -26,8 +26,21 @@ class _StateKey: class CacheableObject(Downloadable): + """This class provides cache capability for managing chunks generated during streaming. + When the object is to be sent to multiple sites, each chunk is generated only once and cached for other + sites. Once all sites received the chunk, it's removed from the cache. + + """ def __init__(self, obj: Any, max_chunk_size: int): + """Constructor of CacheableObject. + + Args: + obj: the object to be downloaded. + max_chunk_size: max number of bytes for each chunk. + + Notes: The object must be able to be divided into multiple items. A chunk is generated for each item. + """ super().__init__(obj) check_non_negative_int("max_chunk_size", max_chunk_size) self.max_chunk_size = max_chunk_size @@ -39,10 +52,23 @@ def __init__(self, obj: Any, max_chunk_size: int): @abstractmethod def get_item_count(self) -> int: + """The subclass must implement this method to return the number of items the object contains. + + Returns: the number of items the object contains + + """ pass @abstractmethod - def produce_item(self, index: int) -> Any: + def produce_item(self, index: int) -> bytes: + """This method is called to produce the chunk for the specified item. + + Args: + index: index of the item. + + Returns: a chunk for the item + + """ pass def set_transaction(self, tx_id, ref_id): @@ -63,17 +89,27 @@ def clear_cache(self): def _get_item(self, index: int, requester: str) -> bytes: with self.lock: - data, _ = self.cache[index] + if not self.cache: + # the cache has been cleared + data = None + else: + data, _ = self.cache[index] + if data is None: data = self.produce_item(index) - self.cache[index] = (data, 0) - self.logger.info(f"created and cached item {index} for {requester}: {len(data)} bytes") + if self.cache: + self.cache[index] = (data, 0) + self.logger.info(f"created and cached item {index} for {requester}: {len(data)} bytes") else: self.logger.info(f"got item {index} from cache for {requester}") return data def _adjust_cache(self, start: int, count: int): with self.lock: + if not self.cache: + # cache has been cleared + return + for i in range(start, start + count): data, num_received = self.cache[i] num_received += 1 @@ -122,6 +158,7 @@ def __init__(self): self.error = None self.result = None + @abstractmethod def consume_items(self, items: List[Any], result: Any) -> Any: """Process items and return updated result.""" pass diff --git a/nvflare/fuel/f3/streaming/download_service.py b/nvflare/fuel/f3/streaming/download_service.py index b76089d9f4..3620053e9d 100644 --- a/nvflare/fuel/f3/streaming/download_service.py +++ b/nvflare/fuel/f3/streaming/download_service.py @@ -25,8 +25,8 @@ from nvflare.fuel.utils.log_utils import get_obj_logger from nvflare.security.logging import secure_format_exception -OBJ_DOWNLOADER_CHANNEL = "obj_downloader__" -OBJ_DOWNLOADER_TOPIC = "obj_downloader__download" +OBJ_DOWNLOADER_CHANNEL = "download_service__" +OBJ_DOWNLOADER_TOPIC = "download_service__download" """ This package provides a framework for building object downloading capability (file download, tensor download, etc.). @@ -194,15 +194,15 @@ class ProduceRC: class DownloadStatus: - """Constants for object download status. - """ + """Constants for object download status.""" + SUCCESS = "success" FAILED = "failed" class TransactionDoneStatus: - """Constants for transaction completion status. - """ + """Constants for transaction completion status.""" + FINISHED = "finished" TIMEOUT = "timeout" DELETED = "deleted" @@ -359,7 +359,7 @@ def add_object( obj: Downloadable, ref_id=None, ) -> str: - if not issubclass(type(obj), Downloadable): + if not isinstance(obj, Downloadable): raise ValueError(f"obj must be of type {Downloadable} but got {type(obj)}") tx = cls._tx_table.get(transaction_id) @@ -555,10 +555,10 @@ def download_object( from_fqcn: the FQCN of the object owner ref_id: reference id of the object to be downloaded per_request_timeout: timeout for each request to the object owner. - cell: the cell to be used for communication withe object owner. + cell: the cell to be used for communication with the object owner. consumer: the Consumer object used for processing received data secure: use P2P private communication with the data owner - optional: supress log messages + optional: suppress log messages abort_signal: for signaling abort Returns: None diff --git a/nvflare/fuel/f3/streaming/file_downloader.py b/nvflare/fuel/f3/streaming/file_downloader.py index 356a10099c..84948e1cbd 100644 --- a/nvflare/fuel/f3/streaming/file_downloader.py +++ b/nvflare/fuel/f3/streaming/file_downloader.py @@ -43,13 +43,32 @@ def __init__( file_downloaded_cb=None, **cb_kwargs, ): - """This is the "object" to be downloaded. + """Constructor of FileDownloadable. Args: - file_name: name of the file. + file_name: name of the file to be downloaded. + chunk_size: size of each chunk + file_downloaded_cb: if specified, the callback to be called when the file is downloaded to a site. + cb_kwargs: kwargs passed to the CB. + + Notes: The file_downloaded_cb will be called as follows: + + file_downloaded_cb(to_site, status, file_name, **cb_kwargs) + + where: to_site is the name of the site that the file is just downloaded to; + status is a value of DownloadStatus as defined in nvflare.fuel.f3.streaming.download_service; + file_name is the name of the file downloaded. + + The file_downloaded_cb is also called after it's downloaded to all sites. In this case, the value of + "to_site" is empty, and the value of "status" is also empty. + """ super().__init__(file_name) self.name = file_name + + if not (os.path.isfile(file_name) and os.path.exists(file_name)): + raise ValueError(f"file {file_name} does not exist or is not a valid file") + self.size = os.path.getsize(file_name) if not chunk_size: diff --git a/nvflare/fuel/f3/streaming/obj_downloader.py b/nvflare/fuel/f3/streaming/obj_downloader.py index d86149ba68..fba0c21b47 100644 --- a/nvflare/fuel/f3/streaming/obj_downloader.py +++ b/nvflare/fuel/f3/streaming/obj_downloader.py @@ -16,9 +16,7 @@ class ObjectDownloader: - """Defines a universal object downloader that can be used to download any Downloadable objects. - - """ + """Defines a universal object downloader that can be used to download any Downloadable objects.""" def __init__( self, diff --git a/nvflare/fuel/hci/server/binary_transfer.py b/nvflare/fuel/hci/server/binary_transfer.py index e23a3ebaba..f402442116 100644 --- a/nvflare/fuel/hci/server/binary_transfer.py +++ b/nvflare/fuel/hci/server/binary_transfer.py @@ -39,12 +39,15 @@ def download_folder(self, conn: Connection, tx_id: str, folder_name: str): tx_path = self.tx_path(conn, tx_id, folder_name) engine = conn.get_prop(ConnProps.ENGINE) + admin = conn.get_prop(ConnProps.ADMIN_SERVER) + timeout = admin.timeout if admin else 5 + cell = engine.get_cell() source_fqcn = cell.get_fqcn() downloader = ObjectDownloader( num_receivers=1, cell=engine.get_cell(), - timeout=5, + timeout=timeout, transaction_done_cb=self._cleanup_tx, tx_path=tx_path, ) diff --git a/nvflare/fuel/utils/fobs/decomposers/via_downloader.py b/nvflare/fuel/utils/fobs/decomposers/via_downloader.py index 97588bba6d..4b26a74163 100644 --- a/nvflare/fuel/utils/fobs/decomposers/via_downloader.py +++ b/nvflare/fuel/utils/fobs/decomposers/via_downloader.py @@ -42,7 +42,7 @@ class EncType: class _RefKey: - FILE_REF_ID = "file_ref_id" + REF_ID = "ref_id" FQCN = "fqcn" @@ -86,10 +86,10 @@ def __init__(self, max_chunk_size: int, config_var_prefix): @abstractmethod def to_downloadable(self, items: dict, max_chunk_size: int, fobs_ctx: dict) -> Downloadable: - """Dump the items to the file with the specified path + """Convert the items Downloadable object. Args: - items: a dict of items of target object type to be dumped to file + items: a dict of items of target object type to be converted max_chunk_size: max size of one chunk. fobs_ctx: FOBS Context @@ -180,7 +180,9 @@ def decompose(self, target: Any, manager: DatumManager = None) -> Any: self._config_var_name(ConfigVarName.DOWNLOAD_CHUNK_SIZE), self.max_chunk_size, ) - if max_chunk_size <= 0: + fobs_ctx = manager.fobs_ctx + use_native = fobs_ctx.get("native", False) + if max_chunk_size <= 0 or use_native: # use native decompose self.logger.info("using native_decompose") data = self.native_decompose(target, manager) @@ -288,7 +290,7 @@ def _create_datum(self, fobs_ctx: dict): ref = { _RefKey.FQCN: cell.get_fqcn(), - _RefKey.FILE_REF_ID: ref_id, + _RefKey.REF_ID: ref_id, } self.logger.info(f"ViaDownloader: created download ref for target type {self.__class__.__name__}: {ref=}") datum = Datum(datum_type=DatumType.TEXT, value=json.dumps(ref), dot=self.get_download_dot()) @@ -302,7 +304,7 @@ def _finalize_download_tx(self, mgr: DatumManager): if downloadable_objs: downloader = self._create_downloader(fobs_ctx) for ref_id, obj in downloadable_objs: - self.logger.debug("ViaDownloader: adding object to downloader: {ref_id=}") + self.logger.debug(f"ViaDownloader: adding object to downloader: {ref_id=}") downloader.add_object(obj, ref_id=ref_id) def _delete_download_tx_on_msg_root(self, msg_root_id: str, downloader: ObjectDownloader): @@ -381,9 +383,9 @@ def _download_from_remote_cell(self, fobs_ctx: dict, ref: dict): self.logger.error("cannot download from remote cell since cell not available in fobs context") raise RuntimeError("FOBS Protocol Error") - ref_id = ref.get(_RefKey.FILE_REF_ID) + ref_id = ref.get(_RefKey.REF_ID) if not ref_id: - self.logger.error(f"missing {_RefKey.FILE_REF_ID} from {ref}") + self.logger.error(f"missing {_RefKey.REF_ID} from {ref}") raise RuntimeError("FOBS Protocol Error") fqcn = ref.get(_RefKey.FQCN) diff --git a/nvflare/fuel/utils/fobs/decomposers/via_file.py b/nvflare/fuel/utils/fobs/decomposers/via_file.py deleted file mode 100644 index b78b78e35d..0000000000 --- a/nvflare/fuel/utils/fobs/decomposers/via_file.py +++ /dev/null @@ -1,532 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import json -import os -import threading -import time -import uuid -from abc import ABC, abstractmethod -from typing import Any, Optional, Tuple - -import nvflare.fuel.utils.app_config_utils as acu -from nvflare.apis.fl_constant import ConfigVarName -from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey -from nvflare.fuel.f3.streaming.file_downloader import ObjectDownloader, add_file, download_file -from nvflare.fuel.utils import fobs -from nvflare.fuel.utils.fobs.datum import Datum, DatumManager, DatumType -from nvflare.fuel.utils.fobs.lobs import get_datum_dir -from nvflare.fuel.utils.log_utils import get_obj_logger -from nvflare.fuel.utils.msg_root_utils import subscribe_to_msg_root - -_MIN_DOWNLOAD_TIMEOUT = 60 # allow at least 1 minute gap between download activities - - -class EncKey: - TYPE = "type" - DATA = "data" - - -class EncType: - NATIVE = "native" - REF = "ref" - - -class _FileRefKey: - LOCATION = "location" - FILE_REF_ID = "file_ref_id" - FQCN = "fqcn" - FILE_META = "file_meta" - - -class _FileLocation: - REMOTE_CELL = "remote_cell" - - -class _CtxKey: - MSG_ROOT_ID = "msg_root_id" - MSG_ROOT_TTL = "msg_root_ttl" - FILES = "files" # files to be downloaded - FINAL_CB_REGISTERED = "final_cb_registered" - - -class _DecomposeCtx: - - def __init__(self): - self.target_to_item = {} # target_id => item_id - self.target_items = {} # item_id => item value - self.last_item_id = 0 - - # some stats - self.file_creation_time = None - self.file_size = 0 - - def set_file_creation_time(self, duration): - self.file_creation_time = duration - - def set_file_size(self, size): - self.file_size = size - - def add_item(self, item: Any): - target_id = id(item) - item_id = self.target_to_item.get(target_id) - if not item_id: - item_id = f"T{self.last_item_id}" - self.last_item_id += 1 - self.target_items[item_id] = item - self.target_to_item[target_id] = item_id - return item_id, target_id - - def get_item_count(self): - return len(self.target_items) - - -class ViaFileDecomposer(fobs.Decomposer, ABC): - - def __init__(self, min_size_for_file, config_var_prefix): - self.logger = get_obj_logger(self) - self.prefix = self.__class__.__name__ - self.decompose_ctx_key = f"{self.prefix}_dc" # kept in fobs_ctx: each target type has its own DecomposeCtx - self.items_key = f"{self.prefix}_items" # in fobs_ctx: each target type has its own set of items - self.min_size_for_file = min_size_for_file - self.config_var_prefix = config_var_prefix - - @abstractmethod - def dump_to_file(self, items: dict, path: str, fobs_ctx: dict) -> Tuple[Optional[str], Optional[dict]]: - """Dump the items to the file with the specified path - - Args: - items: a dict of items of target object type to be dumped to file - path: the path to the file. - fobs_ctx: FOBS Context - - Returns: a tuple of (file name, meta info) - - The "path" is a temporary file name. You should create the file with the specified name. - However, some frameworks (e.g. numpy) may add a special suffix to the name. In this case, you must return the - modified name. - - The "items" is a dict of target objects. The dict contains all objects of the target type in one payload. - The dict could be very big. You must create a file to contain all the objects. - - """ - pass - - @abstractmethod - def load_from_file(self, path: str, fobs_ctx: dict, meta: dict = None) -> dict: - """Load target object items from the specified file - - Args: - path: the absolute path to the file to be loaded. - fobs_ctx: FOBS Context. - meta: meta info of the file. - - Returns: a dict of target objects. - - You must not delete the file after loading. Management of the file is done by the ViaFile class. - - """ - pass - - def supported_dots(self): - return [self.get_bytes_dot(), self.get_file_dot()] - - @abstractmethod - def get_file_dot(self) -> int: - """Get the Datum Object Type to be used for file ref datum - - Returns: the DOT for file ref datum - - """ - pass - - @abstractmethod - def get_bytes_dot(self) -> int: - """Get the Datum Object Type to be used for bytes datum - - Returns: the DOT for bytes datum - - """ - pass - - @abstractmethod - def native_decompose(self, target: Any, manager: DatumManager = None) -> bytes: - pass - - @abstractmethod - def native_recompose(self, data: bytes, manager: DatumManager = None) -> Any: - pass - - def _get_temp_file_name(self): - datum_dir = get_datum_dir() - return os.path.join(datum_dir, f"{self.prefix}_{uuid.uuid4()}") - - def _create_ref(self, target: Any, manager: DatumManager, fobs_ctx: dict): - # create a reference item for the target object. The ref item represents the target object in - # the serialized payload. - dc = fobs_ctx.get(self.decompose_ctx_key) - item_id, target_id = dc.add_item(target) - if dc.get_item_count() == 1: - # register the post_process callback to further process these items. - # only register cb once! - manager.register_post_cb(self._process_items_to_datum) - return item_id, target_id - - def _create_file(self, fobs_ctx: dict) -> Tuple[str, int, dict]: - dc = fobs_ctx.get(self.decompose_ctx_key) - assert isinstance(dc, _DecomposeCtx) - items = dc.target_items - file_name = self._get_temp_file_name() - try: - self.logger.debug(f"ViaFile: dumping {len(items)} items to file {file_name}") - start = time.time() - new_file_name, meta = self.dump_to_file(items, file_name, fobs_ctx) - end = time.time() - dc.set_file_creation_time(end - start) - if new_file_name: - file_name = new_file_name - except Exception as e: - self.logger.error(f"Error dumping {len(items)} items to file {file_name}: {e}") - raise e - - size = os.path.getsize(file_name) - self.logger.info(f"ViaFile: created file {file_name=} {size=}") - dc.set_file_size(size) - return file_name, size, meta - - @staticmethod - def _determine_msg_root(fobs_ctx: dict): - msg_root_id = fobs_ctx.get(_CtxKey.MSG_ROOT_ID) - msg_root_ttl = fobs_ctx.get(_CtxKey.MSG_ROOT_TTL) - - if not msg_root_id: - # try to get from msg - msg = fobs_ctx.get(fobs.FOBSContextKey.MESSAGE) - if msg: - msg_root_id = msg.get_header(MessageHeaderKey.MSG_ROOT_ID) - msg_root_ttl = msg.get_header(MessageHeaderKey.MSG_ROOT_TTL) - return msg_root_id, msg_root_ttl - - def decompose(self, target: Any, manager: DatumManager = None) -> Any: - if not manager: - # this should never happen - raise RuntimeError("FOBS System Error: missing DatumManager") - - min_size_for_file = acu.get_int_var( - self._config_var_name(ConfigVarName.MIN_FILE_SIZE_FOR_STREAMING), self.min_size_for_file - ) - if min_size_for_file <= 0: - # use native decompose - self.logger.debug("using native_decompose") - data = self.native_decompose(target, manager) - return {EncKey.TYPE: EncType.NATIVE, EncKey.DATA: data} - - fobs_ctx = manager.fobs_ctx - - # Create a DecomposeCtx for this target type. - # Note: there could be multiple target types - each target type has its own DecomposeCtx! - dc = fobs_ctx.get(self.decompose_ctx_key) - if not dc: - dc = _DecomposeCtx() - fobs_ctx[self.decompose_ctx_key] = dc - - item_id, target_id = self._create_ref(target, manager, fobs_ctx) - self.logger.debug(f"ViaFile: created ref for target {target_id}: {item_id}") - return {EncKey.TYPE: EncType.REF, EncKey.DATA: item_id} - - def _create_downloader(self, fobs_ctx: dict): - msg_root_id, msg_root_ttl = self._determine_msg_root(fobs_ctx) - - if msg_root_ttl: - timeout = msg_root_ttl - else: - timeout = _MIN_DOWNLOAD_TIMEOUT - - if timeout < _MIN_DOWNLOAD_TIMEOUT: - timeout = _MIN_DOWNLOAD_TIMEOUT - - self.logger.debug(f"determined: {msg_root_id=} {timeout=}") - - downloader = None - cell = fobs_ctx.get(fobs.FOBSContextKey.CELL) - if cell: - downloader = ObjectDownloader( - num_receivers=1, - cell=cell, - timeout=timeout, - transaction_done_cb=self._delete_download_tx, - ) - - if msg_root_id: - subscribe_to_msg_root( - msg_root_id=msg_root_id, - cb=self._delete_download_tx_on_msg_root, - downloader=downloader, - ) - - return downloader - - def _process_items_to_datum(self, mgr: DatumManager): - """This method is called during serialization after all target items are serialized. - For primary msg, we turn the collected items into a file, and add file info as a Datum to the datum manager. - - Args: - mgr: - - Returns: - - """ - fobs_ctx = mgr.fobs_ctx - dc = fobs_ctx.get(self.decompose_ctx_key) - assert isinstance(dc, _DecomposeCtx) - - # create datum for the collected target items - # This is called once for each target object type! - - # register the final CB to be called after the post_process. - # Note that the post_process (this CB) only generates files but does not create download transaction. - # For large object, file generation could take long time. If we create the download transaction, it may - # become expired even before file generation is done! - # This is why we do the file generation in this CB, and then create the transaction in the final_cb! - final_cb_registered = fobs_ctx.get(_CtxKey.FINAL_CB_REGISTERED) - if not final_cb_registered: - # register final_cb - mgr.register_post_cb(self._finalize_download_tx) - fobs_ctx[_CtxKey.FINAL_CB_REGISTERED] = True - - try: - if not mgr.get_error(): - datum = self._create_datum(fobs_ctx) - mgr.add_datum(datum) - except Exception as ex: - self.logger.error(f"exception creating datum: {ex}") - mgr.set_error(f"exception creating datum in {type(self)}") - - def _config_var_name(self, base_name: str): - return f"{self.config_var_prefix}{base_name}" - - def _create_datum(self, fobs_ctx: dict): - file_name, size, meta = self._create_file(fobs_ctx) - cell = fobs_ctx.get(fobs.FOBSContextKey.CELL) - - min_size_for_file = acu.get_int_var( - self._config_var_name(ConfigVarName.MIN_FILE_SIZE_FOR_STREAMING), self.min_size_for_file - ) - self.logger.debug(f"MIN_FILE_SIZE_FOR_STREAMING={min_size_for_file}") - - if meta: - use_file_dot = True - else: - use_file_dot = cell and size > min_size_for_file - - if not use_file_dot: - # use bytes DOT - self.logger.debug("USE bytes DOT!") - with open(file_name, "rb") as f: - data = f.read() - os.remove(file_name) - datum = Datum(datum_type=DatumType.BLOB, value=data, dot=self.get_bytes_dot()) - else: - self.logger.debug("USE FILE DOT!") - # use file DOT - # keep files in fobs_ctx - files = fobs_ctx.get(_CtxKey.FILES) - if not files: - files = [] - fobs_ctx[_CtxKey.FILES] = files - - # create a new ref id - file_ref_id = str(uuid.uuid4()) - files.append((file_ref_id, file_name)) - - file_ref = { - _FileRefKey.LOCATION: _FileLocation.REMOTE_CELL, - _FileRefKey.FQCN: cell.get_fqcn(), - _FileRefKey.FILE_REF_ID: file_ref_id, - _FileRefKey.FILE_META: meta, - } - self.logger.debug(f"created file ref for target type {self.__class__.__name__}: {file_ref=}") - datum = Datum(datum_type=DatumType.TEXT, value=json.dumps(file_ref), dot=self.get_file_dot()) - return datum - - def _finalize_download_tx(self, mgr: DatumManager): - self.logger.debug("ViaFile: finalizing download tx") - fobs_ctx = mgr.fobs_ctx - files = fobs_ctx.get(_CtxKey.FILES) - - if files: - downloader = self._create_downloader(fobs_ctx) - for file_ref_id, file_name in files: - self.logger.debug(f"ViaFile: adding file to downloader: {file_name=}") - add_file( - downloader=downloader, - file_name=file_name, - ref_id=file_ref_id, - ) - - def _delete_download_tx_on_msg_root(self, msg_root_id: str, downloader: ObjectDownloader): - # this CB is triggered when msg root is deleted. - self.logger.debug(f"deleting download transaction associated with {msg_root_id=}") - downloader.delete_transaction() - - def _delete_download_tx(self, tx_id, status, file_names): - # this CB is triggered when download tx times out or is deleted - self.logger.debug(f"ViaFile: deleting download tx: {tx_id} {status=} {file_names=}") - - # delete all files in the tx - for f in file_names: - self.logger.debug(f"ViaFile: deleting download file {f}") - try: - os.remove(f) - except FileNotFoundError: - pass - - def process_datum(self, datum: Datum, manager: DatumManager): - """This is called by the manager to process a datum that has a DOT. - This happens before the recompose processing. - - The datum contains information about where the data is: - For bytes DOT, the data is included in the datum directly. - For file DOT, the data is in a file, and the location of the file is further specified: - - If the location is local, then the file is on local file system; - - If the location is remote_cell, then the file is on a remote cell, and needs to be downloaded. - - Args: - datum: datum to be processed. - manager: the datum manager. - - Returns: None - - """ - self.logger.debug(f"pre-processing datum {datum.dot=} before recompose") - fobs_ctx = manager.fobs_ctx - - if datum.dot == self.get_bytes_dot(): - # data is in the value - items = self._load_from_bytes(datum.value, fobs_ctx) - else: - # data is in a file - file_ref = json.loads(datum.value) - location = file_ref.get(_FileRefKey.LOCATION) - if location == _FileLocation.REMOTE_CELL: - # file is on remote cell - need to download it - file_path = self._download_from_remote_cell(manager.fobs_ctx, file_ref) - remove_after_loading = True - self.logger.debug(f"got downloaded file path: {file_path}") - else: - raise RuntimeError(f"unsupported file location {location}") - - file_meta = file_ref.get(_FileRefKey.FILE_META, None) - items = self._load_items_from_file(file_path, remove_after_loading, fobs_ctx, file_meta) - fobs_ctx[self.items_key] = items - - def recompose(self, data: Any, manager: DatumManager = None) -> Any: - if not manager: - # should never happen! - raise RuntimeError("missing DatumManager") - - if not isinstance(data, dict): - self.logger.error(f"data to be recomposed should be dict but got {type(data)}") - raise RuntimeError("FOBS protocol error") - - enc_type = data.get(EncKey.TYPE) - data = data.get(EncKey.DATA) - if not data: - self.logger.error("missing 'data' property from the recompose data") - raise RuntimeError("FOBS protocol error") - - if enc_type == EncType.NATIVE: - self.logger.debug("using native_recompose") - return self.native_recompose(data, manager) - elif enc_type != EncType.REF: - self.logger.error(f"invalid enc_type {enc_type} in recompose data") - raise RuntimeError("FOBS protocol error") - - if not isinstance(data, str): - self.logger.error(f"ref data must be str but got {type(data)}") - raise RuntimeError("FOBS protocol error") - - # data is the item id - tid = threading.get_ident() - self.logger.debug(f"{tid=} recomposing data item {data}") - item_id = data - fobs_ctx = manager.fobs_ctx - items = fobs_ctx.get(self.items_key) - self.logger.debug(f"trying to get item for {item_id=} from {type(items)=}") - item = items.get(item_id) - self.logger.debug(f"{tid=} found item {item_id}: {type(item)}") - if item is None: - self.logger.error(f"cannot find item {item_id} from loaded data") - return item - - def _load_from_bytes(self, data: bytes, fobs_ctx: dict): - file_path = self._get_temp_file_name() - with open(file_path, "wb") as f: - f.write(data) - self.logger.debug(f"ViaFile recompose: created temp file {file_path}") - return self._load_items_from_file(file_path, True, fobs_ctx, None) - - def _load_items_from_file(self, file_path: str, remove_after_loading: bool, fobs_ctx: dict, file_meta): - items = self.load_from_file(file_path, fobs_ctx, file_meta) - self.logger.debug(f"items loaded from file {file_path}: {type(items)}") - if not isinstance(items, dict): - self.logger.error(f"items loaded from file should be dict but got {type(items)}") - items = {} - else: - self.logger.debug(f"number of items loaded from file: {len(items)}") - self.logger.debug(f"loaded items: {items.keys()}") - if remove_after_loading: - os.remove(file_path) - return items - - def _download_from_remote_cell(self, fobs_ctx: dict, file_ref: dict): - self.logger.debug(f"trying to download_from_remote_cell for {file_ref=}") - cell = fobs_ctx.get(fobs.FOBSContextKey.CELL) - if not cell: - self.logger.error("cannot download from remote cell since cell not available in fobs context") - raise RuntimeError("FOBS Protocol Error") - - file_ref_id = file_ref.get(_FileRefKey.FILE_REF_ID) - if not file_ref_id: - self.logger.error(f"missing {_FileRefKey.FILE_REF_ID} from {file_ref}") - raise RuntimeError("FOBS Protocol Error") - - fqcn = file_ref.get(_FileRefKey.FQCN) - if not fqcn: - self.logger.error(f"missing {_FileRefKey.FQCN} from {file_ref}") - raise RuntimeError("FOBS Protocol Error") - - req_timeout = fobs_ctx.get(fobs.FOBSContextKey.DOWNLOAD_REQ_TIMEOUT, None) - if not req_timeout: - req_timeout = acu.get_positive_float_var( - self._config_var_name(ConfigVarName.STREAMING_PER_REQUEST_TIMEOUT), 10.0 - ) - self.logger.debug(f"DOWNLOAD_REQ_TIMEOUT={req_timeout}") - - abort_signal = fobs_ctx.get(fobs.FOBSContextKey.ABORT_SIGNAL) - - self.logger.debug(f"trying to download file: {file_ref_id=} {fqcn=}") - err, file_path = download_file( - from_fqcn=fqcn, - ref_id=file_ref_id, - location=get_datum_dir(), - per_request_timeout=req_timeout, - cell=cell, - abort_signal=abort_signal, - ) - if err: - self.logger.error(f"failed to download file from {fqcn} for source {file_ref}: {err}") - raise RuntimeError(f"failed to download file from {fqcn}") - else: - self.logger.debug(f"downloaded file to {file_path}") - return file_path From 7bfb48a54210afa2a0a158b96727d335b9975d7c Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Thu, 11 Dec 2025 18:35:03 -0500 Subject: [PATCH 085/102] removed print --- nvflare/private/fed/client/communicator.py | 2 +- nvflare/private/fed/client/utils.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/nvflare/private/fed/client/communicator.py b/nvflare/private/fed/client/communicator.py index 632ec5a803..272a945fe0 100644 --- a/nvflare/private/fed/client/communicator.py +++ b/nvflare/private/fed/client/communicator.py @@ -463,7 +463,7 @@ def submit_update( timeout = self.timeout parent_fqcn = determine_parent_fqcn(self.client_config, fl_ctx) - self.logger.info(f"submitting update to parent FQCN: {parent_fqcn}") + self.logger.debug(f"submitting update to parent FQCN: {parent_fqcn}") fqcn = FQCN.join([parent_fqcn, job_id]) result = self.cell.send_request( diff --git a/nvflare/private/fed/client/utils.py b/nvflare/private/fed/client/utils.py index 6270d83e2d..f5f2d8fc02 100644 --- a/nvflare/private/fed/client/utils.py +++ b/nvflare/private/fed/client/utils.py @@ -51,8 +51,6 @@ def determine_parent_fqcn(client_config: dict, fl_ctx: FLContext) -> str: """ fox_mode = fl_ctx.get_prop(FLContextKey.FOX_MODE, False) - my_identity = fl_ctx.get_identity_name() - print(f"FOX MODE FOR {my_identity}: {fox_mode}") if fox_mode: return FQCN.ROOT_SERVER From c77124d507cf5cc5282a34e5df3b77be4189833e Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Thu, 11 Dec 2025 18:55:19 -0500 Subject: [PATCH 086/102] fix pr comments --- nvflare/fox/api/gcc.py | 10 ++++++++-- nvflare/fox/sim/backend.py | 3 +-- nvflare/fox/sim/simulator.py | 2 +- nvflare/fox/sys/backend.py | 7 +++++-- nvflare/fox/sys/controller.py | 2 +- nvflare/fox/sys/executor.py | 2 +- nvflare/fox/sys/utils.py | 3 ++- 7 files changed, 19 insertions(+), 10 deletions(-) diff --git a/nvflare/fox/api/gcc.py b/nvflare/fox/api/gcc.py index 40c6babae1..d96c71b2be 100644 --- a/nvflare/fox/api/gcc.py +++ b/nvflare/fox/api/gcc.py @@ -149,12 +149,18 @@ def __init__( self.context = context self.waiter = waiter self.send_complete_cb = None - self.cb_kwargs = {} + self.send_complete_cb_kwargs = {} self.logger = get_obj_logger(self) def set_send_complete_cb(self, cb, **cb_kwargs): + if not callable(cb): + raise ValueError("send_complete_cb must be callable") self.send_complete_cb = cb - self.cb_kwargs = cb_kwargs + self.send_complete_cb_kwargs = cb_kwargs + + def send_completed(self): + if self.send_complete_cb: + self.send_complete_cb(**self.send_complete_cb_kwargs) def set_result(self, result): """This is called by the backend to set the result received from the remote app. diff --git a/nvflare/fox/sim/backend.py b/nvflare/fox/sim/backend.py index c6038e41d4..9b27d3720f 100644 --- a/nvflare/fox/sim/backend.py +++ b/nvflare/fox/sim/backend.py @@ -131,8 +131,7 @@ def _run_func_in_group(self, gcc: GroupCallContext, func_name, args, kwargs): try: target_name = gcc.target_name result = self.call_target(target_name, gcc.call_opt, func_name, *args, **kwargs) - if gcc.send_complete_cb: - gcc.send_complete_cb(**gcc.cb_kwargs) + gcc.send_completed() gcc.set_result(result) except Exception as ex: gcc.set_exception(ex) diff --git a/nvflare/fox/sim/simulator.py b/nvflare/fox/sim/simulator.py index 074d7fb6be..4c55e7ae8f 100644 --- a/nvflare/fox/sim/simulator.py +++ b/nvflare/fox/sim/simulator.py @@ -180,7 +180,7 @@ def run(self): self.abort_signal.trigger(True) result = None finally: - self.thread_executor.shutdown(wait=False, cancel_futures=True) + self.thread_executor.shutdown(wait=True, cancel_futures=True) self.logger.info(f"Experiment results are in {self.exp_dir}") return result diff --git a/nvflare/fox/sys/backend.py b/nvflare/fox/sys/backend.py index 14f00c2ac3..a37afbd70e 100644 --- a/nvflare/fox/sys/backend.py +++ b/nvflare/fox/sys/backend.py @@ -116,8 +116,8 @@ def _run_func(self, gcc: GroupCallContext, func_name: str, args, kwargs): target_name=gcc.target_name, call_opt=gcc.call_opt, func_name=func_name, - send_complete_cb=gcc.send_complete_cb, - cb_kwargs=gcc.cb_kwargs, + send_complete_cb=self._msg_sent, + cb_kwargs={"gcc": gcc}, *args, **kwargs, ) @@ -125,6 +125,9 @@ def _run_func(self, gcc: GroupCallContext, func_name: str, args, kwargs): except Exception as ex: gcc.set_exception(ex) + def _msg_sent(self, gcc: GroupCallContext): + gcc.send_completed() + def handle_exception(self, exception: Exception): fl_ctx = self.engine.new_context() secure_log_traceback(self.logger) diff --git a/nvflare/fox/sys/controller.py b/nvflare/fox/sys/controller.py index 11c49dcd5a..82eaf24eb4 100644 --- a/nvflare/fox/sys/controller.py +++ b/nvflare/fox/sys/controller.py @@ -255,4 +255,4 @@ def process_result_of_unknown_task( pass def stop_controller(self, fl_ctx: FLContext): - self.thread_executor.shutdown(wait=False, cancel_futures=True) + self.thread_executor.shutdown(wait=True, cancel_futures=True) diff --git a/nvflare/fox/sys/executor.py b/nvflare/fox/sys/executor.py index 36f8c0b2f5..04a3ba8b36 100644 --- a/nvflare/fox/sys/executor.py +++ b/nvflare/fox/sys/executor.py @@ -85,7 +85,7 @@ def _handle_end_run(self, event_type: str, fl_ctx: FLContext): if self.client_ctx: self.logger.info(f"finalizing client app {self.client_app.name}") self.client_app.finalize(self.client_ctx) - self.thread_executor.shutdown(wait=False, cancel_futures=True) + self.thread_executor.shutdown(wait=True, cancel_futures=True) def _prepare_server_proxy(self, job_id, cell, collab_interface: dict, abort_signal, fl_ctx: FLContext): server_name = "server" diff --git a/nvflare/fox/sys/utils.py b/nvflare/fox/sys/utils.py index ecd06fe20e..96423096d1 100644 --- a/nvflare/fox/sys/utils.py +++ b/nvflare/fox/sys/utils.py @@ -21,6 +21,7 @@ from nvflare.fuel.f3.cellnet.utils import new_cell_message from nvflare.fuel.f3.message import Message +from ...security.logging import secure_log_traceback from .constants import MSG_CHANNEL, MSG_TOPIC, CallReplyKey, ObjectCallKey @@ -122,5 +123,5 @@ def _call_app_method(request: Message, app: App, logger) -> Message: headers={MessageHeaderKey.RETURN_CODE: ReturnCode.OK}, payload={CallReplyKey.RESULT: result} ) except Exception as ex: - traceback.print_exc() + secure_log_traceback(logger) return _error_reply(f"exception {type(ex)}", logger) From eeb496c4af83e243da354103c19da7c12c967d80 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Fri, 12 Dec 2025 17:20:05 -0500 Subject: [PATCH 087/102] address pr comments --- nvflare/fox/api/app.py | 45 ++++++++-------- nvflare/fox/api/backend.py | 2 +- nvflare/fox/api/call_opt.py | 2 +- nvflare/fox/api/ctx.py | 2 + nvflare/fox/api/facade.py | 5 +- nvflare/fox/api/proxy_list.py | 3 +- nvflare/fox/api/run_server.py | 4 +- nvflare/fox/examples/np/algos/avg_stream.py | 10 +++- nvflare/fox/examples/np/algos/client.py | 1 + nvflare/fox/examples/np/algos/filters.py | 3 +- .../fox/examples/np/algos/strategies/avg_h.py | 10 +++- .../np/algos/strategies/avg_intime.py | 36 ++++++------- .../examples/np/algos/strategies/avg_para.py | 10 +++- .../np/algos/strategies/avg_para_tc.py | 7 ++- nvflare/fox/examples/np/algos/utils.py | 2 +- nvflare/fox/examples/np/fed_avg_stream.py | 2 +- nvflare/fox/examples/np/recipe_fed_avg_h.py | 2 +- .../fox/examples/np/recipe_fed_avg_intime.py | 2 +- nvflare/fox/examples/np/recipe_fed_avg_tc.py | 2 +- nvflare/fox/examples/pt/filters.py | 1 - nvflare/fox/examples/pt/filters2.py | 1 - nvflare/fox/examples/pt/pt_avg_filter.py | 53 ++++++------------- nvflare/fox/examples/pt/pt_avg_mixed.py | 10 ++-- nvflare/fox/examples/pt/pt_avg_stream.py | 19 ++++--- nvflare/fox/examples/pt/pt_avg_stream2.py | 3 +- nvflare/fox/examples/pt/pt_np.py | 9 ++-- nvflare/fox/examples/pt/recipe_pt_np.py | 2 +- nvflare/fox/examples/pt/utils.py | 4 +- nvflare/fox/sim/backend.py | 4 +- nvflare/fox/sim/ws.py | 6 +++ nvflare/fox/sys/adaptor.py | 4 +- nvflare/fox/sys/backend.py | 8 +-- nvflare/fox/sys/controller.py | 3 +- nvflare/fox/sys/downloader.py | 2 +- nvflare/fox/sys/recipe.py | 10 ++-- nvflare/fox/sys/utils.py | 5 +- nvflare/fox/sys/ws.py | 3 +- nvflare/fox/utils/tensor_receiver.py | 2 +- 38 files changed, 156 insertions(+), 143 deletions(-) diff --git a/nvflare/fox/api/app.py b/nvflare/fox/api/app.py index 619989febd..b5a5ba4225 100644 --- a/nvflare/fox/api/app.py +++ b/nvflare/fox/api/app.py @@ -41,12 +41,12 @@ class App: def __init__(self, obj, name: str): self.obj = obj - self.name = None - self.fqn = None - self.server = None - self.clients = None - self.client_hierarchy = None - self.backend_type = None + self.name = name + self._fqn = None + self._server_proxy = None + self._client_proxies = None + self._client_hierarchy = None + self._backend_type = None self._me = None self._collab_objs = {} self._abort_signal = None @@ -84,10 +84,10 @@ def get_workspace(self): return self._workspace def get_server_proxy(self): - return self.server + return self._server_proxy def get_client_proxies(self): - return copy.copy(self.clients) + return copy.copy(self._client_proxies) def _add_filters(self, pattern: str, filters, to_list: list, filter_type, incoming): if not filters: @@ -134,7 +134,8 @@ def add_outgoing_result_filters(self, pattern: str, filters: List[object]): def get_outgoing_result_filters(self): return self._outgoing_result_filter_chains - def _find_filter_chain(self, direction, chains: List[FilterChain], target_name: str, func_name: str, ctx: Context): + @staticmethod + def _find_filter_chain(direction, chains: List[FilterChain], target_name: str, func_name: str, ctx: Context): """ Args: @@ -216,7 +217,7 @@ def add_collab_object(self, name: str, obj): if name in self._collab_objs: raise ValueError(f"conflict with existing collab object '{name}' of {type(self._collab_objs[name])}") - if hasattr(obj, name): + if hasattr(self, name): raise ValueError(f"conflict with reserved name {name}") setattr(self, name, obj) @@ -231,10 +232,10 @@ def setup(self, workspace: Workspace, server: Proxy, clients: List[Proxy], abort self._workspace = workspace workspace.resource_dirs = self._resource_dirs - self.server = server + self._server_proxy = server self._abort_signal = abort_signal - self.clients = clients + self._client_proxies = clients self._me = None if not self.name or self.name == "server": self._me = server @@ -248,7 +249,7 @@ def setup(self, workspace: Workspace, server: Proxy, clients: List[Proxy], abort raise ValueError(f"cannot find site for {self.name}") forest = build_forest(objs=clients, get_fqn_f=lambda c: c.fqn, get_name_f=lambda c: c.name) - self.client_hierarchy = forest + self._client_hierarchy = forest def get_my_site(self) -> Proxy: return self._me @@ -308,7 +309,7 @@ def _fox_finalize(self, obj, ctx: Context): def finalize(self, context: Context): self._fox_finalize(self, context) - # initialize target objects + # finalize target objects for obj in self._managed_objects.values(): self._fox_finalize(obj, context) @@ -354,8 +355,8 @@ def has_children(self): return True def get_leaf_clients(self): - assert isinstance(self.client_hierarchy, Forest) - leaf_nodes = [self.client_hierarchy.nodes[n] for n in self.client_hierarchy.leaves] + assert isinstance(self._client_hierarchy, Forest) + leaf_nodes = [self._client_hierarchy.nodes[n] for n in self._client_hierarchy.leaves] return [node.obj for node in leaf_nodes] @@ -370,8 +371,8 @@ def __init__(self, obj, name: str = "server"): raise ValueError("server object must have at least one algo") def get_children(self): - assert isinstance(self.client_hierarchy, Forest) - root_nodes = [self.client_hierarchy.nodes[n] for n in self.client_hierarchy.roots] + assert isinstance(self._client_hierarchy, Forest) + root_nodes = [self._client_hierarchy.nodes[n] for n in self._client_hierarchy.roots] return [node.obj for node in root_nodes] def has_children(self): @@ -386,8 +387,8 @@ def __init__(self, obj, name: str = "client"): super().__init__(obj, name) def get_children(self): - assert isinstance(self.client_hierarchy, Forest) - my_node = self.client_hierarchy.nodes[self.name] + assert isinstance(self._client_hierarchy, Forest) + my_node = self._client_hierarchy.nodes[self.name] assert isinstance(my_node, Node) if my_node.children: return [node.obj for node in my_node.children] @@ -395,8 +396,8 @@ def get_children(self): return None def has_children(self): - assert isinstance(self.client_hierarchy, Forest) - my_node = self.client_hierarchy.nodes[self.name] + assert isinstance(self._client_hierarchy, Forest) + my_node = self._client_hierarchy.nodes[self.name] assert isinstance(my_node, Node) if my_node.children: return True diff --git a/nvflare/fox/api/backend.py b/nvflare/fox/api/backend.py index 40e64a2161..b81fd75e2a 100644 --- a/nvflare/fox/api/backend.py +++ b/nvflare/fox/api/backend.py @@ -55,7 +55,7 @@ def call_target_in_group(self, gcc: GroupCallContext, func_name: str, *args, **k Args: gcc: contextual information about group call. - func_name: mame of the function to be called in the remote app. + func_name: name of the function to be called in the remote app. *args: args to pass to the target function. **kwargs: kwargs to pass to the target function. diff --git a/nvflare/fox/api/call_opt.py b/nvflare/fox/api/call_opt.py index a38aa5c304..f927d17cff 100644 --- a/nvflare/fox/api/call_opt.py +++ b/nvflare/fox/api/call_opt.py @@ -27,7 +27,7 @@ def __init__( Args: expect_result: whether result is expected from the remote object. - blocking: whether rhe call is blocking. Only for group calls. + blocking: whether the call is blocking. Only for group calls. timeout: when expecting result, the max number of secs to wait for result. secure: whether to use P2P secure messaging. optional: whether the call is optional. diff --git a/nvflare/fox/api/ctx.py b/nvflare/fox/api/ctx.py index b03cb53dcb..cf7b2b7f96 100644 --- a/nvflare/fox/api/ctx.py +++ b/nvflare/fox/api/ctx.py @@ -84,6 +84,8 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): if self.parent_ctx: set_call_context(self.parent_ctx) + else: + set_call_context(None) def get_call_context(): diff --git a/nvflare/fox/api/facade.py b/nvflare/fox/api/facade.py index 12f8625414..351c186bf7 100644 --- a/nvflare/fox/api/facade.py +++ b/nvflare/fox/api/facade.py @@ -67,7 +67,7 @@ def site_name(cls): @classproperty def server(cls): ctx = get_call_context() - return ctx.app.server + return ctx.server @classproperty def clients(cls): @@ -77,6 +77,9 @@ def clients(cls): @classproperty def other_clients(cls): ctx = get_call_context() + + # Note that ctx.clients returns a copy of client proxies, not the original client proxy list! + # So it is safe to manipulate the candidates here. candidates = ctx.clients me = ctx.app.get_my_site() if me in candidates: diff --git a/nvflare/fox/api/proxy_list.py b/nvflare/fox/api/proxy_list.py index b41474c5e8..a37c2a4f49 100644 --- a/nvflare/fox/api/proxy_list.py +++ b/nvflare/fox/api/proxy_list.py @@ -19,8 +19,7 @@ class ProxyList(list): def __init__(self, proxies: list): - super().__init__() - self.extend(proxies) + super().__init__(proxies) def __getattr__(self, func_name): """This is called to invoke the specified func without specifying call option. diff --git a/nvflare/fox/api/run_server.py b/nvflare/fox/api/run_server.py index 8c8eab3234..fa836382b1 100644 --- a/nvflare/fox/api/run_server.py +++ b/nvflare/fox/api/run_server.py @@ -13,6 +13,8 @@ # limitations under the License. import traceback +from nvflare.security.logging import secure_log_traceback + from .app import ServerApp from .constants import CollabMethodArgName, ContextKey from .dec import supports_context @@ -39,7 +41,7 @@ def run_server(server_app: ServerApp, logger): result = f(**kwargs) server_ctx.set_prop(ContextKey.RESULT, result) except Exception as ex: - traceback.print_exc() + secure_log_traceback(logger) backend = server_app.get_backend() backend.handle_exception(ex) break diff --git a/nvflare/fox/examples/np/algos/avg_stream.py b/nvflare/fox/examples/np/algos/avg_stream.py index d0411cfffe..ed3472eed4 100644 --- a/nvflare/fox/examples/np/algos/avg_stream.py +++ b/nvflare/fox/examples/np/algos/avg_stream.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import os.path +import threading import uuid from nvflare.fox import fox @@ -26,6 +27,7 @@ class _AggrResult: def __init__(self): self.total = 0 self.count = 0 + self.lock = threading.Lock() class NPFedAvgStream: @@ -73,6 +75,9 @@ def _do_one_round(self, r, current_model): grp.train(r, model, model_type) if file_name: + # train is a blocking call that does not return until train results (success or not) are received + # from all clients. + # remove the file regardless. os.remove(file_name) if aggr_result.count == 0: @@ -94,8 +99,9 @@ def _accept_train_result(self, gcc, result, aggr_result: _AggrResult): model = load_np_model(file_path) os.remove(file_path) - aggr_result.total += model - aggr_result.count += 1 + with aggr_result.lock: + aggr_result.total += model + aggr_result.count += 1 return None def _model_downloaded(self, to_site: str, status: str, file_name): diff --git a/nvflare/fox/examples/np/algos/client.py b/nvflare/fox/examples/np/algos/client.py index 154ab9a6c3..2048341ebd 100644 --- a/nvflare/fox/examples/np/algos/client.py +++ b/nvflare/fox/examples/np/algos/client.py @@ -13,6 +13,7 @@ # limitations under the License. import random import time +import traceback from nvflare.fox import fox from nvflare.fuel.utils.log_utils import get_obj_logger diff --git a/nvflare/fox/examples/np/algos/filters.py b/nvflare/fox/examples/np/algos/filters.py index 3c68a05383..305fda561b 100644 --- a/nvflare/fox/examples/np/algos/filters.py +++ b/nvflare/fox/examples/np/algos/filters.py @@ -19,9 +19,8 @@ class AddNoiseToModel: - def __init__(self, x=0.1): + def __init__(self): self.logger = get_obj_logger(self) - self.x = x @fox.call_filter def add_noise(self, func_kwargs: dict): diff --git a/nvflare/fox/examples/np/algos/strategies/avg_h.py b/nvflare/fox/examples/np/algos/strategies/avg_h.py index c0416782dd..6226acbbcb 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_h.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_h.py @@ -31,6 +31,9 @@ def execute(self): current_model = self._initial_model for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model) + if current_model is None: + self.logger.error(f"training failed in round {i}") + break score = self._do_eval(current_model) self.logger.info(f"[{fox.call_info}]: eval score in round {i}: {score}") self.logger.info(f"FINAL MODEL: {current_model}") @@ -42,7 +45,8 @@ def _do_eval(self, model): for n, v in results: self.logger.info(f"[{fox.call_info}]: got eval result from client {n}: {v}") total += v - return total / len(results) + num_results = len(results) + return total / len(results) if num_results > 0 else 0.0 def _do_one_round(self, r, current_model): total = 0 @@ -50,4 +54,6 @@ def _do_one_round(self, r, current_model): for n, v in results: self.logger.info(f"[{fox.call_info}] round {r}: got group result from client {n}: {v}") total += v - return total / len(results) + + num_results = len(results) + return total / len(results) if num_results > 0 else None diff --git a/nvflare/fox/examples/np/algos/strategies/avg_intime.py b/nvflare/fox/examples/np/algos/strategies/avg_intime.py index 2466e68dbf..b735da7afe 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_intime.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_intime.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import threading + from nvflare.fox import fox from nvflare.fox.api.constants import ContextKey from nvflare.fox.examples.np.algos.utils import parse_array_def @@ -22,6 +24,7 @@ class _AggrResult: def __init__(self): self.total = 0 self.count = 0 + self.lock = threading.Lock() class NPFedAvgInTime: @@ -40,7 +43,9 @@ def execute(self): current_model = fox.get_prop(ContextKey.RESULT, self._init_model) for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model) - # current_model = self._do_one_round_non_blocking(i, current_model) + if current_model is None: + self.logger.error(f"training failed at round {i}") + break score = self._do_eval(current_model) self.logger.info(f"[{fox.call_info}]: eval score in round {i}: {score}") self.logger.info(f"FINAL MODEL: {current_model}") @@ -52,11 +57,15 @@ def _do_eval(self, model): for n, v in results: self.logger.info(f"[{fox.call_info}]: got eval result from client {n}: {v}") total += v - return total / len(results) + + num_results = len(results) + return total / num_results if num_results > 0 else 0.0 def _do_one_round(self, r, current_model): aggr_result = _AggrResult() - timeout = fox.get_app_prop("default_timeout", 10) + + # try to get the configured timeout value + timeout = fox.get_app_prop("default_timeout", self.timeout) self.logger.info(f"got timeout: {timeout}") fox.clients( timeout=timeout, @@ -71,24 +80,9 @@ def _do_one_round(self, r, current_model): self.logger.info(f"[{fox.call_info}] round {r}: aggr result from {aggr_result.count} clients: {result}") return result - def _do_one_round_non_blocking(self, r, current_model): - timeout = fox.get_app_prop("default_timeout", 10) - self.logger.info(f"got timeout: {timeout}") - results = fox.clients( - timeout=timeout, - blocking=False, - ).train(r, current_model) - - total = 0 - for n, v in results: - self.logger.info(f"[{fox.call_info}] round {r}: got group result from client {n}: {v}") - total += v - result = total / len(results) - self.logger.info(f"[{fox.call_info}] round {r}: aggr result from {len(results)} clients: {result}") - return result - def _accept_train_result(self, gcc, result, aggr_result: _AggrResult): self.logger.info(f"[{fox.call_info}] got train result from {fox.caller} {result}") - aggr_result.total += result - aggr_result.count += 1 + with aggr_result.lock: + aggr_result.total += result + aggr_result.count += 1 return None diff --git a/nvflare/fox/examples/np/algos/strategies/avg_para.py b/nvflare/fox/examples/np/algos/strategies/avg_para.py index eaa7ab790a..041e4f8a2f 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_para.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_para.py @@ -31,6 +31,9 @@ def execute(self): current_model = self._initial_model for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model) + if current_model is None: + self.logger.error(f"training failed at round {i}") + break score = self._do_eval(current_model) self.logger.info(f"[{fox.call_info}]: eval score in round {i}: {score}") return current_model @@ -41,7 +44,9 @@ def _do_eval(self, model): for n, v in results: self.logger.info(f"[{fox.call_info}]: got eval result from client {n}: {v}") total += v - return total / len(results) + + num_results = len(results) + return total / len(results) if num_results > 0 else 0.0 def _do_one_round(self, r, current_model): total = 0 @@ -55,4 +60,5 @@ def _do_one_round(self, r, current_model): self.logger.info(f"[{fox.call_info}] round {r}: got group result from client {n}: {v}") total += v - return total / len(results) + num_results = len(results) + return total / len(results) if num_results > 0 else None diff --git a/nvflare/fox/examples/np/algos/strategies/avg_para_tc.py b/nvflare/fox/examples/np/algos/strategies/avg_para_tc.py index dec55989a2..37e1f1a1c8 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_para_tc.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_para_tc.py @@ -31,6 +31,9 @@ def execute(self): current_model = self._initial_model for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model) + if current_model is None: + self.logger.error(f"training failed at round {i}") + break return current_model def _do_one_round(self, r, current_model): @@ -48,4 +51,6 @@ def _do_one_round(self, r, current_model): self.logger.info(f"[{fox.call_info}] round {r}: got group result from client {n}: {v}") total += v - return total / len(results) + + num_results = len(results) + return total / num_results if num_results > 0 else None diff --git a/nvflare/fox/examples/np/algos/utils.py b/nvflare/fox/examples/np/algos/utils.py index f4b056ff3e..7d355a3249 100644 --- a/nvflare/fox/examples/np/algos/utils.py +++ b/nvflare/fox/examples/np/algos/utils.py @@ -32,7 +32,7 @@ def parse_array_def(array_def): raise ValueError(f"unsupported array def: {array_def}") -def parse_state_dict(d: dict[str, list]): +def parse_state_dict(d): result = {} for k, v in d.items(): result[k] = parse_array_def(v) diff --git a/nvflare/fox/examples/np/fed_avg_stream.py b/nvflare/fox/examples/np/fed_avg_stream.py index 5961da7931..e2cc6a2d82 100644 --- a/nvflare/fox/examples/np/fed_avg_stream.py +++ b/nvflare/fox/examples/np/fed_avg_stream.py @@ -23,7 +23,7 @@ def main(): simulator = Simulator( root_dir="/tmp/fox", - experiment_name="fedavg_intime", + experiment_name="fedavg_stream", server=NPFedAvgStream(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=4), client=NPTrainer(delta=1.0), num_clients=2, diff --git a/nvflare/fox/examples/np/recipe_fed_avg_h.py b/nvflare/fox/examples/np/recipe_fed_avg_h.py index e15a861080..9a37ada0c1 100644 --- a/nvflare/fox/examples/np/recipe_fed_avg_h.py +++ b/nvflare/fox/examples/np/recipe_fed_avg_h.py @@ -16,7 +16,7 @@ from nvflare.fox.examples.np.algos.widgets import MetricReceiver from nvflare.fox.sys.recipe import FoxRecipe -JOB_ROOT_DIR = "/sandbox/v27/prod_00/admin@nvidia.com/transfer" +JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" def main(): diff --git a/nvflare/fox/examples/np/recipe_fed_avg_intime.py b/nvflare/fox/examples/np/recipe_fed_avg_intime.py index c617de7fba..946b269ac3 100644 --- a/nvflare/fox/examples/np/recipe_fed_avg_intime.py +++ b/nvflare/fox/examples/np/recipe_fed_avg_intime.py @@ -31,7 +31,7 @@ def main(): ) print_filter = Print() - recipe.add_server_outgoing_call_filters("*.train", [AddNoiseToModel(0.3)]) + recipe.add_server_outgoing_call_filters("*.train", [AddNoiseToModel()]) recipe.add_server_incoming_result_filters("*.train", [print_filter]) recipe.set_server_prop("default_timeout", 5.0) diff --git a/nvflare/fox/examples/np/recipe_fed_avg_tc.py b/nvflare/fox/examples/np/recipe_fed_avg_tc.py index ecc8d67334..64417de596 100644 --- a/nvflare/fox/examples/np/recipe_fed_avg_tc.py +++ b/nvflare/fox/examples/np/recipe_fed_avg_tc.py @@ -16,7 +16,7 @@ from nvflare.fox.examples.np.algos.widgets import MetricReceiver from nvflare.fox.sys.recipe import FoxRecipe -JOB_ROOT_DIR = "/sandbox/v27/prod_00/admin@nvidia.com/transfer" +JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" def main(): diff --git a/nvflare/fox/examples/pt/filters.py b/nvflare/fox/examples/pt/filters.py index 77eaf3d6e9..58cf7aebc7 100644 --- a/nvflare/fox/examples/pt/filters.py +++ b/nvflare/fox/examples/pt/filters.py @@ -60,7 +60,6 @@ def filter_call(self, func_kwargs: dict, context: Context): self.logger.error(f"error filtering call arg {arg_value}: {err}") else: func_kwargs[self.model_arg_name] = model - return func_kwargs return func_kwargs diff --git a/nvflare/fox/examples/pt/filters2.py b/nvflare/fox/examples/pt/filters2.py index 2fbd2ed66e..1db8eb654e 100644 --- a/nvflare/fox/examples/pt/filters2.py +++ b/nvflare/fox/examples/pt/filters2.py @@ -53,7 +53,6 @@ def download_weights(self, func_kwargs: dict): self.logger.error(f"error filtering call arg {arg_value}: {err}") else: func_kwargs[self.model_arg_name] = model - return func_kwargs return func_kwargs @fox.out_result_filter diff --git a/nvflare/fox/examples/pt/pt_avg_filter.py b/nvflare/fox/examples/pt/pt_avg_filter.py index 168d3acb63..659b3ef324 100644 --- a/nvflare/fox/examples/pt/pt_avg_filter.py +++ b/nvflare/fox/examples/pt/pt_avg_filter.py @@ -12,25 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -import threading - -import torch from nvflare.fox import fox from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples.pt.utils import add as add_pt +from nvflare.fox.examples.pt.utils import div as div_pt from nvflare.fox.examples.pt.utils import parse_state_dict from nvflare.fox.sim.simulator import Simulator from nvflare.fuel.utils.log_utils import get_obj_logger -class _AggrResult: - - def __init__(self): - self.total = {} - self.count = 0 - self.lock = threading.Lock() # ensure update integrity - - class PTFedAvg: def __init__(self, initial_model, num_rounds=10, timeout=2.0): @@ -47,37 +38,23 @@ def execute(self): current_model = self._init_model for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model) + if not current_model: + self.logger.error(f"training failed at round {i}") + break self.logger.info(f"FINAL MODEL: {current_model}") return current_model def _do_one_round(self, r, current_model): - aggr_result = _AggrResult() - - fox.clients( - process_resp_cb=self._accept_train_result, - aggr_result=aggr_result, - ).train(r, current_model) - - if aggr_result.count == 0: - return None - else: - result = {} - for k, v in aggr_result.total.items(): - result[k] = torch.div(v, aggr_result.count) - self.logger.info(f"[{fox.call_info}] round {r}: aggr result from {aggr_result.count} clients: {result}") - return result - - def _accept_train_result(self, gcc, result, aggr_result: _AggrResult): - self.logger.info(f"[{fox.call_info}] got train result from {fox.caller}: {result}") - - for k, v in result.items(): - if k not in aggr_result.total: - aggr_result.total[k] = v - else: - aggr_result.total[k] += v - - aggr_result.count += 1 - return None + aggr_result = {} + + results = fox.clients.train(r, current_model) + for n, v in results: + add_pt(v, aggr_result) + + num_results = len(results) + aggr_result = div_pt(aggr_result, num_results) if num_results > 0 else None + self.logger.info(f"[{fox.call_info}] round {r}: aggr result from {num_results} clients: {aggr_result}") + return aggr_result class PTTrainer: diff --git a/nvflare/fox/examples/pt/pt_avg_mixed.py b/nvflare/fox/examples/pt/pt_avg_mixed.py index f9d5a814fe..52085cb019 100644 --- a/nvflare/fox/examples/pt/pt_avg_mixed.py +++ b/nvflare/fox/examples/pt/pt_avg_mixed.py @@ -84,7 +84,7 @@ def _do_one_round(self, r, pt_model, np_model): grp.train(r, pt_model, np_model, model_type) if aggr_result.count == 0: - return None + return None, None else: pt_result = aggr_result.pt_total div_pt(pt_result, aggr_result.count) @@ -122,10 +122,12 @@ def _accept_train_result(self, gcc, result, aggr_result: _AggrResult): if err: raise RuntimeError(f"failed to download NP model file {np_result}: {err}") else: - add_pt(pt_result, aggr_result.pt_total) - add_np(np_result, aggr_result.np_total) + with aggr_result.lock: + add_pt(pt_result, aggr_result.pt_total) + add_np(np_result, aggr_result.np_total) - aggr_result.count += 1 + with aggr_result.lock: + aggr_result.count += 1 return None def _aggregate_tensors(self, td: dict[str, torch.Tensor], aggr_result: _AggrResult): diff --git a/nvflare/fox/examples/pt/pt_avg_stream.py b/nvflare/fox/examples/pt/pt_avg_stream.py index 129e6baf8d..6e5882821d 100644 --- a/nvflare/fox/examples/pt/pt_avg_stream.py +++ b/nvflare/fox/examples/pt/pt_avg_stream.py @@ -101,13 +101,14 @@ def _accept_train_result(self, gcc, result, aggr_result: _AggrResult): if err: raise RuntimeError(f"failed to download model {model}: {err}") else: - for k, v in model.items(): - if k not in aggr_result.total: - aggr_result.total[k] = v - else: - aggr_result.total[k] += v - - aggr_result.count += 1 + with aggr_result.lock: + for k, v in model.items(): + if k not in aggr_result.total: + aggr_result.total[k] = v + else: + aggr_result.total[k] += v + + aggr_result.count += 1 return None def _aggregate_tensors(self, td: dict[str, torch.Tensor], aggr_result: _AggrResult): @@ -118,6 +119,7 @@ def _aggregate_tensors(self, td: dict[str, torch.Tensor], aggr_result: _AggrResu aggr_result.total[k] = v else: aggr_result.total[k] += v + aggr_result.count += 1 class PTTrainer: @@ -130,7 +132,8 @@ def __init__(self, delta: float): def train(self, current_round, model1, model2, model_type: str): if fox.is_aborted: self.logger.debug("training aborted") - return 0 + return None, "model" + self.logger.debug(f"[{fox.call_info}] training round {current_round}: {model_type=} {model1=} {model2=}") if model_type == "ref": err, model1 = download_tensors(ref=model1, per_request_timeout=5.0) diff --git a/nvflare/fox/examples/pt/pt_avg_stream2.py b/nvflare/fox/examples/pt/pt_avg_stream2.py index d5c07b4e80..e369c302da 100644 --- a/nvflare/fox/examples/pt/pt_avg_stream2.py +++ b/nvflare/fox/examples/pt/pt_avg_stream2.py @@ -100,7 +100,8 @@ def __init__(self, delta: float): def train(self, current_round, model, model_type: str): if fox.is_aborted: self.logger.debug("training aborted") - return 0 + return None, "model" + self.logger.debug(f"[{fox.call_info}] training round {current_round}: {model_type=} {model=}") if model_type == "ref": err, model = download_tensors(ref=model, per_request_timeout=5.0) diff --git a/nvflare/fox/examples/pt/pt_np.py b/nvflare/fox/examples/pt/pt_np.py index 62c49381b3..df3539c2b3 100644 --- a/nvflare/fox/examples/pt/pt_np.py +++ b/nvflare/fox/examples/pt/pt_np.py @@ -65,7 +65,7 @@ def _do_one_round(self, r, pt_model, np_model): ).train(r, pt_model, np_model) if aggr_result.count == 0: - return None + return None, None else: pt_result = aggr_result.pt_total div_pt(pt_result, aggr_result.count) @@ -84,9 +84,10 @@ def _accept_train_result(self, gcc, result, aggr_result: _AggrResult): self.logger.info(f"[{fox.call_info}] got train result from {fox.caller}: {result}") pt_result, np_result = result - add_pt(pt_result, aggr_result.pt_total) - add_np(np_result, aggr_result.np_total) - aggr_result.count += 1 + with aggr_result.lock: + add_pt(pt_result, aggr_result.pt_total) + add_np(np_result, aggr_result.np_total) + aggr_result.count += 1 return None diff --git a/nvflare/fox/examples/pt/recipe_pt_np.py b/nvflare/fox/examples/pt/recipe_pt_np.py index 40ad249872..1cd043c5dd 100644 --- a/nvflare/fox/examples/pt/recipe_pt_np.py +++ b/nvflare/fox/examples/pt/recipe_pt_np.py @@ -35,7 +35,7 @@ def main(): ), client=PTTrainer(delta=1.0), ) - recipe.add_decomposers([TensorDecomposer(), NumpyArrayDecomposer]) + recipe.add_decomposers([TensorDecomposer(), NumpyArrayDecomposer()]) recipe.export(JOB_ROOT_DIR) diff --git a/nvflare/fox/examples/pt/utils.py b/nvflare/fox/examples/pt/utils.py index 4d580668f6..df60cb3667 100644 --- a/nvflare/fox/examples/pt/utils.py +++ b/nvflare/fox/examples/pt/utils.py @@ -27,7 +27,7 @@ def parse_array_def(array_def): raise ValueError(f"unsupported array def: {array_def}") -def parse_state_dict(d: dict[str, list]): +def parse_state_dict(d): result = {} for k, v in d.items(): result[k] = parse_array_def(v) @@ -47,8 +47,10 @@ def add(value: dict, to_model: dict): to_model[k] = v else: to_model[k] += v + return to_model def div(model: dict, value): for k, v in model.items(): model[k] = torch.div(v, value) + return model diff --git a/nvflare/fox/sim/backend.py b/nvflare/fox/sim/backend.py index 9b27d3720f..6bc3799087 100644 --- a/nvflare/fox/sim/backend.py +++ b/nvflare/fox/sim/backend.py @@ -15,11 +15,10 @@ import time from nvflare.apis.fl_exception import RunAborted -from nvflare.fox import fox from nvflare.fox.api.app import App from nvflare.fox.api.backend import Backend from nvflare.fox.api.call_opt import CallOption -from nvflare.fox.api.constants import CollabMethodArgName, ContextKey +from nvflare.fox.api.constants import CollabMethodArgName from nvflare.fox.api.dec import adjust_kwargs from nvflare.fox.api.gcc import GroupCallContext from nvflare.fox.api.utils import check_call_args @@ -65,6 +64,7 @@ def call_target(self, target_name: str, call_opt: CallOption, func_name: str, *a while True: if self.abort_signal.triggered: waiter.result = RunAborted("job is aborted") + break ok = waiter.wait(0.1) if ok: diff --git a/nvflare/fox/sim/ws.py b/nvflare/fox/sim/ws.py index 2a57cfbfdb..8b5b6e8638 100644 --- a/nvflare/fox/sim/ws.py +++ b/nvflare/fox/sim/ws.py @@ -23,9 +23,15 @@ def __init__(self, root_dir: str, experiment_name: str, exp_id: str, site_name: if not isinstance(root_dir, str): raise ValueError(f"root_dir must be str but got {type(root_dir)}") + if not isinstance(exp_id, str): + raise ValueError(f"exp_id must be str but got {type(exp_id)}") + if not isinstance(experiment_name, str): raise ValueError(f"experiment_name must be str but got {type(experiment_name)}") + if not isinstance(site_name, str): + raise ValueError(f"site_name must be str but got {type(site_name)}") + self.root_dir = root_dir self.site_name = site_name self.exp_name = experiment_name diff --git a/nvflare/fox/sys/adaptor.py b/nvflare/fox/sys/adaptor.py index 3aafce2f1f..38b3643382 100644 --- a/nvflare/fox/sys/adaptor.py +++ b/nvflare/fox/sys/adaptor.py @@ -79,10 +79,10 @@ def _parse_filters(self, name, add_f, fl_ctx): return f"{name} must be a list but got {type(filters)}" for chain_dict in filters: - pattern, filters, err = self._parse_filter_chain(name, chain_dict, fl_ctx) + pattern, filter_components, err = self._parse_filter_chain(name, chain_dict, fl_ctx) if err: return err - add_f(pattern, filters) + add_f(pattern, filter_components) return None @staticmethod diff --git a/nvflare/fox/sys/backend.py b/nvflare/fox/sys/backend.py index a37afbd70e..3023cd5e97 100644 --- a/nvflare/fox/sys/backend.py +++ b/nvflare/fox/sys/backend.py @@ -80,16 +80,16 @@ def _call_target( assert isinstance(reply, Message) rc = reply.get_header(MessageHeaderKey.RETURN_CODE, ReturnCode.OK) if rc == ReturnCode.TIMEOUT: - return TimeoutError(f"function {func_name} timed out after {timeout} seconds") + raise TimeoutError(f"function {func_name} timed out after {timeout} seconds") elif rc != ReturnCode.OK: - return RuntimeError(f"function {func_name} failed: {rc}") + raise RuntimeError(f"function {func_name} failed: {rc}") if not isinstance(reply.payload, dict): - return RuntimeError(f"function {func_name} failed: reply must be dict but got {type(reply.payload)}") + raise RuntimeError(f"function {func_name} failed: reply must be dict but got {type(reply.payload)}") error = reply.payload.get(CallReplyKey.ERROR) if error: - return RuntimeError(f"function {func_name} failed: {error}") + raise RuntimeError(f"function {func_name} failed: {error}") result = reply.payload.get(CallReplyKey.RESULT) self.logger.info(f"got result from {self.target_fqcn} {func_name=}") diff --git a/nvflare/fox/sys/controller.py b/nvflare/fox/sys/controller.py index 82eaf24eb4..97c70203a3 100644 --- a/nvflare/fox/sys/controller.py +++ b/nvflare/fox/sys/controller.py @@ -192,7 +192,6 @@ def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): all_clients = engine.get_clients() num_clients = len(all_clients) for c in all_clients: - assert isinstance(c, ClientSite) self.client_info[c.name] = None start_time = time.time() @@ -229,7 +228,7 @@ def control_flow(self, abort_signal: Signal, fl_ctx: FLContext): client_proxies = [] for c in all_clients: info = self.client_info[c.name] - assert isinstance(info, _ClientInfo) + # assert isinstance(info, _ClientInfo) client_proxies.append(self._prepare_client_proxy(job_id, c, info.collab_interface, abort_signal, fl_ctx)) ws = FlareWorkspace(fl_ctx) diff --git a/nvflare/fox/sys/downloader.py b/nvflare/fox/sys/downloader.py index a1125ebdbb..8cda392bf4 100644 --- a/nvflare/fox/sys/downloader.py +++ b/nvflare/fox/sys/downloader.py @@ -48,7 +48,7 @@ def __init__( ctx = fox.context backend = ctx.backend if not isinstance(backend, FlareBackend): - raise ValueError(f"backend must be SysBackend but got {type(backend)}") + raise ValueError(f"backend must be FlareBackend but got {type(backend)}") super().__init__( cell=backend.cell, diff --git a/nvflare/fox/sys/recipe.py b/nvflare/fox/sys/recipe.py index 1e088f7768..1d7b43c9fd 100644 --- a/nvflare/fox/sys/recipe.py +++ b/nvflare/fox/sys/recipe.py @@ -167,14 +167,14 @@ def _create_app_args(self, app: App, to_f): self._create_filter_components(to_f, outgoing_result_filters, filter_comp_table) # filters - in_cf_arg = self._create_filer_chain_arg(incoming_call_filters, filter_comp_table) - out_cf_arg = self._create_filer_chain_arg(outgoing_call_filters, filter_comp_table) - in_rf_arg = self._create_filer_chain_arg(incoming_result_filters, filter_comp_table) - out_rf_arg = self._create_filer_chain_arg(outgoing_result_filters, filter_comp_table) + in_cf_arg = self._create_filter_chain_arg(incoming_call_filters, filter_comp_table) + out_cf_arg = self._create_filter_chain_arg(outgoing_call_filters, filter_comp_table) + in_rf_arg = self._create_filter_chain_arg(incoming_result_filters, filter_comp_table) + out_rf_arg = self._create_filter_chain_arg(outgoing_result_filters, filter_comp_table) return collab_obj_ids, in_cf_arg, out_cf_arg, in_rf_arg, out_rf_arg @staticmethod - def _create_filer_chain_arg(filter_chains: list, comp_table: dict): + def _create_filter_chain_arg(filter_chains: list, comp_table: dict): result = [] for chain in filter_chains: assert isinstance(chain, FilterChain) diff --git a/nvflare/fox/sys/utils.py b/nvflare/fox/sys/utils.py index 96423096d1..6342c553e5 100644 --- a/nvflare/fox/sys/utils.py +++ b/nvflare/fox/sys/utils.py @@ -11,8 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import traceback - from nvflare.fox.api.app import App from nvflare.fox.api.constants import CollabMethodArgName from nvflare.fox.api.dec import adjust_kwargs @@ -61,7 +59,8 @@ def _preprocess(app: App, caller, target_obj_name, target_name, func_name, func, def _call_app_method(request: Message, app: App, logger) -> Message: logger.debug("got a remote call") payload = request.payload - assert isinstance(payload, dict) + if not isinstance(payload, dict): + raise RuntimeError(f"request payload must be dict but got {type(payload)}") caller = payload.get(ObjectCallKey.CALLER) if not caller: diff --git a/nvflare/fox/sys/ws.py b/nvflare/fox/sys/ws.py index 4a7f0c2499..99d09d2e9f 100644 --- a/nvflare/fox/sys/ws.py +++ b/nvflare/fox/sys/ws.py @@ -21,7 +21,8 @@ class FlareWorkspace(Workspace): def __init__(self, fl_ctx: FLContext): super().__init__() ws_obj = fl_ctx.get_workspace() - assert isinstance(ws_obj, NVFWorkspace) + if not isinstance(ws_obj, NVFWorkspace): + raise RuntimeError(f"the ws_obj must be NVFWorkspace but got {type(ws_obj)}") self.flare_ws = ws_obj self.job_id = fl_ctx.get_job_id() diff --git a/nvflare/fox/utils/tensor_receiver.py b/nvflare/fox/utils/tensor_receiver.py index 6e7a4f1e0f..f0905bbed8 100644 --- a/nvflare/fox/utils/tensor_receiver.py +++ b/nvflare/fox/utils/tensor_receiver.py @@ -43,7 +43,7 @@ def __call__(self, gcc: GroupCallContext, result): gcc=gcc, ) if err: - return RuntimeError(f"failed to download model {model}: {err}") + raise RuntimeError(f"failed to download model {model}: {err}") else: return None else: From ff423d3c2d0699c567f44ecb2982fc5792bb5632 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Sun, 14 Dec 2025 17:09:17 -0500 Subject: [PATCH 088/102] improve examples --- nvflare/fox/api/run_server.py | 2 -- nvflare/fox/examples/__init__.py | 16 ++++++++++++++++ nvflare/fox/examples/np/cyclic.py | 3 ++- nvflare/fox/examples/np/cyclic_avg.py | 3 ++- nvflare/fox/examples/np/cyclic_file.py | 5 +++-- nvflare/fox/examples/np/cyclic_runner.py | 5 +++-- nvflare/fox/examples/np/fed_avg_h.py | 3 ++- nvflare/fox/examples/np/fed_avg_intime.py | 3 ++- nvflare/fox/examples/np/fed_avg_para.py | 3 ++- nvflare/fox/examples/np/fed_avg_para_tc.py | 5 +++-- nvflare/fox/examples/np/fed_avg_seq.py | 3 ++- nvflare/fox/examples/np/fed_avg_stream.py | 3 ++- nvflare/fox/examples/np/recipe_cyclic.py | 12 +++++++----- nvflare/fox/examples/np/recipe_cyclic_avg.py | 12 +++++++----- nvflare/fox/examples/np/recipe_cyclic_file.py | 12 +++++++----- nvflare/fox/examples/np/recipe_fed_avg_h.py | 12 +++++++----- nvflare/fox/examples/np/recipe_fed_avg_intime.py | 12 +++++++----- nvflare/fox/examples/np/recipe_fed_avg_seq.py | 11 +++++++---- nvflare/fox/examples/np/recipe_fed_avg_stream.py | 12 +++++++----- nvflare/fox/examples/np/recipe_fed_avg_tc.py | 12 +++++++----- nvflare/fox/examples/np/recipe_swarm.py | 12 +++++++----- nvflare/fox/examples/np/swarm.py | 3 ++- nvflare/fox/examples/pt/pt_avg_filter.py | 3 ++- nvflare/fox/examples/pt/pt_avg_mixed.py | 3 ++- nvflare/fox/examples/pt/pt_avg_stream.py | 3 ++- nvflare/fox/examples/pt/pt_avg_stream2.py | 3 ++- nvflare/fox/examples/pt/pt_np.py | 3 ++- nvflare/fox/examples/pt/recipe_pt_avg.py | 11 +++++++---- nvflare/fox/examples/pt/recipe_pt_avg_filter.py | 12 +++++++----- nvflare/fox/examples/pt/recipe_pt_avg_filter2.py | 11 ++++++----- nvflare/fox/examples/pt/recipe_pt_avg_mixed.py | 12 +++++++----- nvflare/fox/examples/pt/recipe_pt_avg_stream.py | 12 +++++++----- nvflare/fox/examples/pt/recipe_pt_avg_stream2.py | 12 +++++++----- nvflare/fox/examples/pt/recipe_pt_np.py | 11 +++++++---- 34 files changed, 162 insertions(+), 98 deletions(-) diff --git a/nvflare/fox/api/run_server.py b/nvflare/fox/api/run_server.py index fa836382b1..1f3eed6641 100644 --- a/nvflare/fox/api/run_server.py +++ b/nvflare/fox/api/run_server.py @@ -11,8 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import traceback - from nvflare.security.logging import secure_log_traceback from .app import ServerApp diff --git a/nvflare/fox/examples/__init__.py b/nvflare/fox/examples/__init__.py index 341a77c5bc..85481fa6cc 100644 --- a/nvflare/fox/examples/__init__.py +++ b/nvflare/fox/examples/__init__.py @@ -11,3 +11,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os + + +def get_job_root_dir(): + return os.getenv("FOX_JOB_ROOT", ".") + + +def get_experiment_root(): + return os.getenv("FOX_EXP_ROOT", ".") + + +def export_recipe(job_name: str, make_recipe_f): + recipe = make_recipe_f(job_name) + job_root = get_job_root_dir() + recipe.export(job_root) + print(f"job exported at {job_root}/{job_name}") diff --git a/nvflare/fox/examples/np/cyclic.py b/nvflare/fox/examples/np/cyclic.py index dfb1d8f33e..70d4575076 100644 --- a/nvflare/fox/examples/np/cyclic.py +++ b/nvflare/fox/examples/np/cyclic.py @@ -14,6 +14,7 @@ import logging from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples import get_experiment_root from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.strategies.cyclic import NPCyclic from nvflare.fox.sim.simulator import Simulator @@ -23,7 +24,7 @@ def main(): simple_logging(logging.DEBUG) simulator = Simulator( - root_dir="/tmp/fox", + root_dir=get_experiment_root(), experiment_name="cyclic", server=NPCyclic(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), client=NPTrainer(delta=1.0), diff --git a/nvflare/fox/examples/np/cyclic_avg.py b/nvflare/fox/examples/np/cyclic_avg.py index 79320411af..65911d95a1 100644 --- a/nvflare/fox/examples/np/cyclic_avg.py +++ b/nvflare/fox/examples/np/cyclic_avg.py @@ -15,6 +15,7 @@ from nvflare.fox import fox from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples import get_experiment_root from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.strategies.avg_para import NPFedAvgParallel from nvflare.fox.examples.np.algos.strategies.cyclic import NPCyclic @@ -56,7 +57,7 @@ def main(): server = Controller(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], cyclic_rounds=2, avg_rounds=3) simulator = Simulator( - root_dir="/tmp/fox", + root_dir=get_experiment_root(), experiment_name=exp_name, server=server, client=NPTrainer(delta=1.0), diff --git a/nvflare/fox/examples/np/cyclic_file.py b/nvflare/fox/examples/np/cyclic_file.py index 796ded9bec..7b395db630 100644 --- a/nvflare/fox/examples/np/cyclic_file.py +++ b/nvflare/fox/examples/np/cyclic_file.py @@ -14,6 +14,7 @@ import logging from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples import get_experiment_root from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.strategies.cyclic import NPCyclic from nvflare.fox.sim.simulator import Simulator @@ -23,8 +24,8 @@ def main(): simple_logging(logging.DEBUG) simulator = Simulator( - root_dir="/tmp/fox", - experiment_name="cyclic", + root_dir=get_experiment_root(), + experiment_name="cyclic_file", server=NPCyclic(initial_model="initial_model.npy", num_rounds=2), client=NPTrainer(delta=1.0), num_clients=2, diff --git a/nvflare/fox/examples/np/cyclic_runner.py b/nvflare/fox/examples/np/cyclic_runner.py index 3eb73b4830..c1c5e2e5d1 100644 --- a/nvflare/fox/examples/np/cyclic_runner.py +++ b/nvflare/fox/examples/np/cyclic_runner.py @@ -15,6 +15,7 @@ from nvflare.fox.api.app import ClientApp, ServerApp from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples import get_experiment_root from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.strategies.cyclic import NPCyclic from nvflare.fox.sim.simulator import AppRunner @@ -24,8 +25,8 @@ def main(): simple_logging(logging.DEBUG) runner = AppRunner( - root_dir="/tmp/fox", - experiment_name="cyclic", + root_dir=get_experiment_root(), + experiment_name="cyclic_runner", server_app=ServerApp(NPCyclic(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2)), client_app=ClientApp(NPTrainer(delta=1.0)), num_clients=2, diff --git a/nvflare/fox/examples/np/fed_avg_h.py b/nvflare/fox/examples/np/fed_avg_h.py index 89f84bf584..fae8983504 100644 --- a/nvflare/fox/examples/np/fed_avg_h.py +++ b/nvflare/fox/examples/np/fed_avg_h.py @@ -14,6 +14,7 @@ import logging from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples import get_experiment_root from nvflare.fox.examples.np.algos.client import NPHierarchicalTrainer from nvflare.fox.examples.np.algos.strategies.avg_h import NPHierarchicalFedAvg from nvflare.fox.examples.np.algos.widgets import MetricReceiver @@ -24,7 +25,7 @@ def main(): simple_logging(logging.DEBUG) simulator = Simulator( - root_dir="/tmp/fox", + root_dir=get_experiment_root(), experiment_name="fedavg_h", server=NPHierarchicalFedAvg(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=3), client=NPHierarchicalTrainer(delta=1.0), diff --git a/nvflare/fox/examples/np/fed_avg_intime.py b/nvflare/fox/examples/np/fed_avg_intime.py index f2746bcc6e..8c1f0fe774 100644 --- a/nvflare/fox/examples/np/fed_avg_intime.py +++ b/nvflare/fox/examples/np/fed_avg_intime.py @@ -14,6 +14,7 @@ import logging from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples import get_experiment_root from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.filters import AddNoiseToModel, Print from nvflare.fox.examples.np.algos.strategies.avg_intime import NPFedAvgInTime @@ -25,7 +26,7 @@ def main(): simple_logging(logging.DEBUG) simulator = Simulator( - root_dir="/tmp/fox", + root_dir=get_experiment_root(), experiment_name="fedavg_intime", server=NPFedAvgInTime(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), client=NPTrainer(delta=1.0), diff --git a/nvflare/fox/examples/np/fed_avg_para.py b/nvflare/fox/examples/np/fed_avg_para.py index 22a9008723..0b3a076dad 100644 --- a/nvflare/fox/examples/np/fed_avg_para.py +++ b/nvflare/fox/examples/np/fed_avg_para.py @@ -14,6 +14,7 @@ import logging from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples import get_experiment_root from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.strategies.avg_para import NPFedAvgParallel from nvflare.fox.examples.np.algos.widgets import MetricReceiver @@ -26,7 +27,7 @@ def main(): server = NPFedAvgParallel(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]], num_rounds=2) simulator = Simulator( - root_dir="/tmp/fox", + root_dir=get_experiment_root(), experiment_name="fedavg_para", server=server, client=NPTrainer(delta=1.0), diff --git a/nvflare/fox/examples/np/fed_avg_para_tc.py b/nvflare/fox/examples/np/fed_avg_para_tc.py index a96f080daa..7c3e73aff4 100644 --- a/nvflare/fox/examples/np/fed_avg_para_tc.py +++ b/nvflare/fox/examples/np/fed_avg_para_tc.py @@ -14,6 +14,7 @@ import logging from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples import get_experiment_root from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.strategies.avg_para_tc import NPFedAvgParallelWithTrafficControl from nvflare.fox.examples.np.algos.widgets import MetricReceiver @@ -30,8 +31,8 @@ def main(): ) simulator = Simulator( - root_dir="/tmp/fox", - experiment_name="fedavg_para", + root_dir=get_experiment_root(), + experiment_name="fedavg_para_tc", server=server, client=NPTrainer(delta=1.0, delay=1.5), server_objects={"metric_receiver": MetricReceiver()}, diff --git a/nvflare/fox/examples/np/fed_avg_seq.py b/nvflare/fox/examples/np/fed_avg_seq.py index 66f4386e90..fb13c72d62 100644 --- a/nvflare/fox/examples/np/fed_avg_seq.py +++ b/nvflare/fox/examples/np/fed_avg_seq.py @@ -14,6 +14,7 @@ import logging from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples import get_experiment_root from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.strategies.avg_seq import NPFedAvgSequential from nvflare.fox.examples.np.algos.widgets import MetricReceiver @@ -29,7 +30,7 @@ def main(): ) simulator = Simulator( - root_dir="/tmp/fox", + root_dir=get_experiment_root(), experiment_name="fedavg_seq", server=server, client=NPTrainer(delta=1.0), diff --git a/nvflare/fox/examples/np/fed_avg_stream.py b/nvflare/fox/examples/np/fed_avg_stream.py index e2cc6a2d82..24c8e6daed 100644 --- a/nvflare/fox/examples/np/fed_avg_stream.py +++ b/nvflare/fox/examples/np/fed_avg_stream.py @@ -14,6 +14,7 @@ import logging from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples import get_experiment_root from nvflare.fox.examples.np.algos.avg_stream import NPFedAvgStream, NPTrainer from nvflare.fox.sim.simulator import Simulator @@ -22,7 +23,7 @@ def main(): simple_logging(logging.DEBUG) simulator = Simulator( - root_dir="/tmp/fox", + root_dir=get_experiment_root(), experiment_name="fedavg_stream", server=NPFedAvgStream(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=4), client=NPTrainer(delta=1.0), diff --git a/nvflare/fox/examples/np/recipe_cyclic.py b/nvflare/fox/examples/np/recipe_cyclic.py index e6f91ce01f..371d529b05 100644 --- a/nvflare/fox/examples/np/recipe_cyclic.py +++ b/nvflare/fox/examples/np/recipe_cyclic.py @@ -11,20 +11,22 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from nvflare.fox.examples import export_recipe from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.strategies.cyclic import NPCyclic from nvflare.fox.sys.recipe import FoxRecipe -JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" - def main(): - recipe = FoxRecipe( - job_name="fox_cyclic", + export_recipe("fox_cyclic", _make_recipe) + + +def _make_recipe(job_name): + return FoxRecipe( + job_name=job_name, server=NPCyclic(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), client=NPTrainer(delta=1.0), ) - recipe.export(JOB_ROOT_DIR) if __name__ == "__main__": diff --git a/nvflare/fox/examples/np/recipe_cyclic_avg.py b/nvflare/fox/examples/np/recipe_cyclic_avg.py index c36de69c69..d111b84eca 100644 --- a/nvflare/fox/examples/np/recipe_cyclic_avg.py +++ b/nvflare/fox/examples/np/recipe_cyclic_avg.py @@ -14,6 +14,7 @@ import os from nvflare.fox import fox +from nvflare.fox.examples import export_recipe from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.strategies.avg_para import NPFedAvgParallel from nvflare.fox.examples.np.algos.strategies.cyclic import NPCyclic @@ -21,8 +22,6 @@ from nvflare.fox.sys.recipe import FoxRecipe from nvflare.fuel.utils.log_utils import get_obj_logger -JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" - class Controller: @@ -60,8 +59,12 @@ def save_result(self): def main(): - recipe = FoxRecipe( - job_name="fox_cyclic_avg", + export_recipe("fox_cyclic_avg", _make_recipe) + + +def _make_recipe(job_name): + return FoxRecipe( + job_name=job_name, server=Controller( initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], cyclic_rounds=2, @@ -69,7 +72,6 @@ def main(): ), client=NPTrainer(delta=1.0), ) - recipe.export(JOB_ROOT_DIR) if __name__ == "__main__": diff --git a/nvflare/fox/examples/np/recipe_cyclic_file.py b/nvflare/fox/examples/np/recipe_cyclic_file.py index 58d7221922..21876fb8a5 100644 --- a/nvflare/fox/examples/np/recipe_cyclic_file.py +++ b/nvflare/fox/examples/np/recipe_cyclic_file.py @@ -11,22 +11,24 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from nvflare.fox.examples import export_recipe from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.strategies.cyclic import NPCyclic from nvflare.fox.sys.recipe import FoxRecipe -JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" - def main(): + export_recipe("fox_cyclic_file", _make_recipe) + + +def _make_recipe(job_name): recipe = FoxRecipe( - job_name="fox_cyclic_file", + job_name=job_name, server=NPCyclic(initial_model="initial_model.npy", num_rounds=2), client=NPTrainer(delta=1.0), ) recipe.set_server_resource_dirs({"data": "/Users/yanc/NVFlare/sandbox/data"}) - - recipe.export(JOB_ROOT_DIR) + return recipe if __name__ == "__main__": diff --git a/nvflare/fox/examples/np/recipe_fed_avg_h.py b/nvflare/fox/examples/np/recipe_fed_avg_h.py index 9a37ada0c1..5439685adc 100644 --- a/nvflare/fox/examples/np/recipe_fed_avg_h.py +++ b/nvflare/fox/examples/np/recipe_fed_avg_h.py @@ -11,22 +11,24 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from nvflare.fox.examples import export_recipe from nvflare.fox.examples.np.algos.client import NPHierarchicalTrainer from nvflare.fox.examples.np.algos.strategies.avg_h import NPHierarchicalFedAvg from nvflare.fox.examples.np.algos.widgets import MetricReceiver from nvflare.fox.sys.recipe import FoxRecipe -JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" - def main(): - recipe = FoxRecipe( - job_name="fox_fedavg_h", + export_recipe("fox_fedavg_h", _make_recipe) + + +def _make_recipe(job_name): + return FoxRecipe( + job_name=job_name, server=NPHierarchicalFedAvg(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), client=NPHierarchicalTrainer(delta=1.0), server_objects={"metric_receiver": MetricReceiver()}, ) - recipe.export(JOB_ROOT_DIR) if __name__ == "__main__": diff --git a/nvflare/fox/examples/np/recipe_fed_avg_intime.py b/nvflare/fox/examples/np/recipe_fed_avg_intime.py index 946b269ac3..627f9c668d 100644 --- a/nvflare/fox/examples/np/recipe_fed_avg_intime.py +++ b/nvflare/fox/examples/np/recipe_fed_avg_intime.py @@ -11,18 +11,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from nvflare.fox.examples import export_recipe from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.filters import AddNoiseToModel, Print from nvflare.fox.examples.np.algos.strategies.avg_intime import NPFedAvgInTime from nvflare.fox.examples.np.algos.widgets import MetricReceiver from nvflare.fox.sys.recipe import FoxRecipe -JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" - def main(): + export_recipe("fox_fedavg_intime", _make_recipe) + + +def _make_recipe(job_name): recipe = FoxRecipe( - job_name="fox_fedavg_intime", + job_name=job_name, server=NPFedAvgInTime(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), server_objects={ "metric_receiver": MetricReceiver(), @@ -38,8 +41,7 @@ def main(): recipe.add_client_incoming_call_filters("*.train", [print_filter]) recipe.add_client_outgoing_result_filters("*.train", [print_filter]) recipe.set_client_prop("default_timeout", 8.0) - - recipe.export(JOB_ROOT_DIR) + return recipe if __name__ == "__main__": diff --git a/nvflare/fox/examples/np/recipe_fed_avg_seq.py b/nvflare/fox/examples/np/recipe_fed_avg_seq.py index a1a360fb24..79d2282411 100644 --- a/nvflare/fox/examples/np/recipe_fed_avg_seq.py +++ b/nvflare/fox/examples/np/recipe_fed_avg_seq.py @@ -11,24 +11,27 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from nvflare.fox.examples import export_recipe from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.strategies.avg_seq import NPFedAvgSequential from nvflare.fox.examples.np.algos.widgets import MetricReceiver from nvflare.fox.sys.recipe import FoxRecipe -JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" - def main(): + export_recipe("fox_fedavg_seq", _make_recipe) + + +def _make_recipe(job_name): recipe = FoxRecipe( - job_name="fox_fedavg_seq", + job_name=job_name, server=NPFedAvgSequential(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), client=NPTrainer(delta=1.0), server_objects={"metric_receiver": MetricReceiver()}, ) recipe.set_server_prop("client_weight_config", {"red": 70, "blue": 100, "silver": 50}) recipe.set_client_prop("client_delta", {"red": 1.0, "blue": 2.0, "silver": 3.0}) - recipe.export(JOB_ROOT_DIR) + return recipe if __name__ == "__main__": diff --git a/nvflare/fox/examples/np/recipe_fed_avg_stream.py b/nvflare/fox/examples/np/recipe_fed_avg_stream.py index ee557541d6..10d899188b 100644 --- a/nvflare/fox/examples/np/recipe_fed_avg_stream.py +++ b/nvflare/fox/examples/np/recipe_fed_avg_stream.py @@ -11,19 +11,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from nvflare.fox.examples import export_recipe from nvflare.fox.examples.np.algos.avg_stream import NPFedAvgStream, NPTrainer from nvflare.fox.sys.recipe import FoxRecipe -JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" - def main(): - recipe = FoxRecipe( - job_name="fox_fedavg_stream", + export_recipe("fox_fedavg_stream", _make_recipe) + + +def _make_recipe(job_name): + return FoxRecipe( + job_name=job_name, server=NPFedAvgStream(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2), client=NPTrainer(delta=1.0), ) - recipe.export(JOB_ROOT_DIR) if __name__ == "__main__": diff --git a/nvflare/fox/examples/np/recipe_fed_avg_tc.py b/nvflare/fox/examples/np/recipe_fed_avg_tc.py index 64417de596..3285622c54 100644 --- a/nvflare/fox/examples/np/recipe_fed_avg_tc.py +++ b/nvflare/fox/examples/np/recipe_fed_avg_tc.py @@ -11,17 +11,20 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from nvflare.fox.examples import export_recipe from nvflare.fox.examples.np.algos.client import NPTrainer from nvflare.fox.examples.np.algos.strategies.avg_para_tc import NPFedAvgParallelWithTrafficControl from nvflare.fox.examples.np.algos.widgets import MetricReceiver from nvflare.fox.sys.recipe import FoxRecipe -JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" - def main(): - recipe = FoxRecipe( - job_name="fox_fedavg_tc", + export_recipe("fox_fedavg_tc", _make_recipe) + + +def _make_recipe(job_name): + return FoxRecipe( + job_name=job_name, server=NPFedAvgParallelWithTrafficControl( initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=2, @@ -30,7 +33,6 @@ def main(): client=NPTrainer(delta=1.0, delay=2.0), server_objects={"metric_receiver": MetricReceiver()}, ) - recipe.export(JOB_ROOT_DIR) if __name__ == "__main__": diff --git a/nvflare/fox/examples/np/recipe_swarm.py b/nvflare/fox/examples/np/recipe_swarm.py index 3bcd59034a..7a6e6903f2 100644 --- a/nvflare/fox/examples/np/recipe_swarm.py +++ b/nvflare/fox/examples/np/recipe_swarm.py @@ -11,19 +11,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from nvflare.fox.examples import export_recipe from nvflare.fox.examples.np.algos.swarm import NPSwarm, NPSwarmClient from nvflare.fox.sys.recipe import FoxRecipe -JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" - def main(): - recipe = FoxRecipe( - job_name="fox_swarm", + export_recipe("fox_swarm", _make_recipe) + + +def _make_recipe(job_name): + return FoxRecipe( + job_name=job_name, server=NPSwarm(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=5), client=NPSwarmClient(delta=1.0), ) - recipe.export(JOB_ROOT_DIR) if __name__ == "__main__": diff --git a/nvflare/fox/examples/np/swarm.py b/nvflare/fox/examples/np/swarm.py index 8bf7a8f9c2..9132ead1cc 100644 --- a/nvflare/fox/examples/np/swarm.py +++ b/nvflare/fox/examples/np/swarm.py @@ -14,6 +14,7 @@ import logging from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples import get_experiment_root from nvflare.fox.examples.np.algos.swarm import NPSwarm, NPSwarmClient from nvflare.fox.sim.simulator import Simulator @@ -22,7 +23,7 @@ def main(): simple_logging(logging.DEBUG) simulator = Simulator( - root_dir="/tmp/fox", + root_dir=get_experiment_root(), experiment_name="swarm", server=NPSwarm(initial_model=[[1, 2, 3], [4, 5, 6], [7, 8, 9]], num_rounds=5), client=NPSwarmClient(delta=1.0), diff --git a/nvflare/fox/examples/pt/pt_avg_filter.py b/nvflare/fox/examples/pt/pt_avg_filter.py index 659b3ef324..465eec81f1 100644 --- a/nvflare/fox/examples/pt/pt_avg_filter.py +++ b/nvflare/fox/examples/pt/pt_avg_filter.py @@ -15,6 +15,7 @@ from nvflare.fox import fox from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples import get_experiment_root from nvflare.fox.examples.pt.utils import add as add_pt from nvflare.fox.examples.pt.utils import div as div_pt from nvflare.fox.examples.pt.utils import parse_state_dict @@ -91,7 +92,7 @@ def main(): client = PTTrainer(delta=1.0) simulator = Simulator( - root_dir="/tmp/fox", + root_dir=get_experiment_root(), experiment_name="pt_fedavg_intime", server=server, client=client, diff --git a/nvflare/fox/examples/pt/pt_avg_mixed.py b/nvflare/fox/examples/pt/pt_avg_mixed.py index 52085cb019..da93f87292 100644 --- a/nvflare/fox/examples/pt/pt_avg_mixed.py +++ b/nvflare/fox/examples/pt/pt_avg_mixed.py @@ -20,6 +20,7 @@ from nvflare.fox import fox from nvflare.fox.api.constants import BackendType from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples import get_experiment_root from nvflare.fox.examples.np.algos.utils import add as add_np from nvflare.fox.examples.np.algos.utils import div as div_np from nvflare.fox.examples.np.algos.utils import parse_state_dict as parse_np @@ -206,7 +207,7 @@ def main(): client = PTTrainer(delta=1.0) simulator = Simulator( - root_dir="/tmp/fox", + root_dir=get_experiment_root(), experiment_name="fedavg_mixed", server=server, client=client, diff --git a/nvflare/fox/examples/pt/pt_avg_stream.py b/nvflare/fox/examples/pt/pt_avg_stream.py index 6e5882821d..6e41e5bae7 100644 --- a/nvflare/fox/examples/pt/pt_avg_stream.py +++ b/nvflare/fox/examples/pt/pt_avg_stream.py @@ -19,6 +19,7 @@ from nvflare.fox import fox from nvflare.fox.api.constants import BackendType from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples import get_experiment_root from nvflare.fox.examples.pt.utils import parse_state_dict from nvflare.fox.sim.simulator import Simulator from nvflare.fox.sys.downloader import Downloader, download_tensors @@ -184,7 +185,7 @@ def main(): client = PTTrainer(delta=1.0) simulator = Simulator( - root_dir="/tmp/fox", + root_dir=get_experiment_root(), experiment_name="pt_fedavg_stream", server=server, client=client, diff --git a/nvflare/fox/examples/pt/pt_avg_stream2.py b/nvflare/fox/examples/pt/pt_avg_stream2.py index e369c302da..67e13cf588 100644 --- a/nvflare/fox/examples/pt/pt_avg_stream2.py +++ b/nvflare/fox/examples/pt/pt_avg_stream2.py @@ -18,6 +18,7 @@ from nvflare.fox import fox from nvflare.fox.api.constants import BackendType from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples import get_experiment_root from nvflare.fox.examples.pt.utils import parse_state_dict from nvflare.fox.sim.simulator import Simulator from nvflare.fox.sys.downloader import Downloader, download_tensors @@ -143,7 +144,7 @@ def main(): client = PTTrainer(delta=1.0) simulator = Simulator( - root_dir="/tmp/fox", + root_dir=get_experiment_root(), experiment_name="pt_fedavg_stream2", server=server, client=client, diff --git a/nvflare/fox/examples/pt/pt_np.py b/nvflare/fox/examples/pt/pt_np.py index df3539c2b3..21ae4a456b 100644 --- a/nvflare/fox/examples/pt/pt_np.py +++ b/nvflare/fox/examples/pt/pt_np.py @@ -16,6 +16,7 @@ from nvflare.fox import fox from nvflare.fox.api.utils import simple_logging +from nvflare.fox.examples import get_experiment_root from nvflare.fox.examples.np.algos.utils import add as add_np from nvflare.fox.examples.np.algos.utils import div as div_np from nvflare.fox.examples.np.algos.utils import parse_state_dict as parse_np @@ -132,7 +133,7 @@ def main(): ) simulator = Simulator( - root_dir="/tmp/fox", + root_dir=get_experiment_root(), experiment_name="pt_np", server=server, client=PTTrainer(delta=1.0), diff --git a/nvflare/fox/examples/pt/recipe_pt_avg.py b/nvflare/fox/examples/pt/recipe_pt_avg.py index d64292ef52..e22978d5aa 100644 --- a/nvflare/fox/examples/pt/recipe_pt_avg.py +++ b/nvflare/fox/examples/pt/recipe_pt_avg.py @@ -12,15 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. from nvflare.app_opt.pt.decomposers import TensorDecomposer +from nvflare.fox.examples import export_recipe from nvflare.fox.examples.pt.pt_avg_filter import PTFedAvg, PTTrainer from nvflare.fox.sys.recipe import FoxRecipe -JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" - def main(): + export_recipe("fox_pt_fedavg", _make_recipe) + + +def _make_recipe(job_name): recipe = FoxRecipe( - job_name="fox_pt_fedavg", + job_name=job_name, server=PTFedAvg( initial_model={ "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], @@ -32,7 +35,7 @@ def main(): client=PTTrainer(delta=1.0), ) recipe.add_decomposers([TensorDecomposer()]) - recipe.export(JOB_ROOT_DIR) + return recipe if __name__ == "__main__": diff --git a/nvflare/fox/examples/pt/recipe_pt_avg_filter.py b/nvflare/fox/examples/pt/recipe_pt_avg_filter.py index c52bacb59e..5f9b2ef74e 100644 --- a/nvflare/fox/examples/pt/recipe_pt_avg_filter.py +++ b/nvflare/fox/examples/pt/recipe_pt_avg_filter.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from nvflare.fox.examples import export_recipe from nvflare.fox.examples.pt.filters import ( IncomingModelCallFilter, IncomingModelResultFilter, @@ -20,12 +21,14 @@ from nvflare.fox.examples.pt.pt_avg_filter import PTFedAvg, PTTrainer from nvflare.fox.sys.recipe import FoxRecipe -JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" - def main(): + export_recipe("fox_pt_fedavg_filter", _make_recipe) + + +def _make_recipe(job_name): recipe = FoxRecipe( - job_name="fox_pt_fedavg_filter", + job_name=job_name, server=PTFedAvg( initial_model={ "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], @@ -47,8 +50,7 @@ def main(): filters=[IncomingModelCallFilter("weights")], ) recipe.add_client_outgoing_result_filters(pattern="*.train", filters=[OutgoingModelResultFilter()]) - - recipe.export(JOB_ROOT_DIR) + return recipe if __name__ == "__main__": diff --git a/nvflare/fox/examples/pt/recipe_pt_avg_filter2.py b/nvflare/fox/examples/pt/recipe_pt_avg_filter2.py index 2764409d24..db744bf08c 100644 --- a/nvflare/fox/examples/pt/recipe_pt_avg_filter2.py +++ b/nvflare/fox/examples/pt/recipe_pt_avg_filter2.py @@ -11,17 +11,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from nvflare.fox.examples import export_recipe from nvflare.fox.examples.pt.filters2 import ModelFilter from nvflare.fox.examples.pt.pt_avg_filter import PTFedAvg, PTTrainer from nvflare.fox.sys.recipe import FoxRecipe -JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" - def main(): + export_recipe("fox_pt_fedavg_filter2", _make_recipe) + +def _make_recipe(job_name): recipe = FoxRecipe( - job_name="fox_pt_fedavg_filter2", + job_name=job_name, server=PTFedAvg( initial_model={ "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], @@ -44,8 +46,7 @@ def main(): filters=[model_filter], ) recipe.add_client_outgoing_result_filters(pattern="*.train", filters=[model_filter]) - - recipe.export(JOB_ROOT_DIR) + return recipe if __name__ == "__main__": diff --git a/nvflare/fox/examples/pt/recipe_pt_avg_mixed.py b/nvflare/fox/examples/pt/recipe_pt_avg_mixed.py index 147224373c..634637cc19 100644 --- a/nvflare/fox/examples/pt/recipe_pt_avg_mixed.py +++ b/nvflare/fox/examples/pt/recipe_pt_avg_mixed.py @@ -11,21 +11,24 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from nvflare.fox.examples import export_recipe from nvflare.fox.examples.pt.pt_avg_mixed import PTFedAvgMixed, PTTrainer from nvflare.fox.sys.recipe import FoxRecipe -JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" - def main(): + export_recipe("fox_pt_fedavg_mixed", _make_recipe) + + +def _make_recipe(job_name): init_model = { "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], "y": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], "z": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], } - recipe = FoxRecipe( - job_name="fox_pt_fedavg_mixed", + return FoxRecipe( + job_name=job_name, server=PTFedAvgMixed( pt_model=init_model, np_model=init_model, @@ -33,7 +36,6 @@ def main(): ), client=PTTrainer(delta=1.0), ) - recipe.export(JOB_ROOT_DIR) if __name__ == "__main__": diff --git a/nvflare/fox/examples/pt/recipe_pt_avg_stream.py b/nvflare/fox/examples/pt/recipe_pt_avg_stream.py index 9699cf04c3..fce248b77e 100644 --- a/nvflare/fox/examples/pt/recipe_pt_avg_stream.py +++ b/nvflare/fox/examples/pt/recipe_pt_avg_stream.py @@ -11,15 +11,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from nvflare.fox.examples import export_recipe from nvflare.fox.examples.pt.pt_avg_stream import PTFedAvgStream, PTTrainer from nvflare.fox.sys.recipe import FoxRecipe -JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" - def main(): - recipe = FoxRecipe( - job_name="fox_pt_fedavg_stream", + export_recipe("fox_pt_fedavg_stream", _make_recipe) + + +def _make_recipe(job_name): + return FoxRecipe( + job_name=job_name, server=PTFedAvgStream( initial_model={ "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], @@ -30,7 +33,6 @@ def main(): ), client=PTTrainer(delta=1.0), ) - recipe.export(JOB_ROOT_DIR) if __name__ == "__main__": diff --git a/nvflare/fox/examples/pt/recipe_pt_avg_stream2.py b/nvflare/fox/examples/pt/recipe_pt_avg_stream2.py index a7a516e2a0..b17d98f6f9 100644 --- a/nvflare/fox/examples/pt/recipe_pt_avg_stream2.py +++ b/nvflare/fox/examples/pt/recipe_pt_avg_stream2.py @@ -11,15 +11,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from nvflare.fox.examples import export_recipe from nvflare.fox.examples.pt.pt_avg_stream2 import PTFedAvgStream, PTTrainer from nvflare.fox.sys.recipe import FoxRecipe -JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" - def main(): - recipe = FoxRecipe( - job_name="fox_pt_fedavg_stream2", + export_recipe("fox_pt_fedavg_stream2", _make_recipe) + + +def _make_recipe(job_name): + return FoxRecipe( + job_name=job_name, server=PTFedAvgStream( initial_model={ "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], @@ -30,7 +33,6 @@ def main(): ), client=PTTrainer(delta=1.0), ) - recipe.export(JOB_ROOT_DIR) if __name__ == "__main__": diff --git a/nvflare/fox/examples/pt/recipe_pt_np.py b/nvflare/fox/examples/pt/recipe_pt_np.py index 1cd043c5dd..65a15ba75f 100644 --- a/nvflare/fox/examples/pt/recipe_pt_np.py +++ b/nvflare/fox/examples/pt/recipe_pt_np.py @@ -13,13 +13,16 @@ # limitations under the License. from nvflare.app_common.decomposers.numpy_decomposers import NumpyArrayDecomposer from nvflare.app_opt.pt.decomposers import TensorDecomposer +from nvflare.fox.examples import export_recipe from nvflare.fox.examples.pt.pt_np import PTFedAvgMixed, PTTrainer from nvflare.fox.sys.recipe import FoxRecipe -JOB_ROOT_DIR = "/Users/yanc/NVFlare/sandbox/v27/prod_00/admin@nvidia.com/transfer" - def main(): + export_recipe("fox_pt_np", _make_recipe) + + +def _make_recipe(job_name): init_model = { "x": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], "y": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], @@ -27,7 +30,7 @@ def main(): } recipe = FoxRecipe( - job_name="fox_pt_np", + job_name=job_name, server=PTFedAvgMixed( pt_model=init_model, np_model=init_model, @@ -36,7 +39,7 @@ def main(): client=PTTrainer(delta=1.0), ) recipe.add_decomposers([TensorDecomposer(), NumpyArrayDecomposer()]) - recipe.export(JOB_ROOT_DIR) + return recipe if __name__ == "__main__": From 898e3f459067a15bd4138ea9eaf36767e27a1b16 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Mon, 15 Dec 2025 10:38:03 -0500 Subject: [PATCH 089/102] remove unused imports --- nvflare/fox/examples/np/algos/client.py | 1 - nvflare/fox/sys/backend.py | 10 ++++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/nvflare/fox/examples/np/algos/client.py b/nvflare/fox/examples/np/algos/client.py index 2048341ebd..154ab9a6c3 100644 --- a/nvflare/fox/examples/np/algos/client.py +++ b/nvflare/fox/examples/np/algos/client.py @@ -13,7 +13,6 @@ # limitations under the License. import random import time -import traceback from nvflare.fox import fox from nvflare.fuel.utils.log_utils import get_obj_logger diff --git a/nvflare/fox/sys/backend.py b/nvflare/fox/sys/backend.py index 3023cd5e97..4a8fa0ccc5 100644 --- a/nvflare/fox/sys/backend.py +++ b/nvflare/fox/sys/backend.py @@ -77,12 +77,18 @@ def _call_target( send_complete_cb=send_complete_cb, **cb_kwargs, ) - assert isinstance(reply, Message) + if not isinstance(reply, Message): + self.logger.error(f"cell message reply must be Message but got {type(reply)}") + raise TimeoutError(f"function {func_name} failed with internal error") + rc = reply.get_header(MessageHeaderKey.RETURN_CODE, ReturnCode.OK) if rc == ReturnCode.TIMEOUT: raise TimeoutError(f"function {func_name} timed out after {timeout} seconds") elif rc != ReturnCode.OK: - raise RuntimeError(f"function {func_name} failed: {rc}") + error = None + if isinstance(reply.payload, dict): + error = reply.payload.get(CallReplyKey.ERROR) + raise RuntimeError(f"function {func_name} failed: {rc=} {error=}") if not isinstance(reply.payload, dict): raise RuntimeError(f"function {func_name} failed: reply must be dict but got {type(reply.payload)}") From a075e5de01d00df6dc631efb145693647edbec6e Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Mon, 15 Dec 2025 12:34:57 -0500 Subject: [PATCH 090/102] add docstring --- nvflare/fox/api/facade.py | 168 ++++++++++++++++++ .../np/algos/strategies/avg_para_tc.py | 1 + 2 files changed, 169 insertions(+) diff --git a/nvflare/fox/api/facade.py b/nvflare/fox/api/facade.py index 351c186bf7..d293bc766e 100644 --- a/nvflare/fox/api/facade.py +++ b/nvflare/fox/api/facade.py @@ -42,40 +42,84 @@ class facade: @classproperty def context(cls): + """Get the call context. + + Returns: a context object + + """ return get_call_context() @classproperty def caller(cls): + """Get the site name of the caller + + Returns: name of the caller + + """ ctx = get_call_context() return ctx.caller @classproperty def callee(cls): + """Get the fully qualified collab object name of the invoked object: [.] + + Returns: fully qualified collab object name of the invoked object + + """ ctx = get_call_context() return ctx.callee @classproperty def call_info(cls): + """Get a string that represents call information + + Returns: a string that represents call information + + The string looks like: + + :=> + + """ ctx = get_call_context() return str(ctx) @classproperty def site_name(cls): + """Get the current site name, which is the name of the "app" object of the current site. + + Returns: the current site name + + """ ctx = get_call_context() return ctx.app.name @classproperty def server(cls): + """Get the server proxy. + + Returns: the server proxy + + """ ctx = get_call_context() return ctx.server @classproperty def clients(cls): + """Get all client proxies. + + Returns: all client proxies as a ProxyList + + """ ctx = get_call_context() return ProxyList(ctx.clients) @classproperty def other_clients(cls): + """Get all client proxies, excluding the site's own proxy. + + Returns: all client proxies, excluding the site's own proxy + + """ ctx = get_call_context() # Note that ctx.clients returns a copy of client proxies, not the original client proxy list! @@ -88,6 +132,11 @@ def other_clients(cls): @classproperty def child_clients(cls): + """Get all child client proxies. + + Returns: all child client proxies if the site has children. An exception is raised if no children. + + """ ctx = get_call_context() candidates = ctx.app.get_children() if not candidates: @@ -96,11 +145,21 @@ def child_clients(cls): @classproperty def has_children(cls): + """Check whether the site has any child proxies. + + Returns: whether the site has any child proxies + + """ ctx = get_call_context() return ctx.app.has_children() @classproperty def leaf_clients(cls): + """Get all leaf client proxies. + + Returns: all leaf client proxies + + """ ctx = get_call_context() candidates = ctx.app.get_leaf_clients() if not candidates: @@ -109,6 +168,14 @@ def leaf_clients(cls): @classmethod def get_clients(cls, names: list[str]): + """Get proxies for specified site names. + + Args: + names: names of the sites for which to get proxies. + + Returns: + + """ ctx = get_call_context() candidates = ctx.clients result = [] @@ -126,49 +193,150 @@ def get_clients(cls, names: list[str]): @classproperty def backend_type(cls): + """Get the backend type of the current site. + + Returns: the backend type of the current site + + """ ctx = get_call_context() return ctx.backend_type @classproperty def is_aborted(cls): + """Check whether the job/experiment has been aborted. + + Returns: whether the job/experiment has been aborted + + """ ctx = get_call_context() return ctx.is_aborted() @classproperty def workspace(cls): + """Get the workspace object. + + Returns: the workspace object + + """ ctx = get_call_context() return ctx.workspace @classproperty def filter_direction(cls): + """Get the direction of filter call (incoming or outgoing). Only available to filter functions. + + Returns: the direction of filter call + + """ ctx = get_call_context() return ctx.get_prop(ContextKey.DIRECTION) @classproperty def qual_func_name(cls): + """Get the filter's qualified function name. Only available to filter functions. + + Returns: the filter's qualified function name + + """ ctx = get_call_context() return ctx.get_prop(ContextKey.QUALIFIED_FUNC_NAME) @staticmethod def fire_event(event_type: str, data): + """Fire an event to listening objects within the site. + + Args: + event_type: type of the event + data: data of the event + + Returns: results from event handlers. + + """ ctx = get_call_context() return ctx.app.fire_event(event_type, data, ctx) @staticmethod def register_event_handler(event_type: str, handler, **handler_kwargs): + """Register an event handler for a specified event type + + Args: + event_type: type of the event + handler: the handler function to be registered + **handler_kwargs: kwargs to be passed to the handler + + Returns: None + + """ ctx = get_call_context() ctx.app.register_event_handler(event_type, handler, **handler_kwargs) @staticmethod def get_app_prop(name: str, default=None): + """Get a specified property from the site's app (usually for configuration properties). + + Args: + name: name of the property. + default: default value if the property does not exist. + + Returns: value of the specified app property, or default value if the property does not exist + + """ ctx = get_call_context() return ctx.app.get_prop(name, default) + @staticmethod + def set_app_prop(name: str, value): + """Set a specified property into the site's app. + Properties in app are permanent during the job/experiment execution. + + Args: + name: name of the property. + value: value of the property. + + Returns: + + """ + ctx = get_call_context() + return ctx.app.set_prop(name, value) + @staticmethod def get_prop(name: str, default=None): + """Get a specified property from the call context. Usually for sharing information during collab function + processing. + + Args: + name: name of the property. + default: default value if the property does not exist. + + Returns: + + """ ctx = get_call_context() return ctx.get_prop(name, default) + @staticmethod + def set_prop(name: str, value): + """Set a specified property into the call context. Usually for sharing information during collab function + processing. + + Args: + name: name of the property. + value: value of the property. + + Returns: + + """ + ctx = get_call_context() + return ctx.set_prop(name, value) + @staticmethod def get_result(default=None): + """Get the last algo execution result from the call context. + + Args: + default: the default value if the result does not exist in the call context. + + Returns: the last algo execution result from the call context + + """ return facade.get_prop(ContextKey.RESULT, default) diff --git a/nvflare/fox/examples/np/algos/strategies/avg_para_tc.py b/nvflare/fox/examples/np/algos/strategies/avg_para_tc.py index 37e1f1a1c8..fb9802ea3f 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_para_tc.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_para_tc.py @@ -34,6 +34,7 @@ def execute(self): if current_model is None: self.logger.error(f"training failed at round {i}") break + self.logger.info(f"FINAL MODEL: {current_model}") return current_model def _do_one_round(self, r, current_model): From bf84d51c3e17ab7e8dc471cc275e4a9d34e4f715 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Mon, 15 Dec 2025 15:30:39 -0500 Subject: [PATCH 091/102] print collab interface in simulator --- nvflare/fox/sim/simulator.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nvflare/fox/sim/simulator.py b/nvflare/fox/sim/simulator.py index 4c55e7ae8f..57a6dfb1e8 100644 --- a/nvflare/fox/sim/simulator.py +++ b/nvflare/fox/sim/simulator.py @@ -173,6 +173,9 @@ def _build_hierarchical_clients(self, height: int, num_children_per_parent: int) return client_apps def run(self): + self.logger.debug(f"Server Collab Interface: {self.server_app.get_collab_interface()}") + self.logger.debug(f"Client Collab Interface: {self.client_app.get_collab_interface()}") + try: result = self._try_run() except KeyboardInterrupt: From ca1c933ab6a26cd51f109daa57c95f95c9ca3da3 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Tue, 16 Dec 2025 13:24:07 -0500 Subject: [PATCH 092/102] address pr comments --- nvflare/fox/api/app.py | 7 ++++-- nvflare/fox/api/backend.py | 4 +++- nvflare/fox/api/constants.py | 3 +++ nvflare/fox/api/dec.py | 10 +++++++++ nvflare/fox/api/gcc.py | 17 +++++++------- nvflare/fox/api/group.py | 3 --- nvflare/fox/api/proxy.py | 7 +++--- nvflare/fox/api/utils.py | 3 ++- nvflare/fox/examples/np/algos/avg_stream.py | 6 ++++- .../examples/np/algos/strategies/cyclic.py | 5 +++++ nvflare/fox/examples/np/algos/swarm.py | 3 ++- nvflare/fox/examples/np/algos/utils.py | 21 ++++++++++++++++++ nvflare/fox/examples/pt/pt_avg_filter.py | 2 +- nvflare/fox/examples/pt/pt_avg_mixed.py | 5 ++++- nvflare/fox/examples/pt/pt_avg_stream.py | 3 +++ nvflare/fox/examples/pt/pt_avg_stream2.py | 3 +++ nvflare/fox/examples/pt/pt_np.py | 5 ++++- nvflare/fox/examples/pt/utils.py | 20 ++++++++++++++++- nvflare/fox/sim/backend.py | 15 ++++++------- nvflare/fox/sim/simulator.py | 22 ++++++++++++++----- nvflare/fox/sys/adaptor.py | 3 +++ nvflare/fox/sys/backend.py | 20 ++++++++++++----- nvflare/fox/sys/controller.py | 4 ++++ nvflare/fox/sys/downloader.py | 6 ++--- nvflare/fox/sys/executor.py | 8 ++++++- nvflare/fox/utils/tensor_receiver.py | 2 +- 26 files changed, 159 insertions(+), 48 deletions(-) diff --git a/nvflare/fox/api/app.py b/nvflare/fox/api/app.py index b5a5ba4225..1a8070d61d 100644 --- a/nvflare/fox/api/app.py +++ b/nvflare/fox/api/app.py @@ -103,9 +103,12 @@ def _add_filters(self, pattern: str, filters, to_list: list, filter_type, incomi filter_obj = filter_type(f, incoming) else: filter_obj = f - self._add_managed_object(f) + filter_objs.append(filter_obj) + # f is a managed object, but the filter_obj (if wrapped) is not! + self._add_managed_object(f) + chain = FilterChain(pattern, filter_type) chain.add_filters(filter_objs) to_list.append(chain) @@ -393,7 +396,7 @@ def get_children(self): if my_node.children: return [node.obj for node in my_node.children] else: - return None + return [] def has_children(self): assert isinstance(self._client_hierarchy, Forest) diff --git a/nvflare/fox/api/backend.py b/nvflare/fox/api/backend.py index b81fd75e2a..5699de4009 100644 --- a/nvflare/fox/api/backend.py +++ b/nvflare/fox/api/backend.py @@ -18,6 +18,7 @@ from nvflare.security.logging import secure_format_traceback from .call_opt import CallOption +from .ctx import Context from .gcc import GroupCallContext @@ -31,11 +32,12 @@ def __init__(self, abort_signal: Signal): self.logger = get_obj_logger(self) @abstractmethod - def call_target(self, target_name: str, call_opt: CallOption, func_name: str, *args, **kwargs): + def call_target(self, context: Context, target_name: str, call_opt: CallOption, func_name: str, *args, **kwargs): """ Call a target function with arguments and return a result. Args: + context: the call context target_name: the fully qualified name of the target object to be called in the remote app. call_opt: call options. func_name: name of the function to be called in the remote app. diff --git a/nvflare/fox/api/constants.py b/nvflare/fox/api/constants.py index 437df6fd1e..863844d45c 100644 --- a/nvflare/fox/api/constants.py +++ b/nvflare/fox/api/constants.py @@ -30,3 +30,6 @@ class FilterDirection: class BackendType: SIMULATION = "simulation" FLARE = "flare" + + +MAKE_CLIENT_APP_METHOD = "make_client_app" diff --git a/nvflare/fox/api/dec.py b/nvflare/fox/api/dec.py index bdc9a8e9fa..1ec7d03c0c 100644 --- a/nvflare/fox/api/dec.py +++ b/nvflare/fox/api/dec.py @@ -210,5 +210,15 @@ def supports_context(func): def adjust_kwargs(func, kwargs): + """Adjust the kwargs and remove keys that are not supported by the func. + + Args: + func: the func to be checked + kwargs: the kwargs to be adjusted + + Returns: the adjusted kwargs + + """ if not supports_context(func): kwargs.pop(CollabMethodArgName.CONTEXT, None) + return kwargs diff --git a/nvflare/fox/api/gcc.py b/nvflare/fox/api/gcc.py index d96c71b2be..f581376432 100644 --- a/nvflare/fox/api/gcc.py +++ b/nvflare/fox/api/gcc.py @@ -183,14 +183,15 @@ def set_result(self, result): if not isinstance(result, Exception): set_call_context(ctx) - result = self.app.apply_incoming_result_filters(self.target_name, self.func_name, result, ctx) - if self.process_cb: - self.cb_kwargs[CollabMethodArgName.CONTEXT] = ctx - check_context_support(self.process_cb, self.cb_kwargs) - result = self.process_cb(self, result, **self.cb_kwargs) - - # set back to original context - set_call_context(self.context) + try: + result = self.app.apply_incoming_result_filters(self.target_name, self.func_name, result, ctx) + if self.process_cb: + self.cb_kwargs[CollabMethodArgName.CONTEXT] = ctx + check_context_support(self.process_cb, self.cb_kwargs) + result = self.process_cb(self, result, **self.cb_kwargs) + finally: + # set back to original context + set_call_context(self.context) except Exception as ex: result = ex finally: diff --git a/nvflare/fox/api/group.py b/nvflare/fox/api/group.py index 8efcd825c7..1afe35088c 100644 --- a/nvflare/fox/api/group.py +++ b/nvflare/fox/api/group.py @@ -21,7 +21,6 @@ from .app import App from .call_opt import CallOption -from .constants import CollabMethodArgName from .ctx import Context from .gcc import GroupCallContext, ResultWaiter from .proxy import Proxy @@ -148,8 +147,6 @@ def method(*args, **kwargs): func_proxy.caller_name, func_proxy.name, target_group=self, set_call_ctx=False ) - call_kwargs[CollabMethodArgName.CONTEXT] = ctx - gcc = GroupCallContext( app=self._app, target_name=func_proxy.target_name, diff --git a/nvflare/fox/api/proxy.py b/nvflare/fox/api/proxy.py index 2d81f041a8..6a72fa33e1 100644 --- a/nvflare/fox/api/proxy.py +++ b/nvflare/fox/api/proxy.py @@ -17,7 +17,6 @@ from .backend import Backend from .call_opt import CallOption -from .constants import CollabMethodArgName from .utils import check_call_args @@ -218,8 +217,7 @@ def call_func(self, call_opt: CallOption, func_name, args, kwargs): call_kwargs = self.app.apply_outgoing_call_filters(p.target_name, func_name, call_kwargs, ctx) check_call_args(func_name, func_itf, call_args, call_kwargs) - call_kwargs[CollabMethodArgName.CONTEXT] = ctx - result = p.backend.call_target(p.target_name, call_opt, func_name, *call_args, **call_kwargs) + result = p.backend.call_target(ctx, p.target_name, call_opt, func_name, *call_args, **call_kwargs) if isinstance(result, Exception): raise result @@ -229,6 +227,9 @@ def call_func(self, call_opt: CallOption, func_name, args, kwargs): return result except Exception as ex: self.backend.handle_exception(ex) + + # Must return the exception as the result of the func call. + # Do NOT raise it! return ex def __getattr__(self, func_name): diff --git a/nvflare/fox/api/utils.py b/nvflare/fox/api/utils.py index 6dbb6860be..2349711e44 100644 --- a/nvflare/fox/api/utils.py +++ b/nvflare/fox/api/utils.py @@ -63,8 +63,9 @@ def check_call_args(func_name, func_itf, call_args, call_kwargs: dict): """ num_call_args = len(call_args) + len(call_kwargs) if num_call_args > len(func_itf): + # For security, collab funcs must only have fixed args - no flexible args are allowed. raise RuntimeError( - f"there are {num_call_args} call args ({call_args=} {call_kwargs=}), " + f"there are {num_call_args} call args ({len(call_args)=} {len(call_kwargs)=}), " f"but function '{func_name}' only supports {len(func_itf)} args ({func_itf})" ) diff --git a/nvflare/fox/examples/np/algos/avg_stream.py b/nvflare/fox/examples/np/algos/avg_stream.py index ed3472eed4..ed205e461a 100644 --- a/nvflare/fox/examples/np/algos/avg_stream.py +++ b/nvflare/fox/examples/np/algos/avg_stream.py @@ -46,6 +46,9 @@ def execute(self): current_model = self._init_model for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model) + if current_model is None: + self.logger.error(f"training failed at round {i}") + break self.logger.info(f"FINAL MODEL: {current_model}") return current_model @@ -118,7 +121,8 @@ def __init__(self, delta: float): def train(self, current_round, weights, model_type: str): if fox.is_aborted: self.logger.debug("training aborted") - return 0 + return None, "" + self.logger.debug(f"[{fox.call_info}] training round {current_round}: {model_type=} {weights=}") if model_type == "ref": err, file_path = download_file(ref=weights, per_request_timeout=5.0) diff --git a/nvflare/fox/examples/np/algos/strategies/cyclic.py b/nvflare/fox/examples/np/algos/strategies/cyclic.py index e2338b0f4f..86eb0c37f6 100644 --- a/nvflare/fox/examples/np/algos/strategies/cyclic.py +++ b/nvflare/fox/examples/np/algos/strategies/cyclic.py @@ -43,6 +43,9 @@ def execute(self): current_model = self._initial_model for current_round in range(self.num_rounds): current_model = self._do_one_round(current_round, current_model) + if current_model is None: + self.logger.error(f"training failed at round {current_round}") + break self.logger.info(f"[{fox.call_info}] final result: {current_model}") self.final_model = current_model return current_model @@ -60,5 +63,7 @@ def _do_one_round(self, current_round, current_model): random.shuffle(clients) for c in clients: current_model = c.train(current_round, current_model) + if current_model is None: + return None self.logger.info(f"[{fox.call_info}] result from {c.name}: {current_model}") return current_model diff --git a/nvflare/fox/examples/np/algos/swarm.py b/nvflare/fox/examples/np/algos/swarm.py index cf73268335..115d2264c6 100644 --- a/nvflare/fox/examples/np/algos/swarm.py +++ b/nvflare/fox/examples/np/algos/swarm.py @@ -89,8 +89,9 @@ def swarm_learn(self, num_rounds, model, current_round): self.logger.info("notify server all done!") try: fox.server(expect_result=False).all_done("OK") - except: + except Exception as ex: traceback.print_exc() + self.logger.error(f"exception occurred in learning: {type(ex)}") self.logger.info("Swarm Training is DONE!") return diff --git a/nvflare/fox/examples/np/algos/utils.py b/nvflare/fox/examples/np/algos/utils.py index 7d355a3249..e5e146c835 100644 --- a/nvflare/fox/examples/np/algos/utils.py +++ b/nvflare/fox/examples/np/algos/utils.py @@ -55,13 +55,34 @@ def load_np_model(file_name: str): def add(model: dict, to_model: dict): + """Add specified model to another model + + Args: + model: the model to be added + to_model: the model to be added to. + + Returns: the updated model + Notes: the to_model is updated + + """ for k, v in model.items(): if k not in to_model: to_model[k] = v else: to_model[k] += v + return to_model def div(model: dict, value): + """Divide a model by a specified value + + Args: + model: the model to be divided + value: the value to divide the model with + + Returns: the updated model + + """ for k, v in model.items(): model[k] = v / value + return model diff --git a/nvflare/fox/examples/pt/pt_avg_filter.py b/nvflare/fox/examples/pt/pt_avg_filter.py index 465eec81f1..8a4bca495f 100644 --- a/nvflare/fox/examples/pt/pt_avg_filter.py +++ b/nvflare/fox/examples/pt/pt_avg_filter.py @@ -39,7 +39,7 @@ def execute(self): current_model = self._init_model for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model) - if not current_model: + if current_model is None: self.logger.error(f"training failed at round {i}") break self.logger.info(f"FINAL MODEL: {current_model}") diff --git a/nvflare/fox/examples/pt/pt_avg_mixed.py b/nvflare/fox/examples/pt/pt_avg_mixed.py index da93f87292..102957b9df 100644 --- a/nvflare/fox/examples/pt/pt_avg_mixed.py +++ b/nvflare/fox/examples/pt/pt_avg_mixed.py @@ -59,6 +59,9 @@ def execute(self): pt_model, np_model = self._pt_model, self._np_model for i in range(self.num_rounds): pt_model, np_model = self._do_one_round(i, pt_model, np_model) + if pt_model is None or np_model is None: + self.logger.error(f"training failed at round {i}") + break self.logger.info(f"FINAL MODEL: {pt_model=} {np_model=}") return pt_model, np_model @@ -152,7 +155,7 @@ def __init__(self, delta: float): def train(self, current_round, pt_model, np_model, model_type: str): if fox.is_aborted: self.logger.debug("training aborted") - return 0 + return None, None, "" self.logger.debug(f"[{fox.call_info}] training round {current_round}: {model_type=} {pt_model=} {np_model=}") diff --git a/nvflare/fox/examples/pt/pt_avg_stream.py b/nvflare/fox/examples/pt/pt_avg_stream.py index 6e41e5bae7..fbb4f02e1a 100644 --- a/nvflare/fox/examples/pt/pt_avg_stream.py +++ b/nvflare/fox/examples/pt/pt_avg_stream.py @@ -50,6 +50,9 @@ def execute(self): current_model = self._init_model for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model) + if current_model is None: + self.logger.error(f"training failed at round {i}") + break self.logger.info(f"FINAL MODEL: {current_model}") return current_model diff --git a/nvflare/fox/examples/pt/pt_avg_stream2.py b/nvflare/fox/examples/pt/pt_avg_stream2.py index 67e13cf588..4ab4dd0a6f 100644 --- a/nvflare/fox/examples/pt/pt_avg_stream2.py +++ b/nvflare/fox/examples/pt/pt_avg_stream2.py @@ -42,6 +42,9 @@ def execute(self): current_model = self._init_model for i in range(self.num_rounds): current_model = self._do_one_round(i, current_model) + if current_model is None: + self.logger.error(f"training failed at round {i}") + break self.logger.info(f"FINAL MODEL: {current_model}") return current_model diff --git a/nvflare/fox/examples/pt/pt_np.py b/nvflare/fox/examples/pt/pt_np.py index 21ae4a456b..8c2f2d82de 100644 --- a/nvflare/fox/examples/pt/pt_np.py +++ b/nvflare/fox/examples/pt/pt_np.py @@ -54,6 +54,9 @@ def execute(self): pt_model, np_model = self._pt_model, self._np_model for i in range(self.num_rounds): pt_model, np_model = self._do_one_round(i, pt_model, np_model) + if pt_model is None or np_model is None: + self.logger.error(f"training failed at round {i}") + break self.logger.info(f"FINAL MODEL: {pt_model=} {np_model=}") return pt_model, np_model @@ -102,7 +105,7 @@ def __init__(self, delta: float): def train(self, current_round, pt_model, np_model): if fox.is_aborted: self.logger.debug("training aborted") - return 0 + return None, None self.logger.debug(f"[{fox.call_info}] training round {current_round}: {pt_model=} {np_model=}") diff --git a/nvflare/fox/examples/pt/utils.py b/nvflare/fox/examples/pt/utils.py index df60cb3667..a41e5cb130 100644 --- a/nvflare/fox/examples/pt/utils.py +++ b/nvflare/fox/examples/pt/utils.py @@ -22,7 +22,7 @@ def parse_array_def(array_def): return array_def if isinstance(array_def, list): - return torch.Tensor(array_def) + return torch.tensor(array_def) else: raise ValueError(f"unsupported array def: {array_def}") @@ -42,6 +42,15 @@ def parse_model_def(model_def): def add(value: dict, to_model: dict): + """Add value to a specified model in-place. + + Args: + value: + to_model: + + Returns: + + """ for k, v in value.items(): if k not in to_model: to_model[k] = v @@ -51,6 +60,15 @@ def add(value: dict, to_model: dict): def div(model: dict, value): + """Divide the model in-place by a specified value. + + Args: + model: + value: + + Returns: + + """ for k, v in model.items(): model[k] = torch.div(v, value) return model diff --git a/nvflare/fox/sim/backend.py b/nvflare/fox/sim/backend.py index 6bc3799087..6f128b4a5a 100644 --- a/nvflare/fox/sim/backend.py +++ b/nvflare/fox/sim/backend.py @@ -43,7 +43,7 @@ def __init__(self, target_obj_name: str, target_app: App, target_obj, abort_sign def _get_func(self, func_name): return self.target_app.find_collab_method(self.target_obj, func_name) - def call_target(self, target_name: str, call_opt: CallOption, func_name: str, *args, **kwargs): + def call_target(self, context, target_name: str, call_opt: CallOption, func_name: str, *args, **kwargs): func = self._get_func(func_name) if not func: raise AttributeError(f"{target_name} does not have method '{func_name}' or it is not collab") @@ -58,7 +58,7 @@ def call_target(self, target_name: str, call_opt: CallOption, func_name: str, *a if expect_result: waiter = _Waiter() - self.executor.submit(self._run_func, waiter, target_name, func_name, func, args, kwargs) + self.executor.submit(self._run_func, waiter, context, target_name, func_name, func, args, kwargs) if waiter: start_time = time.time() while True: @@ -80,8 +80,8 @@ def call_target(self, target_name: str, call_opt: CallOption, func_name: str, *a else: return None - def _preprocess(self, target_name, func_name, func, kwargs): - caller_ctx = kwargs.pop(CollabMethodArgName.CONTEXT) + def _preprocess(self, context, target_name, func_name, func, kwargs): + caller_ctx = context my_ctx = self.target_app.new_context(caller_ctx.caller, caller_ctx.callee) kwargs = self.target_app.apply_incoming_call_filters(target_name, func_name, kwargs, my_ctx) @@ -99,11 +99,10 @@ def _preprocess(self, target_name, func_name, func, kwargs): adjust_kwargs(func, kwargs) return my_ctx, kwargs - def _run_func(self, waiter: _Waiter, target_name, func_name, func, args, kwargs): + def _run_func(self, waiter: _Waiter, context, target_name, func_name, func, args, kwargs): try: - ctx, kwargs = self._preprocess(target_name, func_name, func, kwargs) + ctx, kwargs = self._preprocess(context, target_name, func_name, func, kwargs) result = func(*args, **kwargs) - # set_call_context(ctx) # apply result filter result = self.target_app.apply_outgoing_result_filters(target_name, func_name, result, ctx) @@ -130,7 +129,7 @@ def call_target_in_group(self, gcc: GroupCallContext, func_name: str, *args, **k def _run_func_in_group(self, gcc: GroupCallContext, func_name, args, kwargs): try: target_name = gcc.target_name - result = self.call_target(target_name, gcc.call_opt, func_name, *args, **kwargs) + result = self.call_target(gcc.context, target_name, gcc.call_opt, func_name, *args, **kwargs) gcc.send_completed() gcc.set_result(result) except Exception as ex: diff --git a/nvflare/fox/sim/simulator.py b/nvflare/fox/sim/simulator.py index 57a6dfb1e8..1301fe700a 100644 --- a/nvflare/fox/sim/simulator.py +++ b/nvflare/fox/sim/simulator.py @@ -18,7 +18,7 @@ from nvflare.apis.signal import Signal from nvflare.fox.api.app import App, ClientApp, ServerApp -from nvflare.fox.api.constants import BackendType +from nvflare.fox.api.constants import MAKE_CLIENT_APP_METHOD, BackendType from nvflare.fox.api.dec import get_object_collab_interface from nvflare.fox.api.proxy import Proxy from nvflare.fox.api.run_server import run_server @@ -57,14 +57,26 @@ def _prepare_proxy(for_app: App, target_app: App, backends: dict): app_proxy.add_child(name, p) return app_proxy - def _make_app(self, name, fqn): - make_client_app_f = getattr(self.client_app, "make_client_app", None) + def _make_app(self, site_name, fqn): + """Make a new client app instance for the specified site + + Args: + site_name: nme of the site + fqn: fully qualified name of the site + + Returns: a new instance of the app + + """ + # If the app contains "make_client_app" method, call it to make the app instance! + # Otherwise, make the instance by deep copying. + # If the client_app object cannot be deep-copied, then it must provide the make_client_app method. + make_client_app_f = getattr(self.client_app, MAKE_CLIENT_APP_METHOD, None) if make_client_app_f and callable(make_client_app_f): - app = make_client_app_f(name) + app = make_client_app_f(site_name, BackendType.SIMULATION) else: app = copy.deepcopy(self.client_app) - app.name = name + app.name = site_name app.fqn = fqn app.backend_type = BackendType.SIMULATION return app diff --git a/nvflare/fox/sys/adaptor.py b/nvflare/fox/sys/adaptor.py index 38b3643382..7e6cd9655c 100644 --- a/nvflare/fox/sys/adaptor.py +++ b/nvflare/fox/sys/adaptor.py @@ -98,6 +98,9 @@ def _parse_filter_chain(chain_name, chain_dict: dict, fl_ctx): if not filter_ids: return None, None, f"missing 'filters' in {chain_name}" + if not isinstance(filter_ids, list): + return None, None, f"invalid 'filters' in {chain_name}: expect list got {type(filter_ids)}" + engine = fl_ctx.get_engine() filters = [] for fid in filter_ids: diff --git a/nvflare/fox/sys/backend.py b/nvflare/fox/sys/backend.py index 4a8fa0ccc5..30b338264c 100644 --- a/nvflare/fox/sys/backend.py +++ b/nvflare/fox/sys/backend.py @@ -13,7 +13,6 @@ # limitations under the License. from nvflare.fox.api.backend import Backend from nvflare.fox.api.call_opt import CallOption -from nvflare.fox.api.constants import CollabMethodArgName from nvflare.fox.api.ctx import set_call_context from nvflare.fox.api.gcc import GroupCallContext from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey, ReturnCode @@ -35,8 +34,9 @@ def __init__(self, manager, engine, caller, cell, target_fqcn, abort_signal, thr self.target_fqcn = target_fqcn self.thread_executor = thread_executor - def call_target(self, target_name: str, call_opt: CallOption, func_name: str, *args, **kwargs): + def call_target(self, context, target_name: str, call_opt: CallOption, func_name: str, *args, **kwargs): return self._call_target( + context=context, target_name=target_name, call_opt=call_opt, send_complete_cb=None, @@ -47,10 +47,17 @@ def call_target(self, target_name: str, call_opt: CallOption, func_name: str, *a ) def _call_target( - self, target_name: str, call_opt: CallOption, send_complete_cb, cb_kwargs, func_name: str, *args, **kwargs + self, + context, + target_name: str, + call_opt: CallOption, + send_complete_cb, + cb_kwargs, + func_name: str, + *args, + **kwargs, ): - ctx = kwargs.pop(CollabMethodArgName.CONTEXT) - set_call_context(ctx) + set_call_context(context) payload = { ObjectCallKey.CALLER: self.caller, @@ -79,7 +86,7 @@ def _call_target( ) if not isinstance(reply, Message): self.logger.error(f"cell message reply must be Message but got {type(reply)}") - raise TimeoutError(f"function {func_name} failed with internal error") + raise RuntimeError(f"function {func_name} failed with internal error") rc = reply.get_header(MessageHeaderKey.RETURN_CODE, ReturnCode.OK) if rc == ReturnCode.TIMEOUT: @@ -119,6 +126,7 @@ def call_target_in_group(self, gcc: GroupCallContext, func_name: str, *args, **k def _run_func(self, gcc: GroupCallContext, func_name: str, args, kwargs): try: result = self._call_target( + context=gcc.context, target_name=gcc.target_name, call_opt=gcc.call_opt, func_name=func_name, diff --git a/nvflare/fox/sys/controller.py b/nvflare/fox/sys/controller.py index 97c70203a3..4ac1600283 100644 --- a/nvflare/fox/sys/controller.py +++ b/nvflare/fox/sys/controller.py @@ -82,6 +82,10 @@ def start_controller(self, fl_ctx: FLContext): engine = fl_ctx.get_engine() server_obj = engine.get_component(self.server_obj_id) + if not server_obj: + self.system_panic(f"no component defined for {self.server_obj_id}", fl_ctx) + return + app = ServerApp(server_obj) app.name = "server" diff --git a/nvflare/fox/sys/downloader.py b/nvflare/fox/sys/downloader.py index 8cda392bf4..97d922729d 100644 --- a/nvflare/fox/sys/downloader.py +++ b/nvflare/fox/sys/downloader.py @@ -86,7 +86,7 @@ def download_file(ref: dict, per_request_timeout: float): ctx = fox.context backend = ctx.backend if not isinstance(backend, FlareBackend): - raise ValueError(f"backend must be SysBackend but got {type(backend)}") + raise ValueError(f"backend must be FlareBackend but got {type(backend)}") obj_type = ref.get(DownloadRefKey.OBJECT_TYPE) if obj_type != ObjectType.FILE: @@ -105,7 +105,7 @@ def download_tensors(ref: dict, per_request_timeout: float, tensors_received_cb= ctx = fox.context backend = ctx.backend if not isinstance(backend, FlareBackend): - raise ValueError(f"backend must be SysBackend but got {type(backend)}") + raise ValueError(f"backend must be FlareBackend but got {type(backend)}") obj_type = ref.get(DownloadRefKey.OBJECT_TYPE) if obj_type != ObjectType.TENSORS: @@ -126,7 +126,7 @@ def download_arrays(ref: dict, per_request_timeout: float, arrays_received_cb=No ctx = fox.context backend = ctx.backend if not isinstance(backend, FlareBackend): - raise ValueError(f"backend must be SysBackend but got {type(backend)}") + raise ValueError(f"backend must be FlareBackend but got {type(backend)}") obj_type = ref.get(DownloadRefKey.OBJECT_TYPE) if obj_type != ObjectType.ARRAYS: diff --git a/nvflare/fox/sys/executor.py b/nvflare/fox/sys/executor.py index 04a3ba8b36..3cbb528fa5 100644 --- a/nvflare/fox/sys/executor.py +++ b/nvflare/fox/sys/executor.py @@ -23,7 +23,7 @@ from nvflare.apis.shareable import Shareable, make_reply from nvflare.apis.signal import Signal from nvflare.fox.api.app import ClientApp -from nvflare.fox.api.constants import BackendType +from nvflare.fox.api.constants import MAKE_CLIENT_APP_METHOD, BackendType from nvflare.fox.api.proxy import Proxy from nvflare.fuel.f3.cellnet.fqcn import FQCN @@ -73,6 +73,12 @@ def _handle_start_run(self, event_type: str, fl_ctx: FLContext): client_name = fl_ctx.get_identity_name() app = ClientApp(client_obj) + + # If the app contains "make_client_app" method, call it to make the app instance! + make_client_app_f = getattr(self.client_app, MAKE_CLIENT_APP_METHOD, None) + if make_client_app_f and callable(make_client_app_f): + app = make_client_app_f(client_name, BackendType.FLARE) + app.name = client_name app.backend_type = BackendType.FLARE self.client_app = app diff --git a/nvflare/fox/utils/tensor_receiver.py b/nvflare/fox/utils/tensor_receiver.py index f0905bbed8..d6b3224f3f 100644 --- a/nvflare/fox/utils/tensor_receiver.py +++ b/nvflare/fox/utils/tensor_receiver.py @@ -36,7 +36,7 @@ def __call__(self, gcc: GroupCallContext, result): self.logger.info(f"[{fox.call_info}] got train result from {fox.caller}: {result}") model, model_type = result if model_type == "ref": - err, model = download_tensors( + err, _ = download_tensors( ref=model, per_request_timeout=5.0, tensors_received_cb=self._receive_tensors, From a6bf64c3defb848cbbcbbd9f8ff7e4ff833d64ca Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Tue, 16 Dec 2025 13:58:41 -0500 Subject: [PATCH 093/102] fix client app None issue --- nvflare/fox/sys/executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nvflare/fox/sys/executor.py b/nvflare/fox/sys/executor.py index 3cbb528fa5..9d33994ee0 100644 --- a/nvflare/fox/sys/executor.py +++ b/nvflare/fox/sys/executor.py @@ -75,7 +75,7 @@ def _handle_start_run(self, event_type: str, fl_ctx: FLContext): app = ClientApp(client_obj) # If the app contains "make_client_app" method, call it to make the app instance! - make_client_app_f = getattr(self.client_app, MAKE_CLIENT_APP_METHOD, None) + make_client_app_f = getattr(app, MAKE_CLIENT_APP_METHOD, None) if make_client_app_f and callable(make_client_app_f): app = make_client_app_f(client_name, BackendType.FLARE) From 0326c8ae2db653fb7a38dffa98380ac7e3ec31b0 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Tue, 16 Dec 2025 14:31:45 -0500 Subject: [PATCH 094/102] address pr comments --- nvflare/fox/api/app.py | 2 +- nvflare/fox/api/group.py | 4 ++++ nvflare/fox/sim/simulator.py | 11 ++++++++++- nvflare/fox/sys/executor.py | 6 ++++++ 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/nvflare/fox/api/app.py b/nvflare/fox/api/app.py index 1a8070d61d..1c9f626ebc 100644 --- a/nvflare/fox/api/app.py +++ b/nvflare/fox/api/app.py @@ -213,7 +213,7 @@ def update_props(self, props: dict): def add_collab_object(self, name: str, obj): # name must be acceptable str - pattern = r"^[A-Za-z][A-Za-z0-9_]+$" + pattern = r"^[A-Za-z][A-Za-z0-9_]*$" if not re.match(pattern, name): raise ValueError(f"invalid name {name} for collab object - must be simple name starting with a letter") diff --git a/nvflare/fox/api/group.py b/nvflare/fox/api/group.py index 1afe35088c..4ebd9a3894 100644 --- a/nvflare/fox/api/group.py +++ b/nvflare/fox/api/group.py @@ -161,6 +161,10 @@ def method(*args, **kwargs): gcc.set_send_complete_cb(self._request_sent, waiter=waiter) while True: + # No need to use lock here even if in_sending could change after we read it + # from waiter.in_sending_count and use it to compare with max_parallel. + # This is because waiter.in_sending_count can only decrease (by other threads) + # after we read it here. If this happens, we will catch it in next iteration. in_sending = waiter.in_sending_count if in_sending < max_parallel: waiter.inc_sending() diff --git a/nvflare/fox/sim/simulator.py b/nvflare/fox/sim/simulator.py index 1301fe700a..5e08bfab7f 100644 --- a/nvflare/fox/sim/simulator.py +++ b/nvflare/fox/sim/simulator.py @@ -73,8 +73,17 @@ def _make_app(self, site_name, fqn): make_client_app_f = getattr(self.client_app, MAKE_CLIENT_APP_METHOD, None) if make_client_app_f and callable(make_client_app_f): app = make_client_app_f(site_name, BackendType.SIMULATION) + if not isinstance(app, ClientApp): + raise RuntimeError(f"result returned by {MAKE_CLIENT_APP_METHOD} must be ClientApp but got {type(app)}") else: - app = copy.deepcopy(self.client_app) + try: + app = copy.deepcopy(self.client_app) + except Exception as ex: + self.logger.error( + f"exception occurred {type(ex)} creating client app with deepcopy. " + f"Please implement the {MAKE_CLIENT_APP_METHOD} method in the client app class" + ) + raise ex app.name = site_name app.fqn = fqn diff --git a/nvflare/fox/sys/executor.py b/nvflare/fox/sys/executor.py index 9d33994ee0..7aa4c5c7e4 100644 --- a/nvflare/fox/sys/executor.py +++ b/nvflare/fox/sys/executor.py @@ -70,6 +70,10 @@ def _handle_start_run(self, event_type: str, fl_ctx: FLContext): fl_ctx.set_prop(FLContextKey.FOX_MODE, True, private=True, sticky=True) engine = fl_ctx.get_engine() client_obj = engine.get_component(self.client_obj_id) + if not client_obj: + self.system_panic(f"cannot get client component {self.client_obj_id}", fl_ctx) + return + client_name = fl_ctx.get_identity_name() app = ClientApp(client_obj) @@ -78,6 +82,8 @@ def _handle_start_run(self, event_type: str, fl_ctx: FLContext): make_client_app_f = getattr(app, MAKE_CLIENT_APP_METHOD, None) if make_client_app_f and callable(make_client_app_f): app = make_client_app_f(client_name, BackendType.FLARE) + if not isinstance(app, ClientApp): + raise RuntimeError(f"result returned by {MAKE_CLIENT_APP_METHOD} must be ClientApp but got {type(app)}") app.name = client_name app.backend_type = BackendType.FLARE From 62de0afc2e9f8f3cfaca12034e4748467c5ff267 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Tue, 16 Dec 2025 16:07:51 -0500 Subject: [PATCH 095/102] pr review comments --- nvflare/fox/api/app.py | 5 ++++- nvflare/fox/api/group.py | 5 ++++- nvflare/fox/sim/simulator.py | 2 +- nvflare/fox/sys/controller.py | 2 +- nvflare/fox/sys/executor.py | 2 +- 5 files changed, 11 insertions(+), 5 deletions(-) diff --git a/nvflare/fox/api/app.py b/nvflare/fox/api/app.py index 1c9f626ebc..a90fd7411a 100644 --- a/nvflare/fox/api/app.py +++ b/nvflare/fox/api/app.py @@ -215,7 +215,10 @@ def add_collab_object(self, name: str, obj): # name must be acceptable str pattern = r"^[A-Za-z][A-Za-z0-9_]*$" if not re.match(pattern, name): - raise ValueError(f"invalid name {name} for collab object - must be simple name starting with a letter") + raise ValueError( + f"invalid name {name} for collab object - must start with a letter, " + "followed by one or more alphanumeric and/or underscore chars" + ) if name in self._collab_objs: raise ValueError(f"conflict with existing collab object '{name}' of {type(self._collab_objs[name])}") diff --git a/nvflare/fox/api/group.py b/nvflare/fox/api/group.py index 4ebd9a3894..73690950ff 100644 --- a/nvflare/fox/api/group.py +++ b/nvflare/fox/api/group.py @@ -161,6 +161,9 @@ def method(*args, **kwargs): gcc.set_send_complete_cb(self._request_sent, waiter=waiter) while True: + if self._abort_signal.triggered: + raise RunAborted("run is aborted") + # No need to use lock here even if in_sending could change after we read it # from waiter.in_sending_count and use it to compare with max_parallel. # This is because waiter.in_sending_count can only decrease (by other threads) @@ -171,7 +174,7 @@ def method(*args, **kwargs): func_proxy.backend.call_target_in_group(gcc, func_name, *call_args, **call_kwargs) break else: - # self._logger.debug(f"requests being sent: {in_sending} - wait for runway") + # wait a short time for sending count to decrease and to check abort signal time.sleep(0.1) if not self._call_opt.expect_result: diff --git a/nvflare/fox/sim/simulator.py b/nvflare/fox/sim/simulator.py index 5e08bfab7f..64b0443fab 100644 --- a/nvflare/fox/sim/simulator.py +++ b/nvflare/fox/sim/simulator.py @@ -121,7 +121,7 @@ def __init__( server_app.backend_type = BackendType.SIMULATION self.server_app = server_app self.client_app = client_app - self.thread_executor = ThreadPoolExecutor(max_workers=max_workers) + self.thread_executor = ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="fox_call") if isinstance(num_clients, int): if num_clients <= 0: diff --git a/nvflare/fox/sys/controller.py b/nvflare/fox/sys/controller.py index 4ac1600283..ab49f63eb9 100644 --- a/nvflare/fox/sys/controller.py +++ b/nvflare/fox/sys/controller.py @@ -76,7 +76,7 @@ def __init__( self.server_app = None self.client_info = {} # client name => _ClientInfo self.cell = None - self.thread_executor = ThreadPoolExecutor(max_workers=max_call_threads) + self.thread_executor = ThreadPoolExecutor(max_workers=max_call_threads, thread_name_prefix="fox_call") def start_controller(self, fl_ctx: FLContext): engine = fl_ctx.get_engine() diff --git a/nvflare/fox/sys/executor.py b/nvflare/fox/sys/executor.py index 7aa4c5c7e4..0b9336c064 100644 --- a/nvflare/fox/sys/executor.py +++ b/nvflare/fox/sys/executor.py @@ -64,7 +64,7 @@ def __init__( self.register_event_handler(EventType.END_RUN, self._handle_end_run) self.client_app = None self.client_ctx = None - self.thread_executor = ThreadPoolExecutor(max_workers=max_call_threads) + self.thread_executor = ThreadPoolExecutor(max_workers=max_call_threads, thread_name_prefix="fox_call") def _handle_start_run(self, event_type: str, fl_ctx: FLContext): fl_ctx.set_prop(FLContextKey.FOX_MODE, True, private=True, sticky=True) From 4041c940a9fd0d09b56d082e849bfc5ce5c357db Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Tue, 16 Dec 2025 17:29:05 -0500 Subject: [PATCH 096/102] use condition to manage sending count --- nvflare/fox/api/gcc.py | 28 ++++++++++++++++++++++++++-- nvflare/fox/api/group.py | 23 ++++------------------- 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/nvflare/fox/api/gcc.py b/nvflare/fox/api/gcc.py index f581376432..63b0eea999 100644 --- a/nvflare/fox/api/gcc.py +++ b/nvflare/fox/api/gcc.py @@ -15,6 +15,7 @@ import queue import threading +from nvflare.apis.fl_exception import RunAborted from nvflare.fuel.utils.log_utils import get_obj_logger from .call_opt import CallOption @@ -87,14 +88,37 @@ def __init__(self, sites: list[str]): self.results = ResultQueue(len(sites)) self.in_sending_count = 0 self.lock = threading.Lock() + self.sending_decreased = threading.Condition() def inc_sending(self): - with self.lock: + with self.sending_decreased: self.in_sending_count += 1 def dec_sending(self): - with self.lock: + with self.sending_decreased: self.in_sending_count -= 1 + self.sending_decreased.notify() + + def wait_for_sending_available(self, limit, timeout, abort_signal): + """Wait for sending count to be decreased if current count exceeds limit + + Args: + limit: to limit to check + timeout: time to wait + abort_signal: abort signal + + Returns: the sending count after waiting + + """ + while True: + if abort_signal and abort_signal.triggered: + raise RunAborted("run is aborted") + + with self.sending_decreased: + if self.in_sending_count < limit: + return + else: + self.sending_decreased.wait(timeout) @staticmethod def _get_site_name(target_name: str): diff --git a/nvflare/fox/api/group.py b/nvflare/fox/api/group.py index 73690950ff..c135e3fcb1 100644 --- a/nvflare/fox/api/group.py +++ b/nvflare/fox/api/group.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. import copy -import time from typing import List from nvflare.apis.fl_exception import RunAborted @@ -159,23 +158,9 @@ def method(*args, **kwargs): ) gcc.set_send_complete_cb(self._request_sent, waiter=waiter) - - while True: - if self._abort_signal.triggered: - raise RunAborted("run is aborted") - - # No need to use lock here even if in_sending could change after we read it - # from waiter.in_sending_count and use it to compare with max_parallel. - # This is because waiter.in_sending_count can only decrease (by other threads) - # after we read it here. If this happens, we will catch it in next iteration. - in_sending = waiter.in_sending_count - if in_sending < max_parallel: - waiter.inc_sending() - func_proxy.backend.call_target_in_group(gcc, func_name, *call_args, **call_kwargs) - break - else: - # wait a short time for sending count to decrease and to check abort signal - time.sleep(0.1) + waiter.wait_for_sending_available(max_parallel, 1.0, self._abort_signal) + waiter.inc_sending() + func_proxy.backend.call_target_in_group(gcc, func_name, *call_args, **call_kwargs) if not self._call_opt.expect_result: # do not wait for responses @@ -191,7 +176,7 @@ def method(*args, **kwargs): raise RunAborted("run is aborted") # wait for a short time, so we can check other conditions - done = waiter.wait(0.1) + done = waiter.wait(1.0) if done: break From ed1880e69c3148d442b976eb2a420e80e615a40b Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Wed, 17 Dec 2025 11:08:39 -0500 Subject: [PATCH 097/102] initialize condition with lock --- nvflare/fox/api/gcc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nvflare/fox/api/gcc.py b/nvflare/fox/api/gcc.py index 63b0eea999..04ba8b5bc6 100644 --- a/nvflare/fox/api/gcc.py +++ b/nvflare/fox/api/gcc.py @@ -88,7 +88,7 @@ def __init__(self, sites: list[str]): self.results = ResultQueue(len(sites)) self.in_sending_count = 0 self.lock = threading.Lock() - self.sending_decreased = threading.Condition() + self.sending_decreased = threading.Condition(self.lock) def inc_sending(self): with self.sending_decreased: From ec6db6e652705828e09e7ecd2dfa87d8fd0c23bc Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Wed, 17 Dec 2025 14:27:09 -0500 Subject: [PATCH 098/102] address pr comments --- nvflare/fox/api/app.py | 76 ++++++++++++++----- nvflare/fox/api/ctx.py | 8 +- nvflare/fox/api/facade.py | 2 +- nvflare/fox/api/group.py | 4 +- nvflare/fox/api/run_server.py | 5 +- nvflare/fox/examples/np/algos/filters.py | 3 +- .../fox/examples/np/algos/strategies/avg_h.py | 4 +- .../examples/np/algos/strategies/cyclic.py | 1 + nvflare/fox/examples/np/algos/swarm.py | 5 +- nvflare/fox/examples/pt/filters.py | 2 +- nvflare/fox/examples/pt/pt_avg_filter.py | 4 +- nvflare/fox/sim/simulator.py | 10 +-- nvflare/fox/sys/controller.py | 2 +- nvflare/fox/sys/executor.py | 2 +- 14 files changed, 84 insertions(+), 44 deletions(-) diff --git a/nvflare/fox/api/app.py b/nvflare/fox/api/app.py index a90fd7411a..1acf3eb5c3 100644 --- a/nvflare/fox/api/app.py +++ b/nvflare/fox/api/app.py @@ -20,7 +20,7 @@ from nvflare.fuel.utils.log_utils import get_obj_logger from nvflare.fuel.utils.tree_utils import Forest, Node, build_forest -from .constants import CollabMethodArgName, ContextKey, FilterDirection +from .constants import BackendType, CollabMethodArgName, ContextKey, FilterDirection from .ctx import Context, set_call_context from .dec import ( collab, @@ -56,11 +56,11 @@ def __init__(self, obj, name: str): self._outgoing_call_filter_chains = [] self._incoming_result_filter_chains = [] self._outgoing_result_filter_chains = [] - self._collab_interface = {"": get_object_collab_interface(self)} self._workspace = None self._resource_dirs = {} self._managed_objects = {} # id => obj self.logger = get_obj_logger(self) + self._collab_interface = {"": get_object_collab_interface(self)} self.add_collab_object(name, obj) def set_resource_dirs(self, resource_dirs: dict[str, str]): @@ -77,18 +77,46 @@ def get_resource_dirs(self): def _add_managed_object(self, obj): self._managed_objects[id(obj)] = obj - def get_backend(self): - return self._me.backend + def set_fqn(self, fqn): + self._fqn = fqn + + @property + def fqn(self): + return self._fqn + + @property + def backend(self): + if not self._me: + return None + else: + return self._me.backend + + @property + def backend_type(self): + return self._backend_type - def get_workspace(self): + def set_backend_type(self, t: str): + valid_types = [BackendType.SIMULATION, BackendType.FLARE] + if t not in valid_types: + raise ValueError(f"bad backend type: {t}: must be one of {valid_types}") + self._backend_type = t + + @property + def workspace(self): return self._workspace - def get_server_proxy(self): + @property + def server_proxy(self): return self._server_proxy - def get_client_proxies(self): + @property + def client_proxies(self): return copy.copy(self._client_proxies) + @property + def client_hierarchy(self): + return self._client_hierarchy + def _add_filters(self, pattern: str, filters, to_list: list, filter_type, incoming): if not filters: return @@ -257,7 +285,8 @@ def setup(self, workspace: Workspace, server: Proxy, clients: List[Proxy], abort forest = build_forest(objs=clients, get_fqn_f=lambda c: c.fqn, get_name_f=lambda c: c.name) self._client_hierarchy = forest - def get_my_site(self) -> Proxy: + @property + def my_site(self) -> Proxy: return self._me def find_method(self, target_obj, method_name): @@ -358,7 +387,7 @@ def get_children(self): return [] def has_children(self): - return True + return False def get_leaf_clients(self): assert isinstance(self._client_hierarchy, Forest) @@ -377,7 +406,10 @@ def __init__(self, obj, name: str = "server"): raise ValueError("server object must have at least one algo") def get_children(self): - assert isinstance(self._client_hierarchy, Forest) + if not isinstance(self._client_hierarchy, Forest): + raise RuntimeError( + f"client_hierarchy in app {self.name} must be Forest but got {type(self._client_hierarchy)}" + ) root_nodes = [self._client_hierarchy.nodes[n] for n in self._client_hierarchy.roots] return [node.obj for node in root_nodes] @@ -392,20 +424,24 @@ def __init__(self, obj, name: str = "client"): raise ValueError("client object must be specified") super().__init__(obj, name) + def _get_my_node(self): + if not isinstance(self._client_hierarchy, Forest): + raise RuntimeError( + f"client_hierarchy in app {self.name} must be Forest but got {type(self._client_hierarchy)}" + ) + + node = self._client_hierarchy.nodes.get(self.name) + if not isinstance(node, Node): + raise RuntimeError(f"node for site {self.name} must be a Node but got {type(node)}") + return node + def get_children(self): - assert isinstance(self._client_hierarchy, Forest) - my_node = self._client_hierarchy.nodes[self.name] - assert isinstance(my_node, Node) + my_node = self._get_my_node() if my_node.children: return [node.obj for node in my_node.children] else: return [] def has_children(self): - assert isinstance(self._client_hierarchy, Forest) - my_node = self._client_hierarchy.nodes[self.name] - assert isinstance(my_node, Node) - if my_node.children: - return True - else: - return False + my_node = self._get_my_node() + return True if my_node.children else False diff --git a/nvflare/fox/api/ctx.py b/nvflare/fox/api/ctx.py index cf7b2b7f96..26f9b7ecff 100644 --- a/nvflare/fox/api/ctx.py +++ b/nvflare/fox/api/ctx.py @@ -37,7 +37,7 @@ def __init__(self, app, caller: str, callee: str, abort_signal: Signal, target_g @property def backend(self): - return self.app.get_backend() + return self.app.backend @property def backend_type(self): @@ -45,11 +45,11 @@ def backend_type(self): @property def clients(self): - return self.app.get_client_proxies() + return self.app.client_proxies @property def server(self): - return self.app.get_server_proxy() + return self.app.server_proxy @property def client_hierarchy(self): @@ -57,7 +57,7 @@ def client_hierarchy(self): @property def workspace(self): - return self.app.get_workspace() + return self.app.workspace @property def target_group_size(self): diff --git a/nvflare/fox/api/facade.py b/nvflare/fox/api/facade.py index d293bc766e..27b02abee1 100644 --- a/nvflare/fox/api/facade.py +++ b/nvflare/fox/api/facade.py @@ -125,7 +125,7 @@ def other_clients(cls): # Note that ctx.clients returns a copy of client proxies, not the original client proxy list! # So it is safe to manipulate the candidates here. candidates = ctx.clients - me = ctx.app.get_my_site() + me = ctx.app.my_site if me in candidates: candidates.remove(me) return ProxyList(candidates) diff --git a/nvflare/fox/api/group.py b/nvflare/fox/api/group.py index c135e3fcb1..239fbd7a92 100644 --- a/nvflare/fox/api/group.py +++ b/nvflare/fox/api/group.py @@ -120,7 +120,7 @@ def method(*args, **kwargs): the_backend = p.backend with func_proxy.app.new_context(func_proxy.caller_name, func_proxy.name, target_group=self) as ctx: - self._logger.debug( + self._logger.info( f"[{ctx}] calling {func_name} {self._call_opt} of group {[p.name for p in self._proxies]}" ) @@ -189,7 +189,7 @@ def method(*args, **kwargs): return method def _request_sent(self, waiter: ResultWaiter): - self._logger.info("received _request_sent ...") + self._logger.debug("received _request_sent ...") waiter.dec_sending() diff --git a/nvflare/fox/api/run_server.py b/nvflare/fox/api/run_server.py index 1f3eed6641..194372a087 100644 --- a/nvflare/fox/api/run_server.py +++ b/nvflare/fox/api/run_server.py @@ -40,8 +40,9 @@ def run_server(server_app: ServerApp, logger): server_ctx.set_prop(ContextKey.RESULT, result) except Exception as ex: secure_log_traceback(logger) - backend = server_app.get_backend() - backend.handle_exception(ex) + backend = server_app.backend + if backend: + backend.handle_exception(ex) break logger.info("finalizing server app") diff --git a/nvflare/fox/examples/np/algos/filters.py b/nvflare/fox/examples/np/algos/filters.py index 305fda561b..8faffec79a 100644 --- a/nvflare/fox/examples/np/algos/filters.py +++ b/nvflare/fox/examples/np/algos/filters.py @@ -31,7 +31,7 @@ def add_noise(self, func_kwargs: dict): weights = func_kwargs.get(weights_key) if weights is None: # nothing to filter - print(f"nothing to filter in {func_kwargs}") + self.logger.info(f"nothing to filter in {func_kwargs}") return func_kwargs # add some noise to weights @@ -46,7 +46,6 @@ def add_noise(self, func_kwargs: dict): class Print: def __init__(self): - super().__init__() self.logger = get_obj_logger(self) @fox.call_filter diff --git a/nvflare/fox/examples/np/algos/strategies/avg_h.py b/nvflare/fox/examples/np/algos/strategies/avg_h.py index 6226acbbcb..923270d79b 100644 --- a/nvflare/fox/examples/np/algos/strategies/avg_h.py +++ b/nvflare/fox/examples/np/algos/strategies/avg_h.py @@ -46,7 +46,7 @@ def _do_eval(self, model): self.logger.info(f"[{fox.call_info}]: got eval result from client {n}: {v}") total += v num_results = len(results) - return total / len(results) if num_results > 0 else 0.0 + return total / num_results if num_results > 0 else 0.0 def _do_one_round(self, r, current_model): total = 0 @@ -56,4 +56,4 @@ def _do_one_round(self, r, current_model): total += v num_results = len(results) - return total / len(results) if num_results > 0 else None + return total / num_results if num_results > 0 else None diff --git a/nvflare/fox/examples/np/algos/strategies/cyclic.py b/nvflare/fox/examples/np/algos/strategies/cyclic.py index 86eb0c37f6..440357fbdf 100644 --- a/nvflare/fox/examples/np/algos/strategies/cyclic.py +++ b/nvflare/fox/examples/np/algos/strategies/cyclic.py @@ -64,6 +64,7 @@ def _do_one_round(self, current_round, current_model): for c in clients: current_model = c.train(current_round, current_model) if current_model is None: + self.logger.error(f"training failed on client {c.name} at round {current_round}") return None self.logger.info(f"[{fox.call_info}] result from {c.name}: {current_model}") return current_model diff --git a/nvflare/fox/examples/np/algos/swarm.py b/nvflare/fox/examples/np/algos/swarm.py index 115d2264c6..ec4b0cbfb2 100644 --- a/nvflare/fox/examples/np/algos/swarm.py +++ b/nvflare/fox/examples/np/algos/swarm.py @@ -60,6 +60,7 @@ def __init__(self, delta: float): @fox.init def init(self): + # This example shows that there could be multiple listeners for the same event fox.register_event_handler("final_model", self._accept_final_model) fox.register_event_handler("final_model", self._save_final_model) @@ -69,7 +70,8 @@ def train(self, weights, current_round): return weights + self.delta def sag(self, model, current_round): - results = fox.clients.train(model, current_round) + # results = fox.clients.train(model, current_round) + results = fox.other_clients.train(model, current_round) total = 0 for n, v in results: total += v @@ -104,6 +106,7 @@ def swarm_learn(self, num_rounds, model, current_round): @fox.collab def start(self, num_rounds, initial_model): + self.logger.info(f"[{fox.call_info}]: starting swarm learning") self.swarm_learn(num_rounds, initial_model, 0) def _accept_final_model(self, event_type: str, model): diff --git a/nvflare/fox/examples/pt/filters.py b/nvflare/fox/examples/pt/filters.py index 58cf7aebc7..a54fd5affe 100644 --- a/nvflare/fox/examples/pt/filters.py +++ b/nvflare/fox/examples/pt/filters.py @@ -35,7 +35,7 @@ def filter_call(self, func_kwargs: dict, context: Context): self.logger.info(f"target group size={num_receivers}") downloader = Downloader( - num_receivers=context.target_group_size, + num_receivers=num_receivers, timeout=5.0, ) model = downloader.add_tensors(arg_value, 0) diff --git a/nvflare/fox/examples/pt/pt_avg_filter.py b/nvflare/fox/examples/pt/pt_avg_filter.py index 8a4bca495f..dfd48b8e9a 100644 --- a/nvflare/fox/examples/pt/pt_avg_filter.py +++ b/nvflare/fox/examples/pt/pt_avg_filter.py @@ -68,9 +68,9 @@ def __init__(self, delta: float): def train(self, current_round, weights): if fox.is_aborted: self.logger.debug("training aborted") - return 0 - self.logger.debug(f"[{fox.call_info}] training round {current_round}: {weights=}") + return None + self.logger.debug(f"[{fox.call_info}] training round {current_round}: {weights=}") result = {} for k, v in weights.items(): result[k] = v + self.delta diff --git a/nvflare/fox/sim/simulator.py b/nvflare/fox/sim/simulator.py index 64b0443fab..250805b714 100644 --- a/nvflare/fox/sim/simulator.py +++ b/nvflare/fox/sim/simulator.py @@ -61,7 +61,7 @@ def _make_app(self, site_name, fqn): """Make a new client app instance for the specified site Args: - site_name: nme of the site + site_name: name of the site fqn: fully qualified name of the site Returns: a new instance of the app @@ -86,8 +86,8 @@ def _make_app(self, site_name, fqn): raise ex app.name = site_name - app.fqn = fqn - app.backend_type = BackendType.SIMULATION + app.set_fqn(fqn) + app.set_backend_type(BackendType.SIMULATION) return app def _prepare_proxies(self, for_app: App, server_app: App, client_apps: dict, backends: dict): @@ -117,8 +117,8 @@ def __init__( self.logger = get_obj_logger(self) self.abort_signal = Signal() server_app.name = "server" - server_app.fqn = server_app.name - server_app.backend_type = BackendType.SIMULATION + server_app.set_fqn(server_app.name) + server_app.set_backend_type(BackendType.SIMULATION) self.server_app = server_app self.client_app = client_app self.thread_executor = ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="fox_call") diff --git a/nvflare/fox/sys/controller.py b/nvflare/fox/sys/controller.py index ab49f63eb9..160eca8bd5 100644 --- a/nvflare/fox/sys/controller.py +++ b/nvflare/fox/sys/controller.py @@ -89,7 +89,7 @@ def start_controller(self, fl_ctx: FLContext): app = ServerApp(server_obj) app.name = "server" - app.backend_type = BackendType.FLARE + app.set_backend_type(BackendType.FLARE) err = self.process_config(app, fl_ctx) if err: diff --git a/nvflare/fox/sys/executor.py b/nvflare/fox/sys/executor.py index 0b9336c064..7ade83f405 100644 --- a/nvflare/fox/sys/executor.py +++ b/nvflare/fox/sys/executor.py @@ -86,7 +86,7 @@ def _handle_start_run(self, event_type: str, fl_ctx: FLContext): raise RuntimeError(f"result returned by {MAKE_CLIENT_APP_METHOD} must be ClientApp but got {type(app)}") app.name = client_name - app.backend_type = BackendType.FLARE + app.set_backend_type(BackendType.FLARE) self.client_app = app err = self.process_config(self.client_app, fl_ctx) From f8a8aad78beb68cabb109db71673de9cc3cf1280 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Wed, 17 Dec 2025 14:47:02 -0500 Subject: [PATCH 099/102] address pr comments --- nvflare/fox/api/gcc.py | 3 +++ nvflare/fox/api/proxy.py | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/nvflare/fox/api/gcc.py b/nvflare/fox/api/gcc.py index 04ba8b5bc6..6b2d6fdb22 100644 --- a/nvflare/fox/api/gcc.py +++ b/nvflare/fox/api/gcc.py @@ -45,6 +45,9 @@ def append(self, item, is_whole=True): Returns: whether the queue has received all whole items. + Notes: though this method is not thread safe, it is only called from ResultWaiter, which ensures + thread safety! + """ if self.num_whole_items_received == self.limit: raise RuntimeError(f"queue is full: {self.limit} items are already appended") diff --git a/nvflare/fox/api/proxy.py b/nvflare/fox/api/proxy.py index 6a72fa33e1..6e57cb5024 100644 --- a/nvflare/fox/api/proxy.py +++ b/nvflare/fox/api/proxy.py @@ -226,7 +226,12 @@ def call_func(self, call_opt: CallOption, func_name, args, kwargs): result = self.app.apply_incoming_result_filters(p.target_name, func_name, result, ctx) return result except Exception as ex: - self.backend.handle_exception(ex) + if self.backend: + try: + self.backend.handle_exception(ex) + except Exception as ex2: + # ignore exception from backend handling + self.logger.error(f"ignored backend's exception {type(ex2)}") # Must return the exception as the result of the func call. # Do NOT raise it! From 0318e77b0ee5b52fc1f446a2a6e35ae20ba36517 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Wed, 17 Dec 2025 15:16:16 -0500 Subject: [PATCH 100/102] address meaningful pr comments --- nvflare/fox/api/app.py | 3 ++- nvflare/fox/api/group.py | 1 + nvflare/fox/examples/np/fed_avg_intime.py | 4 ++-- nvflare/fox/examples/pt/pt_avg_filter.py | 4 ++-- nvflare/fox/sys/downloader.py | 5 ++--- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/nvflare/fox/api/app.py b/nvflare/fox/api/app.py index 1acf3eb5c3..282f816a84 100644 --- a/nvflare/fox/api/app.py +++ b/nvflare/fox/api/app.py @@ -390,7 +390,8 @@ def has_children(self): return False def get_leaf_clients(self): - assert isinstance(self._client_hierarchy, Forest) + if not isinstance(self._client_hierarchy, Forest): + raise RuntimeError(f"client_hierarchy must be Forest but got {type(self._client_hierarchy)}") leaf_nodes = [self._client_hierarchy.nodes[n] for n in self._client_hierarchy.leaves] return [node.obj for node in leaf_nodes] diff --git a/nvflare/fox/api/group.py b/nvflare/fox/api/group.py index 239fbd7a92..3daf1d5df1 100644 --- a/nvflare/fox/api/group.py +++ b/nvflare/fox/api/group.py @@ -185,6 +185,7 @@ def method(*args, **kwargs): self._logger.error(f"exception {type(ex)} occurred: {ex}") if the_backend: the_backend.handle_exception(ex) + raise ex return method diff --git a/nvflare/fox/examples/np/fed_avg_intime.py b/nvflare/fox/examples/np/fed_avg_intime.py index 8c1f0fe774..91187da81e 100644 --- a/nvflare/fox/examples/np/fed_avg_intime.py +++ b/nvflare/fox/examples/np/fed_avg_intime.py @@ -36,11 +36,11 @@ def main(): simulator.add_server_outgoing_call_filters("*.train", [AddNoiseToModel()]) simulator.add_server_incoming_result_filters("*.train", [Print()]) - simulator.set_server_prop("default_timeout", 5.0) + simulator.set_server_prop("default_timeout", 8.0) simulator.add_client_incoming_call_filters("*.train", [Print()]) simulator.add_client_outgoing_result_filters("*.train", [Print()]) - simulator.set_client_prop("default_timeout", 8.0) + simulator.set_client_prop("default_timeout", 5.0) result = simulator.run() print(f"final model: {result}") diff --git a/nvflare/fox/examples/pt/pt_avg_filter.py b/nvflare/fox/examples/pt/pt_avg_filter.py index dfd48b8e9a..e631ecb14d 100644 --- a/nvflare/fox/examples/pt/pt_avg_filter.py +++ b/nvflare/fox/examples/pt/pt_avg_filter.py @@ -29,7 +29,7 @@ def __init__(self, initial_model, num_rounds=10, timeout=2.0): self.num_rounds = num_rounds self.initial_model = initial_model self.timeout = timeout - self.name = "PTFedAvgStream" + self.name = "PTFedAvg" self.logger = get_obj_logger(self) self._init_model = parse_state_dict(initial_model) @@ -48,7 +48,7 @@ def execute(self): def _do_one_round(self, r, current_model): aggr_result = {} - results = fox.clients.train(r, current_model) + results = fox.clients(timeout=self.timeout).train(r, current_model) for n, v in results: add_pt(v, aggr_result) diff --git a/nvflare/fox/sys/downloader.py b/nvflare/fox/sys/downloader.py index 97d922729d..09b9fe3501 100644 --- a/nvflare/fox/sys/downloader.py +++ b/nvflare/fox/sys/downloader.py @@ -16,15 +16,14 @@ from nvflare.app_common.np.np_downloader import add_arrays from nvflare.app_common.np.np_downloader import download_arrays as pull_arrays +from nvflare.app_opt.pt.tensor_downloader import add_tensors +from nvflare.app_opt.pt.tensor_downloader import download_tensors as pull_tensors from nvflare.fox import fox from nvflare.fox.sys.backend import FlareBackend from nvflare.fuel.f3.streaming.file_downloader import add_file from nvflare.fuel.f3.streaming.file_downloader import download_file as pull_file from nvflare.fuel.f3.streaming.obj_downloader import ObjectDownloader -from ...app_opt.pt.tensor_downloader import add_tensors -from ...app_opt.pt.tensor_downloader import download_tensors as pull_tensors - class DownloadRefKey: SOURCE = "source" From 5ae50ae85c50d91c95bc07aab6a942c70c614e16 Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Thu, 18 Dec 2025 11:47:24 -0500 Subject: [PATCH 101/102] improve gcc --- nvflare/fox/api/gcc.py | 84 +++++++++++++++++++++++++--------------- nvflare/fox/api/group.py | 26 +++++-------- nvflare/fox/api/utils.py | 2 +- 3 files changed, 63 insertions(+), 49 deletions(-) diff --git a/nvflare/fox/api/gcc.py b/nvflare/fox/api/gcc.py index 6b2d6fdb22..921395c377 100644 --- a/nvflare/fox/api/gcc.py +++ b/nvflare/fox/api/gcc.py @@ -23,6 +23,8 @@ from .ctx import Context, set_call_context from .utils import check_context_support +_SHORT_WAIT = 1.0 + class ResultQueue: def __init__(self, limit: int): @@ -35,6 +37,7 @@ def __init__(self, limit: int): # num_whole_items_received is the number of WHOLE items received. # the queue could contain partial results. self.num_whole_items_received = 0 + self.update_lock = threading.Lock() def append(self, item, is_whole=True): """Append an item to the result queue. @@ -49,14 +52,15 @@ def append(self, item, is_whole=True): thread safety! """ - if self.num_whole_items_received == self.limit: - raise RuntimeError(f"queue is full: {self.limit} items are already appended") - self.q.put_nowait(item) + with self.update_lock: + if self.num_whole_items_received == self.limit: + raise RuntimeError(f"queue is full: {self.limit} items are already appended") + self.q.put_nowait(item) - if is_whole: - # increment num_whole_items_received only if the item is whole! - self.num_whole_items_received += 1 - return self.num_whole_items_received == self.limit + if is_whole: + # increment num_whole_items_received only if the item is whole! + self.num_whole_items_received += 1 + return self.num_whole_items_received == self.limit def __iter__(self): return self @@ -89,39 +93,57 @@ def __init__(self, sites: list[str]): super().__init__() self.sites = sites self.results = ResultQueue(len(sites)) - self.in_sending_count = 0 - self.lock = threading.Lock() - self.sending_decreased = threading.Condition(self.lock) + self.standing_call_count = 0 + self.call_count_decreased = threading.Condition(threading.Lock()) + + def inc_call_count(self): + """Increment standing call count by 1. - def inc_sending(self): - with self.sending_decreased: - self.in_sending_count += 1 + Returns: None - def dec_sending(self): - with self.sending_decreased: - self.in_sending_count -= 1 - self.sending_decreased.notify() + """ + with self.call_count_decreased: + self.standing_call_count += 1 - def wait_for_sending_available(self, limit, timeout, abort_signal): - """Wait for sending count to be decreased if current count exceeds limit + def dec_call_count(self): + """Decrease standing call count by 1, and notify other threads waiting for call count decreased. + + Returns: None + + """ + with self.call_count_decreased: + self.standing_call_count -= 1 + self.call_count_decreased.notify() + + def wait_for_call_permission(self, limit, abort_signal): + """Wait for the permission to make next call. + The permission is granted when parallel call count is lower than the specified limit. Args: limit: to limit to check - timeout: time to wait abort_signal: abort signal - Returns: the sending count after waiting + Returns: None """ while True: - if abort_signal and abort_signal.triggered: - raise RunAborted("run is aborted") + with self.call_count_decreased: + if abort_signal and abort_signal.triggered: + raise RunAborted("run is aborted while waiting for sending availability") - with self.sending_decreased: - if self.in_sending_count < limit: + if self.standing_call_count < limit: return else: - self.sending_decreased.wait(timeout) + self.call_count_decreased.wait(_SHORT_WAIT) + + def wait_for_responses(self, abort_signal): + while True: + if abort_signal.triggered: + raise RunAborted("run is aborted while waiting for remote responses") + + done = self.wait(_SHORT_WAIT) + if done: + break @staticmethod def _get_site_name(target_name: str): @@ -131,15 +153,13 @@ def _get_site_name(target_name: str): def set_result(self, target_name: str, result): site_name = self._get_site_name(target_name) - with self.lock: - all_received = self.results.append((site_name, result)) - if all_received: - self.set() + all_received = self.results.append((site_name, result)) + if all_received: + self.set() def add_partial_result(self, target_name: str, partial_result): site_name = self._get_site_name(target_name) - with self.lock: - self.results.append((site_name, partial_result), is_whole=False) + self.results.append((site_name, partial_result), is_whole=False) class GroupCallContext: diff --git a/nvflare/fox/api/group.py b/nvflare/fox/api/group.py index 3daf1d5df1..a3a46f1c0d 100644 --- a/nvflare/fox/api/group.py +++ b/nvflare/fox/api/group.py @@ -14,7 +14,6 @@ import copy from typing import List -from nvflare.apis.fl_exception import RunAborted from nvflare.apis.signal import Signal from nvflare.fuel.utils.log_utils import get_obj_logger @@ -157,9 +156,12 @@ def method(*args, **kwargs): waiter=waiter, ) - gcc.set_send_complete_cb(self._request_sent, waiter=waiter) - waiter.wait_for_sending_available(max_parallel, 1.0, self._abort_signal) - waiter.inc_sending() + # try to get permission to make next call + gcc.set_send_complete_cb(self._request_sent, gcc=gcc, proxy=func_proxy) + waiter.wait_for_call_permission(max_parallel, self._abort_signal) + + # make next call + waiter.inc_call_count() func_proxy.backend.call_target_in_group(gcc, func_name, *call_args, **call_kwargs) if not self._call_opt.expect_result: @@ -171,15 +173,7 @@ def method(*args, **kwargs): return waiter.results # wait for responses - while True: - if self._abort_signal.triggered: - raise RunAborted("run is aborted") - - # wait for a short time, so we can check other conditions - done = waiter.wait(1.0) - if done: - break - + waiter.wait_for_responses(self._abort_signal) return waiter.results except Exception as ex: self._logger.error(f"exception {type(ex)} occurred: {ex}") @@ -189,9 +183,9 @@ def method(*args, **kwargs): return method - def _request_sent(self, waiter: ResultWaiter): - self._logger.debug("received _request_sent ...") - waiter.dec_sending() + def _request_sent(self, gcc: GroupCallContext, proxy: Proxy): + self._logger.debug(f"[{gcc.context}] call has been sent to '{proxy.name}' for func '{gcc.func_name}'") + gcc.waiter.dec_call_count() def group( diff --git a/nvflare/fox/api/utils.py b/nvflare/fox/api/utils.py index 2349711e44..417ebb2550 100644 --- a/nvflare/fox/api/utils.py +++ b/nvflare/fox/api/utils.py @@ -76,4 +76,4 @@ def check_call_args(func_name, func_itf, call_args, call_kwargs: dict): def simple_logging(level=logging.INFO): - logging.basicConfig(level=level, format="%(asctime)s - %(levelname)s - %(message)s") + logging.basicConfig(level=level, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s") From 3b2a2c92876f78956b24ff406568466fd53bc77e Mon Sep 17 00:00:00 2001 From: Yan Cheng Date: Thu, 18 Dec 2025 12:21:56 -0500 Subject: [PATCH 102/102] fix comments --- nvflare/fox/api/gcc.py | 24 ++++++++++++------------ nvflare/fox/examples/np/algos/client.py | 4 ++-- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/nvflare/fox/api/gcc.py b/nvflare/fox/api/gcc.py index 921395c377..32422f0621 100644 --- a/nvflare/fox/api/gcc.py +++ b/nvflare/fox/api/gcc.py @@ -47,20 +47,20 @@ def append(self, item, is_whole=True): is_whole: whether the item is whole. Returns: whether the queue has received all whole items. - - Notes: though this method is not thread safe, it is only called from ResultWaiter, which ensures - thread safety! - """ with self.update_lock: - if self.num_whole_items_received == self.limit: - raise RuntimeError(f"queue is full: {self.limit} items are already appended") - self.q.put_nowait(item) - - if is_whole: - # increment num_whole_items_received only if the item is whole! - self.num_whole_items_received += 1 - return self.num_whole_items_received == self.limit + if self.num_whole_items_received < self.limit: + self.q.put_nowait(item) + if is_whole: + # increment num_whole_items_received only if the item is whole! + # note: num_whole_items_received is not the number of all items received. + # partial items could be added to the queue but do not count as whole items. + self.num_whole_items_received += 1 + return self.num_whole_items_received == self.limit + else: + # do not allow any items (partial or whole) to be added to the queue if the queue + # has already received all expected whole items. + raise RuntimeError(f"queue is full: {self.limit} whole items are already appended") def __iter__(self): return self diff --git a/nvflare/fox/examples/np/algos/client.py b/nvflare/fox/examples/np/algos/client.py index 154ab9a6c3..887da4207c 100644 --- a/nvflare/fox/examples/np/algos/client.py +++ b/nvflare/fox/examples/np/algos/client.py @@ -64,7 +64,7 @@ def __init__(self, delta: float): def train(self, current_round, weights): if fox.is_aborted: self.logger.debug("training aborted") - return 0 + return None self.logger.debug(f"[{fox.call_info}] training round {current_round}") if fox.has_children: @@ -83,7 +83,7 @@ def train(self, current_round, weights): def _local_train(self, current_round, weights): if fox.is_aborted: self.logger.debug("training aborted") - return 0 + return None self.logger.info(f"[{fox.call_info}] local trained round {current_round} {weights} {type(weights)}") return weights + self.delta