From 6453e33aa0f9fd2cdc748e7b84f60dfb00beb5b9 Mon Sep 17 00:00:00 2001 From: YuanTingHsieh Date: Tue, 7 Jul 2026 16:55:47 -0700 Subject: [PATCH 01/20] Harden outcome recording: registration order, required guard, deep-frozen outcomes Four review findings on the F3-1 outcome-recording path: 1. Register the incarnation BEFORE the tx is monitor-visible. new_transaction() inserted into _tx_table first; a monitor tick in the window could terminate the tx, its outcome dropped by the incarnation guard (terminated-but-unknown gap), and the subsequent registration left a dead _tx_incarnations entry that nothing ever pops. Registration now precedes table insertion. 2. Retire a still-live prior transaction on tx_id reuse. A plain _tx_table overwrite orphaned the old tx: its refs stayed servable in _ref_table forever (the monitor only discovers transactions via _tx_table) and its sources were never released. new_transaction() now delete+transaction_done()s the old tx; its outcome is dropped by the incarnation guard -- the retry is authoritative. 3. Make tx required in _record_outcome() so no call site can opt out of the incarnation guard. With that, the _accept_outcomes flag is provably redundant (shutdown clears all incarnations under the same lock), so it is removed -- one mechanism, one story: outcomes record only for live registered incarnations. 4. Deep-freeze TransferOutcome/RefOutcome: frozen=True only blocks attribute rebinding; refs becomes a tuple and receiver_statuses a MappingProxyType over a private copy, so outcome_cb consumers and pollers cannot mutate the recorded per-receiver truth. Also guard downloaded_to_one/downloaded_to_all with _invoke_cb_safely: a raising user callback no longer loses the EOF reply or permanently skips downloaded_to_all. Tests: ordering regression (incarnation-before-visibility), active-reuse retirement, deep-frozen outcome, raising-callback serving path; _add_tx and the shutdown test updated for the required-tx signature. --- nvflare/fuel/f3/streaming/download_service.py | 93 ++++++++++++------- nvflare/fuel/f3/streaming/transfer_outcome.py | 27 ++++-- .../f3/streaming/download_service_test.py | 90 +++++++++++------- .../fuel/f3/streaming/download_test_utils.py | 1 - .../f3/streaming/transfer_outcome_test.py | 69 +++++++++++++- 5 files changed, 201 insertions(+), 79 deletions(-) diff --git a/nvflare/fuel/f3/streaming/download_service.py b/nvflare/fuel/f3/streaming/download_service.py index e7064878df..b5f4d635d4 100644 --- a/nvflare/fuel/f3/streaming/download_service.py +++ b/nvflare/fuel/f3/streaming/download_service.py @@ -215,12 +215,26 @@ def obj_downloaded(self, to_receiver: str, status: str): if all_done: self._downloaded_to_all_called = True + # Guarded like the terminal callbacks in transaction_done: a raising user + # callback on the serving path must not lose the EOF reply for this attempt, + # and a raising downloaded_to_one must not skip downloaded_to_all (the + # _downloaded_to_all_called latch above is already set and is never retried). assert isinstance(self.obj, Downloadable) - self.obj.downloaded_to_one(to_receiver, status) + _invoke_cb_safely( + self.tx.logger, + f"downloaded_to_one of {type(self.obj)} for ref {self.rid}", + self.obj.downloaded_to_one, + to_receiver, + status, + ) if all_done: # this object is done for all receivers - self.obj.downloaded_to_all() + _invoke_cb_safely( + self.tx.logger, + f"downloaded_to_all of {type(self.obj)} for ref {self.rid}", + self.obj.downloaded_to_all, + ) def snapshot_receiver_statuses(self) -> dict: with self._progress_lock: @@ -612,7 +626,6 @@ class DownloadService: # its outcome over the new incarnation. Guarded by _outcome_lock. _tx_incarnations = {} _outcome_lock = threading.Lock() - _accept_outcomes = True TX_OUTCOME_TTL = 1800.0 _logger = None _tx_monitor = None @@ -629,17 +642,6 @@ def _initialize(cls, cell: Cell): cls._tx_monitor = threading.Thread(target=cls._monitor_tx, daemon=True) cls._tx_monitor.start() - # re-enable outcome recording for the new service lifecycle. Written under - # _outcome_lock only to keep this flag's access uniform (it is read under the - # same lock in _record_outcome). This does NOT by itself close the stale-outcome - # race across shutdown/re-init: a callback that blocked on _outcome_lock during - # shutdown can still win the lock after this write and see the re-enabled flag. - # That race is closed by the live-incarnation guard in _record_outcome, not here. - # Nesting is deadlock-safe: shutdown() acquires _outcome_lock and _init_lock - # sequentially (never nested), so there is no reverse _outcome_lock -> _init_lock. - with cls._outcome_lock: - cls._accept_outcomes = True - initialized = cls._initialized_cells.get(cell) if not initialized: # register CBs @@ -674,14 +676,36 @@ def new_transaction( progress_interval=progress_interval, outcome_cb=outcome_cb, ) - with cls._tx_lock: - cls._tx_table[tx.tid] = tx with cls._outcome_lock: # a reused explicit tx_id must not surface the previous incarnation's # outcome: purge any recorded outcome and register this incarnation as - # current so a concurrently-terminating older incarnation cannot record + # current so a concurrently-terminating older incarnation cannot record. + # Registration happens BEFORE the tx becomes monitor-visible in _tx_table: + # a tx the monitor can terminate is always incarnation-registered, so its + # terminal outcome can never be dropped as stale, and no dead incarnation + # entry can be left behind by a terminate-before-register interleaving. cls._tx_outcomes.pop(tx.tid, None) cls._tx_incarnations[tx.tid] = tx + + old_tx = None + with cls._tx_lock: + old_tx = cls._tx_table.get(tx.tid) + if old_tx: + # A retry reusing a tx_id supersedes a still-live prior transaction. + # Retire it now: a plain overwrite would orphan it -- the monitor only + # discovers transactions through _tx_table, so the old refs would stay + # servable in _ref_table forever and the old sources would never be + # released via transaction_done. + cls._delete_tx(old_tx) + cls._tx_table[tx.tid] = tx + + if old_tx: + # terminal callbacks run outside the lock, as in delete_transaction(); the + # retired transaction's outcome is dropped by the incarnation guard because + # the new incarnation already owns the tid -- the retry is authoritative + old_tx.transaction_done( + TransactionDoneStatus.DELETED, on_outcome=functools.partial(cls._record_outcome, tx=old_tx) + ) return tx.tid @classmethod @@ -730,9 +754,10 @@ def shutdown(cls): cls._finished_refs.clear() with cls._outcome_lock: - # stop recording (a concurrent monitor iteration may be mid-termination) - # and drop recorded outcomes; recording re-enables on next _initialize - cls._accept_outcomes = False + # drop recorded outcomes and clear live incarnations. A monitor iteration + # mid-termination that blocked on _outcome_lock finds its incarnation gone + # once it acquires the lock, so its outcome drops in _record_outcome -- + # the incarnation guard alone gates post-shutdown recording. cls._tx_outcomes.clear() cls._tx_incarnations.clear() @@ -758,25 +783,21 @@ def _delete_tx(cls, tx: _Transaction, tombstone_finished_refs: bool = False): cls._finished_refs.pop(r.rid, None) @classmethod - def _record_outcome(cls, outcome: TransferOutcome, tx: Optional[_Transaction] = None): + def _record_outcome(cls, outcome: TransferOutcome, tx: _Transaction): + # tx is required so no call site can opt out of the incarnation guard: + # recording is legal only for the live registered incarnation. with cls._outcome_lock: - current = cls._tx_incarnations.get(outcome.tx_id) - if tx is not None and current is not tx: - # Record only for the live registered incarnation. current is not tx means - # this outcome belongs to a dead generation and must be dropped, covering: + if cls._tx_incarnations.get(outcome.tx_id) is not tx: + # This outcome belongs to a dead generation and must be dropped: # - a newer same-tx_id incarnation (a retry) registered while this - # transaction was terminating (current is the newer tx), and - # - the incarnation was already cleared -- by shutdown() (which also - # clears the outcome table) or by a prior terminal record (current is - # None). This is what closes the stale-outcome race across shutdown/ - # re-init: a callback that blocked on _outcome_lock during shutdown and - # wins the lock afterward finds no live incarnation and is dropped here, - # regardless of the _accept_outcomes flag below. - return - if current is tx: - cls._tx_incarnations.pop(outcome.tx_id, None) - if not cls._accept_outcomes: + # transaction was terminating, or + # - the incarnation was cleared by shutdown() (which also clears the + # outcome table) or consumed by a prior terminal record. + # This single guard closes the stale-outcome race across shutdown and + # re-init: a recorder that blocked on _outcome_lock during shutdown and + # wins the lock afterward finds no live incarnation and drops here. return + cls._tx_incarnations.pop(outcome.tx_id, None) cls._tx_outcomes[outcome.tx_id] = outcome @classmethod diff --git a/nvflare/fuel/f3/streaming/transfer_outcome.py b/nvflare/fuel/f3/streaming/transfer_outcome.py index 45ed3736ba..beb1f1b06c 100644 --- a/nvflare/fuel/f3/streaming/transfer_outcome.py +++ b/nvflare/fuel/f3/streaming/transfer_outcome.py @@ -44,7 +44,8 @@ import time from dataclasses import dataclass -from typing import Dict, List, Optional +from types import MappingProxyType +from typing import Mapping, Optional, Sequence, Tuple from nvflare.fuel.f3.streaming.transfer_progress import TransferProgressState @@ -92,12 +93,19 @@ class RefOutcome: """Per-object terminal outcome. receiver_statuses maps receiver FQCN to a DownloadStatus value (success / failed) - as recorded by the producer side. Treat the dict as read-only: the same instance - is shared with every consumer of the outcome. + as recorded by the producer side. It is deep-frozen at construction (a + MappingProxyType over a private copy): the same instance is recorded in the + service outcome table and handed to outcome_cb consumers, so a callback must + not be able to mutate the recorded per-receiver truth. If outcomes ever cross + a process boundary, the serializer must materialize it (dict(...)). """ ref_id: str - receiver_statuses: Dict[str, str] + receiver_statuses: Mapping[str, str] + + def __post_init__(self): + # frozen=True only blocks attribute rebinding; freeze the container too + object.__setattr__(self, "receiver_statuses", MappingProxyType(dict(self.receiver_statuses))) @dataclass(frozen=True) @@ -116,9 +124,14 @@ class TransferOutcome: reason: str # a TransferOutcomeReason value done_status: str # the raw TransactionDoneStatus value num_receivers: int - refs: List[RefOutcome] + refs: Tuple[RefOutcome, ...] timestamp: float + def __post_init__(self): + # frozen=True only blocks attribute rebinding; freeze the container too so + # outcome_cb consumers cannot mutate the recorded outcome + object.__setattr__(self, "refs", tuple(self.refs)) + @property def completed(self) -> bool: return self.status == TransferProgressState.COMPLETED @@ -127,7 +140,7 @@ def expired(self, now: float, ttl: float) -> bool: return now - self.timestamp > ttl -def _all_receivers_succeeded(num_receivers: int, refs: List[RefOutcome]) -> bool: +def _all_receivers_succeeded(num_receivers: int, refs: Sequence[RefOutcome]) -> bool: if num_receivers <= 0 or not refs: return False for r in refs: @@ -149,7 +162,7 @@ def compute_transfer_outcome( tx_id: str, done_status: str, num_receivers: int, - refs: List[RefOutcome], + refs: Sequence[RefOutcome], timestamp: Optional[float] = None, ) -> TransferOutcome: """Compute the aggregate terminal outcome for a terminated transaction. diff --git a/tests/unit_test/fuel/f3/streaming/download_service_test.py b/tests/unit_test/fuel/f3/streaming/download_service_test.py index 67c14d88ac..b15c642a73 100644 --- a/tests/unit_test/fuel/f3/streaming/download_service_test.py +++ b/tests/unit_test/fuel/f3/streaming/download_service_test.py @@ -476,50 +476,40 @@ def register_request_cb(self, **kwargs): assert list(service._initialized_cells.keys()) == [] - def test_initialize_reenable_holds_outcome_lock(self): - """The _accept_outcomes re-enable in _initialize() must happen under _outcome_lock. - - This guards lock-discipline uniformity only: the flag is read under _outcome_lock in - _record_outcome(), so its write must be too (it previously ran under _init_lock only). - This is NOT what closes the stale-outcome race -- that is the live-incarnation guard - in _record_outcome(), covered by test_stale_outcome_dropped_after_incarnation_cleared. + def test_new_transaction_registers_incarnation_before_monitor_visible(self): + """A tx must be incarnation-registered before it appears in _tx_table. + + The monitor discovers transactions through _tx_table. If a tx were inserted + there first, a monitor tick landing in the window before incarnation + registration could terminate it: its terminal outcome would be dropped by the + live-incarnation guard (a terminated-but-unknown gap), and new_transaction + would then register a dead incarnation that nothing ever pops. """ service = _make_isolated_download_service() service._tx_monitor = object() # avoid starting a real monitor thread - class FakeCell: - def register_request_cb(self, **kwargs): - pass - - cell = FakeCell() - service._accept_outcomes = False # as left by a prior shutdown() + registered_at_insert = [] - init_done = threading.Event() + class MonitorVisibilityDict(dict): + def __setitem__(self, key, value): + registered_at_insert.append(key in service._tx_incarnations) + super().__setitem__(key, value) - def run_init(): - service._initialize(cell) - init_done.set() + service._tx_table = MonitorVisibilityDict() - with service._outcome_lock: - t = threading.Thread(target=run_init) - t.start() - # while _outcome_lock is held, the guarded re-enable must not complete - assert not init_done.wait(0.2) - assert service._accept_outcomes is False + tx_id = service.new_transaction(cell=Mock(), timeout=10.0, num_receivers=1) - t.join(2.0) - assert init_done.is_set() - assert service._accept_outcomes is True + assert registered_at_insert == [True] + assert service._tx_incarnations[tx_id] is service._tx_table[tx_id] def test_stale_outcome_dropped_after_incarnation_cleared(self): """A terminal outcome for a transaction whose incarnation is no longer live must drop. - This is the invariant that actually closes the cross-lifecycle stale-outcome race: + This is the invariant that closes the cross-lifecycle stale-outcome race: _record_outcome() records only for the live registered incarnation (current is tx). - After shutdown() clears _tx_incarnations, a callback that blocked on _outcome_lock - during shutdown can win the lock afterward and observe _accept_outcomes re-enabled by - a subsequent _initialize() -- so the _accept_outcomes flag alone does not stop it. The - live-incarnation guard does: with no live incarnation for the tid, the outcome drops. + A recorder that blocked on _outcome_lock while shutdown() cleared the tables can + win the lock after a subsequent _initialize(); with no live incarnation for the + tid, its pre-shutdown outcome drops instead of repopulating the cleared table. """ service = _make_isolated_download_service() @@ -529,9 +519,8 @@ def test_stale_outcome_dropped_after_incarnation_cleared(self): old_outcome.tx_id = "tx-old" old_outcome.expired.return_value = False - # shutdown() cleared incarnations; a later _initialize() re-enabled recording + # shutdown() cleared incarnations; recording is gated by incarnation identity alone service._tx_incarnations.clear() - service._accept_outcomes = True # the late callback for the old, now-unregistered transaction must be dropped service._record_outcome(old_outcome, tx=old_tx) @@ -547,6 +536,41 @@ def test_stale_outcome_dropped_after_incarnation_cleared(self): service._record_outcome(live_outcome, tx=live_tx) assert service.get_transaction_outcome("tx-live") is live_outcome + def test_raising_download_callbacks_do_not_break_serving_path(self): + """Raising downloaded_to_one/downloaded_to_all must not propagate into serving. + + A raising downloaded_to_one on the chunk-serving path would lose the EOF reply + for that attempt, and -- because the _downloaded_to_all_called latch is set + before the callbacks run and is never retried -- would permanently skip + downloaded_to_all. Both callbacks are guarded like the terminal callbacks in + transaction_done: the exception is logged, serving and the all-receivers-done + notification proceed. + """ + service = _make_isolated_download_service() + service._tx_monitor = object() # avoid starting a real monitor thread + + calls = [] + + class RaisingDownloadable(MockDownloadable): + def downloaded_to_one(self, to_receiver: str, status: str): + calls.append(("one", to_receiver, status)) + raise RuntimeError("user callback failure") + + def downloaded_to_all(self): + calls.append(("all",)) + raise RuntimeError("user callback failure") + + tx_id = service.new_transaction(cell=Mock(), timeout=10.0, num_receivers=1) + obj = RaisingDownloadable([b"chunk"]) + rid = service.add_object(tx_id, obj) + ref = service._ref_table[rid] + + # must not raise; downloaded_to_all still fires after downloaded_to_one raised + ref.obj_downloaded("r1", DownloadStatus.SUCCESS) + + assert calls == [("one", "r1", DownloadStatus.SUCCESS), ("all",)] + assert ref.snapshot_receiver_statuses() == {"r1": DownloadStatus.SUCCESS} + def test_get_transaction_id_from_ref_id(self, cell): """Test retrieving transaction ID from reference ID.""" tx_id = DownloadService.new_transaction(cell=cell, timeout=10.0, num_receivers=2) diff --git a/tests/unit_test/fuel/f3/streaming/download_test_utils.py b/tests/unit_test/fuel/f3/streaming/download_test_utils.py index 1e218027da..51a83f7f0c 100644 --- a/tests/unit_test/fuel/f3/streaming/download_test_utils.py +++ b/tests/unit_test/fuel/f3/streaming/download_test_utils.py @@ -85,7 +85,6 @@ class IsolatedDownloadService(DownloadService): _tx_outcomes = {} _tx_incarnations = {} _outcome_lock = threading.Lock() - _accept_outcomes = True _logger = Mock() _tx_lock = threading.Lock() _initialized_cells = weakref.WeakKeyDictionary() diff --git a/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py b/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py index 111c0eaf6e..acb90d1f4f 100644 --- a/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py +++ b/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py @@ -35,6 +35,30 @@ def _stub_obj(): return MockDownloadable([b"chunk"]) +class TestOutcomeImmutability: + """The recorded outcome is shared with pollers and outcome_cb consumers: it must be deep-frozen.""" + + def test_outcome_is_deep_frozen(self): + source_statuses = {"r1": DownloadStatus.SUCCESS} + ref = RefOutcome(ref_id="R1", receiver_statuses=source_statuses) + outcome = compute_transfer_outcome("T1", TransactionDoneStatus.FINISHED, 1, [ref], 100.0) + + # containers are frozen, not just the dataclass attributes + assert isinstance(outcome.refs, tuple) + with pytest.raises(TypeError): + outcome.refs[0].receiver_statuses["r2"] = DownloadStatus.SUCCESS + with pytest.raises(dataclasses.FrozenInstanceError): + outcome.refs = () + with pytest.raises(dataclasses.FrozenInstanceError): + outcome.refs[0].receiver_statuses = {} + + # the frozen view is a private copy: mutating the source dict after + # construction cannot rewrite the recorded per-receiver truth + source_statuses["r1"] = DownloadStatus.FAILED + assert outcome.refs[0].receiver_statuses == {"r1": DownloadStatus.SUCCESS} + assert outcome.completed + + class TestComputeTransferOutcome: """Aggregation rules: COMPLETED only when every expected receiver succeeded; receiver truth wins.""" @@ -332,6 +356,43 @@ def test_stale_incarnation_cannot_record_over_live_retry(self): finally: service.shutdown() + def test_reusing_active_tx_id_retires_prior_transaction(self): + # reusing a tx_id while the prior transaction is still LIVE must retire it, + # not orphan it: a plain _tx_table overwrite would leave the old refs servable + # in _ref_table forever (the monitor only sees _tx_table) and never release + # the old sources via transaction_done + from unittest.mock import Mock + + service = make_isolated_download_service() + service._tx_monitor = Mock() # suppress real monitor thread start + cell = Mock() + done = [] + try: + first = service.new_transaction( + cell=cell, + timeout=10.0, + num_receivers=1, + tx_id="TX-DUP", + transaction_done_cb=lambda tid, status, base_objs, **kw: done.append((tid, status)), + ) + obj = _stub_obj() + rid = service.add_object(first, obj) + assert rid in service._ref_table + + second = service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-DUP") + assert second == "TX-DUP" + + # the prior live transaction was retired: refs unservable, done cb fired + assert rid not in service._ref_table + assert done == [("TX-DUP", TransactionDoneStatus.DELETED)] + # its terminal outcome does not shadow the live retry + assert service.get_transaction_outcome("TX-DUP") is None + # the live tx is the new incarnation + assert service._tx_table["TX-DUP"] is service._tx_incarnations["TX-DUP"] + assert service._tx_table["TX-DUP"] is not None + finally: + service.shutdown() + def test_unknown_transaction_has_no_outcome(self): service = make_isolated_download_service() assert service.get_transaction_outcome("no-such-tx") is None @@ -366,7 +427,11 @@ def test_shutdown_clears_outcomes_and_stops_recording(self): service.shutdown() assert service.get_transaction_outcome(tx.tid) is None - # a monitor iteration that was mid-termination during shutdown cannot repopulate + # a monitor iteration that was mid-termination during shutdown cannot repopulate: + # shutdown cleared _tx_incarnations, so the late recorder's tx is not the live + # incarnation and its outcome drops + from unittest.mock import Mock + late = compute_transfer_outcome("T-LATE", TransactionDoneStatus.FINISHED, 1, [], time.time()) - service._record_outcome(late) + service._record_outcome(late, tx=Mock()) assert service.get_transaction_outcome("T-LATE") is None From 51ca37d6388ed2d40edce7caa23ced983076c61c Mon Sep 17 00:00:00 2001 From: YuanTingHsieh Date: Tue, 7 Jul 2026 16:55:53 -0700 Subject: [PATCH 02/20] Design/plan updates: heartbeat phases, per-launch tokens, dependencies, status Address design-review findings: - Forward-path heartbeat rule narrowed: the no-expiry exemption now covers only the payload-materialization phase (TASK_ACCEPTED -> new TASK_PAYLOAD_READY message), bounded by a materialization deadline; heartbeats stay authoritative while user code trains, so a wedged process is detected at heartbeat timeout, not task timeout. Diagrams updated (mermaid re-validated). - launch_once=False moves to launch-scoped tokens: the CJ regenerates the token in each per-launch bootstrap config and stopping a process invalidates its token, so a surviving stale process cannot authenticate against a later launch. - Plan dependencies now enforce the design's payload-safety prerequisites: F3-4 hard-depends on F3-2 + F3-3 (receiver-confirmed, retry-aware outcomes; budget-bounded resolution); critical path updated (F3-2/F3-3 parallel behind F3-1, one M added, still inside the 10-14 week floor). - Explicit mapping between lifecycle transfer states and the F3 TransferOutcome vocabulary (TRANSFER_COMPLETE <=> COMPLETED; TRANSFER_FAILED <=> FAILED or ABORTED; receiver truth applies before the mapping). - Bookkeeping: PR-0 recorded as landing inside the F3-1 PR (#4853); EX-5 moved to Wave 4 (its EP-4 dependency) and assigned P1; CT-5 assigned P1; tier accounting line added (36 PRs, CT-4 halves in P1/P2); P0+P1 ~36 engineer-weeks. - ClientAPIBackendSpec classified as internal (frozen for parallel development, not public API in 2.9). Open Questions annotated with their deciding PRs. - Status set to Approved; Revision 2.2 entry added. - Add Topic.TASK_PAYLOAD_READY to the frozen protocol vocabulary (follow-up to #4856, which merged before the phase-boundary message was introduced). --- docs/design/client_api_execution_modes.md | 48 ++++++++++++------- .../design/client_api_execution_modes_plan.md | 25 +++++----- nvflare/client/cell/defs.py | 5 ++ tests/unit_test/client/cell/defs_test.py | 2 + 4 files changed, 52 insertions(+), 28 deletions(-) diff --git a/docs/design/client_api_execution_modes.md b/docs/design/client_api_execution_modes.md index 71f3380b84..d1914338d4 100644 --- a/docs/design/client_api_execution_modes.md +++ b/docs/design/client_api_execution_modes.md @@ -2,12 +2,14 @@ ## Status -Proposal / design note. Epic: **FLARE-2698** (Client API and 3rd party integration refactoring, fix version 2.9.0). +**Status: Approved** — implementation in progress for 2.9 (see the companion implementation plan, `client_api_execution_modes_plan.md`). Epic: **FLARE-2698** (Client API and 3rd party integration refactoring, fix version 2.9.0). Revision 2 (2026-07-01): incorporates review feedback — universal session setup, owner-death and receive-side contracts, forward-path payload lifecycle, cleanup policy definition, receiver-confirmed terminal outcome, per-receiver transfer budgets, configuration surface, attach auth hardening, and a re-sequenced migration plan. Revision 2.1 (2026-07-06): adds the filters-imply-materializing-hop rule to the payload lifecycle (per-(task,direction) granularity), conversion-filter placement details (site-scope filter ordering, DXO-meta round-trip state, mixed-format cost and bf16 limit, and what selects the regime — `server_expected_format` surviving as a recipe knob), the TensorStream disposition, and a CCWF caveat cross-referencing the materializing-hop rule (a wired conversion filter moves the producer role from the subprocess to the CJ). Claims verified against code. +Revision 2.2 (2026-07-07): status set to Approved. Forward-path heartbeat exemption narrowed to the payload-materialization phase via a new TASK_PAYLOAD_READY control message (heartbeats stay authoritative while user code trains); `launch_once=False` moves to launch-scoped tokens (fresh binding material per launch, invalidated at process stop); explicit mapping between the lifecycle transfer states and the F3 `TransferOutcome` vocabulary; `ClientAPIBackendSpec` classified as internal; Open Questions annotated with their deciding PRs. + ## Background NVFlare supports the Client API so training scripts can interact with FLARE through a small surface: @@ -226,7 +228,9 @@ sequenceDiagram C->>E: task E->>T: TASK_READY (task_id, FLModel ref) T->>E: TASK_ACCEPTED - Note over T: pull task payload, then train() + Note over T: pull task payload + T->>E: TASK_PAYLOAD_READY + Note over T: train() T-->>E: LOG / HEARTBEAT T->>E: RESULT_READY (result_id, transfer_id, manifest) E->>T: RESULT_ACCEPTED (control ack, not payload done) @@ -239,7 +243,10 @@ sequenceDiagram ``` Sequence — `launch_once=False`. A fresh process per task; the HELLO handshake repeats -each round (the job-scoped launch token is reused). The key difference: the per-task +each round with fresh binding material — the CJ regenerates the launch token in the +bootstrap config it already writes per launch, and stopping the previous process +invalidates its token, so a stale process that survived termination cannot +authenticate against a later launch (the token is launch-scoped, not job-scoped). The key difference: the per-task process stop **waits for the terminal transfer outcome** before killing the process — this is what replaces the legacy `download_complete_timeout` deferred stop: @@ -250,8 +257,8 @@ sequenceDiagram participant T as Trainer loop each task C->>E: task - E->>T: write config + Popen(...) [new process] - Note over T: flare.init() (job-scoped token reused) + E->>T: write config (fresh launch token) + Popen(...) [new process] + Note over T: flare.init() (per-launch token) T->>E: HELLO (launch token) E->>T: HELLO_ACCEPTED (fresh session) E->>T: TASK_READY (task_id, FLModel ref) @@ -260,7 +267,7 @@ sequenceDiagram Note over E,T: WAIT_TRANSFER_COMPLETE (hold process until terminal) E->>C: result E->>T: SHUTDOWN - Note over E,T: stop THIS process group,
next task launches a new process + Note over E,T: stop THIS process group and invalidate its token,
next task launches a new process end ``` @@ -307,6 +314,7 @@ sequenceDiagram C->>E: task E->>T: TASK_READY (task_id, FLModel ref) T->>E: TASK_ACCEPTED + T->>E: TASK_PAYLOAD_READY (payload pulled) T-->>E: LOG / HEARTBEAT (session lease) Note over T: train() T->>E: RESULT_READY (result_id, transfer_id, manifest) @@ -410,7 +418,9 @@ subprocess behavior.) ``` CJ -> trainer : TASK_READY (task id, task name, FLModel ref, params) trainer -> CJ : TASK_ACCEPTED -...trainer pulls task payload, trains... +...trainer pulls task payload... +trainer -> CJ : TASK_PAYLOAD_READY (task id — payload materialized or lazily held, training begins) +...trainer trains (heartbeats continue autonomously)... trainer -> CJ : TASK_FAILED (task id, reason — e.g. task payload download failed) [failure path] trainer -> CJ : RESULT_READY (result_id, transfer_id, manifest) CJ -> trainer : RESULT_ACCEPTED (or RESULT_REJECTED) @@ -460,7 +470,7 @@ TASK_ACCEPTED acknowledges control receipt of the task message only — it does - In external_process the executor has two liveness signals: process exit (waitpid on the process tree it owns) and heartbeat. Process exit is authoritative for "trainer is gone"; heartbeat covers a live-but-wedged trainer. - In attach the heartbeat lease is the only liveness signal the executor has. - **Precedence rule for transfers (result path):** active data-plane transfer progress counts as session liveness. While a session has an in-flight result transfer (WAIT_PAYLOAD_ACQUIRED/WAIT_TRANSFER_COMPLETE), the session lease does not expire on missed control heartbeats alone; the transfer's own idle/progress policy (shared F3 layer) governs, and only transfer failure/timeout or an explicit abort ends the session. This resolves the otherwise-contradictory pair "heartbeat timeout invalidates the session" vs "the executor must not revoke the session before terminal payload state" — and heartbeat false-positives during multi-GB result uploads are one failure class this design removes. (The legacy stack solved this with a dedicated STREAM_PROGRESS topic feeding transfer waits, deliberately separate from peer liveness; the same separation is preserved here, just owned by the shared transfer layer.) -- **Precedence rule for the forward (task-delivery) path:** the same false-positive class exists in the *other* direction and needs its own rule, because the CJ cannot see forward bytes. When TASK_READY carries a lazy ref to a multi-GB global model, the control rank pulls those bytes **directly from the producer** (server job process); the CJ that owns the session/heartbeat lease is only a relay and observes no forward transfer (the producer's progress callback fires at the server, not the CJ). A control rank busy materializing 5 GB inside `flare.receive()` therefore has no in-flight-transfer signal to feed the rule above. Rule: **the CJ must not revoke the session on control-heartbeat timeout while a TASK_ACCEPTED is outstanding** — an accepted-but-not-yet-completed task means the trainer is legitimately working (materializing the payload and/or training), and forward materialization is bounded by the trainer-local pull policy (the shared F3 idle/timeout on the trainer→producer transfer), not by the control heartbeat. This is the forward analog of the result-path rule and closes the same 5GB-materialization false-positive that `progress_aware_streaming.md` documents on `task_payload_download`. +- **Precedence rule for the forward (task-delivery) path:** the same false-positive class exists in the *other* direction and needs its own rule, because the CJ cannot see forward bytes. When TASK_READY carries a lazy ref to a multi-GB global model, the control rank pulls those bytes **directly from the producer** (server job process); the CJ that owns the session/heartbeat lease is only a relay and observes no forward transfer (the producer's progress callback fires at the server, not the CJ). A control rank busy materializing 5 GB inside `flare.receive()` therefore has no in-flight-transfer signal to feed the rule above. Rule: **the CJ must not revoke the session on control-heartbeat timeout during the payload-materialization phase — from TASK_ACCEPTED until the trainer sends TASK_PAYLOAD_READY (or TASK_FAILED)** — because materialization is bounded by the trainer-local pull policy (the shared F3 idle/timeout on the trainer→producer transfer), not by the control heartbeat; the CJ additionally bounds the phase with a materialization deadline derived as an envelope over that pull policy, so a trainer that dies mid-pull without emitting TASK_FAILED still fails the task instead of holding the session open. The exemption ends at TASK_PAYLOAD_READY: **while user code trains, heartbeats stay authoritative** — the trainer engine's heartbeat thread runs autonomously alongside training (the same autonomy the send path guarantees above), so a healthy trainer keeps its lease through arbitrarily long training and a wedged or dead one is detected at the normal heartbeat timeout rather than at the task timeout. This matters most in attach, where the heartbeat lease is the only liveness signal. This is the forward analog of the result-path rule and closes the same 5GB-materialization false-positive that `progress_aware_streaming.md` documents on `task_payload_download` — without disabling liveness for the (much longer) training phase. #### CJ Failure (Owner Death) @@ -512,7 +522,7 @@ DONE_CLEANUP_ALLOWED [terminal — producer may exit and free payload] ABORT at any state ────────────────────────────────► ABORTED [terminal] ``` -**Terminal states are DONE_CLEANUP_ALLOWED (success), TRANSFER_FAILED, RESULT_REJECTED, and ABORTED.** (TRANSFER_COMPLETE / TRANSFER_FAILED are the transfer-layer outcomes the machine consumes; TRANSFER_COMPLETE moves the machine to DONE_CLEANUP_ALLOWED. Earlier drafts used the two vocabularies interchangeably; they are now distinct.) RESULT_ACCEPTED is reached well before cleanup is allowed — that gap is the whole point: an accepted control message is not a completed payload transfer. +**Terminal states are DONE_CLEANUP_ALLOWED (success), TRANSFER_FAILED, RESULT_REJECTED, and ABORTED.** (TRANSFER_COMPLETE / TRANSFER_FAILED are the transfer-layer outcomes the machine consumes; TRANSFER_COMPLETE moves the machine to DONE_CLEANUP_ALLOWED. Earlier drafts used the two vocabularies interchangeably; they are now distinct.) The transfer layer itself reports a third, F3-native vocabulary: the aggregate `TransferOutcome.status` values COMPLETED / FAILED / ABORTED (`transfer_outcome.py`, reusing the TransferProgressState terminal set). The mapping at that boundary is fixed and total: TRANSFER_COMPLETE ⇔ COMPLETED; TRANSFER_FAILED ⇔ FAILED **or** ABORTED (the raw `done_status` and `reason` ride along for diagnostics, but the lifecycle machine does not branch on them). Receiver truth is applied *before* the mapping: a transaction deleted by routine cleanup after every expected receiver succeeded resolves COMPLETED, not ABORTED. RESULT_ACCEPTED is reached well before cleanup is allowed — that gap is the whole point: an accepted control message is not a completed payload transfer. Logical ownership: @@ -576,6 +586,7 @@ The forward direction — global model from server/CJ to the trainer — is wher - **TASK_READY is control-only.** It carries the task metadata and FLModel lazy refs; TASK_ACCEPTED acknowledges the control message, before any payload materialization — the same "accepted ≠ transferred" caveat as RESULT_ACCEPTED. - **The trainer's pull is governed by the same shared transfer policy.** `flare.receive()` materializes (or lazily holds) the model by pulling through the shared F3 path; idle/progress/timeout policy is the shared layer's, and the upstream producer (CJ relay or server job process) is held to the same producer-liveness rule — its refs stay alive until the trainer's pull reaches a terminal state. - **Trainer-side download failure is explicit.** If the task payload pull fails or times out, the trainer backend sends TASK_FAILED with the task id and reason; the executor fails or retries the task at the workflow's discretion. There is no silent hang and no ambiguous half-delivered task. +- **Materialization completion is explicit.** When `flare.receive()` hands the task to user code (payload materialized, or lazily held), the trainer backend sends TASK_PAYLOAD_READY with the task id. TASK_PAYLOAD_READY or TASK_FAILED ends the payload-materialization phase — and with it the forward-path heartbeat exemption (see Heartbeat and Liveness); from that point the session lease is governed by heartbeats as normal while user code trains. - **No blind resend.** The CJ does not resend TASK_READY on a timer while the session is alive (Cell request/reply plus TASK_READY idempotency replace the Pipe resend loop that caused duplicate-delivery races). Relationship to `progress_aware_streaming.md`: that design's Phase-1 wait policies are hosted today in TaskExchanger (CJ-side forward waits) and FlareAgent (trainer-side reverse waits) over an internal Pipe topic — exactly the components this design retires. The progress-tracking substrate it defines (per-transfer progress in the shared F3 layer) is the part that survives and is what the waits here consume; the Pipe-topic delivery and TaskExchanger/FlareAgent wait owners die with the legacy stack. The forward-path contract above is the re-homed replacement: readiness comes from HELLO, delivery from Cell request/reply, and materialization waits from the shared transfer layer instead of resend suppression. @@ -733,6 +744,8 @@ silently dropped. `task_script_path`/`task_script_args` (the in_process trainer entry point), the task-name mapping, and the memory knobs are carried forward from today's InProcessClientAPIExecutor/ScriptRunner so existing jobs map without loss. +**`ClientAPIBackendSpec` is internal.** The mode backends behind the executor implement a single `ClientAPIBackendSpec` interface (in `client_api_executor.py`). Its surface is frozen early so the mode backends can be built in parallel, but it is an internal extension interface, not public API in 2.9: users select and configure modes only through the ClientAPIExecutor/ScriptRunner arguments above, and no third-party backend registration point is exposed (see the rejected general Client API Launcher under Alternatives Considered). + **No parameter converters on the executor (per FLARE-2698).** The `params_exchange_format`, `params_transfer_type`, `server_expected_format`, and `from/to_nvflare_converter_id` arguments of the legacy executors are intentionally absent. Conversion between the framework-agnostic @@ -811,7 +824,7 @@ One executor component, no pipes, no launcher, no MetricRelay: LOG messages arri ### Observability -Each session exposes its lifecycle state for operators: session state (waiting HELLO / idle / task active / waiting transfer), current task_id/result_id/transfer_id, per-receiver transfer progress (from the shared layer's progress events), and last heartbeat. These surface through the standard job stats/log channels (CJ logs at state transitions with the ids above; stats pollable via the existing cell/job info mechanisms), so "why is this producer still alive" and "which receiver is stalled" are answerable from the site without a debugger. +Each session exposes its lifecycle state for operators: session state (waiting HELLO / idle / task materializing / task active / waiting transfer), current task_id/result_id/transfer_id, per-receiver transfer progress (from the shared layer's progress events), and last heartbeat. These surface through the standard job stats/log channels (CJ logs at state transitions with the ids above; stats pollable via the existing cell/job info mechanisms), so "why is this producer still alive" and "which receiver is stalled" are answerable from the site without a debugger. ## Scenario Coverage @@ -1020,12 +1033,13 @@ Core external_process and rank contract (steps 3–4): Session and failure contracts (steps 3–4): -- external_process: a HELLO carrying a wrong/stale launch token is rejected (HELLO_REJECTED) and a stale trainer from a previous run cannot bind; attach: a bad token proof fails at HELLO_PROOF (see the attach auth tests) +- external_process: a HELLO carrying a wrong/stale launch token is rejected (HELLO_REJECTED) and a stale trainer from a previous run cannot bind (under `launch_once=False`, the previous launch's token is invalidated at process stop, so a surviving per-task process cannot bind to the next launch); attach: a bad token proof fails at HELLO_PROOF (see the attach auth tests) - HELLO with a mismatched protocol version fails fast with a clear error - duplicate TASK_READY delivery does not double-deliver the task to user code - trainer-side task payload download failure produces TASK_FAILED, not a hang - CJ kill (SIGKILL) leads to trainer process-group self-termination within the grace period; CP reaping covers a disabled trainer-side grace - heartbeat loss during an active multi-GB transfer does not revoke the session while the transfer is progressing; transfer failure/timeout does +- heartbeat loss during payload materialization (TASK_ACCEPTED outstanding, TASK_PAYLOAD_READY not yet sent) does not revoke the session; sustained heartbeat loss after TASK_PAYLOAD_READY, while user code trains, does - `flare.receive()` returns None on SHUTDOWN/job end; raises the session exception on ABORT and on session loss Payload lifecycle (steps 2, 4): @@ -1050,11 +1064,13 @@ CCWF (step 6): ### Open Questions -- Exact public argument names and migration aliases. -- Scheduler batch helper library vs. standard wrapper component. -- Exact factoring of shared code with IPCAgent/IPCExchanger. -- Compatibility flag for selecting the legacy Pipe path during migration. -- Whether partial fan-out (N-1 of N receivers succeeded) should be surfaceable to CCWF workflows as a usable result or always a task failure. +Each open question is tracked to the PR that decides it (plan PR ids); none blocks the interface freezes already landed. + +- Exact public argument names and migration aliases — decided at the EX-2 interface freeze (the frozen `ClientAPIExecutor` constructor is the decision record) and the EX-3 ScriptRunner wiring. +- Scheduler batch helper library vs. standard wrapper component — deferred to Migration Plan step 7 (Future Enhancements); not 2.9-blocking. +- Exact factoring of shared code with IPCAgent/IPCExchanger — decided during the trainer-engine PR (TE-4), where the shared session/receive machinery gets its final home. +- Compatibility flag for selecting the legacy Pipe path during migration — decided in EX-3/EP-4: whether legacy selection stays purely class-name-based (per the Compatibility section's keep-legacy story) or also gets an explicit flag. +- Whether partial fan-out (N-1 of N receivers succeeded) is surfaceable to CCWF workflows as a usable result or always a task failure — decided by F3-3's quorum surface (the plan requires F3-3 to settle it) and surfaced to workflows in CC-1. ## Migration Plan diff --git a/docs/design/client_api_execution_modes_plan.md b/docs/design/client_api_execution_modes_plan.md index 636d574360..68c5631f49 100644 --- a/docs/design/client_api_execution_modes_plan.md +++ b/docs/design/client_api_execution_modes_plan.md @@ -1,6 +1,6 @@ # Client API Execution Modes — 2.9 Implementation Plan -Companion to `client_api_execution_modes.md` (design). Tracks Epic **FLARE-2698** (Client API and 3rd party integration refactoring, 2.9.0). Decomposes the design's 8-step Migration Plan into **36 PRs** with dependencies, sizes, and a release cut line. Scoped against the codebase post-2.8.0; granularity calibrated against the repo's merged-PR history (2026-07-01, see Calibration below). +Companion to `client_api_execution_modes.md` (design). Tracks Epic **FLARE-2698** (Client API and 3rd party integration refactoring, 2.9.0). Decomposes the design's 8-step Migration Plan into **36 PRs** with dependencies, sizes, and a release cut line (the wave plan lists 37 entries; PR-0 is not a separate PR — the doc set lands inside the F3-1 PR, #4853). Scoped against the codebase post-2.8.0; granularity calibrated against the repo's merged-PR history (2026-07-01, see Calibration below). Size guide: **S** <300 changed LOC, **M** 300–800, **L** 800–2000. @@ -24,8 +24,8 @@ Tracks: **F3** payload layer · **EX** executor/ScriptRunner · **TE** trainer e | PR | Track | Size | Notes | |---|---|---|---| -| PR-0 Land design doc + this implementation plan in docs/design/ | all | S | **Lands first, merges fast** — the design is already approved; this just puts the reference material in-repo so every subsequent PR links it. Interface-freeze sign-offs happen on TE-1/EX-2/F3-1 themselves | -| F3-1 Aggregate all-receivers terminal transfer outcome | F3 | M | Interface freeze #3. Purely additive next to FINISHED/TIMEOUT/DELETED | +| PR-0 Land design doc + this implementation plan in docs/design/ | all | S | **Lands with F3-1 in a single PR (#4853)**, not as a separate docs-only PR — the reference material merges together with the first frozen contract, and every subsequent PR links it. Interface-freeze sign-offs happen on TE-1/EX-2/F3-1 themselves | +| F3-1 Aggregate all-receivers terminal transfer outcome | F3 | M | Interface freeze #3. Purely additive next to FINISHED/TIMEOUT/DELETED. Ships as #4853, carrying PR-0's docs | | TE-1 Protocol vocabulary (`client/cell/defs.py`) | TE | S | Interface freeze #1 — the Cell control-protocol Topics/MsgKeys/CHANNEL/version only. The auth **mechanism** is deliberately NOT frozen here: the token is 3 roles (rendezvous, anti-mixup, auth) and whether external_process needs a *secret* HMAC proof at all is a host-trust decision (single-tenant: no; multi-tenant: yes) that EP-3 makes. So the proof helpers (TokenScope, compute/verify_hello_proof, combine_nonces) land with **EP-3**, and the generic generators (generate_session_token/nonce, token_digest) go to the fuel/sec consolidation (**FLARE-3017**). The stateful SessionTokenManager is attach-only (**AT-2**) | | EX-2 ClientAPIExecutor skeleton + backend spec + analytics-event ownership | EX | M | Interface freeze #2 | | TE-2 Bootstrap config schema + NVFLARE_CLIENT_API_CONFIG resolution | TE | M | Additive ConfigKeys; consumes EP-1's 0600 writer. Kept separate from TE-1: touches legacy-shared `client/config.py`, different revert profile | @@ -38,8 +38,8 @@ Tracks: **F3** payload layer · **EX** executor/ScriptRunner · **TE** trainer e | PR | Track | Size | Depends | |---|---|---|---| | F3-2 Receiver-confirmed completion + retry-aware accounting | F3 | M | F3-1. The version-skew wire change — lands as early as possible for maximum soak; capability-flag gated, both skews interop-tested | -| F3-3 Per-(transfer, receiver) acquire/idle budgets | F3 | M | F3-1. Unconditional per-receiver activity tracking. Must also settle the quorum surface for fan-out: workflows with min_responses-style policy (k-of-N receivers suffices) need either an optional min_receivers on the transaction/facade or a documented pattern of evaluating TransferOutcome.refs against their own threshold — `completed` stays the strict all-receivers certificate either way | -| F3-4 Awaitable producer transfer facade + PAYLOAD_ACQUIRED + post-completion linger | F3 | M | F3-1 (and F3-2 for the linger's receiver-confirmed path). Must CHAIN existing DOWNLOAD_COMPLETE_CB, not replace. Folds in the former F3-5 bounded post-completion linger (a leaf with no independent consumer — only this facade's release path uses it); F3-2 still lands as its own isolated PR so the wire-skew change is not welded here | +| F3-3 Per-(transfer, receiver) acquire/idle budgets | F3 | M | F3-1; blocks F3-4, whose aggregate outcome resolves in bounded time only through these budgets (a stalled receiver is marked failed without waiting for the full transaction TTL). Unconditional per-receiver activity tracking. Must also settle the quorum surface for fan-out: workflows with min_responses-style policy (k-of-N receivers suffices) need either an optional min_receivers on the transaction/facade or a documented pattern of evaluating TransferOutcome.refs against their own threshold — `completed` stays the strict all-receivers certificate either way | +| F3-4 Awaitable producer transfer facade + PAYLOAD_ACQUIRED + post-completion linger | F3 | M | F3-1, F3-2, F3-3 — all hard. The facade's returns-means-delivered guarantee holds only over receiver-confirmed, retry-aware outcomes (F3-2), and its aggregate outcome resolves in bounded time only with the per-receiver budgets (F3-3); a facade over producer-served EOF would reintroduce the silent-truncation gap. Must CHAIN existing DOWNLOAD_COMPLETE_CB, not replace. Folds in the former F3-5 bounded post-completion linger (a leaf with no independent consumer — only this facade's release path uses it); F3-2 still lands as its own isolated PR so the wire-skew change is not welded here | | EX-3 in_process backend (consolidate InProcessClientAPIExecutor) | EX | L | EX-2. Behavior-parity bar: "nothing user-visible" | | TE-3 TrainerCellSession engine (handshake, heartbeat, owner-death, trainer-side authenticated teardown) | TE | L | TE-1, TE-2. Injectable clock + kill hook; AT owner co-reviews the teardown-auth tests | | EP-5 CP-side orphan reaping of trainer PGIDs | EP | M | EP-2. PID-reuse guard via start-time record | @@ -48,10 +48,9 @@ Tracks: **F3** payload layer · **EX** executor/ScriptRunner · **TE** trainer e | PR | Track | Size | Depends | |---|---|---|---| -| TE-4 TrainerCellSession task/result contracts (receive queue + TASK_READY idempotency + TASK_FAILED; RESULT_READY flow + terminal-outcome blocking + fan-out drain/shutdown gating) | TE | L | TE-3; F3-4 via a narrow stubbed wait-protocol interface. Merged from two Ms: same class, same owner, no independent consumer of either half. Guard: split fan-out drain back out if the diff passes ~1800 LOC | +| TE-4 TrainerCellSession task/result contracts (receive queue + TASK_READY idempotency + TASK_PAYLOAD_READY/TASK_FAILED; RESULT_READY flow + terminal-outcome blocking + fan-out drain/shutdown gating) | TE | L | TE-3; F3-4 via a narrow stubbed wait-protocol interface. Merged from two Ms: same class, same owner, no independent consumer of either half. Guard: split fan-out drain back out if the diff passes ~1800 LOC | | EP-3 external_process auth + CJ-side HELLO acceptance + session-scoped message enforcement | EP | M | TE-1 (vocabulary). **Owns the auth-model decision**: a per-launch rendezvous id always; a *secret* launch token proven by one-round HMAC on multi-tenant hosts (rendezvous-only + OS isolation may suffice single-tenant). Introduces the proof helpers (TokenScope, compute/verify_hello_proof, combine_nonces) here — reused later by AT-2. Session-scoped message enforcement (accept trainer messages only from the bound session) closes the live IPCAgent any-sender gap; a P0 control, not the P2 attach track | | EX-4 ScriptRunner `execution_mode` param + launch_external_process mapping | EX | M | EX-2/3. Convert ~22 internal recipe call sites in the same PR | -| EX-5 Converter→filter migration (FLARE-2698 bullet 2) | EX | M | EX-3, EP-4. Replace executor-owned ParamsConverters with PT/TF send+receive conversion filters at the client edge (last task-data / first task-result filter); recipes auto-wire per framework only when server format ≠ exchange format; Client API boundary passes through (RAW). Filters delegate to the existing converter classes (no logic rewrite); round-trip state (tensor shapes, excluded entries) rides DXO meta (quantizer pattern), not FLContext — removes the `_ConverterContext` stub. Keep ParamsConverter ABC deprecated + a `ParamsConverterFilter` adapter for custom converter IDs. Wiring the conversion filter makes the CJ a materializing hop (design doc: payload-lifecycle rule); numpy regime excludes bf16 — large models use pytorch end-to-end. Removes params_exchange_format/server_expected_format/converter-id from the surface (already excluded from EX-2's freeze). Transfer type FULL/DIFF stays in model_registry, decided separately | | CC-1 CCWF transfer-declaration plumbing (receiver sets, stage windows, aux passthrough) | CC | M | F3-3. Declaration-only; absent headers preserve today's defaults — its behavior-neutrality is what de-risks the CC track | | CT-8 Session observability (state-transition logs + StatsPoolManager view) | CT | M | EX-2; extends as backends land | @@ -68,6 +67,7 @@ Tracks: **F3** payload layer · **EX** executor/ScriptRunner · **TE** trainer e | PR | Track | Size | Depends | |---|---|---|---| +| EX-5 Converter→filter migration (FLARE-2698 bullet 2) | EX | M | EX-3, EP-4. Replace executor-owned ParamsConverters with PT/TF send+receive conversion filters at the client edge (last task-data / first task-result filter); recipes auto-wire per framework only when server format ≠ exchange format; Client API boundary passes through (RAW). Filters delegate to the existing converter classes (no logic rewrite); round-trip state (tensor shapes, excluded entries) rides DXO meta (quantizer pattern), not FLContext — removes the `_ConverterContext` stub. Keep ParamsConverter ABC deprecated + a `ParamsConverterFilter` adapter for custom converter IDs. Wiring the conversion filter makes the CJ a materializing hop (design doc: payload-lifecycle rule); numpy regime excludes bf16 — large models use pytorch end-to-end. Removes params_exchange_format/server_expected_format/converter-id from the surface (already excluded from EX-2's freeze). Transfer type FULL/DIFF stays in model_registry, decided separately | | CT-2 torchrun 2-rank rank-contract CI tests (CPU/gloo) | CT | M | CT-1, EP-4. EP owner co-reviews | | CT-3 Owner-death (CJ-kill) + payload-lifecycle E2E tests | CT | L | CT-1, F3 track, EP-4. Covers CJ-SIGKILL self-termination + CP reaping; EP owner co-reviews | | CC-2 Swarm onto the transfer contract (remove lazy-ref machinery) | CC | L | CC-1, F3 track, EP-4. Highest-risk PR in the program; maximally isolated revert unit; retires test_lazy_ref_local_aggr / test_msg_root_ttl | @@ -112,18 +112,19 @@ Notes: the one-paragraph "why" for the whole program lives in PR-0 and gets link ## Critical path -F3-1 → F3-4 → TE-4 → TE-5 → EP-4 → CT-1 → CT-3 → release gate. The TE-4 merge removed one review cycle from the path. **EP-4** remains the schedule risk to watch — it's where executor, trainer engine, auth, process runner, and the F3 facade meet; its prerequisites are deliberately extracted to keep it L. +F3-1 → F3-2/F3-3 (parallel) → F3-4 → TE-4 → TE-5 → EP-4 → CT-1 → CT-3 → release gate. F3-2 and F3-3 run in parallel behind F3-1, so the facade's hard dependencies cost the path one M — still inside the 10–14-week floor. The TE-4 merge removed one review cycle from the path. **EP-4** remains the schedule risk to watch — it's where executor, trainer engine, auth, process runner, and the F3 facade meet; its prerequisites are deliberately extracted to keep it L. ## 2.9 cut line (recommendation) -- **P0 — commit for 2.9** (24 PRs): F3 track (4, after folding F3-5 into F3-4) + EX track (4) + TE track (5) + EP track (6) + CT-1/2/3/6/8. Headline: *external_process and in_process on the new Cell stack with an enforceable payload lifecycle, owner-death handling, session-scoped message enforcement, and the config-permission fix*. -- **P1 — strongly target for 2.9** (6 PRs): CC track (5) + the CCWF half of CT-4. Fully severable (SWARM keeps current behavior until CC-2 lands); ship CC-5 flagged experimental. +- **P0 — commit for 2.9** (24 PRs): F3 track (4, after folding F3-5 into F3-4) + EX track (4 — EX-1..4; EX-5 goes to P1) + TE track (5) + EP track (6) + CT-1/2/3/6/8. Headline: *external_process and in_process on the new Cell stack with an enforceable payload lifecycle, owner-death handling, session-scoped message enforcement, and the config-permission fix*. +- **P1 — strongly target for 2.9** (8 PRs): CC track (5) + EX-5 conversion-filter migration (2.9 runs PyTorch end-to-end without it — the numpy regime is what needs the filters) + CT-5 rank-contract example updates (shipped examples must demonstrate the new contract; they do not gate the code cut) + the CCWF half of CT-4. Fully severable (SWARM keeps current behavior until CC-2 lands); ship CC-5 flagged experimental. - **P2 — stretch for 2.9, else 2.9.x/2.10** (4 PRs): AT track (3) + the attach half of CT-4. Attach is the most self-contained step; nothing in P0/P1 depends on it (its reusable auth/proof helpers ship in P0 via EP-3, and session enforcement via EP-3, regardless). - **CT-7 warnings** (1 PR) close 2.9. **Explicitly deferred to 2.10**: the ScriptRunner default flip and any legacy-class removal — ship 2.9 opt-in with warnings; flip after a release of field soak. +- Tier accounting: every one of the 36 PRs has a tier — P0 (24) + P1 unique (7: CC×5, EX-5, CT-5) + P2 unique (3: AT×3) + CT-4 (one PR, halves in P1 and P2) + CT-7 (1) = 36. The P1/P2 headline counts each mention CT-4 once, so the tier lines name 37 items for 36 PRs. ## Effort and staffing -36 PRs. At review-inclusive rates (S ≈ 2 days, M ≈ 1 week, L ≈ 2 weeks) that is ≈ 44 engineer-weeks total; ≈ 34 for P0+P1. With 4–5 engineers owning tracks (F3, TE, EP+EX, CC, AT+CT), full parallelism after Wave 0; calendar floor is the critical path — realistically **10–14 weeks to P0+P1 complete** plus stabilization. If 2.9 code freeze is nearer, cut P2 first, then CC-4/CC-5. +36 PRs. At review-inclusive rates (S ≈ 2 days, M ≈ 1 week, L ≈ 2 weeks) that is ≈ 44 engineer-weeks total; ≈ 36 for P0+P1. With 4–5 engineers owning tracks (F3, TE, EP+EX, CC, AT+CT), full parallelism after Wave 0; calendar floor is the critical path — realistically **10–14 weeks to P0+P1 complete** plus stabilization. If 2.9 code freeze is nearer, cut P2 first, then CC-4/CC-5. ## Guardrails — deliberate non-merges @@ -132,7 +133,7 @@ These pairs are tempting to consolidate and must stay separate: - **EP-1 (0600 fix) with anything**: live credential exposure on today's path; standalone, cherry-pickable, surgical revert. - **EX-3 (in_process backend) + EX-4 (ScriptRunner)**: ScriptRunner is the most-used public entry point (golden exported-config tests, ~22 recipe conversions); a backend parity revert must not drag the public parameter surface. - **Anything into CC-2 (swarm refactor)**: highest-risk PR in the program; must remain a maximally isolated revert unit. -- **F3-3 (budgets) + F3-4 (facade)**: same module, but F3-4 is on the critical path every track waits on; F3-3 serves only CC-1. +- **F3-3 (budgets) + F3-4 (facade)**: same module and strictly sequenced (F3-3 is a hard prerequisite of F3-4), but they stay separate PRs — the budgets land ahead as their own surgical revert unit, and F3-4 is the critical-path PR every track waits on. - **F3-2 (receiver-confirm wire change) must not weld to F3-4's linger/exit-timing policy**: the version-skew change wants early landing and a surgical revert; the linger now lives in F3-4 (former F3-5) but F3-2 still ships isolated ahead of it. - **EP-3 + AT-2**: same CJ-side HELLO/session machinery, but they straddle the P0/P2 boundary — AT-2 consumes EP-3's module instead. - **EP-2 (runner) + EP-5 (CP reaping)**: different processes and blast radii; EP-5's false-positive-reap risk needs an independent revert path. diff --git a/nvflare/client/cell/defs.py b/nvflare/client/cell/defs.py index 57ae47d64f..652e083de5 100644 --- a/nvflare/client/cell/defs.py +++ b/nvflare/client/cell/defs.py @@ -56,6 +56,11 @@ class Topic: # Per task (every round) TASK_READY = "client_api.task_ready" TASK_ACCEPTED = "client_api.task_accepted" + # Sent when the trainer has materialized (or lazily bound) the task payload and hands + # it to user code. Ends the payload-materialization phase: the forward-path heartbeat + # exemption stops here and heartbeats govern the session lease while user code trains + # (design: Heartbeat and Liveness, Revision 2.2). + TASK_PAYLOAD_READY = "client_api.task_payload_ready" TASK_FAILED = "client_api.task_failed" RESULT_READY = "client_api.result_ready" RESULT_ACCEPTED = "client_api.result_accepted" diff --git a/tests/unit_test/client/cell/defs_test.py b/tests/unit_test/client/cell/defs_test.py index b5644a09f2..0b16c0244d 100644 --- a/tests/unit_test/client/cell/defs_test.py +++ b/tests/unit_test/client/cell/defs_test.py @@ -33,6 +33,7 @@ def test_expected_topics_present(self): "HELLO_REJECTED", "TASK_READY", "TASK_ACCEPTED", + "TASK_PAYLOAD_READY", "TASK_FAILED", "RESULT_READY", "RESULT_ACCEPTED", @@ -77,6 +78,7 @@ def test_topic_wire_values(self): "HELLO_REJECTED": "client_api.hello_rejected", "TASK_READY": "client_api.task_ready", "TASK_ACCEPTED": "client_api.task_accepted", + "TASK_PAYLOAD_READY": "client_api.task_payload_ready", "TASK_FAILED": "client_api.task_failed", "RESULT_READY": "client_api.result_ready", "RESULT_ACCEPTED": "client_api.result_accepted", From ce4e95aaa8389f999d2beab09374eca1ccae2f46 Mon Sep 17 00:00:00 2001 From: YuanTingHsieh Date: Tue, 7 Jul 2026 18:05:46 -0700 Subject: [PATCH 03/20] Add F3 payload layer: receiver-confirmed completion, receiver budgets, awaitable facade The consolidated F3/Cell layer the Client API execution modes build on (design: docs/design/client_api_execution_modes.md; plan F3-2 + F3-3 + F3-4). Upper layers (executor backends, trainer engine) consume one primitive: waiter = downloader.get_waiter() outcome = waiter.wait(timeout=...) # returns == delivered F3-2 Receiver-confirmed completion + retry-aware accounting. A confirm-capable receiver's served EOF/ERROR is PROVISIONAL; the receiver's fire-and-forget confirmation -- sent only after Consumer.download_completed() finalization actually succeeds -- finalizes its status. Receiver truth wins (served-EOF + failed finalization = confirmed FAILED, the disk-offload case); retries overwrite provisional records; the first confirmation is final; a confirmation is accepted only against a pending provisional serve on the same ref incarnation, so stale/unsolicited confirms can neither certify nor poison. All wire keys are optional -- both version skews degrade to today's producer-served semantics -- and a runtime kill-switch (streaming_receiver_confirm_enabled, standard application-config resolution) disables the wire behavior on either side without a code revert. F3-3 Per-(transfer, receiver) acquire/idle budgets + quorum surface. Unconditional per-receiver activity tracking (a live receiver no longer masks a stalled one behind the tx-wide timestamp). A receiver that never pulls (acquire, vs workflow-declared receiver_ids, acquired at TRANSACTION level so sequential multi-ref downloads are safe) or goes silent (idle) is finalized FAILED on a monitor pass with a truth-wins re-check -- the aggregate outcome resolves in bounded time, not the transaction TTL. This also bounds a lost confirmation (fail-closed). Expected receiver identities now thread from via_downloader into the transaction. min_receivers surfaces the k-of-N quorum (quorum_met requires the same receiver to succeed on EVERY ref); `completed` stays the strict all-receivers certificate. F3-4 TransferWaiter, the awaitable facade. Event-driven (resolved inside outcome recording; attaches before or after termination), never hangs (unknown/expired/shut-down ids resolve immediately; shutdown releases all waiters), optional FINISHED-gated linger preserving the tombstone replay window for lost terminal replies, acquired_receivers() as the V1 PAYLOAD_ACQUIRED signal, and composes with -- never replaces -- the existing DOWNLOAD_COMPLETE_CB chain. 54 new unit tests (receiver_confirm_test, receiver_budget_test, transfer_waiter_test): skew matrix, kill-switch both sides, receiver-truth- wins, retry healing, mixed fleets, tombstone interplay, unsolicited-confirm guard, bounded lost-confirm, multi-ref acquisition, quorum intersection, config resolution, waiter never-hangs invariants. Full streaming+fobs+legacy progress sweeps green (547). Reviewed by a 28-agent adversarial workflow; all 24 confirmed findings fixed. --- nvflare/fuel/f3/streaming/download_service.py | 489 +++++++++++++++++- nvflare/fuel/f3/streaming/obj_downloader.py | 17 + nvflare/fuel/f3/streaming/transfer_outcome.py | 49 +- .../utils/fobs/decomposers/via_downloader.py | 11 +- .../fuel/f3/streaming/download_test_utils.py | 1 + .../fuel/f3/streaming/receiver_budget_test.py | 348 +++++++++++++ .../f3/streaming/receiver_confirm_test.py | 408 +++++++++++++++ .../fuel/f3/streaming/transfer_waiter_test.py | 245 +++++++++ 8 files changed, 1538 insertions(+), 30 deletions(-) create mode 100644 tests/unit_test/fuel/f3/streaming/receiver_budget_test.py create mode 100644 tests/unit_test/fuel/f3/streaming/receiver_confirm_test.py create mode 100644 tests/unit_test/fuel/f3/streaming/transfer_waiter_test.py diff --git a/nvflare/fuel/f3/streaming/download_service.py b/nvflare/fuel/f3/streaming/download_service.py index b5f4d635d4..c4b384ecc2 100644 --- a/nvflare/fuel/f3/streaming/download_service.py +++ b/nvflare/fuel/f3/streaming/download_service.py @@ -19,6 +19,7 @@ from abc import ABC, abstractmethod from typing import Any, Callable, Optional, Tuple +from nvflare.apis.fl_constant import SystemConfigs from nvflare.apis.signal import Signal from nvflare.fuel.f3.cellnet.cell import Cell from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey, ReturnCode @@ -33,6 +34,7 @@ terminal_state_for_done_status, ) from nvflare.fuel.f3.streaming.transfer_progress import TransferProgressState +from nvflare.fuel.utils.config_service import ConfigService from nvflare.fuel.utils.log_utils import get_obj_logger from nvflare.security.logging import secure_format_exception @@ -172,6 +174,57 @@ class _PropKey: STATE = "state" DATA = "data" STATUS = "status" + # Receiver-confirmed completion (F3-2). All three keys are OPTIONAL on the wire so both + # version skews interop with legacy peers: an old receiver never sends CONFIRM_CAPABLE and + # gets today's producer-served semantics; an old producer never sends CONFIRM_EXPECTED so a + # new receiver never confirms toward it. + CONFIRM = "confirm" # receiver -> producer: terminal receiver truth (a DownloadStatus value) + CONFIRM_CAPABLE = "confirm_capable" # receiver -> producer, per request: will confirm if asked + CONFIRM_EXPECTED = "confirm_expected" # producer -> receiver, per reply: confirmations consumed + + +# Runtime kill-switch for receiver-confirmed completion. The wire behavior is doubly gated -- +# per-message capability advertisement AND this config var on each side -- so a field issue in a +# mixed-version fleet is mitigated by configuration without a code revert. +RECEIVER_CONFIRM_CONFIG_VAR = "streaming_receiver_confirm_enabled" +_receiver_confirm_cached = None + + +def _receiver_confirm_enabled() -> bool: + global _receiver_confirm_cached + if _receiver_confirm_cached is None: + try: + _receiver_confirm_cached = bool( + ConfigService.get_bool_var( + RECEIVER_CONFIRM_CONFIG_VAR, conf=SystemConfigs.APPLICATION_CONF, default=True + ) + ) + except Exception: + # unconfigured environments (e.g. bare unit tests) default to enabled + _receiver_confirm_cached = True + return _receiver_confirm_cached + + +# Per-(transfer, receiver) budgets (F3-3). System defaults resolved from config vars; explicit +# per-transaction values win. None (unset everywhere) disables enforcement for that budget -- +# the whole-transaction timeout then remains the only backstop, exactly today's behavior. +RECEIVER_ACQUIRE_TIMEOUT_CONFIG_VAR = "streaming_receiver_acquire_timeout" +RECEIVER_IDLE_TIMEOUT_CONFIG_VAR = "streaming_receiver_idle_timeout" + + +def _resolve_receiver_budget(explicit, var_name: str): + if explicit is not None: + value = float(explicit) + if value <= 0: + raise ValueError(f"receiver budget must be positive, got {explicit}") + return value + try: + value = ConfigService.get_float_var(var_name, conf=SystemConfigs.APPLICATION_CONF, default=None) + except Exception: + return None + if value is None or value <= 0: + return None + return float(value) class _Ref: @@ -191,6 +244,14 @@ def __init__( self.obj = obj self.num_receivers_done = 0 self.receiver_statuses = {} + # producer-served terminal statuses awaiting the receiver's confirmation; only + # finalized (confirmed or legacy-served) statuses live in receiver_statuses + self._pending_confirms = {} + # unconditional per-receiver liveness (F3-3): receiver -> last activity timestamp, + # updated on every request regardless of whether a progress_cb is configured -- so a + # live receiver can no longer mask a stalled one behind the tx-wide last_active_time + self._receiver_activity = {} + self._created_time = time.time() self._downloaded_to_all_called = False self._receiver_progress = {} self._terminal_progress_state = None @@ -200,12 +261,24 @@ def mark_active(self): self.tx.mark_active() def obj_downloaded(self, to_receiver: str, status: str): + self._finalize_receiver(to_receiver, status) + + def _finalize_receiver(self, to_receiver: str, status: str, require_pending: bool = False) -> bool: # Status recording is guarded so terminal-outcome snapshots taken on the # monitor thread never observe a half-updated map; user callbacks run - # outside the lock. + # outside the lock. The whole decision (dedup, pending-guard, pending pop, + # record, all-done latch) is one critical section, so a duplicate serve can + # never resurrect a pending entry around a racing finalization. with self._progress_lock: if to_receiver in self.receiver_statuses: - return + return False + if require_pending and to_receiver not in self._pending_confirms: + # a legitimate confirmation always follows a provisional terminal serve on + # THIS incarnation of the ref; an unsolicited/stale confirm (e.g. delayed + # across a ref_id reuse) must not certify -- or poison -- this transfer + self.tx.logger.warning(f"dropping unsolicited confirmation from {to_receiver} for ref {self.rid}") + return False + self._pending_confirms.pop(to_receiver, None) self.receiver_statuses[to_receiver] = status self.num_receivers_done = len(self.receiver_statuses) @@ -235,11 +308,126 @@ def obj_downloaded(self, to_receiver: str, status: str): f"downloaded_to_all of {type(self.obj)} for ref {self.rid}", self.obj.downloaded_to_all, ) + return True + + def obj_served(self, to_receiver: str, status: str, expect_confirm: bool): + """Records the producer-served terminal status for a receiver. + + Legacy receivers (expect_confirm=False) finalize immediately: served EOF/ERROR is the + only truth available. Confirm-capable receivers are recorded as PROVISIONAL only -- the + receiver's confirmation (obj_confirmed) finalizes them. This is what makes accounting + retry-aware: while the record is provisional, a later serve for the same receiver + overwrites it -- a lost terminal reply healed by a retry is not stuck at the first + served status -- and the confirmation supersedes any provisional state (a receiver-side + finalization failure after the last chunk turns a served-EOF SUCCESS into a confirmed + FAILED). Once the receiver confirms, its status is final: a receiver that confirms + FAILED has given up (it confirms only on its own terminal exits). + """ + if not expect_confirm: + self.obj_downloaded(to_receiver, status) + return + with self._progress_lock: + if to_receiver in self.receiver_statuses: + # already finalized -- a late duplicate serve must not resurrect a provisional + return + self._pending_confirms[to_receiver] = status + + def obj_confirmed(self, to_receiver: str, status: str) -> bool: + """Records the receiver-confirmed terminal status. Receiver truth wins; first confirm is final. + + Accepted only when a provisional serve is pending for this receiver on THIS + incarnation of the ref -- unsolicited or stale confirmations are dropped, so a + delayed confirm from a previous life of a reused ref_id can neither falsely + certify nor pre-poison the new transfer. + """ + if status not in (DownloadStatus.SUCCESS, DownloadStatus.FAILED): + self.tx.logger.error(f"ignoring confirmation with invalid status '{status}' from {to_receiver}") + return False + accepted = self._finalize_receiver(to_receiver, status, require_pending=True) + if accepted: + # the receiver's truth is the terminal progress state for this receiver + self.emit_progress( + receiver_id=to_receiver, + state=( + TransferProgressState.COMPLETED + if status == DownloadStatus.SUCCESS + else TransferProgressState.FAILED + ), + force=True, + ) + return accepted def snapshot_receiver_statuses(self) -> dict: with self._progress_lock: return dict(self.receiver_statuses) + def snapshot_pending_confirms(self) -> dict: + with self._progress_lock: + return dict(self._pending_confirms) + + def mark_receiver_active(self, receiver: str): + with self._progress_lock: + self._receiver_activity[receiver] = time.time() + + def snapshot_receiver_activity(self) -> dict: + with self._progress_lock: + return dict(self._receiver_activity) + + def enforce_budgets( + self, now: float, acquire_timeout, idle_timeout, expected_receivers, tx_acquired_receivers=None + ) -> list: + """Finalizes FAILED for receivers whose acquire or idle budget is exhausted. + + A budget failure counts toward completion (via obj_downloaded), so the transaction's + aggregate outcome resolves on the next monitor pass instead of waiting for the whole + transaction TTL. This also bounds a lost fire-and-forget confirmation: the receiver + stops making requests after EOF, so its idle budget finalizes it FAILED (fail-closed). + + Returns: list of (receiver, reason) that were failed on this pass. + """ + failures = [] + with self._progress_lock: + final = set(self.receiver_statuses) + activity = dict(self._receiver_activity) + if idle_timeout is not None: + for receiver, last_active in activity.items(): + if receiver in final: + continue + idle = now - last_active + if idle > idle_timeout: + failures.append((receiver, f"idle budget exhausted: {idle:.1f}s > {idle_timeout}s")) + if acquire_timeout is not None and expected_receivers: + waited = now - self._created_time + if waited > acquire_timeout: + for receiver in expected_receivers: + if receiver in final or receiver in activity: + continue + if tx_acquired_receivers is not None and receiver in tx_acquired_receivers: + # acquired at TRANSACTION level: a receiver working through the + # transaction's refs sequentially must not be failed on refs it + # has not reached yet + continue + failures.append( + ( + receiver, + f"acquire budget exhausted: no pull within {acquire_timeout}s (waited {waited:.1f}s)", + ) + ) + enforced = [] + for receiver, reason in failures: + with self._progress_lock: + if receiver in self.receiver_statuses: + continue # finalized (e.g. confirmed) between snapshot and enforcement: truth wins + if self._receiver_activity.get(receiver) != activity.get(receiver): + continue # activity advanced past the snapshot: not actually idle + self._pending_confirms.pop(receiver, None) + if not self._finalize_receiver(receiver, DownloadStatus.FAILED): + continue + self.tx.logger.warning(f"receiver {receiver} failed for ref {self.rid}: {reason}") + self.emit_progress(receiver_id=receiver, state=TransferProgressState.FAILED, force=True) + enforced.append((receiver, reason)) + return enforced + def emit_progress( self, *, @@ -403,6 +591,10 @@ def __init__( progress_cb: Optional[Callable] = None, progress_interval: float = 30.0, outcome_cb: Optional[Callable] = None, + receiver_ids=None, + min_receivers: Optional[int] = None, + receiver_acquire_timeout: Optional[float] = None, + receiver_idle_timeout: Optional[float] = None, ): """Constructor of the transaction object. @@ -420,6 +612,30 @@ def __init__( else: self.tid = "T" + str(uuid.uuid4()) self.timeout = timeout + + # Expected receiver identities (F3-3). Optional: when provided they enable the acquire + # budget (a receiver that never issues its first pull can be failed) and, if + # num_receivers is unknown (0), supply the receiver count. + if receiver_ids: + receiver_ids = tuple(dict.fromkeys(str(r) for r in receiver_ids)) # dedup, keep order + if num_receivers and num_receivers != len(receiver_ids): + raise ValueError( + f"num_receivers ({num_receivers}) does not match receiver_ids count ({len(receiver_ids)})" + ) + num_receivers = len(receiver_ids) + self.receiver_ids = receiver_ids + else: + self.receiver_ids = None + if min_receivers is not None: + if min_receivers <= 0: + raise ValueError(f"min_receivers must be positive, got {min_receivers}") + if num_receivers and min_receivers > num_receivers: + raise ValueError(f"min_receivers ({min_receivers}) exceeds num_receivers ({num_receivers})") + self.min_receivers = min_receivers + self.receiver_acquire_timeout = _resolve_receiver_budget( + receiver_acquire_timeout, RECEIVER_ACQUIRE_TIMEOUT_CONFIG_VAR + ) + self.receiver_idle_timeout = _resolve_receiver_budget(receiver_idle_timeout, RECEIVER_IDLE_TIMEOUT_CONFIG_VAR) self.num_receivers = num_receivers self.last_active_time = time.time() self.start_time = time.time() @@ -486,6 +702,25 @@ def timed_out(self): """ self.transaction_done(TransactionDoneStatus.TIMEOUT) + def enforce_receiver_budgets(self, now: float): + """Evaluates per-receiver budgets across all refs; called by the monitor thread.""" + if self.receiver_acquire_timeout is None and self.receiver_idle_timeout is None: + return + refs = self.snapshot_refs() + # transaction-level acquisition: a first pull on ANY ref acquires the receiver + tx_acquired = set() + for ref in refs: + assert isinstance(ref, _Ref) + tx_acquired.update(ref.snapshot_receiver_activity().keys()) + for ref in refs: + ref.enforce_budgets( + now, + self.receiver_acquire_timeout, + self.receiver_idle_timeout, + self.receiver_ids, + tx_acquired_receivers=tx_acquired, + ) + def is_finished(self): """Check whether the transaction is finished (all objects are downloaded).""" if self.num_receivers <= 0: @@ -518,6 +753,7 @@ def transaction_done(self, status: str, on_outcome=None) -> TransferOutcome: tx_id=self.tid, done_status=status, num_receivers=self.num_receivers, + min_receivers=self.min_receivers, refs=[RefOutcome(ref_id=ref.rid, receiver_statuses=ref.snapshot_receiver_statuses()) for ref in refs], timestamp=time.time(), ) @@ -608,6 +844,66 @@ def __init__(self, tx: _Transaction): self.objects = [r.obj for r in tx.snapshot_refs()] +class TransferWaiter: + """The awaitable facade over a transaction's terminal transfer outcome (F3-4). + + This is the "returns == delivered" primitive the upper layers (executor backends, + trainer engine) consume: wait() blocks -- event-driven, no polling -- until the + transaction's aggregate TransferOutcome is recorded, and the outcome is COMPLETED only + when every expected receiver succeeded (receiver-confirmed where supported, budget- and + TTL-bounded). It attaches to the outcome-recording path directly, so it composes with -- + and never replaces -- transaction_done_cb / outcome_cb / the FOBS-context + DOWNLOAD_COMPLETE_CB chain. + """ + + def __init__(self, transaction_id: str, service=None): + self.transaction_id = transaction_id + self._service = service # the DownloadService class that created this waiter + self._event = threading.Event() + self._outcome: Optional[TransferOutcome] = None + + def _resolve(self, outcome: Optional[TransferOutcome]): + self._outcome = outcome + self._event.set() + + @property + def outcome(self) -> Optional[TransferOutcome]: + """The terminal outcome, or None while the transfer is still in flight.""" + return self._outcome + + def done(self) -> bool: + return self._event.is_set() + + def acquired_receivers(self) -> set: + """Receivers that have issued at least one pull (the PAYLOAD_ACQUIRED signal, V1).""" + service = self._service if self._service is not None else DownloadService + return service.get_acquired_receivers(self.transaction_id) + + def wait(self, timeout: Optional[float] = None, linger: Optional[float] = None) -> Optional[TransferOutcome]: + """Blocks until the terminal transfer outcome is recorded. + + Args: + timeout: max seconds to wait. None waits indefinitely (callers should normally + bound this; the transaction's own TTL and per-receiver budgets bound the + producer side). + linger: optional bounded post-completion linger, applied after any FINISHED + outcome (completed or not). By termination time the sources are already + released and the refs tombstoned; what the linger preserves is the PROCESS + (and with it the tombstone window), so a receiver whose terminal EOF/ERROR + reply was lost can still retry and be replayed its recorded status before + the producer exits. Timed-out/deleted outcomes get no linger. + + Returns: the TransferOutcome; None if the wait timed out (transfer still in flight) + or the service shut down before the transaction terminated. + """ + if not self._event.wait(timeout): + return None + outcome = self._outcome + if outcome is not None and linger and outcome.done_status == TransactionDoneStatus.FINISHED: + time.sleep(linger) + return outcome + + class DownloadService: _init_lock = threading.Lock() @@ -625,6 +921,9 @@ class DownloadService: # transaction that terminates concurrently with a same-id retry cannot record # its outcome over the new incarnation. Guarded by _outcome_lock. _tx_incarnations = {} + # Waiters blocked on a transaction's terminal outcome (the awaitable facade). Guarded by + # _outcome_lock; resolved inside _record_outcome so a waiter can never miss the outcome. + _tx_waiters = {} _outcome_lock = threading.Lock() TX_OUTCOME_TTL = 1800.0 _logger = None @@ -663,6 +962,10 @@ def new_transaction( progress_cb: Optional[Callable] = None, progress_interval: float = 30.0, outcome_cb: Optional[Callable] = None, + receiver_ids=None, + min_receivers: Optional[int] = None, + receiver_acquire_timeout: Optional[float] = None, + receiver_idle_timeout: Optional[float] = None, **cb_kwargs, ): cls._initialize(cell) @@ -675,6 +978,10 @@ def new_transaction( progress_cb=progress_cb, progress_interval=progress_interval, outcome_cb=outcome_cb, + receiver_ids=receiver_ids, + min_receivers=min_receivers, + receiver_acquire_timeout=receiver_acquire_timeout, + receiver_idle_timeout=receiver_idle_timeout, ) with cls._outcome_lock: # a reused explicit tx_id must not surface the previous incarnation's @@ -760,6 +1067,12 @@ def shutdown(cls): # the incarnation guard alone gates post-shutdown recording. cls._tx_outcomes.clear() cls._tx_incarnations.clear() + # unblock the awaitable facade: waiters resolve to None (service shut down + # before the transaction terminated), never hang + for waiters in cls._tx_waiters.values(): + for waiter in waiters: + waiter._resolve(None) + cls._tx_waiters.clear() with cls._init_lock: # Shutdown resets callback-registration state even when a cell is still @@ -782,6 +1095,43 @@ def _delete_tx(cls, tx: _Transaction, tombstone_finished_refs: bool = False): else: cls._finished_refs.pop(r.rid, None) + @classmethod + def get_transfer_waiter(cls, transaction_id: str) -> TransferWaiter: + """Returns an awaitable facade over the transaction's terminal outcome (F3-4). + + Safe to call before or after termination: a waiter created after the outcome was + recorded resolves immediately from the outcome table. + """ + waiter = TransferWaiter(transaction_id, service=cls) + with cls._outcome_lock: + existing = cls._tx_outcomes.get(transaction_id) + if existing is not None: + # resolve even from an expired record: it is still the recorded truth + waiter._resolve(existing) + return waiter + if transaction_id not in cls._tx_incarnations: + # unknown, already-forgotten (outcome expired) or shut down: nothing will + # ever record an outcome for this id, so resolving with None immediately is + # the only way to honor "waiters can never hang". Race-free: _record_outcome + # swaps incarnation -> outcome under this same lock. + waiter._resolve(None) + return waiter + cls._tx_waiters.setdefault(transaction_id, []).append(waiter) + return waiter + + @classmethod + def get_acquired_receivers(cls, transaction_id: str) -> set: + """Receivers that have issued at least one pull on any ref of the transaction.""" + with cls._tx_lock: + tx = cls._tx_table.get(transaction_id) + if tx is None: + return set() + assert isinstance(tx, _Transaction) + acquired = set() + for ref in tx.snapshot_refs(): + acquired.update(ref.snapshot_receiver_activity().keys()) + return acquired + @classmethod def _record_outcome(cls, outcome: TransferOutcome, tx: _Transaction): # tx is required so no call site can opt out of the incarnation guard: @@ -799,6 +1149,10 @@ def _record_outcome(cls, outcome: TransferOutcome, tx: _Transaction): return cls._tx_incarnations.pop(outcome.tx_id, None) cls._tx_outcomes[outcome.tx_id] = outcome + # resolve the awaitable facade: waiters are TransferWaiter objects (no user code + # runs in _resolve), so setting them under the lock is safe and race-free + for waiter in cls._tx_waiters.pop(outcome.tx_id, ()): + waiter._resolve(outcome) @classmethod def get_transaction_outcome(cls, transaction_id: str) -> Optional[TransferOutcome]: @@ -876,6 +1230,10 @@ def _handle_download(cls, request: Message) -> Message: cls._logger.error(f"missing {_PropKey.REF_ID} in request from {requester}") return make_reply(ReturnCode.INVALID_REQUEST) + confirm_status = payload.get(_PropKey.CONFIRM) + if confirm_status is not None: + return cls._handle_confirm(rid, requester, confirm_status) + current_state = payload.get(_PropKey.STATE) with cls._tx_lock: ref = cls._ref_table.get(rid) @@ -893,10 +1251,15 @@ def _handle_download(cls, request: Message) -> Message: assert isinstance(ref, _Ref) ref.mark_active() + ref.mark_receiver_active(requester) ref.emit_progress(receiver_id=requester, state=TransferProgressState.ACTIVE) tx = ref.tx assert isinstance(tx, _Transaction) + # receiver-confirmed completion is armed only when the receiver advertised the + # capability on this request AND the local kill-switch is on + expect_confirm = bool(payload.get(_PropKey.CONFIRM_CAPABLE)) and _receiver_confirm_enabled() + # Keep produce() outside the global transaction lock so slow chunk generation # does not block unrelated downloads. Timeout/delete cleanup can release the # source concurrently; if that happens, the produce exception is reported as @@ -911,16 +1274,28 @@ def _handle_download(cls, request: Message) -> Message: return make_reply(ReturnCode.PROCESS_EXCEPTION) if rc != ProduceRC.OK: - # already done - ref.obj_downloaded( - requester, status=DownloadStatus.SUCCESS if rc == ProduceRC.EOF else DownloadStatus.FAILED - ) - ref.emit_progress( - receiver_id=requester, - state=TransferProgressState.COMPLETED if rc == ProduceRC.EOF else TransferProgressState.FAILED, - force=True, + # already done -- for a confirm-capable receiver this record is PROVISIONAL and the + # receiver's confirmation finalizes it; for a legacy receiver it is final (today's + # producer-served semantics) + ref.obj_served( + requester, + status=DownloadStatus.SUCCESS if rc == ProduceRC.EOF else DownloadStatus.FAILED, + expect_confirm=expect_confirm, ) - return make_reply(ReturnCode.OK, body={_PropKey.STATUS: rc}) + if expect_confirm: + # provisional: the receiver's confirmation carries the terminal truth -- + # do not latch a terminal progress state the confirm may contradict + ref.emit_progress(receiver_id=requester, state=TransferProgressState.ACTIVE, force=True) + else: + ref.emit_progress( + receiver_id=requester, + state=TransferProgressState.COMPLETED if rc == ProduceRC.EOF else TransferProgressState.FAILED, + force=True, + ) + body = {_PropKey.STATUS: rc} + if expect_confirm: + body[_PropKey.CONFIRM_EXPECTED] = True + return make_reply(ReturnCode.OK, body=body) else: # continue — accumulate bytes for timing summary in transaction_done() # CacheableObject returns a list of byte-chunks; FileDownloader returns raw bytes. @@ -935,19 +1310,53 @@ def _handle_download(cls, request: Message) -> Message: bytes_delta=bytes_delta, items_delta=items_delta, ) - return make_reply( - ReturnCode.OK, - body={ - _PropKey.STATUS: rc, - _PropKey.STATE: new_state, - _PropKey.DATA: data, - }, - ) + body = { + _PropKey.STATUS: rc, + _PropKey.STATE: new_state, + _PropKey.DATA: data, + } + if expect_confirm: + body[_PropKey.CONFIRM_EXPECTED] = True + return make_reply(ReturnCode.OK, body=body) + + @classmethod + def _handle_confirm(cls, rid: str, requester: str, status: str) -> Message: + with cls._tx_lock: + ref = cls._ref_table.get(rid) + if ref is None: + # the transaction already terminated/cleaned up: its outcome was computed from what + # was known then (fail-closed for unconfirmed receivers); a late confirm is dropped + cls._logger.debug(f"late confirmation for unknown ref {rid} from {requester} dropped") + return make_reply(ReturnCode.OK) + assert isinstance(ref, _Ref) + # deliberately no unconditional mark_active/mark_receiver_active: a stale or + # unsolicited confirm must not extend the transaction TTL nor reset idle budgets + if ref.obj_confirmed(requester, status): + ref.mark_active() + return make_reply(ReturnCode.OK) @classmethod def _monitor_tx(cls): while True: now = time.time() + + # Per-receiver budget enforcement (F3-3) runs OUTSIDE _tx_lock: finalizing a + # budget-failed receiver fires user callbacks (downloaded_to_one/all), which must + # never run under the global lock. A budget failure recorded here flips + # is_finished() so the classification pass below resolves the tx immediately. + with cls._tx_lock: + budget_txs = list(cls._tx_table.values()) + for tx in budget_txs: + with cls._tx_lock: + if cls._tx_table.get(tx.tid) is not tx: + continue # deleted/replaced since the snapshot: do not touch a dead tx + try: + tx.enforce_receiver_budgets(now) + except Exception as ex: + cls._logger.error( + f"error enforcing receiver budgets for tx {tx.tid}: {secure_format_exception(ex)}" + ) + expired_tx = [] finished_tx = [] with cls._tx_lock: @@ -1076,6 +1485,32 @@ def download_object( # On retry, resend the same state so producer re-generates the same chunk. current_state = None + # Receiver-confirmed completion: we advertise the capability on every request (when the + # kill-switch is on) and learn from each reply whether the producer consumes confirmations. + confirm_enabled = _receiver_confirm_enabled() + producer_expects_confirm = False + + def _send_confirm(receiver_truth: str): + # wire contract: a confirmation is sent ONLY after a producer-served terminal reply + # (EOF/ERROR) -- the producer accepts a confirm only against its pending provisional + # serve, so mid-stream failure exits do not confirm (budgets/TTL handle those) + if not (confirm_enabled and producer_expects_confirm): + return + try: + # fire-and-forget by design: a lost confirmation is backstopped producer-side by + # per-receiver budgets / the transaction timeout, failing closed + cell.fire_and_forget( + channel=OBJ_DOWNLOADER_CHANNEL, + topic=OBJ_DOWNLOADER_TOPIC, + targets=from_fqcn, + message=new_cell_message( + headers={}, payload={_PropKey.REF_ID: ref_id, _PropKey.CONFIRM: receiver_truth} + ), + optional=optional, + ) + except Exception as ex: + logger.warning(f"failed to send download confirmation for ref={ref_id}: {secure_format_exception(ex)}") + def _emit_progress(state: str, force: bool = False): nonlocal progress_sequence, last_progress_emit_time if not progress_cb: @@ -1105,6 +1540,8 @@ def _emit_progress(state: str, force: bool = False): # Build a fresh request each iteration (including retries) # to avoid re-encoding an already-encoded message. request_payload = {_PropKey.REF_ID: ref_id} + if confirm_enabled: + request_payload[_PropKey.CONFIRM_CAPABLE] = True if current_state is not None: request_payload[_PropKey.STATE] = current_state request = new_cell_message(headers={}, payload=request_payload) @@ -1172,6 +1609,8 @@ def _emit_progress(state: str, force: bool = False): payload = reply.payload assert isinstance(payload, dict) + if payload.get(_PropKey.CONFIRM_EXPECTED): + producer_expects_confirm = True status = payload.get(_PropKey.STATUS) if status == ProduceRC.EOF: elapsed = time.time() - download_start @@ -1180,10 +1619,20 @@ def _emit_progress(state: str, force: bool = False): f"[client] download ref={ref_id} done: elapsed={elapsed:.2f}s " f"size={size_mb:.1f}MB ({total_bytes:,} bytes)" ) - consumer.download_completed(ref_id) + try: + consumer.download_completed(ref_id) + except Exception: + # receiver-side finalization failed AFTER the last chunk (e.g. disk-offload + # finalize): exactly what receiver-confirmed completion exists to surface -- + # the producer must not certify this receiver on its served EOF + _send_confirm(DownloadStatus.FAILED) + _emit_progress("failed", force=True) + raise + _send_confirm(DownloadStatus.SUCCESS) _emit_progress("completed", force=True) return elif status == ProduceRC.ERROR: + _send_confirm(DownloadStatus.FAILED) consumer.download_failed(ref_id, f"producer error after {duration} secs") _emit_progress("failed", force=True) return diff --git a/nvflare/fuel/f3/streaming/obj_downloader.py b/nvflare/fuel/f3/streaming/obj_downloader.py index 7fdb7ad8ca..c2b1ea68f5 100644 --- a/nvflare/fuel/f3/streaming/obj_downloader.py +++ b/nvflare/fuel/f3/streaming/obj_downloader.py @@ -28,6 +28,10 @@ def __init__( progress_cb=None, progress_interval: float = 30.0, outcome_cb=None, + receiver_ids=None, + min_receivers=None, + receiver_acquire_timeout=None, + receiver_idle_timeout=None, **cb_kwargs, ): """Constructor of ObjectDownloader. @@ -67,6 +71,10 @@ def __init__( progress_cb=progress_cb, progress_interval=progress_interval, outcome_cb=outcome_cb, + receiver_ids=receiver_ids, + min_receivers=min_receivers, + receiver_acquire_timeout=receiver_acquire_timeout, + receiver_idle_timeout=receiver_idle_timeout, **cb_kwargs, ) @@ -87,6 +95,15 @@ def add_object(self, obj: Downloadable, ref_id=None) -> str: ) return rid + def get_waiter(self): + """Returns the awaitable facade (TransferWaiter) over this transaction's terminal outcome. + + waiter.wait(timeout) blocks until the aggregate TransferOutcome is recorded -- + COMPLETED only when every expected receiver succeeded. This is the primitive + upper layers use to guarantee "send returns only when the payload is delivered". + """ + return DownloadService.get_transfer_waiter(self.tx_id) + def delete_transaction(self): """Delete the download transaction forcefully. You call this method only if you want to stop the downloading process prematurely. diff --git a/nvflare/fuel/f3/streaming/transfer_outcome.py b/nvflare/fuel/f3/streaming/transfer_outcome.py index beb1f1b06c..d926119b17 100644 --- a/nvflare/fuel/f3/streaming/transfer_outcome.py +++ b/nvflare/fuel/f3/streaming/transfer_outcome.py @@ -31,13 +31,15 @@ Outcome status values reuse the TransferProgressState terminal vocabulary (completed / failed / aborted) rather than introducing another status set. -Known limits, resolved by later PRs of the same design (see -docs/design/client_api_execution_modes_plan.md): -- per-receiver statuses are producer-served (recorded when produce() returns EOF), - not receiver-confirmed — a receiver-side finalization failure after the last chunk - is not visible here until receiver-confirmed completion (plan PR F3-2) lands; -- num_receivers is a count without receiver identity — expected-receiver identity - checks arrive with per-receiver budgets (plan PR F3-3); +Semantics with the full payload layer (F3-2/F3-3/F3-4, same design): +- per-receiver statuses are receiver-confirmed where the receiver supports it (a served + EOF is provisional until the receiver confirms its finalization succeeded); legacy + receivers remain producer-served — both skews and the runtime kill-switch degrade to + producer-served semantics (download_service.py, receiver-confirmed completion); +- expected receiver identities and per-receiver acquire/idle budgets bound the outcome's + resolution time (a stalled or never-pulling receiver is finalized FAILED without + waiting the whole-transaction TTL); min_receivers surfaces the optional k-of-N quorum + via quorum_met while `completed` stays the strict all-receivers certificate; - the outcome covers the refs present at termination; adding objects to a transaction after receivers already finished the earlier ones is not supported. """ @@ -92,8 +94,10 @@ class TransferOutcomeReason: class RefOutcome: """Per-object terminal outcome. - receiver_statuses maps receiver FQCN to a DownloadStatus value (success / failed) - as recorded by the producer side. It is deep-frozen at construction (a + receiver_statuses maps receiver FQCN to a DownloadStatus value (success / failed) -- + receiver-confirmed where the receiver supports it, producer-served for legacy peers + or when the receiver-confirm kill-switch is off, and budget-FAILED for receivers that + exhausted their acquire/idle budget. It is deep-frozen at construction (a MappingProxyType over a private copy): the same instance is recorded in the service outcome table and handed to outcome_cb consumers, so a callback must not be able to mutate the recorded per-receiver truth. If outcomes ever cross @@ -126,6 +130,9 @@ class TransferOutcome: num_receivers: int refs: Tuple[RefOutcome, ...] timestamp: float + # optional k-of-N quorum declared by the workflow (F3-3): informational for quorum_met; + # `completed` deliberately stays the strict all-receivers certificate + min_receivers: Optional[int] = None def __post_init__(self): # frozen=True only blocks attribute rebinding; freeze the container too so @@ -136,6 +143,28 @@ def __post_init__(self): def completed(self) -> bool: return self.status == TransferProgressState.COMPLETED + @property + def quorum_met(self) -> bool: + """True if every ref reached at least min_receivers confirmed successes. + + The k-of-N surface for fan-out workflows (min_responses-style policies): `completed` + stays the strict all-receivers certificate; a workflow that accepts partial fan-out + checks quorum_met (or thresholds refs itself). Falls back to `completed` when no + min_receivers was declared; fails closed with no refs. + """ + if self.min_receivers is None: + return self.completed + if not self.refs: + return False + # a receiver counts toward the quorum only if it succeeded on EVERY ref: counting + # per-ref successes independently would let different receiver subsets satisfy each + # ref while no single receiver holds the complete payload + quorum_receivers = None + for r in self.refs: + ref_successes = {rcv for rcv, v in r.receiver_statuses.items() if v == DownloadStatus.SUCCESS} + quorum_receivers = ref_successes if quorum_receivers is None else quorum_receivers & ref_successes + return len(quorum_receivers) >= self.min_receivers + def expired(self, now: float, ttl: float) -> bool: return now - self.timestamp > ttl @@ -164,6 +193,7 @@ def compute_transfer_outcome( num_receivers: int, refs: Sequence[RefOutcome], timestamp: Optional[float] = None, + min_receivers: Optional[int] = None, ) -> TransferOutcome: """Compute the aggregate terminal outcome for a terminated transaction. @@ -213,4 +243,5 @@ def compute_transfer_outcome( num_receivers=num_receivers, refs=refs, timestamp=timestamp, + min_receivers=min_receivers, ) diff --git a/nvflare/fuel/utils/fobs/decomposers/via_downloader.py b/nvflare/fuel/utils/fobs/decomposers/via_downloader.py index a90e1caddb..00b578a636 100644 --- a/nvflare/fuel/utils/fobs/decomposers/via_downloader.py +++ b/nvflare/fuel/utils/fobs/decomposers/via_downloader.py @@ -380,7 +380,9 @@ def decompose(self, target: Any, manager: DatumManager = None) -> Any: self.logger.debug(f"ViaDownloader: created ref for target {target_id}: {item_id}") return {EncKey.TYPE: EncType.REF, EncKey.DATA: item_id} - def _create_downloader(self, fobs_ctx: dict, progress_cb=None, timeout_override=None, num_receivers_override=None): + def _create_downloader( + self, fobs_ctx: dict, progress_cb=None, timeout_override=None, num_receivers_override=None, receiver_ids=None + ): # Transaction lifecycle is managed solely by _monitor_tx() (download_service.py). # We deliberately do NOT subscribe to msg_root deletion here. The msg_root is # deleted as soon as all blobs are delivered, but blob_cb fires asynchronously — @@ -445,6 +447,9 @@ def _create_downloader(self, fobs_ctx: dict, progress_cb=None, timeout_override= transaction_done_cb=on_complete_cb, progress_cb=progress_cb, progress_interval=RESULT_UPLOAD_PROGRESS_INTERVAL, + # expected receiver identities (F3-3): enables the transaction's per-receiver + # acquire budget; None when any identity is unknown + receiver_ids=receiver_ids, ) return downloader @@ -603,11 +608,15 @@ def _finalize_download_tx(self, mgr: DatumManager): progress_context = fobs_ctx.get(RESULT_UPLOAD_PROGRESS_CTX_KEY) or {} timeout_override = progress_context.get(ResultUploadProgressContextKey.STREAMING_IDLE_TIMEOUT) + # forward receiver identities to the transaction only when every identity is + # actually known (a (None,) placeholder means unknown-single-receiver) + known_receiver_ids = receiver_ids if receiver_ids and all(r is not None for r in receiver_ids) else None downloader = self._create_downloader( fobs_ctx, progress_cb=progress_cb if progress_trackable else None, timeout_override=timeout_override, num_receivers_override=download_num_receivers, + receiver_ids=known_receiver_ids, ) if downloader is None: self.logger.warning("download transaction was not created because FOBS context has no cell") diff --git a/tests/unit_test/fuel/f3/streaming/download_test_utils.py b/tests/unit_test/fuel/f3/streaming/download_test_utils.py index 51a83f7f0c..688521c584 100644 --- a/tests/unit_test/fuel/f3/streaming/download_test_utils.py +++ b/tests/unit_test/fuel/f3/streaming/download_test_utils.py @@ -84,6 +84,7 @@ class IsolatedDownloadService(DownloadService): _finished_refs = {} _tx_outcomes = {} _tx_incarnations = {} + _tx_waiters = {} _outcome_lock = threading.Lock() _logger = Mock() _tx_lock = threading.Lock() diff --git a/tests/unit_test/fuel/f3/streaming/receiver_budget_test.py b/tests/unit_test/fuel/f3/streaming/receiver_budget_test.py new file mode 100644 index 0000000000..49bff85ab0 --- /dev/null +++ b/tests/unit_test/fuel/f3/streaming/receiver_budget_test.py @@ -0,0 +1,348 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for per-(transfer, receiver) acquire/idle budgets and the quorum surface (plan: F3-3). + +A receiver that exhausts its acquire budget (never issued its first pull) or idle budget +(stopped making requests) is finalized FAILED for the aggregate outcome on a monitor pass -- +without waiting for the whole-transaction TTL, and without a live receiver masking a stalled +one behind the tx-wide activity timestamp. Budgets are per-transaction opt-in (config-var +defaults); disabled budgets preserve today's TTL-only behavior exactly. +""" + +import time +from unittest.mock import Mock, patch + +import pytest + +from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey +from nvflare.fuel.f3.cellnet.utils import new_cell_message +from nvflare.fuel.f3.streaming import download_service as ds_module +from nvflare.fuel.f3.streaming.download_service import DownloadStatus, ProduceRC, _PropKey +from nvflare.fuel.f3.streaming.transfer_outcome import TransferOutcomeReason, compute_transfer_outcome +from tests.unit_test.fuel.f3.streaming.download_test_utils import ( + MockDownloadable, + make_isolated_download_service, + run_monitor_once, +) + + +@pytest.fixture(autouse=True) +def confirm_switch_on(): + with patch.object(ds_module, "_receiver_confirm_cached", True): + yield + + +def _make_service(): + service = make_isolated_download_service() + service._tx_monitor = Mock() + return service + + +def _new_tx(service, chunks=1, **tx_kwargs): + tx_kwargs.setdefault("timeout", 1000.0) # the whole-tx TTL is deliberately huge: + tx_kwargs.setdefault("num_receivers", 1) # budgets, not the TTL, must resolve these tests + tx_id = service.new_transaction(cell=Mock(), **tx_kwargs) + obj = MockDownloadable([b"chunk"] * chunks) + rid = service.add_object(tx_id, obj) + return tx_id, rid + + +def _pull_once(service, rid, requester, confirm_capable=False, state=None): + payload = {_PropKey.REF_ID: rid} + if confirm_capable: + payload[_PropKey.CONFIRM_CAPABLE] = True + if state is not None: + payload[_PropKey.STATE] = state + request = new_cell_message(headers={MessageHeaderKey.ORIGIN: requester}, payload=payload) + return service._handle_download(request) + + +def _pull_to_terminal(service, rid, requester, confirm_capable=False): + state = None + for _ in range(50): + reply = _pull_once(service, rid, requester, confirm_capable=confirm_capable, state=state) + status = reply.payload.get(_PropKey.STATUS) + if status in (ProduceRC.EOF, ProduceRC.ERROR): + return reply + state = reply.payload.get(_PropKey.STATE) + raise AssertionError("pull loop never reached terminal") + + +class TestIdleBudget: + def test_stalled_receiver_fails_without_waiting_tx_ttl(self): + service = _make_service() + tx_id, rid = _new_tx(service, chunks=3, num_receivers=2, receiver_idle_timeout=5.0) + + _pull_to_terminal(service, rid, "healthy") # legacy receiver, final at serve + _pull_once(service, rid, "stalled") # one pull, then silence + + # a monitor pass past the idle budget (but far inside the 1000s TTL) + run_monitor_once(service, now=time.time() + 30.0) + + outcome = service.get_transaction_outcome(tx_id) + assert outcome is not None, "budget failure must resolve the tx on this pass, not at TTL" + assert not outcome.completed + assert outcome.reason == TransferOutcomeReason.RECEIVER_FAILED + statuses = outcome.refs[0].receiver_statuses + assert statuses["healthy"] == DownloadStatus.SUCCESS + assert statuses["stalled"] == DownloadStatus.FAILED + + def test_lost_confirmation_is_bounded_by_idle_budget(self): + # fire-and-forget confirms can be lost: the receiver stops making requests after its + # served EOF, so its idle budget finalizes it FAILED in bounded time (fail-closed) + service = _make_service() + tx_id, rid = _new_tx(service, receiver_idle_timeout=5.0) + _pull_to_terminal(service, rid, "r1", confirm_capable=True) # provisional, confirm never arrives + + run_monitor_once(service, now=time.time() + 30.0) + + outcome = service.get_transaction_outcome(tx_id) + assert outcome is not None + assert not outcome.completed + assert outcome.refs[0].receiver_statuses == {"r1": DownloadStatus.FAILED} + # the provisional record was cleaned up, not leaked + assert service._tx_outcomes # outcome recorded; ref is gone with the finished tx + + def test_active_receiver_is_not_idle_failed(self): + service = _make_service() + tx_id, rid = _new_tx(service, chunks=3, receiver_idle_timeout=5.0) + reply = _pull_once(service, rid, "r1") + ref = service._ref_table[rid] + # receiver keeps making requests: refresh activity to "now" as seen by the monitor + future = time.time() + 30.0 + with ref._progress_lock: + ref._receiver_activity["r1"] = future - 1.0 + + run_monitor_once(service, now=future) + + assert service.get_transaction_outcome(tx_id) is None # still live + assert ref.snapshot_receiver_statuses() == {} + + +class TestAcquireBudget: + def test_never_pulling_receiver_fails_at_acquire_deadline(self): + service = _make_service() + tx_id, rid = _new_tx( + service, + num_receivers=0, # derived from receiver_ids + receiver_ids=("r1", "r2"), + receiver_acquire_timeout=5.0, + min_receivers=1, + ) + _pull_to_terminal(service, rid, "r1") # r2 never shows up + + run_monitor_once(service, now=time.time() + 30.0) + + outcome = service.get_transaction_outcome(tx_id) + assert outcome is not None + statuses = outcome.refs[0].receiver_statuses + assert statuses == {"r1": DownloadStatus.SUCCESS, "r2": DownloadStatus.FAILED} + # strict certificate fails; the declared k-of-N quorum is satisfied + assert not outcome.completed + assert outcome.quorum_met + assert outcome.min_receivers == 1 + + def test_receiver_with_activity_is_not_acquire_failed(self): + service = _make_service() + tx_id, rid = _new_tx(service, chunks=3, num_receivers=0, receiver_ids=("r1",), receiver_acquire_timeout=5.0) + _pull_once(service, rid, "r1") # first pull happened: acquire satisfied + + run_monitor_once(service, now=time.time() + 30.0) + + # no idle budget configured, so the slow-but-acquired receiver is left alone + assert service.get_transaction_outcome(tx_id) is None + assert service._ref_table[rid].snapshot_receiver_statuses() == {} + + def test_acquire_budget_needs_receiver_identities(self): + # without receiver_ids there is no one to hold to the acquire deadline + service = _make_service() + tx_id, rid = _new_tx(service, num_receivers=2, receiver_acquire_timeout=5.0) + + run_monitor_once(service, now=time.time() + 30.0) + + assert service.get_transaction_outcome(tx_id) is None + assert service._ref_table[rid].snapshot_receiver_statuses() == {} + + +class TestBudgetSemantics: + def test_budgets_disabled_preserve_ttl_only_behavior(self): + service = _make_service() + tx_id, rid = _new_tx(service, num_receivers=2) + _pull_once(service, rid, "r1") + + run_monitor_once(service, now=time.time() + 500.0) # inside the 1000s TTL + + assert service.get_transaction_outcome(tx_id) is None + assert service._ref_table[rid].snapshot_receiver_statuses() == {} + + def test_activity_tracked_without_progress_cb(self): + service = _make_service() + tx_id, rid = _new_tx(service, chunks=2) + _pull_once(service, rid, "r1") + + activity = service._ref_table[rid].snapshot_receiver_activity() + assert "r1" in activity + + def test_budget_failed_receiver_is_final_even_against_late_confirm(self): + service = _make_service() + tx_id, rid = _new_tx(service, num_receivers=2, receiver_idle_timeout=5.0) + _pull_to_terminal(service, rid, "r1", confirm_capable=True) + ref = service._ref_table[rid] + + failures = ref.enforce_budgets(time.time() + 30.0, None, 5.0, None) + assert [r for r, _ in failures] == ["r1"] + assert ref.snapshot_pending_confirms() == {} + + # the straggler confirmation cannot resurrect the budget-failed receiver + ref.obj_confirmed("r1", DownloadStatus.SUCCESS) + assert ref.snapshot_receiver_statuses()["r1"] == DownloadStatus.FAILED + + +class TestMultiRefAcquisition: + def test_sequential_multi_ref_receiver_is_not_acquire_failed(self): + # transaction-level acquisition: a healthy receiver pulling ref 1 must not be + # acquire-failed on ref 2 it has not reached yet + service = _make_service() + tx_id = service.new_transaction( + cell=Mock(), timeout=1000.0, num_receivers=0, receiver_ids=("r1",), receiver_acquire_timeout=5.0 + ) + rid1 = service.add_object(tx_id, MockDownloadable([b"chunk"] * 3)) + rid2 = service.add_object(tx_id, MockDownloadable([b"chunk"] * 3)) + _pull_once(service, rid1, "r1") # busy on ref 1, has not touched ref 2 + + run_monitor_once(service, now=time.time() + 30.0) + + assert service.get_transaction_outcome(tx_id) is None # still live + assert service._ref_table[rid2].snapshot_receiver_statuses() == {} + + def test_confirmed_receiver_wins_over_stale_budget_snapshot(self): + # truth-wins re-check: a receiver finalized (confirmed) between the budget snapshot + # and enforcement must not be flipped to FAILED, and no failure is reported for it + service = _make_service() + tx_id, rid = _new_tx(service, receiver_idle_timeout=5.0) + _pull_to_terminal(service, rid, "r1", confirm_capable=True) + ref = service._ref_table[rid] + service._handle_download( + new_cell_message( + headers={MessageHeaderKey.ORIGIN: "r1"}, + payload={_PropKey.REF_ID: rid, _PropKey.CONFIRM: DownloadStatus.SUCCESS}, + ) + ) + + enforced = ref.enforce_budgets(time.time() + 30.0, None, 5.0, None) + + assert enforced == [] + assert ref.snapshot_receiver_statuses() == {"r1": DownloadStatus.SUCCESS} + + +class TestConfigResolution: + def test_budget_config_var_supplies_default(self): + service = _make_service() + with patch.object(ds_module.ConfigService, "get_float_var", return_value=42.0) as gv: + tx_id = service.new_transaction(cell=Mock(), timeout=10.0, num_receivers=1) + tx = service._tx_table[tx_id] + assert tx.receiver_idle_timeout == 42.0 + assert tx.receiver_acquire_timeout == 42.0 + assert gv.call_args.kwargs.get("conf") == ds_module.SystemConfigs.APPLICATION_CONF + + def test_explicit_budget_overrides_config_var(self): + service = _make_service() + with patch.object(ds_module.ConfigService, "get_float_var", return_value=42.0): + tx_id = service.new_transaction(cell=Mock(), timeout=10.0, num_receivers=1, receiver_idle_timeout=7.0) + assert service._tx_table[tx_id].receiver_idle_timeout == 7.0 + + def test_non_positive_config_var_disables_budget(self): + service = _make_service() + with patch.object(ds_module.ConfigService, "get_float_var", return_value=0.0): + tx_id = service.new_transaction(cell=Mock(), timeout=10.0, num_receivers=1) + tx = service._tx_table[tx_id] + assert tx.receiver_idle_timeout is None + assert tx.receiver_acquire_timeout is None + + +class TestTransactionValidation: + def test_receiver_ids_derive_num_receivers(self): + service = _make_service() + tx_id = service.new_transaction(cell=Mock(), timeout=10.0, num_receivers=0, receiver_ids=("a", "b", "a")) + tx = service._tx_table[tx_id] + assert tx.receiver_ids == ("a", "b") # deduped, order kept + assert tx.num_receivers == 2 + + def test_receiver_ids_num_receivers_mismatch_raises(self): + service = _make_service() + with pytest.raises(ValueError, match="does not match"): + service.new_transaction(cell=Mock(), timeout=10.0, num_receivers=3, receiver_ids=("a", "b")) + + def test_min_receivers_validation(self): + service = _make_service() + with pytest.raises(ValueError, match="must be positive"): + service.new_transaction(cell=Mock(), timeout=10.0, num_receivers=2, min_receivers=0) + with pytest.raises(ValueError, match="exceeds num_receivers"): + service.new_transaction(cell=Mock(), timeout=10.0, num_receivers=2, min_receivers=3) + + def test_negative_budget_rejected(self): + service = _make_service() + with pytest.raises(ValueError, match="must be positive"): + service.new_transaction(cell=Mock(), timeout=10.0, num_receivers=1, receiver_idle_timeout=-1.0) + + +class TestQuorumSurface: + def _refs_outcome(self, statuses, min_receivers, num_receivers=2): + from nvflare.fuel.f3.streaming.transfer_outcome import RefOutcome, TransactionDoneStatus + + refs = [RefOutcome(ref_id="R1", receiver_statuses=statuses)] + return compute_transfer_outcome( + "T1", TransactionDoneStatus.FINISHED, num_receivers, refs, 100.0, min_receivers=min_receivers + ) + + def test_quorum_met_with_partial_fanout(self): + outcome = self._refs_outcome({"a": DownloadStatus.SUCCESS, "b": DownloadStatus.FAILED}, min_receivers=1) + assert not outcome.completed # strict certificate unchanged + assert outcome.quorum_met + + def test_quorum_not_met(self): + outcome = self._refs_outcome({"a": DownloadStatus.SUCCESS, "b": DownloadStatus.FAILED}, min_receivers=2) + assert not outcome.quorum_met + + def test_quorum_falls_back_to_completed_when_unset(self): + success = self._refs_outcome({"a": DownloadStatus.SUCCESS, "b": DownloadStatus.SUCCESS}, min_receivers=None) + assert success.completed and success.quorum_met + partial = self._refs_outcome({"a": DownloadStatus.SUCCESS, "b": DownloadStatus.FAILED}, min_receivers=None) + assert not partial.quorum_met + + def test_quorum_requires_same_receiver_across_all_refs(self): + # disjoint success subsets per ref must NOT satisfy the quorum: no single receiver + # holds the complete (multi-ref) payload + from nvflare.fuel.f3.streaming.transfer_outcome import RefOutcome, TransactionDoneStatus + + refs = [ + RefOutcome(ref_id="R1", receiver_statuses={"a": DownloadStatus.SUCCESS, "b": DownloadStatus.FAILED}), + RefOutcome(ref_id="R2", receiver_statuses={"a": DownloadStatus.FAILED, "b": DownloadStatus.SUCCESS}), + ] + outcome = compute_transfer_outcome("T1", TransactionDoneStatus.FINISHED, 2, refs, 100.0, min_receivers=1) + assert not outcome.quorum_met # each ref has one success, but no common receiver + + refs2 = [ + RefOutcome(ref_id="R1", receiver_statuses={"a": DownloadStatus.SUCCESS, "b": DownloadStatus.FAILED}), + RefOutcome(ref_id="R2", receiver_statuses={"a": DownloadStatus.SUCCESS, "b": DownloadStatus.FAILED}), + ] + outcome2 = compute_transfer_outcome("T1", TransactionDoneStatus.FINISHED, 2, refs2, 100.0, min_receivers=1) + assert outcome2.quorum_met # receiver "a" holds the complete payload + + def test_quorum_fails_closed_with_no_refs(self): + from nvflare.fuel.f3.streaming.transfer_outcome import TransactionDoneStatus + + outcome = compute_transfer_outcome("T1", TransactionDoneStatus.FINISHED, 2, [], 100.0, min_receivers=1) + assert not outcome.quorum_met diff --git a/tests/unit_test/fuel/f3/streaming/receiver_confirm_test.py b/tests/unit_test/fuel/f3/streaming/receiver_confirm_test.py new file mode 100644 index 0000000000..2848bbee8b --- /dev/null +++ b/tests/unit_test/fuel/f3/streaming/receiver_confirm_test.py @@ -0,0 +1,408 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for receiver-confirmed completion + retry-aware accounting (plan: F3-2). + +Producer side: a confirm-capable receiver's served EOF/ERROR is PROVISIONAL; the receiver's +confirmation finalizes it (receiver truth wins, first confirm is final, retries overwrite +provisional state). Legacy receivers keep today's producer-served semantics -- both version +skews degrade to the current behavior, and a runtime kill-switch disables the wire behavior +entirely without a code revert. + +Receiver side: capability is advertised per request, confirmations are sent fire-and-forget +only toward producers that advertised they consume them, and the receiver truth is decided by +Consumer.download_completed() (finalization), not by the served EOF. +""" + +import time +from unittest.mock import Mock, patch + +import pytest + +from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey, ReturnCode +from nvflare.fuel.f3.cellnet.utils import make_reply, new_cell_message +from nvflare.fuel.f3.streaming import download_service as ds_module +from nvflare.fuel.f3.streaming.download_service import Consumer, DownloadStatus, ProduceRC, _PropKey, download_object +from tests.unit_test.fuel.f3.streaming.download_test_utils import ( + MockDownloadable, + make_isolated_download_service, + run_monitor_once, +) + + +@pytest.fixture(autouse=True) +def confirm_switch_on(): + """Pin the kill-switch ON for tests (individual tests patch it OFF explicitly).""" + with patch.object(ds_module, "_receiver_confirm_cached", True): + yield + + +def _make_service(): + service = make_isolated_download_service() + service._tx_monitor = Mock() # suppress the real monitor thread + return service + + +def _new_tx(service, num_receivers=1, timeout=10.0, chunks=1): + tx_id = service.new_transaction(cell=Mock(), timeout=timeout, num_receivers=num_receivers) + obj = MockDownloadable([b"chunk"] * chunks) + rid = service.add_object(tx_id, obj) + return tx_id, rid, obj + + +def _pull_request(rid, requester, confirm_capable=False, state=None): + payload = {_PropKey.REF_ID: rid} + if confirm_capable: + payload[_PropKey.CONFIRM_CAPABLE] = True + if state is not None: + payload[_PropKey.STATE] = state + return new_cell_message(headers={MessageHeaderKey.ORIGIN: requester}, payload=payload) + + +def _confirm_request(rid, requester, status): + return new_cell_message( + headers={MessageHeaderKey.ORIGIN: requester}, payload={_PropKey.REF_ID: rid, _PropKey.CONFIRM: status} + ) + + +def _pull_to_terminal(service, rid, requester, confirm_capable=False): + """Drives the pull loop for one receiver until the producer serves a terminal status.""" + state = None + for _ in range(50): + reply = service._handle_download(_pull_request(rid, requester, confirm_capable=confirm_capable, state=state)) + body = reply.payload + status = body.get(_PropKey.STATUS) + if status in (ProduceRC.EOF, ProduceRC.ERROR): + return reply + state = body.get(_PropKey.STATE) + raise AssertionError("pull loop never reached a terminal status") + + +def _ref(service, rid): + return service._ref_table[rid] + + +class TestProducerSide: + def test_legacy_receiver_finalizes_at_serve(self): + service = _make_service() + tx_id, rid, _ = _new_tx(service) + + reply = _pull_to_terminal(service, rid, "r1", confirm_capable=False) + + assert _PropKey.CONFIRM_EXPECTED not in reply.payload + ref = _ref(service, rid) + assert ref.snapshot_receiver_statuses() == {"r1": DownloadStatus.SUCCESS} + assert ref.snapshot_pending_confirms() == {} + assert service._tx_table[tx_id].is_finished() + + def test_confirm_capable_receiver_is_provisional_at_serve(self): + service = _make_service() + tx_id, rid, _ = _new_tx(service) + + reply = _pull_to_terminal(service, rid, "r1", confirm_capable=True) + + assert reply.payload.get(_PropKey.CONFIRM_EXPECTED) is True + ref = _ref(service, rid) + # served EOF is NOT the receiver's truth yet + assert ref.snapshot_receiver_statuses() == {} + assert ref.snapshot_pending_confirms() == {"r1": DownloadStatus.SUCCESS} + assert not service._tx_table[tx_id].is_finished() + + def test_confirmation_finalizes_and_finishes(self): + service = _make_service() + tx_id, rid, _ = _new_tx(service) + _pull_to_terminal(service, rid, "r1", confirm_capable=True) + + reply = service._handle_download(_confirm_request(rid, "r1", DownloadStatus.SUCCESS)) + + assert reply.get_header(MessageHeaderKey.RETURN_CODE) == ReturnCode.OK + ref = _ref(service, rid) + assert ref.snapshot_receiver_statuses() == {"r1": DownloadStatus.SUCCESS} + assert ref.snapshot_pending_confirms() == {} + assert service._tx_table[tx_id].is_finished() + + def test_receiver_truth_wins_failed_confirm_after_served_eof(self): + # the motivating case: producer served EOF, but the receiver's finalization failed + service = _make_service() + tx_id, rid, _ = _new_tx(service) + _pull_to_terminal(service, rid, "r1", confirm_capable=True) + + service._handle_download(_confirm_request(rid, "r1", DownloadStatus.FAILED)) + + ref = _ref(service, rid) + assert ref.snapshot_receiver_statuses() == {"r1": DownloadStatus.FAILED} + # the transaction reaches its receiver count, but the aggregate outcome must fail + assert service._tx_table[tx_id].is_finished() + run_monitor_once(service, now=time.time()) + outcome = service.get_transaction_outcome(tx_id) + assert not outcome.completed + + def test_retry_overwrites_provisional_and_confirm_wins(self): + # retry-aware accounting: a served ERROR is provisional; a healed retry (EOF) plus a + # SUCCESS confirmation must finalize SUCCESS, not stick at first failure + service = _make_service() + tx_id, rid, obj = _new_tx(service) + ref = _ref(service, rid) + + # simulate a produce-time failure serve, provisionally recorded + ref.obj_served("r1", DownloadStatus.FAILED, expect_confirm=True) + assert ref.snapshot_pending_confirms() == {"r1": DownloadStatus.FAILED} + + # the receiver retries; this time the pull succeeds end-to-end + _pull_to_terminal(service, rid, "r1", confirm_capable=True) + assert ref.snapshot_pending_confirms() == {"r1": DownloadStatus.SUCCESS} + + service._handle_download(_confirm_request(rid, "r1", DownloadStatus.SUCCESS)) + assert ref.snapshot_receiver_statuses() == {"r1": DownloadStatus.SUCCESS} + run_monitor_once(service, now=time.time()) + assert service.get_transaction_outcome(tx_id).completed + + def test_first_confirm_is_final(self): + service = _make_service() + tx_id, rid, _ = _new_tx(service) + _pull_to_terminal(service, rid, "r1", confirm_capable=True) + + service._handle_download(_confirm_request(rid, "r1", DownloadStatus.SUCCESS)) + service._handle_download(_confirm_request(rid, "r1", DownloadStatus.FAILED)) + + assert _ref(service, rid).snapshot_receiver_statuses() == {"r1": DownloadStatus.SUCCESS} + + def test_late_duplicate_serve_cannot_resurrect_provisional(self): + service = _make_service() + tx_id, rid, _ = _new_tx(service) + _pull_to_terminal(service, rid, "r1", confirm_capable=True) + service._handle_download(_confirm_request(rid, "r1", DownloadStatus.SUCCESS)) + + ref = _ref(service, rid) + ref.obj_served("r1", DownloadStatus.FAILED, expect_confirm=True) + + assert ref.snapshot_pending_confirms() == {} + assert ref.snapshot_receiver_statuses() == {"r1": DownloadStatus.SUCCESS} + + def test_invalid_confirm_status_is_ignored(self): + service = _make_service() + tx_id, rid, _ = _new_tx(service) + _pull_to_terminal(service, rid, "r1", confirm_capable=True) + + service._handle_download(_confirm_request(rid, "r1", "bogus")) + + ref = _ref(service, rid) + assert ref.snapshot_receiver_statuses() == {} + assert ref.snapshot_pending_confirms() == {"r1": DownloadStatus.SUCCESS} + + def test_unsolicited_confirm_cannot_certify(self): + # fail-open hole guard: a CONFIRM for a receiver that was never served a terminal + # reply on THIS incarnation of the ref must be dropped -- otherwise a stale confirm + # delayed across a ref_id reuse could certify (or pre-poison) a transfer that never + # delivered a byte + service = _make_service() + tx_id, rid, _ = _new_tx(service) + + service._handle_download(_confirm_request(rid, "r1", DownloadStatus.SUCCESS)) + + ref = _ref(service, rid) + assert ref.snapshot_receiver_statuses() == {} + assert not service._tx_table[tx_id].is_finished() + run_monitor_once(service, now=time.time()) + assert service.get_transaction_outcome(tx_id) is None # still live, nothing certified + + # the FAILED mirror cannot pre-poison either + service._handle_download(_confirm_request(rid, "r1", DownloadStatus.FAILED)) + assert ref.snapshot_receiver_statuses() == {} + + def test_tombstoned_ref_replay_does_not_solicit_confirm(self): + # after cleanup, a confirm-capable receiver retrying a finished ref gets the EOF + # replay WITHOUT CONFIRM_EXPECTED (no live ref to confirm against), and a stray + # confirm for the tombstoned rid is dropped OK without disturbing the outcome + service = _make_service() + tx_id, rid, _ = _new_tx(service) + _pull_to_terminal(service, rid, "r1", confirm_capable=True) + service._handle_download(_confirm_request(rid, "r1", DownloadStatus.SUCCESS)) + run_monitor_once(service, now=time.time()) # FINISHED -> tombstoned + assert rid not in service._ref_table + + replay = service._handle_download(_pull_request(rid, "r1", confirm_capable=True)) + assert replay.payload.get(_PropKey.STATUS) == ProduceRC.EOF + assert _PropKey.CONFIRM_EXPECTED not in replay.payload + + reply = service._handle_download(_confirm_request(rid, "r1", DownloadStatus.FAILED)) + assert reply.get_header(MessageHeaderKey.RETURN_CODE) == ReturnCode.OK + assert service.get_transaction_outcome(tx_id).completed # unchanged + + def test_kill_switch_reads_application_conf(self): + # the switch resolves through the standard application-config source (env var + # NVFLARE_STREAMING_RECEIVER_CONFIRM_ENABLED / job config), like neighboring vars + with patch.object(ds_module, "_receiver_confirm_cached", None): + with patch.object(ds_module.ConfigService, "get_bool_var", return_value=False) as gv: + assert ds_module._receiver_confirm_enabled() is False + assert gv.call_args.kwargs.get("conf") == ds_module.SystemConfigs.APPLICATION_CONF + + def test_late_confirm_for_unknown_ref_is_dropped_ok(self): + service = _make_service() + reply = service._handle_download(_confirm_request("R-GONE", "r1", DownloadStatus.SUCCESS)) + assert reply.get_header(MessageHeaderKey.RETURN_CODE) == ReturnCode.OK + + def test_unconfirmed_receiver_fails_closed_at_timeout(self): + # a lost fire-and-forget confirmation must not certify success: at the transaction + # timeout the unconfirmed receiver has no final status and the outcome fails + service = _make_service() + tx_id, rid, _ = _new_tx(service, timeout=10.0) + _pull_to_terminal(service, rid, "r1", confirm_capable=True) + + tx = service._tx_table[tx_id] + tx.last_active_time = time.time() - 11.0 + run_monitor_once(service, now=time.time()) + + outcome = service.get_transaction_outcome(tx_id) + assert outcome is not None + assert not outcome.completed + + def test_kill_switch_off_restores_legacy_semantics(self): + service = _make_service() + with patch.object(ds_module, "_receiver_confirm_cached", False): + tx_id, rid, _ = _new_tx(service) + reply = _pull_to_terminal(service, rid, "r1", confirm_capable=True) + + # producer ignores the advertised capability entirely + assert _PropKey.CONFIRM_EXPECTED not in reply.payload + ref = _ref(service, rid) + assert ref.snapshot_receiver_statuses() == {"r1": DownloadStatus.SUCCESS} + assert ref.snapshot_pending_confirms() == {} + assert service._tx_table[tx_id].is_finished() + + def test_mixed_fleet_finishes_on_mixed_semantics(self): + # one legacy receiver (final at serve) + one confirm-capable receiver (final at confirm) + service = _make_service() + tx_id, rid, _ = _new_tx(service, num_receivers=2) + tx = service._tx_table[tx_id] + + _pull_to_terminal(service, rid, "legacy", confirm_capable=False) + _pull_to_terminal(service, rid, "modern", confirm_capable=True) + assert not tx.is_finished() # modern receiver not confirmed yet + + service._handle_download(_confirm_request(rid, "modern", DownloadStatus.SUCCESS)) + assert tx.is_finished() + run_monitor_once(service, now=time.time()) + assert service.get_transaction_outcome(tx_id).completed + + +class _ScriptedCell: + """A cell whose send_request returns scripted replies and which records fire_and_forget.""" + + def __init__(self, replies): + self._replies = list(replies) + self.requests = [] + self.confirms = [] + + def send_request(self, channel, target, topic, request, timeout, secure, optional, abort_signal): + self.requests.append(request.payload) + if not self._replies: + raise AssertionError("scripted cell ran out of replies") + return self._replies.pop(0) + + def fire_and_forget(self, channel, topic, targets, message, secure=False, optional=False): + self.confirms.append(message.payload) + + +class _RecordingConsumer(Consumer): + def __init__(self, fail_on_complete=False): + super().__init__() + self.consumed = [] + self.completed = False + self.failed_reason = None + self._fail_on_complete = fail_on_complete + + def consume(self, ref_id, state, data): + self.consumed.append(data) + return state or {} + + def download_completed(self, ref_id): + if self._fail_on_complete: + raise RuntimeError("finalization failed") + self.completed = True + + def download_failed(self, ref_id, reason): + self.failed_reason = reason + + +def _ok_reply(body): + return make_reply(ReturnCode.OK, body=body) + + +def _chunk_reply(confirm_expected=True): + body = {_PropKey.STATUS: ProduceRC.OK, _PropKey.STATE: {"seq": 1}, _PropKey.DATA: b"chunk"} + if confirm_expected: + body[_PropKey.CONFIRM_EXPECTED] = True + return _ok_reply(body) + + +def _terminal_reply(status, confirm_expected=True): + body = {_PropKey.STATUS: status} + if confirm_expected: + body[_PropKey.CONFIRM_EXPECTED] = True + return _ok_reply(body) + + +class TestReceiverSide: + def test_advertises_capability_and_confirms_success_after_finalization(self): + cell = _ScriptedCell([_chunk_reply(), _terminal_reply(ProduceRC.EOF)]) + consumer = _RecordingConsumer() + + download_object(from_fqcn="site-1", ref_id="R1", per_request_timeout=5.0, cell=cell, consumer=consumer) + + assert all(req.get(_PropKey.CONFIRM_CAPABLE) is True for req in cell.requests) + assert consumer.completed + assert cell.confirms == [{_PropKey.REF_ID: "R1", _PropKey.CONFIRM: DownloadStatus.SUCCESS}] + + def test_confirms_failed_when_finalization_raises(self): + # served EOF but download_completed raises: the producer must learn receiver truth + cell = _ScriptedCell([_terminal_reply(ProduceRC.EOF)]) + consumer = _RecordingConsumer(fail_on_complete=True) + + with pytest.raises(RuntimeError, match="finalization failed"): + download_object(from_fqcn="site-1", ref_id="R1", per_request_timeout=5.0, cell=cell, consumer=consumer) + + assert cell.confirms == [{_PropKey.REF_ID: "R1", _PropKey.CONFIRM: DownloadStatus.FAILED}] + + def test_confirms_failed_on_producer_error(self): + cell = _ScriptedCell([_chunk_reply(), _terminal_reply(ProduceRC.ERROR)]) + consumer = _RecordingConsumer() + + download_object(from_fqcn="site-1", ref_id="R1", per_request_timeout=5.0, cell=cell, consumer=consumer) + + assert consumer.failed_reason is not None + assert cell.confirms == [{_PropKey.REF_ID: "R1", _PropKey.CONFIRM: DownloadStatus.FAILED}] + + def test_no_confirm_toward_legacy_producer(self): + # the producer never advertised CONFIRM_EXPECTED: a new receiver must send nothing extra + cell = _ScriptedCell( + [_chunk_reply(confirm_expected=False), _terminal_reply(ProduceRC.EOF, confirm_expected=False)] + ) + consumer = _RecordingConsumer() + + download_object(from_fqcn="site-1", ref_id="R1", per_request_timeout=5.0, cell=cell, consumer=consumer) + + assert consumer.completed + assert cell.confirms == [] + + def test_kill_switch_off_receiver_is_fully_legacy(self): + with patch.object(ds_module, "_receiver_confirm_cached", False): + cell = _ScriptedCell([_terminal_reply(ProduceRC.EOF, confirm_expected=True)]) + consumer = _RecordingConsumer() + + download_object(from_fqcn="site-1", ref_id="R1", per_request_timeout=5.0, cell=cell, consumer=consumer) + + assert all(_PropKey.CONFIRM_CAPABLE not in req for req in cell.requests) + assert cell.confirms == [] + assert consumer.completed diff --git a/tests/unit_test/fuel/f3/streaming/transfer_waiter_test.py b/tests/unit_test/fuel/f3/streaming/transfer_waiter_test.py new file mode 100644 index 0000000000..3cc5da5035 --- /dev/null +++ b/tests/unit_test/fuel/f3/streaming/transfer_waiter_test.py @@ -0,0 +1,245 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the awaitable transfer facade (plan: F3-4). + +TransferWaiter is the "returns == delivered" primitive the upper layers (executor backends, +trainer engine) consume: wait() blocks -- event-driven, resolved inside the outcome-recording +path -- until the aggregate TransferOutcome is recorded; COMPLETED only when every expected +receiver succeeded. It composes with (never replaces) transaction_done_cb / outcome_cb. +""" + +import threading +import time +from unittest.mock import Mock, patch + +import pytest + +from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey +from nvflare.fuel.f3.cellnet.utils import new_cell_message +from nvflare.fuel.f3.streaming import download_service as ds_module +from nvflare.fuel.f3.streaming.download_service import DownloadStatus, ProduceRC, TransferWaiter, _PropKey +from tests.unit_test.fuel.f3.streaming.download_test_utils import ( + MockDownloadable, + make_isolated_download_service, + run_monitor_once, +) + + +@pytest.fixture(autouse=True) +def confirm_switch_on(): + with patch.object(ds_module, "_receiver_confirm_cached", True): + yield + + +def _make_service(): + service = make_isolated_download_service() + service._tx_monitor = Mock() + return service + + +def _new_tx(service, num_receivers=1, timeout=10.0): + tx_id = service.new_transaction(cell=Mock(), timeout=timeout, num_receivers=num_receivers) + rid = service.add_object(tx_id, MockDownloadable([b"chunk"])) + return tx_id, rid + + +def _pull_to_terminal(service, rid, requester, confirm_capable=False): + state = None + for _ in range(50): + payload = {_PropKey.REF_ID: rid} + if confirm_capable: + payload[_PropKey.CONFIRM_CAPABLE] = True + if state is not None: + payload[_PropKey.STATE] = state + reply = service._handle_download( + new_cell_message(headers={MessageHeaderKey.ORIGIN: requester}, payload=payload) + ) + status = reply.payload.get(_PropKey.STATUS) + if status in (ProduceRC.EOF, ProduceRC.ERROR): + return reply + state = reply.payload.get(_PropKey.STATE) + raise AssertionError("pull loop never reached terminal") + + +def _confirm(service, rid, requester, status): + service._handle_download( + new_cell_message( + headers={MessageHeaderKey.ORIGIN: requester}, payload={_PropKey.REF_ID: rid, _PropKey.CONFIRM: status} + ) + ) + + +class TestTransferWaiter: + def test_wait_blocks_until_outcome_and_returns_delivered(self): + # the load-bearing property: a thread blocked in wait() is released by outcome + # recording itself (event-driven, no polling), and gets the COMPLETED certificate + service = _make_service() + tx_id, rid = _new_tx(service) + waiter = service.get_transfer_waiter(tx_id) + assert not waiter.done() + + results = [] + t = threading.Thread(target=lambda: results.append(waiter.wait(timeout=10.0))) + t.start() + + _pull_to_terminal(service, rid, "r1", confirm_capable=True) + _confirm(service, rid, "r1", DownloadStatus.SUCCESS) + run_monitor_once(service, now=time.time()) + + t.join(5.0) + assert not t.is_alive() + assert len(results) == 1 and results[0] is not None + assert results[0].completed + assert results[0].tx_id == tx_id + + def test_waiter_after_termination_resolves_immediately(self): + service = _make_service() + tx_id, rid = _new_tx(service) + _pull_to_terminal(service, rid, "r1") + run_monitor_once(service, now=time.time()) + + waiter = service.get_transfer_waiter(tx_id) + assert waiter.done() + outcome = waiter.wait(timeout=0) + assert outcome is not None and outcome.completed + + def test_wait_timeout_returns_none_then_resolves(self): + service = _make_service() + tx_id, rid = _new_tx(service) + waiter = service.get_transfer_waiter(tx_id) + + assert waiter.wait(timeout=0.05) is None + assert waiter.outcome is None + + _pull_to_terminal(service, rid, "r1") + run_monitor_once(service, now=time.time()) + outcome = waiter.wait(timeout=5.0) + assert outcome is not None and outcome.completed + + def test_failed_outcome_is_returned_not_masked(self): + service = _make_service() + tx_id, rid = _new_tx(service) + waiter = service.get_transfer_waiter(tx_id) + + _pull_to_terminal(service, rid, "r1", confirm_capable=True) + _confirm(service, rid, "r1", DownloadStatus.FAILED) # receiver truth: finalization failed + run_monitor_once(service, now=time.time()) + + outcome = waiter.wait(timeout=5.0) + assert outcome is not None + assert not outcome.completed + + def test_linger_applies_to_finished_outcomes_only(self): + # linger preserves the process/tombstone window so lost terminal replies can be + # replayed: it applies to ANY FINISHED outcome (completed or receiver-failed -- + # the served-SUCCESS receivers of a partial fan-out are exactly who needs it), + # and not to timed-out transactions (no receiver is retrying a served reply). + service = _make_service() + + # completed FINISHED: linger applied + tx_id, rid = _new_tx(service) + _pull_to_terminal(service, rid, "r1") + run_monitor_once(service, now=time.time()) + waiter = service.get_transfer_waiter(tx_id) + with patch.object(ds_module.time, "sleep") as mock_sleep: + outcome = waiter.wait(timeout=5.0, linger=0.2) + assert outcome.completed + mock_sleep.assert_called_once_with(0.2) + + # receiver-failed FINISHED: linger still applied (partial fan-out healing window) + tx_id2, rid2 = _new_tx(service) + _pull_to_terminal(service, rid2, "r1", confirm_capable=True) + _confirm(service, rid2, "r1", DownloadStatus.FAILED) + run_monitor_once(service, now=time.time()) + waiter2 = service.get_transfer_waiter(tx_id2) + with patch.object(ds_module.time, "sleep") as mock_sleep: + outcome2 = waiter2.wait(timeout=5.0, linger=0.2) + assert not outcome2.completed + mock_sleep.assert_called_once_with(0.2) + + # TIMEOUT termination: no linger + tx_id3, rid3 = _new_tx(service) + service._tx_table[tx_id3].last_active_time = time.time() - 100.0 + run_monitor_once(service, now=time.time()) + waiter3 = service.get_transfer_waiter(tx_id3) + with patch.object(ds_module.time, "sleep") as mock_sleep: + outcome3 = waiter3.wait(timeout=5.0, linger=5.0) + assert outcome3 is not None and not outcome3.completed + mock_sleep.assert_not_called() + + def test_waiter_for_unknown_tx_resolves_immediately(self): + # invariant: waiters can never hang -- nothing will ever record an outcome for an + # unknown (or already-forgotten) transaction id + service = _make_service() + waiter = service.get_transfer_waiter("no-such-tx") + assert waiter.done() + assert waiter.wait(timeout=0) is None + assert "no-such-tx" not in service._tx_waiters + + def test_waiter_after_shutdown_resolves_immediately(self): + service = _make_service() + tx_id, _ = _new_tx(service) + service.shutdown() + + waiter = service.get_transfer_waiter(tx_id) + assert waiter.done() + assert waiter.wait(timeout=0) is None + assert not service._tx_waiters + + def test_shutdown_unblocks_waiters_with_none(self): + service = _make_service() + tx_id, _ = _new_tx(service) + waiter = service.get_transfer_waiter(tx_id) + + results = [] + t = threading.Thread(target=lambda: results.append(waiter.wait(timeout=10.0))) + t.start() + service.shutdown() + t.join(5.0) + + assert not t.is_alive(), "shutdown must never leave a waiter hanging" + assert results == [None] + + def test_acquired_receivers_reflects_first_pulls(self): + service = _make_service() + tx_id, rid = _new_tx(service, num_receivers=2) + assert service.get_acquired_receivers(tx_id) == set() + + _pull_to_terminal(service, rid, "r1") + assert service.get_acquired_receivers(tx_id) == {"r1"} + + def test_budget_failure_resolves_waiter_in_bounded_time(self): + # end-to-end with F3-3: a stalled receiver's budget failure terminates the tx and + # releases the waiter -- the producer is never pinned to the full TTL + service = _make_service() + tx_id = service.new_transaction(cell=Mock(), timeout=1000.0, num_receivers=2, receiver_idle_timeout=5.0) + rid = service.add_object(tx_id, MockDownloadable([b"chunk"])) + waiter = service.get_transfer_waiter(tx_id) + + _pull_to_terminal(service, rid, "healthy") + _pull_to_terminal(service, rid, "stalled", confirm_capable=True) # confirm never arrives + run_monitor_once(service, now=time.time() + 30.0) + + outcome = waiter.wait(timeout=5.0) + assert outcome is not None + assert not outcome.completed + assert outcome.refs[0].receiver_statuses["stalled"] == DownloadStatus.FAILED + + def test_waiter_is_a_transfer_waiter(self): + service = _make_service() + tx_id, _ = _new_tx(service) + waiter = service.get_transfer_waiter(tx_id) + assert isinstance(waiter, TransferWaiter) + assert waiter.transaction_id == tx_id From d941683d4a6866202eac314c2f170414e8013a77 Mon Sep 17 00:00:00 2001 From: YuanTingHsieh Date: Tue, 7 Jul 2026 18:32:07 -0700 Subject: [PATCH 04/20] Simplify F3 payload layer: hot-path trims, tx-level acquisition, shared test helpers Cleanup pass over the F3 payload-layer commit (4 parallel review angles: reuse / simplification / efficiency / altitude). No behavior change. - CONFIRM_EXPECTED is serialized only on terminal replies: since confirmations are sent only after terminal serves, the per-chunk advertisement was dead weight on the hottest wire message. Terminal branch folded to one if/else. - Acquisition promoted to the transaction (_acquired_receivers, monotonic set with a double-checked add: at most one lock per (transaction, receiver), not per chunk). Replaces the per-pass union rebuild across refs in both enforce_receiver_budgets and get_acquired_receivers -- acquisition is a transaction-level fact, now modeled as one. - Monitor budget pre-pass pre-filters to transactions that actually have budgets (has_receiver_budgets), so the common no-budget case does zero per-transaction lock reacquisitions per tick. - _resolve_receiver_budget now reuses get_positive_float_var and check_positive_number instead of hand-rolling both branches. - Kill-switch comment corrected: per-process (read once; restart to change), not "runtime". Redundant pending-confirm pop in enforce_budgets removed (_finalize_receiver owns it). - Test scaffolding deduplicated: the pull/confirm wire builders, pull_to_terminal loop, and monitor-suppressed service factory move to download_test_utils; the confirm kill-switch fixture moves to a shared conftest.py. Removes five helpers copy-pasted across three files. Deliberate skips: lru_cache for the kill-switch cache (the module global is a load-bearing test patch point), StreamFuture delegation for TransferWaiter (different resolve semantics; minimal savings), spec-object for the transaction kwargs and a general capability-negotiation mechanism (premature until more consumers/capabilities exist -- noted for later). --- nvflare/fuel/f3/streaming/download_service.py | 81 ++++++++++--------- tests/unit_test/fuel/f3/streaming/conftest.py | 30 +++++++ .../fuel/f3/streaming/download_test_utils.py | 40 ++++++++- .../fuel/f3/streaming/receiver_budget_test.py | 68 ++++------------ .../f3/streaming/receiver_confirm_test.py | 54 ++----------- .../fuel/f3/streaming/transfer_waiter_test.py | 61 +++----------- 6 files changed, 143 insertions(+), 191 deletions(-) create mode 100644 tests/unit_test/fuel/f3/streaming/conftest.py diff --git a/nvflare/fuel/f3/streaming/download_service.py b/nvflare/fuel/f3/streaming/download_service.py index c4b384ecc2..22ebf20937 100644 --- a/nvflare/fuel/f3/streaming/download_service.py +++ b/nvflare/fuel/f3/streaming/download_service.py @@ -34,8 +34,10 @@ terminal_state_for_done_status, ) from nvflare.fuel.f3.streaming.transfer_progress import TransferProgressState +from nvflare.fuel.utils.app_config_utils import get_positive_float_var from nvflare.fuel.utils.config_service import ConfigService from nvflare.fuel.utils.log_utils import get_obj_logger +from nvflare.fuel.utils.validation_utils import check_positive_number from nvflare.security.logging import secure_format_exception OBJ_DOWNLOADER_CHANNEL = "download_service__" @@ -183,9 +185,10 @@ class _PropKey: CONFIRM_EXPECTED = "confirm_expected" # producer -> receiver, per reply: confirmations consumed -# Runtime kill-switch for receiver-confirmed completion. The wire behavior is doubly gated -- -# per-message capability advertisement AND this config var on each side -- so a field issue in a -# mixed-version fleet is mitigated by configuration without a code revert. +# Per-process kill-switch for receiver-confirmed completion (read once at first use; set the +# config var / env before process start and restart to change it). The wire behavior is doubly +# gated -- per-message capability advertisement AND this switch on each side -- so a field issue +# in a mixed-version fleet is mitigated by configuration + restart without a code revert. RECEIVER_CONFIRM_CONFIG_VAR = "streaming_receiver_confirm_enabled" _receiver_confirm_cached = None @@ -214,17 +217,9 @@ def _receiver_confirm_enabled() -> bool: def _resolve_receiver_budget(explicit, var_name: str): if explicit is not None: - value = float(explicit) - if value <= 0: - raise ValueError(f"receiver budget must be positive, got {explicit}") - return value - try: - value = ConfigService.get_float_var(var_name, conf=SystemConfigs.APPLICATION_CONF, default=None) - except Exception: - return None - if value is None or value <= 0: - return None - return float(value) + check_positive_number(var_name, explicit) + return float(explicit) + return get_positive_float_var(var_name, default=None) class _Ref: @@ -368,6 +363,12 @@ def snapshot_pending_confirms(self) -> dict: def mark_receiver_active(self, receiver: str): with self._progress_lock: self._receiver_activity[receiver] = time.time() + tx = self.tx + if receiver not in tx._acquired_receivers: + # double-checked: the set is monotonic, so the lock is taken at most once + # per (transaction, receiver) -- not per chunk + with tx._stats_lock: + tx._acquired_receivers.add(receiver) def snapshot_receiver_activity(self) -> dict: with self._progress_lock: @@ -420,7 +421,7 @@ def enforce_budgets( continue # finalized (e.g. confirmed) between snapshot and enforcement: truth wins if self._receiver_activity.get(receiver) != activity.get(receiver): continue # activity advanced past the snapshot: not actually idle - self._pending_confirms.pop(receiver, None) + # _finalize_receiver pops the pending-confirm entry itself if not self._finalize_receiver(receiver, DownloadStatus.FAILED): continue self.tx.logger.warning(f"receiver {receiver} failed for ref {self.rid}: {reason}") @@ -650,6 +651,9 @@ def __init__( self.progress_interval = float(progress_interval) self.refs = [] self._refs_lock = threading.RLock() + # receivers that have issued at least one pull on ANY ref (monotonic; the + # transaction-level PAYLOAD_ACQUIRED fact the acquire budget and the facade read) + self._acquired_receivers = set() self.logger = get_obj_logger(self) def mark_active(self): @@ -702,17 +706,18 @@ def timed_out(self): """ self.transaction_done(TransactionDoneStatus.TIMEOUT) + @property + def has_receiver_budgets(self) -> bool: + return self.receiver_acquire_timeout is not None or self.receiver_idle_timeout is not None + def enforce_receiver_budgets(self, now: float): """Evaluates per-receiver budgets across all refs; called by the monitor thread.""" - if self.receiver_acquire_timeout is None and self.receiver_idle_timeout is None: + if not self.has_receiver_budgets: return - refs = self.snapshot_refs() - # transaction-level acquisition: a first pull on ANY ref acquires the receiver - tx_acquired = set() - for ref in refs: + with self._stats_lock: + tx_acquired = set(self._acquired_receivers) + for ref in self.snapshot_refs(): assert isinstance(ref, _Ref) - tx_acquired.update(ref.snapshot_receiver_activity().keys()) - for ref in refs: ref.enforce_budgets( now, self.receiver_acquire_timeout, @@ -1127,10 +1132,8 @@ def get_acquired_receivers(cls, transaction_id: str) -> set: if tx is None: return set() assert isinstance(tx, _Transaction) - acquired = set() - for ref in tx.snapshot_refs(): - acquired.update(ref.snapshot_receiver_activity().keys()) - return acquired + with tx._stats_lock: + return set(tx._acquired_receivers) @classmethod def _record_outcome(cls, outcome: TransferOutcome, tx: _Transaction): @@ -1286,15 +1289,14 @@ def _handle_download(cls, request: Message) -> Message: # provisional: the receiver's confirmation carries the terminal truth -- # do not latch a terminal progress state the confirm may contradict ref.emit_progress(receiver_id=requester, state=TransferProgressState.ACTIVE, force=True) + body = {_PropKey.STATUS: rc, _PropKey.CONFIRM_EXPECTED: True} else: ref.emit_progress( receiver_id=requester, state=TransferProgressState.COMPLETED if rc == ProduceRC.EOF else TransferProgressState.FAILED, force=True, ) - body = {_PropKey.STATUS: rc} - if expect_confirm: - body[_PropKey.CONFIRM_EXPECTED] = True + body = {_PropKey.STATUS: rc} return make_reply(ReturnCode.OK, body=body) else: # continue — accumulate bytes for timing summary in transaction_done() @@ -1310,14 +1312,17 @@ def _handle_download(cls, request: Message) -> Message: bytes_delta=bytes_delta, items_delta=items_delta, ) - body = { - _PropKey.STATUS: rc, - _PropKey.STATE: new_state, - _PropKey.DATA: data, - } - if expect_confirm: - body[_PropKey.CONFIRM_EXPECTED] = True - return make_reply(ReturnCode.OK, body=body) + # no CONFIRM_EXPECTED on data chunks: the receiver only consumes it from the + # terminal reply (confirms are sent only after terminal serves), so advertising + # per chunk would be dead weight on the hottest wire message + return make_reply( + ReturnCode.OK, + body={ + _PropKey.STATUS: rc, + _PropKey.STATE: new_state, + _PropKey.DATA: data, + }, + ) @classmethod def _handle_confirm(cls, rid: str, requester: str, status: str) -> Message: @@ -1345,7 +1350,7 @@ def _monitor_tx(cls): # never run under the global lock. A budget failure recorded here flips # is_finished() so the classification pass below resolves the tx immediately. with cls._tx_lock: - budget_txs = list(cls._tx_table.values()) + budget_txs = [tx for tx in cls._tx_table.values() if tx.has_receiver_budgets] for tx in budget_txs: with cls._tx_lock: if cls._tx_table.get(tx.tid) is not tx: diff --git a/tests/unit_test/fuel/f3/streaming/conftest.py b/tests/unit_test/fuel/f3/streaming/conftest.py new file mode 100644 index 0000000000..bba4b164e5 --- /dev/null +++ b/tests/unit_test/fuel/f3/streaming/conftest.py @@ -0,0 +1,30 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest.mock import patch + +import pytest + +from nvflare.fuel.f3.streaming import download_service as ds_module + + +@pytest.fixture(autouse=True) +def confirm_switch_on(): + """Pin the receiver-confirm kill-switch ON for all streaming tests. + + Individual tests patch it OFF explicitly; legacy-path tests are unaffected (their + requests carry no capability key, so the switch is never consulted for them). + """ + with patch.object(ds_module, "_receiver_confirm_cached", True): + yield diff --git a/tests/unit_test/fuel/f3/streaming/download_test_utils.py b/tests/unit_test/fuel/f3/streaming/download_test_utils.py index 688521c584..b9939e798c 100644 --- a/tests/unit_test/fuel/f3/streaming/download_test_utils.py +++ b/tests/unit_test/fuel/f3/streaming/download_test_utils.py @@ -26,7 +26,9 @@ import pytest -from nvflare.fuel.f3.streaming.download_service import Downloadable, DownloadService, ProduceRC +from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey +from nvflare.fuel.f3.cellnet.utils import new_cell_message +from nvflare.fuel.f3.streaming.download_service import Downloadable, DownloadService, ProduceRC, _PropKey class MockDownloadable(Downloadable): @@ -93,6 +95,42 @@ class IsolatedDownloadService(DownloadService): return IsolatedDownloadService +def make_confirm_test_service(): + """An isolated service with the real monitor thread suppressed (budget/confirm/waiter tests).""" + service = make_isolated_download_service() + service._tx_monitor = Mock() + return service + + +def pull_request(rid, requester, confirm_capable=False, state=None): + """Builds one downloader pull request as it appears on the wire.""" + payload = {_PropKey.REF_ID: rid} + if confirm_capable: + payload[_PropKey.CONFIRM_CAPABLE] = True + if state is not None: + payload[_PropKey.STATE] = state + return new_cell_message(headers={MessageHeaderKey.ORIGIN: requester}, payload=payload) + + +def confirm_request(rid, requester, status): + """Builds one receiver-confirmation message as it appears on the wire.""" + return new_cell_message( + headers={MessageHeaderKey.ORIGIN: requester}, payload={_PropKey.REF_ID: rid, _PropKey.CONFIRM: status} + ) + + +def pull_to_terminal(service, rid, requester, confirm_capable=False): + """Drives the pull loop for one receiver until the producer serves a terminal status.""" + state = None + for _ in range(50): + reply = service._handle_download(pull_request(rid, requester, confirm_capable=confirm_capable, state=state)) + status = reply.payload.get(_PropKey.STATUS) + if status in (ProduceRC.EOF, ProduceRC.ERROR): + return reply + state = reply.payload.get(_PropKey.STATE) + raise AssertionError("pull loop never reached a terminal status") + + def run_monitor_once(service_cls, now): from nvflare.fuel.f3.streaming import download_service as download_service_module diff --git a/tests/unit_test/fuel/f3/streaming/receiver_budget_test.py b/tests/unit_test/fuel/f3/streaming/receiver_budget_test.py index 49bff85ab0..14af215cff 100644 --- a/tests/unit_test/fuel/f3/streaming/receiver_budget_test.py +++ b/tests/unit_test/fuel/f3/streaming/receiver_budget_test.py @@ -26,28 +26,14 @@ import pytest -from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey -from nvflare.fuel.f3.cellnet.utils import new_cell_message from nvflare.fuel.f3.streaming import download_service as ds_module -from nvflare.fuel.f3.streaming.download_service import DownloadStatus, ProduceRC, _PropKey +from nvflare.fuel.f3.streaming.download_service import DownloadStatus from nvflare.fuel.f3.streaming.transfer_outcome import TransferOutcomeReason, compute_transfer_outcome -from tests.unit_test.fuel.f3.streaming.download_test_utils import ( - MockDownloadable, - make_isolated_download_service, - run_monitor_once, -) - - -@pytest.fixture(autouse=True) -def confirm_switch_on(): - with patch.object(ds_module, "_receiver_confirm_cached", True): - yield - - -def _make_service(): - service = make_isolated_download_service() - service._tx_monitor = Mock() - return service +from tests.unit_test.fuel.f3.streaming.download_test_utils import MockDownloadable, confirm_request +from tests.unit_test.fuel.f3.streaming.download_test_utils import make_confirm_test_service as _make_service +from tests.unit_test.fuel.f3.streaming.download_test_utils import pull_request +from tests.unit_test.fuel.f3.streaming.download_test_utils import pull_to_terminal as _pull_to_terminal +from tests.unit_test.fuel.f3.streaming.download_test_utils import run_monitor_once def _new_tx(service, chunks=1, **tx_kwargs): @@ -59,34 +45,13 @@ def _new_tx(service, chunks=1, **tx_kwargs): return tx_id, rid -def _pull_once(service, rid, requester, confirm_capable=False, state=None): - payload = {_PropKey.REF_ID: rid} - if confirm_capable: - payload[_PropKey.CONFIRM_CAPABLE] = True - if state is not None: - payload[_PropKey.STATE] = state - request = new_cell_message(headers={MessageHeaderKey.ORIGIN: requester}, payload=payload) - return service._handle_download(request) - - -def _pull_to_terminal(service, rid, requester, confirm_capable=False): - state = None - for _ in range(50): - reply = _pull_once(service, rid, requester, confirm_capable=confirm_capable, state=state) - status = reply.payload.get(_PropKey.STATUS) - if status in (ProduceRC.EOF, ProduceRC.ERROR): - return reply - state = reply.payload.get(_PropKey.STATE) - raise AssertionError("pull loop never reached terminal") - - class TestIdleBudget: def test_stalled_receiver_fails_without_waiting_tx_ttl(self): service = _make_service() tx_id, rid = _new_tx(service, chunks=3, num_receivers=2, receiver_idle_timeout=5.0) _pull_to_terminal(service, rid, "healthy") # legacy receiver, final at serve - _pull_once(service, rid, "stalled") # one pull, then silence + service._handle_download(pull_request(rid, "stalled")) # one pull, then silence # a monitor pass past the idle budget (but far inside the 1000s TTL) run_monitor_once(service, now=time.time() + 30.0) @@ -118,7 +83,7 @@ def test_lost_confirmation_is_bounded_by_idle_budget(self): def test_active_receiver_is_not_idle_failed(self): service = _make_service() tx_id, rid = _new_tx(service, chunks=3, receiver_idle_timeout=5.0) - reply = _pull_once(service, rid, "r1") + reply = service._handle_download(pull_request(rid, "r1")) ref = service._ref_table[rid] # receiver keeps making requests: refresh activity to "now" as seen by the monitor future = time.time() + 30.0 @@ -157,7 +122,7 @@ def test_never_pulling_receiver_fails_at_acquire_deadline(self): def test_receiver_with_activity_is_not_acquire_failed(self): service = _make_service() tx_id, rid = _new_tx(service, chunks=3, num_receivers=0, receiver_ids=("r1",), receiver_acquire_timeout=5.0) - _pull_once(service, rid, "r1") # first pull happened: acquire satisfied + service._handle_download(pull_request(rid, "r1")) # first pull happened: acquire satisfied run_monitor_once(service, now=time.time() + 30.0) @@ -180,7 +145,7 @@ class TestBudgetSemantics: def test_budgets_disabled_preserve_ttl_only_behavior(self): service = _make_service() tx_id, rid = _new_tx(service, num_receivers=2) - _pull_once(service, rid, "r1") + service._handle_download(pull_request(rid, "r1")) run_monitor_once(service, now=time.time() + 500.0) # inside the 1000s TTL @@ -190,7 +155,7 @@ def test_budgets_disabled_preserve_ttl_only_behavior(self): def test_activity_tracked_without_progress_cb(self): service = _make_service() tx_id, rid = _new_tx(service, chunks=2) - _pull_once(service, rid, "r1") + service._handle_download(pull_request(rid, "r1")) activity = service._ref_table[rid].snapshot_receiver_activity() assert "r1" in activity @@ -220,7 +185,7 @@ def test_sequential_multi_ref_receiver_is_not_acquire_failed(self): ) rid1 = service.add_object(tx_id, MockDownloadable([b"chunk"] * 3)) rid2 = service.add_object(tx_id, MockDownloadable([b"chunk"] * 3)) - _pull_once(service, rid1, "r1") # busy on ref 1, has not touched ref 2 + service._handle_download(pull_request(rid1, "r1")) # busy on ref 1, has not touched ref 2 run_monitor_once(service, now=time.time() + 30.0) @@ -234,12 +199,7 @@ def test_confirmed_receiver_wins_over_stale_budget_snapshot(self): tx_id, rid = _new_tx(service, receiver_idle_timeout=5.0) _pull_to_terminal(service, rid, "r1", confirm_capable=True) ref = service._ref_table[rid] - service._handle_download( - new_cell_message( - headers={MessageHeaderKey.ORIGIN: "r1"}, - payload={_PropKey.REF_ID: rid, _PropKey.CONFIRM: DownloadStatus.SUCCESS}, - ) - ) + service._handle_download(confirm_request(rid, "r1", DownloadStatus.SUCCESS)) enforced = ref.enforce_budgets(time.time() + 30.0, None, 5.0, None) @@ -294,7 +254,7 @@ def test_min_receivers_validation(self): def test_negative_budget_rejected(self): service = _make_service() - with pytest.raises(ValueError, match="must be positive"): + with pytest.raises(ValueError, match="must > 0"): service.new_transaction(cell=Mock(), timeout=10.0, num_receivers=1, receiver_idle_timeout=-1.0) diff --git a/tests/unit_test/fuel/f3/streaming/receiver_confirm_test.py b/tests/unit_test/fuel/f3/streaming/receiver_confirm_test.py index 2848bbee8b..e64842a857 100644 --- a/tests/unit_test/fuel/f3/streaming/receiver_confirm_test.py +++ b/tests/unit_test/fuel/f3/streaming/receiver_confirm_test.py @@ -31,27 +31,15 @@ import pytest from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey, ReturnCode -from nvflare.fuel.f3.cellnet.utils import make_reply, new_cell_message +from nvflare.fuel.f3.cellnet.utils import make_reply from nvflare.fuel.f3.streaming import download_service as ds_module from nvflare.fuel.f3.streaming.download_service import Consumer, DownloadStatus, ProduceRC, _PropKey, download_object -from tests.unit_test.fuel.f3.streaming.download_test_utils import ( - MockDownloadable, - make_isolated_download_service, - run_monitor_once, -) - - -@pytest.fixture(autouse=True) -def confirm_switch_on(): - """Pin the kill-switch ON for tests (individual tests patch it OFF explicitly).""" - with patch.object(ds_module, "_receiver_confirm_cached", True): - yield - - -def _make_service(): - service = make_isolated_download_service() - service._tx_monitor = Mock() # suppress the real monitor thread - return service +from tests.unit_test.fuel.f3.streaming.download_test_utils import MockDownloadable +from tests.unit_test.fuel.f3.streaming.download_test_utils import confirm_request as _confirm_request +from tests.unit_test.fuel.f3.streaming.download_test_utils import make_confirm_test_service as _make_service +from tests.unit_test.fuel.f3.streaming.download_test_utils import pull_request as _pull_request +from tests.unit_test.fuel.f3.streaming.download_test_utils import pull_to_terminal as _pull_to_terminal +from tests.unit_test.fuel.f3.streaming.download_test_utils import run_monitor_once def _new_tx(service, num_receivers=1, timeout=10.0, chunks=1): @@ -61,34 +49,6 @@ def _new_tx(service, num_receivers=1, timeout=10.0, chunks=1): return tx_id, rid, obj -def _pull_request(rid, requester, confirm_capable=False, state=None): - payload = {_PropKey.REF_ID: rid} - if confirm_capable: - payload[_PropKey.CONFIRM_CAPABLE] = True - if state is not None: - payload[_PropKey.STATE] = state - return new_cell_message(headers={MessageHeaderKey.ORIGIN: requester}, payload=payload) - - -def _confirm_request(rid, requester, status): - return new_cell_message( - headers={MessageHeaderKey.ORIGIN: requester}, payload={_PropKey.REF_ID: rid, _PropKey.CONFIRM: status} - ) - - -def _pull_to_terminal(service, rid, requester, confirm_capable=False): - """Drives the pull loop for one receiver until the producer serves a terminal status.""" - state = None - for _ in range(50): - reply = service._handle_download(_pull_request(rid, requester, confirm_capable=confirm_capable, state=state)) - body = reply.payload - status = body.get(_PropKey.STATUS) - if status in (ProduceRC.EOF, ProduceRC.ERROR): - return reply - state = body.get(_PropKey.STATE) - raise AssertionError("pull loop never reached a terminal status") - - def _ref(service, rid): return service._ref_table[rid] diff --git a/tests/unit_test/fuel/f3/streaming/transfer_waiter_test.py b/tests/unit_test/fuel/f3/streaming/transfer_waiter_test.py index 3cc5da5035..72d6cc9722 100644 --- a/tests/unit_test/fuel/f3/streaming/transfer_waiter_test.py +++ b/tests/unit_test/fuel/f3/streaming/transfer_waiter_test.py @@ -24,29 +24,12 @@ import time from unittest.mock import Mock, patch -import pytest - -from nvflare.fuel.f3.cellnet.defs import MessageHeaderKey -from nvflare.fuel.f3.cellnet.utils import new_cell_message from nvflare.fuel.f3.streaming import download_service as ds_module -from nvflare.fuel.f3.streaming.download_service import DownloadStatus, ProduceRC, TransferWaiter, _PropKey -from tests.unit_test.fuel.f3.streaming.download_test_utils import ( - MockDownloadable, - make_isolated_download_service, - run_monitor_once, -) - - -@pytest.fixture(autouse=True) -def confirm_switch_on(): - with patch.object(ds_module, "_receiver_confirm_cached", True): - yield - - -def _make_service(): - service = make_isolated_download_service() - service._tx_monitor = Mock() - return service +from nvflare.fuel.f3.streaming.download_service import DownloadStatus, TransferWaiter +from tests.unit_test.fuel.f3.streaming.download_test_utils import MockDownloadable, confirm_request +from tests.unit_test.fuel.f3.streaming.download_test_utils import make_confirm_test_service as _make_service +from tests.unit_test.fuel.f3.streaming.download_test_utils import pull_to_terminal as _pull_to_terminal +from tests.unit_test.fuel.f3.streaming.download_test_utils import run_monitor_once def _new_tx(service, num_receivers=1, timeout=10.0): @@ -55,32 +38,6 @@ def _new_tx(service, num_receivers=1, timeout=10.0): return tx_id, rid -def _pull_to_terminal(service, rid, requester, confirm_capable=False): - state = None - for _ in range(50): - payload = {_PropKey.REF_ID: rid} - if confirm_capable: - payload[_PropKey.CONFIRM_CAPABLE] = True - if state is not None: - payload[_PropKey.STATE] = state - reply = service._handle_download( - new_cell_message(headers={MessageHeaderKey.ORIGIN: requester}, payload=payload) - ) - status = reply.payload.get(_PropKey.STATUS) - if status in (ProduceRC.EOF, ProduceRC.ERROR): - return reply - state = reply.payload.get(_PropKey.STATE) - raise AssertionError("pull loop never reached terminal") - - -def _confirm(service, rid, requester, status): - service._handle_download( - new_cell_message( - headers={MessageHeaderKey.ORIGIN: requester}, payload={_PropKey.REF_ID: rid, _PropKey.CONFIRM: status} - ) - ) - - class TestTransferWaiter: def test_wait_blocks_until_outcome_and_returns_delivered(self): # the load-bearing property: a thread blocked in wait() is released by outcome @@ -95,7 +52,7 @@ def test_wait_blocks_until_outcome_and_returns_delivered(self): t.start() _pull_to_terminal(service, rid, "r1", confirm_capable=True) - _confirm(service, rid, "r1", DownloadStatus.SUCCESS) + service._handle_download(confirm_request(rid, "r1", DownloadStatus.SUCCESS)) run_monitor_once(service, now=time.time()) t.join(5.0) @@ -134,7 +91,9 @@ def test_failed_outcome_is_returned_not_masked(self): waiter = service.get_transfer_waiter(tx_id) _pull_to_terminal(service, rid, "r1", confirm_capable=True) - _confirm(service, rid, "r1", DownloadStatus.FAILED) # receiver truth: finalization failed + service._handle_download( + confirm_request(rid, "r1", DownloadStatus.FAILED) + ) # receiver truth: finalization failed run_monitor_once(service, now=time.time()) outcome = waiter.wait(timeout=5.0) @@ -161,7 +120,7 @@ def test_linger_applies_to_finished_outcomes_only(self): # receiver-failed FINISHED: linger still applied (partial fan-out healing window) tx_id2, rid2 = _new_tx(service) _pull_to_terminal(service, rid2, "r1", confirm_capable=True) - _confirm(service, rid2, "r1", DownloadStatus.FAILED) + service._handle_download(confirm_request(rid2, "r1", DownloadStatus.FAILED)) run_monitor_once(service, now=time.time()) waiter2 = service.get_transfer_waiter(tx_id2) with patch.object(ds_module.time, "sleep") as mock_sleep: From e85d7737f79356a4be29378a04d6d4b460cd3a02 Mon Sep 17 00:00:00 2001 From: YuanTingHsieh Date: Tue, 7 Jul 2026 18:32:11 -0700 Subject: [PATCH 05/20] Remove the implementation-plan doc; design doc stands alone The plan file was a point-in-time PR decomposition, not a durable reference: execution has already deviated from it deliberately (the F3 track shipped as one consolidated PR; the trainer-engine track was re-scoped), and keeping it current would mean plan-only doc churn with every such decision. A stale plan misleads more than no plan; git history preserves it for archaeology. The design doc -- the durable contract reference -- stays, with its plan references decoded into work-item names. --- docs/design/client_api_execution_modes.md | 14 +- .../design/client_api_execution_modes_plan.md | 150 ------------------ 2 files changed, 7 insertions(+), 157 deletions(-) delete mode 100644 docs/design/client_api_execution_modes_plan.md diff --git a/docs/design/client_api_execution_modes.md b/docs/design/client_api_execution_modes.md index d1914338d4..ac8ad80837 100644 --- a/docs/design/client_api_execution_modes.md +++ b/docs/design/client_api_execution_modes.md @@ -2,7 +2,7 @@ ## Status -**Status: Approved** — implementation in progress for 2.9 (see the companion implementation plan, `client_api_execution_modes_plan.md`). Epic: **FLARE-2698** (Client API and 3rd party integration refactoring, fix version 2.9.0). +**Status: Approved** — implementation in progress for 2.9. Epic: **FLARE-2698** (Client API and 3rd party integration refactoring, fix version 2.9.0). Revision 2 (2026-07-01): incorporates review feedback — universal session setup, owner-death and receive-side contracts, forward-path payload lifecycle, cleanup policy definition, receiver-confirmed terminal outcome, per-receiver transfer budgets, configuration surface, attach auth hardening, and a re-sequenced migration plan. @@ -937,7 +937,7 @@ These three are not the same moment — they fire at different workflow stages Two CCWF-specific policies still need to be set (not new transport): - **Fan-out (broadcast/CSE).** The producer stays alive until all N receivers pull; a stuck or dead receiver must not pin it forever. This is enforced with the per-receiver budgets above: each receiver stage gets a workflow-supplied expected-pull window, a receiver that exhausts its budget is marked failed for the aggregate outcome, and the workflow decides whether a partial fan-out (N-1 of N) is a usable result or a task failure. The transaction TTL is the envelope over the stage windows, plus a defined abort path. -- **Disk offload.** Two distinct artifacts must not be conflated. The **producer** serves tensors from its in-memory source (a DownloadService ref, no producer-side disk file); that source is released only on the normalized terminal outcome, which is what keeps the subprocess alive long enough. The **receiver's** offloaded copy (e.g. safetensors written by the receiving side's FOBS decode) is the receiver's own storage, on its own GC lifecycle — it is not gated on the producer's outcome. The historical CCWF fragility was the *producer/subprocess* exiting before the peer finished pulling; that is fixed by the terminal-outcome gate on the producer's source. **Hard dependency:** safely re-enabling disk offload in CCWF requires the receiver-confirmed terminal status (Payload Lifecycle refinement #1, plan PR F3-2) — re-enabling on the current *producer-served EOF* outcome would reintroduce silent truncation (a receiver whose disk write fails after its last pull is invisible to the producer). This is a hard prerequisite, not merely "requires validation." +- **Disk offload.** Two distinct artifacts must not be conflated. The **producer** serves tensors from its in-memory source (a DownloadService ref, no producer-side disk file); that source is released only on the normalized terminal outcome, which is what keeps the subprocess alive long enough. The **receiver's** offloaded copy (e.g. safetensors written by the receiving side's FOBS decode) is the receiver's own storage, on its own GC lifecycle — it is not gated on the producer's outcome. The historical CCWF fragility was the *producer/subprocess* exiting before the peer finished pulling; that is fixed by the terminal-outcome gate on the producer's source. **Hard dependency:** safely re-enabling disk offload in CCWF requires the receiver-confirmed terminal status (Payload Lifecycle refinement #1 — the receiver-confirmed completion wire change) — re-enabling on the current *producer-served EOF* outcome would reintroduce silent truncation (a receiver whose disk write fails after its last pull is invisible to the producer). This is a hard prerequisite, not merely "requires validation." ## Alternatives Considered @@ -1064,13 +1064,13 @@ CCWF (step 6): ### Open Questions -Each open question is tracked to the PR that decides it (plan PR ids); none blocks the interface freezes already landed. +Each open question is tracked to the work item that decides it; none blocks the interface freezes already landed. -- Exact public argument names and migration aliases — decided at the EX-2 interface freeze (the frozen `ClientAPIExecutor` constructor is the decision record) and the EX-3 ScriptRunner wiring. +- Exact public argument names and migration aliases — decided at the `ClientAPIExecutor` interface freeze (the frozen constructor is the decision record) and the ScriptRunner `execution_mode` wiring. - Scheduler batch helper library vs. standard wrapper component — deferred to Migration Plan step 7 (Future Enhancements); not 2.9-blocking. -- Exact factoring of shared code with IPCAgent/IPCExchanger — decided during the trainer-engine PR (TE-4), where the shared session/receive machinery gets its final home. -- Compatibility flag for selecting the legacy Pipe path during migration — decided in EX-3/EP-4: whether legacy selection stays purely class-name-based (per the Compatibility section's keep-legacy story) or also gets an explicit flag. -- Whether partial fan-out (N-1 of N receivers succeeded) is surfaceable to CCWF workflows as a usable result or always a task failure — decided by F3-3's quorum surface (the plan requires F3-3 to settle it) and surfaced to workflows in CC-1. +- Exact factoring of shared code with IPCAgent/IPCExchanger — decided during the trainer-engine work, where the shared session/receive machinery gets its final home. +- Compatibility flag for selecting the legacy Pipe path during migration — decided with the in_process/external_process backend wiring: whether legacy selection stays purely class-name-based (per the Compatibility section's keep-legacy story) or also gets an explicit flag. +- Whether partial fan-out (N-1 of N receivers succeeded) is surfaceable to CCWF workflows as a usable result or always a task failure — decided by the per-receiver budget work's quorum surface (`min_receivers` / `quorum_met`: `completed` stays the strict all-receivers certificate) and surfaced to workflows in the CCWF refactor. ## Migration Plan diff --git a/docs/design/client_api_execution_modes_plan.md b/docs/design/client_api_execution_modes_plan.md deleted file mode 100644 index 68c5631f49..0000000000 --- a/docs/design/client_api_execution_modes_plan.md +++ /dev/null @@ -1,150 +0,0 @@ -# Client API Execution Modes — 2.9 Implementation Plan - -Companion to `client_api_execution_modes.md` (design). Tracks Epic **FLARE-2698** (Client API and 3rd party integration refactoring, 2.9.0). Decomposes the design's 8-step Migration Plan into **36 PRs** with dependencies, sizes, and a release cut line (the wave plan lists 37 entries; PR-0 is not a separate PR — the doc set lands inside the F3-1 PR, #4853). Scoped against the codebase post-2.8.0; granularity calibrated against the repo's merged-PR history (2026-07-01, see Calibration below). - -Size guide: **S** <300 changed LOC, **M** 300–800, **L** 800–2000. - -## Granularity calibration - -Measured against the last ~300 merged PRs on main: median merged PR is ~140 LOC / 4 files; 65% land at ≤300 LOC; only ~13% fall in the 300–800 band. Recent comparable programs shipped as many small PRs over weeks: **recipe API ~28 PRs, tensor-offload/streaming reliability ~20, distributed provisioning/multicloud ~18** — each roughly half this program's scope. So ~36 PRs is in line with how this repo actually ships large features, and the plan's M-heavy granularity is already on the *large* side of repo norms (p75–p85). Consolidating further would create PR shapes the repo rarely merges for core code. - -An adversarial consolidation pass was run on nine candidate merges; four survived (applied below), five were rejected. The pattern: **safe merges are same-owner/same-module dedups of work two tracks scoped twice, or two halves of one contract with no independent consumer. Bad merges drag an early foundation behind a later integration point, or weld a revert-sensitive change (interface freeze, wire-protocol version skew, security fix) to unrelated code.** Deliberate non-merges are listed in Guardrails at the end. - -## Interface freezes (do these first, review across all track owners) - -1. **Protocol vocabulary** (`nvflare/client/cell/defs.py`) — the Cell control-protocol Topics/MsgKeys/CHANNEL/version, consumed by the trainer engine, external_process, and attach. Pure constants, no I/O. (The session-token/HMAC-proof *helpers* are NOT frozen here — see the auth-model note below: they land with EP-3, where the host-trust/auth-strength decision is actually made.) -2. **ClientAPIExecutor skeleton + `ClientAPIBackendSpec`** (`nvflare/app_common/executors/client_api_executor.py`). Freeze the full V1 constructor from the design's Configuration Surface even though only in_process works initially. -3. **F3 aggregate terminal outcome** (`TransferOutcome`, additive `outcome_cb`). Reuse `TransferProgressState` terminal names; `transaction_done_cb` signature untouchable (benign-TIMEOUT callers: `hci/server/binary_transfer.py`, `app_opt/job_launcher/workspace_cell_transfer.py`). - -## Wave plan - -Tracks: **F3** payload layer · **EX** executor/ScriptRunner · **TE** trainer engine · **EP** external_process · **AT** attach · **CC** CCWF · **CT** compat/docs/tests. - -### Wave 0 — foundations (no cross-deps; start immediately, fully parallel) - -| PR | Track | Size | Notes | -|---|---|---|---| -| PR-0 Land design doc + this implementation plan in docs/design/ | all | S | **Lands with F3-1 in a single PR (#4853)**, not as a separate docs-only PR — the reference material merges together with the first frozen contract, and every subsequent PR links it. Interface-freeze sign-offs happen on TE-1/EX-2/F3-1 themselves | -| F3-1 Aggregate all-receivers terminal transfer outcome | F3 | M | Interface freeze #3. Purely additive next to FINISHED/TIMEOUT/DELETED. Ships as #4853, carrying PR-0's docs | -| TE-1 Protocol vocabulary (`client/cell/defs.py`) | TE | S | Interface freeze #1 — the Cell control-protocol Topics/MsgKeys/CHANNEL/version only. The auth **mechanism** is deliberately NOT frozen here: the token is 3 roles (rendezvous, anti-mixup, auth) and whether external_process needs a *secret* HMAC proof at all is a host-trust decision (single-tenant: no; multi-tenant: yes) that EP-3 makes. So the proof helpers (TokenScope, compute/verify_hello_proof, combine_nonces) land with **EP-3**, and the generic generators (generate_session_token/nonce, token_digest) go to the fuel/sec consolidation (**FLARE-3017**). The stateful SessionTokenManager is attach-only (**AT-2**) | -| EX-2 ClientAPIExecutor skeleton + backend spec + analytics-event ownership | EX | M | Interface freeze #2 | -| TE-2 Bootstrap config schema + NVFLARE_CLIENT_API_CONFIG resolution | TE | M | Additive ConfigKeys; consumes EP-1's 0600 writer. Kept separate from TE-1: touches legacy-shared `client/config.py`, different revert profile | -| EP-1 0600 permissions for Client API config files | EP | S | Fixes live exposure in today's `client_api_config.json`; standalone + backportable by design | -| EP-2 TrainerProcessRunner (process-group lifecycle + PGID records) | EP | M | SIGTERM→grace→SIGKILL; preserves SubprocessLauncher's natural-exit window; Windows path platform-guarded | -| EX-1 Export `flare.get_task_name` | EX | S | Ship-today one-liner; kept out of the churn-prone skeleton PR | - -### Wave 1 — parallel tracks behind foundations - -| PR | Track | Size | Depends | -|---|---|---|---| -| F3-2 Receiver-confirmed completion + retry-aware accounting | F3 | M | F3-1. The version-skew wire change — lands as early as possible for maximum soak; capability-flag gated, both skews interop-tested | -| F3-3 Per-(transfer, receiver) acquire/idle budgets | F3 | M | F3-1; blocks F3-4, whose aggregate outcome resolves in bounded time only through these budgets (a stalled receiver is marked failed without waiting for the full transaction TTL). Unconditional per-receiver activity tracking. Must also settle the quorum surface for fan-out: workflows with min_responses-style policy (k-of-N receivers suffices) need either an optional min_receivers on the transaction/facade or a documented pattern of evaluating TransferOutcome.refs against their own threshold — `completed` stays the strict all-receivers certificate either way | -| F3-4 Awaitable producer transfer facade + PAYLOAD_ACQUIRED + post-completion linger | F3 | M | F3-1, F3-2, F3-3 — all hard. The facade's returns-means-delivered guarantee holds only over receiver-confirmed, retry-aware outcomes (F3-2), and its aggregate outcome resolves in bounded time only with the per-receiver budgets (F3-3); a facade over producer-served EOF would reintroduce the silent-truncation gap. Must CHAIN existing DOWNLOAD_COMPLETE_CB, not replace. Folds in the former F3-5 bounded post-completion linger (a leaf with no independent consumer — only this facade's release path uses it); F3-2 still lands as its own isolated PR so the wire-skew change is not welded here | -| EX-3 in_process backend (consolidate InProcessClientAPIExecutor) | EX | L | EX-2. Behavior-parity bar: "nothing user-visible" | -| TE-3 TrainerCellSession engine (handshake, heartbeat, owner-death, trainer-side authenticated teardown) | TE | L | TE-1, TE-2. Injectable clock + kill hook; AT owner co-reviews the teardown-auth tests | -| EP-5 CP-side orphan reaping of trainer PGIDs | EP | M | EP-2. PID-reuse guard via start-time record | - -### Wave 2 — contracts and wiring - -| PR | Track | Size | Depends | -|---|---|---|---| -| TE-4 TrainerCellSession task/result contracts (receive queue + TASK_READY idempotency + TASK_PAYLOAD_READY/TASK_FAILED; RESULT_READY flow + terminal-outcome blocking + fan-out drain/shutdown gating) | TE | L | TE-3; F3-4 via a narrow stubbed wait-protocol interface. Merged from two Ms: same class, same owner, no independent consumer of either half. Guard: split fan-out drain back out if the diff passes ~1800 LOC | -| EP-3 external_process auth + CJ-side HELLO acceptance + session-scoped message enforcement | EP | M | TE-1 (vocabulary). **Owns the auth-model decision**: a per-launch rendezvous id always; a *secret* launch token proven by one-round HMAC on multi-tenant hosts (rendezvous-only + OS isolation may suffice single-tenant). Introduces the proof helpers (TokenScope, compute/verify_hello_proof, combine_nonces) here — reused later by AT-2. Session-scoped message enforcement (accept trainer messages only from the bound session) closes the live IPCAgent any-sender gap; a P0 control, not the P2 attach track | -| EX-4 ScriptRunner `execution_mode` param + launch_external_process mapping | EX | M | EX-2/3. Convert ~22 internal recipe call sites in the same PR | -| CC-1 CCWF transfer-declaration plumbing (receiver sets, stage windows, aux passthrough) | CC | M | F3-3. Declaration-only; absent headers preserve today's defaults — its behavior-neutrality is what de-risks the CC track | -| CT-8 Session observability (state-transition logs + StatsPoolManager view) | CT | M | EX-2; extends as backends land | - -### Wave 3 — integration point - -| PR | Track | Size | Depends | -|---|---|---|---| -| TE-5 CellClientAPI backend (flare.* on the new engine, backend selection) | TE | M | TE-4, TE-2. Opt-in only; legacy defaults untouched | -| EP-4 external_process backend for ClientAPIExecutor | EP | L | EP-2/3, EX-2/3, TE-5, F3-4. The step-4 integration point; watch for XL creep — split dispatch if needed | -| EP-6 Rank contract (torchrun multi-rank, non-control-rank fail-fast) | EP | M | EP-4, TE-5 | -| CT-1 Acceptance-test harness + core external_process suite + simulator/POC smoke | CT | L | EP-4. Absorbs the EP track's E2E validation (incl. POC-mode smoke); EP owner is co-reviewer as the track's sign-off point. tests/integration_test/fast (Blossom premerge) + xdist-safe unit subset | - -### Wave 4 — validation, workflows, attach - -| PR | Track | Size | Depends | -|---|---|---|---| -| EX-5 Converter→filter migration (FLARE-2698 bullet 2) | EX | M | EX-3, EP-4. Replace executor-owned ParamsConverters with PT/TF send+receive conversion filters at the client edge (last task-data / first task-result filter); recipes auto-wire per framework only when server format ≠ exchange format; Client API boundary passes through (RAW). Filters delegate to the existing converter classes (no logic rewrite); round-trip state (tensor shapes, excluded entries) rides DXO meta (quantizer pattern), not FLContext — removes the `_ConverterContext` stub. Keep ParamsConverter ABC deprecated + a `ParamsConverterFilter` adapter for custom converter IDs. Wiring the conversion filter makes the CJ a materializing hop (design doc: payload-lifecycle rule); numpy regime excludes bf16 — large models use pytorch end-to-end. Removes params_exchange_format/server_expected_format/converter-id from the surface (already excluded from EX-2's freeze). Transfer type FULL/DIFF stays in model_registry, decided separately | -| CT-2 torchrun 2-rank rank-contract CI tests (CPU/gloo) | CT | M | CT-1, EP-4. EP owner co-reviews | -| CT-3 Owner-death (CJ-kill) + payload-lifecycle E2E tests | CT | L | CT-1, F3 track, EP-4. Covers CJ-SIGKILL self-termination + CP reaping; EP owner co-reviews | -| CC-2 Swarm onto the transfer contract (remove lazy-ref machinery) | CC | L | CC-1, F3 track, EP-4. Highest-risk PR in the program; maximally isolated revert unit; retires test_lazy_ref_local_aggr / test_msg_root_ttl | -| CC-3 Cyclic onto the transfer contract | CC | S | CC-1, EP-4. Kept out of CC-1 (would drag the behavior-neutral foundation behind EP-4) and out of CC-2 (swarm revert must not drag cyclic); shared broadcast_final_result declaration coordinates behind the CC-1 helper | -| CC-4 CSE / broadcast-best fan-out (N receivers, per-receiver budgets) | CC | L | CC-1/2, TE-4 fan-out drain | -| CC-5 Re-enable CCWF tensor disk offload (terminal-outcome-gated cleanup) | CC | M | CC-2/4. Deliberately last in track; flag experimental in 2.9 | -| AT-2 Attach session manager backend (CJ side) + attach-side session enforcement | AT | L | EX-2, TE-3, EP-3 (reuses EP-3's proof helpers). Adds the stateful `SessionTokenManager` (single-use nonce issuance, attach-window expiry, single-session, invalidation) — required because attach's token is delivered out-of-band by an untrusted starter over a possibly-remote channel, unlike external_process's localhost launch | -| AT-3 Trainer-side attach flow (bootstrap config, ad-hoc cell, HELLO proof) | AT | M | AT-2, TE-5. Connects via CP parent_url (ad-hoc listeners default-disabled) | -| CT-5 Rank-contract example updates (multi-gpu/pt, pt-ddp-docker) + SLURM batch-artifact example | CT | M | EP-4. multi-gpu/pt violates the rank contract today; qwen3-vl is the reference | - -### Wave 5 — release gate - -| PR | Track | Size | Depends | -|---|---|---|---| -| AT-4 Attach hardening (rate limit, bounded proof attempts, reconnect rotation) + E2E smoke + job-config example + delivery docs | AT | L | AT-2/3. Merged from two Ms: same owner, sequential, both inside the P2 tail so they slip together. Full negative-case matrix lives once in CT-4, not here | -| CT-4 Attach + CCWF system-level acceptance suites | CT | L | CT-1, AT track, CC track. Owns the attach negative-case matrix (replay, duplicate attach_id, spoofed teardown) — written once here, deduped from AT-4. CCWF multi-site runs in slow/ (nightly) | -| CT-6 Client API docs overhaul (client_api.rst rewrite, 3rd-party/agent docs) | CT | L | Arg names frozen (steps 3–5 merged). Written against merged code, not the design doc | -| CT-7 Legacy-stack deprecation warnings | CT | S | Coverage gate met. Warnings only in 2.9; the ScriptRunner **default flip is a separate 2.10 PR** by design (different ship dates) | - -## Program PR conventions - -Every PR in the program uses this description skeleton so reviewers always have the map: - -```markdown -## What - - -## Program context -Client API Execution Modes (2.9) — PR , Wave of the plan. -Design: docs/design/client_api_execution_modes.md § -Plan: docs/design/client_api_execution_modes_plan.md -Depends on: · Unblocks: - -## Design contracts implemented - - -## Out of scope (and where it lands instead) - -``` - -Notes: the one-paragraph "why" for the whole program lives in PR-0 and gets linked, not pasted. Interface-freeze PRs (TE-1, EX-2, F3-1) additionally name the tracks that consume the frozen surface and require sign-off from those owners before merge. - -## Critical path - -F3-1 → F3-2/F3-3 (parallel) → F3-4 → TE-4 → TE-5 → EP-4 → CT-1 → CT-3 → release gate. F3-2 and F3-3 run in parallel behind F3-1, so the facade's hard dependencies cost the path one M — still inside the 10–14-week floor. The TE-4 merge removed one review cycle from the path. **EP-4** remains the schedule risk to watch — it's where executor, trainer engine, auth, process runner, and the F3 facade meet; its prerequisites are deliberately extracted to keep it L. - -## 2.9 cut line (recommendation) - -- **P0 — commit for 2.9** (24 PRs): F3 track (4, after folding F3-5 into F3-4) + EX track (4 — EX-1..4; EX-5 goes to P1) + TE track (5) + EP track (6) + CT-1/2/3/6/8. Headline: *external_process and in_process on the new Cell stack with an enforceable payload lifecycle, owner-death handling, session-scoped message enforcement, and the config-permission fix*. -- **P1 — strongly target for 2.9** (8 PRs): CC track (5) + EX-5 conversion-filter migration (2.9 runs PyTorch end-to-end without it — the numpy regime is what needs the filters) + CT-5 rank-contract example updates (shipped examples must demonstrate the new contract; they do not gate the code cut) + the CCWF half of CT-4. Fully severable (SWARM keeps current behavior until CC-2 lands); ship CC-5 flagged experimental. -- **P2 — stretch for 2.9, else 2.9.x/2.10** (4 PRs): AT track (3) + the attach half of CT-4. Attach is the most self-contained step; nothing in P0/P1 depends on it (its reusable auth/proof helpers ship in P0 via EP-3, and session enforcement via EP-3, regardless). -- **CT-7 warnings** (1 PR) close 2.9. **Explicitly deferred to 2.10**: the ScriptRunner default flip and any legacy-class removal — ship 2.9 opt-in with warnings; flip after a release of field soak. -- Tier accounting: every one of the 36 PRs has a tier — P0 (24) + P1 unique (7: CC×5, EX-5, CT-5) + P2 unique (3: AT×3) + CT-4 (one PR, halves in P1 and P2) + CT-7 (1) = 36. The P1/P2 headline counts each mention CT-4 once, so the tier lines name 37 items for 36 PRs. - -## Effort and staffing - -36 PRs. At review-inclusive rates (S ≈ 2 days, M ≈ 1 week, L ≈ 2 weeks) that is ≈ 44 engineer-weeks total; ≈ 36 for P0+P1. With 4–5 engineers owning tracks (F3, TE, EP+EX, CC, AT+CT), full parallelism after Wave 0; calendar floor is the critical path — realistically **10–14 weeks to P0+P1 complete** plus stabilization. If 2.9 code freeze is nearer, cut P2 first, then CC-4/CC-5. - -## Guardrails — deliberate non-merges - -These pairs are tempting to consolidate and must stay separate: - -- **EP-1 (0600 fix) with anything**: live credential exposure on today's path; standalone, cherry-pickable, surgical revert. -- **EX-3 (in_process backend) + EX-4 (ScriptRunner)**: ScriptRunner is the most-used public entry point (golden exported-config tests, ~22 recipe conversions); a backend parity revert must not drag the public parameter surface. -- **Anything into CC-2 (swarm refactor)**: highest-risk PR in the program; must remain a maximally isolated revert unit. -- **F3-3 (budgets) + F3-4 (facade)**: same module and strictly sequenced (F3-3 is a hard prerequisite of F3-4), but they stay separate PRs — the budgets land ahead as their own surgical revert unit, and F3-4 is the critical-path PR every track waits on. -- **F3-2 (receiver-confirm wire change) must not weld to F3-4's linger/exit-timing policy**: the version-skew change wants early landing and a surgical revert; the linger now lives in F3-4 (former F3-5) but F3-2 still ships isolated ahead of it. -- **EP-3 + AT-2**: same CJ-side HELLO/session machinery, but they straddle the P0/P2 boundary — AT-2 consumes EP-3's module instead. -- **EP-2 (runner) + EP-5 (CP reaping)**: different processes and blast radii; EP-5's false-positive-reap risk needs an independent revert path. -- **CT-7 warnings + default flip**: different ship dates by design (2.9 vs 2.10). - -## Cross-cutting risks and constraints (from code scouting) - -- **CI**: GitHub premerge is ubuntu-only, CPU-only, xdist — all acceptance tests must be CPU-feasible and port-dynamic; integration tests run only via Blossom (`/build`, fast/ premerge, slow/ nightly); fork PRs get no integration signal, so protocol contracts need cheap unit-level duplicates. torchrun tests: gloo only; multi-GPU/nccl validation is manual. -- **No Windows CI**: process-group termination Windows path ships platform-guarded with skipped tests — flag as a release-note caveat. -- **Version-skew interop** (F3-2): capability-flag gating with explicit old-producer/new-receiver and new-producer/old-receiver tests; confirm sends fire-and-forget. -- **Behavior parity**: in_process consolidation and ScriptRunner wiring have golden-config tests (exported job JSON byte-compare) to protect the most-used public entry point. -- **0600 writer is single-sourced**: EP-1 owns the hardened writer; TE-2 and attach bootstrap tests consume it rather than re-implementing permission handling. -- **Pass-through registration ownership** moves from ClientAPILauncherExecutor.initialize() into the new backend — coordinate EP-4 with CC-1 to avoid double registration during migration. -- **Trainer exit hang**: FlareAgent's atexit/main-thread-join watcher for non-daemon F3 threads solves a real problem (scripts that never call flare.shutdown()); TE-3/TE-5 must carry an equivalent or trainers hang at exit. From 4ff559c09aa8db981cb18327a3b602b44f7eba43 Mon Sep 17 00:00:00 2001 From: YuanTingHsieh Date: Wed, 8 Jul 2026 16:39:05 -0700 Subject: [PATCH 06/20] Rename _tx_incarnations to _outcome_owners; owner language throughout Behavior-free rename for clarity. The map's purpose is ownership: the transaction object currently entitled to record the outcome for its tx_id. tx_ids are deliberately reused by retries (design: stable transfer ids make retries idempotent), so recording checks object identity against this map -- "if owner is not tx: drop" now reads as what it means. "Incarnation" named the mechanism; "owner" names the purpose. Comments, docstrings, and test names updated; ref-level wording uses "life of the ref" (refs have no owner table; the pending-confirm guard plays that role). --- nvflare/fuel/f3/streaming/download_service.py | 61 ++++++++++--------- .../f3/streaming/download_service_test.py | 34 +++++------ .../fuel/f3/streaming/download_test_utils.py | 2 +- .../f3/streaming/receiver_confirm_test.py | 2 +- .../f3/streaming/transfer_outcome_test.py | 26 ++++---- 5 files changed, 64 insertions(+), 61 deletions(-) diff --git a/nvflare/fuel/f3/streaming/download_service.py b/nvflare/fuel/f3/streaming/download_service.py index 22ebf20937..63b78121d7 100644 --- a/nvflare/fuel/f3/streaming/download_service.py +++ b/nvflare/fuel/f3/streaming/download_service.py @@ -269,7 +269,7 @@ def _finalize_receiver(self, to_receiver: str, status: str, require_pending: boo return False if require_pending and to_receiver not in self._pending_confirms: # a legitimate confirmation always follows a provisional terminal serve on - # THIS incarnation of the ref; an unsolicited/stale confirm (e.g. delayed + # the CURRENT life of this ref; an unsolicited/stale confirm (e.g. delayed # across a ref_id reuse) must not certify -- or poison -- this transfer self.tx.logger.warning(f"dropping unsolicited confirmation from {to_receiver} for ref {self.rid}") return False @@ -331,7 +331,7 @@ def obj_confirmed(self, to_receiver: str, status: str) -> bool: """Records the receiver-confirmed terminal status. Receiver truth wins; first confirm is final. Accepted only when a provisional serve is pending for this receiver on THIS - incarnation of the ref -- unsolicited or stale confirmations are dropped, so a + life of the ref -- unsolicited or stale confirmations are dropped, so a delayed confirm from a previous life of a reused ref_id can neither falsely certify nor pre-poison the new transfer. """ @@ -922,10 +922,13 @@ class DownloadService: # time so producers can query the aggregate result after termination. Guarded by # its own lock so outcome polling never contends with the chunk-serving _tx_lock. _tx_outcomes = {} - # Current live incarnation per tx_id (registered by new_transaction), so a - # transaction that terminates concurrently with a same-id retry cannot record - # its outcome over the new incarnation. Guarded by _outcome_lock. - _tx_incarnations = {} + # The transaction object currently entitled to record the outcome for its tx_id + # (registered by new_transaction). tx_ids are deliberately reused by retries + # (design: stable transfer ids make retries idempotent), so recording checks + # OBJECT identity against this map: a transaction that terminates concurrently + # with a same-id retry is no longer the owner and cannot record its stale outcome + # over the live one. Guarded by _outcome_lock. + _outcome_owners = {} # Waiters blocked on a transaction's terminal outcome (the awaitable facade). Guarded by # _outcome_lock; resolved inside _record_outcome so a waiter can never miss the outcome. _tx_waiters = {} @@ -989,15 +992,15 @@ def new_transaction( receiver_idle_timeout=receiver_idle_timeout, ) with cls._outcome_lock: - # a reused explicit tx_id must not surface the previous incarnation's - # outcome: purge any recorded outcome and register this incarnation as - # current so a concurrently-terminating older incarnation cannot record. - # Registration happens BEFORE the tx becomes monitor-visible in _tx_table: - # a tx the monitor can terminate is always incarnation-registered, so its - # terminal outcome can never be dropped as stale, and no dead incarnation + # a reused explicit tx_id must not surface the previous owner's outcome: + # purge any recorded outcome and register this transaction as the outcome + # owner, so a concurrently-terminating previous owner cannot record. + # Ownership is taken BEFORE the tx becomes monitor-visible in _tx_table: + # a tx the monitor can terminate always owns its outcome slot, so its + # terminal outcome can never be dropped as stale, and no dead owner # entry can be left behind by a terminate-before-register interleaving. cls._tx_outcomes.pop(tx.tid, None) - cls._tx_incarnations[tx.tid] = tx + cls._outcome_owners[tx.tid] = tx old_tx = None with cls._tx_lock: @@ -1013,8 +1016,8 @@ def new_transaction( if old_tx: # terminal callbacks run outside the lock, as in delete_transaction(); the - # retired transaction's outcome is dropped by the incarnation guard because - # the new incarnation already owns the tid -- the retry is authoritative + # retired transaction's outcome is dropped by the owner guard because the + # new transaction already owns the tid -- the retry is authoritative old_tx.transaction_done( TransactionDoneStatus.DELETED, on_outcome=functools.partial(cls._record_outcome, tx=old_tx) ) @@ -1066,12 +1069,12 @@ def shutdown(cls): cls._finished_refs.clear() with cls._outcome_lock: - # drop recorded outcomes and clear live incarnations. A monitor iteration - # mid-termination that blocked on _outcome_lock finds its incarnation gone + # drop recorded outcomes and clear outcome ownership. A monitor iteration + # mid-termination that blocked on _outcome_lock finds its ownership gone # once it acquires the lock, so its outcome drops in _record_outcome -- - # the incarnation guard alone gates post-shutdown recording. + # the owner guard alone gates post-shutdown recording. cls._tx_outcomes.clear() - cls._tx_incarnations.clear() + cls._outcome_owners.clear() # unblock the awaitable facade: waiters resolve to None (service shut down # before the transaction terminated), never hang for waiters in cls._tx_waiters.values(): @@ -1114,11 +1117,11 @@ def get_transfer_waiter(cls, transaction_id: str) -> TransferWaiter: # resolve even from an expired record: it is still the recorded truth waiter._resolve(existing) return waiter - if transaction_id not in cls._tx_incarnations: + if transaction_id not in cls._outcome_owners: # unknown, already-forgotten (outcome expired) or shut down: nothing will # ever record an outcome for this id, so resolving with None immediately is # the only way to honor "waiters can never hang". Race-free: _record_outcome - # swaps incarnation -> outcome under this same lock. + # swaps ownership -> recorded outcome under this same lock. waiter._resolve(None) return waiter cls._tx_waiters.setdefault(transaction_id, []).append(waiter) @@ -1137,20 +1140,20 @@ def get_acquired_receivers(cls, transaction_id: str) -> set: @classmethod def _record_outcome(cls, outcome: TransferOutcome, tx: _Transaction): - # tx is required so no call site can opt out of the incarnation guard: - # recording is legal only for the live registered incarnation. + # tx is required so no call site can opt out of the owner guard: + # recording is legal only for the transaction that owns the outcome slot. with cls._outcome_lock: - if cls._tx_incarnations.get(outcome.tx_id) is not tx: + if cls._outcome_owners.get(outcome.tx_id) is not tx: # This outcome belongs to a dead generation and must be dropped: - # - a newer same-tx_id incarnation (a retry) registered while this - # transaction was terminating, or - # - the incarnation was cleared by shutdown() (which also clears the + # - a newer same-tx_id transaction (a retry) took ownership while this + # one was terminating, or + # - ownership was cleared by shutdown() (which also clears the # outcome table) or consumed by a prior terminal record. # This single guard closes the stale-outcome race across shutdown and # re-init: a recorder that blocked on _outcome_lock during shutdown and - # wins the lock afterward finds no live incarnation and drops here. + # wins the lock afterward finds it no longer owns the slot and drops here. return - cls._tx_incarnations.pop(outcome.tx_id, None) + cls._outcome_owners.pop(outcome.tx_id, None) cls._tx_outcomes[outcome.tx_id] = outcome # resolve the awaitable facade: waiters are TransferWaiter objects (no user code # runs in _resolve), so setting them under the lock is safe and race-free diff --git a/tests/unit_test/fuel/f3/streaming/download_service_test.py b/tests/unit_test/fuel/f3/streaming/download_service_test.py index b15c642a73..e37059eb29 100644 --- a/tests/unit_test/fuel/f3/streaming/download_service_test.py +++ b/tests/unit_test/fuel/f3/streaming/download_service_test.py @@ -476,14 +476,14 @@ def register_request_cb(self, **kwargs): assert list(service._initialized_cells.keys()) == [] - def test_new_transaction_registers_incarnation_before_monitor_visible(self): - """A tx must be incarnation-registered before it appears in _tx_table. + def test_new_transaction_takes_ownership_before_monitor_visible(self): + """A tx must own its outcome slot before it appears in _tx_table. The monitor discovers transactions through _tx_table. If a tx were inserted - there first, a monitor tick landing in the window before incarnation - registration could terminate it: its terminal outcome would be dropped by the - live-incarnation guard (a terminated-but-unknown gap), and new_transaction - would then register a dead incarnation that nothing ever pops. + there first, a monitor tick landing in the window before ownership is taken + could terminate it: its terminal outcome would be dropped by the owner guard + (a terminated-but-unknown gap), and new_transaction would then register a + dead owner entry that nothing ever pops. """ service = _make_isolated_download_service() service._tx_monitor = object() # avoid starting a real monitor thread @@ -492,7 +492,7 @@ def test_new_transaction_registers_incarnation_before_monitor_visible(self): class MonitorVisibilityDict(dict): def __setitem__(self, key, value): - registered_at_insert.append(key in service._tx_incarnations) + registered_at_insert.append(key in service._outcome_owners) super().__setitem__(key, value) service._tx_table = MonitorVisibilityDict() @@ -500,16 +500,16 @@ def __setitem__(self, key, value): tx_id = service.new_transaction(cell=Mock(), timeout=10.0, num_receivers=1) assert registered_at_insert == [True] - assert service._tx_incarnations[tx_id] is service._tx_table[tx_id] + assert service._outcome_owners[tx_id] is service._tx_table[tx_id] - def test_stale_outcome_dropped_after_incarnation_cleared(self): - """A terminal outcome for a transaction whose incarnation is no longer live must drop. + def test_stale_outcome_dropped_after_ownership_cleared(self): + """A terminal outcome for a transaction that no longer owns its tx_id must drop. This is the invariant that closes the cross-lifecycle stale-outcome race: - _record_outcome() records only for the live registered incarnation (current is tx). + _record_outcome() records only for the transaction that owns the outcome slot. A recorder that blocked on _outcome_lock while shutdown() cleared the tables can - win the lock after a subsequent _initialize(); with no live incarnation for the - tid, its pre-shutdown outcome drops instead of repopulating the cleared table. + win the lock after a subsequent _initialize(); no longer owning the slot, its + pre-shutdown outcome drops instead of repopulating the cleared table. """ service = _make_isolated_download_service() @@ -519,20 +519,20 @@ def test_stale_outcome_dropped_after_incarnation_cleared(self): old_outcome.tx_id = "tx-old" old_outcome.expired.return_value = False - # shutdown() cleared incarnations; recording is gated by incarnation identity alone - service._tx_incarnations.clear() + # shutdown() cleared ownership; recording is gated by owner identity alone + service._outcome_owners.clear() # the late callback for the old, now-unregistered transaction must be dropped service._record_outcome(old_outcome, tx=old_tx) assert service.get_transaction_outcome("tx-old") is None - # sanity: a live registered incarnation still records (guard does not over-drop) + # sanity: the owning transaction still records (guard does not over-drop) live_tx = Mock() live_tx.tid = "tx-live" live_outcome = Mock() live_outcome.tx_id = "tx-live" live_outcome.expired.return_value = False - service._tx_incarnations["tx-live"] = live_tx + service._outcome_owners["tx-live"] = live_tx service._record_outcome(live_outcome, tx=live_tx) assert service.get_transaction_outcome("tx-live") is live_outcome diff --git a/tests/unit_test/fuel/f3/streaming/download_test_utils.py b/tests/unit_test/fuel/f3/streaming/download_test_utils.py index b9939e798c..7064f1fbcc 100644 --- a/tests/unit_test/fuel/f3/streaming/download_test_utils.py +++ b/tests/unit_test/fuel/f3/streaming/download_test_utils.py @@ -85,7 +85,7 @@ class IsolatedDownloadService(DownloadService): _ref_table = {} _finished_refs = {} _tx_outcomes = {} - _tx_incarnations = {} + _outcome_owners = {} _tx_waiters = {} _outcome_lock = threading.Lock() _logger = Mock() diff --git a/tests/unit_test/fuel/f3/streaming/receiver_confirm_test.py b/tests/unit_test/fuel/f3/streaming/receiver_confirm_test.py index e64842a857..ab6f390ac5 100644 --- a/tests/unit_test/fuel/f3/streaming/receiver_confirm_test.py +++ b/tests/unit_test/fuel/f3/streaming/receiver_confirm_test.py @@ -163,7 +163,7 @@ def test_invalid_confirm_status_is_ignored(self): def test_unsolicited_confirm_cannot_certify(self): # fail-open hole guard: a CONFIRM for a receiver that was never served a terminal - # reply on THIS incarnation of the ref must be dropped -- otherwise a stale confirm + # reply in this life of the ref must be dropped -- otherwise a stale confirm # delayed across a ref_id reuse could certify (or pre-poison) a transfer that never # delivered a byte service = _make_service() diff --git a/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py b/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py index acb90d1f4f..b1bde7fcac 100644 --- a/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py +++ b/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py @@ -240,11 +240,11 @@ def _add_tx(self, service, num_receivers=1, timeout=10.0): tx = _Transaction(timeout=timeout, num_receivers=num_receivers) with service._tx_lock: service._tx_table[tx.tid] = tx - # mirror new_transaction(): a live tx is registered as the current incarnation. - # _record_outcome() records only for the live incarnation (current is tx), so a tx - # placed directly in _tx_table without this would never record its terminal outcome. + # mirror new_transaction(): a live tx owns its outcome slot. + # _record_outcome() records only for the owning tx, so a tx placed directly in + # _tx_table without ownership would never record its terminal outcome. with service._outcome_lock: - service._tx_incarnations[tx.tid] = tx + service._outcome_owners[tx.tid] = tx obj = _stub_obj() rid = service.add_object(tx.tid, obj) with service._tx_lock: @@ -323,14 +323,14 @@ def test_reused_tx_id_does_not_surface_stale_outcome(self): second = service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-REUSE") assert second == "TX-REUSE" - # the stale terminal outcome of the previous incarnation is purged + # the stale terminal outcome of the previous owner is purged assert service.get_transaction_outcome("TX-REUSE") is None finally: service.shutdown() - def test_stale_incarnation_cannot_record_over_live_retry(self): + def test_stale_owner_cannot_record_over_live_retry(self): # the record-after-purge race: termination removes the old tx from _tx_table, - # a retry registers the same tx_id, THEN the old incarnation records its + # a retry takes ownership of the same tx_id, THEN the old transaction records its # outcome — it must not shadow the live retry from functools import partial from unittest.mock import Mock @@ -343,10 +343,10 @@ def test_stale_incarnation_cannot_record_over_live_retry(self): with service._tx_lock: old_tx = service._tx_table.pop("TX-RACE") # termination step 1, as the monitor does - # the retry registers the same id before the old incarnation records + # the retry takes ownership of the id before the old transaction records service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-RACE") - # the old incarnation now finishes terminating and tries to record + # the old transaction now finishes terminating and tries to record old_tx.transaction_done( TransactionDoneStatus.DELETED, on_outcome=partial(service._record_outcome, tx=old_tx) ) @@ -387,8 +387,8 @@ def test_reusing_active_tx_id_retires_prior_transaction(self): assert done == [("TX-DUP", TransactionDoneStatus.DELETED)] # its terminal outcome does not shadow the live retry assert service.get_transaction_outcome("TX-DUP") is None - # the live tx is the new incarnation - assert service._tx_table["TX-DUP"] is service._tx_incarnations["TX-DUP"] + # the live tx is the new owner + assert service._tx_table["TX-DUP"] is service._outcome_owners["TX-DUP"] assert service._tx_table["TX-DUP"] is not None finally: service.shutdown() @@ -428,8 +428,8 @@ def test_shutdown_clears_outcomes_and_stops_recording(self): assert service.get_transaction_outcome(tx.tid) is None # a monitor iteration that was mid-termination during shutdown cannot repopulate: - # shutdown cleared _tx_incarnations, so the late recorder's tx is not the live - # incarnation and its outcome drops + # shutdown cleared outcome ownership, so the late recorder's tx no longer owns + # the slot and its outcome drops from unittest.mock import Mock late = compute_transfer_outcome("T-LATE", TransactionDoneStatus.FINISHED, 1, [], time.time()) From 5647cca695dcf9954fa96723e86ac1974e2c83bd Mon Sep 17 00:00:00 2001 From: YuanTingHsieh Date: Wed, 8 Jul 2026 16:53:22 -0700 Subject: [PATCH 07/20] Remove plan-id tags from comments and docstrings F3-2/F3-3/F3-4 and "plan:" references decoded against an implementation-plan document that no longer exists; the feature names say what the ids only pointed at. Comments now name the mechanisms directly (receiver-confirmed completion, per-receiver budgets, awaitable transfer facade). --- nvflare/fuel/f3/streaming/download_service.py | 14 +++++++------- nvflare/fuel/f3/streaming/transfer_outcome.py | 5 +++-- .../fuel/utils/fobs/decomposers/via_downloader.py | 2 +- .../fuel/f3/streaming/receiver_budget_test.py | 2 +- .../fuel/f3/streaming/receiver_confirm_test.py | 2 +- .../fuel/f3/streaming/transfer_waiter_test.py | 4 ++-- 6 files changed, 15 insertions(+), 14 deletions(-) diff --git a/nvflare/fuel/f3/streaming/download_service.py b/nvflare/fuel/f3/streaming/download_service.py index 63b78121d7..db74576cba 100644 --- a/nvflare/fuel/f3/streaming/download_service.py +++ b/nvflare/fuel/f3/streaming/download_service.py @@ -176,7 +176,7 @@ class _PropKey: STATE = "state" DATA = "data" STATUS = "status" - # Receiver-confirmed completion (F3-2). All three keys are OPTIONAL on the wire so both + # Receiver-confirmed completion. All three keys are OPTIONAL on the wire so both # version skews interop with legacy peers: an old receiver never sends CONFIRM_CAPABLE and # gets today's producer-served semantics; an old producer never sends CONFIRM_EXPECTED so a # new receiver never confirms toward it. @@ -208,7 +208,7 @@ def _receiver_confirm_enabled() -> bool: return _receiver_confirm_cached -# Per-(transfer, receiver) budgets (F3-3). System defaults resolved from config vars; explicit +# Per-(transfer, receiver) budgets. System defaults resolved from config vars; explicit # per-transaction values win. None (unset everywhere) disables enforcement for that budget -- # the whole-transaction timeout then remains the only backstop, exactly today's behavior. RECEIVER_ACQUIRE_TIMEOUT_CONFIG_VAR = "streaming_receiver_acquire_timeout" @@ -242,7 +242,7 @@ def __init__( # producer-served terminal statuses awaiting the receiver's confirmation; only # finalized (confirmed or legacy-served) statuses live in receiver_statuses self._pending_confirms = {} - # unconditional per-receiver liveness (F3-3): receiver -> last activity timestamp, + # unconditional per-receiver liveness: receiver -> last activity timestamp, # updated on every request regardless of whether a progress_cb is configured -- so a # live receiver can no longer mask a stalled one behind the tx-wide last_active_time self._receiver_activity = {} @@ -614,7 +614,7 @@ def __init__( self.tid = "T" + str(uuid.uuid4()) self.timeout = timeout - # Expected receiver identities (F3-3). Optional: when provided they enable the acquire + # Expected receiver identities. Optional: when provided they enable the acquire # budget (a receiver that never issues its first pull can be failed) and, if # num_receivers is unknown (0), supply the receiver count. if receiver_ids: @@ -850,7 +850,7 @@ def __init__(self, tx: _Transaction): class TransferWaiter: - """The awaitable facade over a transaction's terminal transfer outcome (F3-4). + """The awaitable facade over a transaction's terminal transfer outcome. This is the "returns == delivered" primitive the upper layers (executor backends, trainer engine) consume: wait() blocks -- event-driven, no polling -- until the @@ -1105,7 +1105,7 @@ def _delete_tx(cls, tx: _Transaction, tombstone_finished_refs: bool = False): @classmethod def get_transfer_waiter(cls, transaction_id: str) -> TransferWaiter: - """Returns an awaitable facade over the transaction's terminal outcome (F3-4). + """Returns an awaitable facade over the transaction's terminal outcome. Safe to call before or after termination: a waiter created after the outcome was recorded resolves immediately from the outcome table. @@ -1348,7 +1348,7 @@ def _monitor_tx(cls): while True: now = time.time() - # Per-receiver budget enforcement (F3-3) runs OUTSIDE _tx_lock: finalizing a + # Per-receiver budget enforcement runs OUTSIDE _tx_lock: finalizing a # budget-failed receiver fires user callbacks (downloaded_to_one/all), which must # never run under the global lock. A budget failure recorded here flips # is_finished() so the classification pass below resolves the tx immediately. diff --git a/nvflare/fuel/f3/streaming/transfer_outcome.py b/nvflare/fuel/f3/streaming/transfer_outcome.py index d926119b17..0fa5eaf31f 100644 --- a/nvflare/fuel/f3/streaming/transfer_outcome.py +++ b/nvflare/fuel/f3/streaming/transfer_outcome.py @@ -31,7 +31,8 @@ Outcome status values reuse the TransferProgressState terminal vocabulary (completed / failed / aborted) rather than introducing another status set. -Semantics with the full payload layer (F3-2/F3-3/F3-4, same design): +Semantics with the full payload layer (receiver-confirmed completion, per-receiver +budgets, awaitable transfer facade): - per-receiver statuses are receiver-confirmed where the receiver supports it (a served EOF is provisional until the receiver confirms its finalization succeeded); legacy receivers remain producer-served — both skews and the runtime kill-switch degrade to @@ -130,7 +131,7 @@ class TransferOutcome: num_receivers: int refs: Tuple[RefOutcome, ...] timestamp: float - # optional k-of-N quorum declared by the workflow (F3-3): informational for quorum_met; + # optional k-of-N quorum declared by the workflow: informational for quorum_met; # `completed` deliberately stays the strict all-receivers certificate min_receivers: Optional[int] = None diff --git a/nvflare/fuel/utils/fobs/decomposers/via_downloader.py b/nvflare/fuel/utils/fobs/decomposers/via_downloader.py index 00b578a636..b461e3319d 100644 --- a/nvflare/fuel/utils/fobs/decomposers/via_downloader.py +++ b/nvflare/fuel/utils/fobs/decomposers/via_downloader.py @@ -447,7 +447,7 @@ def _create_downloader( transaction_done_cb=on_complete_cb, progress_cb=progress_cb, progress_interval=RESULT_UPLOAD_PROGRESS_INTERVAL, - # expected receiver identities (F3-3): enables the transaction's per-receiver + # expected receiver identities: enables the transaction's per-receiver # acquire budget; None when any identity is unknown receiver_ids=receiver_ids, ) diff --git a/tests/unit_test/fuel/f3/streaming/receiver_budget_test.py b/tests/unit_test/fuel/f3/streaming/receiver_budget_test.py index 14af215cff..9e5d1aa9f7 100644 --- a/tests/unit_test/fuel/f3/streaming/receiver_budget_test.py +++ b/tests/unit_test/fuel/f3/streaming/receiver_budget_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Tests for per-(transfer, receiver) acquire/idle budgets and the quorum surface (plan: F3-3). +"""Tests for per-(transfer, receiver) acquire/idle budgets and the quorum surface. A receiver that exhausts its acquire budget (never issued its first pull) or idle budget (stopped making requests) is finalized FAILED for the aggregate outcome on a monitor pass -- diff --git a/tests/unit_test/fuel/f3/streaming/receiver_confirm_test.py b/tests/unit_test/fuel/f3/streaming/receiver_confirm_test.py index ab6f390ac5..3b5ac44e70 100644 --- a/tests/unit_test/fuel/f3/streaming/receiver_confirm_test.py +++ b/tests/unit_test/fuel/f3/streaming/receiver_confirm_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Tests for receiver-confirmed completion + retry-aware accounting (plan: F3-2). +"""Tests for receiver-confirmed completion + retry-aware accounting. Producer side: a confirm-capable receiver's served EOF/ERROR is PROVISIONAL; the receiver's confirmation finalizes it (receiver truth wins, first confirm is final, retries overwrite diff --git a/tests/unit_test/fuel/f3/streaming/transfer_waiter_test.py b/tests/unit_test/fuel/f3/streaming/transfer_waiter_test.py index 72d6cc9722..899c6fde70 100644 --- a/tests/unit_test/fuel/f3/streaming/transfer_waiter_test.py +++ b/tests/unit_test/fuel/f3/streaming/transfer_waiter_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Tests for the awaitable transfer facade (plan: F3-4). +"""Tests for the awaitable transfer facade (TransferWaiter). TransferWaiter is the "returns == delivered" primitive the upper layers (executor backends, trainer engine) consume: wait() blocks -- event-driven, resolved inside the outcome-recording @@ -180,7 +180,7 @@ def test_acquired_receivers_reflects_first_pulls(self): assert service.get_acquired_receivers(tx_id) == {"r1"} def test_budget_failure_resolves_waiter_in_bounded_time(self): - # end-to-end with F3-3: a stalled receiver's budget failure terminates the tx and + # end-to-end with the budgets: a stalled receiver's budget failure terminates the tx and # releases the waiter -- the producer is never pinned to the full TTL service = _make_service() tx_id = service.new_transaction(cell=Mock(), timeout=1000.0, num_receivers=2, receiver_idle_timeout=5.0) From ea7a181c8ca2135eb72f0a92563ff763593c2749 Mon Sep 17 00:00:00 2001 From: YuanTingHsieh Date: Wed, 8 Jul 2026 18:24:56 -0700 Subject: [PATCH 08/20] Enforce receiver identities; bind confirmations to serves; settle before resolving waiters Fixes a reproduced review batch on the F3 payload layer. Declared receiver identities are now enforced end to end. Completion (is_finished, the downloaded_to_all latch), the aggregate outcome, and the quorum are judged against receiver_ids when declared: a status from an unexpected receiver can no longer complete -- or certify -- a transfer that a declared receiver never got. Count-based semantics remain only when identities are unknown. (Also makes a ref-less transaction never "finished".) Confirmations are bound to their serve by a per-serve nonce. The terminal reply carries CONFIRM_NONCE; the confirmation must echo it. The pending-entry check alone could not distinguish ref lives: a delayed confirmation from a previous life of a reused ref_id was accepted whenever the new life had its own pending serve for the same receiver. Confirmations also now honor the download's secure flag. Ownership and table registration are one atomic step (_tx_lock nesting _outcome_lock; no reverse nesting exists). Two concurrent same-id constructors could previously interleave across the separate critical sections, leaving the live transaction without outcome ownership while the retired one recorded its DELETED outcome as the live attempt's verdict. Outcomes are recorded -- and waiters released -- only after the transaction settles: terminal progress, done/outcome callbacks, and source release all complete before wait() can return, so an upper layer that stops the producer on wait() return can never preempt them. Idle budgets judge transaction-level activity: a receiver that finished one ref and went silent was exempt from a sibling ref's acquire budget (tx-acquired) while having no per-ref idle timestamp there -- escaping both budgets and pinning the producer to the full TTL. Superseded transactions (tx_id reuse retirement) release their sources but suppress transfer-facing emissions (terminal progress, transaction_done_cb, outcome_cb): their reused tx_id names the live retry, and consumers keyed by tx_id would misattribute them. Regression pins for every reproduced scenario: unexpected-receiver certification, cross-life stale confirmation (with and without a new pending serve), concurrent same-id creation hammer, settle-before-wait-return, multi-ref budget escape, superseded-callback silence, secure confirmations. --- nvflare/fuel/f3/streaming/download_service.py | 265 +++++++++++------- nvflare/fuel/f3/streaming/transfer_outcome.py | 26 +- .../fuel/f3/streaming/download_test_utils.py | 14 +- .../fuel/f3/streaming/receiver_budget_test.py | 82 +++++- .../f3/streaming/receiver_confirm_test.py | 127 +++++++-- .../f3/streaming/transfer_outcome_test.py | 15 +- .../fuel/f3/streaming/transfer_waiter_test.py | 42 ++- 7 files changed, 428 insertions(+), 143 deletions(-) diff --git a/nvflare/fuel/f3/streaming/download_service.py b/nvflare/fuel/f3/streaming/download_service.py index db74576cba..3c3f39f4cf 100644 --- a/nvflare/fuel/f3/streaming/download_service.py +++ b/nvflare/fuel/f3/streaming/download_service.py @@ -181,6 +181,7 @@ class _PropKey: # gets today's producer-served semantics; an old producer never sends CONFIRM_EXPECTED so a # new receiver never confirms toward it. CONFIRM = "confirm" # receiver -> producer: terminal receiver truth (a DownloadStatus value) + CONFIRM_NONCE = "confirm_nonce" # both ways: per-serve nonce binding a confirmation to ITS serve CONFIRM_CAPABLE = "confirm_capable" # receiver -> producer, per request: will confirm if asked CONFIRM_EXPECTED = "confirm_expected" # producer -> receiver, per reply: confirmations consumed @@ -258,7 +259,9 @@ def mark_active(self): def obj_downloaded(self, to_receiver: str, status: str): self._finalize_receiver(to_receiver, status) - def _finalize_receiver(self, to_receiver: str, status: str, require_pending: bool = False) -> bool: + def _finalize_receiver( + self, to_receiver: str, status: str, require_pending: bool = False, nonce: Optional[str] = None + ) -> bool: # Status recording is guarded so terminal-outcome snapshots taken on the # monitor thread never observe a half-updated map; user callbacks run # outside the lock. The whole decision (dedup, pending-guard, pending pop, @@ -267,19 +270,24 @@ def _finalize_receiver(self, to_receiver: str, status: str, require_pending: boo with self._progress_lock: if to_receiver in self.receiver_statuses: return False - if require_pending and to_receiver not in self._pending_confirms: - # a legitimate confirmation always follows a provisional terminal serve on - # the CURRENT life of this ref; an unsolicited/stale confirm (e.g. delayed - # across a ref_id reuse) must not certify -- or poison -- this transfer - self.tx.logger.warning(f"dropping unsolicited confirmation from {to_receiver} for ref {self.rid}") - return False + if require_pending: + pending = self._pending_confirms.get(to_receiver) + if pending is None or pending[1] != nonce: + # a legitimate confirmation always follows a provisional terminal serve on + # the CURRENT life of this ref and echoes that serve's nonce; anything else + # (unsolicited, or delayed across a ref_id reuse) must not certify -- or + # poison -- this transfer + self.tx.logger.warning( + f"dropping unsolicited/stale confirmation from {to_receiver} for ref {self.rid}" + ) + return False self._pending_confirms.pop(to_receiver, None) self.receiver_statuses[to_receiver] = status self.num_receivers_done = len(self.receiver_statuses) assert isinstance(self.tx, _Transaction) - all_done = 0 < self.tx.num_receivers <= self.num_receivers_done and not self._downloaded_to_all_called + all_done = not self._downloaded_to_all_called and self._completion_reached_locked() if all_done: self._downloaded_to_all_called = True @@ -305,6 +313,16 @@ def _finalize_receiver(self, to_receiver: str, status: str, require_pending: boo ) return True + def _completion_reached_locked(self) -> bool: + # Identity-aware when the transaction declared receiver_ids: completion means every + # EXPECTED receiver is final. A status from an unexpected receiver never completes + # the ref -- otherwise (expected "b" absent + unexpected "x" present) would certify + # a delivery "b" never got. Count-based only when identities are unknown. + expected = self.tx.receiver_ids + if expected: + return all(r in self.receiver_statuses for r in expected) + return 0 < self.tx.num_receivers <= self.num_receivers_done + def obj_served(self, to_receiver: str, status: str, expect_confirm: bool): """Records the producer-served terminal status for a receiver. @@ -317,28 +335,37 @@ def obj_served(self, to_receiver: str, status: str, expect_confirm: bool): finalization failure after the last chunk turns a served-EOF SUCCESS into a confirmed FAILED). Once the receiver confirms, its status is final: a receiver that confirms FAILED has given up (it confirms only on its own terminal exits). + + Returns the per-serve nonce the confirmation must echo (None when finalized + immediately or already final). The nonce binds a confirmation to THIS serve of + THIS life of the ref: a stale confirmation from a previous life of a reused + ref_id carries the wrong nonce and is dropped even if the new life has its own + pending serve for the same receiver. """ if not expect_confirm: self.obj_downloaded(to_receiver, status) - return + return None + nonce = uuid.uuid4().hex with self._progress_lock: if to_receiver in self.receiver_statuses: # already finalized -- a late duplicate serve must not resurrect a provisional - return - self._pending_confirms[to_receiver] = status + return None + self._pending_confirms[to_receiver] = (status, nonce) + return nonce - def obj_confirmed(self, to_receiver: str, status: str) -> bool: + def obj_confirmed(self, to_receiver: str, status: str, nonce: Optional[str]) -> bool: """Records the receiver-confirmed terminal status. Receiver truth wins; first confirm is final. - Accepted only when a provisional serve is pending for this receiver on THIS - life of the ref -- unsolicited or stale confirmations are dropped, so a - delayed confirm from a previous life of a reused ref_id can neither falsely - certify nor pre-poison the new transfer. + Accepted only when a provisional serve is pending for this receiver AND the + confirmation echoes that serve's nonce. The nonce is what distinguishes ref + lives: without it, a delayed confirmation from a previous life of a reused + ref_id would be accepted whenever the new life happens to have its own pending + serve for the same receiver -- certifying (or poisoning) a transfer it never saw. """ if status not in (DownloadStatus.SUCCESS, DownloadStatus.FAILED): self.tx.logger.error(f"ignoring confirmation with invalid status '{status}' from {to_receiver}") return False - accepted = self._finalize_receiver(to_receiver, status, require_pending=True) + accepted = self._finalize_receiver(to_receiver, status, require_pending=True, nonce=nonce) if accepted: # the receiver's truth is the terminal progress state for this receiver self.emit_progress( @@ -358,70 +385,79 @@ def snapshot_receiver_statuses(self) -> dict: def snapshot_pending_confirms(self) -> dict: with self._progress_lock: - return dict(self._pending_confirms) + # public shape: receiver -> provisional status (the nonce is internal) + return {r: v[0] for r, v in self._pending_confirms.items()} def mark_receiver_active(self, receiver: str): + now = time.time() with self._progress_lock: - self._receiver_activity[receiver] = time.time() + self._receiver_activity[receiver] = now tx = self.tx - if receiver not in tx._acquired_receivers: - # double-checked: the set is monotonic, so the lock is taken at most once - # per (transaction, receiver) -- not per chunk - with tx._stats_lock: - tx._acquired_receivers.add(receiver) + with tx._stats_lock: + # transaction-level liveness: budgets judge idleness across the WHOLE + # transaction, so a receiver that finished one ref and went silent cannot + # escape a sibling ref's budgets (it has no per-ref timestamp there) + tx._acquired_receivers.add(receiver) + tx._receiver_last_active[receiver] = now def snapshot_receiver_activity(self) -> dict: with self._progress_lock: return dict(self._receiver_activity) def enforce_budgets( - self, now: float, acquire_timeout, idle_timeout, expected_receivers, tx_acquired_receivers=None + self, now: float, acquire_timeout, idle_timeout, expected_receivers, tx_acquired=None, tx_last_active=None ) -> list: """Finalizes FAILED for receivers whose acquire or idle budget is exhausted. - A budget failure counts toward completion (via obj_downloaded), so the transaction's - aggregate outcome resolves on the next monitor pass instead of waiting for the whole - transaction TTL. This also bounds a lost fire-and-forget confirmation: the receiver - stops making requests after EOF, so its idle budget finalizes it FAILED (fail-closed). + Candidates are the declared receiver_ids plus every receiver seen anywhere on the + transaction. Idleness is judged on TRANSACTION-level activity (last request on ANY + ref): a receiver that pulled a sibling ref and went silent has no per-ref timestamp + here, and transaction-level acquisition exempts it from the acquire budget -- judging + idle per-ref would let it escape both budgets and pin the producer to the full TTL. + + A budget failure counts toward completion (via _finalize_receiver), so the aggregate + outcome resolves on the next monitor pass. This also bounds a lost fire-and-forget + confirmation (fail-closed). Returns: list of (receiver, reason) that were failed on this pass. """ - failures = [] + tx_acquired = tx_acquired or set() + tx_last_active = tx_last_active or {} with self._progress_lock: final = set(self.receiver_statuses) - activity = dict(self._receiver_activity) - if idle_timeout is not None: - for receiver, last_active in activity.items(): - if receiver in final: - continue + candidates = set(expected_receivers or ()) | tx_acquired + failures = [] + for receiver in candidates: + if receiver in final: + continue + last_active = tx_last_active.get(receiver) + if last_active is None: + # never pulled anywhere: only the acquire budget (needs declared identities) + if acquire_timeout is not None and expected_receivers and receiver in expected_receivers: + waited = now - self._created_time + if waited > acquire_timeout: + failures.append( + ( + receiver, + f"acquire budget exhausted: no pull within {acquire_timeout}s (waited {waited:.1f}s)", + ) + ) + elif idle_timeout is not None: idle = now - last_active if idle > idle_timeout: - failures.append((receiver, f"idle budget exhausted: {idle:.1f}s > {idle_timeout}s")) - if acquire_timeout is not None and expected_receivers: - waited = now - self._created_time - if waited > acquire_timeout: - for receiver in expected_receivers: - if receiver in final or receiver in activity: - continue - if tx_acquired_receivers is not None and receiver in tx_acquired_receivers: - # acquired at TRANSACTION level: a receiver working through the - # transaction's refs sequentially must not be failed on refs it - # has not reached yet - continue failures.append( ( receiver, - f"acquire budget exhausted: no pull within {acquire_timeout}s (waited {waited:.1f}s)", + f"idle budget exhausted: {idle:.1f}s since last transaction activity > {idle_timeout}s", ) ) enforced = [] for receiver, reason in failures: - with self._progress_lock: - if receiver in self.receiver_statuses: - continue # finalized (e.g. confirmed) between snapshot and enforcement: truth wins - if self._receiver_activity.get(receiver) != activity.get(receiver): + with self.tx._stats_lock: + if self.tx._receiver_last_active.get(receiver) != tx_last_active.get(receiver): continue # activity advanced past the snapshot: not actually idle - # _finalize_receiver pops the pending-confirm entry itself + # _finalize_receiver dedups and pops the pending-confirm entry itself; + # a receiver finalized meanwhile (e.g. confirmed) wins -- truth over budget if not self._finalize_receiver(receiver, DownloadStatus.FAILED): continue self.tx.logger.warning(f"receiver {receiver} failed for ref {self.rid}: {reason}") @@ -652,8 +688,11 @@ def __init__( self.refs = [] self._refs_lock = threading.RLock() # receivers that have issued at least one pull on ANY ref (monotonic; the - # transaction-level PAYLOAD_ACQUIRED fact the acquire budget and the facade read) + # transaction-level PAYLOAD_ACQUIRED fact the acquire budget and the facade read), + # and each receiver's last activity anywhere on the transaction (what the idle + # budget judges -- per-ref timestamps would let a multi-ref receiver escape) self._acquired_receivers = set() + self._receiver_last_active = {} self.logger = get_obj_logger(self) def mark_active(self): @@ -716,6 +755,7 @@ def enforce_receiver_budgets(self, now: float): return with self._stats_lock: tx_acquired = set(self._acquired_receivers) + tx_last_active = dict(self._receiver_last_active) for ref in self.snapshot_refs(): assert isinstance(ref, _Ref) ref.enforce_budgets( @@ -723,50 +763,67 @@ def enforce_receiver_budgets(self, now: float): self.receiver_acquire_timeout, self.receiver_idle_timeout, self.receiver_ids, - tx_acquired_receivers=tx_acquired, + tx_acquired=tx_acquired, + tx_last_active=tx_last_active, ) def is_finished(self): - """Check whether the transaction is finished (all objects are downloaded).""" + """Check whether every expected receiver has a final status on every ref. + + Identity-aware when receiver_ids were declared: statuses from unexpected + receivers never finish a ref. A transaction with no refs yet is never + finished (it terminates via timeout or deletion instead). + """ if self.num_receivers <= 0: return False - for ref in self.snapshot_refs(): + refs = self.snapshot_refs() + if not refs: + return False + for ref in refs: assert isinstance(ref, _Ref) - if ref.num_receivers_done < self.num_receivers: - return False + with ref._progress_lock: + if not ref._completion_reached_locked(): + return False return True - def transaction_done(self, status: str, on_outcome=None) -> TransferOutcome: + def transaction_done(self, status: str, on_outcome=None, superseded: bool = False) -> TransferOutcome: """Called when the transaction is finished. Returns the aggregate TransferOutcome (see transfer_outcome.py): COMPLETED only when every expected receiver succeeded — TransactionDoneStatus.FINISHED alone does not certify that. The existing transaction_done_cb contract is unchanged except that callback exceptions no longer propagate (they would kill the - monitor thread and skip source release); outcome_cb (if set) fires after it - with the computed outcome. on_outcome (used by DownloadService to record the - outcome) is invoked right after the outcome is computed — before the - potentially slow user callbacks — so pollers see the terminal outcome as soon - as the transaction stops being active. + monitor thread and skip source release). + + on_outcome (used by DownloadService to record the outcome and release waiters) + is invoked LAST — after terminal progress, the done/outcome callbacks, and + source release — so an upper layer that acts on waiter.wait() returning (e.g. + stops the producer process) can never preempt the callback chain or the + release of the sources. + + superseded=True (a retry took over this transaction's reused tx_id) suppresses + the transfer-facing emissions — terminal progress, transaction_done_cb and + outcome_cb — because their tx_id now names the LIVE retry; consumers keyed by + tx_id would misattribute them. Sources are still released and the per-object + obj.transaction_done cleanup still runs. """ refs = self.snapshot_refs() - # Compute (and record via on_outcome) the aggregate outcome first, from - # locked per-receiver snapshots, before any user callback runs. + # Compute the aggregate outcome from locked per-receiver snapshots before any + # user callback can observe (or mutate the world around) this transaction. outcome = compute_transfer_outcome( tx_id=self.tid, done_status=status, num_receivers=self.num_receivers, min_receivers=self.min_receivers, + receiver_ids=self.receiver_ids, refs=[RefOutcome(ref_id=ref.rid, receiver_statuses=ref.snapshot_receiver_statuses()) for ref in refs], timestamp=time.time(), ) - if on_outcome: - _invoke_cb_safely(self.logger, f"outcome recording for tx {self.tid}", on_outcome, outcome) progress_state = self._progress_state_for_transaction_status(status) - if progress_state: + if progress_state and not superseded: for ref in refs: ref.emit_terminal_progress_for_started_receivers(progress_state) @@ -796,7 +853,7 @@ def transaction_done(self, status: str, on_outcome=None) -> TransferOutcome: status, ) - if self.transaction_done_cb: + if self.transaction_done_cb and not superseded: _invoke_cb_safely( self.logger, f"transaction done callback for tx {self.tid}", @@ -807,7 +864,7 @@ def transaction_done(self, status: str, on_outcome=None) -> TransferOutcome: **self.cb_kwargs, ) - if self.outcome_cb: + if self.outcome_cb and not superseded: _invoke_cb_safely(self.logger, f"transfer outcome callback for tx {self.tid}", self.outcome_cb, outcome) # Release source objects after the callback so the callback can still @@ -816,6 +873,12 @@ def transaction_done(self, status: str, on_outcome=None) -> TransferOutcome: for ref in refs: ref.obj.release() + # Record the outcome (and release waiters) only now that the transaction is fully + # settled: callbacks done, sources released. waiter.wait() returning therefore + # means there is nothing left in flight on this transaction. + if on_outcome: + _invoke_cb_safely(self.logger, f"outcome recording for tx {self.tid}", on_outcome, outcome) + return outcome def emit_progress_event(self, event: dict): @@ -991,19 +1054,21 @@ def new_transaction( receiver_acquire_timeout=receiver_acquire_timeout, receiver_idle_timeout=receiver_idle_timeout, ) - with cls._outcome_lock: - # a reused explicit tx_id must not surface the previous owner's outcome: - # purge any recorded outcome and register this transaction as the outcome - # owner, so a concurrently-terminating previous owner cannot record. - # Ownership is taken BEFORE the tx becomes monitor-visible in _tx_table: - # a tx the monitor can terminate always owns its outcome slot, so its - # terminal outcome can never be dropped as stale, and no dead owner - # entry can be left behind by a terminate-before-register interleaving. - cls._tx_outcomes.pop(tx.tid, None) - cls._outcome_owners[tx.tid] = tx - old_tx = None + # Ownership and table registration are ONE atomic step (_tx_lock nests + # _outcome_lock here; no path nests them in the reverse order). With separate + # critical sections, two concurrent same-id constructors could interleave so + # that the transaction left in _tx_table is not the outcome owner -- the live + # transaction could then never record, while the retired one recorded its + # DELETED outcome as if it were the live attempt. with cls._tx_lock: + with cls._outcome_lock: + # a reused tx_id must not surface the previous owner's outcome: purge any + # recorded outcome and take ownership. Ownership is taken before the tx + # becomes monitor-visible, so a tx the monitor can terminate always owns + # its outcome slot. + cls._tx_outcomes.pop(tx.tid, None) + cls._outcome_owners[tx.tid] = tx old_tx = cls._tx_table.get(tx.tid) if old_tx: # A retry reusing a tx_id supersedes a still-live prior transaction. @@ -1015,11 +1080,15 @@ def new_transaction( cls._tx_table[tx.tid] = tx if old_tx: - # terminal callbacks run outside the lock, as in delete_transaction(); the - # retired transaction's outcome is dropped by the owner guard because the - # new transaction already owns the tid -- the retry is authoritative + # terminal callbacks run outside the lock, as in delete_transaction(). The + # retired transaction is marked superseded: its sources are released, but its + # transfer-facing emissions (progress, done/outcome callbacks) are suppressed + # -- their reused tx_id now names the live retry -- and its outcome record is + # dropped by the owner guard. The retry is authoritative. old_tx.transaction_done( - TransactionDoneStatus.DELETED, on_outcome=functools.partial(cls._record_outcome, tx=old_tx) + TransactionDoneStatus.DELETED, + on_outcome=functools.partial(cls._record_outcome, tx=old_tx), + superseded=True, ) return tx.tid @@ -1238,7 +1307,7 @@ def _handle_download(cls, request: Message) -> Message: confirm_status = payload.get(_PropKey.CONFIRM) if confirm_status is not None: - return cls._handle_confirm(rid, requester, confirm_status) + return cls._handle_confirm(rid, requester, confirm_status, payload.get(_PropKey.CONFIRM_NONCE)) current_state = payload.get(_PropKey.STATE) with cls._tx_lock: @@ -1283,16 +1352,16 @@ def _handle_download(cls, request: Message) -> Message: # already done -- for a confirm-capable receiver this record is PROVISIONAL and the # receiver's confirmation finalizes it; for a legacy receiver it is final (today's # producer-served semantics) - ref.obj_served( + serve_nonce = ref.obj_served( requester, status=DownloadStatus.SUCCESS if rc == ProduceRC.EOF else DownloadStatus.FAILED, expect_confirm=expect_confirm, ) - if expect_confirm: + if expect_confirm and serve_nonce: # provisional: the receiver's confirmation carries the terminal truth -- # do not latch a terminal progress state the confirm may contradict ref.emit_progress(receiver_id=requester, state=TransferProgressState.ACTIVE, force=True) - body = {_PropKey.STATUS: rc, _PropKey.CONFIRM_EXPECTED: True} + body = {_PropKey.STATUS: rc, _PropKey.CONFIRM_EXPECTED: True, _PropKey.CONFIRM_NONCE: serve_nonce} else: ref.emit_progress( receiver_id=requester, @@ -1328,7 +1397,7 @@ def _handle_download(cls, request: Message) -> Message: ) @classmethod - def _handle_confirm(cls, rid: str, requester: str, status: str) -> Message: + def _handle_confirm(cls, rid: str, requester: str, status: str, nonce: Optional[str]) -> Message: with cls._tx_lock: ref = cls._ref_table.get(rid) if ref is None: @@ -1339,7 +1408,7 @@ def _handle_confirm(cls, rid: str, requester: str, status: str) -> Message: assert isinstance(ref, _Ref) # deliberately no unconditional mark_active/mark_receiver_active: a stale or # unsolicited confirm must not extend the transaction TTL nor reset idle budgets - if ref.obj_confirmed(requester, status): + if ref.obj_confirmed(requester, status, nonce): ref.mark_active() return make_reply(ReturnCode.OK) @@ -1497,6 +1566,7 @@ def download_object( # kill-switch is on) and learn from each reply whether the producer consumes confirmations. confirm_enabled = _receiver_confirm_enabled() producer_expects_confirm = False + confirm_nonce = None def _send_confirm(receiver_truth: str): # wire contract: a confirmation is sent ONLY after a producer-served terminal reply @@ -1512,8 +1582,14 @@ def _send_confirm(receiver_truth: str): topic=OBJ_DOWNLOADER_TOPIC, targets=from_fqcn, message=new_cell_message( - headers={}, payload={_PropKey.REF_ID: ref_id, _PropKey.CONFIRM: receiver_truth} + headers={}, + payload={ + _PropKey.REF_ID: ref_id, + _PropKey.CONFIRM: receiver_truth, + _PropKey.CONFIRM_NONCE: confirm_nonce, + }, ), + secure=secure, optional=optional, ) except Exception as ex: @@ -1619,6 +1695,7 @@ def _emit_progress(state: str, force: bool = False): assert isinstance(payload, dict) if payload.get(_PropKey.CONFIRM_EXPECTED): producer_expects_confirm = True + confirm_nonce = payload.get(_PropKey.CONFIRM_NONCE) status = payload.get(_PropKey.STATUS) if status == ProduceRC.EOF: elapsed = time.time() - download_start diff --git a/nvflare/fuel/f3/streaming/transfer_outcome.py b/nvflare/fuel/f3/streaming/transfer_outcome.py index 0fa5eaf31f..f831ab3469 100644 --- a/nvflare/fuel/f3/streaming/transfer_outcome.py +++ b/nvflare/fuel/f3/streaming/transfer_outcome.py @@ -134,11 +134,17 @@ class TransferOutcome: # optional k-of-N quorum declared by the workflow: informational for quorum_met; # `completed` deliberately stays the strict all-receivers certificate min_receivers: Optional[int] = None + # expected receiver identities declared by the workflow. When present, completion and + # quorum are judged against THESE identities: a status from an unexpected receiver can + # neither complete the transfer nor count toward the quorum. + receiver_ids: Optional[Tuple[str, ...]] = None def __post_init__(self): - # frozen=True only blocks attribute rebinding; freeze the container too so + # frozen=True only blocks attribute rebinding; freeze the containers too so # outcome_cb consumers cannot mutate the recorded outcome object.__setattr__(self, "refs", tuple(self.refs)) + if self.receiver_ids is not None: + object.__setattr__(self, "receiver_ids", tuple(self.receiver_ids)) @property def completed(self) -> bool: @@ -164,15 +170,27 @@ def quorum_met(self) -> bool: for r in self.refs: ref_successes = {rcv for rcv, v in r.receiver_statuses.items() if v == DownloadStatus.SUCCESS} quorum_receivers = ref_successes if quorum_receivers is None else quorum_receivers & ref_successes + if self.receiver_ids is not None: + # only declared receivers count toward the quorum + quorum_receivers &= set(self.receiver_ids) return len(quorum_receivers) >= self.min_receivers def expired(self, now: float, ttl: float) -> bool: return now - self.timestamp > ttl -def _all_receivers_succeeded(num_receivers: int, refs: Sequence[RefOutcome]) -> bool: +def _all_receivers_succeeded(num_receivers: int, refs: Sequence[RefOutcome], receiver_ids=None) -> bool: if num_receivers <= 0 or not refs: return False + if receiver_ids: + # identity mode: every DECLARED receiver must have succeeded on every ref. + # Statuses from unexpected receivers are ignored -- they can never certify + # a transfer that a declared receiver did not actually get. + for r in refs: + for expected in receiver_ids: + if r.receiver_statuses.get(expected) != DownloadStatus.SUCCESS: + return False + return True for r in refs: if len(r.receiver_statuses) < num_receivers: return False @@ -195,6 +213,7 @@ def compute_transfer_outcome( refs: Sequence[RefOutcome], timestamp: Optional[float] = None, min_receivers: Optional[int] = None, + receiver_ids=None, ) -> TransferOutcome: """Compute the aggregate terminal outcome for a terminated transaction. @@ -219,7 +238,7 @@ def compute_transfer_outcome( if done_status not in _KNOWN_DONE_STATUSES: status, reason = TransferProgressState.FAILED, TransferOutcomeReason.UNKNOWN_DONE_STATUS - elif _all_receivers_succeeded(num_receivers, refs): + elif _all_receivers_succeeded(num_receivers, refs, receiver_ids): status, reason = TransferProgressState.COMPLETED, TransferOutcomeReason.ALL_RECEIVERS_SUCCEEDED elif done_status == TransactionDoneStatus.DELETED: status, reason = TransferProgressState.ABORTED, TransferOutcomeReason.DELETED @@ -245,4 +264,5 @@ def compute_transfer_outcome( refs=refs, timestamp=timestamp, min_receivers=min_receivers, + receiver_ids=receiver_ids, ) diff --git a/tests/unit_test/fuel/f3/streaming/download_test_utils.py b/tests/unit_test/fuel/f3/streaming/download_test_utils.py index 7064f1fbcc..85aea82a0d 100644 --- a/tests/unit_test/fuel/f3/streaming/download_test_utils.py +++ b/tests/unit_test/fuel/f3/streaming/download_test_utils.py @@ -112,11 +112,17 @@ def pull_request(rid, requester, confirm_capable=False, state=None): return new_cell_message(headers={MessageHeaderKey.ORIGIN: requester}, payload=payload) -def confirm_request(rid, requester, status): +def confirm_request(rid, requester, status, nonce=None): """Builds one receiver-confirmation message as it appears on the wire.""" - return new_cell_message( - headers={MessageHeaderKey.ORIGIN: requester}, payload={_PropKey.REF_ID: rid, _PropKey.CONFIRM: status} - ) + payload = {_PropKey.REF_ID: rid, _PropKey.CONFIRM: status} + if nonce is not None: + payload[_PropKey.CONFIRM_NONCE] = nonce + return new_cell_message(headers={MessageHeaderKey.ORIGIN: requester}, payload=payload) + + +def serve_nonce(terminal_reply): + """The per-serve nonce a confirmation must echo (from the producer's terminal reply).""" + return terminal_reply.payload.get(_PropKey.CONFIRM_NONCE) def pull_to_terminal(service, rid, requester, confirm_capable=False): diff --git a/tests/unit_test/fuel/f3/streaming/receiver_budget_test.py b/tests/unit_test/fuel/f3/streaming/receiver_budget_test.py index 9e5d1aa9f7..a981029aa2 100644 --- a/tests/unit_test/fuel/f3/streaming/receiver_budget_test.py +++ b/tests/unit_test/fuel/f3/streaming/receiver_budget_test.py @@ -33,7 +33,7 @@ from tests.unit_test.fuel.f3.streaming.download_test_utils import make_confirm_test_service as _make_service from tests.unit_test.fuel.f3.streaming.download_test_utils import pull_request from tests.unit_test.fuel.f3.streaming.download_test_utils import pull_to_terminal as _pull_to_terminal -from tests.unit_test.fuel.f3.streaming.download_test_utils import run_monitor_once +from tests.unit_test.fuel.f3.streaming.download_test_utils import run_monitor_once, serve_nonce def _new_tx(service, chunks=1, **tx_kwargs): @@ -85,10 +85,11 @@ def test_active_receiver_is_not_idle_failed(self): tx_id, rid = _new_tx(service, chunks=3, receiver_idle_timeout=5.0) reply = service._handle_download(pull_request(rid, "r1")) ref = service._ref_table[rid] + tx = service._tx_table[tx_id] # receiver keeps making requests: refresh activity to "now" as seen by the monitor future = time.time() + 30.0 - with ref._progress_lock: - ref._receiver_activity["r1"] = future - 1.0 + with tx._stats_lock: + tx._receiver_last_active["r1"] = future - 1.0 run_monitor_once(service, now=future) @@ -163,15 +164,15 @@ def test_activity_tracked_without_progress_cb(self): def test_budget_failed_receiver_is_final_even_against_late_confirm(self): service = _make_service() tx_id, rid = _new_tx(service, num_receivers=2, receiver_idle_timeout=5.0) - _pull_to_terminal(service, rid, "r1", confirm_capable=True) + terminal = _pull_to_terminal(service, rid, "r1", confirm_capable=True) ref = service._ref_table[rid] - failures = ref.enforce_budgets(time.time() + 30.0, None, 5.0, None) - assert [r for r, _ in failures] == ["r1"] + run_monitor_once(service, now=time.time() + 30.0) # idle budget fires + assert ref.snapshot_receiver_statuses()["r1"] == DownloadStatus.FAILED assert ref.snapshot_pending_confirms() == {} - # the straggler confirmation cannot resurrect the budget-failed receiver - ref.obj_confirmed("r1", DownloadStatus.SUCCESS) + # the straggler confirmation (correct nonce and all) cannot resurrect it + assert ref.obj_confirmed("r1", DownloadStatus.SUCCESS, serve_nonce(terminal)) is False assert ref.snapshot_receiver_statuses()["r1"] == DownloadStatus.FAILED @@ -197,16 +198,75 @@ def test_confirmed_receiver_wins_over_stale_budget_snapshot(self): # and enforcement must not be flipped to FAILED, and no failure is reported for it service = _make_service() tx_id, rid = _new_tx(service, receiver_idle_timeout=5.0) - _pull_to_terminal(service, rid, "r1", confirm_capable=True) + terminal = _pull_to_terminal(service, rid, "r1", confirm_capable=True) ref = service._ref_table[rid] - service._handle_download(confirm_request(rid, "r1", DownloadStatus.SUCCESS)) + service._handle_download(confirm_request(rid, "r1", DownloadStatus.SUCCESS, serve_nonce(terminal))) - enforced = ref.enforce_budgets(time.time() + 30.0, None, 5.0, None) + tx = service._tx_table[tx_id] + with tx._stats_lock: + tx_last_active = dict(tx._receiver_last_active) + enforced = ref.enforce_budgets( + time.time() + 30.0, None, 5.0, None, tx_acquired={"r1"}, tx_last_active=tx_last_active + ) assert enforced == [] assert ref.snapshot_receiver_statuses() == {"r1": DownloadStatus.SUCCESS} +class TestMultiRefIdleEscape: + def test_receiver_idle_after_finishing_one_ref_fails_sibling_ref_in_bounded_time(self): + # P2 pin (reproduced by review): after finishing ref 1, the receiver is tx-acquired + # (exempt from ref 2's acquire budget) and has no per-ref timestamp on ref 2 -- with + # per-ref idle it escaped BOTH budgets and pinned the producer to the full TTL. + # Idle is judged on transaction-level activity, so its silence now fails ref 2. + service = _make_service() + tx_id = service.new_transaction( + cell=Mock(), timeout=1000.0, num_receivers=0, receiver_ids=("r1",), receiver_idle_timeout=5.0 + ) + rid1 = service.add_object(tx_id, MockDownloadable([b"chunk"])) + rid2 = service.add_object(tx_id, MockDownloadable([b"chunk"])) + _pull_to_terminal(service, rid1, "r1") # finishes ref 1 (legacy final), never touches ref 2 + + run_monitor_once(service, now=time.time() + 30.0) + + outcome = service.get_transaction_outcome(tx_id) + assert outcome is not None, "must resolve via the idle budget, not the 1000s TTL" + assert not outcome.completed + statuses = {r.ref_id: r.receiver_statuses for r in outcome.refs} + assert statuses[rid1] == {"r1": DownloadStatus.SUCCESS} + assert statuses[rid2] == {"r1": DownloadStatus.FAILED} + + +class TestConcurrentSameIdCreation: + def test_owner_and_table_stay_consistent_under_concurrent_creation(self): + # P1 pin (reproduced by review): with registration and table insertion in separate + # critical sections, two concurrent same-id constructors could leave the live + # transaction without outcome ownership -- and record the retired transaction's + # DELETED outcome as if it were the live attempt. + import threading as th + + service = _make_service() + for _ in range(30): + barrier = th.Barrier(2) + + def create(): + barrier.wait() + service.new_transaction(cell=Mock(), timeout=10.0, num_receivers=1, tx_id="TX-RACE2") + + threads = [th.Thread(target=create) for _ in range(2)] + for t in threads: + t.start() + for t in threads: + t.join(5.0) + + live = service._tx_table["TX-RACE2"] + with service._outcome_lock: + owner = service._outcome_owners.get("TX-RACE2") + assert owner is live, "the live transaction must own its outcome slot" + # the retired constructor's DELETED outcome must never be recorded for the id + assert service.get_transaction_outcome("TX-RACE2") is None + + class TestConfigResolution: def test_budget_config_var_supplies_default(self): service = _make_service() diff --git a/tests/unit_test/fuel/f3/streaming/receiver_confirm_test.py b/tests/unit_test/fuel/f3/streaming/receiver_confirm_test.py index 3b5ac44e70..d411aeada2 100644 --- a/tests/unit_test/fuel/f3/streaming/receiver_confirm_test.py +++ b/tests/unit_test/fuel/f3/streaming/receiver_confirm_test.py @@ -39,7 +39,7 @@ from tests.unit_test.fuel.f3.streaming.download_test_utils import make_confirm_test_service as _make_service from tests.unit_test.fuel.f3.streaming.download_test_utils import pull_request as _pull_request from tests.unit_test.fuel.f3.streaming.download_test_utils import pull_to_terminal as _pull_to_terminal -from tests.unit_test.fuel.f3.streaming.download_test_utils import run_monitor_once +from tests.unit_test.fuel.f3.streaming.download_test_utils import run_monitor_once, serve_nonce def _new_tx(service, num_receivers=1, timeout=10.0, chunks=1): @@ -82,9 +82,9 @@ def test_confirm_capable_receiver_is_provisional_at_serve(self): def test_confirmation_finalizes_and_finishes(self): service = _make_service() tx_id, rid, _ = _new_tx(service) - _pull_to_terminal(service, rid, "r1", confirm_capable=True) + terminal = _pull_to_terminal(service, rid, "r1", confirm_capable=True) - reply = service._handle_download(_confirm_request(rid, "r1", DownloadStatus.SUCCESS)) + reply = service._handle_download(_confirm_request(rid, "r1", DownloadStatus.SUCCESS, serve_nonce(terminal))) assert reply.get_header(MessageHeaderKey.RETURN_CODE) == ReturnCode.OK ref = _ref(service, rid) @@ -96,9 +96,9 @@ def test_receiver_truth_wins_failed_confirm_after_served_eof(self): # the motivating case: producer served EOF, but the receiver's finalization failed service = _make_service() tx_id, rid, _ = _new_tx(service) - _pull_to_terminal(service, rid, "r1", confirm_capable=True) + terminal = _pull_to_terminal(service, rid, "r1", confirm_capable=True) - service._handle_download(_confirm_request(rid, "r1", DownloadStatus.FAILED)) + service._handle_download(_confirm_request(rid, "r1", DownloadStatus.FAILED, serve_nonce(terminal))) ref = _ref(service, rid) assert ref.snapshot_receiver_statuses() == {"r1": DownloadStatus.FAILED} @@ -120,10 +120,10 @@ def test_retry_overwrites_provisional_and_confirm_wins(self): assert ref.snapshot_pending_confirms() == {"r1": DownloadStatus.FAILED} # the receiver retries; this time the pull succeeds end-to-end - _pull_to_terminal(service, rid, "r1", confirm_capable=True) + terminal = _pull_to_terminal(service, rid, "r1", confirm_capable=True) assert ref.snapshot_pending_confirms() == {"r1": DownloadStatus.SUCCESS} - service._handle_download(_confirm_request(rid, "r1", DownloadStatus.SUCCESS)) + service._handle_download(_confirm_request(rid, "r1", DownloadStatus.SUCCESS, serve_nonce(terminal))) assert ref.snapshot_receiver_statuses() == {"r1": DownloadStatus.SUCCESS} run_monitor_once(service, now=time.time()) assert service.get_transaction_outcome(tx_id).completed @@ -131,18 +131,18 @@ def test_retry_overwrites_provisional_and_confirm_wins(self): def test_first_confirm_is_final(self): service = _make_service() tx_id, rid, _ = _new_tx(service) - _pull_to_terminal(service, rid, "r1", confirm_capable=True) + terminal = _pull_to_terminal(service, rid, "r1", confirm_capable=True) - service._handle_download(_confirm_request(rid, "r1", DownloadStatus.SUCCESS)) - service._handle_download(_confirm_request(rid, "r1", DownloadStatus.FAILED)) + service._handle_download(_confirm_request(rid, "r1", DownloadStatus.SUCCESS, serve_nonce(terminal))) + service._handle_download(_confirm_request(rid, "r1", DownloadStatus.FAILED, serve_nonce(terminal))) assert _ref(service, rid).snapshot_receiver_statuses() == {"r1": DownloadStatus.SUCCESS} def test_late_duplicate_serve_cannot_resurrect_provisional(self): service = _make_service() tx_id, rid, _ = _new_tx(service) - _pull_to_terminal(service, rid, "r1", confirm_capable=True) - service._handle_download(_confirm_request(rid, "r1", DownloadStatus.SUCCESS)) + terminal = _pull_to_terminal(service, rid, "r1", confirm_capable=True) + service._handle_download(_confirm_request(rid, "r1", DownloadStatus.SUCCESS, serve_nonce(terminal))) ref = _ref(service, rid) ref.obj_served("r1", DownloadStatus.FAILED, expect_confirm=True) @@ -187,8 +187,8 @@ def test_tombstoned_ref_replay_does_not_solicit_confirm(self): # confirm for the tombstoned rid is dropped OK without disturbing the outcome service = _make_service() tx_id, rid, _ = _new_tx(service) - _pull_to_terminal(service, rid, "r1", confirm_capable=True) - service._handle_download(_confirm_request(rid, "r1", DownloadStatus.SUCCESS)) + terminal = _pull_to_terminal(service, rid, "r1", confirm_capable=True) + service._handle_download(_confirm_request(rid, "r1", DownloadStatus.SUCCESS, serve_nonce(terminal))) run_monitor_once(service, now=time.time()) # FINISHED -> tombstoned assert rid not in service._ref_table @@ -208,6 +208,72 @@ def test_kill_switch_reads_application_conf(self): assert ds_module._receiver_confirm_enabled() is False assert gv.call_args.kwargs.get("conf") == ds_module.SystemConfigs.APPLICATION_CONF + def test_unexpected_receiver_cannot_complete_declared_identities(self): + # P1 pin: with receiver_ids=("a", "b"), a success from "a" plus a success from an + # UNEXPECTED receiver "x" must not certify -- "b" never received anything + service = _make_service() + tx_id = service.new_transaction(cell=Mock(), timeout=10.0, num_receivers=0, receiver_ids=("a", "b")) + obj = MockDownloadable([b"chunk"]) + rid = service.add_object(tx_id, obj) + + _pull_to_terminal(service, rid, "a") # expected, succeeds + _pull_to_terminal(service, rid, "x") # unexpected, also succeeds + + tx = service._tx_table[tx_id] + assert not tx.is_finished(), "count-based completion would wrongly finish here" + assert not obj.released, "downloaded_to_all/release must not fire without 'b'" + + _pull_to_terminal(service, rid, "b") # the missing expected receiver arrives + assert tx.is_finished() + run_monitor_once(service, now=time.time()) + outcome = service.get_transaction_outcome(tx_id) + assert outcome.completed + assert outcome.receiver_ids == ("a", "b") + + def test_declared_identity_missing_fails_outcome_despite_count(self): + # two successes are recorded ("a" + unexpected "x"), matching the receiver COUNT -- + # but declared receiver "b" got nothing, so the outcome must fail closed (here via + # the transaction TTL, since "b" never pulled and no acquire budget was set) + service = _make_service() + tx_id = service.new_transaction( + cell=Mock(), timeout=10.0, num_receivers=0, receiver_ids=("a", "b"), receiver_idle_timeout=5.0 + ) + rid = service.add_object(tx_id, MockDownloadable([b"chunk"])) + _pull_to_terminal(service, rid, "a") + _pull_to_terminal(service, rid, "x") # unexpected + + run_monitor_once(service, now=time.time() + 30.0) + outcome = service.get_transaction_outcome(tx_id) + assert outcome is not None + assert not outcome.completed, "two successes without declared receiver 'b' must never certify" + assert not outcome.quorum_met or outcome.min_receivers is None + + def test_stale_confirm_from_previous_ref_life_is_dropped_even_with_new_pending(self): + # P1 pin (reproduced by review): old life's confirmation must not finalize the NEW + # life of a reused ref_id even when the new life has its own pending serve for the + # same receiver -- the per-serve nonce is what tells the lives apart + service = _make_service() + tx1 = service.new_transaction(cell=Mock(), timeout=10.0, num_receivers=1, tx_id="TX-LIFE") + rid = service.add_object(tx1, MockDownloadable([b"chunk"]), ref_id="R-LIFE") + old_terminal = _pull_to_terminal(service, rid, "r1", confirm_capable=True) + old_nonce = serve_nonce(old_terminal) + + # the transfer is retried: same tx_id, same ref_id -- a NEW life + service.new_transaction(cell=Mock(), timeout=10.0, num_receivers=1, tx_id="TX-LIFE") + rid2 = service.add_object("TX-LIFE", MockDownloadable([b"chunk"]), ref_id="R-LIFE") + assert rid2 == rid + new_terminal = _pull_to_terminal(service, rid, "r1", confirm_capable=True) # new pending serve exists + + # the OLD life's delayed confirmation arrives: wrong nonce -> dropped + service._handle_download(_confirm_request(rid, "r1", DownloadStatus.SUCCESS, old_nonce)) + ref = _ref(service, rid) + assert ref.snapshot_receiver_statuses() == {} + assert ref.snapshot_pending_confirms() == {"r1": DownloadStatus.SUCCESS} + + # the NEW life's confirmation (correct nonce) finalizes + service._handle_download(_confirm_request(rid, "r1", DownloadStatus.SUCCESS, serve_nonce(new_terminal))) + assert ref.snapshot_receiver_statuses() == {"r1": DownloadStatus.SUCCESS} + def test_late_confirm_for_unknown_ref_is_dropped_ok(self): service = _make_service() reply = service._handle_download(_confirm_request("R-GONE", "r1", DownloadStatus.SUCCESS)) @@ -248,10 +314,10 @@ def test_mixed_fleet_finishes_on_mixed_semantics(self): tx = service._tx_table[tx_id] _pull_to_terminal(service, rid, "legacy", confirm_capable=False) - _pull_to_terminal(service, rid, "modern", confirm_capable=True) + terminal = _pull_to_terminal(service, rid, "modern", confirm_capable=True) assert not tx.is_finished() # modern receiver not confirmed yet - service._handle_download(_confirm_request(rid, "modern", DownloadStatus.SUCCESS)) + service._handle_download(_confirm_request(rid, "modern", DownloadStatus.SUCCESS, serve_nonce(terminal))) assert tx.is_finished() run_monitor_once(service, now=time.time()) assert service.get_transaction_outcome(tx_id).completed @@ -264,6 +330,7 @@ def __init__(self, replies): self._replies = list(replies) self.requests = [] self.confirms = [] + self.confirm_kwargs = [] def send_request(self, channel, target, topic, request, timeout, secure, optional, abort_signal): self.requests.append(request.payload) @@ -273,6 +340,7 @@ def send_request(self, channel, target, topic, request, timeout, secure, optiona def fire_and_forget(self, channel, topic, targets, message, secure=False, optional=False): self.confirms.append(message.payload) + self.confirm_kwargs.append({"secure": secure, "optional": optional}) class _RecordingConsumer(Consumer): @@ -307,10 +375,11 @@ def _chunk_reply(confirm_expected=True): return _ok_reply(body) -def _terminal_reply(status, confirm_expected=True): +def _terminal_reply(status, confirm_expected=True, nonce="n-test"): body = {_PropKey.STATUS: status} if confirm_expected: body[_PropKey.CONFIRM_EXPECTED] = True + body[_PropKey.CONFIRM_NONCE] = nonce return _ok_reply(body) @@ -323,7 +392,9 @@ def test_advertises_capability_and_confirms_success_after_finalization(self): assert all(req.get(_PropKey.CONFIRM_CAPABLE) is True for req in cell.requests) assert consumer.completed - assert cell.confirms == [{_PropKey.REF_ID: "R1", _PropKey.CONFIRM: DownloadStatus.SUCCESS}] + assert cell.confirms == [ + {_PropKey.REF_ID: "R1", _PropKey.CONFIRM: DownloadStatus.SUCCESS, _PropKey.CONFIRM_NONCE: "n-test"} + ] def test_confirms_failed_when_finalization_raises(self): # served EOF but download_completed raises: the producer must learn receiver truth @@ -333,7 +404,9 @@ def test_confirms_failed_when_finalization_raises(self): with pytest.raises(RuntimeError, match="finalization failed"): download_object(from_fqcn="site-1", ref_id="R1", per_request_timeout=5.0, cell=cell, consumer=consumer) - assert cell.confirms == [{_PropKey.REF_ID: "R1", _PropKey.CONFIRM: DownloadStatus.FAILED}] + assert cell.confirms == [ + {_PropKey.REF_ID: "R1", _PropKey.CONFIRM: DownloadStatus.FAILED, _PropKey.CONFIRM_NONCE: "n-test"} + ] def test_confirms_failed_on_producer_error(self): cell = _ScriptedCell([_chunk_reply(), _terminal_reply(ProduceRC.ERROR)]) @@ -342,7 +415,9 @@ def test_confirms_failed_on_producer_error(self): download_object(from_fqcn="site-1", ref_id="R1", per_request_timeout=5.0, cell=cell, consumer=consumer) assert consumer.failed_reason is not None - assert cell.confirms == [{_PropKey.REF_ID: "R1", _PropKey.CONFIRM: DownloadStatus.FAILED}] + assert cell.confirms == [ + {_PropKey.REF_ID: "R1", _PropKey.CONFIRM: DownloadStatus.FAILED, _PropKey.CONFIRM_NONCE: "n-test"} + ] def test_no_confirm_toward_legacy_producer(self): # the producer never advertised CONFIRM_EXPECTED: a new receiver must send nothing extra @@ -356,6 +431,18 @@ def test_no_confirm_toward_legacy_producer(self): assert consumer.completed assert cell.confirms == [] + def test_confirm_honors_secure_flag(self): + # P2 pin: the confirmation must ride with the same security posture as the + # download requests it concludes + cell = _ScriptedCell([_terminal_reply(ProduceRC.EOF)]) + consumer = _RecordingConsumer() + + download_object( + from_fqcn="site-1", ref_id="R1", per_request_timeout=5.0, cell=cell, consumer=consumer, secure=True + ) + + assert cell.confirm_kwargs == [{"secure": True, "optional": False}] + def test_kill_switch_off_receiver_is_fully_legacy(self): with patch.object(ds_module, "_receiver_confirm_cached", False): cell = _ScriptedCell([_terminal_reply(ProduceRC.EOF, confirm_expected=True)]) diff --git a/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py b/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py index b1bde7fcac..e01acdd491 100644 --- a/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py +++ b/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py @@ -175,7 +175,10 @@ def test_transaction_done_returns_outcome_with_receiver_map(self): assert outcome.refs[0].receiver_statuses == {"r1": DownloadStatus.SUCCESS} assert obj.released - def test_on_outcome_fires_before_user_callbacks(self): + def test_on_outcome_fires_after_callbacks_and_release(self): + # settle-then-record: recording (which releases waiters) happens only after the + # callback chain and source release complete, so an upper layer acting on + # waiter.wait() can never preempt them (e.g. by stopping the producer process) order = [] tx = _Transaction( timeout=10.0, @@ -184,12 +187,13 @@ def test_on_outcome_fires_before_user_callbacks(self): outcome_cb=lambda outcome: order.append("outcome_cb"), ) obj = _stub_obj() + obj.release = lambda: order.append("release") # observe source release order ref = tx.add_object(obj) ref.obj_downloaded("r1", DownloadStatus.FAILED) tx.transaction_done(TransactionDoneStatus.FINISHED, on_outcome=lambda outcome: order.append("recorded")) - assert order == ["recorded", "done_cb", "outcome_cb"] + assert order == ["done_cb", "outcome_cb", "release", "recorded"] def test_raising_callbacks_do_not_break_recording_or_release(self): # a raising transaction_done_cb must not skip outcome recording, source @@ -382,9 +386,12 @@ def test_reusing_active_tx_id_retires_prior_transaction(self): second = service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-DUP") assert second == "TX-DUP" - # the prior live transaction was retired: refs unservable, done cb fired + # the prior live transaction was retired: refs unservable, sources released -- + # but its transfer-facing callbacks are SUPPRESSED (superseded): their reused + # tx_id now names the live retry, so firing them would misattribute assert rid not in service._ref_table - assert done == [("TX-DUP", TransactionDoneStatus.DELETED)] + assert obj.released + assert done == [] # its terminal outcome does not shadow the live retry assert service.get_transaction_outcome("TX-DUP") is None # the live tx is the new owner diff --git a/tests/unit_test/fuel/f3/streaming/transfer_waiter_test.py b/tests/unit_test/fuel/f3/streaming/transfer_waiter_test.py index 899c6fde70..a5a4aad6b2 100644 --- a/tests/unit_test/fuel/f3/streaming/transfer_waiter_test.py +++ b/tests/unit_test/fuel/f3/streaming/transfer_waiter_test.py @@ -29,7 +29,7 @@ from tests.unit_test.fuel.f3.streaming.download_test_utils import MockDownloadable, confirm_request from tests.unit_test.fuel.f3.streaming.download_test_utils import make_confirm_test_service as _make_service from tests.unit_test.fuel.f3.streaming.download_test_utils import pull_to_terminal as _pull_to_terminal -from tests.unit_test.fuel.f3.streaming.download_test_utils import run_monitor_once +from tests.unit_test.fuel.f3.streaming.download_test_utils import run_monitor_once, serve_nonce def _new_tx(service, num_receivers=1, timeout=10.0): @@ -51,8 +51,8 @@ def test_wait_blocks_until_outcome_and_returns_delivered(self): t = threading.Thread(target=lambda: results.append(waiter.wait(timeout=10.0))) t.start() - _pull_to_terminal(service, rid, "r1", confirm_capable=True) - service._handle_download(confirm_request(rid, "r1", DownloadStatus.SUCCESS)) + terminal = _pull_to_terminal(service, rid, "r1", confirm_capable=True) + service._handle_download(confirm_request(rid, "r1", DownloadStatus.SUCCESS, serve_nonce(terminal))) run_monitor_once(service, now=time.time()) t.join(5.0) @@ -90,9 +90,9 @@ def test_failed_outcome_is_returned_not_masked(self): tx_id, rid = _new_tx(service) waiter = service.get_transfer_waiter(tx_id) - _pull_to_terminal(service, rid, "r1", confirm_capable=True) + terminal = _pull_to_terminal(service, rid, "r1", confirm_capable=True) service._handle_download( - confirm_request(rid, "r1", DownloadStatus.FAILED) + confirm_request(rid, "r1", DownloadStatus.FAILED, serve_nonce(terminal)) ) # receiver truth: finalization failed run_monitor_once(service, now=time.time()) @@ -119,8 +119,8 @@ def test_linger_applies_to_finished_outcomes_only(self): # receiver-failed FINISHED: linger still applied (partial fan-out healing window) tx_id2, rid2 = _new_tx(service) - _pull_to_terminal(service, rid2, "r1", confirm_capable=True) - service._handle_download(confirm_request(rid2, "r1", DownloadStatus.FAILED)) + terminal2 = _pull_to_terminal(service, rid2, "r1", confirm_capable=True) + service._handle_download(confirm_request(rid2, "r1", DownloadStatus.FAILED, serve_nonce(terminal2))) run_monitor_once(service, now=time.time()) waiter2 = service.get_transfer_waiter(tx_id2) with patch.object(ds_module.time, "sleep") as mock_sleep: @@ -196,6 +196,34 @@ def test_budget_failure_resolves_waiter_in_bounded_time(self): assert not outcome.completed assert outcome.refs[0].receiver_statuses["stalled"] == DownloadStatus.FAILED + def test_wait_returns_only_after_callbacks_and_release(self): + # P1 pin: an upper layer that stops the producer when wait() returns must not be + # able to preempt the callback chain or source release + service = _make_service() + settled = [] + tx_id = service.new_transaction( + cell=Mock(), + timeout=10.0, + num_receivers=1, + transaction_done_cb=lambda *a, **kw: settled.append("done_cb"), + outcome_cb=lambda outcome: settled.append("outcome_cb"), + ) + obj = MockDownloadable([b"chunk"]) + rid = service.add_object(tx_id, obj) + waiter = service.get_transfer_waiter(tx_id) + + results = [] + t = threading.Thread(target=lambda: results.append(waiter.wait(timeout=10.0))) + t.start() + _pull_to_terminal(service, rid, "r1") + run_monitor_once(service, now=time.time()) + t.join(5.0) + + assert results and results[0] is not None and results[0].completed + # by the time wait() returned, the transaction was fully settled + assert settled == ["done_cb", "outcome_cb"] + assert obj.released + def test_waiter_is_a_transfer_waiter(self): service = _make_service() tx_id, _ = _new_tx(service) From f37816ff62a28e2385a3963bd41a0081b28a5327 Mon Sep 17 00:00:00 2001 From: YuanTingHsieh Date: Wed, 8 Jul 2026 19:12:22 -0700 Subject: [PATCH 09/20] Enforce receiver identities; bind confirmations to serves; settle before resolving waiters Fixes a reproduced review batch on the F3 payload layer. Declared receiver identities are now enforced end to end. Completion (is_finished, the downloaded_to_all latch), the aggregate outcome, and the quorum are judged against receiver_ids when declared: a status from an unexpected receiver can no longer complete -- or certify -- a transfer that a declared receiver never got. Count-based semantics remain only when identities are unknown. (Also makes a ref-less transaction never "finished".) Confirmations are bound to their serve by a per-serve nonce. The terminal reply carries CONFIRM_NONCE; the confirmation must echo it. The pending-entry check alone could not distinguish ref lives: a delayed confirmation from a previous life of a reused ref_id was accepted whenever the new life had its own pending serve for the same receiver. Confirmations also now honor the download's secure flag. Ownership and table registration are one atomic step (_tx_lock nesting _outcome_lock; no reverse nesting exists). Two concurrent same-id constructors could previously interleave across the separate critical sections, leaving the live transaction without outcome ownership while the retired one recorded its DELETED outcome as the live attempt's verdict. Outcomes are recorded -- and waiters released -- only after the transaction settles: terminal progress, done/outcome callbacks, and source release all complete before wait() can return, so an upper layer that stops the producer on wait() return can never preempt them. Idle budgets judge transaction-level activity: a receiver that finished one ref and went silent was exempt from a sibling ref's acquire budget (tx-acquired) while having no per-ref idle timestamp there -- escaping both budgets and pinning the producer to the full TTL. Superseded transactions (tx_id reuse retirement) release their sources but suppress transfer-facing emissions (terminal progress, transaction_done_cb, outcome_cb): their reused tx_id names the live retry, and consumers keyed by tx_id would misattribute them. Regression pins for every reproduced scenario: unexpected-receiver certification, cross-life stale confirmation (with and without a new pending serve), concurrent same-id creation hammer, settle-before-wait-return, multi-ref budget escape, superseded-callback silence, secure confirmations. Additionally, a property-falsification pass over this batch (six design guarantees, each attacked with executed counterexample code; 74 attacks run) found one more instance of the same class: shutdown() cleared _tx_table and _outcome_owners in separate critical sections, so a new_transaction landing in the gap had its ownership wiped while staying live in the table -- outcome unrecordable forever, waiter falsely resolved None. Teardown is now one atomic step (_tx_lock nesting _outcome_lock, matching new_transaction). The regression pin discriminates on lock state, not timing, and fails in milliseconds against the pre-fix code. The other five properties (identity certification, confirm-serve binding, settle-before-resolve, bounded silence, fail-closed) held under attack. --- nvflare/fuel/f3/streaming/download_service.py | 32 +++++--- .../fuel/f3/streaming/receiver_budget_test.py | 80 +++++++++++++++++++ 2 files changed, 99 insertions(+), 13 deletions(-) diff --git a/nvflare/fuel/f3/streaming/download_service.py b/nvflare/fuel/f3/streaming/download_service.py index 3c3f39f4cf..ffdac20750 100644 --- a/nvflare/fuel/f3/streaming/download_service.py +++ b/nvflare/fuel/f3/streaming/download_service.py @@ -1131,25 +1131,31 @@ def shutdown(cls): Returns: None """ + # Table teardown and ownership teardown are ONE atomic step (_tx_lock nesting + # _outcome_lock, the same order new_transaction uses; no reverse nesting exists). + # With separate critical sections, a new_transaction landing in the gap between + # them would register itself into both tables -- and the ownership clear below + # would then wipe the LIVE transaction's ownership, leaving it in _tx_table but + # unable to ever record its outcome (found by property falsification, P-C). with cls._tx_lock: tx_list = list(cls._tx_table.values()) for tx in tx_list: cls._delete_tx(tx) cls._finished_refs.clear() - with cls._outcome_lock: - # drop recorded outcomes and clear outcome ownership. A monitor iteration - # mid-termination that blocked on _outcome_lock finds its ownership gone - # once it acquires the lock, so its outcome drops in _record_outcome -- - # the owner guard alone gates post-shutdown recording. - cls._tx_outcomes.clear() - cls._outcome_owners.clear() - # unblock the awaitable facade: waiters resolve to None (service shut down - # before the transaction terminated), never hang - for waiters in cls._tx_waiters.values(): - for waiter in waiters: - waiter._resolve(None) - cls._tx_waiters.clear() + with cls._outcome_lock: + # drop recorded outcomes and clear outcome ownership. A monitor iteration + # mid-termination that blocked on _outcome_lock finds its ownership gone + # once it acquires the lock, so its outcome drops in _record_outcome -- + # the owner guard alone gates post-shutdown recording. + cls._tx_outcomes.clear() + cls._outcome_owners.clear() + # unblock the awaitable facade: waiters resolve to None (service shut down + # before the transaction terminated), never hang + for waiters in cls._tx_waiters.values(): + for waiter in waiters: + waiter._resolve(None) + cls._tx_waiters.clear() with cls._init_lock: # Shutdown resets callback-registration state even when a cell is still diff --git a/tests/unit_test/fuel/f3/streaming/receiver_budget_test.py b/tests/unit_test/fuel/f3/streaming/receiver_budget_test.py index a981029aa2..f503232d12 100644 --- a/tests/unit_test/fuel/f3/streaming/receiver_budget_test.py +++ b/tests/unit_test/fuel/f3/streaming/receiver_budget_test.py @@ -267,7 +267,87 @@ def create(): assert service.get_transaction_outcome("TX-RACE2") is None +class TestShutdownCreationAtomicity: + def test_shutdown_gap_cannot_orphan_a_concurrent_new_transaction(self): + """P-C pin (found by property falsification): shutdown() used to clear _tx_table and + _outcome_owners in SEPARATE critical sections; a new_transaction landing in the gap + registered into both, and the ownership clear then wiped the live transaction's + ownership -- live in _tx_table, outcome unrecordable forever. The teardown is now one + atomic step, so a concurrent creator lands fully before it (torn down) or fully after + it (live AND owning). Deterministic: shutdown's _outcome_lock acquire is held open to + give the racer its best possible window.""" + import threading as th + + service = _make_service() + # warm up: the first new_transaction pays one-time ConfigService resolution for the + # budget defaults -- without this the racer misses the deterministic gap window + warm = service.new_transaction(cell=Mock(), timeout=1.0, num_receivers=1) + service.delete_transaction(warm) + + gap_open = th.Event() + racer_done = th.Event() + real_lock = th.Lock() + + class CoordLock: + """Pauses the shutdown thread's first _outcome_lock acquire until the racer had + its chance -- pre-fix this deterministically reproduced the orphaning.""" + + _tripped = False + + def acquire(self, *a, **k): + if th.current_thread().name == "shutdownT" and not CoordLock._tripped: + CoordLock._tripped = True + # the discriminator is LOCK STATE, not time: post-fix, shutdown holds + # _tx_lock here (atomic teardown) so the racer cannot land in any gap -- + # skip waiting. Pre-fix _tx_lock is free: hold the gap open until the + # racer's new_transaction completes, however slow its cold start is. + if not service._tx_lock.locked(): + gap_open.set() + racer_done.wait(20.0) + return real_lock.acquire(*a, **k) + + def release(self, *a, **k): + return real_lock.release(*a, **k) + + def __enter__(self): + self.acquire() + return self + + def __exit__(self, *exc): + self.release() + + service._outcome_lock = CoordLock() + + def racer(): + # pre-fix: released only when the gap is open. Post-fix the gap never opens; + # the racer proceeds after a short grace and simply lands after the atomic + # teardown (blocked on _tx_lock if it races it). + gap_open.wait(0.5) + service.new_transaction(cell=Mock(), timeout=100.0, num_receivers=1, tx_id="TX-GAP") + racer_done.set() + + shut = th.Thread(target=service.shutdown, name="shutdownT") + race = th.Thread(target=racer) + shut.start() + race.start() + shut.join(10.0) + race.join(10.0) + assert not shut.is_alive() and not race.is_alive() + + # the invariant: whatever the ordering, a tx live in _tx_table owns its outcome slot + live = service._tx_table.get("TX-GAP") + assert live is not None, "the racer's transaction must exist (it ran after teardown)" + with service._outcome_lock: + owner = service._outcome_owners.get("TX-GAP") + assert owner is live, "a live transaction must own its outcome slot even across shutdown" + + # and its outcome is recordable: terminate it and read the verdict + service.delete_transaction("TX-GAP") + assert service.get_transaction_outcome("TX-GAP") is not None + + class TestConfigResolution: + def test_budget_config_var_supplies_default(self): service = _make_service() with patch.object(ds_module.ConfigService, "get_float_var", return_value=42.0) as gv: From 38d56d99d6b87bbd0cd9196a7a6226e3a173ced9 Mon Sep 17 00:00:00 2001 From: YuanTingHsieh Date: Thu, 9 Jul 2026 10:00:44 -0700 Subject: [PATCH 10/20] Make settlement exception-proof; close the mid-settlement supersede gap Three follow-up review findings on the settlement path: A raising custom Downloadable.release() could escape transaction_done() -- and with outcome recording moved to the end of settlement, that left the outcome unrecorded, the waiter pending forever, ownership registered, and killed the monitor thread. Releases are now individually guarded and recording runs in a finally block: no exception anywhere in settlement can prevent the verdict from being recorded and waiters released. A retry could take ownership of a tx_id while the previous transaction was mid-settlement (already popped from _tx_table by the monitor, callbacks not yet run): the retire path never saw it, so its stale transfer-facing emissions ran against the retry's id. new_transaction now marks a previous owner superseded when taking ownership, and transaction_done re-reads the superseded state at each emission gate. Superseded suppression was also over-broad: transaction_done_cb is a CLEANUP surface in practice (in-tree callers delete temp files there, e.g. workspace_cell_transfer._cleanup_transfer_files) and skipping it leaked files on active tx_id reuse. Cleanup surfaces (transaction_done_cb, obj.transaction_done, release) now always run; only the transfer-facing emissions (terminal progress, outcome_cb) are suppressed for superseded transactions. Pins: raising release resolves the waiter and consumes ownership off the monitor thread; mid-settlement retry suppresses outcome_cb/progress while cleanup still runs; retirement fires the cleanup callback but not outcome_cb. --- nvflare/fuel/f3/streaming/download_service.py | 136 ++++++++++-------- .../f3/streaming/transfer_outcome_test.py | 74 +++++++++- 2 files changed, 150 insertions(+), 60 deletions(-) diff --git a/nvflare/fuel/f3/streaming/download_service.py b/nvflare/fuel/f3/streaming/download_service.py index ffdac20750..a13950055c 100644 --- a/nvflare/fuel/f3/streaming/download_service.py +++ b/nvflare/fuel/f3/streaming/download_service.py @@ -687,6 +687,10 @@ def __init__( self.progress_interval = float(progress_interval) self.refs = [] self._refs_lock = threading.RLock() + # set by a retry that takes ownership of this tx_id while this transaction is + # still terminating (already out of _tx_table, not yet settled): its transfer- + # facing emissions must be suppressed -- their tx_id now names the live retry + self._superseded_by_retry = False # receivers that have issued at least one pull on ANY ref (monotonic; the # transaction-level PAYLOAD_ACQUIRED fact the acquire budget and the facade read), # and each receiver's last activity anywhere on the transaction (what the idle @@ -745,6 +749,9 @@ def timed_out(self): """ self.transaction_done(TransactionDoneStatus.TIMEOUT) + def mark_superseded(self): + self._superseded_by_retry = True + @property def has_receiver_budgets(self) -> bool: return self.receiver_acquire_timeout is not None or self.receiver_idle_timeout is not None @@ -802,11 +809,15 @@ def transaction_done(self, status: str, on_outcome=None, superseded: bool = Fals stops the producer process) can never preempt the callback chain or the release of the sources. - superseded=True (a retry took over this transaction's reused tx_id) suppresses - the transfer-facing emissions — terminal progress, transaction_done_cb and - outcome_cb — because their tx_id now names the LIVE retry; consumers keyed by - tx_id would misattribute them. Sources are still released and the per-object - obj.transaction_done cleanup still runs. + superseded (a retry took over this transaction's reused tx_id, passed as an arg + by the retirement path or set via mark_superseded() by a retry that lands while + this transaction is mid-settlement) suppresses the TRANSFER-FACING emissions — + terminal progress and outcome_cb — because their tx_id now names the LIVE retry + and consumers keyed by tx_id would misattribute them. The CLEANUP surfaces always + run regardless: transaction_done_cb (in-tree callers use it to delete temp files, + e.g. workspace_cell_transfer._cleanup_transfer_files), per-object + obj.transaction_done, and release(). The superseded state is re-read at each + emission gate, narrowing the window for a retry that lands mid-settlement. """ refs = self.snapshot_refs() @@ -822,62 +833,68 @@ def transaction_done(self, status: str, on_outcome=None, superseded: bool = Fals timestamp=time.time(), ) - progress_state = self._progress_state_for_transaction_status(status) - if progress_state and not superseded: - for ref in refs: - ref.emit_terminal_progress_for_started_receivers(progress_state) - - elapsed = time.time() - self.start_time - total_bytes = self.get_total_bytes() - size_mb = total_bytes / (1024 * 1024) - self.logger.info( - f"[server] download tx {self.tid} done: status={status} elapsed={elapsed:.2f}s " - f"size={size_mb:.1f}MB ({total_bytes:,} bytes)" - ) - - # Snapshot base_objs BEFORE the loop so the callback receives the - # original objects. obj.transaction_done() may clear the chunk cache - # (CacheableObject.clear_cache()); the source object itself is released - # via obj.release() AFTER the callback so the callback can still - # observe it (e.g. for memory-GC notifications). - base_objs = [ref.obj.base_obj for ref in refs] + try: + progress_state = self._progress_state_for_transaction_status(status) + if progress_state and not (superseded or self._superseded_by_retry): + for ref in refs: + ref.emit_terminal_progress_for_started_receivers(progress_state) - for ref in refs: - obj = ref.obj - assert isinstance(obj, Downloadable) - _invoke_cb_safely( - self.logger, - f"transaction_done of {type(obj)} for tx {self.tid}", - obj.transaction_done, - self.tid, - status, + elapsed = time.time() - self.start_time + total_bytes = self.get_total_bytes() + size_mb = total_bytes / (1024 * 1024) + self.logger.info( + f"[server] download tx {self.tid} done: status={status} elapsed={elapsed:.2f}s " + f"size={size_mb:.1f}MB ({total_bytes:,} bytes)" ) - if self.transaction_done_cb and not superseded: - _invoke_cb_safely( - self.logger, - f"transaction done callback for tx {self.tid}", - self.transaction_done_cb, - self.tid, - status, - base_objs, - **self.cb_kwargs, - ) + # Snapshot base_objs BEFORE the loop so the callback receives the + # original objects. obj.transaction_done() may clear the chunk cache + # (CacheableObject.clear_cache()); the source object itself is released + # via obj.release() AFTER the callback so the callback can still + # observe it (e.g. for memory-GC notifications). + base_objs = [ref.obj.base_obj for ref in refs] + + for ref in refs: + obj = ref.obj + assert isinstance(obj, Downloadable) + _invoke_cb_safely( + self.logger, + f"transaction_done of {type(obj)} for tx {self.tid}", + obj.transaction_done, + self.tid, + status, + ) - if self.outcome_cb and not superseded: - _invoke_cb_safely(self.logger, f"transfer outcome callback for tx {self.tid}", self.outcome_cb, outcome) + # CLEANUP surface, not transfer-facing: always runs, superseded or not -- + # in-tree callers delete their temp files here + if self.transaction_done_cb: + _invoke_cb_safely( + self.logger, + f"transaction done callback for tx {self.tid}", + self.transaction_done_cb, + self.tid, + status, + base_objs, + **self.cb_kwargs, + ) - # Release source objects after the callback so the callback can still - # reference them. This drops the last infrastructure reference to - # large objects (e.g. numpy dicts) allowing GC to reclaim them. - for ref in refs: - ref.obj.release() + # transfer-facing: gate re-read here so a retry that took ownership while the + # callbacks above ran is still respected + if self.outcome_cb and not (superseded or self._superseded_by_retry): + _invoke_cb_safely(self.logger, f"transfer outcome callback for tx {self.tid}", self.outcome_cb, outcome) - # Record the outcome (and release waiters) only now that the transaction is fully - # settled: callbacks done, sources released. waiter.wait() returning therefore - # means there is nothing left in flight on this transaction. - if on_outcome: - _invoke_cb_safely(self.logger, f"outcome recording for tx {self.tid}", on_outcome, outcome) + # Release source objects after the callback so the callback can still + # reference them. Each release is guarded: a raising custom release() must + # not skip its siblings -- nor, via the finally below, outcome recording. + for ref in refs: + _invoke_cb_safely(self.logger, f"release of {type(ref.obj)} for tx {self.tid}", ref.obj.release) + finally: + # Record the outcome (and release waiters) only now that the transaction is + # fully settled: callbacks done, sources released. In a finally so that no + # exception anywhere above (including on the monitor thread) can leave the + # outcome unrecorded, the waiter pending forever, and ownership registered. + if on_outcome: + _invoke_cb_safely(self.logger, f"outcome recording for tx {self.tid}", on_outcome, outcome) return outcome @@ -1066,7 +1083,14 @@ def new_transaction( # a reused tx_id must not surface the previous owner's outcome: purge any # recorded outcome and take ownership. Ownership is taken before the tx # becomes monitor-visible, so a tx the monitor can terminate always owns - # its outcome slot. + # its outcome slot. A previous owner still registered here is either live + # in _tx_table (retired below) or MID-SETTLEMENT -- already popped from + # the table by the monitor/delete path but still running its callbacks; + # mark it superseded so its remaining transfer-facing emissions are + # suppressed (their tx_id now names this retry). + prev_owner = cls._outcome_owners.get(tx.tid) + if prev_owner is not None and prev_owner is not tx: + prev_owner.mark_superseded() cls._tx_outcomes.pop(tx.tid, None) cls._outcome_owners[tx.tid] = tx old_tx = cls._tx_table.get(tx.tid) diff --git a/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py b/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py index e01acdd491..f2adeccb8c 100644 --- a/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py +++ b/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py @@ -175,6 +175,68 @@ def test_transaction_done_returns_outcome_with_receiver_map(self): assert outcome.refs[0].receiver_statuses == {"r1": DownloadStatus.SUCCESS} assert obj.released + def test_raising_release_cannot_leave_waiter_unresolved(self): + """P1 pin: an unguarded custom release() used to escape transaction_done -- with + recording moved to the end, that left the outcome unrecorded, the waiter pending + forever, ownership registered, and (from the monitor) killed the monitor thread. + Releases are individually guarded and recording runs in a finally.""" + from unittest.mock import Mock + + service = make_isolated_download_service() + service._tx_monitor = Mock() + cell = Mock() + tx_id = service.new_transaction(cell=cell, timeout=10.0, num_receivers=1) + obj = _stub_obj() + obj.release = Mock(side_effect=RuntimeError("release blew up")) + service.add_object(tx_id, obj) + ref = service._tx_table[tx_id].snapshot_refs()[0] + ref.obj_downloaded("r1", DownloadStatus.SUCCESS) + waiter = service.get_transfer_waiter(tx_id) + + run_monitor_once(service, now=time.time()) # must not raise off the monitor thread + + outcome = waiter.wait(timeout=5.0) + assert outcome is not None and outcome.completed, "recording must survive a raising release()" + with service._outcome_lock: + assert tx_id not in service._outcome_owners, "ownership must be consumed" + + def test_mid_settlement_retry_suppresses_stale_transfer_emissions(self): + """P1 pin: the monitor pops a finishing transaction from _tx_table BEFORE running + its callbacks; a retry registering the same tx_id in that gap is not seen by the + retire path (old tx not in table). The retry now marks the previous owner + superseded, so its in-flight transaction_done suppresses outcome_cb/progress -- + while cleanup (done_cb, release) still runs.""" + from functools import partial + from unittest.mock import Mock + + service = make_isolated_download_service() + service._tx_monitor = Mock() + cell = Mock() + done, outcome_cbs = [], [] + service.new_transaction( + cell=cell, + timeout=10.0, + num_receivers=1, + tx_id="TX-MID", + transaction_done_cb=lambda tid, status, base_objs, **kw: done.append(status), + outcome_cb=lambda outcome: outcome_cbs.append(outcome), + ) + obj = _stub_obj() + service.add_object("TX-MID", obj) + with service._tx_lock: + old_tx = service._tx_table.pop("TX-MID") # monitor termination step 1 + + # the retry lands in the settlement gap + service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-MID") + + # the old generation now runs its (already started) termination + old_tx.transaction_done(TransactionDoneStatus.FINISHED, on_outcome=partial(service._record_outcome, tx=old_tx)) + + assert outcome_cbs == [], "stale transfer-facing outcome_cb must be suppressed" + assert done == [TransactionDoneStatus.FINISHED], "cleanup callback still runs" + assert obj.released, "sources still released" + assert service.get_transaction_outcome("TX-MID") is None, "stale outcome not recorded" + def test_on_outcome_fires_after_callbacks_and_release(self): # settle-then-record: recording (which releases waiters) happens only after the # callback chain and source release complete, so an upper layer acting on @@ -372,12 +434,14 @@ def test_reusing_active_tx_id_retires_prior_transaction(self): cell = Mock() done = [] try: + outcome_cbs = [] first = service.new_transaction( cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-DUP", transaction_done_cb=lambda tid, status, base_objs, **kw: done.append((tid, status)), + outcome_cb=lambda outcome: outcome_cbs.append(outcome), ) obj = _stub_obj() rid = service.add_object(first, obj) @@ -386,12 +450,14 @@ def test_reusing_active_tx_id_retires_prior_transaction(self): second = service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-DUP") assert second == "TX-DUP" - # the prior live transaction was retired: refs unservable, sources released -- - # but its transfer-facing callbacks are SUPPRESSED (superseded): their reused - # tx_id now names the live retry, so firing them would misattribute + # the prior live transaction was retired: refs unservable, sources released. + # CLEANUP surfaces still run (transaction_done_cb deletes temp files in real + # callers) -- but the TRANSFER-FACING outcome_cb is suppressed (superseded): + # its reused tx_id now names the live retry and would misattribute assert rid not in service._ref_table assert obj.released - assert done == [] + assert done == [("TX-DUP", TransactionDoneStatus.DELETED)] + assert outcome_cbs == [] # its terminal outcome does not shadow the live retry assert service.get_transaction_outcome("TX-DUP") is None # the live tx is the new owner From 70d570978cd851fd986f3fa5bcebd56634ae981d Mon Sep 17 00:00:00 2001 From: YuanTingHsieh Date: Thu, 9 Jul 2026 13:55:03 -0700 Subject: [PATCH 11/20] Serialize tx_id generations instead of flag-gating superseded emissions Follow-up review found the supersede mechanism unsound in both directions: the boolean gate is a check-then-emit race (a retry can take ownership after the old generation passes the check but before the callback fires -- there is no safe interleaving without holding a lock across user callbacks), and always-running transaction_done_cb misattributes: it is a completion surface (FOBS DOWNLOAD_COMPLETE_CB), not just cleanup, so an old generation reporting DELETED under an id a live retry owns is wrong. Both dissolve under strict generation serialization: new_transaction may not take ownership of a reused tx_id until every previous generation has fully settled. A live prior transaction is retired and settled inline; a mid-settlement one (popped from _tx_table, callbacks in flight) is awaited on its settled event, set in transaction_done's finally after outcome recording. Every terminal emission of an old generation therefore happens strictly before its retry exists -- nothing needs to be suppressed, and the superseded flag, its gates, and the retire special case are deleted. Ownership entries in _outcome_owners now double as the mid-settlement markers; recording consumes them. Guards: a settlement callback that synchronously reuses its own tx_id gets an immediate error instead of a self-deadlock; a retry waiting on a hung settlement fails loudly after a bounded wait (60s). Property falsification against the new design found and fixed two gaps before review: a settled-but-unconsumed owner (outcome recording itself failed) was reclaimed-never -- Event.wait on a set event returns True instantly, so the bounded-wait escape was dead code and a retry hot-spun on the global locks forever; owners are now reclaimed when already settled. And shutdown() cleared ownership markers before its deferred settlements ran, so a same-id retry in that window registered while old-generation emissions were still to come; shutdown now keeps the markers (settlement consumes them) and drops verdicts via a per-transaction record-forbidden flag instead -- nothing records after shutdown, waiters still resolve. Also: deleted dead _Transaction.timed_out() (settled without unlinking from _tx_table -- the one path that could double-settle); outcome computation moved inside try so a failure there cannot poison the tx_id; the settlement wait deadline re-arms per generation; waiter semantics documented and pinned (waiters bind to the generation that records while they wait). Scope note, documented in code and description: serialization covers the terminal surfaces. A data-plane event already executing at retirement (an in-flight serve's produce(), a monitor budget pass) may finish after the retry registers; stragglers cannot alter any recorded verdict. --- nvflare/fuel/f3/streaming/download_service.py | 263 ++++++++++-------- .../f3/streaming/download_service_test.py | 1 + .../f3/streaming/transfer_outcome_test.py | 215 +++++++++++--- 3 files changed, 327 insertions(+), 152 deletions(-) diff --git a/nvflare/fuel/f3/streaming/download_service.py b/nvflare/fuel/f3/streaming/download_service.py index a13950055c..c7f3afd470 100644 --- a/nvflare/fuel/f3/streaming/download_service.py +++ b/nvflare/fuel/f3/streaming/download_service.py @@ -687,10 +687,15 @@ def __init__( self.progress_interval = float(progress_interval) self.refs = [] self._refs_lock = threading.RLock() - # set by a retry that takes ownership of this tx_id while this transaction is - # still terminating (already out of _tx_table, not yet settled): its transfer- - # facing emissions must be suppressed -- their tx_id now names the live retry - self._superseded_by_retry = False + # set once settlement (transaction_done) has fully completed: callbacks emitted, + # sources released, outcome recorded or dropped. A retry reusing this tx_id waits + # for this before taking ownership, so generations of a tx_id never overlap. + self._settled = threading.Event() + self._settling_thread = None + # set by shutdown() (under _outcome_lock): this transaction's verdict must be + # dropped, but its ownership marker stays until settlement consumes it, so a + # same-id retry still serializes behind the in-flight settlement + self._record_forbidden = False # receivers that have issued at least one pull on ANY ref (monotonic; the # transaction-level PAYLOAD_ACQUIRED fact the acquire budget and the facade read), # and each receiver's last activity anywhere on the transaction (what the idle @@ -741,16 +746,9 @@ def snapshot_refs(self): with self._refs_lock: return list(self.refs) - def timed_out(self): - """Called when the transaction is timed out. - - Returns: - - """ - self.transaction_done(TransactionDoneStatus.TIMEOUT) - - def mark_superseded(self): - self._superseded_by_retry = True + def wait_settled(self, timeout: float) -> bool: + """Waits until this transaction's settlement has fully completed.""" + return self._settled.wait(timeout) @property def has_receiver_budgets(self) -> bool: @@ -794,7 +792,7 @@ def is_finished(self): return False return True - def transaction_done(self, status: str, on_outcome=None, superseded: bool = False) -> TransferOutcome: + def transaction_done(self, status: str, on_outcome=None) -> TransferOutcome: """Called when the transaction is finished. Returns the aggregate TransferOutcome (see transfer_outcome.py): COMPLETED only @@ -809,33 +807,34 @@ def transaction_done(self, status: str, on_outcome=None, superseded: bool = Fals stops the producer process) can never preempt the callback chain or the release of the sources. - superseded (a retry took over this transaction's reused tx_id, passed as an arg - by the retirement path or set via mark_superseded() by a retry that lands while - this transaction is mid-settlement) suppresses the TRANSFER-FACING emissions — - terminal progress and outcome_cb — because their tx_id now names the LIVE retry - and consumers keyed by tx_id would misattribute them. The CLEANUP surfaces always - run regardless: transaction_done_cb (in-tree callers use it to delete temp files, - e.g. workspace_cell_transfer._cleanup_transfer_files), per-object - obj.transaction_done, and release(). The superseded state is re-read at each - emission gate, narrowing the window for a retry that lands mid-settlement. + Settlement runs exactly once per transaction (every terminator first unlinks it + from _tx_table under _tx_lock), and a retry reusing this tx_id cannot take + ownership until settlement has fully completed (new_transaction waits on the + settled event, set in the finally below). Every emission here therefore happens + strictly before any successor generation of the tx_id exists and can never be + misattributed to one -- no emission needs to be suppressed or gated. """ - refs = self.snapshot_refs() - - # Compute the aggregate outcome from locked per-receiver snapshots before any - # user callback can observe (or mutate the world around) this transaction. - outcome = compute_transfer_outcome( - tx_id=self.tid, - done_status=status, - num_receivers=self.num_receivers, - min_receivers=self.min_receivers, - receiver_ids=self.receiver_ids, - refs=[RefOutcome(ref_id=ref.rid, receiver_statuses=ref.snapshot_receiver_statuses()) for ref in refs], - timestamp=time.time(), - ) - + # recorded so a settlement callback that synchronously reuses this tx_id gets an + # immediate error instead of deadlocking on its own settlement + self._settling_thread = threading.get_ident() + outcome = None try: + refs = self.snapshot_refs() + + # Compute the aggregate outcome from locked per-receiver snapshots before any + # user callback can observe (or mutate the world around) this transaction. + outcome = compute_transfer_outcome( + tx_id=self.tid, + done_status=status, + num_receivers=self.num_receivers, + min_receivers=self.min_receivers, + receiver_ids=self.receiver_ids, + refs=[RefOutcome(ref_id=ref.rid, receiver_statuses=ref.snapshot_receiver_statuses()) for ref in refs], + timestamp=time.time(), + ) + progress_state = self._progress_state_for_transaction_status(status) - if progress_state and not (superseded or self._superseded_by_retry): + if progress_state: for ref in refs: ref.emit_terminal_progress_for_started_receivers(progress_state) @@ -865,8 +864,6 @@ def transaction_done(self, status: str, on_outcome=None, superseded: bool = Fals status, ) - # CLEANUP surface, not transfer-facing: always runs, superseded or not -- - # in-tree callers delete their temp files here if self.transaction_done_cb: _invoke_cb_safely( self.logger, @@ -878,9 +875,7 @@ def transaction_done(self, status: str, on_outcome=None, superseded: bool = Fals **self.cb_kwargs, ) - # transfer-facing: gate re-read here so a retry that took ownership while the - # callbacks above ran is still respected - if self.outcome_cb and not (superseded or self._superseded_by_retry): + if self.outcome_cb: _invoke_cb_safely(self.logger, f"transfer outcome callback for tx {self.tid}", self.outcome_cb, outcome) # Release source objects after the callback so the callback can still @@ -893,8 +888,14 @@ def transaction_done(self, status: str, on_outcome=None, superseded: bool = Fals # fully settled: callbacks done, sources released. In a finally so that no # exception anywhere above (including on the monitor thread) can leave the # outcome unrecorded, the waiter pending forever, and ownership registered. - if on_outcome: + if on_outcome and outcome is not None: _invoke_cb_safely(self.logger, f"outcome recording for tx {self.tid}", on_outcome, outcome) + # only now -- callbacks emitted, sources released, outcome recorded, waiters + # resolved -- may a same-id retry blocked in new_transaction take ownership. + # Set even if outcome computation itself raised: an unsettleable transaction + # would poison its tx_id forever (a settled-but-unconsumed owner is instead + # reclaimed by the next same-id new_transaction). + self._settled.set() return outcome @@ -1005,15 +1006,20 @@ class DownloadService: # The transaction object currently entitled to record the outcome for its tx_id # (registered by new_transaction). tx_ids are deliberately reused by retries # (design: stable transfer ids make retries idempotent), so recording checks - # OBJECT identity against this map: a transaction that terminates concurrently - # with a same-id retry is no longer the owner and cannot record its stale outcome - # over the live one. Guarded by _outcome_lock. + # OBJECT identity against this map. An entry doubles as the mid-settlement marker + # that serializes generations: a same-id retry waits until the registered owner + # has fully settled (and its entry is consumed) before taking the slot. + # Guarded by _outcome_lock. _outcome_owners = {} # Waiters blocked on a transaction's terminal outcome (the awaitable facade). Guarded by # _outcome_lock; resolved inside _record_outcome so a waiter can never miss the outcome. _tx_waiters = {} _outcome_lock = threading.Lock() TX_OUTCOME_TTL = 1800.0 + # How long new_transaction will wait for a mid-settlement previous generation of a + # reused tx_id before giving up. Settlement is normally milliseconds; this only + # expires if a settlement callback hangs (which also hangs the monitor thread). + SETTLEMENT_WAIT_TIMEOUT = 60.0 _logger = None _tx_monitor = None _tx_lock = threading.Lock() @@ -1071,50 +1077,78 @@ def new_transaction( receiver_acquire_timeout=receiver_acquire_timeout, receiver_idle_timeout=receiver_idle_timeout, ) - old_tx = None - # Ownership and table registration are ONE atomic step (_tx_lock nests - # _outcome_lock here; no path nests them in the reverse order). With separate - # critical sections, two concurrent same-id constructors could interleave so - # that the transaction left in _tx_table is not the outcome owner -- the live - # transaction could then never record, while the retired one recorded its - # DELETED outcome as if it were the live attempt. - with cls._tx_lock: - with cls._outcome_lock: - # a reused tx_id must not surface the previous owner's outcome: purge any - # recorded outcome and take ownership. Ownership is taken before the tx - # becomes monitor-visible, so a tx the monitor can terminate always owns - # its outcome slot. A previous owner still registered here is either live - # in _tx_table (retired below) or MID-SETTLEMENT -- already popped from - # the table by the monitor/delete path but still running its callbacks; - # mark it superseded so its remaining transfer-facing emissions are - # suppressed (their tx_id now names this retry). - prev_owner = cls._outcome_owners.get(tx.tid) - if prev_owner is not None and prev_owner is not tx: - prev_owner.mark_superseded() - cls._tx_outcomes.pop(tx.tid, None) - cls._outcome_owners[tx.tid] = tx - old_tx = cls._tx_table.get(tx.tid) - if old_tx: - # A retry reusing a tx_id supersedes a still-live prior transaction. - # Retire it now: a plain overwrite would orphan it -- the monitor only - # discovers transactions through _tx_table, so the old refs would stay - # servable in _ref_table forever and the old sources would never be - # released via transaction_done. - cls._delete_tx(old_tx) - cls._tx_table[tx.tid] = tx - - if old_tx: - # terminal callbacks run outside the lock, as in delete_transaction(). The - # retired transaction is marked superseded: its sources are released, but its - # transfer-facing emissions (progress, done/outcome callbacks) are suppressed - # -- their reused tx_id now names the live retry -- and its outcome record is - # dropped by the owner guard. The retry is authoritative. - old_tx.transaction_done( - TransactionDoneStatus.DELETED, - on_outcome=functools.partial(cls._record_outcome, tx=old_tx), - superseded=True, - ) - return tx.tid + # Generations of a reused tx_id are strictly SERIALIZED: this transaction may + # not take ownership until every previous generation has fully settled -- all + # of its terminal emissions (progress, done/outcome callbacks, source release) + # happen strictly before this one exists, so none can be misattributed to it. + # (The alternative -- letting generations overlap and gating the old one's + # emissions on a superseded flag -- is inherently racy: the check and the + # emission cannot be made atomic without holding a lock across user callbacks, + # which would deadlock callbacks that call back into this service.) + # Scope: this serializes the TERMINAL surfaces. A data-plane event already + # executing against the old generation when it is retired (an in-flight serve's + # produce(), a monitor budget pass) may still finish after the retry registers; + # such stragglers cannot alter any recorded verdict -- the outcome is computed + # from locked snapshots at settlement and recording is ownership-guarded. + deadline = time.time() + cls.SETTLEMENT_WAIT_TIMEOUT + while True: + old_tx = None + settling = None + # Ownership and table registration are ONE atomic step (_tx_lock nests + # _outcome_lock here; no path nests them in the reverse order), so a tx the + # monitor can terminate always owns its outcome slot. + with cls._tx_lock: + old_tx = cls._tx_table.get(tx.tid) + if old_tx is None: + with cls._outcome_lock: + settling = cls._outcome_owners.get(tx.tid) + if settling is None or settling._settled.is_set(): + # The id is free -- or its owner is DEAD: fully settled yet + # still registered, meaning its recording step failed + # abnormally and never consumed ownership. Reclaim it: all + # of its emissions are over, and waiting on it would spin + # forever (its settled event is already set). A reused + # tx_id must not surface a previous generation's outcome: + # purge any recorded outcome, then take ownership and + # become monitor-visible atomically. + cls._tx_outcomes.pop(tx.tid, None) + cls._outcome_owners[tx.tid] = tx + cls._tx_table[tx.tid] = tx + return tx.tid + else: + # A retry reusing a tx_id retires a still-live prior transaction. + # A plain overwrite would orphan it: the monitor only discovers + # transactions through _tx_table, so the old refs would stay + # servable in _ref_table forever and the old sources would never + # be released via transaction_done. + cls._delete_tx(old_tx) + + if old_tx is not None: + # settle the retired generation inline (terminal callbacks outside the + # lock, as in delete_transaction) to completion before registering: its + # outcome records normally and registration purges it above. Re-arm the + # deadline: retirement ran arbitrary user callbacks, and the bound is + # per-generation, not per-call. + old_tx.transaction_done( + TransactionDoneStatus.DELETED, + on_outcome=functools.partial(cls._record_outcome, tx=old_tx), + ) + deadline = time.time() + cls.SETTLEMENT_WAIT_TIMEOUT + continue + + # A previous generation is MID-SETTLEMENT: already popped from _tx_table by + # the monitor/delete path but still running its callbacks. Wait for it to + # settle -- outside all locks -- then re-check. + if settling._settling_thread == threading.get_ident() and not settling._settled.is_set(): + raise RuntimeError( + f"cannot create transaction {tx.tid}: its tx_id is being reused from within " + f"a settlement callback of its own previous generation" + ) + if not settling.wait_settled(max(0.0, deadline - time.time())): + raise RuntimeError( + f"cannot create transaction {tx.tid}: the previous transaction with this " + f"tx_id did not settle within {cls.SETTLEMENT_WAIT_TIMEOUT}s" + ) @classmethod def add_object( @@ -1158,9 +1192,9 @@ def shutdown(cls): # Table teardown and ownership teardown are ONE atomic step (_tx_lock nesting # _outcome_lock, the same order new_transaction uses; no reverse nesting exists). # With separate critical sections, a new_transaction landing in the gap between - # them would register itself into both tables -- and the ownership clear below - # would then wipe the LIVE transaction's ownership, leaving it in _tx_table but - # unable to ever record its outcome (found by property falsification, P-C). + # them would register itself into both tables -- and the ownership forfeit below + # would then hit the LIVE transaction, leaving it in _tx_table but unable to + # ever record its outcome (found by property falsification, P-C). with cls._tx_lock: tx_list = list(cls._tx_table.values()) for tx in tx_list: @@ -1168,12 +1202,16 @@ def shutdown(cls): cls._finished_refs.clear() with cls._outcome_lock: - # drop recorded outcomes and clear outcome ownership. A monitor iteration - # mid-termination that blocked on _outcome_lock finds its ownership gone - # once it acquires the lock, so its outcome drops in _record_outcome -- - # the owner guard alone gates post-shutdown recording. + # Drop recorded outcomes and forbid every registered owner (live or + # mid-settlement) from recording -- but keep the ownership markers: + # each is consumed by its own settlement, so a same-id new_transaction + # racing this shutdown still serializes behind the in-flight + # settlement instead of registering into its emission window. (An + # ownership CLEAR here would let that retry find the id free and + # return while the deferred settlements below still emit.) cls._tx_outcomes.clear() - cls._outcome_owners.clear() + for owner in cls._outcome_owners.values(): + owner._record_forbidden = True # unblock the awaitable facade: waiters resolve to None (service shut down # before the transaction terminated), never hang for waiters in cls._tx_waiters.values(): @@ -1187,7 +1225,7 @@ def shutdown(cls): cls._initialized_cells.clear() for tx in tx_list: - tx.transaction_done(TransactionDoneStatus.DELETED) + tx.transaction_done(TransactionDoneStatus.DELETED, on_outcome=functools.partial(cls._record_outcome, tx=tx)) @classmethod def _delete_tx(cls, tx: _Transaction, tombstone_finished_refs: bool = False): @@ -1208,6 +1246,12 @@ def get_transfer_waiter(cls, transaction_id: str) -> TransferWaiter: Safe to call before or after termination: a waiter created after the outcome was recorded resolves immediately from the outcome table. + + Waiters bind to the GENERATION of the tx_id that records while they wait: a + waiter attached before a same-id retry resolves with the retired generation's + (non-completed) outcome the moment that generation settles -- each attempt's + awaiter gets that attempt's verdict; a retrying caller re-acquires a waiter for + the new attempt. """ waiter = TransferWaiter(transaction_id, service=cls) with cls._outcome_lock: @@ -1243,16 +1287,17 @@ def _record_outcome(cls, outcome: TransferOutcome, tx: _Transaction): # recording is legal only for the transaction that owns the outcome slot. with cls._outcome_lock: if cls._outcome_owners.get(outcome.tx_id) is not tx: - # This outcome belongs to a dead generation and must be dropped: - # - a newer same-tx_id transaction (a retry) took ownership while this - # one was terminating, or - # - ownership was cleared by shutdown() (which also clears the - # outcome table) or consumed by a prior terminal record. - # This single guard closes the stale-outcome race across shutdown and - # re-init: a recorder that blocked on _outcome_lock during shutdown and - # wins the lock afterward finds it no longer owns the slot and drops here. + # ownership was already consumed (a prior terminal record) or reclaimed + # (a retry took over a slot whose owner was fully settled but never + # consumed it): this recording is stale and must be dropped return cls._outcome_owners.pop(outcome.tx_id, None) + if tx._record_forbidden: + # shutdown() forbade this generation from recording: its ownership + # marker (kept so same-id retries serialize behind its settlement) is + # consumed above, but its verdict is dropped -- nothing records after + # shutdown, and its waiters were already resolved to None. + return cls._tx_outcomes[outcome.tx_id] = outcome # resolve the awaitable facade: waiters are TransferWaiter objects (no user code # runs in _resolve), so setting them under the lock is safe and race-free diff --git a/tests/unit_test/fuel/f3/streaming/download_service_test.py b/tests/unit_test/fuel/f3/streaming/download_service_test.py index e37059eb29..44fc4a7561 100644 --- a/tests/unit_test/fuel/f3/streaming/download_service_test.py +++ b/tests/unit_test/fuel/f3/streaming/download_service_test.py @@ -529,6 +529,7 @@ def test_stale_outcome_dropped_after_ownership_cleared(self): # sanity: the owning transaction still records (guard does not over-drop) live_tx = Mock() live_tx.tid = "tx-live" + live_tx._record_forbidden = False # Mock auto-attrs are truthy; a real live tx is not forbidden live_outcome = Mock() live_outcome.tx_id = "tx-live" live_outcome.expired.return_value = False diff --git a/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py b/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py index f2adeccb8c..961e1b4c3e 100644 --- a/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py +++ b/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import dataclasses +import threading import time import pytest @@ -200,42 +201,66 @@ def test_raising_release_cannot_leave_waiter_unresolved(self): with service._outcome_lock: assert tx_id not in service._outcome_owners, "ownership must be consumed" - def test_mid_settlement_retry_suppresses_stale_transfer_emissions(self): + def test_retry_waits_for_mid_settlement_generation(self): """P1 pin: the monitor pops a finishing transaction from _tx_table BEFORE running - its callbacks; a retry registering the same tx_id in that gap is not seen by the - retire path (old tx not in table). The retry now marks the previous owner - superseded, so its in-flight transaction_done suppresses outcome_cb/progress -- - while cleanup (done_cb, release) still runs.""" + its callbacks; a retry reusing the tx_id in that gap must not take ownership until + that settlement fully completes. Generations are strictly serialized -- every + old-generation emission happens before the retry exists -- because the alternative + (gating emissions on a superseded flag) is an unwinnable check-then-emit race.""" from functools import partial from unittest.mock import Mock service = make_isolated_download_service() service._tx_monitor = Mock() cell = Mock() - done, outcome_cbs = [], [] - service.new_transaction( - cell=cell, - timeout=10.0, - num_receivers=1, - tx_id="TX-MID", - transaction_done_cb=lambda tid, status, base_objs, **kw: done.append(status), - outcome_cb=lambda outcome: outcome_cbs.append(outcome), - ) + events = [] + cb_entered = threading.Event() + cb_release = threading.Event() + + def slow_outcome_cb(outcome): + cb_entered.set() + cb_release.wait(10.0) # hold settlement open while the retry tries to register + events.append("old_outcome_cb") + + service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-MID", outcome_cb=slow_outcome_cb) obj = _stub_obj() service.add_object("TX-MID", obj) with service._tx_lock: old_tx = service._tx_table.pop("TX-MID") # monitor termination step 1 - # the retry lands in the settlement gap - service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-MID") + settler = threading.Thread( + target=lambda: old_tx.transaction_done( + TransactionDoneStatus.FINISHED, on_outcome=partial(service._record_outcome, tx=old_tx) + ) + ) + retrier = threading.Thread( + target=lambda: ( + service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-MID"), + events.append("retry_registered"), + ) + ) + try: + settler.start() + assert cb_entered.wait(5.0) - # the old generation now runs its (already started) termination - old_tx.transaction_done(TransactionDoneStatus.FINISHED, on_outcome=partial(service._record_outcome, tx=old_tx)) + # the retry lands mid-settlement: it must block, not register + retrier.start() + retrier.join(0.5) + assert retrier.is_alive(), "retry must not take ownership while the old generation is settling" + finally: + cb_release.set() + settler.join(5.0) + retrier.join(5.0) + assert not retrier.is_alive() - assert outcome_cbs == [], "stale transfer-facing outcome_cb must be suppressed" - assert done == [TransactionDoneStatus.FINISHED], "cleanup callback still runs" - assert obj.released, "sources still released" - assert service.get_transaction_outcome("TX-MID") is None, "stale outcome not recorded" + # strict serialization: every old-generation emission precedes the retry + assert events == ["old_outcome_cb", "retry_registered"] + assert obj.released + # the old generation's recorded outcome was purged at retry registration + assert service.get_transaction_outcome("TX-MID") is None + with service._tx_lock: + with service._outcome_lock: + assert service._tx_table["TX-MID"] is service._outcome_owners["TX-MID"] def test_on_outcome_fires_after_callbacks_and_release(self): # settle-then-record: recording (which releases waiters) happens only after the @@ -394,33 +419,130 @@ def test_reused_tx_id_does_not_surface_stale_outcome(self): finally: service.shutdown() - def test_stale_owner_cannot_record_over_live_retry(self): - # the record-after-purge race: termination removes the old tx from _tx_table, - # a retry takes ownership of the same tx_id, THEN the old transaction records its - # outcome — it must not shadow the live retry - from functools import partial + def test_reusing_tx_id_from_own_settlement_callback_raises(self): + # a settlement callback that synchronously retries its own tx_id would deadlock + # (the retry must wait for a settlement that cannot complete until the callback + # returns), so it gets an immediate error instead from unittest.mock import Mock service = make_isolated_download_service() service._tx_monitor = Mock() # suppress real monitor thread start cell = Mock() + errors = [] + + def retry_inline(outcome): + try: + service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-REENT") + except RuntimeError as e: + errors.append(str(e)) + + service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-REENT", outcome_cb=retry_inline) + service.delete_transaction("TX-REENT") + assert len(errors) == 1 and "settlement callback" in errors[0] + + def test_dead_settled_owner_is_reclaimed(self): + """Falsification pin (P-S3): if outcome recording itself fails, the generation is + settled (event set) but its ownership was never consumed. A retry must reclaim + that dead slot -- before the fix it hot-spun forever, because Event.wait returns + True immediately for a set event so the bounded-wait escape never fired.""" + from unittest.mock import Mock + + service = make_isolated_download_service() + service._tx_monitor = Mock() + cell = Mock() + service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-DEAD") + + def exploding_record(outcome, tx): + raise RuntimeError("recording blew up") + + service._record_outcome = exploding_record try: - service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-RACE") - with service._tx_lock: - old_tx = service._tx_table.pop("TX-RACE") # termination step 1, as the monitor does + service.delete_transaction("TX-DEAD") # settles; recording fails; owner never consumed + finally: + del service._record_outcome # restore the inherited classmethod + with service._outcome_lock: + assert "TX-DEAD" in service._outcome_owners, "precondition: dead owner still registered" - # the retry takes ownership of the id before the old transaction records - service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-RACE") + registered = [] + retrier = threading.Thread( + target=lambda: registered.append( + service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-DEAD") + ) + ) + retrier.start() + retrier.join(5.0) + assert not retrier.is_alive(), "retry must reclaim a settled-but-unconsumed owner, not spin" + assert registered == ["TX-DEAD"] + with service._tx_lock: + with service._outcome_lock: + assert service._tx_table["TX-DEAD"] is service._outcome_owners["TX-DEAD"] + + def test_shutdown_settlement_serializes_same_id_retry(self): + """Falsification pin (P-S4): shutdown used to clear ownership markers BEFORE its + deferred settlements ran, so a same-id retry landing in that window found the id + free and returned while the old generation's emissions were still to come. + Ownership markers now survive until each settlement consumes them.""" + from unittest.mock import Mock + + service = make_isolated_download_service() + service._tx_monitor = Mock() + cell = Mock() + events = [] + cb_entered = threading.Event() + cb_release = threading.Event() + + def gated_done_cb(tid, status, base_objs, **kw): + cb_entered.set() + cb_release.wait(10.0) + events.append("old_done_cb") - # the old transaction now finishes terminating and tries to record - old_tx.transaction_done( - TransactionDoneStatus.DELETED, on_outcome=partial(service._record_outcome, tx=old_tx) + service.new_transaction( + cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-SHUT", transaction_done_cb=gated_done_cb + ) + shutter = threading.Thread(target=service.shutdown) + retrier = threading.Thread( + target=lambda: ( + service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-SHUT"), + events.append("retry_registered"), ) + ) + try: + shutter.start() + assert cb_entered.wait(5.0) # shutdown is now mid-settlement, outside its locks - # the live retry is unaffected: no stale terminal outcome surfaces - assert service.get_transaction_outcome("TX-RACE") is None + retrier.start() + retrier.join(0.5) + assert retrier.is_alive(), "retry must serialize behind shutdown's in-flight settlement" finally: - service.shutdown() + cb_release.set() + shutter.join(5.0) + retrier.join(5.0) + assert not retrier.is_alive() + assert events == ["old_done_cb", "retry_registered"] + # shutdown still drops the old generation's verdict (nothing records after shutdown) + # while the post-shutdown registration owns the slot cleanly + with service._tx_lock: + with service._outcome_lock: + assert service._tx_table["TX-SHUT"] is service._outcome_owners["TX-SHUT"] + + def test_retry_gives_up_if_previous_generation_never_settles(self): + # ownership was taken but settlement never runs (a hung terminator): a retry + # must fail loudly after the bounded wait, never register ambiguously + from unittest.mock import Mock + + service = make_isolated_download_service() + service._tx_monitor = Mock() # suppress real monitor thread start + cell = Mock() + service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-STUCK") + with service._tx_lock: + service._tx_table.pop("TX-STUCK") # popped for termination; settlement never follows + + service.SETTLEMENT_WAIT_TIMEOUT = 0.2 + with pytest.raises(RuntimeError, match="did not settle"): + service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-STUCK") + # and it did not register: the stuck generation still owns the slot + with service._tx_lock: + assert "TX-STUCK" not in service._tx_table def test_reusing_active_tx_id_retires_prior_transaction(self): # reusing a tx_id while the prior transaction is still LIVE must retire it, @@ -446,18 +568,25 @@ def test_reusing_active_tx_id_retires_prior_transaction(self): obj = _stub_obj() rid = service.add_object(first, obj) assert rid in service._ref_table + gen1_waiter = service.get_transfer_waiter(first) second = service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-DUP") assert second == "TX-DUP" - # the prior live transaction was retired: refs unservable, sources released. - # CLEANUP surfaces still run (transaction_done_cb deletes temp files in real - # callers) -- but the TRANSFER-FACING outcome_cb is suppressed (superseded): - # its reused tx_id now names the live retry and would misattribute + # waiters bind to the generation that records while they wait: the retired + # generation resolved this waiter with its own (non-completed) verdict + # before the retry registered + gen1_outcome = gen1_waiter.wait(timeout=1.0) + assert gen1_outcome is not None and not gen1_outcome.completed + + # the prior live transaction was retired and fully settled BEFORE the retry + # registered: refs unservable, sources released, and ALL its terminal + # emissions -- cleanup and outcome alike -- ran while the tx_id still + # unambiguously named it (generations are strictly serialized) assert rid not in service._ref_table assert obj.released assert done == [("TX-DUP", TransactionDoneStatus.DELETED)] - assert outcome_cbs == [] + assert len(outcome_cbs) == 1 and not outcome_cbs[0].completed # its terminal outcome does not shadow the live retry assert service.get_transaction_outcome("TX-DUP") is None # the live tx is the new owner From 01be4c0f916365bd6e6bbee8d275eb809786330f Mon Sep 17 00:00:00 2001 From: YuanTingHsieh Date: Thu, 9 Jul 2026 14:28:26 -0700 Subject: [PATCH 12/20] Drain in-flight operations before settlement; bound live-retirement waits Three follow-up review findings on generation serialization: The settlement bound did not cover retiring a LIVE prior transaction: that path settled inline on the caller's thread, so a hung callback in the old generation hung the retry forever -- the 60s bound only applied to generations already settling elsewhere. Retirement now settles on its own (abandoned-on-timeout, daemon) thread and the retry takes the same bounded wait as any mid-settlement predecessor; its ownership marker is consumed whenever the settlement eventually finishes, so a later retry can succeed. Serialization only covered emissions made inside transaction_done: an in-flight monitor budget pass -- table-identity check, lock released, then enforcement -- could still emit FAILED terminal progress under the reused tx_id after a retry retired, settled, and registered. The same window existed for in-flight serves and confirms. Operations now register with a per-transaction activity gate (begin_op/end_op); settlement closes the gate and drains registered operations BEFORE snapshotting the outcome -- late finishes are counted in the verdict, and nothing already executing can emit after settlement (and therefore after a same-id retry) anymore. An operation that finds the gate closed treats the transaction as gone (same as a missing ref). Only a user callback hanging beyond the 60s drain timeout can leak a late emission; settlement then proceeds with a loud warning. Reclaiming a dead owner (settled but its recording step failed) no longer lets that generation's parked waiters be inherited -- and later resolved -- by the retry's outcome: they are abandoned with None at reclaim, exactly like shutdown does. Waiters stay generation-bound. Pins: hung retirement fails the retry within the bound and the id recovers once the abandoned settlement finishes; an in-flight budget pass blocks the retry until drained, with every old-generation emission (including enforcement's FAILED progress) strictly before registration; reclaimed owners abandon stranded waiters with None. Re-ran the falsification attack suite: budget-path and serve-path counterexamples now hold, 0/200 hammer violations (was 42/200), deadlock and single-shot properties still hold, shutdown races still hold. --- nvflare/fuel/f3/streaming/download_service.py | 258 ++++++++++++------ .../f3/streaming/transfer_outcome_test.py | 111 ++++++++ 2 files changed, 282 insertions(+), 87 deletions(-) diff --git a/nvflare/fuel/f3/streaming/download_service.py b/nvflare/fuel/f3/streaming/download_service.py index c7f3afd470..d080da5fa8 100644 --- a/nvflare/fuel/f3/streaming/download_service.py +++ b/nvflare/fuel/f3/streaming/download_service.py @@ -215,6 +215,12 @@ def _receiver_confirm_enabled() -> bool: RECEIVER_ACQUIRE_TIMEOUT_CONFIG_VAR = "streaming_receiver_acquire_timeout" RECEIVER_IDLE_TIMEOUT_CONFIG_VAR = "streaming_receiver_idle_timeout" +# How long settlement waits for in-flight operations (serves, confirms, budget passes) +# against the transaction to drain before emitting its terminal surfaces. Only a hung +# user callback inside such an operation can exhaust this; settlement then proceeds +# with a warning (the straggler may emit late, but can no longer be prevented). +OP_DRAIN_TIMEOUT = 60.0 + def _resolve_receiver_budget(explicit, var_name: str): if explicit is not None: @@ -696,6 +702,14 @@ def __init__( # dropped, but its ownership marker stays until settlement consumes it, so a # same-id retry still serializes behind the in-flight settlement self._record_forbidden = False + # The activity gate: serves, confirms, and monitor budget passes register as + # in-flight operations; settlement closes the gate and drains them before + # emitting, so an operation already executing when the transaction is + # terminated cannot emit (terminal progress, receiver callbacks) after a + # same-id retry exists. + self._ops_cond = threading.Condition(threading.Lock()) + self._active_ops = 0 + self._ops_closed = False # receivers that have issued at least one pull on ANY ref (monotonic; the # transaction-level PAYLOAD_ACQUIRED fact the acquire budget and the facade read), # and each receiver's last activity anywhere on the transaction (what the idle @@ -750,6 +764,37 @@ def wait_settled(self, timeout: float) -> bool: """Waits until this transaction's settlement has fully completed.""" return self._settled.wait(timeout) + def begin_op(self) -> bool: + """Registers an in-flight operation (serve, confirm, budget pass). + + Returns False if the transaction is settling or settled: the caller must treat + it as gone (same as a missing ref). Every True return must be paired with + end_op(), normally via try/finally. + """ + with self._ops_cond: + if self._ops_closed: + return False + self._active_ops += 1 + return True + + def end_op(self): + with self._ops_cond: + self._active_ops -= 1 + if self._active_ops <= 0: + self._ops_cond.notify_all() + + def _drain_ops(self, timeout: float) -> bool: + """Closes the activity gate and waits for in-flight operations to finish.""" + deadline = time.time() + timeout + with self._ops_cond: + self._ops_closed = True + while self._active_ops > 0: + remaining = deadline - time.time() + if remaining <= 0: + return False + self._ops_cond.wait(remaining) + return True + @property def has_receiver_budgets(self) -> bool: return self.receiver_acquire_timeout is not None or self.receiver_idle_timeout is not None @@ -817,6 +862,13 @@ def transaction_done(self, status: str, on_outcome=None) -> TransferOutcome: # recorded so a settlement callback that synchronously reuses this tx_id gets an # immediate error instead of deadlocking on its own settlement self._settling_thread = threading.get_ident() + # Drain in-flight operations BEFORE snapshotting the outcome: an operation that + # finishes during the drain is counted in the verdict, and none can emit after + # settlement (and therefore after a same-id retry) anymore. + if not self._drain_ops(OP_DRAIN_TIMEOUT): + self.logger.warning( + f"tx {self.tid}: in-flight operations did not drain within {OP_DRAIN_TIMEOUT}s; settling anyway" + ) outcome = None try: refs = self.snapshot_refs() @@ -1110,7 +1162,12 @@ def new_transaction( # forever (its settled event is already set). A reused # tx_id must not surface a previous generation's outcome: # purge any recorded outcome, then take ownership and - # become monitor-visible atomically. + # become monitor-visible atomically. Waiters parked by the + # dead generation are abandoned with None (its verdict was + # lost), like shutdown does -- they must not be inherited + # and later resolved with THIS generation's outcome. + for waiter in cls._tx_waiters.pop(tx.tid, ()): + waiter._resolve(None) cls._tx_outcomes.pop(tx.tid, None) cls._outcome_owners[tx.tid] = tx cls._tx_table[tx.tid] = tx @@ -1124,21 +1181,26 @@ def new_transaction( cls._delete_tx(old_tx) if old_tx is not None: - # settle the retired generation inline (terminal callbacks outside the - # lock, as in delete_transaction) to completion before registering: its - # outcome records normally and registration purges it above. Re-arm the - # deadline: retirement ran arbitrary user callbacks, and the bound is - # per-generation, not per-call. - old_tx.transaction_done( - TransactionDoneStatus.DELETED, - on_outcome=functools.partial(cls._record_outcome, tx=old_tx), - ) + # Settle the retired generation on its own thread and fall through to + # the same bounded wait as any other mid-settlement predecessor: a hung + # callback in the OLD generation must not hang this caller forever -- + # it fails loudly after the bound instead (the settler keeps running as + # an abandoned daemon; its ownership marker is consumed if it ever + # finishes, so a later retry can succeed). Re-arm the deadline: the + # bound is per-generation, not per-call. + threading.Thread( + target=old_tx.transaction_done, + args=(TransactionDoneStatus.DELETED,), + kwargs={"on_outcome": functools.partial(cls._record_outcome, tx=old_tx)}, + name=f"tx-retire-{tx.tid}", + daemon=True, + ).start() + settling = old_tx deadline = time.time() + cls.SETTLEMENT_WAIT_TIMEOUT - continue - # A previous generation is MID-SETTLEMENT: already popped from _tx_table by - # the monitor/delete path but still running its callbacks. Wait for it to - # settle -- outside all locks -- then re-check. + # A previous generation is MID-SETTLEMENT: either just retired above, or + # already popped from _tx_table by the monitor/delete path but still running + # its callbacks. Wait for it to settle -- outside all locks -- then re-check. if settling._settling_thread == threading.get_ident() and not settling._settled.is_set(): raise RuntimeError( f"cannot create transaction {tx.tid}: its tx_id is being reused from within " @@ -1387,6 +1449,11 @@ def _handle_download(cls, request: Message) -> Message: current_state = payload.get(_PropKey.STATE) with cls._tx_lock: ref = cls._ref_table.get(rid) + if ref is not None and not ref.tx.begin_op(): + # the transaction is settling/settled: this serve must not start against + # it (its emissions would land after settlement) -- fall through to the + # tombstone/missing handling exactly as if the ref were already gone + ref = None if not ref: finished_status = cls._get_finished_ref_status(rid, requester) if finished_status == DownloadStatus.SUCCESS: @@ -1399,92 +1466,103 @@ def _handle_download(cls, request: Message) -> Message: cls._logger.error(f"no ref found for {rid} from {requester}") return make_reply(ReturnCode.INVALID_REQUEST) - assert isinstance(ref, _Ref) - ref.mark_active() - ref.mark_receiver_active(requester) - ref.emit_progress(receiver_id=requester, state=TransferProgressState.ACTIVE) - tx = ref.tx - assert isinstance(tx, _Transaction) + try: + assert isinstance(ref, _Ref) + ref.mark_active() + ref.mark_receiver_active(requester) + ref.emit_progress(receiver_id=requester, state=TransferProgressState.ACTIVE) + tx = ref.tx + assert isinstance(tx, _Transaction) - # receiver-confirmed completion is armed only when the receiver advertised the - # capability on this request AND the local kill-switch is on - expect_confirm = bool(payload.get(_PropKey.CONFIRM_CAPABLE)) and _receiver_confirm_enabled() + # receiver-confirmed completion is armed only when the receiver advertised the + # capability on this request AND the local kill-switch is on + expect_confirm = bool(payload.get(_PropKey.CONFIRM_CAPABLE)) and _receiver_confirm_enabled() - # Keep produce() outside the global transaction lock so slow chunk generation - # does not block unrelated downloads. Timeout/delete cleanup can release the - # source concurrently; if that happens, the produce exception is reported as - # a download failure for this requester. - try: - rc, data, new_state = ref.obj.produce(current_state, requester) - except Exception as ex: - ref.emit_progress(receiver_id=requester, state=TransferProgressState.FAILED, force=True) - cls._logger.error( - f"Object {type(ref.obj)} encountered exception when produce: {secure_format_exception(ex)}" - ) - return make_reply(ReturnCode.PROCESS_EXCEPTION) - - if rc != ProduceRC.OK: - # already done -- for a confirm-capable receiver this record is PROVISIONAL and the - # receiver's confirmation finalizes it; for a legacy receiver it is final (today's - # producer-served semantics) - serve_nonce = ref.obj_served( - requester, - status=DownloadStatus.SUCCESS if rc == ProduceRC.EOF else DownloadStatus.FAILED, - expect_confirm=expect_confirm, - ) - if expect_confirm and serve_nonce: - # provisional: the receiver's confirmation carries the terminal truth -- - # do not latch a terminal progress state the confirm may contradict - ref.emit_progress(receiver_id=requester, state=TransferProgressState.ACTIVE, force=True) - body = {_PropKey.STATUS: rc, _PropKey.CONFIRM_EXPECTED: True, _PropKey.CONFIRM_NONCE: serve_nonce} - else: - ref.emit_progress( - receiver_id=requester, - state=TransferProgressState.COMPLETED if rc == ProduceRC.EOF else TransferProgressState.FAILED, - force=True, + # Keep produce() outside the global transaction lock so slow chunk generation + # does not block unrelated downloads. Timeout/delete cleanup can release the + # source concurrently; if that happens, the produce exception is reported as + # a download failure for this requester. + try: + rc, data, new_state = ref.obj.produce(current_state, requester) + except Exception as ex: + ref.emit_progress(receiver_id=requester, state=TransferProgressState.FAILED, force=True) + cls._logger.error( + f"Object {type(ref.obj)} encountered exception when produce: {secure_format_exception(ex)}" ) - body = {_PropKey.STATUS: rc} - return make_reply(ReturnCode.OK, body=body) - else: - # continue — accumulate bytes for timing summary in transaction_done() - # CacheableObject returns a list of byte-chunks; FileDownloader returns raw bytes. - # Sum chunk lengths for lists (len(list) counts items, not bytes). - if data is not None: - bytes_delta = sum(len(c) for c in data) if isinstance(data, list) else len(data) - items_delta = len(data) if isinstance(data, list) else None - tx.add_total_bytes(bytes_delta) - ref.emit_progress( - receiver_id=requester, - state=TransferProgressState.ACTIVE, - bytes_delta=bytes_delta, - items_delta=items_delta, + return make_reply(ReturnCode.PROCESS_EXCEPTION) + + if rc != ProduceRC.OK: + # already done -- for a confirm-capable receiver this record is PROVISIONAL and the + # receiver's confirmation finalizes it; for a legacy receiver it is final (today's + # producer-served semantics) + serve_nonce = ref.obj_served( + requester, + status=DownloadStatus.SUCCESS if rc == ProduceRC.EOF else DownloadStatus.FAILED, + expect_confirm=expect_confirm, ) - # no CONFIRM_EXPECTED on data chunks: the receiver only consumes it from the - # terminal reply (confirms are sent only after terminal serves), so advertising - # per chunk would be dead weight on the hottest wire message - return make_reply( - ReturnCode.OK, - body={ - _PropKey.STATUS: rc, - _PropKey.STATE: new_state, - _PropKey.DATA: data, - }, - ) + if expect_confirm and serve_nonce: + # provisional: the receiver's confirmation carries the terminal truth -- + # do not latch a terminal progress state the confirm may contradict + ref.emit_progress(receiver_id=requester, state=TransferProgressState.ACTIVE, force=True) + body = {_PropKey.STATUS: rc, _PropKey.CONFIRM_EXPECTED: True, _PropKey.CONFIRM_NONCE: serve_nonce} + else: + ref.emit_progress( + receiver_id=requester, + state=TransferProgressState.COMPLETED if rc == ProduceRC.EOF else TransferProgressState.FAILED, + force=True, + ) + body = {_PropKey.STATUS: rc} + return make_reply(ReturnCode.OK, body=body) + else: + # continue — accumulate bytes for timing summary in transaction_done() + # CacheableObject returns a list of byte-chunks; FileDownloader returns raw bytes. + # Sum chunk lengths for lists (len(list) counts items, not bytes). + if data is not None: + bytes_delta = sum(len(c) for c in data) if isinstance(data, list) else len(data) + items_delta = len(data) if isinstance(data, list) else None + tx.add_total_bytes(bytes_delta) + ref.emit_progress( + receiver_id=requester, + state=TransferProgressState.ACTIVE, + bytes_delta=bytes_delta, + items_delta=items_delta, + ) + # no CONFIRM_EXPECTED on data chunks: the receiver only consumes it from the + # terminal reply (confirms are sent only after terminal serves), so advertising + # per chunk would be dead weight on the hottest wire message + return make_reply( + ReturnCode.OK, + body={ + _PropKey.STATUS: rc, + _PropKey.STATE: new_state, + _PropKey.DATA: data, + }, + ) + + finally: + ref.tx.end_op() @classmethod def _handle_confirm(cls, rid: str, requester: str, status: str, nonce: Optional[str]) -> Message: with cls._tx_lock: ref = cls._ref_table.get(rid) + if ref is not None and not ref.tx.begin_op(): + # settling/settled: the outcome snapshot is being (or was) taken -- this + # confirm can no longer influence it and must not emit against the tx + ref = None if ref is None: # the transaction already terminated/cleaned up: its outcome was computed from what # was known then (fail-closed for unconfirmed receivers); a late confirm is dropped cls._logger.debug(f"late confirmation for unknown ref {rid} from {requester} dropped") return make_reply(ReturnCode.OK) assert isinstance(ref, _Ref) - # deliberately no unconditional mark_active/mark_receiver_active: a stale or - # unsolicited confirm must not extend the transaction TTL nor reset idle budgets - if ref.obj_confirmed(requester, status, nonce): - ref.mark_active() + try: + # deliberately no unconditional mark_active/mark_receiver_active: a stale or + # unsolicited confirm must not extend the transaction TTL nor reset idle budgets + if ref.obj_confirmed(requester, status, nonce): + ref.mark_active() + finally: + ref.tx.end_op() return make_reply(ReturnCode.OK) @classmethod @@ -1500,14 +1578,20 @@ def _monitor_tx(cls): budget_txs = [tx for tx in cls._tx_table.values() if tx.has_receiver_budgets] for tx in budget_txs: with cls._tx_lock: - if cls._tx_table.get(tx.tid) is not tx: - continue # deleted/replaced since the snapshot: do not touch a dead tx + # deleted/replaced since the snapshot, or already settling: do not + # touch a dead tx -- begin_op makes the check binding, since + # settlement drains registered operations before emitting + live = cls._tx_table.get(tx.tid) is tx and tx.begin_op() + if not live: + continue try: tx.enforce_receiver_budgets(now) except Exception as ex: cls._logger.error( f"error enforcing receiver budgets for tx {tx.tid}: {secure_format_exception(ex)}" ) + finally: + tx.end_op() expired_tx = [] finished_tx = [] diff --git a/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py b/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py index 961e1b4c3e..1bea726bf2 100644 --- a/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py +++ b/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py @@ -451,6 +451,7 @@ def test_dead_settled_owner_is_reclaimed(self): service._tx_monitor = Mock() cell = Mock() service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-DEAD") + stranded_waiter = service.get_transfer_waiter("TX-DEAD") def exploding_record(outcome, tx): raise RuntimeError("recording blew up") @@ -462,6 +463,7 @@ def exploding_record(outcome, tx): del service._record_outcome # restore the inherited classmethod with service._outcome_lock: assert "TX-DEAD" in service._outcome_owners, "precondition: dead owner still registered" + assert not stranded_waiter.done(), "precondition: the dead generation's waiter is stranded" registered = [] retrier = threading.Thread( @@ -476,6 +478,9 @@ def exploding_record(outcome, tx): with service._tx_lock: with service._outcome_lock: assert service._tx_table["TX-DEAD"] is service._outcome_owners["TX-DEAD"] + # the dead generation's waiters are abandoned with None at reclaim (its verdict + # was lost) -- NOT inherited and later resolved with the retry's outcome + assert stranded_waiter.done() and stranded_waiter.outcome is None def test_shutdown_settlement_serializes_same_id_retry(self): """Falsification pin (P-S4): shutdown used to clear ownership markers BEFORE its @@ -525,6 +530,112 @@ def gated_done_cb(tid, status, base_objs, **kw): with service._outcome_lock: assert service._tx_table["TX-SHUT"] is service._outcome_owners["TX-SHUT"] + def test_hung_retirement_of_live_transaction_fails_bounded(self): + """P1 pin: retiring a LIVE prior transaction used to settle it inline on the + caller's thread, so a hung callback in the old generation hung the retry forever + -- the settlement bound only covered generations already settling elsewhere. + Retirement now settles on its own thread and the retry takes the same bounded + wait as any mid-settlement predecessor.""" + from unittest.mock import Mock + + service = make_isolated_download_service() + service._tx_monitor = Mock() + service.SETTLEMENT_WAIT_TIMEOUT = 0.3 + cell = Mock() + cb_release = threading.Event() + + def hung_done_cb(tid, status, base_objs, **kw): + cb_release.wait(10.0) + + service.new_transaction( + cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-HUNG", transaction_done_cb=hung_done_cb + ) + errors = [] + + def retry(): + try: + service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-HUNG") + except RuntimeError as e: + errors.append(str(e)) + + retrier = threading.Thread(target=retry) + try: + retrier.start() + retrier.join(5.0) + assert not retrier.is_alive(), "retry must fail after the bound, not hang on the hung retirement" + assert len(errors) == 1 and "did not settle" in errors[0] + finally: + cb_release.set() + # once the abandoned settlement eventually completes, the id is usable again + for _ in range(50): + with service._outcome_lock: + consumed = "TX-HUNG" not in service._outcome_owners + if consumed: + break + time.sleep(0.1) + assert service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-HUNG") == "TX-HUNG" + + def test_inflight_budget_pass_drains_before_settlement(self): + """P1 pin: the monitor re-checks table identity, releases _tx_lock, then runs + budget enforcement -- a retry could retire and settle the transaction while + enforcement was mid-flight, and enforcement then emitted FAILED terminal + progress under the reused tx_id. Operations now register with the activity + gate and settlement drains them before emitting or registering the retry.""" + from unittest.mock import Mock + + service = make_isolated_download_service() + service._tx_monitor = Mock() + cell = Mock() + events = [] + cb_entered = threading.Event() + cb_release = threading.Event() + + def gated_downloaded_to_one(receiver, status): + cb_entered.set() + cb_release.wait(10.0) + events.append("old_downloaded_to_one") + + service.new_transaction( + cell=cell, + timeout=1000.0, + num_receivers=1, + tx_id="TX-BUDGET", + receiver_ids=("r1",), + receiver_acquire_timeout=5.0, + progress_cb=lambda **kw: events.append(("old_progress", kw.get("state"))), + ) + obj = _stub_obj() + obj.downloaded_to_one = gated_downloaded_to_one + service.add_object("TX-BUDGET", obj) + + monitor = threading.Thread(target=run_monitor_once, args=(service,), kwargs={"now": time.time() + 100.0}) + retrier = threading.Thread( + target=lambda: ( + service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-BUDGET"), + events.append("retry_registered"), + ) + ) + try: + monitor.start() + assert cb_entered.wait(5.0), "budget enforcement must have finalized r1 as FAILED" + + # the retry lands while enforcement is mid-flight: it must drain first + retrier.start() + retrier.join(0.5) + assert retrier.is_alive(), "retry must not register while a budget pass is in flight" + finally: + cb_release.set() + monitor.join(5.0) + retrier.join(5.0) + assert not retrier.is_alive() + + # every old-generation emission -- including enforcement's FAILED terminal + # progress -- happened strictly before the retry registered + retry_at = events.index("retry_registered") + old_emissions = [i for i, e in enumerate(events) if e != "retry_registered"] + assert old_emissions and all(i < retry_at for i in old_emissions), f"stale emission after retry: {events}" + assert ("old_progress", TransferProgressState.FAILED) in events + def test_retry_gives_up_if_previous_generation_never_settles(self): # ownership was taken but settlement never runs (a hung terminator): a retry # must fail loudly after the bounded wait, never register ambiguously From 11f19a0e44889ae6e9ebb4f42d8a5013435abe54 Mon Sep 17 00:00:00 2001 From: YuanTingHsieh Date: Thu, 9 Jul 2026 15:11:58 -0700 Subject: [PATCH 13/20] Drain waiters parked during the shutdown settlement window shutdown() resolves the waiters it sees and keeps ownership markers so same-id retries serialize behind the deferred settlements -- but that surviving marker makes the tx_id look live to get_transfer_waiter: a waiter created in the window between shutdown's locked teardown and a deferred settlement consuming ownership was parked, and the record-forbidden branch of _record_outcome then consumed ownership without draining it. The waiter hung forever, violating "waiters never hang". The forbidden branch now abandons any window waiters with None, the same verdict shutdown gave the waiters it saw. Every path that consumes ownership now drains waiters: record resolves with the outcome, reclaim and the forbidden branch abandon with None, shutdown resolves wholesale. Pin holds the deferred settlement open with a gated callback, parks a waiter mid-window, and asserts it resolves to None; verified to fail (bounded) with the drain removed. --- nvflare/fuel/f3/streaming/download_service.py | 7 +++- .../f3/streaming/transfer_outcome_test.py | 38 +++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/nvflare/fuel/f3/streaming/download_service.py b/nvflare/fuel/f3/streaming/download_service.py index d080da5fa8..342db8043e 100644 --- a/nvflare/fuel/f3/streaming/download_service.py +++ b/nvflare/fuel/f3/streaming/download_service.py @@ -1358,7 +1358,12 @@ def _record_outcome(cls, outcome: TransferOutcome, tx: _Transaction): # shutdown() forbade this generation from recording: its ownership # marker (kept so same-id retries serialize behind its settlement) is # consumed above, but its verdict is dropped -- nothing records after - # shutdown, and its waiters were already resolved to None. + # shutdown. Waiters shutdown saw were resolved to None then; one that + # parked DURING the settlement window (the surviving marker made the + # id look live to get_transfer_waiter) is drained here, or it would + # hang forever: every path that consumes ownership must drain waiters. + for waiter in cls._tx_waiters.pop(outcome.tx_id, ()): + waiter._resolve(None) return cls._tx_outcomes[outcome.tx_id] = outcome # resolve the awaitable facade: waiters are TransferWaiter objects (no user code diff --git a/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py b/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py index 1bea726bf2..28afecc395 100644 --- a/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py +++ b/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py @@ -636,6 +636,44 @@ def gated_downloaded_to_one(receiver, status): assert old_emissions and all(i < retry_at for i in old_emissions), f"stale emission after retry: {events}" assert ("old_progress", TransferProgressState.FAILED) in events + def test_waiter_parked_during_shutdown_settlement_window_resolves(self): + """P1 pin: shutdown resolves the waiters it sees and keeps ownership markers for + serialization -- but a get_transfer_waiter call in the settlement window (after + shutdown's locked teardown, before the deferred settlement consumes ownership) + found the marker, parked, and hung forever: the record-forbidden branch consumed + ownership without draining waiters. Every ownership-consuming path drains now.""" + from unittest.mock import Mock + + service = make_isolated_download_service() + service._tx_monitor = Mock() + cell = Mock() + cb_entered = threading.Event() + cb_release = threading.Event() + + def gated_done_cb(tid, status, base_objs, **kw): + cb_entered.set() + cb_release.wait(10.0) + + service.new_transaction( + cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-WIN", transaction_done_cb=gated_done_cb + ) + shutter = threading.Thread(target=service.shutdown) + try: + shutter.start() + assert cb_entered.wait(5.0) # locked teardown done; deferred settlement in flight + + # the ownership marker is still visible, so this waiter parks instead of + # resolving immediately -- it must be drained when settlement consumes + # ownership, not stranded + waiter = service.get_transfer_waiter("TX-WIN") + assert not waiter.done(), "precondition: the waiter parked inside the window" + finally: + cb_release.set() + shutter.join(5.0) + outcome = waiter.wait(timeout=5.0) + assert waiter.done(), "window waiter must never hang past shutdown settlement" + assert outcome is None, "shutdown drops verdicts: the window waiter resolves to None" + def test_retry_gives_up_if_previous_generation_never_settles(self): # ownership was taken but settlement never runs (a hung terminator): a retry # must fail loudly after the bounded wait, never register ambiguously From 5c5d76a8e871c7a35a751b924f99cf68b6843c25 Mon Sep 17 00:00:00 2001 From: YuanTingHsieh Date: Thu, 9 Jul 2026 16:51:51 -0700 Subject: [PATCH 14/20] Make tx_ids attempt-scoped: reject duplicates instead of serializing reuse Five review rounds attacked the reused-tx_id retry design; four falsified its generation boundary (mid-settlement supersede, flag-gated suppression races, unbounded retirement, in-flight budget-pass emissions, shutdown window waiters). Each fix was correct, but the pattern was defending a boundary that naming can delete: when two attempts never share an id, a late emission from a dying attempt names that attempt truthfully forever, and misattribution becomes unrepresentable instead of prevented. Grounding facts: the tx_id never crosses the wire in the pull protocol (receivers pull by ref_id), so retry idempotency always belonged to the application-level transfer id carried in caller metadata -- in any design. And id reuse never existed as behavior: the tx_id parameter has been dormant since introduction (every caller takes a fresh uuid). The reuse contract was a plan for the not-yet-written trainer engine, with zero consumers to migrate. new_transaction now registers in one atomic step and raises ValueError on a duplicate id while it is live, settling, or its receipt is retained (TX_OUTCOME_TTL bounds the exclusion window for caller-supplied ids). Deleted with the reuse contract: generation serialization (settled events, mid-settlement waits, settler threads, SETTLEMENT_WAIT_TIMEOUT, the settlement-callback reentrancy guard), dead-owner reclaim and its waiter abandonment, receipt-wipe-on-registration, and the record_forbidden marker dance in shutdown (which reverts to the simple atomic clear; the owner guard alone gates post-shutdown recording, and a waiter arriving in the deferred-settlement window now resolves None immediately instead of parking). Waiters trivially bind to the single attempt their id names. Kept, rationale unchanged: receipts and TTL, TransferWaiter, per-receiver budgets, receiver confirms including the per-serve nonce (ref_id reuse via PASS_THROUGH re-emission is independent of retries), tombstones, the settle-then-resolve ordering, exception-guarded callbacks, and the activity gate -- demoted from misattribution defense to verdict completeness: an operation finishing during termination is counted in the outcome, and nothing emits against a settled transaction. The duplicate check expires receipts inline, so the exclusion window for a caller-supplied id is exactly TX_OUTCOME_TTL (not TTL plus a sweep cycle). The execution-modes design doc is aligned: tx_id never equals transfer_id; each attempt gets a fresh download_tx_id and the manifest carries the transfer_id mapping. Tests updated to pin the new contract: duplicate ids rejected across the whole lifetime (live, settling, receipted) and freed by receipt expiry; racing same-id constructors resolve to exactly one winner and clean errors; settlement callbacks may create new transactions but not their own id; a recording failure leaves the id excluded and shutdown releases its waiter; shutdown-window waiters resolve immediately; the in-flight budget-pass drain now pins that enforcement verdicts are counted in the receipt. Net -293 lines. --- docs/design/client_api_execution_modes.md | 8 +- nvflare/fuel/f3/streaming/download_service.py | 218 +++------ .../f3/streaming/download_service_test.py | 1 - .../fuel/f3/streaming/receiver_budget_test.py | 29 +- .../f3/streaming/receiver_confirm_test.py | 10 +- .../f3/streaming/transfer_outcome_test.py | 438 +++++------------- 6 files changed, 214 insertions(+), 490 deletions(-) diff --git a/docs/design/client_api_execution_modes.md b/docs/design/client_api_execution_modes.md index ac8ad80837..1809301181 100644 --- a/docs/design/client_api_execution_modes.md +++ b/docs/design/client_api_execution_modes.md @@ -501,9 +501,12 @@ For Cell/FOBS-backed streaming, the Client API backend should set: - `MessageHeaderKey.MSG_ROOT_ID = transfer_id` - `MessageHeaderKey.MSG_ROOT_TTL = transfer_timeout` -- DownloadService `tx_id = transfer_id` when a single transaction is used +- DownloadService transaction ids are **attempt-scoped and never reused** (duplicate + registration raises): each attempt gets a fresh `download_tx_id`, and the manifest + carries the `transfer_id -> download_tx_id/download_ref_id` mapping. `transfer_id` + is the stable cross-attempt identity and never doubles as an F3 transaction id. -If multiple DownloadService transactions are needed, the manifest must carry the mapping from transfer_id to all download_tx_id and download_ref_id values. +The manifest must carry the mapping from transfer_id to all download_tx_id and download_ref_id values (one or many transactions alike). **Result send state machine.** The happy path runs down the left; any state can branch to a terminal failure on the right. The producer (trainer) is held alive until a terminal state is reached. @@ -548,6 +551,7 @@ The producer-facing wait is then an awaitable facade over those signals: `flare. - The executor must treat duplicate RESULT_READY messages as idempotent. - If the result was already accepted, reply with the current state rather than creating a second result or second transfer. - Payload download retries happen inside the data-plane mechanism and must not create a new result_id. +- A re-offered payload transfer (transfer-level retry) is a NEW DownloadService transaction under the same transfer_id; F3 tx_ids are attempt-scoped and are never reused across attempts. - TASK_READY redelivery is idempotent by task id (see Control Protocol). **Failure and timeout rules:** diff --git a/nvflare/fuel/f3/streaming/download_service.py b/nvflare/fuel/f3/streaming/download_service.py index 342db8043e..af1c7ac7b6 100644 --- a/nvflare/fuel/f3/streaming/download_service.py +++ b/nvflare/fuel/f3/streaming/download_service.py @@ -693,20 +693,11 @@ def __init__( self.progress_interval = float(progress_interval) self.refs = [] self._refs_lock = threading.RLock() - # set once settlement (transaction_done) has fully completed: callbacks emitted, - # sources released, outcome recorded or dropped. A retry reusing this tx_id waits - # for this before taking ownership, so generations of a tx_id never overlap. - self._settled = threading.Event() - self._settling_thread = None - # set by shutdown() (under _outcome_lock): this transaction's verdict must be - # dropped, but its ownership marker stays until settlement consumes it, so a - # same-id retry still serializes behind the in-flight settlement - self._record_forbidden = False # The activity gate: serves, confirms, and monitor budget passes register as # in-flight operations; settlement closes the gate and drains them before - # emitting, so an operation already executing when the transaction is - # terminated cannot emit (terminal progress, receiver callbacks) after a - # same-id retry exists. + # snapshotting the outcome, so an operation finishing during termination is + # counted in the verdict and nothing emits against the transaction after it + # has settled. self._ops_cond = threading.Condition(threading.Lock()) self._active_ops = 0 self._ops_closed = False @@ -760,10 +751,6 @@ def snapshot_refs(self): with self._refs_lock: return list(self.refs) - def wait_settled(self, timeout: float) -> bool: - """Waits until this transaction's settlement has fully completed.""" - return self._settled.wait(timeout) - def begin_op(self) -> bool: """Registers an in-flight operation (serve, confirm, budget pass). @@ -852,19 +839,14 @@ def transaction_done(self, status: str, on_outcome=None) -> TransferOutcome: stops the producer process) can never preempt the callback chain or the release of the sources. - Settlement runs exactly once per transaction (every terminator first unlinks it - from _tx_table under _tx_lock), and a retry reusing this tx_id cannot take - ownership until settlement has fully completed (new_transaction waits on the - settled event, set in the finally below). Every emission here therefore happens - strictly before any successor generation of the tx_id exists and can never be - misattributed to one -- no emission needs to be suppressed or gated. + Settlement runs exactly once per transaction: every terminator first unlinks it + from _tx_table under _tx_lock, so exactly one caller runs this. tx_ids are + never reused (new_transaction rejects duplicates), so every emission here is + attributable to this transaction alone, forever. """ - # recorded so a settlement callback that synchronously reuses this tx_id gets an - # immediate error instead of deadlocking on its own settlement - self._settling_thread = threading.get_ident() # Drain in-flight operations BEFORE snapshotting the outcome: an operation that - # finishes during the drain is counted in the verdict, and none can emit after - # settlement (and therefore after a same-id retry) anymore. + # finishes during the drain is counted in the verdict, and nothing can emit + # against the transaction after it has settled. if not self._drain_ops(OP_DRAIN_TIMEOUT): self.logger.warning( f"tx {self.tid}: in-flight operations did not drain within {OP_DRAIN_TIMEOUT}s; settling anyway" @@ -939,15 +921,9 @@ def transaction_done(self, status: str, on_outcome=None) -> TransferOutcome: # Record the outcome (and release waiters) only now that the transaction is # fully settled: callbacks done, sources released. In a finally so that no # exception anywhere above (including on the monitor thread) can leave the - # outcome unrecorded, the waiter pending forever, and ownership registered. + # outcome unrecorded and the waiter pending forever. if on_outcome and outcome is not None: _invoke_cb_safely(self.logger, f"outcome recording for tx {self.tid}", on_outcome, outcome) - # only now -- callbacks emitted, sources released, outcome recorded, waiters - # resolved -- may a same-id retry blocked in new_transaction take ownership. - # Set even if outcome computation itself raised: an unsettleable transaction - # would poison its tx_id forever (a settled-but-unconsumed owner is instead - # reclaimed by the next same-id new_transaction). - self._settled.set() return outcome @@ -1055,12 +1031,12 @@ class DownloadService: # time so producers can query the aggregate result after termination. Guarded by # its own lock so outcome polling never contends with the chunk-serving _tx_lock. _tx_outcomes = {} - # The transaction object currently entitled to record the outcome for its tx_id - # (registered by new_transaction). tx_ids are deliberately reused by retries - # (design: stable transfer ids make retries idempotent), so recording checks - # OBJECT identity against this map. An entry doubles as the mid-settlement marker - # that serializes generations: a same-id retry waits until the registered owner - # has fully settled (and its entry is consumed) before taking the slot. + # The transaction object entitled to record the outcome for its tx_id (registered + # by new_transaction, consumed by _record_outcome). Recording checks OBJECT + # identity against this map: a recorder that lost its entry -- shutdown cleared + # ownership, or a prior terminal record already consumed it -- drops its outcome + # instead of recording stale truth. An entry also marks the id as in-use to + # new_transaction's duplicate check while the transaction is settling. # Guarded by _outcome_lock. _outcome_owners = {} # Waiters blocked on a transaction's terminal outcome (the awaitable facade). Guarded by @@ -1068,10 +1044,6 @@ class DownloadService: _tx_waiters = {} _outcome_lock = threading.Lock() TX_OUTCOME_TTL = 1800.0 - # How long new_transaction will wait for a mid-settlement previous generation of a - # reused tx_id before giving up. Settlement is normally milliseconds; this only - # expires if a settlement callback hangs (which also hangs the monitor thread). - SETTLEMENT_WAIT_TIMEOUT = 60.0 _logger = None _tx_monitor = None _tx_lock = threading.Lock() @@ -1129,88 +1101,37 @@ def new_transaction( receiver_acquire_timeout=receiver_acquire_timeout, receiver_idle_timeout=receiver_idle_timeout, ) - # Generations of a reused tx_id are strictly SERIALIZED: this transaction may - # not take ownership until every previous generation has fully settled -- all - # of its terminal emissions (progress, done/outcome callbacks, source release) - # happen strictly before this one exists, so none can be misattributed to it. - # (The alternative -- letting generations overlap and gating the old one's - # emissions on a superseded flag -- is inherently racy: the check and the - # emission cannot be made atomic without holding a lock across user callbacks, - # which would deadlock callbacks that call back into this service.) - # Scope: this serializes the TERMINAL surfaces. A data-plane event already - # executing against the old generation when it is retired (an in-flight serve's - # produce(), a monitor budget pass) may still finish after the retry registers; - # such stragglers cannot alter any recorded verdict -- the outcome is computed - # from locked snapshots at settlement and recording is ownership-guarded. - deadline = time.time() + cls.SETTLEMENT_WAIT_TIMEOUT - while True: - old_tx = None - settling = None - # Ownership and table registration are ONE atomic step (_tx_lock nests - # _outcome_lock here; no path nests them in the reverse order), so a tx the - # monitor can terminate always owns its outcome slot. - with cls._tx_lock: - old_tx = cls._tx_table.get(tx.tid) - if old_tx is None: - with cls._outcome_lock: - settling = cls._outcome_owners.get(tx.tid) - if settling is None or settling._settled.is_set(): - # The id is free -- or its owner is DEAD: fully settled yet - # still registered, meaning its recording step failed - # abnormally and never consumed ownership. Reclaim it: all - # of its emissions are over, and waiting on it would spin - # forever (its settled event is already set). A reused - # tx_id must not surface a previous generation's outcome: - # purge any recorded outcome, then take ownership and - # become monitor-visible atomically. Waiters parked by the - # dead generation are abandoned with None (its verdict was - # lost), like shutdown does -- they must not be inherited - # and later resolved with THIS generation's outcome. - for waiter in cls._tx_waiters.pop(tx.tid, ()): - waiter._resolve(None) - cls._tx_outcomes.pop(tx.tid, None) - cls._outcome_owners[tx.tid] = tx - cls._tx_table[tx.tid] = tx - return tx.tid - else: - # A retry reusing a tx_id retires a still-live prior transaction. - # A plain overwrite would orphan it: the monitor only discovers - # transactions through _tx_table, so the old refs would stay - # servable in _ref_table forever and the old sources would never - # be released via transaction_done. - cls._delete_tx(old_tx) - - if old_tx is not None: - # Settle the retired generation on its own thread and fall through to - # the same bounded wait as any other mid-settlement predecessor: a hung - # callback in the OLD generation must not hang this caller forever -- - # it fails loudly after the bound instead (the settler keeps running as - # an abandoned daemon; its ownership marker is consumed if it ever - # finishes, so a later retry can succeed). Re-arm the deadline: the - # bound is per-generation, not per-call. - threading.Thread( - target=old_tx.transaction_done, - args=(TransactionDoneStatus.DELETED,), - kwargs={"on_outcome": functools.partial(cls._record_outcome, tx=old_tx)}, - name=f"tx-retire-{tx.tid}", - daemon=True, - ).start() - settling = old_tx - deadline = time.time() + cls.SETTLEMENT_WAIT_TIMEOUT - - # A previous generation is MID-SETTLEMENT: either just retired above, or - # already popped from _tx_table by the monitor/delete path but still running - # its callbacks. Wait for it to settle -- outside all locks -- then re-check. - if settling._settling_thread == threading.get_ident() and not settling._settled.is_set(): - raise RuntimeError( - f"cannot create transaction {tx.tid}: its tx_id is being reused from within " - f"a settlement callback of its own previous generation" - ) - if not settling.wait_settled(max(0.0, deadline - time.time())): - raise RuntimeError( - f"cannot create transaction {tx.tid}: the previous transaction with this " - f"tx_id did not settle within {cls.SETTLEMENT_WAIT_TIMEOUT}s" - ) + # tx_ids are ATTEMPT-SCOPED and never reused: a retry of the same logical + # transfer is a NEW transaction with a new id (the stable cross-attempt + # identity is the application-level transfer id carried in the caller's + # metadata -- it never enters this service). Uniqueness is what makes every + # emission of a terminated transaction attributable to it alone, forever: + # nothing a dying attempt does can be confused with a live successor, so no + # suppression, generation serialization, or settlement waiting is needed. + # Registration is therefore a single atomic step: ownership and table entry + # together (_tx_lock nests _outcome_lock here; no path nests them in the + # reverse order), so a tx the monitor can terminate always owns its outcome + # slot -- and a duplicate id is rejected while it is live, still settling, or + # its receipt is retained (receipts expire after TX_OUTCOME_TTL, which bounds + # the exclusion window for caller-supplied ids). + with cls._tx_lock: + with cls._outcome_lock: + # an expired-but-unswept receipt does not exclude the id: expire it + # here (receipts are swept lazily and by the monitor, so without this + # the exclusion window would stretch to TTL + a sweep cycle) + receipt = cls._tx_outcomes.get(tx.tid) + if receipt is not None and receipt.expired(time.time(), cls.TX_OUTCOME_TTL): + cls._tx_outcomes.pop(tx.tid, None) + receipt = None + if tx.tid in cls._tx_table or tx.tid in cls._outcome_owners or receipt is not None: + raise ValueError( + f"transaction id {tx.tid} is already in use: tx_ids are attempt-scoped and " + f"must be unique -- retry with a new id (correlate attempts with an " + f"application-level transfer id instead)" + ) + cls._outcome_owners[tx.tid] = tx + cls._tx_table[tx.tid] = tx + return tx.tid @classmethod def add_object( @@ -1264,16 +1185,14 @@ def shutdown(cls): cls._finished_refs.clear() with cls._outcome_lock: - # Drop recorded outcomes and forbid every registered owner (live or - # mid-settlement) from recording -- but keep the ownership markers: - # each is consumed by its own settlement, so a same-id new_transaction - # racing this shutdown still serializes behind the in-flight - # settlement instead of registering into its emission window. (An - # ownership CLEAR here would let that retry find the id free and - # return while the deferred settlements below still emit.) + # Drop recorded outcomes and clear outcome ownership: a settlement + # mid-flight on another thread finds its ownership gone once it reaches + # _record_outcome, so nothing records after shutdown -- the owner guard + # alone gates post-shutdown recording. (tx_ids are never reused, so a + # new_transaction racing this shutdown registers under a DIFFERENT id + # and cannot collide with the deferred settlements below.) cls._tx_outcomes.clear() - for owner in cls._outcome_owners.values(): - owner._record_forbidden = True + cls._outcome_owners.clear() # unblock the awaitable facade: waiters resolve to None (service shut down # before the transaction terminated), never hang for waiters in cls._tx_waiters.values(): @@ -1287,7 +1206,7 @@ def shutdown(cls): cls._initialized_cells.clear() for tx in tx_list: - tx.transaction_done(TransactionDoneStatus.DELETED, on_outcome=functools.partial(cls._record_outcome, tx=tx)) + tx.transaction_done(TransactionDoneStatus.DELETED) @classmethod def _delete_tx(cls, tx: _Transaction, tombstone_finished_refs: bool = False): @@ -1307,13 +1226,9 @@ def get_transfer_waiter(cls, transaction_id: str) -> TransferWaiter: """Returns an awaitable facade over the transaction's terminal outcome. Safe to call before or after termination: a waiter created after the outcome was - recorded resolves immediately from the outcome table. - - Waiters bind to the GENERATION of the tx_id that records while they wait: a - waiter attached before a same-id retry resolves with the retired generation's - (non-completed) outcome the moment that generation settles -- each attempt's - awaiter gets that attempt's verdict; a retrying caller re-acquires a waiter for - the new attempt. + recorded resolves immediately from the outcome table. tx_ids are attempt-scoped + (never reused), so a waiter always resolves with the verdict of exactly the + attempt it named; a retrying caller acquires a new waiter for the new attempt. """ waiter = TransferWaiter(transaction_id, service=cls) with cls._outcome_lock: @@ -1349,22 +1264,11 @@ def _record_outcome(cls, outcome: TransferOutcome, tx: _Transaction): # recording is legal only for the transaction that owns the outcome slot. with cls._outcome_lock: if cls._outcome_owners.get(outcome.tx_id) is not tx: - # ownership was already consumed (a prior terminal record) or reclaimed - # (a retry took over a slot whose owner was fully settled but never - # consumed it): this recording is stale and must be dropped + # ownership was already consumed (a prior terminal record) or cleared + # by shutdown(): this recording is stale and must be dropped -- the + # owner guard alone gates post-shutdown recording return cls._outcome_owners.pop(outcome.tx_id, None) - if tx._record_forbidden: - # shutdown() forbade this generation from recording: its ownership - # marker (kept so same-id retries serialize behind its settlement) is - # consumed above, but its verdict is dropped -- nothing records after - # shutdown. Waiters shutdown saw were resolved to None then; one that - # parked DURING the settlement window (the surviving marker made the - # id look live to get_transfer_waiter) is drained here, or it would - # hang forever: every path that consumes ownership must drain waiters. - for waiter in cls._tx_waiters.pop(outcome.tx_id, ()): - waiter._resolve(None) - return cls._tx_outcomes[outcome.tx_id] = outcome # resolve the awaitable facade: waiters are TransferWaiter objects (no user code # runs in _resolve), so setting them under the lock is safe and race-free diff --git a/tests/unit_test/fuel/f3/streaming/download_service_test.py b/tests/unit_test/fuel/f3/streaming/download_service_test.py index 44fc4a7561..e37059eb29 100644 --- a/tests/unit_test/fuel/f3/streaming/download_service_test.py +++ b/tests/unit_test/fuel/f3/streaming/download_service_test.py @@ -529,7 +529,6 @@ def test_stale_outcome_dropped_after_ownership_cleared(self): # sanity: the owning transaction still records (guard does not over-drop) live_tx = Mock() live_tx.tid = "tx-live" - live_tx._record_forbidden = False # Mock auto-attrs are truthy; a real live tx is not forbidden live_outcome = Mock() live_outcome.tx_id = "tx-live" live_outcome.expired.return_value = False diff --git a/tests/unit_test/fuel/f3/streaming/receiver_budget_test.py b/tests/unit_test/fuel/f3/streaming/receiver_budget_test.py index f503232d12..9c47af9501 100644 --- a/tests/unit_test/fuel/f3/streaming/receiver_budget_test.py +++ b/tests/unit_test/fuel/f3/streaming/receiver_budget_test.py @@ -238,20 +238,27 @@ def test_receiver_idle_after_finishing_one_ref_fails_sibling_ref_in_bounded_time class TestConcurrentSameIdCreation: - def test_owner_and_table_stay_consistent_under_concurrent_creation(self): - # P1 pin (reproduced by review): with registration and table insertion in separate - # critical sections, two concurrent same-id constructors could leave the live - # transaction without outcome ownership -- and record the retired transaction's - # DELETED outcome as if it were the live attempt. + def test_concurrent_same_id_creation_has_exactly_one_winner(self): + # P1 pin (reproduced by review, then simplified by design): tx_ids are + # attempt-scoped, so racing same-id constructors resolve to exactly ONE + # registered transaction (which owns its outcome slot) and clean ValueErrors + # for every loser -- no interleaving can separate the live transaction from + # its ownership, and no loser leaves any trace. import threading as th service = _make_service() - for _ in range(30): + for round_no in range(30): + tid = f"TX-RACE2-{round_no}" barrier = th.Barrier(2) + results = [] def create(): barrier.wait() - service.new_transaction(cell=Mock(), timeout=10.0, num_receivers=1, tx_id="TX-RACE2") + try: + service.new_transaction(cell=Mock(), timeout=10.0, num_receivers=1, tx_id=tid) + results.append("won") + except ValueError: + results.append("rejected") threads = [th.Thread(target=create) for _ in range(2)] for t in threads: @@ -259,12 +266,12 @@ def create(): for t in threads: t.join(5.0) - live = service._tx_table["TX-RACE2"] + assert sorted(results) == ["rejected", "won"], f"round {round_no}: {results}" + live = service._tx_table[tid] with service._outcome_lock: - owner = service._outcome_owners.get("TX-RACE2") + owner = service._outcome_owners.get(tid) assert owner is live, "the live transaction must own its outcome slot" - # the retired constructor's DELETED outcome must never be recorded for the id - assert service.get_transaction_outcome("TX-RACE2") is None + assert service.get_transaction_outcome(tid) is None class TestShutdownCreationAtomicity: diff --git a/tests/unit_test/fuel/f3/streaming/receiver_confirm_test.py b/tests/unit_test/fuel/f3/streaming/receiver_confirm_test.py index d411aeada2..386781c53d 100644 --- a/tests/unit_test/fuel/f3/streaming/receiver_confirm_test.py +++ b/tests/unit_test/fuel/f3/streaming/receiver_confirm_test.py @@ -258,9 +258,13 @@ def test_stale_confirm_from_previous_ref_life_is_dropped_even_with_new_pending(s old_terminal = _pull_to_terminal(service, rid, "r1", confirm_capable=True) old_nonce = serve_nonce(old_terminal) - # the transfer is retried: same tx_id, same ref_id -- a NEW life - service.new_transaction(cell=Mock(), timeout=10.0, num_receivers=1, tx_id="TX-LIFE") - rid2 = service.add_object("TX-LIFE", MockDownloadable([b"chunk"]), ref_id="R-LIFE") + # the transfer is retried as a NEW attempt (tx_ids are attempt-scoped, never + # reused) that re-serves the SAME ref_id -- the real pattern is PASS_THROUGH + # re-emission, where the ref_id was minted before registration and re-added. + # First retire the old attempt so the rid can be re-registered. + service.delete_transaction(tx1) + tx2 = service.new_transaction(cell=Mock(), timeout=10.0, num_receivers=1, tx_id="TX-LIFE-2") + rid2 = service.add_object(tx2, MockDownloadable([b"chunk"]), ref_id="R-LIFE") assert rid2 == rid new_terminal = _pull_to_terminal(service, rid, "r1", confirm_capable=True) # new pending serve exists diff --git a/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py b/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py index 28afecc395..650588eb94 100644 --- a/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py +++ b/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py @@ -201,67 +201,6 @@ def test_raising_release_cannot_leave_waiter_unresolved(self): with service._outcome_lock: assert tx_id not in service._outcome_owners, "ownership must be consumed" - def test_retry_waits_for_mid_settlement_generation(self): - """P1 pin: the monitor pops a finishing transaction from _tx_table BEFORE running - its callbacks; a retry reusing the tx_id in that gap must not take ownership until - that settlement fully completes. Generations are strictly serialized -- every - old-generation emission happens before the retry exists -- because the alternative - (gating emissions on a superseded flag) is an unwinnable check-then-emit race.""" - from functools import partial - from unittest.mock import Mock - - service = make_isolated_download_service() - service._tx_monitor = Mock() - cell = Mock() - events = [] - cb_entered = threading.Event() - cb_release = threading.Event() - - def slow_outcome_cb(outcome): - cb_entered.set() - cb_release.wait(10.0) # hold settlement open while the retry tries to register - events.append("old_outcome_cb") - - service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-MID", outcome_cb=slow_outcome_cb) - obj = _stub_obj() - service.add_object("TX-MID", obj) - with service._tx_lock: - old_tx = service._tx_table.pop("TX-MID") # monitor termination step 1 - - settler = threading.Thread( - target=lambda: old_tx.transaction_done( - TransactionDoneStatus.FINISHED, on_outcome=partial(service._record_outcome, tx=old_tx) - ) - ) - retrier = threading.Thread( - target=lambda: ( - service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-MID"), - events.append("retry_registered"), - ) - ) - try: - settler.start() - assert cb_entered.wait(5.0) - - # the retry lands mid-settlement: it must block, not register - retrier.start() - retrier.join(0.5) - assert retrier.is_alive(), "retry must not take ownership while the old generation is settling" - finally: - cb_release.set() - settler.join(5.0) - retrier.join(5.0) - assert not retrier.is_alive() - - # strict serialization: every old-generation emission precedes the retry - assert events == ["old_outcome_cb", "retry_registered"] - assert obj.released - # the old generation's recorded outcome was purged at retry registration - assert service.get_transaction_outcome("TX-MID") is None - with service._tx_lock: - with service._outcome_lock: - assert service._tx_table["TX-MID"] is service._outcome_owners["TX-MID"] - def test_on_outcome_fires_after_callbacks_and_release(self): # settle-then-record: recording (which releases waiters) happens only after the # callback chain and source release complete, so an upper layer acting on @@ -400,58 +339,111 @@ def test_delete_after_success_records_completed(self): assert outcome.completed assert outcome.done_status == TransactionDoneStatus.DELETED - def test_reused_tx_id_does_not_surface_stale_outcome(self): - # retry with the same explicit tx_id (design: tx_id = transfer_id, retries reuse it) + def test_new_transaction_rejects_live_id(self): + """tx_ids are attempt-scoped: registering an id that names a LIVE transaction + raises immediately -- and, unlike the old retire-on-reuse design, the live + transaction is completely undisturbed (no callbacks, no release, still servable).""" from unittest.mock import Mock service = make_isolated_download_service() - service._tx_monitor = Mock() # suppress real monitor thread start + service._tx_monitor = Mock() cell = Mock() - try: - first = service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-REUSE") - service.delete_transaction(first) - assert service.get_transaction_outcome("TX-REUSE") is not None - - second = service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-REUSE") - assert second == "TX-REUSE" - # the stale terminal outcome of the previous owner is purged - assert service.get_transaction_outcome("TX-REUSE") is None - finally: - service.shutdown() + done, outcome_cbs = [], [] + service.new_transaction( + cell=cell, + timeout=10.0, + num_receivers=1, + tx_id="TX-DUP", + transaction_done_cb=lambda tid, status, base_objs, **kw: done.append(status), + outcome_cb=lambda outcome: outcome_cbs.append(outcome), + ) + obj = _stub_obj() + rid = service.add_object("TX-DUP", obj) + + with pytest.raises(ValueError, match="already in use"): + service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-DUP") + + # the duplicate attempt left no trace on the live transaction + assert rid in service._ref_table + assert not obj.released + assert done == [] and outcome_cbs == [] + with service._tx_lock: + with service._outcome_lock: + assert service._tx_table["TX-DUP"] is service._outcome_owners["TX-DUP"] + + def test_new_transaction_rejects_settling_and_receipted_id(self): + """The duplicate check covers the id's whole lifetime: live (above), SETTLING + (popped from _tx_table, ownership not yet consumed), and RECEIPTED (outcome + retained for TX_OUTCOME_TTL). Only receipt expiry frees a used id.""" + from functools import partial + from unittest.mock import Mock + + service = make_isolated_download_service() + service._tx_monitor = Mock() + cell = Mock() + service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-LIFE") + with service._tx_lock: + old_tx = service._tx_table.pop("TX-LIFE") # monitor termination step 1 + + # mid-settlement: ownership still registered + with pytest.raises(ValueError, match="already in use"): + service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-LIFE") + + old_tx.transaction_done(TransactionDoneStatus.FINISHED, on_outcome=partial(service._record_outcome, tx=old_tx)) - def test_reusing_tx_id_from_own_settlement_callback_raises(self): - # a settlement callback that synchronously retries its own tx_id would deadlock - # (the retry must wait for a settlement that cannot complete until the callback - # returns), so it gets an immediate error instead + # receipted: the verdict is retained and the id stays excluded + assert service.get_transaction_outcome("TX-LIFE") is not None + with pytest.raises(ValueError, match="already in use"): + service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-LIFE") + + # receipt expiry (TTL) frees the id -- checked INLINE by new_transaction, so an + # expired-but-unswept receipt does not stretch the exclusion window + recorded = service.get_transaction_outcome("TX-LIFE") + assert recorded is not None + with service._outcome_lock: + service._tx_outcomes["TX-LIFE"] = dataclasses.replace( + recorded, timestamp=recorded.timestamp - service.TX_OUTCOME_TTL - 1 + ) + assert service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-LIFE") == "TX-LIFE" + + def test_settlement_callback_can_create_transactions_but_not_its_own_id(self): + """Settlement runs outside all service locks, so callbacks may freely call back + in and create NEW transactions; recreating the settling transaction's own id is + rejected like any other in-use id (its ownership is consumed only after the + callbacks, in on_outcome).""" from unittest.mock import Mock service = make_isolated_download_service() - service._tx_monitor = Mock() # suppress real monitor thread start + service._tx_monitor = Mock() cell = Mock() - errors = [] + errors, created = [], [] - def retry_inline(outcome): + def cb(outcome): try: - service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-REENT") - except RuntimeError as e: + service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-SELF") + except ValueError as e: errors.append(str(e)) + created.append(service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-OTHER")) - service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-REENT", outcome_cb=retry_inline) - service.delete_transaction("TX-REENT") - assert len(errors) == 1 and "settlement callback" in errors[0] + service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-SELF", outcome_cb=cb) + service.delete_transaction("TX-SELF") - def test_dead_settled_owner_is_reclaimed(self): - """Falsification pin (P-S3): if outcome recording itself fails, the generation is - settled (event set) but its ownership was never consumed. A retry must reclaim - that dead slot -- before the fix it hot-spun forever, because Event.wait returns - True immediately for a set event so the bounded-wait escape never fired.""" + assert len(errors) == 1 and "already in use" in errors[0] + assert created == ["TX-OTHER"] + with service._tx_lock: + assert "TX-OTHER" in service._tx_table + + def test_recording_failure_leaves_id_unusable_and_shutdown_frees_waiters(self): + """If outcome recording itself fails abnormally, ownership is never consumed: + the id stays excluded (fail-safe -- fresh uuids never collide with it) and its + waiter is released by shutdown, the terminal never-hang backstop.""" from unittest.mock import Mock service = make_isolated_download_service() service._tx_monitor = Mock() cell = Mock() service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-DEAD") - stranded_waiter = service.get_transfer_waiter("TX-DEAD") + stranded = service.get_transfer_waiter("TX-DEAD") def exploding_record(outcome, tx): raise RuntimeError("recording blew up") @@ -461,126 +453,50 @@ def exploding_record(outcome, tx): service.delete_transaction("TX-DEAD") # settles; recording fails; owner never consumed finally: del service._record_outcome # restore the inherited classmethod - with service._outcome_lock: - assert "TX-DEAD" in service._outcome_owners, "precondition: dead owner still registered" - assert not stranded_waiter.done(), "precondition: the dead generation's waiter is stranded" + assert not stranded.done() + with pytest.raises(ValueError, match="already in use"): + service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-DEAD") - registered = [] - retrier = threading.Thread( - target=lambda: registered.append( - service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-DEAD") - ) - ) - retrier.start() - retrier.join(5.0) - assert not retrier.is_alive(), "retry must reclaim a settled-but-unconsumed owner, not spin" - assert registered == ["TX-DEAD"] - with service._tx_lock: - with service._outcome_lock: - assert service._tx_table["TX-DEAD"] is service._outcome_owners["TX-DEAD"] - # the dead generation's waiters are abandoned with None at reclaim (its verdict - # was lost) -- NOT inherited and later resolved with the retry's outcome - assert stranded_waiter.done() and stranded_waiter.outcome is None - - def test_shutdown_settlement_serializes_same_id_retry(self): - """Falsification pin (P-S4): shutdown used to clear ownership markers BEFORE its - deferred settlements ran, so a same-id retry landing in that window found the id - free and returned while the old generation's emissions were still to come. - Ownership markers now survive until each settlement consumes them.""" + service.shutdown() + assert stranded.done() and stranded.outcome is None + + def test_waiter_during_shutdown_settlement_window_resolves_immediately(self): + """shutdown() clears ownership atomically with the table teardown, so a + get_transfer_waiter call landing in the deferred-settlement window finds the + id unknown and resolves None IMMEDIATELY -- it never parks, so it can never + be stranded by the settlements still running.""" from unittest.mock import Mock service = make_isolated_download_service() service._tx_monitor = Mock() cell = Mock() - events = [] cb_entered = threading.Event() cb_release = threading.Event() def gated_done_cb(tid, status, base_objs, **kw): cb_entered.set() cb_release.wait(10.0) - events.append("old_done_cb") service.new_transaction( - cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-SHUT", transaction_done_cb=gated_done_cb + cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-WIN", transaction_done_cb=gated_done_cb ) shutter = threading.Thread(target=service.shutdown) - retrier = threading.Thread( - target=lambda: ( - service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-SHUT"), - events.append("retry_registered"), - ) - ) try: shutter.start() - assert cb_entered.wait(5.0) # shutdown is now mid-settlement, outside its locks + assert cb_entered.wait(5.0) # locked teardown done; deferred settlement in flight - retrier.start() - retrier.join(0.5) - assert retrier.is_alive(), "retry must serialize behind shutdown's in-flight settlement" + waiter = service.get_transfer_waiter("TX-WIN") + assert waiter.done() and waiter.outcome is None, "window waiter must resolve immediately" finally: cb_release.set() shutter.join(5.0) - retrier.join(5.0) - assert not retrier.is_alive() - assert events == ["old_done_cb", "retry_registered"] - # shutdown still drops the old generation's verdict (nothing records after shutdown) - # while the post-shutdown registration owns the slot cleanly - with service._tx_lock: - with service._outcome_lock: - assert service._tx_table["TX-SHUT"] is service._outcome_owners["TX-SHUT"] - - def test_hung_retirement_of_live_transaction_fails_bounded(self): - """P1 pin: retiring a LIVE prior transaction used to settle it inline on the - caller's thread, so a hung callback in the old generation hung the retry forever - -- the settlement bound only covered generations already settling elsewhere. - Retirement now settles on its own thread and the retry takes the same bounded - wait as any mid-settlement predecessor.""" - from unittest.mock import Mock - - service = make_isolated_download_service() - service._tx_monitor = Mock() - service.SETTLEMENT_WAIT_TIMEOUT = 0.3 - cell = Mock() - cb_release = threading.Event() - - def hung_done_cb(tid, status, base_objs, **kw): - cb_release.wait(10.0) - - service.new_transaction( - cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-HUNG", transaction_done_cb=hung_done_cb - ) - errors = [] - - def retry(): - try: - service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-HUNG") - except RuntimeError as e: - errors.append(str(e)) - - retrier = threading.Thread(target=retry) - try: - retrier.start() - retrier.join(5.0) - assert not retrier.is_alive(), "retry must fail after the bound, not hang on the hung retirement" - assert len(errors) == 1 and "did not settle" in errors[0] - finally: - cb_release.set() - # once the abandoned settlement eventually completes, the id is usable again - for _ in range(50): - with service._outcome_lock: - consumed = "TX-HUNG" not in service._outcome_owners - if consumed: - break - time.sleep(0.1) - assert service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-HUNG") == "TX-HUNG" + assert not shutter.is_alive() def test_inflight_budget_pass_drains_before_settlement(self): - """P1 pin: the monitor re-checks table identity, releases _tx_lock, then runs - budget enforcement -- a retry could retire and settle the transaction while - enforcement was mid-flight, and enforcement then emitted FAILED terminal - progress under the reused tx_id. Operations now register with the activity - gate and settlement drains them before emitting or registering the retry.""" + """The monitor re-checks table identity, releases _tx_lock, then runs budget + enforcement outside it. Settlement drains such in-flight operations (activity + gate) before snapshotting the outcome: the enforcement's verdicts are COUNTED + in the receipt, and none of its emissions land after the transaction settled.""" from unittest.mock import Mock service = make_isolated_download_service() @@ -603,146 +519,36 @@ def gated_downloaded_to_one(receiver, status): receiver_ids=("r1",), receiver_acquire_timeout=5.0, progress_cb=lambda **kw: events.append(("old_progress", kw.get("state"))), + outcome_cb=lambda outcome: events.append("outcome_recorded"), ) obj = _stub_obj() obj.downloaded_to_one = gated_downloaded_to_one service.add_object("TX-BUDGET", obj) monitor = threading.Thread(target=run_monitor_once, args=(service,), kwargs={"now": time.time() + 100.0}) - retrier = threading.Thread( - target=lambda: ( - service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-BUDGET"), - events.append("retry_registered"), - ) - ) + deleter = threading.Thread(target=service.delete_transaction, args=("TX-BUDGET",)) try: monitor.start() assert cb_entered.wait(5.0), "budget enforcement must have finalized r1 as FAILED" - # the retry lands while enforcement is mid-flight: it must drain first - retrier.start() - retrier.join(0.5) - assert retrier.is_alive(), "retry must not register while a budget pass is in flight" + # settlement lands while enforcement is mid-flight: it must drain first + deleter.start() + deleter.join(0.5) + assert deleter.is_alive(), "settlement must not proceed while a budget pass is in flight" finally: cb_release.set() monitor.join(5.0) - retrier.join(5.0) - assert not retrier.is_alive() - - # every old-generation emission -- including enforcement's FAILED terminal - # progress -- happened strictly before the retry registered - retry_at = events.index("retry_registered") - old_emissions = [i for i, e in enumerate(events) if e != "retry_registered"] - assert old_emissions and all(i < retry_at for i in old_emissions), f"stale emission after retry: {events}" - assert ("old_progress", TransferProgressState.FAILED) in events - - def test_waiter_parked_during_shutdown_settlement_window_resolves(self): - """P1 pin: shutdown resolves the waiters it sees and keeps ownership markers for - serialization -- but a get_transfer_waiter call in the settlement window (after - shutdown's locked teardown, before the deferred settlement consumes ownership) - found the marker, parked, and hung forever: the record-forbidden branch consumed - ownership without draining waiters. Every ownership-consuming path drains now.""" - from unittest.mock import Mock - - service = make_isolated_download_service() - service._tx_monitor = Mock() - cell = Mock() - cb_entered = threading.Event() - cb_release = threading.Event() - - def gated_done_cb(tid, status, base_objs, **kw): - cb_entered.set() - cb_release.wait(10.0) - - service.new_transaction( - cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-WIN", transaction_done_cb=gated_done_cb - ) - shutter = threading.Thread(target=service.shutdown) - try: - shutter.start() - assert cb_entered.wait(5.0) # locked teardown done; deferred settlement in flight - - # the ownership marker is still visible, so this waiter parks instead of - # resolving immediately -- it must be drained when settlement consumes - # ownership, not stranded - waiter = service.get_transfer_waiter("TX-WIN") - assert not waiter.done(), "precondition: the waiter parked inside the window" - finally: - cb_release.set() - shutter.join(5.0) - outcome = waiter.wait(timeout=5.0) - assert waiter.done(), "window waiter must never hang past shutdown settlement" - assert outcome is None, "shutdown drops verdicts: the window waiter resolves to None" - - def test_retry_gives_up_if_previous_generation_never_settles(self): - # ownership was taken but settlement never runs (a hung terminator): a retry - # must fail loudly after the bounded wait, never register ambiguously - from unittest.mock import Mock - - service = make_isolated_download_service() - service._tx_monitor = Mock() # suppress real monitor thread start - cell = Mock() - service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-STUCK") - with service._tx_lock: - service._tx_table.pop("TX-STUCK") # popped for termination; settlement never follows - - service.SETTLEMENT_WAIT_TIMEOUT = 0.2 - with pytest.raises(RuntimeError, match="did not settle"): - service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-STUCK") - # and it did not register: the stuck generation still owns the slot - with service._tx_lock: - assert "TX-STUCK" not in service._tx_table - - def test_reusing_active_tx_id_retires_prior_transaction(self): - # reusing a tx_id while the prior transaction is still LIVE must retire it, - # not orphan it: a plain _tx_table overwrite would leave the old refs servable - # in _ref_table forever (the monitor only sees _tx_table) and never release - # the old sources via transaction_done - from unittest.mock import Mock - - service = make_isolated_download_service() - service._tx_monitor = Mock() # suppress real monitor thread start - cell = Mock() - done = [] - try: - outcome_cbs = [] - first = service.new_transaction( - cell=cell, - timeout=10.0, - num_receivers=1, - tx_id="TX-DUP", - transaction_done_cb=lambda tid, status, base_objs, **kw: done.append((tid, status)), - outcome_cb=lambda outcome: outcome_cbs.append(outcome), - ) - obj = _stub_obj() - rid = service.add_object(first, obj) - assert rid in service._ref_table - gen1_waiter = service.get_transfer_waiter(first) - - second = service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-DUP") - assert second == "TX-DUP" - - # waiters bind to the generation that records while they wait: the retired - # generation resolved this waiter with its own (non-completed) verdict - # before the retry registered - gen1_outcome = gen1_waiter.wait(timeout=1.0) - assert gen1_outcome is not None and not gen1_outcome.completed - - # the prior live transaction was retired and fully settled BEFORE the retry - # registered: refs unservable, sources released, and ALL its terminal - # emissions -- cleanup and outcome alike -- ran while the tx_id still - # unambiguously named it (generations are strictly serialized) - assert rid not in service._ref_table - assert obj.released - assert done == [("TX-DUP", TransactionDoneStatus.DELETED)] - assert len(outcome_cbs) == 1 and not outcome_cbs[0].completed - # its terminal outcome does not shadow the live retry - assert service.get_transaction_outcome("TX-DUP") is None - # the live tx is the new owner - assert service._tx_table["TX-DUP"] is service._outcome_owners["TX-DUP"] - assert service._tx_table["TX-DUP"] is not None - finally: - service.shutdown() + deleter.join(5.0) + assert not deleter.is_alive() + + # the enforcement's emissions all precede outcome recording, and its verdict + # was counted: r1 books as FAILED in the receipt + recorded_at = events.index("outcome_recorded") + assert events.index("old_downloaded_to_one") < recorded_at + assert events.index(("old_progress", TransferProgressState.FAILED)) < recorded_at + outcome = service.get_transaction_outcome("TX-BUDGET") + assert outcome is not None and not outcome.completed + assert outcome.refs[0].receiver_statuses.get("r1") == DownloadStatus.FAILED def test_unknown_transaction_has_no_outcome(self): service = make_isolated_download_service() From dc901fb4a2c13d4cfd68e6af59cf359d2ff32ddf Mon Sep 17 00:00:00 2001 From: YuanTingHsieh Date: Thu, 9 Jul 2026 17:03:36 -0700 Subject: [PATCH 15/20] Keep a drain-leaked tx_id excluded until its operations exit Review found the composition gap in the attempt-scoped duplicate check: a settlement drain that times out (OP_DRAIN_TIMEOUT) deliberately proceeds with an operation still in flight, and a caller-supplied tx_id becomes registrable again once its receipt expires (TX_OUTCOME_TTL) -- so a leaked operation that resumes after both windows could emit progress/failure under an id that now names a NEW transaction. Every terminator now registers a drain-timeout leak (tx with in-flight operations after settlement) in _leaked_ops; new_transaction rejects the id -- independently of receipt state -- until the leaked operation count reaches zero, and the monitor reaps entries whose operations exited. The id becomes reusable only after receipt expiry AND all old operations exited; the default fresh uuid never collides with anything. Description wording tightened to match the implementation: ids are single-use while known (live, settling, receipted, or leaked), not "never reused, forever". Pin: an operation held past a shortened drain timeout keeps the id rejected even with the receipt force-expired, and the id frees the moment the operation exits. --- nvflare/fuel/f3/streaming/download_service.py | 56 ++++++++++++++++--- .../fuel/f3/streaming/download_test_utils.py | 1 + .../f3/streaming/transfer_outcome_test.py | 38 +++++++++++++ 3 files changed, 87 insertions(+), 8 deletions(-) diff --git a/nvflare/fuel/f3/streaming/download_service.py b/nvflare/fuel/f3/streaming/download_service.py index af1c7ac7b6..0b6e19b3d1 100644 --- a/nvflare/fuel/f3/streaming/download_service.py +++ b/nvflare/fuel/f3/streaming/download_service.py @@ -1042,6 +1042,13 @@ class DownloadService: # Waiters blocked on a transaction's terminal outcome (the awaitable facade). Guarded by # _outcome_lock; resolved inside _record_outcome so a waiter can never miss the outcome. _tx_waiters = {} + # Transactions whose settlement drain TIMED OUT: an operation (a hung serve or + # budget-pass callback) is still in flight past OP_DRAIN_TIMEOUT and may resume + # and emit later. Its tx_id stays excluded from registration -- even after the + # receipt expires -- until every such operation has exited, so a leaked emission + # can never land under a recycled id. Entries are reaped by the monitor and by + # the registration check once the op count reaches zero. Guarded by _tx_lock. + _leaked_ops = {} _outcome_lock = threading.Lock() TX_OUTCOME_TTL = 1800.0 _logger = None @@ -1101,20 +1108,32 @@ def new_transaction( receiver_acquire_timeout=receiver_acquire_timeout, receiver_idle_timeout=receiver_idle_timeout, ) - # tx_ids are ATTEMPT-SCOPED and never reused: a retry of the same logical - # transfer is a NEW transaction with a new id (the stable cross-attempt - # identity is the application-level transfer id carried in the caller's - # metadata -- it never enters this service). Uniqueness is what makes every - # emission of a terminated transaction attributable to it alone, forever: + # tx_ids are ATTEMPT-SCOPED and single-use while known: a retry of the same + # logical transfer is a NEW transaction with a new id (the stable + # cross-attempt identity is the application-level transfer id carried in the + # caller's metadata -- it never enters this service). Uniqueness is what + # makes every emission of a terminated transaction attributable to it alone: # nothing a dying attempt does can be confused with a live successor, so no # suppression, generation serialization, or settlement waiting is needed. # Registration is therefore a single atomic step: ownership and table entry # together (_tx_lock nests _outcome_lock here; no path nests them in the # reverse order), so a tx the monitor can terminate always owns its outcome - # slot -- and a duplicate id is rejected while it is live, still settling, or - # its receipt is retained (receipts expire after TX_OUTCOME_TTL, which bounds - # the exclusion window for caller-supplied ids). + # slot. A duplicate id is rejected while it is live, still settling, its + # receipt is retained (TX_OUTCOME_TTL bounds that window for caller-supplied + # ids; the default fresh uuid never collides), or a drain-timeout leak from a + # previous attempt is still in flight -- the id becomes reusable only after + # the receipt expired AND every old operation exited. with cls._tx_lock: + leaked = cls._leaked_ops.get(tx.tid) + if leaked is not None: + with leaked._ops_cond: + still_in_flight = leaked._active_ops > 0 + if still_in_flight: + raise ValueError( + f"transaction id {tx.tid} still has in-flight operations from a previous " + f"attempt (its settlement drain timed out): use a new id" + ) + cls._leaked_ops.pop(tx.tid, None) with cls._outcome_lock: # an expired-but-unswept receipt does not exclude the id: expire it # here (receipts are swept lazily and by the monitor, so without this @@ -1164,6 +1183,7 @@ def delete_transaction(cls, transaction_id: str): if tx: tx.transaction_done(TransactionDoneStatus.DELETED, on_outcome=functools.partial(cls._record_outcome, tx=tx)) + cls._register_leaked_ops(tx) @classmethod def shutdown(cls): @@ -1207,6 +1227,23 @@ def shutdown(cls): for tx in tx_list: tx.transaction_done(TransactionDoneStatus.DELETED) + cls._register_leaked_ops(tx) + + @classmethod + def _register_leaked_ops(cls, tx: _Transaction): + """Called by every terminator after settlement: if the drain timed out and + operations are still in flight, exclude the id until they exit.""" + with tx._ops_cond: + leaked = tx._active_ops > 0 + if leaked: + with cls._tx_lock: + cls._leaked_ops[tx.tid] = tx + + @classmethod + def _reap_leaked_ops(cls): + with cls._tx_lock: + for tid in [t for t, tx in cls._leaked_ops.items() if tx._active_ops <= 0]: + cls._leaked_ops.pop(tid, None) @classmethod def _delete_tx(cls, tx: _Transaction, tombstone_finished_refs: bool = False): @@ -1524,16 +1561,19 @@ def _monitor_tx(cls): cls._expire_finished_refs(now) cls._expire_outcomes(now) + cls._reap_leaked_ops() for tx in expired_tx: tx.transaction_done( TransactionDoneStatus.TIMEOUT, on_outcome=functools.partial(cls._record_outcome, tx=tx) ) + cls._register_leaked_ops(tx) for tx in finished_tx: tx.transaction_done( TransactionDoneStatus.FINISHED, on_outcome=functools.partial(cls._record_outcome, tx=tx) ) + cls._register_leaked_ops(tx) time.sleep(5.0) diff --git a/tests/unit_test/fuel/f3/streaming/download_test_utils.py b/tests/unit_test/fuel/f3/streaming/download_test_utils.py index 85aea82a0d..a28bf28f36 100644 --- a/tests/unit_test/fuel/f3/streaming/download_test_utils.py +++ b/tests/unit_test/fuel/f3/streaming/download_test_utils.py @@ -87,6 +87,7 @@ class IsolatedDownloadService(DownloadService): _tx_outcomes = {} _outcome_owners = {} _tx_waiters = {} + _leaked_ops = {} _outcome_lock = threading.Lock() _logger = Mock() _tx_lock = threading.Lock() diff --git a/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py b/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py index 650588eb94..035f6161a2 100644 --- a/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py +++ b/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py @@ -406,6 +406,44 @@ def test_new_transaction_rejects_settling_and_receipted_id(self): ) assert service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-LIFE") == "TX-LIFE" + def test_drain_leak_poisons_id_until_operation_exits(self): + """P1 pin: a drain-timeout leak (operation hung past OP_DRAIN_TIMEOUT) may resume + and emit later -- so its tx_id must stay excluded from registration even after + the receipt expires, until every leaked operation has exited. Otherwise the + leaked emission could land under a recycled id.""" + from unittest.mock import Mock + + import nvflare.fuel.f3.streaming.download_service as ds_module + + service = make_isolated_download_service() + service._tx_monitor = Mock() + cell = Mock() + service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-LEAK") + with service._tx_lock: + tx = service._tx_table["TX-LEAK"] + assert tx.begin_op() # an in-flight operation (serve/budget pass) that will hang + + saved = ds_module.OP_DRAIN_TIMEOUT + ds_module.OP_DRAIN_TIMEOUT = 0.2 + try: + service.delete_transaction("TX-LEAK") # drain times out; settles anyway (logged) + finally: + ds_module.OP_DRAIN_TIMEOUT = saved + + # force-expire the receipt: the id must STILL be excluded by the leak + recorded = service.get_transaction_outcome("TX-LEAK") + assert recorded is not None + with service._outcome_lock: + service._tx_outcomes["TX-LEAK"] = dataclasses.replace( + recorded, timestamp=recorded.timestamp - service.TX_OUTCOME_TTL - 1 + ) + with pytest.raises(ValueError, match="in-flight operations from a previous attempt"): + service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-LEAK") + + # the leaked operation exits: the id becomes usable + tx.end_op() + assert service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-LEAK") == "TX-LEAK" + def test_settlement_callback_can_create_transactions_but_not_its_own_id(self): """Settlement runs outside all service locks, so callbacks may freely call back in and create NEW transactions; recreating the settling transaction's own id is From ee15305eb475ee0aa7f6e51dbbf68c51d5b5515f Mon Sep 17 00:00:00 2001 From: YuanTingHsieh Date: Thu, 9 Jul 2026 17:14:03 -0700 Subject: [PATCH 16/20] Trim comments to constraint-only; one home per invariant Comment/docstring share of download_service.py had grown to a third of the file across the review rounds -- much of it review-history narration and correctness arguments addressed to reviewers, which are noise once merged (that knowledge lives in commit messages and the regression pins). Rules applied: comments state constraints the code cannot show (lock ordering, unlink-before-settle, why callbacks run outside locks, wire semantics); each invariant is documented at exactly one home (table invariants on the class attributes, contracts in method docstrings) with other sites shortened to a pointer; review archaeology and justification-of-change prose deleted. Public API docstrings and test docstrings (each pin documents the bug it guards) are untouched. --- nvflare/fuel/f3/streaming/download_service.py | 179 ++++++------------ 1 file changed, 58 insertions(+), 121 deletions(-) diff --git a/nvflare/fuel/f3/streaming/download_service.py b/nvflare/fuel/f3/streaming/download_service.py index 0b6e19b3d1..0a0fedd6c0 100644 --- a/nvflare/fuel/f3/streaming/download_service.py +++ b/nvflare/fuel/f3/streaming/download_service.py @@ -215,10 +215,9 @@ def _receiver_confirm_enabled() -> bool: RECEIVER_ACQUIRE_TIMEOUT_CONFIG_VAR = "streaming_receiver_acquire_timeout" RECEIVER_IDLE_TIMEOUT_CONFIG_VAR = "streaming_receiver_idle_timeout" -# How long settlement waits for in-flight operations (serves, confirms, budget passes) -# against the transaction to drain before emitting its terminal surfaces. Only a hung -# user callback inside such an operation can exhaust this; settlement then proceeds -# with a warning (the straggler may emit late, but can no longer be prevented). +# How long settlement waits for in-flight operations to drain. Only a hung user +# callback can exhaust it; settlement then proceeds with a warning, and the id +# stays excluded until the leaked operation exits (see _leaked_ops). OP_DRAIN_TIMEOUT = 60.0 @@ -268,11 +267,10 @@ def obj_downloaded(self, to_receiver: str, status: str): def _finalize_receiver( self, to_receiver: str, status: str, require_pending: bool = False, nonce: Optional[str] = None ) -> bool: - # Status recording is guarded so terminal-outcome snapshots taken on the - # monitor thread never observe a half-updated map; user callbacks run - # outside the lock. The whole decision (dedup, pending-guard, pending pop, - # record, all-done latch) is one critical section, so a duplicate serve can - # never resurrect a pending entry around a racing finalization. + # Recording is guarded so outcome snapshots never observe a half-updated map; + # user callbacks run outside the lock. Dedup, pending-guard, pop, record, and + # the all-done latch are ONE critical section, so a duplicate serve can never + # resurrect a pending entry around a racing finalization. with self._progress_lock: if to_receiver in self.receiver_statuses: return False @@ -400,9 +398,8 @@ def mark_receiver_active(self, receiver: str): self._receiver_activity[receiver] = now tx = self.tx with tx._stats_lock: - # transaction-level liveness: budgets judge idleness across the WHOLE - # transaction, so a receiver that finished one ref and went silent cannot - # escape a sibling ref's budgets (it has no per-ref timestamp there) + # tx-level activity: budgets judge the whole transaction (see the + # _receiver_last_active field comment) tx._acquired_receivers.add(receiver) tx._receiver_last_active[receiver] = now @@ -694,10 +691,9 @@ def __init__( self.refs = [] self._refs_lock = threading.RLock() # The activity gate: serves, confirms, and monitor budget passes register as - # in-flight operations; settlement closes the gate and drains them before - # snapshotting the outcome, so an operation finishing during termination is - # counted in the verdict and nothing emits against the transaction after it - # has settled. + # in-flight operations; settlement closes and drains the gate before the + # outcome snapshot, so late finishes are counted and nothing emits against a + # settled transaction. self._ops_cond = threading.Condition(threading.Lock()) self._active_ops = 0 self._ops_closed = False @@ -710,11 +706,6 @@ def __init__( self.logger = get_obj_logger(self) def mark_active(self): - """Called to update the last active time of the transaction. - - Returns: - - """ self.last_active_time = time.time() def add_total_bytes(self, byte_count: int): @@ -732,15 +723,7 @@ def add_object( obj: Downloadable, ref_id=None, ): - """Add a large object (to be downloaded) to the transaction. - - Args: - obj: the large object to be downloaded - ref_id: the ref id to be used, if specified - - Returns: - - """ + """Adds a large object (to be downloaded) to the transaction; returns its ref.""" with self._refs_lock: r = _Ref(self, obj, ref_id) self.refs.append(r) @@ -825,28 +808,16 @@ def is_finished(self): return True def transaction_done(self, status: str, on_outcome=None) -> TransferOutcome: - """Called when the transaction is finished. - - Returns the aggregate TransferOutcome (see transfer_outcome.py): COMPLETED only - when every expected receiver succeeded — TransactionDoneStatus.FINISHED alone - does not certify that. The existing transaction_done_cb contract is unchanged - except that callback exceptions no longer propagate (they would kill the - monitor thread and skip source release). - - on_outcome (used by DownloadService to record the outcome and release waiters) - is invoked LAST — after terminal progress, the done/outcome callbacks, and - source release — so an upper layer that acts on waiter.wait() returning (e.g. - stops the producer process) can never preempt the callback chain or the - release of the sources. - - Settlement runs exactly once per transaction: every terminator first unlinks it - from _tx_table under _tx_lock, so exactly one caller runs this. tx_ids are - never reused (new_transaction rejects duplicates), so every emission here is - attributable to this transaction alone, forever. + """Settles the transaction; returns the aggregate TransferOutcome. + + COMPLETED only when every expected receiver succeeded — FINISHED alone does + not certify that. Callback exceptions never propagate (they would kill the + monitor thread and skip source release). on_outcome (records the outcome, + releasing waiters) is invoked LAST, so acting on waiter.wait() returning can + never preempt the callback chain or source release. Runs exactly once per + transaction: every terminator unlinks it from _tx_table first. """ - # Drain in-flight operations BEFORE snapshotting the outcome: an operation that - # finishes during the drain is counted in the verdict, and nothing can emit - # against the transaction after it has settled. + # drain the activity gate first: in-flight results count in the verdict if not self._drain_ops(OP_DRAIN_TIMEOUT): self.logger.warning( f"tx {self.tid}: in-flight operations did not drain within {OP_DRAIN_TIMEOUT}s; settling anyway" @@ -912,16 +883,13 @@ def transaction_done(self, status: str, on_outcome=None) -> TransferOutcome: if self.outcome_cb: _invoke_cb_safely(self.logger, f"transfer outcome callback for tx {self.tid}", self.outcome_cb, outcome) - # Release source objects after the callback so the callback can still - # reference them. Each release is guarded: a raising custom release() must - # not skip its siblings -- nor, via the finally below, outcome recording. + # release after the callback (so it can still observe the sources); each + # release is guarded so one raising release() cannot skip its siblings for ref in refs: _invoke_cb_safely(self.logger, f"release of {type(ref.obj)} for tx {self.tid}", ref.obj.release) finally: - # Record the outcome (and release waiters) only now that the transaction is - # fully settled: callbacks done, sources released. In a finally so that no - # exception anywhere above (including on the monitor thread) can leave the - # outcome unrecorded and the waiter pending forever. + # in a finally: no exception above may leave the outcome unrecorded and + # waiters pending if on_outcome and outcome is not None: _invoke_cb_safely(self.logger, f"outcome recording for tx {self.tid}", on_outcome, outcome) @@ -1031,23 +999,18 @@ class DownloadService: # time so producers can query the aggregate result after termination. Guarded by # its own lock so outcome polling never contends with the chunk-serving _tx_lock. _tx_outcomes = {} - # The transaction object entitled to record the outcome for its tx_id (registered - # by new_transaction, consumed by _record_outcome). Recording checks OBJECT - # identity against this map: a recorder that lost its entry -- shutdown cleared - # ownership, or a prior terminal record already consumed it -- drops its outcome - # instead of recording stale truth. An entry also marks the id as in-use to - # new_transaction's duplicate check while the transaction is settling. - # Guarded by _outcome_lock. + # The transaction entitled to record the outcome for its tx_id (registered by + # new_transaction, consumed by _record_outcome; object-identity checked, so a + # recorder whose entry is gone drops its stale outcome). Also marks the id as + # in-use while the transaction is settling. Guarded by _outcome_lock. _outcome_owners = {} # Waiters blocked on a transaction's terminal outcome (the awaitable facade). Guarded by # _outcome_lock; resolved inside _record_outcome so a waiter can never miss the outcome. _tx_waiters = {} - # Transactions whose settlement drain TIMED OUT: an operation (a hung serve or - # budget-pass callback) is still in flight past OP_DRAIN_TIMEOUT and may resume - # and emit later. Its tx_id stays excluded from registration -- even after the - # receipt expires -- until every such operation has exited, so a leaked emission - # can never land under a recycled id. Entries are reaped by the monitor and by - # the registration check once the op count reaches zero. Guarded by _tx_lock. + # Transactions whose settlement drain timed out with operations still in flight: + # the tx_id stays excluded from registration -- even past receipt expiry -- until + # every such operation exits (a leaked emission must never land under a recycled + # id). Reaped by the monitor and the registration check. Guarded by _tx_lock. _leaked_ops = {} _outcome_lock = threading.Lock() TX_OUTCOME_TTL = 1800.0 @@ -1108,21 +1071,14 @@ def new_transaction( receiver_acquire_timeout=receiver_acquire_timeout, receiver_idle_timeout=receiver_idle_timeout, ) - # tx_ids are ATTEMPT-SCOPED and single-use while known: a retry of the same - # logical transfer is a NEW transaction with a new id (the stable - # cross-attempt identity is the application-level transfer id carried in the - # caller's metadata -- it never enters this service). Uniqueness is what - # makes every emission of a terminated transaction attributable to it alone: - # nothing a dying attempt does can be confused with a live successor, so no - # suppression, generation serialization, or settlement waiting is needed. - # Registration is therefore a single atomic step: ownership and table entry - # together (_tx_lock nests _outcome_lock here; no path nests them in the - # reverse order), so a tx the monitor can terminate always owns its outcome - # slot. A duplicate id is rejected while it is live, still settling, its - # receipt is retained (TX_OUTCOME_TTL bounds that window for caller-supplied - # ids; the default fresh uuid never collides), or a drain-timeout leak from a - # previous attempt is still in flight -- the id becomes reusable only after - # the receipt expired AND every old operation exited. + # tx_ids are ATTEMPT-SCOPED and single-use while known: a retry is a NEW + # transaction with a new id (the stable cross-attempt identity is the + # caller's application-level transfer id, which never enters this service), + # so nothing a dying attempt emits can be confused with a live successor. + # A duplicate id is rejected while live, settling, receipted + # (TX_OUTCOME_TTL), or leaked (see _leaked_ops). Registration is one atomic + # step -- ownership and table entry together, _tx_lock nesting _outcome_lock + # (no path nests them in the reverse order). with cls._tx_lock: leaked = cls._leaked_ops.get(tx.tid) if leaked is not None: @@ -1135,9 +1091,8 @@ def new_transaction( ) cls._leaked_ops.pop(tx.tid, None) with cls._outcome_lock: - # an expired-but-unswept receipt does not exclude the id: expire it - # here (receipts are swept lazily and by the monitor, so without this - # the exclusion window would stretch to TTL + a sweep cycle) + # expire an unswept receipt inline: the exclusion window is exactly + # TX_OUTCOME_TTL, not TTL plus a sweep cycle receipt = cls._tx_outcomes.get(tx.tid) if receipt is not None and receipt.expired(time.time(), cls.TX_OUTCOME_TTL): cls._tx_outcomes.pop(tx.tid, None) @@ -1187,17 +1142,11 @@ def delete_transaction(cls, transaction_id: str): @classmethod def shutdown(cls): - """Shutdown and clean up resources. - - Returns: None - - """ - # Table teardown and ownership teardown are ONE atomic step (_tx_lock nesting - # _outcome_lock, the same order new_transaction uses; no reverse nesting exists). - # With separate critical sections, a new_transaction landing in the gap between - # them would register itself into both tables -- and the ownership forfeit below - # would then hit the LIVE transaction, leaving it in _tx_table but unable to - # ever record its outcome (found by property falsification, P-C). + """Shuts down the service: terminates all transactions, drops all state.""" + # Table and ownership teardown are ONE atomic step (_tx_lock nesting + # _outcome_lock): a registration landing between separate critical sections + # would enter both tables and then lose its ownership, leaving a live + # transaction that can never record. with cls._tx_lock: tx_list = list(cls._tx_table.values()) for tx in tx_list: @@ -1205,16 +1154,11 @@ def shutdown(cls): cls._finished_refs.clear() with cls._outcome_lock: - # Drop recorded outcomes and clear outcome ownership: a settlement - # mid-flight on another thread finds its ownership gone once it reaches - # _record_outcome, so nothing records after shutdown -- the owner guard - # alone gates post-shutdown recording. (tx_ids are never reused, so a - # new_transaction racing this shutdown registers under a DIFFERENT id - # and cannot collide with the deferred settlements below.) + # Clearing ownership is what stops recording: a settlement mid-flight + # on another thread finds its entry gone at _record_outcome and drops. cls._tx_outcomes.clear() cls._outcome_owners.clear() - # unblock the awaitable facade: waiters resolve to None (service shut down - # before the transaction terminated), never hang + # waiters resolve to None rather than hang for waiters in cls._tx_waiters.values(): for waiter in waiters: waiter._resolve(None) @@ -1275,10 +1219,8 @@ def get_transfer_waiter(cls, transaction_id: str) -> TransferWaiter: waiter._resolve(existing) return waiter if transaction_id not in cls._outcome_owners: - # unknown, already-forgotten (outcome expired) or shut down: nothing will - # ever record an outcome for this id, so resolving with None immediately is - # the only way to honor "waiters can never hang". Race-free: _record_outcome - # swaps ownership -> recorded outcome under this same lock. + # unknown/expired/shut-down: nothing will ever record for this id -- + # resolve None now (waiters never hang); race-free under this lock waiter._resolve(None) return waiter cls._tx_waiters.setdefault(transaction_id, []).append(waiter) @@ -1301,9 +1243,7 @@ def _record_outcome(cls, outcome: TransferOutcome, tx: _Transaction): # recording is legal only for the transaction that owns the outcome slot. with cls._outcome_lock: if cls._outcome_owners.get(outcome.tx_id) is not tx: - # ownership was already consumed (a prior terminal record) or cleared - # by shutdown(): this recording is stale and must be dropped -- the - # owner guard alone gates post-shutdown recording + # ownership consumed (prior record) or cleared (shutdown): stale, drop return cls._outcome_owners.pop(outcome.tx_id, None) cls._tx_outcomes[outcome.tx_id] = outcome @@ -1396,9 +1336,8 @@ def _handle_download(cls, request: Message) -> Message: with cls._tx_lock: ref = cls._ref_table.get(rid) if ref is not None and not ref.tx.begin_op(): - # the transaction is settling/settled: this serve must not start against - # it (its emissions would land after settlement) -- fall through to the - # tombstone/missing handling exactly as if the ref were already gone + # settling/settled: a serve must not start against it -- treat the ref + # as already gone (tombstone/missing handling) ref = None if not ref: finished_status = cls._get_finished_ref_status(rid, requester) @@ -1524,9 +1463,7 @@ def _monitor_tx(cls): budget_txs = [tx for tx in cls._tx_table.values() if tx.has_receiver_budgets] for tx in budget_txs: with cls._tx_lock: - # deleted/replaced since the snapshot, or already settling: do not - # touch a dead tx -- begin_op makes the check binding, since - # settlement drains registered operations before emitting + # dead or settling tx: skip (begin_op makes the table check binding) live = cls._tx_table.get(tx.tid) is tx and tx.begin_op() if not live: continue From 246d11b16dc647680744b214daa45b1eb13425d5 Mon Sep 17 00:00:00 2001 From: YuanTingHsieh Date: Thu, 9 Jul 2026 17:32:06 -0700 Subject: [PATCH 17/20] Close the shutdown window on drain-leaked tx_ids; review polish Review found the leak exclusion's shutdown gap: shutdown() clears the ownership and receipt exclusions up front, but leak registration ran only AFTER the deferred settlement -- so a drain-leaked operation's tx_id looked free for that whole window (unbounded if a settlement callback hangs), and a replacement could register while the leaked operation was still able to emit under the id. delete_transaction and the monitor were not affected: their ownership marker covers settlement and their receipt covers the tail. shutdown() now pre-registers every registered owner (live and mid-settlement) as a potential leak in the same atomic teardown section, before clearing ownership. The per-terminator hook becomes a sync (_update_leaked_ops): it keeps the exclusion while operations are in flight and releases it once none are, so pre-registered ids free normally after a clean drain. Pin drives the exact reviewer sequence: leaked op + shutdown mid-settlement -> same-id registration rejected, still rejected after shutdown returns, freed the moment the operation exits. Also from review: explicit pin that a CONFIRM-keyed message dispatches to confirm handling and never falls through to pull handling (a fall-through would re-serve from scratch); duplicate receiver_ids now log a warning (almost certainly a caller bug) with the dedup pin extended to assert it; make_confirm_test_service renamed to make_service_no_monitor (it was never confirm-specific); conftest notes that its autouse confirm-switch fixture is directory-wide. --- nvflare/fuel/f3/streaming/download_service.py | 38 ++++++++++---- tests/unit_test/fuel/f3/streaming/conftest.py | 5 ++ .../fuel/f3/streaming/download_test_utils.py | 8 ++- .../fuel/f3/streaming/receiver_budget_test.py | 11 ++-- .../f3/streaming/receiver_confirm_test.py | 22 +++++++- .../f3/streaming/transfer_outcome_test.py | 51 +++++++++++++++++++ .../fuel/f3/streaming/transfer_waiter_test.py | 2 +- 7 files changed, 120 insertions(+), 17 deletions(-) diff --git a/nvflare/fuel/f3/streaming/download_service.py b/nvflare/fuel/f3/streaming/download_service.py index 0a0fedd6c0..fcfc075bd2 100644 --- a/nvflare/fuel/f3/streaming/download_service.py +++ b/nvflare/fuel/f3/streaming/download_service.py @@ -652,12 +652,19 @@ def __init__( else: self.tid = "T" + str(uuid.uuid4()) self.timeout = timeout + self.logger = get_obj_logger(self) # Expected receiver identities. Optional: when provided they enable the acquire # budget (a receiver that never issues its first pull can be failed) and, if # num_receivers is unknown (0), supply the receiver count. if receiver_ids: + given = len(receiver_ids) receiver_ids = tuple(dict.fromkeys(str(r) for r in receiver_ids)) # dedup, keep order + if len(receiver_ids) != given: + # almost certainly a caller bug: it believes there are more distinct receivers + self.logger.warning( + f"tx {self.tid}: duplicate receiver_ids deduplicated ({given} -> {len(receiver_ids)})" + ) if num_receivers and num_receivers != len(receiver_ids): raise ValueError( f"num_receivers ({num_receivers}) does not match receiver_ids count ({len(receiver_ids)})" @@ -703,7 +710,6 @@ def __init__( # budget judges -- per-ref timestamps would let a multi-ref receiver escape) self._acquired_receivers = set() self._receiver_last_active = {} - self.logger = get_obj_logger(self) def mark_active(self): self.last_active_time = time.time() @@ -1138,7 +1144,7 @@ def delete_transaction(cls, transaction_id: str): if tx: tx.transaction_done(TransactionDoneStatus.DELETED, on_outcome=functools.partial(cls._record_outcome, tx=tx)) - cls._register_leaked_ops(tx) + cls._update_leaked_ops(tx) @classmethod def shutdown(cls): @@ -1154,6 +1160,13 @@ def shutdown(cls): cls._finished_refs.clear() with cls._outcome_lock: + # Pre-register every registered owner (live and mid-settlement) as a + # potential leak BEFORE clearing its ownership/receipt exclusions: the + # deferred settlements below run outside the locks, and a drain-leaked + # operation's id must never look free in that window. Post-settlement + # _update_leaked_ops releases the ids whose operations drained. + for owner in cls._outcome_owners.values(): + cls._leaked_ops[owner.tid] = owner # Clearing ownership is what stops recording: a settlement mid-flight # on another thread finds its entry gone at _record_outcome and drops. cls._tx_outcomes.clear() @@ -1171,17 +1184,22 @@ def shutdown(cls): for tx in tx_list: tx.transaction_done(TransactionDoneStatus.DELETED) - cls._register_leaked_ops(tx) + cls._update_leaked_ops(tx) @classmethod - def _register_leaked_ops(cls, tx: _Transaction): - """Called by every terminator after settlement: if the drain timed out and - operations are still in flight, exclude the id until they exit.""" + def _update_leaked_ops(cls, tx: _Transaction): + """Called by every terminator after settlement: keep the id excluded while + drain-leaked operations are in flight, release the exclusion once none are. + shutdown() PRE-registers its transactions under the locks -- it clears the + ownership and receipt exclusions up front, so waiting until after settlement + would leave a window where a leaked operation's id looks free.""" with tx._ops_cond: leaked = tx._active_ops > 0 - if leaked: - with cls._tx_lock: + with cls._tx_lock: + if leaked: cls._leaked_ops[tx.tid] = tx + elif cls._leaked_ops.get(tx.tid) is tx: + cls._leaked_ops.pop(tx.tid, None) @classmethod def _reap_leaked_ops(cls): @@ -1504,13 +1522,13 @@ def _monitor_tx(cls): tx.transaction_done( TransactionDoneStatus.TIMEOUT, on_outcome=functools.partial(cls._record_outcome, tx=tx) ) - cls._register_leaked_ops(tx) + cls._update_leaked_ops(tx) for tx in finished_tx: tx.transaction_done( TransactionDoneStatus.FINISHED, on_outcome=functools.partial(cls._record_outcome, tx=tx) ) - cls._register_leaked_ops(tx) + cls._update_leaked_ops(tx) time.sleep(5.0) diff --git a/tests/unit_test/fuel/f3/streaming/conftest.py b/tests/unit_test/fuel/f3/streaming/conftest.py index bba4b164e5..fa33b66f83 100644 --- a/tests/unit_test/fuel/f3/streaming/conftest.py +++ b/tests/unit_test/fuel/f3/streaming/conftest.py @@ -11,6 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +"""DIRECTORY-WIDE fixtures: the autouse fixture below applies to EVERY test in +tests/unit_test/fuel/f3/streaming/, including tests that never touch confirms. +A new test here never exercises the real ConfigService read of the confirm +kill-switch; patch _receiver_confirm_cached explicitly if you need the legacy +(confirm-off) path or the real config read.""" from unittest.mock import patch diff --git a/tests/unit_test/fuel/f3/streaming/download_test_utils.py b/tests/unit_test/fuel/f3/streaming/download_test_utils.py index a28bf28f36..de977c1495 100644 --- a/tests/unit_test/fuel/f3/streaming/download_test_utils.py +++ b/tests/unit_test/fuel/f3/streaming/download_test_utils.py @@ -96,8 +96,12 @@ class IsolatedDownloadService(DownloadService): return IsolatedDownloadService -def make_confirm_test_service(): - """An isolated service with the real monitor thread suppressed (budget/confirm/waiter tests).""" +def make_service_no_monitor(): + """An isolated service with the real monitor thread suppressed. + + Not confirm-specific: use for any test that must not race the real 5s monitor + loop (budgets, confirms, waiters, lifecycle); drive monitor passes explicitly + via run_monitor_once.""" service = make_isolated_download_service() service._tx_monitor = Mock() return service diff --git a/tests/unit_test/fuel/f3/streaming/receiver_budget_test.py b/tests/unit_test/fuel/f3/streaming/receiver_budget_test.py index 9c47af9501..62c8aae5e0 100644 --- a/tests/unit_test/fuel/f3/streaming/receiver_budget_test.py +++ b/tests/unit_test/fuel/f3/streaming/receiver_budget_test.py @@ -30,7 +30,7 @@ from nvflare.fuel.f3.streaming.download_service import DownloadStatus from nvflare.fuel.f3.streaming.transfer_outcome import TransferOutcomeReason, compute_transfer_outcome from tests.unit_test.fuel.f3.streaming.download_test_utils import MockDownloadable, confirm_request -from tests.unit_test.fuel.f3.streaming.download_test_utils import make_confirm_test_service as _make_service +from tests.unit_test.fuel.f3.streaming.download_test_utils import make_service_no_monitor as _make_service from tests.unit_test.fuel.f3.streaming.download_test_utils import pull_request from tests.unit_test.fuel.f3.streaming.download_test_utils import pull_to_terminal as _pull_to_terminal from tests.unit_test.fuel.f3.streaming.download_test_utils import run_monitor_once, serve_nonce @@ -380,12 +380,17 @@ def test_non_positive_config_var_disables_budget(self): class TestTransactionValidation: - def test_receiver_ids_derive_num_receivers(self): + def test_receiver_ids_derive_num_receivers(self, caplog): + import logging + service = _make_service() - tx_id = service.new_transaction(cell=Mock(), timeout=10.0, num_receivers=0, receiver_ids=("a", "b", "a")) + with caplog.at_level(logging.WARNING): + tx_id = service.new_transaction(cell=Mock(), timeout=10.0, num_receivers=0, receiver_ids=("a", "b", "a")) tx = service._tx_table[tx_id] assert tx.receiver_ids == ("a", "b") # deduped, order kept assert tx.num_receivers == 2 + # review-requested pin: duplicates are almost certainly a caller bug -- warn + assert any("deduplicated" in r.message for r in caplog.records) def test_receiver_ids_num_receivers_mismatch_raises(self): service = _make_service() diff --git a/tests/unit_test/fuel/f3/streaming/receiver_confirm_test.py b/tests/unit_test/fuel/f3/streaming/receiver_confirm_test.py index 386781c53d..a84abcf30f 100644 --- a/tests/unit_test/fuel/f3/streaming/receiver_confirm_test.py +++ b/tests/unit_test/fuel/f3/streaming/receiver_confirm_test.py @@ -36,7 +36,7 @@ from nvflare.fuel.f3.streaming.download_service import Consumer, DownloadStatus, ProduceRC, _PropKey, download_object from tests.unit_test.fuel.f3.streaming.download_test_utils import MockDownloadable from tests.unit_test.fuel.f3.streaming.download_test_utils import confirm_request as _confirm_request -from tests.unit_test.fuel.f3.streaming.download_test_utils import make_confirm_test_service as _make_service +from tests.unit_test.fuel.f3.streaming.download_test_utils import make_service_no_monitor as _make_service from tests.unit_test.fuel.f3.streaming.download_test_utils import pull_request as _pull_request from tests.unit_test.fuel.f3.streaming.download_test_utils import pull_to_terminal as _pull_to_terminal from tests.unit_test.fuel.f3.streaming.download_test_utils import run_monitor_once, serve_nonce @@ -278,6 +278,26 @@ def test_stale_confirm_from_previous_ref_life_is_dropped_even_with_new_pending(s service._handle_download(_confirm_request(rid, "r1", DownloadStatus.SUCCESS, serve_nonce(new_terminal))) assert ref.snapshot_receiver_statuses() == {"r1": DownloadStatus.SUCCESS} + def test_confirm_message_dispatches_to_confirm_handling_not_pull(self): + # review-requested pin: a message carrying the CONFIRM key must route to + # _handle_confirm and never fall through to pull handling -- a fall-through + # would treat the confirm as an initial pull request (state=None) and + # re-serve the object from the beginning + service = _make_service() + tx1 = service.new_transaction(cell=Mock(), timeout=10.0, num_receivers=1) + obj = MockDownloadable([b"chunk"]) + rid = service.add_object(tx1, obj, ref_id="R-DISPATCH") + + reply = service._handle_download(_confirm_request(rid, "r1", DownloadStatus.SUCCESS, "bogus-nonce")) + + # confirm-path reply: bare OK, no chunk payload -- and produce() never ran + assert reply.payload is None or _PropKey.DATA not in (reply.payload or {}) + assert obj.current_chunk == 0 + ref = _ref(service, rid) + # the unsolicited confirm was dropped by confirm handling (no pending serve) + assert ref.snapshot_receiver_statuses() == {} + assert ref.snapshot_pending_confirms() == {} + def test_late_confirm_for_unknown_ref_is_dropped_ok(self): service = _make_service() reply = service._handle_download(_confirm_request("R-GONE", "r1", DownloadStatus.SUCCESS)) diff --git a/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py b/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py index 035f6161a2..f4be59cf4c 100644 --- a/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py +++ b/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py @@ -498,6 +498,57 @@ def exploding_record(outcome, tx): service.shutdown() assert stranded.done() and stranded.outcome is None + def test_shutdown_window_keeps_leaked_id_excluded(self): + """P1 pin: shutdown clears the ownership and receipt exclusions up front, and + leak registration used to run only AFTER the deferred settlement -- so a + drain-leaked id looked free for that whole window and a replacement could + register while the leaked operation was still able to emit. Shutdown now + pre-registers every owner as a potential leak under the locks.""" + from unittest.mock import Mock + + import nvflare.fuel.f3.streaming.download_service as ds_module + + service = make_isolated_download_service() + service._tx_monitor = Mock() + cell = Mock() + cb_entered = threading.Event() + cb_release = threading.Event() + + def gated_done_cb(tid, status, base_objs, **kw): + cb_entered.set() + cb_release.wait(10.0) + + service.new_transaction( + cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-SHUTLEAK", transaction_done_cb=gated_done_cb + ) + with service._tx_lock: + tx = service._tx_table["TX-SHUTLEAK"] + assert tx.begin_op() # an in-flight operation that will outlive the drain + + saved = ds_module.OP_DRAIN_TIMEOUT + ds_module.OP_DRAIN_TIMEOUT = 0.2 + shutter = threading.Thread(target=service.shutdown) + try: + shutter.start() + assert cb_entered.wait(5.0) # drain timed out; deferred settlement mid-window + + # the leaked operation could still emit: its id must NOT look free + with pytest.raises(ValueError, match="in-flight operations"): + service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-SHUTLEAK") + finally: + cb_release.set() + shutter.join(5.0) + ds_module.OP_DRAIN_TIMEOUT = saved + assert not shutter.is_alive() + + # still excluded after shutdown while the operation is in flight + with pytest.raises(ValueError, match="in-flight operations"): + service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-SHUTLEAK") + + # the operation exits: the id frees + tx.end_op() + assert service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-SHUTLEAK") == "TX-SHUTLEAK" + def test_waiter_during_shutdown_settlement_window_resolves_immediately(self): """shutdown() clears ownership atomically with the table teardown, so a get_transfer_waiter call landing in the deferred-settlement window finds the diff --git a/tests/unit_test/fuel/f3/streaming/transfer_waiter_test.py b/tests/unit_test/fuel/f3/streaming/transfer_waiter_test.py index a5a4aad6b2..a9aed72948 100644 --- a/tests/unit_test/fuel/f3/streaming/transfer_waiter_test.py +++ b/tests/unit_test/fuel/f3/streaming/transfer_waiter_test.py @@ -27,7 +27,7 @@ from nvflare.fuel.f3.streaming import download_service as ds_module from nvflare.fuel.f3.streaming.download_service import DownloadStatus, TransferWaiter from tests.unit_test.fuel.f3.streaming.download_test_utils import MockDownloadable, confirm_request -from tests.unit_test.fuel.f3.streaming.download_test_utils import make_confirm_test_service as _make_service +from tests.unit_test.fuel.f3.streaming.download_test_utils import make_service_no_monitor as _make_service from tests.unit_test.fuel.f3.streaming.download_test_utils import pull_to_terminal as _pull_to_terminal from tests.unit_test.fuel.f3.streaming.download_test_utils import run_monitor_once, serve_nonce From 49aa1eca75a7bc0549716d3ff38dd4d67352f37b Mon Sep 17 00:00:00 2001 From: YuanTingHsieh Date: Thu, 9 Jul 2026 20:05:35 -0700 Subject: [PATCH 18/20] Make leak markers two-state: release only after settlement completes Review round 8 falsified the shutdown pre-registration's release condition: a marker was releasable whenever _active_ops == 0, but settlement callbacks are not operations -- so with the old transaction blocked inside transaction_done_cb (zero ops), a same-id constructor could pop the marker mid-settlement, register, and then the old outcome_cb fired under an id naming the replacement. The marker is now two-state, as the reviewer prescribed: releasable only when the transaction's settlement has completed (a flag set in transaction_done's finally, after outcome recording -- nothing emits after it) AND no operations are in flight. All three release paths enforce the conjunction: the duplicate check in new_transaction, the per-terminator _update_leaked_ops sync, and the monitor reap. A registration that slips between the flag and the terminator's sync is legal (settlement is fully done), and the sync's identity check keeps it from ever touching the replacement's state. Pin drives the reviewer's exact clean-settlement sequence: shutdown with a gated transaction_done_cb and zero operations -> same-id registration rejected mid-settlement, accepted once settlement completes. --- nvflare/fuel/f3/streaming/download_service.py | 15 ++++--- .../f3/streaming/transfer_outcome_test.py | 44 +++++++++++++++++-- 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/nvflare/fuel/f3/streaming/download_service.py b/nvflare/fuel/f3/streaming/download_service.py index fcfc075bd2..4ddd9de4e6 100644 --- a/nvflare/fuel/f3/streaming/download_service.py +++ b/nvflare/fuel/f3/streaming/download_service.py @@ -704,6 +704,10 @@ def __init__( self._ops_cond = threading.Condition(threading.Lock()) self._active_ops = 0 self._ops_closed = False + # set in transaction_done's finally. A _leaked_ops marker is releasable only + # when this is True AND no operations are in flight: "no ops" alone does not + # mean quiet -- settlement callbacks can still be emitting. + self._settlement_complete = False # receivers that have issued at least one pull on ANY ref (monotonic; the # transaction-level PAYLOAD_ACQUIRED fact the acquire budget and the facade read), # and each receiver's last activity anywhere on the transaction (what the idle @@ -898,6 +902,7 @@ def transaction_done(self, status: str, on_outcome=None) -> TransferOutcome: # waiters pending if on_outcome and outcome is not None: _invoke_cb_safely(self.logger, f"outcome recording for tx {self.tid}", on_outcome, outcome) + self._settlement_complete = True return outcome @@ -1090,10 +1095,10 @@ def new_transaction( if leaked is not None: with leaked._ops_cond: still_in_flight = leaked._active_ops > 0 - if still_in_flight: + if still_in_flight or not leaked._settlement_complete: raise ValueError( - f"transaction id {tx.tid} still has in-flight operations from a previous " - f"attempt (its settlement drain timed out): use a new id" + f"transaction id {tx.tid} from a previous attempt has not fully terminated " + f"(settlement still running or operations still in flight): use a new id" ) cls._leaked_ops.pop(tx.tid, None) with cls._outcome_lock: @@ -1196,7 +1201,7 @@ def _update_leaked_ops(cls, tx: _Transaction): with tx._ops_cond: leaked = tx._active_ops > 0 with cls._tx_lock: - if leaked: + if leaked or not tx._settlement_complete: cls._leaked_ops[tx.tid] = tx elif cls._leaked_ops.get(tx.tid) is tx: cls._leaked_ops.pop(tx.tid, None) @@ -1204,7 +1209,7 @@ def _update_leaked_ops(cls, tx: _Transaction): @classmethod def _reap_leaked_ops(cls): with cls._tx_lock: - for tid in [t for t, tx in cls._leaked_ops.items() if tx._active_ops <= 0]: + for tid in [t for t, tx in cls._leaked_ops.items() if tx._settlement_complete and tx._active_ops <= 0]: cls._leaked_ops.pop(tid, None) @classmethod diff --git a/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py b/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py index f4be59cf4c..de305cc10e 100644 --- a/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py +++ b/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py @@ -437,7 +437,7 @@ def test_drain_leak_poisons_id_until_operation_exits(self): service._tx_outcomes["TX-LEAK"] = dataclasses.replace( recorded, timestamp=recorded.timestamp - service.TX_OUTCOME_TTL - 1 ) - with pytest.raises(ValueError, match="in-flight operations from a previous attempt"): + with pytest.raises(ValueError, match="has not fully terminated"): service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-LEAK") # the leaked operation exits: the id becomes usable @@ -533,7 +533,7 @@ def gated_done_cb(tid, status, base_objs, **kw): assert cb_entered.wait(5.0) # drain timed out; deferred settlement mid-window # the leaked operation could still emit: its id must NOT look free - with pytest.raises(ValueError, match="in-flight operations"): + with pytest.raises(ValueError, match="has not fully terminated"): service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-SHUTLEAK") finally: cb_release.set() @@ -542,13 +542,51 @@ def gated_done_cb(tid, status, base_objs, **kw): assert not shutter.is_alive() # still excluded after shutdown while the operation is in flight - with pytest.raises(ValueError, match="in-flight operations"): + with pytest.raises(ValueError, match="has not fully terminated"): service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-SHUTLEAK") # the operation exits: the id frees tx.end_op() assert service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-SHUTLEAK") == "TX-SHUTLEAK" + def test_shutdown_window_rejects_id_during_clean_settlement(self): + """P1 pin: the shutdown pre-registration used to be releasable whenever + _active_ops == 0 -- but settlement callbacks are not operations, so a same-id + constructor could pop the marker MID-transaction_done and register while the + old outcome_cb was still about to fire under the id. The marker now needs + settlement_complete AND zero operations to release.""" + from unittest.mock import Mock + + service = make_isolated_download_service() + service._tx_monitor = Mock() + cell = Mock() + cb_entered = threading.Event() + cb_release = threading.Event() + + def gated_done_cb(tid, status, base_objs, **kw): + cb_entered.set() + cb_release.wait(10.0) + + service.new_transaction( + cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-CLEANSET", transaction_done_cb=gated_done_cb + ) + # NOTE: no in-flight operation -- the clean-settlement case the ops-only + # release condition missed + shutter = threading.Thread(target=service.shutdown) + try: + shutter.start() + assert cb_entered.wait(5.0) # settlement mid-callbacks, _active_ops == 0 + + with pytest.raises(ValueError, match="has not fully terminated"): + service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-CLEANSET") + finally: + cb_release.set() + shutter.join(5.0) + assert not shutter.is_alive() + + # settlement complete, no operations: the id frees + assert service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-CLEANSET") == "TX-CLEANSET" + def test_waiter_during_shutdown_settlement_window_resolves_immediately(self): """shutdown() clears ownership atomically with the table teardown, so a get_transfer_waiter call landing in the deferred-settlement window finds the From c241b32567a085b6f8bc22b857381feabeeb1780 Mon Sep 17 00:00:00 2001 From: YuanTingHsieh Date: Thu, 9 Jul 2026 22:29:00 -0700 Subject: [PATCH 19/20] Install termination markers at unlink; stamp receipt TTL at recording Review round 9 composed two timing facts into an exposure: the outcome timestamp is captured before the settlement callbacks run, so a settlement slower than TX_OUTCOME_TTL recorded a receipt that was expired at birth -- and the inline expiry check then handed the id to a same-id constructor in the gap after ownership was consumed but before the terminator's marker sync ran. The old transaction's marker then landed on top of the live replacement while its leaked operation could still emit. Two changes, per the reviewer's recommendation: The termination marker is now installed by _delete_tx itself, atomically with the unlink, for EVERY terminator -- not just shutdown. One mechanism covers the entire termination window (settlement plus any drain-leaked tail) on every path; the ownership and receipt exclusions still exist but no longer carry any window alone. Shutdown's separate owner pre-registration loop became redundant and is removed. The overlap state can no longer form: the marker exists before ownership is consumed, and releases only on settlement-complete AND zero operations. Receipts are re-stamped at recording time: the TTL retention window starts when the receipt becomes queryable, matching the documented "kept 30 min" contract, so a slow settlement can never record an already-expired receipt. Pins: slow-settlement receipt is fresh from recording and excludes the id; the reviewer's aged-receipt-plus-leaked-op sequence stays excluded end to end with no overlap; the marker exists mid-settlement on the delete path (not just shutdown). --- nvflare/fuel/f3/streaming/download_service.py | 40 ++++--- .../f3/streaming/download_service_test.py | 10 +- .../f3/streaming/transfer_outcome_test.py | 103 +++++++++++++++++- 3 files changed, 132 insertions(+), 21 deletions(-) diff --git a/nvflare/fuel/f3/streaming/download_service.py b/nvflare/fuel/f3/streaming/download_service.py index 4ddd9de4e6..bba7b4ae7f 100644 --- a/nvflare/fuel/f3/streaming/download_service.py +++ b/nvflare/fuel/f3/streaming/download_service.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import dataclasses import functools import threading import time @@ -1018,10 +1019,13 @@ class DownloadService: # Waiters blocked on a transaction's terminal outcome (the awaitable facade). Guarded by # _outcome_lock; resolved inside _record_outcome so a waiter can never miss the outcome. _tx_waiters = {} - # Transactions whose settlement drain timed out with operations still in flight: - # the tx_id stays excluded from registration -- even past receipt expiry -- until - # every such operation exits (a leaked emission must never land under a recycled - # id). Reaped by the monitor and the registration check. Guarded by _tx_lock. + # Termination markers: installed by _delete_tx at unlink (every terminator) and + # releasable only when the transaction's settlement completed AND no in-flight + # operations remain -- the id stays excluded from registration through the whole + # termination window and any drain-leaked tail, even past receipt expiry (a + # leaked emission must never land under a recycled id). Released by + # _update_leaked_ops, the registration check, or the monitor reap. + # Guarded by _tx_lock. _leaked_ops = {} _outcome_lock = threading.Lock() TX_OUTCOME_TTL = 1800.0 @@ -1165,13 +1169,9 @@ def shutdown(cls): cls._finished_refs.clear() with cls._outcome_lock: - # Pre-register every registered owner (live and mid-settlement) as a - # potential leak BEFORE clearing its ownership/receipt exclusions: the - # deferred settlements below run outside the locks, and a drain-leaked - # operation's id must never look free in that window. Post-settlement - # _update_leaked_ops releases the ids whose operations drained. - for owner in cls._outcome_owners.values(): - cls._leaked_ops[owner.tid] = owner + # every id being torn down already carries its termination marker + # (installed by _delete_tx at unlink), so clearing the ownership and + # receipt exclusions here cannot expose an id mid-settlement. # Clearing ownership is what stops recording: a settlement mid-flight # on another thread finds its entry gone at _record_outcome and drops. cls._tx_outcomes.clear() @@ -1193,11 +1193,9 @@ def shutdown(cls): @classmethod def _update_leaked_ops(cls, tx: _Transaction): - """Called by every terminator after settlement: keep the id excluded while - drain-leaked operations are in flight, release the exclusion once none are. - shutdown() PRE-registers its transactions under the locks -- it clears the - ownership and receipt exclusions up front, so waiting until after settlement - would leave a window where a leaked operation's id looks free.""" + """Called by every terminator after settlement: release the termination marker + (installed at unlink) once settlement completed and no operations remain, or + sustain it while a drain-leaked operation is still in flight.""" with tx._ops_cond: leaked = tx._active_ops > 0 with cls._tx_lock: @@ -1215,6 +1213,12 @@ def _reap_leaked_ops(cls): @classmethod def _delete_tx(cls, tx: _Transaction, tombstone_finished_refs: bool = False): cls._tx_table.pop(tx.tid, None) + # install the termination marker at unlink, for EVERY terminator: it covers + # the whole settlement window and the leaked-operation tail in one mechanism, + # releasable only when settlement completed AND no operations are in flight + # (_update_leaked_ops / the duplicate check / the monitor reap). Ownership and + # receipt exclusions still exist but no longer carry the window alone. + cls._leaked_ops[tx.tid] = tx # remove all refs now = time.time() if tombstone_finished_refs else None @@ -1269,6 +1273,10 @@ def _record_outcome(cls, outcome: TransferOutcome, tx: _Transaction): # ownership consumed (prior record) or cleared (shutdown): stale, drop return cls._outcome_owners.pop(outcome.tx_id, None) + # re-stamp at recording time: the TTL retention window starts when the + # receipt becomes queryable, not when the verdict was computed -- a slow + # settlement must not record a receipt that is already expired + outcome = dataclasses.replace(outcome, timestamp=time.time()) cls._tx_outcomes[outcome.tx_id] = outcome # resolve the awaitable facade: waiters are TransferWaiter objects (no user code # runs in _resolve), so setting them under the lock is safe and race-free diff --git a/tests/unit_test/fuel/f3/streaming/download_service_test.py b/tests/unit_test/fuel/f3/streaming/download_service_test.py index e37059eb29..8cdff791d8 100644 --- a/tests/unit_test/fuel/f3/streaming/download_service_test.py +++ b/tests/unit_test/fuel/f3/streaming/download_service_test.py @@ -13,6 +13,7 @@ # limitations under the License. import threading +import time from typing import Any from unittest.mock import Mock, patch @@ -28,6 +29,7 @@ ProduceRC, TransactionDoneStatus, ) +from nvflare.fuel.f3.streaming.transfer_outcome import compute_transfer_outcome from nvflare.fuel.utils.network_utils import get_open_ports from tests.unit_test.fuel.f3.streaming.download_test_utils import ( MockDownloadable, @@ -529,12 +531,12 @@ def test_stale_outcome_dropped_after_ownership_cleared(self): # sanity: the owning transaction still records (guard does not over-drop) live_tx = Mock() live_tx.tid = "tx-live" - live_outcome = Mock() - live_outcome.tx_id = "tx-live" - live_outcome.expired.return_value = False + live_outcome = compute_transfer_outcome("tx-live", TransactionDoneStatus.FINISHED, 1, [], time.time()) service._outcome_owners["tx-live"] = live_tx service._record_outcome(live_outcome, tx=live_tx) - assert service.get_transaction_outcome("tx-live") is live_outcome + recorded = service.get_transaction_outcome("tx-live") + # recording re-stamps the receipt (TTL starts at recording), so compare identity-free + assert recorded is not None and recorded.tx_id == "tx-live" and recorded.done_status == live_outcome.done_status def test_raising_download_callbacks_do_not_break_serving_path(self): """Raising downloaded_to_one/downloaded_to_all must not propagate into serving. diff --git a/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py b/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py index de305cc10e..c0562b433d 100644 --- a/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py +++ b/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py @@ -466,7 +466,7 @@ def cb(outcome): service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-SELF", outcome_cb=cb) service.delete_transaction("TX-SELF") - assert len(errors) == 1 and "already in use" in errors[0] + assert len(errors) == 1 and "has not fully terminated" in errors[0] assert created == ["TX-OTHER"] with service._tx_lock: assert "TX-OTHER" in service._tx_table @@ -549,6 +549,107 @@ def gated_done_cb(tid, status, base_objs, **kw): tx.end_op() assert service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-SHUTLEAK") == "TX-SHUTLEAK" + def test_receipt_ttl_starts_at_recording_not_at_verdict(self): + """P1 pin: the outcome timestamp is captured before the settlement callbacks + run -- a settlement slower than TX_OUTCOME_TTL used to record a receipt that + was EXPIRED at birth, which the inline expiry then handed to a same-id + constructor in the gap before the terminator's marker sync. Receipts are now + re-stamped at recording time ("kept 30 min" from recording).""" + from unittest.mock import Mock + + service = make_isolated_download_service() + service._tx_monitor = Mock() + service.TX_OUTCOME_TTL = 0.2 + cell = Mock() + + def slow_done_cb(tid, status, base_objs, **kw): + time.sleep(0.4) # slower than the (patched) TTL + + service.new_transaction( + cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-AGED", transaction_done_cb=slow_done_cb + ) + service.delete_transaction("TX-AGED") + + # the receipt is fresh from RECORDING: queryable, and it excludes the id + assert service.get_transaction_outcome("TX-AGED") is not None + with pytest.raises(ValueError, match="already in use"): + service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-AGED") + + def test_aged_receipt_with_leaked_op_cannot_expose_id(self): + """P1 pin (reviewer's exact sequence): slow settlement past the TTL WITH a + drain-leaked operation -- the id must stay excluded end to end; before the + marker moved to _delete_tx, a constructor could slip between ownership + consumption and the marker sync, and the old marker then overlapped the live + replacement.""" + from unittest.mock import Mock + + import nvflare.fuel.f3.streaming.download_service as ds_module + + service = make_isolated_download_service() + service._tx_monitor = Mock() + service.TX_OUTCOME_TTL = 0.2 + cell = Mock() + service.new_transaction( + cell=cell, + timeout=10.0, + num_receivers=1, + tx_id="TX-AGEDLEAK", + transaction_done_cb=lambda tid, status, base_objs, **kw: time.sleep(0.4), + ) + with service._tx_lock: + tx = service._tx_table["TX-AGEDLEAK"] + assert tx.begin_op() # the leak + + saved = ds_module.OP_DRAIN_TIMEOUT + ds_module.OP_DRAIN_TIMEOUT = 0.1 + try: + service.delete_transaction("TX-AGEDLEAK") + finally: + ds_module.OP_DRAIN_TIMEOUT = saved + + # settlement complete, receipt aged past the (patched) TTL -- the leaked op + # still excludes the id, and no overlap state can form + time.sleep(0.25) + with pytest.raises(ValueError, match="has not fully terminated"): + service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-AGEDLEAK") + with service._tx_lock: + assert service._leaked_ops.get("TX-AGEDLEAK") is tx + + tx.end_op() + assert service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-AGEDLEAK") == "TX-AGEDLEAK" + + def test_marker_installed_at_unlink_for_every_terminator(self): + """The termination marker is installed by _delete_tx itself, so EVERY + terminator's id is covered for its whole settlement window -- not just + shutdown's.""" + from unittest.mock import Mock + + service = make_isolated_download_service() + service._tx_monitor = Mock() + cell = Mock() + cb_entered = threading.Event() + cb_release = threading.Event() + + def gated_done_cb(tid, status, base_objs, **kw): + cb_entered.set() + cb_release.wait(10.0) + + service.new_transaction( + cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-UNLINK", transaction_done_cb=gated_done_cb + ) + deleter = threading.Thread(target=service.delete_transaction, args=("TX-UNLINK",)) + try: + deleter.start() + assert cb_entered.wait(5.0) + with service._tx_lock: + assert "TX-UNLINK" in service._leaked_ops, "marker must exist mid-settlement on the delete path" + with pytest.raises(ValueError, match="has not fully terminated"): + service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-UNLINK") + finally: + cb_release.set() + deleter.join(5.0) + assert not deleter.is_alive() + def test_shutdown_window_rejects_id_during_clean_settlement(self): """P1 pin: the shutdown pre-registration used to be releasable whenever _active_ops == 0 -- but settlement callbacks are not operations, so a same-id From 920e913b5149f01451e57ae13fa9a110254b4953 Mon Sep 17 00:00:00 2001 From: YuanTingHsieh Date: Thu, 9 Jul 2026 23:53:36 -0700 Subject: [PATCH 20/20] Record a fail-closed verdict even if outcome computation raises Greptile flagged that the settlement finally's stated invariant -- no exception may leave the outcome unrecorded and waiters pending -- was not upheld if compute_transfer_outcome itself raised: outcome stayed None, recording was skipped, ownership was never consumed, and a waiter parked on the id hung until shutdown (the dead-owner reclaim that once bounded this left with the attempt-scoped rework). Unrealistic in practice (dataclass construction over simple types), but the invariant is now enforced rather than assumed: the finally builds a fail-closed fallback verdict (empty refs certify nothing) whenever outcome is None, recording always runs, and settlement never propagates an exception to its caller (a raise on the monitor thread would have killed the monitor and skipped the terminator's marker sync). Also from the same review: _reap_leaked_ops now reads the op count and settlement flag under the per-transaction gate lock (_tx_lock -> _ops_cond, the established order) instead of relying on CPython read atomicity. Pin: compute_transfer_outcome patched to raise -> waiter still resolves with a fail-closed outcome, ownership consumed, no exception escapes the terminator. --- nvflare/fuel/f3/streaming/download_service.py | 26 ++++++++++++-- .../f3/streaming/transfer_outcome_test.py | 35 +++++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/nvflare/fuel/f3/streaming/download_service.py b/nvflare/fuel/f3/streaming/download_service.py index bba7b4ae7f..f27d7c15c3 100644 --- a/nvflare/fuel/f3/streaming/download_service.py +++ b/nvflare/fuel/f3/streaming/download_service.py @@ -898,10 +898,26 @@ def transaction_done(self, status: str, on_outcome=None) -> TransferOutcome: # release is guarded so one raising release() cannot skip its siblings for ref in refs: _invoke_cb_safely(self.logger, f"release of {type(ref.obj)} for tx {self.tid}", ref.obj.release) + except Exception as ex: + # the ceremony must never raise: a propagating exception would kill the + # monitor thread and skip the terminator's marker sync (the finally below + # still records a fail-closed verdict) + self.logger.error(f"settlement of tx {self.tid} raised: {secure_format_exception(ex)}") finally: # in a finally: no exception above may leave the outcome unrecorded and # waiters pending - if on_outcome and outcome is not None: + if outcome is None: + # outcome computation itself raised: record a fail-closed verdict so + # ownership is consumed and waiters resolve (empty refs cannot certify + # anything and exercises only trivial construction paths) + outcome = compute_transfer_outcome( + tx_id=self.tid, + done_status=status, + num_receivers=self.num_receivers, + refs=[], + timestamp=time.time(), + ) + if on_outcome: _invoke_cb_safely(self.logger, f"outcome recording for tx {self.tid}", on_outcome, outcome) self._settlement_complete = True @@ -1207,7 +1223,13 @@ def _update_leaked_ops(cls, tx: _Transaction): @classmethod def _reap_leaked_ops(cls): with cls._tx_lock: - for tid in [t for t, tx in cls._leaked_ops.items() if tx._settlement_complete and tx._active_ops <= 0]: + released = [] + for tid, tx in cls._leaked_ops.items(): + with tx._ops_cond: # _tx_lock -> _ops_cond is the established order + done = tx._settlement_complete and tx._active_ops <= 0 + if done: + released.append(tid) + for tid in released: cls._leaked_ops.pop(tid, None) @classmethod diff --git a/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py b/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py index c0562b433d..df48579462 100644 --- a/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py +++ b/tests/unit_test/fuel/f3/streaming/transfer_outcome_test.py @@ -549,6 +549,41 @@ def gated_done_cb(tid, status, base_objs, **kw): tx.end_op() assert service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-SHUTLEAK") == "TX-SHUTLEAK" + def test_outcome_computation_failure_still_records_fail_closed(self): + """Greptile pin: if compute_transfer_outcome itself raises, settlement used to + skip recording entirely -- ownership stayed consumed-never, and a waiter parked + on the id hung until shutdown. The finally now records a fail-closed fallback + verdict (empty refs certify nothing), so waiters always resolve.""" + from unittest.mock import Mock, patch + + import nvflare.fuel.f3.streaming.download_service as ds_module + + service = make_isolated_download_service() + service._tx_monitor = Mock() + cell = Mock() + service.new_transaction(cell=cell, timeout=10.0, num_receivers=1, tx_id="TX-COMPUTE") + obj = _stub_obj() + service.add_object("TX-COMPUTE", obj) + waiter = service.get_transfer_waiter("TX-COMPUTE") + + real_compute = ds_module.compute_transfer_outcome + calls = {"n": 0} + + def exploding_first_call(*args, **kwargs): + calls["n"] += 1 + if calls["n"] == 1: + raise RuntimeError("outcome computation blew up") + return real_compute(*args, **kwargs) + + with patch.object(ds_module, "compute_transfer_outcome", side_effect=exploding_first_call): + service.delete_transaction("TX-COMPUTE") + + outcome = waiter.wait(timeout=5.0) + assert outcome is not None, "waiters must resolve even when outcome computation raises" + assert not outcome.completed, "the fallback verdict must be fail-closed" + with service._outcome_lock: + assert "TX-COMPUTE" not in service._outcome_owners, "ownership must be consumed" + def test_receipt_ttl_starts_at_recording_not_at_verdict(self): """P1 pin: the outcome timestamp is captured before the settlement callbacks run -- a settlement slower than TX_OUTCOME_TTL used to record a receipt that