From 2ab3307f4a2e0ea50e9edce7e7455a98749f51a6 Mon Sep 17 00:00:00 2001 From: TianyeDong Date: Sat, 20 Jun 2026 19:27:29 -0400 Subject: [PATCH] fix(miles): track pause/continue refs for timeout cancellation The atomic sync unit records every Ray ObjectRef it issues so a wait_for timeout can fan out ray.cancel(force=True). The pause_generation and continue_generation refs around the finalize fan-out were not recorded, so a timeout landing in that window left those RPCs running on the engines. Co-Authored-By: Claude Opus 4.8 --- rlix/pipeline/miles_model_update_service.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rlix/pipeline/miles_model_update_service.py b/rlix/pipeline/miles_model_update_service.py index 6371f38..9208dcc 100644 --- a/rlix/pipeline/miles_model_update_service.py +++ b/rlix/pipeline/miles_model_update_service.py @@ -300,6 +300,7 @@ async def _run_atomic_unit( # so the subsequent flush_cache returns 200 immediately. try: pause_refs = [h.pause_generation.remote(mode="retract") for h in handles.values()] + inflight_refs.extend(pause_refs) await _ray_get(pause_refs) except Exception as exc: # noqa: BLE001 logger.warning( @@ -313,6 +314,7 @@ async def _run_atomic_unit( # (4.post) resume the engines so subsequent generate calls work. try: cont_refs = [h.continue_generation.remote() for h in handles.values()] + inflight_refs.extend(cont_refs) await _ray_get(cont_refs) except Exception as exc: # noqa: BLE001 logger.warning(