Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions dlrover/python/common/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,10 @@ class NodeEventType(object):
NODE_CHECK_FAILED = "NODE_CHECK_FAILED"
MASTER_CONNECTION_FAILED = "MASTER_CONNECTION_FAILED"
WAIT_PRE_CHECK = "WAIT_PRE_CHECK"
# Synthesized by the master when a pod carries the configured
# eviction label. It is not a k8s event type and bypasses the node
# state machine.
EVICTION = "EVICTION"


class PendingTimeoutStrategyType(object):
Expand Down
11 changes: 11 additions & 0 deletions dlrover/python/common/global_context.py
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,17 @@ def __init__(self):
self.dynamic_failover_extension = None
self.enable_dashboard = False
self.dashboard_port = 8080
# Save checkpoint before pod eviction (wired by the job launcher,
# e.g. dlrover-addon sets these from its --save-ckpt-on-eviction
# master arguments). An empty eviction label key disables the
# label detection in the pod watcher.
self.save_ckpt_on_eviction = False
self.eviction_pod_label_key = ""
self.eviction_pod_label_value = ""
self.eviction_save_timeout = 900
# The checkpoint save directory where the urgent-save marker is
# written for eviction-triggered saves.
self.urgent_save_dir = ""

def set_params_from_brain(self):
self.train_speed_record_num = self.get_param_value_from_brain(
Expand Down
5 changes: 5 additions & 0 deletions dlrover/python/common/node.py
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,10 @@ def __init__(
self.group = node_group
self.group_size = node_group_size
self.group_id = node_group_id
# True when the pod carries the configured eviction label,
# detected by the node watcher. It does not participate in the
# node state machine.
self.is_evicted = False

def exited(self):
return self.status in [
Expand Down Expand Up @@ -307,6 +311,7 @@ def generate_relaunch_node(self, new_id, new_name=None):
new_node.relaunchable = True
new_node.init_time = time.time()
new_node.reported_status = ("", 0)
new_node.is_evicted = False

return new_node

Expand Down
20 changes: 20 additions & 0 deletions dlrover/python/elastic_agent/torch/training.py
Original file line number Diff line number Diff line change
Expand Up @@ -1211,6 +1211,26 @@ def _initialize_workers(self, worker_group, max_errors=3):
def _stop_workers(
self, worker_group: WorkerGroup, is_restart=False, timeout=300
) -> None:
# Stopping the workers (membership-change failover or a failure
# restart) kills any training process that might be in the middle
# of an urgent save. The rank-0 listener removes the marker and
# reports the failure so the master stops waiting immediately.
# No-op when no save is in flight (e.g. the normal post-save FO
# of the eviction flow, where SAVED was already reported).
if self._urgent_save_listener is not None:
try:
aborted = self._urgent_save_listener.abort_in_flight(
"workers stopped for restart/failover"
)
if aborted:
logger.info(
"Aborted the in-flight urgent save because the "
"workers are stopping."
)
except Exception as e:
logger.warning(
f"Failed to abort the in-flight urgent save: {e}"
)
try:
signal.signal(signal.SIGALRM, self._stop_timeout_handler)
signal.alarm(timeout)
Expand Down
56 changes: 53 additions & 3 deletions dlrover/python/elastic_agent/torch/urgent_save_listener.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,9 @@
starts up so a request left by a previous run cannot fire again.

There is no timeout and no detection logic here by design: the observer
waits until the tracker flips; if the training hangs or is relaunched, the
regular dlrover mechanisms (node check / hang diagnosis, independent of
this protocol) take over and the restart clears everything.
waits until the tracker flips. If the workers are stopped mid-save
(failover/restart), ``abort_in_flight`` removes the marker and publishes an
error result so the master stops waiting immediately.
"""

import json
Expand Down Expand Up @@ -74,6 +74,11 @@ def __init__(self, master_client, poll_interval_seconds=5.0):
self._node_name = os.environ.get("POD_NAME") or os.environ.get(
"HOSTNAME", ""
)
# The accepted request whose save has not been confirmed yet:
# (cmd_ts, save_dir). Used to abort the request when the local
# workers are stopped for a failover/restart mid-save.
self._in_flight_lock = threading.Lock()
self._in_flight = None

def run(self):
logger.info(
Expand Down Expand Up @@ -144,6 +149,8 @@ def _maybe_save(self):
self._publish(
cmd_ts, STATUS_RECEIVED, "marker written at {}".format(marker)
)
with self._in_flight_lock:
self._in_flight = (cmd_ts, save_dir)
except Exception as e: # noqa: BLE001 - report instead of dying
logger.error(
"Urgent-save: fail to write the marker in %s: %s", save_dir, e
Expand Down Expand Up @@ -176,11 +183,28 @@ def _observe(self, cmd_ts, save_dir, base_iteration, marker):
)
while True:
time.sleep(self._poll_interval)
with self._in_flight_lock:
in_flight = self._in_flight
if in_flight is None or in_flight[0] != cmd_ts:
# The request was aborted (workers stopped mid-save for a
# failover/restart) or superseded by a newer request.
logger.info(
"Urgent-save: observer of request %s exits because the "
"request is no longer in flight.",
cmd_ts,
)
return
iteration, tracker_mtime = read_tracker(save_dir)
flipped = iteration is not None and (
base_iteration is None or iteration > base_iteration
)
if flipped and tracker_mtime > marker_mtime:
# Clear the in-flight state BEFORE publishing so a
# concurrent abort_in_flight() cannot overwrite this
# success with an error.
with self._in_flight_lock:
if self._in_flight and self._in_flight[0] == cmd_ts:
self._in_flight = None
self._publish(
cmd_ts,
STATUS_SAVED,
Expand All @@ -204,3 +228,29 @@ def _publish(self, cmd_ts, status, message="", step=None):
logger.error(
"Urgent-save: fail to publish '%s' result: %s", status, e
)

def abort_in_flight(self, reason):
"""Abort the in-flight urgent save when workers stop mid-save.

Called by the agent when it stops the local workers for a
failover/restart: the dying training processes can no longer
finish the save, so the marker is removed (a restarted training
must not consume a stale request) and an error result is
published so the master stops waiting immediately instead of
waiting for its deadline. Returns True if a request was aborted.
"""
with self._in_flight_lock:
in_flight = self._in_flight
self._in_flight = None
if in_flight is None:
return False
cmd_ts, save_dir = in_flight
if remove_marker(save_dir):
logger.info(
"Urgent-save: marker removed on abort of request %s: %s",
cmd_ts,
marker_path(save_dir),
)
self._publish(cmd_ts, STATUS_ERROR, "aborted: {}".format(reason))
logger.warning("Urgent-save: request %s aborted (%s).", cmd_ts, reason)
return True
Loading
Loading