HDDS-15715. Add helpful CLI flags to finalization CLIs

hadoop-hdds/interface-client/src/main/proto/hdds.proto

@@ -433,6 +433,10 @@ message UpgradeStatus {
   optional int32 numDatanodesFinalized = 2;
   optional int32 numDatanodesTotal = 3;
   optional bool shouldFinalize = 4;
+  optional uint32 scmSoftwareVersion = 5;
+  optional uint32 scmApparentVersion = 6;
+  optional uint32 minDatanodeApparentVersion = 7;
+  optional uint32 maxDatanodeApparentVersion = 8;
 }
 
 /**

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java

@@ -154,6 +154,8 @@ default int getAllNodeCount() {
   default DatanodeFinalizationCounts getDatanodeFinalizationCounts() {
     int finalizedNodes = 0;
     int totalHealthyNodes = 0;
+    int minApparentVersion = 0;
+    int maxApparentVersion = 0;
 
     for (DatanodeDetails dn : getAllNodes()) {
       try {
@@ -180,6 +182,14 @@ default DatanodeFinalizationCounts getDatanodeFinalizationCounts() {
         ComponentVersion dnApparentVersion = datanodeInfo.getLastKnownApparentVersion();
         ComponentVersion dnSoftwareVersion = datanodeInfo.getLastKnownSoftwareVersion();
 
+        int dnApparentVersionInt = dnApparentVersion.serialize();
+        if (dnApparentVersionInt < minApparentVersion) {
+          minApparentVersion = dnApparentVersionInt;
+        }
+        if (dnApparentVersionInt > maxApparentVersion) {
+          maxApparentVersion = dnApparentVersionInt;
+        }
+
         if (!dnApparentVersion.equals(dnSoftwareVersion)) {
           // Datanode has not yet finalized
           LOG.debug("Datanode {} has not yet finalized: apparent version={}, software version={}",
@@ -194,7 +204,8 @@ default DatanodeFinalizationCounts getDatanodeFinalizationCounts() {
       }
     }
 
-    return new DatanodeFinalizationCounts(finalizedNodes, totalHealthyNodes);
+    return new DatanodeFinalizationCounts(finalizedNodes, totalHealthyNodes,
+        minApparentVersion, maxApparentVersion);
   }
 
   /**
@@ -489,11 +500,17 @@ default void removeNode(DatanodeDetails datanodeDetails) throws NodeNotFoundExce
   final class DatanodeFinalizationCounts {
     private final int numFinalizedDatanodes;
     private final int totalHealthyDatanodes;
+    private final int minApparentVersion;
+    private final int maxApparentVersion;
 
     public DatanodeFinalizationCounts(int numFinalizedDatanodes,
-                                      int totalHealthyDatanodes) {
+                                      int totalHealthyDatanodes,
+                                      int minApparentVersion,
+                                      int maxApparentVersion) {
       this.numFinalizedDatanodes = numFinalizedDatanodes;
       this.totalHealthyDatanodes = totalHealthyDatanodes;
+      this.minApparentVersion = minApparentVersion;
+      this.maxApparentVersion = maxApparentVersion;
     }
 
     public int getNumFinalizedDatanodes() {
@@ -507,5 +524,13 @@ public int getTotalHealthyDatanodes() {
     public boolean allNodesFinalized() {
       return numFinalizedDatanodes == totalHealthyDatanodes;
     }
+
+    public Integer getMinApparentVersion() {
+      return minApparentVersion;
+    }
+
+    public Integer getMaxApparentVersion() {
+      return maxApparentVersion;
+    }
   }
 }

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java

@@ -1226,6 +1226,10 @@ public HddsProtos.UpgradeStatus queryUpgradeStatus() throws IOException {
           .setNumDatanodesFinalized(finalizedDatanodes)
           .setNumDatanodesTotal(healthyDatanodes)
           .setShouldFinalize(shouldFinalize)
+          .setScmSoftwareVersion(scm.getVersionManager().getSoftwareVersion().serialize())
+          .setScmApparentVersion(scm.getVersionManager().getApparentVersion().serialize())
+          .setMinDatanodeApparentVersion(datanodeFinalizationCounts.getMinApparentVersion())
+          .setMaxDatanodeApparentVersion(datanodeFinalizationCounts.getMaxApparentVersion())
           .build();
 
       AUDIT.logReadSuccess(buildAuditMessageForSuccess(SCMAction.QUERY_UPGRADE_STATUS, null));

hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/FinalizeSubCommand.java

@@ -17,13 +17,17 @@
 
 package org.apache.hadoop.ozone.admin.upgrade;
 
+import com.google.common.annotations.VisibleForTesting;
 import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
 import org.apache.hadoop.hdds.cli.AbstractSubcommand;
 import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.ozone.OzoneManagerVersion;
 import org.apache.hadoop.ozone.admin.om.OmAddressOptions;
 import org.apache.hadoop.ozone.client.rpc.RpcClient;
 import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.QueryUpgradeStatusResponse;
 import picocli.CommandLine;
 
 /**
@@ -37,9 +41,19 @@
 )
 public class FinalizeSubCommand extends AbstractSubcommand implements Callable<Integer> {
 
+  /** Poll cadence used by {@code --wait}. Overridable from tests via {@link #setPollIntervalMillis(long)}. */
+  private long pollIntervalMillis = TimeUnit.SECONDS.toMillis(5);
+
   @CommandLine.Mixin
   private OmAddressOptions.OptionalServiceIdOrHostMixin omAddressOptions;
 
+  @CommandLine.Option(names = {"--wait"},
+      defaultValue = "false",
+      description = "After initiating finalization, poll the cluster status until the entire cluster (OM, SCM, "
+          + "and all healthy datanodes) is finalized. Interrupt with Ctrl-C to stop waiting; finalization "
+          + "continues on the server.")
+  private boolean wait;
+
   @Override
   public Integer call() throws Exception {
     try (OzoneManagerProtocol client = getClient()) {
@@ -51,11 +65,65 @@ public Integer call() throws Exception {
       }
       client.finalizeUpgrade();
       out().println("Cluster finalization has been started. Monitor progress with `ozone admin upgrade status`");
+
+      if (wait) {
+        return waitForFinalization(client);
+      }
     }
     return 0;
   }
 
+  /**
+   * Polls the cluster status until OM, SCM and all healthy datanodes report finalized, the operator
+   * interrupts the command, or an RPC fails. {@code --wait} is safe to re-run: if the cluster is already
+   * finalized, the first poll returns done and the command exits 0.
+   */
+  private int waitForFinalization(OzoneManagerProtocol client) {
+    while (true) {
+      try {
+        Thread.sleep(pollIntervalMillis);
+      } catch (InterruptedException e) {
+        Thread.currentThread().interrupt();
+        out().println("Waiting interrupted. Use `ozone admin upgrade status` to monitor progress.");
+        return 0;
+      }
+      QueryUpgradeStatusResponse status;
+      try {
+        status = client.queryUpgradeStatus();
+      } catch (Exception e) {
+        err().println("Failed to query upgrade status: " + e.getMessage()
+            + ". Use `ozone admin upgrade status` to monitor progress.");
+        return 1;
+      }
+      if (isVerbose()) {
+        StatusSubCommand.printVerbose(status, out());
+      } else {
+        HddsProtos.UpgradeStatus hdds = status.getHddsStatus();
+        out().printf("Waiting for finalization: OM=%s, SCM=%s, datanodes=%d/%d%n",
+            status.getOmFinalized(), hdds.getScmFinalized(),
+            hdds.getNumDatanodesFinalized(), hdds.getNumDatanodesTotal());
+      }
+      out().flush();
+      if (isClusterFinalized(status)) {
+        out().println("Finalization complete.");
+        return 0;
+      }
+    }
+  }
+
+  static boolean isClusterFinalized(QueryUpgradeStatusResponse status) {
+    HddsProtos.UpgradeStatus hdds = status.getHddsStatus();
+    return status.getOmFinalized()
+        && hdds.getScmFinalized()
+        && hdds.getNumDatanodesFinalized() == hdds.getNumDatanodesTotal();
+  }
+
   protected OzoneManagerProtocol getClient() throws Exception {
     return omAddressOptions.newClient();
   }
+
+  @VisibleForTesting
+  void setPollIntervalMillis(long millis) {
+    this.pollIntervalMillis = millis;
+  }
 }

hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/StatusSubCommand.java

@@ -17,14 +17,18 @@
 
 package org.apache.hadoop.ozone.admin.upgrade;
 
+import java.io.PrintWriter;
 import java.util.concurrent.Callable;
+import org.apache.hadoop.hdds.HDDSVersion;
 import org.apache.hadoop.hdds.cli.AbstractSubcommand;
 import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.server.JsonUtils;
 import org.apache.hadoop.ozone.OzoneManagerVersion;
 import org.apache.hadoop.ozone.admin.om.OmAddressOptions;
 import org.apache.hadoop.ozone.client.rpc.RpcClient;
 import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.QueryUpgradeStatusResponse;
 import picocli.CommandLine;
 
 /**
@@ -42,6 +46,11 @@ public class StatusSubCommand extends AbstractSubcommand implements Callable<Int
   @CommandLine.Mixin
   private OmAddressOptions.OptionalServiceIdOrHostMixin omAddressOptions;
 
+  @CommandLine.Option(names = {"--json"},
+      defaultValue = "false",
+      description = "Format output as JSON.")
+  private boolean json;
+
   @Override
   public Integer call() throws Exception {
     try (OzoneManagerProtocol client = getClient()) {
@@ -51,20 +60,150 @@ public Integer call() throws Exception {
             "`ozone admin scm finalizationstatus` and `ozone admin om finalizationstatus`");
         return 1;
       }
-      OzoneManagerProtocolProtos.QueryUpgradeStatusResponse status = client.queryUpgradeStatus();
-
+      QueryUpgradeStatusResponse status = client.queryUpgradeStatus();
 
-      out().println("Upgrade status:");
-      out().println("    OM Finalized? " + status.getOmFinalized());
-      out().println("    SCM Finalized? " + status.getHddsStatus().getScmFinalized());
-      out().println("    Datanodes finalized: " + status.getHddsStatus().getNumDatanodesFinalized()
-          + "/" + status.getHddsStatus().getNumDatanodesTotal());
+      if (json) {
+        out().println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(UpgradeStatusDto.from(status)));
+      } else if (isVerbose()) {
+        printVerbose(status, out());
+      } else {
+        printBasic(status, out());
+      }
     }
     return 0;
   }
 
+  /** Basic, non-verbose human-readable status. */
+  static void printBasic(QueryUpgradeStatusResponse status, PrintWriter out) {
+    out.println("Upgrade status:");
+    out.println("    OM Finalized? " + status.getOmFinalized());
+    out.println("    SCM Finalized? " + status.getHddsStatus().getScmFinalized());
+    out.println("    Datanodes finalized: " + status.getHddsStatus().getNumDatanodesFinalized()
+        + "/" + status.getHddsStatus().getNumDatanodesTotal());
+  }
+
+  /**
+   * Verbose human-readable status that includes the software/apparent versions reported by OM
+   * and SCM, and the range of apparent versions across healthy datanodes (last known from heartbeat).
+   * Version fields that are not populated by the server (e.g. when running against an older
+   * cluster that does not yet ship these fields) are reported as "(unavailable)".
+   */
+  static void printVerbose(QueryUpgradeStatusResponse status, PrintWriter out) {
+    HddsProtos.UpgradeStatus hdds = status.getHddsStatus();
+    out.println("Upgrade status:");
+    out.println("    OM Finalized?            " + status.getOmFinalized());
+    out.println("    OM Software Version:     " + formatOmVersion(status.hasOmSoftwareVersion(),
+        status.getOmSoftwareVersion()));
+    out.println("    OM Apparent Version:     " + formatOmVersion(status.hasOmApparentVersion(),
+        status.getOmApparentVersion()));
+    out.println("    SCM Finalized?           " + hdds.getScmFinalized());
+    out.println("    SCM Software Version:    " + formatHddsVersion(hdds.hasScmSoftwareVersion(),
+        hdds.getScmSoftwareVersion()));
+    out.println("    SCM Apparent Version:    " + formatHddsVersion(hdds.hasScmApparentVersion(),
+        hdds.getScmApparentVersion()));
+    out.println("    Datanodes finalized:     " + hdds.getNumDatanodesFinalized() + "/" + hdds.getNumDatanodesTotal());
+    out.println("    Min DN Apparent Version: " + formatHddsVersion(hdds.hasMinDatanodeApparentVersion(),
+        hdds.getMinDatanodeApparentVersion()));
+    out.println("    Max DN Apparent Version: " + formatHddsVersion(hdds.hasMaxDatanodeApparentVersion(),
+        hdds.getMaxDatanodeApparentVersion()));
+  }
+
+  private static String formatOmVersion(boolean present, int serialized) {
+    return present ? OzoneManagerVersion.deserialize(serialized).toString() : "(unavailable)";
+  }
+
+  private static String formatHddsVersion(boolean present, int serialized) {
+    return present ? HDDSVersion.deserialize(serialized).toString() : "(unavailable)";
+  }
+
   protected OzoneManagerProtocol getClient() throws Exception {
     return omAddressOptions.newClient();
   }
 
+  /**
+   * JSON-friendly DTO mirroring {@link QueryUpgradeStatusResponse}. Optional version fields use boxed types
+   * so that values absent from the server's response are omitted from the rendered JSON via
+   * {@link JsonUtils}' NON_NULL inclusion policy.
+   */
+  public static final class UpgradeStatusDto {
+    private boolean omFinalized;
+    private String omSoftwareVersion;
+    private String omApparentVersion;
+    private boolean scmFinalized;
+    private String scmSoftwareVersion;
+    private String scmApparentVersion;
+    private int datanodesFinalized;
+    private int datanodesTotal;
+    private String minDatanodeApparentVersion;
+    private String maxDatanodeApparentVersion;
+
+    public static UpgradeStatusDto from(QueryUpgradeStatusResponse status) {
+      HddsProtos.UpgradeStatus hdds = status.getHddsStatus();
+      UpgradeStatusDto dto = new UpgradeStatusDto();
+      dto.omFinalized = status.getOmFinalized();
+      dto.scmFinalized = hdds.getScmFinalized();
+      dto.datanodesFinalized = hdds.getNumDatanodesFinalized();
+      dto.datanodesTotal = hdds.getNumDatanodesTotal();
+      if (status.hasOmSoftwareVersion()) {
+        dto.omSoftwareVersion = OzoneManagerVersion.deserialize(status.getOmSoftwareVersion()).toString();
+      }
+      if (status.hasOmApparentVersion()) {
+        dto.omApparentVersion = OzoneManagerVersion.deserialize(status.getOmApparentVersion()).toString();
+      }
+      if (hdds.hasScmSoftwareVersion()) {
+        dto.scmSoftwareVersion = HDDSVersion.deserialize(hdds.getScmSoftwareVersion()).toString();
+      }
+      if (hdds.hasScmApparentVersion()) {
+        dto.scmApparentVersion = HDDSVersion.deserialize(hdds.getScmApparentVersion()).toString();
+      }
+      if (hdds.hasMinDatanodeApparentVersion()) {
+        dto.minDatanodeApparentVersion = HDDSVersion.deserialize(hdds.getMinDatanodeApparentVersion()).toString();
+      }
+      if (hdds.hasMaxDatanodeApparentVersion()) {
+        dto.maxDatanodeApparentVersion = HDDSVersion.deserialize(hdds.getMaxDatanodeApparentVersion()).toString();
+      }
+      return dto;
+    }
+
+    public boolean isOmFinalized() {
+      return omFinalized;
+    }
+
+    public String getOmSoftwareVersion() {
+      return omSoftwareVersion;
+    }
+
+    public String getOmApparentVersion() {
+      return omApparentVersion;
+    }
+
+    public boolean isScmFinalized() {
+      return scmFinalized;
+    }
+
+    public String getScmSoftwareVersion() {
+      return scmSoftwareVersion;
+    }
+
+    public String getScmApparentVersion() {
+      return scmApparentVersion;
+    }
+
+    public int getDatanodesFinalized() {
+      return datanodesFinalized;
+    }
+
+    public int getDatanodesTotal() {
+      return datanodesTotal;
+    }
+
+    public String getMinDatanodeApparentVersion() {
+      return minDatanodeApparentVersion;
+    }
+
+    public String getMaxDatanodeApparentVersion() {
+      return maxDatanodeApparentVersion;
+    }
+  }
+
 }