From 1ebc1332640b0b331b3f475a317c74b0636ebcb9 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Wed, 15 Jul 2026 15:00:59 -0400 Subject: [PATCH 1/9] Check OM software versions before starting finalization (cherry picked from commit c7d4e14095fc94d219982ca739c5d20bd44bf0f8) Conflicts: hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OMAdminProtocolServerSideImpl.java hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMStartFinalizeUpgradeRequest.java --- .../ozone/om/protocol/OMAdminProtocol.java | 7 + .../om/protocol/OMPeerUpgradeStatus.java | 39 ++++++ .../OMAdminProtocolClientSideImpl.java | 14 ++ .../src/main/proto/OMAdminProtocol.proto | 14 ++ .../apache/hadoop/ozone/om/OzoneManager.java | 1 - .../OMStartFinalizeUpgradeRequest.java | 37 ++++++ .../OMAdminProtocolServerSideImpl.java | 11 ++ .../TestOMStartFinalizeUpgradeRequest.java | 122 +++++++++++++++++- 8 files changed, 243 insertions(+), 2 deletions(-) create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OMPeerUpgradeStatus.java diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OMAdminProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OMAdminProtocol.java index d6f9395dd37d..2c6e32483abc 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OMAdminProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OMAdminProtocol.java @@ -57,4 +57,11 @@ public interface OMAdminProtocol extends Closeable { * or if the task was triggered successfully (when noWait is true) */ boolean triggerSnapshotDefrag(boolean noWait) throws IOException; + + /** + * Returns the local upgrade status of this OM peer without contacting SCM. + * Intended for use by the OM leader to verify that all Ratis group members + * are running the same software version before accepting a finalize command. + */ + OMPeerUpgradeStatus getUpgradeStatus() throws IOException; } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OMPeerUpgradeStatus.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OMPeerUpgradeStatus.java new file mode 100644 index 000000000000..0887910a848c --- /dev/null +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OMPeerUpgradeStatus.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.protocol; + +import org.apache.hadoop.ozone.OzoneManagerVersion; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.GetUpgradeStatusResponse; + +/** + * Carries the local upgrade status reported by a peer OM via {@link OMAdminProtocol#getUpgradeStatus()}. + * Values are OM-local only: no SCM round-trip is performed by the responding peer. + */ +public final class OMPeerUpgradeStatus { + + private final OzoneManagerVersion softwareVersion; + + public OMPeerUpgradeStatus(GetUpgradeStatusResponse response) { + this.softwareVersion = OzoneManagerVersion.deserialize(response.getOmSoftwareVersion()); + } + + /** The {@link OzoneManagerVersion} of the responding OM binary. */ + public OzoneManagerVersion getSoftwareVersion() { + return softwareVersion; + } +} diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OMAdminProtocolClientSideImpl.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OMAdminProtocolClientSideImpl.java index d248e03b1bdc..42197eeec4dd 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OMAdminProtocolClientSideImpl.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OMAdminProtocolClientSideImpl.java @@ -40,10 +40,13 @@ import org.apache.hadoop.ozone.om.helpers.OMNodeDetails; import org.apache.hadoop.ozone.om.protocol.OMAdminProtocol; import org.apache.hadoop.ozone.om.protocol.OMConfiguration; +import org.apache.hadoop.ozone.om.protocol.OMPeerUpgradeStatus; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.CompactRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.CompactResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.DecommissionOMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.DecommissionOMResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.GetUpgradeStatusRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.GetUpgradeStatusResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.OMConfigurationRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.OMConfigurationResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.OMNodeInfo; @@ -260,6 +263,17 @@ public boolean triggerSnapshotDefrag(boolean noWait) throws IOException { } } + @Override + public OMPeerUpgradeStatus getUpgradeStatus() throws IOException { + try { + GetUpgradeStatusResponse response = rpcProxy.getUpgradeStatus( + NULL_RPC_CONTROLLER, GetUpgradeStatusRequest.newBuilder().build()); + return new OMPeerUpgradeStatus(response); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + } + private void throwException(String errorMsg) throws IOException { throw new IOException("Request Failed. Error: " + errorMsg); diff --git a/hadoop-ozone/interface-client/src/main/proto/OMAdminProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OMAdminProtocol.proto index 4c9a73635bdf..bc3b57207fbd 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OMAdminProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OMAdminProtocol.proto @@ -93,6 +93,16 @@ message TriggerSnapshotDefragResponse { optional bool result = 3; } +// Request from an OM leader to a peer OM to read its local upgrade status. +// Intentionally OM-local: no SCM round-trip is performed by the server. +message GetUpgradeStatusRequest { +} + +message GetUpgradeStatusResponse { + // Serialized OzoneManagerVersion.SOFTWARE_VERSION of the responding OM binary. + optional int32 omSoftwareVersion = 2; +} + /** The service for OM admin operations. */ @@ -113,4 +123,8 @@ service OzoneManagerAdminService { // RPC request from admin to trigger snapshot defragmentation rpc triggerSnapshotDefrag(TriggerSnapshotDefragRequest) returns(TriggerSnapshotDefragResponse); + + // RPC request from the OM leader to a peer OM to read its local upgrade status. + rpc getUpgradeStatus(GetUpgradeStatusRequest) + returns(GetUpgradeStatusResponse); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 423a879e0dd5..0df4911bd78e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -4583,7 +4583,6 @@ public String getOMServiceId() { return omNodeDetails.getServiceId(); } - @VisibleForTesting public List getPeerNodes() { return new ArrayList<>(peerNodesMap.values()); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMStartFinalizeUpgradeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMStartFinalizeUpgradeRequest.java index aa3849d935ce..58d91c645cff 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMStartFinalizeUpgradeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMStartFinalizeUpgradeRequest.java @@ -17,19 +17,26 @@ package org.apache.hadoop.ozone.om.request.upgrade; +import static org.apache.hadoop.hdds.utils.HddsServerUtil.getRemoteUser; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.StartFinalizeUpgrade; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.audit.AuditLogger; import org.apache.hadoop.ozone.audit.OMAction; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.execution.flowcontrol.ExecutionContext; +import org.apache.hadoop.ozone.om.helpers.OMNodeDetails; +import org.apache.hadoop.ozone.om.protocolPB.OMAdminProtocolClientSideImpl; import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.response.OMClientResponse; @@ -61,6 +68,7 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { + "Superuser privilege is required to start finalize upgrade.", OMException.ResultCodes.ACCESS_DENIED); } } + validatePeerOmVersionsBeforeFinalize(ozoneManager.getPeerNodes(), ozoneManager.getConfiguration()); ozoneManager.getScmClient().getContainerClient().finalizeUpgrade(); LOG.info("Successfully triggered the finalize upgrade process in SCM"); return omRequest; @@ -97,4 +105,33 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut return response; } + private static void validatePeerOmVersionsBeforeFinalize(List peerNodes, + OzoneConfiguration configuration) throws OMException { + if (peerNodes.isEmpty()) { + return; + } + OzoneManagerVersion leaderVersion = OzoneManagerVersion.SOFTWARE_VERSION; + List failedPeers = new ArrayList<>(); + for (OMNodeDetails peerDetails : peerNodes) { + String peerId = peerDetails.getNodeId(); + try (OMAdminProtocolClientSideImpl client = + OMAdminProtocolClientSideImpl.createProxyForSingleOM(configuration, getRemoteUser(), peerDetails)) { + OzoneManagerVersion peerVersion = client.getUpgradeStatus().getSoftwareVersion(); + if (!peerVersion.equals(leaderVersion)) { + LOG.warn("OM peer {} is running software version {} but leader is running version {}. " + + "Rejecting finalize command.", peerId, peerVersion, leaderVersion); + failedPeers.add(peerId + " (version: " + peerVersion + ")"); + } + } catch (IOException e) { + LOG.warn("Failed to contact OM peer {} to check software version before finalize.", peerId, e); + failedPeers.add(peerId + " (unreachable: " + e.getMessage() + ")"); + } + } + if (!failedPeers.isEmpty()) { + throw new OMException("Finalize rejected: the following OM peers did not confirm matching software version " + + "(expected version=" + leaderVersion + "): " + String.join(", ", failedPeers), + OMException.ResultCodes.LAYOUT_FEATURE_FINALIZATION_FAILED); + } + } + } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OMAdminProtocolServerSideImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OMAdminProtocolServerSideImpl.java index ba96368cfd5d..d90d6ffb63f4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OMAdminProtocolServerSideImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OMAdminProtocolServerSideImpl.java @@ -26,6 +26,7 @@ import java.util.ArrayList; import java.util.List; import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions; +import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OMNodeDetails; @@ -37,6 +38,8 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.CompactResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.DecommissionOMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.DecommissionOMResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.GetUpgradeStatusRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.GetUpgradeStatusResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.OMConfigurationRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.OMConfigurationResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.OMNodeInfo; @@ -154,4 +157,12 @@ public TriggerSnapshotDefragResponse triggerSnapshotDefrag( .build(); } } + + @Override + public GetUpgradeStatusResponse getUpgradeStatus(RpcController controller, + GetUpgradeStatusRequest request) throws ServiceException { + return GetUpgradeStatusResponse.newBuilder() + .setOmSoftwareVersion(OzoneManagerVersion.SOFTWARE_VERSION.serialize()) + .build(); + } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMStartFinalizeUpgradeRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMStartFinalizeUpgradeRequest.java index 44b206052c74..2d995d1cffdc 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMStartFinalizeUpgradeRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMStartFinalizeUpgradeRequest.java @@ -24,27 +24,43 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.execution.flowcontrol.ExecutionContext; +import org.apache.hadoop.ozone.om.helpers.OMNodeDetails; +import org.apache.hadoop.ozone.om.protocol.OMPeerUpgradeStatus; +import org.apache.hadoop.ozone.om.protocolPB.OMAdminProtocolClientSideImpl; import org.apache.hadoop.ozone.om.request.key.OMKeyRequestTests; import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.GetUpgradeStatusResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.security.UserGroupInformation; import org.apache.ratis.protocol.ClientId; import org.apache.ratis.server.protocol.TermIndex; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; /** * Tests for OMStartFinalizeUpgradeRequest. */ public class TestOMStartFinalizeUpgradeRequest extends OMKeyRequestTests { - + + @BeforeEach + public void stubPeerNodes() { + when(ozoneManager.getPeerNodes()).thenReturn(Collections.emptyList()); + } + @Test public void testPreExecuteCallsScmFinalizeUpgrade() throws IOException { doNothing().when(scmContainerLocationProtocol).finalizeUpgrade(); @@ -99,6 +115,110 @@ public void testAccessDeniedWhenUserIsNotAdmin() throws IOException { verify(scmContainerLocationProtocol, never()).finalizeUpgrade(); } + @Test + public void testPeerVersionCheckPassesWhenNoPeers() throws IOException { + // @BeforeEach already stubs getPeerNodes() to return an empty list. + // preExecute must complete normally and call SCM finalize. + doNothing().when(scmContainerLocationProtocol).finalizeUpgrade(); + + OzoneManagerProtocolProtos.OMRequest original = buildRequest(); + new OMStartFinalizeUpgradeRequest(original).preExecute(ozoneManager); + + verify(scmContainerLocationProtocol).finalizeUpgrade(); + } + + @Test + public void testPeerVersionCheckPassesWhenAllPeersMatch() throws IOException { + doNothing().when(scmContainerLocationProtocol).finalizeUpgrade(); + when(ozoneManager.getPeerNodes()).thenReturn(Arrays.asList(buildPeer("om2"), buildPeer("om3"))); + OMAdminProtocolClientSideImpl matchingClient = peerClientWithVersion(OzoneManagerVersion.SOFTWARE_VERSION); + + try (MockedStatic factory = + mockStatic(OMAdminProtocolClientSideImpl.class)) { + factory.when(() -> OMAdminProtocolClientSideImpl.createProxyForSingleOM(any(), any(), any())) + .thenReturn(matchingClient); + + new OMStartFinalizeUpgradeRequest(buildRequest()).preExecute(ozoneManager); + } + + verify(scmContainerLocationProtocol).finalizeUpgrade(); + } + + @Test + public void testPeerVersionCheckRejectsOneOlderPeer() throws IOException { + when(ozoneManager.getPeerNodes()).thenReturn(Arrays.asList(buildPeer("om2"), buildPeer("om3"))); + OMAdminProtocolClientSideImpl matchingClient = peerClientWithVersion(OzoneManagerVersion.SOFTWARE_VERSION); + OMAdminProtocolClientSideImpl olderClient = peerClientWithVersion(OzoneManagerVersion.HBASE_SUPPORT); + + try (MockedStatic factory = + mockStatic(OMAdminProtocolClientSideImpl.class)) { + factory.when(() -> OMAdminProtocolClientSideImpl.createProxyForSingleOM(any(), any(), any())) + .thenReturn(matchingClient, olderClient); + + OMException ex = assertThrows(OMException.class, + () -> new OMStartFinalizeUpgradeRequest(buildRequest()).preExecute(ozoneManager)); + assertEquals(OMException.ResultCodes.LAYOUT_FEATURE_FINALIZATION_FAILED, ex.getResult()); + } + + verify(scmContainerLocationProtocol, never()).finalizeUpgrade(); + } + + @Test + public void testPeerVersionCheckRejectsOneUnknownFuturePeer() throws IOException { + when(ozoneManager.getPeerNodes()).thenReturn(Arrays.asList(buildPeer("om2"), buildPeer("om3"))); + OMAdminProtocolClientSideImpl matchingClient = peerClientWithVersion(OzoneManagerVersion.SOFTWARE_VERSION); + OMAdminProtocolClientSideImpl unknownClient = peerClientWithVersion(OzoneManagerVersion.UNKNOWN_VERSION); + + try (MockedStatic factory = + mockStatic(OMAdminProtocolClientSideImpl.class)) { + factory.when(() -> OMAdminProtocolClientSideImpl.createProxyForSingleOM(any(), any(), any())) + .thenReturn(matchingClient, unknownClient); + + OMException ex = assertThrows(OMException.class, + () -> new OMStartFinalizeUpgradeRequest(buildRequest()).preExecute(ozoneManager)); + assertEquals(OMException.ResultCodes.LAYOUT_FEATURE_FINALIZATION_FAILED, ex.getResult()); + } + + verify(scmContainerLocationProtocol, never()).finalizeUpgrade(); + } + + @Test + public void testPeerVersionCheckRejectsUnreachablePeer() throws IOException { + when(ozoneManager.getPeerNodes()).thenReturn(Collections.singletonList(buildPeer("om2"))); + OMAdminProtocolClientSideImpl unreachableClient = mock(OMAdminProtocolClientSideImpl.class); + when(unreachableClient.getUpgradeStatus()).thenThrow(new IOException("connection refused")); + + try (MockedStatic factory = + mockStatic(OMAdminProtocolClientSideImpl.class)) { + factory.when(() -> OMAdminProtocolClientSideImpl.createProxyForSingleOM(any(), any(), any())) + .thenReturn(unreachableClient); + + OMException ex = assertThrows(OMException.class, + () -> new OMStartFinalizeUpgradeRequest(buildRequest()).preExecute(ozoneManager)); + assertEquals(OMException.ResultCodes.LAYOUT_FEATURE_FINALIZATION_FAILED, ex.getResult()); + } + + verify(scmContainerLocationProtocol, never()).finalizeUpgrade(); + } + + private static OMNodeDetails buildPeer(String nodeId) { + return new OMNodeDetails.Builder() + .setOMServiceId("testService") + .setOMNodeId(nodeId) + .setHostAddress("127.0.0.1") + .setRpcPort(1) + .build(); + } + + private static OMAdminProtocolClientSideImpl peerClientWithVersion(OzoneManagerVersion version) throws IOException { + OMAdminProtocolClientSideImpl client = mock(OMAdminProtocolClientSideImpl.class); + GetUpgradeStatusResponse response = GetUpgradeStatusResponse.newBuilder() + .setOmSoftwareVersion(version.serialize()) + .build(); + when(client.getUpgradeStatus()).thenReturn(new OMPeerUpgradeStatus(response)); + return client; + } + private OMClientResponse submitRequest() throws IOException { OzoneManagerProtocolProtos.OMRequest original = buildRequest(); OMStartFinalizeUpgradeRequest request = new OMStartFinalizeUpgradeRequest(original); From 3e96948bd3413583be1490ad962e0ec01c398f78 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Thu, 16 Jul 2026 14:02:38 -0400 Subject: [PATCH 2/9] Remove DTO --- .../ozone/om/protocol/OMAdminProtocol.java | 5 ++- .../om/protocol/OMPeerUpgradeStatus.java | 39 ------------------- .../OMAdminProtocolClientSideImpl.java | 6 +-- .../OMStartFinalizeUpgradeRequest.java | 2 +- .../TestOMStartFinalizeUpgradeRequest.java | 7 +--- 5 files changed, 8 insertions(+), 51 deletions(-) delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OMPeerUpgradeStatus.java diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OMAdminProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OMAdminProtocol.java index 2c6e32483abc..1647c760793b 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OMAdminProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OMAdminProtocol.java @@ -19,6 +19,7 @@ import java.io.Closeable; import java.io.IOException; +import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.helpers.OMNodeDetails; import org.apache.hadoop.security.KerberosInfo; @@ -59,9 +60,9 @@ public interface OMAdminProtocol extends Closeable { boolean triggerSnapshotDefrag(boolean noWait) throws IOException; /** - * Returns the local upgrade status of this OM peer without contacting SCM. + * Returns the software version of this OM peer without contacting SCM. * Intended for use by the OM leader to verify that all Ratis group members * are running the same software version before accepting a finalize command. */ - OMPeerUpgradeStatus getUpgradeStatus() throws IOException; + OzoneManagerVersion getUpgradeStatus() throws IOException; } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OMPeerUpgradeStatus.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OMPeerUpgradeStatus.java deleted file mode 100644 index 0887910a848c..000000000000 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OMPeerUpgradeStatus.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.om.protocol; - -import org.apache.hadoop.ozone.OzoneManagerVersion; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.GetUpgradeStatusResponse; - -/** - * Carries the local upgrade status reported by a peer OM via {@link OMAdminProtocol#getUpgradeStatus()}. - * Values are OM-local only: no SCM round-trip is performed by the responding peer. - */ -public final class OMPeerUpgradeStatus { - - private final OzoneManagerVersion softwareVersion; - - public OMPeerUpgradeStatus(GetUpgradeStatusResponse response) { - this.softwareVersion = OzoneManagerVersion.deserialize(response.getOmSoftwareVersion()); - } - - /** The {@link OzoneManagerVersion} of the responding OM binary. */ - public OzoneManagerVersion getSoftwareVersion() { - return softwareVersion; - } -} diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OMAdminProtocolClientSideImpl.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OMAdminProtocolClientSideImpl.java index 42197eeec4dd..9800911d2379 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OMAdminProtocolClientSideImpl.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OMAdminProtocolClientSideImpl.java @@ -37,10 +37,10 @@ import org.apache.hadoop.ozone.om.exceptions.OMLeaderNotReadyException; import org.apache.hadoop.ozone.om.exceptions.OMNotLeaderException; import org.apache.hadoop.ozone.om.ha.HadoopRpcOMFailoverProxyProvider; +import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.om.helpers.OMNodeDetails; import org.apache.hadoop.ozone.om.protocol.OMAdminProtocol; import org.apache.hadoop.ozone.om.protocol.OMConfiguration; -import org.apache.hadoop.ozone.om.protocol.OMPeerUpgradeStatus; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.CompactRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.CompactResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.DecommissionOMRequest; @@ -264,11 +264,11 @@ public boolean triggerSnapshotDefrag(boolean noWait) throws IOException { } @Override - public OMPeerUpgradeStatus getUpgradeStatus() throws IOException { + public OzoneManagerVersion getUpgradeStatus() throws IOException { try { GetUpgradeStatusResponse response = rpcProxy.getUpgradeStatus( NULL_RPC_CONTROLLER, GetUpgradeStatusRequest.newBuilder().build()); - return new OMPeerUpgradeStatus(response); + return OzoneManagerVersion.deserialize(response.getOmSoftwareVersion()); } catch (ServiceException e) { throw ProtobufHelper.getRemoteException(e); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMStartFinalizeUpgradeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMStartFinalizeUpgradeRequest.java index 58d91c645cff..a8c6987b9a97 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMStartFinalizeUpgradeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMStartFinalizeUpgradeRequest.java @@ -116,7 +116,7 @@ private static void validatePeerOmVersionsBeforeFinalize(List pee String peerId = peerDetails.getNodeId(); try (OMAdminProtocolClientSideImpl client = OMAdminProtocolClientSideImpl.createProxyForSingleOM(configuration, getRemoteUser(), peerDetails)) { - OzoneManagerVersion peerVersion = client.getUpgradeStatus().getSoftwareVersion(); + OzoneManagerVersion peerVersion = client.getUpgradeStatus(); if (!peerVersion.equals(leaderVersion)) { LOG.warn("OM peer {} is running software version {} but leader is running version {}. " + "Rejecting finalize command.", peerId, peerVersion, leaderVersion); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMStartFinalizeUpgradeRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMStartFinalizeUpgradeRequest.java index 2d995d1cffdc..cd6f81e741dc 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMStartFinalizeUpgradeRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMStartFinalizeUpgradeRequest.java @@ -38,11 +38,9 @@ import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.execution.flowcontrol.ExecutionContext; import org.apache.hadoop.ozone.om.helpers.OMNodeDetails; -import org.apache.hadoop.ozone.om.protocol.OMPeerUpgradeStatus; import org.apache.hadoop.ozone.om.protocolPB.OMAdminProtocolClientSideImpl; import org.apache.hadoop.ozone.om.request.key.OMKeyRequestTests; import org.apache.hadoop.ozone.om.response.OMClientResponse; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.GetUpgradeStatusResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.security.UserGroupInformation; import org.apache.ratis.protocol.ClientId; @@ -212,10 +210,7 @@ private static OMNodeDetails buildPeer(String nodeId) { private static OMAdminProtocolClientSideImpl peerClientWithVersion(OzoneManagerVersion version) throws IOException { OMAdminProtocolClientSideImpl client = mock(OMAdminProtocolClientSideImpl.class); - GetUpgradeStatusResponse response = GetUpgradeStatusResponse.newBuilder() - .setOmSoftwareVersion(version.serialize()) - .build(); - when(client.getUpgradeStatus()).thenReturn(new OMPeerUpgradeStatus(response)); + when(client.getUpgradeStatus()).thenReturn(version); return client; } From edc5a06e8d30351b00e4179c8228371e1420d2ae Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Thu, 16 Jul 2026 14:08:35 -0400 Subject: [PATCH 3/9] Add missed test --- .../TestOMAdminProtocolServerSideImpl.java | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/protocolPB/TestOMAdminProtocolServerSideImpl.java diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/protocolPB/TestOMAdminProtocolServerSideImpl.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/protocolPB/TestOMAdminProtocolServerSideImpl.java new file mode 100644 index 000000000000..72076a1c4cfa --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/protocolPB/TestOMAdminProtocolServerSideImpl.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.protocolPB; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; + +import com.google.protobuf.ServiceException; +import org.apache.hadoop.ozone.OzoneManagerVersion; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.GetPeerUpgradeStatusRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.GetPeerUpgradeStatusResponse; +import org.junit.jupiter.api.Test; + +/** + * Tests for {@link OMAdminProtocolServerSideImpl#getPeerUpgradeStatus}. + */ +public class TestOMAdminProtocolServerSideImpl { + + @Test + public void testGetPeerUpgradeStatusReturnsSoftwareVersion() throws ServiceException { + OzoneManager om = mock(OzoneManager.class); + + OMAdminProtocolServerSideImpl handler = new OMAdminProtocolServerSideImpl(om); + GetPeerUpgradeStatusResponse response = + handler.getPeerUpgradeStatus(null, GetPeerUpgradeStatusRequest.newBuilder().build()); + + assertEquals(OzoneManagerVersion.SOFTWARE_VERSION.serialize(), response.getOmSoftwareVersion()); + } +} From 9d0f7de79f5d2d4aa9a11da50506630c6c825734 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Thu, 16 Jul 2026 14:08:45 -0400 Subject: [PATCH 4/9] Rename RPC --- .../hadoop/ozone/om/protocol/OMAdminProtocol.java | 2 +- .../om/protocolPB/OMAdminProtocolClientSideImpl.java | 10 +++++----- .../src/main/proto/OMAdminProtocol.proto | 8 ++++---- .../request/upgrade/OMStartFinalizeUpgradeRequest.java | 2 +- .../protocolPB/OMAdminProtocolServerSideImpl.java | 10 +++++----- .../upgrade/TestOMStartFinalizeUpgradeRequest.java | 4 ++-- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OMAdminProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OMAdminProtocol.java index 1647c760793b..68d2c2fafb7b 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OMAdminProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OMAdminProtocol.java @@ -64,5 +64,5 @@ public interface OMAdminProtocol extends Closeable { * Intended for use by the OM leader to verify that all Ratis group members * are running the same software version before accepting a finalize command. */ - OzoneManagerVersion getUpgradeStatus() throws IOException; + OzoneManagerVersion getPeerUpgradeStatus() throws IOException; } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OMAdminProtocolClientSideImpl.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OMAdminProtocolClientSideImpl.java index 9800911d2379..6c1f0c01d954 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OMAdminProtocolClientSideImpl.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OMAdminProtocolClientSideImpl.java @@ -45,8 +45,8 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.CompactResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.DecommissionOMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.DecommissionOMResponse; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.GetUpgradeStatusRequest; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.GetUpgradeStatusResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.GetPeerUpgradeStatusRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.GetPeerUpgradeStatusResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.OMConfigurationRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.OMConfigurationResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.OMNodeInfo; @@ -264,10 +264,10 @@ public boolean triggerSnapshotDefrag(boolean noWait) throws IOException { } @Override - public OzoneManagerVersion getUpgradeStatus() throws IOException { + public OzoneManagerVersion getPeerUpgradeStatus() throws IOException { try { - GetUpgradeStatusResponse response = rpcProxy.getUpgradeStatus( - NULL_RPC_CONTROLLER, GetUpgradeStatusRequest.newBuilder().build()); + GetPeerUpgradeStatusResponse response = rpcProxy.getPeerUpgradeStatus( + NULL_RPC_CONTROLLER, GetPeerUpgradeStatusRequest.newBuilder().build()); return OzoneManagerVersion.deserialize(response.getOmSoftwareVersion()); } catch (ServiceException e) { throw ProtobufHelper.getRemoteException(e); diff --git a/hadoop-ozone/interface-client/src/main/proto/OMAdminProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OMAdminProtocol.proto index bc3b57207fbd..3b852fdbab63 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OMAdminProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OMAdminProtocol.proto @@ -95,10 +95,10 @@ message TriggerSnapshotDefragResponse { // Request from an OM leader to a peer OM to read its local upgrade status. // Intentionally OM-local: no SCM round-trip is performed by the server. -message GetUpgradeStatusRequest { +message GetPeerUpgradeStatusRequest { } -message GetUpgradeStatusResponse { +message GetPeerUpgradeStatusResponse { // Serialized OzoneManagerVersion.SOFTWARE_VERSION of the responding OM binary. optional int32 omSoftwareVersion = 2; } @@ -125,6 +125,6 @@ service OzoneManagerAdminService { returns(TriggerSnapshotDefragResponse); // RPC request from the OM leader to a peer OM to read its local upgrade status. - rpc getUpgradeStatus(GetUpgradeStatusRequest) - returns(GetUpgradeStatusResponse); + rpc getPeerUpgradeStatus(GetPeerUpgradeStatusRequest) + returns(GetPeerUpgradeStatusResponse); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMStartFinalizeUpgradeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMStartFinalizeUpgradeRequest.java index a8c6987b9a97..6fad9761eb1d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMStartFinalizeUpgradeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMStartFinalizeUpgradeRequest.java @@ -116,7 +116,7 @@ private static void validatePeerOmVersionsBeforeFinalize(List pee String peerId = peerDetails.getNodeId(); try (OMAdminProtocolClientSideImpl client = OMAdminProtocolClientSideImpl.createProxyForSingleOM(configuration, getRemoteUser(), peerDetails)) { - OzoneManagerVersion peerVersion = client.getUpgradeStatus(); + OzoneManagerVersion peerVersion = client.getPeerUpgradeStatus(); if (!peerVersion.equals(leaderVersion)) { LOG.warn("OM peer {} is running software version {} but leader is running version {}. " + "Rejecting finalize command.", peerId, peerVersion, leaderVersion); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OMAdminProtocolServerSideImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OMAdminProtocolServerSideImpl.java index d90d6ffb63f4..e245846dc092 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OMAdminProtocolServerSideImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OMAdminProtocolServerSideImpl.java @@ -38,8 +38,8 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.CompactResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.DecommissionOMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.DecommissionOMResponse; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.GetUpgradeStatusRequest; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.GetUpgradeStatusResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.GetPeerUpgradeStatusRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.GetPeerUpgradeStatusResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.OMConfigurationRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.OMConfigurationResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.OMNodeInfo; @@ -159,9 +159,9 @@ public TriggerSnapshotDefragResponse triggerSnapshotDefrag( } @Override - public GetUpgradeStatusResponse getUpgradeStatus(RpcController controller, - GetUpgradeStatusRequest request) throws ServiceException { - return GetUpgradeStatusResponse.newBuilder() + public GetPeerUpgradeStatusResponse getPeerUpgradeStatus(RpcController controller, + GetPeerUpgradeStatusRequest request) throws ServiceException { + return GetPeerUpgradeStatusResponse.newBuilder() .setOmSoftwareVersion(OzoneManagerVersion.SOFTWARE_VERSION.serialize()) .build(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMStartFinalizeUpgradeRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMStartFinalizeUpgradeRequest.java index cd6f81e741dc..2a6ed40f4662 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMStartFinalizeUpgradeRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMStartFinalizeUpgradeRequest.java @@ -184,7 +184,7 @@ public void testPeerVersionCheckRejectsOneUnknownFuturePeer() throws IOException public void testPeerVersionCheckRejectsUnreachablePeer() throws IOException { when(ozoneManager.getPeerNodes()).thenReturn(Collections.singletonList(buildPeer("om2"))); OMAdminProtocolClientSideImpl unreachableClient = mock(OMAdminProtocolClientSideImpl.class); - when(unreachableClient.getUpgradeStatus()).thenThrow(new IOException("connection refused")); + when(unreachableClient.getPeerUpgradeStatus()).thenThrow(new IOException("connection refused")); try (MockedStatic factory = mockStatic(OMAdminProtocolClientSideImpl.class)) { @@ -210,7 +210,7 @@ private static OMNodeDetails buildPeer(String nodeId) { private static OMAdminProtocolClientSideImpl peerClientWithVersion(OzoneManagerVersion version) throws IOException { OMAdminProtocolClientSideImpl client = mock(OMAdminProtocolClientSideImpl.class); - when(client.getUpgradeStatus()).thenReturn(version); + when(client.getPeerUpgradeStatus()).thenReturn(version); return client; } From ab4b4e8434bd2ac7d56d48f21fc5cf4426802c40 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Thu, 16 Jul 2026 17:16:24 -0400 Subject: [PATCH 5/9] Fix exception handling cleanup and change error code --- .../request/upgrade/OMStartFinalizeUpgradeRequest.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMStartFinalizeUpgradeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMStartFinalizeUpgradeRequest.java index 6fad9761eb1d..5c44bb622ba7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMStartFinalizeUpgradeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMStartFinalizeUpgradeRequest.java @@ -25,6 +25,7 @@ import java.util.HashMap; import java.util.List; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.OzoneConsts; @@ -114,8 +115,9 @@ private static void validatePeerOmVersionsBeforeFinalize(List pee List failedPeers = new ArrayList<>(); for (OMNodeDetails peerDetails : peerNodes) { String peerId = peerDetails.getNodeId(); - try (OMAdminProtocolClientSideImpl client = - OMAdminProtocolClientSideImpl.createProxyForSingleOM(configuration, getRemoteUser(), peerDetails)) { + OMAdminProtocolClientSideImpl client = null; + try { + client = OMAdminProtocolClientSideImpl.createProxyForSingleOM(configuration, getRemoteUser(), peerDetails); OzoneManagerVersion peerVersion = client.getPeerUpgradeStatus(); if (!peerVersion.equals(leaderVersion)) { LOG.warn("OM peer {} is running software version {} but leader is running version {}. " @@ -125,12 +127,14 @@ private static void validatePeerOmVersionsBeforeFinalize(List pee } catch (IOException e) { LOG.warn("Failed to contact OM peer {} to check software version before finalize.", peerId, e); failedPeers.add(peerId + " (unreachable: " + e.getMessage() + ")"); + } finally { + IOUtils.cleanupWithLogger(LOG, client); } } if (!failedPeers.isEmpty()) { throw new OMException("Finalize rejected: the following OM peers did not confirm matching software version " + "(expected version=" + leaderVersion + "): " + String.join(", ", failedPeers), - OMException.ResultCodes.LAYOUT_FEATURE_FINALIZATION_FAILED); + OMException.ResultCodes.NOT_SUPPORTED_OPERATION); } } From 34650eb5de0e9c3cf63d53321c9c21b108d52b27 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Thu, 16 Jul 2026 17:33:51 -0400 Subject: [PATCH 6/9] Fix exception check in tests, add test for SCM exceptions --- .../OMAdminProtocolClientSideImpl.java | 2 +- .../TestOMStartFinalizeUpgradeRequest.java | 23 ++++++++++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OMAdminProtocolClientSideImpl.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OMAdminProtocolClientSideImpl.java index 6c1f0c01d954..0b48e51b2987 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OMAdminProtocolClientSideImpl.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OMAdminProtocolClientSideImpl.java @@ -33,11 +33,11 @@ import org.apache.hadoop.ipc_.RPC; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.ozone.OmUtils; +import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.exceptions.OMLeaderNotReadyException; import org.apache.hadoop.ozone.om.exceptions.OMNotLeaderException; import org.apache.hadoop.ozone.om.ha.HadoopRpcOMFailoverProxyProvider; -import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.om.helpers.OMNodeDetails; import org.apache.hadoop.ozone.om.protocol.OMAdminProtocol; import org.apache.hadoop.ozone.om.protocol.OMConfiguration; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMStartFinalizeUpgradeRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMStartFinalizeUpgradeRequest.java index 2a6ed40f4662..eb85e4ee0f86 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMStartFinalizeUpgradeRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMStartFinalizeUpgradeRequest.java @@ -21,9 +21,11 @@ import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.never; @@ -76,6 +78,21 @@ public void testPreExecuteCallsScmFinalizeUpgrade() throws IOException { verify(scmContainerLocationProtocol).finalizeUpgrade(); } + @Test + public void testScmFinalizeFailurePropagatesToClient() throws IOException { + IOException scmFailure = new IOException("SCM finalize upgrade failed"); + doThrow(scmFailure).when(scmContainerLocationProtocol).finalizeUpgrade(); + + OMStartFinalizeUpgradeRequest request = new OMStartFinalizeUpgradeRequest(buildRequest()); + + // The exception raised by SCM must propagate out of preExecute so the OM + // client sees the failure instead of a successful finalize. + IOException ex = assertThrows(IOException.class, () -> request.preExecute(ozoneManager)); + assertSame(scmFailure, ex); + + verify(scmContainerLocationProtocol).finalizeUpgrade(); + } + @Test public void testValidateAndUpdateCacheAddsFinalizationInProgressKey() throws IOException { doNothing().when(scmContainerLocationProtocol).finalizeUpgrade(); @@ -155,7 +172,7 @@ public void testPeerVersionCheckRejectsOneOlderPeer() throws IOException { OMException ex = assertThrows(OMException.class, () -> new OMStartFinalizeUpgradeRequest(buildRequest()).preExecute(ozoneManager)); - assertEquals(OMException.ResultCodes.LAYOUT_FEATURE_FINALIZATION_FAILED, ex.getResult()); + assertEquals(OMException.ResultCodes.NOT_SUPPORTED_OPERATION, ex.getResult()); } verify(scmContainerLocationProtocol, never()).finalizeUpgrade(); @@ -174,7 +191,7 @@ public void testPeerVersionCheckRejectsOneUnknownFuturePeer() throws IOException OMException ex = assertThrows(OMException.class, () -> new OMStartFinalizeUpgradeRequest(buildRequest()).preExecute(ozoneManager)); - assertEquals(OMException.ResultCodes.LAYOUT_FEATURE_FINALIZATION_FAILED, ex.getResult()); + assertEquals(OMException.ResultCodes.NOT_SUPPORTED_OPERATION, ex.getResult()); } verify(scmContainerLocationProtocol, never()).finalizeUpgrade(); @@ -193,7 +210,7 @@ public void testPeerVersionCheckRejectsUnreachablePeer() throws IOException { OMException ex = assertThrows(OMException.class, () -> new OMStartFinalizeUpgradeRequest(buildRequest()).preExecute(ozoneManager)); - assertEquals(OMException.ResultCodes.LAYOUT_FEATURE_FINALIZATION_FAILED, ex.getResult()); + assertEquals(OMException.ResultCodes.NOT_SUPPORTED_OPERATION, ex.getResult()); } verify(scmContainerLocationProtocol, never()).finalizeUpgrade(); From ed8cdfdc7ff89e74f2dd68f39d7be172bef8ee42 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Thu, 16 Jul 2026 17:40:57 -0400 Subject: [PATCH 7/9] Update SCM exception handling and test --- .../OMStartFinalizeUpgradeRequest.java | 10 +++++- .../TestOMStartFinalizeUpgradeRequest.java | 33 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMStartFinalizeUpgradeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMStartFinalizeUpgradeRequest.java index 5c44bb622ba7..7514004a49c5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMStartFinalizeUpgradeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMStartFinalizeUpgradeRequest.java @@ -25,6 +25,7 @@ import java.util.HashMap; import java.util.List; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.scm.exceptions.SCMException; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; @@ -70,7 +71,14 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { } } validatePeerOmVersionsBeforeFinalize(ozoneManager.getPeerNodes(), ozoneManager.getConfiguration()); - ozoneManager.getScmClient().getContainerClient().finalizeUpgrade(); + try { + ozoneManager.getScmClient().getContainerClient().finalizeUpgrade(); + } catch (SCMException e) { + if (e.getResult() == SCMException.ResultCodes.UNSUPPORTED_OPERATION) { + throw new OMException(e.getMessage(), e, OMException.ResultCodes.NOT_SUPPORTED_OPERATION); + } + throw e; + } LOG.info("Successfully triggered the finalize upgrade process in SCM"); return omRequest; } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMStartFinalizeUpgradeRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMStartFinalizeUpgradeRequest.java index eb85e4ee0f86..fe4d9b4fe8f8 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMStartFinalizeUpgradeRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMStartFinalizeUpgradeRequest.java @@ -35,6 +35,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collections; +import org.apache.hadoop.hdds.scm.exceptions.SCMException; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.om.exceptions.OMException; @@ -93,6 +94,38 @@ public void testScmFinalizeFailurePropagatesToClient() throws IOException { verify(scmContainerLocationProtocol).finalizeUpgrade(); } + @Test + public void testScmUnsupportedOperationBecomesOmNotSupportedOperation() throws IOException { + SCMException scmFailure = + new SCMException("SCM version mismatch", SCMException.ResultCodes.UNSUPPORTED_OPERATION); + doThrow(scmFailure).when(scmContainerLocationProtocol).finalizeUpgrade(); + + OMStartFinalizeUpgradeRequest request = new OMStartFinalizeUpgradeRequest(buildRequest()); + + // An SCM UNSUPPORTED_OPERATION is re-mapped to an OM NOT_SUPPORTED_OPERATION, + // preserving the original message and chaining the SCM exception as the cause. + OMException ex = assertThrows(OMException.class, () -> request.preExecute(ozoneManager)); + assertEquals(OMException.ResultCodes.NOT_SUPPORTED_OPERATION, ex.getResult()); + assertEquals(scmFailure.getMessage(), ex.getMessage()); + assertSame(scmFailure, ex.getCause()); + + verify(scmContainerLocationProtocol).finalizeUpgrade(); + } + + @Test + public void testOtherScmExceptionPropagatesUnchanged() throws IOException { + SCMException scmFailure = new SCMException("SCM is in safe mode", SCMException.ResultCodes.SAFE_MODE_EXCEPTION); + doThrow(scmFailure).when(scmContainerLocationProtocol).finalizeUpgrade(); + + OMStartFinalizeUpgradeRequest request = new OMStartFinalizeUpgradeRequest(buildRequest()); + + // Only UNSUPPORTED_OPERATION is re-mapped; any other SCM exception propagates as-is. + SCMException ex = assertThrows(SCMException.class, () -> request.preExecute(ozoneManager)); + assertSame(scmFailure, ex); + + verify(scmContainerLocationProtocol).finalizeUpgrade(); + } + @Test public void testValidateAndUpdateCacheAddsFinalizationInProgressKey() throws IOException { doNothing().when(scmContainerLocationProtocol).finalizeUpgrade(); From da506a6ad283b781c7da3a5b864948a4217af9cf Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Thu, 16 Jul 2026 18:11:09 -0400 Subject: [PATCH 8/9] Deprecate unused OM finalization error result codes --- .../apache/hadoop/ozone/om/exceptions/OMException.java | 4 ++++ .../src/main/proto/OmClientProtocol.proto | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java index 240e99e7d673..942a0b1cf8c1 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java @@ -238,9 +238,13 @@ public enum ResultCodes { DIRECTORY_NOT_EMPTY, + @Deprecated PERSIST_UPGRADE_TO_LAYOUT_VERSION_FAILED, + @Deprecated REMOVE_UPGRADE_TO_LAYOUT_VERSION_FAILED, + @Deprecated UPDATE_LAYOUT_VERSION_FAILED, + @Deprecated LAYOUT_FEATURE_FINALIZATION_FAILED, PREPARE_FAILED, NOT_SUPPORTED_OPERATION_WHEN_PREPARED, diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 72c226b62b30..c3937f0f157d 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -554,10 +554,10 @@ enum Status { DIRECTORY_NOT_EMPTY = 68; - PERSIST_UPGRADE_TO_LAYOUT_VERSION_FAILED = 69; - REMOVE_UPGRADE_TO_LAYOUT_VERSION_FAILED = 70; - UPDATE_LAYOUT_VERSION_FAILED = 71; - LAYOUT_FEATURE_FINALIZATION_FAILED = 72; + PERSIST_UPGRADE_TO_LAYOUT_VERSION_FAILED = 69; // [deprecated = true] + REMOVE_UPGRADE_TO_LAYOUT_VERSION_FAILED = 70; // [deprecated = true] + UPDATE_LAYOUT_VERSION_FAILED = 71; // [deprecated = true] + LAYOUT_FEATURE_FINALIZATION_FAILED = 72; // [deprecated = true] PREPARE_FAILED = 73; NOT_SUPPORTED_OPERATION_WHEN_PREPARED = 74; NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION = 75; From 829ab50b587b6380d6cf4c346656a79211d722b0 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Fri, 17 Jul 2026 15:42:44 -0400 Subject: [PATCH 9/9] Add force flag with tests --- .../admin/upgrade/FinalizeSubCommand.java | 14 +++++- .../admin/upgrade/TestFinalizeSubCommand.java | 13 ++++++ .../om/protocol/OzoneManagerProtocol.java | 12 ++++++ ...ManagerProtocolClientSideTranslatorPB.java | 10 +++++ .../ozone/om/TestOMUpgradeFinalization.java | 2 +- .../src/main/proto/OmClientProtocol.proto | 1 + .../apache/hadoop/ozone/om/OzoneManager.java | 10 ++++- .../OMStartFinalizeUpgradeRequest.java | 13 +++++- .../TestOMStartFinalizeUpgradeRequest.java | 43 +++++++++++++++++++ 9 files changed, 112 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/FinalizeSubCommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/FinalizeSubCommand.java index 411955951df6..b2acbd577592 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/FinalizeSubCommand.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/FinalizeSubCommand.java @@ -40,6 +40,13 @@ public class FinalizeSubCommand extends AbstractSubcommand implements Callable