diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/FinalizeVersionCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/FinalizeVersionCommandHandler.java index a9df014b9ecf..1412859fe36a 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/FinalizeVersionCommandHandler.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/FinalizeVersionCommandHandler.java @@ -18,7 +18,7 @@ package org.apache.hadoop.ozone.container.common.statemachine.commandhandler; import java.util.concurrent.atomic.AtomicLong; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.FinalizeNewLayoutVersionCommandProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.FinalizeNewDatanodeVersionCommandProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto; import org.apache.hadoop.metrics2.lib.MetricsRegistry; import org.apache.hadoop.metrics2.lib.MutableRate; @@ -51,7 +51,7 @@ public FinalizeVersionCommandHandler() { MetricsRegistry registry = new MetricsRegistry( FinalizeVersionCommandHandler.class.getSimpleName()); this.opsLatencyMs = - registry.newRate(SCMCommandProto.Type.finalizeNewLayoutVersionCommand + "Ms"); + registry.newRate(SCMCommandProto.Type.finalizeNewDatanodeVersionCommand + "Ms"); } /** @@ -69,10 +69,10 @@ public void handle(SCMCommand command, OzoneContainer ozoneContainer, invocationCount.incrementAndGet(); final long startTime = Time.monotonicNow(); DatanodeStateMachine dsm = context.getParent(); - final FinalizeNewLayoutVersionCommandProto finalizeCommand = + final FinalizeNewDatanodeVersionCommandProto finalizeCommand = ((FinalizeVersionCommand) command).getProto(); try { - if (finalizeCommand.getFinalizeNewLayoutVersion()) { + if (finalizeCommand.getFinalizeNewDatanodeVersion()) { if (dsm.getVersionManager().needsFinalization()) { LOG.info("Finalize upgrade called."); dsm.getVersionManager().finalizeUpgrade(); @@ -93,7 +93,7 @@ public void handle(SCMCommand command, OzoneContainer ozoneContainer, */ @Override public SCMCommandProto.Type getCommandType() { - return SCMCommandProto.Type.finalizeNewLayoutVersionCommand; + return SCMCommandProto.Type.finalizeNewDatanodeVersionCommand; } /** diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java index e35192daa098..61fb248ef203 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java @@ -37,7 +37,7 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandQueueReportProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerAction; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerActionsProto; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DatanodeVersionProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineAction; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineActionsProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto; @@ -128,13 +128,13 @@ public EndpointStateMachine.EndPointStates call() throws Exception { try { Preconditions.checkState(this.datanodeDetailsProto != null); - LayoutVersionProto versionInfo = toVersionProto( + DatanodeVersionProto versionInfo = toVersionProto( versionManager.getApparentVersion(), versionManager.getSoftwareVersion()); requestBuilder = SCMHeartbeatRequestProto.newBuilder() .setDatanodeDetails(datanodeDetailsProto) - .setDataNodeLayoutVersion(versionInfo); + .setDatanodeVersion(versionInfo); addReports(requestBuilder); addContainerActions(requestBuilder); addPipelineActions(requestBuilder); @@ -363,9 +363,9 @@ private void processResponse(SCMHeartbeatResponseProto response, processCommonCommand(commandResponseProto, setNodeOperationalStateCommand); break; - case finalizeNewLayoutVersionCommand: + case finalizeNewDatanodeVersionCommand: FinalizeVersionCommand finalizeVersionCommand = - FinalizeVersionCommand.getFromProtobuf(commandResponseProto.getFinalizeNewLayoutVersionCommandProto()); + FinalizeVersionCommand.getFromProtobuf(commandResponseProto.getFinalizeNewDatanodeVersionCommandProto()); if (LOG.isDebugEnabled()) { LOG.debug("Received SCM finalize command {}", finalizeVersionCommand.getId()); } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java index cc9466942572..08bff1a422d8 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java @@ -27,7 +27,7 @@ import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DatanodeVersionProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMRegisteredResponseProto; @@ -108,10 +108,10 @@ public EndpointStateMachine.EndPointStates call() throws Exception { if (rpcEndPoint.getState() .equals(EndpointStateMachine.EndPointStates.REGISTER)) { - LayoutVersionProto layoutInfo = LayoutVersionProto.newBuilder() - .setMetadataLayoutVersion( + DatanodeVersionProto versionInfo = DatanodeVersionProto.newBuilder() + .setApparentVersion( versionManager.getApparentVersion().serialize()) - .setSoftwareLayoutVersion( + .setSoftwareVersion( versionManager.getSoftwareVersion().serialize()) .build(); ContainerReportsProto containerReport = @@ -122,7 +122,7 @@ public EndpointStateMachine.EndPointStates call() throws Exception { // TODO : Add responses to the command Queue. SCMRegisteredResponseProto response = rpcEndPoint.getEndPoint() .register(datanodeDetails.getExtendedProtoBufMessage(), - nodeReport, containerReport, pipelineReportsProto, layoutInfo); + nodeReport, containerReport, pipelineReportsProto, versionInfo); Preconditions.assertEquals(datanodeDetails.getUuidString(), response.getDatanodeUUID(), "datanodeID"); Preconditions.assertTrue(!StringUtils.isBlank(response.getClusterID()), "Invalid cluster ID in the response."); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/UpgradeUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/UpgradeUtils.java index 29024891ee41..4901afb7aae6 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/UpgradeUtils.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/UpgradeUtils.java @@ -19,7 +19,7 @@ import org.apache.hadoop.hdds.ComponentVersion; import org.apache.hadoop.hdds.HDDSVersion; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DatanodeVersionProto; /** * Util methods for upgrade. @@ -29,17 +29,18 @@ public final class UpgradeUtils { private UpgradeUtils() { } - public static LayoutVersionProto defaultVersionProto() { + public static DatanodeVersionProto defaultVersionProto() { int softwareVersion = HDDSVersion.SOFTWARE_VERSION.serialize(); - return LayoutVersionProto.newBuilder() - .setMetadataLayoutVersion(softwareVersion) - .setSoftwareLayoutVersion(softwareVersion).build(); + return DatanodeVersionProto.newBuilder() + .setApparentVersion(softwareVersion) + .setSoftwareVersion(softwareVersion).build(); } - public static LayoutVersionProto toVersionProto(ComponentVersion apparentVersion, ComponentVersion softwareVersion) { - return LayoutVersionProto.newBuilder() - .setMetadataLayoutVersion(apparentVersion.serialize()) - .setSoftwareLayoutVersion(softwareVersion.serialize()) + public static DatanodeVersionProto toVersionProto(ComponentVersion apparentVersion, + ComponentVersion softwareVersion) { + return DatanodeVersionProto.newBuilder() + .setApparentVersion(apparentVersion.serialize()) + .setSoftwareVersion(softwareVersion.serialize()) .build(); } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java index a93426213720..1d75ef3208e0 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java @@ -22,7 +22,7 @@ import org.apache.hadoop.hdds.annotation.InterfaceAudience; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ExtendedDatanodeDetailsProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DatanodeVersionProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto; @@ -69,7 +69,7 @@ SCMHeartbeatResponseProto sendHeartbeat(SCMHeartbeatRequestProto heartbeat) * @param extendedDatanodeDetailsProto - extended Datanode Details. * @param nodeReport - Node Report. * @param containerReportsRequestProto - Container Reports. - * @param layoutInfo - Layout Version Information. + * @param versionInfo - Datanode Version Information. * @return SCM Command. */ SCMRegisteredResponseProto register( @@ -77,6 +77,6 @@ SCMRegisteredResponseProto register( NodeReportProto nodeReport, ContainerReportsProto containerReportsRequestProto, PipelineReportsProto pipelineReports, - LayoutVersionProto layoutInfo) throws IOException; + DatanodeVersionProto versionInfo) throws IOException; } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java index 210c87e99bf3..ec728dbf2c26 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java @@ -21,7 +21,7 @@ import org.apache.hadoop.hdds.annotation.InterfaceAudience; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandQueueReportProto; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DatanodeVersionProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMVersionRequestProto; @@ -52,13 +52,13 @@ public interface StorageContainerNodeProtocol { * @param datanodeDetails DatanodeDetails * @param nodeReport NodeReportProto * @param pipelineReport PipelineReportsProto - * @param layoutVersionInfo LayoutVersionProto + * @param versionInfo DatanodeVersionProto * @return SCMRegisteredResponseProto */ RegisteredCommand register(DatanodeDetails datanodeDetails, NodeReportProto nodeReport, PipelineReportsProto pipelineReport, - LayoutVersionProto layoutVersionInfo); + DatanodeVersionProto versionInfo); /** * Send heartbeat to indicate the datanode is alive and doing well. diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/FinalizeVersionCommand.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/FinalizeVersionCommand.java index 181a6f6b1a6b..2e37f1999116 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/FinalizeVersionCommand.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/FinalizeVersionCommand.java @@ -18,31 +18,31 @@ package org.apache.hadoop.ozone.protocol.commands; import java.util.Objects; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.FinalizeNewLayoutVersionCommandProto; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DatanodeVersionProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.FinalizeNewDatanodeVersionCommandProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto; /** * Asks DataNode to finalize new upgrade version. */ public class FinalizeVersionCommand - extends SCMCommand { + extends SCMCommand { private boolean finalizeUpgrade = false; - private LayoutVersionProto versionInfo; + private DatanodeVersionProto versionInfo; - public FinalizeVersionCommand(boolean finalizeNewLayoutVersion, - LayoutVersionProto versionInfo, + public FinalizeVersionCommand(boolean finalizeNewDatanodeVersion, + DatanodeVersionProto versionInfo, long id) { super(id); - finalizeUpgrade = finalizeNewLayoutVersion; + finalizeUpgrade = finalizeNewDatanodeVersion; this.versionInfo = versionInfo; } - public FinalizeVersionCommand(boolean finalizeNewLayoutVersion, - LayoutVersionProto versionInfo) { + public FinalizeVersionCommand(boolean finalizeNewDatanodeVersion, + DatanodeVersionProto versionInfo) { super(); - finalizeUpgrade = finalizeNewLayoutVersion; + finalizeUpgrade = finalizeNewDatanodeVersion; this.versionInfo = versionInfo; } @@ -53,24 +53,24 @@ public FinalizeVersionCommand(boolean finalizeNewLayoutVersion, */ @Override public SCMCommandProto.Type getType() { - return SCMCommandProto.Type.finalizeNewLayoutVersionCommand; + return SCMCommandProto.Type.finalizeNewDatanodeVersionCommand; } @Override - public FinalizeNewLayoutVersionCommandProto getProto() { - return FinalizeNewLayoutVersionCommandProto.newBuilder() - .setFinalizeNewLayoutVersion(finalizeUpgrade) + public FinalizeNewDatanodeVersionCommandProto getProto() { + return FinalizeNewDatanodeVersionCommandProto.newBuilder() + .setFinalizeNewDatanodeVersion(finalizeUpgrade) .setCmdId(getId()) - .setDataNodeLayoutVersion(versionInfo) + .setDatanodeVersion(versionInfo) .build(); } public static FinalizeVersionCommand getFromProtobuf( - FinalizeNewLayoutVersionCommandProto finalizeProto) { + FinalizeNewDatanodeVersionCommandProto finalizeProto) { Objects.requireNonNull(finalizeProto, "finalizeProto == null"); return new FinalizeVersionCommand( - finalizeProto.getFinalizeNewLayoutVersion(), - finalizeProto.getDataNodeLayoutVersion(), finalizeProto.getCmdId()); + finalizeProto.getFinalizeNewDatanodeVersion(), + finalizeProto.getDatanodeVersion(), finalizeProto.getCmdId()); } @Override diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolClientSideTranslatorPB.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolClientSideTranslatorPB.java index 328b2b2e6acd..2e21d7bc2bb3 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolClientSideTranslatorPB.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolClientSideTranslatorPB.java @@ -24,7 +24,7 @@ import java.util.function.Consumer; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ExtendedDatanodeDetailsProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DatanodeVersionProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMDatanodeRequest; @@ -144,7 +144,7 @@ public SCMHeartbeatResponseProto sendHeartbeat( * @param extendedDatanodeDetailsProto - extended Datanode Details * @param nodeReport - Node Report. * @param containerReportsRequestProto - Container Reports. - * @param layoutInfo - Layout Version Information. + * @param versionInfo - Datanode Version Information. * @return SCM Command. */ @Override @@ -153,7 +153,7 @@ public SCMRegisteredResponseProto register( NodeReportProto nodeReport, ContainerReportsProto containerReportsRequestProto, PipelineReportsProto pipelineReportsProto, - LayoutVersionProto layoutInfo) + DatanodeVersionProto versionInfo) throws IOException { SCMRegisterRequestProto.Builder req = SCMRegisterRequestProto.newBuilder(); @@ -161,8 +161,8 @@ public SCMRegisteredResponseProto register( req.setContainerReport(containerReportsRequestProto); req.setPipelineReports(pipelineReportsProto); req.setNodeReport(nodeReport); - if (layoutInfo != null) { - req.setDataNodeLayoutVersion(layoutInfo); + if (versionInfo != null) { + req.setDatanodeVersion(versionInfo); } return submitRequest(Type.Register, (builder) -> builder.setRegisterRequest(req)) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java index 3f91c277bf79..38bc0d32ffd4 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java @@ -25,7 +25,7 @@ import java.io.IOException; import java.util.concurrent.TimeoutException; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DatanodeVersionProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMDatanodeRequest; @@ -71,9 +71,9 @@ public SCMRegisteredResponseProto register( .getContainerReport(); NodeReportProto dnNodeReport = request.getNodeReport(); PipelineReportsProto pipelineReport = request.getPipelineReports(); - LayoutVersionProto versionInfo = null; - if (request.hasDataNodeLayoutVersion()) { - versionInfo = request.getDataNodeLayoutVersion(); + DatanodeVersionProto versionInfo = null; + if (request.hasDatanodeVersion()) { + versionInfo = request.getDatanodeVersion(); } else { // Backward compatibility to make sure old Datanodes can still talk to // SCM. diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java index 902b0ab2168f..ad95e7378226 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java @@ -34,7 +34,7 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandStatusReportsProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DatanodeVersionProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto; @@ -227,7 +227,7 @@ private void sleepIfNeeded() { NodeReportProto nodeReport, ContainerReportsProto containerReportsRequestProto, PipelineReportsProto pipelineReportsProto, - LayoutVersionProto layoutInfo) + DatanodeVersionProto versionInfo) throws IOException { rpcCount.incrementAndGet(); DatanodeDetailsProto datanodeDetailsProto = diff --git a/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto b/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto index 3d08f39992c6..c7a4cd48eac3 100644 --- a/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto +++ b/hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto @@ -77,12 +77,25 @@ enum Status { /** * DataNode will advertise its current layout version through * heartbeat as well as registration mechanism. + * + * Deprecated, please use DatanodeVersionProto instead. The field names + * metadataLayoutVersion and softwareLayoutVersion have been replaced by + * apparentVersion and softwareVersion respectively. */ message LayoutVersionProto { required uint32 metadataLayoutVersion = 1; required uint32 softwareLayoutVersion = 2; } +/** + * Replaces LayoutVersionProto, using the versioning terminology of + * ComponentVersionManager: apparentVersion and softwareVersion. + */ +message DatanodeVersionProto { + optional uint32 apparentVersion = 1; + optional uint32 softwareVersion = 2; +} + /** * Request for version info of the software stack on the server. */ @@ -102,7 +115,8 @@ message SCMRegisterRequestProto { required NodeReportProto nodeReport = 2; required ContainerReportsProto containerReport = 3; required PipelineReportsProto pipelineReports = 4; - optional LayoutVersionProto dataNodeLayoutVersion = 5; + optional LayoutVersionProto dataNodeLayoutVersion = 5 [deprecated = true]; + optional DatanodeVersionProto datanodeVersion = 6; } /** @@ -137,8 +151,9 @@ message SCMHeartbeatRequestProto { optional ContainerActionsProto containerActions = 6; optional PipelineActionsProto pipelineActions = 7; optional PipelineReportsProto pipelineReports = 8; - optional LayoutVersionProto dataNodeLayoutVersion = 9; + optional LayoutVersionProto dataNodeLayoutVersion = 9 [deprecated = true]; optional CommandQueueReportProto commandQueueReport = 10; + optional DatanodeVersionProto datanodeVersion = 11; } message CommandQueueReportProto { @@ -324,10 +339,11 @@ message SCMCommandProto { createPipelineCommand = 6; closePipelineCommand = 7; setNodeOperationalStateCommand = 8; - finalizeNewLayoutVersionCommand = 9; + finalizeNewLayoutVersionCommand = 9 [deprecated = true]; refreshVolumeUsageInfo = 10; reconstructECContainersCommand = 11; reconcileContainerCommand = 12; + finalizeNewDatanodeVersionCommand = 13; } // TODO: once we start using protoc 3.x, refactor this message using "oneof" required Type commandType = 1; @@ -340,10 +356,11 @@ message SCMCommandProto { optional ClosePipelineCommandProto closePipelineCommandProto = 8; optional SetNodeOperationalStateCommandProto setNodeOperationalStateCommandProto = 9; optional FinalizeNewLayoutVersionCommandProto - finalizeNewLayoutVersionCommandProto = 10; + finalizeNewLayoutVersionCommandProto = 10 [deprecated = true]; optional RefreshVolumeUsageCommandProto refreshVolumeUsageCommandProto = 11; optional ReconstructECContainersCommandProto reconstructECContainersCommandProto = 12; optional ReconcileContainerCommandProto reconcileContainerCommandProto = 13; + optional FinalizeNewDatanodeVersionCommandProto finalizeNewDatanodeVersionCommandProto = 14; // If running upon Ratis, holds term of underlying RaftServer iff current // SCM is a leader. If running without Ratis, holds SCMContext.INVALID_TERM. @@ -498,6 +515,9 @@ message ProcessCRLCommandProto { /** * This command asks the DataNode to finalize a new layout version. + * + * Deprecated, please use FinalizeNewDatanodeVersionCommandProto instead, which + * carries a DatanodeVersionProto instead of the deprecated LayoutVersionProto. */ message FinalizeNewLayoutVersionCommandProto { required bool finalizeNewLayoutVersion = 1 [default = false]; @@ -505,6 +525,16 @@ message FinalizeNewLayoutVersionCommandProto { required int64 cmdId = 3; } +/** + * This command asks the DataNode to finalize a new datanode version. + * Replaces FinalizeNewLayoutVersionCommandProto. + */ +message FinalizeNewDatanodeVersionCommandProto { + required bool finalizeNewDatanodeVersion = 1 [default = false]; + optional DatanodeVersionProto datanodeVersion = 2; + required int64 cmdId = 3; +} + /** This command asks the datanode to reconcile its copy of a container with its peer datanodes that also have a copy of the container. diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeInfo.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeInfo.java index 54e996ec3bf2..160f788c90c7 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeInfo.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeInfo.java @@ -27,7 +27,7 @@ import org.apache.hadoop.hdds.ComponentVersion; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandQueueReportProto; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DatanodeVersionProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.MetadataStorageReportProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto; @@ -68,17 +68,17 @@ public class DatanodeInfo extends DatanodeDetails { * Constructs DatanodeInfo from DatanodeDetails. * * @param datanodeDetails Details about the datanode - * @param versionInfo Details about the LayoutVersionProto + * @param versionInfo Details about the DatanodeVersionProto */ public DatanodeInfo(DatanodeDetails datanodeDetails, NodeStatus nodeStatus, - LayoutVersionProto versionInfo, long containerRollIntervalMs) { + DatanodeVersionProto versionInfo, long containerRollIntervalMs) { super(datanodeDetails); this.lock = new ReentrantReadWriteLock(); this.lastHeartbeatTime = Time.monotonicNow(); this.lastKnownSoftwareVersion = - HDDSVersionUtils.deserializeHDDSVersionOrLayoutVersion(versionInfo.getSoftwareLayoutVersion()); + HDDSVersionUtils.deserializeHDDSVersionOrLayoutVersion(versionInfo.getSoftwareVersion()); this.lastKnownApparentVersion = - HDDSVersionUtils.deserializeHDDSVersionOrLayoutVersion(versionInfo.getMetadataLayoutVersion()); + HDDSVersionUtils.deserializeHDDSVersionOrLayoutVersion(versionInfo.getApparentVersion()); this.storageReports = Collections.emptyList(); this.nodeStatus = nodeStatus; this.metadataStorageReports = Collections.emptyList(); @@ -112,16 +112,16 @@ public void updateLastHeartbeatTime(long milliSecondsSinceEpoch) { /** * Updates the last known apparent and software versions for this datanode. */ - public void updateLastKnownVersions(LayoutVersionProto version) { + public void updateLastKnownVersions(DatanodeVersionProto version) { if (version == null) { return; } try { lock.writeLock().lock(); lastKnownSoftwareVersion = - HDDSVersionUtils.deserializeHDDSVersionOrLayoutVersion(version.getSoftwareLayoutVersion()); + HDDSVersionUtils.deserializeHDDSVersionOrLayoutVersion(version.getSoftwareVersion()); lastKnownApparentVersion = - HDDSVersionUtils.deserializeHDDSVersionOrLayoutVersion(version.getMetadataLayoutVersion()); + HDDSVersionUtils.deserializeHDDSVersionOrLayoutVersion(version.getApparentVersion()); } finally { lock.writeLock().unlock(); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java index 96eafa0b2aa8..e8512363c992 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java @@ -32,7 +32,7 @@ import org.apache.hadoop.hdds.protocol.DatanodeID; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DatanodeVersionProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto; @@ -375,10 +375,10 @@ void processNodeReport(DatanodeDetails datanodeDetails, * Process Node LayoutVersion report. * * @param datanodeDetails - * @param layoutReport + * @param versionReport */ void processVersionReport(DatanodeDetails datanodeDetails, - LayoutVersionProto layoutReport); + DatanodeVersionProto versionReport); /** * Get the number of commands of the given type queued on the datanode at the diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java index e23d3f33c102..273edc5fc6ae 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java @@ -44,7 +44,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DatanodeVersionProto; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.events.SCMEvents; @@ -232,24 +232,24 @@ private void initializeStateMachines() { * Adds a new node to the state manager. * * @param datanodeDetails DatanodeDetails - * @param layoutInfo LayoutVersionProto + * @param versionInfo DatanodeVersionProto * * @throws NodeAlreadyExistsException if the node is already present */ public void addNode(DatanodeDetails datanodeDetails, - LayoutVersionProto layoutInfo) throws NodeAlreadyExistsException { - nodeStateMap.addNode(newDatanodeInfo(datanodeDetails, layoutInfo)); + DatanodeVersionProto versionInfo) throws NodeAlreadyExistsException { + nodeStateMap.addNode(newDatanodeInfo(datanodeDetails, versionInfo)); try { - updateLastKnownVersionInfo(datanodeDetails, layoutInfo); + updateLastKnownVersionInfo(datanodeDetails, versionInfo); } catch (NodeNotFoundException ex) { throw new IllegalStateException("Inconsistent NodeStateMap! Datanode " + datanodeDetails.getID() + " was added but not found in map: " + nodeStateMap); } } - private DatanodeInfo newDatanodeInfo(DatanodeDetails datanode, LayoutVersionProto layout) { + private DatanodeInfo newDatanodeInfo(DatanodeDetails datanode, DatanodeVersionProto versionInfo) { final NodeStatus status = newNodeStatus(datanode); - return new DatanodeInfo(datanode, status, layout, containerRollIntervalMs); + return new DatanodeInfo(datanode, status, versionInfo, containerRollIntervalMs); } /** @@ -320,31 +320,31 @@ public void updateLastHeartbeatTime(DatanodeDetails datanodeDetails) /** * Updates the last known layout version of the node. * @param datanodeDetails DataNode Details - * @param layoutInfo DataNode Layout Information + * @param versionInfo DataNode Version Information * * @throws NodeNotFoundException if the node is not present */ public void updateLastKnownVersionInfo(DatanodeDetails datanodeDetails, - LayoutVersionProto layoutInfo) throws NodeNotFoundException { + DatanodeVersionProto versionInfo) throws NodeNotFoundException { nodeStateMap.getNodeInfo(datanodeDetails.getID()) - .updateLastKnownVersions(layoutInfo); + .updateLastKnownVersions(versionInfo); } /** * Update node. * * @param datanodeDetails the datanode details - * @param layoutInfo the layoutInfo + * @param versionInfo the datanode version information * @throws NodeNotFoundException the node not found exception */ public void updateNode(DatanodeDetails datanodeDetails, - LayoutVersionProto layoutInfo) + DatanodeVersionProto versionInfo) throws NodeNotFoundException { - final DatanodeInfo newInfo = newDatanodeInfo(datanodeDetails, layoutInfo); + final DatanodeInfo newInfo = newDatanodeInfo(datanodeDetails, versionInfo); final DatanodeInfo oldInfo = nodeStateMap.updateNode(newInfo); LOG.info("Updated datanode {} {} to {} {}", oldInfo, oldInfo.getNodeStatus(), newInfo, newInfo.getNodeStatus()); - updateLastKnownVersionInfo(datanodeDetails, layoutInfo); + updateLastKnownVersionInfo(datanodeDetails, versionInfo); } /** diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java index 3bcb7fa7f178..135a201bfa7d 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java @@ -60,7 +60,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.StorageTypeProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandQueueReportProto; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DatanodeVersionProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.MetadataStorageReportProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto; @@ -381,9 +381,9 @@ public RegisteredCommand register( DatanodeDetails datanodeDetails, NodeReportProto nodeReport, PipelineReportsProto pipelineReportsProto) { return register(datanodeDetails, nodeReport, pipelineReportsProto, - LayoutVersionProto.newBuilder() - .setMetadataLayoutVersion(versionManager.getApparentVersion().serialize()) - .setSoftwareLayoutVersion(versionManager.getSoftwareVersion().serialize()) + DatanodeVersionProto.newBuilder() + .setApparentVersion(versionManager.getApparentVersion().serialize()) + .setSoftwareVersion(versionManager.getSoftwareVersion().serialize()) .build()); } @@ -403,7 +403,7 @@ public RegisteredCommand register( public RegisteredCommand register( DatanodeDetails datanodeDetails, NodeReportProto nodeReport, PipelineReportsProto pipelineReportsProto, - LayoutVersionProto dnVersionInfo) { + DatanodeVersionProto dnVersionInfo) { if (shouldFenceDatanode(datanodeDetails, dnVersionInfo)) { return RegisteredCommand.newBuilder() .setErrorCode(ErrorCode.errorNodeNotPermitted) @@ -733,7 +733,7 @@ public void processNodeReport(DatanodeDetails datanodeDetails, */ @Override public void processVersionReport(DatanodeDetails datanodeDetails, - LayoutVersionProto versionReport) { + DatanodeVersionProto versionReport) { if (LOG.isDebugEnabled()) { LOG.debug("Processing version report from [datanode={}]", datanodeDetails.getHostName()); @@ -757,11 +757,11 @@ public void processVersionReport(DatanodeDetails datanodeDetails, } protected void sendFinalizeToDatanodeIfNeeded(DatanodeDetails datanodeDetails, - LayoutVersionProto versionReport) { + DatanodeVersionProto versionReport) { ComponentVersion dnSoftwareVersion = HDDSVersionUtils.deserializeHDDSVersionOrLayoutVersion( - versionReport.getSoftwareLayoutVersion()); + versionReport.getSoftwareVersion()); ComponentVersion dnApparentVersion = HDDSVersionUtils.deserializeHDDSVersionOrLayoutVersion( - versionReport.getMetadataLayoutVersion()); + versionReport.getApparentVersion()); ComponentVersion scmSoftwareVersion = versionManager.getSoftwareVersion(); ComponentVersion scmApparentVersion = versionManager.getApparentVersion(); @@ -796,9 +796,9 @@ protected void sendFinalizeToDatanodeIfNeeded(DatanodeDetails datanodeDetails, FinalizeVersionCommand finalizeCmd = new FinalizeVersionCommand(true, - LayoutVersionProto.newBuilder() - .setSoftwareLayoutVersion(dnSoftwareVersion.serialize()) - .setMetadataLayoutVersion(dnSoftwareVersion.serialize()).build()); + DatanodeVersionProto.newBuilder() + .setSoftwareVersion(dnSoftwareVersion.serialize()) + .setApparentVersion(dnSoftwareVersion.serialize()).build()); try { finalizeCmd.setTerm(scmContext.getTermOfLeader()); // Send Finalize command to the data node. It's OK to send Finalize command multiple times. @@ -2041,11 +2041,11 @@ public void removeNode(DatanodeDetails datanodeDetails) throws NodeNotFoundExcep } } - protected boolean shouldFenceDatanode(DatanodeDetails dnDetails, LayoutVersionProto versionReport) { + protected boolean shouldFenceDatanode(DatanodeDetails dnDetails, DatanodeVersionProto versionReport) { ComponentVersion dnSoftwareVersion = HDDSVersionUtils.deserializeHDDSVersionOrLayoutVersion( - versionReport.getSoftwareLayoutVersion()); + versionReport.getSoftwareVersion()); ComponentVersion dnApparentVersion = HDDSVersionUtils.deserializeHDDSVersionOrLayoutVersion( - versionReport.getMetadataLayoutVersion()); + versionReport.getApparentVersion()); return shouldFenceDatanode(dnDetails, dnSoftwareVersion, dnApparentVersion); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java index efb7a5b81166..1c45434cdbb1 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java @@ -37,8 +37,8 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandStatusReportsProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerActionsProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DatanodeVersionProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.IncrementalContainerReportProto; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineActionsProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto; @@ -97,13 +97,13 @@ public List> dispatch(SCMHeartbeatRequestProto heartbeat) { } else { - LayoutVersionProto versionInfo = null; - if (!heartbeat.hasDataNodeLayoutVersion()) { + DatanodeVersionProto versionInfo = null; + if (!heartbeat.hasDatanodeVersion()) { // Backward compatibility to make sure old Datanodes can still talk to // SCM. versionInfo = toVersionProto(INITIAL_VERSION, INITIAL_VERSION); } else { - versionInfo = heartbeat.getDataNodeLayoutVersion(); + versionInfo = heartbeat.getDatanodeVersion(); } LOG.debug("Processing DataNode Layout Report."); @@ -251,13 +251,13 @@ public Map getCommandsToBeSent() { } /** - * Layout report event payload with origin. + * Datanode version report event payload with origin. */ - public static class LayoutReportFromDatanode - extends ReportFromDatanode { + public static class VersionReportFromDatanode + extends ReportFromDatanode { - public LayoutReportFromDatanode(DatanodeDetails datanodeDetails, - LayoutVersionProto report) { + public VersionReportFromDatanode(DatanodeDetails datanodeDetails, + DatanodeVersionProto report) { super(datanodeDetails, report); } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java index f1c4c78168f3..7ca8adccb28b 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java @@ -22,7 +22,7 @@ import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type.createPipelineCommand; import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type.deleteBlocksCommand; import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type.deleteContainerCommand; -import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type.finalizeNewLayoutVersionCommand; +import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type.finalizeNewDatanodeVersionCommand; import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type.reconcileContainerCommand; import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type.reconstructECContainersCommand; import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type.refreshVolumeUsageInfo; @@ -58,7 +58,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DatanodeVersionProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ReconstructECContainersCommandProto; @@ -230,7 +230,7 @@ public SCMRegisteredResponseProto register( NodeReportProto nodeReport, ContainerReportsProto containerReportsProto, PipelineReportsProto pipelineReportsProto, - LayoutVersionProto layoutInfo) + DatanodeVersionProto versionInfo) throws IOException { DatanodeDetails datanodeDetails = DatanodeDetails .getFromProtoBuf(extendedDatanodeDetailsProto); @@ -241,7 +241,7 @@ public SCMRegisteredResponseProto register( // TODO : Return the list of Nodes that forms the SCM HA. RegisteredCommand registeredCommand = scm.getScmNodeManager() .register(datanodeDetails, nodeReport, pipelineReportsProto, - layoutInfo); + versionInfo); if (registeredCommand.getError() == SCMRegisteredResponseProto.ErrorCode.success) { eventPublisher.fireEvent(CONTAINER_REPORT, @@ -419,10 +419,10 @@ public static SCMCommandProto getCommandResponse(SCMCommand cmd, .setSetNodeOperationalStateCommandProto( ((SetNodeOperationalStateCommand)cmd).getProto()) .build(); - case finalizeNewLayoutVersionCommand: + case finalizeNewDatanodeVersionCommand: return builder - .setCommandType(finalizeNewLayoutVersionCommand) - .setFinalizeNewLayoutVersionCommandProto( + .setCommandType(finalizeNewDatanodeVersionCommand) + .setFinalizeNewDatanodeVersionCommandProto( ((FinalizeVersionCommand)cmd).getProto()) .build(); case refreshVolumeUsageInfo: diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java index 4cb0eda95155..04ee7d2b788c 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java @@ -44,7 +44,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandQueueReportProto; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DatanodeVersionProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.MetadataStorageReportProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto; @@ -653,11 +653,11 @@ public void processNodeReport(DatanodeDetails dnUuid, * Empty implementation for processLayoutVersionReport. * * @param dnUuid - * @param layoutReport + * @param versionReport */ @Override public void processVersionReport(DatanodeDetails dnUuid, - LayoutVersionProto layoutReport) { + DatanodeVersionProto versionReport) { // do nothing } @@ -793,9 +793,9 @@ public VersionResponse getVersion(SCMVersionRequestProto versionRequest) { public RegisteredCommand register(DatanodeDetails datanodeDetails, NodeReportProto nodeReport, PipelineReportsProto pipelineReportsProto, - LayoutVersionProto layoutInfo) { + DatanodeVersionProto versionInfo) { final DatanodeInfo info = new DatanodeInfo(datanodeDetails, - NodeStatus.inServiceHealthy(), layoutInfo, HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); + NodeStatus.inServiceHealthy(), versionInfo, HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT); try { node2ContainerMap.addNode(info); addEntryTodnsToUuidMap(datanodeDetails.getIpAddress(), diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java index aa2f1ef94bcd..aea96de4e701 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java @@ -30,7 +30,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandQueueReportProto; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DatanodeVersionProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto; @@ -308,7 +308,7 @@ public void processNodeReport(DatanodeDetails datanodeDetails, @Override public void processVersionReport(DatanodeDetails datanodeDetails, - LayoutVersionProto layoutReport) { + DatanodeVersionProto versionReport) { } /** @@ -436,7 +436,7 @@ public VersionResponse getVersion( public RegisteredCommand register(DatanodeDetails datanodeDetails, NodeReportProto nodeReport, PipelineReportsProto pipelineReport, - LayoutVersionProto layoutreport) { + DatanodeVersionProto versionReport) { return null; } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java index 48b1b813f445..42e811639aca 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java @@ -296,7 +296,7 @@ public void testUpdateNode() throws NodeAlreadyExistsException, UUID dnUuid = UUID.randomUUID(); String ipAddress = "1.2.3.4"; String hostName = "test-host"; - StorageContainerDatanodeProtocolProtos.LayoutVersionProto + StorageContainerDatanodeProtocolProtos.DatanodeVersionProto layoutVersionProto = UpgradeUtils.toVersionProto(HDDSLayoutFeature.INITIAL_VERSION, HDDSLayoutFeature.INITIAL_VERSION); DatanodeDetails dn = DatanodeDetails.newBuilder() @@ -309,7 +309,7 @@ public void testUpdateNode() throws NodeAlreadyExistsException, String newIpAddress = "2.3.4.5"; String newHostName = "new-host"; - StorageContainerDatanodeProtocolProtos.LayoutVersionProto + StorageContainerDatanodeProtocolProtos.DatanodeVersionProto newLayoutVersionProto = UpgradeUtils.defaultVersionProto(); DatanodeDetails newDn = DatanodeDetails.newBuilder() .setUuid(dnUuid) diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java index 8b0ea537d4e4..fb8ac14dcd23 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java @@ -26,7 +26,7 @@ import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.createDatanodeDetails; import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.STALE; -import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type.finalizeNewLayoutVersionCommand; +import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type.finalizeNewDatanodeVersionCommand; import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMRegisteredResponseProto.ErrorCode.errorNodeNotPermitted; import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMRegisteredResponseProto.ErrorCode.success; import static org.apache.hadoop.hdds.scm.HddsTestUtils.getRandomPipelineReports; @@ -78,7 +78,7 @@ import org.apache.hadoop.hdds.protocol.MockDatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandQueueReportProto; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DatanodeVersionProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.MetadataStorageReportProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto; @@ -266,7 +266,7 @@ private DatanodeDetails registerWithCapacity(SCMNodeManager nodeManager) { * @return The created {@link DatanodeDetails}. */ private DatanodeDetails registerWithCapacity(SCMNodeManager nodeManager, - LayoutVersionProto layout, ErrorCode expectedResult) { + DatanodeVersionProto versionInfo, ErrorCode expectedResult) { DatanodeDetails details = MockDatanodeDetails.randomDatanodeDetails(); StorageReportProto storageReport = @@ -280,7 +280,7 @@ private DatanodeDetails registerWithCapacity(SCMNodeManager nodeManager, MockDatanodeDetails.randomDatanodeDetails(), HddsTestUtils.createNodeReport(Arrays.asList(storageReport), Arrays.asList(metadataStorageReport)), - getRandomPipelineReports(), layout); + getRandomPipelineReports(), versionInfo); assertEquals(expectedResult, cmd.getError()); return cmd.getDatanode(); @@ -630,9 +630,9 @@ public void testDatanodeFinalizedCounterTracksVersionReports() // Report a pre-finalized datanode. int softwareVersion = HDDSVersion.SOFTWARE_VERSION.serialize(); nodeManager.processVersionReport(node, - LayoutVersionProto.newBuilder() - .setMetadataLayoutVersion(HDDSLayoutFeature.INITIAL_VERSION.serialize()) - .setSoftwareLayoutVersion(softwareVersion) + DatanodeVersionProto.newBuilder() + .setApparentVersion(HDDSLayoutFeature.INITIAL_VERSION.serialize()) + .setSoftwareVersion(softwareVersion) .build()); assertEquals(0, nodeManager.getDatanodeFinalizationCounts() .getNumFinalizedDatanodes(), @@ -640,9 +640,9 @@ public void testDatanodeFinalizedCounterTracksVersionReports() // Report a finalized datanode. nodeManager.processVersionReport(node, - LayoutVersionProto.newBuilder() - .setMetadataLayoutVersion(softwareVersion) - .setSoftwareLayoutVersion(softwareVersion) + DatanodeVersionProto.newBuilder() + .setApparentVersion(softwareVersion) + .setSoftwareVersion(softwareVersion) .build()); assertEquals(1, nodeManager.getDatanodeFinalizationCounts() .getNumFinalizedDatanodes(), @@ -661,7 +661,7 @@ public void testDatanodeFinalizedCounterTracksRegistrationAndRemoveNode() "Finalized registration should increment finalized count"); - LayoutVersionProto preFinalizedVersionProto = + DatanodeVersionProto preFinalizedVersionProto = toVersionProto(HDDSLayoutFeature.SCM_HA, HDDSVersion.SOFTWARE_VERSION); DatanodeDetails nonFinalizedNode = registerWithCapacity(nodeManager, preFinalizedVersionProto, success); @@ -710,9 +710,9 @@ private static Stream scmDatanodeVersionCombinations() { errorNodeNotPermitted, false), // Newer DN rejected, even though its apparent version matches SCM. Arguments.of(HDDSLayoutFeature.INITIAL_VERSION, - LayoutVersionProto.newBuilder() - .setMetadataLayoutVersion(HDDSLayoutFeature.INITIAL_VERSION.serialize()) - .setSoftwareLayoutVersion(HDDSVersion.SOFTWARE_VERSION.serialize() + 1).build(), + DatanodeVersionProto.newBuilder() + .setApparentVersion(HDDSLayoutFeature.INITIAL_VERSION.serialize()) + .setSoftwareVersion(HDDSVersion.SOFTWARE_VERSION.serialize() + 1).build(), errorNodeNotPermitted, false), /* SCM FINALIZED */ @@ -731,16 +731,16 @@ private static Stream scmDatanodeVersionCombinations() { success, false), // Newer DN rejected, even though its apparent version matches SCM. Arguments.of(HDDSVersion.SOFTWARE_VERSION, - LayoutVersionProto.newBuilder() - .setMetadataLayoutVersion(HDDSVersion.SOFTWARE_VERSION.serialize()) - .setSoftwareLayoutVersion(HDDSVersion.SOFTWARE_VERSION.serialize() + 1).build(), + DatanodeVersionProto.newBuilder() + .setApparentVersion(HDDSVersion.SOFTWARE_VERSION.serialize()) + .setSoftwareVersion(HDDSVersion.SOFTWARE_VERSION.serialize() + 1).build(), errorNodeNotPermitted, false) ); } @ParameterizedTest @MethodSource("scmDatanodeVersionCombinations") - public void testDatanodeFencingOnRegister(ComponentVersion scmApparent, LayoutVersionProto dnVersionProto, + public void testDatanodeFencingOnRegister(ComponentVersion scmApparent, DatanodeVersionProto dnVersionProto, ErrorCode expectedResult, boolean expectFinalizeCmd) throws IOException { OzoneConfiguration conf = new OzoneConfiguration(); SCMStorageConfig scmStorageConfig = mock(SCMStorageConfig.class); @@ -765,7 +765,7 @@ public void testDatanodeFencingOnRegister(ComponentVersion scmApparent, LayoutVe if (expectFinalizeCmd) { verify(eventPublisher, times(1)).fireEvent(eq(DATANODE_COMMAND), captor.capture()); assertEquals(node.getID(), captor.getValue().getDatanodeId()); - assertEquals(finalizeNewLayoutVersionCommand, captor.getValue().getCommand().getType()); + assertEquals(finalizeNewDatanodeVersionCommand, captor.getValue().getCommand().getType()); } else { verify(eventPublisher, times(0)).fireEvent(eq(DATANODE_COMMAND), captor.capture()); } @@ -871,7 +871,7 @@ scmStorageConfig, eventPublisher, new NetworkTopologyImpl(conf), DatanodeDetails node1 = MockDatanodeDetails.randomDatanodeDetails(); StorageReportProto storageReport = HddsTestUtils.createStorageReport( node1.getID(), node1.getNetworkFullPath(), Long.MAX_VALUE); - LayoutVersionProto preFinalizedDNVersion = + DatanodeVersionProto preFinalizedDNVersion = toVersionProto(HDDSLayoutFeature.INITIAL_VERSION, HDDSVersion.SOFTWARE_VERSION); nodeManager.register(node1, HddsTestUtils.createNodeReport(Collections.singletonList(storageReport), emptyList()), diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconNodeManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconNodeManager.java index 143dcef2f0a2..316e5793339c 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconNodeManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconNodeManager.java @@ -36,7 +36,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandQueueReportProto; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DatanodeVersionProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type; @@ -211,7 +211,7 @@ public void refreshAllHealthyDnUsageInfo() { public RegisteredCommand register( DatanodeDetails datanodeDetails, NodeReportProto nodeReport, PipelineReportsProto pipelineReportsProto, - LayoutVersionProto dnVersionInfo) { + DatanodeVersionProto dnVersionInfo) { if (isNodeRegistered(datanodeDetails)) { try { nodeDB.put(datanodeDetails.getID(), datanodeDetails); @@ -297,7 +297,7 @@ public void removeNode(DatanodeDetails datanodeDetails) throws NodeNotFoundExcep @Override protected void sendFinalizeToDatanodeIfNeeded(DatanodeDetails datanodeDetails, - LayoutVersionProto versionReport) { + DatanodeVersionProto versionReport) { // Recon will not send finalize commands to datanodes. } @@ -308,11 +308,11 @@ protected void sendFinalizeToDatanodeIfNeeded(DatanodeDetails datanodeDetails, * it has finalized, unlike SCM. */ @Override - protected boolean shouldFenceDatanode(DatanodeDetails dnDetails, LayoutVersionProto versionReport) { + protected boolean shouldFenceDatanode(DatanodeDetails dnDetails, DatanodeVersionProto versionReport) { ComponentVersion dnSoftwareVersion = HDDSVersionUtils.deserializeHDDSVersionOrLayoutVersion( - versionReport.getSoftwareLayoutVersion()); + versionReport.getSoftwareVersion()); ComponentVersion dnApparentVersion = HDDSVersionUtils.deserializeHDDSVersionOrLayoutVersion( - versionReport.getMetadataLayoutVersion()); + versionReport.getApparentVersion()); ComponentVersion reconSoftwareVersion = versionManager.getSoftwareVersion(); ComponentVersion reconApparentVersion = versionManager.getApparentVersion(); diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java index f44d16d02ac3..d4e73ca32b3c 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java @@ -82,7 +82,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.StorageTypeProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DatanodeVersionProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReport; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto; @@ -428,7 +428,7 @@ public void setUp() throws Exception { NodeReportProto.newBuilder() .addStorageReport(storageReportProto3) .addStorageReport(storageReportProto4).build(); - LayoutVersionProto layoutInfo = defaultVersionProto(); + DatanodeVersionProto versionInfo = defaultVersionProto(); DatanodeDetailsProto datanodeDetailsProto3 = DatanodeDetailsProto.newBuilder() @@ -467,7 +467,7 @@ public void setUp() throws Exception { assertDoesNotThrow(() -> { reconScm.getDatanodeProtocolServer() .register(extendedDatanodeDetailsProto, nodeReportProto, - containerReportsProto, pipelineReportsProto, layoutInfo); + containerReportsProto, pipelineReportsProto, versionInfo); reconScm.getDatanodeProtocolServer() .register(extendedDatanodeDetailsProto2, nodeReportProto2, ContainerReportsProto.newBuilder().build(), @@ -1277,7 +1277,7 @@ private void waitAndCheckConditionAfterHeartbeat(Callable check) .setContainerReport(containerReportsProto) .setDatanodeDetails(extendedDatanodeDetailsProto .getDatanodeDetails()) - .setDataNodeLayoutVersion(defaultVersionProto()) + .setDatanodeVersion(defaultVersionProto()) .build(); reconScm.getDatanodeProtocolServer().sendHeartbeat(heartbeatRequestProto); LambdaTestUtils.await(30000, 1000, check); diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconNodeManager.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconNodeManager.java index df3938b7aa7b..b8dfa05c338b 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconNodeManager.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconNodeManager.java @@ -50,7 +50,7 @@ import org.apache.hadoop.hdds.protocol.DatanodeID; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DatanodeVersionProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMRegisteredResponseProto.ErrorCode; import org.apache.hadoop.hdds.scm.net.NetworkTopology; @@ -271,9 +271,9 @@ private static Stream reconDatanodeVersionCombinations() { errorNodeNotPermitted), // Newer SW DN rejected Arguments.of(HDDSLayoutFeature.INITIAL_VERSION, - LayoutVersionProto.newBuilder() - .setMetadataLayoutVersion(HDDSLayoutFeature.INITIAL_VERSION.serialize()) - .setSoftwareLayoutVersion(HDDSVersion.SOFTWARE_VERSION.serialize() + 1).build(), + DatanodeVersionProto.newBuilder() + .setApparentVersion(HDDSLayoutFeature.INITIAL_VERSION.serialize()) + .setSoftwareVersion(HDDSVersion.SOFTWARE_VERSION.serialize() + 1).build(), errorNodeNotPermitted), /* RECON FINALIZED */ @@ -292,9 +292,9 @@ private static Stream reconDatanodeVersionCombinations() { success), // Newer SW DN rejected Arguments.of(HDDSVersion.SOFTWARE_VERSION, - LayoutVersionProto.newBuilder() - .setMetadataLayoutVersion(HDDSVersion.SOFTWARE_VERSION.serialize()) - .setSoftwareLayoutVersion(HDDSVersion.SOFTWARE_VERSION.serialize() + 1).build(), + DatanodeVersionProto.newBuilder() + .setApparentVersion(HDDSVersion.SOFTWARE_VERSION.serialize()) + .setSoftwareVersion(HDDSVersion.SOFTWARE_VERSION.serialize() + 1).build(), errorNodeNotPermitted) ); } @@ -302,7 +302,7 @@ private static Stream reconDatanodeVersionCombinations() { @ParameterizedTest @MethodSource("reconDatanodeVersionCombinations") public void testDatanodeFencingOnRegister(ComponentVersion reconApparent, - LayoutVersionProto dnVersionProto, ErrorCode expectedResult) throws IOException { + DatanodeVersionProto dnVersionProto, ErrorCode expectedResult) throws IOException { ReconStorageConfig scmStorageConfig = new ReconStorageConfig(conf, new ReconUtils()); EventQueue eventQueue = new EventQueue(); NetworkTopology clusterMap = new NetworkTopologyImpl(conf); diff --git a/hadoop-ozone/vapor/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulationState.java b/hadoop-ozone/vapor/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulationState.java index b9809677855f..6b02a5e8c202 100644 --- a/hadoop-ozone/vapor/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulationState.java +++ b/hadoop-ozone/vapor/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulationState.java @@ -41,8 +41,8 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DatanodeVersionProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.IncrementalContainerReportProto; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto; @@ -126,12 +126,12 @@ public synchronized void ackHeartbeatResponse( } public synchronized SCMHeartbeatRequestProto heartbeatRequest( - InetSocketAddress endpoint, LayoutVersionProto layoutInfo) + InetSocketAddress endpoint, DatanodeVersionProto versionInfo) throws IOException { SCMHeartbeatRequestProto.Builder builder = SCMHeartbeatRequestProto.newBuilder() .setDatanodeDetails(datanodeDetails.getProtoBufMessage()) - .setDataNodeLayoutVersion(layoutInfo) + .setDatanodeVersion(versionInfo) .setNodeReport(createNodeReport()) .setPipelineReports(createPipelineReport()); diff --git a/hadoop-ozone/vapor/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java b/hadoop-ozone/vapor/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java index 4939557f07da..4e46d68d8a08 100644 --- a/hadoop-ozone/vapor/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java +++ b/hadoop-ozone/vapor/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java @@ -59,7 +59,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DatanodeVersionProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto; @@ -135,7 +135,7 @@ public class DatanodeSimulator implements Callable, VaporSubcommand { private Map datanodesMap; private ScheduledExecutorService heartbeatScheduler; - private LayoutVersionProto layoutInfo; + private DatanodeVersionProto versionInfo; @CommandLine.ParentCommand private Freon freonCommand; @@ -396,7 +396,7 @@ private void heartbeat(InetSocketAddress endpoint, DatanodeSimulationState dn) { try { SCMHeartbeatRequestProto heartbeat = dn.heartbeatRequest(endpoint, - layoutInfo); + versionInfo); SCMHeartbeatResponseProto response = client.sendHeartbeat(heartbeat); dn.ackHeartbeatResponse(response); @@ -415,7 +415,7 @@ private void heartbeat(InetSocketAddress endpoint, client.register( dn.getDatanodeDetails().getExtendedProtoBufMessage(), dn.createNodeReport(), dn.createFullContainerReport(), - dn.createPipelineReport(), this.layoutInfo); + dn.createPipelineReport(), this.versionInfo); } } catch (Exception e) { LOGGER.info("Error sending heartbeat for {}: {}", @@ -439,19 +439,19 @@ private void init() throws IOException { scmContainerClient = HAUtils.getScmContainerClient(conf); - this.layoutInfo = createLayoutInfo(); + this.versionInfo = createVersionInfo(); } - private LayoutVersionProto createLayoutInfo() throws IOException { + private DatanodeVersionProto createVersionInfo() throws IOException { DatanodeStorage layoutStorage = new DatanodeStorage(conf, UUID.randomUUID().toString()); DatanodeVersionManager versionManager = new DatanodeVersionManager(layoutStorage, null); - return LayoutVersionProto.newBuilder() - .setMetadataLayoutVersion( + return DatanodeVersionProto.newBuilder() + .setApparentVersion( versionManager.getApparentVersion().serialize()) - .setSoftwareLayoutVersion( + .setSoftwareVersion( versionManager.getSoftwareVersion().serialize()) .build(); } @@ -492,7 +492,7 @@ private boolean registerDataNode(DatanodeSimulationState dn) SCMRegisteredResponseProto response = client.register( dn.getDatanodeDetails().getExtendedProtoBufMessage(), - nodeReport, containerReports, pipelineReports, this.layoutInfo); + nodeReport, containerReports, pipelineReports, this.versionInfo); if (response.hasHostname() && response.hasIpAddress()) { dn.getDatanodeDetails().setHostName(response.getHostname()); dn.getDatanodeDetails().setIpAddress(response.getIpAddress()); @@ -512,7 +512,7 @@ private boolean registerDataNode(DatanodeSimulationState dn) try { reconClient.register(dn.getDatanodeDetails().getExtendedProtoBufMessage(), - nodeReport, containerReports, pipelineReports, this.layoutInfo); + nodeReport, containerReports, pipelineReports, this.versionInfo); } catch (IOException e) { LOGGER.error("Error register datanode to Recon", e); } diff --git a/hadoop-ozone/vapor/src/main/java/org/apache/hadoop/ozone/freon/SCMThroughputBenchmark.java b/hadoop-ozone/vapor/src/main/java/org/apache/hadoop/ozone/freon/SCMThroughputBenchmark.java index 11d3babe39f9..127c305e57b2 100644 --- a/hadoop-ozone/vapor/src/main/java/org/apache/hadoop/ozone/freon/SCMThroughputBenchmark.java +++ b/hadoop-ozone/vapor/src/main/java/org/apache/hadoop/ozone/freon/SCMThroughputBenchmark.java @@ -816,7 +816,7 @@ public void sendHeartbeat() throws IOException, TimeoutException { .newBuilder() .setDatanodeDetails(datanodeDetails.getProtoBufMessage()) .setContainerReport(containerReport) - .setDataNodeLayoutVersion(UpgradeUtils.defaultVersionProto()) + .setDatanodeVersion(UpgradeUtils.defaultVersionProto()) .build(); datanodeScmClient.sendHeartbeat(heartbeatRequest); // scm commands are ignored