This is an automated email from the ASF dual-hosted git repository.
avijayan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new ca25193 HDDS-4177. SCM Container DB bootstrap on Recon startup (#2942)
ca25193 is described below
commit ca25193cc7545f05ae1a12ed389ffc30eedec30c
Author: Aswin Shakil Balasubramanian <[email protected]>
AuthorDate: Fri Jan 7 11:12:21 2022 -0800
HDDS-4177. SCM Container DB bootstrap on Recon startup (#2942)
---
.../protocol/StorageContainerLocationProtocol.java | 2 +
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 +-
.../common/src/main/resources/ozone-default.xml | 35 ++++++
...inerLocationProtocolClientSideTranslatorPB.java | 14 +++
.../src/main/proto/ScmAdminProtocol.proto | 10 ++
...inerLocationProtocolServerSideTranslatorPB.java | 16 +++
.../hdds/scm/server/SCMClientProtocolServer.java | 5 +
.../server/StorageContainerManagerHttpServer.java | 4 +-
.../hadoop/ozone/om/helpers/OMNodeDetails.java | 6 +-
.../dist/src/main/compose/ozone-csi/docker-config | 1 +
.../dist/src/main/compose/ozone/docker-config | 1 +
.../src/main/compose/ozonesecure-mr/docker-config | 1 +
.../src/main/compose/ozonesecure/docker-config | 1 +
.../ozone/recon/TestReconWithOzoneManagerHA.java | 4 +-
.../hadoop/ozone/om/OzoneManagerHttpServer.java | 4 +-
.../apache/hadoop/ozone/recon/ReconConstants.java | 2 +
.../hadoop/ozone/recon/ReconServerConfigKeys.java | 16 +++
.../ozone/recon/fsck/ContainerHealthTask.java | 2 +-
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 5 +
.../scm/ReconStorageContainerManagerFacade.java | 129 ++++++++++++++++++++-
.../recon/spi/StorageContainerServiceProvider.java | 12 ++
.../spi/impl/OzoneManagerServiceProviderImpl.java | 8 +-
.../impl/StorageContainerServiceProviderImpl.java | 107 ++++++++++++++++-
.../TestStorageContainerServiceProviderImpl.java | 12 ++
24 files changed, 381 insertions(+), 18 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
index 7f8663e..a6b9805 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
@@ -365,4 +365,6 @@ public interface StorageContainerLocationProtocol extends
Closeable {
* commands operating on {@code containerID}.
*/
Token<?> getContainerToken(ContainerID containerID) throws IOException;
+
+ long getContainerCount() throws IOException;
}
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index a373fd1..133f39d 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -89,7 +89,7 @@ public final class OzoneConsts {
// OM Http server endpoints
public static final String OZONE_OM_SERVICE_LIST_HTTP_ENDPOINT =
"/serviceList";
- public static final String OZONE_OM_DB_CHECKPOINT_HTTP_ENDPOINT =
+ public static final String OZONE_DB_CHECKPOINT_HTTP_ENDPOINT =
"/dbCheckpoint";
// Ozone File System scheme
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml
b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 057d80c..7b83f60 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -2519,6 +2519,24 @@
</description>
</property>
<property>
+ <name>ozone.recon.scm.connection.request.timeout</name>
+ <value>5s</value>
+ <tag>OZONE, RECON, SCM</tag>
+ <description>
+ Connection request timeout in milliseconds for HTTP call made by Recon to
+ request SCM DB snapshot.
+ </description>
+ </property>
+ <property>
+ <name>ozone.recon.scm.connection.timeout</name>
+ <value>5s</value>
+ <tag>OZONE, RECON, SCM</tag>
+ <description>
+ Connection timeout for HTTP call in milliseconds made by Recon to request
+ SCM snapshot.
+ </description>
+ </property>
+ <property>
<name>ozone.recon.om.socket.timeout</name>
<value>5s</value>
<tag>OZONE, RECON, OM</tag>
@@ -2552,6 +2570,23 @@
</description>
</property>
<property>
+ <name>ozone.recon.scm.container.threshold</name>
+ <value>100</value>
+ <tag>OZONE, RECON, SCM</tag>
+ <description>
+ Threshold value for the difference in number of containers
+ in SCM and RECON.
+ </description>
+ </property>
+ <property>
+ <name>ozone.recon.scm.snapshot.enabled</name>
+ <value>false</value>
+ <tag>OZONE, RECON, SCM</tag>
+ <description>
+ If enabled, SCM DB Snapshot is taken by Recon.
+ </description>
+ </property>
+ <property>
<name>hdds.tracing.enabled</name>
<value>false</value>
<tag>OZONE, HDDS</tag>
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
index c484420..6e5bcf1 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
@@ -52,6 +52,8 @@ import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolPro
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetExistContainerWithPipelinesInBatchRequestProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetPipelineRequestProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetPipelineResponseProto;
+import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerCountRequestProto;
+import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerCountResponseProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.InSafeModeRequestProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ListPipelineRequestProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ListPipelineResponseProto;
@@ -932,6 +934,18 @@ public final class
StorageContainerLocationProtocolClientSideTranslatorPB
}
@Override
+ public long getContainerCount() throws IOException {
+ GetContainerCountRequestProto request =
+ GetContainerCountRequestProto.newBuilder().build();
+
+ GetContainerCountResponseProto response =
+ submitRequest(Type.GetContainerCount,
+ builder -> builder.setGetContainerCountRequest(request))
+ .getGetContainerCountResponse();
+ return response.getContainerCount();
+ }
+
+ @Override
public Object getUnderlyingProxyObject() {
return rpcProxy;
}
diff --git a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
index 4830c11..3211d51 100644
--- a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
+++ b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
@@ -74,6 +74,7 @@ message ScmContainerLocationRequest {
optional ContainerBalancerStatusRequestProto containerBalancerStatusRequest
= 35;
optional FinalizeScmUpgradeRequestProto finalizeScmUpgradeRequest = 36;
optional QueryUpgradeFinalizationProgressRequestProto
queryUpgradeFinalizationProgressRequest = 37;
+ optional GetContainerCountRequestProto getContainerCountRequest = 38;
}
message ScmContainerLocationResponse {
@@ -119,6 +120,7 @@ message ScmContainerLocationResponse {
optional ContainerBalancerStatusResponseProto
containerBalancerStatusResponse = 35;
optional FinalizeScmUpgradeResponseProto finalizeScmUpgradeResponse = 36;
optional QueryUpgradeFinalizationProgressResponseProto
queryUpgradeFinalizationProgressResponse = 37;
+ optional GetContainerCountResponseProto getContainerCountResponse = 38;
enum Status {
OK = 1;
@@ -162,6 +164,7 @@ enum Type {
GetContainerBalancerStatus = 30;
FinalizeScmUpgrade = 31;
QueryUpgradeFinalizationProgress = 32;
+ GetContainerCount = 33;
}
/**
@@ -383,6 +386,13 @@ message GetPipelineResponseProto {
required Pipeline pipeline = 1;
}
+message GetContainerCountRequestProto {
+}
+
+message GetContainerCountResponseProto {
+ required int64 containerCount = 1;
+}
+
message ActivatePipelineRequestProto {
required PipelineID pipelineID = 1;
optional string traceID = 2;
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
index 3a471fb..38be88a 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
@@ -53,6 +53,7 @@ import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolPro
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerWithPipelineResponseProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetExistContainerWithPipelinesInBatchRequestProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetExistContainerWithPipelinesInBatchResponseProto;
+import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerCountResponseProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetPipelineRequestProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetPipelineResponseProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetSafeModeRuleStatusesRequestProto;
@@ -397,6 +398,13 @@ public final class
StorageContainerLocationProtocolServerSideTranslatorPB
.setDatanodeUsageInfoResponse(getDatanodeUsageInfo(
request.getDatanodeUsageInfoRequest()))
.build();
+ case GetContainerCount:
+ return ScmContainerLocationResponse.newBuilder()
+ .setCmdType(request.getCmdType())
+ .setStatus(Status.OK)
+ .setGetContainerCountResponse(getContainerCount(
+ request.getGetContainerCountRequest()))
+ .build();
default:
throw new IllegalArgumentException(
"Unknown command type: " + request.getCmdType());
@@ -831,4 +839,12 @@ public final class
StorageContainerLocationProtocolServerSideTranslatorPB
.build();
}
+ public GetContainerCountResponseProto getContainerCount(
+ StorageContainerLocationProtocolProtos.GetContainerCountRequestProto
+ request) throws IOException {
+
+ return GetContainerCountResponseProto.newBuilder()
+ .setContainerCount(impl.getContainerCount())
+ .build();
+ }
}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index 4006a17..5e3da50 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -945,6 +945,11 @@ public class SCMClientProtocolServer implements
.generateToken(remoteUser.getUserName(), containerID);
}
+ @Override
+ public long getContainerCount() throws IOException {
+ return scm.getContainerManager().getContainers().size();
+ }
+
/**
* Queries a list of Node that match a set of statuses.
*
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
index f7a0761..ae955ee 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
@@ -23,7 +23,7 @@ import org.apache.hadoop.hdds.conf.MutableConfigurationSource;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.server.http.BaseHttpServer;
import org.apache.hadoop.ozone.OzoneConsts;
-import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_OM_DB_CHECKPOINT_HTTP_ENDPOINT;
+import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_HTTP_ENDPOINT;
/**
* HttpServer2 wrapper for the Ozone Storage Container Manager.
@@ -34,7 +34,7 @@ public class StorageContainerManagerHttpServer extends
BaseHttpServer {
StorageContainerManager scm)
throws IOException {
super(conf, "scm");
- addServlet("dbCheckpoint", OZONE_OM_DB_CHECKPOINT_HTTP_ENDPOINT,
+ addServlet("dbCheckpoint", OZONE_DB_CHECKPOINT_HTTP_ENDPOINT,
SCMDBCheckpointServlet.class);
getWebAppContext().setAttribute(OzoneConsts.SCM_CONTEXT_ATTRIBUTE, scm);
}
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OMNodeDetails.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OMNodeDetails.java
index 3de2f12..1a06601 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OMNodeDetails.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OMNodeDetails.java
@@ -31,7 +31,7 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_FLUSH;
-import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_OM_DB_CHECKPOINT_HTTP_ENDPOINT;
+import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_HTTP_ENDPOINT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_RATIS_PORT_DEFAULT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_RATIS_PORT_KEY;
@@ -163,13 +163,13 @@ public final class OMNodeDetails extends NodeDetails {
if (isHttpPolicy) {
if (StringUtils.isNotEmpty(getHttpAddress())) {
return "http://"; + getHttpAddress() +
- OZONE_OM_DB_CHECKPOINT_HTTP_ENDPOINT +
+ OZONE_DB_CHECKPOINT_HTTP_ENDPOINT +
"?" + OZONE_DB_CHECKPOINT_REQUEST_FLUSH + "=true";
}
} else {
if (StringUtils.isNotEmpty(getHttpsAddress())) {
return "https://"; + getHttpsAddress() +
- OZONE_OM_DB_CHECKPOINT_HTTP_ENDPOINT +
+ OZONE_DB_CHECKPOINT_HTTP_ENDPOINT +
"?" + OZONE_DB_CHECKPOINT_REQUEST_FLUSH + "=true";
}
}
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-csi/docker-config
b/hadoop-ozone/dist/src/main/compose/ozone-csi/docker-config
index e3fbb6a..31050a2 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-csi/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozone-csi/docker-config
@@ -21,6 +21,7 @@ OZONE-SITE.XML_ozone.csi.socket=/tmp/csi.sock
OZONE-SITE.XML_ozone.om.address=om
OZONE-SITE.XML_ozone.om.http-address=om:9874
+OZONE-SITE.XML_ozone.scm.http-address=scm:9876
OZONE-SITE.XML_ozone.scm.container.size=1GB
OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-config
b/hadoop-ozone/dist/src/main/compose/ozone/docker-config
index 41c8964..4b59e0e 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-config
@@ -19,6 +19,7 @@ CORE-SITE.XML_fs.trash.interval=1
OZONE-SITE.XML_ozone.om.address=om
OZONE-SITE.XML_ozone.om.http-address=om:9874
+OZONE-SITE.XML_ozone.scm.http-address=scm:9876
OZONE-SITE.XML_ozone.scm.container.size=1GB
OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config
b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config
index 497e0d5..5cfe4f0 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config
@@ -16,6 +16,7 @@
OZONE-SITE.XML_ozone.om.address=om
OZONE-SITE.XML_ozone.om.http-address=om:9874
+OZONE-SITE.XML_ozone.scm.http-address=scm:9876
OZONE-SITE.XML_ozone.scm.container.size=1GB
OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
index 1d38260..97fcde9 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
@@ -19,6 +19,7 @@ CORE-SITE.XML_fs.trash.interval=1
OZONE-SITE.XML_ozone.om.address=om
OZONE-SITE.XML_ozone.om.http-address=om:9874
+OZONE-SITE.XML_ozone.scm.http-address=scm:9876
OZONE-SITE.XML_ozone.scm.container.size=1GB
OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java
index 14ec9ff..1d34000 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.ozone.recon;
import static java.nio.charset.StandardCharsets.UTF_8;
-import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_OM_DB_CHECKPOINT_HTTP_ENDPOINT;
+import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_HTTP_ENDPOINT;
import java.util.HashMap;
import java.util.UUID;
@@ -114,7 +114,7 @@ public class TestReconWithOzoneManagerHA {
String expectedUrl = "http://"; +
(hostname.equals("0.0.0.0") ? "localhost" : hostname) + ":" +
ozoneManager.get().getHttpServer().getHttpAddress().getPort() +
- OZONE_OM_DB_CHECKPOINT_HTTP_ENDPOINT;
+ OZONE_DB_CHECKPOINT_HTTP_ENDPOINT;
String snapshotUrl = impl.getOzoneManagerSnapshotUrl();
Assert.assertEquals("OM Snapshot should be requested from the leader.",
expectedUrl, snapshotUrl);
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
index f24c00d..4250e0f 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
@@ -23,7 +23,7 @@ import org.apache.hadoop.hdds.conf.MutableConfigurationSource;
import org.apache.hadoop.hdds.server.http.BaseHttpServer;
import org.apache.hadoop.ozone.OzoneConsts;
-import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_OM_DB_CHECKPOINT_HTTP_ENDPOINT;
+import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_HTTP_ENDPOINT;
import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_OM_SERVICE_LIST_HTTP_ENDPOINT;
/**
@@ -36,7 +36,7 @@ public class OzoneManagerHttpServer extends BaseHttpServer {
super(conf, "ozoneManager");
addServlet("serviceList", OZONE_OM_SERVICE_LIST_HTTP_ENDPOINT,
ServiceListJSONServlet.class);
- addServlet("dbCheckpoint", OZONE_OM_DB_CHECKPOINT_HTTP_ENDPOINT,
+ addServlet("dbCheckpoint", OZONE_DB_CHECKPOINT_HTTP_ENDPOINT,
OMDBCheckpointServlet.class);
getWebAppContext().setAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE, om);
}
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java
index 2eeb323..5a013dc 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java
@@ -41,6 +41,8 @@ public final class ReconConstants {
public static final String CONTAINER_KEY_COUNT_TABLE =
"containerKeyCountTable";
+ public static final String RECON_SCM_SNAPSHOT_DB = "scm.snapshot.db";
+
// By default, limit the number of results returned
public static final String DEFAULT_FETCH_COUNT = "1000";
public static final String DEFAULT_BATCH_NUMBER = "1";
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServerConfigKeys.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServerConfigKeys.java
index 0cbb9c0..dd9f0c9 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServerConfigKeys.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServerConfigKeys.java
@@ -120,6 +120,22 @@ public final class ReconServerConfigKeys {
public static final String
OZONE_RECON_METRICS_HTTP_CONNECTION_REQUEST_TIMEOUT_DEFAULT = "10s";
+ public static final String OZONE_RECON_SCM_CONTAINER_THRESHOLD =
+ "ozone.recon.scm.container.threshold";
+ public static final int OZONE_RECON_SCM_CONTAINER_THRESHOLD_DEFAULT = 100;
+
+ public static final String OZONE_RECON_SCM_SNAPSHOT_ENABLED =
+ "ozone.recon.scm.snapshot.enabled";
+ public static final boolean OZONE_RECON_SCM_SNAPSHOT_ENABLED_DEFAULT = false;
+
+ public static final String OZONE_RECON_SCM_CONNECTION_TIMEOUT =
+ "ozone.recon.scm.connection.timeout";
+ public static final String OZONE_RECON_SCM_CONNECTION_TIMEOUT_DEFAULT = "5s";
+
+ public static final String OZONE_RECON_SCM_CONNECTION_REQUEST_TIMEOUT =
+ "ozone.recon.scm.connection.request.timeout";
+ public static final String
+ OZONE_RECON_SCM_CONNECTION_REQUEST_TIMEOUT_DEFAULT = "5s";
/**
* Private constructor for utility class.
*/
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthTask.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthTask.java
index b238278..67c6355 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthTask.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthTask.java
@@ -81,6 +81,7 @@ public class ContainerHealthTask extends ReconScmTask {
public synchronized void run() {
try {
while (canRun()) {
+ wait(interval);
long start = Time.monotonicNow();
long currentTime = System.currentTimeMillis();
long existingCount = processExistingDBRecords(currentTime);
@@ -97,7 +98,6 @@ public class ContainerHealthTask extends ReconScmTask {
" processing {} containers.", Time.monotonicNow() - start,
containers.size());
processedContainers.clear();
- wait(interval);
}
} catch (Throwable t) {
LOG.error("Exception in Missing Container task Thread.", t);
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconNodeManager.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconNodeManager.java
index 0f6d795..74029e0 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconNodeManager.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconNodeManager.java
@@ -294,4 +294,9 @@ public class ReconNodeManager extends SCMNodeManager {
return currentTime - getLastHeartbeat(datanodeDetails) >=
reconDatanodeOutdatedTime;
}
+
+ public void reinitialize(Table<UUID, DatanodeDetails> nodeTable) {
+ this.nodeDB = nodeTable;
+ loadExistingNodes();
+ }
}
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
index d0b0d81..081281d 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
@@ -18,14 +18,18 @@
package org.apache.hadoop.ozone.recon.scm;
+import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.UUID;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.block.BlockManager;
import org.apache.hadoop.hdds.scm.container.CloseContainerEventHandler;
@@ -59,9 +63,15 @@ import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.server.events.EventQueue;
import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager;
import org.apache.hadoop.hdds.utils.HddsServerUtil;
+import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
+import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition;
import org.apache.hadoop.hdds.utils.db.DBStore;
import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.Table.KeyValue;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ozone.recon.ReconServerConfigKeys;
import org.apache.hadoop.ozone.recon.fsck.ContainerHealthTask;
import org.apache.hadoop.ozone.recon.persistence.ContainerHealthSchemaManager;
import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager;
@@ -70,6 +80,8 @@ import org.apache.hadoop.ozone.recon.tasks.ReconTaskConfig;
import com.google.inject.Inject;
import static
org.apache.hadoop.hdds.recon.ReconConfigKeys.RECON_SCM_CONFIG_PREFIX;
import static
org.apache.hadoop.hdds.scm.server.StorageContainerManager.buildRpcServerStartMessage;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+
import org.hadoop.ozone.recon.schema.tables.daos.ReconTaskStatusDao;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -90,11 +102,11 @@ public class ReconStorageContainerManagerFacade
private final EventQueue eventQueue;
private final SCMContext scmContext;
private final SCMStorageConfig scmStorageConfig;
- private final DBStore dbStore;
private final SCMNodeDetails reconNodeDetails;
private final SCMHAManager scmhaManager;
private final SequenceIdGenerator sequenceIdGen;
+ private DBStore dbStore;
private ReconNodeManager nodeManager;
private ReconPipelineManager pipelineManager;
private ReconContainerManager containerManager;
@@ -249,7 +261,15 @@ public class ReconStorageContainerManagerFacade
"Recon ScmDatanodeProtocol RPC server",
getDatanodeProtocolServer().getDatanodeRpcAddress()));
}
- initializePipelinesFromScm();
+ boolean isSCMSnapshotEnabled = ozoneConfiguration.getBoolean(
+ ReconServerConfigKeys.OZONE_RECON_SCM_SNAPSHOT_ENABLED,
+ ReconServerConfigKeys.OZONE_RECON_SCM_SNAPSHOT_ENABLED_DEFAULT);
+ if(isSCMSnapshotEnabled) {
+ initializeSCMDB();
+ LOG.info("SCM DB initialized");
+ } else {
+ initializePipelinesFromScm();
+ }
getDatanodeProtocolServer().start();
this.reconScmTasks.forEach(ReconScmTask::start);
}
@@ -307,6 +327,111 @@ public class ReconStorageContainerManagerFacade
}
}
+ private void initializeSCMDB() {
+ try {
+ long scmContainersCount = scmServiceProvider.getContainerCount();
+ long reconContainerCount = containerManager.getContainers().size();
+ long threshold = ozoneConfiguration.getInt(
+ ReconServerConfigKeys.OZONE_RECON_SCM_CONTAINER_THRESHOLD,
+ ReconServerConfigKeys.OZONE_RECON_SCM_CONTAINER_THRESHOLD_DEFAULT);
+
+ if(Math.abs(scmContainersCount - reconContainerCount) > threshold) {
+ LOG.info("Recon Container Count: {}, SCM Container Count: {}",
+ reconContainerCount, scmContainersCount);
+ updateReconSCMDBWithNewSnapshot();
+ LOG.info("Updated Recon DB with SCM DB");
+ } else {
+ initializePipelinesFromScm();
+ }
+ } catch (IOException e) {
+ LOG.error("Exception encountered while getting SCM DB.");
+ }
+ }
+
+ public void updateReconSCMDBWithNewSnapshot() throws IOException {
+ DBCheckpoint dbSnapshot = scmServiceProvider.getSCMDBSnapshot();
+ if (dbSnapshot != null && dbSnapshot.getCheckpointLocation() != null) {
+ LOG.info("Got new checkpoint from SCM : " +
+ dbSnapshot.getCheckpointLocation());
+ try {
+ initializeNewRdbStore(dbSnapshot.getCheckpointLocation().toFile());
+ } catch (IOException e) {
+ LOG.error("Unable to refresh Recon SCM DB Snapshot. ", e);
+ }
+ } else {
+ LOG.error("Null snapshot location got from SCM.");
+ }
+ }
+
+ private void deleteOldSCMDB() throws IOException {
+ if (dbStore != null) {
+ File oldDBLocation = dbStore.getDbLocation();
+ if (oldDBLocation.exists()) {
+ LOG.info("Cleaning up old SCM snapshot db at {}.",
+ oldDBLocation.getAbsolutePath());
+ FileUtils.deleteDirectory(oldDBLocation);
+ }
+ }
+ }
+
+ private void initializeNewRdbStore(File dbFile) throws IOException {
+ try {
+ DBStore newStore = createDBAndAddSCMTablesAndCodecs(
+ dbFile, new ReconSCMDBDefinition());
+ Table<UUID, DatanodeDetails> nodeTable =
+ ReconSCMDBDefinition.NODES.getTable(dbStore);
+ Table<UUID, DatanodeDetails> newNodeTable =
+ ReconSCMDBDefinition.NODES.getTable(newStore);
+ TableIterator<UUID, ? extends KeyValue<UUID,
+ DatanodeDetails>> iterator = nodeTable.iterator();
+ while (iterator.hasNext()) {
+ KeyValue<UUID, DatanodeDetails> keyValue = iterator.next();
+ newNodeTable.put(keyValue.getKey(), keyValue.getValue());
+ }
+ sequenceIdGen.reinitialize(
+ ReconSCMDBDefinition.SEQUENCE_ID.getTable(newStore));
+ pipelineManager.reinitialize(
+ ReconSCMDBDefinition.PIPELINES.getTable(newStore));
+ containerManager.reinitialize(
+ ReconSCMDBDefinition.CONTAINERS.getTable(newStore));
+ nodeManager.reinitialize(
+ ReconSCMDBDefinition.NODES.getTable(newStore));
+ deleteOldSCMDB();
+ setDbStore(newStore);
+ File newDb = new File(dbFile.getParent() +
+ OZONE_URI_DELIMITER + ReconSCMDBDefinition.RECON_SCM_DB_NAME);
+ boolean success = dbFile.renameTo(newDb);
+ if (success) {
+ LOG.info("SCM snapshot linked to Recon DB.");
+ }
+ LOG.info("Created SCM DB handle from snapshot at {}.",
+ dbFile.getAbsolutePath());
+ } catch (IOException ioEx) {
+ LOG.error("Unable to initialize Recon SCM DB snapshot store.", ioEx);
+ }
+ }
+
+ private DBStore createDBAndAddSCMTablesAndCodecs(File dbFile,
+ ReconSCMDBDefinition definition) throws IOException {
+ DBStoreBuilder dbStoreBuilder =
+ DBStoreBuilder.newBuilder(ozoneConfiguration)
+ .setName(dbFile.getName())
+ .setPath(dbFile.toPath().getParent());
+ for (DBColumnFamilyDefinition columnFamily :
+ definition.getColumnFamilies()) {
+ dbStoreBuilder.addTable(columnFamily.getName());
+ dbStoreBuilder.addCodec(columnFamily.getKeyType(),
+ columnFamily.getKeyCodec());
+ dbStoreBuilder.addCodec(columnFamily.getValueType(),
+ columnFamily.getValueCodec());
+ }
+ return dbStoreBuilder.build();
+ }
+
+ public void setDbStore(DBStore dbStore) {
+ this.dbStore = dbStore;
+ }
+
@Override
public NodeManager getScmNodeManager() {
return nodeManager;
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/StorageContainerServiceProvider.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/StorageContainerServiceProvider.java
index be2c7cb..cf57937 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/StorageContainerServiceProvider.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/StorageContainerServiceProvider.java
@@ -23,6 +23,7 @@ import java.util.List;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import
org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
/**
* Interface to access SCM endpoints.
@@ -66,4 +67,15 @@ public interface StorageContainerServiceProvider {
*/
List<HddsProtos.Node> getNodes() throws IOException;
+ /**
+ * Requests SCM for container count.
+ * @return Total number of containers in SCM.
+ */
+ long getContainerCount() throws IOException;
+
+ /**
+ * Requests SCM for DB Snapshot.
+ * @return DBCheckpoint from SCM.
+ */
+ DBCheckpoint getSCMDBSnapshot();
}
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java
index 96263e0..021d014 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java
@@ -58,7 +58,7 @@ import org.apache.hadoop.util.Time;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.io.FileUtils;
import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_FLUSH;
-import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_OM_DB_CHECKPOINT_HTTP_ENDPOINT;
+import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_HTTP_ENDPOINT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_AUTH_TYPE;
import static
org.apache.hadoop.ozone.recon.ReconConstants.RECON_OM_SNAPSHOT_DB;
import static
org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_OM_SNAPSHOT_DB_DIR;
@@ -151,11 +151,11 @@ public class OzoneManagerServiceProviderImpl
HttpConfig.Policy policy = HttpConfig.getHttpPolicy(configuration);
omDBSnapshotUrl = "http://"; + ozoneManagerHttpAddress +
- OZONE_OM_DB_CHECKPOINT_HTTP_ENDPOINT;
+ OZONE_DB_CHECKPOINT_HTTP_ENDPOINT;
if (policy.isHttpsEnabled()) {
omDBSnapshotUrl = "https://"; + ozoneManagerHttpsAddress +
- OZONE_OM_DB_CHECKPOINT_HTTP_ENDPOINT;
+ OZONE_DB_CHECKPOINT_HTTP_ENDPOINT;
}
boolean flushParam = configuration.getBoolean(
@@ -271,7 +271,7 @@ public class OzoneManagerServiceProviderImpl
omLeaderUrl = (policy.isHttpsEnabled() ?
"https://"; + info.getServiceAddress(Type.HTTPS) :
"http://"; + info.getServiceAddress(Type.HTTP)) +
- OZONE_OM_DB_CHECKPOINT_HTTP_ENDPOINT;
+ OZONE_DB_CHECKPOINT_HTTP_ENDPOINT;
}
}
}
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/StorageContainerServiceProviderImpl.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/StorageContainerServiceProviderImpl.java
index 1e609e8..2fe423a 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/StorageContainerServiceProviderImpl.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/StorageContainerServiceProviderImpl.java
@@ -18,18 +18,41 @@
package org.apache.hadoop.ozone.recon.spi.impl;
+import static
org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig.ConfigStrings.HDDS_SCM_HTTP_AUTH_TYPE;
import static org.apache.hadoop.ozone.ClientVersions.CURRENT_VERSION;
+import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_HTTP_ENDPOINT;
+import static
org.apache.hadoop.ozone.recon.ReconConstants.RECON_SCM_SNAPSHOT_DB;
+import static
org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_SCM_CONNECTION_REQUEST_TIMEOUT;
+import static
org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_SCM_CONNECTION_REQUEST_TIMEOUT_DEFAULT;
+import static
org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_SCM_CONNECTION_TIMEOUT;
+import static
org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_SCM_CONNECTION_TIMEOUT_DEFAULT;
+import java.io.File;
import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.List;
+import java.util.concurrent.TimeUnit;
import javax.inject.Inject;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import
org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
+import org.apache.hadoop.hdds.server.http.HttpConfig;
+import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
+import org.apache.hadoop.hdds.utils.db.RocksDBCheckpoint;
+import org.apache.hadoop.hdfs.web.URLConnectionFactory;
+import org.apache.hadoop.ozone.recon.ReconUtils;
import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider;
+import org.apache.hadoop.security.SecurityUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Implementation for StorageContainerServiceProvider that talks with actual
@@ -38,12 +61,53 @@ import
org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider;
public class StorageContainerServiceProviderImpl
implements StorageContainerServiceProvider {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(StorageContainerServiceProviderImpl.class);
private StorageContainerLocationProtocol scmClient;
+ private final OzoneConfiguration configuration;
+ private String scmDBSnapshotUrl;
+ private File scmSnapshotDBParentDir;
+ private URLConnectionFactory connectionFactory;
+ private ReconUtils reconUtils;
@Inject
public StorageContainerServiceProviderImpl(
- StorageContainerLocationProtocol scmClient) {
+ StorageContainerLocationProtocol scmClient,
+ ReconUtils reconUtils,
+ OzoneConfiguration configuration) {
+
+ int connectionTimeout = (int) configuration.getTimeDuration(
+ OZONE_RECON_SCM_CONNECTION_TIMEOUT,
+ OZONE_RECON_SCM_CONNECTION_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS);
+ int connectionRequestTimeout = (int) configuration.getTimeDuration(
+ OZONE_RECON_SCM_CONNECTION_REQUEST_TIMEOUT,
+ OZONE_RECON_SCM_CONNECTION_REQUEST_TIMEOUT_DEFAULT,
+ TimeUnit.MILLISECONDS);
+ connectionFactory =
+ URLConnectionFactory.newDefaultURLConnectionFactory(connectionTimeout,
+ connectionRequestTimeout, configuration);
+
+ String scmHttpAddress = configuration.get(ScmConfigKeys
+ .OZONE_SCM_HTTP_ADDRESS_KEY);
+
+ String scmHttpsAddress = configuration.get(ScmConfigKeys
+ .OZONE_SCM_HTTPS_ADDRESS_KEY);
+
+ HttpConfig.Policy policy = HttpConfig.getHttpPolicy(configuration);
+
+ scmSnapshotDBParentDir = ReconUtils.getReconScmDbDir(configuration);
+
+ scmDBSnapshotUrl = "http://"; + scmHttpAddress +
+ OZONE_DB_CHECKPOINT_HTTP_ENDPOINT;
+
+ if (policy.isHttpsEnabled()) {
+ scmDBSnapshotUrl = "https://"; + scmHttpsAddress +
+ OZONE_DB_CHECKPOINT_HTTP_ENDPOINT;
+ }
+
+ this.reconUtils = reconUtils;
this.scmClient = scmClient;
+ this.configuration = configuration;
}
@Override
@@ -74,4 +138,45 @@ public class StorageContainerServiceProviderImpl
return scmClient.queryNode(null, null, HddsProtos.QueryScope.CLUSTER,
"", CURRENT_VERSION);
}
+
+ @Override
+ public long getContainerCount() throws IOException {
+ return scmClient.getContainerCount();
+ }
+
+ public String getScmDBSnapshotUrl() {
+ return scmDBSnapshotUrl;
+ }
+
+ private boolean isOmSpnegoEnabled() {
+ return configuration.get(HDDS_SCM_HTTP_AUTH_TYPE, "simple")
+ .equals("kerberos");
+ }
+
+ public DBCheckpoint getSCMDBSnapshot() {
+ String snapshotFileName = RECON_SCM_SNAPSHOT_DB + "_" +
+ System.currentTimeMillis();
+ File targetFile = new File(scmSnapshotDBParentDir, snapshotFileName +
+ ".tar.gz");
+
+ try {
+ SecurityUtil.doAsLoginUser(() -> {
+ try (InputStream inputStream = reconUtils.makeHttpCall(
+ connectionFactory, getScmDBSnapshotUrl(),
+ isOmSpnegoEnabled()).getInputStream()) {
+ FileUtils.copyInputStreamToFile(inputStream, targetFile);
+ }
+ return null;
+ });
+
+ Path untarredDbDir = Paths.get(scmSnapshotDBParentDir.getAbsolutePath(),
+ snapshotFileName);
+ reconUtils.untarCheckpointFile(targetFile, untarredDbDir);
+ FileUtils.deleteQuietly(targetFile);
+ return new RocksDBCheckpoint(untarredDbDir);
+ } catch (IOException e) {
+ LOG.error("Unable to obtain SCM DB Snapshot. ", e);
+ }
+ return null;
+ }
}
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestStorageContainerServiceProviderImpl.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestStorageContainerServiceProviderImpl.java
index 71c9bc4..f9d3fc4 100644
---
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestStorageContainerServiceProviderImpl.java
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestStorageContainerServiceProviderImpl.java
@@ -24,13 +24,18 @@ import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import java.io.File;
import java.io.IOException;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
+import org.apache.hadoop.ozone.recon.ReconUtils;
import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider;
+import org.apache.ozone.test.GenericTestUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -55,6 +60,10 @@ public class TestStorageContainerServiceProviderImpl {
try {
StorageContainerLocationProtocol mockScmClient = mock(
StorageContainerLocationProtocol.class);
+ ReconUtils reconUtils = new ReconUtils();
+ File testDir = GenericTestUtils.getRandomizedTestDir();
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getPath());
pipelineID = PipelineID.randomId().getProtobuf();
when(mockScmClient.getPipeline(pipelineID))
.thenReturn(mock(Pipeline.class));
@@ -62,6 +71,9 @@ public class TestStorageContainerServiceProviderImpl {
.toInstance(mockScmClient);
bind(StorageContainerServiceProvider.class)
.to(StorageContainerServiceProviderImpl.class);
+ bind(OzoneConfiguration.class).
+ toInstance(conf);
+ bind(ReconUtils.class).toInstance(reconUtils);
} catch (Exception e) {
Assert.fail();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
