devmadhuu commented on code in PR #11954:
URL: https://github.com/apache/ozone/pull/11954#discussion_r4490737227
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/StorageContainerServiceProviderImpl.java:
##########
@@ -183,9 +190,85 @@ private RocksDBCheckpoint getRocksDBCheckpoint(String
snapshotFileName, File tar
snapshotFileName);
reconUtils.untarCheckpointFile(targetFile, untarredDbDir);
FileUtils.deleteQuietly(targetFile);
+ // SCM writes this flag last, so a checkpoint without it was cut short.
+ if (!ratisSnapshotComplete(untarredDbDir)) {
+ FileUtils.deleteQuietly(untarredDbDir.toFile());
+ throw new IOException("Incomplete SCM DB checkpoint: " +
OZONE_RATIS_SNAPSHOT_COMPLETE_FLAG_NAME + " not found");
+ }
return new RocksDBCheckpoint(untarredDbDir);
}
+ private void downloadSCMDBCheckpointOverGrpc(String hostAddress, File
targetFile, SecurityConfig secConf)
+ throws IOException {
+ int grpcPort =
+ configuration.getInt(ScmConfigKeys.OZONE_SCM_GRPC_PORT_KEY,
ScmConfigKeys.OZONE_SCM_GRPC_PORT_DEFAULT);
+ SCMSecurityProtocolClientSideTranslatorPB scmSecurityClient =
+ getScmSecurityClientWithMaxRetry(configuration, getCurrentUser());
+ try (ReconCertificateClient certClient =
+ new ReconCertificateClient(secConf, scmSecurityClient,
reconStorage, null, null);
+ SCMSnapshotDownloader downloadClient =
+ new InterSCMGrpcClient(hostAddress, grpcPort, configuration,
certClient)) {
+ downloadClient.download(targetFile.toPath()).get();
+ } catch (ExecutionException | InterruptedException e) {
+ LOG.error("Rocks DB checkpoint downloading failed: {}", e);
+ throw new IOException(e);
+ }
+ }
+
+ /**
+ * Downloads the SCM DB checkpoint over HTTP(S), authenticated with SPNEGO
when SCM requires it.
+ * Used when gRPC TLS is enabled, because the inter-SCM gRPC server then
accepts SCM certificates only.
+ */
+ private void downloadSCMDBCheckpoint(String url, File targetFile) throws
IOException {
+ LOG.info("Downloading SCM DB checkpoint from {}", url);
+ URLConnectionFactory connectionFactory =
URLConnectionFactory.newDefaultURLConnectionFactory(configuration);
Review Comment:
Can we make the HTTP connection/read timeouts configurable and suitable for
checkpoint creation? This factory overload fixes both at 60 seconds. SCM
creates the checkpoint before streaming it, so a busy SCM that needs more than
one minute will repeatedly fail full sync. The previous gRPC path allowed a
configurable deadline.
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/StorageContainerServiceProviderImpl.java:
##########
@@ -183,9 +190,85 @@ private RocksDBCheckpoint getRocksDBCheckpoint(String
snapshotFileName, File tar
snapshotFileName);
reconUtils.untarCheckpointFile(targetFile, untarredDbDir);
FileUtils.deleteQuietly(targetFile);
+ // SCM writes this flag last, so a checkpoint without it was cut short.
+ if (!ratisSnapshotComplete(untarredDbDir)) {
+ FileUtils.deleteQuietly(untarredDbDir.toFile());
+ throw new IOException("Incomplete SCM DB checkpoint: " +
OZONE_RATIS_SNAPSHOT_COMPLETE_FLAG_NAME + " not found");
+ }
return new RocksDBCheckpoint(untarredDbDir);
}
+ private void downloadSCMDBCheckpointOverGrpc(String hostAddress, File
targetFile, SecurityConfig secConf)
+ throws IOException {
+ int grpcPort =
+ configuration.getInt(ScmConfigKeys.OZONE_SCM_GRPC_PORT_KEY,
ScmConfigKeys.OZONE_SCM_GRPC_PORT_DEFAULT);
+ SCMSecurityProtocolClientSideTranslatorPB scmSecurityClient =
+ getScmSecurityClientWithMaxRetry(configuration, getCurrentUser());
+ try (ReconCertificateClient certClient =
+ new ReconCertificateClient(secConf, scmSecurityClient,
reconStorage, null, null);
+ SCMSnapshotDownloader downloadClient =
+ new InterSCMGrpcClient(hostAddress, grpcPort, configuration,
certClient)) {
+ downloadClient.download(targetFile.toPath()).get();
+ } catch (ExecutionException | InterruptedException e) {
+ LOG.error("Rocks DB checkpoint downloading failed: {}", e);
+ throw new IOException(e);
+ }
+ }
+
+ /**
+ * Downloads the SCM DB checkpoint over HTTP(S), authenticated with SPNEGO
when SCM requires it.
+ * Used when gRPC TLS is enabled, because the inter-SCM gRPC server then
accepts SCM certificates only.
+ */
+ private void downloadSCMDBCheckpoint(String url, File targetFile) throws
IOException {
+ LOG.info("Downloading SCM DB checkpoint from {}", url);
+ URLConnectionFactory connectionFactory =
URLConnectionFactory.newDefaultURLConnectionFactory(configuration);
+ boolean spnegoEnabled =
"kerberos".equals(configuration.get(HDDS_SCM_HTTP_AUTH_TYPE, "simple"));
+ try {
+ SecurityUtil.doAsLoginUser(() -> {
+ try (InputStream inputStream =
reconUtils.makeHttpCall(connectionFactory, url, spnegoEnabled)
+ .getInputStream()) {
+ FileUtils.copyInputStreamToFile(inputStream, targetFile);
Review Comment:
Please delete targetFile when the HTTP download fails. A timeout or
connection failure during copyInputStreamToFile() leaves a partial tar file,
and the outer error handler does not remove it. Repeated attempts can exhaust
Recon’s disk. The existing gRPC downloader already cleans up its output on
failure.
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/StorageContainerServiceProviderImpl.java:
##########
@@ -183,9 +190,85 @@ private RocksDBCheckpoint getRocksDBCheckpoint(String
snapshotFileName, File tar
snapshotFileName);
reconUtils.untarCheckpointFile(targetFile, untarredDbDir);
FileUtils.deleteQuietly(targetFile);
+ // SCM writes this flag last, so a checkpoint without it was cut short.
+ if (!ratisSnapshotComplete(untarredDbDir)) {
+ FileUtils.deleteQuietly(untarredDbDir.toFile());
+ throw new IOException("Incomplete SCM DB checkpoint: " +
OZONE_RATIS_SNAPSHOT_COMPLETE_FLAG_NAME + " not found");
+ }
return new RocksDBCheckpoint(untarredDbDir);
}
+ private void downloadSCMDBCheckpointOverGrpc(String hostAddress, File
targetFile, SecurityConfig secConf)
+ throws IOException {
+ int grpcPort =
+ configuration.getInt(ScmConfigKeys.OZONE_SCM_GRPC_PORT_KEY,
ScmConfigKeys.OZONE_SCM_GRPC_PORT_DEFAULT);
+ SCMSecurityProtocolClientSideTranslatorPB scmSecurityClient =
+ getScmSecurityClientWithMaxRetry(configuration, getCurrentUser());
+ try (ReconCertificateClient certClient =
+ new ReconCertificateClient(secConf, scmSecurityClient,
reconStorage, null, null);
+ SCMSnapshotDownloader downloadClient =
+ new InterSCMGrpcClient(hostAddress, grpcPort, configuration,
certClient)) {
+ downloadClient.download(targetFile.toPath()).get();
+ } catch (ExecutionException | InterruptedException e) {
+ LOG.error("Rocks DB checkpoint downloading failed: {}", e);
+ throw new IOException(e);
+ }
+ }
+
+ /**
+ * Downloads the SCM DB checkpoint over HTTP(S), authenticated with SPNEGO
when SCM requires it.
+ * Used when gRPC TLS is enabled, because the inter-SCM gRPC server then
accepts SCM certificates only.
+ */
+ private void downloadSCMDBCheckpoint(String url, File targetFile) throws
IOException {
+ LOG.info("Downloading SCM DB checkpoint from {}", url);
+ URLConnectionFactory connectionFactory =
URLConnectionFactory.newDefaultURLConnectionFactory(configuration);
+ boolean spnegoEnabled =
"kerberos".equals(configuration.get(HDDS_SCM_HTTP_AUTH_TYPE, "simple"));
+ try {
+ SecurityUtil.doAsLoginUser(() -> {
+ try (InputStream inputStream =
reconUtils.makeHttpCall(connectionFactory, url, spnegoEnabled)
Review Comment:
Is there any way we can preserve download cancellation on the HTTP path ?
The existing cancel operation uses Future.cancel(true), but this blocking HTTP
copy continues after interruption. Recon can report CANCELLED while still
downloading and holding the sync-running flag. Please make cancellation
close/abort the active transfer and add coverage with a slow response. Recon’s
existing cancellation operation interrupts the worker thread using
cancel(true). The new blocking HTTP copy does not respond to that interruption.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]