smengcl commented on code in PR #11954:
URL: https://github.com/apache/ozone/pull/11954#discussion_r4490832035
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/StorageContainerServiceProviderImpl.java:
##########
@@ -183,9 +190,85 @@ private RocksDBCheckpoint getRocksDBCheckpoint(String
snapshotFileName, File tar
snapshotFileName);
reconUtils.untarCheckpointFile(targetFile, untarredDbDir);
FileUtils.deleteQuietly(targetFile);
+ // SCM writes this flag last, so a checkpoint without it was cut short.
+ if (!ratisSnapshotComplete(untarredDbDir)) {
+ FileUtils.deleteQuietly(untarredDbDir.toFile());
+ throw new IOException("Incomplete SCM DB checkpoint: " +
OZONE_RATIS_SNAPSHOT_COMPLETE_FLAG_NAME + " not found");
+ }
return new RocksDBCheckpoint(untarredDbDir);
}
+ private void downloadSCMDBCheckpointOverGrpc(String hostAddress, File
targetFile, SecurityConfig secConf)
+ throws IOException {
+ int grpcPort =
+ configuration.getInt(ScmConfigKeys.OZONE_SCM_GRPC_PORT_KEY,
ScmConfigKeys.OZONE_SCM_GRPC_PORT_DEFAULT);
+ SCMSecurityProtocolClientSideTranslatorPB scmSecurityClient =
+ getScmSecurityClientWithMaxRetry(configuration, getCurrentUser());
+ try (ReconCertificateClient certClient =
+ new ReconCertificateClient(secConf, scmSecurityClient,
reconStorage, null, null);
+ SCMSnapshotDownloader downloadClient =
+ new InterSCMGrpcClient(hostAddress, grpcPort, configuration,
certClient)) {
+ downloadClient.download(targetFile.toPath()).get();
+ } catch (ExecutionException | InterruptedException e) {
+ LOG.error("Rocks DB checkpoint downloading failed: {}", e);
+ throw new IOException(e);
+ }
+ }
+
+ /**
+ * Downloads the SCM DB checkpoint over HTTP(S), authenticated with SPNEGO
when SCM requires it.
+ * Used when gRPC TLS is enabled, because the inter-SCM gRPC server then
accepts SCM certificates only.
+ */
+ private void downloadSCMDBCheckpoint(String url, File targetFile) throws
IOException {
+ LOG.info("Downloading SCM DB checkpoint from {}", url);
+ URLConnectionFactory connectionFactory =
URLConnectionFactory.newDefaultURLConnectionFactory(configuration);
+ boolean spnegoEnabled =
"kerberos".equals(configuration.get(HDDS_SCM_HTTP_AUTH_TYPE, "simple"));
+ try {
+ SecurityUtil.doAsLoginUser(() -> {
+ try (InputStream inputStream =
reconUtils.makeHttpCall(connectionFactory, url, spnegoEnabled)
Review Comment:
Thanks, done in 0cd851bcfa9: the HTTP download now runs in a helper thread,
and the sync thread waits on it. Cancelling (`cancel(true)`) interrupts the
wait, which disconnects the connection to abort a blocked read and removes the
partial tar. `testGetSCMDBSnapshotCancelAbortsDownload` covers it with a
response that blocks until disconnected (it fails without the change).
_🤖 Addressed by [Claude Code](https://claude.com/claude-code)_
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/StorageContainerServiceProviderImpl.java:
##########
@@ -183,9 +190,85 @@ private RocksDBCheckpoint getRocksDBCheckpoint(String
snapshotFileName, File tar
snapshotFileName);
reconUtils.untarCheckpointFile(targetFile, untarredDbDir);
FileUtils.deleteQuietly(targetFile);
+ // SCM writes this flag last, so a checkpoint without it was cut short.
+ if (!ratisSnapshotComplete(untarredDbDir)) {
+ FileUtils.deleteQuietly(untarredDbDir.toFile());
+ throw new IOException("Incomplete SCM DB checkpoint: " +
OZONE_RATIS_SNAPSHOT_COMPLETE_FLAG_NAME + " not found");
+ }
return new RocksDBCheckpoint(untarredDbDir);
}
+ private void downloadSCMDBCheckpointOverGrpc(String hostAddress, File
targetFile, SecurityConfig secConf)
+ throws IOException {
+ int grpcPort =
+ configuration.getInt(ScmConfigKeys.OZONE_SCM_GRPC_PORT_KEY,
ScmConfigKeys.OZONE_SCM_GRPC_PORT_DEFAULT);
+ SCMSecurityProtocolClientSideTranslatorPB scmSecurityClient =
+ getScmSecurityClientWithMaxRetry(configuration, getCurrentUser());
+ try (ReconCertificateClient certClient =
+ new ReconCertificateClient(secConf, scmSecurityClient,
reconStorage, null, null);
+ SCMSnapshotDownloader downloadClient =
+ new InterSCMGrpcClient(hostAddress, grpcPort, configuration,
certClient)) {
+ downloadClient.download(targetFile.toPath()).get();
+ } catch (ExecutionException | InterruptedException e) {
+ LOG.error("Rocks DB checkpoint downloading failed: {}", e);
+ throw new IOException(e);
+ }
+ }
+
+ /**
+ * Downloads the SCM DB checkpoint over HTTP(S), authenticated with SPNEGO
when SCM requires it.
+ * Used when gRPC TLS is enabled, because the inter-SCM gRPC server then
accepts SCM certificates only.
+ */
+ private void downloadSCMDBCheckpoint(String url, File targetFile) throws
IOException {
+ LOG.info("Downloading SCM DB checkpoint from {}", url);
+ URLConnectionFactory connectionFactory =
URLConnectionFactory.newDefaultURLConnectionFactory(configuration);
Review Comment:
Done in 0cd851bcfa9: added `ozone.recon.scm.snapshot.connection.timeout`
(default 60s, as before) and `ozone.recon.scm.snapshot.read.timeout` (default
10m, to cover checkpoint creation on SCM before the first byte), with entries
in ozone-default.xml.
_🤖 Addressed by [Claude Code](https://claude.com/claude-code)_
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/StorageContainerServiceProviderImpl.java:
##########
@@ -183,9 +190,85 @@ private RocksDBCheckpoint getRocksDBCheckpoint(String
snapshotFileName, File tar
snapshotFileName);
reconUtils.untarCheckpointFile(targetFile, untarredDbDir);
FileUtils.deleteQuietly(targetFile);
+ // SCM writes this flag last, so a checkpoint without it was cut short.
+ if (!ratisSnapshotComplete(untarredDbDir)) {
+ FileUtils.deleteQuietly(untarredDbDir.toFile());
+ throw new IOException("Incomplete SCM DB checkpoint: " +
OZONE_RATIS_SNAPSHOT_COMPLETE_FLAG_NAME + " not found");
+ }
return new RocksDBCheckpoint(untarredDbDir);
}
+ private void downloadSCMDBCheckpointOverGrpc(String hostAddress, File
targetFile, SecurityConfig secConf)
+ throws IOException {
+ int grpcPort =
+ configuration.getInt(ScmConfigKeys.OZONE_SCM_GRPC_PORT_KEY,
ScmConfigKeys.OZONE_SCM_GRPC_PORT_DEFAULT);
+ SCMSecurityProtocolClientSideTranslatorPB scmSecurityClient =
+ getScmSecurityClientWithMaxRetry(configuration, getCurrentUser());
+ try (ReconCertificateClient certClient =
+ new ReconCertificateClient(secConf, scmSecurityClient,
reconStorage, null, null);
+ SCMSnapshotDownloader downloadClient =
+ new InterSCMGrpcClient(hostAddress, grpcPort, configuration,
certClient)) {
+ downloadClient.download(targetFile.toPath()).get();
+ } catch (ExecutionException | InterruptedException e) {
+ LOG.error("Rocks DB checkpoint downloading failed: {}", e);
+ throw new IOException(e);
+ }
+ }
+
+ /**
+ * Downloads the SCM DB checkpoint over HTTP(S), authenticated with SPNEGO
when SCM requires it.
+ * Used when gRPC TLS is enabled, because the inter-SCM gRPC server then
accepts SCM certificates only.
+ */
+ private void downloadSCMDBCheckpoint(String url, File targetFile) throws
IOException {
+ LOG.info("Downloading SCM DB checkpoint from {}", url);
+ URLConnectionFactory connectionFactory =
URLConnectionFactory.newDefaultURLConnectionFactory(configuration);
+ boolean spnegoEnabled =
"kerberos".equals(configuration.get(HDDS_SCM_HTTP_AUTH_TYPE, "simple"));
+ try {
+ SecurityUtil.doAsLoginUser(() -> {
+ try (InputStream inputStream =
reconUtils.makeHttpCall(connectionFactory, url, spnegoEnabled)
+ .getInputStream()) {
+ FileUtils.copyInputStreamToFile(inputStream, targetFile);
Review Comment:
Done in 0cd851bcfa9: the target tar is now deleted when the HTTP download
fails or is cancelled.
_🤖 Addressed by [Claude Code](https://claude.com/claude-code)_
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
