devmadhuu commented on code in PR #11954:
URL: https://github.com/apache/ozone/pull/11954#discussion_r4491133603
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/StorageContainerServiceProviderImpl.java:
##########
@@ -183,9 +201,112 @@ private RocksDBCheckpoint getRocksDBCheckpoint(String
snapshotFileName, File tar
snapshotFileName);
reconUtils.untarCheckpointFile(targetFile, untarredDbDir);
FileUtils.deleteQuietly(targetFile);
+ // SCM writes this flag last, so a checkpoint without it was cut short.
+ if (!ratisSnapshotComplete(untarredDbDir)) {
+ FileUtils.deleteQuietly(untarredDbDir.toFile());
+ throw new IOException("Incomplete SCM DB checkpoint: " +
OZONE_RATIS_SNAPSHOT_COMPLETE_FLAG_NAME + " not found");
+ }
return new RocksDBCheckpoint(untarredDbDir);
}
+ private void downloadSCMDBCheckpointOverGrpc(String hostAddress, File
targetFile, SecurityConfig secConf)
+ throws IOException {
+ int grpcPort =
+ configuration.getInt(ScmConfigKeys.OZONE_SCM_GRPC_PORT_KEY,
ScmConfigKeys.OZONE_SCM_GRPC_PORT_DEFAULT);
+ SCMSecurityProtocolClientSideTranslatorPB scmSecurityClient =
+ getScmSecurityClientWithMaxRetry(configuration, getCurrentUser());
+ try (ReconCertificateClient certClient =
+ new ReconCertificateClient(secConf, scmSecurityClient,
reconStorage, null, null);
+ SCMSnapshotDownloader downloadClient =
+ new InterSCMGrpcClient(hostAddress, grpcPort, configuration,
certClient)) {
+ downloadClient.download(targetFile.toPath()).get();
+ } catch (ExecutionException | InterruptedException e) {
+ LOG.error("Rocks DB checkpoint downloading failed: {}", e);
+ throw new IOException(e);
+ }
+ }
+
+ /**
+ * Downloads the SCM DB checkpoint over HTTP(S), authenticated with SPNEGO
when SCM requires it.
+ * Used when gRPC TLS is enabled, because the inter-SCM gRPC server then
accepts SCM certificates only.
+ */
+ private void downloadSCMDBCheckpoint(String url, File targetFile) throws
IOException {
+ LOG.info("Downloading SCM DB checkpoint from {}", url);
+ int connectTimeout = (int)
configuration.getTimeDuration(OZONE_RECON_SCM_SNAPSHOT_CONNECTION_TIMEOUT,
+ OZONE_RECON_SCM_SNAPSHOT_CONNECTION_TIMEOUT_DEFAULT,
TimeUnit.MILLISECONDS);
+ int readTimeout = (int)
configuration.getTimeDuration(OZONE_RECON_SCM_SNAPSHOT_READ_TIMEOUT,
+ OZONE_RECON_SCM_SNAPSHOT_READ_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS);
+ URLConnectionFactory connectionFactory =
+ URLConnectionFactory.newDefaultURLConnectionFactory(connectTimeout,
readTimeout, configuration);
+ boolean spnegoEnabled =
"kerberos".equals(configuration.get(HDDS_SCM_HTTP_AUTH_TYPE, "simple"));
+ AtomicReference<HttpURLConnection> connection = new AtomicReference<>();
+ // Download in another thread, so that cancelling the sync (an interrupt)
can abort a blocked read by disconnecting.
+ ExecutorService executor = Executors.newSingleThreadExecutor(
+ new
ThreadFactoryBuilder().setNameFormat("ReconSCMDBDownload-%d").setDaemon(true).build());
+ try {
+ executor.submit(() -> SecurityUtil.doAsLoginUser(() -> {
+ try {
+ connection.set(reconUtils.makeHttpCall(connectionFactory, url,
spnegoEnabled));
+ try (InputStream inputStream = connection.get().getInputStream()) {
+ FileUtils.copyInputStreamToFile(inputStream, targetFile);
+ }
+ if (Thread.currentThread().isInterrupted()) {
+ throw new InterruptedIOException("SCM DB checkpoint download
cancelled");
+ }
+ } catch (Exception e) {
+ FileUtils.deleteQuietly(targetFile);
+ throw e;
+ }
+ return null;
+ })).get();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new InterruptedIOException("SCM DB checkpoint download cancelled");
+ } catch (ExecutionException e) {
+ throw e.getCause() instanceof IOException ? (IOException) e.getCause() :
new IOException(e.getCause());
+ } finally {
+ HttpURLConnection activeConnection = connection.get();
+ if (activeConnection != null) {
+ activeConnection.disconnect();
Review Comment:
Thanks for addressing this. Cancellation still blocks when the helper is
waiting for more response-body data. With a real HttpURLConnection, I
reproduced the sync thread blocking inside `disconnect() →
KeepAliveStream.close()` while waiting for the reader’s lock. Consequently,
shutdownNow() is not reached and the sync-running flag remains set.
Could we use an abort mechanism that reliably stops an active read, and
cover this with a local HTTP server that sends some checkpoint bytes and then
stalls? The current mocked disconnect() releases the reader immediately, so it
cannot reproduce this case.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
