smengcl commented on code in PR #11954:
URL: https://github.com/apache/ozone/pull/11954#discussion_r4491171214
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/StorageContainerServiceProviderImpl.java:
##########
@@ -183,9 +201,112 @@ private RocksDBCheckpoint getRocksDBCheckpoint(String
snapshotFileName, File tar
snapshotFileName);
reconUtils.untarCheckpointFile(targetFile, untarredDbDir);
FileUtils.deleteQuietly(targetFile);
+ // SCM writes this flag last, so a checkpoint without it was cut short.
+ if (!ratisSnapshotComplete(untarredDbDir)) {
+ FileUtils.deleteQuietly(untarredDbDir.toFile());
+ throw new IOException("Incomplete SCM DB checkpoint: " +
OZONE_RATIS_SNAPSHOT_COMPLETE_FLAG_NAME + " not found");
+ }
return new RocksDBCheckpoint(untarredDbDir);
}
+ private void downloadSCMDBCheckpointOverGrpc(String hostAddress, File
targetFile, SecurityConfig secConf)
+ throws IOException {
+ int grpcPort =
+ configuration.getInt(ScmConfigKeys.OZONE_SCM_GRPC_PORT_KEY,
ScmConfigKeys.OZONE_SCM_GRPC_PORT_DEFAULT);
+ SCMSecurityProtocolClientSideTranslatorPB scmSecurityClient =
+ getScmSecurityClientWithMaxRetry(configuration, getCurrentUser());
+ try (ReconCertificateClient certClient =
+ new ReconCertificateClient(secConf, scmSecurityClient,
reconStorage, null, null);
+ SCMSnapshotDownloader downloadClient =
+ new InterSCMGrpcClient(hostAddress, grpcPort, configuration,
certClient)) {
+ downloadClient.download(targetFile.toPath()).get();
+ } catch (ExecutionException | InterruptedException e) {
+ LOG.error("Rocks DB checkpoint downloading failed: {}", e);
+ throw new IOException(e);
+ }
+ }
+
+ /**
+ * Downloads the SCM DB checkpoint over HTTP(S), authenticated with SPNEGO
when SCM requires it.
+ * Used when gRPC TLS is enabled, because the inter-SCM gRPC server then
accepts SCM certificates only.
+ */
+ private void downloadSCMDBCheckpoint(String url, File targetFile) throws
IOException {
+ LOG.info("Downloading SCM DB checkpoint from {}", url);
+ int connectTimeout = (int)
configuration.getTimeDuration(OZONE_RECON_SCM_SNAPSHOT_CONNECTION_TIMEOUT,
+ OZONE_RECON_SCM_SNAPSHOT_CONNECTION_TIMEOUT_DEFAULT,
TimeUnit.MILLISECONDS);
+ int readTimeout = (int)
configuration.getTimeDuration(OZONE_RECON_SCM_SNAPSHOT_READ_TIMEOUT,
+ OZONE_RECON_SCM_SNAPSHOT_READ_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS);
+ URLConnectionFactory connectionFactory =
+ URLConnectionFactory.newDefaultURLConnectionFactory(connectTimeout,
readTimeout, configuration);
+ boolean spnegoEnabled =
"kerberos".equals(configuration.get(HDDS_SCM_HTTP_AUTH_TYPE, "simple"));
+ AtomicReference<HttpURLConnection> connection = new AtomicReference<>();
+ // Download in another thread, so that cancelling the sync (an interrupt)
can abort a blocked read by disconnecting.
+ ExecutorService executor = Executors.newSingleThreadExecutor(
+ new
ThreadFactoryBuilder().setNameFormat("ReconSCMDBDownload-%d").setDaemon(true).build());
+ try {
+ executor.submit(() -> SecurityUtil.doAsLoginUser(() -> {
+ try {
+ connection.set(reconUtils.makeHttpCall(connectionFactory, url,
spnegoEnabled));
+ try (InputStream inputStream = connection.get().getInputStream()) {
+ FileUtils.copyInputStreamToFile(inputStream, targetFile);
+ }
+ if (Thread.currentThread().isInterrupted()) {
+ throw new InterruptedIOException("SCM DB checkpoint download
cancelled");
+ }
+ } catch (Exception e) {
+ FileUtils.deleteQuietly(targetFile);
+ throw e;
+ }
+ return null;
+ })).get();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new InterruptedIOException("SCM DB checkpoint download cancelled");
+ } catch (ExecutionException e) {
+ throw e.getCause() instanceof IOException ? (IOException) e.getCause() :
new IOException(e.getCause());
+ } finally {
+ HttpURLConnection activeConnection = connection.get();
+ if (activeConnection != null) {
+ activeConnection.disconnect();
Review Comment:
Thanks for reproducing it, you're right. A blocked `HttpURLConnection` read
can't be aborted from another thread: closing the stream or `disconnect()`
waits for the reader's lock, and for plain HTTP there is no hook to close the
socket. In 08da6f28299, cancelling no longer waits for the download. The sync
thread marks the download cancelled, deletes the partial tar and returns at
once, so the sync-running flag is cleared. The download thread stops at its
next read (it checks the flag) or at `ozone.recon.scm.snapshot.read.timeout`,
and then cleans up and disconnects.
`testGetSCMDBSnapshotCancelDoesNotWaitForStalledDownload` uses a local HTTP
server that sends 1 KB and then stalls. It fails on the previous commit (the
sync thread blocks in `disconnect()`) and passes now. If you'd prefer that the
read itself is cut immediately, that would mean moving off `HttpURLConnection`,
which I'd rather leave out of this fix.
_🤖 Addressed by [Claude Code](https://claude.com/claude-code)_
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
