This is an automated email from the ASF dual-hosted git repository.
jojochuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 4a819cc4eee HDDS-15171. Add available space check on follower during
bootstrap. (#10185)
4a819cc4eee is described below
commit 4a819cc4eee145cac67965f19f81a4186fc430bc
Author: Sadanand Shenoy <[email protected]>
AuthorDate: Thu Jul 16 03:30:52 2026 +0530
HDDS-15171. Add available space check on follower during bootstrap. (#10185)
---
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 8 +
.../common/src/main/resources/ozone-default.xml | 23 ++
.../hadoop/hdds/utils/DBCheckpointServlet.java | 10 +-
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 12 +
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 35 ++-
.../TestOMDbCheckpointServletInodeBasedXfer.java | 58 +++-
.../ozone/om/snapshot/TestOMDBCheckpointUtils.java | 17 ++
.../hadoop/ozone/om/OMDBCheckpointServlet.java | 65 +++--
.../om/OMDBCheckpointServletInodeBasedXfer.java | 26 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 16 +-
.../om/ratis_snapshot/OmRatisSnapshotProvider.java | 300 ++++++++++++++++++---
.../ozone/om/snapshot/OMDBCheckpointUtils.java | 74 +++--
.../TestOmRatisSnapshotProvider.java | 59 ++++
13 files changed, 607 insertions(+), 96 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index bce241f0ded..3b63ecf1974 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -130,6 +130,14 @@ public final class OzoneConsts {
public static final String OZONE_DB_CHECKPOINT_REQUEST_TO_EXCLUDE_SST =
"toExcludeSST";
+ /**
+ * Response header set by OM leader on full checkpoint responses with the
+ * estimated total uncompressed SST bytes (see OMDBCheckpointUtils); used by
+ * followers to pre-check disk space before streaming the tarball body.
+ */
+ public static final String OZONE_OM_CHECKPOINT_ESTIMATED_SST_BYTES_HEADER =
+ "X-Ozone-Om-Checkpoint-Estimated-Sst-Bytes";
+
public static final String RANGER_OZONE_SERVICE_VERSION_KEY =
"#RANGEROZONESERVICEVERSION";
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml
b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index dd33a02b02a..ad8ea45b3d7 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -2405,6 +2405,29 @@
request OM snapshot from OM Leader.
</description>
</property>
+ <property>
+ <name>ozone.om.bootstrap.min.space</name>
+ <value>5GB</value>
+ <tag>OZONE, OM, HA, MANAGEMENT</tag>
+ <description>
+ Minimum free space required on the volume that holds
ozone.om.ratis.snapshot.dir
+ before an OM follower downloads a ratis/bootstrap checkpoint from the
leader,
+ when the leader does not supply the
X-Ozone-Om-Checkpoint-Estimated-Sst-Bytes header
+ (incremental checkpoint or older OM version).
+ Use storage size syntax (e.g. 10GB). Set to 0 to disable this fallback
check.
+ </description>
+ </property>
+
+ <property>
+ <name>ozone.om.bootstrap.checkpoint.estimated.space.headroom.ratio</name>
+ <value>2.0</value>
+ <tag>OZONE, OM, HA, MANAGEMENT</tag>
+ <description>
+ Multiplier applied to the leader-reported estimated uncompressed SST
byte total
+ (X-Ozone-Om-Checkpoint-Estimated-Sst-Bytes) to approximate space needed
for the
+ checkpoint tar and unpack on the follower before streaming the response
body.
+ </description>
+ </property>
<property>
<name>ozone.om.fs.snapshot.max.limit</name>
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java
index a133e5188a2..6941ea3d537 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java
@@ -25,7 +25,6 @@
import com.google.common.annotations.VisibleForTesting;
import java.io.File;
import java.io.IOException;
-import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -238,8 +237,7 @@ public void
processMetadataSnapshotRequest(HttpServletRequest request, HttpServl
file + ".tar\"");
Instant start = Instant.now();
- writeDbDataToStream(checkpoint, request, response.getOutputStream(),
- receivedSstFiles, tmpdir);
+ writeDbDataToStream(checkpoint, request, response, receivedSstFiles,
tmpdir);
Instant end = Instant.now();
long duration = Duration.between(start, end).toMillis();
@@ -368,18 +366,18 @@ public void doPost(HttpServletRequest request,
HttpServletResponse response) {
* @param checkpoint The checkpoint to be written.
* @param ignoredRequest The httpRequest which generated this checkpoint.
* (Parameter is ignored in this class but used in child classes).
- * @param destination The stream to write to.
+ * @param response The HTTP response; the body is written to {@link
HttpServletResponse#getOutputStream()}.
* @param toExcludeList the files to be excluded
*
*/
public void writeDbDataToStream(DBCheckpoint checkpoint,
HttpServletRequest ignoredRequest,
- OutputStream destination,
+ HttpServletResponse response,
Set<String> toExcludeList,
Path tmpdir)
throws IOException, InterruptedException {
Objects.requireNonNull(toExcludeList);
- writeDBCheckpointToStream(checkpoint, destination, toExcludeList);
+ writeDBCheckpointToStream(checkpoint, response.getOutputStream(),
toExcludeList);
}
public DBStore getDbStore() {
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index b1209b6a213..02b270070ed 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@ -291,6 +291,18 @@ public final class OMConfigKeys {
OZONE_OM_SNAPSHOT_PROVIDER_REQUEST_TIMEOUT_DEFAULT =
TimeDuration.valueOf(300000, TimeUnit.MILLISECONDS);
+ public static final String OZONE_OM_BOOTSTRAP_MIN_SPACE_KEY =
+ "ozone.om.bootstrap.min.space";
+ public static final String OZONE_OM_BOOTSTRAP_MIN_SPACE_DEFAULT = "5GB";
+
+ /**
+ * Multiplier applied to the leader-reported estimated SST bytes when
deciding
+ * minimum free space before downloading a checkpoint (tar + unpack
headroom).
+ */
+ public static final String OZONE_OM_BOOTSTRAP_CHECKPOINT_HEADROOM_RATIO_KEY =
+ "ozone.om.bootstrap.checkpoint.estimated.space.headroom.ratio";
+ public static final double
OZONE_OM_BOOTSTRAP_CHECKPOINT_HEADROOM_RATIO_DEFAULT = 2.0D;
+
public static final String OZONE_OM_FS_SNAPSHOT_MAX_LIMIT =
"ozone.om.fs.snapshot.max.limit";
public static final int OZONE_OM_FS_SNAPSHOT_MAX_LIMIT_DEFAULT = 10000;
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java
index 8acc63de9aa..d9fe8a7543d 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java
@@ -47,6 +47,7 @@
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.anyBoolean;
import static org.mockito.Mockito.anyInt;
@@ -54,7 +55,6 @@
import static org.mockito.Mockito.doCallRealMethod;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
@@ -532,8 +532,9 @@ private void testWriteDbDataWithoutOmSnapshot()
// Get the tarball.
Path tmpdir = folder.resolve("bootstrapData");
try (OutputStream fileOutputStream =
Files.newOutputStream(tempFile.toPath())) {
+ HttpServletResponse mockResponse =
mockHttpServletResponse(fileOutputStream);
omDbCheckpointServletMock.writeDbDataToStream(dbCheckpoint, requestMock,
- fileOutputStream, new HashSet<>(), tmpdir);
+ mockResponse, new HashSet<>(), tmpdir);
}
// Untar the file into a temp folder to be examined.
@@ -577,8 +578,9 @@ private void testWriteDbDataWithToExcludeFileList()
// Get the tarball.
Path tmpdir = folder.resolve("bootstrapData");
try (OutputStream fileOutputStream =
Files.newOutputStream(tempFile.toPath())) {
+ HttpServletResponse mockResponse =
mockHttpServletResponse(fileOutputStream);
omDbCheckpointServletMock.writeDbDataToStream(dbCheckpoint, requestMock,
- fileOutputStream, toExcludeList, tmpdir);
+ mockResponse, toExcludeList, tmpdir);
}
// Untar the file into a temp folder to be examined.
@@ -598,6 +600,33 @@ private void testWriteDbDataWithToExcludeFileList()
assertThat(initialCheckpointSet).contains(dummyFile.getName());
}
+ private static HttpServletResponse mockHttpServletResponse(OutputStream out)
+ throws IOException {
+ HttpServletResponse response = mock(HttpServletResponse.class);
+ ServletOutputStream sos = new ServletOutputStream() {
+ @Override
+ public void write(int b) throws IOException {
+ out.write(b);
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ out.write(b, off, len);
+ }
+
+ @Override
+ public boolean isReady() {
+ return true;
+ }
+
+ @Override
+ public void setWriteListener(WriteListener writeListener) {
+ }
+ };
+ when(response.getOutputStream()).thenReturn(sos);
+ return response;
+ }
+
/**
* Calls endpoint in regards to parametrized HTTP method.
*/
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServletInodeBasedXfer.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServletInodeBasedXfer.java
index 791b8e91a86..518c762976a 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServletInodeBasedXfer.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServletInodeBasedXfer.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.ozone.om;
+import static java.net.HttpURLConnection.HTTP_OK;
import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS;
@@ -28,6 +29,9 @@
import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR;
import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA;
import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_FLUSH;
+import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_OM_CHECKPOINT_ESTIMATED_SST_BYTES_HEADER;
+import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_BOOTSTRAP_MIN_SPACE_KEY;
+import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_CHECKPOINT_USE_INODE_BASED_KEY;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_RATIS_SNAPSHOT_MAX_TOTAL_SST_SIZE_KEY;
import static org.apache.ozone.test.OzoneTestBase.uniqueObjectName;
import static org.assertj.core.api.Assertions.assertThat;
@@ -38,10 +42,10 @@
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
+import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyCollection;
import static org.mockito.ArgumentMatchers.anySet;
import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.Mockito.any;
import static org.mockito.Mockito.anyBoolean;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doCallRealMethod;
@@ -55,9 +59,12 @@
import static org.mockito.Mockito.when;
import java.io.BufferedReader;
+import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
@@ -97,6 +104,7 @@
import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
import org.apache.hadoop.hdds.utils.db.DBStore;
import org.apache.hadoop.hdds.utils.db.InodeMetadataRocksDBCheckpoint;
+import org.apache.hadoop.hdfs.web.URLConnectionFactory;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.TestDataUtil;
@@ -105,12 +113,14 @@
import org.apache.hadoop.ozone.client.OzoneSnapshot;
import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
import org.apache.hadoop.ozone.om.codec.OMDBDefinition;
+import org.apache.hadoop.ozone.om.helpers.OMNodeDetails;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.lock.DAGLeveledResource;
import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock;
import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.om.ratis_snapshot.OmRatisSnapshotProvider;
import org.apache.hadoop.ozone.om.snapshot.OmSnapshotUtils;
import org.apache.hadoop.ozone.om.snapshot.SnapshotCache;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
@@ -1043,6 +1053,52 @@ public static Map<String, List<String>>
readFileToMap(String filePath) throws IO
return dataMap;
}
+ /**
+ * Follower bootstrap must abort before streaming when the leader's SST
estimate header
+ * implies more free space than is available (v2 inode-based checkpoint URL).
+ */
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void
testBootstrapSnapshotDownloadAbortsWhenDiskSpaceBelowLeaderSstEstimate(boolean
useInodeBasedTransfer)
+ throws Exception {
+ Path snapshotDir = folder.resolve("ratis-snap-space-v2-" +
UUID.randomUUID());
+ Files.createDirectories(snapshotDir);
+ Path downloadTarget = folder.resolve("checkpoint-target-" +
UUID.randomUUID() + ".tar");
+
+ long usable = Files.getFileStore(snapshotDir).getUsableSpace();
+ long estimatedSstBytes = Math.addExact(Math.min(usable, Long.MAX_VALUE /
4), 1_000_000);
+
+ OzoneConfiguration diskCheckConf = new OzoneConfiguration();
+ diskCheckConf.set(OZONE_OM_BOOTSTRAP_MIN_SPACE_KEY, "0B");
+ diskCheckConf.setBoolean(OZONE_OM_DB_CHECKPOINT_USE_INODE_BASED_KEY,
useInodeBasedTransfer);
+
+ Map<String, OMNodeDetails> peers = new HashMap<>();
+ OMNodeDetails leaderDetails = mock(OMNodeDetails.class);
+ String leaderId = "leader1";
+ peers.put(leaderId, leaderDetails);
+ URL checkpointUrl = mock(URL.class);
+ when(leaderDetails.getOMDBCheckpointEndpointUrl(anyBoolean(),
anyBoolean(), eq(true)))
+ .thenReturn(checkpointUrl);
+
+ HttpURLConnection connection = mock(HttpURLConnection.class);
+ URLConnectionFactory connectionFactory = mock(URLConnectionFactory.class);
+ when(connectionFactory.openConnection(any(URL.class),
anyBoolean())).thenReturn(connection);
+
+ ByteArrayOutputStream uploadBody = new ByteArrayOutputStream();
+ when(connection.getOutputStream()).thenReturn(uploadBody);
+ when(connection.getResponseCode()).thenReturn(HTTP_OK);
+
when(connection.getHeaderField(OZONE_OM_CHECKPOINT_ESTIMATED_SST_BYTES_HEADER))
+ .thenReturn(Long.toString(estimatedSstBytes));
+
+ try (OmRatisSnapshotProvider provider = new
OmRatisSnapshotProvider(diskCheckConf,
+ snapshotDir.toFile(), peers, connectionFactory)) {
+ IOException ex = assertThrows(IOException.class,
+ () -> provider.downloadSnapshot(leaderId, downloadTarget.toFile()));
+
assertTrue(ex.getMessage().contains(OZONE_OM_CHECKPOINT_ESTIMATED_SST_BYTES_HEADER),
+ ex::getMessage);
+ }
+ }
+
private void populateInodesOfFilesInDirectory(DBStore dbStore, Path
dbLocation,
Set<String> inodesFromOmDbCheckpoint, Map<String, List<String>>
hardlinkMap) throws IOException {
try (Stream<Path> filesInOmDb = Files.list(dbLocation)) {
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOMDBCheckpointUtils.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOMDBCheckpointUtils.java
index 249e9285d7d..cda781d0570 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOMDBCheckpointUtils.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOMDBCheckpointUtils.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.om.snapshot;
import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
@@ -89,6 +90,22 @@ private static String getExpectedLogLine(String
expectedDataSize, int expectedSS
return String.format("%s%d, snapshots: %d", baseMessage, expectedSSTFiles,
expectedSnapshots);
}
+ @Test
+ public void testEstimateCheckpointTarballSstDetails() throws IOException {
+ writeSstFilesToDirectory(dbDir, 10, 10 * 1024);
+ Set<Path> snapshotDirs = new HashSet<>();
+ OMDBCheckpointUtils.SstSizeEstimate withoutSnapshots =
+ OMDBCheckpointUtils.estimateCheckpointTarballSstDetails(dbDir,
snapshotDirs);
+ assertEquals(10 * 10 * 1024L, withoutSnapshots.getTotalBytes());
+ assertEquals(10L, withoutSnapshots.getFileCount());
+
+ snapshotDirs.add(dbDir);
+ OMDBCheckpointUtils.SstSizeEstimate withSnapshots =
+ OMDBCheckpointUtils.estimateCheckpointTarballSstDetails(dbDir,
snapshotDirs);
+ assertEquals(20 * 10 * 1024L, withSnapshots.getTotalBytes());
+ assertEquals(20L, withSnapshots.getFileCount());
+ }
+
@Test
public void testIncludeSnapshotData() {
HttpServletRequest httpServletRequest = mock(HttpServletRequest.class);
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java
index b22bf5a6bee..fbb1b56ff16 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java
@@ -24,6 +24,7 @@
import static org.apache.hadoop.ozone.OzoneConsts.OM_CHECKPOINT_DIR;
import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR;
import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR;
+import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_OM_CHECKPOINT_ESTIMATED_SST_BYTES_HEADER;
import static org.apache.hadoop.ozone.OzoneConsts.ROCKSDB_SST_SUFFIX;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_RATIS_SNAPSHOT_MAX_TOTAL_SST_SIZE_DEFAULT;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_RATIS_SNAPSHOT_MAX_TOTAL_SST_SIZE_KEY;
@@ -37,7 +38,6 @@
import jakarta.annotation.Nonnull;
import java.io.File;
import java.io.IOException;
-import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -157,7 +157,7 @@ public void
processMetadataSnapshotRequest(HttpServletRequest request, HttpServl
@Override
public void writeDbDataToStream(DBCheckpoint checkpoint,
HttpServletRequest request,
- OutputStream destination,
+ HttpServletResponse response,
Set<String> toExcludeList,
Path tmpdir)
throws IOException, InterruptedException {
@@ -175,18 +175,35 @@ public void writeDbDataToStream(DBCheckpoint checkpoint,
// Map of link to path.
Map<Path, Path> hardLinkFiles = new HashMap<>();
- try (ArchiveOutputStream<TarArchiveEntry> archiveOutputStream =
tar(destination)) {
- RocksDBCheckpointDiffer differ =
- getDbStore().getRocksDBCheckpointDiffer();
- DirectoryData sstBackupDir = new DirectoryData(tmpdir,
- differ.getSSTBackupDir());
- DirectoryData compactionLogDir = new DirectoryData(tmpdir,
- differ.getCompactionLogDir());
+ RocksDBCheckpointDiffer differ =
+ getDbStore().getRocksDBCheckpointDiffer();
+ DirectoryData sstBackupDir = new DirectoryData(tmpdir,
+ differ.getSSTBackupDir());
+ DirectoryData compactionLogDir = new DirectoryData(tmpdir,
+ differ.getCompactionLogDir());
- // Files to be excluded from tarball
- Map<String, Map<Path, Path>> sstFilesToExclude =
normalizeExcludeList(toExcludeList,
- checkpoint.getCheckpointLocation(), sstBackupDir);
+ // Files to be excluded from tarball
+ Map<String, Map<Path, Path>> sstFilesToExclude =
normalizeExcludeList(toExcludeList,
+ checkpoint.getCheckpointLocation(), sstBackupDir);
+ if (sstFilesToExclude.isEmpty()) {
+ try {
+ Set<Path> snapshotPaths =
+ snapshotPathsForCheckpointEstimate(checkpoint,
includeSnapshotData(request));
+ OMDBCheckpointUtils.SstSizeEstimate estimate =
+ OMDBCheckpointUtils.estimateCheckpointTarballSstDetails(
+ checkpoint.getCheckpointLocation(), snapshotPaths);
+ response.setHeader(OZONE_OM_CHECKPOINT_ESTIMATED_SST_BYTES_HEADER,
+ Long.toString(estimate.getTotalBytes()));
+ OMDBCheckpointUtils.logEstimatedTarballSize(estimate,
snapshotPaths.size());
+ } catch (IOException e) {
+ LOG.warn("Could not estimate checkpoint tarball SST size for response
header: {}",
+ e.getMessage());
+ }
+ }
+
+ try (ArchiveOutputStream<TarArchiveEntry> archiveOutputStream =
+ tar(response.getOutputStream())) {
boolean completed = getFilesForArchive(checkpoint, copyFiles,
hardLinkFiles, sstFilesToExclude, includeSnapshotData(request),
sstBackupDir, compactionLogDir);
@@ -200,6 +217,15 @@ hardLinkFiles, sstFilesToExclude,
includeSnapshotData(request),
}
}
+ private Set<Path> snapshotPathsForCheckpointEstimate(DBCheckpoint checkpoint,
+ boolean includeSnapshotData) throws IOException {
+ Set<Path> snapshotPaths = new HashSet<>();
+ if (includeSnapshotData) {
+ snapshotPaths = getSnapshotDirs(checkpoint, false);
+ }
+ return snapshotPaths;
+ }
+
/**
* Format the list of excluded sst files from follower to match data
* on leader.
@@ -310,11 +336,6 @@ private boolean getFilesForArchive(DBCheckpoint checkpoint,
AtomicLong copySize = new AtomicLong(0L);
- // Log estimated total data transferred on first request.
- if (sstFilesToExclude.isEmpty()) {
- logEstimatedTarballSize(checkpoint, includeSnapshotData);
- }
-
// Get the active fs files.
Path dir = checkpoint.getCheckpointLocation();
if (!processDir(dir, copyFiles, hardLinkFiles, sstFilesToExclude,
@@ -347,16 +368,6 @@ private boolean getFilesForArchive(DBCheckpoint checkpoint,
compactionLogDir.getOriginalDir().toPath());
}
- private void logEstimatedTarballSize(DBCheckpoint checkpoint, boolean
includeSnapshotData)
- throws IOException {
- Set<Path> snapshotPaths = new HashSet<>();
- if (includeSnapshotData) {
- // since this is an estimate we can avoid waiting for dir to exist.
- snapshotPaths = getSnapshotDirs(checkpoint, false);
- }
-
OMDBCheckpointUtils.logEstimatedTarballSize(checkpoint.getCheckpointLocation(),
snapshotPaths);
- }
-
/**
* The snapshotInfo table may contain a snapshot that
* doesn't yet exist on the fs, so wait a few seconds for it.
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServletInodeBasedXfer.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServletInodeBasedXfer.java
index dfe610b0b50..9cea3d5a0ff 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServletInodeBasedXfer.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServletInodeBasedXfer.java
@@ -20,12 +20,12 @@
import static org.apache.hadoop.hdds.utils.Archiver.includeFile;
import static org.apache.hadoop.ozone.OzoneConsts.OM_CHECKPOINT_DIR;
import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_TO_EXCLUDE_SST;
+import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_OM_CHECKPOINT_ESTIMATED_SST_BYTES_HEADER;
import static org.apache.hadoop.ozone.OzoneConsts.ROCKSDB_SST_SUFFIX;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_RATIS_SNAPSHOT_MAX_TOTAL_SST_SIZE_DEFAULT;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_RATIS_SNAPSHOT_MAX_TOTAL_SST_SIZE_KEY;
import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath;
import static
org.apache.hadoop.ozone.om.snapshot.OMDBCheckpointUtils.includeSnapshotData;
-import static
org.apache.hadoop.ozone.om.snapshot.OMDBCheckpointUtils.logEstimatedTarballSize;
import static org.apache.hadoop.ozone.om.snapshot.OmSnapshotUtils.DATA_PREFIX;
import static org.apache.hadoop.ozone.om.snapshot.OmSnapshotUtils.DATA_SUFFIX;
@@ -49,6 +49,7 @@
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
+import java.util.OptionalLong;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
@@ -71,6 +72,7 @@
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import
org.apache.hadoop.ozone.om.lock.HierarchicalResourceLockManager.HierarchicalResourceLock;
import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.om.snapshot.OMDBCheckpointUtils;
import org.apache.hadoop.ozone.om.snapshot.OmSnapshotLocalDataManager;
import org.apache.hadoop.ozone.om.snapshot.OmSnapshotUtils;
import org.apache.hadoop.ozone.om.snapshot.SnapshotCache;
@@ -175,7 +177,10 @@ public void
processMetadataSnapshotRequest(HttpServletRequest request, HttpServl
response.setContentType("application/x-tar");
response.setHeader("Content-Disposition", "attachment; filename=\"" +
tarName + "\"");
Instant start = Instant.now();
- collectDbDataToTransfer(request, receivedSstFiles, omdbArchiver);
+ OptionalLong estimatedSstBytes =
+ collectDbDataToTransfer(request, receivedSstFiles, omdbArchiver);
+ estimatedSstBytes.ifPresent(bytes -> response.setHeader(
+ OZONE_OM_CHECKPOINT_ESTIMATED_SST_BYTES_HEADER,
Long.toString(bytes)));
Instant end = Instant.now();
long duration = Duration.between(start, end).toMillis();
LOG.info("Time taken to collect the DB data : {} milliseconds",
duration);
@@ -234,9 +239,10 @@ Path getCompactionLogDir() {
*
* @param request The HTTP servlet request containing parameters
for the snapshot.
* @param sstFilesToExclude Set of SST file identifiers to exclude from the
archive.
- * @throws IOException if an I/O error occurs during processing or streaming.
+ * @return estimated total uncompressed SST bytes for a full checkpoint
+ * (no SST exclusions), or empty if not computed
*/
- public void collectDbDataToTransfer(HttpServletRequest request,
+ public OptionalLong collectDbDataToTransfer(HttpServletRequest request,
Set<String> sstFilesToExclude, OMDBArchiver omdbArchiver) throws
IOException {
DBCheckpoint checkpoint = null;
OzoneManager om = (OzoneManager)
getServletContext().getAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE);
@@ -254,8 +260,17 @@ public void collectDbDataToTransfer(HttpServletRequest
request,
snapshotPaths = getSnapshotDirsFromDB(omMetadataManager,
omMetadataManager, snapshotLocalDataManager).values();
}
+ OptionalLong estimateForHeader = OptionalLong.empty();
if (sstFilesToExclude.isEmpty()) {
- logEstimatedTarballSize(getDbStore().getDbLocation().toPath(),
snapshotPaths);
+ try {
+ OMDBCheckpointUtils.SstSizeEstimate estimate = OMDBCheckpointUtils
+
.estimateCheckpointTarballSstDetails(getDbStore().getDbLocation().toPath(),
snapshotPaths);
+ OMDBCheckpointUtils.logEstimatedTarballSize(estimate,
snapshotPaths.size());
+ estimateForHeader = OptionalLong.of(estimate.getTotalBytes());
+ } catch (IOException e) {
+ LOG.warn("Could not estimate checkpoint tarball SST size for response
header: {}",
+ e.getMessage());
+ }
}
boolean shouldContinue = true;
@@ -339,6 +354,7 @@ public void collectDbDataToTransfer(HttpServletRequest
request,
} finally {
cleanupCheckpoint(checkpoint);
}
+ return estimateForHeader;
}
/**
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 1910d92e969..5d1e33f7cd8 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -50,6 +50,7 @@
import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
import static org.apache.hadoop.ozone.OzoneConsts.OM_METRICS_FILE;
import static org.apache.hadoop.ozone.OzoneConsts.OM_METRICS_TEMP_FILE;
+import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_OM_CHECKPOINT_ESTIMATED_SST_BYTES_HEADER;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_RATIS_SNAPSHOT_DIR;
import static org.apache.hadoop.ozone.OzoneConsts.PREPARE_MARKER_KEY;
import static org.apache.hadoop.ozone.OzoneConsts.RPC_PORT;
@@ -60,6 +61,8 @@
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_DELETING_LIMIT_PER_TASK;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
+import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_BOOTSTRAP_CHECKPOINT_HEADROOM_RATIO_KEY;
+import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_BOOTSTRAP_MIN_SPACE_KEY;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_EDEKCACHELOADER_INITIAL_DELAY_MS_DEFAULT;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_EDEKCACHELOADER_INITIAL_DELAY_MS_KEY;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_EDEKCACHELOADER_INTERVAL_MS_DEFAULT;
@@ -4115,7 +4118,18 @@ public synchronized TermIndex
installSnapshotFromLeader(String leaderId) throws
omDBCheckpoint = omRatisSnapshotProvider.
downloadDBSnapshotFromLeader(leaderId);
} catch (IOException ex) {
- LOG.error("Failed to download snapshot from Leader {}.", leaderId, ex);
+ if (OmRatisSnapshotProvider.isDiskFullOrQuotaIOException(ex)) {
+ LOG.error(
+ "Failed to download snapshot from leader {}: local disk appears
full or over quota "
+ + "on the OM ratis snapshot volume (see previous ERROR for
path/usable space). "
+ + "Free disk or adjust {}, {}, or {} before bootstrap can
succeed.",
+ leaderId,
+ OZONE_OM_BOOTSTRAP_MIN_SPACE_KEY,
+ OZONE_OM_BOOTSTRAP_CHECKPOINT_HEADROOM_RATIO_KEY,
+ OZONE_OM_CHECKPOINT_ESTIMATED_SST_BYTES_HEADER);
+ } else {
+ LOG.error("Failed to download snapshot from Leader {}.", leaderId, ex);
+ }
cleanupCheckpoint(omDBCheckpoint);
return null;
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java
index 9de1b692c5c..e087a617d01 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java
@@ -22,6 +22,11 @@
import static org.apache.hadoop.ozone.OzoneConsts.MULTIPART_FORM_DATA_BOUNDARY;
import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME;
import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_TO_EXCLUDE_SST;
+import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_OM_CHECKPOINT_ESTIMATED_SST_BYTES_HEADER;
+import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_BOOTSTRAP_CHECKPOINT_HEADROOM_RATIO_DEFAULT;
+import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_BOOTSTRAP_CHECKPOINT_HEADROOM_RATIO_KEY;
+import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_BOOTSTRAP_MIN_SPACE_DEFAULT;
+import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_BOOTSTRAP_MIN_SPACE_KEY;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_CHECKPOINT_USE_INODE_BASED_DEFAULT;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_CHECKPOINT_USE_INODE_BASED_KEY;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_AUTH_TYPE;
@@ -30,6 +35,7 @@
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_PROVIDER_REQUEST_TIMEOUT_DEFAULT;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_PROVIDER_REQUEST_TIMEOUT_KEY;
+import com.google.common.annotations.VisibleForTesting;
import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
@@ -37,14 +43,17 @@
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
+import java.nio.file.FileSystemException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hdds.conf.MutableConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
import org.apache.hadoop.hdds.server.http.HttpConfig;
import org.apache.hadoop.hdds.utils.HAUtils;
import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource;
@@ -54,6 +63,8 @@
import org.apache.hadoop.hdfs.web.URLConnectionFactory;
import org.apache.hadoop.ozone.om.helpers.OMNodeDetails;
import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -86,6 +97,103 @@ public class OmRatisSnapshotProvider extends
RDBSnapshotProvider {
private final boolean spnegoEnabled;
private final URLConnectionFactory connectionFactory;
private final boolean useV2CheckpointApi;
+ /** Minimum usable bytes on snapshot volume before download; 0 = disabled. */
+ private final long bootstrapMinSpaceBytes;
+ /** Applied to leader-reported SST byte estimate to reserve tar/unpack
headroom. */
+ private final double bootstrapCheckpointHeadroomRatio;
+
+ private static final class BootstrapSpaceRequirement {
+ private final long requiredBytes;
+ private final boolean usedLeaderEstimateHeader;
+
+ private BootstrapSpaceRequirement(long requiredBytes, boolean
usedLeaderEstimateHeader) {
+ this.requiredBytes = requiredBytes;
+ this.usedLeaderEstimateHeader = usedLeaderEstimateHeader;
+ }
+ }
+
+ /**
+ * Whether this {@link IOException} (or its causes) typically means the
+ * local filesystem ran out of space or hit a quota while writing.
+ */
+ public static boolean isDiskFullOrQuotaIOException(IOException ioe) {
+ for (Throwable t = ioe; t != null; t = t.getCause()) {
+ if (t instanceof DiskOutOfSpaceException) {
+ return true;
+ }
+ if (matchesDiskFullOrQuotaMessage(t)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Best-effort supplement for JDK {@link FileSystemException} ENOSPC and
+ * quota wording on Linux OM deployments (typically English from libc/JVM).
+ * {@link DiskOutOfSpaceException} in the cause chain is handled by type
+ * in {@link #isDiskFullOrQuotaIOException(IOException)} and does not depend
+ * on message text. Localized OS messages without matching substrings are not
+ * detected here.
+ */
+ private static boolean matchesDiskFullOrQuotaMessage(Throwable throwable) {
+ if (throwable instanceof FileSystemException) {
+ String reason = ((FileSystemException) throwable).getReason();
+ if (reason != null && containsDiskFullOrQuotaText(reason)) {
+ return true;
+ }
+ }
+ String msg = throwable.getMessage();
+ return msg != null && containsDiskFullOrQuotaText(msg);
+ }
+
+ private static boolean containsDiskFullOrQuotaText(String text) {
+ String m = text.toLowerCase(Locale.ROOT);
+ return m.contains("no space left on device")
+ || m.contains("no space")
+ || m.contains("space left")
+ || m.contains("enospc")
+ || m.contains("disk quota exceeded")
+ || m.contains("quota exceeded")
+ || m.contains("quota");
+ }
+
+ private static String formatSnapshotVolumeUsableSpace(File pathOnVolume) {
+ try {
+ Path storePath =
+ pathOnVolume.isDirectory() ? pathOnVolume.toPath() :
pathOnVolume.toPath().getParent();
+ if (storePath == null) {
+ return "unknown";
+ }
+ long usable = Files.getFileStore(storePath).getUsableSpace();
+ return String.format("%s (%d bytes)", StringUtils.byteDesc(usable),
usable);
+ } catch (Exception e) {
+ return "unknown (" + e.getMessage() + ")";
+ }
+ }
+
+ /**
+ * Logs at ERROR when the failure is likely due to disk full / quota, so
+ * operators can distinguish it from network or leader-side errors.
+ */
+ private static void logDiskFullOrQuotaDuringDownload(
+ IOException ioe, File targetFile, String leaderNodeId, URL
checkpointUrl) {
+ if (!isDiskFullOrQuotaIOException(ioe)) {
+ return;
+ }
+ LOG.error(
+ "OM ratis snapshot download from leader {} failed: disk full or
filesystem quota while "
+ + "writing checkpoint file {} (checkpoint URL {}). Usable space on
this volume: {}. "
+ + "Free disk on this OM node or raise {} or adjust {}. Underlying
message: {}",
+ leaderNodeId,
+ targetFile.getAbsolutePath(),
+ checkpointUrl,
+ formatSnapshotVolumeUsableSpace(targetFile),
+ OZONE_OM_BOOTSTRAP_MIN_SPACE_KEY,
+ OZONE_OM_BOOTSTRAP_CHECKPOINT_HEADROOM_RATIO_KEY,
+ ioe.getMessage(),
+ ioe);
+ }
public OmRatisSnapshotProvider(File snapshotDir,
Map<String, OMNodeDetails> peerNodesMap, HttpConfig.Policy httpPolicy,
@@ -96,38 +204,64 @@ public OmRatisSnapshotProvider(File snapshotDir,
this.spnegoEnabled = spnegoEnabled;
this.connectionFactory = connectionFactory;
this.useV2CheckpointApi = OZONE_OM_DB_CHECKPOINT_USE_INODE_BASED_DEFAULT;
+ this.bootstrapMinSpaceBytes = 0L;
+ this.bootstrapCheckpointHeadroomRatio =
OZONE_OM_BOOTSTRAP_CHECKPOINT_HEADROOM_RATIO_DEFAULT;
}
public OmRatisSnapshotProvider(MutableConfigurationSource conf,
File omRatisSnapshotDir, Map<String, OMNodeDetails> peerNodeDetails) {
+ this(conf, omRatisSnapshotDir, peerNodeDetails, null);
+ }
+
+ /**
+ * Same as {@link #OmRatisSnapshotProvider(MutableConfigurationSource, File,
Map)} but allows
+ * tests to inject a {@link URLConnectionFactory} (for example a factory
that returns a mock
+ * {@link HttpURLConnection}).
+ */
+ @VisibleForTesting
+ public OmRatisSnapshotProvider(MutableConfigurationSource conf,
+ File omRatisSnapshotDir,
+ Map<String, OMNodeDetails> peerNodeDetails,
+ URLConnectionFactory connectionFactoryOverride) {
super(omRatisSnapshotDir, OM_DB_NAME);
LOG.info("Initializing OM Snapshot Provider");
this.peerNodesMap = new ConcurrentHashMap<>();
peerNodesMap.putAll(peerNodeDetails);
this.useV2CheckpointApi =
conf.getBoolean(OZONE_OM_DB_CHECKPOINT_USE_INODE_BASED_KEY,
OZONE_OM_DB_CHECKPOINT_USE_INODE_BASED_DEFAULT);
+ this.bootstrapMinSpaceBytes = (long) conf.getStorageSize(
+ OZONE_OM_BOOTSTRAP_MIN_SPACE_KEY,
+ OZONE_OM_BOOTSTRAP_MIN_SPACE_DEFAULT,
+ StorageUnit.BYTES);
+ this.bootstrapCheckpointHeadroomRatio = conf.getDouble(
+ OZONE_OM_BOOTSTRAP_CHECKPOINT_HEADROOM_RATIO_KEY,
+ OZONE_OM_BOOTSTRAP_CHECKPOINT_HEADROOM_RATIO_DEFAULT);
this.httpPolicy = HttpConfig.getHttpPolicy(conf);
this.spnegoEnabled = conf.get(OZONE_OM_HTTP_AUTH_TYPE, "simple")
.equals("kerberos");
- TimeUnit connectionTimeoutUnit =
- OZONE_OM_SNAPSHOT_PROVIDER_CONNECTION_TIMEOUT_DEFAULT.getUnit();
- int connectionTimeoutMS = (int) conf.getTimeDuration(
- OZONE_OM_SNAPSHOT_PROVIDER_CONNECTION_TIMEOUT_KEY,
- OZONE_OM_SNAPSHOT_PROVIDER_CONNECTION_TIMEOUT_DEFAULT.getDuration(),
- connectionTimeoutUnit);
-
- TimeUnit requestTimeoutUnit =
- OZONE_OM_SNAPSHOT_PROVIDER_REQUEST_TIMEOUT_DEFAULT.getUnit();
- int requestTimeoutMS = (int) conf.getTimeDuration(
- OZONE_OM_SNAPSHOT_PROVIDER_REQUEST_TIMEOUT_KEY,
- OZONE_OM_SNAPSHOT_PROVIDER_REQUEST_TIMEOUT_DEFAULT.getDuration(),
- requestTimeoutUnit);
-
- connectionFactory = URLConnectionFactory
- .newDefaultURLConnectionFactory(connectionTimeoutMS, requestTimeoutMS,
- LegacyHadoopConfigurationSource.asHadoopConfiguration(conf));
+ if (connectionFactoryOverride != null) {
+ this.connectionFactory = connectionFactoryOverride;
+ } else {
+ TimeUnit connectionTimeoutUnit =
+ OZONE_OM_SNAPSHOT_PROVIDER_CONNECTION_TIMEOUT_DEFAULT.getUnit();
+ int connectionTimeoutMS = (int) conf.getTimeDuration(
+ OZONE_OM_SNAPSHOT_PROVIDER_CONNECTION_TIMEOUT_KEY,
+ OZONE_OM_SNAPSHOT_PROVIDER_CONNECTION_TIMEOUT_DEFAULT.getDuration(),
+ connectionTimeoutUnit);
+
+ TimeUnit requestTimeoutUnit =
+ OZONE_OM_SNAPSHOT_PROVIDER_REQUEST_TIMEOUT_DEFAULT.getUnit();
+ int requestTimeoutMS = (int) conf.getTimeDuration(
+ OZONE_OM_SNAPSHOT_PROVIDER_REQUEST_TIMEOUT_KEY,
+ OZONE_OM_SNAPSHOT_PROVIDER_REQUEST_TIMEOUT_DEFAULT.getDuration(),
+ requestTimeoutUnit);
+
+ this.connectionFactory = URLConnectionFactory
+ .newDefaultURLConnectionFactory(connectionTimeoutMS,
requestTimeoutMS,
+ LegacyHadoopConfigurationSource.asHadoopConfiguration(conf));
+ }
}
/**
@@ -144,6 +278,88 @@ public void removeDecommissionedPeerNode(String
decommNodeId) {
peerNodesMap.remove(decommNodeId);
}
+ /**
+ * Ensures the filesystem that holds {@link #getSnapshotDir()} has enough
+ * free space for OM bootstrap / install snapshot download and unpack.
+ *
+ * @throws IOException if {@link #bootstrapMinSpaceBytes} is > 0 and
+ * usable space is below the configured minimum
+ */
+ void ensureBootstrapDiskSpace() throws IOException {
+ ensureBootstrapDiskSpaceForRequiredBytes(
+ new BootstrapSpaceRequirement(bootstrapMinSpaceBytes, false));
+ }
+
+ private BootstrapSpaceRequirement resolveBootstrapSpaceRequirement(
+ HttpURLConnection connection) {
+ String headerValue =
+
connection.getHeaderField(OZONE_OM_CHECKPOINT_ESTIMATED_SST_BYTES_HEADER);
+ if (headerValue != null) {
+ String trimmed = headerValue.trim();
+ if (!trimmed.isEmpty()) {
+ try {
+ long estimatedSstBytes = Long.parseLong(trimmed);
+ if (estimatedSstBytes > 0) {
+ long required = (long) Math.ceil(estimatedSstBytes *
bootstrapCheckpointHeadroomRatio);
+ return new BootstrapSpaceRequirement(required, true);
+ }
+ } catch (NumberFormatException e) {
+ LOG.warn("Ignoring invalid {} response header: {}",
+ OZONE_OM_CHECKPOINT_ESTIMATED_SST_BYTES_HEADER, headerValue);
+ }
+ }
+ }
+ return new BootstrapSpaceRequirement(bootstrapMinSpaceBytes, false);
+ }
+
+ private void
ensureBootstrapDiskSpaceForRequiredBytes(BootstrapSpaceRequirement requirement)
+ throws IOException {
+ if (requirement.requiredBytes <= 0) {
+ if (requirement.usedLeaderEstimateHeader) {
+ LOG.debug("Leader returned a non-positive SST size estimate; skipping
disk space check.");
+ } else {
+ LOG.debug("{} is 0 or negative; skipping bootstrap disk space check.",
+ OZONE_OM_BOOTSTRAP_MIN_SPACE_KEY);
+ }
+ return;
+ }
+ File snapshotRoot = getSnapshotDir();
+ if (!snapshotRoot.exists()) {
+ throw new IOException(String.format(
+ "OM ratis snapshot directory %s does not exist; cannot verify
required free space (%s)",
+ snapshotRoot.getAbsolutePath(),
+ StringUtils.byteDesc(requirement.requiredBytes)));
+ }
+ final long usable =
Files.getFileStore(snapshotRoot.toPath()).getUsableSpace();
+ if (usable < requirement.requiredBytes) {
+ String source = requirement.usedLeaderEstimateHeader
+ ? OZONE_OM_CHECKPOINT_ESTIMATED_SST_BYTES_HEADER + " with "
+ + OZONE_OM_BOOTSTRAP_CHECKPOINT_HEADROOM_RATIO_KEY
+ : OZONE_OM_BOOTSTRAP_MIN_SPACE_KEY;
+ String message = String.format(
+ "OM bootstrap / install snapshot aborted: volume containing ratis
snapshot dir "
+ + "%s has usable space %s (%d bytes) but at least %s (%d bytes)
is required "
+ + "(from %s). Free disk on this OM host or adjust
configuration.",
+ snapshotRoot.getAbsolutePath(),
+ StringUtils.byteDesc(usable),
+ usable,
+ StringUtils.byteDesc(requirement.requiredBytes),
+ requirement.requiredBytes,
+ source);
+ LOG.error(message);
+ throw new IOException(message);
+ }
+ LOG.info(
+ "Bootstrap disk space check passed for OM ratis snapshot dir {}:
usable {} >= "
+ + "required {} (from {})",
+ snapshotRoot.getAbsolutePath(),
+ StringUtils.byteDesc(usable),
+ StringUtils.byteDesc(requirement.requiredBytes),
+ requirement.usedLeaderEstimateHeader
+ ? OZONE_OM_CHECKPOINT_ESTIMATED_SST_BYTES_HEADER
+ : OZONE_OM_BOOTSTRAP_MIN_SPACE_KEY);
+ }
+
@Override
public void downloadSnapshot(String leaderNodeID, File targetFile)
throws IOException {
@@ -156,35 +372,43 @@ public void downloadSnapshot(String leaderNodeID, File
targetFile)
HttpURLConnection connection = (HttpURLConnection)
connectionFactory.openConnection(omCheckpointUrl, spnegoEnabled);
- connection.setRequestMethod("POST");
- String contentTypeValue = "multipart/form-data; boundary=" +
- MULTIPART_FORM_DATA_BOUNDARY;
- connection.setRequestProperty("Content-Type", contentTypeValue);
- connection.setDoOutput(true);
-
- List<String> existingFiles = useV2CheckpointApi ?
HAUtils.getExistingFiles(getCandidateDir())
- : HAUtils.getExistingSstFilesRelativeToDbDir(getCandidateDir());
- writeFormData(connection, existingFiles);
-
- connection.connect();
- int errorCode = connection.getResponseCode();
- if ((errorCode != HTTP_OK) && (errorCode != HTTP_CREATED)) {
- throw new IOException("Unexpected exception when trying to reach " +
- "OM to download latest checkpoint. Checkpoint URL: " +
- omCheckpointUrl + ". ErrorCode: " + errorCode);
- }
+ try {
+ connection.setRequestMethod("POST");
+ String contentTypeValue = "multipart/form-data; boundary=" +
+ MULTIPART_FORM_DATA_BOUNDARY;
+ connection.setRequestProperty("Content-Type", contentTypeValue);
+ connection.setDoOutput(true);
+
+ List<String> existingFiles = useV2CheckpointApi ?
HAUtils.getExistingFiles(getCandidateDir())
+ : HAUtils.getExistingSstFilesRelativeToDbDir(getCandidateDir());
+ writeFormData(connection, existingFiles);
+
+ connection.connect();
+ int errorCode = connection.getResponseCode();
+ if ((errorCode != HTTP_OK) && (errorCode != HTTP_CREATED)) {
+ throw new IOException("Unexpected exception when trying to reach " +
+ "OM to download latest checkpoint. Checkpoint URL: " +
+ omCheckpointUrl + ". ErrorCode: " + errorCode);
+ }
- try (InputStream inputStream = connection.getInputStream()) {
- downloadFileWithProgress(inputStream, targetFile);
+ ensureBootstrapDiskSpaceForRequiredBytes(
+ resolveBootstrapSpaceRequirement(connection));
+
+ try (InputStream inputStream = connection.getInputStream()) {
+ downloadFileWithProgress(inputStream, targetFile);
+ }
} catch (IOException ex) {
+ logDiskFullOrQuotaDuringDownload(ex, targetFile, leaderNodeID,
omCheckpointUrl);
boolean deleted = FileUtils.deleteQuietly(targetFile);
- if (!deleted) {
+ if (!deleted && targetFile.exists()) {
LOG.error("OM snapshot which failed to download {} cannot be
deleted",
targetFile);
}
throw ex;
} finally {
- connection.disconnect();
+ if (connection != null) {
+ connection.disconnect();
+ }
}
return null;
});
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OMDBCheckpointUtils.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OMDBCheckpointUtils.java
index e264709bd16..558ff596cbc 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OMDBCheckpointUtils.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OMDBCheckpointUtils.java
@@ -21,6 +21,7 @@
import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA;
import static org.apache.hadoop.ozone.OzoneConsts.ROCKSDB_SST_SUFFIX;
+import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collection;
@@ -49,32 +50,75 @@ public final class OMDBCheckpointUtils {
private OMDBCheckpointUtils() {
}
+ /**
+ * Uncompressed total size of SST files under the DB checkpoint and optional
+ * snapshot paths (used for logging and follower disk checks).
+ */
+ public static final class SstSizeEstimate {
+ private final long totalBytes;
+ private final long fileCount;
+
+ public SstSizeEstimate(long totalBytes, long fileCount) {
+ this.totalBytes = totalBytes;
+ this.fileCount = fileCount;
+ }
+
+ public long getTotalBytes() {
+ return totalBytes;
+ }
+
+ public long getFileCount() {
+ return fileCount;
+ }
+ }
+
public static boolean includeSnapshotData(HttpServletRequest request) {
String includeParam =
request.getParameter(OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA);
return Boolean.parseBoolean(includeParam);
}
+ /**
+ * Walks the given paths and sums logical size of SST files only.
+ *
+ * @throws IOException if the file tree walk fails
+ */
+ public static SstSizeEstimate estimateCheckpointTarballSstDetails(
+ Path dbLocation, Collection<Path> snapshotPaths) throws IOException {
+ Counters.PathCounters counters = Counters.longPathCounters();
+ CountingPathVisitor visitor = new CountingPathVisitor(
+ counters, SST_FILE_FILTER, TRUE);
+ Files.walkFileTree(dbLocation, visitor);
+ boolean includeSnapshotData = !snapshotPaths.isEmpty();
+ if (includeSnapshotData) {
+ for (Path snapshotDir : snapshotPaths) {
+ Files.walkFileTree(snapshotDir, visitor);
+ }
+ }
+ return new SstSizeEstimate(
+ counters.getByteCounter().get(),
+ counters.getFileCounter().get());
+ }
+
public static void logEstimatedTarballSize(Path dbLocation, Collection<Path>
snapshotPaths) {
try {
- Counters.PathCounters counters = Counters.longPathCounters();
- CountingPathVisitor visitor = new CountingPathVisitor(
- counters, SST_FILE_FILTER, TRUE);
- Files.walkFileTree(dbLocation, visitor);
- boolean includeSnapshotData = !snapshotPaths.isEmpty();
- long totalSnapshots = snapshotPaths.size();
- if (includeSnapshotData) {
- for (Path snapshotDir: snapshotPaths) {
- Files.walkFileTree(snapshotDir, visitor);
- }
- }
- LOG.info("Estimates for Checkpoint Tarball Stream - Data size: {} KB,
SST files: {}{}",
- counters.getByteCounter().get() / (1024),
- counters.getFileCounter().get(),
- (includeSnapshotData ? ", snapshots: " + totalSnapshots : ""));
+ SstSizeEstimate estimate =
+ estimateCheckpointTarballSstDetails(dbLocation, snapshotPaths);
+ logEstimatedTarballSize(estimate, snapshotPaths.size());
} catch (Exception e) {
LOG.error("Could not estimate size of transfer to Checkpoint Tarball
Stream for dbLocation:{} snapshotPaths:{}",
dbLocation, snapshotPaths, e);
}
}
+
+ /**
+ * Logs the result of a prior {@link #estimateCheckpointTarballSstDetails}
call.
+ */
+ public static void logEstimatedTarballSize(SstSizeEstimate estimate, int
snapshotDirCount) {
+ boolean includeSnapshotData = snapshotDirCount > 0;
+ LOG.info("Estimates for Checkpoint Tarball Stream - Data size: {} KB, SST
files: {}{}",
+ estimate.getTotalBytes() / (1024),
+ estimate.getFileCount(),
+ (includeSnapshotData ? ", snapshots: " + snapshotDirCount : ""));
+ }
}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis_snapshot/TestOmRatisSnapshotProvider.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis_snapshot/TestOmRatisSnapshotProvider.java
index 2fb0f56ae89..13c6b2355d4 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis_snapshot/TestOmRatisSnapshotProvider.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis_snapshot/TestOmRatisSnapshotProvider.java
@@ -19,7 +19,10 @@
import static java.net.HttpURLConnection.HTTP_OK;
import static org.apache.hadoop.ozone.OzoneConsts.MULTIPART_FORM_DATA_BOUNDARY;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.anyBoolean;
import static org.mockito.Mockito.mock;
@@ -33,15 +36,19 @@
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.StandardCharsets;
+import java.nio.file.FileSystemException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.server.http.HttpConfig;
import org.apache.hadoop.hdfs.web.URLConnectionFactory;
import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.helpers.OMNodeDetails;
import
org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
@@ -80,6 +87,58 @@ public void setup(@TempDir File snapshotDir,
false, connectionFactory);
}
+ @Test
+ public void testIsDiskFullOrQuotaIOExceptionDetectsNoSpaceMessage() {
+ assertThat(OmRatisSnapshotProvider.isDiskFullOrQuotaIOException(
+ new IOException("No space left on device"))).isTrue();
+ }
+
+ @Test
+ public void
testIsDiskFullOrQuotaIOExceptionDetectsFileSystemExceptionReason() {
+ IOException wrapped = new IOException("write failed",
+ new FileSystemException("p", null, "No space left on device"));
+
assertThat(OmRatisSnapshotProvider.isDiskFullOrQuotaIOException(wrapped)).isTrue();
+ }
+
+ @Test
+ public void
testIsDiskFullOrQuotaIOExceptionDetectsDiskOutOfSpaceExceptionInCauseChain() {
+ IOException wrapped = new IOException("write failed", new
DiskOutOfSpaceException("full"));
+
assertThat(OmRatisSnapshotProvider.isDiskFullOrQuotaIOException(wrapped)).isTrue();
+ }
+
+ @Test
+ public void
testIsDiskFullOrQuotaIOExceptionReturnsFalseForNonEnglishFileSystemException() {
+ IOException wrapped = new IOException("write failed",
+ new FileSystemException("p", null, "Kein Speicherplatz mehr auf dem
Gerät"));
+
assertThat(OmRatisSnapshotProvider.isDiskFullOrQuotaIOException(wrapped)).isFalse();
+ }
+
+ @Test
+ public void testIsDiskFullOrQuotaIOExceptionReturnsFalseForOtherErrors() {
+ assertThat(OmRatisSnapshotProvider.isDiskFullOrQuotaIOException(
+ new IOException("Connection reset"))).isFalse();
+ }
+
+ @Test
+ public void testBootstrapDiskSpaceCheckSkippedWhenZero(@TempDir File
snapshotDir) {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(OMConfigKeys.OZONE_OM_BOOTSTRAP_MIN_SPACE_KEY, "0GB");
+ OmRatisSnapshotProvider provider =
+ new OmRatisSnapshotProvider(conf, snapshotDir, new HashMap<>());
+ assertDoesNotThrow(provider::ensureBootstrapDiskSpace);
+ }
+
+ @Test
+ public void testBootstrapDiskSpaceCheckFailsWhenBelowMinimum(@TempDir File
snapshotDir) {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(OMConfigKeys.OZONE_OM_BOOTSTRAP_MIN_SPACE_KEY, "1024EB");
+ OmRatisSnapshotProvider provider =
+ new OmRatisSnapshotProvider(conf, snapshotDir, new HashMap<>());
+ IOException ex =
+ assertThrows(IOException.class, provider::ensureBootstrapDiskSpace);
+
assertThat(ex.getMessage()).contains(OMConfigKeys.OZONE_OM_BOOTSTRAP_MIN_SPACE_KEY);
+ }
+
@Test
public void testDownloadSnapshot() throws IOException,
AuthenticationException {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
