This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 2c578c3ae7 HDDS-8980. RocksDB crash in
RocksDBStoreMetrics.computeSstFileStat (#5034)
2c578c3ae7 is described below
commit 2c578c3ae73417d2fd2f24ab356632ddb5c427fb
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Tue Jul 11 14:24:37 2023 +0200
HDDS-8980. RocksDB crash in RocksDBStoreMetrics.computeSstFileStat (#5034)
---
.../hadoop/hdds/utils/RocksDBStoreMetrics.java | 27 ++++++++++++++-------
.../org/apache/hadoop/hdds/utils/db/DBStore.java | 11 ++-------
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 6 ++---
.../apache/hadoop/hdds/utils/db/RocksDatabase.java | 28 ++++++++++++++--------
.../org/apache/hadoop/ozone/om/TestOmSnapshot.java | 3 ++-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 2 +-
.../protocolPB/OzoneManagerRequestHandler.java | 3 +--
.../recon/recovery/ReconOmMetadataManagerImpl.java | 6 ++++-
8 files changed, 50 insertions(+), 36 deletions(-)
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/RocksDBStoreMetrics.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/RocksDBStoreMetrics.java
index 8e16556b6e..479e88ed3e 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/RocksDBStoreMetrics.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/RocksDBStoreMetrics.java
@@ -230,17 +230,22 @@ public class RocksDBStoreMetrics implements MetricsSource
{
cfPros[index][0], e);
}
- // Calculate number of files per level and size per level
- Map<String, Map<Integer, Map<String, Long>>> data = computeSstFileStat();
+ try {
+ // Calculate number of files per level and size per level
+ Map<String, Map<Integer, Map<String, Long>>> data = computeSstFileStat();
- // Export file number
- exportSstFileStat(rb, data.get(NUM_FILES_AT_LEVEL), NUM_FILES_AT_LEVEL);
+ // Export file number
+ exportSstFileStat(rb, data.get(NUM_FILES_AT_LEVEL), NUM_FILES_AT_LEVEL);
- // Export file total size
- exportSstFileStat(rb, data.get(SIZE_AT_LEVEL), SIZE_AT_LEVEL);
+ // Export file total size
+ exportSstFileStat(rb, data.get(SIZE_AT_LEVEL), SIZE_AT_LEVEL);
+ } catch (IOException e) {
+ LOG.error("Failed to compute sst file stat", e);
+ }
}
- private Map<String, Map<Integer, Map<String, Long>>> computeSstFileStat() {
+ private Map<String, Map<Integer, Map<String, Long>>> computeSstFileStat()
+ throws IOException {
// Calculate number of files per level and size per level
List<LiveFileMetaData> liveFileMetaDataList =
rocksDB.getLiveFilesMetaData();
@@ -292,7 +297,11 @@ public class RocksDBStoreMetrics implements MetricsSource {
}
private void getLatestSequenceNumber(MetricsRecordBuilder rb) {
- rb.addCounter(Interns.info(LAST_SEQUENCE_NUMBER, "RocksDBStat"),
- rocksDB.getLatestSequenceNumber());
+ try {
+ rb.addCounter(Interns.info(LAST_SEQUENCE_NUMBER, "RocksDBStat"),
+ rocksDB.getLatestSequenceNumber());
+ } catch (IOException e) {
+ LOG.error("Failed to get latest sequence number", e);
+ }
}
}
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStore.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStore.java
index f4b740a255..1985562523 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStore.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStore.java
@@ -182,22 +182,15 @@ public interface DBStore extends Closeable,
BatchOperationHandler {
/**
* Get data written to DB since a specific sequence number.
- * @param sequenceNumber
- * @return
- * @throws SequenceNumberNotFoundException
*/
DBUpdatesWrapper getUpdatesSince(long sequenceNumber)
- throws SequenceNumberNotFoundException;
+ throws IOException;
/**
* Get limited data written to DB since a specific sequence number.
- * @param sequenceNumber
- * @param limitCount
- * @return
- * @throws SequenceNumberNotFoundException
*/
DBUpdatesWrapper getUpdatesSince(long sequenceNumber, long limitCount)
- throws SequenceNumberNotFoundException;
+ throws IOException;
/**
* Return if the underlying DB is closed. This call is thread safe.
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
index d20fa13dda..f27ab5e430 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
@@ -347,13 +347,13 @@ public class RDBStore implements DBStore {
@Override
public DBUpdatesWrapper getUpdatesSince(long sequenceNumber)
- throws SequenceNumberNotFoundException {
+ throws IOException {
return getUpdatesSince(sequenceNumber, Long.MAX_VALUE);
}
@Override
public DBUpdatesWrapper getUpdatesSince(long sequenceNumber, long limitCount)
- throws SequenceNumberNotFoundException {
+ throws IOException {
if (limitCount <= 0) {
throw new IllegalArgumentException("Illegal count for getUpdatesSince.");
}
@@ -413,6 +413,7 @@ public class RDBStore implements DBStore {
}
logIterator.get().next();
}
+ dbUpdatesWrapper.setLatestSequenceNumber(db.getLatestSequenceNumber());
} catch (SequenceNumberNotFoundException e) {
LOG.warn("Unable to get delta updates since sequenceNumber {}. "
+ "This exception will be thrown to the client",
@@ -433,7 +434,6 @@ public class RDBStore implements DBStore {
dbUpdatesWrapper.getCurrentSequenceNumber() - sequenceNumber);
}
}
- dbUpdatesWrapper.setLatestSequenceNumber(db.getLatestSequenceNumber());
return dbUpdatesWrapper;
}
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java
index 0d4850027d..2a507b8b53 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java
@@ -243,7 +243,7 @@ public final class RocksDatabase implements Closeable {
}
}
- public long getLatestSequenceNumber() {
+ public long getLatestSequenceNumber() throws IOException {
return RocksDatabase.this.getLatestSequenceNumber();
}
@@ -508,9 +508,9 @@ public final class RocksDatabase implements Closeable {
/**
* @param cfName columnFamily on which flush will run.
- * @throws IOException
*/
public void flush(String cfName) throws IOException {
+ assertClose();
ColumnFamilyHandle handle = getColumnFamilyHandle(cfName);
try (ManagedFlushOptions options = new ManagedFlushOptions()) {
options.setWaitForFlush(true);
@@ -555,6 +555,7 @@ public final class RocksDatabase implements Closeable {
public void compactRangeDefault(final ManagedCompactRangeOptions options)
throws IOException {
+ assertClose();
try {
counter.incrementAndGet();
db.get().compactRange(null, null, null, options);
@@ -567,13 +568,15 @@ public final class RocksDatabase implements Closeable {
}
public void compactDB(ManagedCompactRangeOptions options) throws IOException
{
+ assertClose();
compactRangeDefault(options);
for (RocksDatabase.ColumnFamily columnFamily : getExtraColumnFamilies()) {
compactRange(columnFamily, null, null, options);
}
}
- public int getLiveFilesMetaDataSize() {
+ public int getLiveFilesMetaDataSize() throws IOException {
+ assertClose();
try {
counter.incrementAndGet();
return db.get().getLiveFilesMetaData().size();
@@ -584,9 +587,9 @@ public final class RocksDatabase implements Closeable {
/**
* @param cfName columnFamily on which compaction will run.
- * @throws IOException
*/
public void compactRange(String cfName) throws IOException {
+ assertClose();
ColumnFamilyHandle handle = getColumnFamilyHandle(cfName);
try {
if (handle != null) {
@@ -604,6 +607,7 @@ public final class RocksDatabase implements Closeable {
private ColumnFamilyHandle getColumnFamilyHandle(String cfName)
throws IOException {
+ assertClose();
for (ColumnFamilyHandle cf : getCfHandleMap().get(db.get().getName())) {
try {
String table = new String(cf.getName(), StandardCharsets.UTF_8);
@@ -621,6 +625,7 @@ public final class RocksDatabase implements Closeable {
public void compactRange(ColumnFamily family, final byte[] begin,
final byte[] end, final ManagedCompactRangeOptions options)
throws IOException {
+ assertClose();
try {
counter.incrementAndGet();
db.get().compactRange(family.getHandle(), begin, end, options);
@@ -632,7 +637,8 @@ public final class RocksDatabase implements Closeable {
}
}
- public List<LiveFileMetaData> getLiveFilesMetaData() {
+ public List<LiveFileMetaData> getLiveFilesMetaData() throws IOException {
+ assertClose();
try {
counter.incrementAndGet();
return db.get().getLiveFilesMetaData();
@@ -823,7 +829,8 @@ public final class RocksDatabase implements Closeable {
}
}
- public long getLatestSequenceNumber() {
+ public long getLatestSequenceNumber() throws IOException {
+ assertClose();
try {
counter.incrementAndGet();
return db.get().getLatestSequenceNumber();
@@ -924,7 +931,8 @@ public final class RocksDatabase implements Closeable {
}
@VisibleForTesting
- public List<LiveFileMetaData> getSstFileList() {
+ public List<LiveFileMetaData> getSstFileList() throws IOException {
+ assertClose();
return db.get().getLiveFilesMetaData();
}
@@ -932,7 +940,7 @@ public final class RocksDatabase implements Closeable {
* return the max compaction level of sst files in the db.
* @return level
*/
- private int getLastLevel() {
+ private int getLastLevel() throws IOException {
return getSstFileList().stream()
.max(Comparator.comparing(LiveFileMetaData::level)).get().level();
}
@@ -941,12 +949,12 @@ public final class RocksDatabase implements Closeable {
* Deletes sst files which do not correspond to prefix
* for given table.
* @param prefixPairs, a list of pair (TableName,prefixUsed).
- * @throws RocksDBException
*/
public void deleteFilesNotMatchingPrefix(
List<Pair<String, String>> prefixPairs,
BooleanTriFunction<String, String, String, Boolean> filterFunction)
- throws RocksDBException {
+ throws IOException, RocksDBException {
+ assertClose();
for (LiveFileMetaData liveFileMetaData : getSstFileList()) {
String sstFileColumnFamily =
new String(liveFileMetaData.columnFamilyName(),
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
index 4ae27046c9..de0f8f1f79 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
@@ -1797,7 +1797,8 @@ public class TestOmSnapshot {
}
@NotNull
- private static List<LiveFileMetaData> getKeyTableSstFiles() {
+ private static List<LiveFileMetaData> getKeyTableSstFiles()
+ throws IOException {
if (!bucketLayout.isFileSystemOptimized()) {
return rdbStore.getDb().getSstFileList().stream().filter(
x -> new String(x.columnFamilyName(), UTF_8).equals(
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 92a0d4adaf..3c0ffbdb7b 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -4039,7 +4039,7 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
@Override
public DBUpdates getDBUpdates(
DBUpdatesRequest dbUpdatesRequest)
- throws SequenceNumberNotFoundException {
+ throws IOException {
long limitCount = Long.MAX_VALUE;
if (dbUpdatesRequest.hasLimitCount()) {
limitCount = dbUpdatesRequest.getLimitCount();
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
index 3177c3f4c4..fd099d395e 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
@@ -37,7 +37,6 @@ import
org.apache.hadoop.hdds.protocol.proto.HddsProtos.TransferLeadershipReques
import
org.apache.hadoop.hdds.protocol.proto.HddsProtos.TransferLeadershipResponseProto;
import
org.apache.hadoop.hdds.protocol.proto.HddsProtos.UpgradeFinalizationStatus;
import org.apache.hadoop.hdds.client.ReplicationConfig;
-import org.apache.hadoop.hdds.utils.db.SequenceNumberNotFoundException;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.OzoneManagerPrepareState;
@@ -376,7 +375,7 @@ public class OzoneManagerRequestHandler implements
RequestHandler {
private DBUpdatesResponse getOMDBUpdates(
DBUpdatesRequest dbUpdatesRequest)
- throws SequenceNumberNotFoundException {
+ throws IOException {
DBUpdatesResponse.Builder builder = DBUpdatesResponse
.newBuilder();
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/recovery/ReconOmMetadataManagerImpl.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/recovery/ReconOmMetadataManagerImpl.java
index 88b6cf61c4..ff0c8145f6 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/recovery/ReconOmMetadataManagerImpl.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/recovery/ReconOmMetadataManagerImpl.java
@@ -131,7 +131,11 @@ public class ReconOmMetadataManagerImpl extends
OmMetadataManagerImpl
if (null == rocksDBStore) {
return 0;
} else {
- return rocksDBStore.getDb().getLatestSequenceNumber();
+ try {
+ return rocksDBStore.getDb().getLatestSequenceNumber();
+ } catch (IOException e) {
+ return 0;
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
