AmandeepSingh285 commented on code in PR #3695:
URL: https://github.com/apache/celeborn/pull/3695#discussion_r3280209575
##########
worker/src/main/java/org/apache/celeborn/service/deploy/worker/shuffledb/RocksDBProvider.java:
##########
@@ -48,26 +48,49 @@ public class RocksDBProvider {
private static final Logger logger =
LoggerFactory.getLogger(RocksDBProvider.class);
+ private static Options createDBOptions() {
+ BloomFilter fullFilter = new BloomFilter(10.0D /*
BloomFilter.DEFAULT_BITS_PER_KEY */, false);
+ BlockBasedTableConfig tableFormatConfig =
+ new BlockBasedTableConfig()
+ .setFilterPolicy(fullFilter)
+ .setEnableIndexCompression(false)
+ .setIndexBlockRestartInterval(8)
+ .setFormatVersion(5);
+
+ Options dbOptions = new Options();
+ RocksDBLogger rocksDBLogger = new RocksDBLogger(dbOptions);
+
+ dbOptions.setCreateIfMissing(false);
+ dbOptions.setBottommostCompressionType(CompressionType.ZSTD_COMPRESSION);
+ dbOptions.setCompressionType(CompressionType.LZ4_COMPRESSION);
+ dbOptions.setTableFormatConfig(tableFormatConfig);
+ dbOptions.setLogger(rocksDBLogger);
+
+ return dbOptions;
+ }
+
+ /**
+ * Reopen an existing RocksDB without the delete-and-recreate fallback. Use
this for recovery from
+ * transient errors.
+ */
+ public static org.rocksdb.RocksDB reopenRocksDB(File dbFile) throws
IOException {
+ if (dbFile == null || !dbFile.exists()) {
+ throw new IOException("RocksDB path does not exist: " + dbFile);
+ }
+ Options dbOptions = createDBOptions();
+ try {
+ return org.rocksdb.RocksDB.open(dbOptions, dbFile.toString());
+ } catch (RocksDBException e) {
+ dbOptions.close();
Review Comment:
Good point. Have tried to fix this using ManagedRocksDB which closes all
held resources
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
