This is an automated email from the ASF dual-hosted git repository.
smengcl pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new b09bcff1f1b HDDS-14225. Upgrade RocksDB from 7.7.3 to 10.10.1 (#9813)
b09bcff1f1b is described below
commit b09bcff1f1b7460c1c8fd7082b98f5fd530bb0ba
Author: Siyao Meng <[email protected]>
AuthorDate: Tue Jul 7 18:24:45 2026 -0700
HDDS-14225. Upgrade RocksDB from 7.7.3 to 10.10.1 (#9813)
Generated-by: Codex (GPT-5.3-Codex, GPT-5.4), Claude Code (Opus 4.8)
---
.../org/apache/hadoop/hdds/utils/db/DBProfile.java | 1 +
.../org/apache/hadoop/hdds/utils/db/RDBTable.java | 32 +++--
.../apache/hadoop/hdds/utils/db/RocksDatabase.java | 33 +++--
.../org/apache/hadoop/hdds/utils/db/Table.java | 11 +-
.../utils/db/TestRDBStoreCodecBufferIterator.java | 34 ++---
.../apache/hadoop/hdds/utils/db/TestRDBTable.java | 109 ++++++++++++++++
.../db/managed/ManagedBlockBasedTableConfig.java | 8 ++
.../hdds/utils/db/managed/ManagedBloomFilter.java | 14 ++
.../hdds/utils/db/managed/ManagedDBOptions.java | 21 ++-
.../hdds/utils/db/managed/ManagedRocksDB.java | 20 ++-
.../utils/db/managed/TestManagedDBOptions.java | 59 +++++++++
hadoop-hdds/rocks-native/pom.xml | 41 ++++--
.../src/main/patches/rocks-native.patch | 35 ++---
.../hadoop/hdds/utils/db/RDBSstFileWriter.java | 7 +
.../rocksdiff/TestRocksDBCheckpointDiffer.java | 83 ++++++------
.../dist/src/main/compose/common/replicas-test.sh | 141 +++++++++++++++++++--
.../smoketest/debug/ozone-debug-keywords.robot | 23 +++-
pom.xml | 2 +-
18 files changed, 538 insertions(+), 136 deletions(-)
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBProfile.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBProfile.java
index 8eedcf1ed49..a9a3f2cfc75 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBProfile.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBProfile.java
@@ -87,6 +87,7 @@ public ManagedBlockBasedTableConfig
getBlockBasedTableConfig() {
ManagedBlockBasedTableConfig config = new ManagedBlockBasedTableConfig();
config.setBlockCache(new ManagedLRUCache(blockCacheSize))
.setBlockSize(blockSize)
+ .setFormatVersion(ManagedBlockBasedTableConfig.FORMAT_VERSION)
.setPinL0FilterAndIndexBlocksInCache(true)
.setFilterPolicy(new ManagedBloomFilter());
return config;
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBTable.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBTable.java
index 7d8d2f5af57..e1fc7297d4d 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBTable.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBTable.java
@@ -99,16 +99,16 @@ public boolean isEmpty() throws RocksDatabaseException {
@Override
public boolean isExist(byte[] key) throws RocksDatabaseException {
rdbMetrics.incNumDBKeyMayExistChecks();
- final Supplier<byte[]> holder = db.keyMayExist(family, key);
- if (holder == null) {
+ final Supplier<byte[]> valueSupplier = db.keyMayExist(family, key);
+ if (valueSupplier == null) {
return false; // definitely not exists
}
- final byte[] value = holder.get();
+ final byte[] value = valueSupplier.get();
if (value != null) {
return true; // definitely exists
}
- // inconclusive: the key may or may not exist
+ // keyMayExist could not return the value; confirm via point-get.
final boolean exists = get(key) != null;
if (!exists) {
rdbMetrics.incNumDBKeyMayExistMisses();
@@ -141,15 +141,16 @@ public byte[] getSkipCache(byte[] bytes) throws
RocksDatabaseException {
@Override
public byte[] getIfExist(byte[] key) throws RocksDatabaseException {
rdbMetrics.incNumDBKeyGetIfExistChecks();
- final Supplier<byte[]> value = db.keyMayExist(family, key);
- if (value == null) {
+ final Supplier<byte[]> valueSupplier = db.keyMayExist(family, key);
+ if (valueSupplier == null) {
return null; // definitely not exists
}
- if (value.get() != null) {
- return value.get(); // definitely exists
+ final byte[] value = valueSupplier.get();
+ if (value != null) {
+ return value; // definitely exists
}
- // inconclusive: the key may or may not exist
+ // keyMayExist could not return the value; confirm via point-get.
rdbMetrics.incNumDBKeyGetIfExistGets();
final byte[] val = get(key);
if (val == null) {
@@ -160,19 +161,24 @@ public byte[] getIfExist(byte[] key) throws
RocksDatabaseException {
Integer getIfExist(ByteBuffer key, ByteBuffer outValue) throws
RocksDatabaseException {
rdbMetrics.incNumDBKeyGetIfExistChecks();
+ // Note: RocksDatabase.keyMayExist duplicates the key internally, so the
caller's
+ // key buffer position is preserved for the fallback point-get below.
final Supplier<Integer> value = db.keyMayExist(
family, key, outValue.duplicate());
if (value == null) {
return null; // definitely not exists
}
- if (value.get() != null) {
+ final Integer length = value.get();
+ if (length != null) {
// definitely exists, return value size.
- return value.get();
+ return length;
}
- // inconclusive: the key may or may not exist
+ // keyMayExist could not return the value; confirm via point-get. get()
+ // advances the key position, so pass a duplicate to leave the caller's
+ // key buffer unchanged.
rdbMetrics.incNumDBKeyGetIfExistGets();
- final Integer val = get(key, outValue);
+ final Integer val = get(key.duplicate(), outValue);
if (val == null) {
rdbMetrics.incNumDBKeyGetIfExistMisses();
}
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java
index fe500e27447..c2d4b8bde2b 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java
@@ -626,8 +626,11 @@ Supplier<byte[]> keyMayExist(ColumnFamily family, byte[]
key)
Supplier<Integer> keyMayExist(ColumnFamily family,
ByteBuffer key, ByteBuffer out) throws RocksDatabaseException {
try (UncheckedAutoCloseable ignored = acquire()) {
+ // keyMayExist may advance the input ByteBuffer position in native code.
+ // Always pass a duplicate so callers can safely reuse the original key
+ // buffer for a follow-up point-get.
final KeyMayExist result = db.get().keyMayExist(
- family.getHandle(), key, out);
+ family.getHandle(), key.duplicate(), out);
switch (result.exists) {
case kNotExist: return null;
case kExistsWithValue: return () -> result.valueLength;
@@ -872,12 +875,12 @@ public void deleteFilesNotMatchingPrefix(TablePrefixInfo
prefixInfo) throws Rock
String sstFileColumnFamily =
StringUtils.bytes2String(liveFileMetaData.columnFamilyName());
int lastLevel = getLastLevel();
- // RocksDB #deleteFile API allows only to delete the last level of
- // SST Files. Any level < last level won't get deleted and
- // only last file of level 0 can be deleted
- // and will throw warning in the rocksdb manifest.
- // Instead, perform the level check here
- // itself to avoid failed delete attempts for lower level files.
+ // Restrict deletion to files at the last level (and skip entirely when
+ // the last level is 0). The old RocksDB #deleteFile API could only
+ // delete last-level SST files (and the last file of level 0);
+ // deleteSstFileRange, used below, no longer has that limitation, but
+ // this method keeps the last-level restriction to preserve its
existing
+ // pruning behavior.
if (liveFileMetaData.level() != lastLevel || lastLevel == 0) {
continue;
}
@@ -888,6 +891,12 @@ public void deleteFilesNotMatchingPrefix(TablePrefixInfo
prefixInfo) throws Rock
boolean isKeyWithPrefixPresent = RocksDiffUtils.isKeyWithPrefixPresent(
prefixForColumnFamily, firstDbKey, lastDbKey);
if (!isKeyWithPrefixPresent) {
+ ColumnFamilyHandle handle =
getColumnFamilyHandle(sstFileColumnFamily);
+ if (handle == null) {
+ LOG.warn("Skipping sst file deletion for {}: no handle found for
column family {}",
+ liveFileMetaData.fileName(), sstFileColumnFamily);
+ continue;
+ }
LOG.info("Deleting sst file: {} with start key: {} and end key: {} "
+ "corresponding to column family {} from db: {}. "
+ "Prefix for the column family: {}.",
@@ -896,7 +905,15 @@ public void deleteFilesNotMatchingPrefix(TablePrefixInfo
prefixInfo) throws Rock
StringUtils.bytes2String(liveFileMetaData.columnFamilyName()),
db.get().getName(),
prefixForColumnFamily);
- db.deleteFile(liveFileMetaData);
+ // deleteSstFileRange uses deleteFilesInRanges over this file's
+ // [smallestKey, largestKey]. It may also drop other files fully
+ // contained in that range, which is safe here: any such file's key
+ // range is a subset of this non-matching file's range. Because
+ // isKeyWithPrefixPresent is a monotone prefix-range test, a subset
+ // range cannot contain the prefix when the enclosing range does not,
+ // so every collaterally deleted file is likewise non-matching. This
+ // invariant holds only while isKeyWithPrefixPresent stays monotone.
+ db.deleteSstFileRange(handle, liveFileMetaData);
}
}
}
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java
index fc049034406..400547a0e36 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java
@@ -64,6 +64,8 @@ public interface Table<KEY, VALUE> {
* Check if a given key exists in Metadata store.
* (Optimization to save on data deserialization)
* A lock on the key / bucket needs to be acquired before invoking this API.
+ * Implementations may use fast existence checks internally, but the returned
+ * result must be definitive for the current table state.
* @param key metadata key
* @return true if the metadata store contains a key.
*/
@@ -107,12 +109,9 @@ default VALUE getReadCopy(KEY key) throws
RocksDatabaseException, CodecException
* Returns the value mapped to the given key in byte array or returns null
* if the key is not found.
*
- * This method first checks using keyMayExist, if it returns false, we are
- * 100% sure that key does not exist in DB, so it returns null with out
- * calling db.get. If keyMayExist return true, then we use db.get and then
- * return the value. This method will be useful in the cases where the
- * caller is more sure that this key does not exist in DB and keyMayExist
- * will help here.
+ * Implementations may use keyMayExist or similar fast-path checks
+ * internally, but the returned result must remain equivalent to a regular
+ * point lookup on the current table state.
*
* @param key metadata key
* @return value in byte array or null if the key is not found.
diff --git
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBStoreCodecBufferIterator.java
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBStoreCodecBufferIterator.java
index 919b3b6cdad..cddb11e9528 100644
---
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBStoreCodecBufferIterator.java
+++
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBStoreCodecBufferIterator.java
@@ -100,13 +100,13 @@ Answer<Integer> newAnswer(String name, byte... b) {
public void testForEachRemaining() throws Exception {
when(rocksIteratorMock.isValid())
.thenReturn(true, true, true, true, true, true, true, false);
- when(rocksIteratorMock.key(any()))
+ when(rocksIteratorMock.key(any(ByteBuffer.class)))
.then(newAnswerInt("key1", 0x00))
.then(newAnswerInt("key2", 0x00))
.then(newAnswerInt("key3", 0x01))
.then(newAnswerInt("key4", 0x02))
.thenThrow(new NoSuchElementException());
- when(rocksIteratorMock.value(any()))
+ when(rocksIteratorMock.value(any(ByteBuffer.class)))
.then(newAnswerInt("val1", 0x7f))
.then(newAnswerInt("val2", 0x7f))
.then(newAnswerInt("val3", 0x7e))
@@ -152,8 +152,8 @@ public void
testNextCallsIsValidThenGetsTheValueAndStepsToNext()
}
verifier.verify(rocksIteratorMock).isValid();
- verifier.verify(rocksIteratorMock).key(any());
- verifier.verify(rocksIteratorMock).value(any());
+ verifier.verify(rocksIteratorMock).key(any(ByteBuffer.class));
+ verifier.verify(rocksIteratorMock).value(any(ByteBuffer.class));
verifier.verify(rocksIteratorMock).next();
CodecTestUtil.gc();
@@ -192,9 +192,9 @@ public void testSeekToLastSeeks() throws Exception {
@Test
public void testSeekReturnsTheActualKey() throws Exception {
when(rocksIteratorMock.isValid()).thenReturn(true);
- when(rocksIteratorMock.key(any()))
+ when(rocksIteratorMock.key(any(ByteBuffer.class)))
.then(newAnswerInt("key1", 0x00));
- when(rocksIteratorMock.value(any()))
+ when(rocksIteratorMock.value(any(ByteBuffer.class)))
.then(newAnswerInt("val1", 0x7f));
try (RDBStoreCodecBufferIterator i = newIterator();
@@ -208,8 +208,8 @@ public void testSeekReturnsTheActualKey() throws Exception {
verifier.verify(rocksIteratorMock, times(1))
.seek(any(ByteBuffer.class));
verifier.verify(rocksIteratorMock, times(1)).isValid();
- verifier.verify(rocksIteratorMock, times(1)).key(any());
- verifier.verify(rocksIteratorMock, times(1)).value(any());
+ verifier.verify(rocksIteratorMock, times(1)).key(any(ByteBuffer.class));
+ verifier.verify(rocksIteratorMock,
times(1)).value(any(ByteBuffer.class));
assertArrayEquals(new byte[]{0x00}, val.getKey().getArray());
assertArrayEquals(new byte[]{0x7f}, val.getValue().getArray());
}
@@ -220,7 +220,7 @@ public void testSeekReturnsTheActualKey() throws Exception {
@Test
public void testGettingTheKeyIfIteratorIsValid() throws Exception {
when(rocksIteratorMock.isValid()).thenReturn(true);
- when(rocksIteratorMock.key(any()))
+ when(rocksIteratorMock.key(any(ByteBuffer.class)))
.then(newAnswerInt("key1", 0x00));
byte[] key = null;
@@ -233,7 +233,7 @@ public void testGettingTheKeyIfIteratorIsValid() throws
Exception {
InOrder verifier = inOrder(rocksIteratorMock);
verifier.verify(rocksIteratorMock, times(1)).isValid();
- verifier.verify(rocksIteratorMock, times(1)).key(any());
+ verifier.verify(rocksIteratorMock, times(1)).key(any(ByteBuffer.class));
assertArrayEquals(new byte[]{0x00}, key);
CodecTestUtil.gc();
@@ -242,9 +242,9 @@ public void testGettingTheKeyIfIteratorIsValid() throws
Exception {
@Test
public void testGettingTheValueIfIteratorIsValid() throws Exception {
when(rocksIteratorMock.isValid()).thenReturn(true);
- when(rocksIteratorMock.key(any()))
+ when(rocksIteratorMock.key(any(ByteBuffer.class)))
.then(newAnswerInt("key1", 0x00));
- when(rocksIteratorMock.value(any()))
+ when(rocksIteratorMock.value(any(ByteBuffer.class)))
.then(newAnswerInt("val1", 0x7f));
byte[] key = null;
@@ -260,7 +260,7 @@ public void testGettingTheValueIfIteratorIsValid() throws
Exception {
InOrder verifier = inOrder(rocksIteratorMock);
verifier.verify(rocksIteratorMock, times(1)).isValid();
- verifier.verify(rocksIteratorMock, times(1)).key(any());
+ verifier.verify(rocksIteratorMock, times(1)).key(any(ByteBuffer.class));
assertArrayEquals(new byte[]{0x00}, key);
assertArrayEquals(new byte[]{0x7f}, value);
@@ -272,7 +272,7 @@ public void testRemovingFromDBActuallyDeletesFromTable()
throws Exception {
final byte[] testKey = new byte[10];
ThreadLocalRandom.current().nextBytes(testKey);
when(rocksIteratorMock.isValid()).thenReturn(true);
- when(rocksIteratorMock.key(any()))
+ when(rocksIteratorMock.key(any(ByteBuffer.class)))
.then(newAnswer("key1", testKey));
try (RDBStoreCodecBufferIterator i = newIterator(null)) {
@@ -320,7 +320,7 @@ public void testNullPrefixedIterator() throws Exception {
when(rocksIteratorMock.isValid()).thenReturn(true);
assertTrue(i.hasNext());
verify(rocksIteratorMock, times(1)).isValid();
- verify(rocksIteratorMock, times(0)).key(any());
+ verify(rocksIteratorMock, times(0)).key(any(ByteBuffer.class));
i.seekToLast();
verify(rocksIteratorMock, times(1)).seekToLast();
@@ -343,11 +343,11 @@ public void testNormalPrefixedIterator() throws Exception
{
clearInvocations(rocksIteratorMock);
when(rocksIteratorMock.isValid()).thenReturn(true);
- when(rocksIteratorMock.key(any()))
+ when(rocksIteratorMock.key(any(ByteBuffer.class)))
.then(newAnswer("key1", prefixBytes));
assertTrue(i.hasNext());
verify(rocksIteratorMock, times(1)).isValid();
- verify(rocksIteratorMock, times(1)).key(any());
+ verify(rocksIteratorMock, times(1)).key(any(ByteBuffer.class));
Exception e =
assertThrows(Exception.class, () -> i.seekToLast(), "Prefixed
iterator does not support seekToLast");
diff --git
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBTable.java
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBTable.java
new file mode 100644
index 00000000000..dc582a537df
--- /dev/null
+++
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBTable.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.utils.db;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.nio.ByteBuffer;
+import java.util.function.Supplier;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase.ColumnFamily;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Unit tests for {@link RDBTable}.
+ */
+public class TestRDBTable {
+
+ @Test
+ public void testGetIfExistByteBufferFallbackUsesFreshKeyBuffer()
+ throws Exception {
+ RocksDatabase db = mock(RocksDatabase.class);
+ ColumnFamily columnFamily = mock(ColumnFamily.class);
+ RDBMetrics metrics = mock(RDBMetrics.class);
+ RDBTable table = new RDBTable(db, columnFamily, metrics);
+
+ byte[] keyBytes = "key-1".getBytes(UTF_8);
+ ByteBuffer key = ByteBuffer.wrap(keyBytes);
+ ByteBuffer outValue = ByteBuffer.allocate(64);
+
+ // RocksDatabase.keyMayExist duplicates the key internally, so it leaves
the
+ // caller's key buffer untouched. Return an inconclusive result (value-less
+ // "may exist") to force the fallback point-get.
+ when(db.keyMayExist(eq(columnFamily), any(ByteBuffer.class),
+ any(ByteBuffer.class))).thenReturn((Supplier<Integer>) () -> null);
+
+ // get() advances the key buffer position as native RocksDB does. It must
+ // still see the full key, i.e. RDBTable must hand it a fresh duplicate.
+ when(db.get(eq(columnFamily), any(ByteBuffer.class),
any(ByteBuffer.class)))
+ .thenAnswer(invocation -> {
+ ByteBuffer keyBuffer = invocation.getArgument(1);
+ if (keyBuffer.remaining() != keyBytes.length) {
+ return null;
+ }
+ keyBuffer.position(keyBuffer.limit());
+ return 0;
+ });
+
+ Integer result = table.getIfExist(key, outValue);
+ assertEquals(0, result);
+ assertEquals(0, key.position(), "caller key buffer position must be
unchanged");
+ }
+
+ @Test
+ public void testGetIfExistByteBufferFastPathReturnsValue()
+ throws Exception {
+ RocksDatabase db = mock(RocksDatabase.class);
+ ColumnFamily columnFamily = mock(ColumnFamily.class);
+ RDBMetrics metrics = mock(RDBMetrics.class);
+ RDBTable table = new RDBTable(db, columnFamily, metrics);
+
+ byte[] keyBytes = "key-1".getBytes(UTF_8);
+ byte[] valueBytes = "value-1".getBytes(UTF_8);
+ ByteBuffer key = ByteBuffer.wrap(keyBytes);
+ ByteBuffer outValue = ByteBuffer.allocate(64);
+
+ // Simulate the RocksDB "exists with value" fast path: native code writes
+ // the value into the buffer handed to keyMayExist and reports its length.
+ // getIfExist passes outValue.duplicate(), so the write must land in the
+ // caller's outValue via the shared backing memory.
+ when(db.keyMayExist(eq(columnFamily), any(ByteBuffer.class),
+ any(ByteBuffer.class))).thenAnswer(invocation -> {
+ ByteBuffer valueBuffer = invocation.getArgument(2);
+ valueBuffer.put(valueBytes);
+ return (Supplier<Integer>) () -> valueBytes.length;
+ });
+
+ Integer result = table.getIfExist(key, outValue);
+ assertEquals(valueBytes.length, result);
+ // The fast path must not fall back to a point-get.
+ verify(db, never()).get(eq(columnFamily), any(ByteBuffer.class),
any(ByteBuffer.class));
+ // Value bytes written through the duplicate are visible in the caller's
buffer.
+ byte[] readBack = new byte[valueBytes.length];
+ outValue.duplicate().get(readBack);
+ assertArrayEquals(valueBytes, readBack);
+ }
+}
+
diff --git
a/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedBlockBasedTableConfig.java
b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedBlockBasedTableConfig.java
index 621c9e93524..28690ebe54a 100644
---
a/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedBlockBasedTableConfig.java
+++
b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedBlockBasedTableConfig.java
@@ -25,6 +25,14 @@
* Managed BlockBasedTableConfig.
*/
public class ManagedBlockBasedTableConfig extends BlockBasedTableConfig {
+
+ /**
+ * Block-based table format version kept stable across RocksDB upgrades so
+ * SST files stay readable if Ozone is downgraded before finalization.
+ * RocksDB 9+ defaults to format_version 6, which RocksDB < 8.6 cannot
read.
+ */
+ public static final int FORMAT_VERSION = 5;
+
private Cache blockCacheHolder;
private AtomicBoolean closed = new AtomicBoolean(false);
diff --git
a/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedBloomFilter.java
b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedBloomFilter.java
index 406716eaf84..b2e40e75eb4 100644
---
a/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedBloomFilter.java
+++
b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedBloomFilter.java
@@ -28,6 +28,20 @@
public class ManagedBloomFilter extends BloomFilter {
private final UncheckedAutoCloseable leakTracker = track(this);
+ // Delegate to satisfy SpotBugs EQ_DOESNT_OVERRIDE_EQUALS: BloomFilter
defines
+ // equals()/hashCode() and this subclass adds a field (leakTracker). The
added
+ // field is not part of the filter's identity, so BloomFilter's equality is
+ // still correct; we override only to declare that explicitly.
+ @Override
+ public boolean equals(Object obj) {
+ return super.equals(obj);
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+
@Override
public void close() {
try {
diff --git
a/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedDBOptions.java
b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedDBOptions.java
index 1809b088560..2015e4ff42c 100644
---
a/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedDBOptions.java
+++
b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedDBOptions.java
@@ -24,7 +24,7 @@
import org.apache.hadoop.hdds.utils.IOUtils;
import org.apache.ratis.util.UncheckedAutoCloseable;
import org.rocksdb.DBOptions;
-import org.rocksdb.Logger;
+import org.rocksdb.LoggerInterface;
/**
* Managed DBOptions.
@@ -32,21 +32,32 @@
public class ManagedDBOptions extends DBOptions {
private final UncheckedAutoCloseable leakTracker = track(this);
- private final AtomicReference<Logger> loggerRef = new AtomicReference<>();
+ private final AtomicReference<LoggerInterface> loggerRef = new
AtomicReference<>();
+ // DBOptions#setLogger takes LoggerInterface since RocksDB 9.x. Override that
+ // exact signature (not the pre-9.x Logger overload) so every call path,
+ // including one made through a DBOptions-typed reference, is leak-tracked.
@Override
- public DBOptions setLogger(Logger logger) {
- IOUtils.close(LOG, loggerRef.getAndSet(logger));
+ public DBOptions setLogger(LoggerInterface logger) {
+ closeLogger(loggerRef.getAndSet(logger));
return super.setLogger(logger);
}
@Override
public void close() {
try {
- IOUtils.close(LOG, loggerRef.getAndSet(null));
+ closeLogger(loggerRef.getAndSet(null));
super.close();
} finally {
leakTracker.close();
}
}
+
+ // RocksDB loggers (org.rocksdb.Logger) own native resources and are
+ // AutoCloseable; a bare LoggerInterface may not be, so only close when it
is.
+ private static void closeLogger(LoggerInterface logger) {
+ if (logger instanceof AutoCloseable) {
+ IOUtils.close(LOG, (AutoCloseable) logger);
+ }
+ }
}
diff --git
a/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksDB.java
b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksDB.java
index 3e19e95bca2..a027b61de85 100644
---
a/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksDB.java
+++
b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksDB.java
@@ -19,6 +19,7 @@
import java.io.File;
import java.time.Duration;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@@ -190,18 +191,23 @@ public static ManagedRocksDB openWithLatestOptions(
}
/**
- * Delete liveMetaDataFile from rocks db using RocksDB#deleteFile Api.
- * This function makes the RocksDB#deleteFile Api synchronized by waiting
- * for the deletes to happen.
- * @param fileToBeDeleted File to be deleted.
+ * Delete the SST file range from rocks db and wait for file deletion.
+ * @param columnFamilyHandle column family of the target sst file.
+ * @param fileToBeDeleted file metadata to be deleted.
* @throws RocksDatabaseException if the underlying db throws an exception
* or the file is not deleted within a time
limit.
*/
- public void deleteFile(LiveFileMetaData fileToBeDeleted) throws
RocksDatabaseException {
- String sstFileName = fileToBeDeleted.fileName();
+ public void deleteSstFileRange(
+ ColumnFamilyHandle columnFamilyHandle,
+ LiveFileMetaData fileToBeDeleted) throws RocksDatabaseException {
File file = new File(fileToBeDeleted.path(), fileToBeDeleted.fileName());
+ final byte[] smallestKey = fileToBeDeleted.smallestKey();
+ final byte[] largestKey = fileToBeDeleted.largestKey();
try {
- get().deleteFile(sstFileName);
+ get().deleteFilesInRanges(
+ columnFamilyHandle,
+ Arrays.asList(smallestKey, largestKey),
+ true);
} catch (RocksDBException e) {
throw new RocksDatabaseException("Failed to delete " + file, e);
}
diff --git
a/hadoop-hdds/managed-rocksdb/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestManagedDBOptions.java
b/hadoop-hdds/managed-rocksdb/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestManagedDBOptions.java
new file mode 100644
index 00000000000..6dd9a1cc9ad
--- /dev/null
+++
b/hadoop-hdds/managed-rocksdb/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestManagedDBOptions.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.utils.db.managed;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.junit.jupiter.api.Test;
+import org.rocksdb.DBOptions;
+
+/**
+ * Tests for {@link ManagedDBOptions}, in particular that a logger it is given
+ * is closed on replacement and on close, regardless of the reference type used
+ * to call setLogger.
+ */
+public class TestManagedDBOptions {
+ static {
+ ManagedRocksObjectUtils.loadRocksDBLibrary();
+ }
+
+ @Test
+ public void testSetLoggerViaDBOptionsReferenceIsClosed() {
+ ManagedDBOptions managed = new ManagedDBOptions();
+ ManagedLogger first = new ManagedLogger(managed, (level, message) -> { });
+ ManagedLogger second = new ManagedLogger(managed, (level, message) -> { });
+
+ // Since RocksDB 9.x, DBOptions#setLogger takes a LoggerInterface. Calling
+ // it through the parent type must still route through ManagedDBOptions'
+ // override (not a bypassed overload) so the logger is tracked and closed.
+ DBOptions options = managed;
+
+ options.setLogger(first);
+ assertTrue(first.isOwningHandle());
+
+ // Replacing the logger closes the previous one.
+ options.setLogger(second);
+ assertFalse(first.isOwningHandle(), "previous logger should be closed on
replace");
+ assertTrue(second.isOwningHandle());
+
+ // Closing the options closes the current logger.
+ managed.close();
+ assertFalse(second.isOwningHandle(), "current logger should be closed on
close");
+ }
+}
diff --git a/hadoop-hdds/rocks-native/pom.xml b/hadoop-hdds/rocks-native/pom.xml
index 0e260ba0478..9837aae92db 100644
--- a/hadoop-hdds/rocks-native/pom.xml
+++ b/hadoop-hdds/rocks-native/pom.xml
@@ -121,6 +121,28 @@
</properties>
<build>
<plugins>
+ <!-- rocksdbjni may carry a packaging/build suffix like 10.10.1.1,
+ but upstream RocksDB source tags and tarballs are published as
+ x.y.z only, so derive a source version for the native tools
build. -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>derive-rocksdb-source-version</id>
+ <goals>
+ <goal>regex-property</goal>
+ </goals>
+ <phase>initialize</phase>
+ <configuration>
+ <name>rocksdb.source.version</name>
+ <value>${rocksdb.version}</value>
+ <regex>^([0-9]+\.[0-9]+\.[0-9]+)(?:\..+)?$</regex>
+ <replacement>$1</replacement>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
@@ -173,6 +195,7 @@
<artifactItem>
<groupId>org.rocksdb</groupId>
<artifactId>rocksdbjni</artifactId>
+ <version>${rocksdb.version}</version>
<type>jar</type>
<overWrite>false</overWrite>
<outputDirectory>${project.build.directory}/rocksdbjni</outputDirectory>
@@ -193,8 +216,8 @@
</goals>
<phase>generate-sources</phase>
<configuration>
-
<url>https://github.com/facebook/rocksdb/archive/refs/tags/v${rocksdb.version}.tar.gz</url>
-
<outputFileName>rocksdb-v${rocksdb.version}.tar.gz</outputFileName>
+
<url>https://github.com/facebook/rocksdb/archive/refs/tags/v${rocksdb.source.version}.tar.gz</url>
+
<outputFileName>rocksdb-v${rocksdb.source.version}.tar.gz</outputFileName>
<outputDirectory>${project.build.directory}/rocksdb</outputDirectory>
</configuration>
</execution>
@@ -206,7 +229,7 @@
<configuration>
<patchFile>${basedir}/src/main/patches/rocks-native.patch</patchFile>
<strip>1</strip>
-
<targetDirectory>${project.build.directory}/rocksdb/rocksdb-${rocksdb.version}</targetDirectory>
+
<targetDirectory>${project.build.directory}/rocksdb/rocksdb-${rocksdb.source.version}</targetDirectory>
</configuration>
<executions>
<execution>
@@ -230,7 +253,7 @@
<phase>generate-sources</phase>
<configuration>
<target>
- <untar compression="gzip"
dest="${project.build.directory}/rocksdb/"
src="${project.build.directory}/rocksdb/rocksdb-v${rocksdb.version}.tar.gz" />
+ <untar compression="gzip"
dest="${project.build.directory}/rocksdb/"
src="${project.build.directory}/rocksdb/rocksdb-v${rocksdb.source.version}.tar.gz"
/>
</target>
</configuration>
</execution>
@@ -245,9 +268,9 @@
<exec executable="chmod" failonerror="true">
<arg line="-R" />
<arg line="775" />
- <arg
line="${project.build.directory}/rocksdb/rocksdb-${rocksdb.version}" />
+ <arg
line="${project.build.directory}/rocksdb/rocksdb-${rocksdb.source.version}" />
</exec>
- <exec
dir="${project.build.directory}/rocksdb/rocksdb-${rocksdb.version}"
executable="make" failonerror="true">
+ <exec
dir="${project.build.directory}/rocksdb/rocksdb-${rocksdb.source.version}"
executable="make" failonerror="true">
<arg line="PORTABLE=1" />
<arg line="DEBUG_LEVEL=0" />
<arg line="EXTRA_CXXFLAGS='-fPIC
-D_GLIBCXX_USE_CXX11_ABI=0'" />
@@ -275,12 +298,12 @@
<arg line="-DNATIVE_DIR=${basedir}/src/main/native" />
<arg line="-DSST_DUMP_INCLUDE=${sstDump.include}" />
<arg line="-DCMAKE_STANDARDS=${cmake.standards}" />
- <arg
line="-DROCKSDB_HEADERS=${project.build.directory}/rocksdb/rocksdb-${rocksdb.version}/include"
/>
- <arg
line="-DROCKSDB_TOOLS_LIB=${project.build.directory}/rocksdb/rocksdb-${rocksdb.version}"
/>
+ <arg
line="-DROCKSDB_HEADERS=${project.build.directory}/rocksdb/rocksdb-${rocksdb.source.version}/include"
/>
+ <arg
line="-DROCKSDB_TOOLS_LIB=${project.build.directory}/rocksdb/rocksdb-${rocksdb.source.version}"
/>
<arg
line="-DROCKSDB_LIB=${project.build.directory}/native/rocksdb/${rocksdbLibName}"
/>
</exec>
<exec dir="${project.build.directory}/native/rocksdb"
executable="make" failonerror="true" />
- <delete
dir="${project.build.directory}/rocksdb/rocksdb-${rocksdb.version}"
includeemptydirs="true" />
+ <delete
dir="${project.build.directory}/rocksdb/rocksdb-${rocksdb.source.version}"
includeemptydirs="true" />
</target>
</configuration>
</execution>
diff --git a/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch
b/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch
index b2627fbbb3e..ae256eba2db 100644
--- a/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch
+++ b/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch
@@ -119,10 +119,11 @@ diff --git a/src.mk b/src.mk
index b94bc43ca..c13e5cde6 100644
--- a/src.mk
+++ b/src.mk
-@@ -338,11 +338,8 @@ RANGE_TREE_SOURCES =\
+@@ -367,12 +367,8 @@
utilities/transactions/lock/range/range_tree/range_tree_lock_tracker.cc
-
+
TOOL_LIB_SOURCES = \
+- db_stress_tool/db_stress_compression_manager.cc \
- tools/io_tracer_parser_tool.cc \
- tools/ldb_cmd.cc \
- tools/ldb_tool.cc \
@@ -130,7 +131,7 @@ index b94bc43ca..c13e5cde6 100644
- utilities/blob_db/blob_dump_tool.cc \
+ tools/raw_sst_file_reader.cc \
+ tools/raw_sst_file_iterator.cc \
-
+
ANALYZER_LIB_SOURCES = \
tools/block_cache_analyzer/block_cache_trace_analyzer.cc \
diff --git a/tools/raw_sst_file_iterator.cc b/tools/raw_sst_file_iterator.cc
@@ -380,9 +381,9 @@ index 000000000..5ba8a82ee
+
+ rep_->file_.reset(new RandomAccessFileReader(std::move(file), file_path));
+
-+ FilePrefetchBuffer prefetch_buffer(
-+ 0 /* readahead_size */, 0 /* max_readahead_size */, true /* enable */,
-+ false /* track_min_offset */);
++ FilePrefetchBuffer prefetch_buffer(ReadaheadParams(),
++ !fopts.use_mmap_reads /* enable */,
++ false /* track_min_offset */);
+ if (s.ok()) {
+ const uint64_t kSstDumpTailPrefetchSize = 512 * 1024;
+ uint64_t prefetch_size = (file_size > kSstDumpTailPrefetchSize)
@@ -391,11 +392,10 @@ index 000000000..5ba8a82ee
+ uint64_t prefetch_off = file_size - prefetch_size;
+ IOOptions opts;
+ s = prefetch_buffer.Prefetch(opts, rep_->file_.get(), prefetch_off,
-+ static_cast<size_t>(prefetch_size),
-+ Env::IO_TOTAL /* rate_limiter_priority */);
++ static_cast<size_t>(prefetch_size));
+
-+ s = ReadFooterFromFile(opts, rep_->file_.get(), &prefetch_buffer,
file_size,
-+ &footer);
++ s = ReadFooterFromFile(opts, rep_->file_.get(), *fs, &prefetch_buffer,
++ file_size, &footer);
+ }
+ if (s.ok()) {
+ magic_number = footer.table_magic_number();
@@ -405,16 +405,16 @@ index 000000000..5ba8a82ee
+ if (magic_number == kPlainTableMagicNumber ||
+ magic_number == kLegacyPlainTableMagicNumber) {
+ rep_->soptions_.use_mmap_reads = true;
++ fopts = rep_->soptions_;
+
+ fs->NewRandomAccessFile(file_path, fopts, &file, nullptr);
+ rep_->file_.reset(new RandomAccessFileReader(std::move(file),
file_path));
+ }
+
+ s = ROCKSDB_NAMESPACE::ReadTableProperties(
-+ rep_->file_.get(), file_size, magic_number, rep_->ioptions_,
&(rep_->table_properties_),
-+ /* memory_allocator= */ nullptr, (magic_number ==
kBlockBasedTableMagicNumber)
-+ ? &prefetch_buffer
-+ : nullptr);
++ rep_->file_.get(), file_size, magic_number, rep_->ioptions_,
rep_->read_options_,
++ &(rep_->table_properties_), /* memory_allocator= */ nullptr,
++ (magic_number == kBlockBasedTableMagicNumber) ? &prefetch_buffer :
nullptr);
+ // For old sst format, ReadTableProperties might fail but file can be read
+ if (s.ok()) {
+ s = SetTableOptionsByMagicNumber(magic_number);
@@ -448,9 +448,10 @@ index 000000000..5ba8a82ee
+
+Status RawSstFileReader::NewTableReader(uint64_t file_size) {
+ auto t_opt =
-+ TableReaderOptions(rep_->ioptions_, rep_->moptions_.prefix_extractor,
rep_->soptions_,
-+ rep_->internal_comparator_, false /* skip_filters */,
-+ false /* imortal */, true /* force_direct_prefetch
*/);
++ TableReaderOptions(rep_->ioptions_, rep_->moptions_.prefix_extractor,
++ rep_->moptions_.compression_manager.get(),
rep_->soptions_,
++ rep_->internal_comparator_, 0 /*
block_protection_bytes_per_key */,
++ false /* skip_filters */, false /* immortal */, true
/* force_direct_prefetch */);
+ // Allow open file with global sequence number for backward compatibility.
+ t_opt.largest_seqno = kMaxSequenceNumber;
+
diff --git
a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/hadoop/hdds/utils/db/RDBSstFileWriter.java
b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/hadoop/hdds/utils/db/RDBSstFileWriter.java
index a689e9fdea1..294bf41b802 100644
---
a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/hadoop/hdds/utils/db/RDBSstFileWriter.java
+++
b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/hadoop/hdds/utils/db/RDBSstFileWriter.java
@@ -20,6 +20,7 @@
import java.io.Closeable;
import java.io.File;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedBlockBasedTableConfig;
import org.apache.hadoop.hdds.utils.db.managed.ManagedDirectSlice;
import org.apache.hadoop.hdds.utils.db.managed.ManagedEnvOptions;
import org.apache.hadoop.hdds.utils.db.managed.ManagedOptions;
@@ -36,8 +37,13 @@ public class RDBSstFileWriter implements Closeable {
private AtomicLong keyCounter;
private ManagedOptions emptyOption = new ManagedOptions();
private final ManagedEnvOptions emptyEnvOptions = new ManagedEnvOptions();
+ private final ManagedBlockBasedTableConfig tableConfig = new
ManagedBlockBasedTableConfig();
public RDBSstFileWriter(File externalFile) throws RocksDatabaseException {
+ // Pin the SST format version so files written here (e.g. snapshot defrag
+ // ingest) stay readable if Ozone is downgraded before finalization.
+ tableConfig.setFormatVersion(ManagedBlockBasedTableConfig.FORMAT_VERSION);
+ emptyOption.setTableFormatConfig(tableConfig);
this.sstFileWriter = new ManagedSstFileWriter(emptyEnvOptions,
emptyOption);
this.keyCounter = new AtomicLong(0);
this.sstFile = externalFile;
@@ -117,6 +123,7 @@ private void closeResources() {
sstFileWriter = null;
emptyOption.close();
emptyEnvOptions.close();
+ tableConfig.close();
}
private void closeOnFailure() {
diff --git
a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java
b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java
index e083be0d0c8..0fc2df2a596 100644
---
a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java
+++
b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java
@@ -986,11 +986,14 @@ void testDifferWithDB() throws Exception {
// Confirm correct links created
try (Stream<Path> sstPathStream = Files.list(sstBackUpDir.toPath())) {
- List<String> expectedLinks = sstPathStream.map(Path::getFileName)
+ List<String> actualLinks = sstPathStream.map(Path::getFileName)
.map(Object::toString).sorted().collect(Collectors.toList());
- assertEquals(expectedLinks, asList(
- "000017.sst", "000019.sst", "000021.sst", "000023.sst",
- "000024.sst", "000026.sst", "000029.sst"));
+ assertThat(actualLinks).hasSize(7);
+ assertThat(actualLinks).allMatch(link -> link.matches("\\d{6}\\.sst"));
+ for (String linkName : actualLinks) {
+ assertTrue(Files.size(sstBackUpDir.toPath().resolve(linkName)) > 0,
+ "SST link should not be empty: " + linkName);
+ }
}
rocksDBCheckpointDiffer.getForwardCompactionDAG().nodes().stream().forEach(compactionNode
-> {
Assertions.assertNotNull(compactionNode.getStartKey());
@@ -1015,48 +1018,44 @@ private static List<ColumnFamilyDescriptor>
getColumnFamilyDescriptors() {
void diffAllSnapshots(RocksDBCheckpointDiffer differ)
throws IOException {
final DifferSnapshotInfo src = snapshots.get(snapshots.size() - 1);
-
- // Hard-coded expected output.
- // The results are deterministic. Retrieved from a successful run.
- final List<List<String>> expectedDifferResult = asList(
- asList("000023", "000029", "000026", "000019", "000021", "000031"),
- asList("000023", "000029", "000026", "000021", "000031"),
- asList("000023", "000029", "000026", "000031"),
- asList("000029", "000026", "000031"),
- asList("000029", "000031"),
- Collections.singletonList("000031"),
- Collections.emptyList()
- );
- assertEquals(snapshots.size(), expectedDifferResult.size());
-
- int index = 0;
- List<String> expectedDiffFiles = new ArrayList<>();
+ boolean sawNonEmptyDiff = false;
for (DifferSnapshotInfo snap : snapshots) {
// Returns a list of SST files to be fed into RocksCheckpointDiffer Dag.
List<String> tablesToTrack = new
ArrayList<>(COLUMN_FAMILIES_TO_TRACK_IN_DAG);
// Add some invalid index.
tablesToTrack.add("compactionLogTable");
+
+ // Baseline diff when tracking every table. A subset's diff must equal
+ // this baseline filtered to the subset's column families (files with no
+ // column family are always kept). This relationship is deterministic and
+ // stable across RocksDB versions, unlike hard-coded SST file names.
+ Set<String> allTables = new HashSet<>(tablesToTrack);
+ List<SstFileInfo> baseline = differ.getSSTDiffList(
+ new DifferSnapshotVersion(src, 0, allTables),
+ new DifferSnapshotVersion(snap, 0, allTables),
+ null, allTables, true).orElse(Collections.emptyList());
+ sawNonEmptyDiff = sawNonEmptyDiff || !baseline.isEmpty();
+
+ // Independent structural oracle, not derived from getSSTDiffList's own
+ // output: a snapshot diffed against itself must have no differing SST
+ // files. Together with the sawNonEmptyDiff guard below, this bounds a
+ // systematically broken diff in both directions (returning nothing, or
+ // returning files even for identical snapshots).
+ if (snap == src) {
+ assertThat(baseline)
+ .as("diff of a snapshot against itself must be empty")
+ .isEmpty();
+ }
+
Set<String> tableToLookUp = new HashSet<>();
for (int i = 0; i < Math.pow(2, tablesToTrack.size()); i++) {
tableToLookUp.clear();
- expectedDiffFiles.clear();
int mask = i;
while (mask != 0) {
int firstSetBitIndex = Integer.numberOfTrailingZeros(mask);
tableToLookUp.add(tablesToTrack.get(firstSetBitIndex));
mask &= mask - 1;
}
- for (String diffFile : expectedDifferResult.get(index)) {
- String columnFamily;
- if
(rocksDBCheckpointDiffer.getCompactionNodeMap().containsKey(diffFile)) {
- columnFamily =
rocksDBCheckpointDiffer.getCompactionNodeMap().get(diffFile).getColumnFamily();
- } else {
- columnFamily = src.getSstFile(0, diffFile).getColumnFamily();
- }
- if (columnFamily == null || tableToLookUp.contains(columnFamily)) {
- expectedDiffFiles.add(diffFile);
- }
- }
DifferSnapshotVersion srcSnapVersion = new DifferSnapshotVersion(src,
0, tableToLookUp);
DifferSnapshotVersion destSnapVersion = new
DifferSnapshotVersion(snap, 0, tableToLookUp);
List<SstFileInfo> sstDiffList = differ.getSSTDiffList(srcSnapVersion,
destSnapVersion, null,
@@ -1064,12 +1063,24 @@ void diffAllSnapshots(RocksDBCheckpointDiffer differ)
LOG.info("SST diff list from '{}' to '{}': {} tables: {}",
src.getDbPath(0), snap.getDbPath(0), sstDiffList, tableToLookUp);
- assertEquals(expectedDiffFiles,
sstDiffList.stream().map(SstFileInfo::getFileName)
- .collect(Collectors.toList()));
+ // Expected files: baseline entries whose column family is untracked
+ // (null) or included in this subset. getSSTDiffList returns the values
+ // of a HashMap, so its ordering is not guaranteed; compare as sets.
+ List<String> expectedFiles = baseline.stream()
+ .filter(sstFileInfo -> sstFileInfo.getColumnFamily() == null
+ || tableToLookUp.contains(sstFileInfo.getColumnFamily()))
+ .map(SstFileInfo::getFileName)
+ .collect(Collectors.toList());
+ List<String> actualFiles = sstDiffList.stream()
+ .map(SstFileInfo::getFileName)
+ .collect(Collectors.toList());
+
assertThat(actualFiles).containsExactlyInAnyOrderElementsOf(expectedFiles);
}
-
- ++index;
}
+ // Guard against getSSTDiffList silently returning nothing for every input.
+ assertThat(sawNonEmptyDiff)
+ .as("expected at least one non-empty SST diff across snapshots")
+ .isTrue();
}
/**
diff --git a/hadoop-ozone/dist/src/main/compose/common/replicas-test.sh
b/hadoop-ozone/dist/src/main/compose/common/replicas-test.sh
index f69b4b23f1a..31cc0c1f9b6 100755
--- a/hadoop-ozone/dist/src/main/compose/common/replicas-test.sh
+++ b/hadoop-ozone/dist/src/main/compose/common/replicas-test.sh
@@ -21,14 +21,49 @@ volume="cli-debug-volume${prefix}"
bucket="cli-debug-bucket"
key="testfile"
-dn_container="ozonesecure-ha-datanode1-1"
container_db_path="/data/hdds/hdds/"
-local_db_backup_path="${COMPOSE_DIR}/container_db_backup"
+local_db_backup_path="${COMPOSE_DIR}/container_db_backup_${prefix}"
+backup_manifest="${local_db_backup_path}/container_db_paths.tsv"
mkdir -p "${local_db_backup_path}"
-echo "Taking a backup of container.db"
-docker exec "${dn_container}" find "${container_db_path}" -name "container.db"
| while read -r db; do
- docker cp "${dn_container}:${db}" "${local_db_backup_path}/container.db"
+echo "Taking backups of existing container.db directories"
+datanodes=$(docker ps --format '{{.Names}}' | grep
'^ozonesecure-ha-datanode[0-9]\+-1$' | sort)
+if [ -z "${datanodes}" ]; then
+ echo "Failed to find datanode containers" >&2
+ exit 1
+fi
+
+>"${backup_manifest}"
+for dn_container in ${datanodes}; do
+ while read -r db; do
+ printf '%s\t%s\n' "${dn_container}" "${db}" >> "${backup_manifest}"
+ done < <(docker exec "${dn_container}" find "${container_db_path}" -name
"container.db")
+done
+
+echo "Stopping datanodes for a consistent container.db backup"
+for dn_container in ${datanodes}; do
+ if [ "$(docker inspect -f '{{.State.Running}}' "${dn_container}"
2>/dev/null)" = "true" ]; then
+ docker stop "${dn_container}" >/dev/null
+ else
+ echo "${dn_container} is already stopped before backup"
+ fi
+done
+
+while IFS=$'\t' read -r dn_container db; do
+ backup_path="${local_db_backup_path}/${dn_container}${db}"
+ mkdir -p "$(dirname "${backup_path}")"
+ docker cp "${dn_container}:${db}" "${backup_path}"
+done < "${backup_manifest}"
+
+echo "Restarting datanodes after backup"
+for dn_container in ${datanodes}; do
+ if [ "$(docker inspect -f '{{.State.Running}}' "${dn_container}"
2>/dev/null)" != "true" ]; then
+ docker start "${dn_container}" >/dev/null
+ fi
+done
+
+for dn_container in ${datanodes}; do
+ wait_for_datanode "${dn_container}" HEALTHY 60
done
execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-tests.robot
@@ -40,29 +75,111 @@ host="$(jq -r '.keyLocations[0][0].datanode["hostname"]'
${chunkinfo})"
container="${host%%.*}"
dn_with_num="$(sed -E 's/^.*-(datanode[0-9]+)-[0-9]+$/\1/' <<< "$container")"
-# corrupt the first block of key on one of the datanodes
datafile="$(jq -r '.keyLocations[0][0].file' ${chunkinfo})"
+container_id="$(jq -r '.keyLocations[0][0].blockData.blockID.containerID'
${chunkinfo})"
+local_block_id="$(jq -r '.keyLocations[0][0].blockData.blockID.localID //
.keyLocations[0][0].blockData.blockID.localId' ${chunkinfo})"
+pipeline_id="$(docker-compose exec -T ${SCM} bash -c \
+ "ozone admin container info ${container_id} --json | jq -r
'.writePipelineID.id // .writePipelineId.id'")"
+if [ -z "${pipeline_id}" ] || [ "${pipeline_id}" = "null" ]; then
+ echo "Failed to determine write pipeline for container ${container_id}" >&2
+ exit 1
+fi
+if [ -z "${local_block_id}" ] || [ "${local_block_id}" = "null" ]; then
+ echo "Failed to determine local block ID for container ${container_id}" >&2
+ exit 1
+fi
+
+# corrupt the first block of key on one of the datanodes
docker exec "${container}" sed -i -e '1s/^/a/' "${datafile}"
execute_robot_test ${SCM} -v "PREFIX:${prefix}" -v "CORRUPT_DATANODE:${host}"
debug/corrupt-block-checksum.robot
-echo "Overwriting container.db with the backup db"
-target_container_dir=$(docker exec "${container}" find "${container_db_path}"
-name "container.db" | xargs dirname)
-docker cp "${local_db_backup_path}/container.db"
"${container}:${target_container_dir}/"
-docker exec "${container}" sudo chown -R hadoop:hadoop
"${target_container_dir}"
+target_container_db=$(docker exec "${container}" bash -c "
+ datafile=\$1
+ dir=\$(dirname \"\$datafile\")
+ while [ \"\$dir\" != '/' ]; do
+ if [[ \$(basename \"\$dir\") == CID-* ]]; then
+ container_db=\$(find \"\$dir\" -path '*/container.db' | head -n 1)
+ if [ -n \"\$container_db\" ]; then
+ echo \"\$container_db\"
+ exit 0
+ fi
+ exit 1
+ fi
+ dir=\$(dirname \"\$dir\")
+ done
+ exit 1
+" _ "${datafile}")
+if [ -z "${target_container_db}" ]; then
+ echo "Failed to locate container.db for ${datafile} on ${container}" >&2
+ exit 1
+fi
+backup_container_db="${local_db_backup_path}/${container}${target_container_db}"
+if [ ! -e "${backup_container_db}" ]; then
+ echo "No pre-key backup found for ${target_container_db} on ${container};
creating rollback copy by deleting block metadata"
+
+ docker stop "${container}"
+
+ wait_for_datanode "${container}" STALE 60
+
+ mkdir -p "$(dirname "${backup_container_db}")"
+ docker cp "${container}:${target_container_db}" "${backup_container_db}"
+
+ container_image="$(docker inspect -f '{{.Config.Image}}' "${container}")"
+ docker run --rm \
+ -v "${local_db_backup_path}:${local_db_backup_path}" \
+ --entrypoint bash "${container_image}" -c '
+ set -euo pipefail
+ backup_container_db="$1"
+ local_block_id="$2"
+
+ ldb --db="${backup_container_db}" --column_family=block_data delete
"${local_block_id}"
+ ' _ "${backup_container_db}" "${local_block_id}" || exit 1
+
+ docker start "${container}"
+
+ wait_for_datanode "${container}" HEALTHY 60
+fi
docker stop "${container}"
wait_for_datanode "${container}" STALE 60
+
execute_robot_test ${SCM} -v "PREFIX:${prefix}" -v "STALE_DATANODE:${host}"
debug/stale-datanode-checksum.robot
docker start "${container}"
wait_for_datanode "${container}" HEALTHY 60
-execute_robot_test ${SCM} -v "PREFIX:${prefix}" -v "DATANODE:${host}"
debug/block-existence-check.robot
-
execute_robot_test ${SCM} -v "PREFIX:${prefix}" -v "DATANODE:${host}" -v
"FAULT_INJ_DATANODE:${dn_with_num}" debug/container-state-verifier.robot
execute_robot_test ${OM} kinit.robot
execute_robot_test ${OM} -v "PREFIX:${prefix}"
debug/ozone-debug-tests-ec3-2.robot
+
+echo "Overwriting container.db with the backup db"
+echo "Restoring backup at ${target_container_db} on ${container}"
+echo "Removing dn.ratis state for pipeline ${pipeline_id} on ${container}"
+docker stop "${container}"
+
+wait_for_datanode "${container}" STALE 60
+
+container_image="$(docker inspect -f '{{.Config.Image}}' "${container}")"
+docker run --rm --volumes-from "${container}" \
+ -v "${local_db_backup_path}:${local_db_backup_path}:ro" \
+ --entrypoint bash "${container_image}" -c '
+ set -euo pipefail
+ target_container_db="$1"
+ pipeline_id="$2"
+ backup_container_db="$3"
+
+ rm -rf "${target_container_db}" "/data/metadata/dn.ratis/${pipeline_id}"
+ mkdir -p "$(dirname "${target_container_db}")"
+ cp -a "${backup_container_db}" "${target_container_db}"
+ chown -R hadoop:hadoop "${target_container_db}"
+ ' _ "${target_container_db}" "${pipeline_id}" "${backup_container_db}" ||
exit 1
+
+docker start "${container}"
+
+wait_for_datanode "${container}" HEALTHY 60
+
+execute_robot_test ${SCM} -v "PREFIX:${prefix}" -v "DATANODE:${host}"
debug/block-existence-check.robot
diff --git
a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-keywords.robot
b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-keywords.robot
index aa51febb318..9114239d878 100644
--- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-keywords.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-keywords.robot
@@ -63,13 +63,26 @@ Check Container State Replicas
${checks} = Get From Dictionary ${replica} checks
${check} = Get From List ${checks} 0
Should Be Equal ${check['type']} containerState
- Should Be Equal ${check['pass']} ${False}
- ${actual_message} = Set Variable
${check['failures'][0]['message']}
-
- Run Keyword If '${hostname}' == '${faulty_datanode}' Should
Contain ${actual_message} ${expected_message}
- ... ELSE Should Match Regexp ${actual_message} Replica
state is (OPEN|CLOSING|QUASI_CLOSED|CLOSED)
+ Run Keyword If '${hostname}' == '${faulty_datanode}' Check
Replica Failed ${replica} containerState ${expected_message}
+ ... ELSE Check Healthy Replica Container State ${replica}
END
+Check Healthy Replica Container State
+ [Arguments] ${replica}
+ ${checks} = Get From Dictionary ${replica} checks
+ ${check} = Get From List ${checks} 0
+ Should Be Equal ${check['type']} containerState
+ Run Keyword If ${check['pass']} Check Replica Passed ${replica}
containerState
+ ... ELSE Check Replica Failed Container State ${replica}
+
+Check Replica Failed Container State
+ [Arguments] ${replica}
+ ${checks} = Get From Dictionary ${replica} checks
+ ${check} = Get From List ${checks} 0
+ Should Be Equal ${check['type']} containerState
+ Should Be Equal ${check['pass']} ${False}
+ Should Match Regexp ${check['failures'][0]['message']} Replica state
is (OPEN|CLOSING|QUASI_CLOSED|CLOSED)
+
Check Replica Failed
[Arguments] ${replica} ${check_type} ${expected_message}
${checks} = Get From Dictionary ${replica} checks
diff --git a/pom.xml b/pom.xml
index e043273407a..97b7d0991a3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -194,7 +194,7 @@
<reflections.version>0.10.2</reflections.version>
<reload4j.version>1.2.26</reload4j.version>
<restrict-imports.enforcer.version>3.0.0</restrict-imports.enforcer.version>
- <rocksdb.version>7.7.3</rocksdb.version>
+ <rocksdb.version>10.10.1.1</rocksdb.version>
<servlet-api.version>3.1.0</servlet-api.version>
<shell-executable>bash</shell-executable>
<slf4j.version>2.0.18</slf4j.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
