This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 8e32bc46964 HDDS-14957. Separate OpenKeyInfo to differentiate KeyInfo
(#10181)
8e32bc46964 is described below
commit 8e32bc46964000d42768f1e99b7dbf8036b202b9
Author: Bolin Lin <[email protected]>
AuthorDate: Tue Jul 14 02:28:50 2026 -0400
HDDS-14957. Separate OpenKeyInfo to differentiate KeyInfo (#10181)
---
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 71 +++++++++--
.../hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java | 44 ++++++-
.../ozone/om/helpers/TestOmKeyInfoCodec.java | 132 ++++++++++++++++++++-
.../om/helpers/TestRepeatedOmKeyInfoCodec.java | 109 ++++++++++++++++-
.../hadoop/ozone/om/codec/OMDBDefinition.java | 23 ++--
.../ozone/om/request/key/OMKeyCommitRequest.java | 3 -
.../om/request/key/OMKeyCommitRequestWithFSO.java | 4 +-
.../om/request/key/TestOMKeyCommitRequest.java | 2 -
8 files changed, 347 insertions(+), 41 deletions(-)
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
index 4d8d3d4c7eb..ab4da4badd9 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
@@ -60,7 +60,8 @@ public final class OmKeyInfo extends WithParentObjectId
implements CopyObject<OmKeyInfo>, WithTags {
private static final Logger LOG = LoggerFactory.getLogger(OmKeyInfo.class);
- private static final Codec<OmKeyInfo> CODEC = newCodec();
+ private static final Codec<OmKeyInfo> CODEC = newCodec(true);
+ private static final Codec<OmKeyInfo> CODEC_KEY_TABLE = newCodec(false);
/**
* Metadata key flag to indicate whether a deleted key was a committed key.
* The flag is set when a committed key is deleted from AOS but still held in
@@ -109,7 +110,7 @@ public final class OmKeyInfo extends WithParentObjectId
// generation unchanged.
// This allows a key to be created an committed atomically if the original
has not
// been modified.
- private Long expectedDataGeneration = null;
+ private final Long expectedDataGeneration;
private OmKeyInfo(Builder b) {
super(b);
@@ -131,18 +132,41 @@ private OmKeyInfo(Builder b) {
this.expectedDataGeneration = b.expectedDataGeneration;
}
- private static Codec<OmKeyInfo> newCodec() {
+ /**
+ * Creates a new codec for OmKeyInfo.
+ *
+ * @param isOpenKey true for openKeyTable (includes expectedDataGeneration),
+ * false for keyTable (excludes these fields)
+ * @return the codec
+ */
+ private static Codec<OmKeyInfo> newCodec(boolean isOpenKey) {
return new DelegatedCodec<>(
Proto2Codec.get(KeyInfo.getDefaultInstance()),
OmKeyInfo::getFromProtobuf,
- k -> k.getProtobuf(true, ClientVersion.CURRENT_VERSION),
+ k -> k.getProtobuf(true, ClientVersion.CURRENT_VERSION, isOpenKey),
OmKeyInfo.class);
}
- public static Codec<OmKeyInfo> getCodec() {
+ /**
+ * Gets the codec for openKeyTable. This codec includes
expectedDataGeneration
+ * field during serialization.
+ *
+ * @return the codec for openKeyTable
+ */
+ public static Codec<OmKeyInfo> getOpenKeyTableCodec() {
return CODEC;
}
+ /**
+ * Gets the codec for keyTable. This codec excludes fields that are only
+ * meaningful for open keys.
+ *
+ * @return the codec for keyTable
+ */
+ public static Codec<OmKeyInfo> getKeyTableCodec() {
+ return CODEC_KEY_TABLE;
+ }
+
public String getVolumeName() {
return volumeName;
}
@@ -180,10 +204,6 @@ public String getFileName() {
return fileName;
}
- public void setExpectedDataGeneration(Long generation) {
- this.expectedDataGeneration = generation;
- }
-
public Long getExpectedDataGeneration() {
return expectedDataGeneration;
}
@@ -737,7 +757,20 @@ public KeyInfo getNetworkProtobuf(String fullKeyName, int
clientVersion,
* @return KeyInfo
*/
public KeyInfo getProtobuf(boolean ignorePipeline, int clientVersion) {
- return getProtobuf(ignorePipeline, null, clientVersion, false);
+ return getProtobuf(ignorePipeline, null, clientVersion, false, true);
+ }
+
+ /**
+ * Gets KeyInfo for persistence with control over fields only used in
openKeyTable.
+ *
+ * @param ignorePipeline true for persist to DB, false for network transmit.
+ * @param clientVersion the client version
+ * @param isOpenKey true for openKeyTable, false for keyTable
+ * @return KeyInfo
+ */
+ public KeyInfo getProtobuf(boolean ignorePipeline, int clientVersion,
+ boolean isOpenKey) {
+ return getProtobuf(ignorePipeline, null, clientVersion, false, isOpenKey);
}
/**
@@ -749,6 +782,22 @@ public KeyInfo getProtobuf(boolean ignorePipeline, int
clientVersion) {
*/
private KeyInfo getProtobuf(boolean ignorePipeline, String fullKeyName,
int clientVersion, boolean latestVersionBlocks) {
+ return getProtobuf(ignorePipeline, fullKeyName, clientVersion,
latestVersionBlocks, true);
+ }
+
+ /**
+ * Gets KeyInfo with all parameters.
+ *
+ * @param ignorePipeline ignore pipeline flag
+ * @param fullKeyName user given key name
+ * @param clientVersion the client version
+ * @param latestVersionBlocks whether to include only latest version blocks
+ * @param isOpenKey true for openKeyTable, false for keyTable
+ * @return key info object
+ */
+ private KeyInfo getProtobuf(boolean ignorePipeline, String fullKeyName,
+ int clientVersion, boolean latestVersionBlocks,
+ boolean isOpenKey) {
long latestVersion = keyLocationVersions.isEmpty() ? -1 :
keyLocationVersions.get(keyLocationVersions.size() - 1).getVersion();
@@ -800,7 +849,7 @@ private KeyInfo getProtobuf(boolean ignorePipeline, String
fullKeyName,
kb.setFileEncryptionInfo(OMPBHelper.convert(encInfo));
}
kb.setIsFile(isFile);
- if (expectedDataGeneration != null) {
+ if (isOpenKey && expectedDataGeneration != null) {
kb.setExpectedDataGeneration(expectedDataGeneration);
}
if (ownerName != null) {
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java
index 0f10832114e..cf37681a793 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java
@@ -37,8 +37,13 @@
* admin wants to confirm if a given key is deleted from deletedTable metadata.
*/
public class RepeatedOmKeyInfo implements CopyObject<RepeatedOmKeyInfo> {
- private static final Codec<RepeatedOmKeyInfo> CODEC_TRUE = newCodec(true);
- private static final Codec<RepeatedOmKeyInfo> CODEC_FALSE = newCodec(false);
+
+ private static final Codec<RepeatedOmKeyInfo> CODEC_TRUE = newCodec(true,
true);
+ private static final Codec<RepeatedOmKeyInfo> CODEC_FALSE = newCodec(false,
true);
+
+ // Codecs for deletedTable - exclude fields only used in openKeyTable
+ private static final Codec<RepeatedOmKeyInfo> CODEC_DELETED_TABLE_TRUE =
newCodec(true, false);
+ private static final Codec<RepeatedOmKeyInfo> CODEC_DELETED_TABLE_FALSE =
newCodec(false, false);
private final List<OmKeyInfo> omKeyInfoList;
/**
@@ -51,18 +56,36 @@ public class RepeatedOmKeyInfo implements
CopyObject<RepeatedOmKeyInfo> {
*/
private final long bucketId;
- private static Codec<RepeatedOmKeyInfo> newCodec(boolean ignorePipeline) {
+ private static Codec<RepeatedOmKeyInfo> newCodec(boolean ignorePipeline,
boolean isOpenKey) {
return new DelegatedCodec<>(
Proto2Codec.get(RepeatedKeyInfo.getDefaultInstance()),
RepeatedOmKeyInfo::getFromProto,
- k -> k.getProto(ignorePipeline, ClientVersion.CURRENT_VERSION),
+ k -> k.getProto(ignorePipeline, ClientVersion.CURRENT_VERSION,
isOpenKey),
RepeatedOmKeyInfo.class);
}
- public static Codec<RepeatedOmKeyInfo> getCodec(boolean ignorePipeline) {
+ /**
+ * Gets the codec for openKeyTable. This codec includes fields only used in
+ * openKeyTable during serialization.
+ *
+ * @param ignorePipeline whether to ignore pipeline info
+ * @return the codec for openKeyTable
+ */
+ public static Codec<RepeatedOmKeyInfo> getOpenKeyTableCodec(boolean
ignorePipeline) {
return ignorePipeline ? CODEC_TRUE : CODEC_FALSE;
}
+ /**
+ * Gets the codec for deletedTable. This codec excludes fields only used in
+ * openKeyTable during serialization, as deleted keys are committed keys.
+ *
+ * @param ignorePipeline whether to ignore pipeline info
+ * @return the codec for deletedTable
+ */
+ public static Codec<RepeatedOmKeyInfo> getDeletedTableCodec(boolean
ignorePipeline) {
+ return ignorePipeline ? CODEC_DELETED_TABLE_TRUE :
CODEC_DELETED_TABLE_FALSE;
+ }
+
public RepeatedOmKeyInfo(long bucketId) {
this.omKeyInfoList = new ArrayList<>();
this.bucketId = bucketId;
@@ -129,9 +152,18 @@ public static RepeatedOmKeyInfo
getFromProto(RepeatedKeyInfo repeatedKeyInfo) {
* @param compact true for persistence, false for network transmit
*/
public RepeatedKeyInfo getProto(boolean compact, int clientVersion) {
+ return getProto(compact, clientVersion, true);
+ }
+
+ /**
+ * @param compact true for persistence, false for network transmit
+ * @param clientVersion the client version
+ * @param isOpenKey true for openKeyTable, false for keyTable/deletedTable
+ */
+ public RepeatedKeyInfo getProto(boolean compact, int clientVersion, boolean
isOpenKey) {
List<KeyInfo> list = new ArrayList<>();
for (OmKeyInfo k : cloneOmKeyInfoList()) {
- list.add(k.getProtobuf(compact, clientVersion));
+ list.add(k.getProtobuf(compact, clientVersion, isOpenKey));
}
RepeatedKeyInfo.Builder builder = RepeatedKeyInfo.newBuilder()
diff --git
a/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfoCodec.java
b/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfoCodec.java
index bc568eea837..3de8f4677d8 100644
---
a/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfoCodec.java
+++
b/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfoCodec.java
@@ -18,8 +18,10 @@
package org.apache.hadoop.ozone.om.helpers;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.util.ArrayList;
@@ -35,6 +37,7 @@
import org.apache.hadoop.hdds.utils.db.Codec;
import org.apache.hadoop.hdds.utils.db.Proto2CodecTestBase;
import org.apache.hadoop.io.MD5Hash;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo;
import org.apache.hadoop.util.Time;
import org.junit.jupiter.api.Test;
@@ -51,7 +54,7 @@ public class TestOmKeyInfoCodec extends
Proto2CodecTestBase<OmKeyInfo> {
@Override
public Codec<OmKeyInfo> getCodec() {
- return OmKeyInfo.getCodec();
+ return OmKeyInfo.getOpenKeyTableCodec();
}
private static FileChecksum createEmptyChecksum() {
@@ -92,6 +95,42 @@ private OmKeyInfo getKeyInfo(int chunkNum) {
.build();
}
+ /**
+ * Creates an OmKeyInfo with fields only used in openKeyTable set.
+ * The expectedDataGeneration field is only meaningful for keys in
openKeyTable.
+ */
+ private OmKeyInfo getKeyInfoWithOpenKeyFields(int chunkNum) {
+ List<OmKeyLocationInfo> omKeyLocationInfoList = new ArrayList<>();
+ Pipeline pipeline = HddsTestUtils.getRandomPipeline();
+ for (int i = 0; i < chunkNum; i++) {
+ BlockID blockID = new BlockID(i, i);
+ OmKeyLocationInfo keyLocationInfo = new OmKeyLocationInfo.Builder()
+ .setBlockID(blockID)
+ .setPipeline(pipeline)
+ .build();
+ omKeyLocationInfoList.add(keyLocationInfo);
+ }
+ OmKeyLocationInfoGroup omKeyLocationInfoGroup = new
+ OmKeyLocationInfoGroup(0, omKeyLocationInfoList);
+
+ return new OmKeyInfo.Builder()
+ .setCreationTime(Time.now())
+ .setModificationTime(Time.now())
+ .setReplicationConfig(RatisReplicationConfig
+ .getInstance(HddsProtos.ReplicationFactor.THREE))
+ .setVolumeName(VOLUME)
+ .setBucketName(BUCKET)
+ .setKeyName(KEYNAME)
+ .setObjectID(Time.now())
+ .setUpdateID(Time.now())
+ .setDataSize(100)
+ .setOmKeyLocationInfos(
+ Collections.singletonList(omKeyLocationInfoGroup))
+ .setFileChecksum(checksum)
+ .setExpectedDataGeneration(12345L)
+ .build();
+ }
+
@Test
public void test() throws IOException {
testOmKeyInfoCodecWithoutPipeline(1);
@@ -100,7 +139,7 @@ public void test() throws IOException {
public void testOmKeyInfoCodecWithoutPipeline(int chunkNum)
throws IOException {
- final Codec<OmKeyInfo> codec = OmKeyInfo.getCodec();
+ final Codec<OmKeyInfo> codec = OmKeyInfo.getOpenKeyTableCodec();
OmKeyInfo originKey = getKeyInfo(chunkNum);
byte[] rawData = codec.toPersistedFormat(originKey);
OmKeyInfo key = codec.fromPersistedFormat(rawData);
@@ -111,4 +150,93 @@ public void testOmKeyInfoCodecWithoutPipeline(int chunkNum)
assertNotNull(key.getFileChecksum());
assertEquals(key.getFileChecksum(), checksum);
}
+
+ @Test
+ public void testOpenKeyTableCodecIncludesOpenKeyFields() throws IOException {
+ final Codec<OmKeyInfo> openKeyCodec = OmKeyInfo.getOpenKeyTableCodec();
+ OmKeyInfo originKey = getKeyInfoWithOpenKeyFields(1);
+
+ assertEquals(12345L, originKey.getExpectedDataGeneration());
+
+ byte[] rawData = openKeyCodec.toPersistedFormat(originKey);
+ OmKeyInfo deserializedKey = openKeyCodec.fromPersistedFormat(rawData);
+
+ assertEquals(12345L, deserializedKey.getExpectedDataGeneration());
+
+ KeyInfo keyInfo = KeyInfo.parseFrom(rawData);
+ assertTrue(keyInfo.hasExpectedDataGeneration(),
+ "openKeyTable codec should include expectedDataGeneration in proto");
+ assertEquals(12345L, keyInfo.getExpectedDataGeneration());
+ }
+
+ @Test
+ public void testKeyTableCodecExcludesOpenKeyFields() throws IOException {
+ final Codec<OmKeyInfo> keyTableCodec = OmKeyInfo.getKeyTableCodec();
+ OmKeyInfo originKey = getKeyInfoWithOpenKeyFields(1);
+ assertEquals(12345L, originKey.getExpectedDataGeneration());
+
+ byte[] rawData = keyTableCodec.toPersistedFormat(originKey);
+ KeyInfo keyInfo = KeyInfo.parseFrom(rawData);
+ assertFalse(keyInfo.hasExpectedDataGeneration(),
+ "keyTable codec should NOT include expectedDataGeneration in proto");
+
+ OmKeyInfo deserializedKey = keyTableCodec.fromPersistedFormat(rawData);
+ assertEquals(VOLUME, deserializedKey.getVolumeName());
+ assertEquals(BUCKET, deserializedKey.getBucketName());
+ assertEquals(KEYNAME, deserializedKey.getKeyName());
+ assertEquals(100, deserializedKey.getDataSize());
+
+ assertNull(deserializedKey.getExpectedDataGeneration(),
+ "Deserialized key from keyTable should have null
expectedDataGeneration");
+ }
+
+ @Test
+ public void testKeyTableCodecCanReadOpenKeyTableData() throws IOException {
+ final Codec<OmKeyInfo> openKeyCodec = OmKeyInfo.getOpenKeyTableCodec();
+ final Codec<OmKeyInfo> keyTableCodec = OmKeyInfo.getKeyTableCodec();
+
+ OmKeyInfo originKey = getKeyInfoWithOpenKeyFields(1);
+ byte[] rawData = openKeyCodec.toPersistedFormat(originKey);
+ OmKeyInfo deserializedKey = keyTableCodec.fromPersistedFormat(rawData);
+
+ assertEquals(VOLUME, deserializedKey.getVolumeName());
+ assertEquals(BUCKET, deserializedKey.getBucketName());
+ assertEquals(12345L, deserializedKey.getExpectedDataGeneration());
+ }
+
+ @Test
+ public void testCodecsWithKeyWithoutOpenKeyFields() throws IOException {
+ final Codec<OmKeyInfo> openKeyCodec = OmKeyInfo.getOpenKeyTableCodec();
+ final Codec<OmKeyInfo> keyTableCodec = OmKeyInfo.getKeyTableCodec();
+
+ OmKeyInfo originKey = getKeyInfo(1);
+ assertNull(originKey.getExpectedDataGeneration());
+
+ byte[] openKeyData = openKeyCodec.toPersistedFormat(originKey);
+ byte[] keyTableData = keyTableCodec.toPersistedFormat(originKey);
+
+ OmKeyInfo fromOpenKeyCodec = openKeyCodec.fromPersistedFormat(openKeyData);
+ OmKeyInfo fromKeyTableCodec =
keyTableCodec.fromPersistedFormat(keyTableData);
+
+ assertEquals(VOLUME, fromOpenKeyCodec.getVolumeName());
+ assertEquals(VOLUME, fromKeyTableCodec.getVolumeName());
+ assertEquals(BUCKET, fromOpenKeyCodec.getBucketName());
+ assertEquals(BUCKET, fromKeyTableCodec.getBucketName());
+ assertNull(fromOpenKeyCodec.getExpectedDataGeneration());
+ assertNull(fromKeyTableCodec.getExpectedDataGeneration());
+ }
+
+ @Test
+ public void testKeyTableCodecProducesSmallerOutput() throws IOException {
+ final Codec<OmKeyInfo> openKeyCodec = OmKeyInfo.getOpenKeyTableCodec();
+ final Codec<OmKeyInfo> keyTableCodec = OmKeyInfo.getKeyTableCodec();
+
+ OmKeyInfo keyWithOpenFields = getKeyInfoWithOpenKeyFields(1);
+
+ byte[] openKeyData = openKeyCodec.toPersistedFormat(keyWithOpenFields);
+ byte[] keyTableData = keyTableCodec.toPersistedFormat(keyWithOpenFields);
+
+ assertTrue(keyTableData.length < openKeyData.length,
+ "keyTable codec should produce smaller serialized output when
openKeyTable-only fields are set");
+ }
}
diff --git
a/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestRepeatedOmKeyInfoCodec.java
b/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestRepeatedOmKeyInfoCodec.java
index 3227cfbe6a4..fdd7c64a730 100644
---
a/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestRepeatedOmKeyInfoCodec.java
+++
b/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestRepeatedOmKeyInfoCodec.java
@@ -21,6 +21,7 @@
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.io.IOException;
@@ -36,11 +37,12 @@
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.utils.db.Codec;
import org.apache.hadoop.hdds.utils.db.Proto2CodecTestBase;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RepeatedKeyInfo;
import org.apache.hadoop.util.Time;
import org.junit.jupiter.api.Test;
/**
- * Test {@link RepeatedOmKeyInfo#getCodec(boolean)}.
+ * Test {@link RepeatedOmKeyInfo#getOpenKeyTableCodec(boolean)}.
*/
public class TestRepeatedOmKeyInfoCodec
extends Proto2CodecTestBase<RepeatedOmKeyInfo> {
@@ -51,7 +53,7 @@ public class TestRepeatedOmKeyInfoCodec
@Override
public Codec<RepeatedOmKeyInfo> getCodec() {
- return RepeatedOmKeyInfo.getCodec(true);
+ return RepeatedOmKeyInfo.getOpenKeyTableCodec(true);
}
private OmKeyInfo getKeyInfo(int chunkNum) {
@@ -94,7 +96,7 @@ void test() throws Exception {
}
public void testWithoutPipeline(int chunkNum) throws IOException {
- final Codec<RepeatedOmKeyInfo> codec = RepeatedOmKeyInfo.getCodec(true);
+ final Codec<RepeatedOmKeyInfo> codec =
RepeatedOmKeyInfo.getOpenKeyTableCodec(true);
OmKeyInfo originKey = getKeyInfo(chunkNum);
long bucketId = Time.now();
RepeatedOmKeyInfo repeatedOmKeyInfo = new RepeatedOmKeyInfo(originKey,
bucketId);
@@ -108,9 +110,9 @@ public void testWithoutPipeline(int chunkNum) throws
IOException {
public void testCompatibility(int chunkNum) throws IOException {
final Codec<RepeatedOmKeyInfo> codecWithoutPipeline
- = RepeatedOmKeyInfo.getCodec(true);
+ = RepeatedOmKeyInfo.getOpenKeyTableCodec(true);
final Codec<RepeatedOmKeyInfo> codecWithPipeline
- = RepeatedOmKeyInfo.getCodec(false);
+ = RepeatedOmKeyInfo.getOpenKeyTableCodec(false);
OmKeyInfo originKey = getKeyInfo(chunkNum);
long bucketId = Time.now();
RepeatedOmKeyInfo repeatedOmKeyInfo = new RepeatedOmKeyInfo(originKey,
bucketId);
@@ -125,7 +127,7 @@ public void threadSafety() throws InterruptedException {
final OmKeyInfo key = getKeyInfo(1);
long bucketId = Time.now();
final RepeatedOmKeyInfo subject = new RepeatedOmKeyInfo(key, bucketId);
- final Codec<RepeatedOmKeyInfo> codec = RepeatedOmKeyInfo.getCodec(true);
+ final Codec<RepeatedOmKeyInfo> codec =
RepeatedOmKeyInfo.getOpenKeyTableCodec(true);
final AtomicBoolean failed = new AtomicBoolean();
ThreadFactory threadFactory = new ThreadFactoryBuilder().setDaemon(true)
.build();
@@ -150,4 +152,99 @@ public void threadSafety() throws InterruptedException {
}
assertFalse(failed.get());
}
+
+ private OmKeyInfo getKeyInfoWithOpenKeyFields(int chunkNum) {
+ List<OmKeyLocationInfo> omKeyLocationInfoList = new ArrayList<>();
+ Pipeline pipeline = HddsTestUtils.getRandomPipeline();
+ for (int i = 0; i < chunkNum; i++) {
+ BlockID blockID = new BlockID(i, i);
+ OmKeyLocationInfo keyLocationInfo = new OmKeyLocationInfo.Builder()
+ .setBlockID(blockID)
+ .setPipeline(pipeline)
+ .build();
+ omKeyLocationInfoList.add(keyLocationInfo);
+ }
+ OmKeyLocationInfoGroup omKeyLocationInfoGroup = new
+ OmKeyLocationInfoGroup(0, omKeyLocationInfoList);
+ return new OmKeyInfo.Builder()
+ .setCreationTime(Time.now())
+ .setModificationTime(Time.now())
+ .setReplicationConfig(
+ RatisReplicationConfig
+ .getInstance(HddsProtos.ReplicationFactor.THREE))
+ .setVolumeName(VOLUME)
+ .setBucketName(BUCKET)
+ .setKeyName(KEYNAME)
+ .setObjectID(Time.now())
+ .setUpdateID(Time.now())
+ .setDataSize(100)
+ .setOmKeyLocationInfos(
+ Collections.singletonList(omKeyLocationInfoGroup))
+ .setExpectedDataGeneration(12345L)
+ .build();
+ }
+
+ @Test
+ void testRegularCodecIncludesOpenKeyFields() throws IOException {
+ final Codec<RepeatedOmKeyInfo> codec =
RepeatedOmKeyInfo.getOpenKeyTableCodec(true);
+ OmKeyInfo keyWithFields = getKeyInfoWithOpenKeyFields(1);
+ long bucketId = Time.now();
+ RepeatedOmKeyInfo repeatedOmKeyInfo = new RepeatedOmKeyInfo(keyWithFields,
bucketId);
+
+ byte[] rawData = codec.toPersistedFormat(repeatedOmKeyInfo);
+ RepeatedKeyInfo proto = RepeatedKeyInfo.parseFrom(rawData);
+
+ assertTrue(proto.getKeyInfo(0).hasExpectedDataGeneration());
+ assertEquals(12345L, proto.getKeyInfo(0).getExpectedDataGeneration());
+ }
+
+ @Test
+ void testDeletedTableCodecExcludesOpenKeyFields() throws IOException {
+ final Codec<RepeatedOmKeyInfo> codec =
RepeatedOmKeyInfo.getDeletedTableCodec(true);
+ OmKeyInfo keyWithFields = getKeyInfoWithOpenKeyFields(1);
+ long bucketId = Time.now();
+ RepeatedOmKeyInfo repeatedOmKeyInfo = new RepeatedOmKeyInfo(keyWithFields,
bucketId);
+
+ byte[] rawData = codec.toPersistedFormat(repeatedOmKeyInfo);
+ RepeatedKeyInfo proto = RepeatedKeyInfo.parseFrom(rawData);
+
+ assertFalse(proto.getKeyInfo(0).hasExpectedDataGeneration());
+
+ RepeatedOmKeyInfo deserialized = codec.fromPersistedFormat(rawData);
+ assertEquals(VOLUME,
deserialized.getOmKeyInfoList().get(0).getVolumeName());
+ assertEquals(BUCKET,
deserialized.getOmKeyInfoList().get(0).getBucketName());
+ assertEquals(bucketId, deserialized.getBucketId());
+ }
+
+ @Test
+ void testDeletedTableCodecCanReadRegularCodecData() throws IOException {
+ final Codec<RepeatedOmKeyInfo> regularCodec =
RepeatedOmKeyInfo.getOpenKeyTableCodec(true);
+ final Codec<RepeatedOmKeyInfo> deletedTableCodec =
RepeatedOmKeyInfo.getDeletedTableCodec(true);
+
+ OmKeyInfo keyWithFields = getKeyInfoWithOpenKeyFields(1);
+ long bucketId = Time.now();
+ RepeatedOmKeyInfo repeatedOmKeyInfo = new RepeatedOmKeyInfo(keyWithFields,
bucketId);
+
+ byte[] rawData = regularCodec.toPersistedFormat(repeatedOmKeyInfo);
+ RepeatedOmKeyInfo deserialized =
deletedTableCodec.fromPersistedFormat(rawData);
+
+ assertEquals(VOLUME,
deserialized.getOmKeyInfoList().get(0).getVolumeName());
+ assertEquals(12345L,
deserialized.getOmKeyInfoList().get(0).getExpectedDataGeneration());
+ }
+
+ @Test
+ void testDeletedTableCodecProducesSmallerOutput() throws IOException {
+ final Codec<RepeatedOmKeyInfo> regularCodec =
RepeatedOmKeyInfo.getOpenKeyTableCodec(true);
+ final Codec<RepeatedOmKeyInfo> deletedTableCodec =
RepeatedOmKeyInfo.getDeletedTableCodec(true);
+
+ OmKeyInfo keyWithFields = getKeyInfoWithOpenKeyFields(1);
+ long bucketId = Time.now();
+ RepeatedOmKeyInfo repeatedOmKeyInfo = new RepeatedOmKeyInfo(keyWithFields,
bucketId);
+
+ byte[] regularData = regularCodec.toPersistedFormat(repeatedOmKeyInfo);
+ byte[] deletedTableData =
deletedTableCodec.toPersistedFormat(repeatedOmKeyInfo);
+
+ assertTrue(deletedTableData.length < regularData.length,
+ "deletedTable codec should produce smaller output");
+ }
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
index e41a7b15445..02e32edec46 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
@@ -204,25 +204,25 @@ public final class OMDBDefinition extends
DBDefinition.WithMap {
//---------------------------------------------------------------------------
// Object Store (OBS) Tables:
public static final String KEY_TABLE = "keyTable";
- /** keyTable: /volume/bucket/key :- KeyInfo. */
+ /** keyTable: /volume/bucket/key :- KeyInfo (excludes fields only used in
openKeyTable). */
public static final DBColumnFamilyDefinition<String, OmKeyInfo> KEY_TABLE_DEF
= new DBColumnFamilyDefinition<>(KEY_TABLE,
StringCodec.get(),
- OmKeyInfo.getCodec());
+ OmKeyInfo.getKeyTableCodec());
public static final String DELETED_TABLE = "deletedTable";
- /** deletedTable: /volume/bucket/key :- RepeatedKeyInfo. */
+ /** deletedTable: /volume/bucket/key :- RepeatedKeyInfo (excludes fields
only used in openKeyTable). */
public static final DBColumnFamilyDefinition<String, RepeatedOmKeyInfo>
DELETED_TABLE_DEF
= new DBColumnFamilyDefinition<>(DELETED_TABLE,
StringCodec.get(),
- RepeatedOmKeyInfo.getCodec(true));
+ RepeatedOmKeyInfo.getDeletedTableCodec(true));
public static final String OPEN_KEY_TABLE = "openKeyTable";
/** openKeyTable: /volume/bucket/key/id :- KeyInfo. */
public static final DBColumnFamilyDefinition<String, OmKeyInfo>
OPEN_KEY_TABLE_DEF
= new DBColumnFamilyDefinition<>(OPEN_KEY_TABLE,
StringCodec.get(),
- OmKeyInfo.getCodec());
+ OmKeyInfo.getOpenKeyTableCodec());
public static final String MULTIPART_INFO_TABLE = "multipartInfoTable";
/** multipartInfoTable: /volume/bucket/key/uploadId :- parts. */
@@ -241,18 +241,18 @@ public final class OMDBDefinition extends
DBDefinition.WithMap {
//---------------------------------------------------------------------------
// File System Optimized (FSO) Tables:
public static final String FILE_TABLE = "fileTable";
- /** fileTable: /volumeId/bucketId/parentId/fileName :- KeyInfo. */
+ /** fileTable: /volumeId/bucketId/parentId/fileName :- KeyInfo (excludes
fields only used in openKeyTable). */
public static final DBColumnFamilyDefinition<String, OmKeyInfo>
FILE_TABLE_DEF
= new DBColumnFamilyDefinition<>(FILE_TABLE,
StringCodec.get(),
- OmKeyInfo.getCodec());
+ OmKeyInfo.getKeyTableCodec());
public static final String OPEN_FILE_TABLE = "openFileTable";
/** openFileTable: /volumeId/bucketId/parentId/fileName/id :- KeyInfo. */
public static final DBColumnFamilyDefinition<String, OmKeyInfo>
OPEN_FILE_TABLE_DEF
= new DBColumnFamilyDefinition<>(OPEN_FILE_TABLE,
StringCodec.get(),
- OmKeyInfo.getCodec());
+ OmKeyInfo.getOpenKeyTableCodec());
public static final String DIRECTORY_TABLE = "directoryTable";
/** directoryTable: /volumeId/bucketId/parentId/dirName :- DirInfo. */
@@ -262,11 +262,14 @@ public final class OMDBDefinition extends
DBDefinition.WithMap {
OmDirectoryInfo.getCodec());
public static final String DELETED_DIR_TABLE = "deletedDirectoryTable";
- /** deletedDirectoryTable: /volumeId/bucketId/parentId/dirName/objectId :-
KeyInfo. */
+ /**
+ * deletedDirectoryTable: /volumeId/bucketId/parentId/dirName/objectId :-
KeyInfo
+ * (excludes fields only used in openKeyTable).
+ */
public static final DBColumnFamilyDefinition<String, OmKeyInfo>
DELETED_DIR_TABLE_DEF
= new DBColumnFamilyDefinition<>(DELETED_DIR_TABLE,
StringCodec.get(),
- OmKeyInfo.getCodec());
+ OmKeyInfo.getKeyTableCodec());
//---------------------------------------------------------------------------
// S3 Multi-Tenancy Tables
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
index 4e1d3603d73..6c34443058b 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
@@ -303,9 +303,6 @@ public OMClientResponse validateAndUpdateCache(OzoneManager
ozoneManager, Execut
}
validateAtomicRewrite(keyToDelete, omKeyInfo, auditMap);
- // Optimistic locking validation has passed. Now set the rewrite fields
to null so they are
- // not persisted in the key table.
- // Combination
// Set the UpdateID to current transactionLogIndex
omKeyInfo = omKeyInfo.toBuilder()
.setExpectedDataGeneration(null)
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
index 25b5a4b15d4..f9aa275ef75 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
@@ -245,7 +245,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager
ozoneManager, Execut
validateAtomicRewrite(keyToDelete, omKeyInfo, auditMap);
// Optimistic locking validation has passed. Now set the rewrite fields
to null so they are
// not persisted in the key table.
- omKeyInfo.setExpectedDataGeneration(null);
+ omKeyInfo = omKeyInfo.toBuilder()
+ .setExpectedDataGeneration(null)
+ .build();
long correctedSpace = omKeyInfo.getReplicatedSize();
// if keyToDelete isn't null, usedNamespace shouldn't check and increase.
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java
index c64196c590f..de009bf8fe4 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java
@@ -274,7 +274,6 @@ public void testAtomicRewrite() throws Exception {
assertEquals(OK, omClientResponse.getOMResponse().getStatus());
OmKeyInfo committedKey = closedKeyTable.get(getOzonePathKey());
- assertNull(committedKey.getExpectedDataGeneration());
// Generation should be changed
assertNotEquals(closedKeyInfo.getGeneration(),
committedKey.getGeneration());
assertEquals(acls, committedKey.getAcls());
@@ -312,7 +311,6 @@ public void testAtomicCreateIfNotExistsCommitKeyAbsent()
throws Exception {
OmKeyInfo committedKey = closedKeyTable.get(getOzonePathKey());
assertNotNull(committedKey);
- assertNull(committedKey.getExpectedDataGeneration());
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
