This is an automated email from the ASF dual-hosted git repository.
SaketaChalamchala pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 87d2bd8577b HDDS-15387. Selectively deserialize KeyInfo/DirectoryInfo
fields for snapshot diff (#10603)
87d2bd8577b is described below
commit 87d2bd8577b3d30843e8e2b600acc18944cd3d0e
Author: SaketaChalamchala <[email protected]>
AuthorDate: Mon Jul 13 14:52:17 2026 -0700
HDDS-15387. Selectively deserialize KeyInfo/DirectoryInfo fields for
snapshot diff (#10603)
Co-authored-by: Wei-Chiu Chuang <[email protected]>
Co-authored-by: Copilot Autofix powered by AI
<[email protected]>
---
.../src/main/proto/OmClientProtocol.proto | 4 +
.../ozone/om/snapshot/SnapshotDiffValueParser.java | 359 +++++++++++++++++++++
.../om/snapshot/TestSnapshotDiffValueParser.java | 249 ++++++++++++++
3 files changed, 612 insertions(+)
diff --git
a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index f50d812e51f..bb8f54c79c5 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -1187,6 +1187,8 @@ message FileChecksumProto {
}
message KeyInfo {
+ // When adding a new field, update
SnapshotDiffValueParser.computeKeyInfoCompareSignature()
+ // if it is required for snapshot diff comparisons.
required string volumeName = 1;
required string bucketName = 2;
required string keyName = 3;
@@ -1251,6 +1253,8 @@ message BasicKeyInfo {
}
message DirectoryInfo {
+ // When adding a new field, update
SnapshotDiffValueParser.computeDirectoryInfoCompareSignature()
+ // if it is required for snapshot diff comparisons.
required string name = 1;
required uint64 creationTime = 2;
required uint64 modificationTime = 3;
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffValueParser.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffValueParser.java
new file mode 100644
index 00000000000..b1c09cd082a
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffValueParser.java
@@ -0,0 +1,359 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.snapshot;
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.CodedInputStream;
+import com.google.protobuf.WireFormat;
+import java.io.IOException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.KeyValue;
+import org.apache.hadoop.ozone.OzoneConsts;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DirectoryInfo;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo;
+
+/**
+ * Parses snapshot diff values without full deserialization.
+ */
+public final class SnapshotDiffValueParser {
+ private static final int INT_BYTES = 4;
+ private static final int LONG_BYTES = 8;
+ private static final int HSYNC_METADATA_PRESENT_TAG = 1001;
+ private static final String DIGEST_ALGORITHM = "SHA-256";
+
+ private SnapshotDiffValueParser() {
+ }
+
+ public static ParsedRequiredInfo parseKeyInfoRequiredFields(byte[] value,
boolean includeUpdateId)
+ throws IOException {
+ CodedInputStream input = CodedInputStream.newInstance(value);
+ long updateId = 0L;
+ long objectId = 0L;
+ long parentId = 0L;
+ boolean hasUpdateId = false;
+ String keyName = null;
+
+ int tag;
+ while ((tag = input.readTag()) != 0) {
+ int fieldNumber = WireFormat.getTagFieldNumber(tag);
+ switch (fieldNumber) {
+ case KeyInfo.KEYNAME_FIELD_NUMBER:
+ keyName = input.readString();
+ break;
+ case KeyInfo.OBJECTID_FIELD_NUMBER:
+ objectId = input.readUInt64();
+ break;
+ case KeyInfo.UPDATEID_FIELD_NUMBER:
+ if (includeUpdateId) {
+ updateId = input.readUInt64();
+ hasUpdateId = true;
+ } else {
+ input.skipField(tag);
+ }
+ break;
+ case KeyInfo.PARENTID_FIELD_NUMBER:
+ parentId = input.readUInt64();
+ break;
+ default:
+ input.skipField(tag);
+ break;
+ }
+ }
+
+ return new ParsedRequiredInfo(updateId, hasUpdateId, objectId, parentId,
keyName);
+ }
+
+ public static byte[] computeKeyInfoCompareSignature(byte[] value) throws
IOException {
+ CodedInputStream input = CodedInputStream.newInstance(value);
+ MessageDigest digest = newDigest();
+ AtomicBoolean hasHsyncMetadata = new AtomicBoolean(false);
+ int keyLocationListCount = 0;
+ ByteString latestKeyLocationList = null;
+ List<byte[]> metadataSignatures = new ArrayList<>();
+ List<byte[]> tagSignatures = new ArrayList<>();
+
+ int tag;
+ while ((tag = input.readTag()) != 0) {
+ int fieldNumber = WireFormat.getTagFieldNumber(tag);
+ switch (fieldNumber) {
+ case KeyInfo.DATASIZE_FIELD_NUMBER:
+ updateDigestWithLong(digest, fieldNumber, input.readUInt64());
+ break;
+ case KeyInfo.KEYLOCATIONLIST_FIELD_NUMBER:
+ latestKeyLocationList = input.readBytes();
+ keyLocationListCount++;
+ break;
+ case KeyInfo.METADATA_FIELD_NUMBER:
+ byte[] metadataDigest =
parseKeyValueDigest(input.readBytes().toByteArray(), true, hasHsyncMetadata);
+ if (metadataDigest != null) {
+ metadataSignatures.add(metadataDigest);
+ }
+ break;
+ case KeyInfo.FILECHECKSUM_FIELD_NUMBER:
+ case KeyInfo.ACLS_FIELD_NUMBER:
+ updateDigestWithBytes(digest, fieldNumber, input.readBytes());
+ break;
+ case KeyInfo.TAGS_FIELD_NUMBER:
+ byte[] tagDigest =
parseKeyValueDigest(input.readBytes().toByteArray(), false, null);
+ if (tagDigest != null) {
+ tagSignatures.add(tagDigest);
+ }
+ break;
+ default:
+ input.skipField(tag);
+ break;
+ }
+ }
+
+ if (latestKeyLocationList != null) {
+ updateDigestWithBytes(digest, KeyInfo.KEYLOCATIONLIST_FIELD_NUMBER,
latestKeyLocationList);
+ }
+ addCanonicalizedDigest(digest, KeyInfo.METADATA_FIELD_NUMBER,
metadataSignatures);
+ addCanonicalizedDigest(digest, KeyInfo.TAGS_FIELD_NUMBER, tagSignatures);
+ updateDigestWithBoolean(digest, HSYNC_METADATA_PRESENT_TAG,
hasHsyncMetadata.get());
+ updateDigestWithInt(digest, keyLocationListCount);
+
+ return digest.digest();
+ }
+
+ public static ParsedRequiredInfo parseDirectoryInfoRequiredFields(byte[]
value, boolean includeUpdateId)
+ throws IOException {
+ CodedInputStream input = CodedInputStream.newInstance(value);
+ long updateId = 0L;
+ long objectId = 0L;
+ long parentId = 0L;
+ boolean hasUpdateId = false;
+ String name = null;
+
+ int tag;
+ while ((tag = input.readTag()) != 0) {
+ int fieldNumber = WireFormat.getTagFieldNumber(tag);
+ switch (fieldNumber) {
+ case DirectoryInfo.NAME_FIELD_NUMBER:
+ name = input.readString();
+ break;
+ case DirectoryInfo.OBJECTID_FIELD_NUMBER:
+ objectId = input.readUInt64();
+ break;
+ case DirectoryInfo.UPDATEID_FIELD_NUMBER:
+ if (includeUpdateId) {
+ updateId = input.readUInt64();
+ hasUpdateId = true;
+ } else {
+ input.skipField(tag);
+ }
+ break;
+ case DirectoryInfo.PARENTID_FIELD_NUMBER:
+ parentId = input.readUInt64();
+ break;
+ default:
+ input.skipField(tag);
+ break;
+ }
+ }
+
+ return new ParsedRequiredInfo(updateId, hasUpdateId, objectId, parentId,
name);
+ }
+
+ public static byte[] computeDirectoryInfoCompareSignature(byte[] value)
throws IOException {
+ CodedInputStream input = CodedInputStream.newInstance(value);
+ MessageDigest digest = newDigest();
+ List<byte[]> metadataSignatures = new ArrayList<>();
+
+ int tag;
+ while ((tag = input.readTag()) != 0) {
+ int fieldNumber = WireFormat.getTagFieldNumber(tag);
+ switch (fieldNumber) {
+ case DirectoryInfo.METADATA_FIELD_NUMBER:
+ byte[] metadataDigest =
parseKeyValueDigest(input.readBytes().toByteArray(), false, null);
+ if (metadataDigest != null) {
+ metadataSignatures.add(metadataDigest);
+ }
+ break;
+ case DirectoryInfo.ACLS_FIELD_NUMBER:
+ updateDigestWithBytes(digest, fieldNumber, input.readBytes());
+ break;
+ default:
+ input.skipField(tag);
+ break;
+ }
+ }
+
+ addCanonicalizedDigest(digest, DirectoryInfo.METADATA_FIELD_NUMBER,
metadataSignatures);
+
+ return digest.digest();
+ }
+
+ private static void updateDigestWithLong(MessageDigest digest, int
fieldNumber, long value) {
+ updateDigestWithInt(digest, fieldNumber);
+ byte[] buffer = new byte[LONG_BYTES];
+ for (int i = LONG_BYTES - 1; i >= 0; i--) {
+ buffer[i] = (byte) (value & 0xFFL);
+ value >>>= 8;
+ }
+ digest.update(buffer);
+ }
+
+ private static void updateDigestWithBytes(MessageDigest digest, int
fieldNumber, ByteString value) {
+ updateDigestWithInt(digest, fieldNumber);
+ updateDigestWithInt(digest, value.size());
+ digest.update(value.toByteArray());
+ }
+
+ private static void updateTaggedString(MessageDigest digest, int
fieldNumber, String value) {
+ updateDigestWithInt(digest, fieldNumber);
+ if (value == null) {
+ updateDigestWithInt(digest, 0);
+ return;
+ }
+ byte[] bytes = value.getBytes(java.nio.charset.StandardCharsets.UTF_8);
+ updateDigestWithInt(digest, bytes.length);
+ digest.update(bytes);
+ }
+
+ private static void updateDigestWithBoolean(MessageDigest digest, int
fieldNumber, boolean value) {
+ updateDigestWithInt(digest, fieldNumber);
+ updateDigestWithInt(digest, value ? 1 : 0);
+ }
+
+ private static void updateDigestWithRawBytes(MessageDigest digest, int
fieldNumber, byte[] value) {
+ updateDigestWithInt(digest, fieldNumber);
+ updateDigestWithInt(digest, value.length);
+ digest.update(value);
+ }
+
+ private static void updateDigestWithInt(MessageDigest digest, int value) {
+ byte[] buffer = new byte[INT_BYTES];
+ for (int i = INT_BYTES - 1; i >= 0; i--) {
+ buffer[i] = (byte) (value & 0xFF);
+ value >>>= 8;
+ }
+ digest.update(buffer);
+ }
+
+ private static MessageDigest newDigest() {
+ try {
+ return MessageDigest.getInstance(DIGEST_ALGORITHM);
+ } catch (NoSuchAlgorithmException e) {
+ throw new IllegalStateException("SHA-256 is not available", e);
+ }
+ }
+
+ private static void addCanonicalizedDigest(
+ MessageDigest digest, int fieldNumber, List<byte[]> metadataEntries) {
+ if (metadataEntries.isEmpty()) {
+ return;
+ }
+ metadataEntries.sort(SnapshotDiffValueParser::compareBytes);
+ MessageDigest metadataDigest = newDigest();
+ for (byte[] metadataEntry : metadataEntries) {
+ metadataDigest.update(metadataEntry);
+ }
+ updateDigestWithRawBytes(digest, fieldNumber, metadataDigest.digest());
+ }
+
+ private static byte[] parseKeyValueDigest(byte[] keyValueBytes, boolean
computeHsync, AtomicBoolean hasHsync)
+ throws IOException {
+ if (keyValueBytes == null || keyValueBytes.length == 0) {
+ return null;
+ }
+ CodedInputStream input = CodedInputStream.newInstance(keyValueBytes);
+ String key = null;
+ String value = null;
+ while (!input.isAtEnd()) {
+ int tag = input.readTag();
+ if (tag == 0) {
+ break;
+ }
+ int fieldNumber = WireFormat.getTagFieldNumber(tag);
+ switch (fieldNumber) {
+ case KeyValue.KEY_FIELD_NUMBER:
+ key = input.readString();
+ break;
+ case KeyValue.VALUE_FIELD_NUMBER:
+ value = input.readString();
+ break;
+ default:
+ input.skipField(tag);
+ break;
+ }
+ }
+ if (computeHsync && hasHsync != null &&
OzoneConsts.HSYNC_CLIENT_ID.equals(key)) {
+ hasHsync.set(true);
+ }
+ MessageDigest entryDigest = newDigest();
+ updateTaggedString(entryDigest, KeyValue.KEY_FIELD_NUMBER, key);
+ updateTaggedString(entryDigest, KeyValue.VALUE_FIELD_NUMBER, value);
+ return entryDigest.digest();
+ }
+
+ private static int compareBytes(byte[] left, byte[] right) {
+ int length = Math.min(left.length, right.length);
+ for (int i = 0; i < length; i++) {
+ int diff = (left[i] & 0xFF) - (right[i] & 0xFF);
+ if (diff != 0) {
+ return diff;
+ }
+ }
+ return left.length - right.length;
+ }
+
+ /**
+ * Parsed fields shared by key and directory entries.
+ * Holds IDs and name with optional updateID when requested.
+ */
+ public static final class ParsedRequiredInfo {
+ private final long updateId;
+ private final boolean hasUpdateId;
+ private final long objectId;
+ private final long parentId;
+ private final String name;
+
+ private ParsedRequiredInfo(long updateId, boolean hasUpdateId, long
objectId, long parentId, String name) {
+ this.updateId = updateId;
+ this.hasUpdateId = hasUpdateId;
+ this.objectId = objectId;
+ this.parentId = parentId;
+ this.name = name;
+ }
+
+ public long getUpdateId() {
+ return updateId;
+ }
+
+ public boolean hasUpdateId() {
+ return hasUpdateId;
+ }
+
+ public long getObjectId() {
+ return objectId;
+ }
+
+ public long getParentId() {
+ return parentId;
+ }
+
+ public String getName() {
+ return name;
+ }
+ }
+}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffValueParser.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffValueParser.java
new file mode 100644
index 00000000000..261a97a130d
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffValueParser.java
@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.snapshot;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.hadoop.fs.FileChecksum;
+import org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum;
+import org.apache.hadoop.hdds.client.BlockID;
+import org.apache.hadoop.hdds.client.RatisReplicationConfig;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
+import org.apache.hadoop.io.MD5Hash;
+import org.apache.hadoop.ozone.OzoneAcl;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.junit.jupiter.api.Test;
+
+class TestSnapshotDiffValueParser {
+ private static final String VOLUME = "volume";
+ private static final String BUCKET = "bucket";
+ private static final String KEY_NAME = "dir/file";
+ private static final String DIR_NAME = "dir";
+ private static final long OBJECT_ID = 10L;
+ private static final long PARENT_ID = 20L;
+ private static final long UPDATE_ID = 30L;
+
+ @Test
+ void testKeyInfoParserSignatureAndUpdateId() throws Exception {
+ OmKeyInfo keyInfo = createKeyInfo(100L, 200L, 1024L, createChecksum((byte)
1),
+ createMetadata("meta", "one"), createTags("tag", "one"), createAcls(),
+ Collections.singletonList(createKeyLocationGroup(1L)));
+ byte[] rawData = OmKeyInfo.getCodec().toPersistedFormat(keyInfo);
+
+ SnapshotDiffValueParser.ParsedRequiredInfo parsed =
+ SnapshotDiffValueParser.parseKeyInfoRequiredFields(rawData, true);
+ assertEquals(UPDATE_ID, parsed.getUpdateId());
+ assertEquals(OBJECT_ID, parsed.getObjectId());
+ assertEquals(PARENT_ID, parsed.getParentId());
+ assertEquals(KEY_NAME, parsed.getName());
+ assertFalse(SnapshotDiffValueParser.parseKeyInfoRequiredFields(rawData,
false).hasUpdateId());
+
+ OmKeyInfo metadataChanged = createKeyInfo(100L, 200L, 1024L,
createChecksum((byte) 1),
+ createMetadata("meta", "two"), createTags("tag", "one"), createAcls(),
+ Collections.singletonList(createKeyLocationGroup(1L)));
+ byte[] metadataRaw =
OmKeyInfo.getCodec().toPersistedFormat(metadataChanged);
+
assertFalse(Arrays.equals(SnapshotDiffValueParser.computeKeyInfoCompareSignature(rawData),
+ SnapshotDiffValueParser.computeKeyInfoCompareSignature(metadataRaw)));
+
+ Map<String, String> hsyncMetadata = createMetadata("meta", "one");
+ hsyncMetadata.put(OzoneConsts.HSYNC_CLIENT_ID, "client1");
+ OmKeyInfo hsyncChanged = createKeyInfo(100L, 200L, 1024L,
createChecksum((byte) 1),
+ hsyncMetadata, createTags("tag", "one"), createAcls(),
+ Collections.singletonList(createKeyLocationGroup(1L)));
+ byte[] hsyncRaw = OmKeyInfo.getCodec().toPersistedFormat(hsyncChanged);
+
assertFalse(Arrays.equals(SnapshotDiffValueParser.computeKeyInfoCompareSignature(rawData),
+ SnapshotDiffValueParser.computeKeyInfoCompareSignature(hsyncRaw)));
+
+ OmKeyInfo tagsChanged = createKeyInfo(100L, 200L, 1024L,
createChecksum((byte) 1),
+ createMetadata("meta", "one"), createTags("tag", "two"), createAcls(),
+ Collections.singletonList(createKeyLocationGroup(1L)));
+ byte[] tagsRaw = OmKeyInfo.getCodec().toPersistedFormat(tagsChanged);
+
assertFalse(Arrays.equals(SnapshotDiffValueParser.computeKeyInfoCompareSignature(rawData),
+ SnapshotDiffValueParser.computeKeyInfoCompareSignature(tagsRaw)));
+
+ OmKeyInfo aclsChanged = createKeyInfo(100L, 200L, 1024L,
createChecksum((byte) 1),
+ createMetadata("meta", "one"), createTags("tag", "one"),
createAcls("user:other:rw"),
+ Collections.singletonList(createKeyLocationGroup(1L)));
+ byte[] aclsRaw = OmKeyInfo.getCodec().toPersistedFormat(aclsChanged);
+
assertFalse(Arrays.equals(SnapshotDiffValueParser.computeKeyInfoCompareSignature(rawData),
+ SnapshotDiffValueParser.computeKeyInfoCompareSignature(aclsRaw)));
+
+ OmKeyInfo checksumChanged = createKeyInfo(100L, 200L, 1024L,
createChecksum((byte) 2),
+ createMetadata("meta", "one"), createTags("tag", "one"), createAcls(),
+ Collections.singletonList(createKeyLocationGroup(1L)));
+ byte[] checksumRaw =
OmKeyInfo.getCodec().toPersistedFormat(checksumChanged);
+
assertFalse(Arrays.equals(SnapshotDiffValueParser.computeKeyInfoCompareSignature(rawData),
+ SnapshotDiffValueParser.computeKeyInfoCompareSignature(checksumRaw)));
+
+ OmKeyInfo dataSizeChanged = createKeyInfo(100L, 200L, 2048L,
createChecksum((byte) 1),
+ createMetadata("meta", "one"), createTags("tag", "one"), createAcls(),
+ Collections.singletonList(createKeyLocationGroup(1L)));
+ byte[] dataSizeRaw =
OmKeyInfo.getCodec().toPersistedFormat(dataSizeChanged);
+
assertFalse(Arrays.equals(SnapshotDiffValueParser.computeKeyInfoCompareSignature(rawData),
+ SnapshotDiffValueParser.computeKeyInfoCompareSignature(dataSizeRaw)));
+
+ OmKeyInfo locationCountChanged = createKeyInfo(100L, 200L, 1024L,
createChecksum((byte) 1),
+ createMetadata("meta", "one"), createTags("tag", "one"), createAcls(),
+ createKeyLocationGroups(1L, 2L));
+ byte[] locationCountRaw =
OmKeyInfo.getCodec().toPersistedFormat(locationCountChanged);
+
assertFalse(Arrays.equals(SnapshotDiffValueParser.computeKeyInfoCompareSignature(rawData),
+
SnapshotDiffValueParser.computeKeyInfoCompareSignature(locationCountRaw)));
+
+ OmKeyInfo latestLocationChanged = createKeyInfo(100L, 200L, 1024L,
createChecksum((byte) 1),
+ createMetadata("meta", "one"), createTags("tag", "one"), createAcls(),
+ createKeyLocationGroups(1L, 3L));
+ byte[] latestLocationRaw =
OmKeyInfo.getCodec().toPersistedFormat(latestLocationChanged);
+
assertFalse(Arrays.equals(SnapshotDiffValueParser.computeKeyInfoCompareSignature(locationCountRaw),
+
SnapshotDiffValueParser.computeKeyInfoCompareSignature(latestLocationRaw)));
+ }
+
+ @Test
+ void testKeyInfoIgnoresVolatileTimes() throws Exception {
+ OmKeyInfo keyInfo = createKeyInfo(100L, 200L, 1024L, createChecksum((byte)
1),
+ createMetadata("meta", "one"), createTags("tag", "one"), createAcls(),
+ Collections.singletonList(createKeyLocationGroup(1L)));
+ OmKeyInfo timeChanged = createKeyInfo(110L, 220L, 1024L,
createChecksum((byte) 1),
+ createMetadata("meta", "one"), createTags("tag", "one"), createAcls(),
+ Collections.singletonList(createKeyLocationGroup(1L)));
+
+ byte[] rawData = OmKeyInfo.getCodec().toPersistedFormat(keyInfo);
+ byte[] rawTimeChanged =
OmKeyInfo.getCodec().toPersistedFormat(timeChanged);
+
+ assertArrayEquals(
+ SnapshotDiffValueParser.computeKeyInfoCompareSignature(rawData),
+
SnapshotDiffValueParser.computeKeyInfoCompareSignature(rawTimeChanged));
+ }
+
+ @Test
+ void testDirectoryInfoSignatureAndParsing() throws Exception {
+ OmDirectoryInfo dirInfo = createDirectoryInfo(100L, 200L,
createMetadata("meta", "one"), createAcls());
+ byte[] rawData = OmDirectoryInfo.getCodec().toPersistedFormat(dirInfo);
+ SnapshotDiffValueParser.ParsedRequiredInfo parsed =
+ SnapshotDiffValueParser.parseDirectoryInfoRequiredFields(rawData,
true);
+ assertEquals(UPDATE_ID, parsed.getUpdateId());
+ assertEquals(OBJECT_ID, parsed.getObjectId());
+ assertEquals(PARENT_ID, parsed.getParentId());
+ assertEquals(DIR_NAME, parsed.getName());
+
assertFalse(SnapshotDiffValueParser.parseDirectoryInfoRequiredFields(rawData,
false).hasUpdateId());
+
+ OmDirectoryInfo metadataChanged = createDirectoryInfo(100L, 200L,
createMetadata("meta", "two"), createAcls());
+ byte[] rawMetadataChanged =
OmDirectoryInfo.getCodec().toPersistedFormat(metadataChanged);
+
assertFalse(Arrays.equals(SnapshotDiffValueParser.computeDirectoryInfoCompareSignature(rawData),
+
SnapshotDiffValueParser.computeDirectoryInfoCompareSignature(rawMetadataChanged)));
+
+ OmDirectoryInfo timeChanged = createDirectoryInfo(110L, 220L,
createMetadata("meta", "one"), createAcls());
+ byte[] rawTimeChanged =
OmDirectoryInfo.getCodec().toPersistedFormat(timeChanged);
+ assertArrayEquals(
+ SnapshotDiffValueParser.computeDirectoryInfoCompareSignature(rawData),
+
SnapshotDiffValueParser.computeDirectoryInfoCompareSignature(rawTimeChanged));
+ }
+
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ private static OmKeyInfo createKeyInfo(long creationTime, long
modificationTime, long dataSize, FileChecksum checksum,
+ Map<String, String> metadata, Map<String, String> tags, List<OzoneAcl>
acls,
+ List<OmKeyLocationInfoGroup> keyLocationGroups) {
+ return new OmKeyInfo.Builder()
+ .setVolumeName(VOLUME)
+ .setBucketName(BUCKET)
+ .setKeyName(KEY_NAME)
+ .setCreationTime(creationTime)
+ .setModificationTime(modificationTime)
+
.setReplicationConfig(RatisReplicationConfig.getInstance(ReplicationFactor.ONE))
+ .setObjectID(OBJECT_ID)
+ .setParentObjectID(PARENT_ID)
+ .setUpdateID(UPDATE_ID)
+ .setDataSize(dataSize)
+ .setOmKeyLocationInfos(keyLocationGroups)
+ .setFileChecksum(checksum)
+ .addAllMetadata(metadata)
+ .setTags(tags)
+ .setAcls(acls)
+ .build();
+ }
+
+ private static OmDirectoryInfo createDirectoryInfo(long creationTime, long
modificationTime,
+ Map<String, String> metadata, List<OzoneAcl> acls) {
+ return OmDirectoryInfo.newBuilder()
+ .setName(DIR_NAME)
+ .setCreationTime(creationTime)
+ .setModificationTime(modificationTime)
+ .setObjectID(OBJECT_ID)
+ .setParentObjectID(PARENT_ID)
+ .setUpdateID(UPDATE_ID)
+ .addAllMetadata(metadata)
+ .setAcls(acls)
+ .build();
+ }
+
+ private static OmKeyLocationInfoGroup createKeyLocationGroup(long blockId) {
+ OmKeyLocationInfo location = new OmKeyLocationInfo.Builder()
+ .setBlockID(new BlockID(blockId, blockId))
+ .build();
+ return new OmKeyLocationInfoGroup(0, Collections.singletonList(location));
+ }
+
+ private static List<OmKeyLocationInfoGroup> createKeyLocationGroups(long...
blockIds) {
+ List<OmKeyLocationInfoGroup> groups = new java.util.ArrayList<>();
+ int version = 0;
+ for (long blockId : blockIds) {
+ OmKeyLocationInfo location = new OmKeyLocationInfo.Builder()
+ .setBlockID(new BlockID(blockId, blockId))
+ .build();
+ groups.add(new OmKeyLocationInfoGroup(version++,
Collections.singletonList(location)));
+ }
+ return groups;
+ }
+
+ private static FileChecksum createChecksum(byte value) {
+ byte[] bytes = new byte[32];
+ Arrays.fill(bytes, value);
+ MD5Hash fileMd5 = MD5Hash.digest(bytes);
+ return new MD5MD5CRC32GzipFileChecksum(0, 0, fileMd5);
+ }
+
+ private static Map<String, String> createMetadata(String key, String value) {
+ Map<String, String> metadata = new LinkedHashMap<>();
+ metadata.put(key, value);
+ return metadata;
+ }
+
+ private static Map<String, String> createTags(String key, String value) {
+ Map<String, String> tags = new LinkedHashMap<>();
+ tags.put(key, value);
+ return tags;
+ }
+
+ private static List<OzoneAcl> createAcls() {
+ return Collections.singletonList(OzoneAcl.parseAcl("user:test:rw"));
+ }
+
+ private static List<OzoneAcl> createAcls(String acl) {
+ return Collections.singletonList(OzoneAcl.parseAcl(acl));
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
