This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new cfe95f4ba4 Add id to MetadataKey so that we can change the key order
(#9159)
cfe95f4ba4 is described below
commit cfe95f4ba49a1dc95b18bddc744ecde731a7e6a4
Author: Xiaotian (Jackie) Jiang <[email protected]>
AuthorDate: Mon Aug 8 13:53:56 2022 -0700
Add id to MetadataKey so that we can change the key order (#9159)
---
.../org/apache/pinot/common/utils/DataTable.java | 116 +++++++++++++--------
.../pinot/core/common/datablock/BaseDataBlock.java | 4 +-
.../core/common/datatable/DataTableImplV3.java | 4 +-
.../core/common/datatable/DataTableSerDeTest.java | 2 +-
4 files changed, 76 insertions(+), 50 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/utils/DataTable.java
b/pinot-common/src/main/java/org/apache/pinot/common/utils/DataTable.java
index df0c021c5a..90d0a1128f 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/utils/DataTable.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/DataTable.java
@@ -18,9 +18,9 @@
*/
package org.apache.pinot.common.utils;
+import com.google.common.base.Preconditions;
import java.io.IOException;
import java.math.BigDecimal;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import javax.annotation.Nullable;
@@ -90,62 +90,81 @@ public interface DataTable {
INT, LONG, STRING
}
- /* The MetadataKey is used in V3, where we present metadata as
Map<MetadataKey, String>
+ /* The MetadataKey is used since V3, where we present metadata as
Map<MetadataKey, String>
* ATTENTION:
- * - Don't change existing keys.
- * - Don't remove existing keys.
- * - Always add new keys to the end.
+ * - DO NOT change the id for the existing keys
+ * - To add a new key, add it with the current MAX_ID, and increase MAX_ID
+ * - NEVER decrease MAX_ID
* Otherwise, backward compatibility will be broken.
*/
enum MetadataKey {
- UNKNOWN("unknown", MetadataValueType.STRING),
- TABLE("table", MetadataValueType.STRING), // NOTE: this key is only used
in PrioritySchedulerTest
- NUM_DOCS_SCANNED("numDocsScanned", MetadataValueType.LONG),
- NUM_ENTRIES_SCANNED_IN_FILTER("numEntriesScannedInFilter",
MetadataValueType.LONG),
- NUM_ENTRIES_SCANNED_POST_FILTER("numEntriesScannedPostFilter",
MetadataValueType.LONG),
- NUM_SEGMENTS_QUERIED("numSegmentsQueried", MetadataValueType.INT),
- NUM_SEGMENTS_PROCESSED("numSegmentsProcessed", MetadataValueType.INT),
- NUM_SEGMENTS_MATCHED("numSegmentsMatched", MetadataValueType.INT),
- NUM_CONSUMING_SEGMENTS_QUERIED("numConsumingSegmentsQueried",
MetadataValueType.INT),
- MIN_CONSUMING_FRESHNESS_TIME_MS("minConsumingFreshnessTimeMs",
MetadataValueType.LONG),
- TOTAL_DOCS("totalDocs", MetadataValueType.LONG),
- NUM_GROUPS_LIMIT_REACHED("numGroupsLimitReached",
MetadataValueType.STRING),
- TIME_USED_MS("timeUsedMs", MetadataValueType.LONG),
- TRACE_INFO("traceInfo", MetadataValueType.STRING),
- REQUEST_ID("requestId", MetadataValueType.LONG),
- NUM_RESIZES("numResizes", MetadataValueType.INT),
- RESIZE_TIME_MS("resizeTimeMs", MetadataValueType.LONG),
- THREAD_CPU_TIME_NS("threadCpuTimeNs", MetadataValueType.LONG),
- SYSTEM_ACTIVITIES_CPU_TIME_NS("systemActivitiesCpuTimeNs",
MetadataValueType.LONG),
- RESPONSE_SER_CPU_TIME_NS("responseSerializationCpuTimeNs",
MetadataValueType.LONG),
- NUM_SEGMENTS_PRUNED_BY_SERVER("numSegmentsPrunedByServer",
MetadataValueType.INT),
- NUM_SEGMENTS_PRUNED_INVALID("numSegmentsPrunedByInvalid",
MetadataValueType.INT),
- NUM_SEGMENTS_PRUNED_BY_LIMIT("numSegmentsPrunedByLimit",
MetadataValueType.INT),
- NUM_SEGMENTS_PRUNED_BY_VALUE("numSegmentsPrunedByValue",
MetadataValueType.INT),
-
EXPLAIN_PLAN_NUM_EMPTY_FILTER_SEGMENTS("explainPlanNumEmptyFilterSegments",
MetadataValueType.INT),
-
EXPLAIN_PLAN_NUM_MATCH_ALL_FILTER_SEGMENTS("explainPlanNumMatchAllFilterSegments",
MetadataValueType.INT),
- NUM_CONSUMING_SEGMENTS_PROCESSED("numConsumingSegmentsProcessed",
MetadataValueType.INT),
- NUM_CONSUMING_SEGMENTS_MATCHED("numConsumingSegmentsMatched",
MetadataValueType.INT);
-
- private static final MetadataKey[] VALUES;
+ UNKNOWN(0, "unknown", MetadataValueType.STRING),
+ TABLE(1, "table", MetadataValueType.STRING), // NOTE: this key is only
used in PrioritySchedulerTest
+ NUM_DOCS_SCANNED(2, "numDocsScanned", MetadataValueType.LONG),
+ NUM_ENTRIES_SCANNED_IN_FILTER(3, "numEntriesScannedInFilter",
MetadataValueType.LONG),
+ NUM_ENTRIES_SCANNED_POST_FILTER(4, "numEntriesScannedPostFilter",
MetadataValueType.LONG),
+ NUM_SEGMENTS_QUERIED(5, "numSegmentsQueried", MetadataValueType.INT),
+ NUM_SEGMENTS_PROCESSED(6, "numSegmentsProcessed", MetadataValueType.INT),
+ NUM_SEGMENTS_MATCHED(7, "numSegmentsMatched", MetadataValueType.INT),
+ NUM_CONSUMING_SEGMENTS_QUERIED(8, "numConsumingSegmentsQueried",
MetadataValueType.INT),
+ NUM_CONSUMING_SEGMENTS_PROCESSED(26, "numConsumingSegmentsProcessed",
MetadataValueType.INT),
+ NUM_CONSUMING_SEGMENTS_MATCHED(27, "numConsumingSegmentsMatched",
MetadataValueType.INT),
+ MIN_CONSUMING_FRESHNESS_TIME_MS(9, "minConsumingFreshnessTimeMs",
MetadataValueType.LONG),
+ TOTAL_DOCS(10, "totalDocs", MetadataValueType.LONG),
+ NUM_GROUPS_LIMIT_REACHED(11, "numGroupsLimitReached",
MetadataValueType.STRING),
+ TIME_USED_MS(12, "timeUsedMs", MetadataValueType.LONG),
+ TRACE_INFO(13, "traceInfo", MetadataValueType.STRING),
+ REQUEST_ID(14, "requestId", MetadataValueType.LONG),
+ NUM_RESIZES(15, "numResizes", MetadataValueType.INT),
+ RESIZE_TIME_MS(16, "resizeTimeMs", MetadataValueType.LONG),
+ THREAD_CPU_TIME_NS(17, "threadCpuTimeNs", MetadataValueType.LONG),
+ SYSTEM_ACTIVITIES_CPU_TIME_NS(18, "systemActivitiesCpuTimeNs",
MetadataValueType.LONG),
+ RESPONSE_SER_CPU_TIME_NS(19, "responseSerializationCpuTimeNs",
MetadataValueType.LONG),
+ NUM_SEGMENTS_PRUNED_BY_SERVER(20, "numSegmentsPrunedByServer",
MetadataValueType.INT),
+ NUM_SEGMENTS_PRUNED_INVALID(21, "numSegmentsPrunedByInvalid",
MetadataValueType.INT),
+ NUM_SEGMENTS_PRUNED_BY_LIMIT(22, "numSegmentsPrunedByLimit",
MetadataValueType.INT),
+ NUM_SEGMENTS_PRUNED_BY_VALUE(23, "numSegmentsPrunedByValue",
MetadataValueType.INT),
+ EXPLAIN_PLAN_NUM_EMPTY_FILTER_SEGMENTS(24,
"explainPlanNumEmptyFilterSegments", MetadataValueType.INT),
+ EXPLAIN_PLAN_NUM_MATCH_ALL_FILTER_SEGMENTS(25,
"explainPlanNumMatchAllFilterSegments", MetadataValueType.INT);
+
+ // We keep this constant to track the max id added so far for backward
compatibility.
+ // Increase it when adding new keys, but NEVER DECREASE IT!!!
+ private static final int MAX_ID = 27;
+
+ private static final MetadataKey[] ID_TO_ENUM_KEY_MAP = new
MetadataKey[MAX_ID + 1];
private static final Map<String, MetadataKey> NAME_TO_ENUM_KEY_MAP = new
HashMap<>();
+
+ private final int _id;
private final String _name;
private final MetadataValueType _valueType;
- MetadataKey(String name, MetadataValueType valueType) {
+ MetadataKey(int id, String name, MetadataValueType valueType) {
+ _id = id;
_name = name;
_valueType = valueType;
}
- // getByOrdinal returns an enum key for a given ordinal or null if the key
does not exist.
+ /**
+ * Returns the MetadataKey for the given id, or {@code null} if the id
does not exist.
+ */
@Nullable
- public static MetadataKey getByOrdinal(int ordinal) {
- return VALUES[Math.min(ordinal, VALUES.length - 1)];
+ public static MetadataKey getById(int id) {
+ if (id >= 0 && id < ID_TO_ENUM_KEY_MAP.length) {
+ return ID_TO_ENUM_KEY_MAP[id];
+ }
+ return null;
}
- // getByName returns an enum key for a given name or null if the key does
not exist.
+ /**
+ * Returns the MetadataKey for the given name, or {@code null} if the name
does not exist.
+ */
+ @Nullable
public static MetadataKey getByName(String name) {
- return NAME_TO_ENUM_KEY_MAP.getOrDefault(name, null);
+ return NAME_TO_ENUM_KEY_MAP.get(name);
+ }
+
+ public int getId() {
+ return _id;
}
// getName returns the associated name(string) of the enum key.
@@ -161,11 +180,18 @@ public interface DataTable {
static {
MetadataKey[] values = values();
for (MetadataKey key : values) {
- if (NAME_TO_ENUM_KEY_MAP.put(key.getName(), key) != null) {
- throw new IllegalArgumentException("Duplicate name defined in the
MetadataKey definition: " + key.getName());
- }
+ int id = key.getId();
+ Preconditions.checkArgument(id >= 0 && id <= MAX_ID,
+ "Invalid id: %s for MetadataKey: %s, must be in the range of [0,
MAX_ID(%s)]", id, key, MAX_ID);
+ Preconditions.checkArgument(ID_TO_ENUM_KEY_MAP[id] == null,
+ "Duplicate id: %s defined for MetadataKey: %s and %s", id,
ID_TO_ENUM_KEY_MAP[id], key);
+ ID_TO_ENUM_KEY_MAP[id] = key;
+
+ String name = key.getName();
+ MetadataKey oldKey = NAME_TO_ENUM_KEY_MAP.put(name, key);
+ Preconditions.checkArgument(oldKey == null, "Duplicate name: %s
defined for MetadataKey: %s and %s", name,
+ oldKey, key);
}
- VALUES = Arrays.copyOf(values, values.length + 1);
}
}
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/common/datablock/BaseDataBlock.java
b/pinot-core/src/main/java/org/apache/pinot/core/common/datablock/BaseDataBlock.java
index 395bc49d05..74d42f6628 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/common/datablock/BaseDataBlock.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/common/datablock/BaseDataBlock.java
@@ -539,7 +539,7 @@ public abstract class BaseDataBlock implements DataTable {
continue;
}
String value = entry.getValue();
- dataOutputStream.writeInt(key.ordinal());
+ dataOutputStream.writeInt(key.getId());
if (key.getValueType() == MetadataValueType.INT) {
dataOutputStream.write(Ints.toByteArray(Integer.parseInt(value)));
} else if (key.getValueType() == MetadataValueType.LONG) {
@@ -570,7 +570,7 @@ public abstract class BaseDataBlock implements DataTable {
Map<String, String> metadata = new HashMap<>();
for (int i = 0; i < numEntries; i++) {
int keyId = buffer.getInt();
- MetadataKey key = MetadataKey.getByOrdinal(keyId);
+ MetadataKey key = MetadataKey.getById(keyId);
// Ignore unknown keys.
if (key == null) {
continue;
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/common/datatable/DataTableImplV3.java
b/pinot-core/src/main/java/org/apache/pinot/core/common/datatable/DataTableImplV3.java
index 433167808a..0e214f3a47 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/common/datatable/DataTableImplV3.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/common/datatable/DataTableImplV3.java
@@ -322,7 +322,7 @@ public class DataTableImplV3 extends BaseDataTable {
continue;
}
String value = entry.getValue();
- dataOutputStream.writeInt(key.ordinal());
+ dataOutputStream.writeInt(key.getId());
if (key.getValueType() == MetadataValueType.INT) {
dataOutputStream.write(Ints.toByteArray(Integer.parseInt(value)));
} else if (key.getValueType() == MetadataValueType.LONG) {
@@ -353,7 +353,7 @@ public class DataTableImplV3 extends BaseDataTable {
Map<String, String> metadata = new HashMap<>();
for (int i = 0; i < numEntries; i++) {
int keyId = buffer.getInt();
- MetadataKey key = MetadataKey.getByOrdinal(keyId);
+ MetadataKey key = MetadataKey.getById(keyId);
// Ignore unknown keys.
if (key == null) {
continue;
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/common/datatable/DataTableSerDeTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/common/datatable/DataTableSerDeTest.java
index e0750eb9a0..cbb3c34992 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/common/datatable/DataTableSerDeTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/common/datatable/DataTableSerDeTest.java
@@ -513,7 +513,7 @@ public class DataTableSerDeTest {
Assert.assertEquals(numEntries, EXPECTED_METADATA.size());
for (int i = 0; i < numEntries; i++) {
int keyOrdinal = dataInputStream.readInt();
- DataTable.MetadataKey key = MetadataKey.getByOrdinal(keyOrdinal);
+ DataTable.MetadataKey key = MetadataKey.getById(keyOrdinal);
Assert.assertNotEquals(key, null);
if (key.getValueType() == DataTable.MetadataValueType.INT) {
byte[] actualBytes = new byte[Integer.BYTES];
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
