siddharthteotia commented on a change in pull request #6710:
URL: https://github.com/apache/incubator-pinot/pull/6710#discussion_r603692307
##########
File path:
pinot-core/src/main/java/org/apache/pinot/core/common/datatable/DataTableUtils.java
##########
@@ -233,4 +244,170 @@ private static DataTable
buildEmptyDataTableForDistinctQuery(QueryContext queryC
dataTableBuilder.finishRow();
return dataTableBuilder.build();
}
+
+ /**
+ * Helper method to decode string.
+ */
+ public static String decodeString(DataInputStream dataInputStream)
+ throws IOException {
+ int length = dataInputStream.readInt();
+ if (length == 0) {
+ return StringUtils.EMPTY;
+ } else {
+ byte[] buffer = new byte[length];
+ int numBytesRead = dataInputStream.read(buffer);
+ assert numBytesRead == length;
+ return StringUtil.decodeUtf8(buffer);
+ }
+ }
+
+ /**
+ * Helper method to decode int.
+ */
+ public static int decodeInt(DataInputStream dataInputStream)
+ throws IOException {
+ int length = Integer.BYTES;
+ byte[] buffer = new byte[length];
+ int numBytesRead = dataInputStream.read(buffer);
+ assert numBytesRead == length;
+ return Ints.fromByteArray(buffer);
+ }
+
+ /**
+ * Helper method to decode long.
+ */
+ public static long decodeLong(DataInputStream dataInputStream)
+ throws IOException {
+ int length = Long.BYTES;
+ byte[] buffer = new byte[length];
+ int numBytesRead = dataInputStream.read(buffer);
+ assert numBytesRead == length;
+ return Longs.fromByteArray(buffer);
+ }
+
+ /**
+ * Helper method to serialize dictionary map.
+ */
+ public static byte[] serializeDictionaryMap(Map<String, Map<Integer,
String>> dictionaryMap)
+ throws IOException {
+ ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+ DataOutputStream dataOutputStream = new
DataOutputStream(byteArrayOutputStream);
+
+ dataOutputStream.writeInt(dictionaryMap.size());
+ for (Map.Entry<String, Map<Integer, String>> dictionaryMapEntry :
dictionaryMap.entrySet()) {
+ String columnName = dictionaryMapEntry.getKey();
+ Map<Integer, String> dictionary = dictionaryMapEntry.getValue();
+ byte[] bytes = StringUtil.encodeUtf8(columnName);
+ dataOutputStream.writeInt(bytes.length);
+ dataOutputStream.write(bytes);
+ dataOutputStream.writeInt(dictionary.size());
+
+ for (Map.Entry<Integer, String> dictionaryEntry : dictionary.entrySet())
{
+ dataOutputStream.writeInt(dictionaryEntry.getKey());
+ byte[] valueBytes = StringUtil.encodeUtf8(dictionaryEntry.getValue());
+ dataOutputStream.writeInt(valueBytes.length);
+ dataOutputStream.write(valueBytes);
+ }
+ }
+
+ return byteArrayOutputStream.toByteArray();
+ }
+
+ /**
+ * Helper method to deserialize dictionary map.
+ */
+ public static Map<String, Map<Integer, String>>
deserializeDictionaryMap(byte[] bytes)
+ throws IOException {
+ try (ByteArrayInputStream byteArrayInputStream = new
ByteArrayInputStream(bytes);
+ DataInputStream dataInputStream = new
DataInputStream(byteArrayInputStream)) {
+ int numDictionaries = dataInputStream.readInt();
+ Map<String, Map<Integer, String>> dictionaryMap = new
HashMap<>(numDictionaries);
+
+ for (int i = 0; i < numDictionaries; i++) {
+ String column = decodeString(dataInputStream);
+ int dictionarySize = dataInputStream.readInt();
+ Map<Integer, String> dictionary = new HashMap<>(dictionarySize);
+ for (int j = 0; j < dictionarySize; j++) {
+ int key = dataInputStream.readInt();
+ String value = decodeString(dataInputStream);
+ dictionary.put(key, value);
+ }
+ dictionaryMap.put(column, dictionary);
+ }
+
+ return dictionaryMap;
+ }
+ }
+
+ /**
Review comment:
We don't need to do this conversion. The factory class should read the
version as the first 4 bytes from the incoming ByteBuffer and accordingly
create DataTableImplV2 or DataTableImplV3
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
