siddharthteotia commented on a change in pull request #6710:
URL: https://github.com/apache/incubator-pinot/pull/6710#discussion_r606092101
##########
File path:
pinot-core/src/test/java/org/apache/pinot/core/common/datatable/DataTableSerDeTest.java
##########
@@ -96,22 +131,267 @@ public void testAllDataTypes()
for (int i = 0; i < numColumns; i++) {
columnNames[i] = columnDataTypes[i].name();
}
+
+ int[] ints = new int[NUM_ROWS];
+ long[] longs = new long[NUM_ROWS];
+ float[] floats = new float[NUM_ROWS];
+ double[] doubles = new double[NUM_ROWS];
+ String[] strings = new String[NUM_ROWS];
+ byte[][] bytes = new byte[NUM_ROWS][];
+ Object[] objects = new Object[NUM_ROWS];
+ int[][] intArrays = new int[NUM_ROWS][];
+ long[][] longArrays = new long[NUM_ROWS][];
+ float[][] floatArrays = new float[NUM_ROWS][];
+ double[][] doubleArrays = new double[NUM_ROWS][];
+ String[][] stringArrays = new String[NUM_ROWS][];
+
DataSchema dataSchema = new DataSchema(columnNames, columnDataTypes);
+ DataTableBuilder dataTableBuilder = new DataTableBuilder(dataSchema);
+ fillDataTableWithRandomData(dataTableBuilder, columnDataTypes, numColumns,
ints, longs, floats, doubles, strings,
+ bytes, objects, intArrays, longArrays, floatArrays, doubleArrays,
stringArrays);
+
+ DataTable dataTable = dataTableBuilder.build();
+ DataTable newDataTable =
DataTableFactory.getDataTable(dataTable.toBytes());
+ Assert.assertEquals(newDataTable.getDataSchema(), dataSchema,
ERROR_MESSAGE);
+ Assert.assertEquals(newDataTable.getNumberOfRows(), NUM_ROWS,
ERROR_MESSAGE);
+ verifyDataIsSame(newDataTable, columnDataTypes, numColumns, ints, longs,
floats, doubles, strings, bytes, objects,
+ intArrays, longArrays, floatArrays, doubleArrays, stringArrays);
+ }
+
+ @Test
+ public void testV2V3Compatibility()
+ throws IOException {
+ DataSchema.ColumnDataType[] columnDataTypes =
DataSchema.ColumnDataType.values();
+ int numColumns = columnDataTypes.length;
+ String[] columnNames = new String[numColumns];
+ for (int i = 0; i < numColumns; i++) {
+ columnNames[i] = columnDataTypes[i].name();
+ }
+
+ int[] ints = new int[NUM_ROWS];
+ long[] longs = new long[NUM_ROWS];
+ float[] floats = new float[NUM_ROWS];
+ double[] doubles = new double[NUM_ROWS];
+ String[] strings = new String[NUM_ROWS];
+ byte[][] bytes = new byte[NUM_ROWS][];
+ Object[] objects = new Object[NUM_ROWS];
+ int[][] intArrays = new int[NUM_ROWS][];
+ long[][] longArrays = new long[NUM_ROWS][];
+ float[][] floatArrays = new float[NUM_ROWS][];
+ double[][] doubleArrays = new double[NUM_ROWS][];
+ String[][] stringArrays = new String[NUM_ROWS][];
+
+ DataSchema dataSchema = new DataSchema(columnNames, columnDataTypes);
+
+ // Verify V3 broker can deserialize data table (has data, but has no
metadata) send by V2 server
+ DataTableBuilder.setCurrentDataTableVersion(DataTableBuilder.VERSION_2);
+ DataTableBuilder dataTableBuilderV2WithDataOnly = new
DataTableBuilder(dataSchema);
+ fillDataTableWithRandomData(dataTableBuilderV2WithDataOnly,
columnDataTypes, numColumns, ints, longs, floats,
+ doubles, strings, bytes, objects, intArrays, longArrays, floatArrays,
doubleArrays, stringArrays);
+
+ DataTable dataTableV2 = dataTableBuilderV2WithDataOnly.build(); // create
a V2 data table
+ DataTable newDataTable =
+ DataTableFactory.getDataTable(dataTableV2.toBytes()); // Broker
deserialize data table bytes as V2
+ Assert.assertEquals(newDataTable.getDataSchema(), dataSchema,
ERROR_MESSAGE);
+ Assert.assertEquals(newDataTable.getNumberOfRows(), NUM_ROWS,
ERROR_MESSAGE);
+ verifyDataIsSame(newDataTable, columnDataTypes, numColumns, ints, longs,
floats, doubles, strings, bytes, objects,
+ intArrays, longArrays, floatArrays, doubleArrays, stringArrays);
+ Assert.assertEquals(newDataTable.getMetadata().size(), 0);
+
+ // Verify V3 broker can deserialize data table (has data and metadata)
send by V2 server
+ for (String key : EXPECTED_METADATA.keySet()) {
+ dataTableV2.getMetadata().put(key, EXPECTED_METADATA.get(key));
+ }
+ newDataTable = DataTableFactory.getDataTable(dataTableV2.toBytes()); //
Broker deserialize data table bytes as V2
+ Assert.assertEquals(newDataTable.getDataSchema(), dataSchema,
ERROR_MESSAGE);
+ Assert.assertEquals(newDataTable.getNumberOfRows(), NUM_ROWS,
ERROR_MESSAGE);
+ verifyDataIsSame(newDataTable, columnDataTypes, numColumns, ints, longs,
floats, doubles, strings, bytes, objects,
+ intArrays, longArrays, floatArrays, doubleArrays, stringArrays);
+ Assert.assertEquals(newDataTable.getMetadata(), EXPECTED_METADATA);
+
+ // Verify V3 broker can deserialize data table (only has metadata) send by
V2 server
+ DataTableBuilder dataTableBuilderV2WithMetadataDataOnly = new
DataTableBuilder(dataSchema);
+ dataTableV2 = dataTableBuilderV2WithMetadataDataOnly.build(); // create a
V2 data table
+ for (String key : EXPECTED_METADATA.keySet()) {
+ dataTableV2.getMetadata().put(key, EXPECTED_METADATA.get(key));
+ }
+ newDataTable = DataTableFactory.getDataTable(dataTableV2.toBytes()); //
Broker deserialize data table bytes as V2
+ Assert.assertEquals(newDataTable.getDataSchema(), dataSchema,
ERROR_MESSAGE);
+ Assert.assertEquals(newDataTable.getNumberOfRows(), 0, 0);
+ Assert.assertEquals(newDataTable.getMetadata(), EXPECTED_METADATA);
+
+ // Verify V3 broker can deserialize (has data, but has no metadata) send
by V3 server.
+ DataTableBuilder.setCurrentDataTableVersion(VERSION_3);
+ DataTableBuilder dataTableBuilderV3WithDataOnly = new
DataTableBuilder(dataSchema);
+ fillDataTableWithRandomData(dataTableBuilderV3WithDataOnly,
columnDataTypes, numColumns, ints, longs, floats,
+ doubles, strings, bytes, objects, intArrays, longArrays, floatArrays,
doubleArrays, stringArrays);
+ DataTable dataTableV3 = dataTableBuilderV3WithDataOnly.build(); // create
a V3 data table
+ // Deserialize data table bytes as V3
+ newDataTable = DataTableFactory.getDataTable(dataTableV3.toBytes());
+ Assert.assertEquals(newDataTable.getDataSchema(), dataSchema,
ERROR_MESSAGE);
+ Assert.assertEquals(newDataTable.getNumberOfRows(), NUM_ROWS,
ERROR_MESSAGE);
+ verifyDataIsSame(newDataTable, columnDataTypes, numColumns, ints, longs,
floats, doubles, strings, bytes, objects,
+ intArrays, longArrays, floatArrays, doubleArrays, stringArrays);
+ // DataTable V3 serialization logic will add an extra THREAD_CPU_TIME_NS
KV pair into metadata
+ Assert.assertEquals(newDataTable.getMetadata().size(), 1);
+
Assert.assertTrue(newDataTable.getMetadata().containsKey(THREAD_CPU_TIME_NS.getName()));
+
+ // Verify V3 broker can deserialize data table (has data and metadata)
send by V3 server
+ for (String key : EXPECTED_METADATA.keySet()) {
+ dataTableV3.getMetadata().put(key, EXPECTED_METADATA.get(key));
+ }
+ newDataTable = DataTableFactory.getDataTable(dataTableV3.toBytes()); //
Broker deserialize data table bytes as V3
+ Assert.assertEquals(newDataTable.getDataSchema(), dataSchema,
ERROR_MESSAGE);
+ Assert.assertEquals(newDataTable.getNumberOfRows(), NUM_ROWS,
ERROR_MESSAGE);
+ verifyDataIsSame(newDataTable, columnDataTypes, numColumns, ints, longs,
floats, doubles, strings, bytes, objects,
+ intArrays, longArrays, floatArrays, doubleArrays, stringArrays);
+ newDataTable.getMetadata().remove(THREAD_CPU_TIME_NS.getName());
+ Assert.assertEquals(newDataTable.getMetadata(), EXPECTED_METADATA);
+ // Verify V3 broker can deserialize data table (only has metadata) send by
V3 server
+ DataTableBuilder dataTableBuilderV3WithMetadataDataOnly = new
DataTableBuilder(dataSchema);
+ dataTableV3 = dataTableBuilderV3WithMetadataDataOnly.build(); // create a
V2 data table
+ for (String key : EXPECTED_METADATA.keySet()) {
+ dataTableV3.getMetadata().put(key, EXPECTED_METADATA.get(key));
+ }
+ newDataTable = DataTableFactory.getDataTable(dataTableV3.toBytes()); //
Broker deserialize data table bytes as V2
+ Assert.assertEquals(newDataTable.getDataSchema(), dataSchema,
ERROR_MESSAGE);
Review comment:
@mqliang , can you please fix the typo? this should be V3
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
