jt2594838 commented on code in PR #16936:
URL: https://github.com/apache/iotdb/pull/16936#discussion_r2680679506
##########
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java:
##########
@@ -103,13 +109,145 @@ public void rewriteAlignedChunkMetadataStatistics() {
AbstractAlignedChunkMetadata abstractAlignedChunkMetadata =
SchemaUtils.rewriteAlignedChunkMetadataStatistics(
alignedChunkMetadata, targetDataType);
- Assert.assertEquals(
- targetDataType,
-
abstractAlignedChunkMetadata.getValueChunkMetadataList().get(0).getDataType());
+ if
(!abstractAlignedChunkMetadata.getValueChunkMetadataList().isEmpty()) {
+ Assert.assertEquals(
+ targetDataType,
+
abstractAlignedChunkMetadata.getValueChunkMetadataList().get(0).getDataType());
+ }
} catch (ClassCastException e) {
Assert.fail(e.getMessage());
}
}
}
}
+
+ @Test
+ public void mergeMetadataStatistics() throws Exception {
+ Set<TSDataType> unsupportTsDataType = new HashSet<>();
+ unsupportTsDataType.add(TSDataType.UNKNOWN);
+ unsupportTsDataType.add(TSDataType.VECTOR);
+ for (TSDataType sourceDataType : Arrays.asList(TSDataType.DOUBLE)) {
+ for (TSDataType targetDataType : Arrays.asList(TSDataType.TEXT,
TSDataType.BLOB)) {
+
+ if (sourceDataType.equals(targetDataType)) {
+ continue;
+ }
+ if (unsupportTsDataType.contains(sourceDataType)
+ || unsupportTsDataType.contains(targetDataType)) {
+ continue;
+ }
+
+ System.out.println("from " + sourceDataType + " to " + targetDataType);
+
+ // Aligned series
+ Statistics<?> s1 = Statistics.getStatsByType(sourceDataType);
+ s1.update(new long[] {1, 2}, new double[] {1.0, 2.0}, 2);
+ Statistics<?> s2 = Statistics.getStatsByType(TSDataType.DOUBLE);
+ s2.update(new long[] {1, 2}, new double[] {1.0, 2.0}, 2);
+ List<IChunkMetadata> valueChunkMetadatas =
+ Arrays.asList(
+ new ChunkMetadata(
+ "s0",
+ sourceDataType,
+ SchemaUtils.getDataTypeCompatibleEncoding(sourceDataType,
TSEncoding.RLE),
+ CompressionType.LZ4,
+ 0,
+ s1),
+ new ChunkMetadata(
+ "s1",
+ TSDataType.DOUBLE,
+
SchemaUtils.getDataTypeCompatibleEncoding(TSDataType.DOUBLE, TSEncoding.RLE),
+ CompressionType.LZ4,
+ 0,
+ s2));
+ IChunkMetadata alignedChunkMetadata =
+ new AlignedChunkMetadata(new ChunkMetadata(), valueChunkMetadatas);
+
+ Statistics<?> s3 = Statistics.getStatsByType(targetDataType);
+ if (targetDataType == TSDataType.BLOB) {
+ s3.update(3, new Binary("3", StandardCharsets.UTF_8));
+ s3.update(4, new Binary("4", StandardCharsets.UTF_8));
+ } else {
+ s3.update(
+ new long[] {1, 2},
+ new Binary[] {
+ new Binary("3", StandardCharsets.UTF_8),
+ new Binary("4", StandardCharsets.UTF_8),
+ new Binary("3", StandardCharsets.UTF_8),
+ new Binary("4", StandardCharsets.UTF_8)
+ },
+ 2);
Review Comment:
The last two values may be ignored if you provide a length of 2.
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java:
##########
@@ -403,25 +404,29 @@ public static AbstractAlignedChunkMetadata
rewriteAlignedChunkMetadataStatistics
AbstractAlignedChunkMetadata alignedChunkMetadata, TSDataType
targetDataType) {
List<IChunkMetadata> newValueChunkMetadataList = new ArrayList<>();
for (IChunkMetadata valueChunkMetadata :
alignedChunkMetadata.getValueChunkMetadataList()) {
- Statistics<?> statistics = Statistics.getStatsByType(targetDataType);
- statistics = getNewStatistics(valueChunkMetadata, targetDataType,
statistics);
-
- ChunkMetadata newChunkMetadata = (ChunkMetadata) valueChunkMetadata;
- newChunkMetadata.setTsDataType(targetDataType);
- newChunkMetadata.setStatistics(statistics);
- newValueChunkMetadataList.add(newChunkMetadata);
+ if (targetDataType.isCompatible(valueChunkMetadata.getDataType())) {
+ Statistics<?> statistics = Statistics.getStatsByType(targetDataType);
+ statistics = getNewStatistics(valueChunkMetadata, targetDataType,
statistics);
+
+ ChunkMetadata newChunkMetadata = (ChunkMetadata) valueChunkMetadata;
+ newChunkMetadata.setTsDataType(targetDataType);
+ newChunkMetadata.setStatistics(statistics);
+ newValueChunkMetadataList.add(newChunkMetadata);
+ }
}
Review Comment:
May need to add a null as a place holder.
The same for other places.
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java:
##########
@@ -460,11 +464,9 @@ public static Statistics<?> getNewStatistics(
new Binary(
chunkMetadata.getStatistics().getMaxValue().toString(),
StandardCharsets.UTF_8);
}
- long[] longValues = new long[4];
+ long[] longValues = new long[2];
longValues[0] = chunkMetadata.getStatistics().getStartTime();
longValues[1] = chunkMetadata.getStatistics().getEndTime();
- longValues[2] = longValues[1];
- longValues[3] = longValues[1];
statistics.update(longValues, binaryValues, binaryValues.length);
Review Comment:
Will it not be problematic if `longValues` and `binaryValues` have different
length.
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java:
##########
@@ -403,25 +404,29 @@ public static AbstractAlignedChunkMetadata
rewriteAlignedChunkMetadataStatistics
AbstractAlignedChunkMetadata alignedChunkMetadata, TSDataType
targetDataType) {
List<IChunkMetadata> newValueChunkMetadataList = new ArrayList<>();
for (IChunkMetadata valueChunkMetadata :
alignedChunkMetadata.getValueChunkMetadataList()) {
- Statistics<?> statistics = Statistics.getStatsByType(targetDataType);
- statistics = getNewStatistics(valueChunkMetadata, targetDataType,
statistics);
-
- ChunkMetadata newChunkMetadata = (ChunkMetadata) valueChunkMetadata;
- newChunkMetadata.setTsDataType(targetDataType);
- newChunkMetadata.setStatistics(statistics);
- newValueChunkMetadataList.add(newChunkMetadata);
+ if (targetDataType.isCompatible(valueChunkMetadata.getDataType())) {
Review Comment:
Only the target component should be written, not all of them.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
