jt2594838 commented on code in PR #17601:
URL: https://github.com/apache/iotdb/pull/17601#discussion_r3205737456
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java:
##########
@@ -689,6 +808,15 @@ private static long getTableWritePointCount(
:
alignedChunkMetadata.getTimeChunkMetadata().getStatistics().getCount();
}
+ private static long getTableWritePointCount(
+ final List<TimeseriesMetadata> timeseriesMetadataList) {
+ return timeseriesMetadataList.stream()
+ .filter(Objects::nonNull)
+ .filter(timeseriesMetadata ->
Objects.nonNull(timeseriesMetadata.getStatistics()))
+ .mapToLong(timeseriesMetadata ->
timeseriesMetadata.getStatistics().getCount())
+ .sum();
+ }
Review Comment:
No need to filter time column here?
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java:
##########
@@ -611,18 +670,77 @@ private long updateTableTsFileResourceAndVerifySchema(
tsFileResource.updateStartTime(deviceId,
alignedChunkMetadata.getStartTime());
tsFileResource.updateEndTime(deviceId,
alignedChunkMetadata.getEndTime());
writePointCount += getTableWritePointCount(alignedChunkMetadata);
+ if (Objects.nonNull(lastValueCollector)) {
+ lastValueCollector.update(deviceId,
alignedChunkMetadata.getValueChunkMetadataList());
+ }
+ }
+
+ if (!hasChunk) {
+ if (Objects.isNull(allTimeseriesMetadataByDevice)) {
+ allTimeseriesMetadataByDevice =
reader.getAllTimeseriesMetadata(true);
+ }
+
+ final List<TimeseriesMetadata> timeseriesMetadataList =
+ getTimeseriesMetadata(allTimeseriesMetadataByDevice, deviceId);
+ if (Objects.nonNull(timeseriesMetadataList)) {
+ for (final TimeseriesMetadata timeseriesMetadata :
timeseriesMetadataList) {
+ if (Objects.isNull(timeseriesMetadata)
+ || Objects.isNull(timeseriesMetadata.getStatistics())) {
+ continue;
+ }
+
+ hasChunk = true;
+ tsFileResource.updateStartTime(
+ deviceId, timeseriesMetadata.getStatistics().getStartTime());
+ tsFileResource.updateEndTime(
+ deviceId, timeseriesMetadata.getStatistics().getEndTime());
+ }
+
+ if (hasChunk) {
+ writePointCount +=
getTableWritePointCount(timeseriesMetadataList);
+ if (Objects.nonNull(lastValueCollector)) {
+ lastValueCollector.update(deviceId, timeseriesMetadataList);
+ }
+ }
+ }
}
if (hasChunk) {
getOrCreateTableSchemaCache().setCurrentTimeIndex(tsFileResource.getTimeIndex());
- getOrCreateTableSchemaCache().autoCreateAndVerify(deviceId);
+ if (!getOrCreateTableSchemaCache().isDeviceDeletedByMods(deviceId)) {
+ getOrCreateTableSchemaCache().autoCreateAndVerify(deviceId);
+ }
}
}
}
+ if (hasFallbackProcessedDevice && Objects.nonNull(lastValueCollector)) {
+
tsFileResource.setLastValues(lastValueCollector.toTsFileResourceLastValues());
+ }
return writePointCount;
}
+ private static List<TimeseriesMetadata> getTimeseriesMetadata(
+ final Map<IDeviceID, List<TimeseriesMetadata>> device2TimeseriesMetadata,
+ final IDeviceID deviceId) {
+ final List<TimeseriesMetadata> timeseriesMetadataList =
device2TimeseriesMetadata.get(deviceId);
+ if (Objects.nonNull(timeseriesMetadataList)) {
+ return timeseriesMetadataList;
+ }
+
+ for (final Map.Entry<IDeviceID, List<TimeseriesMetadata>> entry :
+ device2TimeseriesMetadata.entrySet()) {
+ if (Arrays.equals(entry.getKey().getSegments(), deviceId.getSegments()))
{
+ return entry.getValue();
+ }
+ }
Review Comment:
In what case can you not use `device2TimeseriesMetadata.get(deviceId)` but
can use an array comparison?
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java:
##########
@@ -611,18 +670,77 @@ private long updateTableTsFileResourceAndVerifySchema(
tsFileResource.updateStartTime(deviceId,
alignedChunkMetadata.getStartTime());
tsFileResource.updateEndTime(deviceId,
alignedChunkMetadata.getEndTime());
writePointCount += getTableWritePointCount(alignedChunkMetadata);
+ if (Objects.nonNull(lastValueCollector)) {
+ lastValueCollector.update(deviceId,
alignedChunkMetadata.getValueChunkMetadataList());
+ }
+ }
+
+ if (!hasChunk) {
+ if (Objects.isNull(allTimeseriesMetadataByDevice)) {
+ allTimeseriesMetadataByDevice =
reader.getAllTimeseriesMetadata(true);
+ }
+
+ final List<TimeseriesMetadata> timeseriesMetadataList =
+ getTimeseriesMetadata(allTimeseriesMetadataByDevice, deviceId);
+ if (Objects.nonNull(timeseriesMetadataList)) {
+ for (final TimeseriesMetadata timeseriesMetadata :
timeseriesMetadataList) {
+ if (Objects.isNull(timeseriesMetadata)
+ || Objects.isNull(timeseriesMetadata.getStatistics())) {
+ continue;
+ }
Review Comment:
If you cannot find the chunk from `reader.getAlignedChunkMetadata(deviceId,
false)`, how can you find it from `reader.getAllTimeseriesMetadata(true)`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
