jt2594838 commented on code in PR #17770:
URL: https://github.com/apache/iotdb/pull/17770#discussion_r3303162119
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java:
##########
@@ -412,37 +418,68 @@ private boolean putValueToColumns(final BatchData data,
final Tablet tablet, fin
case TEXT:
case BLOB:
case STRING:
- tablet.addValue(rowIndex, i, Binary.EMPTY_VALUE.getValues());
+ PipeTabletUtils.putValue(
+ tablet, rowIndex, i, tablet.getSchemas().get(i).getType(),
Binary.EMPTY_VALUE);
}
- tablet.getBitMaps()[i].mark(rowIndex);
+ PipeTabletUtils.markNullValue(tablet, rowIndex, i);
continue;
}
isNeedFillTime = true;
switch (tablet.getSchemas().get(i).getType()) {
case BOOLEAN:
- tablet.addValue(rowIndex, i, primitiveType.getBoolean());
+ PipeTabletUtils.putValue(
+ tablet,
+ rowIndex,
+ i,
+ tablet.getSchemas().get(i).getType(),
+ primitiveType.getBoolean());
Review Comment:
What is the point?
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java:
##########
@@ -319,8 +324,8 @@ private Tablet getNextTablet() {
PipeMemoryWeightUtil.calculateTabletRowCountAndMemory(data);
tablet =
new Tablet(
- currentDevice.toString(), currentMeasurements,
rowCountAndMemorySize.getLeft());
- tablet.initBitMaps();
+ currentDeviceString, currentMeasurements,
rowCountAndMemorySize.getLeft());
+ PipeTabletUtils.initEmptyBitMaps(tablet);
Review Comment:
Why is an empty bitmap needed here, there, or anywhere,
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java:
##########
@@ -304,8 +308,9 @@ private Tablet getNextTablet() {
Tablet tablet = null;
if (!data.hasCurrent()) {
- tablet = new Tablet(currentDevice.toString(), currentMeasurements, 1);
- tablet.initBitMaps();
+ tablet = new Tablet(currentDeviceString, currentMeasurements, 1);
+ PipeTabletUtils.initEmptyBitMaps(tablet);
+ PipeTabletUtils.compactBitMaps(tablet);
return tablet;
Review Comment:
What is the point
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeTabletUtils.java:
##########
@@ -0,0 +1,256 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.pipe.event.common.tablet;
+
+import org.apache.iotdb.db.i18n.DataNodePipeMessages;
+
+import org.apache.tsfile.common.conf.TSFileConfig;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.utils.Binary;
+import org.apache.tsfile.utils.BitMap;
+import org.apache.tsfile.write.UnSupportedDataTypeException;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+
+import java.time.LocalDate;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+public final class PipeTabletUtils {
+
+ private PipeTabletUtils() {}
+
+ public static final class TabletStringInternPool {
+
+ private final Map<String, String> internedStrings = new HashMap<>();
+
+ public String intern(final String value) {
+ if (Objects.isNull(value)) {
+ return null;
+ }
+
+ final String internedValue = internedStrings.get(value);
+ if (Objects.nonNull(internedValue)) {
+ return internedValue;
+ }
+
+ internedStrings.put(value, value);
+ return value;
+ }
+
+ public void intern(final String[] values) {
+ if (Objects.isNull(values)) {
+ return;
+ }
+
+ for (int i = 0; i < values.length; ++i) {
+ values[i] = intern(values[i]);
+ }
+ }
+
+ public void intern(final List<String> values) {
+ if (Objects.isNull(values)) {
+ return;
+ }
+
+ for (int i = 0; i < values.size(); ++i) {
+ values.set(i, intern(values.get(i)));
+ }
+ }
+
+ public Tablet intern(final Tablet tablet) {
+ if (Objects.isNull(tablet)) {
+ return null;
+ }
+
+ tablet.setInsertTargetName(intern(tablet.getDeviceId()));
+ internMeasurementSchemas(tablet.getSchemas());
+ return tablet;
+ }
+
+ public void internMeasurementSchemas(final List<IMeasurementSchema>
schemas) {
+ if (Objects.isNull(schemas)) {
+ return;
+ }
+
+ for (final IMeasurementSchema schema : schemas) {
+ if (schema instanceof MeasurementSchema) {
+ intern((MeasurementSchema) schema);
+ }
+ }
+ }
+
+ public MeasurementSchema intern(final MeasurementSchema schema) {
+ if (Objects.isNull(schema)) {
+ return null;
+ }
+
+ schema.setMeasurementName(intern(schema.getMeasurementName()));
+ schema.setProps(intern(schema.getProps()));
+ return schema;
+ }
+
+ private Map<String, String> intern(final Map<String, String> props) {
+ if (Objects.isNull(props) || props.isEmpty()) {
+ return props;
+ }
+
+ final Map<String, String> internedProps = new HashMap<>(props.size());
+ for (final Map.Entry<String, String> entry : props.entrySet()) {
+ internedProps.put(intern(entry.getKey()), intern(entry.getValue()));
+ }
+ return internedProps;
+ }
+ }
+
+ public static Tablet internTablet(
+ final Tablet tablet, final TabletStringInternPool
tabletStringInternPool) {
+ return Objects.nonNull(tabletStringInternPool) ?
tabletStringInternPool.intern(tablet) : tablet;
+ }
+
+ public static void initEmptyBitMaps(final Tablet tablet) {
+ tablet.setBitMaps(new BitMap[getColumnCount(tablet)]);
+ }
+
+ public static void compactBitMaps(final Tablet tablet) {
+ if (Objects.isNull(tablet)) {
+ return;
+ }
+ tablet.setBitMaps(compactBitMaps(tablet.getBitMaps(),
tablet.getRowSize()));
+ }
+
+ public static BitMap[] compactBitMaps(final BitMap[] bitMaps, final int
rowCount) {
+ if (Objects.isNull(bitMaps)) {
+ return null;
+ }
+
+ boolean hasMarkedBitMap = false;
+ for (int i = 0; i < bitMaps.length; ++i) {
+ if (Objects.nonNull(bitMaps[i])
+ && bitMaps[i].isAllUnmarked(Math.min(rowCount,
bitMaps[i].getSize()))) {
+ bitMaps[i] = null;
+ }
+ if (Objects.nonNull(bitMaps[i])) {
+ hasMarkedBitMap = true;
+ }
+ }
+
+ return hasMarkedBitMap ? bitMaps : null;
+ }
+
+ public static BitMap[] copyBitMapsOrCreateEmpty(final Tablet tablet) {
+ final BitMap[] bitMaps = tablet.getBitMaps();
+ return Objects.nonNull(bitMaps)
+ ? Arrays.copyOf(bitMaps, bitMaps.length)
+ : new BitMap[getColumnCount(tablet)];
+ }
+
+ public static void markNullValue(final Tablet tablet, final int rowIndex,
final int columnIndex) {
+ final BitMap[] bitMaps = ensureBitMaps(tablet, columnIndex + 1);
+ if (Objects.isNull(bitMaps[columnIndex])) {
+ bitMaps[columnIndex] = new BitMap(tablet.getMaxRowNumber());
+ }
+ bitMaps[columnIndex].mark(rowIndex);
+ }
+
+ public static void putTimestamp(final Tablet tablet, final int rowIndex,
final long timestamp) {
+ tablet.getTimestamps()[rowIndex] = timestamp;
+ tablet.setRowSize(Math.max(tablet.getRowSize(), rowIndex + 1));
+ }
+
+ public static void putValue(
+ final Tablet tablet,
+ final int rowIndex,
+ final int columnIndex,
+ final TSDataType dataType,
+ final Object value) {
+ switch (dataType) {
Review Comment:
What is the point of the two methods?
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java:
##########
@@ -441,6 +445,42 @@ protected BitMap[] initBitmaps(int columnSize, int
rowSize) {
return bitMaps;
}
+ protected BitMap[] initBitmapsForSplit(int columnSize, int rowSize) {
+ if (this.bitMaps == null) {
+ return null;
+ }
+
+ final int sourceRowCount = rowCount > 0 ? rowCount : times == null ? 0 :
times.length;
+ final BitMap[] splitBitMaps = new BitMap[columnSize];
+ boolean hasBitMap = false;
+ for (int i = 0; i < columnSize && i < this.bitMaps.length; ++i) {
+ if (this.bitMaps[i] != null
+ && !this.bitMaps[i].isAllUnmarked(Math.min(sourceRowCount,
this.bitMaps[i].getSize()))) {
+ splitBitMaps[i] = new BitMap(rowSize);
+ hasBitMap = true;
+ }
+ }
+ return hasBitMap ? splitBitMaps : null;
+ }
+
+ protected static BitMap[] compactBitMaps(final BitMap[] bitMaps, final int
rowCount) {
+ if (bitMaps == null) {
+ return null;
+ }
+
+ boolean hasBitMap = false;
+ for (int i = 0; i < bitMaps.length; ++i) {
+ if (bitMaps[i] != null
+ && bitMaps[i].isAllUnmarked(Math.min(rowCount,
bitMaps[i].getSize()))) {
+ bitMaps[i] = null;
+ }
+ if (bitMaps[i] != null) {
+ hasBitMap = true;
+ }
+ }
+ return hasBitMap ? bitMaps : null;
+ }
Review Comment:
Duplicated?
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java:
##########
@@ -900,6 +900,24 @@ private Object convertColumnToTablet(
return columnValue;
}
+ private static BitMap[] compactBitMaps(final BitMap[] bitMaps, final int
rowCount) {
+ if (bitMaps == null) {
+ return null;
+ }
+
+ boolean hasBitMap = false;
+ for (int i = 0; i < bitMaps.length; ++i) {
+ if (bitMaps[i] != null
+ && bitMaps[i].isAllUnmarked(Math.min(rowCount,
bitMaps[i].getSize()))) {
+ bitMaps[i] = null;
+ }
+ if (bitMaps[i] != null) {
+ hasBitMap = true;
+ }
+ }
+ return hasBitMap ? bitMaps : null;
+ }
+
Review Comment:
Here too?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
