This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new c7e2d8e67c5 Optimize insertRelationalTablet performance (#14197)
c7e2d8e67c5 is described below
commit c7e2d8e67c5efafaf7b16bddb576b998881106cc
Author: Haonan <[email protected]>
AuthorDate: Tue Nov 26 14:31:13 2024 +0800
Optimize insertRelationalTablet performance (#14197)
* Optimize single device insert relation tablet performance
* optimize ttl check
---
.../planner/plan/node/write/InsertTabletNode.java | 12 +--
.../node/write/RelationalInsertTabletNode.java | 88 +++++++++++++++-------
.../plan/relational/planner/RelationPlanner.java | 3 +
.../plan/relational/sql/ast/InsertTablet.java | 3 +
.../plan/statement/crud/InsertTabletStatement.java | 10 +++
.../db/storageengine/dataregion/DataRegion.java | 2 +-
6 files changed, 80 insertions(+), 38 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java
index 093f3715957..2773206a9db 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java
@@ -70,7 +70,6 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
-import java.util.function.IntToLongFunction;
import static org.apache.iotdb.db.utils.CommonUtils.isAlive;
@@ -1229,26 +1228,23 @@ public class InsertTabletNode extends InsertNode
implements WALEntryValue {
/**
* @param results insertion result of each row
- * @param rowTTLGetter the ttl associated with each row
+ * @param ttl the ttl
* @return the position of the first alive row
* @throws OutOfTTLException if all rows have expired the TTL
*/
- public int checkTTL(TSStatus[] results, IntToLongFunction rowTTLGetter)
throws OutOfTTLException {
- return checkTTLInternal(results, rowTTLGetter, true);
+ public int checkTTL(TSStatus[] results, long ttl) throws OutOfTTLException {
+ return checkTTLInternal(results, ttl, true);
}
- protected int checkTTLInternal(
- TSStatus[] results, IntToLongFunction rowTTLGetter, boolean
breakOnFirstAlive)
+ protected int checkTTLInternal(TSStatus[] results, long ttl, boolean
breakOnFirstAlive)
throws OutOfTTLException {
/*
* assume that batch has been sorted by client
*/
int loc = 0;
- long ttl = 0;
int firstAliveLoc = -1;
while (loc < getRowCount()) {
- ttl = rowTTLGetter.applyAsLong(loc);
long currTime = getTimes()[loc];
// skip points that do not satisfy TTL
if (!isAlive(currTime, ttl)) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java
index e4f7647ccbd..ed88d5cf191 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
+import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.exception.query.OutOfTTLException;
import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
@@ -49,27 +50,13 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.function.IntToLongFunction;
public class RelationalInsertTabletNode extends InsertTabletNode {
// deviceId cache for Table-view insertion
private IDeviceID[] deviceIDs;
- public RelationalInsertTabletNode(
- PlanNodeId id,
- PartialPath devicePath,
- boolean isAligned,
- String[] measurements,
- TSDataType[] dataTypes,
- long[] times,
- BitMap[] bitMaps,
- Object[] columns,
- int rowCount,
- TsTableColumnCategory[] columnCategories) {
- super(id, devicePath, isAligned, measurements, dataTypes, times, bitMaps,
columns, rowCount);
- setColumnCategories(columnCategories);
- }
+ private boolean singleDevice;
public RelationalInsertTabletNode(
PlanNodeId id,
@@ -101,8 +88,46 @@ public class RelationalInsertTabletNode extends
InsertTabletNode {
super(id);
}
+ @TestOnly
+ public RelationalInsertTabletNode(
+ PlanNodeId id,
+ PartialPath devicePath,
+ boolean isAligned,
+ String[] measurements,
+ TSDataType[] dataTypes,
+ long[] times,
+ BitMap[] bitMaps,
+ Object[] columns,
+ int rowCount,
+ TsTableColumnCategory[] columnCategories) {
+ super(id, devicePath, isAligned, measurements, dataTypes, times, bitMaps,
columns, rowCount);
+ setColumnCategories(columnCategories);
+ }
+
+ public void setSingleDevice() {
+ this.singleDevice = true;
+ }
+
@Override
public IDeviceID getDeviceID(int rowIdx) {
+ if (singleDevice) {
+ if (deviceIDs == null) {
+ deviceIDs = new IDeviceID[1];
+ }
+ if (deviceIDs[0] == null) {
+ String[] deviceIdSegments = new String[idColumnIndices.size() + 1];
+ deviceIdSegments[0] = this.getTableName();
+ for (int i = 0; i < idColumnIndices.size(); i++) {
+ final Integer columnIndex = idColumnIndices.get(i);
+ Object idSeg = ((Object[]) columns[columnIndex])[0];
+ boolean isNull =
+ bitMaps != null && bitMaps[columnIndex] != null &&
bitMaps[columnIndex].isMarked(0);
+ deviceIdSegments[i + 1] = !isNull && idSeg != null ?
idSeg.toString() : null;
+ }
+ deviceIDs[0] = Factory.DEFAULT_FACTORY.create(deviceIdSegments);
+ }
+ return deviceIDs[0];
+ }
if (deviceIDs == null) {
deviceIDs = new IDeviceID[rowCount];
}
@@ -139,18 +164,23 @@ public class RelationalInsertTabletNode extends
InsertTabletNode {
long[] subTimes = new long[count];
Object[] values = initTabletValues(dataTypes.length, count, dataTypes);
BitMap[] newBitMaps = this.bitMaps == null ? null :
initBitmaps(dataTypes.length, count);
- return new RelationalInsertTabletNode(
- getPlanNodeId(),
- targetPath,
- isAligned,
- measurements,
- dataTypes,
- measurementSchemas,
- subTimes,
- newBitMaps,
- values,
- subTimes.length,
- columnCategories);
+ RelationalInsertTabletNode split =
+ new RelationalInsertTabletNode(
+ getPlanNodeId(),
+ targetPath,
+ isAligned,
+ measurements,
+ dataTypes,
+ measurementSchemas,
+ subTimes,
+ newBitMaps,
+ values,
+ subTimes.length,
+ columnCategories);
+ if (singleDevice) {
+ split.setSingleDevice();
+ }
+ return split;
}
protected Map<TRegionReplicaSet, List<Integer>> splitByReplicaSet(
@@ -303,8 +333,8 @@ public class RelationalInsertTabletNode extends
InsertTabletNode {
}
@Override
- public int checkTTL(TSStatus[] results, IntToLongFunction rowTTLGetter)
throws OutOfTTLException {
- return checkTTLInternal(results, rowTTLGetter, false);
+ public int checkTTL(TSStatus[] results, long ttl) throws OutOfTTLException {
+ return checkTTLInternal(results, ttl, false);
}
@Override
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java
index 469c571fe3b..04fa9b2ee49 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java
@@ -654,6 +654,9 @@ public class RelationPlanner extends
AstVisitor<RelationPlan, Void> {
insertTabletStatement.getRowCount(),
insertTabletStatement.getColumnCategories());
insertNode.setFailedMeasurementNumber(insertTabletStatement.getFailedMeasurementNumber());
+ if (insertTabletStatement.isSingleDevice()) {
+ insertNode.setSingleDevice();
+ }
return new RelationPlan(
insertNode, analysis.getRootScope(), Collections.emptyList(),
outerContext);
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java
index 86de18bf0b9..9894adb4d01 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java
@@ -107,5 +107,8 @@ public class InsertTablet extends WrappedInsertStatement {
IDeviceID deviceID = insertTabletStatement.getTableDeviceID(i);
deviceID2LastIdxMap.put(deviceID, i);
}
+ if (deviceID2LastIdxMap.size() == 1) {
+ insertTabletStatement.setSingleDevice();
+ }
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java
index fe632bc4978..eec9350ce5e 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java
@@ -71,6 +71,8 @@ public class InsertTabletStatement extends
InsertBaseStatement implements ISchem
private IDeviceID[] deviceIDs;
+ private boolean singleDevice;
+
protected int rowCount = 0;
/**
@@ -471,6 +473,14 @@ public class InsertTabletStatement extends
InsertBaseStatement implements ISchem
return deviceIDs[rowIdx];
}
+ public void setSingleDevice() {
+ singleDevice = true;
+ }
+
+ public boolean isSingleDevice() {
+ return singleDevice;
+ }
+
@Override
public void insertColumn(int pos, ColumnSchema columnSchema) {
super.insertColumn(pos, columnSchema);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 3b3bb70e329..468f217f7ae 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -1237,7 +1237,7 @@ public class DataRegion implements IDataRegionForQuery {
InsertTabletNode insertTabletNode, TSStatus[] results, long[]
infoForMetrics)
throws OutOfTTLException {
boolean noFailure;
- int loc = insertTabletNode.checkTTL(results, i ->
getTTL(insertTabletNode));
+ int loc = insertTabletNode.checkTTL(results, getTTL(insertTabletNode));
noFailure = loc == 0;
List<Pair<IDeviceID, Integer>> deviceEndOffsetPairs =
insertTabletNode.splitByDevice(loc, insertTabletNode.getRowCount());
