This is an automated email from the ASF dual-hosted git repository. jiangtian pushed a commit to branch table_model_auto_cast in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b8b9928296decdfc76504413ebe4db513e7c222c Author: Tian Jiang <[email protected]> AuthorDate: Thu Dec 12 10:56:51 2024 +0800 Add table model auto-cast --- .../it/session}/IoTDBSessionRelationalIT.java | 182 ++++++++++++++++++++- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 8 + .../relational/sql/ast/WrappedInsertStatement.java | 20 ++- .../plan/statement/crud/InsertBaseStatement.java | 2 +- .../plan/statement/crud/InsertRowStatement.java | 5 +- .../plan/statement/crud/InsertTabletStatement.java | 5 +- .../dataregion/memtable/AbstractMemTable.java | 4 +- .../memtable/AlignedWritableMemChunk.java | 4 + .../memtable/AlignedWritableMemChunkGroup.java | 8 +- .../memtable/IWritableMemChunkGroup.java | 2 + .../dataregion/memtable/WritableMemChunkGroup.java | 5 + .../org/apache/iotdb/db/utils/CommonUtils.java | 86 ---------- .../db/utils/datastructure/AlignedTVList.java | 26 ++- .../resources/conf/iotdb-system.properties | 118 +++++++++++++ .../conf/iotdb-system.properties.template | 2 +- pom.xml | 2 +- 16 files changed, 371 insertions(+), 108 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionRelationalIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java similarity index 86% rename from integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionRelationalIT.java rename to integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java index 438c5844ca5..fdffb7c9312 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionRelationalIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.session.it; +package org.apache.iotdb.relational.it.session; import org.apache.iotdb.isession.ISession; import org.apache.iotdb.isession.ITableSession; @@ -30,6 +30,7 @@ import org.apache.iotdb.rpc.StatementExecutionException; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.record.Tablet.ColumnCategory; import org.apache.tsfile.write.schema.IMeasurementSchema; @@ -42,11 +43,17 @@ import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; import static org.junit.Assert.fail; @@ -473,8 +480,25 @@ public class IoTDBSessionRelationalIT { cnt++; } assertEquals(30, cnt); + + // partial insert is disabled + session.executeNonQueryStatement("SET CONFIGURATION \"enable_partial_insert\"=\"false\""); + int rowIndex = 0; + tablet.addTimestamp(rowIndex, timestamp + rowIndex); + tablet.addValue("id1", rowIndex, "id:" + rowIndex); + tablet.addValue("attr1", rowIndex, "attr:" + rowIndex); + tablet.addValue("m1", rowIndex, rowIndex * 1.0); + tablet.addValue("m2", rowIndex, rowIndex * 1.0); + try { + session.insert(tablet); + fail("Exception expected"); + } catch (StatementExecutionException e) { + assertEquals("", e.getMessage()); + } + } finally { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement("SET CONFIGURATION \"enable_partial_insert\"=\"true\""); session.executeNonQueryStatement( "SET CONFIGURATION \"enable_auto_create_schema\"=\"true\""); } @@ -1104,4 +1128,160 @@ public class IoTDBSessionRelationalIT { assertEquals(1, cnt); } } + + private void testOneCast(TSDataType from, TSDataType to, int testNum, boolean partialInsert) + throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db1\""); + // create a column with type of "to" + session.executeNonQueryStatement( + "CREATE TABLE table" + + testNum + + " (id1 string id," + + "m1 " + + to.toString() + + " measurement)"); + if (partialInsert) { + session.executeNonQueryStatement("SET CONFIGURATION enable_partial_insert='true' ON 1"); + } else { + session.executeNonQueryStatement("SET CONFIGURATION enable_partial_insert='false' ON 1"); + } + + // insert a tablet with type of "from" + List<IMeasurementSchema> schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("id1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("m1", from)); + final List<ColumnCategory> columnTypes = + Arrays.asList(ColumnCategory.ID, ColumnCategory.MEASUREMENT); + Tablet tablet = + new Tablet( + "table" + testNum, + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes, + 15); + + tablet.addTimestamp(0, 0); + tablet.addTimestamp(1, 1); + tablet.addValue(0, 0, "d1"); + tablet.addValue(1, 0, "d1"); + // the measurement in the first row is null + tablet.addValue("m1", 1, genValue(from, 1)); + System.out.println("from=" + from + ", to=" + to); + if (to.isCompatible(from)) { + // can cast, insert and check the result + session.insert(tablet); + // time, id1, m1 + SessionDataSet dataSet = + session.executeQueryStatement("select * from table" + testNum + " order by time"); + RowRecord rec = dataSet.next(); + System.out.println(rec); + assertEquals(0, rec.getFields().get(0).getLongV()); + assertEquals("d1", rec.getFields().get(1).toString()); + assertNull(rec.getFields().get(2).getDataType()); + rec = dataSet.next(); + System.out.println(rec); + assertEquals(1, rec.getFields().get(0).getLongV()); + assertEquals("d1", rec.getFields().get(1).toString()); + if (to == TSDataType.BLOB) { + assertEquals(genValue(to, 1), rec.getFields().get(2).getBinaryV()); + } else if (to == TSDataType.DATE) { + assertEquals(genValue(to, 1), rec.getFields().get(2).getDateV()); + } else { + assertEquals(genValue(to, 1).toString(), rec.getFields().get(2).toString()); + } + assertFalse(dataSet.hasNext()); + } else { + if (partialInsert) { + // cannot cast, but partial insert + try { + session.insert(tablet); + fail("Exception expected"); + } catch (StatementExecutionException e) { + assertEquals( + "507: Fail to insert measurements [m1] caused by [Incompatible data type of column m1: " + + from + + "/" + + to + + "]", + e.getMessage()); + } + // time, id1, m1 + SessionDataSet dataSet = + session.executeQueryStatement("select * from table" + testNum + " order by time"); + RowRecord rec = dataSet.next(); + System.out.println(rec); + assertEquals(0, rec.getFields().get(0).getLongV()); + assertEquals("d1", rec.getFields().get(1).toString()); + assertNull(rec.getFields().get(2).getDataType()); + rec = dataSet.next(); + System.out.println(rec); + assertEquals(1, rec.getFields().get(0).getLongV()); + assertEquals("d1", rec.getFields().get(1).toString()); + assertNull(rec.getFields().get(2).getDataType()); + assertFalse(dataSet.hasNext()); + } else { + // cannot cast, expect an exception + try { + session.insert(tablet); + fail("Exception expected"); + } catch (StatementExecutionException e) { + assertEquals( + "614: Incompatible data type of column m1: " + from + "/" + to, e.getMessage()); + } + } + } + + session.executeNonQueryStatement("DROP TABLE table" + testNum); + } + } + + @SuppressWarnings("SameParameterValue") + private Object genValue(TSDataType dataType, int i) { + switch (dataType) { + case INT32: + return i; + case DATE: + return LocalDate.ofEpochDay(i); + case TIMESTAMP: + case INT64: + return (long) i; + case BOOLEAN: + return i % 2 == 0; + case FLOAT: + return i * 1.0f; + case DOUBLE: + return i * 1.0; + case STRING: + case TEXT: + case BLOB: + return new Binary(Integer.toString(i), StandardCharsets.UTF_8); + case UNKNOWN: + case VECTOR: + default: + throw new IllegalArgumentException("Unsupported data type: " + dataType); + } + } + + @Test + @Category({LocalStandaloneIT.class, ClusterIT.class}) + public void insertRelationalTabletWithAutoCastTest() + throws IoTDBConnectionException, StatementExecutionException { + int testNum = 14; + Set<TSDataType> dataTypes = new HashSet<>(); + Collections.addAll(dataTypes, TSDataType.values()); + dataTypes.remove(TSDataType.VECTOR); + dataTypes.remove(TSDataType.UNKNOWN); + + for (TSDataType from : dataTypes) { + for (TSDataType to : dataTypes) { + testOneCast(from, to, testNum, false); + testOneCast(from, to, testNum, true); + } + } + + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement("SET CONFIGURATION \"enable_partial_insert\"=\"true\" ON 1"); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index b652acf7329..e112e5efb96 100755 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -2870,6 +2870,14 @@ public class IoTDBDescriptor { } else { BinaryAllocator.getInstance().close(true); } + + conf.setEnablePartialInsert( + Boolean.parseBoolean( + Optional.ofNullable( + properties.getProperty( + "enable_partial_insert", String.valueOf(conf.isEnablePartialInsert()))) + .map(String::trim) + .orElse(String.valueOf(conf.isEnablePartialInsert())))); } catch (Exception e) { if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WrappedInsertStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WrappedInsertStatement.java index 06d0003a9f5..bd53aaf0131 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WrappedInsertStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WrappedInsertStatement.java @@ -20,6 +20,9 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException; +import org.apache.iotdb.db.exception.metadata.PathNotExistException; import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; @@ -168,7 +171,8 @@ public abstract class WrappedInsertStatement extends WrappedStatement new SemanticException( "Column " + incoming.getName() + " does not exists or fails to be " + "created", TSStatusCode.COLUMN_NOT_EXISTS.getStatusCode()); - if (incoming.getColumnCategory() != TsTableColumnCategory.MEASUREMENT) { + if (incoming.getColumnCategory() != TsTableColumnCategory.MEASUREMENT + || !IoTDBDescriptor.getInstance().getConfig().isEnablePartialInsert()) { // non-measurement columns cannot be partially inserted throw semanticException; } else { @@ -182,19 +186,22 @@ public abstract class WrappedInsertStatement extends WrappedStatement // sql insertion does not provide type // the type is inferred and can be inconsistent with the existing one innerTreeStatement.setDataType(InternalTypeManager.getTSDataType(real.getType()), i); - } else if (!incoming.getType().equals(real.getType())) { + } else if (!InternalTypeManager.getTSDataType(real.getType()) + .isCompatible(InternalTypeManager.getTSDataType(incoming.getType()))) { SemanticException semanticException = new SemanticException( String.format( - "Inconsistent data type of column %s: %s/%s", + "Incompatible data type of column %s: %s/%s", incoming.getName(), incoming.getType(), real.getType()), TSStatusCode.DATA_TYPE_MISMATCH.getStatusCode()); - if (incoming.getColumnCategory() != TsTableColumnCategory.MEASUREMENT) { + if (incoming.getColumnCategory() != TsTableColumnCategory.MEASUREMENT + || !IoTDBDescriptor.getInstance().getConfig().isEnablePartialInsert()) { // non-measurement columns cannot be partially inserted throw semanticException; } else { // partial insertion innerTreeStatement.markFailedMeasurement(i, semanticException); + return; } } if (incoming.getColumnCategory() == null) { @@ -215,6 +222,11 @@ public abstract class WrappedInsertStatement extends WrappedStatement getDefaultEncoding(tsDataType), TSFileDescriptor.getInstance().getConfig().getCompressor()); innerTreeStatement.setMeasurementSchema(measurementSchema, i); + try { + innerTreeStatement.selfCheckDataTypes(i); + } catch (DataTypeMismatchException | PathNotExistException e) { + throw new SemanticException(e); + } } public void validateDeviceSchema(Metadata metadata, MPPQueryContext context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java index 1856d8968ae..7df56bbce88 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java @@ -192,7 +192,7 @@ public abstract class InsertBaseStatement extends Statement { public void updateAfterSchemaValidation(MPPQueryContext context) throws QueryProcessException {} /** Check whether data types are matched with measurement schemas */ - protected void selfCheckDataTypes(int index) + public void selfCheckDataTypes(int index) throws DataTypeMismatchException, PathNotExistException { if (IoTDBDescriptor.getInstance().getConfig().isEnablePartialInsert()) { // if enable partial insert, mark failed measurements with exception diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java index 8c832a4e6f3..00be61c6b80 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java @@ -190,9 +190,8 @@ public class InsertRowStatement extends InsertBaseStatement implements ISchemaVa @Override protected boolean checkAndCastDataType(int columnIndex, TSDataType dataType) { - if (CommonUtils.checkCanCastType(dataTypes[columnIndex], dataType)) { - values[columnIndex] = - CommonUtils.castValue(dataTypes[columnIndex], dataType, values[columnIndex]); + if (dataType.isCompatible(dataTypes[columnIndex])) { + values[columnIndex] = dataType.castFromSingleValue(dataType, values[columnIndex]); dataTypes[columnIndex] = dataType; return true; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java index f61db241355..e6f6f5f1a20 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java @@ -191,9 +191,8 @@ public class InsertTabletStatement extends InsertBaseStatement implements ISchem @Override protected boolean checkAndCastDataType(int columnIndex, TSDataType dataType) { - if (CommonUtils.checkCanCastType(dataTypes[columnIndex], dataType)) { - columns[columnIndex] = - CommonUtils.castArray(dataTypes[columnIndex], dataType, columns[columnIndex]); + if (dataType.isCompatible(dataTypes[columnIndex])) { + columns[columnIndex] = dataType.castFromArray(dataTypes[columnIndex], columns[columnIndex]); dataTypes[columnIndex] = dataType; return true; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java index fff93680c9c..b46f1a7612a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java @@ -796,8 +796,8 @@ public abstract class AbstractMemTable implements IMemTable { } else { pointDeleted += pair.right.delete(modEntry); } - if (pair.right.getMemChunkMap().isEmpty()) { - memTableMap.remove(pair.left); + if (pair.right.isEmpty()) { + memTableMap.remove(pair.left).release(); } } return pointDeleted; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java index 74c4682de6a..81105bbc686 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java @@ -554,4 +554,8 @@ public class AlignedWritableMemChunk implements IWritableMemChunk { public List<IMeasurementSchema> getSchemaList() { return schemaList; } + + public boolean isAllDeleted() { + return list.isAllDeleted(); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java index 4e079b61787..bcba1b41ad2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java @@ -98,6 +98,11 @@ public class AlignedWritableMemChunkGroup implements IWritableMemChunkGroup { return Collections.singletonMap("", memChunk); } + @Override + public boolean isEmpty() { + return memChunk.isEmpty() || memChunk.isAllDeleted(); + } + @Override public long delete(ModEntry modEntry) { int deletedPointsNumber = 0; @@ -124,7 +129,8 @@ public class AlignedWritableMemChunkGroup implements IWritableMemChunkGroup { } public long deleteTime(ModEntry modEntry) { - return memChunk.deleteTime(modEntry.getStartTime(), modEntry.getEndTime()); + int pointNumDeleted = memChunk.deleteTime(modEntry.getStartTime(), modEntry.getEndTime()); + return pointNumDeleted; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java index 353da05bf28..a2d23dc9db0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java @@ -51,6 +51,8 @@ public interface IWritableMemChunkGroup extends WALEntryValue { Map<String, IWritableMemChunk> getMemChunkMap(); + boolean isEmpty(); + long delete(ModEntry modEntry); long deleteTime(ModEntry modEntry); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java index b3fade015a3..ac5d588eb4f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java @@ -116,6 +116,11 @@ public class WritableMemChunkGroup implements IWritableMemChunkGroup { return memChunkMap; } + @Override + public boolean isEmpty() { + return memChunkMap.isEmpty(); + } + @Override public long delete(ModEntry modEntry) { Iterator<Entry<String, IWritableMemChunk>> iter = memChunkMap.entrySet().iterator(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java index 3b3e6311164..b54838b4735 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java @@ -57,9 +57,7 @@ import org.apache.tsfile.write.UnSupportedDataTypeException; import java.time.LocalDate; import java.time.ZoneId; -import java.util.Arrays; import java.util.List; -import java.util.Objects; import java.util.concurrent.TimeUnit; @SuppressWarnings("java:S106") // for console outputs @@ -190,90 +188,6 @@ public class CommonUtils { } } - public static boolean checkCanCastType(TSDataType src, TSDataType dest) { - if (Objects.isNull(src)) { - return true; - } - switch (src) { - case INT32: - if (dest == TSDataType.INT64 || dest == TSDataType.FLOAT || dest == TSDataType.DOUBLE) { - return true; - } - case INT64: - if (dest == TSDataType.DOUBLE) { - return true; - } - case FLOAT: - if (dest == TSDataType.DOUBLE) { - return true; - } - } - return false; - } - - public static Object castValue(TSDataType srcDataType, TSDataType destDataType, Object value) { - if (Objects.isNull(value)) { - return null; - } - switch (srcDataType) { - case INT32: - if (destDataType == TSDataType.INT64) { - value = (long) ((int) value); - } else if (destDataType == TSDataType.FLOAT) { - value = (float) ((int) value); - } else if (destDataType == TSDataType.DOUBLE) { - value = (double) ((int) value); - } - break; - case INT64: - if (destDataType == TSDataType.DOUBLE) { - value = (double) ((long) value); - } - break; - case FLOAT: - if (destDataType == TSDataType.DOUBLE) { - value = (double) ((float) value); - } - break; - } - return value; - } - - public static Object castArray(TSDataType srcDataType, TSDataType destDataType, Object value) { - switch (srcDataType) { - case INT32: - if (destDataType == TSDataType.INT64) { - value = Arrays.stream((int[]) value).mapToLong(Long::valueOf).toArray(); - } else if (destDataType == TSDataType.FLOAT) { - int[] tmp = (int[]) value; - float[] result = new float[tmp.length]; - for (int i = 0; i < tmp.length; i++) { - result[i] = (float) tmp[i]; - } - value = result; - } else if (destDataType == TSDataType.DOUBLE) { - value = Arrays.stream((int[]) value).mapToDouble(Double::valueOf).toArray(); - } - break; - case INT64: - if (destDataType == TSDataType.DOUBLE) { - value = Arrays.stream((long[]) value).mapToDouble(Double::valueOf).toArray(); - } - break; - case FLOAT: - if (destDataType == TSDataType.DOUBLE) { - float[] tmp = (float[]) value; - double[] result = new double[tmp.length]; - for (int i = 0; i < tmp.length; i++) { - result[i] = tmp[i]; - } - value = result; - } - break; - } - return value; - } - private static boolean parseBoolean(String value) throws QueryProcessException { value = value.toLowerCase(); if (SqlConstant.BOOLEAN_FALSE_NUM.equals(value) || SqlConstant.BOOLEAN_FALSE.equals(value)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java index f0a645bf3ec..5d5a6e2b41c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java @@ -86,6 +86,8 @@ public abstract class AlignedTVList extends TVList { // constructed after deletion BitMap timeColDeletedMap; + protected int timeDeletedCnt = 0; + AlignedTVList(List<TSDataType> types) { super(); indices = new ArrayList<>(types.size()); @@ -141,6 +143,7 @@ public abstract class AlignedTVList extends TVList { // for table model, we won't discard any row even if all value columns are null alignedTvList.allValueColDeletedMap = ignoreAllNullRows ? getAllValueColDeletedMap() : null; alignedTvList.timeColDeletedMap = this.timeColDeletedMap; + alignedTvList.timeDeletedCnt = this.timeDeletedCnt; return alignedTvList; } @@ -149,6 +152,7 @@ public abstract class AlignedTVList extends TVList { public AlignedTVList clone() { AlignedTVList cloneList = AlignedTVList.newAlignedList(dataTypes); cloneAs(cloneList); + cloneList.timeDeletedCnt = this.timeDeletedCnt; System.arraycopy( memoryBinaryChunkSize, 0, cloneList.memoryBinaryChunkSize, 0, dataTypes.size()); for (int[] indicesArray : indices) { @@ -538,19 +542,18 @@ public abstract class AlignedTVList extends TVList { public int deleteTime(long lowerBound, long upperBound) { delete(lowerBound, upperBound); - int deletedNumber = 0; + int prevDeletedCnt = this.timeDeletedCnt; for (int i = 0; i < rowCount; i++) { long time = getTime(i); if (time >= lowerBound && time <= upperBound) { markRowNull(i); - deletedNumber++; } } boolean needUpdateMaxTime = lowerBound <= maxTime && maxTime <= upperBound; if (needUpdateMaxTime) { updateMaxTime(); } - return deletedNumber; + return timeDeletedCnt - prevDeletedCnt; } private void updateMaxTime() { @@ -707,7 +710,10 @@ public abstract class AlignedTVList extends TVList { } } - private void markRowNull(int i) { + /** + * @return true if the row is marked, false if it is already marked + */ + private boolean markRowNull(int i) { if (timeColDeletedMap == null) { timeColDeletedMap = new BitMap(rowCount); } else if (timeColDeletedMap.getSize() < rowCount) { @@ -717,7 +723,13 @@ public abstract class AlignedTVList extends TVList { timeColDeletedMap = new BitMap(rowCount, newBytes); } // use value index so that sorts will not change the nullability - timeColDeletedMap.mark(getValueIndex(i)); + if (timeColDeletedMap.isMarked(i)) { + return false; + } else { + timeColDeletedMap.mark(getValueIndex(i)); + timeDeletedCnt++; + return true; + } } /** @@ -1478,4 +1490,8 @@ public abstract class AlignedTVList extends TVList { public List<List<BitMap>> getBitMaps() { return bitMaps; } + + public boolean isAllDeleted() { + return timeDeletedCnt == rowCount; + } } diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties index c51f948ad25..f99015dbe25 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties @@ -70,3 +70,121 @@ cn_metric_prometheus_reporter_port=9091 # dn_metric_reporter_list= dn_metric_prometheus_reporter_port=9092 +#Thu Dec 12 10:40:29 HKT 2024 +#enable_partial_insert=false +#Thu Dec 12 10:40:33 HKT 2024 +#enable_partial_insert=true +#Thu Dec 12 10:40:33 HKT 2024 +#enable_partial_insert=false +#Thu Dec 12 10:40:37 HKT 2024 +#enable_partial_insert=true +#Thu Dec 12 10:40:38 HKT 2024 +#enable_partial_insert=false +#Thu Dec 12 10:42:19 HKT 2024 +#enable_partial_insert=true +#Thu Dec 12 10:42:19 HKT 2024 +#enable_partial_insert=false +#Thu Dec 12 10:42:19 HKT 2024 +#enable_partial_insert=true +#Thu Dec 12 10:42:19 HKT 2024 +#enable_partial_insert=false +#Thu Dec 12 10:42:19 HKT 2024 +#enable_partial_insert=true +#Thu Dec 12 10:42:19 HKT 2024 +#enable_partial_insert=false +#Thu Dec 12 10:42:19 HKT 2024 +#enable_partial_insert=true +#Thu Dec 12 10:42:19 HKT 2024 +#enable_partial_insert=false +#Thu Dec 12 10:42:19 HKT 2024 +#enable_partial_insert=true +#Thu Dec 12 10:42:20 HKT 2024 +#enable_partial_insert=false +#Thu Dec 12 10:42:20 HKT 2024 +#enable_partial_insert=true +#Thu Dec 12 10:42:20 HKT 2024 +#enable_partial_insert=false +#Thu Dec 12 10:42:20 HKT 2024 +#enable_partial_insert=true +#Thu Dec 12 10:42:20 HKT 2024 +#enable_partial_insert=false +#Thu Dec 12 10:42:20 HKT 2024 +#enable_partial_insert=true +#Thu Dec 12 10:42:20 HKT 2024 +#enable_partial_insert=false +#Thu Dec 12 10:42:20 HKT 2024 +#enable_partial_insert=true +#Thu Dec 12 10:42:20 HKT 2024 +#enable_partial_insert=false +#Thu Dec 12 10:42:20 HKT 2024 +#enable_partial_insert=true +#Thu Dec 12 10:42:20 HKT 2024 +#enable_partial_insert=false +#Thu Dec 12 10:43:29 HKT 2024 +#enable_partial_insert=true +#Thu Dec 12 10:43:29 HKT 2024 +#enable_partial_insert=false +#Thu Dec 12 10:43:29 HKT 2024 +#enable_partial_insert=true +#Thu Dec 12 10:43:29 HKT 2024 +#enable_partial_insert=false +#Thu Dec 12 10:43:29 HKT 2024 +#enable_partial_insert=true +#Thu Dec 12 10:43:29 HKT 2024 +#enable_partial_insert=false +#Thu Dec 12 10:43:29 HKT 2024 +#enable_partial_insert=true +#Thu Dec 12 10:43:30 HKT 2024 +#enable_partial_insert=false +#Thu Dec 12 10:43:30 HKT 2024 +#enable_partial_insert=true +#Thu Dec 12 10:43:30 HKT 2024 +#enable_partial_insert=false +#Thu Dec 12 10:43:30 HKT 2024 +#enable_partial_insert=true +#Thu Dec 12 10:43:30 HKT 2024 +#enable_partial_insert=false +#Thu Dec 12 10:43:30 HKT 2024 +#enable_partial_insert=true +#Thu Dec 12 10:43:30 HKT 2024 +#enable_partial_insert=false +#Thu Dec 12 10:43:30 HKT 2024 +#enable_partial_insert=true +#Thu Dec 12 10:43:30 HKT 2024 +#enable_partial_insert=false +#Thu Dec 12 10:43:30 HKT 2024 +#enable_partial_insert=true +#Thu Dec 12 10:43:30 HKT 2024 +#enable_partial_insert=false +#Thu Dec 12 10:43:30 HKT 2024 +#enable_partial_insert=true +#Thu Dec 12 10:43:30 HKT 2024 +#enable_partial_insert=false +#Thu Dec 12 10:43:31 HKT 2024 +#enable_partial_insert=true +#Thu Dec 12 10:43:31 HKT 2024 +#enable_partial_insert=false +#Thu Dec 12 10:43:31 HKT 2024 +#enable_partial_insert=true +#Thu Dec 12 10:43:31 HKT 2024 +#enable_partial_insert=false +#Thu Dec 12 10:43:31 HKT 2024 +#enable_partial_insert=true +#Thu Dec 12 10:43:31 HKT 2024 +#enable_partial_insert=false +#Thu Dec 12 10:45:59 HKT 2024 +#enable_partial_insert=true +#Thu Dec 12 10:46:00 HKT 2024 +#enable_partial_insert=false +#Thu Dec 12 10:46:00 HKT 2024 +#enable_partial_insert=true +#Thu Dec 12 10:46:00 HKT 2024 +#enable_partial_insert=false +#Thu Dec 12 10:46:02 HKT 2024 +#enable_partial_insert=true +#Thu Dec 12 10:46:02 HKT 2024 +#enable_partial_insert=false +#Thu Dec 12 10:46:02 HKT 2024 +#enable_partial_insert=true +#Thu Dec 12 10:46:02 HKT 2024 +enable_partial_insert=false diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index 2efae33aae9..fa0ab8407c5 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -1137,7 +1137,7 @@ flush_thread_count=0 # In one insert (one device, one timestamp, multiple measurements), # if enable partial insert, one measurement failure will not impact other measurements -# effectiveMode: restart +# effectiveMode: hot_reload # Datatype: boolean enable_partial_insert=true diff --git a/pom.xml b/pom.xml index ad0eaea6bcf..112468fa607 100644 --- a/pom.xml +++ b/pom.xml @@ -166,7 +166,7 @@ <thrift.version>0.14.1</thrift.version> <xz.version>1.9</xz.version> <zstd-jni.version>1.5.6-3</zstd-jni.version> - <tsfile.version>1.2.0-241129-SNAPSHOT</tsfile.version> + <tsfile.version>1.2.0-241211-SNAPSHOT</tsfile.version> </properties> <!-- if we claim dependencies in dependencyManagement, then we do not claim
