This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rel/1.2
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/1.2 by this push:
new ed725b4eb4b [To rel/1.2][IOTDB-6157] Check timestamp precision when
insert (#11239)
ed725b4eb4b is described below
commit ed725b4eb4b538d918529b2ad4ce4234f8892c90
Author: Haonan <[email protected]>
AuthorDate: Fri Oct 6 23:58:32 2023 -0500
[To rel/1.2][IOTDB-6157] Check timestamp precision when insert (#11239)
---
.../iotdb/it/env/cluster/MppCommonConfig.java | 7 ++
.../it/env/cluster/MppSharedCommonConfig.java | 7 ++
.../iotdb/it/env/remote/RemoteCommonConfig.java | 5 ++
.../org/apache/iotdb/itbase/env/CommonConfig.java | 2 +
.../org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java | 38 +++++++++-
.../apache/iotdb/db/it/IoTDBDatetimeFormatIT.java | 1 +
.../apache/iotdb/db/it/IoTDBInsertMultiRowIT.java | 11 +++
.../iotdb/session/it/IoTDBSessionSimpleIT.java | 36 +++++++++
.../iotdb/db/protocol/mqtt/MPPPublishHandler.java | 2 +
.../v1/handler/StatementConstructionHandler.java | 3 +
.../v2/handler/StatementConstructionHandler.java | 4 +
.../plan/analyze/LoadTsfileAnalyzer.java | 5 +-
.../db/queryengine/plan/parser/ASTVisitor.java | 2 +
.../plan/parser/StatementGenerator.java | 25 ++++++-
.../iotdb/db/utils/TimestampPrecisionUtils.java | 38 +++++++++-
.../db/utils/TimestampPrecisionUtilsTest.java | 87 ++++++++++++++++++++++
.../resources/conf/iotdb-common.properties | 6 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 13 ++++
.../iotdb/commons/conf/CommonDescriptor.java | 6 ++
pom.xml | 4 +-
20 files changed, 289 insertions(+), 13 deletions(-)
diff --git
a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppCommonConfig.java
b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppCommonConfig.java
index 5d1714f6347..34eae508be8 100644
---
a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppCommonConfig.java
+++
b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppCommonConfig.java
@@ -217,6 +217,13 @@ public class MppCommonConfig extends MppBaseConfig
implements CommonConfig {
return this;
}
+ @Override
+ public CommonConfig setTimestampPrecisionCheckEnabled(boolean
timestampPrecisionCheckEnabled) {
+ setProperty(
+ "timestamp_precision_check_enabled",
String.valueOf(timestampPrecisionCheckEnabled));
+ return this;
+ }
+
@Override
public CommonConfig setEnableMemControl(boolean enableMemControl) {
setProperty("enable_mem_control", String.valueOf(enableMemControl));
diff --git
a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppSharedCommonConfig.java
b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppSharedCommonConfig.java
index 0b1daea98d3..e21dfca3b00 100644
---
a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppSharedCommonConfig.java
+++
b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppSharedCommonConfig.java
@@ -228,6 +228,13 @@ public class MppSharedCommonConfig implements CommonConfig
{
return this;
}
+ @Override
+ public CommonConfig setTimestampPrecisionCheckEnabled(boolean
timestampPrecisionCheckEnabled) {
+ cnConfig.setTimestampPrecisionCheckEnabled(timestampPrecisionCheckEnabled);
+ dnConfig.setTimestampPrecisionCheckEnabled(timestampPrecisionCheckEnabled);
+ return this;
+ }
+
@Override
public CommonConfig setConfigNodeRatisSnapshotTriggerThreshold(
int ratisSnapshotTriggerThreshold) {
diff --git
a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/RemoteCommonConfig.java
b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/RemoteCommonConfig.java
index 177b4b1af3d..7c6615753eb 100644
---
a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/RemoteCommonConfig.java
+++
b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/RemoteCommonConfig.java
@@ -163,6 +163,11 @@ public class RemoteCommonConfig implements CommonConfig {
return this;
}
+ @Override
+ public CommonConfig setTimestampPrecisionCheckEnabled(boolean
timestampPrecisionCheckEnabled) {
+ return this;
+ }
+
@Override
public CommonConfig setConfigNodeRatisSnapshotTriggerThreshold(
int ratisSnapshotTriggerThreshold) {
diff --git
a/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java
b/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java
index df302e79891..943ab1d7c24 100644
---
a/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java
+++
b/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java
@@ -78,6 +78,8 @@ public interface CommonConfig {
CommonConfig setTimestampPrecision(String timestampPrecision);
+ CommonConfig setTimestampPrecisionCheckEnabled(boolean
timestampPrecisionCheckEnabled);
+
CommonConfig setConfigNodeRatisSnapshotTriggerThreshold(int
ratisSnapshotTriggerThreshold);
CommonConfig setMaxDegreeOfIndexNode(int maxDegreeOfIndexNode);
diff --git
a/integration-test/src/test/java/org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java
b/integration-test/src/test/java/org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java
index 780aa75e8c5..19170f7964c 100644
---
a/integration-test/src/test/java/org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java
@@ -133,8 +133,7 @@ public class IOTDBLoadTsFileIT {
statement.execute(String.format("delete database %s",
SchemaConfig.STORAGE_GROUP_0));
statement.execute(String.format("delete database %s",
SchemaConfig.STORAGE_GROUP_1));
- } catch (IoTDBSQLException e) {
- LOGGER.info(String.format("delete storage group message : %s",
e.getMessage()));
+ } catch (IoTDBSQLException ignored) {
}
}
@@ -570,7 +569,7 @@ public class IOTDBLoadTsFileIT {
@Test
public void testLoadWithEmptyTsFile() throws Exception {
- try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir,
"1-0-0-0.tsfile"))) {}
+ try (TsFileGenerator ignored = new TsFileGenerator(new File(tmpDir,
"1-0-0-0.tsfile"))) {}
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
@@ -583,6 +582,39 @@ public class IOTDBLoadTsFileIT {
}
}
+ @Test
+ public void testLoadTsFileWithWrongTimestampPrecision() throws Exception {
+ try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir,
"1-0-0-0.tsfile"))) {
+ generator.registerTimeseries(
+ SchemaConfig.DEVICE_0,
+ Arrays.asList(
+ SchemaConfig.MEASUREMENT_00,
+ SchemaConfig.MEASUREMENT_01,
+ SchemaConfig.MEASUREMENT_02,
+ SchemaConfig.MEASUREMENT_03));
+ generator.registerAlignedTimeseries(
+ SchemaConfig.DEVICE_1,
+ Arrays.asList(
+ SchemaConfig.MEASUREMENT_10,
+ SchemaConfig.MEASUREMENT_11,
+ SchemaConfig.MEASUREMENT_12,
+ SchemaConfig.MEASUREMENT_13));
+ // generate ns timestamp
+ generator.generateData(
+ SchemaConfig.DEVICE_0, 100000, PARTITION_INTERVAL / 10_000, false,
1694689856546000000L);
+ generator.generateData(
+ SchemaConfig.DEVICE_1, 100000, PARTITION_INTERVAL / 10_000, true,
1694689856546000000L);
+ }
+
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+
+ statement.execute(String.format("load \"%s\"",
tmpDir.getAbsolutePath()));
+ } catch (IoTDBSQLException e) {
+ Assert.assertTrue(e.getMessage().contains("Current system timestamp
precision is ms"));
+ }
+ }
+
private static class SchemaConfig {
private static final String STORAGE_GROUP_0 = "root.sg.test_0";
private static final String STORAGE_GROUP_1 = "root.sg.test_1";
diff --git
a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDatetimeFormatIT.java
b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDatetimeFormatIT.java
index 5812d6b432e..bb3a8de7d12 100644
---
a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDatetimeFormatIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDatetimeFormatIT.java
@@ -44,6 +44,7 @@ public class IoTDBDatetimeFormatIT {
@Before
public void setUp() throws Exception {
+
EnvFactory.getEnv().getConfig().getCommonConfig().setTimestampPrecisionCheckEnabled(false);
EnvFactory.getEnv().initClusterEnvironment();
}
diff --git
a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertMultiRowIT.java
b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertMultiRowIT.java
index ef3eec20ad1..faff5a18172 100644
---
a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertMultiRowIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertMultiRowIT.java
@@ -159,4 +159,15 @@ public class IoTDBInsertMultiRowIT {
fail();
}
}
+
+ @Test
+ public void testInsertMultiRowWithWrongTimestampPrecision() {
+ try (Statement st1 = connection.createStatement()) {
+ st1.execute(
+ "insert into root.t1.d99.wt01(timestamp, s1, s2)
values(1618283005586000, 1, 1), (1618283005586001, 1, 2)");
+ fail();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("Current system timestamp precision
is ms"));
+ }
+ }
}
diff --git
a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java
b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java
index acf80a2dfc4..d637fed1f5c 100644
---
a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java
@@ -537,6 +537,42 @@ public class IoTDBSessionSimpleIT {
}
}
+ @Test
+ @Category({LocalStandaloneIT.class, ClusterIT.class})
+ public void insertTabletWithWrongTimestampPrecisionTest() {
+ try (ISession session = EnvFactory.getEnv().getSessionConnection()) {
+ List<MeasurementSchema> schemaList = new ArrayList<>();
+ schemaList.add(new MeasurementSchema("s0", TSDataType.DOUBLE,
TSEncoding.RLE));
+ schemaList.add(new MeasurementSchema("s1", TSDataType.FLOAT,
TSEncoding.RLE));
+ schemaList.add(new MeasurementSchema("s2", TSDataType.INT64,
TSEncoding.RLE));
+ schemaList.add(new MeasurementSchema("s3", TSDataType.INT32,
TSEncoding.RLE));
+ schemaList.add(new MeasurementSchema("s4", TSDataType.BOOLEAN,
TSEncoding.RLE));
+ schemaList.add(new MeasurementSchema("s5", TSDataType.TEXT,
TSEncoding.RLE));
+ schemaList.add(new MeasurementSchema("s6", TSDataType.TEXT,
TSEncoding.RLE));
+
+ Tablet tablet = new Tablet("root.sg1.d1", schemaList);
+ for (long time = 1694689856546000000L; time < 1694689856546000010L;
time++) {
+ int rowIndex = tablet.rowSize++;
+ tablet.addTimestamp(rowIndex, time);
+
+ tablet.addValue(schemaList.get(0).getMeasurementId(), rowIndex,
(double) time);
+ tablet.addValue(schemaList.get(1).getMeasurementId(), rowIndex,
(float) time);
+ tablet.addValue(schemaList.get(2).getMeasurementId(), rowIndex, time);
+ tablet.addValue(schemaList.get(3).getMeasurementId(), rowIndex, (int)
time);
+ tablet.addValue(schemaList.get(4).getMeasurementId(), rowIndex, time %
2 == 0);
+ tablet.addValue(schemaList.get(5).getMeasurementId(), rowIndex, new
Binary("Text" + time));
+ tablet.addValue(schemaList.get(6).getMeasurementId(), rowIndex, "Text"
+ time);
+ }
+
+ if (tablet.rowSize != 0) {
+ session.insertTablet(tablet);
+ tablet.reset();
+ }
+ } catch (Exception e) {
+ Assert.assertTrue(e.getMessage().contains("Current system timestamp
precision is ms"));
+ }
+ }
+
@Test
@Category({LocalStandaloneIT.class, ClusterIT.class})
public void createTimeSeriesWithDoubleTicksTest() {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/mqtt/MPPPublishHandler.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/mqtt/MPPPublishHandler.java
index a23df1d55d8..4063c7c44d1 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/mqtt/MPPPublishHandler.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/mqtt/MPPPublishHandler.java
@@ -34,6 +34,7 @@ import
org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaFetcher;
import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult;
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement;
import org.apache.iotdb.db.utils.CommonUtils;
+import org.apache.iotdb.db.utils.TimestampPrecisionUtils;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.service.rpc.thrift.TSProtocolVersion;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -134,6 +135,7 @@ public class MPPPublishHandler extends
AbstractInterceptHandler {
InsertRowStatement statement = new InsertRowStatement();
statement.setDevicePath(
DataNodeDevicePathCache.getInstance().getPartialPath(event.getDevice()));
+ TimestampPrecisionUtils.checkTimestampPrecision(event.getTimestamp());
statement.setTime(event.getTimestamp());
statement.setMeasurements(event.getMeasurements().toArray(new
String[0]));
if (event.getDataTypes() == null) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v1/handler/StatementConstructionHandler.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v1/handler/StatementConstructionHandler.java
index 9e91aedaa32..eb2d32df72e 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v1/handler/StatementConstructionHandler.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v1/handler/StatementConstructionHandler.java
@@ -23,6 +23,7 @@ import
org.apache.iotdb.db.exception.WriteProcessRejectException;
import org.apache.iotdb.db.protocol.rest.v1.model.InsertTabletRequest;
import
org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache;
import
org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement;
+import org.apache.iotdb.db.utils.TimestampPrecisionUtils;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.utils.Binary;
import org.apache.iotdb.tsfile.utils.BitMap;
@@ -37,6 +38,8 @@ public class StatementConstructionHandler {
public static InsertTabletStatement constructInsertTabletStatement(
InsertTabletRequest insertTabletRequest)
throws MetadataException, WriteProcessRejectException {
+ TimestampPrecisionUtils.checkTimestampPrecision(
+
insertTabletRequest.getTimestamps().get(insertTabletRequest.getTimestamps().size()
- 1));
// construct insert statement
InsertTabletStatement insertStatement = new InsertTabletStatement();
insertStatement.setDevicePath(
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/handler/StatementConstructionHandler.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/handler/StatementConstructionHandler.java
index 2f7fb2bc2b9..928f7d75329 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/handler/StatementConstructionHandler.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/handler/StatementConstructionHandler.java
@@ -27,6 +27,7 @@ import
org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDeviceP
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement;
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement;
import
org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement;
+import org.apache.iotdb.db.utils.TimestampPrecisionUtils;
import org.apache.iotdb.rpc.IoTDBConnectionException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.utils.Binary;
@@ -43,6 +44,8 @@ public class StatementConstructionHandler {
public static InsertTabletStatement constructInsertTabletStatement(
InsertTabletRequest insertTabletRequest)
throws MetadataException, WriteProcessRejectException {
+ TimestampPrecisionUtils.checkTimestampPrecision(
+
insertTabletRequest.getTimestamps().get(insertTabletRequest.getTimestamps().size()
- 1));
// construct insert statement
InsertTabletStatement insertStatement = new InsertTabletStatement();
insertStatement.setDevicePath(
@@ -210,6 +213,7 @@ public class StatementConstructionHandler {
PathUtils.checkIsLegalSingleMeasurementsAndUpdate(
insertRecordsRequest.getMeasurementsList().get(i))
.toArray(new String[0]));
+
TimestampPrecisionUtils.checkTimestampPrecision(insertRecordsRequest.getTimestamps().get(i));
statement.setTime(insertRecordsRequest.getTimestamps().get(i));
statement.setDataTypes(dataTypesList.get(i).toArray(new TSDataType[0]));
List<Object> values =
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java
index 13bd528e56b..5f507dbe1d9 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java
@@ -40,6 +40,7 @@ import
org.apache.iotdb.db.queryengine.plan.statement.metadata.DatabaseSchemaSta
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
import org.apache.iotdb.db.utils.FileLoaderUtils;
+import org.apache.iotdb.db.utils.TimestampPrecisionUtils;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
@@ -125,7 +126,8 @@ public class LoadTsfileAnalyzer {
} catch (Exception e) {
LOGGER.warn(String.format("Parse file %s to resource error.",
tsFile.getPath()), e);
throw new SemanticException(
- String.format("Parse file %s to resource error",
tsFile.getPath()));
+ String.format(
+ "Parse file %s to resource error, because %s",
tsFile.getPath(), e.getMessage()));
}
}
@@ -171,6 +173,7 @@ public class LoadTsfileAnalyzer {
} else {
tsFileResource.deserialize();
}
+
TimestampPrecisionUtils.checkTimestampPrecision(tsFileResource.getFileEndTime());
tsFileResource.setStatus(TsFileResourceStatus.NORMAL);
loadTsFileStatement.addTsFileResource(tsFileResource);
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
index f3e394c5808..2a22cbb2342 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
@@ -196,6 +196,7 @@ import
org.apache.iotdb.db.queryengine.plan.statement.sys.quota.ShowSpaceQuotaSt
import
org.apache.iotdb.db.queryengine.plan.statement.sys.quota.ShowThrottleQuotaStatement;
import org.apache.iotdb.db.schemaengine.template.TemplateAlterOperationType;
import org.apache.iotdb.db.utils.DateTimeUtils;
+import org.apache.iotdb.db.utils.TimestampPrecisionUtils;
import org.apache.iotdb.db.utils.constant.SqlConstant;
import org.apache.iotdb.trigger.api.enums.TriggerEvent;
import org.apache.iotdb.trigger.api.enums.TriggerType;
@@ -1844,6 +1845,7 @@ public class ASTVisitor extends
IoTDBSqlParserBaseVisitor<Statement> {
} else {
timestamp =
parseTimeValue(insertMultiValues.get(i).timeValue(),
DateTimeUtils.currentTime());
+ TimestampPrecisionUtils.checkTimestampPrecision(timestamp);
}
} else {
if (!isTimeDefault) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java
index 74cb4fb8aa2..4e90fdf6e24 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java
@@ -66,6 +66,7 @@ import
org.apache.iotdb.db.queryengine.plan.statement.metadata.template.UnsetSch
import org.apache.iotdb.db.schemaengine.schemaregion.utils.MetaFormatUtils;
import org.apache.iotdb.db.schemaengine.template.TemplateQueryType;
import org.apache.iotdb.db.utils.QueryDataSetUtils;
+import org.apache.iotdb.db.utils.TimestampPrecisionUtils;
import org.apache.iotdb.db.utils.constant.SqlConstant;
import org.apache.iotdb.mpp.rpc.thrift.TDeleteModelMetricsReq;
import org.apache.iotdb.mpp.rpc.thrift.TFetchTimeseriesReq;
@@ -283,6 +284,7 @@ public class StatementGenerator {
InsertRowStatement insertStatement = new InsertRowStatement();
insertStatement.setDevicePath(
DEVICE_PATH_CACHE.getPartialPath(insertRecordReq.getPrefixPath()));
+
TimestampPrecisionUtils.checkTimestampPrecision(insertRecordReq.getTimestamp());
insertStatement.setTime(insertRecordReq.getTimestamp());
insertStatement.setMeasurements(insertRecordReq.getMeasurements().toArray(new
String[0]));
insertStatement.setAligned(insertRecordReq.isAligned);
@@ -298,6 +300,7 @@ public class StatementGenerator {
InsertRowStatement insertStatement = new InsertRowStatement();
insertStatement.setDevicePath(
DEVICE_PATH_CACHE.getPartialPath(insertRecordReq.getPrefixPath()));
+
TimestampPrecisionUtils.checkTimestampPrecision(insertRecordReq.getTimestamp());
insertStatement.setTime(insertRecordReq.getTimestamp());
insertStatement.setMeasurements(insertRecordReq.getMeasurements().toArray(new
String[0]));
insertStatement.setDataTypes(new
TSDataType[insertStatement.getMeasurements().length]);
@@ -316,8 +319,12 @@ public class StatementGenerator {
insertStatement.setDevicePath(
DEVICE_PATH_CACHE.getPartialPath(insertTabletReq.getPrefixPath()));
insertStatement.setMeasurements(insertTabletReq.getMeasurements().toArray(new
String[0]));
- insertStatement.setTimes(
- QueryDataSetUtils.readTimesFromBuffer(insertTabletReq.timestamps,
insertTabletReq.size));
+ long[] timestamps =
+ QueryDataSetUtils.readTimesFromBuffer(insertTabletReq.timestamps,
insertTabletReq.size);
+ if (timestamps.length != 0) {
+
TimestampPrecisionUtils.checkTimestampPrecision(timestamps[timestamps.length -
1]);
+ }
+ insertStatement.setTimes(timestamps);
insertStatement.setColumns(
QueryDataSetUtils.readTabletValuesFromBuffer(
insertTabletReq.values,
@@ -349,8 +356,12 @@ public class StatementGenerator {
InsertTabletStatement insertTabletStatement = new
InsertTabletStatement();
insertTabletStatement.setDevicePath(DEVICE_PATH_CACHE.getPartialPath(req.prefixPaths.get(i)));
insertTabletStatement.setMeasurements(req.measurementsList.get(i).toArray(new
String[0]));
- insertTabletStatement.setTimes(
- QueryDataSetUtils.readTimesFromBuffer(req.timestampsList.get(i),
req.sizeList.get(i)));
+ long[] timestamps =
+ QueryDataSetUtils.readTimesFromBuffer(req.timestampsList.get(i),
req.sizeList.get(i));
+ if (timestamps.length != 0) {
+
TimestampPrecisionUtils.checkTimestampPrecision(timestamps[timestamps.length -
1]);
+ }
+ insertTabletStatement.setTimes(timestamps);
insertTabletStatement.setColumns(
QueryDataSetUtils.readTabletValuesFromBuffer(
req.valuesList.get(i),
@@ -389,6 +400,7 @@ public class StatementGenerator {
InsertRowStatement statement = new InsertRowStatement();
statement.setDevicePath(DEVICE_PATH_CACHE.getPartialPath(req.getPrefixPaths().get(i)));
statement.setMeasurements(req.getMeasurementsList().get(i).toArray(new
String[0]));
+
TimestampPrecisionUtils.checkTimestampPrecision(req.getTimestamps().get(i));
statement.setTime(req.getTimestamps().get(i));
statement.fillValues(req.valuesList.get(i));
statement.setAligned(req.isAligned);
@@ -415,6 +427,7 @@ public class StatementGenerator {
addMeasurementAndValue(
statement, req.getMeasurementsList().get(i),
req.getValuesList().get(i));
statement.setDataTypes(new
TSDataType[statement.getMeasurements().length]);
+
TimestampPrecisionUtils.checkTimestampPrecision(req.getTimestamps().get(i));
statement.setTime(req.getTimestamps().get(i));
statement.setNeedInferType(true);
statement.setAligned(req.isAligned);
@@ -436,6 +449,8 @@ public class StatementGenerator {
InsertRowsOfOneDeviceStatement insertStatement = new
InsertRowsOfOneDeviceStatement();
insertStatement.setDevicePath(DEVICE_PATH_CACHE.getPartialPath(req.prefixPath));
List<InsertRowStatement> insertRowStatementList = new ArrayList<>();
+ // req.timestamps sorted on session side
+
TimestampPrecisionUtils.checkTimestampPrecision(req.timestamps.get(req.timestamps.size()
- 1));
for (int i = 0; i < req.timestamps.size(); i++) {
InsertRowStatement statement = new InsertRowStatement();
statement.setDevicePath(insertStatement.getDevicePath());
@@ -461,6 +476,8 @@ public class StatementGenerator {
InsertRowsOfOneDeviceStatement insertStatement = new
InsertRowsOfOneDeviceStatement();
insertStatement.setDevicePath(DEVICE_PATH_CACHE.getPartialPath(req.prefixPath));
List<InsertRowStatement> insertRowStatementList = new ArrayList<>();
+ // req.timestamps sorted on session side
+
TimestampPrecisionUtils.checkTimestampPrecision(req.timestamps.get(req.timestamps.size()
- 1));
for (int i = 0; i < req.timestamps.size(); i++) {
InsertRowStatement statement = new InsertRowStatement();
statement.setDevicePath(insertStatement.getDevicePath());
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimestampPrecisionUtils.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimestampPrecisionUtils.java
index e613dacf908..d5fdf658178 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimestampPrecisionUtils.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimestampPrecisionUtils.java
@@ -19,12 +19,15 @@
package org.apache.iotdb.db.utils;
import org.apache.iotdb.commons.conf.CommonDescriptor;
+import org.apache.iotdb.db.exception.sql.SemanticException;
import java.util.concurrent.TimeUnit;
public class TimestampPrecisionUtils {
- private static final String timestampPrecision =
+ static String TIMESTAMP_PRECISION =
CommonDescriptor.getInstance().getConfig().getTimestampPrecision();
+ private static boolean isTimestampPrecisionCheckEnabled =
+
CommonDescriptor.getInstance().getConfig().isTimestampPrecisionCheckEnabled();
@FunctionalInterface
private interface ConvertFunction<T1, T2, R> {
@@ -34,7 +37,7 @@ public class TimestampPrecisionUtils {
private static final ConvertFunction<Long, TimeUnit, Long> convertFunction;
static {
- switch (timestampPrecision) {
+ switch (TIMESTAMP_PRECISION) {
case "ms":
convertFunction = TimeUnit.MILLISECONDS::convert;
break;
@@ -54,4 +57,35 @@ public class TimestampPrecisionUtils {
public static long convertToCurrPrecision(long sourceTime, TimeUnit
sourceUnit) {
return convertFunction.apply(sourceTime, sourceUnit);
}
+
+ /** check whether the input timestamp match the current system timestamp
precision. */
+ public static void checkTimestampPrecision(long time) {
+ if (!isTimestampPrecisionCheckEnabled) {
+ return;
+ }
+ switch (TIMESTAMP_PRECISION) {
+ case "ms":
+ if (time > 10_000_000_000_000L) {
+ throw new SemanticException(
+ String.format(
+ "Current system timestamp precision is %s, "
+ + "please check whether the timestamp %s is correct.",
+ TIMESTAMP_PRECISION, time));
+ }
+ break;
+ case "us":
+ if (time > 10_000_000_000_000_000L) {
+ throw new SemanticException(
+ String.format(
+ "Current system timestamp precision is %s, "
+ + "please check whether the timestamp %s is correct.",
+ TIMESTAMP_PRECISION, time));
+ }
+ break;
+ // Long.MaxValue is 19 digits, therefore no problem when the precision
is ns.
+ case "ns":
+ default:
+ break;
+ }
+ }
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/TimestampPrecisionUtilsTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/TimestampPrecisionUtilsTest.java
new file mode 100644
index 00000000000..87585cc663f
--- /dev/null
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/TimestampPrecisionUtilsTest.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.utils;
+
+import org.junit.Test;
+
+import static org.junit.Assert.fail;
+
+public class TimestampPrecisionUtilsTest {
+
+ @Test
+ public void testCheckMsTimestampPrecision() {
+ TimestampPrecisionUtils.TIMESTAMP_PRECISION = "ms";
+ try {
+ TimestampPrecisionUtils.checkTimestampPrecision(-1L);
+ TimestampPrecisionUtils.checkTimestampPrecision(0L);
+ TimestampPrecisionUtils.checkTimestampPrecision(1694689856546L);
+ } catch (Exception e) {
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCheckIllegalMsTimestampPrecision() {
+ TimestampPrecisionUtils.TIMESTAMP_PRECISION = "ms";
+ try {
+ TimestampPrecisionUtils.checkTimestampPrecision(1694689856546000L);
+ fail();
+ } catch (Exception ignored) {
+ }
+ }
+
+ @Test
+ public void testCheckUsTimestampPrecision() {
+ TimestampPrecisionUtils.TIMESTAMP_PRECISION = "us";
+ try {
+ TimestampPrecisionUtils.checkTimestampPrecision(-1L);
+ TimestampPrecisionUtils.checkTimestampPrecision(0L);
+ TimestampPrecisionUtils.checkTimestampPrecision(1694689856546L);
+ TimestampPrecisionUtils.checkTimestampPrecision(1694689856546000L);
+ } catch (Exception e) {
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCheckIllegalUsTimestampPrecision() {
+ TimestampPrecisionUtils.TIMESTAMP_PRECISION = "us";
+ try {
+ TimestampPrecisionUtils.checkTimestampPrecision(1694689856546000000L);
+ fail();
+ } catch (Exception ignored) {
+ }
+ }
+
+ @Test
+ public void testCheckNsTimestampPrecision() {
+ TimestampPrecisionUtils.TIMESTAMP_PRECISION = "ns";
+ try {
+ TimestampPrecisionUtils.checkTimestampPrecision(-1L);
+ TimestampPrecisionUtils.checkTimestampPrecision(0L);
+ TimestampPrecisionUtils.checkTimestampPrecision(1694689856546L);
+ TimestampPrecisionUtils.checkTimestampPrecision(1694689856546000L);
+ TimestampPrecisionUtils.checkTimestampPrecision(1694689856546000000L);
+ TimestampPrecisionUtils.checkTimestampPrecision(Long.MAX_VALUE);
+ } catch (Exception e) {
+ fail(e.getMessage());
+ }
+ }
+}
diff --git
a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties
b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties
index e27e572b213..149c97e5674 100644
---
a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties
+++
b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties
@@ -449,7 +449,11 @@ cluster_name=defaultCluster
# Use this value to set timestamp precision as "ms", "us" or "ns".
# Once the precision has been set, it can not be changed.
# Datatype: String
-# timestamp_precision=ms
+timestamp_precision=ms
+
+# When the timestamp precision check is enabled, the timestamps those are over
13 digits for ms precision, or over 16 digits for us precision are not allowed
to be inserted.
+# Datatype: Boolean
+# timestamp_precision_check_enabled=true
# Default TTL for databases that are not set TTL by statements, If not set
(default), the TTL will be unlimited.
# Negative value means the TTL is unlimited.
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
index adc5cee2158..2556b6155d4 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
@@ -137,6 +137,11 @@ public class CommonConfig {
/** This variable set timestamp precision as millisecond, microsecond or
nanosecond. */
private String timestampPrecision = "ms";
+ private boolean timestampPrecisionCheckEnabled = true;
+
+ /** The number of threads in the thread pool that execute model inference
tasks. */
+ private int modelInferenceExecutionThreadCount = 5;
+
/**
* The name of the directory that stores the tsfiles temporarily hold or
generated by the pipe
* module. The directory is located in the data directory of IoTDB.
@@ -472,6 +477,14 @@ public class CommonConfig {
return timestampPrecision;
}
+ public void setTimestampPrecisionCheckEnabled(boolean
timestampPrecisionCheckEnabled) {
+ this.timestampPrecisionCheckEnabled = timestampPrecisionCheckEnabled;
+ }
+
+ public boolean isTimestampPrecisionCheckEnabled() {
+ return timestampPrecisionCheckEnabled;
+ }
+
public String getPipeHardlinkBaseDirName() {
return pipeHardlinkBaseDirName;
}
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java
index 76925891899..1c008e7e8e7 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java
@@ -200,6 +200,12 @@ public class CommonDescriptor {
config.setTimestampPrecision(
properties.getProperty("timestamp_precision",
config.getTimestampPrecision()).trim());
+ config.setTimestampPrecisionCheckEnabled(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "timestamp_precision_check_enabled",
+ String.valueOf(config.isTimestampPrecisionCheckEnabled()))));
+
String endPointUrl =
properties.getProperty(
"target_ml_node_endpoint",
diff --git a/pom.xml b/pom.xml
index ded8698fcf8..9530aa5fcd9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1194,6 +1194,7 @@
</activation>
<properties>
<maven.compiler.release>8</maven.compiler.release>
+ <argLine>--illegal-access=permit
--add-opens=java.base/java.util.concurrent=ALL-UNNAMED
--add-opens=java.base/java.lang=ALL-UNNAMED
--add-opens=java.base/java.util=ALL-UNNAMED
--add-opens=java.base/java.nio=ALL-UNNAMED
--add-opens=java.base/java.io=ALL-UNNAMED
--add-opens=java.base/java.net=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.java [...]
<!-- change to 1.15.0 will modify many codes (all are
javadocs), we change it to 1.15.0
until: iotdb decides to do not support jdk8-->
<google.java.format.version>1.7</google.java.format.version>
@@ -1223,8 +1224,7 @@
<jdk>[16,20)</jdk>
</activation>
<properties>
- <maven.compiler.release>8</maven.compiler.release>
- <argLine>--illegal-access=permit
--add-opens=java.base/java.util.concurrent=ALL-UNNAMED
--add-opens=java.base/java.lang=ALL-UNNAMED
--add-opens=java.base/java.util=ALL-UNNAMED
--add-opens=java.base/java.nio=ALL-UNNAMED
--add-opens=java.base/java.io=ALL-UNNAMED
--add-opens=java.base/java.net=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.java [...]
+
<argLine>--add-opens=java.base/java.util.concurrent=ALL-UNNAMED
--add-opens=java.base/java.lang=ALL-UNNAMED
--add-opens=java.base/java.util=ALL-UNNAMED
--add-opens=java.base/java.nio=ALL-UNNAMED
--add-opens=java.base/java.io=ALL-UNNAMED
--add-opens=java.base/java.net=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add [...]
</properties>
</profile>
<!--
