This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch timestamp_check12 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 7d5469791b4a99c8e952825a2767acc3ad2976d4 Author: Haonan <[email protected]> AuthorDate: Mon Sep 18 17:26:51 2023 +0800 [IOTDB-6157] Check timestamp precision when insert (#11151) --- .../iotdb/it/env/cluster/MppCommonConfig.java | 7 ++ .../it/env/cluster/MppSharedCommonConfig.java | 7 ++ .../iotdb/it/env/remote/RemoteCommonConfig.java | 5 ++ .../org/apache/iotdb/itbase/env/CommonConfig.java | 2 + .../org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java | 38 +++++++++- .../apache/iotdb/db/it/IoTDBDatetimeFormatIT.java | 1 + .../apache/iotdb/db/it/IoTDBInsertMultiRowIT.java | 11 +++ .../iotdb/session/it/IoTDBSessionSimpleIT.java | 36 +++++++++ .../iotdb/db/protocol/mqtt/MPPPublishHandler.java | 2 + .../v1/handler/StatementConstructionHandler.java | 3 + .../v2/handler/StatementConstructionHandler.java | 6 +- .../plan/analyze/LoadTsfileAnalyzer.java | 5 +- .../db/queryengine/plan/parser/ASTVisitor.java | 2 + .../plan/parser/StatementGenerator.java | 25 ++++++- .../iotdb/db/utils/TimestampPrecisionUtils.java | 36 ++++++++- .../db/utils/TimestampPrecisionUtilsTest.java | 87 ++++++++++++++++++++++ .../resources/conf/iotdb-common.properties | 6 +- .../apache/iotdb/commons/conf/CommonConfig.java | 13 ++++ .../iotdb/commons/conf/CommonDescriptor.java | 10 +++ pom.xml | 4 +- 20 files changed, 293 insertions(+), 13 deletions(-) diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppCommonConfig.java index 5d1714f6347..34eae508be8 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppCommonConfig.java @@ -217,6 +217,13 @@ public class MppCommonConfig extends MppBaseConfig implements CommonConfig { return this; } + @Override + public CommonConfig setTimestampPrecisionCheckEnabled(boolean timestampPrecisionCheckEnabled) { + setProperty( + "timestamp_precision_check_enabled", String.valueOf(timestampPrecisionCheckEnabled)); + return this; + } + @Override public CommonConfig setEnableMemControl(boolean enableMemControl) { setProperty("enable_mem_control", String.valueOf(enableMemControl)); diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppSharedCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppSharedCommonConfig.java index 0b1daea98d3..e21dfca3b00 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppSharedCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppSharedCommonConfig.java @@ -228,6 +228,13 @@ public class MppSharedCommonConfig implements CommonConfig { return this; } + @Override + public CommonConfig setTimestampPrecisionCheckEnabled(boolean timestampPrecisionCheckEnabled) { + cnConfig.setTimestampPrecisionCheckEnabled(timestampPrecisionCheckEnabled); + dnConfig.setTimestampPrecisionCheckEnabled(timestampPrecisionCheckEnabled); + return this; + } + @Override public CommonConfig setConfigNodeRatisSnapshotTriggerThreshold( int ratisSnapshotTriggerThreshold) { diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/RemoteCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/RemoteCommonConfig.java index 177b4b1af3d..7c6615753eb 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/RemoteCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/RemoteCommonConfig.java @@ -163,6 +163,11 @@ public class RemoteCommonConfig implements CommonConfig { return this; } + @Override + public CommonConfig setTimestampPrecisionCheckEnabled(boolean timestampPrecisionCheckEnabled) { + return this; + } + @Override public CommonConfig setConfigNodeRatisSnapshotTriggerThreshold( int ratisSnapshotTriggerThreshold) { diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java index df302e79891..943ab1d7c24 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java @@ -78,6 +78,8 @@ public interface CommonConfig { CommonConfig setTimestampPrecision(String timestampPrecision); + CommonConfig setTimestampPrecisionCheckEnabled(boolean timestampPrecisionCheckEnabled); + CommonConfig setConfigNodeRatisSnapshotTriggerThreshold(int ratisSnapshotTriggerThreshold); CommonConfig setMaxDegreeOfIndexNode(int maxDegreeOfIndexNode); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java index 780aa75e8c5..19170f7964c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java @@ -133,8 +133,7 @@ public class IOTDBLoadTsFileIT { statement.execute(String.format("delete database %s", SchemaConfig.STORAGE_GROUP_0)); statement.execute(String.format("delete database %s", SchemaConfig.STORAGE_GROUP_1)); - } catch (IoTDBSQLException e) { - LOGGER.info(String.format("delete storage group message : %s", e.getMessage())); + } catch (IoTDBSQLException ignored) { } } @@ -570,7 +569,7 @@ public class IOTDBLoadTsFileIT { @Test public void testLoadWithEmptyTsFile() throws Exception { - try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) {} + try (TsFileGenerator ignored = new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) {} try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -583,6 +582,39 @@ public class IOTDBLoadTsFileIT { } } + @Test + public void testLoadTsFileWithWrongTimestampPrecision() throws Exception { + try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) { + generator.registerTimeseries( + SchemaConfig.DEVICE_0, + Arrays.asList( + SchemaConfig.MEASUREMENT_00, + SchemaConfig.MEASUREMENT_01, + SchemaConfig.MEASUREMENT_02, + SchemaConfig.MEASUREMENT_03)); + generator.registerAlignedTimeseries( + SchemaConfig.DEVICE_1, + Arrays.asList( + SchemaConfig.MEASUREMENT_10, + SchemaConfig.MEASUREMENT_11, + SchemaConfig.MEASUREMENT_12, + SchemaConfig.MEASUREMENT_13)); + // generate ns timestamp + generator.generateData( + SchemaConfig.DEVICE_0, 100000, PARTITION_INTERVAL / 10_000, false, 1694689856546000000L); + generator.generateData( + SchemaConfig.DEVICE_1, 100000, PARTITION_INTERVAL / 10_000, true, 1694689856546000000L); + } + + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + statement.execute(String.format("load \"%s\"", tmpDir.getAbsolutePath())); + } catch (IoTDBSQLException e) { + Assert.assertTrue(e.getMessage().contains("Current system timestamp precision is ms")); + } + } + private static class SchemaConfig { private static final String STORAGE_GROUP_0 = "root.sg.test_0"; private static final String STORAGE_GROUP_1 = "root.sg.test_1"; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDatetimeFormatIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDatetimeFormatIT.java index 5812d6b432e..bb3a8de7d12 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDatetimeFormatIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDatetimeFormatIT.java @@ -44,6 +44,7 @@ public class IoTDBDatetimeFormatIT { @Before public void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setTimestampPrecisionCheckEnabled(false); EnvFactory.getEnv().initClusterEnvironment(); } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertMultiRowIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertMultiRowIT.java index ef3eec20ad1..faff5a18172 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertMultiRowIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertMultiRowIT.java @@ -159,4 +159,15 @@ public class IoTDBInsertMultiRowIT { fail(); } } + + @Test + public void testInsertMultiRowWithWrongTimestampPrecision() { + try (Statement st1 = connection.createStatement()) { + st1.execute( + "insert into root.t1.d99.wt01(timestamp, s1, s2) values(1618283005586000, 1, 1), (1618283005586001, 1, 2)"); + fail(); + } catch (SQLException e) { + assertTrue(e.getMessage().contains("Current system timestamp precision is ms")); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java index acf80a2dfc4..d637fed1f5c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java @@ -537,6 +537,42 @@ public class IoTDBSessionSimpleIT { } } + @Test + @Category({LocalStandaloneIT.class, ClusterIT.class}) + public void insertTabletWithWrongTimestampPrecisionTest() { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + List<MeasurementSchema> schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("s0", TSDataType.DOUBLE, TSEncoding.RLE)); + schemaList.add(new MeasurementSchema("s1", TSDataType.FLOAT, TSEncoding.RLE)); + schemaList.add(new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.RLE)); + schemaList.add(new MeasurementSchema("s3", TSDataType.INT32, TSEncoding.RLE)); + schemaList.add(new MeasurementSchema("s4", TSDataType.BOOLEAN, TSEncoding.RLE)); + schemaList.add(new MeasurementSchema("s5", TSDataType.TEXT, TSEncoding.RLE)); + schemaList.add(new MeasurementSchema("s6", TSDataType.TEXT, TSEncoding.RLE)); + + Tablet tablet = new Tablet("root.sg1.d1", schemaList); + for (long time = 1694689856546000000L; time < 1694689856546000010L; time++) { + int rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, time); + + tablet.addValue(schemaList.get(0).getMeasurementId(), rowIndex, (double) time); + tablet.addValue(schemaList.get(1).getMeasurementId(), rowIndex, (float) time); + tablet.addValue(schemaList.get(2).getMeasurementId(), rowIndex, time); + tablet.addValue(schemaList.get(3).getMeasurementId(), rowIndex, (int) time); + tablet.addValue(schemaList.get(4).getMeasurementId(), rowIndex, time % 2 == 0); + tablet.addValue(schemaList.get(5).getMeasurementId(), rowIndex, new Binary("Text" + time)); + tablet.addValue(schemaList.get(6).getMeasurementId(), rowIndex, "Text" + time); + } + + if (tablet.rowSize != 0) { + session.insertTablet(tablet); + tablet.reset(); + } + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("Current system timestamp precision is ms")); + } + } + @Test @Category({LocalStandaloneIT.class, ClusterIT.class}) public void createTimeSeriesWithDoubleTicksTest() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/mqtt/MPPPublishHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/mqtt/MPPPublishHandler.java index a23df1d55d8..4063c7c44d1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/mqtt/MPPPublishHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/mqtt/MPPPublishHandler.java @@ -34,6 +34,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaFetcher; import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.iotdb.db.utils.CommonUtils; +import org.apache.iotdb.db.utils.TimestampPrecisionUtils; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.service.rpc.thrift.TSProtocolVersion; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; @@ -134,6 +135,7 @@ public class MPPPublishHandler extends AbstractInterceptHandler { InsertRowStatement statement = new InsertRowStatement(); statement.setDevicePath( DataNodeDevicePathCache.getInstance().getPartialPath(event.getDevice())); + TimestampPrecisionUtils.checkTimestampPrecision(event.getTimestamp()); statement.setTime(event.getTimestamp()); statement.setMeasurements(event.getMeasurements().toArray(new String[0])); if (event.getDataTypes() == null) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v1/handler/StatementConstructionHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v1/handler/StatementConstructionHandler.java index 9e91aedaa32..eb2d32df72e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v1/handler/StatementConstructionHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v1/handler/StatementConstructionHandler.java @@ -23,6 +23,7 @@ import org.apache.iotdb.db.exception.WriteProcessRejectException; import org.apache.iotdb.db.protocol.rest.v1.model.InsertTabletRequest; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; +import org.apache.iotdb.db.utils.TimestampPrecisionUtils; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.utils.Binary; import org.apache.iotdb.tsfile.utils.BitMap; @@ -37,6 +38,8 @@ public class StatementConstructionHandler { public static InsertTabletStatement constructInsertTabletStatement( InsertTabletRequest insertTabletRequest) throws MetadataException, WriteProcessRejectException { + TimestampPrecisionUtils.checkTimestampPrecision( + insertTabletRequest.getTimestamps().get(insertTabletRequest.getTimestamps().size() - 1)); // construct insert statement InsertTabletStatement insertStatement = new InsertTabletStatement(); insertStatement.setDevicePath( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/handler/StatementConstructionHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/handler/StatementConstructionHandler.java index 2f7fb2bc2b9..ed5f1b05ca4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/handler/StatementConstructionHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/handler/StatementConstructionHandler.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDeviceP import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; +import org.apache.iotdb.db.utils.TimestampPrecisionUtils; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.utils.Binary; @@ -42,7 +43,9 @@ public class StatementConstructionHandler { public static InsertTabletStatement constructInsertTabletStatement( InsertTabletRequest insertTabletRequest) - throws MetadataException, WriteProcessRejectException { + throws IllegalPathException, WriteProcessRejectException { + TimestampPrecisionUtils.checkTimestampPrecision( + insertTabletRequest.getTimestamps().get(insertTabletRequest.getTimestamps().size() - 1)); // construct insert statement InsertTabletStatement insertStatement = new InsertTabletStatement(); insertStatement.setDevicePath( @@ -210,6 +213,7 @@ public class StatementConstructionHandler { PathUtils.checkIsLegalSingleMeasurementsAndUpdate( insertRecordsRequest.getMeasurementsList().get(i)) .toArray(new String[0])); + TimestampPrecisionUtils.checkTimestampPrecision(insertRecordsRequest.getTimestamps().get(i)); statement.setTime(insertRecordsRequest.getTimestamps().get(i)); statement.setDataTypes(dataTypesList.get(i).toArray(new TSDataType[0])); List<Object> values = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java index 13bd528e56b..5f507dbe1d9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java @@ -40,6 +40,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.DatabaseSchemaSta import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; import org.apache.iotdb.db.utils.FileLoaderUtils; +import org.apache.iotdb.db.utils.TimestampPrecisionUtils; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.tsfile.common.constant.TsFileConstant; import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; @@ -125,7 +126,8 @@ public class LoadTsfileAnalyzer { } catch (Exception e) { LOGGER.warn(String.format("Parse file %s to resource error.", tsFile.getPath()), e); throw new SemanticException( - String.format("Parse file %s to resource error", tsFile.getPath())); + String.format( + "Parse file %s to resource error, because %s", tsFile.getPath(), e.getMessage())); } } @@ -171,6 +173,7 @@ public class LoadTsfileAnalyzer { } else { tsFileResource.deserialize(); } + TimestampPrecisionUtils.checkTimestampPrecision(tsFileResource.getFileEndTime()); tsFileResource.setStatus(TsFileResourceStatus.NORMAL); loadTsFileStatement.addTsFileResource(tsFileResource); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java index f3e394c5808..2a22cbb2342 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java @@ -196,6 +196,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.quota.ShowSpaceQuotaSt import org.apache.iotdb.db.queryengine.plan.statement.sys.quota.ShowThrottleQuotaStatement; import org.apache.iotdb.db.schemaengine.template.TemplateAlterOperationType; import org.apache.iotdb.db.utils.DateTimeUtils; +import org.apache.iotdb.db.utils.TimestampPrecisionUtils; import org.apache.iotdb.db.utils.constant.SqlConstant; import org.apache.iotdb.trigger.api.enums.TriggerEvent; import org.apache.iotdb.trigger.api.enums.TriggerType; @@ -1844,6 +1845,7 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> { } else { timestamp = parseTimeValue(insertMultiValues.get(i).timeValue(), DateTimeUtils.currentTime()); + TimestampPrecisionUtils.checkTimestampPrecision(timestamp); } } else { if (!isTimeDefault) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java index 74cb4fb8aa2..4e90fdf6e24 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java @@ -66,6 +66,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.UnsetSch import org.apache.iotdb.db.schemaengine.schemaregion.utils.MetaFormatUtils; import org.apache.iotdb.db.schemaengine.template.TemplateQueryType; import org.apache.iotdb.db.utils.QueryDataSetUtils; +import org.apache.iotdb.db.utils.TimestampPrecisionUtils; import org.apache.iotdb.db.utils.constant.SqlConstant; import org.apache.iotdb.mpp.rpc.thrift.TDeleteModelMetricsReq; import org.apache.iotdb.mpp.rpc.thrift.TFetchTimeseriesReq; @@ -283,6 +284,7 @@ public class StatementGenerator { InsertRowStatement insertStatement = new InsertRowStatement(); insertStatement.setDevicePath( DEVICE_PATH_CACHE.getPartialPath(insertRecordReq.getPrefixPath())); + TimestampPrecisionUtils.checkTimestampPrecision(insertRecordReq.getTimestamp()); insertStatement.setTime(insertRecordReq.getTimestamp()); insertStatement.setMeasurements(insertRecordReq.getMeasurements().toArray(new String[0])); insertStatement.setAligned(insertRecordReq.isAligned); @@ -298,6 +300,7 @@ public class StatementGenerator { InsertRowStatement insertStatement = new InsertRowStatement(); insertStatement.setDevicePath( DEVICE_PATH_CACHE.getPartialPath(insertRecordReq.getPrefixPath())); + TimestampPrecisionUtils.checkTimestampPrecision(insertRecordReq.getTimestamp()); insertStatement.setTime(insertRecordReq.getTimestamp()); insertStatement.setMeasurements(insertRecordReq.getMeasurements().toArray(new String[0])); insertStatement.setDataTypes(new TSDataType[insertStatement.getMeasurements().length]); @@ -316,8 +319,12 @@ public class StatementGenerator { insertStatement.setDevicePath( DEVICE_PATH_CACHE.getPartialPath(insertTabletReq.getPrefixPath())); insertStatement.setMeasurements(insertTabletReq.getMeasurements().toArray(new String[0])); - insertStatement.setTimes( - QueryDataSetUtils.readTimesFromBuffer(insertTabletReq.timestamps, insertTabletReq.size)); + long[] timestamps = + QueryDataSetUtils.readTimesFromBuffer(insertTabletReq.timestamps, insertTabletReq.size); + if (timestamps.length != 0) { + TimestampPrecisionUtils.checkTimestampPrecision(timestamps[timestamps.length - 1]); + } + insertStatement.setTimes(timestamps); insertStatement.setColumns( QueryDataSetUtils.readTabletValuesFromBuffer( insertTabletReq.values, @@ -349,8 +356,12 @@ public class StatementGenerator { InsertTabletStatement insertTabletStatement = new InsertTabletStatement(); insertTabletStatement.setDevicePath(DEVICE_PATH_CACHE.getPartialPath(req.prefixPaths.get(i))); insertTabletStatement.setMeasurements(req.measurementsList.get(i).toArray(new String[0])); - insertTabletStatement.setTimes( - QueryDataSetUtils.readTimesFromBuffer(req.timestampsList.get(i), req.sizeList.get(i))); + long[] timestamps = + QueryDataSetUtils.readTimesFromBuffer(req.timestampsList.get(i), req.sizeList.get(i)); + if (timestamps.length != 0) { + TimestampPrecisionUtils.checkTimestampPrecision(timestamps[timestamps.length - 1]); + } + insertTabletStatement.setTimes(timestamps); insertTabletStatement.setColumns( QueryDataSetUtils.readTabletValuesFromBuffer( req.valuesList.get(i), @@ -389,6 +400,7 @@ public class StatementGenerator { InsertRowStatement statement = new InsertRowStatement(); statement.setDevicePath(DEVICE_PATH_CACHE.getPartialPath(req.getPrefixPaths().get(i))); statement.setMeasurements(req.getMeasurementsList().get(i).toArray(new String[0])); + TimestampPrecisionUtils.checkTimestampPrecision(req.getTimestamps().get(i)); statement.setTime(req.getTimestamps().get(i)); statement.fillValues(req.valuesList.get(i)); statement.setAligned(req.isAligned); @@ -415,6 +427,7 @@ public class StatementGenerator { addMeasurementAndValue( statement, req.getMeasurementsList().get(i), req.getValuesList().get(i)); statement.setDataTypes(new TSDataType[statement.getMeasurements().length]); + TimestampPrecisionUtils.checkTimestampPrecision(req.getTimestamps().get(i)); statement.setTime(req.getTimestamps().get(i)); statement.setNeedInferType(true); statement.setAligned(req.isAligned); @@ -436,6 +449,8 @@ public class StatementGenerator { InsertRowsOfOneDeviceStatement insertStatement = new InsertRowsOfOneDeviceStatement(); insertStatement.setDevicePath(DEVICE_PATH_CACHE.getPartialPath(req.prefixPath)); List<InsertRowStatement> insertRowStatementList = new ArrayList<>(); + // req.timestamps sorted on session side + TimestampPrecisionUtils.checkTimestampPrecision(req.timestamps.get(req.timestamps.size() - 1)); for (int i = 0; i < req.timestamps.size(); i++) { InsertRowStatement statement = new InsertRowStatement(); statement.setDevicePath(insertStatement.getDevicePath()); @@ -461,6 +476,8 @@ public class StatementGenerator { InsertRowsOfOneDeviceStatement insertStatement = new InsertRowsOfOneDeviceStatement(); insertStatement.setDevicePath(DEVICE_PATH_CACHE.getPartialPath(req.prefixPath)); List<InsertRowStatement> insertRowStatementList = new ArrayList<>(); + // req.timestamps sorted on session side + TimestampPrecisionUtils.checkTimestampPrecision(req.timestamps.get(req.timestamps.size() - 1)); for (int i = 0; i < req.timestamps.size(); i++) { InsertRowStatement statement = new InsertRowStatement(); statement.setDevicePath(insertStatement.getDevicePath()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimestampPrecisionUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimestampPrecisionUtils.java index e613dacf908..5b7bafc3b34 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimestampPrecisionUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimestampPrecisionUtils.java @@ -19,12 +19,15 @@ package org.apache.iotdb.db.utils; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.db.exception.sql.SemanticException; import java.util.concurrent.TimeUnit; public class TimestampPrecisionUtils { - private static final String timestampPrecision = + static String TIMESTAMP_PRECISION = CommonDescriptor.getInstance().getConfig().getTimestampPrecision(); + private static boolean isTimestampPrecisionCheckEnabled = + CommonDescriptor.getInstance().getConfig().isTimestampPrecisionCheckEnabled(); @FunctionalInterface private interface ConvertFunction<T1, T2, R> { @@ -54,4 +57,35 @@ public class TimestampPrecisionUtils { public static long convertToCurrPrecision(long sourceTime, TimeUnit sourceUnit) { return convertFunction.apply(sourceTime, sourceUnit); } + + /** check whether the input timestamp match the current system timestamp precision. */ + public static void checkTimestampPrecision(long time) { + if (!isTimestampPrecisionCheckEnabled) { + return; + } + switch (TIMESTAMP_PRECISION) { + case "ms": + if (time > 10_000_000_000_000L) { + throw new SemanticException( + String.format( + "Current system timestamp precision is %s, " + + "please check whether the timestamp %s is correct.", + TIMESTAMP_PRECISION, time)); + } + break; + case "us": + if (time > 10_000_000_000_000_000L) { + throw new SemanticException( + String.format( + "Current system timestamp precision is %s, " + + "please check whether the timestamp %s is correct.", + TIMESTAMP_PRECISION, time)); + } + break; + // Long.MaxValue is 19 digits, therefore no problem when the precision is ns. + case "ns": + default: + break; + } + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/TimestampPrecisionUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/TimestampPrecisionUtilsTest.java new file mode 100644 index 00000000000..87585cc663f --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/TimestampPrecisionUtilsTest.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.utils; + +import org.junit.Test; + +import static org.junit.Assert.fail; + +public class TimestampPrecisionUtilsTest { + + @Test + public void testCheckMsTimestampPrecision() { + TimestampPrecisionUtils.TIMESTAMP_PRECISION = "ms"; + try { + TimestampPrecisionUtils.checkTimestampPrecision(-1L); + TimestampPrecisionUtils.checkTimestampPrecision(0L); + TimestampPrecisionUtils.checkTimestampPrecision(1694689856546L); + } catch (Exception e) { + fail(e.getMessage()); + } + } + + @Test + public void testCheckIllegalMsTimestampPrecision() { + TimestampPrecisionUtils.TIMESTAMP_PRECISION = "ms"; + try { + TimestampPrecisionUtils.checkTimestampPrecision(1694689856546000L); + fail(); + } catch (Exception ignored) { + } + } + + @Test + public void testCheckUsTimestampPrecision() { + TimestampPrecisionUtils.TIMESTAMP_PRECISION = "us"; + try { + TimestampPrecisionUtils.checkTimestampPrecision(-1L); + TimestampPrecisionUtils.checkTimestampPrecision(0L); + TimestampPrecisionUtils.checkTimestampPrecision(1694689856546L); + TimestampPrecisionUtils.checkTimestampPrecision(1694689856546000L); + } catch (Exception e) { + fail(e.getMessage()); + } + } + + @Test + public void testCheckIllegalUsTimestampPrecision() { + TimestampPrecisionUtils.TIMESTAMP_PRECISION = "us"; + try { + TimestampPrecisionUtils.checkTimestampPrecision(1694689856546000000L); + fail(); + } catch (Exception ignored) { + } + } + + @Test + public void testCheckNsTimestampPrecision() { + TimestampPrecisionUtils.TIMESTAMP_PRECISION = "ns"; + try { + TimestampPrecisionUtils.checkTimestampPrecision(-1L); + TimestampPrecisionUtils.checkTimestampPrecision(0L); + TimestampPrecisionUtils.checkTimestampPrecision(1694689856546L); + TimestampPrecisionUtils.checkTimestampPrecision(1694689856546000L); + TimestampPrecisionUtils.checkTimestampPrecision(1694689856546000000L); + TimestampPrecisionUtils.checkTimestampPrecision(Long.MAX_VALUE); + } catch (Exception e) { + fail(e.getMessage()); + } + } +} diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties index e27e572b213..149c97e5674 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties @@ -449,7 +449,11 @@ cluster_name=defaultCluster # Use this value to set timestamp precision as "ms", "us" or "ns". # Once the precision has been set, it can not be changed. # Datatype: String -# timestamp_precision=ms +timestamp_precision=ms + +# When the timestamp precision check is enabled, the timestamps those are over 13 digits for ms precision, or over 16 digits for us precision are not allowed to be inserted. +# Datatype: Boolean +# timestamp_precision_check_enabled=true # Default TTL for databases that are not set TTL by statements, If not set (default), the TTL will be unlimited. # Negative value means the TTL is unlimited. diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java index adc5cee2158..2556b6155d4 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java @@ -137,6 +137,11 @@ public class CommonConfig { /** This variable set timestamp precision as millisecond, microsecond or nanosecond. */ private String timestampPrecision = "ms"; + private boolean timestampPrecisionCheckEnabled = true; + + /** The number of threads in the thread pool that execute model inference tasks. */ + private int modelInferenceExecutionThreadCount = 5; + /** * The name of the directory that stores the tsfiles temporarily hold or generated by the pipe * module. The directory is located in the data directory of IoTDB. @@ -472,6 +477,14 @@ public class CommonConfig { return timestampPrecision; } + public void setTimestampPrecisionCheckEnabled(boolean timestampPrecisionCheckEnabled) { + this.timestampPrecisionCheckEnabled = timestampPrecisionCheckEnabled; + } + + public boolean isTimestampPrecisionCheckEnabled() { + return timestampPrecisionCheckEnabled; + } + public String getPipeHardlinkBaseDirName() { return pipeHardlinkBaseDirName; } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java index 76925891899..5431647a7e4 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java @@ -200,6 +200,16 @@ public class CommonDescriptor { config.setTimestampPrecision( properties.getProperty("timestamp_precision", config.getTimestampPrecision()).trim()); + config.setTimestampPrecisionCheckEnabled( + Boolean.parseBoolean( + properties.getProperty( + "timestamp_precision_check_enabled", + String.valueOf(config.isTimestampPrecisionCheckEnabled())))); + + config.setDatanodeTokenTimeoutMS( + Integer.parseInt( + properties.getProperty("datanode_token_timeout", String.valueOf(3 * 60 * 1000)))); + String endPointUrl = properties.getProperty( "target_ml_node_endpoint", diff --git a/pom.xml b/pom.xml index ded8698fcf8..9530aa5fcd9 100644 --- a/pom.xml +++ b/pom.xml @@ -1194,6 +1194,7 @@ </activation> <properties> <maven.compiler.release>8</maven.compiler.release> + <argLine>--illegal-access=permit --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.java [...] <!-- change to 1.15.0 will modify many codes (all are javadocs), we change it to 1.15.0 until: iotdb decides to do not support jdk8--> <google.java.format.version>1.7</google.java.format.version> @@ -1223,8 +1224,7 @@ <jdk>[16,20)</jdk> </activation> <properties> - <maven.compiler.release>8</maven.compiler.release> - <argLine>--illegal-access=permit --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.java [...] + <argLine>--add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add [...] </properties> </profile> <!--
