This is an automated email from the ASF dual-hosted git repository.
hxd pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 5cb248b IOTDB-734 Add Support for NaN in Double / Floats in SQL
Syntax. (#1305)
5cb248b is described below
commit 5cb248b004e1922d86f723e96a3fde456fe67b81
Author: Julian <[email protected]>
AuthorDate: Tue Jun 9 05:24:56 2020 +0200
IOTDB-734 Add Support for NaN in Double / Floats in SQL Syntax. (#1305)
* IOTDB-734 Add Support for NaN in Double / Floats in SQL Syntax.
---
docs/UserGuide/Server/Config Manual.md | 55 ++++++++
docs/zh/UserGuide/Server/Config Manual.md | 58 +++++++++
.../resources/conf/iotdb-engine.properties | 3 +
.../org/apache/iotdb/db/qp/strategy/SqlBase.g4 | 4 +
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 18 +++
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 2 +
.../apache/iotdb/db/utils/TypeInferenceUtils.java | 6 +-
.../iotdb/db/integration/IoTDBInsertNaNIT.java | 139 +++++++++++++++++++++
.../java/org/apache/iotdb/db/qp/PlannerTest.java | 13 ++
9 files changed, 297 insertions(+), 1 deletion(-)
diff --git a/docs/UserGuide/Server/Config Manual.md
b/docs/UserGuide/Server/Config Manual.md
index a436d35..6046275 100644
--- a/docs/UserGuide/Server/Config Manual.md
+++ b/docs/UserGuide/Server/Config Manual.md
@@ -582,6 +582,61 @@ The permission definitions are in
${IOTDB\_CONF}/conf/jmx.access.
|Default|no |
|Effective|After restart system|
+## Automatic Schema Creation and Type Inference
+
+* enable\_auto\_create\_schema
+
+|Name| enable\_auto\_create\_schema |
+|:---:|:---|
+|Description| whether auto create the time series when a non-existed time
series data comes|
+|Type| true or false |
+|Default|true |
+|Effective|After restart system|
+
+* default\_storage\_group\_level
+
+|Name| default\_storage\_group\_level |
+|:---:|:---|
+|Description| Storage group level when creating schema automatically is
enabled. For example, if we receives a data point from root.sg0.d1.s2, we will
set root.sg0 as the storage group if storage group level is 1. (root is level
0)|
+|Type| integer |
+|Default|1 |
+|Effective|After restart system|
+
+* boolean\_string\_infer\_type
+
+|Name| boolean\_string\_infer\_type |
+|:---:|:---|
+|Description| To which type the values "true" and "false" should be reslved|
+|Type| BOOLEAN or TEXT |
+|Default|BOOLEAN |
+|Effective|After restart system|
+
+* integer\_string\_infer\_type
+
+|Name| integer\_string\_infer\_type |
+|:---:|:---|
+|Description| To which type an integer string like "67" in a query should be
resolved|
+|Type| INT32, INT64, DOUBLE, FLOAT or TEXT |
+|Default|DOUBLE |
+|Effective|After restart system|
+
+* nan\_string\_infer\_type
+
+|Name| nan\_string\_infer\_type |
+|:---:|:---|
+|Description| To which type the value NaN in a query should be resolved|
+|Type| DOUBLE, FLOAT or TEXT |
+|Default|FLOAT |
+|Effective|After restart system|
+
+* floating\_string\_infer\_type
+
+|Name| floating\_string\_infer\_type |
+|:---:|:---|
+|Description| To which type a floating number string like "6.7" in a query
should be resolved|
+|Type| DOUBLE, FLOAT or TEXT |
+|Default|FLOAT |
+|Effective|After restart system|
## Enable GC log
GC log is off by default.
diff --git a/docs/zh/UserGuide/Server/Config Manual.md
b/docs/zh/UserGuide/Server/Config Manual.md
index 227df61..6c53b6c 100644
--- a/docs/zh/UserGuide/Server/Config Manual.md
+++ b/docs/zh/UserGuide/Server/Config Manual.md
@@ -528,6 +528,64 @@
|改后生效方式|重启服务器生效|
+## 数据类型自动推断
+
+
+* enable\_auto\_create\_schema
+
+|名字| enable\_auto\_create\_schema |
+|:---:|:---|
+|描述| 当写入的序列不存在时，是否自动创建序列到Schema|
+|取值| true or false |
+|默认值|true |
+|改后生效方式|重启服务器生效|
+
+* default\_storage\_group\_level
+
+|名字| default\_storage\_group\_level |
+|:---:|:---|
+|描述| 当写入的数据不存在且自动创建序列时，若需要创建相应的存储组，将序列路径的哪一层当做存储组. 例如, 如果我们接到一个新序列
root.sg0.d1.s2, 并且level=1， 那么root.sg0被视为存储组（因为root是level 0 层）|
+|取值| 整数 |
+|默认值|1 |
+|改后生效方式|重启服务器生效|
+
+* boolean\_string\_infer\_type
+
+|名字| boolean\_string\_infer\_type |
+|:---:|:---|
+|描述| "true" 或者 "false" 被视为什么数据|
+|取值| BOOLEAN 或者 TEXT |
+|默认值|BOOLEAN |
+|改后生效方式|重启服务器生效|
+
+* integer\_string\_infer\_type
+
+|名字| integer\_string\_infer\_type |
+|:---:|:---|
+|描述| 整数型数据被推断成什么 |
+|取值| INT32, INT64, FLOAT, DOUBLE, TEXT |
+|默认值|FLOAT |
+|改后生效方式|重启服务器生效|
+
+* nan\_string\_infer\_type
+
+|名字| nan\_string\_infer\_type |
+|:---:|:---|
+|描述| NaN 字符串被推断为什么|
+|取值| DOUBLE, FLOAT or TEXT |
+|默认值|FLOAT |
+|改后生效方式|重启服务器生效|
+
+* floating\_string\_infer\_type
+
+|名字| floating\_string\_infer\_type |
+|:---:|:---|
+|描述| "6.7"等浮点数被推断为什么|
+|取值| DOUBLE, FLOAT or TEXT |
+|默认值|FLOAT |
+|改后生效方式|重启服务器生效|
+
+
## 开启GC日志
GC日志默认是关闭的。为了性能调优，用户可能会需要手机GC信息。
若要打开GC日志，则需要在启动IoTDB Server的时候加上"printgc"参数：
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties
b/server/src/assembly/resources/conf/iotdb-engine.properties
index ea78585..bb78b41 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -398,6 +398,9 @@ integer_string_infer_type=FLOAT
# register time series as which type when receiving a floating number string
"6.7"
floating_string_infer_type=FLOAT
+# register time series as which type when receiving the Literal NaN. Values
can be DOUBLE, FLOAT or TEXT
+nan_string_infer_type=DOUBLE
+
# BOOLEAN encoding when creating schema automatically is enabled
default_boolean_encoding=RLE
diff --git a/server/src/main/antlr4/org/apache/iotdb/db/qp/strategy/SqlBase.g4
b/server/src/main/antlr4/org/apache/iotdb/db/qp/strategy/SqlBase.g4
index 7d5b9f2..d6ef3dd 100644
--- a/server/src/main/antlr4/org/apache/iotdb/db/qp/strategy/SqlBase.g4
+++ b/server/src/main/antlr4/org/apache/iotdb/db/qp/strategy/SqlBase.g4
@@ -394,6 +394,7 @@ dateFormat
constant
: dateExpression
+ | NaN
| MINUS? realLiteral
| MINUS? INT
| STRING_LITERAL
@@ -876,6 +877,7 @@ TRUE
FALSE
: F A L S E
;
+
//============================
// End of the keywords list
//============================
@@ -937,6 +939,8 @@ R_BRACKET : '}';
UNDERLINE : '_';
+NaN : 'NaN';
+
STRING_LITERAL
: DOUBLE_QUOTE_STRING_LITERAL
| SINGLE_QUOTE_STRING_LITERAL
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 2d26c3c..23c111a 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -386,6 +386,11 @@ public class IoTDBConfig {
private TSDataType floatingStringInferType = TSDataType.FLOAT;
/**
+ * register time series as which type when receiving the Literal NaN. Values
can be DOUBLE, FLOAT or TEXT
+ */
+ private TSDataType nanStringInferType = TSDataType.DOUBLE;
+
+ /**
* Storage group level when creating schema automatically is enabled
*/
private int defaultStorageGroupLevel = 1;
@@ -1323,6 +1328,19 @@ public class IoTDBConfig {
this.floatingStringInferType = floatingNumberStringInferType;
}
+ public TSDataType getNanStringInferType() {
+ return nanStringInferType;
+ }
+
+ public void setNanStringInferType(TSDataType nanStringInferType) {
+ if (nanStringInferType != TSDataType.DOUBLE &&
+ nanStringInferType != TSDataType.FLOAT &&
+ nanStringInferType != TSDataType.TEXT) {
+ throw new IllegalArgumentException("Config Property
nan_string_infer_type can only be FLOAT, DOUBLE or TEXT but is " +
nanStringInferType);
+ }
+ this.nanStringInferType = nanStringInferType;
+ }
+
public int getDefaultStorageGroupLevel() {
return defaultStorageGroupLevel;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index fd25b14..333b2d8 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -499,6 +499,8 @@ public class IoTDBDescriptor {
conf.getIntegerStringInferType().toString())));
conf.setFloatingStringInferType(TSDataType.valueOf(properties.getProperty("floating_string_infer_type",
conf.getFloatingStringInferType().toString())));
+
conf.setNanStringInferType(TSDataType.valueOf(properties.getProperty("nan_string_infer_type",
+ conf.getNanStringInferType().toString())));
conf.setDefaultStorageGroupLevel(
Integer.parseInt(properties.getProperty("default_storage_group_level",
Integer.toString(conf.getDefaultStorageGroupLevel()))));
diff --git
a/server/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java
b/server/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java
index 2e9981d..e328874 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java
@@ -22,7 +22,6 @@ package org.apache.iotdb.db.utils;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.qp.constant.SQLConstant;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.utils.Binary;
public class TypeInferenceUtils {
@@ -32,6 +31,8 @@ public class TypeInferenceUtils {
private static TSDataType floatingStringInferType =
IoTDBDescriptor.getInstance().getConfig().getFloatingStringInferType();
+ private static TSDataType nanStringInferType =
IoTDBDescriptor.getInstance().getConfig().getNanStringInferType();
+
private TypeInferenceUtils() {
}
@@ -65,6 +66,9 @@ public class TypeInferenceUtils {
} else {
return floatingStringInferType;
}
+ // "NaN" is returned if the NaN Literal is given in Parser
+ } else if ("NaN".equals(strValue)) {
+ return nanStringInferType;
} else {
return TSDataType.TEXT;
}
diff --git
a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBInsertNaNIT.java
b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBInsertNaNIT.java
new file mode 100644
index 0000000..0e6fd16
--- /dev/null
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBInsertNaNIT.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.integration;
+
+import org.apache.iotdb.db.utils.EnvironmentUtils;
+import org.apache.iotdb.db.utils.MathUtils;
+import org.apache.iotdb.jdbc.Config;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.iotdb.db.constant.TestConstant.TIMESTAMP_STR;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * Notice that, all test begins with "IoTDB" is integration test. All test
which will start the IoTDB server should be
+ * defined as integration test.
+ *
+ * This test stores NaN Values and retrieves them via SQL Interface.
+ */
+public class IoTDBInsertNaNIT {
+
+ private static final String CREATE_TEMPLATE_SQL = "CREATE TIMESERIES
root.vehicle.%s.%s WITH DATATYPE=%s, ENCODING=%s, MAX_POINT_NUMBER=%d";
+ private static final String INSERT_TEMPLATE_SQL = "insert into
root.vehicle.%s(timestamp,%s) values(%d,%s)";
+ private static List<String> sqls = new ArrayList<>();
+ private static final int TIMESTAMP = 10;
+ private static final String VALUE = "NaN";
+ private static final float DELTA_FLOAT = 0.0000001f;
+ private static final double DELTA_DOUBLE = 0.0000001d;
+
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ EnvironmentUtils.closeStatMonitor();
+ initCreateSQLStatement();
+ EnvironmentUtils.envSetUp();
+ insertData();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ EnvironmentUtils.cleanEnv();
+ }
+
+ private static void initCreateSQLStatement(){
+ sqls.add("SET STORAGE GROUP TO root.vehicle.f0");
+ sqls.add("SET STORAGE GROUP TO root.vehicle.d0");
+ for(int i = 0; i < 10; i++){
+ sqls.add(String.format(CREATE_TEMPLATE_SQL, "f0", "s"+i+"rle", "FLOAT",
"RLE", i));
+ sqls.add(String.format(CREATE_TEMPLATE_SQL, "f0", "s"+i+"2f", "FLOAT",
"TS_2DIFF", i));
+ sqls.add(String.format(CREATE_TEMPLATE_SQL, "d0", "s"+i+"rle", "DOUBLE",
"RLE", i));
+ sqls.add(String.format(CREATE_TEMPLATE_SQL, "d0", "s"+i+"2f", "DOUBLE",
"TS_2DIFF", i));
+ }
+ for(int i = 0; i < 10; i++){
+ sqls.add(String.format(INSERT_TEMPLATE_SQL, "f0", "s"+i+"rle",
TIMESTAMP, VALUE));
+ sqls.add(String.format(INSERT_TEMPLATE_SQL, "f0", "s"+i+"2f", TIMESTAMP,
VALUE));
+ sqls.add(String.format(INSERT_TEMPLATE_SQL, "d0", "s"+i+"rle",
TIMESTAMP, VALUE));
+ sqls.add(String.format(INSERT_TEMPLATE_SQL, "d0", "s"+i+"2f", TIMESTAMP,
VALUE));
+ }
+
+ }
+
+ private static void insertData() throws ClassNotFoundException {
+ Class.forName(Config.JDBC_DRIVER_NAME);
+ try (Connection connection = DriverManager
+ .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root",
"root");
+ Statement statement = connection.createStatement();) {
+
+ for (String sql : sqls) {
+ statement.execute(sql);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Test
+ public void selectAllSQLTest() throws ClassNotFoundException {
+ Class.forName(Config.JDBC_DRIVER_NAME);
+ try (Connection connection = DriverManager
+ .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root",
"root");
+ Statement statement = connection.createStatement()) {
+ boolean hasResultSet = statement.execute("select * from root");
+ Assert.assertTrue(hasResultSet);
+ int cnt;
+ try (ResultSet resultSet = statement.getResultSet()) {
+ cnt = 0;
+ while (resultSet.next()) {
+ assertEquals(TIMESTAMP + "", resultSet.getString(TIMESTAMP_STR));
+ for (int i = 0; i < 10; i++) {
+
Assert.assertEquals(MathUtils.roundWithGivenPrecision(Float.parseFloat(VALUE),
i),
+ resultSet.getFloat(String.format("root.vehicle.%s.%s", "f0",
"s" + i + "rle")),
+ DELTA_FLOAT);
+
Assert.assertEquals(MathUtils.roundWithGivenPrecision(Float.parseFloat(VALUE),
i),
+ resultSet.getFloat(String.format("root.vehicle.%s.%s", "f0",
"s" + i + "2f")),
+ DELTA_FLOAT);
+
Assert.assertEquals(MathUtils.roundWithGivenPrecision(Double.parseDouble(VALUE),
i),
+ resultSet.getDouble(String.format("root.vehicle.%s.%s", "d0",
"s" + i + "rle")),
+ DELTA_DOUBLE);
+
Assert.assertEquals(MathUtils.roundWithGivenPrecision(Double.parseDouble(VALUE),
i),
+ resultSet.getDouble(String.format("root.vehicle.%s.%s", "d0",
"s" + i + "2f")),
+ DELTA_DOUBLE);
+ }
+ cnt++;
+ }
+ Assert.assertEquals(1, cnt);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+}
diff --git a/server/src/test/java/org/apache/iotdb/db/qp/PlannerTest.java
b/server/src/test/java/org/apache/iotdb/db/qp/PlannerTest.java
index e4396a0..ec05a13 100644
--- a/server/src/test/java/org/apache/iotdb/db/qp/PlannerTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/qp/PlannerTest.java
@@ -23,6 +23,7 @@ import
org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.qp.logical.Operator.OperatorType;
import org.apache.iotdb.db.qp.physical.PhysicalPlan;
+import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
@@ -35,6 +36,7 @@ import org.junit.Test;
import java.util.Collections;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
public class PlannerTest {
@@ -151,4 +153,15 @@ public class PlannerTest {
String createTSStatement = "create timeseriess root.vehicle.d1.s1 with
datatype=INT32,encoding=RLE";
processor.parseSQLToPhysicalPlan(createTSStatement);
}
+
+ @Test
+ public void insertStatementWithNullValue() throws QueryProcessException {
+ String createTSStatement = "insert into root.vehicle.d0(time,s0)
values(10,NaN)";
+ PhysicalPlan physicalPlan =
processor.parseSQLToPhysicalPlan(createTSStatement);
+
+ assertTrue(physicalPlan instanceof InsertPlan);
+ assertEquals("NaN", ((InsertPlan) physicalPlan).getValues()[0]);
+ // Later we will use Double.parseDouble so we have to ensure that it is
parsed right
+ assertEquals(Double.NaN, Double.parseDouble("NaN"), 1e-15);
+ }
}
\ No newline at end of file
