This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 4a1dc2d [IOTDB-886]fix float loss precision (#1796)
4a1dc2d is described below
commit 4a1dc2d89be12214ba7a2256b79f91a190a83f4b
Author: gwmh <[email protected]>
AuthorDate: Sun Oct 25 23:50:21 2020 +0800
[IOTDB-886]fix float loss precision (#1796)
---
.../src/assembly/resources/conf/iotdb-engine.properties | 4 ++++
.../main/java/org/apache/iotdb/db/conf/IoTDBConfig.java | 15 +++++++++++++++
.../java/org/apache/iotdb/db/conf/IoTDBDescriptor.java | 3 +++
.../org/apache/iotdb/db/utils/TypeInferenceUtils.java | 9 +++++++++
.../org/apache/iotdb/db/utils/TypeInferenceUtilsTest.java | 4 +++-
5 files changed, 34 insertions(+), 1 deletion(-)
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties
b/server/src/assembly/resources/conf/iotdb-engine.properties
index 5332cff..3045cce 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -454,6 +454,10 @@ boolean_string_infer_type=BOOLEAN
# register time series as which type when receiving an integer string "67"
integer_string_infer_type=FLOAT
+# register time series as which type when receiving an integer string and
using float may lose precision
+# num > 2 ^ 24
+long_string_infer_type=DOUBLE
+
# register time series as which type when receiving a floating number string
"6.7"
floating_string_infer_type=FLOAT
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index c65b744..89f053b 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -446,6 +446,12 @@ public class IoTDBConfig {
private TSDataType integerStringInferType = TSDataType.FLOAT;
/**
+ * register time series as which type when receiving an integer string and
using float may lose precision
+ * num > 2 ^ 24
+ */
+ private TSDataType longStringInferType = TSDataType.DOUBLE;
+
+ /**
* register time series as which type when receiving a floating number
string "6.7"
*/
private TSDataType floatingStringInferType = TSDataType.FLOAT;
@@ -1542,6 +1548,15 @@ public class IoTDBConfig {
this.integerStringInferType = integerStringInferType;
}
+ public void setLongStringInferType(
+ TSDataType longStringInferType) {
+ this.longStringInferType = longStringInferType;
+ }
+
+ public TSDataType getLongStringInferType() {
+ return longStringInferType;
+ }
+
public TSDataType getFloatingStringInferType() {
return floatingStringInferType;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 7b324d7..4fe9ecb 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -578,6 +578,9 @@ public class IoTDBDescriptor {
conf.setIntegerStringInferType(
TSDataType.valueOf(properties.getProperty("integer_string_infer_type",
conf.getIntegerStringInferType().toString())));
+ conf.setLongStringInferType(
+ TSDataType.valueOf(properties.getProperty("long_string_infer_type",
+ conf.getLongStringInferType().toString())));
conf.setFloatingStringInferType(
TSDataType.valueOf(properties.getProperty("floating_string_infer_type",
conf.getFloatingStringInferType().toString())));
diff --git
a/server/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java
b/server/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java
index 04580fd..a9174d1 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java
@@ -30,6 +30,8 @@ public class TypeInferenceUtils {
private static TSDataType integerStringInferType =
IoTDBDescriptor.getInstance().getConfig().getIntegerStringInferType();
+ private static TSDataType longStringInferType =
IoTDBDescriptor.getInstance().getConfig().getLongStringInferType();
+
private static TSDataType floatingStringInferType =
IoTDBDescriptor.getInstance().getConfig().getFloatingStringInferType();
private static TSDataType nanStringInferType =
IoTDBDescriptor.getInstance().getConfig().getNanStringInferType();
@@ -52,6 +54,10 @@ public class TypeInferenceUtils {
.equalsIgnoreCase(SQLConstant.BOOLEAN_FALSE);
}
+ private static boolean isConvertFloatPrecisionLack(String s){
+ return Long.parseLong(s) > (2 << 24);
+ }
+
/**
* Get predicted DataType of the given value
*/
@@ -63,6 +69,9 @@ public class TypeInferenceUtils {
return booleanStringInferType;
} else if (isNumber(strValue)){
if (!strValue.contains(TsFileConstant.PATH_SEPARATOR)) {
+ if (isConvertFloatPrecisionLack(strValue)) {
+ return longStringInferType;
+ }
return integerStringInferType;
} else {
return floatingStringInferType;
diff --git
a/server/src/test/java/org/apache/iotdb/db/utils/TypeInferenceUtilsTest.java
b/server/src/test/java/org/apache/iotdb/db/utils/TypeInferenceUtilsTest.java
index 5df9b5d..efc854e 100644
--- a/server/src/test/java/org/apache/iotdb/db/utils/TypeInferenceUtilsTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/utils/TypeInferenceUtilsTest.java
@@ -44,12 +44,14 @@ public class TypeInferenceUtilsTest {
@Test
public void testInferType() {
- Object[] values = {123, "abc", 123.123d, true, 123.1f, "123", "12.2",
"true"};
+ Object[] values = {123, "abc", 123.123d, true, 123.1f, "123", "12.2",
"9999999999999999",
+ "true"};
TSDataType[] encodings =
{IoTDBDescriptor.getInstance().getConfig().getIntegerStringInferType(),
TSDataType.TEXT,
IoTDBDescriptor.getInstance().getConfig().getFloatingStringInferType(),
TSDataType.BOOLEAN, TSDataType.FLOAT,
IoTDBDescriptor.getInstance().getConfig().getIntegerStringInferType(),
IoTDBDescriptor.getInstance().getConfig().getFloatingStringInferType(),
+ IoTDBDescriptor.getInstance().getConfig().getLongStringInferType(),
IoTDBDescriptor.getInstance().getConfig().getBooleanStringInferType()
};
