This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 29adcf6 [IOTDB-615] fix infer type from session (#1282)
29adcf6 is described below
commit 29adcf648be1792bfac54987cfeb2b35b06008e2
Author: Jialin Qiao <[email protected]>
AuthorDate: Thu May 28 23:07:56 2020 +0800
[IOTDB-615] fix infer type from session (#1282)
* fix infer type from session
---
.../main/java/org/apache/iotdb/SessionExample.java | 16 ++++++++++
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 13 +++-----
.../iotdb/db/qp/physical/crud/InsertPlan.java | 34 +++++++++++---------
.../org/apache/iotdb/db/service/TSServiceImpl.java | 2 ++
.../apache/iotdb/db/utils/TypeInferenceUtils.java | 8 ++---
.../iotdb/db/writelog/recover/LogReplayer.java | 2 +-
.../iotdb/db/engine/storagegroup/TTLTest.java | 4 +--
.../iotdb/db/utils/TypeInferenceUtilsTest.java | 19 ++++++++++--
service-rpc/src/main/thrift/rpc.thrift | 2 ++
.../java/org/apache/iotdb/session/Session.java | 2 ++
.../org/apache/iotdb/session/SessionDataSet.java | 10 ++++--
.../org/apache/iotdb/session/IoTDBSessionIT.java | 36 ++++++++++++----------
12 files changed, 96 insertions(+), 52 deletions(-)
diff --git a/example/session/src/main/java/org/apache/iotdb/SessionExample.java
b/example/session/src/main/java/org/apache/iotdb/SessionExample.java
index f7e7db9..01b05e5 100644
--- a/example/session/src/main/java/org/apache/iotdb/SessionExample.java
+++ b/example/session/src/main/java/org/apache/iotdb/SessionExample.java
@@ -153,6 +153,22 @@ public class SessionExample {
}
}
+ private static void insertStrRecord() throws IoTDBConnectionException,
StatementExecutionException {
+ String deviceId = "root.sg1.d1";
+ List<String> measurements = new ArrayList<>();
+ measurements.add("s1");
+ measurements.add("s2");
+ measurements.add("s3");
+
+ for (long time = 0; time < 10; time++) {
+ List<String> values = new ArrayList<>();
+ values.add("1");
+ values.add("2");
+ values.add("3");
+ session.insertRecord(deviceId, time, measurements, values);
+ }
+ }
+
private static void insertRecordInObject()
throws IoTDBConnectionException, StatementExecutionException {
String deviceId = "root.sg1.d1";
diff --git
a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
index ca9c13b..899b486 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
@@ -877,13 +877,8 @@ public class PlanExecutor implements IPlanExecutor {
if
(!IoTDBDescriptor.getInstance().getConfig().isAutoCreateSchemaEnabled()) {
throw new PathNotExistException(deviceId + PATH_SEPARATOR +
measurement);
}
- TSDataType dataType;
- if (insertPlan.getStrValues() != null) {
- // infer type for insert sql
- dataType =
TypeInferenceUtils.getPredictedDataType(insertPlan.getStrValues()[i]);
- } else {
- dataType =
TypeInferenceUtils.getPredictedDataType(insertPlan.getValues()[i]);
- }
+ TSDataType dataType = TypeInferenceUtils
+ .getPredictedDataType(insertPlan.getValues()[i],
insertPlan.isInferType());
Path path = new Path(deviceId, measurement);
internalCreateTimeseries(path.toString(), dataType);
}
@@ -892,13 +887,13 @@ public class PlanExecutor implements IPlanExecutor {
// reset measurement to common name instead of alias
measurementList[i] = measurementNode.getName();
- if(insertPlan.getStrValues() == null) {
+ if(!insertPlan.isInferType()) {
checkType(insertPlan, i, measurementNode.getSchema().getType());
}
}
insertPlan.setMeasurements(measurementList);
- insertPlan.setSchemas(schemas);
+ insertPlan.setSchemasAndTransferType(schemas);
StorageEngine.getInstance().insert(insertPlan);
} catch (StorageEngineException | MetadataException e) {
throw new QueryProcessException(e);
diff --git
a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertPlan.java
b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertPlan.java
index b2c2c2a..116ca7a 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertPlan.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertPlan.java
@@ -50,16 +50,10 @@ public class InsertPlan extends PhysicalPlan {
private TSDataType[] types;
private MeasurementSchema[] schemas;
- public String[] getStrValues() {
- return strValues;
- }
-
- public void setStrValues(String[] strValues) {
- this.strValues = strValues;
- }
+ // if inferType is false, use the type of values directly
+ // if inferType is true, values is String[], and infer types from them
+ private boolean inferType = false;
- // only for sql
- private String[] strValues;
public InsertPlan() {
super(false, OperatorType.INSERT);
@@ -140,7 +134,8 @@ public class InsertPlan extends PhysicalPlan {
// build types and values
this.types = new TSDataType[measurements.length];
this.values = new Object[measurements.length];
- this.strValues = insertValues;
+ System.arraycopy(insertValues, 0, values, 0, measurements.length);
+ inferType = true;
canbeSplit = false;
}
@@ -153,18 +148,29 @@ public class InsertPlan extends PhysicalPlan {
this.time = time;
}
+ public boolean isInferType() {
+ return inferType;
+ }
+
+ public void setInferType(boolean inferType) {
+ this.inferType = inferType;
+ }
+
public MeasurementSchema[] getSchemas() {
return schemas;
}
- public void setSchemas(MeasurementSchema[] schemas) throws
QueryProcessException {
+ /**
+ * if inferType is true,
+ * transfer String[] values to specific data types (Integer, Long, Float,
Double, Binary)
+ */
+ public void setSchemasAndTransferType(MeasurementSchema[] schemas) throws
QueryProcessException {
this.schemas = schemas;
- if (strValues != null) {
+ if (inferType) {
for (int i = 0; i < schemas.length; i++) {
types[i] = schemas[i].getType();
- values[i] = CommonUtils.parseValue(types[i], strValues[i]);
+ values[i] = CommonUtils.parseValue(types[i], values[i].toString());
}
- strValues = null;
}
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
index fef8785..9d7c505 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
@@ -1076,6 +1076,7 @@ public class TSServiceImpl implements TSIService.Iface,
ServerContext {
plan.setTypes(new TSDataType[plan.getMeasurements().length]);
plan.setValues(new Object[plan.getMeasurements().length]);
plan.setValues(req.valuesList.get(i));
+ plan.setInferType(req.isInferType());
TSStatus status = checkAuthority(plan, req.getSessionId());
if (status != null) {
resp.addToStatusList(status);
@@ -1124,6 +1125,7 @@ public class TSServiceImpl implements TSIService.Iface,
ServerContext {
plan.setTypes(new TSDataType[plan.getMeasurements().length]);
plan.setValues(new Object[plan.getMeasurements().length]);
plan.setValues(req.values);
+ plan.setInferType(req.isInferType());
TSStatus status = checkAuthority(plan, req.getSessionId());
if (status != null) {
diff --git
a/server/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java
b/server/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java
index 9e2bba1..2e9981d 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java
@@ -53,10 +53,10 @@ public class TypeInferenceUtils {
/**
* Get predicted DataType of the given value
*/
- public static TSDataType getPredictedDataType(Object value) {
+ public static TSDataType getPredictedDataType(Object value, boolean
inferType) {
- if (value instanceof String) {
- String strValue = (String) value;
+ if (inferType) {
+ String strValue = value.toString();
if (isBoolean(strValue)) {
return booleanStringInferType;
} else if (isNumber(strValue)){
@@ -78,8 +78,6 @@ public class TypeInferenceUtils {
return TSDataType.FLOAT;
} else if (value instanceof Double) {
return TSDataType.DOUBLE;
- } else if (value instanceof Binary) {
- return TSDataType.TEXT;
}
return TSDataType.TEXT;
diff --git
a/server/src/main/java/org/apache/iotdb/db/writelog/recover/LogReplayer.java
b/server/src/main/java/org/apache/iotdb/db/writelog/recover/LogReplayer.java
index 0f32e8e..b5574a4 100644
--- a/server/src/main/java/org/apache/iotdb/db/writelog/recover/LogReplayer.java
+++ b/server/src/main/java/org/apache/iotdb/db/writelog/recover/LogReplayer.java
@@ -172,7 +172,7 @@ public class LogReplayer {
try {
MeasurementSchema[] schemas =
MManager.getInstance().getSchemas(insertPlan.getDeviceId(),
insertPlan.getMeasurements());
- insertPlan.setSchemas(schemas);
+ insertPlan.setSchemasAndTransferType(schemas);
recoverMemTable.insert(insertPlan);
} catch (Exception e) {
logger.error(
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TTLTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TTLTest.java
index 3023367..a48e969 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TTLTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TTLTest.java
@@ -133,7 +133,7 @@ public class TTLTest {
insertPlan.setMeasurements(new String[]{"s1"});
insertPlan.setTypes(new TSDataType[]{TSDataType.INT64});
insertPlan.setValues(new Object[]{1L});
- insertPlan.setSchemas(
+ insertPlan.setSchemasAndTransferType(
new MeasurementSchema[]{new MeasurementSchema("s1", TSDataType.INT64,
TSEncoding.PLAIN)});
// ok without ttl
@@ -160,7 +160,7 @@ public class TTLTest {
insertPlan.setMeasurements(new String[]{"s1"});
insertPlan.setTypes(new TSDataType[]{TSDataType.INT64});
insertPlan.setValues(new Object[]{1L});
- insertPlan.setSchemas(
+ insertPlan.setSchemasAndTransferType(
new MeasurementSchema[]{new MeasurementSchema("s1", TSDataType.INT64,
TSEncoding.PLAIN)});
long initTime = System.currentTimeMillis();
diff --git
a/server/src/test/java/org/apache/iotdb/db/utils/TypeInferenceUtilsTest.java
b/server/src/test/java/org/apache/iotdb/db/utils/TypeInferenceUtilsTest.java
index 69b4e5c..267bafc 100644
--- a/server/src/test/java/org/apache/iotdb/db/utils/TypeInferenceUtilsTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/utils/TypeInferenceUtilsTest.java
@@ -42,9 +42,10 @@ public class TypeInferenceUtilsTest {
}
@Test
- public void getPredictedDataTypeTest() {
+ public void testInferType() {
Object[] values = {123, "abc", 123.123d, true, 123.1f, "123", "12.2",
"true"};
- TSDataType[] encodings = {TSDataType.INT32, TSDataType.TEXT,
TSDataType.DOUBLE,
+ TSDataType[] encodings =
{IoTDBDescriptor.getInstance().getConfig().getIntegerStringInferType(),
+ TSDataType.TEXT,
IoTDBDescriptor.getInstance().getConfig().getFloatingStringInferType(),
TSDataType.BOOLEAN, TSDataType.FLOAT,
IoTDBDescriptor.getInstance().getConfig().getIntegerStringInferType(),
IoTDBDescriptor.getInstance().getConfig().getFloatingStringInferType(),
@@ -52,7 +53,19 @@ public class TypeInferenceUtilsTest {
};
for (int i = 0; i < values.length; i++) {
- assertEquals(encodings[i],
TypeInferenceUtils.getPredictedDataType(values[i]));
+ assertEquals(encodings[i],
TypeInferenceUtils.getPredictedDataType(values[i], true));
+ }
+ }
+
+ @Test
+ public void testNotInferType() {
+ Object[] values = {123, "abc", 123.123d, true, 123.1f, "123", "12.2",
"true"};
+ TSDataType[] encodings = {TSDataType.INT32, TSDataType.TEXT,
TSDataType.DOUBLE,
+ TSDataType.BOOLEAN, TSDataType.FLOAT, TSDataType.TEXT,
TSDataType.TEXT, TSDataType.TEXT
+ };
+
+ for (int i = 0; i < values.length; i++) {
+ assertEquals(encodings[i],
TypeInferenceUtils.getPredictedDataType(values[i], false));
}
}
}
diff --git a/service-rpc/src/main/thrift/rpc.thrift
b/service-rpc/src/main/thrift/rpc.thrift
index 876dac4..0073669 100644
--- a/service-rpc/src/main/thrift/rpc.thrift
+++ b/service-rpc/src/main/thrift/rpc.thrift
@@ -171,6 +171,7 @@ struct TSInsertRecordReq {
3: required list<string> measurements
4: required binary values
5: required i64 timestamp
+ 6: optional bool inferType
}
struct TSInsertTabletReq {
@@ -199,6 +200,7 @@ struct TSInsertRecordsReq {
3: required list<list<string>> measurementsList
4: required list<binary> valuesList
5: required list<i64> timestamps
+ 6: optional bool inferType
}
struct TSDeleteDataReq {
diff --git a/session/src/main/java/org/apache/iotdb/session/Session.java
b/session/src/main/java/org/apache/iotdb/session/Session.java
index 244b602..f3e36ae 100644
--- a/session/src/main/java/org/apache/iotdb/session/Session.java
+++ b/session/src/main/java/org/apache/iotdb/session/Session.java
@@ -375,6 +375,7 @@ public class Session {
request.setDeviceIds(deviceIds);
request.setTimestamps(times);
request.setMeasurementsList(measurementsList);
+ request.setInferType(true);
List<ByteBuffer> buffersList = new ArrayList<>();
for (int i = 0; i < measurementsList.size(); i++) {
ByteBuffer buffer =
ByteBuffer.allocate(calculateStrLength(valuesList.get(i)));
@@ -433,6 +434,7 @@ public class Session {
request.setDeviceId(deviceId);
request.setTimestamp(time);
request.setMeasurements(measurements);
+ request.setInferType(true);
ByteBuffer buffer = ByteBuffer.allocate(calculateStrLength(values));
putStrValues(values, buffer);
buffer.flip();
diff --git a/session/src/main/java/org/apache/iotdb/session/SessionDataSet.java
b/session/src/main/java/org/apache/iotdb/session/SessionDataSet.java
index bc60c5b..b1708f8 100644
--- a/session/src/main/java/org/apache/iotdb/session/SessionDataSet.java
+++ b/session/src/main/java/org/apache/iotdb/session/SessionDataSet.java
@@ -76,11 +76,17 @@ public class SessionDataSet {
for (int i = 0; i < ioTDBRpcDataSet.columnSize; i++) {
Field field;
+ int index = i + 1;
+ int datasetColumnIndex = i + START_INDEX;
+ if (ioTDBRpcDataSet.ignoreTimeStamp) {
+ index--;
+ datasetColumnIndex--;
+ }
int loc =
-
ioTDBRpcDataSet.columnOrdinalMap.get(ioTDBRpcDataSet.columnNameList.get(i + 1))
+
ioTDBRpcDataSet.columnOrdinalMap.get(ioTDBRpcDataSet.columnNameList.get(index))
- START_INDEX;
- if (!ioTDBRpcDataSet.isNull(i + START_INDEX)) {
+ if (!ioTDBRpcDataSet.isNull(datasetColumnIndex)) {
byte[] valueBytes = ioTDBRpcDataSet.values[loc];
TSDataType dataType =
ioTDBRpcDataSet.columnTypeDeduplicatedList.get(loc);
field = new Field(dataType);
diff --git a/session/src/test/java/org/apache/iotdb/session/IoTDBSessionIT.java
b/session/src/test/java/org/apache/iotdb/session/IoTDBSessionIT.java
index 8097e37..a6c197e 100644
--- a/session/src/test/java/org/apache/iotdb/session/IoTDBSessionIT.java
+++ b/session/src/test/java/org/apache/iotdb/session/IoTDBSessionIT.java
@@ -44,6 +44,7 @@ import
org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.read.common.Field;
+import org.apache.iotdb.tsfile.read.common.RowRecord;
import org.apache.iotdb.tsfile.write.record.Tablet;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
import org.junit.After;
@@ -90,6 +91,25 @@ public class IoTDBSessionIT {
}
+ private void insertByStr() throws IoTDBConnectionException,
StatementExecutionException {
+ String deviceId = "root.sg1.d1";
+ List<String> measurements = new ArrayList<>();
+ measurements.add("s1");
+ measurements.add("s2");
+ measurements.add("s3");
+
+ for (long time = 0; time < 100; time++) {
+ List<String> values = new ArrayList<>();
+ values.add("1");
+ values.add("2");
+ values.add("3");
+ session.insertRecord(deviceId, time, measurements, values);
+ }
+
+ SessionDataSet dataSet = session.executeQueryStatement("show timeseries
root");
+ assertTrue(dataSet.hasNext());
+ RowRecord record = dataSet.next();
+ }
@Test
public void testInsertByObject() throws IoTDBConnectionException,
StatementExecutionException {
@@ -676,22 +696,6 @@ public class IoTDBSessionIT {
}
}
- private void insertByStr() throws IoTDBConnectionException,
StatementExecutionException {
- String deviceId = "root.sg1.d1";
- List<String> measurements = new ArrayList<>();
- measurements.add("s1");
- measurements.add("s2");
- measurements.add("s3");
-
- for (long time = 0; time < 100; time++) {
- List<String> values = new ArrayList<>();
- values.add("1");
- values.add("2");
- values.add("3");
- session.insertRecord(deviceId, time, measurements, values);
- }
- }
-
private void insertTabletTest1(String deviceId)
throws IoTDBConnectionException, BatchExecutionException {
