This is an automated email from the ASF dual-hosted git repository. xuekaifeng pushed a commit to branch IOTDB-615-Use-binary-rather-than-string-in-insert-plan in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit f4b19f8fd1d2f5562048bab855ff7752a451ed4a Merge: d4a408b 7585123 Author: 151250176 <[email protected]> AuthorDate: Fri May 15 09:38:30 2020 +0800 Merge branch 'master' of https://github.com/apache/incubator-iotdb into IOTDB-615-Use-binary-rather-than-string-in-insert-plan # Conflicts: # server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java # session/src/main/java/org/apache/iotdb/session/Session.java # session/src/test/java/org/apache/iotdb/session/pool/SessionPoolTest.java .gitignore | 3 + LICENSE-binary | 2 +- NOTICE | 10 - NOTICE-binary | 10 - cli/src/assembly/resources/sbin/start-cli.bat | 3 - .../java/org/apache/iotdb/cli/AbstractCli.java | 45 ++- cli/src/main/java/org/apache/iotdb/cli/WinCli.java | 19 +- .../org/apache/iotdb/cli/StartClientScriptIT.java | 3 +- docs/Community/Community-Powered By.md | 17 + docs/Download/README.md | 40 +- docs/SystemDesign/1-TsFile/2-Format.md | 278 +++++++++---- .../2-QueryEngine/4-ResultSetConstruction.md | 177 +++++++++ docs/SystemDesign/5-DataQuery/9-FillFunction.md | 12 +- docs/UserGuide/1-Overview/2-Architecture.md | 6 +- docs/UserGuide/1-Overview/3-Scenario.md | 4 +- .../UserGuide/4-Client/1-Command Line Interface.md | 11 +- .../4-Client/2-Programming - Native API.md | 4 +- docs/UserGuide/4-Client/3-Programming - JDBC.md | 72 ---- docs/UserGuide/4-Client/7-Status Codes.md | 117 +++--- .../1-DDL Data Definition Language.md | 31 ++ .../2-DML Data Manipulation Language.md | 7 +- ...anagement Statements.md => 3-Administration.md} | 2 +- docs/UserGuide/6-System Tools/3-JMX Tool.md | 47 ++- .../7-Ecosystem Integration/2-MapReduce TsFile.md | 2 +- .../8-Architecture/3-Shared Nothing Cluster.md | 2 +- docs/zh/Community/Community-Powered By.md | 39 +- docs/zh/Download/README.md | 42 +- docs/zh/SystemDesign/1-TsFile/2-Format.md | 279 +++++++++---- .../2-QueryEngine/4-ResultSetConstruction.md | 175 +++++++++ docs/zh/SystemDesign/5-DataQuery/9-FillFunction.md | 13 +- docs/zh/UserGuide/1-Overview/2-Architecture.md | 6 +- docs/zh/UserGuide/1-Overview/3-Scenario.md | 10 +- .../UserGuide/4-Client/1-Command Line Interface.md | 11 +- .../4-Client/2-Programming - Native API.md | 4 +- docs/zh/UserGuide/4-Client/3-Programming - JDBC.md | 74 ---- docs/zh/UserGuide/4-Client/7-Status Codes.md | 118 +++--- .../1-DDL Data Definition Language.md | 24 ++ .../2-DML Data Manipulation Language.md | 7 +- ...anagement Statements.md => 3-Administration.md} | 4 +- docs/zh/UserGuide/6-System Tools/3-JMX Tool.md | 59 +-- example/flink/README.md | 2 + .../apache/iotdb/flink/FlinkTsFileBatchSink.java | 114 ++++++ .../apache/iotdb/flink/FlinkTsFileBatchSource.java | 2 +- .../apache/iotdb/flink/FlinkTsFileStreamSink.java | 115 ++++++ .../iotdb/flink/FlinkTsFileStreamSource.java | 2 +- .../java/org/apache/iotdb/flink/TsFileUtils.java | 103 +++++ .../java/org/apache/iotdb/flink/TsFlieUtils.java | 72 ---- example/mqtt/README.md | 3 +- .../main/java/org/apache/iotdb/SessionExample.java | 60 ++- .../java/org/apache/iotdb/SessionPoolExample.java | 114 ++++++ flink-tsfile-connector/README.md | 86 ++++- .../iotdb/flink/tsfile/RowTSRecordConverter.java | 163 ++++++++ .../iotdb/flink/tsfile/TSRecordConverter.java | 61 +++ .../iotdb/flink/tsfile/TSRecordOutputFormat.java | 99 +++++ .../iotdb/flink/tsfile/TsFileOutputFormat.java | 122 ++++++ .../tsfile/RowTSRecordOutputFormatITCase.java | 56 +++ .../flink/tsfile/RowTSRecordOutputFormatTest.java | 77 ++++ .../flink/tsfile/RowTsFileConnectorTestBase.java | 87 +++++ .../flink/tsfile/RowTsFileInputFormatTestBase.java | 72 +--- .../tsfile/RowTsFileOutputFormatTestBase.java | 117 ++++++ .../apache/iotdb/flink/util/TsFileWriteUtil.java | 2 +- .../apache/iotdb/hadoop/fileSystem/HDFSFile.java | 26 +- .../apache/iotdb/hadoop/fileSystem/HDFSInput.java | 45 +-- .../apache/iotdb/hadoop/fileSystem/HDFSOutput.java | 15 +- ...ultSet.java => AbstractIoTDBJDBCResultSet.java} | 273 +++++-------- .../org/apache/iotdb/jdbc/IoTDBConnection.java | 1 + .../org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java | 93 +++++ ...ultSet.java => IoTDBNonAlignJDBCResultSet.java} | 110 +++--- .../org/apache/iotdb/jdbc/IoTDBQueryResultSet.java | 203 ---------- .../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 28 +- ...ultSetTest.java => IoTDBJDBCResultSetTest.java} | 2 +- pom.xml | 2 +- .../resources/conf/iotdb-engine.properties | 12 +- .../org/apache/iotdb/db/qp/strategy/SqlBase.g4 | 7 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 12 - .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 14 +- .../db/engine/cache/CacheHitRatioMonitor.java | 65 +++- .../engine/cache/CacheHitRatioMonitorMXBean.java | 27 +- .../apache/iotdb/db/engine/cache/ChunkCache.java | 25 +- .../iotdb/db/engine/cache/ChunkMetadataCache.java | 53 ++- .../iotdb/db/engine/cache/LRULinkedHashMap.java | 31 +- .../db/engine/cache/TimeSeriesMetadataCache.java | 89 +++-- .../iotdb/db/engine/cache/TsFileMetaDataCache.java | 174 --------- .../iotdb/db/engine/merge/task/MergeFileTask.java | 34 +- .../engine/storagegroup/StorageGroupProcessor.java | 14 +- .../db/engine/storagegroup/TsFileProcessor.java | 2 +- .../db/engine/storagegroup/TsFileResource.java | 44 ++- .../org/apache/iotdb/db/metadata/MManager.java | 14 + .../java/org/apache/iotdb/db/metadata/MTree.java | 20 + .../org/apache/iotdb/db/mqtt/PublishHandler.java | 7 +- .../main/java/org/apache/iotdb/db/qp/Planner.java | 2 + .../apache/iotdb/db/qp/constant/SQLConstant.java | 4 + .../apache/iotdb/db/qp/executor/IPlanExecutor.java | 4 +- .../apache/iotdb/db/qp/executor/PlanExecutor.java | 50 ++- .../org/apache/iotdb/db/qp/logical/Operator.java | 2 +- .../apache/iotdb/db/qp/logical/RootOperator.java | 1 - .../iotdb/db/qp/logical/crud/QueryOperator.java | 2 +- .../{RootOperator.java => sys/FlushOperator.java} | 37 +- .../logical/sys/MergeOperator.java} | 19 +- .../iotdb/db/qp/physical/crud/FillQueryPlan.java | 2 +- .../iotdb/db/qp/physical/crud/GroupByFillPlan.java | 2 +- .../iotdb/db/qp/physical/crud/QueryPlan.java | 2 +- .../db/qp/physical/crud/RawDataQueryPlan.java | 16 +- .../GroupByFillPlan.java => sys/FlushPlan.java} | 34 +- .../sys/MergePlan.java} | 26 +- .../iotdb/db/qp/strategy/LogicalGenerator.java | 42 +- .../iotdb/db/qp/strategy/PhysicalGenerator.java | 86 +++-- .../qp/strategy/optimizer/ConcatPathOptimizer.java | 27 +- .../query/dataset/groupby/GroupByFillDataSet.java | 7 +- .../dataset/groupby/LocalGroupByExecutor.java | 6 +- .../db/query/executor/AggregationExecutor.java | 31 +- .../iotdb/db/query/executor/FillQueryExecutor.java | 4 +- .../iotdb/db/query/executor/LastQueryExecutor.java | 26 +- .../iotdb/db/query/executor/QueryRouter.java | 2 +- .../iotdb/db/query/{ => executor}/fill/IFill.java | 16 +- .../fill/LastPointReader.java} | 89 +---- .../db/query/{ => executor}/fill/LinearFill.java | 93 +++-- .../iotdb/db/query/executor/fill/PreviousFill.java | 115 ++++++ .../reader/series/SeriesRawDataBatchReader.java | 24 +- .../query/timegenerator/ServerTimeGenerator.java | 5 +- .../org/apache/iotdb/db/service/TSServiceImpl.java | 123 ++---- .../apache/iotdb/db/tools/TsFileSketchTool.java | 129 ++++--- .../writelog/recover/TsFileRecoverPerformer.java | 2 +- .../apache/iotdb/db/engine/merge/MergeTest.java | 2 - .../apache/iotdb/db/integration/IoTDBAliasIT.java | 273 +++++++++++++ .../db/integration/IoTDBEngineTimeGeneratorIT.java | 37 +- .../apache/iotdb/db/integration/IoTDBFillIT.java | 52 ++- .../iotdb/db/integration/IoTDBGroupByFillIT.java | 60 ++- .../apache/iotdb/db/integration/IoTDBLastIT.java | 2 + .../IoTDBMultiOverlappedChunkInUnseqIT.java | 17 +- .../db/integration/IoTDBMultiOverlappedPageIT.java | 42 +- .../apache/iotdb/db/integration/IoTDBTagIT.java | 32 ++ .../org/apache/iotdb/db/metadata/MTreeTest.java | 14 + .../apache/iotdb/db/qp/plan/PhysicalPlanTest.java | 4 +- .../query/reader/series/SeriesReaderTestUtil.java | 3 - .../apache/iotdb/db/utils/EnvironmentUtils.java | 3 - .../recover/RecoverResourceFromReaderTest.java | 189 +++++++++ service-rpc/pom.xml | 5 + .../java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java | 430 +++++++++++++++++++++ .../java/org/apache/iotdb/session/Session.java | 161 +++++++- .../org/apache/iotdb/session/SessionDataSet.java | 311 ++++++++------- .../iotdb/session/pool/SessionDataSetWrapper.java | 17 +- .../org/apache/iotdb/session/pool/SessionPool.java | 4 +- .../org/apache/iotdb/session/IoTDBSessionIT.java | 19 +- .../iotdb/session/IoTDBSessionIteratorIT.java | 175 +++++++++ .../java/org/apache/iotdb/session/SessionUT.java | 2 + .../apache/iotdb/session/pool/SessionPoolTest.java | 14 +- site/src/main/.vuepress/config.js | 10 +- .../org/apache/iotdb/spark/db/Converter.scala | 4 +- .../apache/iotdb/spark/db/EnvironmentUtils.java | 2 - .../file/metadata/MetadataIndexConstructor.java | 31 +- .../tsfile/file/metadata/MetadataIndexEntry.java | 32 +- .../tsfile/file/metadata/MetadataIndexNode.java | 23 +- .../file/metadata/enums/MetadataIndexNodeType.java | 9 + .../file/metadata/statistics/BinaryStatistics.java | 2 +- .../iotdb/tsfile/read/TsFileSequenceReader.java | 208 +++++++--- .../org/apache/iotdb/tsfile/read/common/Path.java | 8 +- .../read/controller/MetadataQuerierByFileImpl.java | 10 +- .../apache/iotdb/tsfile/write/schema/Schema.java | 4 +- .../file/metadata/MetadataIndexNodeTest.java | 14 +- .../tsfile/file/metadata/utils/TestHelper.java | 5 +- 161 files changed, 5740 insertions(+), 2280 deletions(-) diff --cc session/src/main/java/org/apache/iotdb/session/Session.java index 213a55e,89f32b6..69ff7dd --- a/session/src/main/java/org/apache/iotdb/session/Session.java +++ b/session/src/main/java/org/apache/iotdb/session/Session.java @@@ -188,29 -191,90 +194,53 @@@ public class Session } } + private boolean reconnect() { + boolean flag = false; + for (int i = 1; i <= RETRY_NUM; i++) { + try { + if (transport != null) { + transport.close(); + } + isClosed = true; + open(enableRPCCompression, connectionTimeoutInMs); + flag = true; + break; + } catch (Exception e) { + try { + Thread.sleep(RETRY_INTERVAL); + } catch (InterruptedException e1) { + Thread.currentThread().interrupt(); + logger.error("reconnection is interrupted.", e1); + } + } + } + return flag; + } + /** - * insert data in one row, if you want to improve your performance, please use insertRecords method - * or insertTablet method + * insert data in one row, if you want to improve your performance, please use insertRecords + * method or insertTablet method * - * @see Session#insertRecords(List, List, List, List) + * @see Session#insertRecords(List, List, List, List, List) * @see Session#insertTablet(Tablet) */ public void insertRecord(String deviceId, long time, List<String> measurements, + List<TSDataType> types, Object... values) throws IoTDBConnectionException, StatementExecutionException { - List<String> stringValues = new ArrayList<>(); - for (Object o : values) { - stringValues.add(o.toString()); - } + List<Object> valuesList = new ArrayList<>(Arrays.asList(values)); - insertRecord(deviceId, time, measurements, stringValues); + insertRecord(deviceId, time, measurements, types, valuesList); } - /** - * insert data in one row, if you want to improve your performance, please use insertRecords method - * or insertTablet method - * - * @see Session#insertRecords(List, List, List, List) - * @see Session#insertTablet(Tablet) - */ - public void insertRecord(String deviceId, long time, List<String> measurements, - List<String> values) throws IoTDBConnectionException, StatementExecutionException { - TSInsertRecordReq request = new TSInsertRecordReq(); - request.setSessionId(sessionId); - request.setDeviceId(deviceId); - request.setTimestamp(time); - request.setMeasurements(measurements); - request.setValues(values); - - try { - RpcUtils.verifySuccess(client.insertRecord(request)); - } catch (TException e) { - if (reconnect()) { - try { - RpcUtils.verifySuccess(client.insertRecord(request)); - } catch (TException e1) { - throw new IoTDBConnectionException(e1); - } - } else { - throw new IoTDBConnectionException("Fail to reconnect to server," - + " please check server status", e); - } - } - } + /** * insert the data of a device. For each timestamp, the number of measurements is the same. - * - * a Tablet example: - * - * device1 - * time s1, s2, s3 - * 1, 1, 1, 1 - * 2, 2, 2, 2 - * 3, 3, 3, 3 - * + * <p> + * a Tablet example: + * <p> + * device1 time s1, s2, s3 1, 1, 1, 1 2, 2, 2, 2 3, 3, 3, 3 + * <p> * times in Tablet may be not in ascending order * * @param tablet data batch
