This is an automated email from the ASF dual-hosted git repository. east pushed a commit to branch nvmlogging in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit cb66b434f33b3abd3c482a58224aabc369f69c6d Merge: fec0d2d df5dd1b Author: mdf369 <[email protected]> AuthorDate: Mon Dec 16 20:44:14 2019 +0800 Merge branch 'master' into nvmlogging # Conflicts: # server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java .travis.yml | 16 +- README.md | 4 +- client-py/compile.bat | 4 +- client-py/compile.sh | 4 +- client-py/readme.md | 2 +- client-py/src/client_example.py | 133 +- client/pom.xml | 2 +- client/src/test/resources/logback.xml | 2 +- distribution/pom.xml | 2 +- docker/src/main/Dockerfile | 4 +- docs/Development-Contributing.md | 6 +- docs/Development-Document.md | 48 + .../UserGuide/2-Concept/3-Encoding.md | 4 +- .../UserGuide/2-Concept/4-Compression.md | 2 +- .../UserGuide/4-Client/2-Programming - JDBC.md | 1 - .../UserGuide/4-Client/3-Programming - Session.md | 2 +- .../1-DDL (Data Definition Language).md | 4 +- .../2-DML (Data Manipulation Language).md | 96 +- .../5-Operation Manual/4-SQL Reference.md | 33 +- .../UserGuide/6-System Tools/5-Log Visualizer.md | 2 +- .../6-System Tools/8-Load External Tsfile.md | 78 + .../7-Ecosystem Integration/2-MapReduce TsFile.md | 2 +- .../7-Ecosystem Integration/5-Hive TsFile.md | 8 +- .../8-System Design (Developer)/1-Hierarchy.md | 8 +- .../3-Writing Data on HDFS.md | 6 +- .../0-Get Started/2-Frequently asked questions.md | 2 +- .../UserGuide/2-Concept/3-Encoding.md | 2 - .../UserGuide/2-Concept/4-Compression.md | 2 +- .../UserGuide/3-Server/5-Docker Image.md | 2 +- .../UserGuide/4-Client/2-Programming - JDBC.md | 3 +- .../UserGuide/4-Client/3-Programming - Session.md | 14 +- .../4-Client/5-Programming - TsFile API.md | 6 +- .../2-DML (Data Manipulation Language).md | 73 +- .../5-Operation Manual/4-SQL Reference.md | 32 +- .../UserGuide/6-System Tools/5-Log Visualizer.md | 2 +- .../6-System Tools/8-Load External Tsfile.md | 72 + .../7-Ecosystem Integration/2-MapReduce TsFile.md | 2 +- .../7-Ecosystem Integration/3-Spark TsFile.md | 6 +- .../7-Ecosystem Integration/4-Spark IoTDB.md | 8 +- .../7-Ecosystem Integration/5-Hive TsFile.md | 10 +- .../8-System Design (Developer)/1-Hierarchy.md | 8 +- .../3-Writing Data on HDFS.md | 6 +- example/hadoop/pom.xml | 6 +- example/jdbc/pom.xml | 2 +- .../main/java/org/apache/iotdb/JDBCExample.java | 7 +- example/kafka/pom.xml | 4 +- example/kafka/readme.md | 4 +- example/pom.xml | 2 +- example/rocketmq/pom.xml | 2 +- example/rocketmq/readme.md | 4 +- example/session/pom.xml | 2 +- .../main/java/org/apache/iotdb/SessionExample.java | 4 +- example/tsfile/pom.xml | 2 +- example/tsfile/readme.md | 2 +- .../iotdb/tsfile/TsFileWriteWithTSRecord.java | 4 +- grafana/conf/application.properties | 5 +- grafana/pom.xml | 2 +- .../iotdb/web/grafana/dao/impl/BasicDaoImpl.java | 39 +- hadoop/README.md | 2 +- hadoop/pom.xml | 4 +- .../apache/iotdb/hadoop/tsfile/TSFHadoopTest.java | 3 +- .../iotdb/hadoop/tsfile/constant/TestConstant.java | 18 +- hadoop/src/test/resources/logback.xml | 2 +- hive-connector/pom.xml | 12 +- .../apache/iotdb/hive/TSFHiveInputFormatTest.java | 9 +- .../apache/iotdb/hive/TSFHiveRecordReaderTest.java | 3 +- .../apache/iotdb/hive/constant/TestConstant.java | 18 +- jdbc/README.md | 3 +- jdbc/pom.xml | 2 +- .../main/java/org/apache/iotdb/jdbc/Constant.java | 3 - .../org/apache/iotdb/jdbc/IoTDBConnection.java | 150 +- .../apache/iotdb/jdbc/IoTDBDatabaseMetadata.java | 44 +- .../iotdb/jdbc/IoTDBMetadataResultMetadata.java | 2 +- .../apache/iotdb/jdbc/IoTDBMetadataResultSet.java | 5 - .../jdbc/IoTDBPreparedInsertionStatement.java | 15 +- .../apache/iotdb/jdbc/IoTDBPreparedStatement.java | 65 +- .../org/apache/iotdb/jdbc/IoTDBQueryResultSet.java | 321 ++-- .../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 125 +- .../src/main/java/org/apache/iotdb/jdbc/Utils.java | 170 +- .../test/java/org/apache/iotdb/jdbc/BatchTest.java | 34 +- .../org/apache/iotdb/jdbc/IoTDBConnectionTest.java | 10 +- .../iotdb/jdbc/IoTDBDatabaseMetadataTest.java | 4 +- .../iotdb/jdbc/IoTDBPreparedStatementTest.java | 38 +- .../apache/iotdb/jdbc/IoTDBQueryResultSetTest.java | 142 +- .../org/apache/iotdb/jdbc/IoTDBStatementTest.java | 35 +- .../test/java/org/apache/iotdb/jdbc/UtilsTest.java | 228 +-- pom.xml | 19 +- server/pom.xml | 15 +- .../resources/conf/iotdb-engine.properties | 2 +- .../org/apache/iotdb/db/sql/parse/TqlLexer.g | 577 ------ .../org/apache/iotdb/db/sql/parse/TqlParser.g | 824 -------- .../org/apache/iotdb/db/qp/strategy/SqlBase.g4 | 862 +++++++++ .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 27 +- .../org/apache/iotdb/db/conf/IoTDBConstant.java | 19 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 4 +- .../iotdb/db/conf/adapter/CompressionRatio.java | 12 +- .../org/apache/iotdb/db/engine/StorageEngine.java | 30 +- .../engine/storagegroup/StorageGroupProcessor.java | 300 ++- .../db/engine/storagegroup/TsFileResource.java | 8 + .../exception/query/IllegalASTFormatException.java | 68 - .../runtime/SQLParserException.java} | 30 +- .../org/apache/iotdb/db/metadata/MManager.java | 23 +- .../org/apache/iotdb/db/qp/QueryProcessor.java | 52 +- .../apache/iotdb/db/qp/constant/SQLConstant.java | 29 +- .../iotdb/db/qp/constant/TqlParserConstant.java | 71 - .../qp/executor/AbstractQueryProcessExecutor.java | 113 +- .../db/qp/executor/IQueryProcessExecutor.java | 3 +- .../iotdb/db/qp/executor/QueryProcessExecutor.java | 263 ++- .../org/apache/iotdb/db/qp/logical/Operator.java | 2 +- .../db/qp/logical/crud/BasicFunctionOperator.java | 5 +- .../db/qp/logical/crud/BasicOperatorType.java | 5 +- .../iotdb/db/qp/logical/crud/QueryOperator.java | 70 +- .../iotdb/db/qp/logical/sys/LoadFilesOperator.java | 67 + .../sys/MoveFileOperator.java} | 33 +- .../logical/sys/RemoveFileOperator.java} | 37 +- .../iotdb/db/qp/physical/crud/GroupByPlan.java | 41 +- .../iotdb/db/qp/physical/crud/QueryPlan.java | 23 + .../db/qp/physical/sys/LoadConfigurationPlan.java | 3 +- .../sys/{ShowPlan.java => OperateFilePlan.java} | 56 +- .../apache/iotdb/db/qp/physical/sys/ShowPlan.java | 2 +- .../iotdb/db/qp/strategy/LogicalGenerator.java | 1952 +++++++++---------- .../strategy/LogicalGeneratorError.java} | 29 +- .../apache/iotdb/db/qp/strategy/ParseDriver.java | 76 + .../iotdb/db/qp/strategy/PhysicalGenerator.java | 41 +- .../qp/strategy/optimizer/ConcatPathOptimizer.java | 3 +- .../iotdb/db/query/context/QueryContext.java | 10 +- .../{JobFileManager.java => QueryFileManager.java} | 50 +- .../db/query/control/QueryResourceManager.java | 35 +- .../db/query/dataset/DeviceIterateDataSet.java | 24 +- .../dataset/EngineDataSetWithValueFilter.java | 4 +- .../dataset/EngineDataSetWithoutValueFilter.java | 13 +- .../apache/iotdb/db/query/dataset/ListDataSet.java | 5 +- .../{ListDataSet.java => SingleDataSet.java} | 25 +- .../dataset/groupby/GroupByEngineDataSet.java | 67 +- .../groupby/GroupByWithValueFilterDataSet.java | 30 +- .../groupby/GroupByWithoutValueFilterDataSet.java | 19 +- .../db/query/executor/AggregateEngineExecutor.java | 2 +- .../iotdb/db/query/executor/EngineExecutor.java | 2 +- .../iotdb/db/query/executor/EngineQueryRouter.java | 61 +- .../db/query/executor/FillEngineExecutor.java | 5 +- .../db/query/executor/IEngineQueryRouter.java | 15 +- .../db/query/externalsort/ExternalSortJob.java | 12 +- .../externalsort/ExternalSortJobScheduler.java | 10 +- .../externalsort/SimpleExternalSortEngine.java | 2 +- .../java/org/apache/iotdb/db/query/fill/IFill.java | 2 +- .../resourceRelated/UnseqResourceMergeReader.java | 2 +- .../UnseqResourceReaderByTimestamp.java | 2 +- .../SeriesReaderWithoutValueFilter.java | 9 +- .../java/org/apache/iotdb/db/service/IoTDB.java | 1 - .../apache/iotdb/db/service/MetricsService.java | 290 +-- .../org/apache/iotdb/db/service/StaticResps.java | 89 + .../org/apache/iotdb/db/service/TSServiceImpl.java | 792 ++++---- .../apache/iotdb/db/sql/parse/AstErrorNode.java | 58 - .../org/apache/iotdb/db/sql/parse/AstNode.java | 362 ---- .../apache/iotdb/db/sql/parse/AstNodeOrigin.java | 110 -- .../java/org/apache/iotdb/db/sql/parse/Node.java | 42 - .../org/apache/iotdb/db/sql/parse/ParseDriver.java | 192 -- .../org/apache/iotdb/db/sql/parse/ParseError.java | 52 - .../org/apache/iotdb/db/sql/parse/ParseUtils.java | 43 - .../iotdb/db/sync/receiver/load/FileLoader.java | 43 +- .../{MemEst => memestimation}/MemEstTool.java | 2 +- .../{MemEst => memestimation}/MemEstToolCmd.java | 2 +- .../org/apache/iotdb/db/utils/FileLoaderUtils.java | 62 + .../apache/iotdb/db/utils/QueryDataSetUtils.java | 90 +- .../writelog/recover/TsFileRecoverPerformer.java | 6 +- .../iotdb/db/auth/LocalFIleRoleAccessorTest.java | 3 +- .../iotdb/db/auth/LocalFileRoleManagerTest.java | 3 +- .../iotdb/db/auth/LocalFileUserAccessorTest.java | 3 +- .../iotdb/db/auth/LocalFileUserManagerTest.java | 3 +- .../strategy/DirectoryStrategyTest.java | 4 +- .../org/apache/iotdb/db/constant/TestConstant.java | 15 +- .../db/engine/cache/DeviceMetaDataCacheTest.java | 5 +- .../db/engine/memtable/MemTableFlushTaskTest.java | 5 +- .../apache/iotdb/db/engine/merge/MergeLogTest.java | 3 +- .../iotdb/db/engine/merge/MergeOverLapTest.java | 19 +- .../iotdb/db/engine/merge/MergePerfTest.java | 3 +- .../iotdb/db/engine/merge/MergeTaskTest.java | 3 +- .../apache/iotdb/db/engine/merge/MergeTest.java | 13 +- .../iotdb/db/engine/merge/MergeUpgradeTest.java | 13 +- .../engine/modification/DeletionFileNodeTest.java | 4 +- .../engine/modification/ModificationFileTest.java | 6 +- .../io/LocalTextModificationAccessorTest.java | 6 +- .../storagegroup/StorageGroupProcessorTest.java | 5 +- .../iotdb/db/engine/storagegroup/TTLTest.java | 2 +- .../engine/storagegroup/TsFileProcessorTest.java | 6 +- .../version/SimpleFileVersionControllerTest.java | 3 +- .../iotdb/db/integration/IOTDBGroupByIT.java | 269 +-- .../integration/IOTDBGroupByInnerIntervalIT.java | 296 +++ .../apache/iotdb/db/integration/IoTDBCloseIT.java | 12 +- .../apache/iotdb/db/integration/IoTDBDaemonIT.java | 22 +- .../iotdb/db/integration/IoTDBGroupbyDeviceIT.java | 28 +- .../integration/IoTDBLoadExternalTsfileTest.java | 500 +++++ .../iotdb/db/integration/IoTDBMetadataFetchIT.java | 4 +- .../db/integration/IoTDBMultiStatementsIT.java | 12 +- .../iotdb/db/integration/IoTDBPreparedStmtIT.java | 2 - .../iotdb/db/integration/IoTDBQueryDemoIT.java | 94 +- .../db/integration/IoTDBSequenceDataQueryIT.java | 12 +- .../iotdb/db/integration/IoTDBSeriesReaderIT.java | 16 +- .../iotdb/db/metadata/MManagerImproveTest.java | 2 +- .../org/apache/iotdb/db/qp/QueryProcessorTest.java | 16 +- .../iotdb/db/qp/plan/LogicalPlanSmallTest.java | 237 ++- .../apache/iotdb/db/qp/plan/PhysicalPlanTest.java | 95 +- .../org/apache/iotdb/db/qp/plan/QPUpdateTest.java | 7 +- .../iotdb/db/qp/plan/TestConcatOptimizer.java | 5 +- .../iotdb/db/qp/strategy/LogicalGeneratorTest.java | 5 +- .../apache/iotdb/db/qp/utils/MemIntQpExecutor.java | 19 +- .../db/query/control/FileReaderManagerTest.java | 9 +- .../query/executor/GroupByEngineDataSetTest.java | 117 +- .../query/externalsort/ExternalSortEngineTest.java | 6 +- ...ExternalSortFileSerializerDeserializerTest.java | 3 +- .../iotdb/db/query/reader/ReaderTestHelper.java | 3 +- .../org/apache/iotdb/db/sql/TqlParserTest.java | 2021 -------------------- .../apache/iotdb/db/tools/IoTDBWatermarkTest.java | 5 +- .../org/apache/iotdb/db/tools/WalCheckerTest.java | 9 +- .../apache/iotdb/db/utils/EnvironmentUtils.java | 15 +- .../apache/iotdb/db/utils/OpenFileNumUtilTest.java | 3 +- server/src/test/resources/iotdb-engine.properties | 21 + server/src/test/resources/logback.xml | 2 +- service-rpc/pom.xml | 50 +- service-rpc/rpc-changelist.md | 26 + .../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 - service-rpc/src/main/thrift/rpc.thrift | 185 +- service-rpc/src/pypi/README.md | 63 + service-rpc/src/pypi/setup.py | 55 + session/pom.xml | 2 +- .../java/org/apache/iotdb/session/Session.java | 113 +- .../org/apache/iotdb/session/SessionDataSet.java | 159 +- .../org/apache/iotdb/session/SessionUtils.java | 78 +- .../org/apache/iotdb/session/IoTDBSessionIT.java | 84 +- .../iotdb/session/utils/EnvironmentUtils.java | 20 +- session/src/test/resources/iotdb-engine.properties | 20 + spark-iotdb-connector/Readme.md | 8 +- spark-iotdb-connector/pom.xml | 2 +- .../src/test/resources/iotdb-engine.properties | 21 + .../apache/iotdb/spark/db/EnvironmentUtils.java | 13 +- .../org/apache/iotdb/spark/db/IoTDBTest.scala | 2 + spark-tsfile/README.md | 6 +- spark-tsfile/pom.xml | 4 +- .../apache/iotdb/spark/constant/TestConstant.java | 17 +- .../apache/iotdb/spark/tsfile/ConverterTest.scala | 3 +- .../apache/iotdb/spark/tsfile/HDFSInputTest.java | 4 +- .../org/apache/iotdb/spark/tsfile/TSFileSuit.scala | 3 +- tsfile/README.md | 2 +- tsfile/pom.xml | 2 +- .../iotdb/tsfile/common/conf/TSFileConfig.java | 6 +- .../iotdb/tsfile/read/filter/GroupByFilter.java | 101 + .../tsfile/read/filter/factory/FilterType.java | 2 +- .../query/dataset/DataSetWithTimeGenerator.java | 5 +- .../query/dataset/DataSetWithoutTimeGenerator.java | 4 +- .../tsfile/read/query/dataset/QueryDataSet.java | 59 +- ...meseriesTestConstant.java => TestConstant.java} | 14 +- .../iotdb/tsfile/file/header/PageHeaderTest.java | 6 +- .../file/metadata/ChunkGroupMetaDataTest.java | 3 +- .../file/metadata/TimeSeriesMetadataTest.java | 4 +- .../file/metadata/TsDeviceMetadataIndexTest.java | 4 +- .../tsfile/file/metadata/TsFileMetaDataTest.java | 202 +- .../org/apache/iotdb/tsfile/read/ReadTest.java | 750 ++++---- .../iotdb/tsfile/read/TimePlainEncodeReadTest.java | 733 +++---- .../tsfile/read/filter/GroupByFilterTest.java | 126 ++ .../TsFileGeneratorForSeriesReaderByTimestamp.java | 451 ++--- .../apache/iotdb/tsfile/utils/BytesUtilsTest.java | 22 +- .../apache/iotdb/tsfile/utils/FileGenerator.java | 429 ++--- .../apache/iotdb/tsfile/utils/FileUtilsTest.java | 10 +- .../iotdb/tsfile/utils/TsFileGeneratorForTest.java | 7 +- .../org/apache/iotdb/tsfile/write/PerfTest.java | 7 +- .../iotdb/tsfile/write/ReadPageInMemTest.java | 3 +- .../iotdb/tsfile/write/TsFileIOWriterTest.java | 3 +- .../iotdb/tsfile/write/TsFileReadWriteTest.java | 3 +- .../org/apache/iotdb/tsfile/write/WriteTest.java | 7 +- .../iotdb/tsfile/write/writer/PageWriterTest.java | 4 +- .../write/writer/RestorableTsFileIOWriterTest.java | 4 +- tsfile/src/test/resources/logback.xml | 2 +- 272 files changed, 8641 insertions(+), 9986 deletions(-) diff --cc server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java index 8feb826,f71b8f3..eaee4bb --- a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java +++ b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java @@@ -68,9 -69,8 +70,9 @@@ public class EnvironmentUtils private static long oldGroupSizeInByte = config.getMemtableSizeThreshold(); public static void cleanEnv() throws IOException, StorageEngineException { + NVMSpaceManager.close(); - QueryResourceManager.getInstance().endQueryForGivenJob(TEST_QUERY_JOB_ID); + QueryResourceManager.getInstance().endQuery(TEST_QUERY_JOB_ID); // clear opened file streams FileReaderManager.getInstance().closeAndRemoveAllOpenedReaders(); @@@ -145,8 -141,8 +145,8 @@@ /** * disable memory control</br> this function should be called before all code in the setup */ - public static void envSetUp() throws StartupException, IOException { + public static void envSetUp() throws StartupException { - IoTDBDescriptor.getInstance().getConfig().setEnableParameterAdapter(false); + config.setEnableParameterAdapter(false); MManager.getInstance().init(); IoTDBConfigDynamicAdapter.getInstance().setInitialized(true); @@@ -168,10 -164,8 +168,10 @@@ MultiFileLogNodeManager.getInstance().start(); FlushManager.getInstance().start(); MergeManager.getINSTANCE().start(); - TEST_QUERY_JOB_ID = QueryResourceManager.getInstance().assignJobId(); + TEST_QUERY_JOB_ID = QueryResourceManager.getInstance().assignQueryId(); TEST_QUERY_CONTEXT = new QueryContext(TEST_QUERY_JOB_ID); + + NVMSpaceManager.init(config.getNvmDir()); } private static void createAllDir() { @@@ -187,10 -181,9 +187,11 @@@ createDir(config.getSystemDir()); // create wal createDir(config.getWalFolder()); + // create index + createDir(config.getIndexFileDir()); // create query createDir(config.getQueryDir()); + createDir(TestConstant.OUTPUT_DATA_DIR); // create data for (String dataDir : config.getDataDirs()) { createDir(dataDir);
