This is an automated email from the ASF dual-hosted git repository. qiaojialin pushed a commit to branch fix_recover_delete_bug in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 897543f4720002fc9edc3abfb6367be2d571c69f Author: qiaojialin <[email protected]> AuthorDate: Fri Apr 10 16:10:06 2020 +0800 fix recover delete bug --- .../writelog/recover/TsFileRecoverPerformer.java | 1 + .../iotdb/db/integration/IoTDBRestartIT.java | 43 ++++++++++++++++++++++ .../iotdb/tsfile/read/TsFileSequenceReader.java | 16 ++------ .../apache/iotdb/tsfile/utils/VersionUtils.java | 36 ++++++++++++++++++ .../write/writer/RestorableTsFileIOWriter.java | 4 ++ .../iotdb/tsfile/write/writer/TsFileIOWriter.java | 2 +- 6 files changed, 88 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java index aff32c8..71726cb 100644 --- a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java +++ b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java @@ -192,6 +192,7 @@ public class TsFileRecoverPerformer { private void redoLogs(RestorableTsFileIOWriter restorableTsFileIOWriter) throws StorageGroupProcessorException { IMemTable recoverMemTable = new PrimitiveMemTable(); + recoverMemTable.setVersion(versionController.nextVersion()); LogReplayer logReplayer = new LogReplayer(logNodePrefix, filePath, resource.getModFile(), versionController, resource, recoverMemTable, acceptUnseq); logReplayer.replayLogs(); diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBRestartIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBRestartIT.java index 931ecb2..a9854cd 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBRestartIT.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBRestartIT.java @@ -84,4 +84,47 @@ public class IoTDBRestartIT { EnvironmentUtils.cleanEnv(); } + + + @Test + public void testRestartDelete() + throws SQLException, ClassNotFoundException, IOException, StorageEngineException { + EnvironmentUtils.envSetUp(); + Class.forName(Config.JDBC_DRIVER_NAME); + + try(Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", + "root"); + Statement statement = connection.createStatement()){ + statement.execute("insert into root.turbine.d1(timestamp,s1) values(1,1)"); + statement.execute("insert into root.turbine.d1(timestamp,s1) values(2,2)"); + statement.execute("insert into root.turbine.d1(timestamp,s1) values(3,3)"); + } + + EnvironmentUtils.restartDaemon(); + + try(Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", + "root"); + Statement statement = connection.createStatement()){ + statement.execute("delete from root.turbine.d1.s1 where time<=1"); + statement.execute("flush"); + statement.execute("delete from root.turbine.d1.s1 where time<=2"); + + boolean hasResultSet = statement.execute("SELECT s1 FROM root.turbine.d1"); + assertTrue(hasResultSet); + String[] exp = new String[]{ + "3,3" + }; + ResultSet resultSet = statement.getResultSet(); + int cnt = 0; + while (resultSet.next()) { + String result = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(2); + assertEquals(exp[cnt], result); + cnt++; + } + } + + EnvironmentUtils.cleanEnv(); + } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java index 999ac28..79ccd96 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java @@ -40,6 +40,7 @@ import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl; import org.apache.iotdb.tsfile.read.reader.TsFileInput; import org.apache.iotdb.tsfile.utils.Pair; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; +import org.apache.iotdb.tsfile.utils.VersionUtils; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -347,22 +348,11 @@ public class TsFileSequenceReader implements AutoCloseable { // set version in ChunkMetadata List<Pair<Long, Long>> versionInfo = tsFileMetaData.getVersionInfo(); for (Entry<String, List<ChunkMetadata>> entry : seriesMetadata.entrySet()) { - setVersion(entry.getValue(), versionInfo); + VersionUtils.applyVersion(entry.getValue(), versionInfo); } return seriesMetadata; } - - private void setVersion(List<ChunkMetadata> chunkMetadataList, List<Pair<Long, Long>> versionInfo) { - int versionIndex = 0; - for (ChunkMetadata chunkMetadata : chunkMetadataList) { - while (chunkMetadata.getOffsetOfChunkHeader() >= versionInfo.get(versionIndex).left) { - versionIndex++; - } - chunkMetadata.setVersion(versionInfo.get(versionIndex).right); - } - } - /** * this function return all timeseries names in this file * @@ -721,7 +711,7 @@ public class TsFileSequenceReader implements AutoCloseable { chunkMetadataList.add(ChunkMetadata.deserializeFrom(buffer)); } - setVersion(chunkMetadataList, versionInfo); + VersionUtils.applyVersion(chunkMetadataList, versionInfo); return chunkMetadataList; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/VersionUtils.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/VersionUtils.java new file mode 100644 index 0000000..4948f68 --- /dev/null +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/VersionUtils.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.tsfile.utils; + +import java.util.List; +import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; + +public class VersionUtils { + + public static void applyVersion(List<ChunkMetadata> chunkMetadataList, List<Pair<Long, Long>> versionInfo) { + int versionIndex = 0; + for (ChunkMetadata chunkMetadata : chunkMetadataList) { + while (chunkMetadata.getOffsetOfChunkHeader() >= versionInfo.get(versionIndex).left) { + versionIndex++; + } + chunkMetadata.setVersion(versionInfo.get(versionIndex).right); + } + } + +} diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java index 3af18fc..f2755f5 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java @@ -39,6 +39,7 @@ import org.apache.iotdb.tsfile.read.TsFileCheckStatus; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.common.Path; import org.apache.iotdb.tsfile.utils.Pair; +import org.apache.iotdb.tsfile.utils.VersionUtils; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -195,6 +196,9 @@ public class RestorableTsFileIOWriter extends TsFileIOWriter { if (!newlyFlushedMetadataList.isEmpty()) { for (ChunkGroupMetadata chunkGroupMetadata : newlyFlushedMetadataList) { List<ChunkMetadata> rowMetaDataList = chunkGroupMetadata.getChunkMetadataList(); + + VersionUtils.applyVersion(rowMetaDataList, versionInfo); + String device = chunkGroupMetadata.getDevice(); for (ChunkMetadata chunkMetaData : rowMetaDataList) { String measurementId = chunkMetaData.getMeasurementUid(); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java index 6f0ed79..87ee259 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java @@ -83,7 +83,7 @@ public class TsFileIOWriter { private long markedPosition; private String currentChunkGroupDeviceId; private long currentChunkGroupStartOffset; - private List<Pair<Long, Long>> versionInfo = new ArrayList<>(); + protected List<Pair<Long, Long>> versionInfo = new ArrayList<>(); /** * empty construct function.
