This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch TyUnseqTimeSeriesMetadata in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 247ab2cbeeca19d72ea86edf692be914ebe62e79 Author: JackieTien97 <[email protected]> AuthorDate: Tue Apr 14 20:15:27 2020 +0800 Solve it --- .../apache/iotdb/db/query/fill/PreviousFill.java | 6 +- .../iotdb/db/query/reader/series/SeriesReader.java | 8 +- .../org/apache/iotdb/db/utils/FileLoaderUtils.java | 5 +- .../IoTDBMultiOverlappedChunkInUnseqIT.java | 109 +++++++++++++++++++++ 4 files changed, 120 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/query/fill/PreviousFill.java b/server/src/main/java/org/apache/iotdb/db/query/fill/PreviousFill.java index b209c58..fdd9642 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/fill/PreviousFill.java +++ b/server/src/main/java/org/apache/iotdb/db/query/fill/PreviousFill.java @@ -139,7 +139,7 @@ public class PreviousFill extends IFill { TsFileResource resource = seqFileResource.get(index); TimeseriesMetadata timeseriesMetadata = FileLoaderUtils.loadTimeSeriesMetadata( - resource, seriesPath, context, timeFilter, allSensors); + resource, seriesPath, context, timeFilter, allSensors, false); if (timeseriesMetadata != null) { if (timeseriesMetadata.getStatistics().canUseStatistics() && endtimeContainedByTimeFilter(timeseriesMetadata.getStatistics())) { @@ -180,7 +180,7 @@ public class PreviousFill extends IFill { } TimeseriesMetadata timeseriesMetadata = FileLoaderUtils.loadTimeSeriesMetadata( - unseqFileResource.poll(), seriesPath, context, timeFilter, allSensors); + unseqFileResource.poll(), seriesPath, context, timeFilter, allSensors, true); if (timeseriesMetadata != null && timeseriesMetadata.getStatistics().canUseStatistics() && lBoundTime <= timeseriesMetadata.getStatistics().getEndTime()) { // The last timeseriesMetadata will be used as a pivot to filter the rest unseq files. @@ -196,7 +196,7 @@ public class PreviousFill extends IFill { && (lBoundTime <= unseqFileResource.peek().getEndTimeMap().get(seriesPath.getDevice()))) { TimeseriesMetadata timeseriesMetadata = FileLoaderUtils.loadTimeSeriesMetadata( - unseqFileResource.poll(), seriesPath, context, timeFilter, allSensors); + unseqFileResource.poll(), seriesPath, context, timeFilter, allSensors, true); unseqTimeseriesMetadataList.add(timeseriesMetadata); // update lBoundTime if current unseq timeseriesMetadata's last point is a valid result if (timeseriesMetadata.getStatistics().canUseStatistics() diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java index 5590615..04c3ff9 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java @@ -543,7 +543,7 @@ class SeriesReader { * Fill sequence TimeSeriesMetadata List until it is not empty */ while (seqTimeSeriesMetadata.isEmpty() && !seqFileResource.isEmpty()) { - TimeseriesMetadata timeseriesMetadata = FileLoaderUtils.loadTimeSeriesMetadata(seqFileResource.remove(0), seriesPath, context, timeFilter, allSensors); + TimeseriesMetadata timeseriesMetadata = FileLoaderUtils.loadTimeSeriesMetadata(seqFileResource.remove(0), seriesPath, context, timeFilter, allSensors, false); if (timeseriesMetadata != null) { seqTimeSeriesMetadata.add(timeseriesMetadata); } @@ -553,7 +553,7 @@ class SeriesReader { * Fill unSequence TimeSeriesMetadata Priority Queue until it is not empty */ while (unSeqTimeSeriesMetadata.isEmpty() && !unseqFileResource.isEmpty()) { - TimeseriesMetadata timeseriesMetadata = FileLoaderUtils.loadTimeSeriesMetadata(unseqFileResource.remove(0), seriesPath, context, timeFilter, allSensors); + TimeseriesMetadata timeseriesMetadata = FileLoaderUtils.loadTimeSeriesMetadata(unseqFileResource.remove(0), seriesPath, context, timeFilter, allSensors, true); if (timeseriesMetadata != null) { unSeqTimeSeriesMetadata.add(timeseriesMetadata); } @@ -608,13 +608,13 @@ class SeriesReader { private void unpackAllOverlappedTsFilesToTimeSeriesMetadata(long endTime) throws IOException { while (!unseqFileResource.isEmpty() && endTime >= unseqFileResource.get(0).getStartTimeMap().get(seriesPath.getDevice())) { - TimeseriesMetadata timeseriesMetadata = FileLoaderUtils.loadTimeSeriesMetadata(unseqFileResource.remove(0), seriesPath, context, timeFilter, allSensors); + TimeseriesMetadata timeseriesMetadata = FileLoaderUtils.loadTimeSeriesMetadata(unseqFileResource.remove(0), seriesPath, context, timeFilter, allSensors, true); if (timeseriesMetadata != null) { unSeqTimeSeriesMetadata.add(timeseriesMetadata); } } while (!seqFileResource.isEmpty() && endTime >= seqFileResource.get(0).getStartTimeMap().get(seriesPath.getDevice())) { - TimeseriesMetadata timeseriesMetadata = FileLoaderUtils.loadTimeSeriesMetadata(seqFileResource.remove(0), seriesPath, context, timeFilter, allSensors); + TimeseriesMetadata timeseriesMetadata = FileLoaderUtils.loadTimeSeriesMetadata(seqFileResource.remove(0), seriesPath, context, timeFilter, allSensors, false); if (timeseriesMetadata != null) { seqTimeSeriesMetadata.add(timeseriesMetadata); } diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java index 8884a7d..fa1f9bc 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java @@ -90,7 +90,7 @@ public class FileLoaderUtils { * @param allSensors measurements queried at the same time of this device */ public static TimeseriesMetadata loadTimeSeriesMetadata(TsFileResource resource, Path seriesPath, - QueryContext context, Filter timeFilter, Set<String> allSensors) throws IOException { + QueryContext context, Filter timeFilter, Set<String> allSensors, boolean isUnseq) throws IOException { TimeseriesMetadata timeSeriesMetadata; if (resource.isClosed()) { timeSeriesMetadata = TimeSeriesMetadataCache.getInstance() @@ -122,6 +122,9 @@ public class FileLoaderUtils { return null; } } + if (timeSeriesMetadata != null && timeSeriesMetadata.getStatistics() != null && isUnseq) { + timeSeriesMetadata.getStatistics().setCanUseStatistics(false); + } return timeSeriesMetadata; } diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBMultiOverlappedChunkInUnseqIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBMultiOverlappedChunkInUnseqIT.java new file mode 100644 index 0000000..294481d --- /dev/null +++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBMultiOverlappedChunkInUnseqIT.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.integration; + +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.utils.EnvironmentUtils; +import org.apache.iotdb.jdbc.Config; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; + +import static org.apache.iotdb.db.constant.TestConstant.count; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * Notice that, all test begins with "IoTDB" is integration test. All test which will start the + * IoTDB server should be defined as integration test. + */ +public class IoTDBMultiOverlappedChunkInUnseqIT { + + private static long beforeMemtableSizeThreshold; + + @BeforeClass + public static void setUp() throws Exception { + EnvironmentUtils.closeStatMonitor(); + beforeMemtableSizeThreshold = IoTDBDescriptor.getInstance().getConfig().getMemtableSizeThreshold(); + IoTDBDescriptor.getInstance().getConfig().setMemtableSizeThreshold(1024); + EnvironmentUtils.envSetUp(); + Class.forName(Config.JDBC_DRIVER_NAME); + insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + // recovery value + EnvironmentUtils.cleanEnv(); + IoTDBDescriptor.getInstance().getConfig().setMemtableSizeThreshold(beforeMemtableSizeThreshold); + } + + @Test + public void selectOverlappedPageTest() { + + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + String sql = "select count(s0) from root.vehicle.d0 where time < 1000000"; + ResultSet resultSet = statement.executeQuery(sql); + while (resultSet.next()) { + String ans = resultSet.getString(count("root.vehicle.d0.s0")); + assertEquals("1000", ans); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + private static void insertData() { + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + + statement.execute("CREATE TIMESERIES root.vehicle.d0.s0 WITH DATATYPE=INT32, ENCODING=RLE"); + + String sql = String + .format("insert into root.vehicle.d0(timestamp,s0) values(%s,%s)", 1000000, 1000000); + statement.execute(sql); + + statement.execute("flush"); + for (long time = 1; time <= 1000; time++) { + sql = String + .format("insert into root.vehicle.d0(timestamp,s0) values(%s,%s)", time, time); + statement.execute(sql); + } + for (long time = 2; time <= 1000; time++) { + sql = String + .format("insert into root.vehicle.d0(timestamp,s0) values(%s,%s)", time, 1000); + statement.execute(sql); + } + statement.execute("flush"); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } +} \ No newline at end of file
