This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch OverlappedPageBug in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 09b68bb7696d7282e6d12a187ed0f9ccbdb33dd9 Author: JackieTien97 <[email protected]> AuthorDate: Wed Jan 26 15:05:49 2022 +0800 [To rel/0.12] [IOTDB-2445] Fix overlapped data should be consumed first bug --- .../iotdb/db/query/reader/series/SeriesReader.java | 12 ++- .../db/integration/IoTDBUnseqOverlappedPageIT.java | 116 +++++++++++++++++++++ 2 files changed, 127 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java index 64141d6..f25f37a 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java @@ -816,6 +816,17 @@ public class SeriesReader { return; } + long currentPageEndpointTime; + + if (!unSeqPageReaders.isEmpty()) { + currentPageEndpointTime = + orderUtils.getOverlapCheckTime(unSeqPageReaders.peek().getStatistics()); + // unpack all overlapped data for first unseq page + unpackAllOverlappedTsFilesToTimeSeriesMetadata(currentPageEndpointTime); + unpackAllOverlappedTimeSeriesMetadataToCachedChunkMetadata(currentPageEndpointTime, false); + unpackAllOverlappedChunkMetadataToPageReaders(currentPageEndpointTime, false); + } + /* * init firstPageReader */ @@ -823,7 +834,6 @@ public class SeriesReader { initFirstPageReader(); } - long currentPageEndpointTime; if (mergeReader.hasNextTimeValuePair()) { currentPageEndpointTime = mergeReader.getCurrentReadStopTime(); } else { diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBUnseqOverlappedPageIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBUnseqOverlappedPageIT.java new file mode 100644 index 0000000..211aa7b --- /dev/null +++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBUnseqOverlappedPageIT.java @@ -0,0 +1,116 @@ +package org.apache.iotdb.db.integration; + +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.utils.EnvironmentUtils; +import org.apache.iotdb.jdbc.Config; +import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; + +import static org.apache.iotdb.db.constant.TestConstant.TIMESTAMP_STR; +import static org.junit.Assert.fail; + +/** + * Notice that, all test begins with "IoTDB" is integration test. All test which will start the + * IoTDB server should be defined as integration test. + * + * <p>You can comprehensively view the generated data in the following online doc: + * + * <p>https://docs.google.com/spreadsheets/d/1kfrSR1_paSd9B1Z0jnPBD3WQIMDslDuNm4R0mpWx9Ms/edit?usp=sharing + */ +public class IoTDBUnseqOverlappedPageIT { + + private static int beforeMaxNumberOfPointsInPage; + + private static String[] dataSet1 = + new String[] { + "SET STORAGE GROUP TO root.sg1", + "CREATE TIMESERIES root.sg1.d1.s1 WITH DATATYPE=INT32, ENCODING=PLAIN", + "INSERT INTO root.sg1.d1(time,s1) values(1, 1)", + "INSERT INTO root.sg1.d1(time,s1) values(2, 2)", + "INSERT INTO root.sg1.d1(time,s1) values(3, 3)", + "INSERT INTO root.sg1.d1(time,s1) values(4, 4)", + "INSERT INTO root.sg1.d1(time,s1) values(5, 5)", + "INSERT INTO root.sg1.d1(time,s1) values(6, 6)", + "flush", + "INSERT INTO root.sg1.d1(time,s1) values(8, 8)", + "INSERT INTO root.sg1.d1(time,s1) values(9, 9)", + "INSERT INTO root.sg1.d1(time,s1) values(10, 10)", + "INSERT INTO root.sg1.d1(time,s1) values(11, 11)", + "INSERT INTO root.sg1.d1(time,s1) values(12, 12)", + "INSERT INTO root.sg1.d1(time,s1) values(13, 13)", + "flush", + "INSERT INTO root.sg1.d1(time,s1) values(5, 50)", + "INSERT INTO root.sg1.d1(time,s1) values(7, 70)", + "INSERT INTO root.sg1.d1(time,s1) values(11, 110)", + "INSERT INTO root.sg1.d1(time,s1) values(12, 120)", + "flush", + }; + + @BeforeClass + public static void setUp() throws Exception { + Class.forName(Config.JDBC_DRIVER_NAME); + EnvironmentUtils.closeStatMonitor(); + // max_number_of_points_in_page = 10 + beforeMaxNumberOfPointsInPage = + TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage(); + TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(3); + EnvironmentUtils.envSetUp(); + try (Connection connection = + DriverManager.getConnection( + Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + for (String insertSql : dataSet1) { + statement.execute(insertSql); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @AfterClass + public static void tearDown() throws Exception { + // recovery value + EnvironmentUtils.cleanEnv(); + IoTDBDescriptor.getInstance() + .getConfig() + .setMemtableSizeThreshold(beforeMaxNumberOfPointsInPage); + } + + @Test + public void selectOverlappedPageTest() { + String[] res = { + "1,1", "2,2", "3,3", "4,4", "5,50", "6,6", "7,70", "8,8", "9,9", "10,10", "11,110", "12,120", + "13,13" + }; + + try (Connection connection = + DriverManager.getConnection( + Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + + String sql = "select s1 from root.sg1.d1"; + try (ResultSet resultSet = statement.executeQuery(sql)) { + int cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString("root.sg1.d1.s1"); + Assert.assertEquals(res[cnt], ans); + cnt++; + } + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } +}
