This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch IOTDB-6243 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 281b2c10620b974f8ad3cea88c4efd3c7ed6a819 Author: JackieTien97 <[email protected]> AuthorDate: Mon Nov 13 18:17:20 2023 +0800 add It --- .../db/it/fill/IoTDBFillWithThresholdInMSIT.java | 243 +++++++++++++++++++++ .../fill/filter/MonthIntervalNSFillFilter.java | 3 +- .../fill/filter/MonthIntervalUSFillFilter.java | 5 +- .../fill/filter/MonthIntervalFillFilterTest.java | 163 ++++++++++++++ .../fill/filter/MonthIntervalMSFillFilterTest.java | 55 ----- 5 files changed, 411 insertions(+), 58 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/fill/IoTDBFillWithThresholdInMSIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/fill/IoTDBFillWithThresholdInMSIT.java new file mode 100644 index 00000000000..bf425e3bbe9 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/fill/IoTDBFillWithThresholdInMSIT.java @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.it.fill; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.itbase.category.LocalStandaloneIT; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Locale; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({LocalStandaloneIT.class, ClusterIT.class}) +public class IoTDBFillWithThresholdInMSIT { + + private static String[] creationSqls = + new String[] { + "CREATE DATABASE root.fillTest", + "CREATE TIMESERIES root.fillTest.d0.s0 WITH DATATYPE=INT32, ENCODING=PLAIN", + "CREATE TIMESERIES root.fillTest.d0.s1 WITH DATATYPE=INT64, ENCODING=PLAIN", + "CREATE TIMESERIES root.fillTest.d0.s2 WITH DATATYPE=FLOAT, ENCODING=PLAIN", + "CREATE TIMESERIES root.fillTest.d0.s3 WITH DATATYPE=DOUBLE, ENCODING=PLAIN", + "CREATE TIMESERIES root.fillTest.d0.s4 WITH DATATYPE=TEXT, ENCODING=PLAIN", + "CREATE TIMESERIES root.fillTest.d0.s5 WITH DATATYPE=BOOLEAN, ENCODING=PLAIN", + }; + + @Before + public void setUp() throws Exception { + Locale.setDefault(Locale.ENGLISH); + + EnvFactory.getEnv().getConfig().getCommonConfig().setTimestampPrecision("ms"); + EnvFactory.getEnv().initClusterEnvironment(); + prepareData(); + } + + @After + public void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testFill() { + + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + String[] ans = { + "1675223280000,4,3,4.0,4.0,on,false", + "1675223340000,5,5,5.0,5.0,on,true", + "1675223400000,6,null,6.0,6.0,null,false", + "1675223460000,null,null,null,null,null,null", + "1675223520000,null,null,null,null,null,null", + "1675223580000,null,null,null,null,null,null", + "1675223640000,null,null,null,null,null,null", + "1675223700000,null,null,null,null,null,null", + "1675223760000,7,7,7.0,null,off,false", + "1675223820000,null,null,null,null,null,null", + "1675223880000,8,8,8.0,null,on,true" + }; + + try (ResultSet set = + statement.executeQuery( + "SELECT last_value(*) FROM root.fillTest.d0 group by((2023-02-01T11:47:00.000+08:00, 2023-02-01T11:58:00.000+08:00], 1m)")) { + int cnt = 0; + while (set.next()) { + String row = + set.getString("Time") + + "," + + set.getString("last_value(root.fillTest.d0.s0)") + + "," + + set.getString("last_value(root.fillTest.d0.s1)") + + "," + + set.getString("last_value(root.fillTest.d0.s2)") + + "," + + set.getString("last_value(root.fillTest.d0.s3)") + + "," + + set.getString("last_value(root.fillTest.d0.s4)") + + "," + + set.getString("last_value(root.fillTest.d0.s5)"); + + assertEquals(ans[cnt], row); + cnt++; + } + assertEquals(ans.length, cnt); + } + + ans = + new String[] { + "1675223280000,4,3,4.0,4.0,on,false", + "1675223340000,5,5,5.0,5.0,on,true", + "1675223400000,6,5,6.0,6.0,on,false", + "1675223460000,6,5,6.0,6.0,on,false", + "1675223520000,6,5,6.0,6.0,on,false", + "1675223580000,6,5,6.0,6.0,on,false", + "1675223640000,6,5,6.0,6.0,on,false", + "1675223700000,6,5,6.0,6.0,on,false", + "1675223760000,7,7,7.0,6.0,off,false", + "1675223820000,7,7,7.0,6.0,off,false", + "1675223880000,8,8,8.0,6.0,on,true" + }; + + try (ResultSet set = + statement.executeQuery( + "SELECT last_value(*) FROM root.fillTest.d0 group by((2023-02-01T11:47:00.000+08:00, 2023-02-01T11:58:00.000+08:00], 1m) FILL(PREVIOUS)")) { + int cnt = 0; + while (set.next()) { + String row = + set.getString("Time") + + "," + + set.getString("last_value(root.fillTest.d0.s0)") + + "," + + set.getString("last_value(root.fillTest.d0.s1)") + + "," + + set.getString("last_value(root.fillTest.d0.s2)") + + "," + + set.getString("last_value(root.fillTest.d0.s3)") + + "," + + set.getString("last_value(root.fillTest.d0.s4)") + + "," + + set.getString("last_value(root.fillTest.d0.s5)"); + assertEquals(ans[cnt], row); + cnt++; + } + assertEquals(ans.length, cnt); + } + + ans = + new String[] { + "1675223280000,4,3,4.0,4.0,on,false", + "1675223340000,5,5,5.0,5.0,on,true", + "1675223400000,6,5,6.0,6.0,on,false", + "1675223460000,6,5,6.0,6.0,on,false", + "1675223520000,6,5,6.0,6.0,on,false", + "1675223580000,6,5,6.0,6.0,on,false", + "1675223640000,6,5,6.0,6.0,on,false", + "1675223700000,6,5,6.0,6.0,on,false", + "1675223760000,7,7,7.0,6.0,off,false", + "1675223820000,7,7,7.0,6.0,off,false", + "1675223880000,8,8,8.0,6.0,off,false" + }; + + try (ResultSet set = + statement.executeQuery( + "SELECT last_value(*) FROM root.fillTest.d0 group by((2023-02-01T11:47:00.000+08:00, 2023-02-01T11:58:00.000+08:00], 1m) FILL(PREVIOUS, 2m)")) { + int cnt = 0; + while (set.next()) { + String row = + set.getString("Time") + + "," + + set.getString("last_value(root.fillTest.d0.s0)") + + "," + + set.getString("last_value(root.fillTest.d0.s1)") + + "," + + set.getString("last_value(root.fillTest.d0.s2)") + + "," + + set.getString("last_value(root.fillTest.d0.s3)") + + "," + + set.getString("last_value(root.fillTest.d0.s4)") + + "," + + set.getString("last_value(root.fillTest.d0.s5)"); + System.out.println("\"" + row + "\","); + // assertEquals(ans[cnt], row); + cnt++; + } + assertEquals(ans.length, cnt); + } + } catch (SQLException e) { + fail(e.getMessage()); + } + } + + private void prepareData() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + for (String sql : creationSqls) { + statement.addBatch(sql); + } + statement.executeBatch(); + statement.clearBatch(); + + // 2023-02-01T11:47:30.000+08:00 + statement.addBatch( + "INSERT INTO root.fillTest.d0(timestamp,s0,s1,s2,s3,s4,s5) VALUES(1675223250000, 1, 1, 1.0, null, 'on', true)"); + // 2023-02-01T11:47:40.001+08:00 + statement.addBatch( + "INSERT INTO root.fillTest.d0(timestamp,s0,s1,s2,s3,s4,s5) VALUES(1675223260001, null, 2, 2.0, 2.0, null, false)"); + // 2023-02-01T11:47:50.100+08:00 + statement.addBatch( + "INSERT INTO root.fillTest.d0(timestamp,s0,s1,s2,s3,s4,s5) VALUES(1675223270100, null, 3, null, 3.0, 'off', false)"); + // 2023-02-01T11:48:00.000+08:00 + statement.addBatch( + "INSERT INTO root.fillTest.d0(timestamp,s0,s1,s2,s3,s4,s5) VALUES(1675223280000, 4, null, 4.0, 4.0, 'on', null)"); + // 2023-02-01T11:48:50.000+08:00 + statement.addBatch( + "INSERT INTO root.fillTest.d0(timestamp,s0,s1,s2,s3,s4,s5) VALUES(1675223330000, 5, 5, 5.0, 5.0, 'on', true)"); + // 2023-02-01T11:49:00.001+08:00 + statement.addBatch( + "INSERT INTO root.fillTest.d0(timestamp,s0,s1,s2,s3,s4,s5) VALUES(1675223340001, 6, null, 6.0, 6.0, null, false)"); + // 2023-02-01T11:55:50.000+08:00 + statement.addBatch( + "INSERT INTO root.fillTest.d0(timestamp,s0,s1,s2,s3,s4,s5) VALUES(1675223750000, 7, 7, 7.0, null, 'off', false)"); + // 2023-02-01T11:57:50.000+08:00 + statement.addBatch( + "INSERT INTO root.fillTest.d0(timestamp,s0,s1,s2,s3,s4,s5) VALUES(1675223870000, 8, 8, 8.0, null, 'on', true)"); + + statement.executeBatch(); + + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalNSFillFilter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalNSFillFilter.java index 5fad98dcaa8..563fb941eb7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalNSFillFilter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalNSFillFilter.java @@ -38,7 +38,8 @@ public class MonthIntervalNSFillFilter extends AbstractMonthIntervalFillFilter { LocalDateTime localDateTime = LocalDateTime.ofInstant(instant, zone); Instant upper = localDateTime.plusMonths(monthDuration).plusNanos(nonMonthDuration).toInstant(zoneOffset); - long timeInNs = upper.getLong(ChronoField.NANO_OF_SECOND) + upper.getEpochSecond(); + long timeInNs = + upper.getLong(ChronoField.NANO_OF_SECOND) + upper.getEpochSecond() * 1_000_000_000L; return timeInNs >= greater; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalUSFillFilter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalUSFillFilter.java index d5f0647d0b4..afde3e5141f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalUSFillFilter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalUSFillFilter.java @@ -36,14 +36,15 @@ public class MonthIntervalUSFillFilter extends AbstractMonthIntervalFillFilter { public boolean needFill(long time, long previousTime) { long smaller = Math.min(time, previousTime); long greater = Math.max(time, previousTime); - Instant instant = Instant.ofEpochSecond(smaller / 1_000_000_000L, smaller % 1_000_000_000L); + Instant instant = Instant.ofEpochSecond(smaller / 1_000_000L, smaller % 1_000_000L); LocalDateTime localDateTime = LocalDateTime.ofInstant(instant, zone); Instant upper = localDateTime .plusMonths(monthDuration) .plus(nonMonthDuration, MICROS) .toInstant(zoneOffset); - long timeInUs = upper.getLong(ChronoField.MICRO_OF_SECOND) + instant.getEpochSecond(); + long timeInUs = + upper.getLong(ChronoField.MICRO_OF_SECOND) + upper.getEpochSecond() * 1_000_000L; return timeInUs >= greater; } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalFillFilterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalFillFilterTest.java new file mode 100644 index 00000000000..04c81eb15f7 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalFillFilterTest.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.fill.filter; + +import org.junit.Test; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoField; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class MonthIntervalFillFilterTest { + + @Test + public void testMonthIntervalMSFillFilter() { + MonthIntervalMSFillFilter fillFilter1 = + new MonthIntervalMSFillFilter(1, 0, ZoneId.systemDefault()); + MonthIntervalMSFillFilter fillFilter2 = + new MonthIntervalMSFillFilter(1, 86_400_000, ZoneId.systemDefault()); + DateTimeFormatter formatter = DateTimeFormatter.ISO_LOCAL_DATE_TIME; + LocalDateTime localDateTime = LocalDateTime.parse("2023-02-01T11:47:30", formatter); + + ZoneOffset zoneOffset = ZoneId.systemDefault().getRules().getOffset(localDateTime); + + Instant instant = localDateTime.toInstant(zoneOffset); + + long previousTime = instant.toEpochMilli(); + + assertTrue( + fillFilter1.needFill( + localDateTime.plusMonths(1).toInstant(zoneOffset).toEpochMilli(), previousTime)); + assertFalse( + fillFilter1.needFill( + localDateTime.plusMonths(1).toInstant(zoneOffset).toEpochMilli() + 1, previousTime)); + + assertTrue( + fillFilter2.needFill( + localDateTime.plusMonths(1).toInstant(zoneOffset).toEpochMilli() + 1, previousTime)); + + assertTrue( + fillFilter2.needFill( + localDateTime.plusMonths(1).plusDays(1).toInstant(zoneOffset).toEpochMilli(), + previousTime)); + assertFalse( + fillFilter2.needFill( + localDateTime.plusMonths(1).plusDays(1).toInstant(zoneOffset).toEpochMilli() + 1, + previousTime)); + } + + @Test + public void testMonthIntervalUSFillFilter() { + + MonthIntervalUSFillFilter fillFilter1 = + new MonthIntervalUSFillFilter(1, 0, ZoneId.systemDefault()); + MonthIntervalUSFillFilter fillFilter2 = + new MonthIntervalUSFillFilter(1, 86_400_000_000L, ZoneId.systemDefault()); + DateTimeFormatter formatter = DateTimeFormatter.ISO_LOCAL_DATE_TIME; + LocalDateTime localDateTime = LocalDateTime.parse("2023-02-01T11:47:30", formatter); + + ZoneOffset zoneOffset = ZoneId.systemDefault().getRules().getOffset(localDateTime); + + Instant instant = localDateTime.toInstant(zoneOffset); + + long previousTime = + instant.getEpochSecond() * 1_000_000L + instant.getLong(ChronoField.MICRO_OF_SECOND); + + Instant current = localDateTime.plusMonths(1).toInstant(zoneOffset); + assertTrue( + fillFilter1.needFill( + current.getEpochSecond() * 1_000_000 + current.getLong(ChronoField.MICRO_OF_SECOND), + previousTime)); + assertFalse( + fillFilter1.needFill( + current.getEpochSecond() * 1_000_000 + current.getLong(ChronoField.MICRO_OF_SECOND) + 1, + previousTime)); + + assertTrue( + fillFilter2.needFill( + current.getEpochSecond() * 1_000_000 + current.getLong(ChronoField.MICRO_OF_SECOND) + 1, + previousTime)); + + current = localDateTime.plusMonths(1).plusDays(1).toInstant(zoneOffset); + assertTrue( + fillFilter2.needFill( + current.getEpochSecond() * 1_000_000 + current.getLong(ChronoField.MICRO_OF_SECOND), + previousTime)); + assertFalse( + fillFilter2.needFill( + current.getEpochSecond() * 1_000_000 + current.getLong(ChronoField.MICRO_OF_SECOND) + 1, + previousTime)); + } + + @Test + public void testMonthIntervalNSFillFilter() { + + MonthIntervalNSFillFilter fillFilter1 = + new MonthIntervalNSFillFilter(1, 0, ZoneId.systemDefault()); + MonthIntervalNSFillFilter fillFilter2 = + new MonthIntervalNSFillFilter(1, 86_400_000_000_000L, ZoneId.systemDefault()); + DateTimeFormatter formatter = DateTimeFormatter.ISO_LOCAL_DATE_TIME; + LocalDateTime localDateTime = LocalDateTime.parse("2023-02-01T11:47:30", formatter); + + ZoneOffset zoneOffset = ZoneId.systemDefault().getRules().getOffset(localDateTime); + + Instant instant = localDateTime.toInstant(zoneOffset); + + long previousTime = + instant.getEpochSecond() * 1_000_000_000L + instant.getLong(ChronoField.NANO_OF_SECOND); + + Instant current = localDateTime.plusMonths(1).toInstant(zoneOffset); + assertTrue( + fillFilter1.needFill( + current.getEpochSecond() * 1_000_000_000 + current.getLong(ChronoField.NANO_OF_SECOND), + previousTime)); + assertFalse( + fillFilter1.needFill( + current.getEpochSecond() * 1_000_000_000 + + current.getLong(ChronoField.NANO_OF_SECOND) + + 1, + previousTime)); + + assertTrue( + fillFilter2.needFill( + current.getEpochSecond() * 1_000_000_000 + + current.getLong(ChronoField.NANO_OF_SECOND) + + 1, + previousTime)); + + current = localDateTime.plusMonths(1).plusDays(1).toInstant(zoneOffset); + assertTrue( + fillFilter2.needFill( + current.getEpochSecond() * 1_000_000_000 + current.getLong(ChronoField.NANO_OF_SECOND), + previousTime)); + assertFalse( + fillFilter2.needFill( + current.getEpochSecond() * 1_000_000_000 + + current.getLong(ChronoField.NANO_OF_SECOND) + + 1, + previousTime)); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalMSFillFilterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalMSFillFilterTest.java deleted file mode 100644 index f5dbc378123..00000000000 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalMSFillFilterTest.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.execution.operator.process.fill.filter; - -import org.junit.Test; - -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -public class MonthIntervalMSFillFilterTest { - - @Test - public void testMonthIntervalMSFillFilter() { - MonthIntervalMSFillFilter fillFilter = - new MonthIntervalMSFillFilter(1, 0, ZoneId.systemDefault()); - DateTimeFormatter formatter = DateTimeFormatter.ISO_LOCAL_DATE_TIME; - LocalDateTime localDateTime = LocalDateTime.parse("2023-02-01T11:47:30", formatter); - - ZoneOffset zoneOffset = ZoneId.systemDefault().getRules().getOffset(localDateTime); - - Instant instant = localDateTime.toInstant(zoneOffset); - - long previousTime = instant.toEpochMilli(); - - assertTrue( - fillFilter.needFill( - localDateTime.plusMonths(1).toInstant(zoneOffset).toEpochMilli(), previousTime)); - assertFalse( - fillFilter.needFill( - localDateTime.plusMonths(1).toInstant(zoneOffset).toEpochMilli() + 1, previousTime)); - } -}
