JackieTien97 commented on code in PR #18589:
URL: https://github.com/apache/iotdb/pull/18589#discussion_r4475309537


##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/FixedIntervalFillFilter.java:
##########
@@ -34,14 +36,6 @@ public FixedIntervalFillFilter(long timeInterval) {
   public boolean needFill(long time, long previousTime) {
     // the reason that we use Math.abs is that we may use order by time desc 
which will cause
     // previousTime is larger than time
-    return isTimeDistanceLessThanOrEqual(time, previousTime, timeInterval);
-  }
-
-  private boolean isTimeDistanceLessThanOrEqual(long left, long right, long 
maxDistance) {
-    if (maxDistance < 0) {
-      return false;
-    }
-    long distance = left >= right ? left - right : right - left;
-    return Long.compareUnsigned(distance, maxDistance) <= 0;
+    return isTimeDistanceLessThanOrEqualTo(time, previousTime, timeInterval);

Review Comment:
   Verified. With `timeInterval = 15552000000000000L` (180 days in ns) and 
`previousTime = 1700000000000000000L`, a gap of `timeInterval + 1` is no longer 
filled in either time order. Please add a regression test, for example:
   
   ```java
   long interval = 15552000000000000L;
   long previousTime = 1700000000000000000L;
   FixedIntervalFillFilter filter = new FixedIntervalFillFilter(interval);
   assertTrue(filter.needFill(previousTime + interval, previousTime));
   assertFalse(filter.needFill(previousTime + interval + 1, previousTime));
   assertFalse(filter.needFill(previousTime, previousTime + interval + 1)); // 
order by time desc
   assertFalse(
       new FixedIntervalFillFilter(Long.MAX_VALUE).needFill(Long.MAX_VALUE, 
Long.MIN_VALUE));
   ```



##########
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketOutlierSample.java:
##########
@@ -354,14 +368,12 @@ public void outlierSampleLong(RowWindow rowWindow, 
PointCollector collector)
         currentValue = rowWindow.getRow(i).getLong(0);
         nextValue = rowWindow.getRow(i + 1).getLong(0);
 
-        x1 = currentTime - lastTime;
-        x2 = nextTime - currentTime;
-        y1 = currentValue - lastValue;
-        y2 = nextValue - currentValue;
+        x1 = timeDifferenceAsDouble(currentTime, lastTime);
+        x2 = timeDifferenceAsDouble(nextTime, currentTime);
+        y1 = timeDifferenceAsDouble(currentValue, lastValue);
+        y2 = timeDifferenceAsDouble(nextValue, currentValue);

Review Comment:
   Verified. With timestamps `[0, 1, 2, 3, 4]`, values `B + [0, 1, 10, 1, 0]` 
(`B = 1L << 60`), `number=1` and `proportion=0.2`, INT64 `cos` and `prenextdis` 
(lines 528-529) now emit only `(2, B + 10)`; on f24d8a4 both emitted `(1, B + 
1)`. The rewritten INT64 `stendis` path, which also emitted `(1, B + 1)` on 
f24d8a4, now picks `(2, B + 10)` as well.
   
   Please add this as a regression test for INT64 `cos` and `prenextdis`, plus 
`stendis` if you keep that rewrite. Mocked `RowWindow` / `Row` / 
`PointCollector` are enough, and node-commons already has Mockito:
   
   <details>
   <summary>Test sketch</summary>
   
   ```java
   private static void assertSelectsCentralSpike(String type) throws Exception {
     long b = 1L << 60;
     Map<String, String> attributes = new HashMap<>();
     attributes.put("type", type);
     attributes.put("number", "1");
     attributes.put("proportion", "0.2");
     UDFParameters parameters =
         new UDFParameters(
             Collections.singletonList("s1"), 
Collections.singletonList(Type.INT64), attributes);
     UDTFEqualSizeBucketOutlierSample udf = new 
UDTFEqualSizeBucketOutlierSample();
     udf.validate(new UDFParameterValidator(parameters));
     udf.beforeStart(parameters, new UDTFConfigurations(ZoneId.of("UTC")));
   
     long[] values = {b, b + 1, b + 10, b + 1, b};
     RowWindow window = Mockito.mock(RowWindow.class);
     Mockito.when(window.windowSize()).thenReturn(values.length);
     for (int i = 0; i < values.length; i++) {
       Row row = Mockito.mock(Row.class);
       Mockito.when(row.getTime()).thenReturn((long) i);
       Mockito.when(row.getLong(0)).thenReturn(values[i]);
       Mockito.when(window.getRow(i)).thenReturn(row);
     }
     PointCollector collector = Mockito.mock(PointCollector.class);
     udf.transform(window, collector);
   
     Mockito.verify(collector).putLong(2L, b + 10);
     Mockito.verify(collector, Mockito.times(1)).putLong(Mockito.anyLong(), 
Mockito.anyLong());
   }
   ```
   
   </details>



##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java:
##########
@@ -591,32 +592,43 @@ public static long convertDurationStrToLong(
    */
   public static long convertDurationStrToLong(
       long currentTime, String duration, String timestampPrecision, boolean 
convertYearToMonth) {
-    long total = 0;
-    long temp = 0;
-    for (int i = 0; i < duration.length(); i++) {
-      char ch = duration.charAt(i);
-      if (Character.isDigit(ch)) {
-        temp *= 10;
-        temp += (ch - '0');
-      } else {
-        String unit = String.valueOf(duration.charAt(i));
-        // This is to identify units with two letters.
-        if (i + 1 < duration.length() && !Character.isDigit(duration.charAt(i 
+ 1))) {
-          i++;
-          unit += duration.charAt(i);
-        }
-        unit = unit.toLowerCase();
-        if (convertYearToMonth && unit.equals("y")) {
-          temp *= 12;
-          unit = "mo";
+    try {
+      long total = 0;
+      long temp = 0;
+      for (int i = 0; i < duration.length(); i++) {
+        char ch = duration.charAt(i);
+        if (Character.isDigit(ch)) {
+          temp = Math.addExact(Math.multiplyExact(temp, 10), 
Character.digit(ch, 10));
+        } else {
+          String unit = String.valueOf(duration.charAt(i));
+          // This is to identify units with two letters.
+          if (i + 1 < duration.length() && 
!Character.isDigit(duration.charAt(i + 1))) {
+            i++;
+            unit += duration.charAt(i);
+          }
+          unit = unit.toLowerCase();
+          if (convertYearToMonth && unit.equals("y")) {
+            temp = Math.multiplyExact(temp, 12);
+            unit = "mo";
+          }
+          long componentCurrentTime =
+              isMonthUnit(unit) ? (currentTime == -1 ? -1 : 
Math.addExact(currentTime, total)) : -1;

Review Comment:
   Verified. At `ns` precision with UTC, `where time > 
2262-04-11T23:47:16.854775806 - 1s1ns` now parses to `Time > 
9223372035854775805`, the same predicate as `- 1000000001ns`, and `+ 1s1ns` is 
still rejected with `Date expression is out of range`. Please cover this with a 
regression test, for example:
   
   ```java
   // with the timestamp precision set to "ns" (and restored afterwards)
   assertEquals(
       1_000_000_001L,
       DateTimeUtils.convertDurationStrToLong(9223372036854775806L, "1s1ns", 
"ns", false));
   ```
   
   Please also add a `StatementGenerator.createStatement(sql, 
ZoneId.of("UTC"))` case that checks the `- 1s1ns` and `- 1000000001ns` queries 
produce the same `where` predicate.



##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/FixedIntervalFillFilter.java:
##########
@@ -21,6 +21,8 @@
 
 import 
org.apache.iotdb.db.queryengine.execution.operator.process.fill.IFillFilter;
 
+import static 
org.apache.iotdb.db.pipe.processor.downsampling.DownSamplingTimeUtils.isTimeDistanceLessThanOrEqualTo;

Review Comment:
   **Non-blocking:** this makes the query-engine fill filter depend on a helper 
in the pipe down-sampling package. On master, `FixedIntervalFillFilter` keeps 
the same unsigned comparison in a private static 
`isTimeDistanceLessThanOrEqual` method. Mirroring that would remove the 
cross-package dependency and keep this class aligned with master.



##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java:
##########
@@ -628,37 +640,46 @@ public static long convertDurationStrToLongForTest(
   /** convert duration string to millisecond, microsecond or nanosecond. */
   public static long convertDurationStrToLong(
       long currentTime, long value, String unit, String timestampPrecision) {
+    try {
+      return convertDurationStrToLongInternal(currentTime, value, unit, 
timestampPrecision);
+    } catch (ArithmeticException e) {
+      throw new SemanticException("Time duration is out of range.");
+    }
+  }
+
+  private static long convertDurationStrToLongInternal(
+      long currentTime, long value, String unit, String timestampPrecision) {
     DurationUnit durationUnit = DurationUnit.valueOf(unit);
     long res = value;
     switch (durationUnit) {
       case y:
-        res *= 365 * 86_400_000L;
+        res = Math.multiplyExact(value, 365 * 86_400_000L);
         break;
       case mo:
         if (currentTime == -1) {
-          res *= 30 * 86_400_000L;
+          res = Math.multiplyExact(value, 30 * 86_400_000L);
         } else {
           Calendar calendar = Calendar.getInstance();
           
calendar.setTimeZone(SessionManager.getInstance().getSessionTimeZone());
           calendar.setTimeInMillis(currentTime);
-          calendar.add(Calendar.MONTH, (int) (value));
-          res = calendar.getTimeInMillis() - currentTime;
+          calendar.add(Calendar.MONTH, Math.toIntExact(value));
+          res = Math.subtractExact(calendar.getTimeInMillis(), currentTime);

Review Comment:
   **Non-blocking (master behaves the same):** month components can still 
reject a valid subtraction near the upper bound. At `ns` precision with UTC, 
the current `dev/1.3` accepts these expressions, while this PR rejects them 
with `Time duration is out of range.`:
   
   | Expression | `dev/1.3` (a42374d) | This PR |
   | --- | --- | --- |
   | `2262-04-11T23:47:16.854775807 - 1mo` | `Time > 9220693636854775807` | 
rejected |
   | `2262-04-11T23:47:16.854775807 - 1ns1mo` | `Time > 9220693636854775806` | 
rejected |
   | `2262-04-11T23:47:15.854775807 - 2s1mo` | `Time > 9220693633854775807` | 
rejected |
   
   There are two causes. The reference time for a month component is still 
`currentTime + total`, even for subtraction. And because `Calendar` treats the 
ns value as milliseconds, adding a month overflows `getTimeInMillis()`, which 
this `subtractExact` now turns into an error. Only the very end of the `ns` 
range is affected: the fixed-duration part of the expression plus a few seconds 
per month. Because master behaves the same way, I'm not blocking on this. If 
it's worth handling, please do it on master first.



##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/ai/InferenceOperator.java:
##########
@@ -374,4 +378,34 @@ public long ramBytesUsed() {
             ? 0
             : 
targetColumnNames.stream().mapToLong(RamUsageEstimator::sizeOf).sum());
   }
+
+  static long calculateGeneratedTimeInterval(long minTimestamp, long 
maxTimestamp, long totalRow) {
+    try {
+      BigInteger interval =
+          BigInteger.valueOf(maxTimestamp)
+              .subtract(BigInteger.valueOf(minTimestamp))
+              .divide(BigInteger.valueOf(totalRow));
+      if (interval.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) {
+        throw new ArithmeticException();
+      }
+      return interval.longValue();
+    } catch (ArithmeticException e) {
+      throw new 
ModelInferenceProcessException(GENERATED_TIME_COLUMN_OUT_OF_RANGE_MESSAGE);
+    }
+  }
+
+  static long calculateGeneratedTime(long maxTimestamp, long interval, long 
currentRowIndex) {

Review Comment:
   Verified. `calculateGeneratedTimeInterval(-6000000000000000000L, 
-4000000000000000000L, 2)` returns `1000000000000000000L`, 
`calculateGeneratedTime(-4000000000000000000L, 1000000000000000000L, 10)` now 
returns `6000000000000000000L`, and index `14` (`1e19`) is still rejected. None 
of the existing `InferenceOperatorTest` cases uses a negative `maxTimestamp` 
whose product with the row index exceeds `Long.MAX_VALUE`, so please add this 
one:
   
   ```java
   long interval =
       InferenceOperator.calculateGeneratedTimeInterval(
           -6000000000000000000L, -4000000000000000000L, 2);
   assertEquals(1000000000000000000L, interval);
   assertEquals(
       6000000000000000000L,
       InferenceOperator.calculateGeneratedTime(-4000000000000000000L, 
interval, 10));
   assertThrows(
       ModelInferenceProcessException.class,
       () -> InferenceOperator.calculateGeneratedTime(-4000000000000000000L, 
interval, 14));
   ```



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to