JackieTien97 commented on code in PR #18589:
URL: https://github.com/apache/iotdb/pull/18589#discussion_r4475309537
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/FixedIntervalFillFilter.java:
##########
@@ -34,14 +36,6 @@ public FixedIntervalFillFilter(long timeInterval) {
public boolean needFill(long time, long previousTime) {
// the reason that we use Math.abs is that we may use order by time desc
which will cause
// previousTime is larger than time
- return isTimeDistanceLessThanOrEqual(time, previousTime, timeInterval);
- }
-
- private boolean isTimeDistanceLessThanOrEqual(long left, long right, long
maxDistance) {
- if (maxDistance < 0) {
- return false;
- }
- long distance = left >= right ? left - right : right - left;
- return Long.compareUnsigned(distance, maxDistance) <= 0;
+ return isTimeDistanceLessThanOrEqualTo(time, previousTime, timeInterval);
Review Comment:
Verified. With `timeInterval = 15552000000000000L` (180 days in ns) and
`previousTime = 1700000000000000000L`, a gap of `timeInterval + 1` is no longer
filled in either time order. Please add a regression test, for example:
```java
long interval = 15552000000000000L;
long previousTime = 1700000000000000000L;
FixedIntervalFillFilter filter = new FixedIntervalFillFilter(interval);
assertTrue(filter.needFill(previousTime + interval, previousTime));
assertFalse(filter.needFill(previousTime + interval + 1, previousTime));
assertFalse(filter.needFill(previousTime, previousTime + interval + 1)); //
order by time desc
assertFalse(
new FixedIntervalFillFilter(Long.MAX_VALUE).needFill(Long.MAX_VALUE,
Long.MIN_VALUE));
```
##########
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketOutlierSample.java:
##########
@@ -354,14 +368,12 @@ public void outlierSampleLong(RowWindow rowWindow,
PointCollector collector)
currentValue = rowWindow.getRow(i).getLong(0);
nextValue = rowWindow.getRow(i + 1).getLong(0);
- x1 = currentTime - lastTime;
- x2 = nextTime - currentTime;
- y1 = currentValue - lastValue;
- y2 = nextValue - currentValue;
+ x1 = timeDifferenceAsDouble(currentTime, lastTime);
+ x2 = timeDifferenceAsDouble(nextTime, currentTime);
+ y1 = timeDifferenceAsDouble(currentValue, lastValue);
+ y2 = timeDifferenceAsDouble(nextValue, currentValue);
Review Comment:
Verified. With timestamps `[0, 1, 2, 3, 4]`, values `B + [0, 1, 10, 1, 0]`
(`B = 1L << 60`), `number=1` and `proportion=0.2`, INT64 `cos` and `prenextdis`
(lines 528-529) now emit only `(2, B + 10)`; on f24d8a4 both emitted `(1, B +
1)`. The rewritten INT64 `stendis` path, which also emitted `(1, B + 1)` on
f24d8a4, now picks `(2, B + 10)` as well.
Please add this as a regression test for INT64 `cos` and `prenextdis`, plus
`stendis` if you keep that rewrite. Mocked `RowWindow` / `Row` /
`PointCollector` are enough, and node-commons already has Mockito:
<details>
<summary>Test sketch</summary>
```java
private static void assertSelectsCentralSpike(String type) throws Exception {
long b = 1L << 60;
Map<String, String> attributes = new HashMap<>();
attributes.put("type", type);
attributes.put("number", "1");
attributes.put("proportion", "0.2");
UDFParameters parameters =
new UDFParameters(
Collections.singletonList("s1"),
Collections.singletonList(Type.INT64), attributes);
UDTFEqualSizeBucketOutlierSample udf = new
UDTFEqualSizeBucketOutlierSample();
udf.validate(new UDFParameterValidator(parameters));
udf.beforeStart(parameters, new UDTFConfigurations(ZoneId.of("UTC")));
long[] values = {b, b + 1, b + 10, b + 1, b};
RowWindow window = Mockito.mock(RowWindow.class);
Mockito.when(window.windowSize()).thenReturn(values.length);
for (int i = 0; i < values.length; i++) {
Row row = Mockito.mock(Row.class);
Mockito.when(row.getTime()).thenReturn((long) i);
Mockito.when(row.getLong(0)).thenReturn(values[i]);
Mockito.when(window.getRow(i)).thenReturn(row);
}
PointCollector collector = Mockito.mock(PointCollector.class);
udf.transform(window, collector);
Mockito.verify(collector).putLong(2L, b + 10);
Mockito.verify(collector, Mockito.times(1)).putLong(Mockito.anyLong(),
Mockito.anyLong());
}
```
</details>
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java:
##########
@@ -591,32 +592,43 @@ public static long convertDurationStrToLong(
*/
public static long convertDurationStrToLong(
long currentTime, String duration, String timestampPrecision, boolean
convertYearToMonth) {
- long total = 0;
- long temp = 0;
- for (int i = 0; i < duration.length(); i++) {
- char ch = duration.charAt(i);
- if (Character.isDigit(ch)) {
- temp *= 10;
- temp += (ch - '0');
- } else {
- String unit = String.valueOf(duration.charAt(i));
- // This is to identify units with two letters.
- if (i + 1 < duration.length() && !Character.isDigit(duration.charAt(i
+ 1))) {
- i++;
- unit += duration.charAt(i);
- }
- unit = unit.toLowerCase();
- if (convertYearToMonth && unit.equals("y")) {
- temp *= 12;
- unit = "mo";
+ try {
+ long total = 0;
+ long temp = 0;
+ for (int i = 0; i < duration.length(); i++) {
+ char ch = duration.charAt(i);
+ if (Character.isDigit(ch)) {
+ temp = Math.addExact(Math.multiplyExact(temp, 10),
Character.digit(ch, 10));
+ } else {
+ String unit = String.valueOf(duration.charAt(i));
+ // This is to identify units with two letters.
+ if (i + 1 < duration.length() &&
!Character.isDigit(duration.charAt(i + 1))) {
+ i++;
+ unit += duration.charAt(i);
+ }
+ unit = unit.toLowerCase();
+ if (convertYearToMonth && unit.equals("y")) {
+ temp = Math.multiplyExact(temp, 12);
+ unit = "mo";
+ }
+ long componentCurrentTime =
+ isMonthUnit(unit) ? (currentTime == -1 ? -1 :
Math.addExact(currentTime, total)) : -1;
Review Comment:
Verified. At `ns` precision with UTC, `where time >
2262-04-11T23:47:16.854775806 - 1s1ns` now parses to `Time >
9223372035854775805`, the same predicate as `- 1000000001ns`, and `+ 1s1ns` is
still rejected with `Date expression is out of range`. Please cover this with a
regression test, for example:
```java
// with the timestamp precision set to "ns" (and restored afterwards)
assertEquals(
1_000_000_001L,
DateTimeUtils.convertDurationStrToLong(9223372036854775806L, "1s1ns",
"ns", false));
```
Please also add a `StatementGenerator.createStatement(sql,
ZoneId.of("UTC"))` case that checks the `- 1s1ns` and `- 1000000001ns` queries
produce the same `where` predicate.
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/FixedIntervalFillFilter.java:
##########
@@ -21,6 +21,8 @@
import
org.apache.iotdb.db.queryengine.execution.operator.process.fill.IFillFilter;
+import static
org.apache.iotdb.db.pipe.processor.downsampling.DownSamplingTimeUtils.isTimeDistanceLessThanOrEqualTo;
Review Comment:
**Non-blocking:** this makes the query-engine fill filter depend on a helper
in the pipe down-sampling package. On master, `FixedIntervalFillFilter` keeps
the same unsigned comparison in a private static
`isTimeDistanceLessThanOrEqual` method. Mirroring that would remove the
cross-package dependency and keep this class aligned with master.
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java:
##########
@@ -628,37 +640,46 @@ public static long convertDurationStrToLongForTest(
/** convert duration string to millisecond, microsecond or nanosecond. */
public static long convertDurationStrToLong(
long currentTime, long value, String unit, String timestampPrecision) {
+ try {
+ return convertDurationStrToLongInternal(currentTime, value, unit,
timestampPrecision);
+ } catch (ArithmeticException e) {
+ throw new SemanticException("Time duration is out of range.");
+ }
+ }
+
+ private static long convertDurationStrToLongInternal(
+ long currentTime, long value, String unit, String timestampPrecision) {
DurationUnit durationUnit = DurationUnit.valueOf(unit);
long res = value;
switch (durationUnit) {
case y:
- res *= 365 * 86_400_000L;
+ res = Math.multiplyExact(value, 365 * 86_400_000L);
break;
case mo:
if (currentTime == -1) {
- res *= 30 * 86_400_000L;
+ res = Math.multiplyExact(value, 30 * 86_400_000L);
} else {
Calendar calendar = Calendar.getInstance();
calendar.setTimeZone(SessionManager.getInstance().getSessionTimeZone());
calendar.setTimeInMillis(currentTime);
- calendar.add(Calendar.MONTH, (int) (value));
- res = calendar.getTimeInMillis() - currentTime;
+ calendar.add(Calendar.MONTH, Math.toIntExact(value));
+ res = Math.subtractExact(calendar.getTimeInMillis(), currentTime);
Review Comment:
**Non-blocking (master behaves the same):** month components can still
reject a valid subtraction near the upper bound. At `ns` precision with UTC,
the current `dev/1.3` accepts these expressions, while this PR rejects them
with `Time duration is out of range.`:
| Expression | `dev/1.3` (a42374d) | This PR |
| --- | --- | --- |
| `2262-04-11T23:47:16.854775807 - 1mo` | `Time > 9220693636854775807` |
rejected |
| `2262-04-11T23:47:16.854775807 - 1ns1mo` | `Time > 9220693636854775806` |
rejected |
| `2262-04-11T23:47:15.854775807 - 2s1mo` | `Time > 9220693633854775807` |
rejected |
There are two causes. The reference time for a month component is still
`currentTime + total`, even for subtraction. And because `Calendar` treats the
ns value as milliseconds, adding a month overflows `getTimeInMillis()`, which
this `subtractExact` now turns into an error. Only the very end of the `ns`
range is affected: the fixed-duration part of the expression plus a few seconds
per month. Because master behaves the same way, I'm not blocking on this. If
it's worth handling, please do it on master first.
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/ai/InferenceOperator.java:
##########
@@ -374,4 +378,34 @@ public long ramBytesUsed() {
? 0
:
targetColumnNames.stream().mapToLong(RamUsageEstimator::sizeOf).sum());
}
+
+ static long calculateGeneratedTimeInterval(long minTimestamp, long
maxTimestamp, long totalRow) {
+ try {
+ BigInteger interval =
+ BigInteger.valueOf(maxTimestamp)
+ .subtract(BigInteger.valueOf(minTimestamp))
+ .divide(BigInteger.valueOf(totalRow));
+ if (interval.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) {
+ throw new ArithmeticException();
+ }
+ return interval.longValue();
+ } catch (ArithmeticException e) {
+ throw new
ModelInferenceProcessException(GENERATED_TIME_COLUMN_OUT_OF_RANGE_MESSAGE);
+ }
+ }
+
+ static long calculateGeneratedTime(long maxTimestamp, long interval, long
currentRowIndex) {
Review Comment:
Verified. `calculateGeneratedTimeInterval(-6000000000000000000L,
-4000000000000000000L, 2)` returns `1000000000000000000L`,
`calculateGeneratedTime(-4000000000000000000L, 1000000000000000000L, 10)` now
returns `6000000000000000000L`, and index `14` (`1e19`) is still rejected. None
of the existing `InferenceOperatorTest` cases uses a negative `maxTimestamp`
whose product with the row index exceeds `Long.MAX_VALUE`, so please add this
one:
```java
long interval =
InferenceOperator.calculateGeneratedTimeInterval(
-6000000000000000000L, -4000000000000000000L, 2);
assertEquals(1000000000000000000L, interval);
assertEquals(
6000000000000000000L,
InferenceOperator.calculateGeneratedTime(-4000000000000000000L,
interval, 10));
assertThrows(
ModelInferenceProcessException.class,
() -> InferenceOperator.calculateGeneratedTime(-4000000000000000000L,
interval, 14));
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]