This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 9fe87a56bdc [cherry-pick](branch-21) support lead/lag function input
column as third params (#49381) (#50653)
9fe87a56bdc is described below
commit 9fe87a56bdcbc7a56b30f47eea20cc9c7bb990f7
Author: zhangstar333 <[email protected]>
AuthorDate: Wed May 7 23:08:56 2025 +0800
[cherry-pick](branch-21) support lead/lag function input column as third
params (#49381) (#50653)
### What problem does this PR solve?
cherry-pick from master (#49381)
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
.../aggregate_function_window.h | 64 ++++++++++-----------
.../trees/expressions/functions/window/Lag.java | 5 +-
.../trees/expressions/functions/window/Lead.java | 5 +-
.../functions/window/WindowFunction.java | 6 +-
.../data/correctness_p0/test_lag_lead_window.out | Bin 1402 -> 1797 bytes
.../correctness_p0/test_lag_lead_window.groovy | 39 +++++++++++++
6 files changed, 74 insertions(+), 45 deletions(-)
diff --git a/be/src/vec/aggregate_functions/aggregate_function_window.h
b/be/src/vec/aggregate_functions/aggregate_function_window.h
index 7e22d6c0e1f..9be169d0a20 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_window.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_window.h
@@ -21,13 +21,11 @@
#pragma once
#include <glog/logging.h>
-#include <stddef.h>
-#include <stdint.h>
#include <algorithm>
#include <boost/iterator/iterator_facade.hpp>
+#include <cstdint>
#include <memory>
-#include <ostream>
#include "gutil/integral_types.h"
#include "vec/aggregate_functions/aggregate_function.h"
@@ -358,15 +356,10 @@ public:
static constexpr bool result_nullable = result_is_nullable;
void reset() {
_data_value.reset();
- _default_value.reset();
_is_inited = false;
+ _offset_value = 0;
}
- bool default_is_null() { return _default_value.is_null(); }
-
- // here _ptr pointer default column from third
- void set_value_from_default() { this->_data_value = _default_value; }
-
void insert_result_into(IColumn& to) const {
if constexpr (result_is_nullable) {
if (_data_value.is_null()) {
@@ -383,8 +376,6 @@ public:
}
}
- void set_is_null() { this->_data_value.reset(); }
-
void set_value(const IColumn** columns, size_t pos) {
if constexpr (arg_is_nullable) {
if (assert_cast<const
ColumnNullable*>(columns[0])->is_null_at(pos)) {
@@ -397,39 +388,46 @@ public:
_data_value.set_value(columns[0], pos);
}
- void check_default(const IColumn* column) {
- if (!_is_inited) {
- if (is_column_nullable(*column)) {
- const auto* nullable_column = assert_cast<const
ColumnNullable*>(column);
- if (nullable_column->is_null_at(0)) {
- _default_value.reset();
- } else {
-
_default_value.set_value(nullable_column->get_nested_column_ptr(), 0);
- }
+ void set_value_from_default(const IColumn* column, size_t pos) {
+ DCHECK_GE(pos, 0);
+ if (is_column_nullable(*column)) {
+ const auto* nullable_column = assert_cast<const
ColumnNullable*>(column);
+ if (nullable_column->is_null_at(pos)) {
+ this->_data_value.reset();
} else {
- _default_value.set_value(column, 0);
+
this->_data_value.set_value(nullable_column->get_nested_column_ptr().get(),
pos);
}
+ } else {
+ this->_data_value.set_value(column, pos);
+ }
+ }
+
+ void set_offset_value(const IColumn* column) {
+ if (!_is_inited) {
+ const auto* column_number = assert_cast<const
ColumnInt64*>(column);
+ _offset_value = column_number->get_data()[0];
_is_inited = true;
}
}
+ int64_t get_offset_value() const { return _offset_value; }
+
private:
BaseValue<ColVecType, arg_is_nullable> _data_value;
- BaseValue<ColVecType, arg_is_nullable> _default_value;
bool _is_inited = false;
+ int64_t _offset_value = 0;
};
template <typename Data, bool = false>
struct WindowFunctionLeadImpl : Data {
void add_range_single_place(int64_t partition_start, int64_t
partition_end, int64_t frame_start,
int64_t frame_end, const IColumn** columns) {
- this->check_default(columns[2]);
if (frame_end > partition_end) { //output default value, win end is
under partition
- if (this->default_is_null()) {
- this->set_is_null();
- } else {
- this->set_value_from_default();
- }
+ this->set_offset_value(columns[1]);
+ // eg: lead(column, 10, default_value), column size maybe 3 rows
+ // offset value 10 is from second argument, pos: 11 is calculated
as frame_end
+ auto pos = frame_end - 1 - this->get_offset_value();
+ this->set_value_from_default(columns[2], pos);
return;
}
this->set_value(columns, frame_end - 1);
@@ -442,13 +440,11 @@ template <typename Data, bool = false>
struct WindowFunctionLagImpl : Data {
void add_range_single_place(int64_t partition_start, int64_t
partition_end, int64_t frame_start,
int64_t frame_end, const IColumn** columns) {
- this->check_default(columns[2]);
+ // window start is beyond partition
if (partition_start >= frame_end) { //[unbound preceding(0), offset
preceding(-123)]
- if (this->default_is_null()) { // win start is beyond partition
- this->set_is_null();
- } else {
- this->set_value_from_default();
- }
+ this->set_offset_value(columns[1]);
+ auto pos = frame_end - 1 + this->get_offset_value();
+ this->set_value_from_default(columns[2], pos);
return;
}
this->set_value(columns, frame_end - 1);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/window/Lag.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/window/Lag.java
index fc74cadebfc..58ff2b820f0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/window/Lag.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/window/Lag.java
@@ -99,7 +99,7 @@ public class Lag extends WindowFunction implements
TernaryExpression, Explicitly
return;
}
if (children().size() >= 2) {
- checkValidParams(getOffset(), true);
+ checkValidParams(getOffset());
if (getOffset() instanceof Literal) {
if (((Literal) getOffset()).getDouble() < 0) {
throw new AnalysisException(
@@ -109,9 +109,6 @@ public class Lag extends WindowFunction implements
TernaryExpression, Explicitly
throw new AnalysisException(
"The offset parameter of LAG must be a constant positive
integer: " + this.toSql());
}
- if (children().size() >= 3) {
- checkValidParams(getDefaultValue(), false);
- }
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/window/Lead.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/window/Lead.java
index 251141a68cb..b0de4ad571b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/window/Lead.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/window/Lead.java
@@ -94,7 +94,7 @@ public class Lead extends WindowFunction implements
TernaryExpression, Explicitl
return;
}
if (children().size() >= 2) {
- checkValidParams(getOffset(), true);
+ checkValidParams(getOffset());
if (getOffset() instanceof Literal) {
if (((Literal) getOffset()).getDouble() < 0) {
throw new AnalysisException(
@@ -104,9 +104,6 @@ public class Lead extends WindowFunction implements
TernaryExpression, Explicitl
throw new AnalysisException(
"The offset parameter of LAG must be a constant positive
integer: " + this.toSql());
}
- if (children().size() >= 3) {
- checkValidParams(getDefaultValue(), false);
- }
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/window/WindowFunction.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/window/WindowFunction.java
index 1265f685b26..13b4ec58476 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/window/WindowFunction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/window/WindowFunction.java
@@ -59,14 +59,14 @@ public abstract class WindowFunction extends BoundFunction
implements SupportWin
/**
* LAG/LEAD param must be const, and offset must be number
*/
- protected void checkValidParams(Expression param, boolean isOffset) {
+ protected void checkValidParams(Expression param) {
DataType type = param.getDataType();
- if (isOffset == true && !type.isNumericType()) {
+ if (!type.isNumericType()) {
throw new AnalysisException("The offset of LAG/LEAD must be a
number: " + this.toSql());
}
if (!param.isConstant()) {
throw new AnalysisException(
- "The parameter 2 or parameter 3 of LAG/LEAD must be a
constant value: " + this.toSql());
+ "The parameter 2 of LAG/LEAD must be a constant value: " +
this.toSql());
}
}
}
diff --git a/regression-test/data/correctness_p0/test_lag_lead_window.out
b/regression-test/data/correctness_p0/test_lag_lead_window.out
index 041314a1c65..f9927b708c9 100644
Binary files a/regression-test/data/correctness_p0/test_lag_lead_window.out and
b/regression-test/data/correctness_p0/test_lag_lead_window.out differ
diff --git a/regression-test/suites/correctness_p0/test_lag_lead_window.groovy
b/regression-test/suites/correctness_p0/test_lag_lead_window.groovy
index 1dfccca58ee..0cf731e7d9e 100644
--- a/regression-test/suites/correctness_p0/test_lag_lead_window.groovy
+++ b/regression-test/suites/correctness_p0/test_lag_lead_window.groovy
@@ -41,6 +41,12 @@ suite("test_lag_lead_window") {
lead(cc,1,'') over (PARTITION by cc order
by aa) as lead_cc
from ${tableName}
order by aa; """
+
+ qt_select_default3 """ select aa,cc,bb,lead(cc,1,bb) over (PARTITION by cc
order by aa) as lead_res,
+ lag(cc,1,bb) over (PARTITION by cc order by
aa) as lag_res
+ from ${tableName}
+ order by aa; """
+
sql """ DROP TABLE IF EXISTS test1 """
sql """ CREATE TABLE IF NOT EXISTS test1 (id varchar(255), create_time
datetime)
DISTRIBUTED BY HASH(id) PROPERTIES("replication_num" = "1"); """
@@ -61,4 +67,37 @@ suite("test_lag_lead_window") {
sql """ DROP TABLE IF EXISTS test1 """
+
+ qt_select_lead_7 """ SELECT
+ sale_date,
+ product_id,
+ quantity,
+ LEAD (quantity, 1, quantity) OVER (
+ PARTITION BY
+ product_id
+ ORDER BY
+ sale_date
+ ) AS next_day_quantity,
+ LAG (quantity, 1, quantity) OVER (
+ PARTITION BY
+ product_id
+ ORDER BY
+ sale_date
+ ) AS pre_day_quantity
+ FROM
+ (
+ select 1 AS product_id, '2023-01-01' AS sale_date, 10 AS
quantity
+ UNION ALL
+ select 1, '2023-01-02', 20
+ UNION ALL
+ select 1, '2023-01-03', 30
+ UNION ALL
+ select 1, '2023-01-04', NULL
+ ) AS t
+ ORDER BY
+ product_id,
+ sale_date;
+ """
+
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
