Repository: impala
Updated Branches:
refs/heads/master ea8d2ba7f -> d428c16f1
IMPALA-5017: Error on decimal overflow
Before this patch, decimal operations would either silently overflow (in
the case of sum() and avg()), or produce a warning.
In this patch, the behaviour is changed so that an error is produced in
the case of overflow when DECIMAL_v2 is enabled. Decimal v1 behaviour is
unchanged.
We introduce overflow checks when computing sum() and avg(). This
results in a ~30% performance regression when we are in decimal v2 mode
compared to decimal v1.
Benchmarks:
Query:
select sum(dec_38_19) from decimal_tbl
Decimal v1: 11.57s
Decimal v2: 16.58s
Query:
select avg(dec_38_19) from decimal_tbl
Decimal v1: 12.08s
Decimal v2: 17.08s
The performance regression is not as bad if we are computing the sum or
average of decimal column with a lower precision:
Query:
select sum(dec_9_5) from decimal_tbl
Decimal v1: 11.06s
Decimal v2: 13.08s
Query:
select avg(dec_9_5) from decimal_tbl
Decimal v1: 11.56s
Decimal v2: 13.57s
Testing:
- Added several end to end tests.
- Updated Expr tests to check for error in case of overflow.
Change-Id: Id98a92c9a9469ec8cf14e518c741a2dab7053019
Reviewed-on: http://gerrit.cloudera.org:8080/8404
Reviewed-by: Taras Bobrovytsky <tbobrovyt...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/575b5a20
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/575b5a20
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/575b5a20
Branch: refs/heads/master
Commit: 575b5a20e6ddc72b7cefcdd27ae722a8ad0530ee
Parents: ea8d2ba
Author: Taras Bobrovytsky <tbobrovyt...@cloudera.com>
Authored: Thu Oct 19 16:50:17 2017 -0700
Committer: Impala Public Jenkins <impala-public-jenk...@gerrit.cloudera.org>
Committed: Fri Dec 1 23:23:01 2017 +0000
----------------------------------------------------------------------
be/src/exprs/aggregate-functions-ir.cc | 49 ++++++-
be/src/exprs/decimal-operators-ir.cc | 14 +-
be/src/exprs/expr-codegen-test.cc | 2 +-
be/src/exprs/expr-test.cc | 131 +++++++++++--------
.../queries/QueryTest/decimal-exprs.test | 100 ++++++++++++++
.../queries/QueryTest/decimal.test | 11 --
tests/hs2/test_hs2.py | 2 +-
7 files changed, 231 insertions(+), 78 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/575b5a20/be/src/exprs/aggregate-functions-ir.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/aggregate-functions-ir.cc
b/be/src/exprs/aggregate-functions-ir.cc
index 049344e..d5f946a 100644
--- a/be/src/exprs/aggregate-functions-ir.cc
+++ b/be/src/exprs/aggregate-functions-ir.cc
@@ -402,6 +402,7 @@ IR_ALWAYS_INLINE void
AggregateFunctions::DecimalAvgAddOrRemove(FunctionContext*
DCHECK(dst->ptr != NULL);
DCHECK_EQ(sizeof(DecimalAvgState), dst->len);
DecimalAvgState* avg = reinterpret_cast<DecimalAvgState*>(dst->ptr);
+ bool decimal_v2 =
ctx->impl()->GetConstFnAttr(FunctionContextImpl::DECIMAL_V2);
// Since the src and dst are guaranteed to be the same scale, we can just
// do a simple add.
@@ -409,11 +410,25 @@ IR_ALWAYS_INLINE void
AggregateFunctions::DecimalAvgAddOrRemove(FunctionContext*
switch (ctx->impl()->GetConstFnAttr(FunctionContextImpl::ARG_TYPE_SIZE, 0)) {
case 4:
avg->sum_val16 += m * src.val4;
+ if (UNLIKELY(decimal_v2 &&
+ abs(avg->sum_val16) > DecimalUtil::MAX_UNSCALED_DECIMAL16)) {
+ ctx->SetError("Avg computation overflowed");
+ }
break;
case 8:
avg->sum_val16 += m * src.val8;
+ if (UNLIKELY(decimal_v2 &&
+ abs(avg->sum_val16) > DecimalUtil::MAX_UNSCALED_DECIMAL16)) {
+ ctx->SetError("Avg computation overflowed");
+ }
break;
case 16:
+ if (UNLIKELY(decimal_v2 && (avg->sum_val16 >= 0) == (src.val16 >= 0) &&
+ abs(avg->sum_val16) > DecimalUtil::MAX_UNSCALED_DECIMAL16 -
abs(src.val16))) {
+ // We can't check for overflow after performing the addition like in
the other
+ // cases because the result may not fit into int128.
+ ctx->SetError("Avg computation overflowed");
+ }
avg->sum_val16 += m * src.val16;
break;
default:
@@ -434,6 +449,11 @@ void AggregateFunctions::DecimalAvgMerge(FunctionContext*
ctx,
DCHECK(dst->ptr != NULL);
DCHECK_EQ(sizeof(DecimalAvgState), dst->len);
DecimalAvgState* dst_struct = reinterpret_cast<DecimalAvgState*>(dst->ptr);
+ bool decimal_v2 =
ctx->impl()->GetConstFnAttr(FunctionContextImpl::DECIMAL_V2);
+ bool overflow = decimal_v2 &&
+ abs(dst_struct->sum_val16) >
+ DecimalUtil::MAX_UNSCALED_DECIMAL16 - abs(src_struct->sum_val16);
+ if (UNLIKELY(overflow)) ctx->SetError("Avg computation overflowed");
dst_struct->sum_val16 += src_struct->sum_val16;
dst_struct->count += src_struct->count;
}
@@ -452,12 +472,16 @@ DecimalVal
AggregateFunctions::DecimalAvgGetValue(FunctionContext* ctx,
int sum_scale =
ctx->impl()->GetConstFnAttr(FunctionContextImpl::ARG_TYPE_SCALE, 0);
bool is_nan = false;
bool overflow = false;
- bool round = ctx->impl()->GetConstFnAttr(FunctionContextImpl::DECIMAL_V2);
+ bool decimal_v2 =
ctx->impl()->GetConstFnAttr(FunctionContextImpl::DECIMAL_V2);
Decimal16Value result = sum.Divide<int128_t>(sum_scale, count, 0 /* count's
scale */,
- output_precision, output_scale, round, &is_nan, &overflow);
+ output_precision, output_scale, decimal_v2, &is_nan, &overflow);
if (UNLIKELY(is_nan)) return DecimalVal::null();
if (UNLIKELY(overflow)) {
- ctx->AddWarning("Avg computation overflowed, returning NULL");
+ if (decimal_v2) {
+ ctx->SetError("Avg computation overflowed");
+ } else {
+ ctx->AddWarning("Avg computation overflowed, returning NULL");
+ }
return DecimalVal::null();
}
return DecimalVal(result.value());
@@ -516,14 +540,29 @@ IR_ALWAYS_INLINE void
AggregateFunctions::SumDecimalAddOrSubtract(FunctionContex
if (src.is_null) return;
if (dst->is_null) InitZero<DecimalVal>(ctx, dst);
int precision =
ctx->impl()->GetConstFnAttr(FunctionContextImpl::ARG_TYPE_PRECISION, 0);
+ bool decimal_v2 =
ctx->impl()->GetConstFnAttr(FunctionContextImpl::DECIMAL_V2);
// Since the src and dst are guaranteed to be the same scale, we can just
// do a simple add.
int m = subtract ? -1 : 1;
if (precision <= 9) {
dst->val16 += m * src.val4;
+ if (UNLIKELY(decimal_v2 &&
+ abs(dst->val16) > DecimalUtil::MAX_UNSCALED_DECIMAL16)) {
+ ctx->SetError("Sum computation overflowed");
+ }
} else if (precision <= 19) {
dst->val16 += m * src.val8;
+ if (UNLIKELY(decimal_v2 &&
+ abs(dst->val16) > DecimalUtil::MAX_UNSCALED_DECIMAL16)) {
+ ctx->SetError("Sum computation overflowed");
+ }
} else {
+ if (UNLIKELY(decimal_v2 && (dst->val16 >= 0) == (src.val16 >= 0) &&
+ abs(dst->val16) > DecimalUtil::MAX_UNSCALED_DECIMAL16 -
abs(src.val16))) {
+ // We can't check for overflow after performing the addition like in the
other
+ // cases because the result may not fit into int128.
+ ctx->SetError("Sum computation overflowed");
+ }
dst->val16 += m * src.val16;
}
}
@@ -532,6 +571,10 @@ void AggregateFunctions::SumDecimalMerge(FunctionContext*
ctx,
const DecimalVal& src, DecimalVal* dst) {
if (src.is_null) return;
if (dst->is_null) InitZero<DecimalVal>(ctx, dst);
+ bool decimal_v2 =
ctx->impl()->GetConstFnAttr(FunctionContextImpl::DECIMAL_V2);
+ bool overflow = decimal_v2 &&
+ abs(dst->val16) > DecimalUtil::MAX_UNSCALED_DECIMAL16 - abs(src.val16);
+ if (UNLIKELY(overflow)) ctx->SetError("Sum computation overflowed");
dst->val16 += src.val16;
}
http://git-wip-us.apache.org/repos/asf/impala/blob/575b5a20/be/src/exprs/decimal-operators-ir.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/decimal-operators-ir.cc
b/be/src/exprs/decimal-operators-ir.cc
index 0133317..bb54a0f 100644
--- a/be/src/exprs/decimal-operators-ir.cc
+++ b/be/src/exprs/decimal-operators-ir.cc
@@ -33,11 +33,15 @@
namespace impala {
#define RETURN_IF_OVERFLOW(ctx, overflow, return_type) \
- do {\
- if (UNLIKELY(overflow)) {\
- ctx->AddWarning("Expression overflowed, returning NULL");\
- return return_type::null();\
- }\
+ do { \
+ if (UNLIKELY(overflow)) { \
+ if (ctx->impl()->GetConstFnAttr(FunctionContextImpl::DECIMAL_V2)) { \
+ ctx->SetError("Decimal expression overflowed"); \
+ } else { \
+ ctx->AddWarning("Decimal expression overflowed, returning NULL"); \
+ } \
+ return return_type::null(); \
+ } \
} while (false)
// Inline in IR module so branches can be optimised out.
http://git-wip-us.apache.org/repos/asf/impala/blob/575b5a20/be/src/exprs/expr-codegen-test.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/expr-codegen-test.cc
b/be/src/exprs/expr-codegen-test.cc
index 781e457..e1c5c50 100644
--- a/be/src/exprs/expr-codegen-test.cc
+++ b/be/src/exprs/expr-codegen-test.cc
@@ -324,7 +324,7 @@ TEST_F(ExprCodegenTest, TestInlineConstFnAttrs) {
// Call InlineConstFnAttrs() and rerun verification
int replaced = InlineConstFnAttrs(expr, codegen.get(), fn);
- EXPECT_EQ(replaced, 9);
+ EXPECT_EQ(replaced, 19);
ResetVerification(codegen.get());
verification_succeeded = VerifyFunction(codegen.get(), fn);
EXPECT_TRUE(verification_succeeded) << CodeGenUtil::Print(fn);
http://git-wip-us.apache.org/repos/asf/impala/blob/575b5a20/be/src/exprs/expr-test.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/expr-test.cc b/be/src/exprs/expr-test.cc
index 341a6e2..4b7bcd0 100644
--- a/be/src/exprs/expr-test.cc
+++ b/be/src/exprs/expr-test.cc
@@ -1569,10 +1569,10 @@ DecimalTestCase decimal_cases[] = {
{ "cast(1 as decimal(38,0)) + cast(0.2 as decimal(38,1))", {{ false, false,
12, 38, 1 }}},
{ "cast(100000000000000000000000000000000 as decimal(38,0)) + cast(0 as
decimal(38,38))",
{{ false, true, 0, 38, 38 },
- { false, true, 0, 38, 6 }}},
+ { true, false, 0, 38, 6 }}},
{ "cast(-100000000000000000000000000000000 as decimal(38,0)) + cast(0 as
decimal(38,38))",
{{ false, true, 0, 38, 38 },
- { false, true, 0, 38, 6 }}},
+ { true, false, 0, 38, 6 }}},
{ "cast(99999999999999999999999999999999 as decimal(38,0)) + cast(0 as
decimal(38,38))",
{{ false, true, 0, 38, 38 },
{ false, false, StringToInt128("99999999999999999999999999999999000000"),
38, 6 }}},
@@ -1614,10 +1614,12 @@ DecimalTestCase decimal_cases[] = {
{{ false, false, StringToInt128("1"), 38, 0 }}},
{ "cast(99999999999999999999999999999999.999999 as decimal(38,6)) + "
"cast(8899999999999999999999999999999.999999 as decimal(38,6))",
- {{ false, true, 0, 38, 6 }}},
+ {{ false, true, 0, 38, 6 },
+ { true, false, 0, 38, 6 }}},
{ "cast(-99999999999999999999999999999999.999999 as decimal(38,6)) + "
"cast(-8899999999999999999999999999999.999999 as decimal(38,6))",
- {{ false, true, 0, 38, 6 }}},
+ {{ false, true, 0, 38, 6 },
+ { true, false, 0, 38, 6 }}},
{ "cast(-99999999999999999999999999999999.999999 as decimal(38,6)) + "
"cast(8899999999999999999999999999999.999999 as decimal(38,6))",
{{ false, false,
StringToInt128("-91100000000000000000000000000000000000"), 38, 6 }}},
@@ -1634,10 +1636,12 @@ DecimalTestCase decimal_cases[] = {
// Smallest result that overflows.
{ "cast(77777777777777777777777777777777.777777 as decimal(38,6)) + "
"cast(22222222222222222222222222222222.222223 as decimal(38,6))",
- {{ false, true, 0, 38, 6 }}},
+ {{ false, true, 0, 38, 6 },
+ { true, false, 0, 38, 6 }}},
{ "cast(-77777777777777777777777777777777.777777 as decimal(38,6)) + "
"cast(-22222222222222222222222222222222.222223 as decimal(38,6))",
- {{ false, true, 0, 38, 6 }}},
+ {{ false, true, 0, 38, 6 },
+ { true, false, 0, 38, 6 }}},
{ "cast(11111111111111111111111111111111.777777 as decimal(38,6)) + "
"cast(11111111111111111111111111111111.555555 as decimal(38,6))",
{{ false, false, StringToInt128("22222222222222222222222222222223333332"),
38, 6 }}},
@@ -1670,11 +1674,11 @@ DecimalTestCase decimal_cases[] = {
{ "cast(99999999999999999999999999999999.999999 as decimal(38,6)) + "
"cast(0.0000005 as decimal(38,7))",
{{ false, true, 0, 38, 7 },
- { false, true, 0, 38, 6 }}},
+ { true, false, 0, 38, 6 }}},
{ "cast(-99999999999999999999999999999999.999999 as decimal(38,6)) + "
"cast(-0.0000005 as decimal(38,7))",
{{ false, true, 0, 38, 7 },
- { false, true, 0, 38, 6 }}},
+ { true, false, 0, 38, 6 }}},
{ "cast(99999999999999999999999999999999.999999 as decimal(38,6)) + "
"cast(-0.0000006 as decimal(38,7))",
{{ false, true, 0, 38, 7 },
@@ -1698,11 +1702,11 @@ DecimalTestCase decimal_cases[] = {
{ "cast(99999999999999999999999999999999 as decimal(38,0)) + "
"cast(0.9999995 as decimal(38,38))",
{{ false, true, 0, 38, 38 },
- { false, true, 0, 38, 6 }}},
+ { true, false, 0, 38, 6 }}},
{ "cast(-99999999999999999999999999999999 as decimal(38,0)) + "
"cast(-0.9999995 as decimal(38,38))",
{{ false, true, 0, 38, 38 },
- { false, true, 0, 38, 6 }}},
+ { true, false, 0, 38, 6 }}},
{ "cast(0.99999999999999999999999999999999999999 as decimal(38,38)) + "
"cast(-5e-38 as decimal(38,38))",
{{ false, false, StringToInt128("99999999999999999999999999999999999994"),
38, 38 },
@@ -1832,17 +1836,21 @@ DecimalTestCase decimal_cases[] = {
// MAX_UNSCALED_DECIMAL16.
{ "cast(18446744073709551615 as decimal(38,0)) * "
"cast(9223372036854775807 as decimal(38,0))",
- {{ false, true, 0, 38, 0 }}},
+ {{ false, true, 0, 38, 0 },
+ { true, false, 0, 38, 0 }}},
{ "cast(-18446744073709551615 as decimal(38,0)) * "
"cast(9223372036854775807 as decimal(38,0))",
- {{ false, true, 0, 38, 0 }}},
+ {{ false, true, 0, 38, 0 },
+ { true, false, 0, 38, 0 }}},
// int256 is required. We are multiplying (2^64 - 1) * (2^63) here.
{ "cast(18446744073709551615 as decimal(38,0)) * "
"cast(9223372036854775808 as decimal(38,0))",
- {{ false, true, 0, 38, 0 }}},
+ {{ false, true, 0, 38, 0 },
+ { true, false, 0, 38, 0 }}},
{ "cast(-18446744073709551615 as decimal(38,0)) * "
"cast(9223372036854775808 as decimal(38,0))",
- {{ false, true, 0, 38, 0 }}},
+ {{ false, true, 0, 38, 0 },
+ { true, false, 0, 38, 0 }}},
// Largest intermediate result that does not require int256.
{ "cast(1844674407370.9551615 as decimal(38,7)) * "
"cast(9223372036854775807 as decimal(38,0))",
@@ -1879,7 +1887,8 @@ DecimalTestCase decimal_cases[] = {
{ false, false, StringToInt128("1000000000000000000000000000000000000"),
38, 37 }}},
{ "cast(10000000000000000000 as decimal(21,0)) * "
"cast(10000000000000000000 as decimal(21,0))",
- {{ false, true, 0, 38, 0 }}},
+ {{ false, true, 0, 38, 0 },
+ { true, false, 0, 38, 0 }}},
{ "cast(1000000000000000.0000 as decimal(38,4)) * "
"cast(1000000000000000.0000 as decimal(38,4))",
{{ false, true, 0, 38, 8 },
@@ -1892,11 +1901,11 @@ DecimalTestCase decimal_cases[] = {
{ "cast(10000000000000000.0000 as decimal(38,4)) * "
"cast(10000000000000000.0000 as decimal(38,4))",
{{ false, true, 0, 38, 8 },
- { false, true, 0, 38, 6 }}},
+ { true, false, 0, 38, 6 }}},
{ "cast(-10000000000000000.0000 as decimal(38,4)) * "
"cast(10000000000000000.0000 as decimal(38,4))",
{{ false, true, 0, 38, 8 },
- { false, true, 0, 38, 6 }}},
+ { true, false, 0, 38, 6 }}},
// The reason why the result of (38,38) * (38,38) is (38,37).
{ "cast(0.99999999999999999999999999999999999999 as decimal(38,38)) * "
"cast(0.99999999999999999999999999999999999999 as decimal(38,38))",
@@ -1944,7 +1953,7 @@ DecimalTestCase decimal_cases[] = {
{ "cast(99999999999999999999999999999999999999 as decimal(38,0)) / "
"cast(0.00000000000000000000000000000000000001 as decimal(38,38))",
{{ false, true, 0, 38, 38 },
- { false, true, 0, 38, 6 }}},
+ { true, false, 0, 38, 6 }}},
{ "cast(0.00000000000000000000000000000000000001 as decimal(38,38)) / "
"cast(99999999999999999999999999999999999999 as decimal(38,0))",
{{ false, false, 0, 38, 38 }}},
@@ -2485,7 +2494,8 @@ DecimalTestCase decimal_cases[] = {
{ "cast('10' as decimal(2,0)) / cast('0' as decimal(38,19))",
{{ false, true, 0, 38, 19 },
{ true, false, 0, 38, 19 }}},
- { "mod(cast(NULL as decimal(2,0)), NULL)", {{ false, true, 0, 2, 0 }}},
+ { "mod(cast(NULL as decimal(2,0)), NULL)",
+ {{ false, true, 0, 2, 0 }}},
// Test CAST DECIMAL -> DECIMAL
{ "cast(cast(0.12344 as decimal(6,5)) as decimal(6,4))",
{{ false, false, 1234, 6, 4 }}},
@@ -2497,10 +2507,10 @@ DecimalTestCase decimal_cases[] = {
{ false, false, 1, 1, 0 }}},
{ "cast(cast(999999999.99 as DECIMAL(11,2)) as DECIMAL(9,0))",
{{ false, false, 999999999, 9, 0 },
- { false, true, 0, 9, 0 }}},
+ { true, false, 0, 9, 0 }}},
{ "cast(cast(-999999999.99 as DECIMAL(11,2)) as DECIMAL(9,0))",
{{ false, false, -999999999, 9, 0 },
- { false, true, 0, 9, 0 }}},
+ { true, false, 0, 9, 0 }}},
// IMPALA-2233: Test that implicit casts do not lose precision.
// The overload greatest(decimal(*,*)) is available and should be used.
{ "greatest(0, cast('99999.1111' as decimal(30,10)))",
@@ -2511,7 +2521,7 @@ DecimalTestCase decimal_cases[] = {
"as d))) as t",
{{ false, false, static_cast<int128_t>(10000000ll) *
10000000000ll * 10000000000ll * 10000000000ll, 38, 5 },
- { false, true, 0, 38, 6}}},
+ { true, false, 0, 38, 6}}},
{ "avg(d) from (values((cast(1234567890 as DECIMAL(10,0)) as d))) as t",
{{ false, false, 1234567890, 10, 0},
{ false, false, 1234567890000000, 16, 6}}},
@@ -2528,17 +2538,17 @@ DecimalTestCase decimal_cases[] = {
"as d))) as t",
{{ false, false, static_cast<int128_t>(100) *
10000000000ll * 10000000000ll * 10000000000ll, 33, 0},
- { false, true, 0, 38, 6}}},
+ { true, false, 0, 38, 6}}},
{ "avg(d) from (values((cast(100000000000000000000000000000000.0 as
DECIMAL(34,1)) "
"as d))) as t",
{{ false, false, static_cast<int128_t>(1000) *
10000000000ll * 10000000000ll * 10000000000ll, 34, 1},
- { false, true, 0, 38, 6}}},
+ { true, false, 0, 38, 6}}},
{ "avg(d) from (values((cast(100000000000000000000000000000000.00000 as
DECIMAL(38,5)) "
"as d))) as t",
{{ false, false, static_cast<int128_t>(10000000) *
10000000000ll * 10000000000ll * 10000000000ll, 38, 5},
- { false, true, 0, 38, 6}}},
+ { true, false, 0, 38, 6}}},
{ "avg(d) from (values((cast(10000000000000000000000000000000.000000 as
DECIMAL(38,6)) "
"as d))) as t",
{{ false, false, static_cast<int128_t>(10000000) *
@@ -2555,7 +2565,7 @@ DecimalTestCase decimal_cases[] = {
{{ false, false, 9999999900, 10, 2 }}},
{ "cast(cast(99999999.5999999 AS int) AS decimal(10,2))",
{{ false, false, 9999999900, 10, 2 },
- { false, true, 0, 10, 2 }}},
+ { true, false, 0, 10, 2 }}},
{ "cast(cast(10000.5999999 as int) as decimal(30,6))",
{{ false, false, 10000000000, 30, 6 },
{ false, false, 10001000000, 30, 6 }}},
@@ -2567,76 +2577,76 @@ DecimalTestCase decimal_cases[] = {
{ false, false, -100010, 6, 1 }}},
{ "cast(cast(9999.5 AS int) AS decimal(4,0))",
{{ false, false, 9999, 4, 0 },
- { false, true, 0, 4, 0 }}},
+ { true, false, 0, 4, 0 }}},
{ "cast(cast(-9999.5 AS int) AS decimal(4,0))",
{{ false, false, -9999, 4, 0 },
- { false, true, 0, 4, 0 }}},
+ { true, false, 0, 4, 0 }}},
{ "cast(cast(127.4999 AS tinyint) AS decimal(30,0))",
{{ false, false, 127, 30, 0 }}},
{ "cast(cast(127.5 AS tinyint) AS decimal(30,0))",
{{ false, false, 127, 30, 0 },
- { false, true, 0, 30, 0 }}},
+ { true, false, 0, 30, 0 }}},
{ "cast(cast(128.0 AS tinyint) AS decimal(30,0))",
{{ false, false, -128, 30, 0 }, // BUG: JIRA: IMPALA-865
- { false, true, 0, 30, 0 }}},
+ { true, false, 0, 30, 0 }}},
{ "cast(cast(-128.4999 AS tinyint) AS decimal(30,0))",
{{ false, false, -128, 30, 0 }}},
{ "cast(cast(-128.5 AS tinyint) AS decimal(30,0))",
{{ false, false, -128, 30, 0 },
- { false, true, 0, 30, 0 }}},
+ { true, false, 0, 30, 0 }}},
{ "cast(cast(-129.0 AS tinyint) AS decimal(30,0))",
{{ false, false, 127, 30, 0 }, // BUG: JIRA: IMPALA-865
- { false, true, 0, 30, 0 }}},
+ { true, false, 0, 30, 0 }}},
{ "cast(cast(32767.4999 AS smallint) AS decimal(30,0))",
{{ false, false, 32767, 30, 0 }}},
{ "cast(cast(32767.5 AS smallint) AS decimal(30,0))",
{{ false, false, 32767, 30, 0 },
- { false, true, 0, 30, 0 }}},
+ { true, false, 0, 30, 0 }}},
{ "cast(cast(32768.0 AS smallint) AS decimal(30,0))",
{{ false, false, -32768, 30, 0 }, // BUG: JIRA: IMPALA-865
- { false, true, 0, 30, 0 }}},
+ { true, false, 0, 30, 0 }}},
{ "cast(cast(-32768.4999 AS smallint) AS decimal(30,0))",
{{ false, false, -32768, 30, 0 }}},
{ "cast(cast(-32768.5 AS smallint) AS decimal(30,0))",
{{ false, false, -32768, 30, 0 },
- { false, true, 0, 30, 0 }}},
+ { true, false, 0, 30, 0 }}},
{ "cast(cast(-32769.0 AS smallint) AS decimal(30,0))",
{{ false, false, 32767, 30, 0 }, // BUG: JIRA: IMPALA-865
- { false, true, 0, 30, 0 }}},
+ { true, false, 0, 30, 0 }}},
{ "cast(cast(2147483647.4999 AS int) AS decimal(30,0))",
{{ false, false, 2147483647, 30, 0 }}},
{ "cast(cast(2147483647.5 AS int) AS decimal(30,0))",
{{ false, false, 2147483647, 30, 0 },
- { false, true, 0, 30, 0 }}},
+ { true, false, 0, 30, 0 }}},
{ "cast(cast(2147483648.0 AS int) AS decimal(30,0))",
{{ false, false, -2147483648, 30, 0 }, // BUG: JIRA: IMPALA-865
- { false, true, 0, 30, 0 }}},
+ { true, false, 0, 30, 0 }}},
{ "cast(cast(-2147483648.4999 AS int) AS decimal(30,0))",
{{ false, false, -2147483648, 30, 0 }}},
{ "cast(cast(-2147483648.5 AS int) AS decimal(30,0))",
{{ false, false, -2147483648, 30, 0 },
- { false, true, 0, 30, 0 }}},
+ { true, false, 0, 30, 0 }}},
{ "cast(cast(-2147483649.0 AS int) AS decimal(30,0))",
{{ false, false, 2147483647, 30, 0 }, // BUG: JIRA: IMPALA-865
- { false, true, 0, 30, 0 }}},
+ { true, false, 0, 30, 0 }}},
{ "cast(cast(9223372036854775807.4999 AS bigint) AS decimal(30,0))",
{{ false, false, 9223372036854775807, 30, 0 }}},
{ "cast(cast(9223372036854775807.5 AS bigint) AS decimal(30,0))",
{{ false, false, 9223372036854775807, 30, 0 },
- { false, true, 0, 30, 0 }}},
+ { true, false, 0, 30, 0 }}},
{ "cast(cast(9223372036854775808.0 AS bigint) AS decimal(30,0))",
// BUG; also GCC workaround with -1
{{ false, false, -9223372036854775807 - 1, 30, 0 },
// error: integer constant is so large that it is unsigned
- { false, true, 0, 30, 0 }}},
+ { true, false, 0, 30, 0 }}},
{ "cast(cast(-9223372036854775808.4999 AS bigint) AS decimal(30,0))",
{{ false, false, -9223372036854775807 - 1, 30, 0 }}},
{ "cast(cast(-9223372036854775808.5 AS bigint) AS decimal(30,0))",
{{ false, false, -9223372036854775807 - 1, 30, 0 },
- { false, true, 0, 30, 0 }}},
+ { true, false, 0, 30, 0 }}},
{ "cast(cast(-9223372036854775809.0 AS bigint) AS decimal(30,0))",
{{ false, false, 9223372036854775807, 30, 0 }, // BUG: JIRA: IMPALA-865
- { false, true, 0, 30, 0 }}},
+ { true, false, 0, 30, 0 }}},
{ "cast(cast(cast(pow(1, -38) as decimal(38,38)) as bigint) as
decimal(18,10))",
{{ false, false, 0, 18, 10 },
{ false, false, 10000000000, 18, 10 }}},
@@ -2651,7 +2661,7 @@ DecimalTestCase decimal_cases[] = {
{ false, false, 10000, 5, 1 }}},
{ "cast(cast(power(10, 3) - power(10, -2) as float) as decimal(4,1))",
{{ false, false, 9999, 4, 1 },
- { false, true, 0, 4, 1 }}},
+ { true, false, 0, 4, 1 }}},
{ "cast(cast(-power(10, 3) + power(10, -1) as float) as decimal(4,1))",
{{ false, false, -9999, 4, 1 }}},
{ "cast(cast(-power(10, 3) + power(10, -2) as float) as decimal(5,1))",
@@ -2659,7 +2669,7 @@ DecimalTestCase decimal_cases[] = {
{ false, false, -10000, 5, 1 }}},
{ "cast(cast(-power(10, 3) + power(10, -2) as float) as decimal(4,1))",
{{ false, false, -9999, 4, 1 },
- { false, true, 0, 4, 1 }}},
+ { true, false, 0, 4, 1 }}},
{ "cast(cast(power(10, 3) - 0.45 as double) as decimal(4,1))",
{{ false, false, 9995, 4, 1 },
{ false, false, 9996, 4, 1 }}},
@@ -2670,7 +2680,7 @@ DecimalTestCase decimal_cases[] = {
{ false, false, 1000, 5, 0 }}},
{ "cast(cast(power(10, 3) - 0.45 as double) as decimal(3,0))",
{{ false, false, 999, 3, 0 },
- { false, true, 0, 3, 0 }}},
+ { true, false, 0, 3, 0 }}},
{ "cast(cast(-power(10, 3) + 0.45 as double) as decimal(4,1))",
{{ false, false, -9995, 4, 1 },
{ false, false, -9996, 4, 1 }}},
@@ -2681,7 +2691,7 @@ DecimalTestCase decimal_cases[] = {
{ false, false, -1000, 5, 0 }}},
{ "cast(cast(-power(10, 3) + 0.45 as double) as decimal(3,0))",
{{ false, false, -999, 3, 0 },
- { false, true, 0, 3, 0 }}},
+ { true, false, 0, 3, 0 }}},
{ "cast(cast(power(10, 3) - 0.5 as double) as decimal(4,1))",
{{ false, false, 9995, 4, 1 }}},
{ "cast(cast(power(10, 3) - 0.5 as double) as decimal(5,2))",
@@ -2691,7 +2701,7 @@ DecimalTestCase decimal_cases[] = {
{ false, false, 1000, 5, 0 }}},
{ "cast(cast(power(10, 3) - 0.5 as double) as decimal(3,0))",
{{ false, false, 999, 3, 0 },
- { false, true, 0, 3, 0 }}},
+ { true, false, 0, 3, 0 }}},
{ "cast(cast(-power(10, 3) + 0.5 as double) as decimal(4,1))",
{{ false, false, -9995, 4, 1 }}},
{ "cast(cast(-power(10, 3) + 0.5 as double) as decimal(5,2))",
@@ -2701,7 +2711,7 @@ DecimalTestCase decimal_cases[] = {
{ false, false, -1000, 5, 0 }}},
{ "cast(cast(-power(10, 3) + 0.5 as double) as decimal(3,0))",
{{ false, false, -999, 3, 0 },
- { false, true, 0, 3, 0 }}},
+ { true, false, 0, 3, 0 }}},
{ "cast(cast(power(10, 3) - 0.55 as double) as decimal(4,1))",
{{ false, false, 9994, 4, 1 },
{ false, false, 9995, 4, 1 }}},
@@ -2721,20 +2731,27 @@ DecimalTestCase decimal_cases[] = {
{ "cast(cast(-power(10, 3) + 0.55 as double) as decimal(3,0))",
{{ false, false, -999, 3, 0 }}},
{ "cast(power(2, 1023) * 100 as decimal(38,0))",
- {{ false, true, 0, 38, 0 }}},
+ {{ false, true, 0, 38, 0 },
+ { true, false, 0, 38, 0 }}},
{ "cast(power(2, 1023) * 100 as decimal(18,0))",
- {{ false, true, 0, 18, 0 }}},
+ {{ false, true, 0, 18, 0 },
+ { true, false, 0, 18, 0 }}},
{ "cast(power(2, 1023) * 100 as decimal(9,0))",
- {{ false, true, 0, 9, 0 }}},
+ {{ false, true, 0, 9, 0 },
+ { true, false, 0, 9, 0 }}},
{ "cast(0/0 as decimal(38,0))",
- {{ false, true, 0, 38, 0 }}},
+ {{ false, true, 0, 38, 0 },
+ { true, false, 0, 38, 0 }}},
{ "cast(0/0 as decimal(18,0))",
- {{ false, true, 0, 18, 0 }}},
+ {{ false, true, 0, 18, 0 },
+ { true, false, 0, 18, 0 }}},
{ "cast(0/0 as decimal(9,0))",
- {{ false, true, 0, 9, 0 }}},
+ {{ false, true, 0, 9, 0 },
+ { true, false, 0, 9, 0 }}},
// 39 5's - legal double but will overflow in decimal
{ "cast(555555555555555555555555555555555555555 as decimal(38,0))",
- {{ false, true, 0, 38, 0 }}},
+ {{ false, true, 0, 38, 0 },
+ { true, false, 0, 38, 0 }}},
};
TEST_F(ExprTest, DecimalArithmeticExprs) {
http://git-wip-us.apache.org/repos/asf/impala/blob/575b5a20/testdata/workloads/functional-query/queries/QueryTest/decimal-exprs.test
----------------------------------------------------------------------
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/decimal-exprs.test
b/testdata/workloads/functional-query/queries/QueryTest/decimal-exprs.test
index b34cb78..328fbaf 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/decimal-exprs.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/decimal-exprs.test
@@ -94,6 +94,106 @@ select 10.0 % 0 from decimal_tbl;
UDF ERROR: Cannot divide decimal by zero
====
---- QUERY
+# Test DECIMAL V1 decimal overflow
+set decimal_v2=false;
+select
+ cast(9999999999999999999999999999 as decimal(38, 6)) *
+ cast(9999999999999999999999999999 as decimal(38, 6))
+---- RESULTS
+NULL
+---- TYPES
+DECIMAL
+---- ERRORS
+UDF WARNING: Decimal expression overflowed, returning NULL
+====
+---- QUERY
+# Test DECIMAL V2 decimal overflow
+set decimal_v2=true;
+select
+ cast(9999999999999999999999999999 as decimal(38, 6)) *
+ cast(9999999999999999999999999999 as decimal(38, 6))
+---- CATCH
+UDF ERROR: Decimal expression overflowed
+====
+---- QUERY
+# IMPALA-1837: Handle loss of precision when implicitly casting a literal to a
decimal.
+# Here "1.8" will be implicitly cast to a decimal(38,38), losing precision.
+set decimal_v2=false;
+select coalesce(1.8, cast(0 as decimal(38,38)));
+---- TYPES
+DECIMAL
+---- RESULTS
+0.00000000000000000000000000000000000000
+---- ERRORS
+UDF WARNING: Decimal expression overflowed, returning NULL
+====
+---- QUERY
+set decimal_v2=true;
+select coalesce(1.8, cast(0 as decimal(38,38)));
+---- CATCH
+UDF ERROR: Decimal expression overflowed
+====
+---- QUERY
+# DECIMAL v1 sum() overflow. A negative number is incorrectly returned due to
overflow.
+set decimal_v2=false;
+select sum(d6 * cast(4e37 as decimal(38,0))) from decimal_tbl;
+---- RESULTS
+-40282366920938463463374607431768211456
+---- TYPES
+DECIMAL
+====
+---- QUERY
+# DECIMAL v2 sum() overflow.
+set decimal_v2=true;
+select sum(d6 * cast(4e37 as decimal(38,0))) from decimal_tbl;
+---- CATCH
+UDF ERROR: Sum computation overflowed
+====
+---- QUERY
+# DECIMAL v1 avg() overflow. A negative number is incorrectly returned due to
overflow.
+set decimal_v2=false;
+select avg(d6 * cast(4e37 as decimal(38,0))) from decimal_tbl;
+---- RESULTS
+-28056473384187692692674921486353642291
+---- TYPES
+DECIMAL
+====
+---- QUERY
+# DECIMAL v2 avg() overflow.
+set decimal_v2=true;
+select avg(d6 * cast(4e37 as decimal(38,0))) from decimal_tbl;
+---- CATCH
+UDF ERROR: Avg computation overflowed
+====
+---- QUERY
+# DECIMAL v1 avg() and sum() could incorrecly indicate an overflow error if
+# only the absolute values are considered.
+set decimal_v2=false;
+with t as (
+ select cast(99999999999999999999999999999999999999 as decimal(38, 0)) as c
+ union all
+ select cast(-99999999999999999999999999999999999999 as decimal(38, 0)) as c)
+select sum(c), avg(c) from t;
+---- RESULTS
+0,0
+---- TYPES
+DECIMAL,DECIMAL
+====
+---- QUERY
+# DECIMAL v2 avg() and sum() could incorrecly indicate an overflow error if
+# only the absolute values are considered.
+set decimal_v2=true;
+with t as (
+ select cast(99999999999999999999999999999999999999 as decimal(38, 0)) as c
+ union all
+ select cast(-99999999999999999999999999999999999999 as decimal(38, 0)) as c)
+select sum(c), avg(c) from t;
+---- RESULTS
+0,0.000000
+---- TYPES
+DECIMAL,DECIMAL
+====
+---- QUERY
# Test casting behavior without decimal_v2 query option set.
set decimal_v2=false;
select cast(d3 as decimal(20, 3)) from decimal_tbl;
http://git-wip-us.apache.org/repos/asf/impala/blob/575b5a20/testdata/workloads/functional-query/queries/QueryTest/decimal.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/decimal.test
b/testdata/workloads/functional-query/queries/QueryTest/decimal.test
index 1f0fa4c..e2958df 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/decimal.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/decimal.test
@@ -368,17 +368,6 @@ STRING
---- QUERY
# IMPALA-1837: Handle loss of precision when implicitly casting a literal to a
decimal.
# Here "1.8" will be implicitly cast to a decimal(38,38), losing precision.
-select coalesce(1.8, cast(0 as decimal(38,38)))
----- TYPES
-DECIMAL
----- RESULTS
-0.00000000000000000000000000000000000000
----- ERRORS
-UDF WARNING: Expression overflowed, returning NULL
-====
----- QUERY
-# IMPALA-1837: Handle loss of precision when implicitly casting a literal to a
decimal.
-# Here "1.8" will be implicitly cast to a decimal(38,38), losing precision.
# No error is reported because the overflowed expr is never evaluated.
select coalesce(cast(0 as decimal(38,38)), 1.8)
---- TYPES
http://git-wip-us.apache.org/repos/asf/impala/blob/575b5a20/tests/hs2/test_hs2.py
----------------------------------------------------------------------
diff --git a/tests/hs2/test_hs2.py b/tests/hs2/test_hs2.py
index 7d7f9e9..5fa8fda 100644
--- a/tests/hs2/test_hs2.py
+++ b/tests/hs2/test_hs2.py
@@ -441,7 +441,7 @@ class TestHS2(HS2TestSuite):
# Test overflow warning
log = self.get_log("select cast(1000 as decimal(2, 1))")
- assert "Expression overflowed, returning NULL" in log
+ assert "Decimal expression overflowed, returning NULL" in log
@needs_session()
def test_get_exec_summary(self):
