http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f982c3f7/be/src/exprs/expr.h ---------------------------------------------------------------------- diff --git a/be/src/exprs/expr.h b/be/src/exprs/expr.h index 13ba312..3a92c21 100644 --- a/be/src/exprs/expr.h +++ b/be/src/exprs/expr.h @@ -260,55 +260,8 @@ class Expr { /// not strip these symbols. static void InitBuiltinsDummy(); - // Any additions to this enum must be reflected in both GetConstant*() and - // GetIrConstant(). - enum ExprConstant { - // RETURN_TYPE_*: properties of FunctionContext::GetReturnType(). - RETURN_TYPE_SIZE, // int - RETURN_TYPE_PRECISION, // int - RETURN_TYPE_SCALE, // int - // ARG_TYPE_* with parameter i: properties of FunctionContext::GetArgType(i). - ARG_TYPE_SIZE, // int[] - ARG_TYPE_PRECISION, // int[] - ARG_TYPE_SCALE, // int[] - }; - - // Static function for obtaining a runtime constant. Expr compute functions and - // builtins implementing the UDF interface should use this function, rather than - // accessing runtime constants directly, so any recognized constants can be inlined via - // InlineConstants() in the codegen path. In the interpreted path, this function will - // work as-is. - // - // 'c' determines which constant is returned. The type of the constant is annotated in - // the ExprConstant enum above. If the constant is an array, 'i' must be specified and - // indicates which element to return. 'i' must always be an immediate integer value so - // InlineConstants() can resolve the index, e.g., it cannot be a variable or an - // expression like "1 + 1". For example, if 'c' = ARG_TYPE_SIZE, then 'T' = int and - // 0 <= i < children_.size(). - // - // InlineConstants() can be run on the function to replace recognized constants. The - // constants are only replaced in the function itself, so any callee functions with - // constants to be replaced must be inlined into the function that InlineConstants() - // is run on (e.g. by annotating them with IR_ALWAYS_INLINE). - // - // TODO: implement a loop unroller (or use LLVM's) so we can use GetConstantInt() in - // loops - static int GetConstantInt(const FunctionContext& ctx, ExprConstant c, int i = -1); - - /// Finds all calls to Expr::GetConstantInt() in 'fn' and replaces them with the - /// appropriate runtime constants based on the arguments. 'return_type' is the - /// return type of the UDF or UDAF, i.e. the value of FunctionContext::GetReturnType(). - /// 'arg_types' are the argument types of the UDF or UDAF, i.e. the values of - /// FunctionContext::GetArgType(). - static int InlineConstants(const FunctionContext::TypeDesc& return_type, - const std::vector<FunctionContext::TypeDesc>& arg_types, LlvmCodeGen* codegen, - llvm::Function* fn); - static const char* LLVM_CLASS_NAME; - // Expr::GetConstantInt() symbol prefix. - static const char* GET_CONSTANT_INT_SYMBOL_PREFIX; - protected: friend class AggFnEvaluator; friend class CastExpr; @@ -409,11 +362,6 @@ class Expr { /// functions (e.g. in ScalarFnCall() when codegen is disabled). llvm::Function* GetStaticGetValWrapper(ColumnType type, LlvmCodeGen* codegen); - /// Replace all calls to Expr::GetConstant() in 'fn' based on the types of the - /// expr and its children. This is a convenience method that invokes the static - /// InlineConstants() function with the correct arguments for the expr. - int InlineConstants(LlvmCodeGen* codegen, llvm::Function* fn); - /// Simple debug string that provides no expr subclass-specific information std::string DebugString(const std::string& expr_name) const; @@ -457,13 +405,6 @@ class Expr { static StringVal GetStringVal(Expr* expr, ExprContext* context, const TupleRow* row); static TimestampVal GetTimestampVal(Expr* expr, ExprContext* context, const TupleRow* row); static DecimalVal GetDecimalVal(Expr* expr, ExprContext* context, const TupleRow* row); - - - // Helper function for GetConstantInt() and InlineConstants(): return the constant value - // given the specific argument and return types. - static int GetConstantInt(const FunctionContext::TypeDesc& return_type, - const std::vector<FunctionContext::TypeDesc>& arg_types, ExprConstant c, - int i = -1); }; }
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f982c3f7/be/src/exprs/math-functions-ir.cc ---------------------------------------------------------------------- diff --git a/be/src/exprs/math-functions-ir.cc b/be/src/exprs/math-functions-ir.cc index da8df36..6a331ba 100644 --- a/be/src/exprs/math-functions-ir.cc +++ b/be/src/exprs/math-functions-ir.cc @@ -433,7 +433,7 @@ template <typename T> T MathFunctions::Negative(FunctionContext* ctx, const T& v template <> DecimalVal MathFunctions::Negative(FunctionContext* ctx, const DecimalVal& val) { if (val.is_null) return val; - int type_byte_size = Expr::GetConstantInt(*ctx, Expr::RETURN_TYPE_SIZE); + int type_byte_size = ctx->impl()->GetConstFnAttr(FunctionContextImpl::RETURN_TYPE_SIZE); switch (type_byte_size) { case 4: return DecimalVal(-val.val4); @@ -519,7 +519,7 @@ template <bool ISLEAST> DecimalVal MathFunctions::LeastGreatest( DCHECK_GT(num_args, 0); if (args[0].is_null) return DecimalVal::null(); DecimalVal result_val = args[0]; - int type_byte_size = Expr::GetConstantInt(*ctx, Expr::RETURN_TYPE_SIZE); + int type_byte_size = ctx->impl()->GetConstFnAttr(FunctionContextImpl::RETURN_TYPE_SIZE); for (int i = 1; i < num_args; ++i) { if (args[i].is_null) return DecimalVal::null(); switch (type_byte_size) { http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f982c3f7/be/src/exprs/scalar-fn-call.cc ---------------------------------------------------------------------- diff --git a/be/src/exprs/scalar-fn-call.cc b/be/src/exprs/scalar-fn-call.cc index 06830b7..59aac6d 100644 --- a/be/src/exprs/scalar-fn-call.cc +++ b/be/src/exprs/scalar-fn-call.cc @@ -316,8 +316,8 @@ Status ScalarFnCall::GetCodegendComputeFn(LlvmCodeGen* codegen, Function** fn) { NumFixedArgs(), vararg_start_idx_ != -1, &udf, &cache_entry_)); // Inline constants into the function if it has an IR body. if (!udf->isDeclaration()) { - InlineConstants(AnyValUtil::ColumnTypeToTypeDesc(type_), - AnyValUtil::ColumnTypesToTypeDescs(arg_types), codegen, udf); + codegen->InlineConstFnAttrs(AnyValUtil::ColumnTypeToTypeDesc(type_), + AnyValUtil::ColumnTypesToTypeDescs(arg_types), udf); udf = codegen->FinalizeFunction(udf); if (udf == NULL) { return Status( http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f982c3f7/be/src/runtime/decimal-value.inline.h ---------------------------------------------------------------------- diff --git a/be/src/runtime/decimal-value.inline.h b/be/src/runtime/decimal-value.inline.h index f886787..3a65a16 100644 --- a/be/src/runtime/decimal-value.inline.h +++ b/be/src/runtime/decimal-value.inline.h @@ -79,7 +79,8 @@ inline const T DecimalValue<T>::fractional_part(int scale) const { template<typename T> inline DecimalValue<T> DecimalValue<T>::ScaleTo(int src_scale, int dst_scale, - int dst_precision, bool* overflow) const { int delta_scale = src_scale - dst_scale; + int dst_precision, bool* overflow) const { + int delta_scale = src_scale - dst_scale; T result = value(); T max_value = DecimalUtil::GetScaleMultiplier<T>(dst_precision); if (delta_scale >= 0) { http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f982c3f7/be/src/runtime/lib-cache.cc ---------------------------------------------------------------------- diff --git a/be/src/runtime/lib-cache.cc b/be/src/runtime/lib-cache.cc index 00c1159..9a8d873 100644 --- a/be/src/runtime/lib-cache.cc +++ b/be/src/runtime/lib-cache.cc @@ -400,12 +400,9 @@ Status LibCache::GetCacheEntryInternal(const string& hdfs_lib_file, LibType type DynamicOpen((*entry)->local_path.c_str(), &(*entry)->shared_object_handle)); } else if (type == TYPE_IR) { // Load the module temporarily and populate all symbols. - ObjectPool pool; - scoped_ptr<LlvmCodeGen> codegen; - string module_id = filesystem::path((*entry)->local_path).stem().string(); - RETURN_IF_ERROR(LlvmCodeGen::CreateFromFile( - &pool, NULL, (*entry)->local_path, module_id, &codegen)); - codegen->GetSymbols(&(*entry)->symbols); + const string file = (*entry)->local_path; + const string module_id = filesystem::path(file).stem().string(); + RETURN_IF_ERROR(LlvmCodeGen::GetSymbols(file, module_id, &(*entry)->symbols)); } else { DCHECK_EQ(type, TYPE_JAR); // Nothing to do. http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f982c3f7/be/src/runtime/runtime-state.cc ---------------------------------------------------------------------- diff --git a/be/src/runtime/runtime-state.cc b/be/src/runtime/runtime-state.cc index de0fc97..0d7f262 100644 --- a/be/src/runtime/runtime-state.cc +++ b/be/src/runtime/runtime-state.cc @@ -156,7 +156,7 @@ Status RuntimeState::CreateBlockMgr() { Status RuntimeState::CreateCodegen() { if (codegen_.get() != NULL) return Status::OK(); // TODO: add the fragment ID to the codegen ID as well - RETURN_IF_ERROR(LlvmCodeGen::CreateImpalaCodegen(obj_pool_.get(), + RETURN_IF_ERROR(LlvmCodeGen::CreateImpalaCodegen(this, instance_mem_tracker_.get(), PrintId(fragment_instance_id()), &codegen_)); codegen_->EnableOptimizations(true); profile_.AddChild(codegen_->runtime_profile()); http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f982c3f7/be/src/runtime/runtime-state.h ---------------------------------------------------------------------- diff --git a/be/src/runtime/runtime-state.h b/be/src/runtime/runtime-state.h index 14f9d38..97014aa 100644 --- a/be/src/runtime/runtime-state.h +++ b/be/src/runtime/runtime-state.h @@ -104,6 +104,7 @@ class RuntimeState { bool abort_on_default_limit_exceeded() const { return query_options().abort_on_default_limit_exceeded; } + bool decimal_v2() const { return query_options().decimal_v2; } const TQueryCtx& query_ctx() const; const TPlanFragmentInstanceCtx& instance_ctx() const { return *instance_ctx_; } const TUniqueId& session_id() const { return query_ctx().session.session_id; } http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f982c3f7/be/src/service/frontend.cc ---------------------------------------------------------------------- diff --git a/be/src/service/frontend.cc b/be/src/service/frontend.cc index b83d091..e48cb1e 100644 --- a/be/src/service/frontend.cc +++ b/be/src/service/frontend.cc @@ -82,7 +82,7 @@ Frontend::Frontend() { {"getTableFiles", "([B)[B", &get_table_files_id_}, {"showCreateFunction", "([B)Ljava/lang/String;", &show_create_function_id_}, {"buildTestDescriptorTable", "([B)[B", &build_test_descriptor_table_id_} -}; + }; JNIEnv* jni_env = getJNIEnv(); // create instance of java class JniFrontend http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f982c3f7/be/src/udf/udf-internal.h ---------------------------------------------------------------------- diff --git a/be/src/udf/udf-internal.h b/be/src/udf/udf-internal.h index 96d0fc7..907bb22 100644 --- a/be/src/udf/udf-internal.h +++ b/be/src/udf/udf-internal.h @@ -132,12 +132,53 @@ class FunctionContextImpl { return arg_types_; } - // UDFs may manipulate DecimalVal arguments via SIMD instructions such as 'movaps' - // that require 16-byte memory alignment. + RuntimeState* state() { return state_; } + + /// Various static attributes of the UDF/UDA that can be injected as constants + /// by codegen. Note that the argument types refer to those in the UDF/UDA signature, + /// not the arguments of the C++ functions implementing the UDF/UDA. Any change to + /// this enum must be reflected in FunctionContextImpl::GetConstFnAttr(). + enum ConstFnAttr { + /// RETURN_TYPE_*: properties of FunctionContext::GetReturnType() + RETURN_TYPE_SIZE, // int + RETURN_TYPE_PRECISION, // int + RETURN_TYPE_SCALE, // int + /// ARG_TYPE_* with parameter i: properties of FunctionContext::GetArgType(i) + ARG_TYPE_SIZE, // int[] + ARG_TYPE_PRECISION, // int[] + ARG_TYPE_SCALE, // int[] + /// True if decimal_v2 query option is set. + DECIMAL_V2, + }; + + /// This function returns the various static attributes of the UDF/UDA. Calls to this + /// function are replaced by constants injected by codegen. If codegen is disabled, + /// this function is interpreted as-is. + /// + /// 't' is the static function attribute defined in the ConstFnAttr enum above. + /// For function attributes of arguments, 'i' holds the argument number (0 indexed). + /// Please note that argument refers to the arguments in the signature of the UDF or UDA. + /// 'i' must always be an immediate integer value in order to utilize the constant + /// replacement when codegen is enabled. e.g., it cannot be a variable or an expression + /// like "1 + 1". + /// + int GetConstFnAttr(ConstFnAttr t, int i = -1); + + /// Return the function attribute 't' defined in ConstFnAttr above. + static int GetConstFnAttr(const RuntimeState* state, + const impala_udf::FunctionContext::TypeDesc& return_type, + const std::vector<impala_udf::FunctionContext::TypeDesc>& arg_types, + ConstFnAttr t, int i = -1); + + /// UDFs may manipulate DecimalVal arguments via SIMD instructions such as 'movaps' + /// that require 16-byte memory alignment. static const int VARARGS_BUFFER_ALIGNMENT = 16; + + /// The LLVM class name for FunctionContext. Used for handcrafted IR. static const char* LLVM_FUNCTIONCONTEXT_NAME; - RuntimeState* state() { return state_; } + /// FunctionContextImpl::GetConstFnAttr() symbol. Used for call sites replacement. + static const char* GET_CONST_FN_ATTR_SYMBOL; private: friend class impala_udf::FunctionContext; http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f982c3f7/be/src/udf/udf-test-harness.cc ---------------------------------------------------------------------- diff --git a/be/src/udf/udf-test-harness.cc b/be/src/udf/udf-test-harness.cc index 1c3737d..d34b5ee 100644 --- a/be/src/udf/udf-test-harness.cc +++ b/be/src/udf/udf-test-harness.cc @@ -18,6 +18,7 @@ #include "udf/udf-test-harness.h" #include <vector> +#include "runtime/runtime-state.h" #include "udf/udf-internal.h" #include "common/names.h" @@ -27,8 +28,8 @@ using namespace impala; FunctionContext* UdfTestHarness::CreateTestContext( const FunctionContext::TypeDesc& return_type, - const vector<FunctionContext::TypeDesc>& arg_types) { - return FunctionContextImpl::CreateContext(NULL, NULL, return_type, arg_types, 0, true); + const vector<FunctionContext::TypeDesc>& arg_types, RuntimeState* state) { + return FunctionContextImpl::CreateContext(state, NULL, return_type, arg_types, 0, true); } void UdfTestHarness::SetConstantArgs( http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f982c3f7/be/src/udf/udf-test-harness.h ---------------------------------------------------------------------- diff --git a/be/src/udf/udf-test-harness.h b/be/src/udf/udf-test-harness.h index 67ba8fc..668a44d 100644 --- a/be/src/udf/udf-test-harness.h +++ b/be/src/udf/udf-test-harness.h @@ -24,6 +24,7 @@ #include <boost/function.hpp> #include <boost/scoped_ptr.hpp> +#include "runtime/runtime-state.h" #include "udf/udf.h" #include "udf/udf-debug.h" @@ -36,7 +37,8 @@ class UdfTestHarness { /// argument of the UDF not including the FunctionContext*. The caller is responsible /// for calling delete on it. This context has additional debugging validation enabled. static FunctionContext* CreateTestContext(const FunctionContext::TypeDesc& return_type, - const std::vector<FunctionContext::TypeDesc>& arg_types); + const std::vector<FunctionContext::TypeDesc>& arg_types, + impala::RuntimeState* state = nullptr); /// Use with test contexts to test use of IsArgConstant() and GetConstantArg(). /// constant_args should contain an AnyVal* for each argument of the UDF not including http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f982c3f7/be/src/udf/udf.cc ---------------------------------------------------------------------- diff --git a/be/src/udf/udf.cc b/be/src/udf/udf.cc index cf1fa0f..c2a5270 100644 --- a/be/src/udf/udf.cc +++ b/be/src/udf/udf.cc @@ -82,7 +82,12 @@ class RuntimeState { assert(false); } - bool abort_on_error() { + bool abort_on_error() const { + assert(false); + return false; + } + + bool decimal_v2() const { assert(false); return false; } @@ -99,6 +104,7 @@ class RuntimeState { } #else +#include "exprs/anyval-util.h" #include "runtime/free-pool.h" #include "runtime/mem-tracker.h" #include "runtime/runtime-state.h" @@ -114,6 +120,8 @@ using std::pair; const int FunctionContextImpl::VARARGS_BUFFER_ALIGNMENT; const char* FunctionContextImpl::LLVM_FUNCTIONCONTEXT_NAME = "class.impala_udf::FunctionContext"; +const char* FunctionContextImpl::GET_CONST_FN_ATTR_SYMBOL = + "_ZN6impala19FunctionContextImpl14GetConstFnAttrENS0_11ConstFnAttrEi"; static const int MAX_WARNINGS = 1000; @@ -501,3 +509,53 @@ const FunctionContext::TypeDesc* FunctionContext::GetArgType(int arg_idx) const if (arg_idx < 0 || arg_idx >= impl_->arg_types_.size()) return NULL; return &impl_->arg_types_[arg_idx]; } + +static int GetTypeByteSize(const FunctionContext::TypeDesc& type) { +#if defined(IMPALA_UDF_SDK_BUILD) && IMPALA_UDF_SDK_BUILD + return 0; +#else + return AnyValUtil::TypeDescToColumnType(type).GetByteSize(); +#endif +} + +int FunctionContextImpl::GetConstFnAttr(FunctionContextImpl::ConstFnAttr t, int i) { + return GetConstFnAttr(state_, return_type_, arg_types_, t, i); +} + +int FunctionContextImpl::GetConstFnAttr(const RuntimeState* state, + const FunctionContext::TypeDesc& return_type, + const vector<FunctionContext::TypeDesc>& arg_types, + ConstFnAttr t, int i) { + switch (t) { + case RETURN_TYPE_SIZE: + assert(i == -1); + return GetTypeByteSize(return_type); + case RETURN_TYPE_PRECISION: + assert(i == -1); + assert(return_type.type == FunctionContext::TYPE_DECIMAL); + return return_type.precision; + case RETURN_TYPE_SCALE: + assert(i == -1); + assert(return_type.type == FunctionContext::TYPE_DECIMAL); + return return_type.scale; + case ARG_TYPE_SIZE: + assert(i >= 0); + assert(i < arg_types.size()); + return GetTypeByteSize(arg_types[i]); + case ARG_TYPE_PRECISION: + assert(i >= 0); + assert(i < arg_types.size()); + assert(arg_types[i].type == FunctionContext::TYPE_DECIMAL); + return arg_types[i].precision; + case ARG_TYPE_SCALE: + assert(i >= 0); + assert(i < arg_types.size()); + assert(arg_types[i].type == FunctionContext::TYPE_DECIMAL); + return arg_types[i].scale; + case DECIMAL_V2: + return state->decimal_v2(); + default: + assert(false); + return -1; + } +} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f982c3f7/testdata/workloads/functional-query/queries/QueryTest/decimal.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/decimal.test b/testdata/workloads/functional-query/queries/QueryTest/decimal.test index 6e7cbae..4a1316b 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/decimal.test +++ b/testdata/workloads/functional-query/queries/QueryTest/decimal.test @@ -449,3 +449,47 @@ NULL,0.0000 ---- TYPES DECIMAL, DECIMAL ==== +---- QUERY +# Test casting behavior without decimal_v2 query option set. +set decimal_v2=false; +select cast(d3 as decimal(20, 3)) from functional.decimal_tbl; +---- RESULTS +1.234 +12.345 +123.456 +1234.567 +12345.678 +---- TYPES +DECIMAL +==== +---- QUERY +# Test casting behavior with decimal_v2 query option set. +set decimal_v2=true; +select cast(d3 as decimal(20, 3)) from functional.decimal_tbl; +---- RESULTS +1.235 +12.346 +123.457 +1234.568 +12345.679 +---- TYPES +DECIMAL +==== +---- QUERY +# Test casting behavior without decimal_v2 query option set. +set decimal_v2=false; +select sum(cast(d3 as DECIMAL(20,2)) + cast(d5 as DECIMAL(20,4))) from functional.decimal_tbl; +---- RESULTS +26078.2788 +---- TYPES +DECIMAL +==== +---- QUERY +# Test casting behavior with decimal_v2 query option set. +set decimal_v2=true; +select sum(cast(d3 as DECIMAL(20,2)) + cast(d5 as DECIMAL(20,4))) from functional.decimal_tbl; +---- RESULTS +26078.3189 +---- TYPES +DECIMAL +==== http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f982c3f7/tests/query_test/test_decimal_casting.py ---------------------------------------------------------------------- diff --git a/tests/query_test/test_decimal_casting.py b/tests/query_test/test_decimal_casting.py index b23e790..fac49c5 100644 --- a/tests/query_test/test_decimal_casting.py +++ b/tests/query_test/test_decimal_casting.py @@ -23,6 +23,7 @@ from metacomm.combinatorics.all_pairs2 import all_pairs2 as all_pairs from random import randint from tests.common.impala_test_suite import ImpalaTestSuite +from tests.common.test_dimensions import create_exec_option_dimension_from_dict from tests.common.test_vector import ImpalaTestDimension, ImpalaTestMatrix class TestDecimalCasting(ImpalaTestSuite): @@ -42,8 +43,8 @@ class TestDecimalCasting(ImpalaTestSuite): # mimics test_vectors.py and takes a subset of all decimal types 'pairwise' : all_pairs([(p, s) for p in xrange(1, 39) for s in xrange(0, p + 1)]) } - # We can cast for numerrics or string types. - CAST_FROM = ['string', 'number'] + # We can cast for numerics, string or decimal types. + CAST_FROM = ['string', 'number', 'decimal'] # Set the default precisin to 38 to operate on decimal values. getcontext().prec = 38 # Represents a 0 in decimal @@ -60,6 +61,8 @@ class TestDecimalCasting(ImpalaTestSuite): *TestDecimalCasting.DECIMAL_TYPES_MAP[cls.exploration_strategy()])) cls.ImpalaTestMatrix.add_dimension( ImpalaTestDimension('cast_from', *TestDecimalCasting.CAST_FROM)) + cls.ImpalaTestMatrix.add_dimension(create_exec_option_dimension_from_dict( + {'decimal_v2': ['false']})) cls.iterations = 1 def setup_method(self, method): @@ -79,15 +82,21 @@ class TestDecimalCasting(ImpalaTestSuite): assert actual == expected, "Cast: {0}, Expected: {1}, Actual: {2}".format(cast,\ expected, actual) - def _normalize_cast_expr(self, decimal_val, scale, from_string=False): + def _normalize_cast_expr(self, decimal_val, scale, cast_from): """Convert the decimal value to a string litetal to avoid overflow. If an integer literal is greater than the max bigint supported by Impala, it overflows. This methods replaces it with a string literal. """ - if (scale == 0 and abs(decimal_val) > self.max_bigint) or from_string: - return "select cast('{0}' as Decimal({1}, {2}))" - return "select cast({0} as Decimal({1}, {2}))" + # Decimal({1},{2}) is the target type to cast to. If casting from decimal, + # Decimal({3},{4}) is the intermediate type to cast {0} to. + if (scale == 0 and abs(decimal_val) > self.max_bigint) or (cast_from == 'string'): + return "select cast('{0}' as Decimal({1},{2}))" + elif cast_from == 'decimal': + base_cast = "cast('{0}' as Decimal({3},{4}))" + return "select cast(" + base_cast + " as Decimal({1},{2}))" + else: + return "select cast({0} as Decimal({1},{2}))" def test_min_max_zero_null(self, vector): """Sanity test at limits. @@ -98,20 +107,21 @@ class TestDecimalCasting(ImpalaTestSuite): - NULL is expressible in all decimal types """ precision, scale = vector.get_value('decimal_type') - from_string = vector.get_value('cast_from') == 'string' dec_max = Decimal('{0}.{1}'.format('9' * (precision - scale), '9' * scale)) # Multiplying large values eith -1 can produce an overflow. dec_min = Decimal('-{0}'.format(str(dec_max))) - cast = self._normalize_cast_expr(dec_max, scale, from_string=from_string) + cast = self._normalize_cast_expr(dec_max, scale, vector.get_value('cast_from')) # Test max - res = Decimal(self.execute_scalar(cast.format(dec_max, precision, scale))) + res = Decimal(self.execute_scalar(\ + cast.format(dec_max, precision, scale, precision, scale))) self._assert_decimal_result(cast, res, dec_max) # Test Min - res = Decimal(self.execute_scalar(cast.format(dec_min, precision, scale))) + res = Decimal(self.execute_scalar(\ + cast.format(dec_min, precision, scale, precision, scale))) self._assert_decimal_result(cast, res, dec_min) # Test zero - res = Decimal(self.execute_scalar(cast.format(TestDecimalCasting.DECIMAL_ZERO, - precision, scale))) + res = Decimal(self.execute_scalar(\ + cast.format(TestDecimalCasting.DECIMAL_ZERO, precision, scale, precision, scale))) self._assert_decimal_result(cast, res, TestDecimalCasting.DECIMAL_ZERO) # Test NULL null_cast = "select cast(NULL as Decimal({0}, {1}))".format(precision, scale) @@ -122,11 +132,12 @@ class TestDecimalCasting(ImpalaTestSuite): """Test to verify that an exact representation of the desired Decimal type is maintained.""" precision, scale = vector.get_value('decimal_type') - from_string = vector.get_value('cast_from') == 'string' + if vector.get_value('cast_from') == 'decimal': + pytest.skip("Casting between the same decimal type isn't interesting") for i in xrange(self.iterations): val = self._gen_decimal_val(precision, scale) - cast = self._normalize_cast_expr(val, scale, from_string=from_string)\ - .format(val, precision, scale) + cast = self._normalize_cast_expr(val, scale, vector.get_value('cast_from'))\ + .format(val, precision, scale, precision, scale) res = Decimal(self.execute_scalar(cast)) self._assert_decimal_result(cast, res, val) @@ -134,12 +145,12 @@ class TestDecimalCasting(ImpalaTestSuite): """Test to verify that we always return NULL when trying to cast a number with greater precision that its intended decimal type""" precision, scale = vector.get_value('decimal_type') - from_string = vector.get_value('cast_from') == 'string' for i in xrange(self.iterations): # Generate a decimal with a larger precision than the one we're casting to. - val = self._gen_decimal_val(randint(precision + 1, 39), scale) - cast = self._normalize_cast_expr(val, scale, from_string=from_string)\ - .format(val, precision, scale) + from_precision = randint(precision + 1, 39) + val = self._gen_decimal_val(from_precision, scale) + cast = self._normalize_cast_expr(val, scale, vector.get_value('cast_from'))\ + .format(val, precision, scale, min(38, from_precision), scale) res = self.execute_scalar(cast) self._assert_decimal_result(cast, res, 'NULL') @@ -148,16 +159,17 @@ class TestDecimalCasting(ImpalaTestSuite): than the target decimal type (with no change in precision). """ precision, scale = vector.get_value('decimal_type') - from_string = vector.get_value('cast_from') == 'string' + is_decimal_v2 = vector.get_value('exec_option')['decimal_v2'] == 'true' if precision == scale: pytest.skip("Cannot underflow scale when precision and scale are equal") for i in xrange(self.iterations): - new_scale = randint(scale + 1, precision) - val = self._gen_decimal_val(precision, randint(new_scale, precision)) - # We don't need to normalize the cast expr because scale will never be zero - cast = self._normalize_cast_expr(val, scale, from_string=from_string)\ - .format(val, precision, scale) - res = Decimal(self.execute_scalar(cast)) - # Truncate the decimal value to its target scale with quantize. - self._assert_decimal_result(cast, res, val.quantize(Decimal('0e-%s' % scale), - rounding=ROUND_DOWN)) + from_scale = randint(scale + 1, precision) + val = self._gen_decimal_val(precision, from_scale) + cast = self._normalize_cast_expr(val, scale, vector.get_value('cast_from'))\ + .format(val, precision, scale, precision, from_scale) + res = Decimal(self.execute_scalar(cast, vector.get_value('exec_option'))) + if is_decimal_v2: + expected_val = val.quantize(Decimal('0e-%s' % scale)) + else: + expected_val = val.quantize(Decimal('0e-%s' % scale), rounding=ROUND_DOWN) + self._assert_decimal_result(cast, res, expected_val)
