This is an automated email from the ASF dual-hosted git repository.
praveenbingo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 996235d ARROW-7300: [C++][Gandiva] Implement functions to cast from
strings to integers/floats
996235d is described below
commit 996235d081fc6397286871103c4cd5c5fde93279
Author: Projjal Chanda <[email protected]>
AuthorDate: Wed May 20 10:25:23 2020 +0530
ARROW-7300: [C++][Gandiva] Implement functions to cast from strings to
integers/floats
Closes #5947 from projjal/cast_from_string and squashes the following
commits:
bef542ee2 <Projjal Chanda> some fixes
e35c9d47c <Projjal Chanda> clang format
69d10c415 <Projjal Chanda> Added benchmark tests
bd6edff2f <Projjal Chanda> added helper parsing functions to execution
context
22a9e8133 <Projjal Chanda> implemented cast functions without holder
50654b53f <Projjal Chanda> Revert "Implemented cast from string to integers
and floats functions"
eb9a55ffa <Projjal Chanda> Implemented cast from string to integers and
floats functions
Authored-by: Projjal Chanda <[email protected]>
Signed-off-by: Praveen <[email protected]>
---
cpp/src/gandiva/context_helper.cc | 28 +++++
cpp/src/gandiva/execution_context.h | 14 +++
cpp/src/gandiva/function_registry_arithmetic.cc | 5 +
cpp/src/gandiva/gdv_function_stubs.h | 6 +
cpp/src/gandiva/precompiled/CMakeLists.txt | 3 +-
cpp/src/gandiva/precompiled/arithmetic_ops.cc | 35 ++++++
cpp/src/gandiva/precompiled/arithmetic_ops_test.cc | 129 +++++++++++++++++++++
cpp/src/gandiva/precompiled/types.h | 21 ++++
cpp/src/gandiva/tests/generate_data.h | 24 ++++
cpp/src/gandiva/tests/micro_benchmarks.cc | 46 ++++++++
10 files changed, 310 insertions(+), 1 deletion(-)
diff --git a/cpp/src/gandiva/context_helper.cc
b/cpp/src/gandiva/context_helper.cc
index 224bfd8..22c79c6 100644
--- a/cpp/src/gandiva/context_helper.cc
+++ b/cpp/src/gandiva/context_helper.cc
@@ -50,6 +50,22 @@ void ExportedContextFunctions::AddMappings(Engine* engine)
const {
engine->AddGlobalMappingForFunc("gdv_fn_context_arena_reset",
types->void_type(), args,
reinterpret_cast<void*>(gdv_fn_context_arena_reset));
+
+ args = {types->i64_type(), // int64_t context_ptr
+ types->i8_ptr_type(), // const char* data
+ types->i32_type(), // int32_t lenr
+ types->ptr_type(types->float_type())}; // float* val
+
+ engine->AddGlobalMappingForFunc("gdv_fn_context_parse_float32",
types->i1_type(), args,
+
reinterpret_cast<void*>(gdv_fn_context_parse_float32));
+
+ args = {types->i64_type(), // int64_t context_ptr
+ types->i8_ptr_type(), // const char* data
+ types->i32_type(), // int32_t lenr
+ types->ptr_type(types->double_type())}; // double* val
+
+ engine->AddGlobalMappingForFunc("gdv_fn_context_parse_float64",
types->i1_type(), args,
+
reinterpret_cast<void*>(gdv_fn_context_parse_float64));
}
} // namespace gandiva
@@ -73,4 +89,16 @@ void gdv_fn_context_arena_reset(int64_t context_ptr) {
auto context = reinterpret_cast<gandiva::ExecutionContext*>(context_ptr);
return context->arena()->Reset();
}
+
+bool gdv_fn_context_parse_float32(int64_t context_ptr, const char* data,
int32_t len,
+ float* val) {
+ auto context = reinterpret_cast<gandiva::ExecutionContext*>(context_ptr);
+ return context->parse_float(data, len, val);
+}
+
+bool gdv_fn_context_parse_float64(int64_t context_ptr, const char* data,
int32_t len,
+ double* val) {
+ auto context = reinterpret_cast<gandiva::ExecutionContext*>(context_ptr);
+ return context->parse_double(data, len, val);
+}
}
diff --git a/cpp/src/gandiva/execution_context.h
b/cpp/src/gandiva/execution_context.h
index efa5468..0844c08 100644
--- a/cpp/src/gandiva/execution_context.h
+++ b/cpp/src/gandiva/execution_context.h
@@ -19,6 +19,7 @@
#include <memory>
#include <string>
+#include "arrow/util/parsing.h"
#include "gandiva/simple_arena.h"
namespace gandiva {
@@ -46,9 +47,22 @@ class ExecutionContext {
arena_.Reset();
}
+ bool parse_float(const char* data, int32_t len, float* val) {
+ return float_converter_(data, len, val);
+ }
+
+ bool parse_double(const char* data, int32_t len, double* val) {
+ return double_converter_(data, len, val);
+ }
+
private:
std::string error_msg_;
SimpleArena arena_;
+
+ // Keeping float_converters as part of execution context since they have
non-trivial
+ // construction cost
+ arrow::internal::StringConverter<arrow::FloatType> float_converter_;
+ arrow::internal::StringConverter<arrow::DoubleType> double_converter_;
};
} // namespace gandiva
diff --git a/cpp/src/gandiva/function_registry_arithmetic.cc
b/cpp/src/gandiva/function_registry_arithmetic.cc
index dca273e..85b6648 100644
--- a/cpp/src/gandiva/function_registry_arithmetic.cc
+++ b/cpp/src/gandiva/function_registry_arithmetic.cc
@@ -64,6 +64,11 @@ std::vector<NativeFunction> GetArithmeticFunctionRegistry() {
UNARY_SAFE_NULL_IF_NULL(castDATE, {}, int32, date32),
UNARY_SAFE_NULL_IF_NULL(castDATE, {}, date32, date64),
+ UNARY_UNSAFE_NULL_IF_NULL(castINT, {}, utf8, int32),
+ UNARY_UNSAFE_NULL_IF_NULL(castBIGINT, {}, utf8, int64),
+ UNARY_UNSAFE_NULL_IF_NULL(castFLOAT4, {}, utf8, float32),
+ UNARY_UNSAFE_NULL_IF_NULL(castFLOAT8, {}, utf8, float64),
+
// add/sub/multiply/divide/mod
BINARY_SYMMETRIC_FN(add, {}), BINARY_SYMMETRIC_FN(subtract, {}),
BINARY_SYMMETRIC_FN(multiply, {}),
diff --git a/cpp/src/gandiva/gdv_function_stubs.h
b/cpp/src/gandiva/gdv_function_stubs.h
index 4d66aa3..0c7e31a 100644
--- a/cpp/src/gandiva/gdv_function_stubs.h
+++ b/cpp/src/gandiva/gdv_function_stubs.h
@@ -37,6 +37,12 @@ uint8_t* gdv_fn_context_arena_malloc(int64_t context_ptr,
int32_t data_len);
void gdv_fn_context_arena_reset(int64_t context_ptr);
+bool gdv_fn_context_parse_float32(int64_t context_ptr, const char* data,
int32_t len,
+ float* val);
+
+bool gdv_fn_context_parse_float64(int64_t context_ptr, const char* data,
int32_t len,
+ double* val);
+
bool in_expr_lookup_int32(int64_t ptr, int32_t value, bool in_validity);
bool in_expr_lookup_int64(int64_t ptr, int64_t value, bool in_validity);
diff --git a/cpp/src/gandiva/precompiled/CMakeLists.txt
b/cpp/src/gandiva/precompiled/CMakeLists.txt
index dff8517..b74b8fb 100644
--- a/cpp/src/gandiva/precompiled/CMakeLists.txt
+++ b/cpp/src/gandiva/precompiled/CMakeLists.txt
@@ -75,7 +75,8 @@ foreach(SRC_FILE ${PRECOMPILED_SRCS})
-o
${BC_FILE}
${ARROW_GANDIVA_PC_CXX_FLAGS}
- -I${CMAKE_SOURCE_DIR}/src)
+ -I${CMAKE_SOURCE_DIR}/src
+ -I${ARROW_BINARY_DIR}/src)
add_custom_command(OUTPUT ${BC_FILE} COMMAND ${PRECOMPILE_COMMAND} DEPENDS
${SRC_FILE})
list(APPEND BC_FILES ${BC_FILE})
endforeach()
diff --git a/cpp/src/gandiva/precompiled/arithmetic_ops.cc
b/cpp/src/gandiva/precompiled/arithmetic_ops.cc
index afeb2f9..443db24 100644
--- a/cpp/src/gandiva/precompiled/arithmetic_ops.cc
+++ b/cpp/src/gandiva/precompiled/arithmetic_ops.cc
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+#include "arrow/util/parsing.h"
+
extern "C" {
#include <math.h>
@@ -232,6 +234,37 @@ DIV(int64)
DIV_FLOAT(float32)
DIV_FLOAT(float64)
+#define CAST_INT_FROM_STRING(OUT_TYPE, ARROW_TYPE, TYPE_NAME)
\
+ FORCE_INLINE
\
+ gdv_##OUT_TYPE cast##TYPE_NAME##_utf8(int64_t context, const char* data,
\
+ int32_t len) {
\
+ gdv_##OUT_TYPE val;
\
+ arrow::internal::StringConverter<ARROW_TYPE> converter_;
\
+ if (!converter_(data, len, &val)) {
\
+ gdv_fn_context_set_error_msg(context,
\
+ "Failed parsing the string to required
format"); \
+ }
\
+ return val;
\
+ }
+
+CAST_INT_FROM_STRING(int32, arrow::Int32Type, INT)
+CAST_INT_FROM_STRING(int64, arrow::Int64Type, BIGINT)
+
+#define CAST_FLOAT_FROM_STRING(OUT_TYPE, ARROW_TYPE, TYPE_NAME)
\
+ FORCE_INLINE
\
+ gdv_##OUT_TYPE cast##TYPE_NAME##_utf8(int64_t context, const char* data,
\
+ int32_t len) {
\
+ gdv_##OUT_TYPE val;
\
+ if (!gdv_fn_context_parse_##OUT_TYPE(context, data, len, &val)) {
\
+ gdv_fn_context_set_error_msg(context,
\
+ "Failed parsing the string to required
format"); \
+ }
\
+ return val;
\
+ }
+
+CAST_FLOAT_FROM_STRING(float32, arrow::FloatType, FLOAT4)
+CAST_FLOAT_FROM_STRING(float64, arrow::DoubleType, FLOAT8)
+
#undef DIV_FLOAT
#undef DATE_FUNCTION
@@ -240,5 +273,7 @@ DIV_FLOAT(float64)
#undef NUMERIC_DATE_TYPES
#undef NUMERIC_FUNCTION
#undef NUMERIC_TYPES
+#undef CAST_INT_FROM_STRING
+#undef CAST_FLOAT_FROM_STRING
} // extern "C"
diff --git a/cpp/src/gandiva/precompiled/arithmetic_ops_test.cc
b/cpp/src/gandiva/precompiled/arithmetic_ops_test.cc
index a9f2c52..7b61b5b 100644
--- a/cpp/src/gandiva/precompiled/arithmetic_ops_test.cc
+++ b/cpp/src/gandiva/precompiled/arithmetic_ops_test.cc
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include "../execution_context.h"
#include "gandiva/precompiled/types.h"
@@ -100,4 +101,132 @@ TEST(TestArithmeticOps, TestDiv) {
context.Reset();
}
+TEST(TestArithmeticOps, TestCastINT) {
+ gandiva::ExecutionContext ctx;
+
+ int64_t ctx_ptr = reinterpret_cast<int64_t>(&ctx);
+
+ EXPECT_EQ(castINT_utf8(ctx_ptr, "-45", 3), -45);
+ EXPECT_EQ(castINT_utf8(ctx_ptr, "0", 1), 0);
+ EXPECT_EQ(castINT_utf8(ctx_ptr, "2147483647", 10), 2147483647);
+ EXPECT_EQ(castINT_utf8(ctx_ptr, "02147483647", 11), 2147483647);
+ EXPECT_EQ(castINT_utf8(ctx_ptr, "-2147483648", 11), -2147483648LL);
+ EXPECT_EQ(castINT_utf8(ctx_ptr, "-02147483648", 12), -2147483648LL);
+
+ castINT_utf8(ctx_ptr, "2147483648", 10);
+ EXPECT_THAT(ctx.get_error(),
+ ::testing::HasSubstr("Failed parsing the string to required
format"));
+ ctx.Reset();
+
+ castINT_utf8(ctx_ptr, "-2147483649", 11);
+ EXPECT_THAT(ctx.get_error(),
+ ::testing::HasSubstr("Failed parsing the string to required
format"));
+ ctx.Reset();
+
+ castINT_utf8(ctx_ptr, "12.34", 5);
+ EXPECT_THAT(ctx.get_error(),
+ ::testing::HasSubstr("Failed parsing the string to required
format"));
+ ctx.Reset();
+
+ castINT_utf8(ctx_ptr, "abc", 3);
+ EXPECT_THAT(ctx.get_error(),
+ ::testing::HasSubstr("Failed parsing the string to required
format"));
+ ctx.Reset();
+
+ castINT_utf8(ctx_ptr, "", 0);
+ EXPECT_THAT(ctx.get_error(),
+ ::testing::HasSubstr("Failed parsing the string to required
format"));
+ ctx.Reset();
+
+ castINT_utf8(ctx_ptr, "-", 1);
+ EXPECT_THAT(ctx.get_error(),
+ ::testing::HasSubstr("Failed parsing the string to required
format"));
+ ctx.Reset();
+}
+
+TEST(TestArithmeticOps, TestCastBIGINT) {
+ gandiva::ExecutionContext ctx;
+
+ int64_t ctx_ptr = reinterpret_cast<int64_t>(&ctx);
+
+ EXPECT_EQ(castBIGINT_utf8(ctx_ptr, "-45", 3), -45);
+ EXPECT_EQ(castBIGINT_utf8(ctx_ptr, "0", 1), 0);
+ EXPECT_EQ(castBIGINT_utf8(ctx_ptr, "9223372036854775807", 19),
9223372036854775807LL);
+ EXPECT_EQ(castBIGINT_utf8(ctx_ptr, "09223372036854775807", 20),
9223372036854775807LL);
+ EXPECT_EQ(castBIGINT_utf8(ctx_ptr, "-9223372036854775808", 20),
+ -9223372036854775807LL - 1);
+ EXPECT_EQ(castBIGINT_utf8(ctx_ptr, "-009223372036854775808", 22),
+ -9223372036854775807LL - 1);
+
+ castBIGINT_utf8(ctx_ptr, "9223372036854775808", 19);
+ EXPECT_THAT(ctx.get_error(),
+ ::testing::HasSubstr("Failed parsing the string to required
format"));
+ ctx.Reset();
+
+ castBIGINT_utf8(ctx_ptr, "-9223372036854775809", 20);
+ EXPECT_THAT(ctx.get_error(),
+ ::testing::HasSubstr("Failed parsing the string to required
format"));
+ ctx.Reset();
+
+ castBIGINT_utf8(ctx_ptr, "12.34", 5);
+ EXPECT_THAT(ctx.get_error(),
+ ::testing::HasSubstr("Failed parsing the string to required
format"));
+ ctx.Reset();
+
+ castBIGINT_utf8(ctx_ptr, "abc", 3);
+ EXPECT_THAT(ctx.get_error(),
+ ::testing::HasSubstr("Failed parsing the string to required
format"));
+ ctx.Reset();
+
+ castBIGINT_utf8(ctx_ptr, "", 0);
+ EXPECT_THAT(ctx.get_error(),
+ ::testing::HasSubstr("Failed parsing the string to required
format"));
+ ctx.Reset();
+
+ castBIGINT_utf8(ctx_ptr, "-", 1);
+ EXPECT_THAT(ctx.get_error(),
+ ::testing::HasSubstr("Failed parsing the string to required
format"));
+ ctx.Reset();
+}
+
+TEST(TestArithmeticOps, TestCastFloat4) {
+ gandiva::ExecutionContext ctx;
+
+ int64_t ctx_ptr = reinterpret_cast<int64_t>(&ctx);
+
+ EXPECT_EQ(castFLOAT4_utf8(ctx_ptr, "-45.34", 6), -45.34f);
+ EXPECT_EQ(castFLOAT4_utf8(ctx_ptr, "0", 1), 0.0f);
+ EXPECT_EQ(castFLOAT4_utf8(ctx_ptr, "5", 1), 5.0f);
+
+ castFLOAT4_utf8(ctx_ptr, "", 0);
+ EXPECT_THAT(ctx.get_error(),
+ ::testing::HasSubstr("Failed parsing the string to required
format"));
+ ctx.Reset();
+
+ castFLOAT4_utf8(ctx_ptr, "e", 1);
+ EXPECT_THAT(ctx.get_error(),
+ ::testing::HasSubstr("Failed parsing the string to required
format"));
+ ctx.Reset();
+}
+
+TEST(TestParseStringHolder, TestCastFloat8) {
+ gandiva::ExecutionContext ctx;
+
+ int64_t ctx_ptr = reinterpret_cast<int64_t>(&ctx);
+
+ EXPECT_EQ(castFLOAT8_utf8(ctx_ptr, "-45.34", 6), -45.34);
+ EXPECT_EQ(castFLOAT8_utf8(ctx_ptr, "0", 1), 0.0);
+ EXPECT_EQ(castFLOAT8_utf8(ctx_ptr, "5", 1), 5.0);
+
+ castFLOAT8_utf8(ctx_ptr, "", 0);
+ EXPECT_THAT(ctx.get_error(),
+ ::testing::HasSubstr("Failed parsing the string to required
format"));
+ ctx.Reset();
+
+ castFLOAT8_utf8(ctx_ptr, "e", 1);
+ EXPECT_THAT(ctx.get_error(),
+ ::testing::HasSubstr("Failed parsing the string to required
format"));
+ ctx.Reset();
+}
+
} // namespace gandiva
diff --git a/cpp/src/gandiva/precompiled/types.h
b/cpp/src/gandiva/precompiled/types.h
index 11529bf..09ee596 100644
--- a/cpp/src/gandiva/precompiled/types.h
+++ b/cpp/src/gandiva/precompiled/types.h
@@ -216,4 +216,25 @@ const char* replace_utf8_utf8_utf8(gdv_int64 context,
const char* text,
gdv_int32 text_len, const char* from_str,
gdv_int32 from_str_len, const char* to_str,
gdv_int32 to_str_len, gdv_int32* out_len);
+
+const char* castVARCHAR_int32_int64(int64_t context, int32_t value, int64_t
len,
+ int32_t* out_len);
+
+const char* castVARCHAR_int64_int64(int64_t context, int64_t value, int64_t
len,
+ int32_t* out_len);
+
+const char* castVARCHAR_float32_int64(int64_t context, float value, int64_t
len,
+ int32_t* out_len);
+
+const char* castVARCHAR_float64_int64(int64_t context, double value, int64_t
len,
+ int32_t* out_len);
+
+int32_t castINT_utf8(int64_t context, const char* data, int32_t len);
+
+int64_t castBIGINT_utf8(int64_t context, const char* data, int32_t len);
+
+float castFLOAT4_utf8(int64_t context, const char* data, int32_t len);
+
+double castFLOAT8_utf8(int64_t context, const char* data, int32_t len);
+
} // extern "C"
diff --git a/cpp/src/gandiva/tests/generate_data.h
b/cpp/src/gandiva/tests/generate_data.h
index bd7bbb8..519541f 100644
--- a/cpp/src/gandiva/tests/generate_data.h
+++ b/cpp/src/gandiva/tests/generate_data.h
@@ -129,4 +129,28 @@ class FastUtf8DataGenerator : public
DataGenerator<std::string> {
char cur_char_;
};
+class Utf8IntDataGenerator : public DataGenerator<std::string> {
+ public:
+ Utf8IntDataGenerator() {}
+
+ std::string GenerateData() { return std::to_string(random_.next()); }
+
+ private:
+ Random random_;
+};
+
+class Utf8FloatDataGenerator : public DataGenerator<std::string> {
+ public:
+ Utf8FloatDataGenerator() {}
+
+ std::string GenerateData() {
+ return std::to_string(
+ static_cast<float>(random_.next()) /
+ static_cast<float>(RAND_MAX / 100)); // random float between 0.0 to
100.0
+ }
+
+ private:
+ Random random_;
+};
+
} // namespace gandiva
diff --git a/cpp/src/gandiva/tests/micro_benchmarks.cc
b/cpp/src/gandiva/tests/micro_benchmarks.cc
index 6b4dddf..35c77e3 100644
--- a/cpp/src/gandiva/tests/micro_benchmarks.cc
+++ b/cpp/src/gandiva/tests/micro_benchmarks.cc
@@ -180,6 +180,50 @@ static void TimedTestFilterLike(benchmark::State& state) {
ASSERT_TRUE(status.ok());
}
+static void TimedTestCastFloatFromString(benchmark::State& state) {
+ auto field_a = field("a", utf8());
+ auto schema = arrow::schema({field_a});
+ auto pool = arrow::default_memory_pool();
+
+ auto field_result = field("res", arrow::float64());
+
+ auto node_a = TreeExprBuilder::MakeField(field_a);
+ auto fn = TreeExprBuilder::MakeFunction("castFLOAT8", {node_a},
arrow::float64());
+ auto expr = TreeExprBuilder::MakeExpression(fn, field_result);
+
+ std::shared_ptr<Projector> projector;
+ ASSERT_OK(Projector::Make(schema, {expr}, TestConfiguration(), &projector));
+
+ Utf8FloatDataGenerator data_generator;
+ ProjectEvaluator evaluator(projector);
+
+ Status status = TimedEvaluate<arrow::StringType, std::string>(
+ schema, evaluator, data_generator, pool, 1 * MILLION, 16 * THOUSAND,
state);
+ ASSERT_TRUE(status.ok());
+}
+
+static void TimedTestCastIntFromString(benchmark::State& state) {
+ auto field_a = field("a", utf8());
+ auto schema = arrow::schema({field_a});
+ auto pool = arrow::default_memory_pool();
+
+ auto field_result = field("res", int32());
+
+ auto node_a = TreeExprBuilder::MakeField(field_a);
+ auto fn = TreeExprBuilder::MakeFunction("castINT", {node_a}, int32());
+ auto expr = TreeExprBuilder::MakeExpression(fn, field_result);
+
+ std::shared_ptr<Projector> projector;
+ ASSERT_OK(Projector::Make(schema, {expr}, TestConfiguration(), &projector));
+
+ Utf8IntDataGenerator data_generator;
+ ProjectEvaluator evaluator(projector);
+
+ Status status = TimedEvaluate<arrow::StringType, std::string>(
+ schema, evaluator, data_generator, pool, 1 * MILLION, 16 * THOUSAND,
state);
+ ASSERT_TRUE(status.ok());
+}
+
static void TimedTestAllocs(benchmark::State& state) {
// schema for input fields
auto field_a = field("a", arrow::utf8());
@@ -395,6 +439,8 @@
BENCHMARK(TimedTestBigNested)->MinTime(1.0)->Unit(benchmark::kMicrosecond);
BENCHMARK(TimedTestExtractYear)->MinTime(1.0)->Unit(benchmark::kMicrosecond);
BENCHMARK(TimedTestFilterAdd2)->MinTime(1.0)->Unit(benchmark::kMicrosecond);
BENCHMARK(TimedTestFilterLike)->MinTime(1.0)->Unit(benchmark::kMicrosecond);
+BENCHMARK(TimedTestCastFloatFromString)->MinTime(1.0)->Unit(benchmark::kMicrosecond);
+BENCHMARK(TimedTestCastIntFromString)->MinTime(1.0)->Unit(benchmark::kMicrosecond);
BENCHMARK(TimedTestAllocs)->MinTime(1.0)->Unit(benchmark::kMicrosecond);
BENCHMARK(TimedTestMultiOr)->MinTime(1.0)->Unit(benchmark::kMicrosecond);
BENCHMARK(TimedTestInExpr)->MinTime(1.0)->Unit(benchmark::kMicrosecond);
