Repository: arrow Updated Branches: refs/heads/master bea5ab037 -> 243328931
ARROW-696: [C++] Support decimals in IPC and JSON reader/writer to enable integration tests Also adds `arrow::decimal` function and renames `decimal` namespace to `DecimalUtil`. The decimal array tests have been consolidated in array-test.cc. Fixed a signedness bug in HexEncode used in JSON write and pretty printing Author: Wes McKinney <[email protected]> Closes #1049 from wesm/ARROW-696 and squashes the following commits: 73b767cf [Wes McKinney] Support + test decimals in JSON reader/writer c8ca538e [Wes McKinney] Implement IPC roundtrip for decimals 9f7cef6b [Wes McKinney] Some decimal API refinements. Add failing IPC test for decimals Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/24332893 Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/24332893 Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/24332893 Branch: refs/heads/master Commit: 2433289314249ec8ad4b6fb503fde782372ac8df Parents: bea5ab0 Author: Wes McKinney <[email protected]> Authored: Tue Sep 5 17:21:08 2017 -0400 Committer: Wes McKinney <[email protected]> Committed: Tue Sep 5 17:21:08 2017 -0400 ---------------------------------------------------------------------- cpp/src/arrow/CMakeLists.txt | 1 - cpp/src/arrow/array-decimal-test.cc | 132 -------------------------- cpp/src/arrow/array-test.cc | 110 +++++++++++++++++++-- cpp/src/arrow/array.cc | 4 +- cpp/src/arrow/builder.cc | 2 +- cpp/src/arrow/builder.h | 11 +-- cpp/src/arrow/gpu/cuda_context.cc | 2 +- cpp/src/arrow/ipc/ipc-json-test.cc | 3 +- cpp/src/arrow/ipc/ipc-read-write-test.cc | 2 +- cpp/src/arrow/ipc/json-internal.cc | 42 +++++--- cpp/src/arrow/ipc/metadata-internal.cc | 13 ++- cpp/src/arrow/ipc/reader.cc | 2 - cpp/src/arrow/ipc/test-common.h | 35 ++++++- cpp/src/arrow/ipc/writer.cc | 1 + cpp/src/arrow/pretty_print.cc | 5 +- cpp/src/arrow/python/arrow_to_pandas.cc | 4 +- cpp/src/arrow/python/builtin_convert.cc | 4 +- cpp/src/arrow/python/helpers.cc | 2 +- cpp/src/arrow/python/pandas_to_arrow.cc | 4 +- cpp/src/arrow/status.h | 20 ++-- cpp/src/arrow/type.cc | 2 +- cpp/src/arrow/type.h | 37 +++++--- cpp/src/arrow/util/bit-util.cc | 4 +- cpp/src/arrow/util/bit-util.h | 5 +- cpp/src/arrow/util/decimal-test.cc | 20 ++-- cpp/src/arrow/util/decimal.cc | 4 +- cpp/src/arrow/util/decimal.h | 6 +- cpp/src/arrow/util/int128.cc | 2 - cpp/src/arrow/util/int128.h | 2 - cpp/src/arrow/util/string.h | 2 +- integration/integration_test.py | 2 +- 31 files changed, 256 insertions(+), 229 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 55fab2d..6963b11 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -50,7 +50,6 @@ install( ADD_ARROW_TEST(allocator-test) ADD_ARROW_TEST(array-test) -ADD_ARROW_TEST(array-decimal-test) ADD_ARROW_TEST(buffer-test) ADD_ARROW_TEST(memory_pool-test) ADD_ARROW_TEST(pretty_print-test) http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/array-decimal-test.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/array-decimal-test.cc b/cpp/src/arrow/array-decimal-test.cc deleted file mode 100644 index 5f3a4f1..0000000 --- a/cpp/src/arrow/array-decimal-test.cc +++ /dev/null @@ -1,132 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include <cstddef> -#include <cstdint> -#include <memory> -#include <vector> - -#include <gtest/gtest.h> - -#include "arrow/array.h" -#include "arrow/builder.h" -#include "arrow/test-util.h" -#include "arrow/util/decimal.h" -#include "arrow/util/int128.h" - -using std::size_t; - -namespace arrow { - -class Buffer; - -namespace decimal { - -using DecimalVector = std::vector<Int128>; - -class DecimalTest : public ::testing::TestWithParam<int> { - public: - DecimalTest() {} - - template <size_t BYTE_WIDTH = 16> - void MakeData(const DecimalVector& input, std::vector<uint8_t>* out) const { - out->reserve(input.size() * BYTE_WIDTH); - - std::array<uint8_t, BYTE_WIDTH> bytes{{0}}; - - for (const auto& value : input) { - ASSERT_OK(value.ToBytes(&bytes)); - out->insert(out->end(), bytes.cbegin(), bytes.cend()); - } - } - - template <size_t BYTE_WIDTH = 16> - void TestCreate(int precision, const DecimalVector& draw, - const std::vector<uint8_t>& valid_bytes, int64_t offset) const { - auto type = std::make_shared<DecimalType>(precision, 4); - - auto builder = std::make_shared<DecimalBuilder>(type); - - size_t null_count = 0; - - const size_t size = draw.size(); - - ASSERT_OK(builder->Reserve(size)); - - for (size_t i = 0; i < size; ++i) { - if (valid_bytes[i]) { - ASSERT_OK(builder->Append(draw[i])); - } else { - ASSERT_OK(builder->AppendNull()); - ++null_count; - } - } - - std::shared_ptr<Array> out; - ASSERT_OK(builder->Finish(&out)); - - std::vector<uint8_t> raw_bytes; - - raw_bytes.reserve(size * BYTE_WIDTH); - MakeData<BYTE_WIDTH>(draw, &raw_bytes); - - auto expected_data = std::make_shared<Buffer>(raw_bytes.data(), BYTE_WIDTH); - std::shared_ptr<Buffer> expected_null_bitmap; - ASSERT_OK(BitUtil::BytesToBits(valid_bytes, &expected_null_bitmap)); - - int64_t expected_null_count = test::null_count(valid_bytes); - auto expected = std::make_shared<DecimalArray>( - type, size, expected_data, expected_null_bitmap, expected_null_count); - - std::shared_ptr<Array> lhs = out->Slice(offset); - std::shared_ptr<Array> rhs = expected->Slice(offset); - bool result = lhs->Equals(rhs); - ASSERT_TRUE(result); - } -}; - -TEST_P(DecimalTest, NoNulls) { - int precision = GetParam(); - std::vector<Int128> draw = {Int128(1), Int128(-2), Int128(2389), Int128(4), - Int128(-12348)}; - std::vector<uint8_t> valid_bytes = {true, true, true, true, true}; - this->TestCreate(precision, draw, valid_bytes, 0); - this->TestCreate(precision, draw, valid_bytes, 2); -} - -TEST_P(DecimalTest, WithNulls) { - int precision = GetParam(); - std::vector<Int128> draw = {Int128(1), Int128(2), Int128(-1), Int128(4), - Int128(-1), Int128(1), Int128(2)}; - Int128 big; - ASSERT_OK(FromString("230342903942.234234", &big)); - draw.push_back(big); - - Int128 big_negative; - ASSERT_OK(FromString("-23049302932.235234", &big_negative)); - draw.push_back(big_negative); - - std::vector<uint8_t> valid_bytes = {true, true, false, true, false, - true, true, true, true}; - this->TestCreate(precision, draw, valid_bytes, 0); - this->TestCreate(precision, draw, valid_bytes, 2); -} - -INSTANTIATE_TEST_CASE_P(DecimalTest, DecimalTest, ::testing::Range(1, 38)); - -} // namespace decimal -} // namespace arrow http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/array-test.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc index 3a3d28c..a73a73e 100644 --- a/cpp/src/arrow/array-test.cc +++ b/cpp/src/arrow/array-test.cc @@ -33,6 +33,8 @@ #include "arrow/test-util.h" #include "arrow/type.h" #include "arrow/type_traits.h" +#include "arrow/util/decimal.h" +#include "arrow/util/int128.h" namespace arrow { @@ -69,7 +71,7 @@ Status MakeArrayFromValidBytes(const vector<uint8_t>& v, MemoryPool* pool, int64_t null_count = v.size() - std::accumulate(v.begin(), v.end(), 0); std::shared_ptr<Buffer> null_buf; - RETURN_NOT_OK(BitUtil::BytesToBits(v, &null_buf)); + RETURN_NOT_OK(BitUtil::BytesToBits(v, default_memory_pool(), &null_buf)); TypedBufferBuilder<int32_t> value_builder(pool); for (size_t i = 0; i < v.size(); ++i) { @@ -157,7 +159,7 @@ TEST_F(TestArray, TestIsNull) { } std::shared_ptr<Buffer> null_buf; - ASSERT_OK(BitUtil::BytesToBits(null_bitmap, &null_buf)); + ASSERT_OK(BitUtil::BytesToBits(null_bitmap, default_memory_pool(), &null_buf)); std::unique_ptr<Array> arr; arr.reset(new Int32Array(null_bitmap.size(), nullptr, null_buf, null_count)); @@ -236,7 +238,8 @@ class TestPrimitiveBuilder : public TestBuilder { int64_t ex_null_count = 0; if (nullable) { - ASSERT_OK(BitUtil::BytesToBits(valid_bytes_, &ex_null_bitmap)); + ASSERT_OK( + BitUtil::BytesToBits(valid_bytes_, default_memory_pool(), &ex_null_bitmap)); ex_null_count = test::null_count(valid_bytes_); } else { ex_null_bitmap = nullptr; @@ -327,13 +330,13 @@ void TestPrimitiveBuilder<PBoolean>::Check(const std::unique_ptr<BooleanBuilder> int64_t size = builder->length(); std::shared_ptr<Buffer> ex_data; - ASSERT_OK(BitUtil::BytesToBits(draws_, &ex_data)); + ASSERT_OK(BitUtil::BytesToBits(draws_, default_memory_pool(), &ex_data)); std::shared_ptr<Buffer> ex_null_bitmap; int64_t ex_null_count = 0; if (nullable) { - ASSERT_OK(BitUtil::BytesToBits(valid_bytes_, &ex_null_bitmap)); + ASSERT_OK(BitUtil::BytesToBits(valid_bytes_, default_memory_pool(), &ex_null_bitmap)); ex_null_count = test::null_count(valid_bytes_); } else { ex_null_bitmap = nullptr; @@ -753,7 +756,7 @@ class TestStringArray : public ::testing::Test { length_ = static_cast<int64_t>(offsets_.size()) - 1; value_buf_ = test::GetBufferFromVector(chars_); offsets_buf_ = test::GetBufferFromVector(offsets_); - ASSERT_OK(BitUtil::BytesToBits(valid_bytes_, &null_bitmap_)); + ASSERT_OK(BitUtil::BytesToBits(valid_bytes_, default_memory_pool(), &null_bitmap_)); null_count_ = test::null_count(valid_bytes_); strings_ = std::make_shared<StringArray>(length_, offsets_buf_, value_buf_, @@ -967,7 +970,7 @@ class TestBinaryArray : public ::testing::Test { value_buf_ = test::GetBufferFromVector(chars_); offsets_buf_ = test::GetBufferFromVector(offsets_); - ASSERT_OK(BitUtil::BytesToBits(valid_bytes_, &null_bitmap_)); + ASSERT_OK(BitUtil::BytesToBits(valid_bytes_, default_memory_pool(), &null_bitmap_)); null_count_ = test::null_count(valid_bytes_); strings_ = std::make_shared<BinaryArray>(length_, offsets_buf_, value_buf_, @@ -2464,4 +2467,97 @@ TEST(TestUnionArrayAdHoc, TestSliceEquals) { CheckUnion(batch->column(2)); } +using DecimalVector = std::vector<Int128>; + +class DecimalTest : public ::testing::TestWithParam<int> { + public: + DecimalTest() {} + + template <size_t BYTE_WIDTH = 16> + void MakeData(const DecimalVector& input, std::vector<uint8_t>* out) const { + out->reserve(input.size() * BYTE_WIDTH); + + std::array<uint8_t, BYTE_WIDTH> bytes{{0}}; + + for (const auto& value : input) { + ASSERT_OK(value.ToBytes(&bytes)); + out->insert(out->end(), bytes.cbegin(), bytes.cend()); + } + } + + template <size_t BYTE_WIDTH = 16> + void TestCreate(int32_t precision, const DecimalVector& draw, + const std::vector<uint8_t>& valid_bytes, int64_t offset) const { + auto type = std::make_shared<DecimalType>(precision, 4); + + auto builder = std::make_shared<DecimalBuilder>(type); + + size_t null_count = 0; + + const size_t size = draw.size(); + + ASSERT_OK(builder->Reserve(size)); + + for (size_t i = 0; i < size; ++i) { + if (valid_bytes[i]) { + ASSERT_OK(builder->Append(draw[i])); + } else { + ASSERT_OK(builder->AppendNull()); + ++null_count; + } + } + + std::shared_ptr<Array> out; + ASSERT_OK(builder->Finish(&out)); + + std::vector<uint8_t> raw_bytes; + + raw_bytes.reserve(size * BYTE_WIDTH); + MakeData<BYTE_WIDTH>(draw, &raw_bytes); + + auto expected_data = std::make_shared<Buffer>(raw_bytes.data(), BYTE_WIDTH); + std::shared_ptr<Buffer> expected_null_bitmap; + ASSERT_OK( + BitUtil::BytesToBits(valid_bytes, default_memory_pool(), &expected_null_bitmap)); + + int64_t expected_null_count = test::null_count(valid_bytes); + auto expected = std::make_shared<DecimalArray>( + type, size, expected_data, expected_null_bitmap, expected_null_count); + + std::shared_ptr<Array> lhs = out->Slice(offset); + std::shared_ptr<Array> rhs = expected->Slice(offset); + bool result = lhs->Equals(rhs); + ASSERT_TRUE(result); + } +}; + +TEST_P(DecimalTest, NoNulls) { + int32_t precision = GetParam(); + std::vector<Int128> draw = {Int128(1), Int128(-2), Int128(2389), Int128(4), + Int128(-12348)}; + std::vector<uint8_t> valid_bytes = {true, true, true, true, true}; + this->TestCreate(precision, draw, valid_bytes, 0); + this->TestCreate(precision, draw, valid_bytes, 2); +} + +TEST_P(DecimalTest, WithNulls) { + int32_t precision = GetParam(); + std::vector<Int128> draw = {Int128(1), Int128(2), Int128(-1), Int128(4), + Int128(-1), Int128(1), Int128(2)}; + Int128 big; + ASSERT_OK(DecimalUtil::FromString("230342903942.234234", &big)); + draw.push_back(big); + + Int128 big_negative; + ASSERT_OK(DecimalUtil::FromString("-23049302932.235234", &big_negative)); + draw.push_back(big_negative); + + std::vector<uint8_t> valid_bytes = {true, true, false, true, false, + true, true, true, true}; + this->TestCreate(precision, draw, valid_bytes, 0); + this->TestCreate(precision, draw, valid_bytes, 2); +} + +INSTANTIATE_TEST_CASE_P(DecimalTest, DecimalTest, ::testing::Range(1, 38)); + } // namespace arrow http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/array.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc index babf326..34f0868 100644 --- a/cpp/src/arrow/array.cc +++ b/cpp/src/arrow/array.cc @@ -319,8 +319,8 @@ DecimalArray::DecimalArray(const std::shared_ptr<internal::ArrayData>& data) std::string DecimalArray::FormatValue(int64_t i) const { const auto& type_ = static_cast<const DecimalType&>(*type()); - decimal::Int128 value(GetValue(i)); - return decimal::ToString(value, type_.precision(), type_.scale()); + Int128 value(GetValue(i)); + return DecimalUtil::ToString(value, type_.precision(), type_.scale()); } // ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/builder.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc index 24d26d4..0129dc8 100644 --- a/cpp/src/arrow/builder.cc +++ b/cpp/src/arrow/builder.cc @@ -1032,7 +1032,7 @@ DecimalBuilder::DecimalBuilder(MemoryPool* pool, const std::shared_ptr<DataType> : DecimalBuilder(type, pool) {} #endif -Status DecimalBuilder::Append(const decimal::Int128& value) { +Status DecimalBuilder::Append(const Int128& value) { RETURN_NOT_OK(FixedSizeBinaryBuilder::Reserve(1)); std::array<uint8_t, 16> bytes; RETURN_NOT_OK(value.ToBytes(&bytes)); http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/builder.h ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h index a649d62..a99e682 100644 --- a/cpp/src/arrow/builder.h +++ b/cpp/src/arrow/builder.h @@ -38,6 +38,7 @@ namespace arrow { class Array; +class Int128; namespace internal { @@ -45,12 +46,6 @@ struct ArrayData; } // namespace internal -namespace decimal { - -class Int128; - -} // namespace decimal - static constexpr int64_t kMinBuilderCapacity = 1 << 5; /// Base class for all data array builders. @@ -747,7 +742,9 @@ class ARROW_EXPORT DecimalBuilder : public FixedSizeBinaryBuilder { explicit DecimalBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type); #endif - Status Append(const decimal::Int128& val); + using FixedSizeBinaryBuilder::Append; + + Status Append(const Int128& val); Status Finish(std::shared_ptr<Array>* out) override; }; http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/gpu/cuda_context.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/gpu/cuda_context.cc b/cpp/src/arrow/gpu/cuda_context.cc index 430ecab..42315cd 100644 --- a/cpp/src/arrow/gpu/cuda_context.cc +++ b/cpp/src/arrow/gpu/cuda_context.cc @@ -185,7 +185,7 @@ class CudaDeviceManager::CudaDeviceManagerImpl { // device_number -> CudaContext std::unordered_map<int, std::shared_ptr<CudaContext>> contexts_; - int host_bytes_allocated_; + int64_t host_bytes_allocated_; }; CudaDeviceManager::CudaDeviceManager() { impl_.reset(new CudaDeviceManagerImpl()); } http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/ipc/ipc-json-test.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/ipc/ipc-json-test.cc b/cpp/src/arrow/ipc/ipc-json-test.cc index 1967e24..7855aea 100644 --- a/cpp/src/arrow/ipc/ipc-json-test.cc +++ b/cpp/src/arrow/ipc/ipc-json-test.cc @@ -376,7 +376,8 @@ TEST(TestJsonFileReadWrite, MinimalFormatExample) { ::testing::Values(&MakeIntRecordBatch, &MakeListRecordBatch, &MakeNonNullRecordBatch, \ &MakeZeroLengthRecordBatch, &MakeDeeplyNestedList, \ &MakeStringTypesRecordBatch, &MakeStruct, &MakeUnion, &MakeDates, \ - &MakeTimestamps, &MakeTimes, &MakeFWBinary, &MakeDictionary); + &MakeTimestamps, &MakeTimes, &MakeFWBinary, &MakeDecimal, \ + &MakeDictionary); class TestJsonRoundTrip : public ::testing::TestWithParam<MakeRecordBatch*> { public: http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/ipc/ipc-read-write-test.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc index 94d8eda..ac4d36e 100644 --- a/cpp/src/arrow/ipc/ipc-read-write-test.cc +++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc @@ -121,7 +121,7 @@ TEST_F(TestSchemaMetadata, NestedFields) { &MakeZeroLengthRecordBatch, &MakeDeeplyNestedList, \ &MakeStringTypesRecordBatch, &MakeStruct, &MakeUnion, \ &MakeDictionary, &MakeDates, &MakeTimestamps, &MakeTimes, \ - &MakeFWBinary, &MakeBooleanBatch); + &MakeFWBinary, &MakeDecimal, &MakeBooleanBatch); static int g_file_number = 0; http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/ipc/json-internal.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc index f0fb963..025f6c2 100644 --- a/cpp/src/arrow/ipc/json-internal.cc +++ b/cpp/src/arrow/ipc/json-internal.cc @@ -346,6 +346,7 @@ class SchemaWriter { return WritePrimitive("fixedsizebinary", type); } + Status Visit(const DecimalType& type) { return WritePrimitive("decimal", type); } Status Visit(const TimestampType& type) { return WritePrimitive("timestamp", type); } Status Visit(const IntervalType& type) { return WritePrimitive("interval", type); } @@ -364,8 +365,6 @@ class SchemaWriter { return Status::OK(); } - Status Visit(const DecimalType& type) { return Status::NotImplemented("decimal"); } - Status Visit(const DictionaryType& type) { return VisitType(*type.dictionary()->type()); } @@ -437,10 +436,11 @@ class ArrayWriter { WriteDataValues(const T& arr) { for (int64_t i = 0; i < arr.length(); ++i) { int32_t length; - const char* buf = reinterpret_cast<const char*>(arr.GetValue(i, &length)); + const uint8_t* buf = arr.GetValue(i, &length); if (std::is_base_of<StringArray, T>::value) { - writer_->String(buf, length); + // Presumed UTF-8 + writer_->String(reinterpret_cast<const char*>(buf), length); } else { writer_->String(HexEncode(buf, length)); } @@ -450,8 +450,9 @@ class ArrayWriter { void WriteDataValues(const FixedSizeBinaryArray& arr) { int32_t width = arr.byte_width(); for (int64_t i = 0; i < arr.length(); ++i) { - const char* buf = reinterpret_cast<const char*>(arr.GetValue(i)); - writer_->String(HexEncode(buf, width)); + const uint8_t* buf = arr.GetValue(i); + std::string encoded = HexEncode(buf, width); + writer_->String(encoded); } } @@ -535,8 +536,6 @@ class ArrayWriter { return Status::OK(); } - Status Visit(const DecimalArray& array) { return Status::NotImplemented("decimal"); } - Status Visit(const DictionaryArray& array) { return VisitArrayValues(*array.indices()); } @@ -666,6 +665,17 @@ static Status GetFixedSizeBinary(const RjObject& json_type, return Status::OK(); } +static Status GetDecimal(const RjObject& json_type, std::shared_ptr<DataType>* type) { + const auto& it_precision = json_type.FindMember("precision"); + const auto& it_scale = json_type.FindMember("scale"); + + RETURN_NOT_INT("precision", it_precision, json_type); + RETURN_NOT_INT("scale", it_scale, json_type); + + *type = decimal(it_precision->value.GetInt(), it_scale->value.GetInt()); + return Status::OK(); +} + static Status GetDate(const RjObject& json_type, std::shared_ptr<DataType>* type) { const auto& it_unit = json_type.FindMember("unit"); RETURN_NOT_STRING("unit", it_unit, json_type); @@ -802,6 +812,8 @@ static Status GetType(const RjObject& json_type, *type = binary(); } else if (type_name == "fixedsizebinary") { return GetFixedSizeBinary(json_type, type); + } else if (type_name == "decimal") { + return GetDecimal(json_type, type); } else if (type_name == "null") { *type = null(); } else if (type_name == "date") { @@ -817,8 +829,12 @@ static Status GetType(const RjObject& json_type, *type = list(children[0]); } else if (type_name == "struct") { *type = struct_(children); - } else { + } else if (type_name == "union") { return GetUnion(json_type, children, type); + } else { + std::stringstream ss; + ss << "Unrecognized type name: " << type_name; + return Status::Invalid(ss.str()); } return Status::OK(); } @@ -1039,7 +1055,7 @@ class ArrayReader { template <typename T> typename std::enable_if<std::is_base_of<FixedSizeBinaryType, T>::value, Status>::type Visit(const T& type) { - FixedSizeBinaryBuilder builder(type_, pool_); + typename TypeTraits<T>::BuilderType builder(type_, pool_); const auto& json_data = obj_->FindMember("DATA"); RETURN_NOT_ARRAY("DATA", json_data, *obj_); @@ -1063,8 +1079,10 @@ class ArrayReader { const rj::Value& val = json_data_arr[i]; DCHECK(val.IsString()); std::string hex_string = val.GetString(); - DCHECK_EQ(static_cast<int32_t>(hex_string.size()), byte_width * 2) - << "Expected size: " << byte_width * 2 << " got: " << hex_string.size(); + if (static_cast<int32_t>(hex_string.size()) != byte_width * 2) { + DCHECK(false) << "Expected size: " << byte_width * 2 + << " got: " << hex_string.size(); + } const char* hex_data = hex_string.c_str(); for (int32_t j = 0; j < byte_width; ++j) { http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/ipc/metadata-internal.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc index 1f2340d..962fe79 100644 --- a/cpp/src/arrow/ipc/metadata-internal.cc +++ b/cpp/src/arrow/ipc/metadata-internal.cc @@ -248,8 +248,11 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data, case flatbuf::Type_Bool: *out = boolean(); return Status::OK(); - case flatbuf::Type_Decimal: - return Status::NotImplemented("Decimal"); + case flatbuf::Type_Decimal: { + auto dec_type = static_cast<const flatbuf::Decimal*>(type_data); + *out = decimal(dec_type->precision(), dec_type->scale()); + return Status::OK(); + } case flatbuf::Type_Date: { auto date_type = static_cast<const flatbuf::Date*>(type_data); if (date_type->unit() == flatbuf::DateUnit_DAY) { @@ -420,6 +423,12 @@ static Status TypeToFlatbuffer(FBB& fbb, const DataType& type, } *offset = flatbuf::CreateTimestamp(fbb, fb_unit, fb_timezone).Union(); } break; + case Type::DECIMAL: { + const auto& dec_type = static_cast<const DecimalType&>(*value_type); + *out_type = flatbuf::Type_Decimal; + *offset = + flatbuf::CreateDecimal(fbb, dec_type.precision(), dec_type.scale()).Union(); + } break; case Type::LIST: *out_type = flatbuf::Type_List; return ListToFlatbuffer(fbb, *value_type, children, dictionary_memo, offset); http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/ipc/reader.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc index a4cac0b..919dd7a 100644 --- a/cpp/src/arrow/ipc/reader.cc +++ b/cpp/src/arrow/ipc/reader.cc @@ -189,8 +189,6 @@ class ArrayLoader { Status Visit(const NullType& type) { return Status::NotImplemented("null"); } - Status Visit(const DecimalType& type) { return Status::NotImplemented("decimal"); } - template <typename T> typename std::enable_if<std::is_base_of<FixedWidthType, T>::value && !std::is_base_of<FixedSizeBinaryType, T>::value && http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/ipc/test-common.h ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h index c1e79d4..ae36b5f 100644 --- a/cpp/src/arrow/ipc/test-common.h +++ b/cpp/src/arrow/ipc/test-common.h @@ -161,12 +161,12 @@ Status MakeRandomBooleanArray(const int length, bool include_nulls, std::vector<uint8_t> values(length); test::random_null_bytes(length, 0.5, values.data()); std::shared_ptr<Buffer> data; - RETURN_NOT_OK(BitUtil::BytesToBits(values, &data)); + RETURN_NOT_OK(BitUtil::BytesToBits(values, default_memory_pool(), &data)); if (include_nulls) { std::vector<uint8_t> valid_bytes(length); std::shared_ptr<Buffer> null_bitmap; - RETURN_NOT_OK(BitUtil::BytesToBits(valid_bytes, &null_bitmap)); + RETURN_NOT_OK(BitUtil::BytesToBits(valid_bytes, default_memory_pool(), &null_bitmap)); test::random_null_bytes(length, 0.1, valid_bytes.data()); *out = std::make_shared<BooleanArray>(length, data, null_bitmap, -1); } else { @@ -371,7 +371,7 @@ Status MakeStruct(std::shared_ptr<RecordBatch>* out) { std::vector<uint8_t> null_bytes(list_batch->num_rows(), 1); null_bytes[0] = 0; std::shared_ptr<Buffer> null_bitmask; - RETURN_NOT_OK(BitUtil::BytesToBits(null_bytes, &null_bitmask)); + RETURN_NOT_OK(BitUtil::BytesToBits(null_bytes, default_memory_pool(), &null_bitmask)); std::shared_ptr<Array> with_nulls( new StructArray(type, list_batch->num_rows(), columns, null_bitmask, 1)); @@ -431,7 +431,7 @@ Status MakeUnion(std::shared_ptr<RecordBatch>* out) { std::vector<uint8_t> null_bytes(length, 1); null_bytes[2] = 0; std::shared_ptr<Buffer> null_bitmask; - RETURN_NOT_OK(BitUtil::BytesToBits(null_bytes, &null_bitmask)); + RETURN_NOT_OK(BitUtil::BytesToBits(null_bytes, default_memory_pool(), &null_bitmask)); // construct individual nullable/non-nullable struct arrays auto sparse_no_nulls = @@ -670,6 +670,33 @@ Status MakeFWBinary(std::shared_ptr<RecordBatch>* out) { return Status::OK(); } +Status MakeDecimal(std::shared_ptr<RecordBatch>* out) { + auto f0 = field("f0", decimal(19, 4)); + auto schema = ::arrow::schema({f0, f0}); + + constexpr int kDecimalSize = 16; + constexpr int length = 10; + + std::shared_ptr<Buffer> data, is_valid; + std::vector<uint8_t> is_valid_bytes(length); + + RETURN_NOT_OK(AllocateBuffer(default_memory_pool(), kDecimalSize * length, &data)); + + test::random_bytes(kDecimalSize * length, 0, data->mutable_data()); + test::random_null_bytes(length, 0.1, is_valid_bytes.data()); + + RETURN_NOT_OK(BitUtil::BytesToBits(is_valid_bytes, default_memory_pool(), &is_valid)); + + auto a1 = std::make_shared<DecimalArray>(f0->type(), length, data, is_valid, + kUnknownNullCount); + + auto a2 = std::make_shared<DecimalArray>(f0->type(), length, data); + + ArrayVector arrays = {a1, a2}; + *out = std::make_shared<RecordBatch>(schema, a1->length(), arrays); + return Status::OK(); +} + } // namespace ipc } // namespace arrow http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/ipc/writer.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc index 7ac6380..3f3d168 100644 --- a/cpp/src/arrow/ipc/writer.cc +++ b/cpp/src/arrow/ipc/writer.cc @@ -341,6 +341,7 @@ class RecordBatchSerializer : public ArrayVisitor { VISIT_FIXED_WIDTH(Time32Array); VISIT_FIXED_WIDTH(Time64Array); VISIT_FIXED_WIDTH(FixedSizeBinaryArray); + VISIT_FIXED_WIDTH(DecimalArray); #undef VISIT_FIXED_WIDTH http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/pretty_print.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc index aedad12..7794f8d 100644 --- a/cpp/src/arrow/pretty_print.cc +++ b/cpp/src/arrow/pretty_print.cc @@ -99,7 +99,7 @@ class ArrayPrinter { if (array.IsNull(i)) { Write("null"); } else { - const char* buf = reinterpret_cast<const char*>(array.GetValue(i, &length)); + const uint8_t* buf = array.GetValue(i, &length); (*sink_) << HexEncode(buf, length); } } @@ -116,8 +116,7 @@ class ArrayPrinter { if (array.IsNull(i)) { Write("null"); } else { - const char* buf = reinterpret_cast<const char*>(array.GetValue(i)); - (*sink_) << HexEncode(buf, width); + (*sink_) << HexEncode(array.GetValue(i), width); } } } http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/python/arrow_to_pandas.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc index d2d6fb4..769f609 100644 --- a/cpp/src/arrow/python/arrow_to_pandas.cc +++ b/cpp/src/arrow/python/arrow_to_pandas.cc @@ -606,8 +606,8 @@ static Status ConvertTimes(PandasOptions options, const ChunkedArray& data, static Status RawDecimalToString(const uint8_t* bytes, int precision, int scale, std::string* result) { DCHECK_NE(result, nullptr); - decimal::Int128 decimal(bytes); - *result = decimal::ToString(decimal, precision, scale); + Int128 decimal(bytes); + *result = DecimalUtil::ToString(decimal, precision, scale); return Status::OK(); } http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/python/builtin_convert.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc index dca3f45..eb17f4e 100644 --- a/cpp/src/arrow/python/builtin_convert.cc +++ b/cpp/src/arrow/python/builtin_convert.cc @@ -589,8 +589,8 @@ class DecimalConverter std::string string; RETURN_NOT_OK(PythonDecimalToString(item.obj(), &string)); - decimal::Int128 value; - RETURN_NOT_OK(decimal::FromString(string, &value)); + Int128 value; + RETURN_NOT_OK(DecimalUtil::FromString(string, &value)); return typed_builder_->Append(value); } http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/python/helpers.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/python/helpers.cc b/cpp/src/arrow/python/helpers.cc index 67f0655..deda93f 100644 --- a/cpp/src/arrow/python/helpers.cc +++ b/cpp/src/arrow/python/helpers.cc @@ -103,7 +103,7 @@ Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int* precision, auto size = str.size; std::string c_string(bytes, size); - return decimal::FromString(c_string, nullptr, precision, scale); + return DecimalUtil::FromString(c_string, nullptr, precision, scale); } Status DecimalFromString(PyObject* decimal_constructor, const std::string& decimal_string, http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/python/pandas_to_arrow.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/python/pandas_to_arrow.cc b/cpp/src/arrow/python/pandas_to_arrow.cc index fe8ae80..8410381 100644 --- a/cpp/src/arrow/python/pandas_to_arrow.cc +++ b/cpp/src/arrow/python/pandas_to_arrow.cc @@ -634,8 +634,8 @@ Status PandasConverter::ConvertDecimals() { std::string string; RETURN_NOT_OK(PythonDecimalToString(object, &string)); - decimal::Int128 value; - RETURN_NOT_OK(decimal::FromString(string, &value)); + Int128 value; + RETURN_NOT_OK(DecimalUtil::FromString(string, &value)); RETURN_NOT_OK(builder.Append(value)); } else if (PandasObjectIsNull(object)) { RETURN_NOT_OK(builder.AppendNull()); http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/status.h ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/status.h b/cpp/src/arrow/status.h index 0cb0fb1..80285da 100644 --- a/cpp/src/arrow/status.h +++ b/cpp/src/arrow/status.h @@ -51,16 +51,14 @@ namespace arrow { #ifdef ARROW_EXTRA_ERROR_CONTEXT -#define RETURN_NOT_OK(s) \ - do { \ - Status _s = (s); \ - if (ARROW_PREDICT_FALSE(!_s.ok())) { \ - std::stringstream ss; \ - ss << __FILE__ << ":" << __LINE__ \ - << " code: " << #s \ - << "\n" << _s.message(); \ - return Status(_s.code(), ss.str()); \ - } \ +#define RETURN_NOT_OK(s) \ + do { \ + Status _s = (s); \ + if (ARROW_PREDICT_FALSE(!_s.ok())) { \ + std::stringstream ss; \ + ss << __FILE__ << ":" << __LINE__ << " code: " << #s << "\n" << _s.message(); \ + return Status(_s.code(), ss.str()); \ + } \ } while (0) #else @@ -73,7 +71,7 @@ namespace arrow { } \ } while (0) -#endif // ARROW_EXTRA_ERROR_CONTEXT +#endif // ARROW_EXTRA_ERROR_CONTEXT #define RETURN_NOT_OK_ELSE(s, else_) \ do { \ http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/type.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc index b7963b8..87bb73b 100644 --- a/cpp/src/arrow/type.cc +++ b/cpp/src/arrow/type.cc @@ -455,7 +455,7 @@ std::shared_ptr<Field> field(const std::string& name, return std::make_shared<Field>(name, type, nullable, metadata); } -std::shared_ptr<DataType> decimal(int precision, int scale) { +std::shared_ptr<DataType> decimal(int32_t precision, int32_t scale) { return std::make_shared<DecimalType>(precision, scale); } http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/type.h ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h index 2a89423..d197817 100644 --- a/cpp/src/arrow/type.h +++ b/cpp/src/arrow/type.h @@ -461,19 +461,19 @@ class ARROW_EXPORT DecimalType : public FixedSizeBinaryType { public: static constexpr Type::type type_id = Type::DECIMAL; - explicit DecimalType(int precision, int scale) + explicit DecimalType(int32_t precision, int32_t scale) : FixedSizeBinaryType(16, Type::DECIMAL), precision_(precision), scale_(scale) {} Status Accept(TypeVisitor* visitor) const override; std::string ToString() const override; static std::string name() { return "decimal"; } - int precision() const { return precision_; } - int scale() const { return scale_; } + int32_t precision() const { return precision_; } + int32_t scale() const { return scale_; } private: - int precision_; - int scale_; + int32_t precision_; + int32_t scale_; }; enum class UnionMode : char { SPARSE, DENSE }; @@ -753,14 +753,29 @@ class ARROW_EXPORT Schema { // ---------------------------------------------------------------------- // Factory functions -std::shared_ptr<DataType> ARROW_EXPORT fixed_size_binary(int32_t byte_width); +/// \brief Make an instance of FixedSizeBinaryType +ARROW_EXPORT +std::shared_ptr<DataType> fixed_size_binary(int32_t byte_width); -std::shared_ptr<DataType> ARROW_EXPORT list(const std::shared_ptr<Field>& value_type); -std::shared_ptr<DataType> ARROW_EXPORT list(const std::shared_ptr<DataType>& value_type); +/// \brief Make an instance of DecimalType +ARROW_EXPORT +std::shared_ptr<DataType> decimal(int32_t precision, int32_t scale); -std::shared_ptr<DataType> ARROW_EXPORT timestamp(TimeUnit::type unit); -std::shared_ptr<DataType> ARROW_EXPORT timestamp(TimeUnit::type unit, - const std::string& timezone); +/// \brief Make an instance of ListType +ARROW_EXPORT +std::shared_ptr<DataType> list(const std::shared_ptr<Field>& value_type); + +/// \brief Make an instance of ListType +ARROW_EXPORT +std::shared_ptr<DataType> list(const std::shared_ptr<DataType>& value_type); + +/// \brief Make an instance of TimestampType +ARROW_EXPORT +std::shared_ptr<DataType> timestamp(TimeUnit::type unit); + +/// \brief Make an instance of TimestampType +ARROW_EXPORT +std::shared_ptr<DataType> timestamp(TimeUnit::type unit, const std::string& timezone); /// \brief Create an instance of 32-bit time type /// Unit can be either SECOND or MILLI http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/util/bit-util.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/util/bit-util.cc b/cpp/src/arrow/util/bit-util.cc index 0984055..15bf359 100644 --- a/cpp/src/arrow/util/bit-util.cc +++ b/cpp/src/arrow/util/bit-util.cc @@ -42,12 +42,12 @@ void BitUtil::FillBitsFromBytes(const std::vector<uint8_t>& bytes, uint8_t* bits } } -Status BitUtil::BytesToBits(const std::vector<uint8_t>& bytes, +Status BitUtil::BytesToBits(const std::vector<uint8_t>& bytes, MemoryPool* pool, std::shared_ptr<Buffer>* out) { int64_t bit_length = BitUtil::BytesForBits(bytes.size()); std::shared_ptr<Buffer> buffer; - RETURN_NOT_OK(AllocateBuffer(default_memory_pool(), bit_length, &buffer)); + RETURN_NOT_OK(AllocateBuffer(pool, bit_length, &buffer)); memset(buffer->mutable_data(), 0, static_cast<size_t>(bit_length)); FillBitsFromBytes(bytes, buffer->mutable_data()); http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/util/bit-util.h ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h index da2826f..aa83746 100644 --- a/cpp/src/arrow/util/bit-util.h +++ b/cpp/src/arrow/util/bit-util.h @@ -380,7 +380,10 @@ static T ShiftRightLogical(T v, int shift) { } void FillBitsFromBytes(const std::vector<uint8_t>& bytes, uint8_t* bits); -ARROW_EXPORT Status BytesToBits(const std::vector<uint8_t>&, std::shared_ptr<Buffer>*); + +/// \brief Convert vector of bytes to bitmap buffer +ARROW_EXPORT +Status BytesToBits(const std::vector<uint8_t>&, MemoryPool*, std::shared_ptr<Buffer>*); } // namespace BitUtil http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/util/decimal-test.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/util/decimal-test.cc b/cpp/src/arrow/util/decimal-test.cc index 5162971..97af088 100644 --- a/cpp/src/arrow/util/decimal-test.cc +++ b/cpp/src/arrow/util/decimal-test.cc @@ -27,7 +27,7 @@ #include "arrow/util/int128.h" namespace arrow { -namespace decimal { +namespace DecimalUtil { class DecimalTestFixture : public ::testing::Test { public: @@ -163,35 +163,35 @@ TEST(DecimalTest, TestDecimalStringAndBytesRoundTrip) { TEST(DecimalTest, TestInvalidInputMinus) { std::string invalid_value("-"); Int128 out; - Status status = decimal::FromString(invalid_value, &out); + Status status = DecimalUtil::FromString(invalid_value, &out); ASSERT_RAISES(Invalid, status); } TEST(DecimalTest, TestInvalidInputDot) { std::string invalid_value("0.0.0"); Int128 out; - Status status = decimal::FromString(invalid_value, &out); + Status status = DecimalUtil::FromString(invalid_value, &out); ASSERT_RAISES(Invalid, status); } TEST(DecimalTest, TestInvalidInputEmbeddedMinus) { std::string invalid_value("0-13-32"); Int128 out; - Status status = decimal::FromString(invalid_value, &out); + Status status = DecimalUtil::FromString(invalid_value, &out); ASSERT_RAISES(Invalid, status); } TEST(DecimalTest, TestInvalidInputSingleChar) { std::string invalid_value("a"); Int128 out; - Status status = decimal::FromString(invalid_value, &out); + Status status = DecimalUtil::FromString(invalid_value, &out); ASSERT_RAISES(Invalid, status); } TEST(DecimalTest, TestInvalidInputWithValidSubstring) { std::string invalid_value("-23092.235-"); Int128 out; - Status status = decimal::FromString(invalid_value, &out); + Status status = DecimalUtil::FromString(invalid_value, &out); auto msg = status.message(); ASSERT_RAISES(Invalid, status); } @@ -199,21 +199,21 @@ TEST(DecimalTest, TestInvalidInputWithValidSubstring) { TEST(DecimalTest, TestInvalidInputWithMinusPlus) { std::string invalid_value("-+23092.235"); Int128 out; - Status status = decimal::FromString(invalid_value, &out); + Status status = DecimalUtil::FromString(invalid_value, &out); ASSERT_RAISES(Invalid, status); } TEST(DecimalTest, TestInvalidInputWithPlusMinus) { std::string invalid_value("+-23092.235"); Int128 out; - Status status = decimal::FromString(invalid_value, &out); + Status status = DecimalUtil::FromString(invalid_value, &out); ASSERT_RAISES(Invalid, status); } TEST(DecimalTest, TestInvalidInputWithLeadingZeros) { std::string invalid_value("00a"); Int128 out; - Status status = decimal::FromString(invalid_value, &out); + Status status = DecimalUtil::FromString(invalid_value, &out); ASSERT_RAISES(Invalid, status); } @@ -282,5 +282,5 @@ TEST(Int128TestFalse, ConstructibleFromBool) { ASSERT_EQ(0, value.low_bits()); } -} // namespace decimal +} // namespace DecimalUtil } // namespace arrow http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/util/decimal.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc index bfd87a5..3e17bf6 100644 --- a/cpp/src/arrow/util/decimal.cc +++ b/cpp/src/arrow/util/decimal.cc @@ -26,7 +26,7 @@ #include "arrow/util/logging.h" namespace arrow { -namespace decimal { +namespace DecimalUtil { void StringToInteger(const std::string& whole, const std::string& fractional, int8_t sign, Int128* out) { @@ -206,5 +206,5 @@ std::string ToString(const Int128& decimal_value, int precision, int scale) { return str; } -} // namespace decimal +} // namespace DecimalUtil } // namespace arrow http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/util/decimal.h ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/util/decimal.h b/cpp/src/arrow/util/decimal.h index 4b7cac8..2e94877 100644 --- a/cpp/src/arrow/util/decimal.h +++ b/cpp/src/arrow/util/decimal.h @@ -23,10 +23,11 @@ #include "arrow/status.h" namespace arrow { -namespace decimal { class Int128; +namespace DecimalUtil { + ARROW_EXPORT void StringToInteger(const std::string& whole, const std::string& fractional, int8_t sign, Int128* out); @@ -35,6 +36,7 @@ ARROW_EXPORT Status FromString(const std::string& string, Int128* out, ARROW_EXPORT std::string ToString(const Int128& decimal_value, int precision, int scale); -} // namespace decimal +} // namespace DecimalUtil } // namespace arrow + #endif // ARROW_DECIMAL_H http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/util/int128.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/util/int128.cc b/cpp/src/arrow/util/int128.cc index 552c6cc..d5659a8 100644 --- a/cpp/src/arrow/util/int128.cc +++ b/cpp/src/arrow/util/int128.cc @@ -29,7 +29,6 @@ #include "arrow/util/logging.h" namespace arrow { -namespace decimal { static constexpr uint64_t kIntMask = 0xFFFFFFFF; static constexpr auto kCarryBit = static_cast<uint64_t>(1) << static_cast<uint64_t>(32); @@ -525,5 +524,4 @@ Int128 operator%(const Int128& left, const Int128& right) { return remainder; } -} // namespace decimal } // namespace arrow http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/util/int128.h ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/util/int128.h b/cpp/src/arrow/util/int128.h index f6fafd7..dd2ec52 100644 --- a/cpp/src/arrow/util/int128.h +++ b/cpp/src/arrow/util/int128.h @@ -27,7 +27,6 @@ #include "arrow/util/visibility.h" namespace arrow { -namespace decimal { /// Represents a signed 128-bit integer in two's complement. /// Calculations wrap around and overflow is ignored. @@ -125,7 +124,6 @@ ARROW_EXPORT Int128 operator*(const Int128& left, const Int128& right); ARROW_EXPORT Int128 operator/(const Int128& left, const Int128& right); ARROW_EXPORT Int128 operator%(const Int128& left, const Int128& right); -} // namespace decimal } // namespace arrow #endif // ARROW_INT128_H http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/util/string.h ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/util/string.h b/cpp/src/arrow/util/string.h index 6e70ddc..a2af87c 100644 --- a/cpp/src/arrow/util/string.h +++ b/cpp/src/arrow/util/string.h @@ -27,7 +27,7 @@ namespace arrow { static const char* kAsciiTable = "0123456789ABCDEF"; -static inline std::string HexEncode(const char* data, int32_t length) { +static inline std::string HexEncode(const uint8_t* data, int32_t length) { std::string hex_string; hex_string.reserve(length * 2); for (int32_t j = 0; j < length; ++j) { http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/integration/integration_test.py ---------------------------------------------------------------------- diff --git a/integration/integration_test.py b/integration/integration_test.py index 4653948..4c17345 100644 --- a/integration/integration_test.py +++ b/integration/integration_test.py @@ -896,7 +896,7 @@ class CPPTester(Tester): EXE_PATH = os.environ.get( 'ARROW_CPP_EXE_PATH', - os.path.join(ARROW_HOME, 'cpp/test-build/debug')) + os.path.join(ARROW_HOME, 'cpp/build/debug')) CPP_INTEGRATION_EXE = os.path.join(EXE_PATH, 'json-integration-test') STREAM_TO_FILE = os.path.join(EXE_PATH, 'stream-to-file')
