[arrow] branch master updated: ARROW-1716: [Format/JSON] Use string integer value for Decimals in JSON

2017-11-07 Thread wesm 
This is an automated email from the ASF dual-hosted git repository.

wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git


The following commit(s) were added to refs/heads/master by this push:
 new 3188d70  ARROW-1716: [Format/JSON] Use string integer value for 
Decimals in JSON
3188d70 is described below

commit 3188d70202795d8e0a8092ec5685d859b02e366d
Author: Phillip Cloud 
AuthorDate: Tue Nov 7 16:22:20 2017 -0500

ARROW-1716: [Format/JSON] Use string integer value for Decimals in JSON

Author: Phillip Cloud 

Closes #1267 from cpcloud/ARROW-1716 and squashes the following commits:

b4f3aed2 [Phillip Cloud] Add cases for every valid precision
f8d4391f [Phillip Cloud] Use the full range of decimal values in 
integration tests
6fef5f71 [Phillip Cloud] ARROW-1716: [Format/JSON] Use string integer value 
for Decimals in JSON
---
 cpp/src/arrow/ipc/ipc-read-write-test.cc   |   4 +-
 cpp/src/arrow/ipc/json-internal.cc |  69 +++---
 cpp/src/arrow/ipc/test-common.h|  13 ++-
 integration/integration_test.py| 105 -
 .../arrow/vector/file/json/JsonFileReader.java |   9 +-
 .../arrow/vector/file/json/JsonFileWriter.java |   8 +-
 .../apache/arrow/vector/util/DecimalUtility.java   |  32 ++-
 7 files changed, 165 insertions(+), 75 deletions(-)

diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc 
b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index 6f2f5cf..40cd3f0 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -727,7 +727,7 @@ TEST_F(TestTensorRoundTrip, BasicRoundtrip) {
   int64_t size = 24;
 
   std::vector values;
-  test::randint(size, 0, 100, );
+  test::randint(size, 0, 100, );
 
   auto data = test::GetBufferFromVector(values);
 
@@ -748,7 +748,7 @@ TEST_F(TestTensorRoundTrip, NonContiguous) {
   ASSERT_OK(io::MemoryMapFixture::InitMemoryMap(kBufferSize, path, _));
 
   std::vector values;
-  test::randint(24, 0, 100, );
+  test::randint(24, 0, 100, );
 
   auto data = test::GetBufferFromVector(values);
   Tensor tensor(int64(), data, {4, 3}, {48, 16});
diff --git a/cpp/src/arrow/ipc/json-internal.cc 
b/cpp/src/arrow/ipc/json-internal.cc
index 025f6c2..c1c0661 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -33,6 +33,7 @@
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
 #include "arrow/util/bit-util.h"
+#include "arrow/util/decimal.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/string.h"
 #include "arrow/visitor_inline.h"
@@ -448,7 +449,8 @@ class ArrayWriter {
   }
 
   void WriteDataValues(const FixedSizeBinaryArray& arr) {
-int32_t width = arr.byte_width();
+const int32_t width = arr.byte_width();
+
 for (int64_t i = 0; i < arr.length(); ++i) {
   const uint8_t* buf = arr.GetValue(i);
   std::string encoded = HexEncode(buf, width);
@@ -456,6 +458,13 @@ class ArrayWriter {
 }
   }
 
+  void WriteDataValues(const DecimalArray& arr) {
+for (int64_t i = 0; i < arr.length(); ++i) {
+  const Decimal128 value(arr.GetValue(i));
+  writer_->String(value.ToIntegerString());
+}
+  }
+
   void WriteDataValues(const BooleanArray& arr) {
 for (int i = 0; i < arr.length(); ++i) {
   writer_->Bool(arr.Value(i));
@@ -1053,7 +1062,9 @@ class ArrayReader {
   }
 
   template 
-  typename std::enable_if::value, 
Status>::type
+  typename std::enable_if::value &&
+  !std::is_base_of::value,
+  Status>::type
   Visit(const T& type) {
 typename TypeTraits::BuilderType builder(type_, pool_);
 
@@ -1073,22 +1084,52 @@ class ArrayReader {
 for (int i = 0; i < length_; ++i) {
   if (!is_valid_[i]) {
 RETURN_NOT_OK(builder.AppendNull());
-continue;
-  }
+  } else {
+const rj::Value& val = json_data_arr[i];
+DCHECK(val.IsString())
+<< "Found non-string JSON value when parsing FixedSizeBinary 
value";
+std::string hex_string = val.GetString();
+if (static_cast(hex_string.size()) != byte_width * 2) {
+  DCHECK(false) << "Expected size: " << byte_width * 2
+<< " got: " << hex_string.size();
+}
+const char* hex_data = hex_string.c_str();
 
-  const rj::Value& val = json_data_arr[i];
-  DCHECK(val.IsString());
-  std::string hex_string = val.GetString();
-  if (static_cast(hex_string.size()) != byte_width * 2) {
-DCHECK(false) << "Expected size: " << byte_width * 2
-  << " got: " << hex_string.size();
+for (int32_t j = 0; j < byte_width; ++j) {
+  RETURN_NOT_OK(ParseHexValue(hex_data + j * 2, _buffer_data[j]));
+

[arrow] branch master updated: [Format] Fix link to Flatbuffers project in IPC.md

2017-11-07 Thread wesm 
This is an automated email from the ASF dual-hosted git repository.

wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git


The following commit(s) were added to refs/heads/master by this push:
 new e631119  [Format] Fix link to Flatbuffers project in IPC.md
e631119 is described below

commit e631119d853a3182fa4e2d8cd980440f59c7a679
Author: Victor Uriarte 
AuthorDate: Tue Nov 7 11:57:43 2017 -0700

[Format] Fix link to Flatbuffers project in IPC.md

Close #1291

Change-Id: Ibae55c472b08cedcf22bd783c7a7fce1449051fe
---
 format/IPC.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/format/IPC.md b/format/IPC.md
index f3b4885..5a5d3ae 100644
--- a/format/IPC.md
+++ b/format/IPC.md
@@ -252,5 +252,5 @@ shared memory region) to be a multiple of 8:
 
 [1]: https://github.com/apache/arrow/blob/master/format/File.fbs
 [2]: https://github.com/apache/arrow/blob/master/format/Message.fbs
-[3]: https://github.com/google]/flatbuffers
+[3]: https://github.com/google/flatbuffers
 [4]: https://github.com/apache/arrow/blob/master/format/Layout.md

-- 
To stop receiving notification emails like this one, please contact
['"commits@arrow.apache.org" '].