[arrow] Git Push Summary

2017-08-09 Thread wesm 
Repository: arrow
Updated Branches:
  refs/heads/object-store [deleted] 4915ecf1e

arrow git commit: ARROW-1342: [Python] Support strided ndarrays in pandas conversion from nested lists

2017-08-09 Thread wesm 
Repository: arrow
Updated Branches:
  refs/heads/master e44ede87c -> 2972c9d3a


ARROW-1342: [Python] Support strided ndarrays in pandas conversion from nested 
lists

This does drop the vector append to the builder that was there before. I'm 
going to do some local benchmarking to make sure this doesn't degrade 
performance unacceptably, will report back here

Author: Wes McKinney 

Closes #956 from wesm/ARROW-1342 and squashes the following commits:

f2ebeba8 [Wes McKinney] Fix cpplint issue
f403f9dd [Wes McKinney] Fix test case to be platform independent, note 
ARROW-1345. Improve quality of error message
f4f44c18 [Wes McKinney] Fix test case where inferred list type is null
ae5c8312 [Wes McKinney] Drop striding check
b4aecd3a [Wes McKinney] Support strided ndarrays in pandas conversion from 
nested lists


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/2972c9d3
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/2972c9d3
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/2972c9d3

Branch: refs/heads/master
Commit: 2972c9d3a0d371dbdcf69c68a0109b83aa6fd944
Parents: e44ede8
Author: Wes McKinney 
Authored: Wed Aug 9 17:47:30 2017 -0400
Committer: Wes McKinney 
Committed: Wed Aug 9 17:47:30 2017 -0400

--
 cpp/src/arrow/python/numpy-internal.h   | 41 +++
 cpp/src/arrow/python/pandas_to_arrow.cc | 67 ++--
 cpp/src/arrow/table.cc  |  4 +-
 python/pyarrow/error.pxi|  3 +-
 python/pyarrow/includes/common.pxd  |  1 +
 python/pyarrow/tests/pandas_examples.py | 10 ++--
 python/pyarrow/tests/test_convert_pandas.py |  9 
 7 files changed, 91 insertions(+), 44 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/arrow/blob/2972c9d3/cpp/src/arrow/python/numpy-internal.h
--
diff --git a/cpp/src/arrow/python/numpy-internal.h 
b/cpp/src/arrow/python/numpy-internal.h
index f1ef7da..db34d24 100644
--- a/cpp/src/arrow/python/numpy-internal.h
+++ b/cpp/src/arrow/python/numpy-internal.h
@@ -25,6 +25,7 @@
 #include "arrow/python/platform.h"
 
 #include 
+#include 
 
 namespace arrow {
 namespace py {
@@ -51,7 +52,12 @@ class Ndarray1DIndexer {
 
   int64_t size() const { return PyArray_SIZE(arr_); }
 
+  T* data() const { return data_; }
+
+  bool is_strided() const { return stride_ == 1; }
+
   T& operator[](size_type index) { return *(data_ + index * stride_); }
+  T& operator[](size_type index) const { return *(data_ + index * stride_); }
 
  private:
   PyArrayObject* arr_;
@@ -59,6 +65,41 @@ class Ndarray1DIndexer {
   int64_t stride_;
 };
 
+static inline std::string GetNumPyTypeName(int npy_type) {
+#define TYPE_CASE(TYPE, NAME) \
+  case NPY_##TYPE:\
+return NAME;
+
+  switch (npy_type) {
+TYPE_CASE(BOOL, "bool")
+TYPE_CASE(INT8, "int8")
+TYPE_CASE(INT16, "int16")
+TYPE_CASE(INT32, "int32")
+TYPE_CASE(INT64, "int64")
+#if (NPY_INT64 != NPY_LONGLONG)
+TYPE_CASE(LONGLONG, "longlong")
+#endif
+TYPE_CASE(UINT8, "uint8")
+TYPE_CASE(UINT16, "uint16")
+TYPE_CASE(UINT32, "uint32")
+TYPE_CASE(UINT64, "uint64")
+#if (NPY_UINT64 != NPY_ULONGLONG)
+TYPE_CASE(ULONGLONG, "ulonglong")
+#endif
+TYPE_CASE(FLOAT16, "float16")
+TYPE_CASE(FLOAT32, "float32")
+TYPE_CASE(FLOAT64, "float64")
+TYPE_CASE(DATETIME, "datetime64")
+TYPE_CASE(OBJECT, "object")
+TYPE_CASE(VOID, "void")
+default:
+  break;
+  }
+
+#undef TYPE_CASE
+  return "unrecognized type in GetNumPyTypeName";
+}
+
 }  // namespace py
 }  // namespace arrow
 

http://git-wip-us.apache.org/repos/asf/arrow/blob/2972c9d3/cpp/src/arrow/python/pandas_to_arrow.cc
--
diff --git a/cpp/src/arrow/python/pandas_to_arrow.cc 
b/cpp/src/arrow/python/pandas_to_arrow.cc
index 060fcb2..b6cc16b 100644
--- a/cpp/src/arrow/python/pandas_to_arrow.cc
+++ b/cpp/src/arrow/python/pandas_to_arrow.cc
@@ -97,8 +97,6 @@ static int64_t ValuesToBitmap(PyArrayObject* arr, uint8_t* 
bitmap) {
   int64_t null_count = 0;
 
   Ndarray1DIndexer values(arr);
-
-  // TODO(wesm): striding
   for (int i = 0; i < values.size(); ++i) {
 if (traits::isnull(values[i])) {
   ++null_count;
@@ -125,22 +123,27 @@ static int64_t MaskToBitmap(PyArrayObject* mask, int64_t 
length, uint8_t* bitmap
   return null_count;
 }
 
-template 
-static int64_t ValuesToValidBytes(const void* data, int64_t length,
-  uint8_t* valid_bytes) {
+template 
+static Status AppendNdarrayToBuilder(PyArrayObject* array, BuilderType* 
builder) {
   typedef internal::npy_traits traits;
   typedef

svn commit: r20913 - in /dev/arrow/apache-arrow-0.6.0-rc0: ./ apache-arrow-0.6.0.tar.gz apache-arrow-0.6.0.tar.gz.asc apache-arrow-0.6.0.tar.gz.md5 apache-arrow-0.6.0.tar.gz.sha

2017-08-09 Thread wesm 
Author: wesm
Date: Wed Aug  9 22:00:34 2017
New Revision: 20913

Log:
Apache Arrow ${version} RC${rc}

Added:
dev/arrow/apache-arrow-0.6.0-rc0/
dev/arrow/apache-arrow-0.6.0-rc0/apache-arrow-0.6.0.tar.gz   (with props)
dev/arrow/apache-arrow-0.6.0-rc0/apache-arrow-0.6.0.tar.gz.asc
dev/arrow/apache-arrow-0.6.0-rc0/apache-arrow-0.6.0.tar.gz.md5
dev/arrow/apache-arrow-0.6.0-rc0/apache-arrow-0.6.0.tar.gz.sha

Added: dev/arrow/apache-arrow-0.6.0-rc0/apache-arrow-0.6.0.tar.gz
==
Binary file - no diff available.

Propchange: dev/arrow/apache-arrow-0.6.0-rc0/apache-arrow-0.6.0.tar.gz
--
svn:mime-type = application/octet-stream

Added: dev/arrow/apache-arrow-0.6.0-rc0/apache-arrow-0.6.0.tar.gz.asc
==
--- dev/arrow/apache-arrow-0.6.0-rc0/apache-arrow-0.6.0.tar.gz.asc (added)
+++ dev/arrow/apache-arrow-0.6.0-rc0/apache-arrow-0.6.0.tar.gz.asc Wed Aug  9 
22:00:34 2017
@@ -0,0 +1,17 @@
+-BEGIN PGP SIGNATURE-
+Version: GnuPG v1
+
+iQIcBAABCgAGBQJZi4X6AAoJEPEFiDoXNWI98xoP/3I876G77beZAwr4vcHy803b
+swcPJZ1IQVwKF7W8IMc3o0eOutxMwWzXhRWzLtjbkUAEyG/Qs68sNzcSx/qzZUkb
+YHzKDuobChg31G3PiNiDS5lgKeYYpNHp0tlDKYWq3SiuvOxLwMlFLP6sAhKdNjte
+jLILPi2HdsxvEX3AJB4zEmKfsjp3UZWDZaoJ5vwGkK0Z1myZpoxCUAU+a3RN/ynd
+hDob084iRhYPwHgbVNL+/Lzkl4lUmEgDuAfDjWVnA4BNrwZHHdddf8a9nRAV9CXh
+ypSqqhJMDtfxPijgim0EL4piKz3Kst9jE25xGGOQFTuC16lRuoNfOochOXCoCi0v
+KwO9X/Zju0CDc4FWRAMZs9FiC+/oNIIShQT5NVoj1st7Q6dKeAxJvE9/KsI5nnma
+mZa6RfjaJE7lq42jP+JTmdMIYA9AWYtFxGypp+mg+47094zF6nCLkgkGXu0HOZh8
+SiJ8FUWkQMltnbxl3aOjhZDrNOALTdhMnIXaTj2pxVTqZT8JM9XRn/O9BT85e7Wv
+b+5DJW5RLhLSKaomBZo6aDqH64mE55lGxlAEeTnDW1jNPdgmtzNYi7ycAdXl7Cid
+ifeWyTu7c/VeZKFHvQPYNOIopYwu98HBtIMdgyqUJ4UbENSoZeyVb9YT+OiSRzpe
+AvbZTrpiATOvrpyV71Vo
+=fHuv
+-END PGP SIGNATURE-

Added: dev/arrow/apache-arrow-0.6.0-rc0/apache-arrow-0.6.0.tar.gz.md5
==
--- dev/arrow/apache-arrow-0.6.0-rc0/apache-arrow-0.6.0.tar.gz.md5 (added)
+++ dev/arrow/apache-arrow-0.6.0-rc0/apache-arrow-0.6.0.tar.gz.md5 Wed Aug  9 
22:00:34 2017
@@ -0,0 +1 @@
+apache-arrow-0.6.0.tar.gz: 47 BC 48 19 2A 0B CF 67  DC F8 39 C0 09 01 1B AF

Added: dev/arrow/apache-arrow-0.6.0-rc0/apache-arrow-0.6.0.tar.gz.sha
==
--- dev/arrow/apache-arrow-0.6.0-rc0/apache-arrow-0.6.0.tar.gz.sha (added)
+++ dev/arrow/apache-arrow-0.6.0-rc0/apache-arrow-0.6.0.tar.gz.sha Wed Aug  9 
22:00:34 2017
@@ -0,0 +1 @@
+b7de0b78042f1770b984ee1e6ba6f90cca9d752a  apache-arrow-0.6.0.tar.gz

arrow git commit: ARROW-1243: [JAVA] update all libs to latest versions

2017-08-09 Thread wesm 
Repository: arrow
Updated Branches:
  refs/heads/master a9c2f196e -> 7fdbcc606


ARROW-1243: [JAVA] update all libs to latest versions

NB this commit excludes Jackson and logback upgrades, since they are dealt with 
in 871 and 872

Author: Matt Darwin <(none)>
Author: Matt Darwin 
Author: Matt 

Closes #873 from mattdarwin/upgrade-libs and squashes the following commits:

9b51f46e [Matt Darwin] Merge branch 'master' into upgrade-libs
284a4ce3 [Matt Darwin] Merge branch 'master' of https://github.com/apache/arrow
79550b15 [Matt Darwin] rolling back lilith to 0.9.44 since 8 doesn't support 
java 7
c63eef64 [Matt Darwin] Merge branch 'master' into upgrade-libs
bc3b6a07 [Matt] Merge pull request #1 from apache/master
8599ba06 [Matt Darwin] backing out guava upgrade
80d81e64 [Matt Darwin] downgrading guava to 20 for java 7 compatibility
806f3489 [Matt Darwin] Merge branch 'master' into upgrade-libs
8aafb7e9 [Matt Darwin] correcting indentation in BaseValueVector
94c14698 [Matt Darwin] upgrading netty to 4.0.49
cff5596a [Matt Darwin] reverting to netty 4.0.41.Final
568737d3 [Matt Darwin] switching to Collections from Guava for empty iterator
c194e48d [Matt Darwin] upgraded hppc to 0.7.2
38be468c [Matt Darwin] upgrading libs except jackson and logback


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/7fdbcc60
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/7fdbcc60
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/7fdbcc60

Branch: refs/heads/master
Commit: 7fdbcc60693884f701b40ee74ff86f5da71976af
Parents: a9c2f19
Author: Matt Darwin <(none)>
Authored: Wed Aug 9 09:10:08 2017 -0400
Committer: Wes McKinney 
Committed: Wed Aug 9 09:10:08 2017 -0400

--
 java/memory/pom.xml| 4 ++--
 java/pom.xml   | 6 +++---
 java/tools/pom.xml | 4 ++--
 java/vector/pom.xml| 6 +++---
 .../src/main/java/org/apache/arrow/vector/BaseValueVector.java | 3 ++-
 5 files changed, 12 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/arrow/blob/7fdbcc60/java/memory/pom.xml
--
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index 9a8d2d7..1070747 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -23,13 +23,13 @@
 
   com.google.code.findbugs
   jsr305
-  3.0.1
+  3.0.2
 
 
 
   com.carrotsearch
   hppc
-  0.7.1
+  0.7.2
 
   
 

http://git-wip-us.apache.org/repos/asf/arrow/blob/7fdbcc60/java/pom.xml
--
diff --git a/java/pom.xml b/java/pom.xml
index 5702e96..b0621c5 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -454,7 +454,7 @@
 
   io.netty
   netty-handler
-  4.0.41.Final
+  4.0.49.Final
 
 
 
@@ -495,7 +495,7 @@
   
   com.googlecode.jmockit
   jmockit
-  1.3
+  1.7
   test
 
 
@@ -507,7 +507,7 @@
 
   org.mockito
   mockito-core
-  1.9.5
+  2.7.22
   test
   
   

http://git-wip-us.apache.org/repos/asf/arrow/blob/7fdbcc60/java/tools/pom.xml
--
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index 9d067ef..8aadb44 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -38,12 +38,12 @@
 
 org.apache.commons
 commons-lang3
-3.4
+3.6
 
 
 commons-cli
 commons-cli
-1.2
+1.4
 
 
   ch.qos.logback

http://git-wip-us.apache.org/repos/asf/arrow/blob/7fdbcc60/java/vector/pom.xml
--
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index e15ab9a..a81bbd0 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -34,7 +34,7 @@
 
   joda-time
   joda-time
-  2.9
+  2.9.9
 
 
   com.fasterxml.jackson.core
@@ -49,12 +49,12 @@
 
   com.carrotsearch
   hppc
-  0.7.1
+  0.7.2
 
 
   org.apache.commons
   commons-lang3
-  3.4
+  3.6
 
 
   commons-codec

http://git-wip-us.apache.org/repos/asf/arrow/blob/7fdbcc60/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
--
diff --git 
a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java

arrow git commit: ARROW-1340: [Java] Fix NullableMapVector field metadata

2017-08-09 Thread wesm 
Repository: arrow
Updated Branches:
  refs/heads/master 7fdbcc606 -> 86154f0be


ARROW-1340: [Java] Fix NullableMapVector field metadata

Author: Emilio Lahr-Vivaz 

Closes #953 from elahrvivaz/ARROW-1340 and squashes the following commits:

a307779e [Emilio Lahr-Vivaz] ARROW-1340: [Java] Fix NullableMapVector field 
metadata


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/86154f0b
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/86154f0b
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/86154f0b

Branch: refs/heads/master
Commit: 86154f0be3fbafcd27716f3b3f7058c31242a52f
Parents: 7fdbcc6
Author: Emilio Lahr-Vivaz 
Authored: Wed Aug 9 09:11:52 2017 -0400
Committer: Wes McKinney 
Committed: Wed Aug 9 09:11:52 2017 -0400

--
 .../arrow/vector/complex/NullableMapVector.java |  3 +-
 .../org/apache/arrow/vector/TestMapVector.java  | 57 
 2 files changed, 59 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/arrow/blob/86154f0b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
--
diff --git 
a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
 
b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
index e70a915..fda9c14 100644
--- 
a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
+++ 
b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
@@ -86,7 +86,8 @@ public class NullableMapVector extends MapVector implements 
FieldVector {
   @Override
   public Field getField() {
 Field f = super.getField();
-return new Field(f.getName(), true, f.getType(), f.getChildren());
+FieldType type = new FieldType(true, f.getType(), 
f.getFieldType().getDictionary(), f.getFieldType().getMetadata());
+return new Field(f.getName(), type, f.getChildren());
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/arrow/blob/86154f0b/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java
--
diff --git 
a/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java 
b/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java
new file mode 100644
index 000..357df96
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+
+public class TestMapVector {
+
+  private BufferAllocator allocator;
+
+  @Before
+  public void init() {
+allocator = new DirtyRootAllocator(Long.MAX_VALUE, (byte) 100);
+  }
+
+  @After
+  public void terminate() throws Exception {
+allocator.close();
+  }
+
+  @Test
+  public void testFieldMetadata() throws Exception {
+Map metadata = new HashMap<>();
+metadata.put("k1", "v1");
+FieldType type = new FieldType(true, Struct.INSTANCE, null, metadata);
+try (NullableMapVector vector = new NullableMapVector("map", allocator, 
type, null)) {
+  Assert.assertEquals(vector.getField().getMetadata(), type.getMetadata());
+}
+  }
+}