arrow git commit: ARROW-326: Include scale and precision when materializing decimal writer

2016-10-10 Thread smp 
Repository: arrow
Updated Branches:
  refs/heads/master fb799bc8f -> 8c8d341e1


ARROW-326: Include scale and precision when materializing decimal writer

closes #166


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/8c8d341e
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/8c8d341e
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/8c8d341e

Branch: refs/heads/master
Commit: 8c8d341e12efcedecd3c2545aaf349bf5f899bc1
Parents: fb799bc
Author: Steven Phillips 
Authored: Mon Oct 10 13:42:41 2016 -0700
Committer: Steven Phillips 
Committed: Mon Oct 10 20:09:32 2016 -0700

--
 java/vector/src/main/codegen/templates/MapWriters.java  | 5 +
 .../arrow/vector/complex/impl/TestPromotableWriter.java | 9 ++---
 2 files changed, 11 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/arrow/blob/8c8d341e/java/vector/src/main/codegen/templates/MapWriters.java
--
diff --git a/java/vector/src/main/codegen/templates/MapWriters.java 
b/java/vector/src/main/codegen/templates/MapWriters.java
index 9fe20df..696bbf6 100644
--- a/java/vector/src/main/codegen/templates/MapWriters.java
+++ b/java/vector/src/main/codegen/templates/MapWriters.java
@@ -73,7 +73,12 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
 <#if lowerName == "int" ><#assign lowerName = "integer" />
 <#assign upperName = minor.class?upper_case />
   case ${upperName}:
+<#if lowerName == "decimal" >
+Decimal decimal = (Decimal)child.getType();
+decimal(child.getName(), decimal.getScale(), decimal.getPrecision());
+<#else>
 ${lowerName}(child.getName());
+   
 break;
 
   }

http://git-wip-us.apache.org/repos/asf/arrow/blob/8c8d341e/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
--
diff --git 
a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
 
b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
index d439ceb..176ad51 100644
--- 
a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
+++ 
b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
@@ -67,6 +67,8 @@ public class TestPromotableWriter {
   writer.setPosition(1);
   writer.bit("A").writeBit(1);
 
+  writer.decimal("dec", 10,10);
+
   writer.setPosition(2);
   writer.integer("A").writeInt(10);
 
@@ -108,9 +110,10 @@ public class TestPromotableWriter {
   newMapWriter.setPosition(2);
   newMapWriter.integer("A").writeInt(10);
 
-  Field childField = 
container.getField().getChildren().get(0).getChildren().get(0);
-  assertEquals("Child field should be union type: " + 
childField.getName(), Type.Union, childField.getType().getTypeType());
-
+  Field childField1 = 
container.getField().getChildren().get(0).getChildren().get(0);
+  Field childField2 = 
container.getField().getChildren().get(0).getChildren().get(1);
+  assertEquals("Child field should be union type: " + 
childField1.getName(), Type.Union, childField1.getType().getTypeType());
+  assertEquals("Child field should be decimal type: " + 
childField2.getName(), Type.Decimal, childField2.getType().getTypeType());
 }
   }
 }

[1/2] arrow git commit: ARROW-312: Read and write Arrow IPC file format from Python

2016-10-10 Thread smp 
Repository: arrow
Updated Branches:
  refs/heads/arrow-326 [created] fb29195e0


ARROW-312: Read and write Arrow IPC file format from Python

This also adds some IO scaffolding for interacting with `arrow::Buffer` objects 
from Python and assorted additions to help with testing.

Author: Wes McKinney 

Closes #164 from wesm/ARROW-312 and squashes the following commits:

7df3e5f [Wes McKinney] Set BUILD_WITH_INSTALL_RPATH on arrow_ipc
be8cee0 [Wes McKinney] Link Cython modules to libarrow* libraries
5716601 [Wes McKinney] Fix accidental deletion
77fb03b [Wes McKinney] Add / test Buffer wrapper. Test that we can write an 
arrow file to a wrapped buffer. Resize buffer in BufferOutputStream on close
316537d [Wes McKinney] Get ready to wrap Arrow buffers in a Python object
4822d32 [Wes McKinney] Implement RecordBatch::Equals, compare in Python ipc 
file writes
a931e49 [Wes McKinney] Permit buffers (write padding) in a non-multiple of 64 
in an IPC context, to allow zero-copy writing of NumPy arrays
2c49cd4 [Wes McKinney] Some debugging
ca1562b [Wes McKinney] Draft implementations of Arrow file read/write from 
Python


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/772800ac
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/772800ac
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/772800ac

Branch: refs/heads/arrow-326
Commit: 772800ace3706b8f169803b3ef8932e04ec54619
Parents: eb1491a
Author: Wes McKinney 
Authored: Mon Oct 10 11:21:49 2016 -0400
Committer: Wes McKinney 
Committed: Mon Oct 10 11:21:49 2016 -0400

--
 cpp/src/arrow/io/io-memory-test.cc   |  25 ++
 cpp/src/arrow/io/memory.cc   |  13 +-
 cpp/src/arrow/ipc/CMakeLists.txt |   7 +
 cpp/src/arrow/ipc/adapter.cc |  16 +-
 cpp/src/arrow/ipc/util.h |   6 +-
 cpp/src/arrow/table-test.cc  |  27 ++
 cpp/src/arrow/table.cc   |  16 ++
 cpp/src/arrow/table.h|   2 +
 cpp/src/arrow/types/primitive-test.cc|   3 +-
 cpp/src/arrow/util/bit-util.h|  13 +
 cpp/src/arrow/util/buffer.cc |  16 +-
 cpp/src/arrow/util/buffer.h  |   1 -
 cpp/src/arrow/util/logging.h |   4 +-
 python/CMakeLists.txt|   8 +-
 python/cmake_modules/FindArrow.cmake |  11 +
 python/pyarrow/__init__.py   |   3 +-
 python/pyarrow/array.pyx |  44 +---
 python/pyarrow/includes/common.pxd   |   4 -
 python/pyarrow/includes/libarrow.pxd |  29 ++-
 python/pyarrow/includes/libarrow_io.pxd  |  14 +-
 python/pyarrow/includes/libarrow_ipc.pxd |  52 
 python/pyarrow/includes/pyarrow.pxd  |  13 +-
 python/pyarrow/io.pxd|   6 +
 python/pyarrow/io.pyx| 340 --
 python/pyarrow/ipc.pyx   | 155 
 python/pyarrow/table.pxd |  17 +-
 python/pyarrow/table.pyx | 194 +++
 python/pyarrow/tests/test_array.py   |   4 +
 python/pyarrow/tests/test_io.py  |  41 
 python/pyarrow/tests/test_ipc.py | 116 +
 python/pyarrow/tests/test_table.py   |  82 +++
 python/setup.py  |   1 +
 python/src/pyarrow/adapters/builtin.cc   |   2 +-
 python/src/pyarrow/adapters/pandas.cc|   8 +
 python/src/pyarrow/common.cc |   2 +-
 python/src/pyarrow/common.h  |  20 +-
 python/src/pyarrow/io.cc |   6 +-
 37 files changed, 1012 insertions(+), 309 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/arrow/blob/772800ac/cpp/src/arrow/io/io-memory-test.cc
--
diff --git a/cpp/src/arrow/io/io-memory-test.cc 
b/cpp/src/arrow/io/io-memory-test.cc
index 6de35da..a49faf3 100644
--- a/cpp/src/arrow/io/io-memory-test.cc
+++ b/cpp/src/arrow/io/io-memory-test.cc
@@ -121,5 +121,30 @@ TEST_F(TestMemoryMappedFile, InvalidFile) {
   IOError, MemoryMappedFile::Open(non_existent_path, FileMode::READ, 
));
 }
 
+class TestBufferOutputStream : public ::testing::Test {
+ public:
+  void SetUp() {
+buffer_.reset(new PoolBuffer(default_memory_pool()));
+stream_.reset(new BufferOutputStream(buffer_));
+  }
+
+ protected:
+  std::shared_ptr buffer_;
+  std::unique_ptr stream_;
+};
+
+TEST_F(TestBufferOutputStream, CloseResizes) {
+  std::string data = "data123456";
+
+  const int64_t nbytes = static_cast(data.size());
+  const int K = 100;
+  for (int i = 0; i < K; ++i) {
+EXPECT_OK(stream_->Write(reinterpret_cast(data.c_str()), 
nbytes));
+  }
+
+  ASSERT_OK(stream_->Close());
+  ASSERT_EQ(K * nbytes, buffer_->size());
+}
+
 }  //

[2/2] arrow git commit: ARROW-326: Include scale and precision when materializing decimal writer

2016-10-10 Thread smp 
ARROW-326: Include scale and precision when materializing decimal writer

closes #166


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/fb29195e
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/fb29195e
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/fb29195e

Branch: refs/heads/arrow-326
Commit: fb29195e0040143375340ab4172bb419fdebaa8f
Parents: 772800a
Author: Steven Phillips 
Authored: Mon Oct 10 13:42:41 2016 -0700
Committer: Steven Phillips 
Committed: Mon Oct 10 20:06:23 2016 -0700

--
 java/vector/src/main/codegen/templates/MapWriters.java  | 5 +
 .../arrow/vector/complex/impl/TestPromotableWriter.java | 9 ++---
 2 files changed, 11 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/arrow/blob/fb29195e/java/vector/src/main/codegen/templates/MapWriters.java
--
diff --git a/java/vector/src/main/codegen/templates/MapWriters.java 
b/java/vector/src/main/codegen/templates/MapWriters.java
index 9fe20df..696bbf6 100644
--- a/java/vector/src/main/codegen/templates/MapWriters.java
+++ b/java/vector/src/main/codegen/templates/MapWriters.java
@@ -73,7 +73,12 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
 <#if lowerName == "int" ><#assign lowerName = "integer" />
 <#assign upperName = minor.class?upper_case />
   case ${upperName}:
+<#if lowerName == "decimal" >
+Decimal decimal = (Decimal)child.getType();
+decimal(child.getName(), decimal.getScale(), decimal.getPrecision());
+<#else>
 ${lowerName}(child.getName());
+   
 break;
 
   }

http://git-wip-us.apache.org/repos/asf/arrow/blob/fb29195e/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
--
diff --git 
a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
 
b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
index d439ceb..176ad51 100644
--- 
a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
+++ 
b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
@@ -67,6 +67,8 @@ public class TestPromotableWriter {
   writer.setPosition(1);
   writer.bit("A").writeBit(1);
 
+  writer.decimal("dec", 10,10);
+
   writer.setPosition(2);
   writer.integer("A").writeInt(10);
 
@@ -108,9 +110,10 @@ public class TestPromotableWriter {
   newMapWriter.setPosition(2);
   newMapWriter.integer("A").writeInt(10);
 
-  Field childField = 
container.getField().getChildren().get(0).getChildren().get(0);
-  assertEquals("Child field should be union type: " + 
childField.getName(), Type.Union, childField.getType().getTypeType());
-
+  Field childField1 = 
container.getField().getChildren().get(0).getChildren().get(0);
+  Field childField2 = 
container.getField().getChildren().get(0).getChildren().get(1);
+  assertEquals("Child field should be union type: " + 
childField1.getName(), Type.Union, childField1.getType().getTypeType());
+  assertEquals("Child field should be decimal type: " + 
childField2.getName(), Type.Decimal, childField2.getType().getTypeType());
 }
   }
 }

[arrow] Git Push Summary

2016-10-10 Thread julien 
Repository: arrow
Updated Tags:  refs/tags/apache-arrow-0.1.0 [created] 4fc8a8173

[2/3] arrow git commit: [maven-release-plugin] prepare for next development iteration

2016-10-10 Thread julien 
[maven-release-plugin] prepare for next development iteration


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/17cd7a64
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/17cd7a64
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/17cd7a64

Branch: refs/heads/master
Commit: 17cd7a6466741d22053d132ea306ad6f05351419
Parents: e7080ef
Author: Julien Le Dem 
Authored: Fri Oct 7 17:15:08 2016 -0700
Committer: Julien Le Dem 
Committed: Fri Oct 7 17:15:08 2016 -0700

--
 java/format/pom.xml | 2 +-
 java/memory/pom.xml | 2 +-
 java/pom.xml| 4 ++--
 java/vector/pom.xml | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/arrow/blob/17cd7a64/java/format/pom.xml
--
diff --git a/java/format/pom.xml b/java/format/pom.xml
index c81cfed..eb045d6 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -15,7 +15,7 @@
 
   arrow-java-root
   org.apache.arrow
-  0.1.0
+  0.1.1-SNAPSHOT
 
 
 arrow-format

http://git-wip-us.apache.org/repos/asf/arrow/blob/17cd7a64/java/memory/pom.xml
--
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index 8af2313..6ed1448 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -14,7 +14,7 @@
   
 org.apache.arrow
 arrow-java-root
-0.1.0
+0.1.1-SNAPSHOT
   
   arrow-memory
   Arrow Memory

http://git-wip-us.apache.org/repos/asf/arrow/blob/17cd7a64/java/pom.xml
--
diff --git a/java/pom.xml b/java/pom.xml
index 8ca8eac..0147de7 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -20,7 +20,7 @@
 
   org.apache.arrow
   arrow-java-root
-  0.1.0
+  0.1.1-SNAPSHOT
   pom
 
   Apache Arrow Java Root POM
@@ -41,7 +41,7 @@
 
scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git
 
scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git
 https://github.com/apache/arrow
-apache-arrow-0.1.0
+HEAD
   
 
   

http://git-wip-us.apache.org/repos/asf/arrow/blob/17cd7a64/java/vector/pom.xml
--
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index ae48d22..1d06bde 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -14,7 +14,7 @@
   
 org.apache.arrow
 arrow-java-root
-0.1.0
+0.1.1-SNAPSHOT
   
   arrow-vector
   Arrow Vectors

[1/3] arrow git commit: [maven-release-plugin] prepare release apache-arrow-0.1.0 [Forced Update!]

2016-10-10 Thread julien 
Repository: arrow
Updated Branches:
  refs/heads/master 772800ace -> a9747ceac (forced update)


[maven-release-plugin] prepare release apache-arrow-0.1.0


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/e7080ef9
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/e7080ef9
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/e7080ef9

Branch: refs/heads/master
Commit: e7080ef9f1bd91505996edd4e4b7643cc54f6b5f
Parents: eb1491a
Author: Julien Le Dem 
Authored: Fri Oct 7 17:14:58 2016 -0700
Committer: Julien Le Dem 
Committed: Fri Oct 7 17:14:58 2016 -0700

--
 java/format/pom.xml | 5 ++---
 java/memory/pom.xml | 5 ++---
 java/pom.xml| 7 +++
 java/vector/pom.xml | 5 ++---
 4 files changed, 9 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/arrow/blob/e7080ef9/java/format/pom.xml
--
diff --git a/java/format/pom.xml b/java/format/pom.xml
index 7830004..c81cfed 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -9,14 +9,13 @@
   License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS 
   OF ANY KIND, either express or implied. See the License for the specific 
   language governing permissions and limitations under the License. -->
-http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
-  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
 4.0.0
 
 
   arrow-java-root
   org.apache.arrow
-  0.1-SNAPSHOT
+  0.1.0
 
 
 arrow-format

http://git-wip-us.apache.org/repos/asf/arrow/blob/e7080ef9/java/memory/pom.xml
--
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index b91b598..8af2313 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -9,13 +9,12 @@
   License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS 
   OF ANY KIND, either express or implied. See the License for the specific 
   language governing permissions and limitations under the License. -->
-http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
-  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
   4.0.0
   
 org.apache.arrow
 arrow-java-root
-0.1-SNAPSHOT
+0.1.0
   
   arrow-memory
   Arrow Memory

http://git-wip-us.apache.org/repos/asf/arrow/blob/e7080ef9/java/pom.xml
--
diff --git a/java/pom.xml b/java/pom.xml
index fc2c18d..8ca8eac 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -9,8 +9,7 @@
   License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
   OF ANY KIND, either express or implied. See the License for the specific
   language governing permissions and limitations under the License. -->
-http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
-  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
   4.0.0
 
   
@@ -21,7 +20,7 @@
 
   org.apache.arrow
   arrow-java-root
-  0.1-SNAPSHOT
+  0.1.0
   pom
 
   Apache Arrow Java Root POM
@@ -42,7 +41,7 @@
 
scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git
 
scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git
 https://github.com/apache/arrow
-HEAD
+apache-arrow-0.1.0
   
 
   

http://git-wip-us.apache.org/repos/asf/arrow/blob/e7080ef9/java/vector/pom.xml
--
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index 08f9bc8..ae48d22 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -9,13 +9,12 @@
   License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS 
   OF ANY KIND, either express or implied. See the License for the specific 
   language governing permissions and limitations under the License. -->
-http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
-  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0

[3/3] arrow git commit: ARROW-312: Read and write Arrow IPC file format from Python

2016-10-10 Thread julien 
ARROW-312: Read and write Arrow IPC file format from Python

This also adds some IO scaffolding for interacting with `arrow::Buffer` objects 
from Python and assorted additions to help with testing.

Author: Wes McKinney 

Closes #164 from wesm/ARROW-312 and squashes the following commits:

7df3e5f [Wes McKinney] Set BUILD_WITH_INSTALL_RPATH on arrow_ipc
be8cee0 [Wes McKinney] Link Cython modules to libarrow* libraries
5716601 [Wes McKinney] Fix accidental deletion
77fb03b [Wes McKinney] Add / test Buffer wrapper. Test that we can write an 
arrow file to a wrapped buffer. Resize buffer in BufferOutputStream on close
316537d [Wes McKinney] Get ready to wrap Arrow buffers in a Python object
4822d32 [Wes McKinney] Implement RecordBatch::Equals, compare in Python ipc 
file writes
a931e49 [Wes McKinney] Permit buffers (write padding) in a non-multiple of 64 
in an IPC context, to allow zero-copy writing of NumPy arrays
2c49cd4 [Wes McKinney] Some debugging
ca1562b [Wes McKinney] Draft implementations of Arrow file read/write from 
Python


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/a9747cea
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/a9747cea
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/a9747cea

Branch: refs/heads/master
Commit: a9747ceac2b6399c6acf027de8074d8661d5eb1d
Parents: 17cd7a6
Author: Wes McKinney 
Authored: Mon Oct 10 11:21:49 2016 -0400
Committer: Julien Le Dem 
Committed: Mon Oct 10 18:42:05 2016 -0700

--
 cpp/src/arrow/io/io-memory-test.cc   |  25 ++
 cpp/src/arrow/io/memory.cc   |  13 +-
 cpp/src/arrow/ipc/CMakeLists.txt |   7 +
 cpp/src/arrow/ipc/adapter.cc |  16 +-
 cpp/src/arrow/ipc/util.h |   6 +-
 cpp/src/arrow/table-test.cc  |  27 ++
 cpp/src/arrow/table.cc   |  16 ++
 cpp/src/arrow/table.h|   2 +
 cpp/src/arrow/types/primitive-test.cc|   3 +-
 cpp/src/arrow/util/bit-util.h|  13 +
 cpp/src/arrow/util/buffer.cc |  16 +-
 cpp/src/arrow/util/buffer.h  |   1 -
 cpp/src/arrow/util/logging.h |   4 +-
 python/CMakeLists.txt|   8 +-
 python/cmake_modules/FindArrow.cmake |  11 +
 python/pyarrow/__init__.py   |   3 +-
 python/pyarrow/array.pyx |  44 +---
 python/pyarrow/includes/common.pxd   |   4 -
 python/pyarrow/includes/libarrow.pxd |  29 ++-
 python/pyarrow/includes/libarrow_io.pxd  |  14 +-
 python/pyarrow/includes/libarrow_ipc.pxd |  52 
 python/pyarrow/includes/pyarrow.pxd  |  13 +-
 python/pyarrow/io.pxd|   6 +
 python/pyarrow/io.pyx| 340 --
 python/pyarrow/ipc.pyx   | 155 
 python/pyarrow/table.pxd |  17 +-
 python/pyarrow/table.pyx | 194 +++
 python/pyarrow/tests/test_array.py   |   4 +
 python/pyarrow/tests/test_io.py  |  41 
 python/pyarrow/tests/test_ipc.py | 116 +
 python/pyarrow/tests/test_table.py   |  82 +++
 python/setup.py  |   1 +
 python/src/pyarrow/adapters/builtin.cc   |   2 +-
 python/src/pyarrow/adapters/pandas.cc|   8 +
 python/src/pyarrow/common.cc |   2 +-
 python/src/pyarrow/common.h  |  20 +-
 python/src/pyarrow/io.cc |   6 +-
 37 files changed, 1012 insertions(+), 309 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/arrow/blob/a9747cea/cpp/src/arrow/io/io-memory-test.cc
--
diff --git a/cpp/src/arrow/io/io-memory-test.cc 
b/cpp/src/arrow/io/io-memory-test.cc
index 6de35da..a49faf3 100644
--- a/cpp/src/arrow/io/io-memory-test.cc
+++ b/cpp/src/arrow/io/io-memory-test.cc
@@ -121,5 +121,30 @@ TEST_F(TestMemoryMappedFile, InvalidFile) {
   IOError, MemoryMappedFile::Open(non_existent_path, FileMode::READ, 
));
 }
 
+class TestBufferOutputStream : public ::testing::Test {
+ public:
+  void SetUp() {
+buffer_.reset(new PoolBuffer(default_memory_pool()));
+stream_.reset(new BufferOutputStream(buffer_));
+  }
+
+ protected:
+  std::shared_ptr buffer_;
+  std::unique_ptr stream_;
+};
+
+TEST_F(TestBufferOutputStream, CloseResizes) {
+  std::string data = "data123456";
+
+  const int64_t nbytes = static_cast(data.size());
+  const int K = 100;
+  for (int i = 0; i < K; ++i) {
+EXPECT_OK(stream_->Write(reinterpret_cast(data.c_str()), 
nbytes));
+  }
+
+  ASSERT_OK(stream_->Close());
+  ASSERT_EQ(K * nbytes, buffer_->size());
+}
+
 }  // namespace io
 }  // namespace arrow

svn commit: r16464 - in /release/arrow: apache-arrow-0.1.0.tar.gz apache-arrow-0.1.0.tar.gz.asc apache-arrow-0.1.0.tar.gz.md5 apache-arrow-0.1.0.tar.gz.sha

2016-10-10 Thread julien 
Author: julien
Date: Tue Oct 11 01:29:30 2016
New Revision: 16464

Log:
apache arrow release 0.1.0

Added:
release/arrow/apache-arrow-0.1.0.tar.gz   (with props)
release/arrow/apache-arrow-0.1.0.tar.gz.asc
release/arrow/apache-arrow-0.1.0.tar.gz.md5
release/arrow/apache-arrow-0.1.0.tar.gz.sha

Added: release/arrow/apache-arrow-0.1.0.tar.gz
==
Binary file - no diff available.

Propchange: release/arrow/apache-arrow-0.1.0.tar.gz
--
svn:mime-type = application/octet-stream

Added: release/arrow/apache-arrow-0.1.0.tar.gz.asc
==
--- release/arrow/apache-arrow-0.1.0.tar.gz.asc (added)
+++ release/arrow/apache-arrow-0.1.0.tar.gz.asc Tue Oct 11 01:29:30 2016
@@ -0,0 +1,16 @@
+-BEGIN PGP SIGNATURE-
+
+iQIcBAABAgAGBQJX+D6DAAoJEALav98WedGU5LkQALgZgp9MYSgaDCwWHNrvGn6I
+awXEjXg7TOhZdDBBM7MyT/WGBV42npwdB/vC37bXccCsNszp5urx8tiZVd1VoPGB
+tN/+UEe9ktcxADIqA6G937vKM8nMZP5vQGecnIrDVpM9PsaFucQfY/mDCrNpm3Pl
+rvYEfGzaAnxQ5MsqibwaH2MSUerKaWN3uukI3XQYeXuG1yIopcmaV1w7vlRTRRaO
+HTeeB0dNGiI+TChXhiRe9VQhEo8wRyoIkXgA2q5lf+BvLEYQs/x6Gl6w0GkKf1Ry
+50HiGfw518iHZfsuDer4zY/DgzzkfpYs5n6jhT23ExtwdnLhKtURBeLpSoZP/HXp
+1KiMpuK2SuH7VwsWaOTf2HoEbIukZNlyvnzjCCBWMPpR0xUzj5NgeHjJXOaNtrdS
+qDbxEuVCE1HIxe+UMYqNhkl3vE4Lp+hIrxTF7VOhOQSm7dHoE8vtRvf3Q0CiujkH
+rmIl3c8vlklN5MnqFbfs/WiqPZKU+53j/6IpySOs2CcpQWEw0gP6VvDm0VY12N/W
+U1ce52dB0pLlGXAPkbzBHK8IXgaHxevjMlvDl6JnAVWFSbdqvoQlFDDc5ssj1itu
+DFGr883PXKKthcPJ90eqr/kGxaPBs+BW2v+hhjh3Xqb8bIsoOTeAAwZVxqr8i+t6
+ZPIwi9jCXEXbdPHiWkDu
+=kZtB
+-END PGP SIGNATURE-

Added: release/arrow/apache-arrow-0.1.0.tar.gz.md5
==
--- release/arrow/apache-arrow-0.1.0.tar.gz.md5 (added)
+++ release/arrow/apache-arrow-0.1.0.tar.gz.md5 Tue Oct 11 01:29:30 2016
@@ -0,0 +1 @@
+apache-arrow-0.1.0.tar.gz: FF EA 47 12 B8 B6 16 65  84 16 70 AD F9 B7 7A 2D

Added: release/arrow/apache-arrow-0.1.0.tar.gz.sha
==
--- release/arrow/apache-arrow-0.1.0.tar.gz.sha (added)
+++ release/arrow/apache-arrow-0.1.0.tar.gz.sha Tue Oct 11 01:29:30 2016
@@ -0,0 +1 @@
+f33e61fcd58ed2279c075934be63c2a2678137d9  apache-arrow-0.1.0.tar.gz

arrow git commit: ARROW-312: Read and write Arrow IPC file format from Python

2016-10-10 Thread wesm 
Repository: arrow
Updated Branches:
  refs/heads/master eb1491a96 -> 772800ace


ARROW-312: Read and write Arrow IPC file format from Python

This also adds some IO scaffolding for interacting with `arrow::Buffer` objects 
from Python and assorted additions to help with testing.

Author: Wes McKinney 

Closes #164 from wesm/ARROW-312 and squashes the following commits:

7df3e5f [Wes McKinney] Set BUILD_WITH_INSTALL_RPATH on arrow_ipc
be8cee0 [Wes McKinney] Link Cython modules to libarrow* libraries
5716601 [Wes McKinney] Fix accidental deletion
77fb03b [Wes McKinney] Add / test Buffer wrapper. Test that we can write an 
arrow file to a wrapped buffer. Resize buffer in BufferOutputStream on close
316537d [Wes McKinney] Get ready to wrap Arrow buffers in a Python object
4822d32 [Wes McKinney] Implement RecordBatch::Equals, compare in Python ipc 
file writes
a931e49 [Wes McKinney] Permit buffers (write padding) in a non-multiple of 64 
in an IPC context, to allow zero-copy writing of NumPy arrays
2c49cd4 [Wes McKinney] Some debugging
ca1562b [Wes McKinney] Draft implementations of Arrow file read/write from 
Python


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/772800ac
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/772800ac
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/772800ac

Branch: refs/heads/master
Commit: 772800ace3706b8f169803b3ef8932e04ec54619
Parents: eb1491a
Author: Wes McKinney 
Authored: Mon Oct 10 11:21:49 2016 -0400
Committer: Wes McKinney 
Committed: Mon Oct 10 11:21:49 2016 -0400

--
 cpp/src/arrow/io/io-memory-test.cc   |  25 ++
 cpp/src/arrow/io/memory.cc   |  13 +-
 cpp/src/arrow/ipc/CMakeLists.txt |   7 +
 cpp/src/arrow/ipc/adapter.cc |  16 +-
 cpp/src/arrow/ipc/util.h |   6 +-
 cpp/src/arrow/table-test.cc  |  27 ++
 cpp/src/arrow/table.cc   |  16 ++
 cpp/src/arrow/table.h|   2 +
 cpp/src/arrow/types/primitive-test.cc|   3 +-
 cpp/src/arrow/util/bit-util.h|  13 +
 cpp/src/arrow/util/buffer.cc |  16 +-
 cpp/src/arrow/util/buffer.h  |   1 -
 cpp/src/arrow/util/logging.h |   4 +-
 python/CMakeLists.txt|   8 +-
 python/cmake_modules/FindArrow.cmake |  11 +
 python/pyarrow/__init__.py   |   3 +-
 python/pyarrow/array.pyx |  44 +---
 python/pyarrow/includes/common.pxd   |   4 -
 python/pyarrow/includes/libarrow.pxd |  29 ++-
 python/pyarrow/includes/libarrow_io.pxd  |  14 +-
 python/pyarrow/includes/libarrow_ipc.pxd |  52 
 python/pyarrow/includes/pyarrow.pxd  |  13 +-
 python/pyarrow/io.pxd|   6 +
 python/pyarrow/io.pyx| 340 --
 python/pyarrow/ipc.pyx   | 155 
 python/pyarrow/table.pxd |  17 +-
 python/pyarrow/table.pyx | 194 +++
 python/pyarrow/tests/test_array.py   |   4 +
 python/pyarrow/tests/test_io.py  |  41 
 python/pyarrow/tests/test_ipc.py | 116 +
 python/pyarrow/tests/test_table.py   |  82 +++
 python/setup.py  |   1 +
 python/src/pyarrow/adapters/builtin.cc   |   2 +-
 python/src/pyarrow/adapters/pandas.cc|   8 +
 python/src/pyarrow/common.cc |   2 +-
 python/src/pyarrow/common.h  |  20 +-
 python/src/pyarrow/io.cc |   6 +-
 37 files changed, 1012 insertions(+), 309 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/arrow/blob/772800ac/cpp/src/arrow/io/io-memory-test.cc
--
diff --git a/cpp/src/arrow/io/io-memory-test.cc 
b/cpp/src/arrow/io/io-memory-test.cc
index 6de35da..a49faf3 100644
--- a/cpp/src/arrow/io/io-memory-test.cc
+++ b/cpp/src/arrow/io/io-memory-test.cc
@@ -121,5 +121,30 @@ TEST_F(TestMemoryMappedFile, InvalidFile) {
   IOError, MemoryMappedFile::Open(non_existent_path, FileMode::READ, 
));
 }
 
+class TestBufferOutputStream : public ::testing::Test {
+ public:
+  void SetUp() {
+buffer_.reset(new PoolBuffer(default_memory_pool()));
+stream_.reset(new BufferOutputStream(buffer_));
+  }
+
+ protected:
+  std::shared_ptr buffer_;
+  std::unique_ptr stream_;
+};
+
+TEST_F(TestBufferOutputStream, CloseResizes) {
+  std::string data = "data123456";
+
+  const int64_t nbytes = static_cast(data.size());
+  const int K = 100;
+  for (int i = 0; i < K; ++i) {
+EXPECT_OK(stream_->Write(reinterpret_cast(data.c_str()), 
nbytes));
+  }
+
+  ASSERT_OK(stream_->Close());
+  ASSERT_EQ(K * nbytes, buffer_->size());
+}
+
 }  // namespace