subject:"arrow git commit\: ARROW\-312\: Read and write Arrow IPC file format from Python"

[1/2] arrow git commit: ARROW-312: Read and write Arrow IPC file format from Python

2016-10-10 Thread smp

Repository: arrow
Updated Branches:
  refs/heads/arrow-326 [created] fb29195e0


ARROW-312: Read and write Arrow IPC file format from Python

This also adds some IO scaffolding for interacting with `arrow::Buffer` objects 
from Python and assorted additions to help with testing.

Author: Wes McKinney 

Closes #164 from wesm/ARROW-312 and squashes the following commits:

7df3e5f [Wes McKinney] Set BUILD_WITH_INSTALL_RPATH on arrow_ipc
be8cee0 [Wes McKinney] Link Cython modules to libarrow* libraries
5716601 [Wes McKinney] Fix accidental deletion
77fb03b [Wes McKinney] Add / test Buffer wrapper. Test that we can write an 
arrow file to a wrapped buffer. Resize buffer in BufferOutputStream on close
316537d [Wes McKinney] Get ready to wrap Arrow buffers in a Python object
4822d32 [Wes McKinney] Implement RecordBatch::Equals, compare in Python ipc 
file writes
a931e49 [Wes McKinney] Permit buffers (write padding) in a non-multiple of 64 
in an IPC context, to allow zero-copy writing of NumPy arrays
2c49cd4 [Wes McKinney] Some debugging
ca1562b [Wes McKinney] Draft implementations of Arrow file read/write from 
Python


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/772800ac
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/772800ac
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/772800ac

Branch: refs/heads/arrow-326
Commit: 772800ace3706b8f169803b3ef8932e04ec54619
Parents: eb1491a
Author: Wes McKinney 
Authored: Mon Oct 10 11:21:49 2016 -0400
Committer: Wes McKinney 
Committed: Mon Oct 10 11:21:49 2016 -0400

--
 cpp/src/arrow/io/io-memory-test.cc   |  25 ++
 cpp/src/arrow/io/memory.cc   |  13 +-
 cpp/src/arrow/ipc/CMakeLists.txt |   7 +
 cpp/src/arrow/ipc/adapter.cc |  16 +-
 cpp/src/arrow/ipc/util.h |   6 +-
 cpp/src/arrow/table-test.cc  |  27 ++
 cpp/src/arrow/table.cc   |  16 ++
 cpp/src/arrow/table.h|   2 +
 cpp/src/arrow/types/primitive-test.cc|   3 +-
 cpp/src/arrow/util/bit-util.h|  13 +
 cpp/src/arrow/util/buffer.cc |  16 +-
 cpp/src/arrow/util/buffer.h  |   1 -
 cpp/src/arrow/util/logging.h |   4 +-
 python/CMakeLists.txt|   8 +-
 python/cmake_modules/FindArrow.cmake |  11 +
 python/pyarrow/__init__.py   |   3 +-
 python/pyarrow/array.pyx |  44 +---
 python/pyarrow/includes/common.pxd   |   4 -
 python/pyarrow/includes/libarrow.pxd |  29 ++-
 python/pyarrow/includes/libarrow_io.pxd  |  14 +-
 python/pyarrow/includes/libarrow_ipc.pxd |  52 
 python/pyarrow/includes/pyarrow.pxd  |  13 +-
 python/pyarrow/io.pxd|   6 +
 python/pyarrow/io.pyx| 340 --
 python/pyarrow/ipc.pyx   | 155 
 python/pyarrow/table.pxd |  17 +-
 python/pyarrow/table.pyx | 194 +++
 python/pyarrow/tests/test_array.py   |   4 +
 python/pyarrow/tests/test_io.py  |  41 
 python/pyarrow/tests/test_ipc.py | 116 +
 python/pyarrow/tests/test_table.py   |  82 +++
 python/setup.py  |   1 +
 python/src/pyarrow/adapters/builtin.cc   |   2 +-
 python/src/pyarrow/adapters/pandas.cc|   8 +
 python/src/pyarrow/common.cc |   2 +-
 python/src/pyarrow/common.h  |  20 +-
 python/src/pyarrow/io.cc |   6 +-
 37 files changed, 1012 insertions(+), 309 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/arrow/blob/772800ac/cpp/src/arrow/io/io-memory-test.cc
--
diff --git a/cpp/src/arrow/io/io-memory-test.cc 
b/cpp/src/arrow/io/io-memory-test.cc
index 6de35da..a49faf3 100644
--- a/cpp/src/arrow/io/io-memory-test.cc
+++ b/cpp/src/arrow/io/io-memory-test.cc
@@ -121,5 +121,30 @@ TEST_F(TestMemoryMappedFile, InvalidFile) {
   IOError, MemoryMappedFile::Open(non_existent_path, FileMode::READ, 
));
 }
 
+class TestBufferOutputStream : public ::testing::Test {
+ public:
+  void SetUp() {
+buffer_.reset(new PoolBuffer(default_memory_pool()));
+stream_.reset(new BufferOutputStream(buffer_));
+  }
+
+ protected:
+  std::shared_ptr buffer_;
+  std::unique_ptr stream_;
+};
+
+TEST_F(TestBufferOutputStream, CloseResizes) {
+  std::string data = "data123456";
+
+  const int64_t nbytes = static_cast(data.size());
+  const int K = 100;
+  for (int i = 0; i < K; ++i) {
+EXPECT_OK(stream_->Write(reinterpret_cast(data.c_str()), 
nbytes));
+  }
+
+  ASSERT_OK(stream_->Close());
+  ASSERT_EQ(K * nbytes, buffer_->size());
+}
+
 }  //

[3/3] arrow git commit: ARROW-312: Read and write Arrow IPC file format from Python

2016-10-10 Thread julien

ARROW-312: Read and write Arrow IPC file format from Python

This also adds some IO scaffolding for interacting with `arrow::Buffer` objects 
from Python and assorted additions to help with testing.

Author: Wes McKinney 

Closes #164 from wesm/ARROW-312 and squashes the following commits:

7df3e5f [Wes McKinney] Set BUILD_WITH_INSTALL_RPATH on arrow_ipc
be8cee0 [Wes McKinney] Link Cython modules to libarrow* libraries
5716601 [Wes McKinney] Fix accidental deletion
77fb03b [Wes McKinney] Add / test Buffer wrapper. Test that we can write an 
arrow file to a wrapped buffer. Resize buffer in BufferOutputStream on close
316537d [Wes McKinney] Get ready to wrap Arrow buffers in a Python object
4822d32 [Wes McKinney] Implement RecordBatch::Equals, compare in Python ipc 
file writes
a931e49 [Wes McKinney] Permit buffers (write padding) in a non-multiple of 64 
in an IPC context, to allow zero-copy writing of NumPy arrays
2c49cd4 [Wes McKinney] Some debugging
ca1562b [Wes McKinney] Draft implementations of Arrow file read/write from 
Python


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/a9747cea
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/a9747cea
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/a9747cea

Branch: refs/heads/master
Commit: a9747ceac2b6399c6acf027de8074d8661d5eb1d
Parents: 17cd7a6
Author: Wes McKinney 
Authored: Mon Oct 10 11:21:49 2016 -0400
Committer: Julien Le Dem 
Committed: Mon Oct 10 18:42:05 2016 -0700

--
 cpp/src/arrow/io/io-memory-test.cc   |  25 ++
 cpp/src/arrow/io/memory.cc   |  13 +-
 cpp/src/arrow/ipc/CMakeLists.txt |   7 +
 cpp/src/arrow/ipc/adapter.cc |  16 +-
 cpp/src/arrow/ipc/util.h |   6 +-
 cpp/src/arrow/table-test.cc  |  27 ++
 cpp/src/arrow/table.cc   |  16 ++
 cpp/src/arrow/table.h|   2 +
 cpp/src/arrow/types/primitive-test.cc|   3 +-
 cpp/src/arrow/util/bit-util.h|  13 +
 cpp/src/arrow/util/buffer.cc |  16 +-
 cpp/src/arrow/util/buffer.h  |   1 -
 cpp/src/arrow/util/logging.h |   4 +-
 python/CMakeLists.txt|   8 +-
 python/cmake_modules/FindArrow.cmake |  11 +
 python/pyarrow/__init__.py   |   3 +-
 python/pyarrow/array.pyx |  44 +---
 python/pyarrow/includes/common.pxd   |   4 -
 python/pyarrow/includes/libarrow.pxd |  29 ++-
 python/pyarrow/includes/libarrow_io.pxd  |  14 +-
 python/pyarrow/includes/libarrow_ipc.pxd |  52 
 python/pyarrow/includes/pyarrow.pxd  |  13 +-
 python/pyarrow/io.pxd|   6 +
 python/pyarrow/io.pyx| 340 --
 python/pyarrow/ipc.pyx   | 155 
 python/pyarrow/table.pxd |  17 +-
 python/pyarrow/table.pyx | 194 +++
 python/pyarrow/tests/test_array.py   |   4 +
 python/pyarrow/tests/test_io.py  |  41 
 python/pyarrow/tests/test_ipc.py | 116 +
 python/pyarrow/tests/test_table.py   |  82 +++
 python/setup.py  |   1 +
 python/src/pyarrow/adapters/builtin.cc   |   2 +-
 python/src/pyarrow/adapters/pandas.cc|   8 +
 python/src/pyarrow/common.cc |   2 +-
 python/src/pyarrow/common.h  |  20 +-
 python/src/pyarrow/io.cc |   6 +-
 37 files changed, 1012 insertions(+), 309 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/arrow/blob/a9747cea/cpp/src/arrow/io/io-memory-test.cc
--
diff --git a/cpp/src/arrow/io/io-memory-test.cc 
b/cpp/src/arrow/io/io-memory-test.cc
index 6de35da..a49faf3 100644
--- a/cpp/src/arrow/io/io-memory-test.cc
+++ b/cpp/src/arrow/io/io-memory-test.cc
@@ -121,5 +121,30 @@ TEST_F(TestMemoryMappedFile, InvalidFile) {
   IOError, MemoryMappedFile::Open(non_existent_path, FileMode::READ, 
));
 }
 
+class TestBufferOutputStream : public ::testing::Test {
+ public:
+  void SetUp() {
+buffer_.reset(new PoolBuffer(default_memory_pool()));
+stream_.reset(new BufferOutputStream(buffer_));
+  }
+
+ protected:
+  std::shared_ptr buffer_;
+  std::unique_ptr stream_;
+};
+
+TEST_F(TestBufferOutputStream, CloseResizes) {
+  std::string data = "data123456";
+
+  const int64_t nbytes = static_cast(data.size());
+  const int K = 100;
+  for (int i = 0; i < K; ++i) {
+EXPECT_OK(stream_->Write(reinterpret_cast(data.c_str()), 
nbytes));
+  }
+
+  ASSERT_OK(stream_->Close());
+  ASSERT_EQ(K * nbytes, buffer_->size());
+}
+
 }  // namespace io
 }  // namespace arrow

arrow git commit: ARROW-312: Read and write Arrow IPC file format from Python

2016-10-10 Thread wesm

Repository: arrow
Updated Branches:
  refs/heads/master eb1491a96 -> 772800ace


ARROW-312: Read and write Arrow IPC file format from Python

This also adds some IO scaffolding for interacting with `arrow::Buffer` objects 
from Python and assorted additions to help with testing.

Author: Wes McKinney 

Closes #164 from wesm/ARROW-312 and squashes the following commits:

7df3e5f [Wes McKinney] Set BUILD_WITH_INSTALL_RPATH on arrow_ipc
be8cee0 [Wes McKinney] Link Cython modules to libarrow* libraries
5716601 [Wes McKinney] Fix accidental deletion
77fb03b [Wes McKinney] Add / test Buffer wrapper. Test that we can write an 
arrow file to a wrapped buffer. Resize buffer in BufferOutputStream on close
316537d [Wes McKinney] Get ready to wrap Arrow buffers in a Python object
4822d32 [Wes McKinney] Implement RecordBatch::Equals, compare in Python ipc 
file writes
a931e49 [Wes McKinney] Permit buffers (write padding) in a non-multiple of 64 
in an IPC context, to allow zero-copy writing of NumPy arrays
2c49cd4 [Wes McKinney] Some debugging
ca1562b [Wes McKinney] Draft implementations of Arrow file read/write from 
Python


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/772800ac
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/772800ac
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/772800ac

Branch: refs/heads/master
Commit: 772800ace3706b8f169803b3ef8932e04ec54619
Parents: eb1491a
Author: Wes McKinney 
Authored: Mon Oct 10 11:21:49 2016 -0400
Committer: Wes McKinney 
Committed: Mon Oct 10 11:21:49 2016 -0400

--
 cpp/src/arrow/io/io-memory-test.cc   |  25 ++
 cpp/src/arrow/io/memory.cc   |  13 +-
 cpp/src/arrow/ipc/CMakeLists.txt |   7 +
 cpp/src/arrow/ipc/adapter.cc |  16 +-
 cpp/src/arrow/ipc/util.h |   6 +-
 cpp/src/arrow/table-test.cc  |  27 ++
 cpp/src/arrow/table.cc   |  16 ++
 cpp/src/arrow/table.h|   2 +
 cpp/src/arrow/types/primitive-test.cc|   3 +-
 cpp/src/arrow/util/bit-util.h|  13 +
 cpp/src/arrow/util/buffer.cc |  16 +-
 cpp/src/arrow/util/buffer.h  |   1 -
 cpp/src/arrow/util/logging.h |   4 +-
 python/CMakeLists.txt|   8 +-
 python/cmake_modules/FindArrow.cmake |  11 +
 python/pyarrow/__init__.py   |   3 +-
 python/pyarrow/array.pyx |  44 +---
 python/pyarrow/includes/common.pxd   |   4 -
 python/pyarrow/includes/libarrow.pxd |  29 ++-
 python/pyarrow/includes/libarrow_io.pxd  |  14 +-
 python/pyarrow/includes/libarrow_ipc.pxd |  52 
 python/pyarrow/includes/pyarrow.pxd  |  13 +-
 python/pyarrow/io.pxd|   6 +
 python/pyarrow/io.pyx| 340 --
 python/pyarrow/ipc.pyx   | 155 
 python/pyarrow/table.pxd |  17 +-
 python/pyarrow/table.pyx | 194 +++
 python/pyarrow/tests/test_array.py   |   4 +
 python/pyarrow/tests/test_io.py  |  41 
 python/pyarrow/tests/test_ipc.py | 116 +
 python/pyarrow/tests/test_table.py   |  82 +++
 python/setup.py  |   1 +
 python/src/pyarrow/adapters/builtin.cc   |   2 +-
 python/src/pyarrow/adapters/pandas.cc|   8 +
 python/src/pyarrow/common.cc |   2 +-
 python/src/pyarrow/common.h  |  20 +-
 python/src/pyarrow/io.cc |   6 +-
 37 files changed, 1012 insertions(+), 309 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/arrow/blob/772800ac/cpp/src/arrow/io/io-memory-test.cc
--
diff --git a/cpp/src/arrow/io/io-memory-test.cc 
b/cpp/src/arrow/io/io-memory-test.cc
index 6de35da..a49faf3 100644
--- a/cpp/src/arrow/io/io-memory-test.cc
+++ b/cpp/src/arrow/io/io-memory-test.cc
@@ -121,5 +121,30 @@ TEST_F(TestMemoryMappedFile, InvalidFile) {
   IOError, MemoryMappedFile::Open(non_existent_path, FileMode::READ, 
));
 }
 
+class TestBufferOutputStream : public ::testing::Test {
+ public:
+  void SetUp() {
+buffer_.reset(new PoolBuffer(default_memory_pool()));
+stream_.reset(new BufferOutputStream(buffer_));
+  }
+
+ protected:
+  std::shared_ptr buffer_;
+  std::unique_ptr stream_;
+};
+
+TEST_F(TestBufferOutputStream, CloseResizes) {
+  std::string data = "data123456";
+
+  const int64_t nbytes = static_cast(data.size());
+  const int K = 100;
+  for (int i = 0; i < K; ++i) {
+EXPECT_OK(stream_->Write(reinterpret_cast(data.c_str()), 
nbytes));
+  }
+
+  ASSERT_OK(stream_->Close());
+  ASSERT_EQ(K * nbytes, buffer_->size());
+}
+
 }  // namespace

[1/2] arrow git commit: ARROW-312: Read and write Arrow IPC file format from Python

[3/3] arrow git commit: ARROW-312: Read and write Arrow IPC file format from Python

arrow git commit: ARROW-312: Read and write Arrow IPC file format from Python

3 matches

Site Navigation

Mail list logo

Footer information