[1/2] arrow git commit: ARROW-312: Read and write Arrow IPC file format from Python
Repository: arrow Updated Branches: refs/heads/arrow-326 [created] fb29195e0 ARROW-312: Read and write Arrow IPC file format from Python This also adds some IO scaffolding for interacting with `arrow::Buffer` objects from Python and assorted additions to help with testing. Author: Wes McKinney Closes #164 from wesm/ARROW-312 and squashes the following commits: 7df3e5f [Wes McKinney] Set BUILD_WITH_INSTALL_RPATH on arrow_ipc be8cee0 [Wes McKinney] Link Cython modules to libarrow* libraries 5716601 [Wes McKinney] Fix accidental deletion 77fb03b [Wes McKinney] Add / test Buffer wrapper. Test that we can write an arrow file to a wrapped buffer. Resize buffer in BufferOutputStream on close 316537d [Wes McKinney] Get ready to wrap Arrow buffers in a Python object 4822d32 [Wes McKinney] Implement RecordBatch::Equals, compare in Python ipc file writes a931e49 [Wes McKinney] Permit buffers (write padding) in a non-multiple of 64 in an IPC context, to allow zero-copy writing of NumPy arrays 2c49cd4 [Wes McKinney] Some debugging ca1562b [Wes McKinney] Draft implementations of Arrow file read/write from Python Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/772800ac Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/772800ac Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/772800ac Branch: refs/heads/arrow-326 Commit: 772800ace3706b8f169803b3ef8932e04ec54619 Parents: eb1491a Author: Wes McKinney Authored: Mon Oct 10 11:21:49 2016 -0400 Committer: Wes McKinney Committed: Mon Oct 10 11:21:49 2016 -0400 -- cpp/src/arrow/io/io-memory-test.cc | 25 ++ cpp/src/arrow/io/memory.cc | 13 +- cpp/src/arrow/ipc/CMakeLists.txt | 7 + cpp/src/arrow/ipc/adapter.cc | 16 +- cpp/src/arrow/ipc/util.h | 6 +- cpp/src/arrow/table-test.cc | 27 ++ cpp/src/arrow/table.cc | 16 ++ cpp/src/arrow/table.h| 2 + cpp/src/arrow/types/primitive-test.cc| 3 +- cpp/src/arrow/util/bit-util.h| 13 + cpp/src/arrow/util/buffer.cc | 16 +- cpp/src/arrow/util/buffer.h | 1 - cpp/src/arrow/util/logging.h | 4 +- python/CMakeLists.txt| 8 +- python/cmake_modules/FindArrow.cmake | 11 + python/pyarrow/__init__.py | 3 +- python/pyarrow/array.pyx | 44 +--- python/pyarrow/includes/common.pxd | 4 - python/pyarrow/includes/libarrow.pxd | 29 ++- python/pyarrow/includes/libarrow_io.pxd | 14 +- python/pyarrow/includes/libarrow_ipc.pxd | 52 python/pyarrow/includes/pyarrow.pxd | 13 +- python/pyarrow/io.pxd| 6 + python/pyarrow/io.pyx| 340 -- python/pyarrow/ipc.pyx | 155 python/pyarrow/table.pxd | 17 +- python/pyarrow/table.pyx | 194 +++ python/pyarrow/tests/test_array.py | 4 + python/pyarrow/tests/test_io.py | 41 python/pyarrow/tests/test_ipc.py | 116 + python/pyarrow/tests/test_table.py | 82 +++ python/setup.py | 1 + python/src/pyarrow/adapters/builtin.cc | 2 +- python/src/pyarrow/adapters/pandas.cc| 8 + python/src/pyarrow/common.cc | 2 +- python/src/pyarrow/common.h | 20 +- python/src/pyarrow/io.cc | 6 +- 37 files changed, 1012 insertions(+), 309 deletions(-) -- http://git-wip-us.apache.org/repos/asf/arrow/blob/772800ac/cpp/src/arrow/io/io-memory-test.cc -- diff --git a/cpp/src/arrow/io/io-memory-test.cc b/cpp/src/arrow/io/io-memory-test.cc index 6de35da..a49faf3 100644 --- a/cpp/src/arrow/io/io-memory-test.cc +++ b/cpp/src/arrow/io/io-memory-test.cc @@ -121,5 +121,30 @@ TEST_F(TestMemoryMappedFile, InvalidFile) { IOError, MemoryMappedFile::Open(non_existent_path, FileMode::READ, &result)); } +class TestBufferOutputStream : public ::testing::Test { + public: + void SetUp() { +buffer_.reset(new PoolBuffer(default_memory_pool())); +stream_.reset(new BufferOutputStream(buffer_)); + } + + protected: + std::shared_ptr buffer_; + std::unique_ptr stream_; +}; + +TEST_F(TestBufferOutputStream, CloseResizes) { + std::string data = "data123456"; + + const int64_t nbytes = static_cast(data.size()); + const int K = 100; + for (int i = 0; i < K; ++i) { +EXPECT_OK(stream_->Write(reinterpret_cast(data.c_str()), nbytes)); + } + + ASSERT_OK(stream_->Close()); + ASSERT_EQ(K * nbytes, buffer_->size()); +} + } // namespace io } // namespace arrow http://git-wip-us.apache.org/repos/asf/arrow
[3/3] arrow git commit: ARROW-312: Read and write Arrow IPC file format from Python
ARROW-312: Read and write Arrow IPC file format from Python This also adds some IO scaffolding for interacting with `arrow::Buffer` objects from Python and assorted additions to help with testing. Author: Wes McKinney Closes #164 from wesm/ARROW-312 and squashes the following commits: 7df3e5f [Wes McKinney] Set BUILD_WITH_INSTALL_RPATH on arrow_ipc be8cee0 [Wes McKinney] Link Cython modules to libarrow* libraries 5716601 [Wes McKinney] Fix accidental deletion 77fb03b [Wes McKinney] Add / test Buffer wrapper. Test that we can write an arrow file to a wrapped buffer. Resize buffer in BufferOutputStream on close 316537d [Wes McKinney] Get ready to wrap Arrow buffers in a Python object 4822d32 [Wes McKinney] Implement RecordBatch::Equals, compare in Python ipc file writes a931e49 [Wes McKinney] Permit buffers (write padding) in a non-multiple of 64 in an IPC context, to allow zero-copy writing of NumPy arrays 2c49cd4 [Wes McKinney] Some debugging ca1562b [Wes McKinney] Draft implementations of Arrow file read/write from Python Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/a9747cea Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/a9747cea Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/a9747cea Branch: refs/heads/master Commit: a9747ceac2b6399c6acf027de8074d8661d5eb1d Parents: 17cd7a6 Author: Wes McKinney Authored: Mon Oct 10 11:21:49 2016 -0400 Committer: Julien Le Dem Committed: Mon Oct 10 18:42:05 2016 -0700 -- cpp/src/arrow/io/io-memory-test.cc | 25 ++ cpp/src/arrow/io/memory.cc | 13 +- cpp/src/arrow/ipc/CMakeLists.txt | 7 + cpp/src/arrow/ipc/adapter.cc | 16 +- cpp/src/arrow/ipc/util.h | 6 +- cpp/src/arrow/table-test.cc | 27 ++ cpp/src/arrow/table.cc | 16 ++ cpp/src/arrow/table.h| 2 + cpp/src/arrow/types/primitive-test.cc| 3 +- cpp/src/arrow/util/bit-util.h| 13 + cpp/src/arrow/util/buffer.cc | 16 +- cpp/src/arrow/util/buffer.h | 1 - cpp/src/arrow/util/logging.h | 4 +- python/CMakeLists.txt| 8 +- python/cmake_modules/FindArrow.cmake | 11 + python/pyarrow/__init__.py | 3 +- python/pyarrow/array.pyx | 44 +--- python/pyarrow/includes/common.pxd | 4 - python/pyarrow/includes/libarrow.pxd | 29 ++- python/pyarrow/includes/libarrow_io.pxd | 14 +- python/pyarrow/includes/libarrow_ipc.pxd | 52 python/pyarrow/includes/pyarrow.pxd | 13 +- python/pyarrow/io.pxd| 6 + python/pyarrow/io.pyx| 340 -- python/pyarrow/ipc.pyx | 155 python/pyarrow/table.pxd | 17 +- python/pyarrow/table.pyx | 194 +++ python/pyarrow/tests/test_array.py | 4 + python/pyarrow/tests/test_io.py | 41 python/pyarrow/tests/test_ipc.py | 116 + python/pyarrow/tests/test_table.py | 82 +++ python/setup.py | 1 + python/src/pyarrow/adapters/builtin.cc | 2 +- python/src/pyarrow/adapters/pandas.cc| 8 + python/src/pyarrow/common.cc | 2 +- python/src/pyarrow/common.h | 20 +- python/src/pyarrow/io.cc | 6 +- 37 files changed, 1012 insertions(+), 309 deletions(-) -- http://git-wip-us.apache.org/repos/asf/arrow/blob/a9747cea/cpp/src/arrow/io/io-memory-test.cc -- diff --git a/cpp/src/arrow/io/io-memory-test.cc b/cpp/src/arrow/io/io-memory-test.cc index 6de35da..a49faf3 100644 --- a/cpp/src/arrow/io/io-memory-test.cc +++ b/cpp/src/arrow/io/io-memory-test.cc @@ -121,5 +121,30 @@ TEST_F(TestMemoryMappedFile, InvalidFile) { IOError, MemoryMappedFile::Open(non_existent_path, FileMode::READ, &result)); } +class TestBufferOutputStream : public ::testing::Test { + public: + void SetUp() { +buffer_.reset(new PoolBuffer(default_memory_pool())); +stream_.reset(new BufferOutputStream(buffer_)); + } + + protected: + std::shared_ptr buffer_; + std::unique_ptr stream_; +}; + +TEST_F(TestBufferOutputStream, CloseResizes) { + std::string data = "data123456"; + + const int64_t nbytes = static_cast(data.size()); + const int K = 100; + for (int i = 0; i < K; ++i) { +EXPECT_OK(stream_->Write(reinterpret_cast(data.c_str()), nbytes)); + } + + ASSERT_OK(stream_->Close()); + ASSERT_EQ(K * nbytes, buffer_->size()); +} + } // namespace io } // namespace arrow http://git-wip-us.apache.org/repos/asf/arrow/blob/a9747cea/cpp/src/arrow/io/memory.cc -
arrow git commit: ARROW-312: Read and write Arrow IPC file format from Python
Repository: arrow Updated Branches: refs/heads/master eb1491a96 -> 772800ace ARROW-312: Read and write Arrow IPC file format from Python This also adds some IO scaffolding for interacting with `arrow::Buffer` objects from Python and assorted additions to help with testing. Author: Wes McKinney Closes #164 from wesm/ARROW-312 and squashes the following commits: 7df3e5f [Wes McKinney] Set BUILD_WITH_INSTALL_RPATH on arrow_ipc be8cee0 [Wes McKinney] Link Cython modules to libarrow* libraries 5716601 [Wes McKinney] Fix accidental deletion 77fb03b [Wes McKinney] Add / test Buffer wrapper. Test that we can write an arrow file to a wrapped buffer. Resize buffer in BufferOutputStream on close 316537d [Wes McKinney] Get ready to wrap Arrow buffers in a Python object 4822d32 [Wes McKinney] Implement RecordBatch::Equals, compare in Python ipc file writes a931e49 [Wes McKinney] Permit buffers (write padding) in a non-multiple of 64 in an IPC context, to allow zero-copy writing of NumPy arrays 2c49cd4 [Wes McKinney] Some debugging ca1562b [Wes McKinney] Draft implementations of Arrow file read/write from Python Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/772800ac Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/772800ac Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/772800ac Branch: refs/heads/master Commit: 772800ace3706b8f169803b3ef8932e04ec54619 Parents: eb1491a Author: Wes McKinney Authored: Mon Oct 10 11:21:49 2016 -0400 Committer: Wes McKinney Committed: Mon Oct 10 11:21:49 2016 -0400 -- cpp/src/arrow/io/io-memory-test.cc | 25 ++ cpp/src/arrow/io/memory.cc | 13 +- cpp/src/arrow/ipc/CMakeLists.txt | 7 + cpp/src/arrow/ipc/adapter.cc | 16 +- cpp/src/arrow/ipc/util.h | 6 +- cpp/src/arrow/table-test.cc | 27 ++ cpp/src/arrow/table.cc | 16 ++ cpp/src/arrow/table.h| 2 + cpp/src/arrow/types/primitive-test.cc| 3 +- cpp/src/arrow/util/bit-util.h| 13 + cpp/src/arrow/util/buffer.cc | 16 +- cpp/src/arrow/util/buffer.h | 1 - cpp/src/arrow/util/logging.h | 4 +- python/CMakeLists.txt| 8 +- python/cmake_modules/FindArrow.cmake | 11 + python/pyarrow/__init__.py | 3 +- python/pyarrow/array.pyx | 44 +--- python/pyarrow/includes/common.pxd | 4 - python/pyarrow/includes/libarrow.pxd | 29 ++- python/pyarrow/includes/libarrow_io.pxd | 14 +- python/pyarrow/includes/libarrow_ipc.pxd | 52 python/pyarrow/includes/pyarrow.pxd | 13 +- python/pyarrow/io.pxd| 6 + python/pyarrow/io.pyx| 340 -- python/pyarrow/ipc.pyx | 155 python/pyarrow/table.pxd | 17 +- python/pyarrow/table.pyx | 194 +++ python/pyarrow/tests/test_array.py | 4 + python/pyarrow/tests/test_io.py | 41 python/pyarrow/tests/test_ipc.py | 116 + python/pyarrow/tests/test_table.py | 82 +++ python/setup.py | 1 + python/src/pyarrow/adapters/builtin.cc | 2 +- python/src/pyarrow/adapters/pandas.cc| 8 + python/src/pyarrow/common.cc | 2 +- python/src/pyarrow/common.h | 20 +- python/src/pyarrow/io.cc | 6 +- 37 files changed, 1012 insertions(+), 309 deletions(-) -- http://git-wip-us.apache.org/repos/asf/arrow/blob/772800ac/cpp/src/arrow/io/io-memory-test.cc -- diff --git a/cpp/src/arrow/io/io-memory-test.cc b/cpp/src/arrow/io/io-memory-test.cc index 6de35da..a49faf3 100644 --- a/cpp/src/arrow/io/io-memory-test.cc +++ b/cpp/src/arrow/io/io-memory-test.cc @@ -121,5 +121,30 @@ TEST_F(TestMemoryMappedFile, InvalidFile) { IOError, MemoryMappedFile::Open(non_existent_path, FileMode::READ, &result)); } +class TestBufferOutputStream : public ::testing::Test { + public: + void SetUp() { +buffer_.reset(new PoolBuffer(default_memory_pool())); +stream_.reset(new BufferOutputStream(buffer_)); + } + + protected: + std::shared_ptr buffer_; + std::unique_ptr stream_; +}; + +TEST_F(TestBufferOutputStream, CloseResizes) { + std::string data = "data123456"; + + const int64_t nbytes = static_cast(data.size()); + const int K = 100; + for (int i = 0; i < K; ++i) { +EXPECT_OK(stream_->Write(reinterpret_cast(data.c_str()), nbytes)); + } + + ASSERT_OK(stream_->Close()); + ASSERT_EQ(K * nbytes, buffer_->size()); +} + } // namespace io } // namespace arrow http://git-wip-us.apache.org/repos/asf/arrow/bl