Repository: arrow Updated Branches: refs/heads/master e15c6a0b3 -> e8b6231b2
ARROW-450: Fixes for PARQUET-818 Author: Uwe L. Korn <[email protected]> Closes #263 from xhochy/ARROW-450 and squashes the following commits: 287015a [Uwe L. Korn] ARROW-450: Fixes for PARQUET-818 Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/e8b6231b Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/e8b6231b Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/e8b6231b Branch: refs/heads/master Commit: e8b6231b29f59b2978b78a33eff73697d537c5dd Parents: e15c6a0 Author: Uwe L. Korn <[email protected]> Authored: Mon Jan 2 02:37:37 2017 -0500 Committer: Wes McKinney <[email protected]> Committed: Mon Jan 2 02:37:37 2017 -0500 ---------------------------------------------------------------------- python/pyarrow/includes/parquet.pxd | 19 ++----------------- python/pyarrow/parquet.pyx | 15 ++++++--------- 2 files changed, 8 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/arrow/blob/e8b6231b/python/pyarrow/includes/parquet.pxd ---------------------------------------------------------------------- diff --git a/python/pyarrow/includes/parquet.pxd b/python/pyarrow/includes/parquet.pxd index cb791e1..b4d127c 100644 --- a/python/pyarrow/includes/parquet.pxd +++ b/python/pyarrow/includes/parquet.pxd @@ -120,24 +120,9 @@ cdef extern from "parquet/api/writer.h" namespace "parquet" nogil: shared_ptr[WriterProperties] build() -cdef extern from "parquet/arrow/io.h" namespace "parquet::arrow" nogil: - cdef cppclass ParquetAllocator: - ParquetAllocator() - ParquetAllocator(MemoryPool* pool) - MemoryPool* pool() - void set_pool(MemoryPool* pool) - - cdef cppclass ParquetReadSource: - ParquetReadSource(ParquetAllocator* allocator) - Open(const shared_ptr[ReadableFileInterface]& file) - - cdef cppclass ParquetWriteSink: - ParquetWriteSink(const shared_ptr[OutputStream]& file) - - cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil: CStatus OpenFile(const shared_ptr[ReadableFileInterface]& file, - ParquetAllocator* allocator, + MemoryPool* allocator, unique_ptr[FileReader]* reader) cdef cppclass FileReader: @@ -157,6 +142,6 @@ cdef extern from "parquet/arrow/schema.h" namespace "parquet::arrow" nogil: cdef extern from "parquet/arrow/writer.h" namespace "parquet::arrow" nogil: cdef CStatus WriteFlatTable( const CTable* table, MemoryPool* pool, - const shared_ptr[ParquetWriteSink]& sink, + const shared_ptr[OutputStream]& sink, int64_t chunk_size, const shared_ptr[WriterProperties]& properties) http://git-wip-us.apache.org/repos/asf/arrow/blob/e8b6231b/python/pyarrow/parquet.pyx ---------------------------------------------------------------------- diff --git a/python/pyarrow/parquet.pyx b/python/pyarrow/parquet.pyx index 043ccf1..7379456 100644 --- a/python/pyarrow/parquet.pyx +++ b/python/pyarrow/parquet.pyx @@ -42,12 +42,12 @@ __all__ = [ cdef class ParquetReader: cdef: - ParquetAllocator allocator + MemoryPool* allocator unique_ptr[FileReader] reader column_idx_map def __cinit__(self): - self.allocator.set_pool(default_memory_pool()) + self.allocator = default_memory_pool() def open(self, source): self._open(source) @@ -69,7 +69,7 @@ cdef class ParquetReader: ParquetFileReader.OpenFile(path))) else: get_reader(source, &rd_handle) - check_status(OpenFile(rd_handle, &self.allocator, &self.reader)) + check_status(OpenFile(rd_handle, self.allocator, &self.reader)) def read_all(self): cdef: @@ -174,10 +174,8 @@ def write_table(table, sink, chunk_size=None, version=None, """ cdef Table table_ = table cdef CTable* ctable_ = table_.table - cdef shared_ptr[ParquetWriteSink] sink_ - cdef shared_ptr[FileOutputStream] filesink_ - cdef shared_ptr[OutputStream] general_sink + cdef shared_ptr[OutputStream] sink_ cdef WriterProperties.Builder properties_builder cdef int64_t chunk_size_ = 0 @@ -237,10 +235,9 @@ def write_table(table, sink, chunk_size=None, version=None, if isinstance(sink, six.string_types): check_status(FileOutputStream.Open(tobytes(sink), &filesink_)) - sink_.reset(new ParquetWriteSink(<shared_ptr[OutputStream]>filesink_)) + sink_ = <shared_ptr[OutputStream]>filesink_ else: - get_writer(sink, &general_sink) - sink_.reset(new ParquetWriteSink(general_sink)) + get_writer(sink, &sink_) with nogil: check_status(WriteFlatTable(ctable_, default_memory_pool(), sink_,
