Repository: arrow Updated Branches: refs/heads/master 312a66535 -> 7238d544c
ARROW-734: [C++/Python] Support building PyArrow on MSVC Author: Wes McKinney <wes.mckin...@twosigma.com> Closes #553 from wesm/ARROW-734 and squashes the following commits: 6e00485 [Wes McKinney] Restore -undefined,dynamic_lookup linker logic on Linux for Python extensions 5be7e31 [Wes McKinney] do_import_numpy.h no longer needed 2d00e6b [Wes McKinney] Fix Unix library names 1e6bb6e [Wes McKinney] typos 8f4928e [Wes McKinney] More build fixes. Can now import pyarrow 5162203 [Wes McKinney] Remove unneeded exports 024579e [Wes McKinney] Wow, MSVC mangles the name CreateDirectory 990fdc2 [Wes McKinney] Install DLLs fixes in FindArrow.cmake ccf941e [Wes McKinney] Restore CompilerInfo to original state 1e657ad [Wes McKinney] More fixes. Change TimeUnit to struct-based enum 2be93f0 [Wes McKinney] NumPy initialization / build fixes 1744f83 [Wes McKinney] Add new files 68e2d5b [Wes McKinney] Move NumPy API initialization into libarrow_python 0a2d387 [Wes McKinney] WIP MSVC support for PyArrow. Linker errors Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/7238d544 Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/7238d544 Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/7238d544 Branch: refs/heads/master Commit: 7238d544c1f0b05a393cdf68b2e2c9485bdb154e Parents: 312a665 Author: Wes McKinney <wes.mckin...@twosigma.com> Authored: Mon Apr 17 17:46:11 2017 -0400 Committer: Wes McKinney <wes.mckin...@twosigma.com> Committed: Mon Apr 17 17:46:11 2017 -0400 ---------------------------------------------------------------------- cpp/cmake_modules/BuildUtils.cmake | 4 +++ cpp/cmake_modules/CompilerInfo.cmake | 5 ++- cpp/cmake_modules/FindPythonLibsNew.cmake | 15 +++++--- cpp/src/arrow/io/hdfs-internal.cc | 2 +- cpp/src/arrow/io/hdfs-internal.h | 2 +- cpp/src/arrow/io/hdfs.cc | 8 ++--- cpp/src/arrow/io/hdfs.h | 2 +- cpp/src/arrow/io/io-hdfs-test.cc | 8 ++--- cpp/src/arrow/ipc/feather-internal.h | 16 ++++----- cpp/src/arrow/ipc/feather.cc | 8 ++--- cpp/src/arrow/ipc/json-internal.cc | 4 +-- cpp/src/arrow/ipc/metadata.cc | 8 ++--- cpp/src/arrow/python/CMakeLists.txt | 34 +++++++++--------- cpp/src/arrow/python/builtin_convert.cc | 3 +- cpp/src/arrow/python/builtin_convert.h | 3 +- cpp/src/arrow/python/common.h | 6 ++-- cpp/src/arrow/python/config.cc | 4 +-- cpp/src/arrow/python/config.h | 6 +--- cpp/src/arrow/python/do_import_numpy.h | 21 ----------- cpp/src/arrow/python/helpers.cc | 6 ++-- cpp/src/arrow/python/helpers.h | 12 +++---- cpp/src/arrow/python/init.cc | 34 ++++++++++++++++++ cpp/src/arrow/python/init.h | 35 +++++++++++++++++++ cpp/src/arrow/python/io.h | 2 +- cpp/src/arrow/python/numpy-internal.h | 7 ++-- cpp/src/arrow/python/numpy_convert.cc | 9 +++-- cpp/src/arrow/python/numpy_convert.h | 7 +++- cpp/src/arrow/python/numpy_interop.h | 2 +- cpp/src/arrow/python/pandas_convert.cc | 5 ++- cpp/src/arrow/python/pandas_convert.h | 2 +- cpp/src/arrow/python/platform.h | 32 +++++++++++++++++ cpp/src/arrow/python/python-test.cc | 2 +- cpp/src/arrow/python/type_traits.h | 2 +- cpp/src/arrow/python/util/datetime.h | 2 +- cpp/src/arrow/python/util/test_main.cc | 7 ++-- cpp/src/arrow/type.cc | 14 ++++---- cpp/src/arrow/type.h | 32 +++++++++-------- python/CMakeLists.txt | 34 +++++++++--------- python/cmake_modules/CompilerInfo.cmake | 48 -------------------------- python/cmake_modules/FindArrow.cmake | 25 ++++++++------ python/cmake_modules/UseCython.cmake | 8 +++-- python/pyarrow/_config.pyx | 11 ++---- python/pyarrow/_io.pyx | 2 +- python/pyarrow/includes/common.pxd | 3 +- python/pyarrow/includes/libarrow.pxd | 4 +-- python/setup.py | 28 ++++++++------- 46 files changed, 289 insertions(+), 245 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/cmake_modules/BuildUtils.cmake ---------------------------------------------------------------------- diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake index 3a3b536..4e6532b 100644 --- a/cpp/cmake_modules/BuildUtils.cmake +++ b/cpp/cmake_modules/BuildUtils.cmake @@ -102,6 +102,8 @@ function(ADD_ARROW_LIB LIB_NAME) # Necessary to make static linking into other shared libraries work properly set_property(TARGET ${LIB_NAME}_objlib PROPERTY POSITION_INDEPENDENT_CODE 1) + set(RUNTIME_INSTALL_DIR bin) + if (ARROW_BUILD_SHARED) add_library(${LIB_NAME}_shared SHARED $<TARGET_OBJECTS:${LIB_NAME}_objlib>) @@ -139,6 +141,7 @@ function(ADD_ARROW_LIB LIB_NAME) endif() install(TARGETS ${LIB_NAME}_shared + RUNTIME DESTINATION ${RUNTIME_INSTALL_DIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) endif() @@ -155,6 +158,7 @@ function(ADD_ARROW_LIB LIB_NAME) LINK_PRIVATE ${ARG_STATIC_PRIVATE_LINK_LIBS}) install(TARGETS ${LIB_NAME}_static + RUNTIME DESTINATION ${RUNTIME_INSTALL_DIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) endif() http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/cmake_modules/CompilerInfo.cmake ---------------------------------------------------------------------- diff --git a/cpp/cmake_modules/CompilerInfo.cmake b/cpp/cmake_modules/CompilerInfo.cmake index 079d9d1..3c60391 100644 --- a/cpp/cmake_modules/CompilerInfo.cmake +++ b/cpp/cmake_modules/CompilerInfo.cmake @@ -19,8 +19,8 @@ # Sets COMPILER_VERSION to the version execute_process(COMMAND "${CMAKE_CXX_COMPILER}" -v ERROR_VARIABLE COMPILER_VERSION_FULL) -message(INFO " ${COMPILER_VERSION_FULL}") -message(INFO " ${CMAKE_CXX_COMPILER_ID}") +message(INFO "Compiler version: ${COMPILER_VERSION_FULL}") +message(INFO "Compiler id: ${CMAKE_CXX_COMPILER_ID}") string(TOLOWER "${COMPILER_VERSION_FULL}" COMPILER_VERSION_FULL_LOWER) if(MSVC) @@ -62,4 +62,3 @@ else() message(FATAL_ERROR "Unknown compiler. Version info:\n${COMPILER_VERSION_FULL}") endif() message("Selected compiler ${COMPILER_FAMILY} ${COMPILER_VERSION}") - http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/cmake_modules/FindPythonLibsNew.cmake ---------------------------------------------------------------------- diff --git a/cpp/cmake_modules/FindPythonLibsNew.cmake b/cpp/cmake_modules/FindPythonLibsNew.cmake index d9cc4b3..9610816 100644 --- a/cpp/cmake_modules/FindPythonLibsNew.cmake +++ b/cpp/cmake_modules/FindPythonLibsNew.cmake @@ -233,12 +233,17 @@ FUNCTION(PYTHON_ADD_MODULE _NAME ) # segfaults, so do this dynamic lookup instead. SET_TARGET_PROPERTIES(${_NAME} PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") + ELSEIF(MSVC) + target_link_libraries(${_NAME} ${PYTHON_LIBRARIES}) ELSE() - # In general, we should not link against libpython as we do not embed - # the Python interpreter. The python binary itself can then define where - # the symbols should loaded from. - SET_TARGET_PROPERTIES(${_NAME} PROPERTIES LINK_FLAGS - "-Wl,-undefined,dynamic_lookup") + # In general, we should not link against libpython as we do not embed the + # Python interpreter. The python binary itself can then define where the + # symbols should loaded from. For being manylinux1 compliant, one is not + # allowed to link to libpython. Partly because not all systems ship it, + # also because the interpreter ABI/API was not stable between patch + # releases for Python < 3.5. + SET_TARGET_PROPERTIES(${_NAME} PROPERTIES LINK_FLAGS + "-Wl,-undefined,dynamic_lookup") ENDIF() IF(PYTHON_MODULE_${_NAME}_BUILD_SHARED) SET_TARGET_PROPERTIES(${_NAME} PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}") http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/io/hdfs-internal.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/io/hdfs-internal.cc b/cpp/src/arrow/io/hdfs-internal.cc index e4b2cd5..e67419b 100644 --- a/cpp/src/arrow/io/hdfs-internal.cc +++ b/cpp/src/arrow/io/hdfs-internal.cc @@ -409,7 +409,7 @@ int LibHdfsShim::SetWorkingDirectory(hdfsFS fs, const char* path) { } } -int LibHdfsShim::CreateDirectory(hdfsFS fs, const char* path) { +int LibHdfsShim::MakeDirectory(hdfsFS fs, const char* path) { return this->hdfsCreateDirectory(fs, path); } http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/io/hdfs-internal.h ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/io/hdfs-internal.h b/cpp/src/arrow/io/hdfs-internal.h index 01cf149..c5ea397 100644 --- a/cpp/src/arrow/io/hdfs-internal.h +++ b/cpp/src/arrow/io/hdfs-internal.h @@ -173,7 +173,7 @@ struct LibHdfsShim { int SetWorkingDirectory(hdfsFS fs, const char* path); - int CreateDirectory(hdfsFS fs, const char* path); + int MakeDirectory(hdfsFS fs, const char* path); int SetReplication(hdfsFS fs, const char* path, int16_t replication); http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/io/hdfs.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/io/hdfs.cc b/cpp/src/arrow/io/hdfs.cc index 3510ba1..a27e132 100644 --- a/cpp/src/arrow/io/hdfs.cc +++ b/cpp/src/arrow/io/hdfs.cc @@ -347,8 +347,8 @@ class HdfsClient::HdfsClientImpl { return Status::OK(); } - Status CreateDirectory(const std::string& path) { - int ret = driver_->CreateDirectory(fs_, path.c_str()); + Status MakeDirectory(const std::string& path) { + int ret = driver_->MakeDirectory(fs_, path.c_str()); CHECK_FAILURE(ret, "create directory"); return Status::OK(); } @@ -505,8 +505,8 @@ Status HdfsClient::Connect( return Status::OK(); } -Status HdfsClient::CreateDirectory(const std::string& path) { - return impl_->CreateDirectory(path); +Status HdfsClient::MakeDirectory(const std::string& path) { + return impl_->MakeDirectory(path); } Status HdfsClient::Delete(const std::string& path, bool recursive) { http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/io/hdfs.h ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/io/hdfs.h b/cpp/src/arrow/io/hdfs.h index e3f5442..f3de4a2 100644 --- a/cpp/src/arrow/io/hdfs.h +++ b/cpp/src/arrow/io/hdfs.h @@ -82,7 +82,7 @@ class ARROW_EXPORT HdfsClient : public FileSystemClient { // // @param path (in): absolute HDFS path // @returns Status - Status CreateDirectory(const std::string& path); + Status MakeDirectory(const std::string& path); // Delete file or directory // @param path: absolute path to data http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/io/io-hdfs-test.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/io/io-hdfs-test.cc b/cpp/src/arrow/io/io-hdfs-test.cc index 0a9f5d9..0fdb897 100644 --- a/cpp/src/arrow/io/io-hdfs-test.cc +++ b/cpp/src/arrow/io/io-hdfs-test.cc @@ -45,7 +45,7 @@ class TestHdfsClient : public ::testing::Test { if (client_->Exists(scratch_dir_)) { RETURN_NOT_OK((client_->Delete(scratch_dir_, true))); } - return client_->CreateDirectory(scratch_dir_); + return client_->MakeDirectory(scratch_dir_); } Status WriteDummyFile(const std::string& path, const uint8_t* buffer, int64_t size, @@ -161,14 +161,14 @@ TYPED_TEST(TestHdfsClient, ConnectsAgain) { ASSERT_OK(client->Disconnect()); } -TYPED_TEST(TestHdfsClient, CreateDirectory) { +TYPED_TEST(TestHdfsClient, MakeDirectory) { SKIP_IF_NO_DRIVER(); std::string path = this->ScratchPath("create-directory"); if (this->client_->Exists(path)) { ASSERT_OK(this->client_->Delete(path, true)); } - ASSERT_OK(this->client_->CreateDirectory(path)); + ASSERT_OK(this->client_->MakeDirectory(path)); ASSERT_TRUE(this->client_->Exists(path)); std::vector<HdfsPathInfo> listing; EXPECT_OK(this->client_->ListDirectory(path, &listing)); @@ -253,7 +253,7 @@ TYPED_TEST(TestHdfsClient, ListDirectory) { ASSERT_OK(this->MakeScratchDir()); ASSERT_OK(this->WriteDummyFile(p1, data.data(), size)); ASSERT_OK(this->WriteDummyFile(p2, data.data(), size / 2)); - ASSERT_OK(this->client_->CreateDirectory(d1)); + ASSERT_OK(this->client_->MakeDirectory(d1)); std::vector<HdfsPathInfo> listing; ASSERT_OK(this->client_->ListDirectory(this->scratch_dir_, &listing)); http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/ipc/feather-internal.h ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/ipc/feather-internal.h b/cpp/src/arrow/ipc/feather-internal.h index 6847445..646c3b2 100644 --- a/cpp/src/arrow/ipc/feather-internal.h +++ b/cpp/src/arrow/ipc/feather-internal.h @@ -75,7 +75,7 @@ struct ARROW_EXPORT CategoryMetadata { }; struct ARROW_EXPORT TimestampMetadata { - TimeUnit unit; + TimeUnit::type unit; // A timezone name known to the Olson timezone database. For display purposes // because the actual data is all UTC @@ -83,7 +83,7 @@ struct ARROW_EXPORT TimestampMetadata { }; struct ARROW_EXPORT TimeMetadata { - TimeUnit unit; + TimeUnit::type unit; }; static constexpr const char* kFeatherMagicBytes = "FEA1"; @@ -156,12 +156,12 @@ static inline flatbuffers::Offset<fbs::PrimitiveArray> GetPrimitiveArray( array.length, array.null_count, array.total_bytes); } -static inline fbs::TimeUnit ToFlatbufferEnum(TimeUnit unit) { +static inline fbs::TimeUnit ToFlatbufferEnum(TimeUnit::type unit) { return static_cast<fbs::TimeUnit>(static_cast<int>(unit)); } -static inline TimeUnit FromFlatbufferEnum(fbs::TimeUnit unit) { - return static_cast<TimeUnit>(static_cast<int>(unit)); +static inline TimeUnit::type FromFlatbufferEnum(fbs::TimeUnit unit) { + return static_cast<TimeUnit::type>(static_cast<int>(unit)); } // Convert Feather enums to Flatbuffer enums @@ -197,10 +197,10 @@ class ARROW_EXPORT ColumnBuilder { void SetValues(const ArrayMetadata& values); void SetUserMetadata(const std::string& data); void SetCategory(const ArrayMetadata& levels, bool ordered = false); - void SetTimestamp(TimeUnit unit); - void SetTimestamp(TimeUnit unit, const std::string& timezone); + void SetTimestamp(TimeUnit::type unit); + void SetTimestamp(TimeUnit::type unit, const std::string& timezone); void SetDate(); - void SetTime(TimeUnit unit); + void SetTime(TimeUnit::type unit); FBB& fbb(); private: http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/ipc/feather.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc index 5dc0396..7d0abdd 100644 --- a/cpp/src/arrow/ipc/feather.cc +++ b/cpp/src/arrow/ipc/feather.cc @@ -184,12 +184,12 @@ void ColumnBuilder::SetCategory(const ArrayMetadata& levels, bool ordered) { meta_category_.ordered = ordered; } -void ColumnBuilder::SetTimestamp(TimeUnit unit) { +void ColumnBuilder::SetTimestamp(TimeUnit::type unit) { type_ = ColumnType::TIMESTAMP; meta_timestamp_.unit = unit; } -void ColumnBuilder::SetTimestamp(TimeUnit unit, const std::string& timezone) { +void ColumnBuilder::SetTimestamp(TimeUnit::type unit, const std::string& timezone) { SetTimestamp(unit); meta_timestamp_.timezone = timezone; } @@ -198,7 +198,7 @@ void ColumnBuilder::SetDate() { type_ = ColumnType::DATE; } -void ColumnBuilder::SetTime(TimeUnit unit) { +void ColumnBuilder::SetTime(TimeUnit::type unit) { type_ = ColumnType::TIME; meta_time_.unit = unit; } @@ -279,7 +279,7 @@ class TableReader::TableReaderImpl { } case fbs::TypeMetadata_TimestampMetadata: { auto meta = static_cast<const fbs::TimestampMetadata*>(metadata); - TimeUnit unit = FromFlatbufferEnum(meta->unit()); + TimeUnit::type unit = FromFlatbufferEnum(meta->unit()); std::string tz; // flatbuffer non-null if (meta->timezone() != 0) { http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/ipc/json-internal.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc index 18ee834..2ab3acb 100644 --- a/cpp/src/arrow/ipc/json-internal.cc +++ b/cpp/src/arrow/ipc/json-internal.cc @@ -77,7 +77,7 @@ static std::string GetFloatingPrecisionName(FloatingPoint::Precision precision) return "UNKNOWN"; } -static std::string GetTimeUnitName(TimeUnit unit) { +static std::string GetTimeUnitName(TimeUnit::type unit) { switch (unit) { case TimeUnit::SECOND: return "SECOND"; @@ -645,7 +645,7 @@ static Status GetTimestamp(const RjObject& json_type, std::shared_ptr<DataType>* std::string unit_str = json_unit->value.GetString(); - TimeUnit unit; + TimeUnit::type unit; if (unit_str == "SECOND") { unit = TimeUnit::SECOND; } else if (unit_str == "MILLISECOND") { http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/ipc/metadata.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc index ee21156..791948b 100644 --- a/cpp/src/arrow/ipc/metadata.cc +++ b/cpp/src/arrow/ipc/metadata.cc @@ -188,7 +188,7 @@ static Status UnionToFlatBuffer(FBB& fbb, const std::shared_ptr<DataType>& type, *offset = IntToFlatbuffer(fbb, BIT_WIDTH, IS_SIGNED); \ break; -static inline flatbuf::TimeUnit ToFlatbufferUnit(TimeUnit unit) { +static inline flatbuf::TimeUnit ToFlatbufferUnit(TimeUnit::type unit) { switch (unit) { case TimeUnit::SECOND: return flatbuf::TimeUnit_SECOND; @@ -204,7 +204,7 @@ static inline flatbuf::TimeUnit ToFlatbufferUnit(TimeUnit unit) { return flatbuf::TimeUnit_MIN; } -static inline TimeUnit FromFlatbufferUnit(flatbuf::TimeUnit unit) { +static inline TimeUnit::type FromFlatbufferUnit(flatbuf::TimeUnit unit) { switch (unit) { case flatbuf::TimeUnit_SECOND: return TimeUnit::SECOND; @@ -258,7 +258,7 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data, } case flatbuf::Type_Time: { auto time_type = static_cast<const flatbuf::Time*>(type_data); - TimeUnit unit = FromFlatbufferUnit(time_type->unit()); + TimeUnit::type unit = FromFlatbufferUnit(time_type->unit()); int32_t bit_width = time_type->bitWidth(); switch (unit) { case TimeUnit::SECOND: @@ -279,7 +279,7 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data, } case flatbuf::Type_Timestamp: { auto ts_type = static_cast<const flatbuf::Timestamp*>(type_data); - TimeUnit unit = FromFlatbufferUnit(ts_type->unit()); + TimeUnit::type unit = FromFlatbufferUnit(ts_type->unit()); if (ts_type->timezone() != 0 && ts_type->timezone()->Length() > 0) { *out = timestamp(unit, ts_type->timezone()->str()); } else { http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/python/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/python/CMakeLists.txt b/cpp/src/arrow/python/CMakeLists.txt index 8f7991e..607a1c4 100644 --- a/cpp/src/arrow/python/CMakeLists.txt +++ b/cpp/src/arrow/python/CMakeLists.txt @@ -17,18 +17,18 @@ if (ARROW_BUILD_TESTS) add_library(arrow_python_test_main STATIC util/test_main.cc) + target_link_libraries(arrow_python_test_main + gtest) + if (APPLE) target_link_libraries(arrow_python_test_main - gtest - dl) + ${CMAKE_DL_LIBS}) set_target_properties(arrow_python_test_main PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") - else() + elseif(NOT MSVC) target_link_libraries(arrow_python_test_main - gtest pthread - dl - ) + ${CMAKE_DL_LIBS}) endif() endif() @@ -38,12 +38,6 @@ set(ARROW_PYTHON_MIN_TEST_LIBS arrow_static ${BOOST_REGEX_LIBRARY}) -if(ARROW_BUILD_TESTS) - ADD_THIRDPARTY_LIB(python - SHARED_LIB "${PYTHON_LIBRARIES}") - list(APPEND ARROW_PYTHON_MIN_TEST_LIBS python) -endif() - set(ARROW_PYTHON_TEST_LINK_LIBS ${ARROW_PYTHON_MIN_TEST_LIBS}) # ---------------------------------------------------------------------- @@ -53,6 +47,7 @@ set(ARROW_PYTHON_SRCS common.cc config.cc helpers.cc + init.cc io.cc numpy_convert.cc pandas_convert.cc @@ -66,9 +61,14 @@ ADD_ARROW_LIB(arrow_python SOURCES ${ARROW_PYTHON_SRCS} SHARED_LINK_FLAGS "" SHARED_LINK_LIBS ${ARROW_PYTHON_SHARED_LINK_LIBS} - STATIC_LINK_LIBS ${ARROW_IO_SHARED_PRIVATE_LINK_LIBS} + STATIC_LINK_LIBS "" ) +if (MSVC) + target_link_libraries(arrow_python_shared + ${PYTHON_LIBRARIES}) +endif() + if ("${COMPILER_FAMILY}" STREQUAL "clang") # Clang, be quiet. Python C API has lots of macros set_property(SOURCE ${ARROW_PYTHON_SRCS} @@ -82,19 +82,19 @@ install(FILES builtin_convert.h common.h config.h - do_import_numpy.h helpers.h + init.h io.h numpy_convert.h numpy_interop.h pandas_convert.h + platform.h type_traits.h DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/arrow/python") -# set_target_properties(arrow_python_shared PROPERTIES -# INSTALL_RPATH "\$ORIGIN") - if (ARROW_BUILD_TESTS) ADD_ARROW_TEST(python-test STATIC_LINK_LIBS "${ARROW_PYTHON_TEST_LINK_LIBS}") + target_link_libraries(python-test + ${PYTHON_LIBRARIES}) endif() http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/python/builtin_convert.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc index 8cc9876..137937c 100644 --- a/cpp/src/arrow/python/builtin_convert.cc +++ b/cpp/src/arrow/python/builtin_convert.cc @@ -15,7 +15,8 @@ // specific language governing permissions and limitations // under the License. -#include <Python.h> +#include "arrow/python/platform.h" + #include <datetime.h> #include <algorithm> http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/python/builtin_convert.h ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/python/builtin_convert.h b/cpp/src/arrow/python/builtin_convert.h index 2141c25..a6180d4 100644 --- a/cpp/src/arrow/python/builtin_convert.h +++ b/cpp/src/arrow/python/builtin_convert.h @@ -21,13 +21,12 @@ #ifndef ARROW_PYTHON_ADAPTERS_BUILTIN_H #define ARROW_PYTHON_ADAPTERS_BUILTIN_H -#include <Python.h> +#include "arrow/python/platform.h" #include <memory> #include <string> #include "arrow/type.h" - #include "arrow/util/visibility.h" #include "arrow/python/common.h" http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/python/common.h ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/python/common.h b/cpp/src/arrow/python/common.h index a6806ab..882bb15 100644 --- a/cpp/src/arrow/python/common.h +++ b/cpp/src/arrow/python/common.h @@ -32,7 +32,7 @@ class MemoryPool; namespace py { -class PyAcquireGIL { +class ARROW_EXPORT PyAcquireGIL { public: PyAcquireGIL() { state_ = PyGILState_Ensure(); } @@ -45,7 +45,7 @@ class PyAcquireGIL { #define PYARROW_IS_PY2 PY_MAJOR_VERSION <= 2 -class OwnedRef { +class ARROW_EXPORT OwnedRef { public: OwnedRef() : obj_(nullptr) {} @@ -70,7 +70,7 @@ class OwnedRef { PyObject* obj_; }; -struct PyObjectStringify { +struct ARROW_EXPORT PyObjectStringify { OwnedRef tmp_obj; const char* bytes; Py_ssize_t size; http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/python/config.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/python/config.cc b/cpp/src/arrow/python/config.cc index c2a6916..3cec7c4 100644 --- a/cpp/src/arrow/python/config.cc +++ b/cpp/src/arrow/python/config.cc @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include <Python.h> +#include "arrow/python/platform.h" #include <datetime.h> #include "arrow/python/config.h" @@ -23,8 +23,6 @@ namespace arrow { namespace py { -void Init() {} - PyObject* numpy_nan = nullptr; void set_numpy_nan(PyObject* obj) { http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/python/config.h ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/python/config.h b/cpp/src/arrow/python/config.h index c132726..c2b089d 100644 --- a/cpp/src/arrow/python/config.h +++ b/cpp/src/arrow/python/config.h @@ -18,8 +18,7 @@ #ifndef ARROW_PYTHON_CONFIG_H #define ARROW_PYTHON_CONFIG_H -#include <iostream> -#include <Python.h> +#include "arrow/python/platform.h" #include "arrow/python/numpy_interop.h" #include "arrow/util/visibility.h" @@ -35,9 +34,6 @@ ARROW_EXPORT extern PyObject* numpy_nan; ARROW_EXPORT -void Init(); - -ARROW_EXPORT void set_numpy_nan(PyObject* obj); } // namespace py http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/python/do_import_numpy.h ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/python/do_import_numpy.h b/cpp/src/arrow/python/do_import_numpy.h deleted file mode 100644 index bb4a382..0000000 --- a/cpp/src/arrow/python/do_import_numpy.h +++ /dev/null @@ -1,21 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// Trick borrowed from dynd-python for initializing the NumPy array API - -// Trigger the array import (inversion of NO_IMPORT_ARRAY) -#define NUMPY_IMPORT_ARRAY http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/python/helpers.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/python/helpers.cc b/cpp/src/arrow/python/helpers.cc index 3d3d07a..f7c73a8 100644 --- a/cpp/src/arrow/python/helpers.cc +++ b/cpp/src/arrow/python/helpers.cc @@ -92,11 +92,11 @@ Status PythonDecimalToArrowDecimal( return FromString(c_string, arrow_decimal); } -template Status PythonDecimalToArrowDecimal( +template Status ARROW_TEMPLATE_EXPORT PythonDecimalToArrowDecimal( PyObject* python_decimal, decimal::Decimal32* arrow_decimal); -template Status PythonDecimalToArrowDecimal( +template Status ARROW_TEMPLATE_EXPORT PythonDecimalToArrowDecimal( PyObject* python_decimal, decimal::Decimal64* arrow_decimal); -template Status PythonDecimalToArrowDecimal( +template Status ARROW_TEMPLATE_EXPORT PythonDecimalToArrowDecimal( PyObject* python_decimal, decimal::Decimal128* arrow_decimal); Status InferDecimalPrecisionAndScale( http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/python/helpers.h ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/python/helpers.h b/cpp/src/arrow/python/helpers.h index 77fde26..c6402d8 100644 --- a/cpp/src/arrow/python/helpers.h +++ b/cpp/src/arrow/python/helpers.h @@ -18,7 +18,7 @@ #ifndef PYARROW_HELPERS_H #define PYARROW_HELPERS_H -#include <Python.h> +#include "arrow/python/platform.h" #include <memory> #include <string> @@ -42,18 +42,18 @@ class OwnedRef; ARROW_EXPORT std::shared_ptr<DataType> GetPrimitiveType(Type::type type); -Status ImportModule(const std::string& module_name, OwnedRef* ref); -Status ImportFromModule( +Status ARROW_EXPORT ImportModule(const std::string& module_name, OwnedRef* ref); +Status ARROW_EXPORT ImportFromModule( const OwnedRef& module, const std::string& module_name, OwnedRef* ref); template <typename T> -Status PythonDecimalToArrowDecimal( +Status ARROW_EXPORT PythonDecimalToArrowDecimal( PyObject* python_decimal, decimal::Decimal<T>* arrow_decimal); -Status InferDecimalPrecisionAndScale( +Status ARROW_EXPORT InferDecimalPrecisionAndScale( PyObject* python_decimal, int* precision = nullptr, int* scale = nullptr); -Status DecimalFromString( +Status ARROW_EXPORT DecimalFromString( PyObject* decimal_constructor, const std::string& decimal_string, PyObject** out); } // namespace py http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/python/init.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/python/init.cc b/cpp/src/arrow/python/init.cc new file mode 100644 index 0000000..fa70af7 --- /dev/null +++ b/cpp/src/arrow/python/init.cc @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/python/platform.h" + +// Trigger the array import (inversion of NO_IMPORT_ARRAY) +#define NUMPY_IMPORT_ARRAY + +#include "arrow/python/init.h" +#include "arrow/python/numpy_interop.h" + +namespace arrow { +namespace py { + +void InitNumPy() { + import_numpy(); +} + +} // namespace py +} // namespace arrow http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/python/init.h ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/python/init.h b/cpp/src/arrow/python/init.h new file mode 100644 index 0000000..a2533d8 --- /dev/null +++ b/cpp/src/arrow/python/init.h @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef ARROW_PYTHON_INIT_H +#define ARROW_PYTHON_INIT_H + +#include "arrow/python/platform.h" + +#include "arrow/python/numpy_interop.h" +#include "arrow/util/visibility.h" + +namespace arrow { +namespace py { + +ARROW_EXPORT +void InitNumPy(); + +} // namespace py +} // namespace arrow + +#endif // ARROW_PYTHON_INIT_H http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/python/io.h ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/python/io.h b/cpp/src/arrow/python/io.h index 905bd6c..bf14cd6 100644 --- a/cpp/src/arrow/python/io.h +++ b/cpp/src/arrow/python/io.h @@ -34,7 +34,7 @@ namespace py { // A common interface to a Python file-like object. Must acquire GIL before // calling any methods -class PythonFile { +class ARROW_EXPORT PythonFile { public: explicit PythonFile(PyObject* file); ~PythonFile(); http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/python/numpy-internal.h ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/python/numpy-internal.h b/cpp/src/arrow/python/numpy-internal.h index fcc6a58..f1ef7da 100644 --- a/cpp/src/arrow/python/numpy-internal.h +++ b/cpp/src/arrow/python/numpy-internal.h @@ -20,12 +20,11 @@ #ifndef ARROW_PYTHON_NUMPY_INTERNAL_H #define ARROW_PYTHON_NUMPY_INTERNAL_H -#include <Python.h> +#include "arrow/python/numpy_interop.h" -#include <cstdint> +#include "arrow/python/platform.h" -#include "arrow/python/numpy_convert.h" -#include "arrow/python/numpy_interop.h" +#include <cstdint> namespace arrow { namespace py { http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/python/numpy_convert.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/python/numpy_convert.cc b/cpp/src/arrow/python/numpy_convert.cc index ab79e17..2c1a591 100644 --- a/cpp/src/arrow/python/numpy_convert.cc +++ b/cpp/src/arrow/python/numpy_convert.cc @@ -15,10 +15,9 @@ // specific language governing permissions and limitations // under the License. -#include <Python.h> +#include "arrow/python/numpy_interop.h" #include "arrow/python/numpy_convert.h" -#include "arrow/python/numpy_interop.h" #include <cstdint> #include <memory> @@ -38,8 +37,8 @@ namespace py { bool is_contiguous(PyObject* array) { if (PyArray_Check(array)) { - return PyArray_FLAGS(reinterpret_cast<PyArrayObject*>(array)) & - (NPY_ARRAY_C_CONTIGUOUS | NPY_ARRAY_F_CONTIGUOUS); + return (PyArray_FLAGS(reinterpret_cast<PyArrayObject*>(array)) & + (NPY_ARRAY_C_CONTIGUOUS | NPY_ARRAY_F_CONTIGUOUS)) != 0; } else { return false; } @@ -167,7 +166,7 @@ Status NumPyDtypeToArrow(PyObject* dtype, std::shared_ptr<DataType>* out) { case NPY_DATETIME: { auto date_dtype = reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(descr->c_metadata); - TimeUnit unit; + TimeUnit::type unit; switch (date_dtype->meta.base) { case NPY_FR_s: unit = TimeUnit::SECOND; http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/python/numpy_convert.h ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/python/numpy_convert.h b/cpp/src/arrow/python/numpy_convert.h index c252640..a486646 100644 --- a/cpp/src/arrow/python/numpy_convert.h +++ b/cpp/src/arrow/python/numpy_convert.h @@ -21,7 +21,7 @@ #ifndef ARROW_PYTHON_NUMPY_CONVERT_H #define ARROW_PYTHON_NUMPY_CONVERT_H -#include <Python.h> +#include "arrow/python/platform.h" #include <memory> #include <string> @@ -48,14 +48,19 @@ class ARROW_EXPORT NumPyBuffer : public Buffer { }; // Handle misbehaved types like LONGLONG and ULONGLONG +ARROW_EXPORT int cast_npy_type_compat(int type_num); +ARROW_EXPORT bool is_contiguous(PyObject* array); ARROW_EXPORT Status NumPyDtypeToArrow(PyObject* dtype, std::shared_ptr<DataType>* out); +ARROW_EXPORT Status GetTensorType(PyObject* dtype, std::shared_ptr<DataType>* out); + +ARROW_EXPORT Status GetNumPyType(const DataType& type, int* type_num); ARROW_EXPORT Status NdarrayToTensor( http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/python/numpy_interop.h ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/python/numpy_interop.h b/cpp/src/arrow/python/numpy_interop.h index 0a4b425..b93200c 100644 --- a/cpp/src/arrow/python/numpy_interop.h +++ b/cpp/src/arrow/python/numpy_interop.h @@ -18,7 +18,7 @@ #ifndef PYARROW_NUMPY_INTEROP_H #define PYARROW_NUMPY_INTEROP_H -#include <Python.h> +#include "arrow/python/platform.h" #include <numpy/numpyconfig.h> http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/python/pandas_convert.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc index 5cdcb6f..636a3fd 100644 --- a/cpp/src/arrow/python/pandas_convert.cc +++ b/cpp/src/arrow/python/pandas_convert.cc @@ -17,9 +17,8 @@ // Functions for pandas conversion via NumPy -#include <Python.h> - #include "arrow/python/numpy_interop.h" + #include "arrow/python/pandas_convert.h" #include <algorithm> @@ -490,7 +489,7 @@ struct UnboxDate {}; template <> struct UnboxDate<Date32Type> { - static int64_t Unbox(PyObject* obj) { + static int32_t Unbox(PyObject* obj) { return PyDate_to_days(reinterpret_cast<PyDateTime_Date*>(obj)); } }; http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/python/pandas_convert.h ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/python/pandas_convert.h b/cpp/src/arrow/python/pandas_convert.h index fd901d8..45c8a1a 100644 --- a/cpp/src/arrow/python/pandas_convert.h +++ b/cpp/src/arrow/python/pandas_convert.h @@ -21,7 +21,7 @@ #ifndef ARROW_PYTHON_ADAPTERS_PANDAS_H #define ARROW_PYTHON_ADAPTERS_PANDAS_H -#include <Python.h> +#include "arrow/python/platform.h" #include <memory> #include <string> http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/python/platform.h ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/python/platform.h b/cpp/src/arrow/python/platform.h new file mode 100644 index 0000000..38f8e0f --- /dev/null +++ b/cpp/src/arrow/python/platform.h @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Functions for converting between pandas's NumPy-based data representation +// and Arrow data structures + +#ifndef ARROW_PYTHON_PLATFORM_H +#define ARROW_PYTHON_PLATFORM_H + +#include <Python.h> +#include <iostream> + +// Work around C2528 error +#if _MSC_VER >= 1900 +#undef timezone +#endif + +#endif // ARROW_PYTHON_PLATFORM_H http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/python/python-test.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/python/python-test.cc b/cpp/src/arrow/python/python-test.cc index a4a11c0..cbc9377 100644 --- a/cpp/src/arrow/python/python-test.cc +++ b/cpp/src/arrow/python/python-test.cc @@ -19,7 +19,7 @@ #include <memory> -#include <Python.h> +#include "arrow/python/platform.h" #include "arrow/array.h" #include "arrow/builder.h" http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/python/type_traits.h ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/python/type_traits.h b/cpp/src/arrow/python/type_traits.h index c464d65..26b15bd 100644 --- a/cpp/src/arrow/python/type_traits.h +++ b/cpp/src/arrow/python/type_traits.h @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include <Python.h> +#include "arrow/python/platform.h" #include <cstdint> #include <limits> http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/python/util/datetime.h ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/python/util/datetime.h b/cpp/src/arrow/python/util/datetime.h index 82cf6fc..852f426 100644 --- a/cpp/src/arrow/python/util/datetime.h +++ b/cpp/src/arrow/python/util/datetime.h @@ -18,7 +18,7 @@ #ifndef PYARROW_UTIL_DATETIME_H #define PYARROW_UTIL_DATETIME_H -#include <Python.h> +#include "arrow/python/platform.h" #include <datetime.h> namespace arrow { http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/python/util/test_main.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/python/util/test_main.cc b/cpp/src/arrow/python/util/test_main.cc index c83514d..c24da40 100644 --- a/cpp/src/arrow/python/util/test_main.cc +++ b/cpp/src/arrow/python/util/test_main.cc @@ -15,18 +15,17 @@ // specific language governing permissions and limitations // under the License. -#include <Python.h> +#include "arrow/python/platform.h" #include <gtest/gtest.h> -#include "arrow/python/do_import_numpy.h" -#include "arrow/python/numpy_interop.h" +#include "arrow/python/init.h" int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); Py_Initialize(); - arrow::py::import_numpy(); + arrow::py::InitNumPy(); int ret = RUN_ALL_TESTS(); http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/type.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc index a2300d6..2e454ae 100644 --- a/cpp/src/arrow/type.cc +++ b/cpp/src/arrow/type.cc @@ -134,10 +134,10 @@ std::string Date32Type::ToString() const { // ---------------------------------------------------------------------- // Time types -TimeType::TimeType(Type::type type_id, TimeUnit unit) +TimeType::TimeType(Type::type type_id, TimeUnit::type unit) : FixedWidthType(type_id), unit_(unit) {} -Time32Type::Time32Type(TimeUnit unit) : TimeType(Type::TIME32, unit) { +Time32Type::Time32Type(TimeUnit::type unit) : TimeType(Type::TIME32, unit) { DCHECK(unit == TimeUnit::SECOND || unit == TimeUnit::MILLI) << "Must be seconds or milliseconds"; } @@ -148,7 +148,7 @@ std::string Time32Type::ToString() const { return ss.str(); } -Time64Type::Time64Type(TimeUnit unit) : TimeType(Type::TIME64, unit) { +Time64Type::Time64Type(TimeUnit::type unit) : TimeType(Type::TIME64, unit) { DCHECK(unit == TimeUnit::MICRO || unit == TimeUnit::NANO) << "Must be microseconds or nanoseconds"; } @@ -338,19 +338,19 @@ std::shared_ptr<DataType> fixed_size_binary(int32_t byte_width) { return std::make_shared<FixedSizeBinaryType>(byte_width); } -std::shared_ptr<DataType> timestamp(TimeUnit unit) { +std::shared_ptr<DataType> timestamp(TimeUnit::type unit) { return std::make_shared<TimestampType>(unit); } -std::shared_ptr<DataType> timestamp(TimeUnit unit, const std::string& timezone) { +std::shared_ptr<DataType> timestamp(TimeUnit::type unit, const std::string& timezone) { return std::make_shared<TimestampType>(unit, timezone); } -std::shared_ptr<DataType> time32(TimeUnit unit) { +std::shared_ptr<DataType> time32(TimeUnit::type unit) { return std::make_shared<Time32Type>(unit); } -std::shared_ptr<DataType> time64(TimeUnit unit) { +std::shared_ptr<DataType> time64(TimeUnit::type unit) { return std::make_shared<Time64Type>(unit); } http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/cpp/src/arrow/type.h ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h index 6810b35..ea4ea03 100644 --- a/cpp/src/arrow/type.h +++ b/cpp/src/arrow/type.h @@ -538,9 +538,11 @@ class ARROW_EXPORT Date64Type : public DateType { static std::string name() { return "date"; } }; -enum class TimeUnit : char { SECOND = 0, MILLI = 1, MICRO = 2, NANO = 3 }; +struct TimeUnit { + enum type { SECOND = 0, MILLI = 1, MICRO = 2, NANO = 3 }; +}; -static inline std::ostream& operator<<(std::ostream& os, TimeUnit unit) { +static inline std::ostream& operator<<(std::ostream& os, TimeUnit::type unit) { switch (unit) { case TimeUnit::SECOND: os << "s"; @@ -560,11 +562,11 @@ static inline std::ostream& operator<<(std::ostream& os, TimeUnit unit) { class ARROW_EXPORT TimeType : public FixedWidthType { public: - TimeUnit unit() const { return unit_; } + TimeUnit::type unit() const { return unit_; } protected: - TimeType(Type::type type_id, TimeUnit unit); - TimeUnit unit_; + TimeType(Type::type type_id, TimeUnit::type unit); + TimeUnit::type unit_; }; class ARROW_EXPORT Time32Type : public TimeType { @@ -574,7 +576,7 @@ class ARROW_EXPORT Time32Type : public TimeType { int bit_width() const override { return static_cast<int>(sizeof(c_type) * CHAR_BIT); } - explicit Time32Type(TimeUnit unit = TimeUnit::MILLI); + explicit Time32Type(TimeUnit::type unit = TimeUnit::MILLI); Status Accept(TypeVisitor* visitor) const override; std::string ToString() const override; @@ -587,7 +589,7 @@ class ARROW_EXPORT Time64Type : public TimeType { int bit_width() const override { return static_cast<int>(sizeof(c_type) * CHAR_BIT); } - explicit Time64Type(TimeUnit unit = TimeUnit::MILLI); + explicit Time64Type(TimeUnit::type unit = TimeUnit::MILLI); Status Accept(TypeVisitor* visitor) const override; std::string ToString() const override; @@ -602,21 +604,21 @@ class ARROW_EXPORT TimestampType : public FixedWidthType { int bit_width() const override { return static_cast<int>(sizeof(int64_t) * CHAR_BIT); } - explicit TimestampType(TimeUnit unit = TimeUnit::MILLI) + explicit TimestampType(TimeUnit::type unit = TimeUnit::MILLI) : FixedWidthType(Type::TIMESTAMP), unit_(unit) {} - explicit TimestampType(TimeUnit unit, const std::string& timezone) + explicit TimestampType(TimeUnit::type unit, const std::string& timezone) : FixedWidthType(Type::TIMESTAMP), unit_(unit), timezone_(timezone) {} Status Accept(TypeVisitor* visitor) const override; std::string ToString() const override; static std::string name() { return "timestamp"; } - TimeUnit unit() const { return unit_; } + TimeUnit::type unit() const { return unit_; } const std::string& timezone() const { return timezone_; } private: - TimeUnit unit_; + TimeUnit::type unit_; std::string timezone_; }; @@ -710,15 +712,15 @@ std::shared_ptr<DataType> ARROW_EXPORT fixed_size_binary(int32_t byte_width); std::shared_ptr<DataType> ARROW_EXPORT list(const std::shared_ptr<Field>& value_type); std::shared_ptr<DataType> ARROW_EXPORT list(const std::shared_ptr<DataType>& value_type); -std::shared_ptr<DataType> ARROW_EXPORT timestamp(TimeUnit unit); +std::shared_ptr<DataType> ARROW_EXPORT timestamp(TimeUnit::type unit); std::shared_ptr<DataType> ARROW_EXPORT timestamp( - TimeUnit unit, const std::string& timezone); + TimeUnit::type unit, const std::string& timezone); /// Unit can be either SECOND or MILLI -std::shared_ptr<DataType> ARROW_EXPORT time32(TimeUnit unit); +std::shared_ptr<DataType> ARROW_EXPORT time32(TimeUnit::type unit); /// Unit can be either MICRO or NANO -std::shared_ptr<DataType> ARROW_EXPORT time64(TimeUnit unit); +std::shared_ptr<DataType> ARROW_EXPORT time64(TimeUnit::type unit); std::shared_ptr<DataType> ARROW_EXPORT struct_( const std::vector<std::shared_ptr<Field>>& fields); http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/python/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 36052bc..c1431af 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -28,7 +28,7 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/../cpp/cmake_mod include(CMakeParseArguments) -set(BUILD_SUPPORT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../cpp/build-support) +set(BUILD_SUPPORT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../cpp/build-support") # Allow "make install" to not depend on all targets. # @@ -58,10 +58,6 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") OFF) endif() -if(NOT PYARROW_BUILD_TESTS) - set(NO_TESTS 1) -endif() - find_program(CCACHE_FOUND ccache) if(CCACHE_FOUND) set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache) @@ -73,18 +69,19 @@ endif(CCACHE_FOUND) ############################################################ include(BuildUtils) -include(CompilerInfo) include(SetupCxxFlags) +include(CompilerInfo) # Add common flags set(CMAKE_CXX_FLAGS "${CXX_COMMON_FLAGS} ${CMAKE_CXX_FLAGS}") -# Enable perf and other tools to work properly -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer") - -# Suppress Cython warnings -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-variable") +if (NOT MSVC) + # Enable perf and other tools to work properly + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer") + # Suppress Cython warnings + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-variable") +endif() if ("${COMPILER_FAMILY}" STREQUAL "clang") # Using Clang with ccache causes a bunch of spurious warnings that are @@ -146,9 +143,10 @@ endif() # # The gold linker is only for ELF binaries, which OSX doesn't use. We can # just skip. -if (NOT APPLE) +if (NOT APPLE AND NOT MSVC) execute_process(COMMAND ${CMAKE_CXX_COMPILER} -Wl,--version OUTPUT_VARIABLE LINKER_OUTPUT) endif () + if (LINKER_OUTPUT MATCHES "gold") if ("${PYARROW_LINK}" STREQUAL "d" AND "${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE") @@ -166,9 +164,6 @@ endif() # act on its value. if ("${PYARROW_LINK}" STREQUAL "d") set(BUILD_SHARED_LIBS ON) - - # Position independent code is only necessary when producing shared objects. - add_definitions(-fPIC) endif() # set compile output directory @@ -188,9 +183,16 @@ if (${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_CURRENT_BINARY_DIR}) EXECUTE_PROCESS(COMMAND ln ${MORE_ARGS} -sf ${BUILD_OUTPUT_ROOT_DIRECTORY} ${CMAKE_CURRENT_BINARY_DIR}/build/latest) else() - set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${BUILD_SUBDIR_NAME}/") + if (MSVC) + # MSVC makes its own output directories based on the build configuration + set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/") + else() + set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${BUILD_SUBDIR_NAME}/") + endif() endif() +message(STATUS "Build output directory: ${BUILD_OUTPUT_ROOT_DIRECTORY}") + # where to put generated archives (.a files) set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}") set(ARCHIVE_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}") http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/python/cmake_modules/CompilerInfo.cmake ---------------------------------------------------------------------- diff --git a/python/cmake_modules/CompilerInfo.cmake b/python/cmake_modules/CompilerInfo.cmake deleted file mode 100644 index 8e85bde..0000000 --- a/python/cmake_modules/CompilerInfo.cmake +++ /dev/null @@ -1,48 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# Sets COMPILER_FAMILY to 'clang' or 'gcc' -# Sets COMPILER_VERSION to the version -execute_process(COMMAND "${CMAKE_CXX_COMPILER}" -v - ERROR_VARIABLE COMPILER_VERSION_FULL) -message(INFO " ${COMPILER_VERSION_FULL}") - -# clang on Linux and Mac OS X before 10.9 -if("${COMPILER_VERSION_FULL}" MATCHES ".*clang version.*") - set(COMPILER_FAMILY "clang") - string(REGEX REPLACE ".*clang version ([0-9]+\\.[0-9]+).*" "\\1" - COMPILER_VERSION "${COMPILER_VERSION_FULL}") -# clang on Mac OS X 10.9 and later -elseif("${COMPILER_VERSION_FULL}" MATCHES ".*based on LLVM.*") - set(COMPILER_FAMILY "clang") - string(REGEX REPLACE ".*based on LLVM ([0-9]+\\.[0.9]+).*" "\\1" - COMPILER_VERSION "${COMPILER_VERSION_FULL}") - -# clang on Mac OS X, XCode 7+. No version replacement is done -# because Apple no longer advertises the upstream LLVM version. -elseif("${COMPILER_VERSION_FULL}" MATCHES "clang-.*") - set(COMPILER_FAMILY "clang") - -# gcc -elseif("${COMPILER_VERSION_FULL}" MATCHES ".*gcc version.*") - set(COMPILER_FAMILY "gcc") - string(REGEX REPLACE ".*gcc version ([0-9\\.]+).*" "\\1" - COMPILER_VERSION "${COMPILER_VERSION_FULL}") -else() - message(FATAL_ERROR "Unknown compiler. Version info:\n${COMPILER_VERSION_FULL}") -endif() -message("Selected compiler ${COMPILER_FAMILY} ${COMPILER_VERSION}") http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/python/cmake_modules/FindArrow.cmake ---------------------------------------------------------------------- diff --git a/python/cmake_modules/FindArrow.cmake b/python/cmake_modules/FindArrow.cmake index 51a8871..8e13dd6 100644 --- a/python/cmake_modules/FindArrow.cmake +++ b/python/cmake_modules/FindArrow.cmake @@ -67,18 +67,23 @@ find_library(ARROW_PYTHON_LIB_PATH NAMES arrow_python if (ARROW_INCLUDE_DIR AND ARROW_LIBS) set(ARROW_FOUND TRUE) - set(ARROW_LIB_NAME libarrow) - set(ARROW_JEMALLOC_LIB_NAME libarrow_jemalloc) - set(ARROW_PYTHON_LIB_NAME libarrow_python) - set(ARROW_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_LIB_NAME}.a) - set(ARROW_SHARED_LIB ${ARROW_LIBS}/${ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}) + if (MSVC) + set(ARROW_STATIC_LIB ${ARROW_LIB_PATH}) + set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}) + set(ARROW_JEMALLOC_STATIC_LIB ${ARROW_JEMALLOC_LIB_PATH}) + set(ARROW_SHARED_LIB ${ARROW_STATIC_LIB}) + set(ARROW_PYTHON_SHARED_LIB ${ARROW_PYTHON_STATIC_LIB}) + set(ARROW_JEMALLOC_SHARED_LIB ${ARROW_JEMALLOC_STATIC_LIB}) + else() + set(ARROW_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow.a) + set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow_python.a) + set(ARROW_JEMALLOC_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow_jemalloc.a) - set(ARROW_JEMALLOC_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_JEMALLOC_LIB_NAME}.a) - set(ARROW_JEMALLOC_SHARED_LIB ${ARROW_LIBS}/${ARROW_JEMALLOC_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}) - - set(ARROW_PYTHON_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_PYTHON_LIB_NAME}.a) - set(ARROW_PYTHON_SHARED_LIB ${ARROW_LIBS}/${ARROW_PYTHON_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}) + set(ARROW_SHARED_LIB ${ARROW_LIBS}/libarrow${CMAKE_SHARED_LIBRARY_SUFFIX}) + set(ARROW_JEMALLOC_SHARED_LIB ${ARROW_LIBS}/libarrow_jemalloc${CMAKE_SHARED_LIBRARY_SUFFIX}) + set(ARROW_PYTHON_SHARED_LIB ${ARROW_LIBS}/libarrow_python${CMAKE_SHARED_LIBRARY_SUFFIX}) + endif() if (NOT Arrow_FIND_QUIETLY) message(STATUS "Found the Arrow core library: ${ARROW_LIB_PATH}") http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/python/cmake_modules/UseCython.cmake ---------------------------------------------------------------------- diff --git a/python/cmake_modules/UseCython.cmake b/python/cmake_modules/UseCython.cmake index 7c06b02..7920940 100644 --- a/python/cmake_modules/UseCython.cmake +++ b/python/cmake_modules/UseCython.cmake @@ -122,9 +122,11 @@ function( compile_pyx _name pyx_target_name generated_files pyx_file) endif() set_source_files_properties( ${_generated_files} PROPERTIES GENERATED TRUE ) - # Cython creates a lot of compiler warning detritus on clang - set_source_files_properties(${_generated_files} PROPERTIES - COMPILE_FLAGS -Wno-unused-function) + if (NOT WIN32) + # Cython creates a lot of compiler warning detritus on clang + set_source_files_properties(${_generated_files} PROPERTIES + COMPILE_FLAGS -Wno-unused-function) + endif() set( ${generated_files} ${_generated_files} PARENT_SCOPE ) http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/python/pyarrow/_config.pyx ---------------------------------------------------------------------- diff --git a/python/pyarrow/_config.pyx b/python/pyarrow/_config.pyx index 536f278..2c1e6bf 100644 --- a/python/pyarrow/_config.pyx +++ b/python/pyarrow/_config.pyx @@ -14,18 +14,13 @@ # distutils: language = c++ # cython: embedsignature = True -cdef extern from 'arrow/python/do_import_numpy.h': - pass - -cdef extern from 'arrow/python/numpy_interop.h' namespace 'arrow::py': - int import_numpy() +cdef extern from 'arrow/python/init.h' namespace 'arrow::py': + void InitNumPy() cdef extern from 'arrow/python/config.h' namespace 'arrow::py': - void Init() void set_numpy_nan(object o) -import_numpy() -Init() +InitNumPy() import numpy as np set_numpy_nan(np.nan) http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/python/pyarrow/_io.pyx ---------------------------------------------------------------------- diff --git a/python/pyarrow/_io.pyx b/python/pyarrow/_io.pyx index ec37de0..09e8233 100644 --- a/python/pyarrow/_io.pyx +++ b/python/pyarrow/_io.pyx @@ -807,7 +807,7 @@ cdef class _HdfsClient: cdef c_string c_path = tobytes(path) with nogil: check_status(self.client.get() - .CreateDirectory(c_path)) + .MakeDirectory(c_path)) def delete(self, path, bint recursive=False): """ http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/python/pyarrow/includes/common.pxd ---------------------------------------------------------------------- diff --git a/python/pyarrow/includes/common.pxd b/python/pyarrow/includes/common.pxd index ab38ff3..44723fa 100644 --- a/python/pyarrow/includes/common.pxd +++ b/python/pyarrow/includes/common.pxd @@ -26,8 +26,7 @@ from libcpp.vector cimport vector from cpython cimport PyObject cimport cpython -# This must be included for cerr and other things to work -cdef extern from "<iostream>": +cdef extern from "arrow/python/platform.h": pass cdef extern from "<Python.h>": http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/python/pyarrow/includes/libarrow.pxd ---------------------------------------------------------------------- diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index b8aa24c..ea835f6 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -54,7 +54,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: _Type_STRUCT" arrow::Type::STRUCT" _Type_DICTIONARY" arrow::Type::DICTIONARY" - enum TimeUnit" arrow::TimeUnit": + enum TimeUnit" arrow::TimeUnit::type": TimeUnit_SECOND" arrow::TimeUnit::SECOND" TimeUnit_MILLI" arrow::TimeUnit::MILLI" TimeUnit_MICRO" arrow::TimeUnit::MICRO" @@ -435,7 +435,7 @@ cdef extern from "arrow/io/hdfs.h" namespace "arrow::io" nogil: CStatus Connect(const HdfsConnectionConfig* config, shared_ptr[CHdfsClient]* client) - CStatus CreateDirectory(const c_string& path) + CStatus MakeDirectory(const c_string& path) CStatus Delete(const c_string& path, c_bool recursive) http://git-wip-us.apache.org/repos/asf/arrow/blob/7238d544/python/setup.py ---------------------------------------------------------------------- diff --git a/python/setup.py b/python/setup.py index 3991856..ab71e78 100644 --- a/python/setup.py +++ b/python/setup.py @@ -91,6 +91,13 @@ class build_ext(_build_ext): _build_ext.initialize_options(self) self.extra_cmake_args = os.environ.get('PYARROW_CMAKE_OPTIONS', '') self.build_type = os.environ.get('PYARROW_BUILD_TYPE', 'debug').lower() + + if sys.platform == 'win32': + # Cannot do debug builds in Windows unless Python itself is a debug + # build + if not hasattr(sys, 'gettotalrefcount'): + self.build_type = 'release' + self.with_parquet = strtobool( os.environ.get('PYARROW_WITH_PARQUET', '0')) self.with_jemalloc = strtobool( @@ -132,13 +139,10 @@ class build_ext(_build_ext): return static_lib_option = '' - build_tests_option = '' cmake_options = [ '-DPYTHON_EXECUTABLE=%s' % sys.executable, - '-DPYARROW_BUILD_TESTS=off', static_lib_option, - build_tests_option, ] if self.with_parquet: @@ -150,10 +154,10 @@ class build_ext(_build_ext): if self.bundle_arrow_cpp: cmake_options.append('-DPYARROW_BUNDLE_ARROW_CPP=ON') - if sys.platform != 'win32': - cmake_options.append('-DCMAKE_BUILD_TYPE={0}' - .format(self.build_type)) + cmake_options.append('-DCMAKE_BUILD_TYPE={0}' + .format(self.build_type)) + if sys.platform != 'win32': cmake_command = (['cmake', self.extra_cmake_args] + cmake_options + [source]) @@ -167,15 +171,15 @@ class build_ext(_build_ext): self.spawn(args) else: import shlex - cmake_generator = 'Visual Studio 14 2015' - if is_64_bit: - cmake_generator += ' Win64' + cmake_generator = 'Visual Studio 14 2015 Win64' + if not is_64_bit: + raise RuntimeError('Not supported on 32-bit Windows') + # Generate the build files extra_cmake_args = shlex.split(self.extra_cmake_args) cmake_command = (['cmake'] + extra_cmake_args + cmake_options + - [source, - '-G', cmake_generator]) + [source, '-G', cmake_generator]) if "-G" in self.extra_cmake_args: cmake_command = cmake_command[:-2] @@ -336,7 +340,7 @@ setup( use_scm_version={"root": "..", "relative_to": __file__}, setup_requires=['setuptools_scm', 'cython >= 0.23'], install_requires=['numpy >= 1.9', 'six >= 1.0.0'], - test_requires=['pytest'], + tests_require=['pytest'], description="Python library for Apache Arrow", long_description=long_description, classifiers=[