(arrow) branch ARROW-40181-glog07 deleted (was 147504673e)
This is an automated email from the ASF dual-hosted git repository. uwe pushed a change to branch ARROW-40181-glog07 in repository https://gitbox.apache.org/repos/asf/arrow.git was 147504673e GH-40181: [C++] Support glog 0.7 build The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository.
(arrow) 01/01: GH-40181: [C++] Support glog 0.7 build
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch ARROW-40181-glog07
in repository https://gitbox.apache.org/repos/asf/arrow.git
commit 147504673e409fc5267f0422a58f170f442b5455
Author: Uwe L. Korn
AuthorDate: Mon Feb 26 09:21:31 2024 +0100
GH-40181: [C++] Support glog 0.7 build
---
cpp/cmake_modules/FindGLOG.cmake | 8 +++-
cpp/src/arrow/util/logging.cc| 6 +++---
2 files changed, 10 insertions(+), 4 deletions(-)
diff --git a/cpp/cmake_modules/FindGLOG.cmake b/cpp/cmake_modules/FindGLOG.cmake
index 61b7d0694e..62b235ee91 100644
--- a/cpp/cmake_modules/FindGLOG.cmake
+++ b/cpp/cmake_modules/FindGLOG.cmake
@@ -17,6 +17,11 @@
#
# find_package(GLOG)
+find_package(glog CONFIG)
+if(glog_FOUND)
+ return()
+endif()
+
if(GLOG_FOUND)
return()
endif()
@@ -56,5 +61,6 @@ if(GLOG_FOUND)
add_library(glog::glog UNKNOWN IMPORTED)
set_target_properties(glog::glog
PROPERTIES IMPORTED_LOCATION "${GLOG_LIB}"
- INTERFACE_INCLUDE_DIRECTORIES
"${GLOG_INCLUDE_DIR}")
+ INTERFACE_INCLUDE_DIRECTORIES
"${GLOG_INCLUDE_DIR}"
+ INTERFACE_COMPILE_DEFINITIONS
"GLOG_USE_GLOG_EXPORT")
endif()
diff --git a/cpp/src/arrow/util/logging.cc b/cpp/src/arrow/util/logging.cc
index d293113237..25c336a6d2 100644
--- a/cpp/src/arrow/util/logging.cc
+++ b/cpp/src/arrow/util/logging.cc
@@ -116,7 +116,7 @@ static std::unique_ptr log_dir_;
#ifdef ARROW_USE_GLOG
// Glog's severity map.
-static int GetMappedSeverity(ArrowLogLevel severity) {
+static google::LogSeverity GetMappedSeverity(ArrowLogLevel severity) {
switch (severity) {
case ArrowLogLevel::ARROW_DEBUG:
return google::GLOG_INFO;
@@ -148,7 +148,7 @@ void ArrowLog::StartArrowLog(const std::string& app_name,
app_name_.reset(new std::string(app_name));
log_dir_.reset(new std::string(log_dir));
#ifdef ARROW_USE_GLOG
- int mapped_severity_threshold = GetMappedSeverity(severity_threshold_);
+ google::LogSeverity mapped_severity_threshold =
GetMappedSeverity(severity_threshold_);
google::SetStderrLogging(mapped_severity_threshold);
// Enable log file if log_dir is not empty.
if (!log_dir.empty()) {
@@ -173,7 +173,7 @@ void ArrowLog::StartArrowLog(const std::string& app_name,
google::SetLogFilenameExtension(app_name_without_path.c_str());
for (int i = static_cast(severity_threshold_);
i <= static_cast(ArrowLogLevel::ARROW_FATAL); ++i) {
- int level = GetMappedSeverity(static_cast(i));
+ google::LogSeverity level =
GetMappedSeverity(static_cast(i));
google::SetLogDestination(level, dir_ends_with_slash.c_str());
}
}
(arrow) branch ARROW-40181-glog07 created (now 147504673e)
This is an automated email from the ASF dual-hosted git repository. uwe pushed a change to branch ARROW-40181-glog07 in repository https://gitbox.apache.org/repos/asf/arrow.git at 147504673e GH-40181: [C++] Support glog 0.7 build This branch includes the following new commits: new 147504673e GH-40181: [C++] Support glog 0.7 build The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[arrow] branch conda-gcs-update updated (ec1246d -> bfb7b0f)
This is an automated email from the ASF dual-hosted git repository. uwe pushed a change to branch conda-gcs-update in repository https://gitbox.apache.org/repos/asf/arrow.git. from ec1246d ARROW-15670: [C++/Python/Packaging] Update conda pinnings and enable GCS on Windows add bfb7b0f aws-sdk-cpp 1.8.* doesn't work with newer CMake No new revisions were added by this update. Summary of changes: dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
[arrow] branch conda-gcs-update created (now ec1246d)
This is an automated email from the ASF dual-hosted git repository. uwe pushed a change to branch conda-gcs-update in repository https://gitbox.apache.org/repos/asf/arrow.git. at ec1246d ARROW-15670: [C++/Python/Packaging] Update conda pinnings and enable GCS on Windows This branch includes the following new commits: new ec1246d ARROW-15670: [C++/Python/Packaging] Update conda pinnings and enable GCS on Windows The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[arrow] 01/01: ARROW-15670: [C++/Python/Packaging] Update conda pinnings and enable GCS on Windows
This is an automated email from the ASF dual-hosted git repository. uwe pushed a commit to branch conda-gcs-update in repository https://gitbox.apache.org/repos/asf/arrow.git commit ec1246d845abaf94693ec5520c5b647754253539 Author: Uwe L. Korn AuthorDate: Sun Feb 13 09:04:31 2022 +0100 ARROW-15670: [C++/Python/Packaging] Update conda pinnings and enable GCS on Windows --- ...on10.2cxx_compiler_version7numpy1.18python3.7.cpython.yaml | 8 +--- ...on10.2cxx_compiler_version7numpy1.18python3.8.cpython.yaml | 8 +--- ...on10.2cxx_compiler_version7numpy1.19python3.9.cpython.yaml | 8 +--- ...n10.2cxx_compiler_version7numpy1.21python3.10.cpython.yaml | 8 +--- ...onNonecxx_compiler_version9numpy1.18python3.7.cpython.yaml | 8 +--- ...onNonecxx_compiler_version9numpy1.18python3.8.cpython.yaml | 8 +--- ...onNonecxx_compiler_version9numpy1.19python3.9.cpython.yaml | 8 +--- ...nNonecxx_compiler_version9numpy1.21python3.10.cpython.yaml | 8 +--- .../.ci_support/linux_aarch64_numpy1.18python3.7.cpython.yaml | 8 +--- .../.ci_support/linux_aarch64_numpy1.18python3.8.cpython.yaml | 8 +--- .../.ci_support/linux_aarch64_numpy1.19python3.9.cpython.yaml | 8 +--- .../linux_aarch64_numpy1.21python3.10.cpython.yaml| 8 +--- .../.ci_support/linux_ppc64le_numpy1.18python3.7.cpython.yaml | 8 +--- .../.ci_support/linux_ppc64le_numpy1.18python3.8.cpython.yaml | 8 +--- .../.ci_support/linux_ppc64le_numpy1.19python3.9.cpython.yaml | 8 +--- .../linux_ppc64le_numpy1.21python3.10.cpython.yaml| 8 +--- .../.ci_support/osx_64_numpy1.18python3.7.cpython.yaml| 8 +--- .../.ci_support/osx_64_numpy1.18python3.8.cpython.yaml| 8 +--- .../.ci_support/osx_64_numpy1.19python3.9.cpython.yaml| 8 +--- .../.ci_support/osx_64_numpy1.21python3.10.cpython.yaml | 8 +--- .../.ci_support/osx_arm64_numpy1.19python3.8.cpython.yaml | 8 +--- .../.ci_support/osx_arm64_numpy1.19python3.9.cpython.yaml | 8 +--- .../.ci_support/osx_arm64_numpy1.21python3.10.cpython.yaml| 8 +--- ...4_cuda_compiler_version10.2numpy1.18python3.7.cpython.yaml | 6 -- ...4_cuda_compiler_version10.2numpy1.18python3.8.cpython.yaml | 6 -- ...4_cuda_compiler_version10.2numpy1.19python3.9.cpython.yaml | 6 -- ..._cuda_compiler_version10.2numpy1.21python3.10.cpython.yaml | 6 -- ...4_cuda_compiler_versionNonenumpy1.18python3.7.cpython.yaml | 6 -- ...4_cuda_compiler_versionNonenumpy1.18python3.8.cpython.yaml | 6 -- ...4_cuda_compiler_versionNonenumpy1.19python3.9.cpython.yaml | 6 -- ..._cuda_compiler_versionNonenumpy1.21python3.10.cpython.yaml | 6 -- dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat | 1 + dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 2 +- 33 files changed, 149 insertions(+), 86 deletions(-) diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version7cuda_compiler_version10.2cxx_compiler_version7numpy1.18python3.7.cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version7cuda_compiler_version10.2cxx_compiler_version7numpy1.18python3.7.cpython.yaml index 4273c9b..9c6fe5c 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version7cuda_compiler_version10.2cxx_compiler_version7numpy1.18python3.7.cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version7cuda_compiler_version10.2cxx_compiler_version7numpy1.18python3.7.cpython.yaml @@ -24,8 +24,10 @@ gflags: - '2.2' glog: - '0.5' +google_cloud_cpp: +- '1.35' grpc_cpp: -- '1.42' +- '1.43' libprotobuf: - '3.19' lz4_c: @@ -35,7 +37,7 @@ numpy: openssl: - 1.1.1 orc: -- 1.7.2 +- 1.7.3 pin_run_as_build: bzip2: max_pin: x @@ -49,7 +51,7 @@ pin_run_as_build: python: - 3.7.* *_cpython re2: -- 2021.11.01 +- 2022.02.01 snappy: - '1' target_platform: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version7cuda_compiler_version10.2cxx_compiler_version7numpy1.18python3.8.cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version7cuda_compiler_version10.2cxx_compiler_version7numpy1.18python3.8.cpython.yaml index 7108ddf..4a1f196 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version7cuda_compiler_version10.2cxx_compiler_version7numpy1.18python3.8.cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_c_compiler_version7cuda_compiler_version10.2cxx_compiler_version7numpy1.18python3.8.cpython.yaml @@ -24,8 +24,10 @@ gflags: - '2.2' glog: - '0.5' +google_cloud_cpp: +- '1.35' grpc_cpp: -- '1.42' +- '1.43' libprotobuf: - '3.19'
[arrow] branch master updated (f878ee5 -> 0ef2080)
This is an automated email from the ASF dual-hosted git repository. uwe pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git. from f878ee5 ARROW-12150: [Python] Correctly infer type of mixed-precision Decimals add 0ef2080 ARROW-12738: [C++/Python/R] Update conda variant files No new revisions were added by this update. Summary of changes: ...ler_version10.2numpy1.17python3.6.cpython.yaml | 14 +++--- ...ler_version10.2numpy1.17python3.7.cpython.yaml | 14 +++--- ...ler_version10.2numpy1.17python3.8.cpython.yaml | 14 +++--- ...ler_version10.2numpy1.19python3.9.cpython.yaml | 14 +++--- ...ler_versionNonenumpy1.17python3.6.cpython.yaml | 14 +++--- ...ler_versionNonenumpy1.17python3.7.cpython.yaml | 4 ++-- ...ler_versionNonenumpy1.17python3.8.cpython.yaml | 14 +++--- ...ler_versionNonenumpy1.19python3.9.cpython.yaml | 14 +++--- .../linux_aarch64_numpy1.17python3.6.cpython.yaml | 12 ++-- .../linux_aarch64_numpy1.17python3.7.cpython.yaml | 12 ++-- .../linux_aarch64_numpy1.17python3.8.cpython.yaml | 12 ++-- .../linux_aarch64_numpy1.19python3.9.cpython.yaml | 12 ++-- .../osx_64_numpy1.17python3.6.cpython.yaml| 4 ++-- .../osx_64_numpy1.17python3.7.cpython.yaml| 4 ++-- .../osx_64_numpy1.17python3.8.cpython.yaml| 14 +++--- .../osx_64_numpy1.19python3.9.cpython.yaml| 14 +++--- .../.ci_support/osx_arm64_python3.8.cpython.yaml | 14 +++--- .../.ci_support/osx_arm64_python3.9.cpython.yaml | 14 +++--- .../.ci_support/r/linux_64_r_base4.0.yaml | 13 ++--- .../.ci_support/r/linux_64_r_base4.1.yaml | 2 +- .../conda-recipes/.ci_support/r/osx_64_r_base4.0.yaml | 2 +- .../conda-recipes/.ci_support/r/osx_64_r_base4.1.yaml | 2 +- ...ler_versionNonenumpy1.17python3.6.cpython.yaml | 12 ++-- ...ler_versionNonenumpy1.17python3.7.cpython.yaml | 12 ++-- ...ler_versionNonenumpy1.17python3.8.cpython.yaml | 12 ++-- ...ler_versionNonenumpy1.19python3.9.cpython.yaml | 12 ++-- dev/tasks/conda-recipes/azure.osx.yml | 19 +++ 27 files changed, 155 insertions(+), 145 deletions(-)
[arrow] branch master updated (60cfcf8 -> 979a080)
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git.
from 60cfcf8 ARROW-12648: [C++][FlightRPC] Enable TLS for Flight benchmark
add 979a080 ARROW-12649: [Python/Packaging] Move conda-aarch64 to Azure
with cross-compilation
No new revisions were added by this update.
Summary of changes:
...ux_aarch64_numpy1.17python3.6.cpython.yaml} | 19 +++---
...ux_aarch64_numpy1.17python3.7.cpython.yaml} | 19 +++---
...ux_aarch64_numpy1.17python3.8.cpython.yaml} | 19 +++---
...ux_aarch64_numpy1.19python3.9.cpython.yaml} | 19 +++---
.../linux_aarch64_python3.6.cpython.yaml | 71 --
.../linux_aarch64_python3.7.cpython.yaml | 71 --
.../linux_aarch64_python3.8.cpython.yaml | 71 --
.../linux_aarch64_python3.9.cpython.yaml | 71 --
dev/tasks/conda-recipes/drone-steps.sh | 33 --
dev/tasks/conda-recipes/drone.yml | 43 -
dev/tasks/tasks.yml| 6 +-
11 files changed, 39 insertions(+), 403 deletions(-)
copy
dev/tasks/conda-recipes/.ci_support/{linux_64_cuda_compiler_versionNonenumpy1.17python3.6.cpython.yaml
=> linux_aarch64_numpy1.17python3.6.cpython.yaml} (83%)
copy
dev/tasks/conda-recipes/.ci_support/{linux_64_cuda_compiler_versionNonenumpy1.17python3.7.cpython.yaml
=> linux_aarch64_numpy1.17python3.7.cpython.yaml} (83%)
copy
dev/tasks/conda-recipes/.ci_support/{linux_64_cuda_compiler_versionNonenumpy1.17python3.8.cpython.yaml
=> linux_aarch64_numpy1.17python3.8.cpython.yaml} (83%)
copy
dev/tasks/conda-recipes/.ci_support/{linux_64_cuda_compiler_versionNonenumpy1.19python3.9.cpython.yaml
=> linux_aarch64_numpy1.19python3.9.cpython.yaml} (83%)
delete mode 100644
dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.6.cpython.yaml
delete mode 100644
dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.7.cpython.yaml
delete mode 100644
dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.8.cpython.yaml
delete mode 100644
dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.9.cpython.yaml
delete mode 100755 dev/tasks/conda-recipes/drone-steps.sh
delete mode 100644 dev/tasks/conda-recipes/drone.yml
[arrow] branch master updated (26a5a04 -> c394a20)
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git.
from 26a5a04 ARROW-12390: [Rust] Inline from_trusted_len_iter,
try_from_trusted_len_iter, extend_from_slice
add c394a20 ARROW-12246: [CI] Sync conda recipes with upstream feedstock
No new revisions were added by this update.
Summary of changes:
...version10.2numpy1.17python3.6.cpython.yaml} | 35
...version10.2numpy1.17python3.7.cpython.yaml} | 35
...version10.2numpy1.17python3.8.cpython.yaml} | 35
...version10.2numpy1.19python3.9.cpython.yaml} | 33
...versionNonenumpy1.17python3.6.cpython.yaml} | 33
...versionNonenumpy1.17python3.7.cpython.yaml} | 33
...versionNonenumpy1.17python3.8.cpython.yaml} | 33
...versionNonenumpy1.19python3.9.cpython.yaml} | 31 +++
... => osx_64_numpy1.17python3.6.cpython.yaml} | 28 +++
... => osx_64_numpy1.17python3.7.cpython.yaml} | 28 +++
... => osx_64_numpy1.17python3.8.cpython.yaml} | 29 ---
... => osx_64_numpy1.19python3.9.cpython.yaml} | 26 +++---
...n.yaml => osx_arm64_python3.8.cpython.yaml} | 36
...n.yaml => osx_arm64_python3.9.cpython.yaml} | 34
...versionNonenumpy1.17python3.6.cpython.yaml} | 20 ++---
...versionNonenumpy1.17python3.7.cpython.yaml} | 20 ++---
...versionNonenumpy1.17python3.8.cpython.yaml} | 20 ++---
...versionNonenumpy1.19python3.9.cpython.yaml} | 18 ++--
dev/tasks/conda-recipes/.scripts/logging_utils.sh | 30 +++
dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat| 14
dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat | 14 +++-
dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh | 39 +
dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh | 14 ++--
dev/tasks/conda-recipes/arrow-cpp/meta.yaml| 97 --
dev/tasks/conda-recipes/azure.osx.yml | 11 ++-
dev/tasks/conda-recipes/build_steps.sh | 4 +
dev/tasks/conda-recipes/run_docker_build.sh| 4 +
dev/tasks/tasks.yml| 57 +
28 files changed, 460 insertions(+), 351 deletions(-)
rename
dev/tasks/conda-recipes/.ci_support/{linux_cuda_compiler_version9.2python3.6.cpython.yaml
=> linux_64_cuda_compiler_version10.2numpy1.17python3.6.cpython.yaml} (76%)
rename
dev/tasks/conda-recipes/.ci_support/{linux_cuda_compiler_version9.2python3.7.cpython.yaml
=> linux_64_cuda_compiler_version10.2numpy1.17python3.7.cpython.yaml} (76%)
rename
dev/tasks/conda-recipes/.ci_support/{linux_cuda_compiler_version9.2python3.8.cpython.yaml
=> linux_64_cuda_compiler_version10.2numpy1.17python3.8.cpython.yaml} (76%)
rename
dev/tasks/conda-recipes/.ci_support/{linux_cuda_compiler_version9.2python3.9.cpython.yaml
=> linux_64_cuda_compiler_version10.2numpy1.19python3.9.cpython.yaml} (77%)
rename
dev/tasks/conda-recipes/.ci_support/{linux_cuda_compiler_versionNonepython3.6.cpython.yaml
=> linux_64_cuda_compiler_versionNonenumpy1.17python3.6.cpython.yaml} (77%)
rename
dev/tasks/conda-recipes/.ci_support/{linux_cuda_compiler_versionNonepython3.7.cpython.yaml
=> linux_64_cuda_compiler_versionNonenumpy1.17python3.7.cpython.yaml} (77%)
rename
dev/tasks/conda-recipes/.ci_support/{linux_cuda_compiler_versionNonepython3.8.cpython.yaml
=> linux_64_cuda_compiler_versionNonenumpy1.17python3.8.cpython.yaml} (77%)
rename
dev/tasks/conda-recipes/.ci_support/{linux_cuda_compiler_versionNonepython3.9.cpython.yaml
=> linux_64_cuda_compiler_versionNonenumpy1.19python3.9.cpython.yaml} (78%)
rename dev/tasks/conda-recipes/.ci_support/{osx_python3.6.cpython.yaml =>
osx_64_numpy1.17python3.6.cpython.yaml} (81%)
rename dev/tasks/conda-recipes/.ci_support/{osx_python3.7.cpython.yaml =>
osx_64_numpy1.17python3.7.cpython.yaml} (81%)
rename dev/tasks/conda-recipes/.ci_support/{osx_python3.8.cpython.yaml =>
osx_64_numpy1.17python3.8.cpython.yaml} (77%)
copy dev/tasks/conda-recipes/.ci_support/{osx_python3.9.cpython.yaml =>
osx_64_numpy1.19python3.9.cpython.yaml} (82%)
copy dev/tasks/conda-recipes/.ci_support/{osx_python3.9.cpython.yaml =>
osx_arm64_python3.8.cpython.yaml} (71%)
rename dev/tasks/conda-recipes/.ci_support/{osx_python3.9.cpython.yaml =>
osx_arm64_python3.9.cpython.yaml} (74%)
rename dev/tasks/conda-recipes/.ci_support/{win_python3.6.cpython.yaml =>
win_64_cuda_compiler_versionNonenumpy1.17python3.6.cpython.yaml} (82%)
rename dev/tasks/conda-recipes/.ci_support/{win_python3.7.cpython.yaml =>
win_64_cuda_compiler_versionNonenumpy1.17python3.7.cpython.yaml} (82%)
rename dev/tasks/conda-recipes/.ci_support/{win_python3.8.cpython.yaml =>
[arrow] branch master updated (0b020a1 -> d7da16e)
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git.
from 0b020a1 ARROW-11683: [R] Support dplyr::mutate()
add d7da16e ARROW-11695: [C++][FlightRPC] fix option to disable TLS
verification
No new revisions were added by this update.
Summary of changes:
cpp/src/arrow/flight/CMakeLists.txt| 87 ++
cpp/src/arrow/flight/client.cc | 38 +++---
...check_tls_opts_132.cc => check_tls_opts_134.cc} | 20 +++--
...check_tls_opts_132.cc => check_tls_opts_136.cc} | 16 ++--
4 files changed, 107 insertions(+), 54 deletions(-)
copy cpp/src/arrow/flight/try_compile/{check_tls_opts_132.cc =>
check_tls_opts_134.cc} (64%)
copy cpp/src/arrow/flight/try_compile/{check_tls_opts_132.cc =>
check_tls_opts_136.cc} (69%)
[arrow] branch master updated (dbc77d2 -> 26a7dcd)
This is an automated email from the ASF dual-hosted git repository. uwe pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git. from dbc77d2 ARROW-11350: [C++] Bump dependency versions add 26a7dcd ARROW-11472: [Python][CI] Temporary pin numpy on kartothek integration builds No new revisions were added by this update. Summary of changes: ci/docker/conda-python-kartothek.dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
[arrow] branch master updated (5b9ce64 -> d713be4)
This is an automated email from the ASF dual-hosted git repository. uwe pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git. from 5b9ce64 ARROW-11406: [CI][C++] Fix ccache caching on Travis-CI add d713be4 ARROW-11372: [Release] Support RC verification on macOS-ARM64 No new revisions were added by this update. Summary of changes: dev/release/verify-release-candidate.sh | 8 ++-- 1 file changed, 6 insertions(+), 2 deletions(-)
[arrow] branch master updated (9560204 -> 98f01c5)
This is an automated email from the ASF dual-hosted git repository. uwe pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git. from 9560204 ARROW-10350: [Rust] Fixes to publication metadata in Cargo.toml add 98f01c5 ARROW-10180: [C++][Doc] Update dependency management docs No new revisions were added by this update. Summary of changes: docs/source/developers/cpp/building.rst | 32 ++-- 1 file changed, 18 insertions(+), 14 deletions(-)
[arrow] branch master updated (36d38b5 -> d1ffe72)
This is an automated email from the ASF dual-hosted git repository. uwe pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git. from 36d38b5 ARROW-7363: [Python] add combine_chunks method to ChunkedArray add d1ffe72 ARROW-10833: [Python] Allow pyarrow to be compiled on NumPy <1.16.6 and work on 1.20+ No new revisions were added by this update. Summary of changes: cpp/src/arrow/python/numpy_convert.cc | 4 ++-- cpp/src/arrow/python/numpy_to_arrow.cc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-)
[arrow] branch master updated (e58e607 -> 2cae0b3)
This is an automated email from the ASF dual-hosted git repository. uwe pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git. from e58e607 ARROW-10742: [Python] Check mask when creating array from numpy add 2cae0b3 ARROW-10843: [C++] Add support for temporal types in sort family kernels No new revisions were added by this update. Summary of changes: cpp/src/arrow/compute/kernels/vector_sort.cc | 50 ++--- cpp/src/arrow/compute/kernels/vector_sort_test.cc | 127 -- cpp/src/arrow/testing/gtest_util.h| 15 +-- cpp/src/arrow/type_fwd.h | 2 +- 4 files changed, 162 insertions(+), 32 deletions(-)
[arrow] branch master updated: ARROW-5679: [Python][CI] Remove Python 3.5 support
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new fd8843d ARROW-5679: [Python][CI] Remove Python 3.5 support
fd8843d is described below
commit fd8843dd6573fafc7babb84c53e9ee7accbb2724
Author: Antoine Pitrou
AuthorDate: Wed Dec 2 14:45:49 2020 +0100
ARROW-5679: [Python][CI] Remove Python 3.5 support
Closes #8776 from pitrou/ARROW-5679-drop-py-35
Authored-by: Antoine Pitrou
Signed-off-by: Uwe L. Korn
---
.github/workflows/archery.yml | 2 +-
.github/workflows/python.yml| 7 ---
dev/archery/setup.py| 4 +-
dev/release/verify-release-candidate-wheels.bat | 8
dev/tasks/tasks.yml | 62 -
docs/source/developers/archery.rst | 2 +-
docs/source/python/benchmarks.rst | 3 +-
docs/source/python/install.rst | 2 +-
python/manylinux1/build_arrow.sh| 2 +-
python/manylinux1/scripts/build_python.sh | 2 +-
python/manylinux201x/build_arrow.sh | 2 +-
python/pyarrow/tests/test_fs.py | 4 --
python/pyarrow/tests/test_ipc.py| 6 +--
python/pyarrow/tests/test_pandas.py | 3 +-
python/pyarrow/tests/test_serialization.py | 3 +-
python/pyarrow/tests/test_types.py | 2 +-
python/requirements-build.txt | 1 -
python/requirements-test.txt| 6 +--
python/requirements-wheel-test.txt | 6 +--
python/setup.py | 3 +-
20 files changed, 18 insertions(+), 112 deletions(-)
diff --git a/.github/workflows/archery.yml b/.github/workflows/archery.yml
index b230bce..7dd75b5 100644
--- a/.github/workflows/archery.yml
+++ b/.github/workflows/archery.yml
@@ -51,7 +51,7 @@ jobs:
- name: Setup Python
uses: actions/setup-python@v1
with:
- python-version: '3.5'
+ python-version: '3.6'
- name: Install Archery, Crossbow- and Test Dependencies
working-directory: dev/archery
run: pip install pytest responses toolz jinja2 -e .[all]
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
index 6b84ed2..02829eb 100644
--- a/.github/workflows/python.yml
+++ b/.github/workflows/python.yml
@@ -47,18 +47,11 @@ jobs:
fail-fast: false
matrix:
name:
- - ubuntu-16.04-python-3
- conda-python-3.8-nopandas
- conda-python-3.6-pandas-0.23
- conda-python-3.7-pandas-latest
- centos-python-3.6-manylinux1
include:
- - name: ubuntu-16.04-python-3
-cache: ubuntu-16.04-python-3
-image: ubuntu-python
-# this image always builds with python 3.5
-title: AMD64 Ubuntu 16.04 Python 3.5
-ubuntu: 16.04
- name: conda-python-3.8-nopandas
cache: conda-python-3.8
image: conda-python
diff --git a/dev/archery/setup.py b/dev/archery/setup.py
index 8823ace..27d7d4d 100755
--- a/dev/archery/setup.py
+++ b/dev/archery/setup.py
@@ -21,8 +21,8 @@ import operator
import sys
from setuptools import setup
-if sys.version_info < (3, 5):
-sys.exit('Python < 3.5 is not supported')
+if sys.version_info < (3, 6):
+sys.exit('Python < 3.6 is not supported')
extras = {
'benchmark': ['pandas'],
diff --git a/dev/release/verify-release-candidate-wheels.bat
b/dev/release/verify-release-candidate-wheels.bat
index daf8487..2b57113 100644
--- a/dev/release/verify-release-candidate-wheels.bat
+++ b/dev/release/verify-release-candidate-wheels.bat
@@ -49,9 +49,6 @@ call deactivate
set ARROW_TEST_DATA=%cd%\arrow\testing\data
-CALL :verify_wheel 3.5 %1 %2 m
-if errorlevel 1 GOTO error
-
CALL :verify_wheel 3.6 %1 %2 m
if errorlevel 1 GOTO error
@@ -100,11 +97,6 @@ py.test %CONDA_ENV_PATH%\Lib\site-packages\pyarrow --pdb -v
|| EXIT /B 1
python -c "import pyarrow" || EXIT /B 1
python -c "import pyarrow.parquet" || EXIT /B 1
-
-if "%PY_VERSION%"=="3.5" GOTO done
-
-:python36_and_higher_checks
-
python -c "import pyarrow.flight" || EXIT /B 1
python -c "import pyarrow.dataset" || EXIT /B 1
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index c5a6948..b955948 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -296,20 +296,6 @@ tasks:
## Wheel Linux ##
- wheel-manylinux1-cp35m:
-ci: azure
-template: python-wheels/azure.linux.yml
-params:
- python_version: 3.5
- unicode_width: 16
- whee
[arrow] branch master updated (2b68a7b -> e74891e)
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git.
from 2b68a7b ARROW-10690: [Java] Fix ComplexCopier bug for list vector
add e74891e ARROW-10724: [Dev Tools] Added labeler to PRs that need
rebase.
No new revisions were added by this update.
Summary of changes:
.../{dev_labeler.yml => dev_labeler_pr_rebase.yml} | 18 +-
1 file changed, 9 insertions(+), 9 deletions(-)
copy .github/workflows/{dev_labeler.yml => dev_labeler_pr_rebase.yml} (74%)
[arrow] branch master updated (b26478a -> ddda960)
This is an automated email from the ASF dual-hosted git repository. uwe pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git. from b26478a ARROW-10467: [FlightRPC][Java] Add the ability to pass arbitrary client headers. add ddda960 ARROW-10482: [Python] Fix compression per column in Parquet writing No new revisions were added by this update. Summary of changes: python/pyarrow/_parquet.pyx | 2 +- python/pyarrow/tests/test_parquet.py | 5 + 2 files changed, 6 insertions(+), 1 deletion(-)
[arrow] branch master updated (b26478a -> ddda960)
This is an automated email from the ASF dual-hosted git repository. uwe pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git. from b26478a ARROW-10467: [FlightRPC][Java] Add the ability to pass arbitrary client headers. add ddda960 ARROW-10482: [Python] Fix compression per column in Parquet writing No new revisions were added by this update. Summary of changes: python/pyarrow/_parquet.pyx | 2 +- python/pyarrow/tests/test_parquet.py | 5 + 2 files changed, 6 insertions(+), 1 deletion(-)
[arrow] branch master updated (590541d -> c49b5da)
This is an automated email from the ASF dual-hosted git repository. uwe pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git. from 590541d ARROW-9747: [Java][C++] Initial Support for 256-bit Decimals add c49b5da ARROW-10302: [Python] Don't double-package plasma-store-server No new revisions were added by this update. Summary of changes: dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh | 1 + dev/tasks/conda-recipes/arrow-cpp/meta.yaml| 1 + python/pyarrow/plasma.py | 4 python/setup.py| 8 ++-- 4 files changed, 12 insertions(+), 2 deletions(-)
[arrow] branch master updated (4fd0664 -> 09dc0cc)
This is an automated email from the ASF dual-hosted git repository. uwe pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git. from 4fd0664 ARROW-10090: [C++][Compute] Improve mode kernel add 09dc0cc ARROW-10104: [Python] Separate tests into its own conda package No new revisions were added by this update. Summary of changes: .../linux_aarch64_python3.6.cpython.yaml | 6 ++- .../linux_aarch64_python3.7.cpython.yaml | 6 ++- .../linux_aarch64_python3.8.cpython.yaml | 6 ++- ...a_compiler_version9.2python3.6.cpython.yaml | 6 ++- ...a_compiler_version9.2python3.7.cpython.yaml | 6 ++- ...a_compiler_version9.2python3.8.cpython.yaml | 6 ++- ..._compiler_versionNonepython3.6.cpython.yaml | 6 ++- ..._compiler_versionNonepython3.7.cpython.yaml | 6 ++- ..._compiler_versionNonepython3.8.cpython.yaml | 6 ++- .../.ci_support/osx_python3.6.cpython.yaml | 6 ++- .../.ci_support/osx_python3.7.cpython.yaml | 6 ++- .../.ci_support/osx_python3.8.cpython.yaml | 6 ++- .../.ci_support/win_python3.6.cpython.yaml | 4 +- .../.ci_support/win_python3.7.cpython.yaml | 4 +- .../.ci_support/win_python3.8.cpython.yaml | 4 +- dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat | 4 ++ dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh | 4 ++ dev/tasks/conda-recipes/arrow-cpp/meta.yaml| 50 ++ 18 files changed, 112 insertions(+), 30 deletions(-)
[arrow] branch master updated (c0dd2e2 -> 6a35f8a)
This is an automated email from the ASF dual-hosted git repository. uwe pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git. from c0dd2e2 ARROW-8601: [Go][Flight] Implementations Flight RPC server and client add 6a35f8a ARROW-10081: [C++/Python] Fix bash syntax in drone.io conda builds No new revisions were added by this update. Summary of changes: dev/tasks/conda-recipes/drone-steps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[arrow] branch master updated (8595406 -> 69d57d4)
This is an automated email from the ASF dual-hosted git repository. uwe pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git. from 8595406 ARROW-10060: [Rust] [DataFusion] Fixed error on which Err were discarded in MergeExec. add 69d57d4 ARROW-10064: [C++] Resolve compile warnings on Apple Clang 12 No new revisions were added by this update. Summary of changes: cpp/src/arrow/ipc/metadata_internal.cc | 2 +- cpp/src/arrow/util/uri.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)
[arrow] branch master updated (9ea2409 -> da641aa)
This is an automated email from the ASF dual-hosted git repository. uwe pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git. from 9ea2409 ARROW-9919: [Rust][DataFusion] Speedup math operations by 15%+ add da641aa ARROW-9920: [Python] Validate input to pa.concat_arrays() to avoid segfault No new revisions were added by this update. Summary of changes: python/pyarrow/array.pxi | 3 +++ python/pyarrow/tests/test_array.py | 11 +++ 2 files changed, 14 insertions(+)
[arrow] branch master updated (e5c12bc -> 0891feb)
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git.
from e5c12bc ARROW-7012: [C++] Add comments explaining high level detail
about ChunkedArray class and questions about chunk sizes
add 0891feb ARROW-8149: [C++/Python] Enable CUDA Support in conda recipes
No new revisions were added by this update.
Summary of changes:
...ml => linux_aarch64_python3.6.cpython.yaml} | 24 +-
...ml => linux_aarch64_python3.7.cpython.yaml} | 24 +-
...ml => linux_aarch64_python3.8.cpython.yaml} | 24 +-
..._compiler_version9.2python3.6.cpython.yaml} | 23 +-
..._compiler_version9.2python3.7.cpython.yaml} | 23 +-
..._compiler_version9.2python3.8.cpython.yaml} | 23 +-
...compiler_versionNonepython3.6.cpython.yaml} | 21 +-
...compiler_versionNonepython3.7.cpython.yaml} | 21 +-
...compiler_versionNonepython3.8.cpython.yaml} | 21 +-
...thon3.7.yaml => osx_python3.6.cpython.yaml} | 14 +-
...thon3.8.yaml => osx_python3.7.cpython.yaml} | 14 +-
...thon3.6.yaml => osx_python3.8.cpython.yaml} | 14 +-
...3.7vc14.yaml => win_python3.6.cpython.yaml} | 27 +-
...3.6vc14.yaml => win_python3.7.cpython.yaml} | 27 +-
...3.8vc14.yaml => win_python3.8.cpython.yaml} | 27 +-
dev/tasks/conda-recipes/README.md | 10 +-
dev/tasks/conda-recipes/arrow-cpp/LLVM_LICENSE.txt | 68 +
.../arrow-cpp/{bld.bat => bld-arrow.bat} | 39 +--
.../{pyarrow/bld.bat => arrow-cpp/bld-pyarrow.bat} | 2 +-
.../arrow-cpp/{build.sh => build-arrow.sh} | 36 ++-
.../build.sh => arrow-cpp/build-pyarrow.sh}| 15 +
dev/tasks/conda-recipes/arrow-cpp/meta.yaml| 322 ++---
dev/tasks/conda-recipes/azure.linux.yml| 13 +-
dev/tasks/conda-recipes/azure.osx.yml | 2 +-
dev/tasks/conda-recipes/azure.win.yml | 11 +-
dev/tasks/conda-recipes/build_steps.sh | 3 +-
dev/tasks/conda-recipes/pyarrow/meta.yaml | 79 -
dev/tasks/tasks.yml| 87 --
28 files changed, 623 insertions(+), 391 deletions(-)
copy dev/tasks/conda-recipes/.ci_support/{linux_python3.8.yaml =>
linux_aarch64_python3.6.cpython.yaml} (70%)
copy dev/tasks/conda-recipes/.ci_support/{linux_python3.8.yaml =>
linux_aarch64_python3.7.cpython.yaml} (70%)
copy dev/tasks/conda-recipes/.ci_support/{linux_python3.8.yaml =>
linux_aarch64_python3.8.cpython.yaml} (70%)
copy dev/tasks/conda-recipes/.ci_support/{linux_python3.6.yaml =>
linux_cuda_compiler_version9.2python3.6.cpython.yaml} (69%)
copy dev/tasks/conda-recipes/.ci_support/{linux_python3.6.yaml =>
linux_cuda_compiler_version9.2python3.7.cpython.yaml} (69%)
copy dev/tasks/conda-recipes/.ci_support/{linux_python3.6.yaml =>
linux_cuda_compiler_version9.2python3.8.cpython.yaml} (69%)
rename dev/tasks/conda-recipes/.ci_support/{linux_python3.6.yaml =>
linux_cuda_compiler_versionNonepython3.6.cpython.yaml} (74%)
rename dev/tasks/conda-recipes/.ci_support/{linux_python3.7.yaml =>
linux_cuda_compiler_versionNonepython3.7.cpython.yaml} (74%)
rename dev/tasks/conda-recipes/.ci_support/{linux_python3.8.yaml =>
linux_cuda_compiler_versionNonepython3.8.cpython.yaml} (74%)
rename dev/tasks/conda-recipes/.ci_support/{osx_python3.7.yaml =>
osx_python3.6.cpython.yaml} (88%)
rename dev/tasks/conda-recipes/.ci_support/{osx_python3.8.yaml =>
osx_python3.7.cpython.yaml} (88%)
rename dev/tasks/conda-recipes/.ci_support/{osx_python3.6.yaml =>
osx_python3.8.cpython.yaml} (88%)
rename
dev/tasks/conda-recipes/.ci_support/{win_c_compilervs2015cxx_compilervs2015python3.7vc14.yaml
=> win_python3.6.cpython.yaml} (71%)
rename
dev/tasks/conda-recipes/.ci_support/{win_c_compilervs2015cxx_compilervs2015python3.6vc14.yaml
=> win_python3.7.cpython.yaml} (71%)
rename
dev/tasks/conda-recipes/.ci_support/{win_c_compilervs2015cxx_compilervs2015python3.8vc14.yaml
=> win_python3.8.cpython.yaml} (71%)
create mode 100644 dev/tasks/conda-recipes/arrow-cpp/LLVM_LICENSE.txt
rename dev/tasks/conda-recipes/arrow-cpp/{bld.bat => bld-arrow.bat} (90%)
rename dev/tasks/conda-recipes/{pyarrow/bld.bat => arrow-cpp/bld-pyarrow.bat}
(97%)
rename dev/tasks/conda-recipes/arrow-cpp/{build.sh => build-arrow.sh} (51%)
rename dev/tasks/conda-recipes/{pyarrow/build.sh =>
arrow-cpp/build-pyarrow.sh} (57%)
mode change 100644 => 100755
delete mode 100644 dev/tasks/conda-recipes/pyarrow/meta.yaml
[arrow] branch master updated (04a1867 -> 705e194)
This is an automated email from the ASF dual-hosted git repository. uwe pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git. from 04a1867 ARROW-8993: [Rust] support reading non-seekable sources add 705e194 ARROW-8965: [Python][Doc] Pyarrow documentation for pip nightlies references 404'd location No new revisions were added by this update. Summary of changes: docs/source/python/install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[arrow] branch master updated (04a1867 -> 705e194)
This is an automated email from the ASF dual-hosted git repository. uwe pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git. from 04a1867 ARROW-8993: [Rust] support reading non-seekable sources add 705e194 ARROW-8965: [Python][Doc] Pyarrow documentation for pip nightlies references 404'd location No new revisions were added by this update. Summary of changes: docs/source/python/install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[arrow] branch master updated (20c0858 -> bd58907)
This is an automated email from the ASF dual-hosted git repository. uwe pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git. from 20c0858 ARROW-9073: [C++] Fix RapidJSON include directory detection with RapidJSONConfig.cmake add bd58907 ARROW-9074: [GLib] Add missing arrow-json check No new revisions were added by this update. Summary of changes: c_glib/configure.ac | 1 + c_glib/meson.build | 1 + 2 files changed, 2 insertions(+)
[arrow] branch master updated (d00c50a -> 20c0858)
This is an automated email from the ASF dual-hosted git repository. uwe pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git. from d00c50a ARROW-3688: [Rust] Add append_values for primitive builders add 20c0858 ARROW-9073: [C++] Fix RapidJSON include directory detection with RapidJSONConfig.cmake No new revisions were added by this update. Summary of changes: cpp/cmake_modules/ThirdpartyToolchain.cmake | 8 ++-- 1 file changed, 6 insertions(+), 2 deletions(-)
[arrow] branch master updated (076a0f6 -> 0876ee5)
This is an automated email from the ASF dual-hosted git repository. uwe pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git. from 076a0f6 ARROW-8978: [C++][CI] Fix valgrind warnings in cpp-conda-valgrind nightly build add 0876ee5 ARROW-8984: [R] Revise install guides now that Windows conda package exists No new revisions were added by this update. Summary of changes: r/R/install-arrow.R | 2 +- r/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)
[arrow] branch master updated (bdfdc79 -> ce4fa01)
This is an automated email from the ASF dual-hosted git repository. uwe pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git. from bdfdc79 ARROW-7490: [Java] Avro converter should convert attributes and props to FieldType metadata add ce4fa01 ARROW-6195: [C++] Detect Apache mirror without Python No new revisions were added by this update. Summary of changes: cpp/build-support/get_apache_mirror.py | 50 - cpp/cmake_modules/ThirdpartyToolchain.cmake | 30 ++--- 2 files changed, 26 insertions(+), 54 deletions(-) delete mode 100755 cpp/build-support/get_apache_mirror.py
[arrow] branch master updated (74fa956 -> 56adb6a)
This is an automated email from the ASF dual-hosted git repository. uwe pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git. from 74fa956 ARROW-7072: [Java] Support concating validity bits efficiently add 56adb6a PARQUET-1689: [C++] Stream API: Allow for columns/rows to be skipped when reading No new revisions were added by this update. Summary of changes: cpp/src/parquet/stream_reader.cc | 111 ++- cpp/src/parquet/stream_reader.h | 33 +- cpp/src/parquet/stream_reader_test.cc | 201 ++ 3 files changed, 338 insertions(+), 7 deletions(-)
[arrow] branch master updated (a179933 -> b74b027)
This is an automated email from the ASF dual-hosted git repository. uwe pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git. from a179933 ARROW-6392: [FlightRPC][Python] check type of list_flights result add b74b027 ARROW-6403: [Python] Expose FileReader::ReadRowGroups() to Python No new revisions were added by this update. Summary of changes: python/pyarrow/_parquet.pxd | 6 + python/pyarrow/_parquet.pyx | 13 +-- python/pyarrow/parquet.py| 30 python/pyarrow/tests/test_parquet.py | 44 4 files changed, 91 insertions(+), 2 deletions(-)
[arrow] branch master updated: ARROW-5919: [R] Test R-in-conda as a nightly build
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new b4698e6 ARROW-5919: [R] Test R-in-conda as a nightly build
b4698e6 is described below
commit b4698e6e7513dd83053542e195cd5c112e8b8947
Author: Uwe L. Korn
AuthorDate: Mon Jul 15 11:16:11 2019 +0200
ARROW-5919: [R] Test R-in-conda as a nightly build
Author: Uwe L. Korn
Author: Uwe L. Korn
Closes #4855 from xhochy/conda-r-docker and squashes the following commits:
b3189b863 Add LICENSE to conda_env_r
971181dec Add to nightlies
743aa5955 Update Dockerfile.conda
b79e0c08e ARROW-: Test R-in-conda as a nightly build
---
ci/conda_env_r.yml | 36
dev/tasks/tests.yml | 11 +++
docker-compose.yml | 15 +++
r/Dockerfile.conda | 34 ++
4 files changed, 96 insertions(+)
diff --git a/ci/conda_env_r.yml b/ci/conda_env_r.yml
new file mode 100644
index 000..0eff8a3
--- /dev/null
+++ b/ci/conda_env_r.yml
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+r-assertthat
+r-base
+r-bit64
+r-fs
+r-purrr
+r-r6
+r-rcpp >=0.12.18.2
+r-rlang
+r-tidyselect
+# Test dependencies
+pandoc
+r-covr
+r-hms
+r-lubridate
+r-pkgdown
+r-rmarkdown
+r-roxygen2
+r-testthat
+r-tibble
diff --git a/dev/tasks/tests.yml b/dev/tasks/tests.yml
index 32e2ed5..9674437 100644
--- a/dev/tasks/tests.yml
+++ b/dev/tasks/tests.yml
@@ -20,6 +20,7 @@ groups:
# makes it easier to submit related tasks
docker:
- docker-r
+- docker-r-conda
- docker-rust
- docker-cpp
- docker-cpp-alpine
@@ -88,6 +89,16 @@ tasks:
- docker-compose build r
- docker-compose run r
+ docker-r-conda:
+ci: circle
+platform: linux
+template: docker-tests/circle.linux.yml
+params:
+ commands:
+- docker-compose build cpp
+- docker-compose build r-conda
+- docker-compose run r-conda
+
docker-rust:
ci: circle
platform: linux
diff --git a/docker-compose.yml b/docker-compose.yml
index 3b50cd1..d2c6029 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -400,6 +400,21 @@ services:
dockerfile: r/Dockerfile
volumes: *ubuntu-volumes
+ r-conda:
+# Usage:
+# export R_VERSION=3.5.1
+# docker-compose build cpp
+# docker-compose build r-conda
+# docker-compose run r-conda
+image: arrow:r-conda-${R_VERSION:-3.5.1}
+shm_size: 2G
+build:
+ context: .
+ dockerfile: r/Dockerfile.conda
+ args:
+R_VERSION: ${R_VERSION:-3.5.1}
+volumes: *ubuntu-volumes
+
# Tools and Linters #
# TODO(kszucs): site
diff --git a/r/Dockerfile.conda b/r/Dockerfile.conda
new file mode 100644
index 000..2c664eb
--- /dev/null
+++ b/r/Dockerfile.conda
@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM arrow:cpp
+
+# install R specific packages
+ARG R_VERSION=3.5.1
+COPY ci/conda_env_r.yml /arrow/ci/
+RUN conda install -q \
+--file arrow/ci/conda_env_r.yml \
+r-base=$R_VERSION \
+nomkl && \
+conda clean --all
+
+ENV ARROW_PYTHON=OFF \
+ARROW_BUILD_TESTS=OFF
+
+# build, install,
[arrow] branch master updated: ARROW-5874: [Python] Fix macOS wheels to depend on system or Homebrew OpenSSL
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 00505b5 ARROW-5874: [Python] Fix macOS wheels to depend on system or
Homebrew OpenSSL
00505b5 is described below
commit 00505b55625ed47d239ca2fb8087bd708097b414
Author: Krisztián Szűcs
AuthorDate: Mon Jul 8 19:51:01 2019 +0200
ARROW-5874: [Python] Fix macOS wheels to depend on system or Homebrew
OpenSSL
The wheel build and test build succeeds, just the deployments fails because
of removed openssl for testing:
https://travis-ci.org/ursa-labs/crossbow/builds/555822505
The currently running build should pass:
https://travis-ci.org/ursa-labs/crossbow/builds/555849713
although we have a bunch of warnings ` was built for newer OSX version
(10.12) than being linked (10.9)` for the dependencies coming from brew.
cc @xhochy
Author: Krisztián Szűcs
Closes #4823 from kszucs/osx-wheel-openssl and squashes the following
commits:
eb45ae47d reinstall openssl
ca8d177a6 ignore deps
be4ab0f77 openssl
---
dev/tasks/python-wheels/osx-build.sh | 1 +
dev/tasks/python-wheels/travis.osx.yml | 4 +++-
2 files changed, 4 insertions(+), 1 deletion(-)
diff --git a/dev/tasks/python-wheels/osx-build.sh
b/dev/tasks/python-wheels/osx-build.sh
index b299646..517a63a 100755
--- a/dev/tasks/python-wheels/osx-build.sh
+++ b/dev/tasks/python-wheels/osx-build.sh
@@ -138,6 +138,7 @@ function build_wheel {
-DgRPC_SOURCE=SYSTEM \
-Dc-ares_SOURCE=BUNDLED \
-DARROW_PROTOBUF_USE_SHARED=OFF \
+ -DOPENSSL_USE_STATIC_LIBS=ON \
-DMAKE=make \
..
make -j5
diff --git a/dev/tasks/python-wheels/travis.osx.yml
b/dev/tasks/python-wheels/travis.osx.yml
index 980d122..8bea257 100644
--- a/dev/tasks/python-wheels/travis.osx.yml
+++ b/dev/tasks/python-wheels/travis.osx.yml
@@ -68,11 +68,13 @@ install:
# test the built wheels, remove llvm and grpc dependencies to ensure
# things are properly statically-linked
- - brew uninstall llvm@7 grpc c-ares
+ - brew uninstall --ignore-dependencies llvm@7 grpc c-ares openssl
- install_run arrow
# move built wheels to a top level directory
- mv -v arrow/python/dist/* dist/
+ # reinstall openssl because travis' deployment script depends on it
+ - brew install openssl
deploy:
provider: releases
[arrow] branch master updated: ARROW-5565: [Python][Docs] Add instructions how to use gdb to debug C++ libraries when running Python unit tests
This is an automated email from the ASF dual-hosted git repository. uwe pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git The following commit(s) were added to refs/heads/master by this push: new c938464 ARROW-5565: [Python][Docs] Add instructions how to use gdb to debug C++ libraries when running Python unit tests c938464 is described below commit c9384641e44707c41f78703a8be738e77a072896 Author: Wes McKinney AuthorDate: Fri Jun 14 13:43:11 2019 +0200 ARROW-5565: [Python][Docs] Add instructions how to use gdb to debug C++ libraries when running Python unit tests Author: Wes McKinney Closes #4560 from wesm/ARROW-5565 and squashes the following commits: 325b3670 Add docs section about how to use gdb to debug from Python --- docs/source/developers/python.rst | 26 ++ 1 file changed, 26 insertions(+) diff --git a/docs/source/developers/python.rst b/docs/source/developers/python.rst index 69bd59d..0242714 100644 --- a/docs/source/developers/python.rst +++ b/docs/source/developers/python.rst @@ -341,6 +341,32 @@ environment variable when building pyarrow: export PYARROW_WITH_CUDA=1 +Debugging +- + +Since pyarrow depends on the Arrow C++ libraries, debugging can +frequently involve crossing between Python and C++ shared libraries. + +Using gdb on Linux +~~ + +To debug the C++ libraries with gdb while running the Python unit + test, first start pytest with gdb: + +.. code-block:: shell + + gdb --args python -m pytest pyarrow/tests/test_to_run.py -k $TEST_TO_MATCH + +To set a breakpoint, use the same gdb syntax that you would when +debugging a C++ unitttest, for example: + +.. code-block:: shell + + (gdb) b src/arrow/python/arrow_to_pandas.cc:1874 + No source file named src/arrow/python/arrow_to_pandas.cc. + Make breakpoint pending on future shared library load? (y or [n]) y + Breakpoint 1 (src/arrow/python/arrow_to_pandas.cc:1874) pending. + Building on Windows ===
[arrow] branch master updated: ARROW-5436: [Python] parquet.read_table add filters keyword
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new d235f69 ARROW-5436: [Python] parquet.read_table add filters keyword
d235f69 is described below
commit d235f69bc7ebfbbd03f031b2191c69244861cf4f
Author: Joris Van den Bossche
AuthorDate: Thu Jun 6 11:35:34 2019 -0400
ARROW-5436: [Python] parquet.read_table add filters keyword
https://issues.apache.org/jira/browse/ARROW-5436
I suppose the fact that `parquet.read_table` dispatched to
FileSystem.read_parquet was for historical reasons (that function was added
before ParquetDataset was added), but directly calling ParquetDataset there
looks cleaner instead of going through FileSystem.read_parquet. So therefore I
also changed that.
In addition, I made sure the `memory_map` keyword was actually passed
through, I think an oversight of https://github.com/apache/arrow/pull/2954.
(those two changes should be useful anyway, regardless of adding `filters`
keyword or not)
Author: Joris Van den Bossche
Closes #4409 from jorisvandenbossche/ARROW-5436-parquet-read_table and
squashes the following commits:
85e5b0e1 lint
0ae1488d add test with nested list
9baf420b add filters to read_pandas
0df8c881 Merge remote-tracking branch
'upstream/master' into ARROW-5436-parquet-read_table
4ea7b77d fix test
4eb2ea7f add filters keyword
9c10f700 fix passing of memory_map (leftover from
ARROW-2807)
896abb2a simplify read_table (use ParquetDataset
directly)
---
python/pyarrow/parquet.py| 20
python/pyarrow/tests/test_parquet.py | 32
2 files changed, 44 insertions(+), 8 deletions(-)
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index d44deee..34a9c42 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -1181,6 +1181,11 @@ memory_map : boolean, default True
If the source is a file path, use a memory map to read file, which can
improve performance in some environments
{1}
+filters : List[Tuple] or List[List[Tuple]] or None (default)
+List of filters to apply, like ``[[('x', '=', 0), ...], ...]``. This
+implements partition-level (hive) filtering only, i.e., to prevent the
+loading of some files of the dataset if `source` is a directory.
+See the docstring of ParquetDataset for more details.
Returns
---
@@ -1190,14 +1195,12 @@ Returns
def read_table(source, columns=None, use_threads=True, metadata=None,
use_pandas_metadata=False, memory_map=True,
- filesystem=None):
+ filesystem=None, filters=None):
if _is_path_like(source):
-fs, path = _get_filesystem_and_path(filesystem, source)
-return fs.read_parquet(path, columns=columns,
- use_threads=use_threads, metadata=metadata,
- use_pandas_metadata=use_pandas_metadata)
-
-pf = ParquetFile(source, metadata=metadata)
+pf = ParquetDataset(source, metadata=metadata, memory_map=memory_map,
+filesystem=filesystem, filters=filters)
+else:
+pf = ParquetFile(source, metadata=metadata, memory_map=memory_map)
return pf.read(columns=columns, use_threads=use_threads,
use_pandas_metadata=use_pandas_metadata)
@@ -1212,10 +1215,11 @@ read_table.__doc__ = _read_table_docstring.format(
def read_pandas(source, columns=None, use_threads=True, memory_map=True,
-metadata=None):
+metadata=None, filters=None):
return read_table(source, columns=columns,
use_threads=use_threads,
metadata=metadata, memory_map=True,
+ filters=filters,
use_pandas_metadata=True)
diff --git a/python/pyarrow/tests/test_parquet.py
b/python/pyarrow/tests/test_parquet.py
index 76ec864..a97e885 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -1523,6 +1523,38 @@ def test_invalid_pred_op(tempdir):
])
+@pytest.mark.pandas
+def test_filters_read_table(tempdir):
+# test that filters keyword is passed through in read_table
+fs = LocalFileSystem.get_instance()
+base_path = tempdir
+
+integer_keys = [0, 1, 2, 3, 4]
+partition_spec = [
+['integers', integer_keys],
+]
+N = 5
+
+df = pd.DataFrame({
+'index': np.arange(N),
+'integers': np.array(integer_keys, dtype='i4'),
+}, columns=['index', 'integers'])
+
+_generate_partition_directories(fs, base_path, partition_spec, df)
+
+table = pq.
[arrow] branch master updated: ARROW-5521: [Packaging] Use Apache RAT 0.13
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 052130a ARROW-5521: [Packaging] Use Apache RAT 0.13
052130a is described below
commit 052130ad4980b70ace2074220ee2db7b912b2c57
Author: Antoine Pitrou
AuthorDate: Thu Jun 6 10:28:50 2019 -0400
ARROW-5521: [Packaging] Use Apache RAT 0.13
Author: Antoine Pitrou
Closes #4486 from pitrou/ARROW-5521-rat-0-13 and squashes the following
commits:
62debb51 ARROW-5521: Use Apache RAT 0.13
---
dev/release/rat_exclude_files.txt | 1 +
dev/release/run-rat.sh| 2 +-
js/test/inference/column.ts | 17 +
js/test/inference/nested.ts | 17 +
js/test/inference/visitor/get.ts | 17 +
5 files changed, 53 insertions(+), 1 deletion(-)
diff --git a/dev/release/rat_exclude_files.txt
b/dev/release/rat_exclude_files.txt
index bd1123d..4c5aef3 100644
--- a/dev/release/rat_exclude_files.txt
+++ b/dev/release/rat_exclude_files.txt
@@ -134,6 +134,7 @@ go/*.tmpldata
go/*.s
js/.npmignore
js/closure-compiler-scripts/*
+js/src/fb/*.ts
python/cmake_modules/FindPythonLibsNew.cmake
python/cmake_modules/SnappyCMakeLists.txt
python/cmake_modules/SnappyConfig.h
diff --git a/dev/release/run-rat.sh b/dev/release/run-rat.sh
index 587e93a..94fa55f 100755
--- a/dev/release/run-rat.sh
+++ b/dev/release/run-rat.sh
@@ -18,7 +18,7 @@
# under the License.
#
-RAT_VERSION=0.12
+RAT_VERSION=0.13
# download apache rat
if [ ! -f apache-rat-${RAT_VERSION}.jar ]; then
diff --git a/js/test/inference/column.ts b/js/test/inference/column.ts
index 44d7fab..7065d0e 100644
--- a/js/test/inference/column.ts
+++ b/js/test/inference/column.ts
@@ -1,3 +1,20 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
import { Data } from '../../src/data';
import { Field } from '../../src/schema';
import { Column } from '../../src/column';
diff --git a/js/test/inference/nested.ts b/js/test/inference/nested.ts
index ca164f6..10621c5 100644
--- a/js/test/inference/nested.ts
+++ b/js/test/inference/nested.ts
@@ -1,3 +1,20 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
import { Data } from '../../src/data';
import { Field } from '../../src/schema';
import { DataType } from '../../src/type';
diff --git a/js/test/inference/visitor/get.ts b/js/test/inference/visitor/get.ts
index d4a996a..ad7605f 100644
--- a/js/test/inference/visitor/get.ts
+++ b/js/test/inference/visitor/get.ts
@@ -1,3 +1,20 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either expres
[arrow] branch master updated: ARROW-5449: [C++] Test extended-length paths on Windows
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new b492975 ARROW-5449: [C++] Test extended-length paths on Windows
b492975 is described below
commit b492975faa68913a91c7a4758f125b4678f06a8b
Author: Antoine Pitrou
AuthorDate: Thu Jun 6 10:27:31 2019 -0400
ARROW-5449: [C++] Test extended-length paths on Windows
This helps validate that UNC paths (e.g. //server/share/...) should work.
Backslashes are still untested.
Author: Antoine Pitrou
Closes #4487 from pitrou/ARROW-5449-win-paths and squashes the following
commits:
539ef4ab ARROW-5449: Test extended-length paths on Windows
---
cpp/src/arrow/filesystem/localfs-test.cc | 87 +---
cpp/src/arrow/filesystem/test-util.h | 44 +---
2 files changed, 94 insertions(+), 37 deletions(-)
diff --git a/cpp/src/arrow/filesystem/localfs-test.cc
b/cpp/src/arrow/filesystem/localfs-test.cc
index ef36ea4..cd2dfc5 100644
--- a/cpp/src/arrow/filesystem/localfs-test.cc
+++ b/cpp/src/arrow/filesystem/localfs-test.cc
@@ -33,6 +33,7 @@ namespace arrow {
namespace fs {
namespace internal {
+using ::arrow::internal::PlatformFilename;
using ::arrow::internal::TemporaryDir;
TimePoint CurrentTimePoint() {
@@ -41,51 +42,101 @@ TimePoint CurrentTimePoint() {
std::chrono::duration_cast(now.time_since_epoch()));
}
+class LocalFSTestMixin : public ::testing::Test {
+ public:
+ void SetUp() override { ASSERT_OK(TemporaryDir::Make("test-localfs-",
&temp_dir_)); }
+
+ protected:
+ std::unique_ptr temp_dir_;
+};
+
+struct CommonPathFormatter {
+ std::string operator()(const PlatformFilename& fn) { return fn.ToString(); }
+};
+
+#ifdef _WIN32
+struct ExtendedLengthPathFormatter {
+ std::string operator()(const PlatformFilename& fn) { return "//?/" +
fn.ToString(); }
+};
+
+using PathFormatters = ::testing::Types;
+#else
+using PathFormatters = ::testing::Types;
+#endif
+
// Generic LocalFileSystem tests
-class TestLocalFSGeneric : public ::testing::Test, public
GenericFileSystemTest {
+template
+class TestLocalFSGeneric : public LocalFSTestMixin, public
GenericFileSystemTest {
public:
void SetUp() override {
-ASSERT_OK(TemporaryDir::Make("test-localfs-", &temp_dir_));
+LocalFSTestMixin::SetUp();
local_fs_ = std::make_shared();
-fs_ = std::make_shared(temp_dir_->path().ToString(),
local_fs_);
+auto path = PathFormatter()(temp_dir_->path());
+fs_ = std::make_shared(path, local_fs_);
}
protected:
std::shared_ptr GetEmptyFileSystem() override { return fs_; }
- std::unique_ptr temp_dir_;
std::shared_ptr local_fs_;
std::shared_ptr fs_;
};
-GENERIC_FS_TEST_FUNCTIONS(TestLocalFSGeneric);
+TYPED_TEST_CASE(TestLocalFSGeneric, PathFormatters);
+
+GENERIC_FS_TYPED_TEST_FUNCTIONS(TestLocalFSGeneric);
// Concrete LocalFileSystem tests
-class TestLocalFS : public ::testing::Test {
+template
+class TestLocalFS : public LocalFSTestMixin {
public:
void SetUp() {
-ASSERT_OK(TemporaryDir::Make("test-localfs-", &temp_dir_));
+LocalFSTestMixin::SetUp();
local_fs_ = std::make_shared();
-fs_ = std::make_shared(temp_dir_->path().ToString(),
local_fs_);
+auto path = PathFormatter()(temp_dir_->path());
+fs_ = std::make_shared(path, local_fs_);
}
protected:
- std::unique_ptr temp_dir_;
std::shared_ptr local_fs_;
std::shared_ptr fs_;
};
-TEST_F(TestLocalFS, DirectoryMTime) {
+TYPED_TEST_CASE(TestLocalFS, PathFormatters);
+
+TYPED_TEST(TestLocalFS, CorrectPathExists) {
+ // Test that the right location on disk is accessed
+ std::shared_ptr stream;
+ ASSERT_OK(this->fs_->OpenOutputStream("abc", &stream));
+ std::string data = "some data";
+ auto data_size = static_cast(data.size());
+ ASSERT_OK(stream->Write(data.data(), data_size));
+ ASSERT_OK(stream->Close());
+
+ // Now check the file's existence directly, bypassing the FileSystem
abstraction
+ auto path = this->temp_dir_->path().ToString() + "/abc";
+ PlatformFilename fn;
+ int fd;
+ int64_t size = -1;
+ ASSERT_OK(PlatformFilename::FromString(path, &fn));
+ ASSERT_OK(::arrow::internal::FileOpenReadable(fn, &fd));
+ Status st = ::arrow::internal::FileGetSize(fd, &size);
+ ASSERT_OK(::arrow::internal::FileClose(fd));
+ ASSERT_OK(st);
+ ASSERT_EQ(size, data_size);
+}
+
+TYPED_TEST(TestLocalFS, DirectoryMTime) {
TimePoint t1 = CurrentTimePoint();
- ASSERT_OK(fs_->CreateDir("AB/CD/EF"));
+ ASSERT_OK(this->fs_->Create
[arrow] branch master updated: ARROW-5349: [C++][Parquet] Add method to set file path in a parquet::FileMetaData instance
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new f82af62 ARROW-5349: [C++][Parquet] Add method to set file path in a
parquet::FileMetaData instance
f82af62 is described below
commit f82af62dfde364f3eeea2015586ec5846a78ec8f
Author: Wes McKinney
AuthorDate: Sun May 26 14:22:27 2019 +0200
ARROW-5349: [C++][Parquet] Add method to set file path in a
parquet::FileMetaData instance
This is an RFC based on the discussion ongoing. If this seems like the
accepted approach I'll add a Python unit test
Author: Wes McKinney
Closes #4386 from wesm/parquet-metadata-set-file-path and squashes the
following commits:
e77cf7a8a Add method to set file path in a
parquet::FileMetaData instance
---
cpp/src/parquet/metadata-test.cc | 5 +
cpp/src/parquet/metadata.cc | 10 ++
cpp/src/parquet/metadata.h | 5 +
python/pyarrow/_parquet.pxd | 2 ++
python/pyarrow/_parquet.pyx | 9 +
5 files changed, 31 insertions(+)
diff --git a/cpp/src/parquet/metadata-test.cc b/cpp/src/parquet/metadata-test.cc
index 7115876..8b6ebc1 100644
--- a/cpp/src/parquet/metadata-test.cc
+++ b/cpp/src/parquet/metadata-test.cc
@@ -163,6 +163,11 @@ TEST(Metadata, TestBuildAccess) {
ASSERT_EQ(16, rg2_column2->dictionary_page_offset());
ASSERT_EQ(10, rg2_column1->data_page_offset());
ASSERT_EQ(26, rg2_column2->data_page_offset());
+
+ // Test FileMetaData::set_file_path
+ ASSERT_TRUE(rg2_column1->file_path().empty());
+ f_accessor->set_file_path("/foo/bar/bar.parquet");
+ ASSERT_EQ("/foo/bar/bar.parquet", rg2_column1->file_path());
}
TEST(Metadata, TestV1Version) {
diff --git a/cpp/src/parquet/metadata.cc b/cpp/src/parquet/metadata.cc
index f596061..5d701a7 100644
--- a/cpp/src/parquet/metadata.cc
+++ b/cpp/src/parquet/metadata.cc
@@ -386,6 +386,14 @@ class FileMetaData::FileMetaDataImpl {
return key_value_metadata_;
}
+ void set_file_path(const std::string& path) {
+for (format::RowGroup& row_group : metadata_->row_groups) {
+ for (format::ColumnChunk& chunk : row_group.columns) {
+chunk.__set_file_path(path);
+ }
+}
+ }
+
private:
friend FileMetaDataBuilder;
uint32_t metadata_len_;
@@ -483,6 +491,8 @@ std::shared_ptr
FileMetaData::key_value_metadata() const
return impl_->key_value_metadata();
}
+void FileMetaData::set_file_path(const std::string& path) {
impl_->set_file_path(path); }
+
void FileMetaData::WriteTo(OutputStream* dst) const { return
impl_->WriteTo(dst); }
ApplicationVersion::ApplicationVersion(const std::string& application, int
major,
diff --git a/cpp/src/parquet/metadata.h b/cpp/src/parquet/metadata.h
index cd31a3c..fe67bc0 100644
--- a/cpp/src/parquet/metadata.h
+++ b/cpp/src/parquet/metadata.h
@@ -110,8 +110,10 @@ class PARQUET_EXPORT ColumnChunkMetaData {
// column chunk
int64_t file_offset() const;
+
// parameter is only used when a dataset is spread across multiple files
const std::string& file_path() const;
+
// column metadata
Type::type type() const;
int64_t num_values() const;
@@ -190,6 +192,9 @@ class PARQUET_EXPORT FileMetaData {
std::shared_ptr key_value_metadata() const;
+ // Set file_path ColumnChunk fields to a particular value
+ void set_file_path(const std::string& path);
+
private:
friend FileMetaDataBuilder;
explicit FileMetaData(const void* serialized_metadata, uint32_t*
metadata_len);
diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd
index f1b44b0..75c0015 100644
--- a/python/pyarrow/_parquet.pxd
+++ b/python/pyarrow/_parquet.pxd
@@ -222,6 +222,8 @@ cdef extern from "parquet/api/reader.h" namespace "parquet"
nogil:
const c_string created_by()
int num_schema_elements()
+void set_file_path(const c_string& path)
+
unique_ptr[CRowGroupMetaData] RowGroup(int i)
const SchemaDescriptor* schema()
shared_ptr[const CKeyValueMetadata] key_value_metadata() const
diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index f074852..db7f0c4 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -522,6 +522,15 @@ cdef class FileMetaData:
def row_group(self, int i):
return RowGroupMetaData(self, i)
+def set_file_path(self, path):
+"""
+Modify the file_path field of each ColumnChunk in the
+FileMetaData to be a particular value
+"""
+cdef:
+c_string c_path = tobytes(path)
+self._metadata.set_file_path(c_path)
+
cdef class ParquetSchema:
cdef:
[arrow] branch master updated: PARQUET-1586: [C++] Add --dump options to parquet-reader tool to dump def/rep levels
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new d82ac40 PARQUET-1586: [C++] Add --dump options to parquet-reader tool
to dump def/rep levels
d82ac40 is described below
commit d82ac407fab1d4b28669b8f7a940f88d39dfd874
Author: Renat Valiullin
AuthorDate: Sun May 26 14:21:27 2019 +0200
PARQUET-1586: [C++] Add --dump options to parquet-reader tool to dump
def/rep levels
Author: Renat Valiullin
Closes #4385 from rip-nsk/PARQUET-1586 and squashes the following commits:
2d5c9e38f fixes for lint
1e370238b fix lint
a7bcacee3 parquet-reader --dump
---
cpp/src/parquet/column_scanner.h| 17 --
cpp/src/parquet/printer.cc | 44 +---
cpp/src/parquet/printer.h | 3 +-
cpp/src/parquet/reader-test.cc | 101 +---
cpp/tools/parquet/parquet-reader.cc | 9 ++--
5 files changed, 140 insertions(+), 34 deletions(-)
diff --git a/cpp/src/parquet/column_scanner.h b/cpp/src/parquet/column_scanner.h
index cb0da2c..1e084a9 100644
--- a/cpp/src/parquet/column_scanner.h
+++ b/cpp/src/parquet/column_scanner.h
@@ -62,7 +62,7 @@ class PARQUET_EXPORT Scanner {
int64_t batch_size = DEFAULT_SCANNER_BATCH_SIZE,
::arrow::MemoryPool* pool = ::arrow::default_memory_pool());
- virtual void PrintNext(std::ostream& out, int width) = 0;
+ virtual void PrintNext(std::ostream& out, int width, bool with_levels =
false) = 0;
bool HasNext() { return level_offset_ < levels_buffered_ ||
reader_->HasNext(); }
@@ -171,15 +171,24 @@ class PARQUET_TEMPLATE_CLASS_EXPORT TypedScanner : public
Scanner {
return true;
}
- virtual void PrintNext(std::ostream& out, int width) {
+ virtual void PrintNext(std::ostream& out, int width, bool with_levels =
false) {
T val;
+int16_t def_level = -1;
+int16_t rep_level = -1;
bool is_null = false;
-char buffer[25];
+char buffer[80];
-if (!NextValue(&val, &is_null)) {
+if (!Next(&val, &def_level, &rep_level, &is_null)) {
throw ParquetException("No more values buffered");
}
+if (with_levels) {
+ out << " D:" << def_level << " R:" << rep_level << " ";
+ if (!is_null) {
+out << "V:";
+ }
+}
+
if (is_null) {
std::string null_fmt = format_fwf(width);
snprintf(buffer, sizeof(buffer), null_fmt.c_str(), "NULL");
diff --git a/cpp/src/parquet/printer.cc b/cpp/src/parquet/printer.cc
index 82c52ff..d78d9b9 100644
--- a/cpp/src/parquet/printer.cc
+++ b/cpp/src/parquet/printer.cc
@@ -42,11 +42,11 @@ class ColumnReader;
// ParquetFilePrinter::DebugPrint
// the fixed initial size is just for an example
-#define COL_WIDTH "30"
+#define COL_WIDTH 30
void ParquetFilePrinter::DebugPrint(std::ostream& stream, std::list
selected_columns,
-bool print_values, bool
print_key_value_metadata,
-const char* filename) {
+bool print_values, bool format_dump,
+bool print_key_value_metadata, const char*
filename) {
const FileMetaData* file_metadata = fileReader->metadata().get();
stream << "File Name: " << filename << "\n";
@@ -54,7 +54,7 @@ void ParquetFilePrinter::DebugPrint(std::ostream& stream,
std::list selecte
stream << "Created By: " << file_metadata->created_by() << "\n";
stream << "Total rows: " << file_metadata->num_rows() << "\n";
- if (print_key_value_metadata) {
+ if (print_key_value_metadata && file_metadata->key_value_metadata()) {
auto key_value_metadata = file_metadata->key_value_metadata();
int64_t size_of_key_value_metadata = key_value_metadata->size();
stream << "Key Value File Metadata: " << size_of_key_value_metadata << "
entries\n";
@@ -95,7 +95,7 @@ void ParquetFilePrinter::DebugPrint(std::ostream& stream,
std::list selecte
std::unique_ptr group_metadata =
file_metadata->RowGroup(r);
stream << "--- Total Bytes " << group_metadata->total_byte_size() << "
---\n";
-stream << " Rows: " << group_metadata->num_rows() << "---\n";
+stream << "--- Rows: " << group_metadata->num_rows() << " ---\n";
// Print column metadata
for (auto i : selected_columns) {
@@ -103,7 +103,7 @@ void ParquetFilePrinter:
[arrow] branch master updated: ARROW-5245: [CI][C++] Unpin cmake format (current version is 5.1)
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 21e9d74 ARROW-5245: [CI][C++] Unpin cmake format (current version is
5.1)
21e9d74 is described below
commit 21e9d74b951d9b65ca0c186c8b6de8cc53cc225f
Author: Micah Kornfield
AuthorDate: Sun May 26 14:15:50 2019 +0200
ARROW-5245: [CI][C++] Unpin cmake format (current version is 5.1)
It looks like some formatting has changed going from 4 to 5. For the most
part it looks more readable, but I'm not sure.
Author: Micah Kornfield
Closes #4388 from emkornfield/unpin_cmake and squashes the following
commits:
e2d48fd2f run cmake format and turn off autosort
c965fc994 ARROW-5245: Unpin cmake format
---
ci/travis_lint.sh | 2 +-
cpp/CMakeLists.txt | 18 +-
cpp/cmake_modules/BuildUtils.cmake | 13 +-
cpp/cmake_modules/DefineOptions.cmake | 2 +-
cpp/cmake_modules/FindClangTools.cmake | 5 +-
cpp/cmake_modules/ThirdpartyToolchain.cmake| 467 -
cpp/cmake_modules/san-config.cmake | 12 +-
cpp/src/arrow/adapters/orc/CMakeLists.txt | 8 +-
cpp/src/arrow/dbi/hiveserver2/CMakeLists.txt | 7 +-
.../arrow/dbi/hiveserver2/thrift/CMakeLists.txt| 23 +-
cpp/src/arrow/flight/CMakeLists.txt| 44 +-
cpp/src/arrow/ipc/CMakeLists.txt | 2 +-
cpp/src/arrow/python/CMakeLists.txt| 7 +-
cpp/src/gandiva/CMakeLists.txt | 6 +-
cpp/src/gandiva/precompiled/CMakeLists.txt | 13 +-
cpp/src/gandiva/tests/CMakeLists.txt | 6 +-
cpp/src/parquet/CMakeLists.txt | 22 +-
cpp/src/plasma/CMakeLists.txt | 21 +-
python/CMakeLists.txt | 14 +-
run-cmake-format.py| 4 +-
20 files changed, 294 insertions(+), 402 deletions(-)
diff --git a/ci/travis_lint.sh b/ci/travis_lint.sh
index eb0ef43..1c36eaf 100755
--- a/ci/travis_lint.sh
+++ b/ci/travis_lint.sh
@@ -31,7 +31,7 @@ pre-commit install
pre-commit run hadolint -a
# CMake formatting check
-pip install cmake_format==0.4.5
+pip install cmake_format
$TRAVIS_BUILD_DIR/run-cmake-format.py --check
# C++ code linting
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 6be40c7..4c70a38 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -361,7 +361,7 @@ if(${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_CURRENT_BINARY_DIR})
set(MORE_ARGS "-T")
endif()
execute_process(COMMAND ln ${MORE_ARGS} -sf ${BUILD_OUTPUT_ROOT_DIRECTORY}
- ${CMAKE_CURRENT_BINARY_DIR}/build/latest)
+
${CMAKE_CURRENT_BINARY_DIR}/build/latest)
else()
set(BUILD_OUTPUT_ROOT_DIRECTORY
"${CMAKE_CURRENT_BINARY_DIR}/${BUILD_SUBDIR_NAME}/")
endif()
@@ -595,11 +595,8 @@ if(ARROW_STATIC_LINK_LIBS)
add_dependencies(arrow_dependencies ${ARROW_STATIC_LINK_LIBS})
endif()
-set(ARROW_SHARED_PRIVATE_LINK_LIBS
-${ARROW_STATIC_LINK_LIBS}
-${BOOST_SYSTEM_LIBRARY}
-${BOOST_FILESYSTEM_LIBRARY}
-${BOOST_REGEX_LIBRARY})
+set(ARROW_SHARED_PRIVATE_LINK_LIBS ${ARROW_STATIC_LINK_LIBS}
${BOOST_SYSTEM_LIBRARY}
+ ${BOOST_FILESYSTEM_LIBRARY}
${BOOST_REGEX_LIBRARY})
list(APPEND ARROW_STATIC_LINK_LIBS ${BOOST_SYSTEM_LIBRARY}
${BOOST_FILESYSTEM_LIBRARY}
${BOOST_REGEX_LIBRARY})
@@ -623,11 +620,8 @@ if(ARROW_BUILD_BENCHMARKS)
toolchain-benchmarks)
endif()
-set(ARROW_TEST_STATIC_LINK_LIBS
-arrow_testing_static
-arrow_static
-${ARROW_LINK_LIBS}
-${ARROW_TEST_LINK_TOOLCHAIN})
+set(ARROW_TEST_STATIC_LINK_LIBS arrow_testing_static arrow_static
${ARROW_LINK_LIBS}
+${ARROW_TEST_LINK_TOOLCHAIN})
set(ARROW_TEST_SHARED_LINK_LIBS
arrow_testing_shared
@@ -670,7 +664,7 @@ if(ARROW_BUILD_BENCHMARKS)
endif()
set(ARROW_BENCHMARK_LINK_LIBS benchmark::benchmark_main benchmark::benchmark
- ${ARROW_TEST_LINK_LIBS})
+${ARROW_TEST_LINK_LIBS})
endif()
set(ARROW_SYSTEM_LINK_LIBS)
diff --git a/cpp/cmake_modules/BuildUtils.cmake
b/cpp/cmake_modules/BuildUtils.cmake
index 7f9af7f..45cff6e 100644
--- a/cpp/cmake_modules/BuildUtils.cmake
+++ b/cpp/cmake_modules/BuildUtils.cmake
@@ -281,8 +281,7 @@ function(ADD_ARROW_LIB LIB_NAME)
RUNTIME DESTINATION ${RUNTIME_INSTALL_DIR}
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
-INCLUDES
-DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+
[arrow] branch master updated: PARQUET-1583: [C++] Remove superfluous parquet::Vector class
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 2d4fe01 PARQUET-1583: [C++] Remove superfluous parquet::Vector class
2d4fe01 is described below
commit 2d4fe0153cd3edec6ddc8ca742d05c8e3a98f768
Author: Wes McKinney
AuthorDate: Tue May 21 08:38:09 2019 +0200
PARQUET-1583: [C++] Remove superfluous parquet::Vector class
This class which is a syntactic sugar layer on `arrow::ResizableBuffer` was
only being used in one place, for dictionary decoding. I don't think we need to
maintain this, and it also features possible instantiated template symbol issues
Author: Wes McKinney
Closes #4354 from wesm/PARQUET-1583 and squashes the following commits:
24535175 lint
29af0799 Remove superfluous parquet::Vector class
---
cpp/src/parquet/encoding.cc| 55 +-
cpp/src/parquet/util/memory.cc | 50 --
cpp/src/parquet/util/memory.h | 22 -
3 files changed, 33 insertions(+), 94 deletions(-)
diff --git a/cpp/src/parquet/encoding.cc b/cpp/src/parquet/encoding.cc
index 3a4b342..ebb7aea 100644
--- a/cpp/src/parquet/encoding.cc
+++ b/cpp/src/parquet/encoding.cc
@@ -781,7 +781,7 @@ class DictDecoderImpl : public DecoderImpl, virtual public
DictDecoder {
explicit DictDecoderImpl(const ColumnDescriptor* descr,
::arrow::MemoryPool* pool =
::arrow::default_memory_pool())
: DecoderImpl(descr, Encoding::RLE_DICTIONARY),
-dictionary_(0, pool),
+dictionary_(AllocateBuffer(pool, 0)),
byte_array_data_(AllocateBuffer(pool, 0)) {}
// Perform type-specific initiatialization
@@ -798,8 +798,8 @@ class DictDecoderImpl : public DecoderImpl, virtual public
DictDecoder {
int Decode(T* buffer, int max_values) override {
max_values = std::min(max_values, num_values_);
-int decoded_values =
-idx_decoder_.GetBatchWithDict(dictionary_.data(), buffer, max_values);
+int decoded_values = idx_decoder_.GetBatchWithDict(
+reinterpret_cast(dictionary_->data()), buffer, max_values);
if (decoded_values != max_values) {
ParquetException::EofException();
}
@@ -809,9 +809,9 @@ class DictDecoderImpl : public DecoderImpl, virtual public
DictDecoder {
int DecodeSpaced(T* buffer, int num_values, int null_count, const uint8_t*
valid_bits,
int64_t valid_bits_offset) override {
-int decoded_values =
-idx_decoder_.GetBatchWithDictSpaced(dictionary_.data(), buffer,
num_values,
-null_count, valid_bits,
valid_bits_offset);
+int decoded_values = idx_decoder_.GetBatchWithDictSpaced(
+reinterpret_cast(dictionary_->data()), buffer, num_values,
null_count,
+valid_bits, valid_bits_offset);
if (decoded_values != num_values) {
ParquetException::EofException();
}
@@ -819,8 +819,15 @@ class DictDecoderImpl : public DecoderImpl, virtual public
DictDecoder {
}
protected:
+ inline void DecodeDict(TypedDecoder* dictionary) {
+int num_dictionary_values = dictionary->values_left();
+PARQUET_THROW_NOT_OK(dictionary_->Resize(num_dictionary_values *
sizeof(T)));
+dictionary->Decode(reinterpret_cast(dictionary_->mutable_data()),
+ num_dictionary_values);
+ }
+
// Only one is set.
- Vector dictionary_;
+ std::shared_ptr dictionary_;
// Data that contains the byte array data (byte_array_dictionary_ just has
the
// pointers).
@@ -831,9 +838,7 @@ class DictDecoderImpl : public DecoderImpl, virtual public
DictDecoder {
template
inline void DictDecoderImpl::SetDict(TypedDecoder* dictionary) {
- int num_dictionary_values = dictionary->values_left();
- dictionary_.Resize(num_dictionary_values);
- dictionary->Decode(dictionary_.data(), num_dictionary_values);
+ DecodeDict(dictionary);
}
template <>
@@ -845,12 +850,13 @@ template <>
inline void DictDecoderImpl::SetDict(
TypedDecoder* dictionary) {
int num_dictionary_values = dictionary->values_left();
- dictionary_.Resize(num_dictionary_values);
- dictionary->Decode(dictionary_.data(), num_dictionary_values);
+ DecodeDict(dictionary);
+
+ auto dict_values = reinterpret_cast(dictionary_->mutable_data());
int total_size = 0;
for (int i = 0; i < num_dictionary_values; ++i) {
-total_size += dictionary_[i].len;
+total_size += dict_values[i].len;
}
if (total_size > 0) {
PARQUET_THROW_NOT_OK(byte_array_data_->Resize(total_size, false));
@@ -859,17 +865,18 @@ inline void DictDecoderImpl::SetDict(
int offset = 0;
uint8_t* bytes_data = byte_array_data_->mutable_data();
for (int i = 0; i < num_diction
[arrow] branch master updated: ARROW-5167: [C++] Upgrade string-view-light to latest
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 85db347 ARROW-5167: [C++] Upgrade string-view-light to latest
85db347 is described below
commit 85db347db08a851a6fe482ef277c969531da58e3
Author: Antoine Pitrou
AuthorDate: Mon Apr 22 22:20:05 2019 +0200
ARROW-5167: [C++] Upgrade string-view-light to latest
Author: Antoine Pitrou
Closes #4182 from pitrou/ARROW-5167-string-view-latest and squashes the
following commits:
1e59cb82 ARROW-5167: Upgrade string-view-light to latest
---
cpp/src/arrow/vendored/string_view.hpp | 76 +-
1 file changed, 57 insertions(+), 19 deletions(-)
diff --git a/cpp/src/arrow/vendored/string_view.hpp
b/cpp/src/arrow/vendored/string_view.hpp
index 1647c93..d912ee8 100644
--- a/cpp/src/arrow/vendored/string_view.hpp
+++ b/cpp/src/arrow/vendored/string_view.hpp
@@ -1,11 +1,11 @@
-// Vendored from git tag 54a90f61ccb08dbd9870d24f735ded0daa659341
+// Vendored from git tag 062acda5e35c8922dbbccf81300a58edea521b45
-// Copyright 2017-2018 by Martin Moene
+// Copyright 2017-2019 by Martin Moene
//
// string-view lite, a C++17-like string_view for C++98 and later.
// For more information see https://github.com/martinmoene/string-view-lite
//
-// Distributed under the Boost Software License, Version 1.0.
+// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE.txt or copy at
http://www.boost.org/LICENSE_1_0.txt)
#pragma once
@@ -57,6 +57,16 @@
# define nssv_CONFIG_CONVERSION_STD_STRING_FREE_FUNCTIONS 1
#endif
+// Control presence of exception handling (try and auto discover):
+
+#ifndef nssv_CONFIG_NO_EXCEPTIONS
+# if defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)
+# define nssv_CONFIG_NO_EXCEPTIONS 0
+# else
+# define nssv_CONFIG_NO_EXCEPTIONS 1
+# endif
+#endif
+
// C++ language version detection (C++20 is speculative):
// Note: VC14.0/1900 (VS2015) lacks too much from C++14.
@@ -210,7 +220,7 @@ using std::operator<<;
# define nssv_COMPILER_MSVC_VERSION 0
#endif
-#define nssv_COMPILER_VERSION( major, minor, patch ) (10 * ( 10 * major +
minor) + patch)
+#define nssv_COMPILER_VERSION( major, minor, patch ) ( 10 * ( 10 * (major) +
(minor) ) + (patch) )
#if defined(__clang__)
# define nssv_COMPILER_CLANG_VERSION nssv_COMPILER_VERSION(__clang_major__,
__clang_minor__, __clang_patchlevel__)
@@ -265,8 +275,10 @@ using std::operator<<;
#define nssv_HAVE_WCHAR16_T nssv_CPP11_100
#define nssv_HAVE_WCHAR32_T nssv_CPP11_100
-#if ! ( ( nssv_CPP11 && nssv_COMPILER_CLANG_VERSION ) || nssv_BETWEEN(
nssv_COMPILER_CLANG_VERSION, 300, 400 ) )
+#if ! ( ( nssv_CPP11_OR_GREATER && nssv_COMPILER_CLANG_VERSION ) ||
nssv_BETWEEN( nssv_COMPILER_CLANG_VERSION, 300, 400 ) )
# define nssv_HAVE_STD_DEFINED_LITERALS nssv_CPP11_140
+#else
+# define nssv_HAVE_STD_DEFINED_LITERALS 0
#endif
// Presence of C++14 language features:
@@ -340,9 +352,12 @@ using std::operator<<;
#include
#include
#include
-#include
#include// std::char_traits<>
+#if ! nssv_CONFIG_NO_EXCEPTIONS
+# include
+#endif
+
#if nssv_CPP11_OR_GREATER
# include
#endif
@@ -444,12 +459,12 @@ public:
{}
#endif
-nssv_constexpr basic_string_view( CharT const * s, size_type count )
+nssv_constexpr basic_string_view( CharT const * s, size_type count )
nssv_noexcept // non-standard noexcept
: data_( s )
, size_( count )
{}
-nssv_constexpr basic_string_view( CharT const * s)
+nssv_constexpr basic_string_view( CharT const * s) nssv_noexcept //
non-standard noexcept
: data_( s )
, size_( Traits::length(s) )
{}
@@ -502,12 +517,15 @@ public:
nssv_constexpr14 const_reference at( size_type pos ) const
{
-if ( pos < size() )
+#if nssv_CONFIG_NO_EXCEPTIONS
+assert( pos < size() );
+#else
+if ( pos >= size() )
{
-return data_at( pos );
+throw std::out_of_range("nonst::string_view::at()");
}
-
-throw std::out_of_range("nonst::string_view::at()");
+#endif
+return data_at( pos );
}
nssv_constexpr const_reference front() const { return data_at( 0 );
}
@@ -541,9 +559,14 @@ public:
size_type copy( CharT * dest, size_type n, size_type pos = 0 ) const
{
+#if nssv_CONFIG_NO_EXCEPTIONS
+assert( pos <= size() );
+#else
if ( pos > size() )
+{
throw std::out_of_range("nonst::string_view::copy()");
-
+}
+#endif
const size_type rlen = (std::min)( n, size() - pos );
(void) Traits::copy( dest, data() + pos, rlen );
@@
[arrow] branch master updated: ARROW-4824: [Python] Fix error checking in read_csv()
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new e441291 ARROW-4824: [Python] Fix error checking in read_csv()
e441291 is described below
commit e441291468a0d2082d72bd52406391ccc2172f3c
Author: Antoine Pitrou
AuthorDate: Mon Apr 22 22:17:08 2019 +0200
ARROW-4824: [Python] Fix error checking in read_csv()
Raise a TypeError when a text file is given to read_csv(), as only binary
files are allowed.
Also fix a systemic issue where Python exceptions could be swallowed by C++
destructors that could call back into Python and clear the current error
status. The solution is to define a "safe" facility to call into Python code
from C++ without clobbering the current error status.
Author: Antoine Pitrou
Closes #4183 from pitrou/ARROW-4824-read-csv-error-message and squashes the
following commits:
ed3f931f ARROW-4824: Fix error checking in read_csv()
---
cpp/src/arrow/python/common.h| 19 +++
cpp/src/arrow/python/flight.cc | 84 +-
cpp/src/arrow/python/io.cc | 108 ++-
python/pyarrow/io.pxi| 6 ++-
python/pyarrow/tests/test_csv.py | 11
python/pyarrow/tests/test_io.py | 6 +++
6 files changed, 149 insertions(+), 85 deletions(-)
diff --git a/cpp/src/arrow/python/common.h b/cpp/src/arrow/python/common.h
index 6e41bed..27661a9 100644
--- a/cpp/src/arrow/python/common.h
+++ b/cpp/src/arrow/python/common.h
@@ -54,6 +54,7 @@ ARROW_PYTHON_EXPORT Status PassPyError();
#define PY_RETURN_IF_ERROR(CODE) ARROW_RETURN_NOT_OK(CheckPyError(CODE));
+// A RAII-style helper that ensures the GIL is acquired inside a lexical block.
class ARROW_PYTHON_EXPORT PyAcquireGIL {
public:
PyAcquireGIL() : acquired_gil_(false) { acquire(); }
@@ -81,6 +82,24 @@ class ARROW_PYTHON_EXPORT PyAcquireGIL {
ARROW_DISALLOW_COPY_AND_ASSIGN(PyAcquireGIL);
};
+// A helper to call safely into the Python interpreter from arbitrary C++ code.
+// The GIL is acquired, and the current thread's error status is preserved.
+template
+Status SafeCallIntoPython(Function&& func) {
+ PyAcquireGIL lock;
+ PyObject* exc_type;
+ PyObject* exc_value;
+ PyObject* exc_traceback;
+ PyErr_Fetch(&exc_type, &exc_value, &exc_traceback);
+ Status st = std::forward(func)();
+ // If the return Status is a "Python error", the current Python error status
+ // describes the error and shouldn't be clobbered.
+ if (!st.IsPythonError() && exc_type != NULLPTR) {
+PyErr_Restore(exc_type, exc_value, exc_traceback);
+ }
+ return st;
+}
+
#define PYARROW_IS_PY2 PY_MAJOR_VERSION <= 2
// A RAII primitive that DECREFs the underlying PyObject* when it
diff --git a/cpp/src/arrow/python/flight.cc b/cpp/src/arrow/python/flight.cc
index 33a063a..861aa8d 100644
--- a/cpp/src/arrow/python/flight.cc
+++ b/cpp/src/arrow/python/flight.cc
@@ -35,16 +35,18 @@ PyServerAuthHandler::PyServerAuthHandler(PyObject* handler,
Status PyServerAuthHandler::Authenticate(arrow::flight::ServerAuthSender*
outgoing,
arrow::flight::ServerAuthReader*
incoming) {
- PyAcquireGIL lock;
- vtable_.authenticate(handler_.obj(), outgoing, incoming);
- return CheckPyError();
+ return SafeCallIntoPython([=] {
+vtable_.authenticate(handler_.obj(), outgoing, incoming);
+return CheckPyError();
+ });
}
Status PyServerAuthHandler::IsValid(const std::string& token,
std::string* peer_identity) {
- PyAcquireGIL lock;
- vtable_.is_valid(handler_.obj(), token, peer_identity);
- return CheckPyError();
+ return SafeCallIntoPython([=] {
+vtable_.is_valid(handler_.obj(), token, peer_identity);
+return CheckPyError();
+ });
}
PyClientAuthHandler::PyClientAuthHandler(PyObject* handler,
@@ -56,15 +58,17 @@ PyClientAuthHandler::PyClientAuthHandler(PyObject* handler,
Status PyClientAuthHandler::Authenticate(arrow::flight::ClientAuthSender*
outgoing,
arrow::flight::ClientAuthReader*
incoming) {
- PyAcquireGIL lock;
- vtable_.authenticate(handler_.obj(), outgoing, incoming);
- return CheckPyError();
+ return SafeCallIntoPython([=] {
+vtable_.authenticate(handler_.obj(), outgoing, incoming);
+return CheckPyError();
+ });
}
Status PyClientAuthHandler::GetToken(std::string* token) {
- PyAcquireGIL lock;
- vtable_.get_token(handler_.obj(), token);
- return CheckPyError();
+ return SafeCallIntoPython([=] {
+vtable_.get_token(handler_.obj(), token);
+return CheckPyError();
+ });
}
PyFlightServer::PyFlightServer(PyObject* server, PyFlightServerVtable vtable)
@@ -77,47 +81,53 @@ Status PyFlightServer::
[arrow] branch master updated: ARROW-3208: [C++] Fix Cast dictionary to numeric segfault
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 37f898f ARROW-3208: [C++] Fix Cast dictionary to numeric segfault
37f898f is described below
commit 37f898f9ca7d76448223407d436aeee6a81a8f7d
Author: François Saint-Jacques
AuthorDate: Wed Mar 20 09:48:37 2019 +0100
ARROW-3208: [C++] Fix Cast dictionary to numeric segfault
Author: François Saint-Jacques
Closes #3978 from
fsaintjacques/ARROW-3208-python-parquet-segfault-partition and squashes the
following commits:
8a4c2613 ARROW-3208: Fix Cast dictionary to
numeric segfault with nullptr Buffer
---
cpp/src/arrow/compute/kernels/cast-test.cc | 17 +
cpp/src/arrow/compute/kernels/cast.cc | 19 +--
python/pyarrow/tests/test_parquet.py | 16
3 files changed, 46 insertions(+), 6 deletions(-)
diff --git a/cpp/src/arrow/compute/kernels/cast-test.cc
b/cpp/src/arrow/compute/kernels/cast-test.cc
index 271b233..8cf71d1 100644
--- a/cpp/src/arrow/compute/kernels/cast-test.cc
+++ b/cpp/src/arrow/compute/kernels/cast-test.cc
@@ -33,6 +33,7 @@
#include "arrow/table.h"
#include "arrow/testing/gtest_common.h"
#include "arrow/testing/gtest_util.h"
+#include "arrow/testing/random.h"
#include "arrow/type.h"
#include "arrow/type_fwd.h"
#include "arrow/type_traits.h"
@@ -1116,6 +1117,22 @@ TYPED_TEST(TestDictionaryCast, Basic) {
this->CheckPass(*MakeArray(out.array()), *plain_array, plain_array->type(),
options);
}
+TEST_F(TestCast, DictToNumericNoNulls) {
+ // ARROW-3208
+ CastOptions options;
+
+ // Convoluted way to create an array with nullptr bitmap buffer
+ auto array_ = _MakeArray(int32(), {1, 2, 3, 4, 5, 6},
{});
+ auto data = array_->data();
+ data->buffers[0] = nullptr;
+ auto array = MakeArray(data);
+
+ Datum encoded;
+ ASSERT_OK(DictionaryEncode(&this->ctx_, array->data(), &encoded));
+
+ this->CheckPass(*MakeArray(encoded.array()), *array, array->type(), options);
+}
+
TEST_F(TestCast, DictToNonDictNoNulls) {
vector dict_values = {"foo", "bar", "baz"};
auto ex_dict = _MakeArray(utf8(), dict_values, {});
diff --git a/cpp/src/arrow/compute/kernels/cast.cc
b/cpp/src/arrow/compute/kernels/cast.cc
index ee21803..2d3e1a8 100644
--- a/cpp/src/arrow/compute/kernels/cast.cc
+++ b/cpp/src/arrow/compute/kernels/cast.cc
@@ -842,15 +842,22 @@ struct CastFunctor
void UnpackPrimitiveDictionary(const Array& indices, const c_type* dictionary,
c_type* out) {
- internal::BitmapReader valid_bits_reader(indices.null_bitmap_data(),
indices.offset(),
- indices.length());
+ const auto& in = indices.data()->GetValues(1);
+ int64_t length = indices.length();
- auto in = indices.data()->GetValues(1);
- for (int64_t i = 0; i < indices.length(); ++i) {
-if (valid_bits_reader.IsSet()) {
+ if (indices.null_count() == 0) {
+for (int64_t i = 0; i < length; ++i) {
out[i] = dictionary[in[i]];
}
-valid_bits_reader.Next();
+ } else {
+auto null_bitmap = indices.null_bitmap_data();
+internal::BitmapReader valid_bits_reader(null_bitmap, indices.offset(),
length);
+for (int64_t i = 0; i < length; ++i) {
+ if (valid_bits_reader.IsSet()) {
+out[i] = dictionary[in[i]];
+ }
+ valid_bits_reader.Next();
+}
}
}
diff --git a/python/pyarrow/tests/test_parquet.py
b/python/pyarrow/tests/test_parquet.py
index 8406f36..34d0956 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -2479,3 +2479,19 @@ def test_write_nested_zero_length_array_chunk_failure():
for batch in my_arrays]
tbl = pa.Table.from_batches(my_batches, pa.schema(cols))
_check_roundtrip(tbl)
+
+
+def test_partitioned_dataset(tempdir):
+# ARROW-3208: Segmentation fault when reading a Parquet partitioned dataset
+# to a Parquet file
+path = tempdir / "ARROW-3208"
+df = pd.DataFrame({
+'one': [-1, 10, 2.5, 100, 1000, 1, 29.2],
+'two': [-1, 10, 2, 100, 1000, 1, 11],
+'three': [0, 0, 0, 0, 0, 0, 0]
+})
+table = pa.Table.from_pandas(df)
+pq.write_to_dataset(table, root_path=str(path),
+partition_cols=['one', 'two'])
+table = pq.ParquetDataset(path).read()
+pq.write_table(table, path / "output.parquet")
[arrow] branch master updated: ARROW-4697: [C++] Add URI parsing facility
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new ca23513 ARROW-4697: [C++] Add URI parsing facility
ca23513 is described below
commit ca2351363ba1724de17eda3dd8ef334d7231f4f8
Author: Antoine Pitrou
AuthorDate: Wed Mar 20 09:23:27 2019 +0100
ARROW-4697: [C++] Add URI parsing facility
Using the [uriparser](https://uriparser.github.io/) C library.
Author: Antoine Pitrou
Author: Uwe L. Korn
Closes #3779 from pitrou/ARROW-4697-cpp-uri-parsing and squashes the
following commits:
5ad5d1b6 Set CMAKE_INSTALL_LIBDIR when building uriparser
ff41cef7 Fix AppVeyor failure?
6348b896 - Work around a CMake issue - Add docstrings -
Add more path tests
3f5c5899 Explicitly bundle uriparser if conda was set as
default source
a1714902 ARROW-4697: Add URI parsing facility
---
ci/appveyor-cpp-build.bat | 2 +-
cpp/CMakeLists.txt | 3 +
cpp/cmake_modules/ThirdpartyToolchain.cmake | 75
cpp/src/arrow/CMakeLists.txt| 1 +
cpp/src/arrow/symbols.map | 2 +
cpp/src/arrow/util/CMakeLists.txt | 1 +
cpp/src/arrow/util/uri-test.cc | 182
cpp/src/arrow/util/uri.cc | 147 ++
cpp/src/arrow/util/uri.h| 67 ++
cpp/thirdparty/versions.txt | 3 +
10 files changed, 482 insertions(+), 1 deletion(-)
diff --git a/ci/appveyor-cpp-build.bat b/ci/appveyor-cpp-build.bat
index b735d0f..0320db9 100644
--- a/ci/appveyor-cpp-build.bat
+++ b/ci/appveyor-cpp-build.bat
@@ -96,7 +96,7 @@ if "%JOB%" == "Build_Debug" (
exit /B 0
)
-set CONDA_PACKAGES=--file=ci\conda_env_python.yml python=%PYTHON% numpy=1.14
thrift-cpp=0.11 boost-cpp
+set CONDA_PACKAGES=--file=ci\conda_env_python.yml python=%PYTHON% numpy=1.14
boost-cpp
if "%ARROW_BUILD_GANDIVA%" == "ON" (
@rem Install llvmdev in the toolchain if building gandiva.dll
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 3886743..d80fe70 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -757,6 +757,9 @@ set(ARROW_LINK_LIBS ${double-conversion_LIBRARIES})
set(ARROW_STATIC_LINK_LIBS ${double-conversion_LIBRARIES})
set(ARROW_STATIC_INSTALL_INTERFACE_LIBS ${double-conversion_LIBRARIES})
+list(APPEND ARROW_STATIC_LINK_LIBS uriparser::uriparser)
+list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS uriparser::uriparser)
+
if(ARROW_WITH_BROTLI)
# Order is important for static linking
list(APPEND ARROW_LINK_LIBS Brotli::brotlienc Brotli::brotlidec
Brotli::brotlicommon)
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake
b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 29b6173..bdb8a98 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -133,6 +133,8 @@ macro(build_dependency DEPENDENCY_NAME)
build_grpc()
elseif("${DEPENDENCY_NAME}" STREQUAL "BZip2")
build_bzip2()
+ elseif("${DEPENDENCY_NAME}" STREQUAL "uriparser")
+build_uriparser()
else()
message(FATAL_ERROR "Unknown thirdparty dependency to build:
${DEPENDENCY_NAME}")
endif()
@@ -341,6 +343,13 @@ else()
)
endif()
+if(DEFINED ENV{ARROW_URIPARSER_URL})
+ set(URIPARSER_SOURCE_URL "$ENV{ARROW_URIPARSER_URL}")
+else()
+ set(URIPARSER_SOURCE_URL
+
"https://github.com/uriparser/uriparser/archive/${URIPARSER_VERSION}.tar.gz";)
+endif()
+
if(DEFINED ENV{ARROW_ZLIB_URL})
set(ZLIB_SOURCE_URL "$ENV{ARROW_ZLIB_URL}")
else()
@@ -516,6 +525,72 @@ include_directories(SYSTEM
${double-conversion_INCLUDE_DIRS})
double_conversion_compability()
# --
+# uriparser library
+
+macro(build_uriparser)
+ message(STATUS "Building uriparser from source")
+ set(URIPARSER_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/uriparser_ep-install")
+ set(
+URIPARSER_STATIC_LIB
+
"${URIPARSER_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}uriparser${CMAKE_STATIC_LIBRARY_SUFFIX}"
+)
+ set(URIPARSER_INCLUDE_DIRS "${URIPARSER_PREFIX}/include")
+
+ set(URIPARSER_CMAKE_ARGS
+ ${EP_COMMON_CMAKE_ARGS}
+ "-DURIPARSER_BUILD_DOCS=off"
+ "-DURIPARSER_BUILD_TESTS=off"
+ "-DURIPARSER_BUILD_TOOLS=off"
+ "-DURIPARSER_BUILD_WCHAR_T=off"
+ "-DBUILD_SHARED_LIBS=off"
+ "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
+ "-DCMAKE_INSTALL_LIBDIR=lib"
+ "-DCMAKE_POSITION_INDEPENDENT_CODE=on"
+ "-DCMAKE_INSTALL_PREFIX
[arrow] branch master updated: ARROW-4740: [Java] Upgrade to JUnit 5.
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 18b3fc1 ARROW-4740: [Java] Upgrade to JUnit 5.
18b3fc1 is described below
commit 18b3fc196fd981ad477bf57b19d0d7723769ae93
Author: Micah Kornfield
AuthorDate: Mon Mar 18 11:19:37 2019 +0100
ARROW-4740: [Java] Upgrade to JUnit 5.
- Followed recommended POM Recipes from web.
- Remove POM references that no longer seem valid (and update others test
related packages to be there latest).
- Modified TestAuth.java to demonstrate new exception assertions.
Author: Micah Kornfield
Closes #3955 from emkornfield/upgrade_junit and squashes the following
commits:
d8d8a5ec ARROW-4740: Upgrade to JUnit 5.
---
java/adapter/jdbc/pom.xml | 8
.../org/apache/arrow/flight/auth/TestAuth.java | 25 +++--
java/gandiva/pom.xml | 6 ---
java/pom.xml | 43 ++
4 files changed, 42 insertions(+), 40 deletions(-)
diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml
index dc91950..7915bb6 100644
--- a/java/adapter/jdbc/pom.xml
+++ b/java/adapter/jdbc/pom.xml
@@ -46,14 +46,6 @@
${dep.guava.version}
-
-
-junit
-junit
-${dep.junit.version}
-test
-
-
com.h2database
diff --git
a/java/flight/src/test/java/org/apache/arrow/flight/auth/TestAuth.java
b/java/flight/src/test/java/org/apache/arrow/flight/auth/TestAuth.java
index 384190f..20a2814 100644
--- a/java/flight/src/test/java/org/apache/arrow/flight/auth/TestAuth.java
+++ b/java/flight/src/test/java/org/apache/arrow/flight/auth/TestAuth.java
@@ -17,6 +17,8 @@
package org.apache.arrow.flight.auth;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
import java.io.IOException;
import java.util.Arrays;
@@ -33,16 +35,14 @@ import org.apache.arrow.util.AutoCloseables;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
-import org.junit.Rule;
import org.junit.Test;
-import org.junit.rules.ExpectedException;
import com.google.common.collect.ImmutableList;
-public class TestAuth {
+import io.grpc.StatusRuntimeException;
- @Rule
- public ExpectedException thrown = ExpectedException.none();
+public class TestAuth {
+ final String PERMISSION_DENIED = "PERMISSION_DENIED";
private static final String USERNAME = "flight";
private static final String PASSWORD = "woohoo";
@@ -62,17 +62,20 @@ public class TestAuth {
@Test
public void invalidAuth() {
-thrown.expectMessage("PERMISSION_DENIED");
-client.authenticateBasic(USERNAME, "WRONG");
+assertThrows(StatusRuntimeException.class, () -> {
+ client.authenticateBasic(USERNAME, "WRONG");
+}, PERMISSION_DENIED);
-thrown.expectMessage("PERMISSION_DENIED");
-client.listFlights(Criteria.ALL);
+assertThrows(StatusRuntimeException.class, () -> {
+ client.listFlights(Criteria.ALL);
+}, PERMISSION_DENIED);
}
@Test
public void didntAuth() {
-thrown.expectMessage("PERMISSION_DENIED");
-client.listFlights(Criteria.ALL);
+assertThrows(StatusRuntimeException.class, () -> {
+ client.listFlights(Criteria.ALL);
+}, PERMISSION_DENIED);
}
@Before
diff --git a/java/gandiva/pom.xml b/java/gandiva/pom.xml
index ded334c..c0addcb 100644
--- a/java/gandiva/pom.xml
+++ b/java/gandiva/pom.xml
@@ -48,12 +48,6 @@
${protobuf.version}
-junit
-junit
-4.12
-test
-
-
com.google.guava
guava
23.0
diff --git a/java/pom.xml b/java/pom.xml
index 076f642..17f91bd 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -29,7 +29,8 @@
${project.basedir}/target/generated-sources
-4.11
+1.4.0
+5.4.0
1.7.25
20.0
4.1.22.Final
@@ -565,31 +566,43 @@
test
+
+
+ org.junit.platform
+ junit-platform-runner
+ ${dep.junit.platform.version}
+ test
+
+
+ org.junit.jupiter
+ junit-jupiter-engine
+ ${dep.junit.jupiter.version}
+ test
+
+
+ org.junit.jupiter
+ junit-jupiter-api
+ ${dep.junit.jupiter.version}
+ test
+
-
- org.jmockit
- jmockit
- 1.33
+ org.junit.vintage
+ junit-vintage-engine
+ ${dep.junit.jupiter.version}
test
+
junit
junit
- ${dep.junit.version}
+ 4.12
test
org.mockito
[arrow] branch master updated: ARROW-4900: [C++] polyfill __cpuidex on mingw-w64
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 100dab0 ARROW-4900: [C++] polyfill __cpuidex on mingw-w64
100dab0 is described below
commit 100dab0cb7e61e3871023a95cd26037ff08fdca7
Author: Jeroen Ooms
AuthorDate: Sat Mar 16 17:38:52 2019 +0100
ARROW-4900: [C++] polyfill __cpuidex on mingw-w64
Author: Jeroen Ooms
Closes #3923 from jeroen/cpuidex and squashes the following commits:
59429f02 Mention mingw-w64 polyfill in LICENSE.txt
28619330 run clang-format
9e780465 polyfill for __cpuidex on mingw-w64
---
LICENSE.txt| 10 ++
cpp/src/arrow/util/cpu-info.cc | 9 +
2 files changed, 19 insertions(+)
diff --git a/LICENSE.txt b/LICENSE.txt
index 1605d4c..d66d4ba 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -847,3 +847,13 @@ This project include code from CMake.
Copyright: Copyright 2000-2019 Kitware, Inc. and Contributors
Homepage: https://gitlab.kitware.com/cmake/cmake
License: 3-clause BSD
+
+
+
+This project include code from mingw-w64.
+
+* cpp/src/arrow/util/cpu-info.cc has a polyfill for mingw-w64 < 5
+
+Copyright (c) 2009 - 2013 by the mingw-w64 project
+Homepage: https://mingw-w64.org
+License: Zope Public License (ZPL) Version 2.1.
diff --git a/cpp/src/arrow/util/cpu-info.cc b/cpp/src/arrow/util/cpu-info.cc
index 41f240b..9a8cde6 100644
--- a/cpp/src/arrow/util/cpu-info.cc
+++ b/cpp/src/arrow/util/cpu-info.cc
@@ -54,6 +54,15 @@ using boost::algorithm::contains;
using boost::algorithm::trim;
using std::max;
+#if defined(__MINGW64_VERSION_MAJOR) && __MINGW64_VERSION_MAJOR < 5
+void __cpuidex(int CPUInfo[4], int function_id, int subfunction_id) {
+ __asm__ __volatile__("cpuid"
+ : "=a"(CPUInfo[0]), "=b"(CPUInfo[1]), "=c"(CPUInfo[2]),
+ "=d"(CPUInfo[3])
+ : "a"(function_id), "c"(subfunction_id));
+}
+#endif
+
namespace arrow {
namespace internal {
[arrow] branch master updated: ARROW-4922: [Packaging] Use system libraries for .deb and .rpm
This is an automated email from the ASF dual-hosted git repository. uwe pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git The following commit(s) were added to refs/heads/master by this push: new 9bc58a1 ARROW-4922: [Packaging] Use system libraries for .deb and .rpm 9bc58a1 is described below commit 9bc58a104cc91f3666776ea462c03c86f5afd9c5 Author: Kouhei Sutou AuthorDate: Sat Mar 16 17:36:57 2019 +0100 ARROW-4922: [Packaging] Use system libraries for .deb and .rpm Author: Kouhei Sutou Closes #3934 from kou/linux-packages and squashes the following commits: 1db208df Use system libraries for .deb and .rpm --- .../linux-packages/apt/debian-stretch/Dockerfile | 7 +++ .../linux-packages/apt/ubuntu-bionic/Dockerfile| 10 +- .../linux-packages/apt/ubuntu-cosmic/Dockerfile| 11 +-- .../linux-packages/apt/ubuntu-xenial/Dockerfile| 7 +++ dev/tasks/linux-packages/debian/control| 7 +++ dev/tasks/linux-packages/debian/rules | 3 --- dev/tasks/linux-packages/yum/arrow.spec.in | 23 ++ dev/tasks/linux-packages/yum/centos-6/Dockerfile | 7 ++- dev/tasks/linux-packages/yum/centos-7/Dockerfile | 11 ++- 9 files changed, 78 insertions(+), 8 deletions(-) diff --git a/dev/tasks/linux-packages/apt/debian-stretch/Dockerfile b/dev/tasks/linux-packages/apt/debian-stretch/Dockerfile index 3431451..ab0e990 100644 --- a/dev/tasks/linux-packages/apt/debian-stretch/Dockerfile +++ b/dev/tasks/linux-packages/apt/debian-stretch/Dockerfile @@ -50,10 +50,17 @@ RUN \ libboost-filesystem-dev \ libboost-regex-dev \ libboost-system-dev \ +libbrotli-dev \ +libdouble-conversion-dev \ libgirepository1.0-dev \ libglib2.0-doc \ +libgoogle-glog-dev \ +liblz4-dev \ libprotobuf-dev \ libprotoc-dev \ +libre2-dev \ +libsnappy-dev \ +libzstd-dev \ lsb-release \ nvidia-cuda-toolkit \ pkg-config \ diff --git a/dev/tasks/linux-packages/apt/ubuntu-bionic/Dockerfile b/dev/tasks/linux-packages/apt/ubuntu-bionic/Dockerfile index 02d330a..767f8c1 100644 --- a/dev/tasks/linux-packages/apt/ubuntu-bionic/Dockerfile +++ b/dev/tasks/linux-packages/apt/ubuntu-bionic/Dockerfile @@ -38,15 +38,23 @@ RUN \ libboost-filesystem-dev \ libboost-regex-dev \ libboost-system-dev \ +libbrotli-dev \ +libdouble-conversion-dev \ libgirepository1.0-dev \ libglib2.0-doc \ +libgoogle-glog-dev \ +liblz4-dev \ libprotobuf-dev \ libprotoc-dev \ +libre2-dev \ +libsnappy-dev \ +libzstd-dev \ lsb-release \ nvidia-cuda-toolkit \ pkg-config \ protobuf-compiler \ python3-dev \ -python3-numpy && \ +python3-numpy \ +rapidjson-dev && \ apt clean && \ rm -rf /var/lib/apt/lists/* diff --git a/dev/tasks/linux-packages/apt/ubuntu-cosmic/Dockerfile b/dev/tasks/linux-packages/apt/ubuntu-cosmic/Dockerfile index 736..4272690 100644 --- a/dev/tasks/linux-packages/apt/ubuntu-cosmic/Dockerfile +++ b/dev/tasks/linux-packages/apt/ubuntu-cosmic/Dockerfile @@ -38,16 +38,23 @@ RUN \ libboost-filesystem-dev \ libboost-regex-dev \ libboost-system-dev \ +libbrotli-dev \ +libdouble-conversion-dev \ libgirepository1.0-dev \ libglib2.0-doc \ -libjemalloc-dev \ +libgoogle-glog-dev \ +liblz4-dev \ libprotobuf-dev \ libprotoc-dev \ +libre2-dev \ +libsnappy-dev \ +libzstd-dev \ lsb-release \ nvidia-cuda-toolkit \ pkg-config \ protobuf-compiler \ python3-dev \ -python3-numpy && \ +python3-numpy \ +rapidjson-dev && \ apt clean && \ rm -rf /var/lib/apt/lists/* diff --git a/dev/tasks/linux-packages/apt/ubuntu-xenial/Dockerfile b/dev/tasks/linux-packages/apt/ubuntu-xenial/Dockerfile index 51dfcf9..520fe13 100644 --- a/dev/tasks/linux-packages/apt/ubuntu-xenial/Dockerfile +++ b/dev/tasks/linux-packages/apt/ubuntu-xenial/Dockerfile @@ -46,10 +46,17 @@ RUN \ libboost-filesystem-dev \ libboost-regex-dev \ libboost-system-dev \ +libbrotli-dev \ +libdouble-conversion-dev \ libgirepository1.0-dev \ libglib2.0-doc \ +libgoogle-glog-dev \ +liblz4-dev \ libprotobuf-dev \ libprotoc-dev \ +libre2-dev \ +libsnappy-dev \ +libzstd-dev \ lsb-release \ nvidia-cuda-toolkit \ pkg-config \ diff --git a/dev/tasks/linux-packages/debian/control b/dev/tasks/linux-packages/debian/control index 5fcbb5c..ba15b2b 100644 --- a/dev/tasks/linux-packages/debian/control +++ b/dev/tasks/linux-packages/debian/control @@ -16,9 +16,16 @@ Build-Depends: libboost-filesystem-dev, libboost-regex-dev, libboost-system-dev, + libbrotli-dev, + libdouble-conversion-dev, libgirepository1.0-dev, + libgoogle-glog-dev, + liblz4-dev,
[arrow] branch master updated: ARROW-4867: [Python] Respect ordering of columns argument passed to Table.from_pandas
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 76e8fe9 ARROW-4867: [Python] Respect ordering of columns argument
passed to Table.from_pandas
76e8fe9 is described below
commit 76e8fe98d9d61a58ed706c448697e8474fabd30f
Author: Wes McKinney
AuthorDate: Sat Mar 16 17:33:26 2019 +0100
ARROW-4867: [Python] Respect ordering of columns argument passed to
Table.from_pandas
I read through the discussion on ARROW-3766 where this change was
originally made, and I think there was a misunderstanding about a comment I
made where I said "The columns argument in Table.from_pandas is just for column
filtering". I admit it's a big ambiguous what's the right thing to do, but it
seems like the user intent of passing `columns` is to use that order in the
resulting schema, but not error on columns that are not found. We could also
introduce "null" type columns for "n [...]
Author: Wes McKinney
Closes #3930 from wesm/ARROW-4867 and squashes the following commits:
4b4ad64e Respect ordering of columns argument passed to
Table.from_pandas
---
python/pyarrow/pandas_compat.py | 4 +---
python/pyarrow/tests/test_convert_pandas.py | 9 +
2 files changed, 6 insertions(+), 7 deletions(-)
diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index 10038de..90a0ad6 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -411,9 +411,7 @@ def _resolve_columns_of_interest(df, schema, columns):
elif schema is not None:
columns = schema.names
elif columns is not None:
-# columns is only for filtering, the function must keep the column
-# ordering of either the dataframe or the passed schema
-columns = [c for c in df.columns if c in columns]
+columns = [c for c in columns if c in df.columns]
else:
columns = df.columns
diff --git a/python/pyarrow/tests/test_convert_pandas.py
b/python/pyarrow/tests/test_convert_pandas.py
index 16e2a76..0aab444 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -140,7 +140,8 @@ class TestConvertMetadata(object):
assert table.column(0).name == '0'
def test_from_pandas_with_columns(self):
-df = pd.DataFrame({0: [1, 2, 3], 1: [1, 3, 3], 2: [2, 4, 5]})
+df = pd.DataFrame({0: [1, 2, 3], 1: [1, 3, 3], 2: [2, 4, 5]},
+ columns=[1, 0])
table = pa.Table.from_pandas(df, columns=[0, 1])
expected = pa.Table.from_pandas(df[[0, 1]])
@@ -2495,15 +2496,15 @@ def
test_table_from_pandas_columns_argument_only_does_filtering():
columns1 = ['arrays', 'floats', 'partition']
schema1 = pa.schema([
-('partition', pa.int64()),
('arrays', pa.list_(pa.int64())),
('floats', pa.float64()),
+('partition', pa.int64())
])
columns2 = ['floats', 'partition']
schema2 = pa.schema([
-('partition', pa.int64()),
-('floats', pa.float64())
+('floats', pa.float64()),
+('partition', pa.int64())
])
table1 = pa.Table.from_pandas(df, columns=columns1, preserve_index=False)
[arrow] branch master updated: Fix link to Flight.proto (#3915)
This is an automated email from the ASF dual-hosted git repository. uwe pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git The following commit(s) were added to refs/heads/master by this push: new afffe3a Fix link to Flight.proto (#3915) afffe3a is described below commit afffe3af09a7e5d77404d139fea933c120361a3f Author: Maxim Novikov AuthorDate: Fri Mar 15 13:20:22 2019 +0100 Fix link to Flight.proto (#3915) --- java/flight/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/flight/README.md b/java/flight/README.md index da91a4f..00922bf 100644 --- a/java/flight/README.md +++ b/java/flight/README.md @@ -23,7 +23,7 @@ Exposing Apache Arrow data on the wire. [Protocol Description Slides](https://www.slideshare.net/JacquesNadeau5/apache-arrow-flight-overview) -[GRPC Protocol Definition](https://github.com/jacques-n/arrow/blob/flight/java/flight/src/main/protobuf/flight.proto) +[GRPC Protocol Definition](https://github.com/apache/arrow/blob/master/format/Flight.proto) ## Example usage
[arrow] branch master updated: ARROW-4776: [C++] Add DictionaryBuilder constructor which takes a dictionary array
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 65d0e19 ARROW-4776: [C++] Add DictionaryBuilder constructor which
takes a dictionary array
65d0e19 is described below
commit 65d0e1959a3a2a52acae85e9f5b465f345585476
Author: Benjamin Kietzman
AuthorDate: Wed Mar 13 16:53:05 2019 +0100
ARROW-4776: [C++] Add DictionaryBuilder constructor which takes a
dictionary array
DictionaryBuilder can now be constructed with a user provided dictionary.
This is also available through MakeBuilder
Author: Benjamin Kietzman
Author: Uwe L. Korn
Closes #3867 from
bkietz/ARROW-4776-dictionarybuilder-should-support-bootstrapping-from-an-existing-dict-type
and squashes the following commits:
d2551c99 Use {String,Binary}DictionaryBuilder for those
types
91da93b9 remove unused std::stringstream
4514860b Add tests for MakeBuilder(dictionary type)
5fcbe263 Add dictionary constructor to DictionaryBuilder
---
cpp/src/arrow/array-dict-test.cc| 136 +++-
cpp/src/arrow/array/builder_dict.cc | 38 --
cpp/src/arrow/array/builder_dict.h | 4 ++
cpp/src/arrow/builder.cc| 41 +++
4 files changed, 210 insertions(+), 9 deletions(-)
diff --git a/cpp/src/arrow/array-dict-test.cc b/cpp/src/arrow/array-dict-test.cc
index fcc9fea..8289aee 100644
--- a/cpp/src/arrow/array-dict-test.cc
+++ b/cpp/src/arrow/array-dict-test.cc
@@ -63,17 +63,67 @@ TYPED_TEST(TestDictionaryBuilder, Basic) {
ASSERT_EQ(builder.length(), 4);
ASSERT_EQ(builder.null_count(), 1);
+ // Build expected data
+ auto dict_array = ArrayFromJSON(std::make_shared(), "[1, 2]");
+ auto dict_type = dictionary(int8(), dict_array);
+
std::shared_ptr result;
ASSERT_OK(builder.Finish(&result));
+ auto int_array = ArrayFromJSON(int8(), "[0, 1, 0, null]");
+ DictionaryArray expected(dict_type, int_array);
+
+ ASSERT_TRUE(expected.Equals(result));
+}
+
+TYPED_TEST(TestDictionaryBuilder, ArrayInit) {
+ auto dict_array = ArrayFromJSON(std::make_shared(), "[1, 2]");
+ auto dict_type = dictionary(int8(), dict_array);
+
+ DictionaryBuilder builder(dict_array, default_memory_pool());
+ ASSERT_OK(builder.Append(static_cast(1)));
+ ASSERT_OK(builder.Append(static_cast(2)));
+ ASSERT_OK(builder.Append(static_cast(1)));
+ ASSERT_OK(builder.AppendNull());
+
+ ASSERT_EQ(builder.length(), 4);
+ ASSERT_EQ(builder.null_count(), 1);
+
// Build expected data
+
+ std::shared_ptr result;
+ ASSERT_OK(builder.Finish(&result));
+
+ auto int_array = ArrayFromJSON(int8(), "[0, 1, 0, null]");
+ DictionaryArray expected(dict_type, int_array);
+
+ AssertArraysEqual(expected, *result);
+}
+
+TYPED_TEST(TestDictionaryBuilder, MakeBuilder) {
auto dict_array = ArrayFromJSON(std::make_shared(), "[1, 2]");
- auto dict_type = std::make_shared(int8(), dict_array);
+ auto dict_type = dictionary(int8(), dict_array);
+ std::unique_ptr boxed_builder;
+ ASSERT_OK(MakeBuilder(default_memory_pool(), dict_type, &boxed_builder));
+ auto& builder = checked_cast&>(*boxed_builder);
+
+ ASSERT_OK(builder.Append(static_cast(1)));
+ ASSERT_OK(builder.Append(static_cast(2)));
+ ASSERT_OK(builder.Append(static_cast(1)));
+ ASSERT_OK(builder.AppendNull());
+
+ ASSERT_EQ(builder.length(), 4);
+ ASSERT_EQ(builder.null_count(), 1);
+
+ // Build expected data
+
+ std::shared_ptr result;
+ ASSERT_OK(builder.Finish(&result));
auto int_array = ArrayFromJSON(int8(), "[0, 1, 0, null]");
DictionaryArray expected(dict_type, int_array);
- ASSERT_TRUE(expected.Equals(result));
+ AssertArraysEqual(expected, *result);
}
TYPED_TEST(TestDictionaryBuilder, ArrayConversion) {
@@ -87,7 +137,7 @@ TYPED_TEST(TestDictionaryBuilder, ArrayConversion) {
// Build expected data
auto dict_array = ArrayFromJSON(type, "[1, 2]");
- auto dict_type = std::make_shared(int8(), dict_array);
+ auto dict_type = dictionary(int8(), dict_array);
auto int_array = ArrayFromJSON(int8(), "[0, 1, 0]");
DictionaryArray expected(dict_type, int_array);
@@ -239,6 +289,47 @@ TEST(TestStringDictionaryBuilder, Basic) {
ASSERT_TRUE(expected.Equals(result));
}
+TEST(TestStringDictionaryBuilder, ArrayInit) {
+ auto dict_array = ArrayFromJSON(utf8(), R"(["test", "test2"])");
+ auto int_array = ArrayFromJSON(int8(), "[0, 1, 0]");
+
+ // Build the dictionary Array
+ StringDictionaryBuilder builder(dict_array, default_memory_pool());
+ ASSERT_OK(builder.Append("test"));
+ ASSERT_OK(builder.Append("test2"));
+ ASSERT_OK(builder.Append("test"));
+
+ std::shared_ptr result;
+ ASSERT_OK(bu
[arrow] branch master updated: ARROW-4620: [C#] Add unit tests for "Types" in arrow/csharp
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 781d67e ARROW-4620: [C#] Add unit tests for "Types" in arrow/csharp
781d67e is described below
commit 781d67ead08f4193a98167a102509dbcb63b2dd5
Author: Prashanth Govindarajan
AuthorDate: Tue Mar 12 17:14:37 2019 +0100
ARROW-4620: [C#] Add unit tests for "Types" in arrow/csharp
A port of most of the unit tests under arrow/cpp/type-test.cc to csharp.
Author: Prashanth Govindarajan
Closes #3662 from pgovind/TypeUnitTests and squashes the following commits:
29d7497d sq
9ef3eeac Move SchemaComparer and FieldComparer to
their own files
a18da7ad Address commend and update Eric's PR to
use the new compare
ba8c6f5d Rename dict to dictionary
d1f6ec9b Address nits and comments
1b48a856 More unit tests
08dbc836 Type unit tests
---
csharp/src/Apache.Arrow/Field.Builder.cs | 13 +++
csharp/src/Apache.Arrow/Schema.Builder.cs | 13 +++
.../Apache.Arrow.Tests/ArrowStreamReaderTests.cs | 3 +-
csharp/test/Apache.Arrow.Tests/FieldComparer.cs| 42 +
.../test/Apache.Arrow.Tests/SchemaBuilderTests.cs | 72 ++
csharp/test/Apache.Arrow.Tests/SchemaComparer.cs | 48 ++
csharp/test/Apache.Arrow.Tests/TypeTests.cs| 103 +
7 files changed, 292 insertions(+), 2 deletions(-)
diff --git a/csharp/src/Apache.Arrow/Field.Builder.cs
b/csharp/src/Apache.Arrow/Field.Builder.cs
index fede4ea..6d820b6 100644
--- a/csharp/src/Apache.Arrow/Field.Builder.cs
+++ b/csharp/src/Apache.Arrow/Field.Builder.cs
@@ -70,6 +70,19 @@ namespace Apache.Arrow
return this;
}
+public Builder Metadata(IEnumerable>
dictionary)
+{
+if (dictionary == null)
+{
+throw new ArgumentNullException(nameof(dictionary));
+}
+foreach (KeyValuePair entry in dictionary)
+{
+Metadata(entry.Key, entry.Value);
+}
+return this;
+}
+
public Field Build()
{
return new Field(_name, _type, _nullable, _metadata);
diff --git a/csharp/src/Apache.Arrow/Schema.Builder.cs
b/csharp/src/Apache.Arrow/Schema.Builder.cs
index 58ee7f3..560958c 100644
--- a/csharp/src/Apache.Arrow/Schema.Builder.cs
+++ b/csharp/src/Apache.Arrow/Schema.Builder.cs
@@ -63,6 +63,19 @@ namespace Apache.Arrow
return this;
}
+public Builder Metadata(IEnumerable>
dictionary)
+{
+if (dictionary == null)
+{
+throw new ArgumentNullException(nameof(dictionary));
+}
+foreach (KeyValuePair entry in dictionary)
+{
+this.Metadata(entry.Key, entry.Value);
+}
+return this;
+}
+
public Schema Build()
{
return new Schema(_fields, _metadata);
diff --git a/csharp/test/Apache.Arrow.Tests/ArrowStreamReaderTests.cs
b/csharp/test/Apache.Arrow.Tests/ArrowStreamReaderTests.cs
index af68a85..3a1dae6 100644
--- a/csharp/test/Apache.Arrow.Tests/ArrowStreamReaderTests.cs
+++ b/csharp/test/Apache.Arrow.Tests/ArrowStreamReaderTests.cs
@@ -87,8 +87,7 @@ namespace Apache.Arrow.Tests
private void CompareSchemas(Schema expectedSchema, Schema actualSchema)
{
-Assert.Equal(expectedSchema.Fields.Count,
actualSchema.Fields.Count);
-// TODO: compare fields once
https://github.com/apache/arrow/pull/3662 is in
+Assert.True(SchemaComparer.Equals(expectedSchema, actualSchema));
}
private class ArrayComparer :
diff --git a/csharp/test/Apache.Arrow.Tests/FieldComparer.cs
b/csharp/test/Apache.Arrow.Tests/FieldComparer.cs
new file mode 100644
index 000..97a303f
--- /dev/null
+++ b/csharp/test/Apache.Arrow.Tests/FieldComparer.cs
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY K
[arrow] branch master updated: ARROW-4790: [Python/Packaging] Update manylinux docker image in crossbow task
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 3db5797 ARROW-4790: [Python/Packaging] Update manylinux docker image
in crossbow task
3db5797 is described below
commit 3db5797171616f1387ef8ce8ed1183e96e68dcdf
Author: Krisztián Szűcs
AuthorDate: Mon Mar 11 11:28:31 2019 +0100
ARROW-4790: [Python/Packaging] Update manylinux docker image in crossbow
task
Centralize the docker command into docker-compose. Now it's enough to
change the docker tag in a single place.
crossbow task:
[kszucs/crossbow/build-447](https://github.com/kszucs/crossbow/branches/all?utf8=%E2%9C%93&query=build-447)
Author: Krisztián Szűcs
Closes #3830 from kszucs/ARROW-4790 and squashes the following commits:
5cf85c9c comments
89da1791 run manylinux image from docker-compose
---
.travis.yml | 3 +--
ci/travis_script_manylinux.sh| 7 ++-
dev/tasks/python-wheels/travis.linux.yml | 12 ++--
docker-compose.yml | 24
4 files changed, 33 insertions(+), 13 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index afddeea..de9832a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -226,9 +226,8 @@ matrix:
language: cpp
env:
- PYTHON_VERSIONS="2.7,32 3.6,16"
-- MANYLINUX1_IMAGE="quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-4778"
before_script:
-- if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then docker pull
$MANYLINUX1_IMAGE; fi
+- if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then docker-compose pull
python-manylinux1; fi
script:
- if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then
$TRAVIS_BUILD_DIR/ci/travis_script_manylinux.sh; fi
- name: "Java w/ OpenJDK 8"
diff --git a/ci/travis_script_manylinux.sh b/ci/travis_script_manylinux.sh
index 9f05f26..fe6c108 100755
--- a/ci/travis_script_manylinux.sh
+++ b/ci/travis_script_manylinux.sh
@@ -47,14 +47,11 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do
sudo rm -rf dist
# build the wheels
- docker run --shm-size=2g --rm \
+ docker-compose run \
-e PYARROW_PARALLEL=3 \
-e PYTHON_VERSION=$PYTHON_VERSION \
-e UNICODE_WIDTH=$UNICODE_WIDTH \
--v $PWD:/io \
--v $PWD/../../:/arrow \
-$MANYLINUX1_IMAGE \
-/io/build_arrow.sh
+python-manylinux1
# create a testing conda environment
CONDA_ENV_DIR=$TRAVIS_BUILD_DIR/pyarrow-test-$PYTHON_VERSION
diff --git a/dev/tasks/python-wheels/travis.linux.yml
b/dev/tasks/python-wheels/travis.linux.yml
index 240c523..64b9e7e 100644
--- a/dev/tasks/python-wheels/travis.linux.yml
+++ b/dev/tasks/python-wheels/travis.linux.yml
@@ -38,15 +38,15 @@ script:
- mkdir -p dist
# build wheel
- - pushd arrow/python/manylinux1
- - docker run --shm-size=2g
+ - pushd arrow
+ # don't attempt to build it, because the base image by pypa is updated
+ # regularly and would cause undeterministic builds
+ - docker-compose pull python-manylinux1
+ - docker-compose run
-e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }}
-e PYTHON_VERSION="{{ python_version }}"
-e UNICODE_WIDTH="{{ unicode_width }}"
- -v $PWD:/io
- -v $PWD/../../:/arrow
- quay.io/xhochy/arrow_manylinux1_x86_64_base:llvm-7-manylinux1
- /io/build_arrow.sh
+ python-manylinux1
- popd
# test on multiple distributions
diff --git a/docker-compose.yml b/docker-compose.yml
index 926e3b5..5581061 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -232,6 +232,30 @@ services:
dockerfile: docs/Dockerfile
volumes: *ubuntu-volumes
+ # Package Builder ###
+
+ python-manylinux1:
+# Usage:
+# either build:
+# $ docker-compose build python-manylinux1
+# or pull:
+# $ docker-compose pull python-manylinux1
+# an then run:
+# $ docker-compose run -e PYTHON_VERSION=3.7 python-manylinux1
+image: quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-4778
+build:
+ context: .
+ dockerfile: python/manylinux1/Dockerfile-x86_64_base
+shm_size: 2G
+environment:
+ PYARROW_PARALLEL: 3
+ PYTHON_VERSION: ${PYTHON_VERSION:-3.6}
+ UNICODE_WIDTH: ${UNICODE_WIDTH:-16}
+volumes:
+ - .:/arrow:delegated
+ - ./python/manylinux1:/io:delegated
+command: /io/build_arrow.sh
+
# Integration Tests #
# impala:
[arrow] branch master updated: ARROW-4538: [Python] Remove index column from subschema in write_to_dataframe
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new d05efe6 ARROW-4538: [Python] Remove index column from subschema in
write_to_dataframe
d05efe6 is described below
commit d05efe680f4ea60d164ccda641451db06c9d20d7
Author: Christian Thiel
AuthorDate: Wed Mar 6 12:17:04 2019 +0100
ARROW-4538: [Python] Remove index column from subschema in
write_to_dataframe
I noticed that `parquet.write_to_dataset` failes if the index name is not
None. This is due to `write_to_dataframe` only removing the old `__index_level`
prefixed column names.
In this PR I determined the dataframe index names from the metadata just
like in `pyarrow.pandas_compat.table_to_blockmanager`. I believe that this is a
good thing to do, as the extra trip via pandas in `write_to_dataframe` will
hopefully be superfluous in the future.
This is my first contribution to arrow (appart from some issues), so please
let me know if this is what you're looking for.
Author: Christian Thiel
Closes #3744 from c-thiel/master and squashes the following commits:
a66b5f85 ARROW-4538: Remove index columns from subschema
in write_to_dataframe
---
python/pyarrow/parquet.py| 10 +-
python/pyarrow/tests/test_parquet.py | 18 --
2 files changed, 25 insertions(+), 3 deletions(-)
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index f5a98eb..fe602bb 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -1249,10 +1249,18 @@ def write_to_dataset(table, root_path,
partition_cols=None,
raise ValueError('No data left to save outside partition columns')
subschema = table.schema
+# ARROW-4538: Remove index column from subschema in write_to_dataframe
+metadata = subschema.metadata
+has_pandas_metadata = (metadata is not None and b'pandas' in metadata)
+index_columns = []
+if has_pandas_metadata:
+pandas_metadata = json.loads(metadata[b'pandas'].decode('utf8'))
+index_columns = pandas_metadata['index_columns']
# ARROW-2891: Ensure the output_schema is preserved when writing a
# partitioned dataset
for col in table.schema.names:
-if (col.startswith('__index_level_') or col in partition_cols):
+if (col.startswith('__index_level_') or col in partition_cols or
+col in index_columns):
subschema = subschema.remove(subschema.get_field_index(col))
for keys, subgroup in data_df.groupby(partition_keys):
diff --git a/python/pyarrow/tests/test_parquet.py
b/python/pyarrow/tests/test_parquet.py
index f4ced93..77b9ead 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -1900,7 +1900,8 @@ def test_read_table_doesnt_warn(datadir):
def _test_write_to_dataset_with_partitions(base_path,
filesystem=None,
- schema=None):
+ schema=None,
+ index_name=None):
# ARROW-1400
output_df = pd.DataFrame({'group1': list('aaaccc'),
'group2': list('eefeffgeee'),
@@ -1908,9 +1909,13 @@ def _test_write_to_dataset_with_partitions(base_path,
'nan': [pd.np.nan] * 10,
'date': np.arange('2017-01-01', '2017-01-11',
dtype='datetime64[D]')})
+# ARROW-4538
+output_df.index.name = index_name
+
cols = output_df.columns.tolist()
partition_by = ['group1', 'group2']
-output_table = pa.Table.from_pandas(output_df, schema=schema, safe=False)
+output_table = pa.Table.from_pandas(output_df, schema=schema, safe=False,
+preserve_index=True)
pq.write_to_dataset(output_table, base_path, partition_by,
filesystem=filesystem)
@@ -1931,6 +1936,10 @@ def _test_write_to_dataset_with_partitions(base_path,
dataset_cols = set(dataset.schema.to_arrow_schema().names)
assert dataset_cols == set(output_table.schema.names)
+# ARROW-4538
+if index_name is not None:
+assert index_name in dataset_cols
+
input_table = dataset.read()
input_df = input_table.to_pandas()
@@ -1991,6 +2000,11 @@ def
test_write_to_dataset_with_partitions_and_schema(tempdir):
_test_write_to_dataset_with_partitions(str(tempdir), schema=schema)
+def test_write_to
[arrow] branch master updated: ARROW-4758: [C++][Flight] Fix intermittent build failure
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new dd3ba0c ARROW-4758: [C++][Flight] Fix intermittent build failure
dd3ba0c is described below
commit dd3ba0c1f63f94415ac2886c28f9b880f1116764
Author: Pindikura Ravindra
AuthorDate: Tue Mar 5 20:09:50 2019 +0100
ARROW-4758: [C++][Flight] Fix intermittent build failure
Author: Pindikura Ravindra
Closes #3803 from pravindra/flight and squashes the following commits:
a20c3468 switch dep to metadata_fbs
f1a07ba7 ARROW-4758: Fix intermittent build failure
---
cpp/src/arrow/flight/CMakeLists.txt | 1 +
1 file changed, 1 insertion(+)
diff --git a/cpp/src/arrow/flight/CMakeLists.txt
b/cpp/src/arrow/flight/CMakeLists.txt
index 6f44c01..8bd23e5 100644
--- a/cpp/src/arrow/flight/CMakeLists.txt
+++ b/cpp/src/arrow/flight/CMakeLists.txt
@@ -77,6 +77,7 @@ add_arrow_lib(arrow_flight
${ARROW_FLIGHT_SRCS}
DEPENDENCIES
flight_grpc_gen
+ metadata_fbs
SHARED_LINK_LIBS
arrow_shared
${ARROW_FLIGHT_STATIC_LINK_LIBS}
[arrow] branch master updated: ARROW-4775: [Site] Site navbar cannot be expanded
This is an automated email from the ASF dual-hosted git repository. uwe pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git The following commit(s) were added to refs/heads/master by this push: new 0e673f8 ARROW-4775: [Site] Site navbar cannot be expanded 0e673f8 is described below commit 0e673f812b5148daa224dfc607a6af835f7b38ac Author: Kenta Murata AuthorDate: Tue Mar 5 19:56:06 2019 +0100 ARROW-4775: [Site] Site navbar cannot be expanded I found that the navbar at the top of the page cannot be expanded when the page is narrow. This pull request fixes this problem. Author: Kenta Murata Closes #3814 from mrkn/fix_site_navbar_expanding_behavior and squashes the following commits: 8fb38ac0 Fix navbar expanding behavior --- site/_includes/header.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/_includes/header.html b/site/_includes/header.html index 39a1d05..7c55a2c 100644 --- a/site/_includes/header.html +++ b/site/_includes/header.html @@ -1,6 +1,6 @@ Apache Arrow™ - +
[arrow] branch master updated: ARROW-4756: Update readme for triggering docker builds
This is an automated email from the ASF dual-hosted git repository. uwe pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git The following commit(s) were added to refs/heads/master by this push: new 5df33ad ARROW-4756: Update readme for triggering docker builds 5df33ad is described below commit 5df33ada68354b8e9e69abc258a2aee0c2be33d8 Author: Pindikura Ravindra AuthorDate: Tue Mar 5 19:45:32 2019 +0100 ARROW-4756: Update readme for triggering docker builds Author: Pindikura Ravindra Closes #3802 from pravindra/manylinux_doc and squashes the following commits: b5c675b3 ARROW-4756: Update readme for triggering docker builds --- python/manylinux1/README.md | 24 1 file changed, 24 insertions(+) diff --git a/python/manylinux1/README.md b/python/manylinux1/README.md index 06288f4..3fcc4f7 100644 --- a/python/manylinux1/README.md +++ b/python/manylinux1/README.md @@ -59,6 +59,30 @@ dependency is persisted in the docker image. When you do local adjustments to this image, you need to change the name of the docker image in the `docker run` command. +### Using quay.io to trigger and build the docker image + +1. Make the change in the build scripts (eg. to modify the boost build, update `scripts/boost.sh`). + +2. Setup an account on quay.io and link to your GitHub account + +3. In quay.io, Add a new repository using : + +1. Link to GitHub repository push +2. Trigger build on changes to a specific branch (eg. myquay) of the repo (eg. `pravindra/arrow`) +3. Set Dockerfile location to `/python/manylinux1/Dockerfile-x86_64_base` +4. Set Context location to `/python/manylinux1` + +4. Push change (in step 1) to the branch specified in step 3.ii + +* This should trigger a build in quay.io, the build takes about 2 hrs to finish. + +5. Add a tag `latest` to the build after step 4 finishes, save the build ID (eg. `quay.io/pravindra/arrow_manylinux1_x86_64_base:latest`) + +6. In your arrow PR, + +* include the change from 1. +* modify `travis_script_manylinux.sh` to switch to the location from step 5 for the docker image. + ## TensorFlow compatible wheels for Arrow As TensorFlow is not compatible with the manylinux1 standard, the above
[arrow] branch master updated: ARROW-4737: run C# tests in CI
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new c35a3e6 ARROW-4737: run C# tests in CI
c35a3e6 is described below
commit c35a3e6e6c5e87490489f4709997646f7c0ec7e8
Author: Eric Erhardt
AuthorDate: Sun Mar 3 19:25:32 2019 +0100
ARROW-4737: run C# tests in CI
The C# tests aren't running in CI because they are being filtered out.
Adding detection logic for C#, so the tests run when `csharp` code changes.
@wesm @chutchinson @pgovind
Author: Eric Erhardt
Closes #3788 from eerhardt/FixCSharpCI and squashes the following commits:
7aaf8aa5 ARROW-4737: run C# tests in CI
---
ci/appveyor-filter-changes.bat | 7 +++
ci/detect-changes.py | 6 --
2 files changed, 11 insertions(+), 2 deletions(-)
diff --git a/ci/appveyor-filter-changes.bat b/ci/appveyor-filter-changes.bat
index e78f91f..e506e11 100644
--- a/ci/appveyor-filter-changes.bat
+++ b/ci/appveyor-filter-changes.bat
@@ -29,6 +29,13 @@ if "%JOB%" == "Rust" (
echo ===
appveyor exit
)
+) else if "%JOB%" == "C#" (
+if "%ARROW_CI_CSHARP_AFFECTED%" == "0" (
+echo ===
+echo === No C# changes, exiting job
+echo ===
+appveyor exit
+)
) else (
if "%ARROW_CI_PYTHON_AFFECTED%" == "0" (
echo ===
diff --git a/ci/detect-changes.py b/ci/detect-changes.py
index 8e24c3f..8c7ce17 100644
--- a/ci/detect-changes.py
+++ b/ci/detect-changes.py
@@ -108,7 +108,7 @@ def list_appveyor_affected_files():
LANGUAGE_TOPICS = ['c_glib', 'cpp', 'docs', 'go', 'java', 'js', 'python',
- 'r', 'ruby', 'rust']
+ 'r', 'ruby', 'rust', 'csharp']
ALL_TOPICS = LANGUAGE_TOPICS + ['integration', 'site', 'dev']
@@ -124,7 +124,7 @@ AFFECTED_DEPENDENCIES = {
}
COMPONENTS = {'cpp', 'java', 'c_glib', 'r', 'ruby', 'integration', 'js',
- 'rust', 'site', 'go', 'docs', 'python', 'dev'}
+ 'rust', 'csharp', 'site', 'go', 'docs', 'python', 'dev'}
def get_affected_topics(affected_files):
@@ -241,6 +241,7 @@ def test_get_affected_topics():
'r': True,
'ruby': True,
'rust': False,
+'csharp': False,
'integration': True,
'site': False,
'dev': False
@@ -258,6 +259,7 @@ def test_get_affected_topics():
'r': True,
'ruby': True,
'rust': True,
+'csharp': True,
'integration': True,
'site': False,
'dev': False
svn commit: r32627 - in /release/arrow: arrow-0.12.0/ arrow-0.12.1/ arrow-0.12.1/apache-arrow-0.12.1.tar.gz arrow-0.12.1/apache-arrow-0.12.1.tar.gz.asc arrow-0.12.1/apache-arrow-0.12.1.tar.gz.sha256 a
Author: uwe Date: Mon Feb 25 15:06:20 2019 New Revision: 32627 Log: Apache Arrow 0.12.1 Added: release/arrow/arrow-0.12.1/ release/arrow/arrow-0.12.1/apache-arrow-0.12.1.tar.gz (with props) release/arrow/arrow-0.12.1/apache-arrow-0.12.1.tar.gz.asc release/arrow/arrow-0.12.1/apache-arrow-0.12.1.tar.gz.sha256 release/arrow/arrow-0.12.1/apache-arrow-0.12.1.tar.gz.sha512 Removed: release/arrow/arrow-0.12.0/ Added: release/arrow/arrow-0.12.1/apache-arrow-0.12.1.tar.gz == Binary file - no diff available. Propchange: release/arrow/arrow-0.12.1/apache-arrow-0.12.1.tar.gz -- svn:mime-type = application/octet-stream Added: release/arrow/arrow-0.12.1/apache-arrow-0.12.1.tar.gz.asc == --- release/arrow/arrow-0.12.1/apache-arrow-0.12.1.tar.gz.asc (added) +++ release/arrow/arrow-0.12.1/apache-arrow-0.12.1.tar.gz.asc Mon Feb 25 15:06:20 2019 @@ -0,0 +1,17 @@ +-BEGIN PGP SIGNATURE- +Version: GnuPG v1 + +iQIcBAABAgAGBQJcbsq1AAoJECnZTiKMqtYCOkgQAJV7rX9aHs3hNnzNPhUtywi7 +LMg9wjfIpF/7pHr/hnQLB1PsZjTtyepZCqZ+YOGxjG0ZkUgTCcvsl4Lj866LShuw +14OxX1KW6H21zVUXnOrMGEYf1V8KFZRmN6aHzGUOzy51Q4+hc9fiGKvb0qgZFU0l +5Qw1MryZ22i/7ryiORdIlvof8y7BbiRvUQ9041aM/00JyLcKfbKyrGdld5In8ohN +roZLhmYq52iNGhIkvJ9tXGVl+zg2bz+2Bx5ex1j0b0mEVCpDUcgkN3VT0qAXAwJs +Y382GQpSdRm1JzERduZwTUha/CS6KwvgKUhK1xh3cSYJPAgshjaIY6zgvUdYC3kR +5ICn1FykngjlcVLHMqtFHyuCA5stqpIOCplqUclsy6LHgl8W/ZikOVvSIrTFt4Oc +auDhVc7IVtYjlxd24uOgjRgM/Pu3iL68PVZoZcbWUaPPPpmBXC6xl9EGYLx8Jfhn +rSoSKwFGI0ZJPdzNoZGQqlsoiZw8gw3Z8n1pX24ZMchbkJ/NJfqu8uXLVTjaJjXm +twiQGsZ9iw4Bo1jdfCQ+gy+4m9g+ix0WFxOXFIi6rrnZ6u1K3sGYPpY5hpooKS7u +HDDVPIRyU4H1RkoV6IoHqgMcldeOCXOLlnO0D1LWPbjDnC0zUULkVqNUF9IrN1tN +RrQ93LWtBUXIoYHeUYST +=/XOB +-END PGP SIGNATURE- Added: release/arrow/arrow-0.12.1/apache-arrow-0.12.1.tar.gz.sha256 == --- release/arrow/arrow-0.12.1/apache-arrow-0.12.1.tar.gz.sha256 (added) +++ release/arrow/arrow-0.12.1/apache-arrow-0.12.1.tar.gz.sha256 Mon Feb 25 15:06:20 2019 @@ -0,0 +1 @@ +e93e43343544e344bbc912b89da01d8abf66596f029d26b2b135b102a9e39895 apache-arrow-0.12.1.tar.gz Added: release/arrow/arrow-0.12.1/apache-arrow-0.12.1.tar.gz.sha512 == --- release/arrow/arrow-0.12.1/apache-arrow-0.12.1.tar.gz.sha512 (added) +++ release/arrow/arrow-0.12.1/apache-arrow-0.12.1.tar.gz.sha512 Mon Feb 25 15:06:20 2019 @@ -0,0 +1 @@ +45a07daae97e956a64f6a6cbc7a4339245d2bf6c4efcd7457c4eed4900f0a62e9433add5c1a01b734788fdec804b5ce49bdc6d309e7e397bae640bc379510fa1 apache-arrow-0.12.1.tar.gz
[arrow] branch maint-0.12.x updated (cef7e71 -> 90d42ef)
This is an automated email from the ASF dual-hosted git repository. uwe pushed a change to branch maint-0.12.x in repository https://gitbox.apache.org/repos/asf/arrow.git. from cef7e71 ARROW-4501: Fix out-of-bounds read in DoubleCrcHash new 0f7f8d4 [Release] Update CHANGELOG.md for 0.12.1 new 2c4d0ad [Release] Update .deb/.rpm changelogs for 0.12.1 new 495c50d [Release] Update versions for 0.12.1 new ba09a9e [maven-release-plugin] prepare release apache-arrow-0.12.1 new 8610051 [maven-release-plugin] prepare for next development iteration new 90d42ef [Release] Update versions for 0.12.2-SNAPSHOT The 3388 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: CHANGELOG.md | 24 ++ c_glib/configure.ac| 2 +- c_glib/meson.build | 2 +- cpp/CMakeLists.txt | 2 +- .../linux-packages/debian.ubuntu-trusty/changelog | 6 ++ dev/tasks/linux-packages/debian/changelog | 6 ++ dev/tasks/linux-packages/yum/arrow.spec.in | 3 +++ java/adapter/jdbc/pom.xml | 2 +- java/flight/pom.xml| 2 +- java/format/pom.xml| 2 +- java/memory/pom.xml| 2 +- java/plasma/pom.xml| 2 +- java/pom.xml | 2 +- java/tools/pom.xml | 2 +- java/vector/pom.xml| 2 +- matlab/CMakeLists.txt | 2 +- r/DESCRIPTION | 2 +- ruby/red-arrow-cuda/lib/arrow-cuda/version.rb | 2 +- ruby/red-arrow/lib/arrow/version.rb| 2 +- ruby/red-gandiva/lib/gandiva/version.rb| 2 +- ruby/red-parquet/lib/parquet/version.rb| 2 +- ruby/red-plasma/lib/plasma/version.rb | 2 +- rust/arrow/Cargo.toml | 2 +- rust/parquet/Cargo.toml| 2 +- 24 files changed, 59 insertions(+), 20 deletions(-)
[arrow] branch maint-0.12.x updated (cef7e71 -> 90d42ef)
This is an automated email from the ASF dual-hosted git repository. uwe pushed a change to branch maint-0.12.x in repository https://gitbox.apache.org/repos/asf/arrow.git. from cef7e71 ARROW-4501: Fix out-of-bounds read in DoubleCrcHash new 0f7f8d4 [Release] Update CHANGELOG.md for 0.12.1 new 2c4d0ad [Release] Update .deb/.rpm changelogs for 0.12.1 new 495c50d [Release] Update versions for 0.12.1 new ba09a9e [maven-release-plugin] prepare release apache-arrow-0.12.1 new 8610051 [maven-release-plugin] prepare for next development iteration new 90d42ef [Release] Update versions for 0.12.2-SNAPSHOT The 3388 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: CHANGELOG.md | 24 ++ c_glib/configure.ac| 2 +- c_glib/meson.build | 2 +- cpp/CMakeLists.txt | 2 +- .../linux-packages/debian.ubuntu-trusty/changelog | 6 ++ dev/tasks/linux-packages/debian/changelog | 6 ++ dev/tasks/linux-packages/yum/arrow.spec.in | 3 +++ java/adapter/jdbc/pom.xml | 2 +- java/flight/pom.xml| 2 +- java/format/pom.xml| 2 +- java/memory/pom.xml| 2 +- java/plasma/pom.xml| 2 +- java/pom.xml | 2 +- java/tools/pom.xml | 2 +- java/vector/pom.xml| 2 +- matlab/CMakeLists.txt | 2 +- r/DESCRIPTION | 2 +- ruby/red-arrow-cuda/lib/arrow-cuda/version.rb | 2 +- ruby/red-arrow/lib/arrow/version.rb| 2 +- ruby/red-gandiva/lib/gandiva/version.rb| 2 +- ruby/red-parquet/lib/parquet/version.rb| 2 +- ruby/red-plasma/lib/plasma/version.rb | 2 +- rust/arrow/Cargo.toml | 2 +- rust/parquet/Cargo.toml| 2 +- 24 files changed, 59 insertions(+), 20 deletions(-)
[arrow] branch master updated: ARROW-4654: [C++] Explicit flight.cc source dependencies
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 25b566c ARROW-4654: [C++] Explicit flight.cc source dependencies
25b566c is described below
commit 25b566cbbc9eabbac63cb41c8240aaa51a3c0c98
Author: François Saint-Jacques
AuthorDate: Fri Feb 22 17:17:22 2019 +0100
ARROW-4654: [C++] Explicit flight.cc source dependencies
When building protobuf from sources and using ninja, python/flight.cc
would build before the generated sources. This explicit a dependency
between the generated files.
Author: François Saint-Jacques
Closes #3729 from fsaintjacques/ARROW-4654-fix-flight-cmake and squashes
the following commits:
304cb8eb ARROW-4654: Explicit flight.cc source
dependencies
---
cpp/src/arrow/python/CMakeLists.txt | 1 +
1 file changed, 1 insertion(+)
diff --git a/cpp/src/arrow/python/CMakeLists.txt
b/cpp/src/arrow/python/CMakeLists.txt
index 9cf7eeb..f77545c 100644
--- a/cpp/src/arrow/python/CMakeLists.txt
+++ b/cpp/src/arrow/python/CMakeLists.txt
@@ -46,6 +46,7 @@ set(ARROW_PYTHON_SRCS
if(ARROW_FLIGHT)
set(ARROW_PYTHON_SRCS ${ARROW_PYTHON_SRCS} flight.cc)
+ set_source_files_properties(flight.cc PROPERTIES OBJECT_DEPENDS
flight_grpc_gen)
endif()
if("${COMPILER_FAMILY}" STREQUAL "clang")
[arrow] branch master updated: ARROW-4643: [C++] Force compiler diagnostic colors
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 9814605 ARROW-4643: [C++] Force compiler diagnostic colors
9814605 is described below
commit 9814605c25ab0e8736bae9ccad89c334647da660
Author: François Saint-Jacques
AuthorDate: Fri Feb 22 13:26:09 2019 +0100
ARROW-4643: [C++] Force compiler diagnostic colors
Author: François Saint-Jacques
Closes #3724 from fsaintjacques/cmake-cxx-color and squashes the following
commits:
bdda9cdb Fix guard take 2
2ce859e1 Add gcc version guard
b2acfdb8 ARROW-4643: Force compiler diagnostic
colors
---
cpp/cmake_modules/SetupCxxFlags.cmake | 16
1 file changed, 12 insertions(+), 4 deletions(-)
diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake
b/cpp/cmake_modules/SetupCxxFlags.cmake
index c71bf15..2876d74 100644
--- a/cpp/cmake_modules/SetupCxxFlags.cmake
+++ b/cpp/cmake_modules/SetupCxxFlags.cmake
@@ -207,10 +207,16 @@ if ("${COMPILER_FAMILY}" STREQUAL "msvc")
set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /wd4996")
endif()
-if ("${COMPILER_FAMILY}" STREQUAL "gcc" AND
-"${COMPILER_VERSION}" VERSION_GREATER "6.0")
- # Without this, gcc >= 7 warns related to changes in C++17
- set(CXX_ONLY_FLAGS "${CXX_ONLY_FLAGS} -Wno-noexcept-type")
+if ("${COMPILER_FAMILY}" STREQUAL "gcc")
+ if ("${COMPILER_VERSION}" VERSION_GREATER "6.0")
+# Without this, gcc >= 7 warns related to changes in C++17
+set(CXX_ONLY_FLAGS "${CXX_ONLY_FLAGS} -Wno-noexcept-type")
+ endif()
+
+ if ("${COMPILER_VERSION}" VERSION_GREATER "4.9")
+# Add colors when paired with ninja
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-color=always")
+ endif()
endif()
# Clang options for all builds
@@ -225,6 +231,8 @@ if ("${COMPILER_FAMILY}" STREQUAL "clang")
# Avoid clang error when an unknown warning flag is passed
set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-unknown-warning-option")
+ # Add colors when paired with ninja
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fcolor-diagnostics")
endif()
# if build warning flags is set, add to CXX_COMMON_FLAGS
svn commit: r32580 - in /dev/arrow: 0.12.1-rc0/ apache-arrow-0.12.1-rc0/
Author: uwe Date: Thu Feb 21 16:26:58 2019 New Revision: 32580 Log: Apache Arrow 0.12.1 RC0 Added: dev/arrow/apache-arrow-0.12.1-rc0/ - copied from r32579, dev/arrow/0.12.1-rc0/ Removed: dev/arrow/0.12.1-rc0/
svn commit: r32579 - in /dev/arrow/0.12.1-rc0: ./ apache-arrow-0.12.1.tar.gz apache-arrow-0.12.1.tar.gz.asc apache-arrow-0.12.1.tar.gz.sha256 apache-arrow-0.12.1.tar.gz.sha512
Author: uwe Date: Thu Feb 21 16:00:35 2019 New Revision: 32579 Log: Apache Arrow 0.12.1 RC0 Added: dev/arrow/0.12.1-rc0/ dev/arrow/0.12.1-rc0/apache-arrow-0.12.1.tar.gz (with props) dev/arrow/0.12.1-rc0/apache-arrow-0.12.1.tar.gz.asc dev/arrow/0.12.1-rc0/apache-arrow-0.12.1.tar.gz.sha256 dev/arrow/0.12.1-rc0/apache-arrow-0.12.1.tar.gz.sha512 Added: dev/arrow/0.12.1-rc0/apache-arrow-0.12.1.tar.gz == Binary file - no diff available. Propchange: dev/arrow/0.12.1-rc0/apache-arrow-0.12.1.tar.gz -- svn:mime-type = application/octet-stream Added: dev/arrow/0.12.1-rc0/apache-arrow-0.12.1.tar.gz.asc == --- dev/arrow/0.12.1-rc0/apache-arrow-0.12.1.tar.gz.asc (added) +++ dev/arrow/0.12.1-rc0/apache-arrow-0.12.1.tar.gz.asc Thu Feb 21 16:00:35 2019 @@ -0,0 +1,17 @@ +-BEGIN PGP SIGNATURE- +Version: GnuPG v1 + +iQIcBAABAgAGBQJcbsq1AAoJECnZTiKMqtYCOkgQAJV7rX9aHs3hNnzNPhUtywi7 +LMg9wjfIpF/7pHr/hnQLB1PsZjTtyepZCqZ+YOGxjG0ZkUgTCcvsl4Lj866LShuw +14OxX1KW6H21zVUXnOrMGEYf1V8KFZRmN6aHzGUOzy51Q4+hc9fiGKvb0qgZFU0l +5Qw1MryZ22i/7ryiORdIlvof8y7BbiRvUQ9041aM/00JyLcKfbKyrGdld5In8ohN +roZLhmYq52iNGhIkvJ9tXGVl+zg2bz+2Bx5ex1j0b0mEVCpDUcgkN3VT0qAXAwJs +Y382GQpSdRm1JzERduZwTUha/CS6KwvgKUhK1xh3cSYJPAgshjaIY6zgvUdYC3kR +5ICn1FykngjlcVLHMqtFHyuCA5stqpIOCplqUclsy6LHgl8W/ZikOVvSIrTFt4Oc +auDhVc7IVtYjlxd24uOgjRgM/Pu3iL68PVZoZcbWUaPPPpmBXC6xl9EGYLx8Jfhn +rSoSKwFGI0ZJPdzNoZGQqlsoiZw8gw3Z8n1pX24ZMchbkJ/NJfqu8uXLVTjaJjXm +twiQGsZ9iw4Bo1jdfCQ+gy+4m9g+ix0WFxOXFIi6rrnZ6u1K3sGYPpY5hpooKS7u +HDDVPIRyU4H1RkoV6IoHqgMcldeOCXOLlnO0D1LWPbjDnC0zUULkVqNUF9IrN1tN +RrQ93LWtBUXIoYHeUYST +=/XOB +-END PGP SIGNATURE- Added: dev/arrow/0.12.1-rc0/apache-arrow-0.12.1.tar.gz.sha256 == --- dev/arrow/0.12.1-rc0/apache-arrow-0.12.1.tar.gz.sha256 (added) +++ dev/arrow/0.12.1-rc0/apache-arrow-0.12.1.tar.gz.sha256 Thu Feb 21 16:00:35 2019 @@ -0,0 +1 @@ +e93e43343544e344bbc912b89da01d8abf66596f029d26b2b135b102a9e39895 apache-arrow-0.12.1.tar.gz Added: dev/arrow/0.12.1-rc0/apache-arrow-0.12.1.tar.gz.sha512 == --- dev/arrow/0.12.1-rc0/apache-arrow-0.12.1.tar.gz.sha512 (added) +++ dev/arrow/0.12.1-rc0/apache-arrow-0.12.1.tar.gz.sha512 Thu Feb 21 16:00:35 2019 @@ -0,0 +1 @@ +45a07daae97e956a64f6a6cbc7a4339245d2bf6c4efcd7457c4eed4900f0a62e9433add5c1a01b734788fdec804b5ce49bdc6d309e7e397bae640bc379510fa1 apache-arrow-0.12.1.tar.gz
[arrow] annotated tag apache-arrow-0.12.1 updated (ba09a9e -> a5b0d36)
This is an automated email from the ASF dual-hosted git repository. uwe pushed a change to annotated tag apache-arrow-0.12.1 in repository https://gitbox.apache.org/repos/asf/arrow.git. *** WARNING: tag apache-arrow-0.12.1 was modified! *** from ba09a9e (commit) to a5b0d36 (tag) tagging ba09a9e93dc28da629f63e101e231c8b8df942d3 (commit) replaces apache-arrow-0.12.0 by Uwe L. Korn on Thu Feb 21 14:17:46 2019 +0100 - Log - [maven-release-plugin] copy for tag apache-arrow-0.12.1 --- No new revisions were added by this update. Summary of changes:
[arrow] branch master updated: ARROW-4629: [Python] Pandas arrow conversion slowed down by imports
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 957fe15 ARROW-4629: [Python] Pandas arrow conversion slowed down by
imports
957fe15 is described below
commit 957fe15bb3dafadda4d0a6f9359e4f40107440bb
Author: fjetter
AuthorDate: Wed Feb 20 15:20:35 2019 +0100
ARROW-4629: [Python] Pandas arrow conversion slowed down by imports
The local imports slow down the conversion from pandas to arrow
significantly (see [here](https://issues.apache.org/jira/browse/ARROW-4629))
Author: fjetter
Author: Uwe L. Korn
Closes #3706 from fjetter/local_imports and squashes the following commits:
eb5c8bad Apply suggestions from code review
b4604bec Only import pandas_compat if pandas is available
f1c8b401 Don't use local imports
---
python/pyarrow/array.pxi| 11 ---
python/pyarrow/pandas_compat.py | 6 +-
2 files changed, 9 insertions(+), 8 deletions(-)
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index cc77798..324c140 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -15,6 +15,11 @@
# specific language governing permissions and limitations
# under the License.
+from pyarrow.compat import HAVE_PANDAS
+
+if HAVE_PANDAS:
+import pyarrow.pandas_compat as pdcompat
+
cdef _sequence_to_array(object sequence, object mask, object size,
DataType type, CMemoryPool* pool, c_bool from_pandas):
@@ -165,9 +170,9 @@ def array(object obj, type=None, mask=None, size=None, bint
from_pandas=False,
from_pandas=True, safe=safe,
memory_pool=memory_pool)
else:
-import pyarrow.pandas_compat as pdcompat
-values, type = pdcompat.get_datetimetz_type(values, obj.dtype,
-type)
+if HAVE_PANDAS:
+values, type = pdcompat.get_datetimetz_type(
+values, obj.dtype, type)
return _ndarray_to_array(values, mask, type, from_pandas, safe,
pool)
else:
diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index 403f15d..86e826f 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -28,7 +28,7 @@ import pandas as pd
import six
import pyarrow as pa
-from pyarrow.compat import builtin_pickle, PY2, zip_longest # noqa
+from pyarrow.compat import builtin_pickle, DatetimeTZDtype, PY2, zip_longest
# noqa
def infer_dtype(column):
@@ -447,8 +447,6 @@ def dataframe_to_arrays(df, schema, preserve_index,
nthreads=1, columns=None,
def get_datetimetz_type(values, dtype, type_):
-from pyarrow.compat import DatetimeTZDtype
-
if values.dtype.type != np.datetime64:
return values, type_
@@ -543,7 +541,6 @@ def _reconstruct_block(item):
def _make_datetimetz(tz):
-from pyarrow.compat import DatetimeTZDtype
tz = pa.lib.string_to_tzinfo(tz)
return DatetimeTZDtype('ns', tz=tz)
@@ -554,7 +551,6 @@ def _make_datetimetz(tz):
def table_to_blockmanager(options, table, categories=None,
ignore_metadata=False):
-from pyarrow.compat import DatetimeTZDtype
index_columns = []
columns = []
[arrow] branch master updated: ARROW-4265: [C++] Automatic conversion between Table and std::vector>
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 240c469 ARROW-4265: [C++] Automatic conversion between Table and
std::vector>
240c469 is described below
commit 240c46959ac631d646d98ac57aaa719a63a9dc97
Author: Korn, Uwe
AuthorDate: Mon Feb 18 16:04:17 2019 +0100
ARROW-4265: [C++] Automatic conversion between Table and
std::vector>
This enables conversions between a `std::vector>` like and
`arrow::Table`.
tuple to Table:
```cpp
std::vector> rows = ..
std::shared_ptr table;
if (!arrow::stl::TableFromTupleRange(
arrow::default_memory_pool(),
rows, names, &table).ok()
) {
// Error handling code should go here.
}
```
Table to tuple:
```cpp
// An important aspect here is that the table columns need to be in the
// same order as the columns will later appear in the tuple. As the tuple
// is unnamed, matching is done on positions.
std::shared_ptr table = ..
// The range needs to be pre-allocated to the respective amount of rows.
// This allows us to pass in an arbitrary range object, not only
// `std::vector`.
std::vector> rows(2);
if (!arrow::stl::TupleRangeFromTable(*table, &rows).ok()) {
// Error handling code should go here.
}
```
Author: Korn, Uwe
Author: Uwe L. Korn
Closes #3404 from xhochy/stl-extension and squashes the following commits:
4856260a Cast to size_t to compare on equal signedness
aaeacfd1 docker-compose run clang-format
386e5bc9 Check size of target
8b472da8 Update documentation
1a3743e0 Allow building shared libs without tests
9a08a3eb Use full path to checked_cast
e037507c Use ArrayFromJSON
1ab23f87 Move to type_singleton
30e66f9e Add additional STL conversions
---
cpp/CMakeLists.txt | 2 +-
cpp/src/arrow/stl-test.cc | 169 --
cpp/src/arrow/stl.h| 255 -
cpp/src/arrow/type.cc | 12 +
cpp/src/arrow/type.h | 5 +
dev/lint/Dockerfile| 3 +-
docs/source/cpp/{ => examples}/index.rst | 17 +-
.../row_columnar_conversion.rst} | 7 +-
.../source/cpp/examples/tuple_range_conversion.rst | 106 +
docs/source/cpp/index.rst | 2 +-
10 files changed, 541 insertions(+), 37 deletions(-)
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 7d1fee0..ab9ff39 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -812,7 +812,7 @@ pass ARROW_BUILD_SHARED=on")
# Use shared linking for unit tests if it's available
set(ARROW_TEST_LINK_LIBS ${ARROW_TEST_SHARED_LINK_LIBS})
set(ARROW_EXAMPLE_LINK_LIBS arrow_shared)
-else()
+elseif(ARROW_BUILD_TESTS)
if(NOT ARROW_BUILD_STATIC)
message(FATAL_ERROR "If using static linkage for unit tests, must also \
pass ARROW_BUILD_STATIC=on")
diff --git a/cpp/src/arrow/stl-test.cc b/cpp/src/arrow/stl-test.cc
index 6c33098..04c4cf4 100644
--- a/cpp/src/arrow/stl-test.cc
+++ b/cpp/src/arrow/stl-test.cc
@@ -23,8 +23,13 @@
#include
#include "arrow/stl.h"
+#include "arrow/table.h"
+#include "arrow/testing/gtest_util.h"
#include "arrow/type.h"
+using primitive_types_tuple = std::tuple;
+
namespace arrow {
namespace stl {
@@ -36,12 +41,9 @@ TEST(TestSchemaFromTuple, PrimitiveTypesVector) {
field("column7", uint32(), false), field("column8", uint64(), false),
field("column9", boolean(), false), field("column10", utf8(), false)});
- std::shared_ptr schema =
- SchemaFromTuple>::
- MakeSchema(std::vector({"column1", "column2",
"column3", "column4",
- "column5", "column6",
"column7", "column8",
- "column9", "column10"}));
+ std::shared_ptr schema =
SchemaFromTuple::MakeSchema(
+ std::vector({"column1", "column2", "column3", "column4",
"column5",
+"column6", "column7", "column8", "column9",
"column10"}));
ASSERT_TRUE(expected_schema.Equals(*schema));
}
@@ -53,13 +55,9 @@ TEST(TestSchemaFromTuple, PrimitiveTypesTuple) {
field("column7", uint32(), false), field("column8", uint64(), false),
field("col
[arrow] branch master updated: ARROW-4601: [Python] Add license header to dockerignore
This is an automated email from the ASF dual-hosted git repository. uwe pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git The following commit(s) were added to refs/heads/master by this push: new 5e2445b ARROW-4601: [Python] Add license header to dockerignore 5e2445b is described below commit 5e2445b63d46b85a9a1dadeceee5e6159e10cee3 Author: Uwe L. Korn AuthorDate: Sun Feb 17 19:46:16 2019 +0100 ARROW-4601: [Python] Add license header to dockerignore Author: Uwe L. Korn Closes #3679 from xhochy/ARROW-4601 and squashes the following commits: 7b5544fd ARROW-4601: Add license header to dockerignore --- python/manylinux1/.dockerignore | 17 + 1 file changed, 17 insertions(+) diff --git a/python/manylinux1/.dockerignore b/python/manylinux1/.dockerignore index 1521c8b..e1cab17 100644 --- a/python/manylinux1/.dockerignore +++ b/python/manylinux1/.dockerignore @@ -1 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + dist
[arrow] branch master updated: ARROW-4577: [C++] Don't set interface link libs on arrow_shared where there are none
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 524a9dc ARROW-4577: [C++] Don't set interface link libs on
arrow_shared where there are none
524a9dc is described below
commit 524a9dc79556a15d1f56c094913f0e0682358844
Author: Uwe L. Korn
AuthorDate: Fri Feb 15 18:25:15 2019 +0100
ARROW-4577: [C++] Don't set interface link libs on arrow_shared where there
are none
Author: Uwe L. Korn
Closes #3652 from xhochy/ARROW-4577 and squashes the following commits:
e9d6b14c ARROW-4577: Don't set interface link libs on
arrow_shared where there are none
---
cpp/CMakeLists.txt | 1 -
cpp/cmake_modules/BuildUtils.cmake | 8 +---
2 files changed, 1 insertion(+), 8 deletions(-)
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 17582c9..7d1fee0 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -844,7 +844,6 @@ endif()
list(APPEND ARROW_LINK_LIBS ${ARROW_SYSTEM_LINK_LIBS})
list(APPEND ARROW_STATIC_LINK_LIBS ${ARROW_SYSTEM_LINK_LIBS})
-list(APPEND ARROW_SHARED_INSTALL_INTERFACE_LIBS ${ARROW_SYSTEM_LINK_LIBS})
list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${ARROW_SYSTEM_LINK_LIBS})
#
diff --git a/cpp/cmake_modules/BuildUtils.cmake
b/cpp/cmake_modules/BuildUtils.cmake
index 1591d86..5a0f2b8 100644
--- a/cpp/cmake_modules/BuildUtils.cmake
+++ b/cpp/cmake_modules/BuildUtils.cmake
@@ -206,16 +206,10 @@ function(ADD_ARROW_LIB LIB_NAME)
VERSION "${ARROW_FULL_SO_VERSION}"
SOVERSION "${ARROW_SO_VERSION}")
-if (ARG_SHARED_INSTALL_INTERFACE_LIBS)
- set(INTERFACE_LIBS ${ARG_SHARED_INSTALL_INTERFACE_LIBS})
-else()
- set(INTERFACE_LIBS ${ARG_SHARED_LINK_LIBS})
-endif()
-
target_link_libraries(${LIB_NAME}_shared
LINK_PUBLIC
"$"
- "$"
+ "$"
LINK_PRIVATE ${ARG_SHARED_PRIVATE_LINK_LIBS})
if (ARROW_RPATH_ORIGIN)
[arrow] branch master updated: ARROW-4532: [Java] fix bug causing very large varchar value buffers
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 61b2926 ARROW-4532: [Java] fix bug causing very large varchar value
buffers
61b2926 is described below
commit 61b2926fdbd07dae3bfefae6aabdb2dc4a18f48c
Author: Pindikura Ravindra
AuthorDate: Tue Feb 12 15:35:56 2019 +0100
ARROW-4532: [Java] fix bug causing very large varchar value buffers
The varchar/varbinary vectors have a setSafe variant that takes an arrow
buf and a start/end offsets. The method needs to expand the buffer to make
space for 'end - start' bytes. but, due to a bug, it was expanding it to
accommodate 'end' bytes, thereby making the value buffer much larger than
required.
Author: Pindikura Ravindra
Closes #3613 from pravindra/varchar and squashes the following commits:
1b4f2243 ARROW-4532: fix review comments
8f88e3aa ARROW-4532: fix a size compute bug
---
.../arrow/vector/BaseVariableWidthVector.java | 2 +-
.../org/apache/arrow/vector/TestValueVector.java | 45 ++
2 files changed, 46 insertions(+), 1 deletion(-)
diff --git
a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
index ac148a2..a5b550a 100644
---
a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
+++
b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
@@ -1100,7 +1100,7 @@ public abstract class BaseVariableWidthVector extends
BaseValueVector
assert index >= 0;
final int dataLength = end - start;
fillEmpties(index);
-handleSafe(index, end);
+handleSafe(index, dataLength);
BitVectorHelper.setValidityBit(validityBuffer, index, isSet);
final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH);
offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength);
diff --git
a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index 30fe23c..f0cc4c4 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -1395,6 +1395,51 @@ public class TestValueVector {
}
@Test
+ public void testSetSafeWithArrowBufNoExcessAllocs() {
+final int numValues = BaseFixedWidthVector.INITIAL_VALUE_ALLOCATION * 2;
+final byte[] valueBytes = "hello world".getBytes();
+final int valueBytesLength = valueBytes.length;
+final int isSet = 1;
+
+try (
+final VarCharVector fromVector = newVector(VarCharVector.class,
EMPTY_SCHEMA_PATH,
+MinorType.VARCHAR, allocator);
+final VarCharVector toVector = newVector(VarCharVector.class,
EMPTY_SCHEMA_PATH,
+MinorType.VARCHAR, allocator)) {
+ /*
+ * Populate the from vector with 'numValues' with byte-arrays, each of
size 'valueBytesLength'.
+ */
+ fromVector.setInitialCapacity(numValues);
+ fromVector.allocateNew();
+ for (int i = 0; i < numValues; ++i) {
+fromVector.setSafe(i, valueBytes, 0 /*start*/, valueBytesLength);
+ }
+ fromVector.setValueCount(numValues);
+ ArrowBuf fromDataBuffer = fromVector.getDataBuffer();
+ assertTrue(numValues * valueBytesLength <= fromDataBuffer.capacity());
+
+ /*
+ * Copy the entries one-by-one from 'fromVector' to 'toVector', but use
the setSafe with
+ * ArrowBuf API (instead of setSafe with byte-array).
+ */
+ toVector.setInitialCapacity(numValues);
+ toVector.allocateNew();
+ for (int i = 0; i < numValues; i++) {
+int start = fromVector.getstartOffset(i);
+int end = fromVector.getstartOffset(i + 1);
+toVector.setSafe(i, isSet, start, end, fromDataBuffer);
+ }
+
+ /*
+ * Since the 'fromVector' and 'toVector' have the same initial capacity,
and were populated
+ * with the same varchar elements, the allocations and hence, the final
capacity should be
+ * the same.
+ */
+ assertEquals(fromDataBuffer.capacity(),
toVector.getDataBuffer().capacity());
+}
+ }
+
+ @Test
public void testCopyFromWithNulls() {
try (final VarCharVector vector = newVector(VarCharVector.class,
EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator);
final VarCharVector vector2 =
[arrow] branch master updated: ARROW-4181: [Python] Fixes for Numpy struct array conversion
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new a5d8ccc ARROW-4181: [Python] Fixes for Numpy struct array conversion
a5d8ccc is described below
commit a5d8ccc3fc7671986c38cfe3558b77e0fe157053
Author: Antoine Pitrou
AuthorDate: Tue Feb 12 15:32:35 2019 +0100
ARROW-4181: [Python] Fixes for Numpy struct array conversion
This fixes two issues:
- object fields inside numpy structs should be allowed
- buggy ndarray indexing if stride % itemsize != 0
Author: Antoine Pitrou
Closes #3614 from pitrou/ARROW-4181-py-struct-conversion-fixes and squashes
the following commits:
9886e29e ARROW-4181: Fixes for Numpy struct array
conversion
---
cpp/src/arrow/python/numpy-internal.h | 20 +++-
cpp/src/arrow/python/numpy_to_arrow.cc | 23 ---
python/pyarrow/tests/test_convert_pandas.py | 18 +-
3 files changed, 36 insertions(+), 25 deletions(-)
diff --git a/cpp/src/arrow/python/numpy-internal.h
b/cpp/src/arrow/python/numpy-internal.h
index 6954e35..e27ae5c 100644
--- a/cpp/src/arrow/python/numpy-internal.h
+++ b/cpp/src/arrow/python/numpy-internal.h
@@ -40,14 +40,12 @@ class Ndarray1DIndexer {
Ndarray1DIndexer() : arr_(NULLPTR), data_(NULLPTR) {}
- explicit Ndarray1DIndexer(PyArrayObject* arr) : Ndarray1DIndexer() {
Init(arr); }
-
- void Init(PyArrayObject* arr) {
+ explicit Ndarray1DIndexer(PyArrayObject* arr) : Ndarray1DIndexer() {
arr_ = arr;
DCHECK_EQ(1, PyArray_NDIM(arr)) << "Only works with 1-dimensional arrays";
Py_INCREF(arr);
-data_ = reinterpret_cast(PyArray_DATA(arr));
-stride_ = PyArray_STRIDES(arr)[0] / sizeof(T);
+data_ = reinterpret_cast(PyArray_DATA(arr));
+stride_ = PyArray_STRIDES(arr)[0];
}
~Ndarray1DIndexer() { Py_XDECREF(arr_); }
@@ -56,14 +54,18 @@ class Ndarray1DIndexer {
T* data() const { return data_; }
- bool is_strided() const { return stride_ != 1; }
+ bool is_strided() const { return stride_ != sizeof(T); }
- T& operator[](size_type index) { return data_[index * stride_]; }
- T& operator[](size_type index) const { return data_[index * stride_]; }
+ T& operator[](size_type index) {
+return *reinterpret_cast(data_ + index * stride_);
+ }
+ const T& operator[](size_type index) const {
+return *reinterpret_cast(data_ + index * stride_);
+ }
private:
PyArrayObject* arr_;
- T* data_;
+ uint8_t* data_;
int64_t stride_;
};
diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc
b/cpp/src/arrow/python/numpy_to_arrow.cc
index a944b80..ef63ccf 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -318,8 +318,18 @@ Status NumPyConverter::Convert() {
return Status::Invalid("only handle 1-dimensional arrays");
}
- DCHECK_NE(dtype_->type_num, NPY_OBJECT)
- << "This class does not handle NPY_OBJECT arrays";
+ if (dtype_->type_num == NPY_OBJECT) {
+// If an object array, convert it like a normal Python sequence
+PyConversionOptions py_options;
+py_options.type = type_;
+py_options.from_pandas = from_pandas_;
+std::shared_ptr res;
+RETURN_NOT_OK(ConvertPySequence(reinterpret_cast(arr_),
+reinterpret_cast(mask_),
py_options,
+&res));
+out_arrays_ = res->chunks();
+return Status::OK();
+ }
if (type_ == nullptr) {
return Status::Invalid("Must pass data type for non-object arrays");
@@ -790,15 +800,6 @@ Status NdarrayToArrow(MemoryPool* pool, PyObject* ao,
PyObject* mo, bool from_pa
return Status::Invalid("Input object was not a NumPy array");
}
- PyArrayObject* arr = reinterpret_cast(ao);
-
- if (PyArray_DESCR(arr)->type_num == NPY_OBJECT) {
-PyConversionOptions py_options;
-py_options.type = type;
-py_options.from_pandas = from_pandas;
-return ConvertPySequence(ao, mo, py_options, out);
- }
-
NumPyConverter converter(pool, ao, mo, type, from_pandas, cast_options);
RETURN_NOT_OK(converter.Convert());
const auto& output_arrays = converter.result();
diff --git a/python/pyarrow/tests/test_convert_pandas.py
b/python/pyarrow/tests/test_convert_pandas.py
index 8f2c2eb..e904486 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -1851,21 +1851,29 @@ class TestConvertStructTypes(object):
assert arr.to_pylist() == [{}, {}]
def test_from_numpy_nested(self):
+# Note: an object field inside a struct
dt = np.dtype([('x', np.dtype([('xx', np.int8),
('yy
[arrow] branch master updated: ARROW-3653: [C++][Python] Support data copying between different GPU devices
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 41e3bff ARROW-3653: [C++][Python] Support data copying between
different GPU devices
41e3bff is described below
commit 41e3bff035c8f44ff3cb7794b68ebb4d08cb641c
Author: Pearu Peterson
AuthorDate: Tue Feb 12 15:30:22 2019 +0100
ARROW-3653: [C++][Python] Support data copying between different GPU devices
This PR introduces new C++ methods
`CudaContext::CopyDeviceToAnotherDevice`
`CudaBuffer::CopyFromAnotherDevice`
that are used to implement the Python support for copying CUDA buffer data
between different devices.
Author: Pearu Peterson
Closes #3617 from pearu/arrow-3653 and squashes the following commits:
4de6c973 Skip tests on copying data between buffers of
different devices on a single GPU system.
b10afc34 Support copying data in buffers of different
devices.
---
cpp/src/arrow/gpu/cuda_context.cc | 16
cpp/src/arrow/gpu/cuda_context.h | 2 ++
cpp/src/arrow/gpu/cuda_memory.cc | 8
cpp/src/arrow/gpu/cuda_memory.h | 11 ++-
python/pyarrow/_cuda.pyx | 23 +--
python/pyarrow/includes/libarrow_cuda.pxd | 3 +++
python/pyarrow/tests/test_cuda.py | 14 --
7 files changed, 64 insertions(+), 13 deletions(-)
diff --git a/cpp/src/arrow/gpu/cuda_context.cc
b/cpp/src/arrow/gpu/cuda_context.cc
index 2f3f1bd..93caa52 100644
--- a/cpp/src/arrow/gpu/cuda_context.cc
+++ b/cpp/src/arrow/gpu/cuda_context.cc
@@ -112,6 +112,16 @@ class CudaContext::CudaContextImpl {
return Status::OK();
}
+ Status CopyDeviceToAnotherDevice(const std::shared_ptr&
dst_ctx, void* dst,
+ const void* src, int64_t nbytes) {
+ContextSaver set_temporary(context_);
+CU_RETURN_NOT_OK(cuMemcpyPeer(reinterpret_cast(dst),
+
reinterpret_cast(dst_ctx->handle()),
+ reinterpret_cast(src),
context_,
+ static_cast(nbytes)));
+return Status::OK();
+ }
+
Status Synchronize(void) {
ContextSaver set_temporary(context_);
CU_RETURN_NOT_OK(cuCtxSynchronize());
@@ -301,6 +311,12 @@ Status CudaContext::CopyDeviceToDevice(void* dst, const
void* src, int64_t nbyte
return impl_->CopyDeviceToDevice(dst, src, nbytes);
}
+Status CudaContext::CopyDeviceToAnotherDevice(const
std::shared_ptr& dst_ctx,
+ void* dst, const void* src,
+ int64_t nbytes) {
+ return impl_->CopyDeviceToAnotherDevice(dst_ctx, dst, src, nbytes);
+}
+
Status CudaContext::Synchronize(void) { return impl_->Synchronize(); }
Status CudaContext::Close() { return impl_->Close(); }
diff --git a/cpp/src/arrow/gpu/cuda_context.h b/cpp/src/arrow/gpu/cuda_context.h
index 938a815..682cbd8 100644
--- a/cpp/src/arrow/gpu/cuda_context.h
+++ b/cpp/src/arrow/gpu/cuda_context.h
@@ -141,6 +141,8 @@ class ARROW_EXPORT CudaContext : public
std::enable_shared_from_this&
dst_ctx, void* dst,
+ const void* src, int64_t nbytes);
Status Free(void* device_ptr, int64_t nbytes);
class CudaContextImpl;
diff --git a/cpp/src/arrow/gpu/cuda_memory.cc b/cpp/src/arrow/gpu/cuda_memory.cc
index a0da580..e91fef2 100644
--- a/cpp/src/arrow/gpu/cuda_memory.cc
+++ b/cpp/src/arrow/gpu/cuda_memory.cc
@@ -163,6 +163,14 @@ Status CudaBuffer::CopyFromDevice(const int64_t position,
const void* data,
return context_->CopyDeviceToDevice(mutable_data_ + position, data, nbytes);
}
+Status CudaBuffer::CopyFromAnotherDevice(const std::shared_ptr&
src_ctx,
+ const int64_t position, const void*
data,
+ int64_t nbytes) {
+ DCHECK_LE(nbytes, size_ - position) << "Copy would overflow buffer";
+ return src_ctx->CopyDeviceToAnotherDevice(context_, mutable_data_ +
position, data,
+nbytes);
+}
+
Status CudaBuffer::ExportForIpc(std::shared_ptr* handle) {
if (is_ipc_) {
return Status::Invalid("Buffer has already been exported for IPC");
diff --git a/cpp/src/arrow/gpu/cuda_memory.h b/cpp/src/arrow/gpu/cuda_memory.h
index 193deed..6b9f04c 100644
--- a/cpp/src/arrow/gpu/cuda_memory.h
+++ b/cpp/src/arrow/gpu/cuda_memory.h
@@ -64,7 +64,7 @@ class ARROW_EXPORT CudaBuffer : public Buffer {
Status CopyToHost(const int64_t position, const int64_t nbytes, void* out)
const;
/// \brief Copy memory to device at position
- /// \param[in] position start position to copy bytes
+ /// \param[in] position start positio
[arrow] branch master updated: ARROW-4535: [C++] Fix MakeBuilder to preserve ListType's field name
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 35b2871 ARROW-4535: [C++] Fix MakeBuilder to preserve ListType's
field name
35b2871 is described below
commit 35b28719fbf2fe7bea47f17ee93bd771020bd9ce
Author: Kenta Murata
AuthorDate: Tue Feb 12 15:28:56 2019 +0100
ARROW-4535: [C++] Fix MakeBuilder to preserve ListType's field name
MakeBuilder doesn't preserve the field name in the given ListType.
I think this is a bug.
Author: Kenta Murata
Closes #3619 from mrkn/fix_make_builder_to_preserve_field_name_of_list_type
and squashes the following commits:
e4d85978 Fix c_glib's test
8fc747a6 Fix MakeBuilder to preserve ListType's field name
---
c_glib/test/helper/buildable.rb | 4 ++--
c_glib/test/test-list-array.rb | 2 +-
c_glib/test/test-orc-file-reader.rb | 4 ++--
cpp/src/arrow/array-list-test.cc| 17 +
cpp/src/arrow/builder.cc| 2 +-
5 files changed, 23 insertions(+), 6 deletions(-)
diff --git a/c_glib/test/helper/buildable.rb b/c_glib/test/helper/buildable.rb
index f3ae709..a9c514c 100644
--- a/c_glib/test/helper/buildable.rb
+++ b/c_glib/test/helper/buildable.rb
@@ -101,8 +101,8 @@ module Helper
build_array(Arrow::StringArrayBuilder.new, values)
end
-def build_list_array(value_data_type, values_list)
- value_field = Arrow::Field.new("value", value_data_type)
+def build_list_array(value_data_type, values_list, field_name: "value")
+ value_field = Arrow::Field.new(field_name, value_data_type)
data_type = Arrow::ListDataType.new(value_field)
builder = Arrow::ListArrayBuilder.new(data_type)
values_list.each do |values|
diff --git a/c_glib/test/test-list-array.rb b/c_glib/test/test-list-array.rb
index abbd004..eea16cc 100644
--- a/c_glib/test/test-list-array.rb
+++ b/c_glib/test/test-list-array.rb
@@ -19,7 +19,7 @@ class TestListArray < Test::Unit::TestCase
include Helper::Buildable
def test_new
-field = Arrow::Field.new("item", Arrow::Int8DataType.new)
+field = Arrow::Field.new("value", Arrow::Int8DataType.new)
data_type = Arrow::ListDataType.new(field)
value_offsets = Arrow::Buffer.new([0, 2, 5, 5].pack("l*"))
data = Arrow::Buffer.new([1, 2, 3, 4, 5].pack("c*"))
diff --git a/c_glib/test/test-orc-file-reader.rb
b/c_glib/test/test-orc-file-reader.rb
index 544112c..8b4f08f 100644
--- a/c_glib/test/test-orc-file-reader.rb
+++ b/c_glib/test/test-orc-file-reader.rb
@@ -64,7 +64,7 @@ map: list>
end
def build_items_array(items_array)
-build_list_array(item_data_type, items_array)
+build_list_array(item_data_type, items_array, field_name: "item")
end
def items_data_type
@@ -93,7 +93,7 @@ map: list>
end
def build_key_value_array(key_value_array)
-build_list_array(key_value_data_type, key_value_array)
+build_list_array(key_value_data_type, key_value_array, field_name: "item")
end
def middle_array
diff --git a/cpp/src/arrow/array-list-test.cc b/cpp/src/arrow/array-list-test.cc
index e616674..cb84896 100644
--- a/cpp/src/arrow/array-list-test.cc
+++ b/cpp/src/arrow/array-list-test.cc
@@ -296,4 +296,21 @@ TEST_F(TestListArray, TestZeroLength) {
ASSERT_OK(ValidateArray(*result_));
}
+TEST_F(TestListArray, TestBuilderPreserveFieleName) {
+ auto list_type_with_name = list(field("counts", int32()));
+
+ std::unique_ptr tmp;
+ ASSERT_OK(MakeBuilder(pool_, list_type_with_name, &tmp));
+ builder_.reset(checked_cast(tmp.release()));
+
+ vector values = {1, 2, 4, 8};
+ ASSERT_OK(builder_->AppendValues(values.data(), values.size()));
+
+ std::shared_ptr list_array;
+ ASSERT_OK(builder_->Finish(&list_array));
+
+ const auto& type = checked_cast(*list_array->type());
+ ASSERT_EQ("counts", type.value_field()->name());
+}
+
} // namespace arrow
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 2072edc..8d0ab19 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -75,7 +75,7 @@ Status MakeBuilder(MemoryPool* pool, const
std::shared_ptr& type,
std::shared_ptr value_type =
internal::checked_cast(*type).value_type();
RETURN_NOT_OK(MakeBuilder(pool, value_type, &value_builder));
- out->reset(new ListBuilder(pool, std::move(value_builder)));
+ out->reset(new ListBuilder(pool, std::move(value_builder), type));
return Status::OK();
}
[arrow] branch master updated: ARROW-4481: [Website] Remove generated specification docs from site after docs migration
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new b31845d ARROW-4481: [Website] Remove generated specification docs
from site after docs migration
b31845d is described below
commit b31845d23228f99543e6c7e403f311680c7e16c7
Author: Wes McKinney
AuthorDate: Tue Feb 12 15:28:04 2019 +0100
ARROW-4481: [Website] Remove generated specification docs from site after
docs migration
We no longer need to render the format docs as part of the Jekyll site, as
they were ported to the master Sphinx project. As a result of this, the
instructions to build the website were broken
Author: Wes McKinney
Closes #3620 from wesm/ARROW-4481 and squashes the following commits:
013d13bf Remove generated specification docs from site after
docs migration
---
site/README.md | 7 ---
site/_config.yml| 1 -
site/_docs/ipc.md | 25 -
site/_docs/memory_layout.md | 25 -
site/_docs/metadata.md | 25 -
site/_includes/header.html | 12 +++-
6 files changed, 3 insertions(+), 92 deletions(-)
diff --git a/site/README.md b/site/README.md
index 7a287ec..73fd185 100644
--- a/site/README.md
+++ b/site/README.md
@@ -44,13 +44,6 @@ subdirectory of `site`.
git clone --branch=asf-site https://github.com/apache/arrow-site.git asf-site
```
-Also, from the `site` directory, run the following command to import the format
-markdown documents so that they will rendered.
-
-```
-scripts/sync_format_docs.sh
-```
-
## Previewing the site
From the `site` directory, run the following to generate HTML files and run the
diff --git a/site/_config.yml b/site/_config.yml
index b7ec25b..ae420e4 100644
--- a/site/_config.yml
+++ b/site/_config.yml
@@ -30,7 +30,6 @@ include:
exclude:
- Gemfile
- Gemfile.lock
- - _docs/format/*
- ruby
- asf-site
- scripts
diff --git a/site/_docs/ipc.md b/site/_docs/ipc.md
deleted file mode 100644
index bc22dc3..000
--- a/site/_docs/ipc.md
+++ /dev/null
@@ -1,25 +0,0 @@
-layout: docs
-title: Arrow Messaging and IPC
-permalink: /docs/ipc.html
-
-
-{% include_relative format/IPC.md %}
\ No newline at end of file
diff --git a/site/_docs/memory_layout.md b/site/_docs/memory_layout.md
deleted file mode 100644
index 74cd7ed..000
--- a/site/_docs/memory_layout.md
+++ /dev/null
@@ -1,25 +0,0 @@
-layout: docs
-title: Physical Memory Layout
-permalink: /docs/memory_layout.html
-
-
-{% include_relative format/Layout.md %}
\ No newline at end of file
diff --git a/site/_docs/metadata.md b/site/_docs/metadata.md
deleted file mode 100644
index 382ab0e..000
--- a/site/_docs/metadata.md
+++ /dev/null
@@ -1,25 +0,0 @@
-layout: docs
-title: Arrow Metadata
-permalink: /docs/metadata.html
-
-
-{% include_relative format/Metadata.md %}
\ No newline at end of file
diff --git a/site/_includes/header.html b/site/_includes/header.html
index 2abf60f..39a1d05 100644
--- a/site/_includes/header.html
+++ b/site/_includes/header.html
@@ -35,17 +35,11 @@
Powered By
-
-
+
+
Specification
-
-Memory Layout
-Metadata
-Messaging / IPC
-
[arrow] branch master updated: ARROW-4537: [CI] Suppress shell warning on travis-ci
This is an automated email from the ASF dual-hosted git repository. uwe pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git The following commit(s) were added to refs/heads/master by this push: new 000d529 ARROW-4537: [CI] Suppress shell warning on travis-ci 000d529 is described below commit 000d52904aede585714347a630f2d2d1f5f94824 Author: Kenta Murata AuthorDate: Tue Feb 12 15:27:01 2019 +0100 ARROW-4537: [CI] Suppress shell warning on travis-ci Enclose optional shell variable names by double quotations to suppress shell warnings that is like below: ``` +'[' == 1 ']' /home/travis/build/apache/arrow/ci/travis_before_script_cpp.sh: line 81: [: ==: unary operator expected ``` Author: Kenta Murata Closes #3623 from mrkn/suppress_shell_warning_on_ci and squashes the following commits: 4db2e53d Suppress shell warning on travis-ci --- ci/travis_before_script_cpp.sh | 26 +- ci/travis_install_toolchain.sh | 4 ++-- ci/travis_script_python.sh | 6 +++--- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh index 76ae9a6..6c65cd6 100755 --- a/ci/travis_before_script_cpp.sh +++ b/ci/travis_before_script_cpp.sh @@ -78,7 +78,7 @@ fi ARROW_CXXFLAGS="" # Use Ninja for faster builds when using toolchain -if [ $ARROW_TRAVIS_USE_TOOLCHAIN == "1" ]; then +if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -GNinja" if [ "$DISTRO_CODENAME" != "trusty" ]; then # Make sure the toolchain linker (from binutils package) is picked up by clang @@ -86,53 +86,53 @@ if [ $ARROW_TRAVIS_USE_TOOLCHAIN == "1" ]; then fi fi -if [ $ARROW_TRAVIS_PLASMA == "1" ]; then +if [ "$ARROW_TRAVIS_PLASMA" == "1" ]; then CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_PLASMA=ON" fi -if [ $ARROW_TRAVIS_PLASMA_JAVA_CLIENT == "1" ]; then +if [ "$ARROW_TRAVIS_PLASMA_JAVA_CLIENT" == "1" ]; then CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_PLASMA_JAVA_CLIENT=ON" fi -if [ $ARROW_TRAVIS_ORC == "1" ]; then +if [ "$ARROW_TRAVIS_ORC" == "1" ]; then CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_ORC=ON" fi -if [ $ARROW_TRAVIS_PARQUET == "1" ]; then +if [ "$ARROW_TRAVIS_PARQUET" == "1" ]; then CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS \ -DARROW_PARQUET=ON \ -DPARQUET_BUILD_EXAMPLES=ON \ -DPARQUET_BUILD_EXECUTABLES=ON" fi -if [ $ARROW_TRAVIS_GANDIVA == "1" ]; then +if [ "$ARROW_TRAVIS_GANDIVA" == "1" ]; then CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_GANDIVA=ON" - if [ $ARROW_TRAVIS_GANDIVA_JAVA == "1" ]; then + if [ "$ARROW_TRAVIS_GANDIVA_JAVA" == "1" ]; then CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_GANDIVA_JAVA=ON" fi fi -if [ $ARROW_TRAVIS_VALGRIND == "1" ]; then +if [ "$ARROW_TRAVIS_VALGRIND" == "1" ]; then CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_TEST_MEMCHECK=ON" fi -if [ $ARROW_TRAVIS_COVERAGE == "1" ]; then +if [ "$ARROW_TRAVIS_COVERAGE" == "1" ]; then CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_GENERATE_COVERAGE=ON" fi -if [ $ARROW_TRAVIS_VERBOSE == "1" ]; then +if [ "$ARROW_TRAVIS_VERBOSE" == "1" ]; then CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_VERBOSE_THIRDPARTY_BUILD=ON" fi -if [ $ARROW_TRAVIS_VENDORED_BOOST == "1" ]; then +if [ "$ARROW_TRAVIS_VENDORED_BOOST" == "1" ]; then CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_BOOST_VENDORED=ON" fi -if [ $ARROW_TRAVIS_STATIC_BOOST == "1" ]; then +if [ "$ARROW_TRAVIS_STATIC_BOOST" == "1" ]; then CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_BOOST_USE_SHARED=OFF" fi -if [ $ARROW_TRAVIS_OPTIONAL_INSTALL == "1" ]; then +if [ "$ARROW_TRAVIS_OPTIONAL_INSTALL" == "1" ]; then CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_OPTIONAL_INSTALL=ON" fi diff --git a/ci/travis_install_toolchain.sh b/ci/travis_install_toolchain.sh index defc7ec..4d674a1 100755 --- a/ci/travis_install_toolchain.sh +++ b/ci/travis_install_toolchain.sh @@ -25,7 +25,7 @@ if [ ! -e $CPP_TOOLCHAIN ]; then CONDA_PACKAGES="" CONDA_LABEL="" -if [ $ARROW_TRAVIS_GANDIVA == "1" ] && [ $TRAVIS_OS_NAME == "osx" ]; then +if [ "$ARROW_TRAVIS_GANDIVA" == "1" ] && [ $TRAVIS_OS_NAME == &q
[arrow] branch master updated (29f76df -> 2b9155a)
This is an automated email from the ASF dual-hosted git repository. uwe pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git. discard 29f76df ARROW-4446: [C++][Python] Run Gandiva C++ unit tests in Appveyor, get build and tests working in Python discard 27ff2b3 ARROW-1425: [Python][Documentation] Examples of convert Timestamps to/from pandas via arrow discard f3ee51f ARROW-4476: [Rust] [DataFusion] Update README to cover DataFusion and new testing git submodule discard 2f6329f ARROW-4483: [Website] Add myself to contributors.yaml to fix broken link in blog post discard 4131484 ARROW-4500: [C++] Remove pthread / librt hacks causing linking issues in some Linux environments discard 7fd18a1 ARROW-4318: [C++] Add Tensor::CountNonZero discard 86df479 ARROW-3606: [Crossbow] Fix flake8 crossbow warnings discard 886ae72 ARROW-3422: [C++] Uniformly add ExternalProject builds to the "toolchain" target. Fix gRPC EP build on Linux discard e7816b8 ARROW-4496: [Python] Pin to gfortran<4 discard 75f4159 ARROW-4406: [Python] Exclude HDFS directories in S3 from ParquetManifest discard f957b5b ARROW-3903: [Python] Random array generator for Arrow conversion and Parquet testing discard 64c1a65 ARROW-4061: [Rust] [Parquet] Implement spaced version for non-diction… discard 5fc09d4 ARROW-4449: [Rust] Convert File to T: Read + Seek for schema inference discard c81fbaa ARROW-3972: [C++] Migrate to LLVM 7. Add option to disable using ld.gold discard 54182b2 ARROW-3966 [Java] JDBC Column Metadata in Arrow Field Metadata discard 4c680ca ARROW-4488: [Rust] From AsRef<[u8]> for Buffer does not ensure correct padding discard 4004b72 ARROW-3289: [C++] Implement Flight DoPut discard 623deef PARQUET-1525: [C++] remove dependency on getopt in parquet tools discard e533a9e [Website] Edits to Python string blog post discard 9af5a70 ARROW-4472: [Website][Python] Blog post about string memory use work in Arrow 0.12 discard c693dde ARROW-4460: [Website] DataFusion Blog Post discard 7ce2655 PARQUET-1521: [C++] Use pure virtual interfaces for parquet::TypedColumnWriter, remove use of 'extern template class' discard 5f0ff7f ARROW-3239: [C++] Implement simple random array generation discard 927cfef ARROW-4477: [JS] remove constructor override in the bignum mixins discard c93e2ae ARROW-3923: [Java] JDBC Time Fetches Without Timezone discard 0c55b25 ARROW-4294: [C++] [Plasma] Add support for evicting Plasma objects to external store discard a6ae348 ARROW-4475: [Python] Fix recursive serialization of self-containing objects discard 53bf5bf ARROW-4440: [C++] Revert recent changes to flatbuffers EP causing flakiness discard b0ac234 ARROW-4442: [JS] Add explicit type annotation to Chunked typeId getter discard 2e16122 ARROW-4469: [CI] Pin conda-forge binutils version to 2.31 for now discard 170a7ff ARROW-4471: [C++] Pass AR and RANLIB to all external projects discard 29f14ca ARROW-4263: [Rust] Donate DataFusion discard 7f96b6f ARROW-3481: [Java] Fix java building failure with Maven 3.5.4 discard 07ab9cf ARROW-4435: Minor fixups to csharp .sln and .csproj file discard 9f34a41 ARROW-4436: [Documentation] Update building.rst to reflect pyarrow req discard 9bd5294 ARROW-4459: [Testing] Add arrow-testing repo as submodule discard 9b4b471 ARROW-4454: [C++] fix unused parameter warnings discard f08e109 ARROW-4455: [Plasma] Suppress class-memaccess warnings discard fb11322 ARROW-4235: [GLib] Use "column_builder" in GArrowRecordBatchBuilder discard 8ab0212 ARROW-4277: [C++] Add gmock to the toolchain discard caa3a92 ARROW-4397: [C++] Add dim_names in Tensor and SparseTensor discard 76da3cd [Python] Use 'latest' manylinux1 build image discard 48de821 ARROW-3846: [Gandiva][C++] Build Gandiva C++ libraries and get unit tests passing on Windows discard 7d1a9e7 ARROW-4410: [C++] Fix edge cases in InvertKernel discard e59bf77 ARROW-4430: [C++] Fix untested TypedByteBuffer::Append method discard 36e26fe ARROW-4431: [C++] Fixes for gRPC vendored builds new 6009eaa [Release] Apache Arrow JavaScript 0.4.0 new 5d742dd ARROW-4431: [C++] Fixes for gRPC vendored builds new 4a80fd9 ARROW-4430: [C++] Fix untested TypedByteBuffer::Append method new 0f8534b ARROW-4410: [C++] Fix edge cases in InvertKernel new 5232a4c ARROW-3846: [Gandiva][C++] Build Gandiva C++ libraries and get unit tests passing on Windows new e323711 [Python] Use 'latest' manylinux1 build image new cffe275 ARROW-4397: [C++] Add dim_names in Tensor and SparseTensor new 8982ac1 ARROW-4277: [C++] Add gmock to the toolchain new 726ae00 ARROW-4235: [GLib] Use "column_builder" in GArrowRecordBatchBuilder new 3573236 ARROW-4455: [Plasma] Suppress class-memaccess warnings new c0ad567 ARROW-4454: [C++] fix unused parameter wa
svn commit: r32384 - /release/arrow/arrow-js-0.3.1/
Author: uwe Date: Thu Feb 7 08:53:38 2019 New Revision: 32384 Log: Remove Arrow JS 0.3.1 Removed: release/arrow/arrow-js-0.3.1/
svn commit: r32383 - in /release/arrow: apache-arrow-js-0.4.0/ arrow-js-0.4.0/
Author: uwe Date: Thu Feb 7 08:51:14 2019 New Revision: 32383 Log: Apache Arrow JS 0.4.0 Added: release/arrow/arrow-js-0.4.0/ release/arrow/arrow-js-0.4.0/apache-arrow-js-0.4.0.tar.gz (with props) release/arrow/arrow-js-0.4.0/apache-arrow-js-0.4.0.tar.gz.asc release/arrow/arrow-js-0.4.0/apache-arrow-js-0.4.0.tar.gz.sha256 release/arrow/arrow-js-0.4.0/apache-arrow-js-0.4.0.tar.gz.sha512 Removed: release/arrow/apache-arrow-js-0.4.0/ Added: release/arrow/arrow-js-0.4.0/apache-arrow-js-0.4.0.tar.gz == Binary file - no diff available. Propchange: release/arrow/arrow-js-0.4.0/apache-arrow-js-0.4.0.tar.gz -- svn:mime-type = application/octet-stream Added: release/arrow/arrow-js-0.4.0/apache-arrow-js-0.4.0.tar.gz.asc == --- release/arrow/arrow-js-0.4.0/apache-arrow-js-0.4.0.tar.gz.asc (added) +++ release/arrow/arrow-js-0.4.0/apache-arrow-js-0.4.0.tar.gz.asc Thu Feb 7 08:51:14 2019 @@ -0,0 +1,17 @@ +-BEGIN PGP SIGNATURE- +Version: GnuPG v1 + +iQIcBAABAgAGBQJcUw9RAAoJECnZTiKMqtYC/FAQAJ6ArEy1lt+DbyMoPvT+CLnK +EzdHwyPr169FZx1rakj745RUpVYawLGN+4xqop4iO4V+y7g0dOegwgfyyFY9ANXX +cQxuf6R2zY8nGjS9t38UliYNHCW/N/LkiQbGvt1VmTZanRiigQ/O8cFGXoxIGqsl +GwJ1giOBVhKvWu8zYpje3zprxOnFCjQDFkxvhu0d7ZJzVpR5KR7MDLRc8EPeYgv+ +30VDFRvJXN1wAT7iwBt76A/ZttXlUcjWbHgzo0d27GGzPmgFCHAY8iV1w/1ww3g2 +0pYDtDXoWFzFBqD+ChWEhnwfj1F01rqjzxQTR1XzezL76NxGxXPON9DyXb7ii2ap +2SCjPUl/8NvxbAhiKqBZ+7lxYCBtb1Uy/1Z01MNHShPXcn2z6Bo8VviUxKpLzV9q +eKbsddSQ/dE2aIRoa1ZGJv8molLJR849P0aK++puksMZxANPeBTG38bBvwBHuUJc +iJfR6fU/i9cKSbsqthCT5qg3n6nXV2J/bOa7N3nbOLiin0Icl08p1YMzGsobFsSR +BJESkPBmivVSaMLitnPw8PCR/HzFAT3RX2eRpZ7WZRlFguFLESapmQMrwkB3R2o3 +RqUxnvdlIaTP9JKOE9hrBd0xGMWMGZjoCkcq/RgfsIBh1LLN6Few85ceFwVGC72D +aA2DVLVfxjPJ9bUH2tKa +=fXSW +-END PGP SIGNATURE- Added: release/arrow/arrow-js-0.4.0/apache-arrow-js-0.4.0.tar.gz.sha256 == --- release/arrow/arrow-js-0.4.0/apache-arrow-js-0.4.0.tar.gz.sha256 (added) +++ release/arrow/arrow-js-0.4.0/apache-arrow-js-0.4.0.tar.gz.sha256 Thu Feb 7 08:51:14 2019 @@ -0,0 +1 @@ +d53c6a1d68b87a529e1bdd31313c0130eb2c2ce762087448b65f66903ec31901 apache-arrow-js-0.4.0.tar.gz Added: release/arrow/arrow-js-0.4.0/apache-arrow-js-0.4.0.tar.gz.sha512 == --- release/arrow/arrow-js-0.4.0/apache-arrow-js-0.4.0.tar.gz.sha512 (added) +++ release/arrow/arrow-js-0.4.0/apache-arrow-js-0.4.0.tar.gz.sha512 Thu Feb 7 08:51:14 2019 @@ -0,0 +1 @@ +ed61153bd87fec48a55ae4294fe6787231b51e04718b6498fca15eb5d36e34c9724e2b16fb1c3ef4058fe3c5d8e18b97c4903acca6fd33bdb67e4d7176b2d470 apache-arrow-js-0.4.0.tar.gz
svn commit: r32382 - in /release/arrow/apache-arrow-js-0.4.0: ./ apache-arrow-js-0.4.0.tar.gz apache-arrow-js-0.4.0.tar.gz.asc apache-arrow-js-0.4.0.tar.gz.sha256 apache-arrow-js-0.4.0.tar.gz.sha512
Author: uwe Date: Thu Feb 7 08:49:48 2019 New Revision: 32382 Log: Apache Arrow JS 0.4.0 Added: release/arrow/apache-arrow-js-0.4.0/ release/arrow/apache-arrow-js-0.4.0/apache-arrow-js-0.4.0.tar.gz (with props) release/arrow/apache-arrow-js-0.4.0/apache-arrow-js-0.4.0.tar.gz.asc release/arrow/apache-arrow-js-0.4.0/apache-arrow-js-0.4.0.tar.gz.sha256 release/arrow/apache-arrow-js-0.4.0/apache-arrow-js-0.4.0.tar.gz.sha512 Added: release/arrow/apache-arrow-js-0.4.0/apache-arrow-js-0.4.0.tar.gz == Binary file - no diff available. Propchange: release/arrow/apache-arrow-js-0.4.0/apache-arrow-js-0.4.0.tar.gz -- svn:mime-type = application/octet-stream Added: release/arrow/apache-arrow-js-0.4.0/apache-arrow-js-0.4.0.tar.gz.asc == --- release/arrow/apache-arrow-js-0.4.0/apache-arrow-js-0.4.0.tar.gz.asc (added) +++ release/arrow/apache-arrow-js-0.4.0/apache-arrow-js-0.4.0.tar.gz.asc Thu Feb 7 08:49:48 2019 @@ -0,0 +1,17 @@ +-BEGIN PGP SIGNATURE- +Version: GnuPG v1 + +iQIcBAABAgAGBQJcUw9RAAoJECnZTiKMqtYC/FAQAJ6ArEy1lt+DbyMoPvT+CLnK +EzdHwyPr169FZx1rakj745RUpVYawLGN+4xqop4iO4V+y7g0dOegwgfyyFY9ANXX +cQxuf6R2zY8nGjS9t38UliYNHCW/N/LkiQbGvt1VmTZanRiigQ/O8cFGXoxIGqsl +GwJ1giOBVhKvWu8zYpje3zprxOnFCjQDFkxvhu0d7ZJzVpR5KR7MDLRc8EPeYgv+ +30VDFRvJXN1wAT7iwBt76A/ZttXlUcjWbHgzo0d27GGzPmgFCHAY8iV1w/1ww3g2 +0pYDtDXoWFzFBqD+ChWEhnwfj1F01rqjzxQTR1XzezL76NxGxXPON9DyXb7ii2ap +2SCjPUl/8NvxbAhiKqBZ+7lxYCBtb1Uy/1Z01MNHShPXcn2z6Bo8VviUxKpLzV9q +eKbsddSQ/dE2aIRoa1ZGJv8molLJR849P0aK++puksMZxANPeBTG38bBvwBHuUJc +iJfR6fU/i9cKSbsqthCT5qg3n6nXV2J/bOa7N3nbOLiin0Icl08p1YMzGsobFsSR +BJESkPBmivVSaMLitnPw8PCR/HzFAT3RX2eRpZ7WZRlFguFLESapmQMrwkB3R2o3 +RqUxnvdlIaTP9JKOE9hrBd0xGMWMGZjoCkcq/RgfsIBh1LLN6Few85ceFwVGC72D +aA2DVLVfxjPJ9bUH2tKa +=fXSW +-END PGP SIGNATURE- Added: release/arrow/apache-arrow-js-0.4.0/apache-arrow-js-0.4.0.tar.gz.sha256 == --- release/arrow/apache-arrow-js-0.4.0/apache-arrow-js-0.4.0.tar.gz.sha256 (added) +++ release/arrow/apache-arrow-js-0.4.0/apache-arrow-js-0.4.0.tar.gz.sha256 Thu Feb 7 08:49:48 2019 @@ -0,0 +1 @@ +d53c6a1d68b87a529e1bdd31313c0130eb2c2ce762087448b65f66903ec31901 apache-arrow-js-0.4.0.tar.gz Added: release/arrow/apache-arrow-js-0.4.0/apache-arrow-js-0.4.0.tar.gz.sha512 == --- release/arrow/apache-arrow-js-0.4.0/apache-arrow-js-0.4.0.tar.gz.sha512 (added) +++ release/arrow/apache-arrow-js-0.4.0/apache-arrow-js-0.4.0.tar.gz.sha512 Thu Feb 7 08:49:48 2019 @@ -0,0 +1 @@ +ed61153bd87fec48a55ae4294fe6787231b51e04718b6498fca15eb5d36e34c9724e2b16fb1c3ef4058fe3c5d8e18b97c4903acca6fd33bdb67e4d7176b2d470 apache-arrow-js-0.4.0.tar.gz
[arrow] annotated tag apache-arrow-js-0.4.0 updated (6009eaa -> 0ed4275)
This is an automated email from the ASF dual-hosted git repository. uwe pushed a change to annotated tag apache-arrow-js-0.4.0 in repository https://gitbox.apache.org/repos/asf/arrow.git. *** WARNING: tag apache-arrow-js-0.4.0 was modified! *** from 6009eaa (commit) to 0ed4275 (tag) tagging 6009eaa49ae29826764eb6e626bf0d12b83f3481 (commit) replaces apache-arrow-0.12.0 by Uwe L. Korn on Thu Jan 31 16:06:44 2019 +0100 - Log - Apache Arrow JS 0.4.0 --- No new revisions were added by this update. Summary of changes:
[arrow] branch master updated: ARROW-3966 [Java] JDBC Column Metadata in Arrow Field Metadata
This is an automated email from the ASF dual-hosted git repository. uwe pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git The following commit(s) were added to refs/heads/master by this push: new 54182b2 ARROW-3966 [Java] JDBC Column Metadata in Arrow Field Metadata 54182b2 is described below commit 54182b2fef6e0003e8dce232261644fc3a9fc9b3 Author: Mike Pigott AuthorDate: Wed Feb 6 18:10:04 2019 +0100 ARROW-3966 [Java] JDBC Column Metadata in Arrow Field Metadata https://issues.apache.org/jira/browse/ARROW-3966 This change includes #3133, and supports a new configuration item called "Include Metadata." If true, metadata from the JDBC ResultSetMetaData object is pulled along to the Schema Field Metadata. For now, this includes: * Catalog Name * Table Name * Column Name * Column Type Name Author: Mike Pigott Author: Michael Pigott Closes #3134 from mikepigott/jdbc-column-metadata and squashes the following commits: 02f2f348 ARROW-3966: Picking up lost change to support null calendars. 7049c36a Merge branch 'master' into jdbc-column-metadata e9a9b2b1 Merge pull request #6 from apache/master 65741a9e ARROW-3966: Code review feedback cc6cc88a ARROW-3966: Using a 1:N loop instead of a 0:N-1 loop for fewer index offsets in code. cfb2ba68 ARROW-3966: Using a helper method for building a UTC calendar with root locale. 29285132 ARROW-3966: Moving the metadata flag assignment into the builder. 69022c26 ARROW-3966: Fixing merge. 4a6de863 Merge branch 'master' into jdbc-column-metadata 509a1cc5 Merge pull request #5 from apache/master 789c8c84 Merge pull request #4 from apache/master e5b19eee Merge pull request #3 from apache/master 3b17c297 Merge pull request #2 from apache/master d847ebc4 Fixing file location 1ceac9eb Merge branch 'master' into jdbc-column-metadata 881c6c83 Merge pull request #1 from apache/master 03091a86 Unit tests for including result set metadata. 72d64cc6 Affirming the field metadata is empty when the configuration excludes field metadata. 7b4527c0 Test for the include-metadata flag in the configuration. 7e9ce373 Merge branch 'jdbc-to-arrow-config' into jdbc-column-metadata bb3165b9 Updating the function calls to use the JdbcToArrowConfig versions. a6fb1be4 Fixing function call 5bfd6a29 Merge branch 'jdbc-to-arrow-config' into jdbc-column-metadata 68c91e7a Modifying the jdbcToArrowSchema and jdbcToArrowVectors methods to receive JdbcToArrowConfig objects. b5b0cb11 Merge branch 'jdbc-to-arrow-config' into jdbc-column-metadata 8d6cf008 Documentation for public static VectorSchemaRoot sqlToArrow(Connection connection, String query, JdbcToArrowConfig config) 4f1260ce Adding documentation for public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, JdbcToArrowConfig config) e34a9e79 Fixing formatting. fe097c88 Merge branch 'jdbc-to-arrow-config' into jdbc-column-metadata df632e36 Updating the SQL tests to include JdbcToArrowConfig versions. b2700448 Updated validaton & documentation, and unit tests for the new JdbcToArrowConfig. da77cbe8 Creating a configuration class for the JDBC-to-Arrow converter. a78c7704 Updating Javadocs. 523387f3 Updating the API to support an optional 'includeMetadata' field. 5af1b5b2 Separating out the field-type creation from the field creation. --- .../org/apache/arrow/adapter/jdbc/Constants.java | 27 +++ .../org/apache/arrow/adapter/jdbc/JdbcToArrow.java | 20 ++--- .../arrow/adapter/jdbc/JdbcToArrowConfig.java | 18 - .../adapter/jdbc/JdbcToArrowConfigBuilder.java | 41 +- .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 88 +++--- .../arrow/adapter/jdbc/JdbcToArrowConfigTest.java | 26 ++- .../arrow/adapter/jdbc/JdbcToArrowTestHelper.java | 42 +++ .../adapter/jdbc/h2/JdbcToArrowCharSetTest.java| 15 .../adapter/jdbc/h2/JdbcToArrowDataTypesTest.java | 15 .../arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java | 14 .../arrow/adapter/jdbc/h2/JdbcToArrowTest.java | 15 .../adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java | 16 12 files changed, 295 insertions(+), 42 deletions(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/Constants.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/Constants.java new file mode 100644 index 000..c6aa018 --- /dev/null +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/Constants.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work f
[arrow] branch master updated: PARQUET-1521: [C++] Use pure virtual interfaces for parquet::TypedColumnWriter, remove use of 'extern template class'
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 7ce2655 PARQUET-1521: [C++] Use pure virtual interfaces for
parquet::TypedColumnWriter, remove use of 'extern template class'
7ce2655 is described below
commit 7ce26553b8ce78085751e4a4ae603d4043abf337
Author: Wes McKinney
AuthorDate: Tue Feb 5 15:15:43 2019 +0100
PARQUET-1521: [C++] Use pure virtual interfaces for
parquet::TypedColumnWriter, remove use of 'extern template class'
This follows corresponding work in TypedColumnReader. The public API is
unchanged as can be verified by lack of changes to the unit tests
Author: Wes McKinney
Closes #3551 from wesm/PARQUET-1521 and squashes the following commits:
aa6687a9 Fix clang warnings
33555044 Print build warning level
b657ac93 Fix parquet-column-io-benchmark
61204dec Refactor TypedColumnWriter implementation to be
based on pure virtual interface, remove use of extern template class
---
cpp/cmake_modules/SetupCxxFlags.cmake | 2 +
cpp/src/parquet/column-io-benchmark.cc | 26 +-
cpp/src/parquet/column_writer.cc | 450 ++---
cpp/src/parquet/column_writer.h| 208 +++
4 files changed, 350 insertions(+), 336 deletions(-)
diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake
b/cpp/cmake_modules/SetupCxxFlags.cmake
index 44ca22f..43dab02 100644
--- a/cpp/cmake_modules/SetupCxxFlags.cmake
+++ b/cpp/cmake_modules/SetupCxxFlags.cmake
@@ -111,6 +111,8 @@ if (NOT BUILD_WARNING_LEVEL)
endif(NOT BUILD_WARNING_LEVEL)
string(TOUPPER ${BUILD_WARNING_LEVEL} BUILD_WARNING_LEVEL)
+message(STATUS "Arrow build warning level: ${BUILD_WARNING_LEVEL}")
+
if ("${BUILD_WARNING_LEVEL}" STREQUAL "CHECKIN")
# Pre-checkin builds
if ("${COMPILER_FAMILY}" STREQUAL "msvc")
diff --git a/cpp/src/parquet/column-io-benchmark.cc
b/cpp/src/parquet/column-io-benchmark.cc
index c648d56..762bcb7 100644
--- a/cpp/src/parquet/column-io-benchmark.cc
+++ b/cpp/src/parquet/column-io-benchmark.cc
@@ -30,14 +30,15 @@ using schema::PrimitiveNode;
namespace benchmark {
-std::unique_ptr BuildWriter(int64_t output_size, OutputStream*
dst,
+std::shared_ptr BuildWriter(int64_t output_size, OutputStream*
dst,
ColumnChunkMetaDataBuilder* metadata,
ColumnDescriptor* schema,
const WriterProperties* properties) {
std::unique_ptr pager =
PageWriter::Open(dst, Compression::UNCOMPRESSED, metadata);
- return std::unique_ptr(new Int64Writer(
- metadata, std::move(pager), false /*use_dictionary*/, Encoding::PLAIN,
properties));
+ std::shared_ptr writer =
+ ColumnWriter::Make(metadata, std::move(pager), properties);
+ return std::static_pointer_cast(writer);
}
std::shared_ptr Int64Schema(Repetition::type repetition) {
@@ -65,14 +66,17 @@ static void BM_WriteInt64Column(::benchmark::State& state) {
std::vector definition_levels(state.range(0), 1);
std::vector repetition_levels(state.range(0), 0);
std::shared_ptr schema = Int64Schema(repetition);
- WriterProperties::Builder builder;
- std::shared_ptr properties =
builder.compression(codec)->build();
+ std::shared_ptr properties = WriterProperties::Builder()
+ .compression(codec)
+
->encoding(Encoding::PLAIN)
+ ->disable_dictionary()
+ ->build();
auto metadata = ColumnChunkMetaDataBuilder::Make(
properties, schema.get(), reinterpret_cast(&thrift_metadata));
while (state.KeepRunning()) {
InMemoryOutputStream stream;
-std::unique_ptr writer = BuildWriter(
+std::shared_ptr writer = BuildWriter(
state.range(0), &stream, metadata.get(), schema.get(),
properties.get());
writer->WriteBatch(values.size(), definition_levels.data(),
repetition_levels.data(),
values.data());
@@ -125,13 +129,17 @@ static void BM_ReadInt64Column(::benchmark::State& state)
{
std::vector definition_levels(state.range(0), 1);
std::vector repetition_levels(state.range(0), 0);
std::shared_ptr schema = Int64Schema(repetition);
- WriterProperties::Builder builder;
- std::shared_ptr properties =
builder.compression(codec)->build();
+ std::shared_ptr properties = WriterProperties::Builder()
+ .compression(codec)
+
->encoding(Encoding::PLAIN)
+
[arrow] branch master updated: ARROW-3239: [C++] Implement simple random array generation
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 5f0ff7f ARROW-3239: [C++] Implement simple random array generation
5f0ff7f is described below
commit 5f0ff7fcce18cc2afed8cf1165696946adb517a4
Author: François Saint-Jacques
AuthorDate: Tue Feb 5 14:37:08 2019 +0100
ARROW-3239: [C++] Implement simple random array generation
This implement the following API.
```
random::RandomArrayGenerator rand(seed);
auto bool_array = rand.Boolean(num_rows, 0.75, null_prob);
auto u8_array = rand.Int8(num_rows, 0, 100, null_prob);
```
Author: François Saint-Jacques
Closes #3533 from fsaintjacques/ARROW-3239-random-utils and squashes the
following commits:
a806b1ff Add ARROW_EXPORT to RandomArrayGenerator
63d9103b Fix GenerateOptions seed type
59c3a3bb Add undef to macro
22eca801 Handle special case with MSVC
728aadcd Fix downcasting issues
4840ac0e ARROW-3239: Implement simple random
array generation
---
cpp/src/arrow/CMakeLists.txt | 2 +-
cpp/src/arrow/ipc/json-test.cc| 43 +++-
cpp/src/arrow/ipc/read-write-benchmark.cc | 33 ++
cpp/src/arrow/ipc/test-common.h | 21 ++--
cpp/src/arrow/test-random.cc | 149 ++
cpp/src/arrow/test-random.h | 169 ++
cpp/src/arrow/test-util.h | 5 +-
7 files changed, 354 insertions(+), 68 deletions(-)
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index 1dba589..c65824f 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -243,7 +243,7 @@ endif()
if (ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS)
# that depend on gtest
ADD_ARROW_LIB(arrow_testing
-SOURCES test-util.cc
+SOURCES test-util.cc test-random.cc
OUTPUTS ARROW_TESTING_LIBRARIES
DEPENDENCIES ${GTEST_LIBRARY}
SHARED_LINK_LIBS arrow_shared ${GTEST_LIBRARY}
diff --git a/cpp/src/arrow/ipc/json-test.cc b/cpp/src/arrow/ipc/json-test.cc
index 47a0a29..bea6fbb 100644
--- a/cpp/src/arrow/ipc/json-test.cc
+++ b/cpp/src/arrow/ipc/json-test.cc
@@ -32,6 +32,7 @@
#include "arrow/ipc/test-common.h"
#include "arrow/memory_pool.h"
#include "arrow/record_batch.h"
+#include "arrow/test-random.h"
#include "arrow/test-util.h"
#include "arrow/type.h"
#include "arrow/type_traits.h"
@@ -216,48 +217,38 @@ TEST(TestJsonArrayWriter, Unions) {
// Data generation for test case below
void MakeBatchArrays(const std::shared_ptr& schema, const int num_rows,
std::vector>* arrays) {
- std::vector is_valid;
- random_is_valid(num_rows, 0.25, &is_valid);
+ const float null_prob = 0.25f;
+ random::RandomArrayGenerator rand(0x564a3bf0);
- std::vector v1_values;
- std::vector v2_values;
-
- randint(num_rows, 0, 100, &v1_values);
- randint(num_rows, 0, 100, &v2_values);
-
- std::shared_ptr v1;
- ArrayFromVector(is_valid, v1_values, &v1);
-
- std::shared_ptr v2;
- ArrayFromVector(is_valid, v2_values, &v2);
+ *arrays = {rand.Boolean(num_rows, 0.75, null_prob),
+ rand.Int8(num_rows, 0, 100, null_prob),
+ rand.Int32(num_rows, -1000, 1000, null_prob),
+ rand.UInt64(num_rows, 0, 1UL << 16, null_prob)};
static const int kBufferSize = 10;
static uint8_t buffer[kBufferSize];
static uint32_t seed = 0;
StringBuilder string_builder;
for (int i = 0; i < num_rows; ++i) {
-if (!is_valid[i]) {
- ASSERT_OK(string_builder.AppendNull());
-} else {
- random_ascii(kBufferSize, seed++, buffer);
- ASSERT_OK(string_builder.Append(buffer, kBufferSize));
-}
+random_ascii(kBufferSize, seed++, buffer);
+ASSERT_OK(string_builder.Append(buffer, kBufferSize));
}
std::shared_ptr v3;
ASSERT_OK(string_builder.Finish(&v3));
- arrays->emplace_back(v1);
- arrays->emplace_back(v2);
arrays->emplace_back(v3);
}
TEST(TestJsonFileReadWrite, BasicRoundTrip) {
- auto v1_type = int8();
- auto v2_type = int32();
- auto v3_type = utf8();
+ auto v1_type = boolean();
+ auto v2_type = int8();
+ auto v3_type = int32();
+ auto v4_type = uint64();
+ auto v5_type = utf8();
auto schema =
- ::arrow::schema({field("f1", v1_type), field("f2", v2_type), field("f3",
v3_type)});
+ ::arrow::schema({field("f1", v1_type), field("f2", v2_type), field("f3",
v3_type),
+ field("f4", v4_type), field("f5", v5_type)});
std::unique_ptr writer;
ASSERT_OK(JsonWriter::Open(schema, &writer));
@@ -289,7 +280,7 @@ TEST
[arrow] branch master updated: ARROW-4477: [JS] remove constructor override in the bignum mixins
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 927cfef ARROW-4477: [JS] remove constructor override in the bignum
mixins
927cfef is described below
commit 927cfeff875e557e28649891ea20ca38cb9d1536
Author: ptaylor
AuthorDate: Tue Feb 5 11:00:10 2019 +0100
ARROW-4477: [JS] remove constructor override in the bignum mixins
Removes the constructor override leftover from the original
`Object.create()` implementation of the bignum mixins. Closes
https://issues.apache.org/jira/browse/ARROW-4477
Author: ptaylor
Closes #3557 from trxcllnt/js/fix-bn-constructor and squashes the following
commits:
3a468594 remove constructor override in the bignum mixins
---
js/src/util/bn.ts | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/js/src/util/bn.ts b/js/src/util/bn.ts
index 74e3e82..c49c342 100644
--- a/js/src/util/bn.ts
+++ b/js/src/util/bn.ts
@@ -38,9 +38,9 @@ const BigNumNMixin = {
};
/** @ignore */
-const SignedBigNumNMixin: any = Object.assign({}, BigNumNMixin, { signed:
true, constructor: undefined });
+const SignedBigNumNMixin: any = Object.assign({}, BigNumNMixin, { signed: true
});
/** @ignore */
-const UnsignedBigNumNMixin: any = Object.assign({}, BigNumNMixin, { signed:
false, constructor: undefined });
+const UnsignedBigNumNMixin: any = Object.assign({}, BigNumNMixin, { signed:
false });
/** @ignore */
export class BN {
[arrow] branch master updated: ARROW-3923: [Java] JDBC Time Fetches Without Timezone
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new c93e2ae ARROW-3923: [Java] JDBC Time Fetches Without Timezone
c93e2ae is described below
commit c93e2ae14f4d35cf367817a2d2c65024c50dbecc
Author: Michael Pigott
AuthorDate: Tue Feb 5 10:18:41 2019 +0100
ARROW-3923: [Java] JDBC Time Fetches Without Timezone
https://issues.apache.org/jira/browse/ARROW-3923
Hello! I was reading through the JDBC source code and I noticed that a
java.util.Calendar was required for creating an Arrow Schema and Arrow Vectors
from a JDBC ResultSet, when none is required.
This change makes the Calendar optional.
Unit Tests:
The existing SureFire plugin configuration uses a UTC calendar for the
database, which is the default Calendar in the existing code. Likewise, no
changes to the unit tests are required to provide adequate coverage for the
change.
Author: Michael Pigott
Author: Mike Pigott
Closes #3066 from mikepigott/jdbc-timestamp-no-calendar and squashes the
following commits:
4d95da0a ARROW-3923: Supporting a null Calendar in the
config, and reverting the breaking change.
cd9a2306 Merge branch 'master' into jdbc-timestamp-no-calendar
509a1cc5 Merge pull request #5 from apache/master
789c8c84 Merge pull request #4 from apache/master
e5b19eee Merge pull request #3 from apache/master
3b17c297 Merge pull request #2 from apache/master
881c6c83 Merge pull request #1 from apache/master
089cff4d Format fixes
a58a4a5f Fixing calendar usage.
e12832a3 Allowing for timestamps without a time zone.
---
.../org/apache/arrow/adapter/jdbc/JdbcToArrow.java | 6 ++--
.../arrow/adapter/jdbc/JdbcToArrowConfig.java | 4 +--
.../adapter/jdbc/JdbcToArrowConfigBuilder.java | 11 +++
.../arrow/adapter/jdbc/JdbcToArrowUtils.java | 38 +-
.../arrow/adapter/jdbc/JdbcToArrowConfigTest.java | 15 +
5 files changed, 46 insertions(+), 28 deletions(-)
diff --git
a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java
b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java
index fd32036..ddf70df 100644
---
a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java
+++
b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java
@@ -179,13 +179,12 @@ public class JdbcToArrow {
* For the given JDBC {@link ResultSet}, fetch the data from Relational DB
and convert it to Arrow objects.
*
* @param resultSet ResultSet to use to fetch the data from underlying
database
- * @param calendar Calendar instance to use for Date, Time and Timestamp
datasets.
+ * @param calendar Calendar instance to use for Date, Time and Timestamp
datasets, or null if none.
* @return Arrow Data Objects {@link VectorSchemaRoot}
* @throws SQLException on error
*/
public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, Calendar
calendar) throws SQLException, IOException {
Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be
null");
-Preconditions.checkNotNull(calendar, "Calendar object can not be null");
return sqlToArrow(resultSet, new JdbcToArrowConfig(new
RootAllocator(Integer.MAX_VALUE), calendar));
}
@@ -195,7 +194,7 @@ public class JdbcToArrow {
*
* @param resultSet ResultSet to use to fetch the data from underlying
database
* @param allocator Memory allocator to use.
- * @param calendar Calendar instance to use for Date, Time and Timestamp
datasets.
+ * @param calendar Calendar instance to use for Date, Time and Timestamp
datasets, or null if none.
* @return Arrow Data Objects {@link VectorSchemaRoot}
* @throws SQLException on error
*/
@@ -203,7 +202,6 @@ public class JdbcToArrow {
throws SQLException, IOException {
Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be
null");
Preconditions.checkNotNull(allocator, "Memory Allocator object can not be
null");
-Preconditions.checkNotNull(calendar, "Calendar object can not be null");
return sqlToArrow(resultSet, new JdbcToArrowConfig(allocator, calendar));
}
diff --git
a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfig.java
b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfig.java
index 59813a8..e9fcffb 100644
---
a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfig.java
+++
b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfig.java
@@ -48,7 +48,6 @@ public final class JdbcToArrowConfig {
*/
[arrow] branch master updated: ARROW-3481: [Java] Fix java building failure with Maven 3.5.4
This is an automated email from the ASF dual-hosted git repository. uwe pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git The following commit(s) were added to refs/heads/master by this push: new 7f96b6f ARROW-3481: [Java] Fix java building failure with Maven 3.5.4 7f96b6f is described below commit 7f96b6feb1d12d2bccba70b7c1aadc52ff4e337e Author: Yuhong Guo AuthorDate: Mon Feb 4 16:39:40 2019 +0100 ARROW-3481: [Java] Fix java building failure with Maven 3.5.4 This problem happens in some rare case. When there are arrow lib build by old maven, but now current maven is updated to 3.5.4. This problem could happen and raise a building failure. Author: Yuhong Guo Closes #2738 from guoyuhong/fixJavaBuild and squashes the following commits: 74665109 Fix java build with Maven 3.5.4 --- java/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/pom.xml b/java/pom.xml index 9093bfa..57005b9 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -368,7 +368,7 @@ maven-enforcer-plugin - 3.0.0-M1 + 3.0.0-M2 maven-surefire-plugin
[arrow] branch master updated: ARROW-4435: Minor fixups to csharp .sln and .csproj file
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 07ab9cf ARROW-4435: Minor fixups to csharp .sln and .csproj file
07ab9cf is described below
commit 07ab9cf29ac65e119975d285e5d68be09e28562e
Author: Eric Erhardt
AuthorDate: Mon Feb 4 16:34:57 2019 +0100
ARROW-4435: Minor fixups to csharp .sln and .csproj file
1. Add .sln file to the repo.
2. Minor clean up of .csproj to move properties out of the
Configuration/Platform PropertyGroups, which is the standard way of setting
properties in SDK style projects.
Author: Eric Erhardt
Closes #3529 from eerhardt/SolutionProjectFixUp and squashes the following
commits:
88a7c6ef Add RAT exclude for csharp sln file
e9036b38 Minor fixups to csharp .sln and .csproj file
---
csharp/.gitignore | 5 -
csharp/Apache.Arrow.sln | 31 +
csharp/src/Apache.Arrow/Apache.Arrow.csproj | 14 +++--
dev/release/rat_exclude_files.txt | 1 +
4 files changed, 39 insertions(+), 12 deletions(-)
diff --git a/csharp/.gitignore b/csharp/.gitignore
index e5b411f..4283568 100644
--- a/csharp/.gitignore
+++ b/csharp/.gitignore
@@ -261,4 +261,7 @@ __pycache__/
*.pyc
# Project-specific
-artifacts/
\ No newline at end of file
+artifacts/
+
+# add .sln files back because they are ignored by the root .gitignore file
+!*.sln
\ No newline at end of file
diff --git a/csharp/Apache.Arrow.sln b/csharp/Apache.Arrow.sln
new file mode 100644
index 000..53b463c
--- /dev/null
+++ b/csharp/Apache.Arrow.sln
@@ -0,0 +1,31 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 15
+VisualStudioVersion = 15.0.28307.357
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Apache.Arrow",
"src\Apache.Arrow\Apache.Arrow.csproj", "{BA6B2B0D-EAAE-4183-8A39-1B9CF571F71F}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Apache.Arrow.Tests",
"test\Apache.Arrow.Tests\Apache.Arrow.Tests.csproj",
"{9CCEC01B-E67A-4726-BE72-7B514F76163F}"
+EndProject
+Global
+ GlobalSection(SolutionConfigurationPlatforms) = preSolution
+ Debug|Any CPU = Debug|Any CPU
+ Release|Any CPU = Release|Any CPU
+ EndGlobalSection
+ GlobalSection(ProjectConfigurationPlatforms) = postSolution
+ {BA6B2B0D-EAAE-4183-8A39-1B9CF571F71F}.Debug|Any CPU.ActiveCfg
= Debug|Any CPU
+ {BA6B2B0D-EAAE-4183-8A39-1B9CF571F71F}.Debug|Any CPU.Build.0 =
Debug|Any CPU
+ {BA6B2B0D-EAAE-4183-8A39-1B9CF571F71F}.Release|Any
CPU.ActiveCfg = Release|Any CPU
+ {BA6B2B0D-EAAE-4183-8A39-1B9CF571F71F}.Release|Any CPU.Build.0
= Release|Any CPU
+ {9CCEC01B-E67A-4726-BE72-7B514F76163F}.Debug|Any CPU.ActiveCfg
= Debug|Any CPU
+ {9CCEC01B-E67A-4726-BE72-7B514F76163F}.Debug|Any CPU.Build.0 =
Debug|Any CPU
+ {9CCEC01B-E67A-4726-BE72-7B514F76163F}.Release|Any
CPU.ActiveCfg = Release|Any CPU
+ {9CCEC01B-E67A-4726-BE72-7B514F76163F}.Release|Any CPU.Build.0
= Release|Any CPU
+ EndGlobalSection
+ GlobalSection(SolutionProperties) = preSolution
+ HideSolutionNode = FALSE
+ EndGlobalSection
+ GlobalSection(ExtensibilityGlobals) = postSolution
+ SolutionGuid = {FD0BB617-6031-4844-B99D-B331E335B572}
+ EndGlobalSection
+EndGlobal
diff --git a/csharp/src/Apache.Arrow/Apache.Arrow.csproj
b/csharp/src/Apache.Arrow/Apache.Arrow.csproj
index adc21c9..c2d73ec 100644
--- a/csharp/src/Apache.Arrow/Apache.Arrow.csproj
+++ b/csharp/src/Apache.Arrow/Apache.Arrow.csproj
@@ -1,9 +1,11 @@
-
+
netstandard1.3
+7.2
+true
Apache
Apache Arrow library
2018 Apache Software Foundation
@@ -15,16 +17,6 @@
0.0.1
-
-7.2
-true
-
-
-
-true
-7.2
-
-
diff --git a/dev/release/rat_exclude_files.txt
b/dev/release/rat_exclude_files.txt
index 4866ec2..6bd62c4 100644
--- a/dev/release/rat_exclude_files.txt
+++ b/dev/release/rat_exclude_files.txt
@@ -164,6 +164,7 @@ c_glib/gtk-doc.make
csharp/.gitattributes
csharp/src/Apache.Arrow/Flatbuf/*
csharp/build/Common.props
+csharp/Apache.Arrow.sln
csharp/src/Apache.Arrow/Apache.Arrow.csproj
csharp/src/Apache.Arrow/Properties/Resources.Designer.cs
csharp/src/Apache.Arrow/Properties/Resources.resx
[arrow] branch master updated: ARROW-4436: [Documentation] Update building.rst to reflect pyarrow req
This is an automated email from the ASF dual-hosted git repository. uwe pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git The following commit(s) were added to refs/heads/master by this push: new 9f34a41 ARROW-4436: [Documentation] Update building.rst to reflect pyarrow req 9f34a41 is described below commit 9f34a417a37f6d1f7061cf793292123d9749f9a9 Author: Micah Kornfield AuthorDate: Mon Feb 4 16:28:58 2019 +0100 ARROW-4436: [Documentation] Update building.rst to reflect pyarrow req Make note of the fact that pyarrow has to be installed for sphinx to successfully build the documentation. Author: Micah Kornfield Closes #3539 from emkornfield/update_build_doc and squashes the following commits: 8a3e5418 Fix install instructions 9468abe5 convert to note --- docs/source/building.rst | 10 +- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/source/building.rst b/docs/source/building.rst index c6ff974..2239a19 100644 --- a/docs/source/building.rst +++ b/docs/source/building.rst @@ -59,7 +59,15 @@ These two steps are mandatory and must be executed in order. doxygen popd -#. Build the complete documentation using Sphinx +#. Build the complete documentation using Sphinx. + + .. note:: + + This step requires the the pyarrow library is installed + in your python environment. One way to accomplish + this is to follow the build instructions at :ref:`development` + and then run `python setup.py install` in arrow/python + (it is best to do this in a dedicated conda/virtual environment). .. code-block:: shell
[arrow] branch master updated: ARROW-4459: [Testing] Add arrow-testing repo as submodule
This is an automated email from the ASF dual-hosted git repository. uwe pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git The following commit(s) were added to refs/heads/master by this push: new 9bd5294 ARROW-4459: [Testing] Add arrow-testing repo as submodule 9bd5294 is described below commit 9bd5294277e516c0f7a9caaa9ff2f1346f7297b3 Author: Andy Grove AuthorDate: Mon Feb 4 16:03:31 2019 +0100 ARROW-4459: [Testing] Add arrow-testing repo as submodule This PR adds the arrow-testing repo as a submodule in the directory `testing`. Author: Andy Grove Closes #3547 from andygrove/ARROW-4459 and squashes the following commits: 522ac269 Add arrow-testing submodule --- .gitmodules | 3 +++ testing | 1 + 2 files changed, 4 insertions(+) diff --git a/.gitmodules b/.gitmodules index 71722b2..6efc487 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "cpp/submodules/parquet-testing"] path = cpp/submodules/parquet-testing url = https://github.com/apache/parquet-testing.git +[submodule "testing"] + path = testing + url = https://github.com/apache/arrow-testing diff --git a/testing b/testing new file mode 16 index 000..6ee39a9 --- /dev/null +++ b/testing @@ -0,0 +1 @@ +Subproject commit 6ee39a9d17b0902b7372c22b9ff823304c69d709
[arrow] branch master updated: ARROW-4277: [C++] Add gmock to the toolchain
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 8ab0212 ARROW-4277: [C++] Add gmock to the toolchain
8ab0212 is described below
commit 8ab0212de91d4b2179e14a6d451c1aa3a440c296
Author: Micah Kornfield
AuthorDate: Fri Feb 1 10:03:36 2019 +0100
ARROW-4277: [C++] Add gmock to the toolchain
To some extent this was an educated guess on CMake settings (with a lot of
copy and pasting, so I expect things could be done slightly better).
Author: Micah Kornfield
Closes #3416 from emkornfield/gmock and squashes the following commits:
d6b143fe try to fix mac
9f8854eb revert version bump for gtest and append GMOCK
directory before other includes
e3b3a32c try to use vendored gmock on trusty
4c2425bf reorder gtest library for trusty?
367abc9f Can more typos
2cf4a268 Remove force found and add more messaging
43f47af2 remove unused GMOCK_HOME
0a738b9e remove prefix
7c78244a force gmock true
218506e3 fix condition
84bab120 fix environment variable
f7f23a0d remove duplicate gmock
0c0ff927 fix some typos
812f4142 update gmock output description in gmock modules
626621a1 cleanup bad suffixes and use gtest_roots
4d69be85 fix cmake gmock message and format
7c05e73d update cmake recipe with copy/paste bug fixes
be8020a4 add gmock to conda package list
e4d081ab add mock
f2c47681 replace DummyBinaryKernel
3c51702c add many linux build
911f093a Add gmock (mostly guesswork and copy/paste)
---
ci/conda_env_cpp.yml| 1 +
cpp/CMakeLists.txt | 10 ++--
cpp/cmake_modules/FindGTest.cmake | 93 -
cpp/cmake_modules/ThirdpartyToolchain.cmake | 47 ++-
cpp/src/arrow/compute/compute-test.cc | 9 +--
cpp/src/arrow/compute/test-util.h | 13
python/manylinux1/scripts/build_gtest.sh| 4 +-
7 files changed, 158 insertions(+), 19 deletions(-)
diff --git a/ci/conda_env_cpp.yml b/ci/conda_env_cpp.yml
index 87523b3..3d50b21 100644
--- a/ci/conda_env_cpp.yml
+++ b/ci/conda_env_cpp.yml
@@ -25,6 +25,7 @@ double-conversion
flatbuffers
gflags
glog
+gmock
gtest
libprotobuf
lz4-c
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 9cdbe7d..e0dbcd3 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -800,8 +800,9 @@ set(ARROW_TEST_STATIC_LINK_LIBS
arrow_testing_static
arrow_static
${ARROW_LINK_LIBS}
- ${GTEST_MAIN_LIBRARY}
- ${GTEST_LIBRARY})
+ ${GTEST_LIBRARY}
+ ${GMOCK_MAIN_LIBRARY}
+ ${GMOCK_LIBRARY})
set(ARROW_TEST_SHARED_LINK_LIBS
arrow_testing_shared
@@ -811,8 +812,9 @@ set(ARROW_TEST_SHARED_LINK_LIBS
${BOOST_SYSTEM_LIBRARY}
${BOOST_FILESYSTEM_LIBRARY}
${BOOST_REGEX_LIBRARY}
- ${GTEST_MAIN_LIBRARY}
- ${GTEST_LIBRARY})
+ ${GTEST_LIBRARY}
+ ${GMOCK_MAIN_LIBRARY}
+ ${GMOCK_LIBRARY})
if(NOT MSVC)
set(ARROW_TEST_SHARED_LINK_LIBS
diff --git a/cpp/cmake_modules/FindGTest.cmake
b/cpp/cmake_modules/FindGTest.cmake
index c7496c6..6ddb14a 100644
--- a/cpp/cmake_modules/FindGTest.cmake
+++ b/cpp/cmake_modules/FindGTest.cmake
@@ -21,8 +21,8 @@
# to be set before calling find_package:
#
# GTest_HOME - When set, this path is inspected instead of standard library
-#locations as the root of the GTest installation.
-#The environment variable GTEST_HOME overrides this veriable.
+#locations as the root of the GTest/Gmock installation.
+#The environment variable GTEST_HOME overrides this variable.
#
# This module defines
# GTEST_INCLUDE_DIR, directory containing headers
@@ -32,6 +32,14 @@
# GTEST_SHARED_LIB, path to libgtest's shared library
# GTEST_SHARED_MAIN_LIB, path to libgtest_main's shared library
# GTEST_FOUND, whether gtest has been found
+#
+# GMOCK_INCLUDE_DIR, directory containing headers
+# GMOCK_LIBS, directory containing gmock libraries
+# GMOCK_STATIC_LIB, path to libgmock.a
+# GMOCK_STATIC_MAIN_LIB, path to libgmock_main.a
+# GMOCK_SHARED_LIB, path to libgmock's shared library
+# GMOCK_SHARED_MAIN_LIB, path to libgmock_main's shared library
+# GMOCK_FOUND, whether gmock has been found
if( NOT "${GTEST_HOME}" STREQUAL "")
file( TO_CMAKE_PATH "${GTEST_HOME}" _native_path )
@@ -48,6 +56,15 @@ set(GTEST_SHARED_LIB_NAME
${CMAKE_SHARED_LIBRARY_PREFIX}gtest${CMAKE_SHARED_LIBRARY_SUFFIX})
set(GTEST_MAIN_SHARED_LIB_NAME
${CMAKE_SHARED_LIBRARY_PREFIX}gtest_main${CMAKE_SHARED_LIBRARY_SUFFIX})
+set(GMOCK_STATIC_LIB_NAME
+ ${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX})
+set(GMOCK_MAIN_STATIC_LIB_NAME
+ ${CMAKE_STATIC_LIBRARY_PREFIX}gmock_main${CMAKE_
svn commit: r32239 - in /dev/arrow/apache-arrow-js-0.4.0-rc1: ./ apache-arrow-js-0.4.0.tar.gz apache-arrow-js-0.4.0.tar.gz.asc apache-arrow-js-0.4.0.tar.gz.sha256 apache-arrow-js-0.4.0.tar.gz.sha512
Author: uwe
Date: Thu Jan 31 15:08:09 2019
New Revision: 32239
Log:
Apache Arrow JavaScript ${version} RC${rc}
Added:
dev/arrow/apache-arrow-js-0.4.0-rc1/
dev/arrow/apache-arrow-js-0.4.0-rc1/apache-arrow-js-0.4.0.tar.gz (with
props)
dev/arrow/apache-arrow-js-0.4.0-rc1/apache-arrow-js-0.4.0.tar.gz.asc
dev/arrow/apache-arrow-js-0.4.0-rc1/apache-arrow-js-0.4.0.tar.gz.sha256
dev/arrow/apache-arrow-js-0.4.0-rc1/apache-arrow-js-0.4.0.tar.gz.sha512
Added: dev/arrow/apache-arrow-js-0.4.0-rc1/apache-arrow-js-0.4.0.tar.gz
==
Binary file - no diff available.
Propchange: dev/arrow/apache-arrow-js-0.4.0-rc1/apache-arrow-js-0.4.0.tar.gz
--
svn:mime-type = application/octet-stream
Added: dev/arrow/apache-arrow-js-0.4.0-rc1/apache-arrow-js-0.4.0.tar.gz.asc
==
--- dev/arrow/apache-arrow-js-0.4.0-rc1/apache-arrow-js-0.4.0.tar.gz.asc (added)
+++ dev/arrow/apache-arrow-js-0.4.0-rc1/apache-arrow-js-0.4.0.tar.gz.asc Thu
Jan 31 15:08:09 2019
@@ -0,0 +1,17 @@
+-BEGIN PGP SIGNATURE-
+Version: GnuPG v1
+
+iQIcBAABAgAGBQJcUw9RAAoJECnZTiKMqtYC/FAQAJ6ArEy1lt+DbyMoPvT+CLnK
+EzdHwyPr169FZx1rakj745RUpVYawLGN+4xqop4iO4V+y7g0dOegwgfyyFY9ANXX
+cQxuf6R2zY8nGjS9t38UliYNHCW/N/LkiQbGvt1VmTZanRiigQ/O8cFGXoxIGqsl
+GwJ1giOBVhKvWu8zYpje3zprxOnFCjQDFkxvhu0d7ZJzVpR5KR7MDLRc8EPeYgv+
+30VDFRvJXN1wAT7iwBt76A/ZttXlUcjWbHgzo0d27GGzPmgFCHAY8iV1w/1ww3g2
+0pYDtDXoWFzFBqD+ChWEhnwfj1F01rqjzxQTR1XzezL76NxGxXPON9DyXb7ii2ap
+2SCjPUl/8NvxbAhiKqBZ+7lxYCBtb1Uy/1Z01MNHShPXcn2z6Bo8VviUxKpLzV9q
+eKbsddSQ/dE2aIRoa1ZGJv8molLJR849P0aK++puksMZxANPeBTG38bBvwBHuUJc
+iJfR6fU/i9cKSbsqthCT5qg3n6nXV2J/bOa7N3nbOLiin0Icl08p1YMzGsobFsSR
+BJESkPBmivVSaMLitnPw8PCR/HzFAT3RX2eRpZ7WZRlFguFLESapmQMrwkB3R2o3
+RqUxnvdlIaTP9JKOE9hrBd0xGMWMGZjoCkcq/RgfsIBh1LLN6Few85ceFwVGC72D
+aA2DVLVfxjPJ9bUH2tKa
+=fXSW
+-END PGP SIGNATURE-
Added: dev/arrow/apache-arrow-js-0.4.0-rc1/apache-arrow-js-0.4.0.tar.gz.sha256
==
--- dev/arrow/apache-arrow-js-0.4.0-rc1/apache-arrow-js-0.4.0.tar.gz.sha256
(added)
+++ dev/arrow/apache-arrow-js-0.4.0-rc1/apache-arrow-js-0.4.0.tar.gz.sha256 Thu
Jan 31 15:08:09 2019
@@ -0,0 +1 @@
+d53c6a1d68b87a529e1bdd31313c0130eb2c2ce762087448b65f66903ec31901
apache-arrow-js-0.4.0.tar.gz
Added: dev/arrow/apache-arrow-js-0.4.0-rc1/apache-arrow-js-0.4.0.tar.gz.sha512
==
--- dev/arrow/apache-arrow-js-0.4.0-rc1/apache-arrow-js-0.4.0.tar.gz.sha512
(added)
+++ dev/arrow/apache-arrow-js-0.4.0-rc1/apache-arrow-js-0.4.0.tar.gz.sha512 Thu
Jan 31 15:08:09 2019
@@ -0,0 +1 @@
+ed61153bd87fec48a55ae4294fe6787231b51e04718b6498fca15eb5d36e34c9724e2b16fb1c3ef4058fe3c5d8e18b97c4903acca6fd33bdb67e4d7176b2d470
apache-arrow-js-0.4.0.tar.gz
[arrow] branch master updated: include new jest configs to ensure verify release script works (#3532)
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 28c9eeb include new jest configs to ensure verify release script
works (#3532)
28c9eeb is described below
commit 28c9eeb2991a588080346d2550aa955cd37ef693
Author: Paul Taylor
AuthorDate: Thu Jan 31 14:47:01 2019 +
include new jest configs to ensure verify release script works (#3532)
---
js/package.json | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/js/package.json b/js/package.json
index fe37d69..22a1df9 100644
--- a/js/package.json
+++ b/js/package.json
@@ -53,7 +53,9 @@
"tsconfig",
"README.md",
"gulpfile.js",
-"npm-release.sh"
+"npm-release.sh",
+"jest.config.js",
+"jest.coverage.config.js"
],
"dependencies": {
"@types/flatbuffers": "^1.9.0",
svn commit: r32237 - in /dev/arrow/apache-arrow-js-0.4.0-rc0: ./ apache-arrow-js-0.4.0.tar.gz apache-arrow-js-0.4.0.tar.gz.asc apache-arrow-js-0.4.0.tar.gz.sha256 apache-arrow-js-0.4.0.tar.gz.sha512
Author: uwe
Date: Thu Jan 31 10:21:44 2019
New Revision: 32237
Log:
Apache Arrow JavaScript ${version} RC${rc}
Added:
dev/arrow/apache-arrow-js-0.4.0-rc0/
dev/arrow/apache-arrow-js-0.4.0-rc0/apache-arrow-js-0.4.0.tar.gz (with
props)
dev/arrow/apache-arrow-js-0.4.0-rc0/apache-arrow-js-0.4.0.tar.gz.asc
dev/arrow/apache-arrow-js-0.4.0-rc0/apache-arrow-js-0.4.0.tar.gz.sha256
dev/arrow/apache-arrow-js-0.4.0-rc0/apache-arrow-js-0.4.0.tar.gz.sha512
Added: dev/arrow/apache-arrow-js-0.4.0-rc0/apache-arrow-js-0.4.0.tar.gz
==
Binary file - no diff available.
Propchange: dev/arrow/apache-arrow-js-0.4.0-rc0/apache-arrow-js-0.4.0.tar.gz
--
svn:mime-type = application/octet-stream
Added: dev/arrow/apache-arrow-js-0.4.0-rc0/apache-arrow-js-0.4.0.tar.gz.asc
==
--- dev/arrow/apache-arrow-js-0.4.0-rc0/apache-arrow-js-0.4.0.tar.gz.asc (added)
+++ dev/arrow/apache-arrow-js-0.4.0-rc0/apache-arrow-js-0.4.0.tar.gz.asc Thu
Jan 31 10:21:44 2019
@@ -0,0 +1,17 @@
+-BEGIN PGP SIGNATURE-
+Version: GnuPG v1
+
+iQIcBAABAgAGBQJcUswwAAoJECnZTiKMqtYCA8IQAKRlr53QjLFSCgncGSZ2T2gS
+gq///8D09rsd1joMT73v9ohtKqu6CEMBzJ1jsn6we/eIHqd6nntgagISgwnp5CAE
+Sq87n4ikcj8MfrbzrU4/E/v65bCfY0lScHHclWsDGr+tFHmizN6GMshRk0a/HEUv
+QoVTCpb7/l/9JMLdrdgqPhB7T+OvEjIzCsWGzHku44J0bqAQHjcwoikjd7jjiawh
+EltMlAfnp4mNSlV0H7QLZPcQm/VCMJ/H9nPc2pTSTGRWua5f7ARHlouQYLAVUwTS
+UnlRPo3rlkoGDx/eSeoWKbRGUhIEaw7bCHPVKtRKnw7Ri6kNX+6ZBveAZGxp0Vz5
+wNvmb0iKbKEozQqFumqhNqAKJ3m0e2hvDwL0jrWe0prgppmXq6LI07si/5QHkRiZ
+rCsCvMOja+g8db0qaocX+eqkpZL6FJHDbEqNhvy0vGV+OMTMyQ99Pf7mvARryh1s
+ywAqV6JcylYHbKxU493zrDf36IthISaZqV51w8XlvhjBb5vaV5fzaoHxrQ+Mnzp5
+SsCJh1cw8/LbY4QIVh5Qaoh6KvoC7wMkrGATozVD0XE7WDYeprCXGmjD/AipAuWd
+zikn+y2AscunCdsWltVtuP3oidXr1Pn8ybjsCTGKsgl2sHfQV8LR5as52V7d9fHt
+fRIm/JzG+/SRbNl2RBbQ
+=eEJU
+-END PGP SIGNATURE-
Added: dev/arrow/apache-arrow-js-0.4.0-rc0/apache-arrow-js-0.4.0.tar.gz.sha256
==
--- dev/arrow/apache-arrow-js-0.4.0-rc0/apache-arrow-js-0.4.0.tar.gz.sha256
(added)
+++ dev/arrow/apache-arrow-js-0.4.0-rc0/apache-arrow-js-0.4.0.tar.gz.sha256 Thu
Jan 31 10:21:44 2019
@@ -0,0 +1 @@
+8dfe8969f2cd39d04b38c685536fee96a47ad402b1de858d1d66875750e4b6f2
apache-arrow-js-0.4.0.tar.gz
Added: dev/arrow/apache-arrow-js-0.4.0-rc0/apache-arrow-js-0.4.0.tar.gz.sha512
==
--- dev/arrow/apache-arrow-js-0.4.0-rc0/apache-arrow-js-0.4.0.tar.gz.sha512
(added)
+++ dev/arrow/apache-arrow-js-0.4.0-rc0/apache-arrow-js-0.4.0.tar.gz.sha512 Thu
Jan 31 10:21:44 2019
@@ -0,0 +1 @@
+0cfd37b7bee91c0c0f63db9e2940e791c0b3e19290adc44f3c8776b6888a2791b7403b4493c334f528c111ec49e0428834a2fab37bf3dc084b5effb3746f75b2
apache-arrow-js-0.4.0.tar.gz
[arrow] branch master updated: ARROW-4328: Add a ARROW_USE_OLD_CXXABI configure var to R
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new cf7bd13 ARROW-4328: Add a ARROW_USE_OLD_CXXABI configure var to R
cf7bd13 is described below
commit cf7bd131949366b8df901a89e31651f227f13197
Author: Peter
AuthorDate: Wed Jan 30 15:51:05 2019 +0100
ARROW-4328: Add a ARROW_USE_OLD_CXXABI configure var to R
Sets GLIBCXX_USE_CXX11_ABI=0 so that the R lib will be compatible with a
shared library built
with -DARROW_TENSORFLOW=ON
Author: Peter
Closes #3409 from cHYzZQo/r-with-tensorflow and squashes the following
commits:
d7dc98b7 ARROW-4328: Add a ARROW_USE_OLD_CXXABI configure var to R
---
r/README.md | 7 +++
r/configure | 8 +++-
r/src/Makevars.in | 2 +-
3 files changed, 15 insertions(+), 2 deletions(-)
diff --git a/r/README.md b/r/README.md
index 987d0c2..1e2cb2c 100644
--- a/r/README.md
+++ b/r/README.md
@@ -24,6 +24,13 @@ Then the R package:
devtools::install_github("apache/arrow/r")
```
+If libarrow was built with the old CXXABI then you need to pass
+the ARROW_USE_OLD_CXXABI configuration variable.
+
+``` r
+devtools::install_github("apache/arrow/r",
args=c("--configure-vars=ARROW_USE_OLD_CXXABI=1"))
+```
+
## Example
``` r
diff --git a/r/configure b/r/configure
index c17fd4c..9df1f9d 100755
--- a/r/configure
+++ b/r/configure
@@ -71,6 +71,12 @@ CXX11FLAGS=$("${R_HOME}"/bin/R CMD config CXX11FLAGS)
CXX11STD=$("${R_HOME}"/bin/R CMD config CXX11STD)
CPPFLAGS=$("${R_HOME}"/bin/R CMD config CPPFLAGS)
+# If libarrow uses the old GLIBCXX ABI, so we have to use it too
+PKG_CXXFLAGS="$C_VISIBILITY"
+if [ "$ARROW_USE_OLD_CXXABI" ]; then
+ PKG_CXXFLAGS="$PKG_CXXFLAGS -D_GLIBCXX_USE_CXX11_ABI=0"
+fi
+
# Test configuration
echo "#include $PKG_TEST_HEADER" | ${CXXCPP} ${CPPFLAGS} ${PKG_CFLAGS}
${CXX11FLAGS} ${CXX11STD} -xc++ - >/dev/null 2>&1
@@ -91,7 +97,7 @@ if [ $? -ne 0 ]; then
fi
# Write to Makevars
-sed -e "s|@cflags@|$PKG_CFLAGS|" -e "s|@libs@|$PKG_LIBS|" -e
"s|@visibility@|$C_VISIBILITY|" src/Makevars.in > src/Makevars
+sed -e "s|@cflags@|$PKG_CFLAGS|" -e "s|@libs@|$PKG_LIBS|" -e
"s|@pkgcxxflags@|$PKG_CXXFLAGS|" src/Makevars.in > src/Makevars
# Success
exit 0
diff --git a/r/src/Makevars.in b/r/src/Makevars.in
index a0d5fed..0d28087 100644
--- a/r/src/Makevars.in
+++ b/r/src/Makevars.in
@@ -16,7 +16,7 @@
# under the License.
PKG_CPPFLAGS=@cflags@
-PKG_CXXFLAGS=@visibility@
+PKG_CXXFLAGS=@pkgcxxflags@
CXX_STD=CXX11
PKG_LIBS=@libs@ -Wl,-rpath,/usr/local/lib
#CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0"
[arrow] branch master updated: ARROW-4414: [C++] Stop using cmake COMMAND_EXPAND_LISTS because it breaks package builds for older distros
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 6626c00 ARROW-4414: [C++] Stop using cmake COMMAND_EXPAND_LISTS
because it breaks package builds for older distros
6626c00 is described below
commit 6626c009ac19be2fdf5815b70eabbdfdf2b0e434
Author: Krisztián Szűcs
AuthorDate: Wed Jan 30 13:33:23 2019 +0100
ARROW-4414: [C++] Stop using cmake COMMAND_EXPAND_LISTS because it breaks
package builds for older distros
COMMAND_EXPAND_LISTS option of add_custom_command is too new on Ubuntu
Xenial and Debian stretch. It's available since CMake 3.8
Linux package builds:
[kszucs/crossbow/build-434](https://github.com/kszucs/crossbow/branches/all?utf8=%E2%9C%93&query=434)
(the xenial crossbow error is unrelated)
Author: Krisztián Szűcs
Closes #3522 from kszucs/ARROW-4414 and squashes the following commits:
23dde8a3 revert removing cmake from conda_env_cpp
ca52b309 use system cmake on travis and in docker images;
remove cmake_expand_lists
---
ci/travis_script_python.sh | 1 -
cpp/src/gandiva/precompiled/CMakeLists.txt | 3 +--
2 files changed, 1 insertion(+), 3 deletions(-)
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 60335d9..27d75da 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -48,7 +48,6 @@ fi
conda create -y -q -p $CONDA_ENV_DIR \
--file $TRAVIS_BUILD_DIR/ci/conda_env_python.yml \
nomkl \
- cmake \
pip \
numpy=1.14 \
python=${PYTHON_VERSION} \
diff --git a/cpp/src/gandiva/precompiled/CMakeLists.txt
b/cpp/src/gandiva/precompiled/CMakeLists.txt
index 6e0a092..2b629c7 100644
--- a/cpp/src/gandiva/precompiled/CMakeLists.txt
+++ b/cpp/src/gandiva/precompiled/CMakeLists.txt
@@ -44,8 +44,7 @@ foreach(SRC_FILE ${PRECOMPILED_SRCS})
-O3 -c ${ABSOLUTE_SRC} -o ${BC_FILE}
${ARROW_GANDIVA_PC_CXX_FLAGS}
-I${CMAKE_SOURCE_DIR}/src
-DEPENDS ${SRC_FILE}
-COMMAND_EXPAND_LISTS)
+DEPENDS ${SRC_FILE})
list(APPEND BC_FILES ${BC_FILE})
endforeach()
[arrow] branch master updated: ARROW-4389: [R] Don't install clang-tools in test job
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 8f6e0c2 ARROW-4389: [R] Don't install clang-tools in test job
8f6e0c2 is described below
commit 8f6e0c2ced6a7d7e0e9105eff32a42512d541a60
Author: Uwe L. Korn
AuthorDate: Sun Jan 27 21:41:41 2019 +0100
ARROW-4389: [R] Don't install clang-tools in test job
They aren't actually used.
Author: Uwe L. Korn
Closes #3500 from xhochy/ARROW-4389 and squashes the following commits:
f5d66ca2 ARROW-4389: Don't install clang-tools in test job
---
.travis.yml | 1 -
1 file changed, 1 deletion(-)
diff --git a/.travis.yml b/.travis.yml
index 0ebd3ca..31cf0a8 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -340,7 +340,6 @@ matrix:
fi
- $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
- $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
-- $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
- export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$TRAVIS_BUILD_DIR/cpp-install/lib
- export
PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$TRAVIS_BUILD_DIR/cpp-install/lib/pkgconfig
- pushd ${TRAVIS_BUILD_DIR}/r
[arrow] branch master updated: ARROW-4330: [C++] More robust discovery of pthreads
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new b427864 ARROW-4330: [C++] More robust discovery of pthreads
b427864 is described below
commit b4278641a6a56c56d2007469b0eb840d52cc007d
Author: Wes McKinney
AuthorDate: Sun Jan 27 21:38:37 2019 +0100
ARROW-4330: [C++] More robust discovery of pthreads
I found when using a non-system gcc (e.g. the `gcc_linux-64` and
`gxx_linux-64` gcc7 conda packages) that CMake is not able to discover
pthreads. The conda-forge community regards this as a bug in CMake, but there
is another better way to do this using the built-in `FindThreads.cmake`.
There's a chance this might also resolve the missing thread symbol in the
Gandiva windows build, but I haven't tried it yet
Author: Wes McKinney
Closes #3494 from wesm/pthreads-cmake and squashes the following commits:
828d6bf1 Code review
38349ad8 Restore jemalloc dependencies on pthreads
122e2647 Clean up system link libraries for pthread, librt
7522a60a Use FindThreads.cmake to handle -pthread
---
cpp/CMakeLists.txt | 52 +
cpp/cmake_modules/ThirdpartyToolchain.cmake | 14 +++-
2 files changed, 19 insertions(+), 47 deletions(-)
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 5309c35..3ec430e 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -839,50 +839,28 @@ if (ARROW_BUILD_BENCHMARKS)
${ARROW_TEST_LINK_LIBS})
endif()
+set(ARROW_SYSTEM_LINK_LIBS)
+
if (ARROW_JEMALLOC)
add_definitions(-DARROW_JEMALLOC)
add_definitions(-DARROW_JEMALLOC_INCLUDE_DIR=${JEMALLOC_INCLUDE_DIR})
-
- if (NOT WIN32 AND NOT APPLE)
-set(ARROW_JEMALLOC_LINK_LIBS
- jemalloc_static
- # For glibc <2.17 we need to link to librt.
- # As we compile with --as-needed by default, the linker will omit this
- # dependency if not required.
- rt
- )
- else()
-set(ARROW_JEMALLOC_LINK_LIBS
- jemalloc_static
- )
- endif()
- list(APPEND
-ARROW_SHARED_PRIVATE_LINK_LIBS
-${ARROW_JEMALLOC_LINK_LIBS})
- list(APPEND
-ARROW_STATIC_LINK_LIBS
-${ARROW_JEMALLOC_LINK_LIBS})
- list(APPEND
-ARROW_STATIC_INSTALL_INTERFACE_LIBS
-jemalloc
-rt)
+ list(APPEND ARROW_SYSTEM_LINK_LIBS jemalloc_static)
endif(ARROW_JEMALLOC)
-if (PTHREAD_LIBRARY)
- list(APPEND
-ARROW_LINK_LIBS
-pthreadshared)
- list(APPEND
-ARROW_SHARED_INSTALL_INTERFACE_LIBS
-pthread)
- list(APPEND
-ARROW_STATIC_LINK_LIBS
-pthreadshared)
- list(APPEND
-ARROW_STATIC_INSTALL_INTERFACE_LIBS
-pthread)
+if (THREADS_FOUND)
+ list(APPEND ARROW_SYSTEM_LINK_LIBS Threads::Threads)
endif()
+if (NOT WIN32 AND NOT APPLE)
+ # Pass -lrt on Linux only
+ list(APPEND ARROW_SYSTEM_LINK_LIBS rt)
+endif()
+
+list(APPEND ARROW_LINK_LIBS ${ARROW_SYSTEM_LINK_LIBS})
+list(APPEND ARROW_STATIC_LINK_LIBS ${ARROW_SYSTEM_LINK_LIBS})
+list(APPEND ARROW_SHARED_INSTALL_INTERFACE_LIBS ${ARROW_SYSTEM_LINK_LIBS})
+list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${ARROW_SYSTEM_LINK_LIBS})
+
# Subdirectories
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake
b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 5a75a9b..13a6a47 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -334,14 +334,8 @@ endif()
# --
# Find pthreads
-if (WIN32)
- set(PTHREAD_LIBRARY "PTHREAD_LIBRARY-NOTFOUND")
-else()
- find_library(PTHREAD_LIBRARY pthread)
- message(STATUS "Found pthread: ${PTHREAD_LIBRARY}")
- add_library(pthreadshared SHARED IMPORTED)
- set_target_properties(pthreadshared PROPERTIES IMPORTED_LOCATION
${PTHREAD_LIBRARY})
-endif()
+set(THREADS_PREFER_PTHREAD_FLAG ON)
+find_package(Threads REQUIRED)
# --
# Add Boost dependencies (code adapted from Apache Kudu (incubating))
@@ -802,7 +796,7 @@ if (ARROW_JEMALLOC)
ADD_THIRDPARTY_LIB(jemalloc
STATIC_LIB ${JEMALLOC_STATIC_LIB}
SHARED_LIB ${JEMALLOC_SHARED_LIB}
-DEPS ${PTHREAD_LIBRARY})
+DEPS Threads::Threads)
add_dependencies(jemalloc_static jemalloc_ep)
endif()
@@ -1532,7 +1526,7 @@ if (ARROW_USE_GLOG)
set(GLOG_STATIC_LIB
"${GLOG_BUILD_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}glog${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(GLOG_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
set(GLOG_CMAKE_C_FLAGS "${EP_C_FLAGS} -fPIC")
-if (PTHREAD_LIBRARY)
+if (Threads::Threads)
set(GLOG_CMAKE_CXX_FLAGS &qu
