This is an automated email from the ASF dual-hosted git repository.
SYaoJun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-graphar.git
The following commit(s) were added to refs/heads/main by this push:
new b6214906 fix: upgrade to apache-arrow 24.0.0 and enable CI run success
(#922)
b6214906 is described below
commit b62149061c7f8261fdffa1a18c92eeaf00837a26
Author: Jason <[email protected]>
AuthorDate: Mon May 11 11:53:16 2026 +0800
fix: upgrade to apache-arrow 24.0.0 and enable CI run success (#922)
* fix: upgrade to apache-arrow 24.0.0 and enable CI run success
Signed-off-by: Jason <[email protected]>
* fix: keep the java ci still use arrow-17
Signed-off-by: Jason <[email protected]>
* fix: restore python upload scripts
Signed-off-by: Jason <[email protected]>
---------
Signed-off-by: Jason <[email protected]>
---
.github/workflows/ci.yml | 18 +++++----------
.github/workflows/pre-commit.yml | 14 +++++++----
.github/workflows/python-wheel-workflow.yml | 24 ++++++++-----------
.github/workflows/python.yml | 12 +++++-----
.github/workflows/rust.yaml | 10 ++++----
cpp/CMakeLists.txt | 4 ++--
cpp/cmake/apache-arrow.cmake | 4 ++--
cpp/src/CMakeLists.txt | 4 +++-
cpp/src/graphar/arrow/chunk_reader.cc | 10 ++++++++
cpp/src/graphar/arrow/chunk_writer.cc | 3 +--
cpp/src/graphar/expression.cc | 19 +++++++++++++++
cpp/src/graphar/expression.h | 2 ++
cpp/src/graphar/filesystem.cc | 36 +++++++++++++++++++----------
cpp/src/graphar/high-level/graph_reader.cc | 8 +++++++
cpp/src/graphar/reader_util.cc | 5 ++++
cpp/test/test_arrow_chunk_writer.cc | 26 ++++++++++-----------
cpp/test/test_builder.cc | 11 +++++----
17 files changed, 133 insertions(+), 77 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 81cf716c..fb7c4059 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -46,7 +46,7 @@ jobs:
env:
GAR_TEST_DATA: ${{ github.workspace }}/graphar-testing/
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
with:
submodules: true
@@ -67,10 +67,10 @@ jobs:
-P /tmp/
sudo apt-get install -y
/tmp/apache-arrow-apt-source-latest-"$(lsb_release --codename --short)".deb
sudo apt-get update -y
- sudo apt install -y libarrow-dev=17.0.0-1 \
- libarrow-dataset-dev=17.0.0-1 \
- libarrow-acero-dev=17.0.0-1 \
- libparquet-dev=17.0.0-1
+ sudo apt install -y libarrow-dev=24.0.0-1 \
+ libarrow-dataset-dev=24.0.0-1 \
+ libarrow-acero-dev=24.0.0-1 \
+ libparquet-dev=24.0.0-1
sudo apt-get install -y libboost-graph-dev ccache libcurl4-openssl-dev
doxygen lcov ninja-build
# install benchmark
@@ -96,12 +96,6 @@ jobs:
cd build
cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=ON
-DBUILD_EXAMPLES=ON -DBUILD_BENCHMARKS=ON -DGRAPHAR_ENABLE_COVERAGE=ON
- - name: clang-format
- run: |
- pip install pre-commit
- pre-commit install
- pre-commit run clang-format -a
-
- name: cpplint
working-directory: "cpp/build"
run: |
@@ -199,7 +193,7 @@ jobs:
fail-fast: false
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
with:
submodules: true
diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml
index fd5a4cad..7627841e 100644
--- a/.github/workflows/pre-commit.yml
+++ b/.github/workflows/pre-commit.yml
@@ -18,7 +18,7 @@
name: Pre-commit Checks
on:
- pull_request_target:
+ pull_request:
branches:
- main
types:
@@ -29,7 +29,7 @@ on:
- ready_for_review
concurrency:
- group: ${{ github.repository }}-${{ github.event.number || github.head_ref
|| github.sha }}-${{ github.workflow }}
+ group: ${{ github.repository }}-${{ github.event.number }}-${{
github.workflow }}-${{ github.event_name }}
cancel-in-progress: true
permissions:
@@ -41,9 +41,15 @@ jobs:
runs-on: ubuntu-slim
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
with:
submodules: true
- name: cmake-format check
- run: pre-commit run cmake-format -a
\ No newline at end of file
+ run: |
+ pip install pre-commit
+ pre-commit install
+ pre-commit run cmake-format -a
+
+ - name: clang-format
+ run: pre-commit run clang-format -a
\ No newline at end of file
diff --git a/.github/workflows/python-wheel-workflow.yml
b/.github/workflows/python-wheel-workflow.yml
index c8040325..ad44f9c5 100644
--- a/.github/workflows/python-wheel-workflow.yml
+++ b/.github/workflows/python-wheel-workflow.yml
@@ -94,20 +94,17 @@ jobs:
runner: ubuntu-latest # This is the standard x86_64 runner
os: linux
manylinux: _2_28
- deployment-target: ''
# Job 2: Native aarch64 build
- platform: aarch64
runner: ubuntu-22.04-arm # This is a native ARM64 runner
os: linux
manylinux: _2_28
- deployment-target: ''
# Job 3: macOS arm64 build
- platform: arm64
runner: macos-latest
os: macos
- deployment-target: '11.0'
env:
@@ -121,7 +118,7 @@ jobs:
CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.os == 'linux' &&
format('manylinux{0}', matrix.manylinux) || '' }}
CIBW_MANYLINUX_AARCH64_IMAGE: ${{ matrix.os == 'linux' &&
format('manylinux{0}', matrix.manylinux) || '' }}
CIBW_ENVIRONMENT_WINDOWS: DISTUTILS_USE_SDK=1 MSSdk=1
- CIBW_ENVIRONMENT_MACOS: ${{ matrix.os == 'macos' &&
format('MACOSX_DEPLOYMENT_TARGET={0} CMAKE_OSX_DEPLOYMENT_TARGET={0}
CFLAGS=-mmacosx-version-min={0} CXXFLAGS=-mmacosx-version-min={0}
LDFLAGS=-mmacosx-version-min={0}', matrix.deployment-target) || '' }}
+ CIBW_ENVIRONMENT_MACOS: ${{ matrix.os == 'macos' &&
'MACOSX_DEPLOYMENT_TARGET=$(sw_vers -productVersion)
CMAKE_OSX_DEPLOYMENT_TARGET=$(sw_vers -productVersion)
CFLAGS=-mmacosx-version-min=$(sw_vers -productVersion)
CXXFLAGS=-mmacosx-version-min=$(sw_vers -productVersion)
LDFLAGS=-mmacosx-version-min=$(sw_vers -productVersion)' || '' }}
CIBW_BEFORE_BUILD_LINUX: |
set -eux
if [ -f /etc/system-release-cpe ]; then
@@ -163,8 +160,8 @@ jobs:
with:
python-version: "3.9"
- - name: Set up Miniconda (macOS/Windows)
- if: matrix.os == 'windows' || matrix.os == 'macos'
+ - name: Set up Miniconda (Windows)
+ if: matrix.os == 'windows'
uses: conda-incubator/setup-miniconda@v3
with:
auto-activate-base: true
@@ -176,15 +173,13 @@ jobs:
shell: bash
run: |
set -euxo pipefail
- mamba install -y -c conda-forge arrow-cpp
- # Note: CONDA_PREFIX may be unset unless conda is activated in this
shell.
- # setup-miniconda exports CONDA (base install prefix), which is
sufficient here.
- echo "CMAKE_PREFIX_PATH=$CONDA" >> "$GITHUB_ENV"
+ brew install apache-arrow
+ echo "CMAKE_PREFIX_PATH=/usr/local" >> "$GITHUB_ENV"
# Optional sanity check: ensure Arrow dylib isn't built for a newer
macOS than deployment target.
if command -v otool >/dev/null 2>&1; then
- ls -lah "$CONDA/lib" || true
- if [ -f "$CONDA/lib/libarrow.dylib" ]; then
- otool -l "$CONDA/lib/libarrow.dylib" | (grep -A3 -E
'LC_BUILD_VERSION|LC_VERSION_MIN_MACOSX' || true)
+ ls -lah /usr/local/lib || true
+ if [ -f "/usr/local/lib/libarrow.dylib" ]; then
+ otool -l "/usr/local/lib/libarrow.dylib" | (grep -A3 -E
'LC_BUILD_VERSION|LC_VERSION_MIN_MACOSX' || true)
fi
fi
@@ -259,6 +254,7 @@ jobs:
python -c "import graphar; import graphar._core; print('GraphAr
imported successfully')"
graphar --help >/dev/null
+
upload_test_pypi:
name: Publish to TestPyPI (auto)
needs: [build_wheels, build_sdist]
@@ -302,6 +298,6 @@ jobs:
mkdir -p python/dist
find dist -name "*" -type f -exec mv {} python/dist/ \;
- name: Publish to PyPI
- uses: pypa/gh-action-pypi-publish@release/v1
+ uses:
pypa/gh-action-pypi-publish@release/2473ec6c6aa87f38946284d51289219fd0b87264
with:
packages-dir: python/dist/
\ No newline at end of file
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
index 02f60634..4a8962e0 100644
--- a/.github/workflows/python.yml
+++ b/.github/workflows/python.yml
@@ -48,7 +48,7 @@ jobs:
GAR_TEST_DATA: ${{ github.workspace }}/graphar-testing/
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
with:
submodules: true
@@ -65,10 +65,10 @@ jobs:
-P /tmp/
sudo apt-get install -y
/tmp/apache-arrow-apt-source-latest-"$(lsb_release --codename --short)".deb
sudo apt-get update -y
- sudo apt install -y libarrow-dev=17.0.0-1 \
- libarrow-dataset-dev=17.0.0-1 \
- libarrow-acero-dev=17.0.0-1 \
- libparquet-dev=17.0.0-1
+ sudo apt install -y libarrow-dev=24.0.0-1 \
+ libarrow-dataset-dev=24.0.0-1 \
+ libarrow-acero-dev=24.0.0-1 \
+ libparquet-dev=24.0.0-1
sudo apt-get install -y ccache libcurl4-openssl-dev
git clone https://github.com/apache/incubator-graphar-testing.git
$GAR_TEST_DATA --depth 1
@@ -115,7 +115,7 @@ jobs:
GAR_TEST_DATA: ${{ github.workspace }}/graphar-testing/
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
with:
submodules: true
diff --git a/.github/workflows/rust.yaml b/.github/workflows/rust.yaml
index c95d2549..ff8d3e8e 100644
--- a/.github/workflows/rust.yaml
+++ b/.github/workflows/rust.yaml
@@ -97,11 +97,11 @@ jobs:
-P /tmp/
sudo apt-get install -y
/tmp/apache-arrow-apt-source-latest-"$(lsb_release --codename --short)".deb
sudo apt-get update -y
- sudo apt install -y libarrow-dev=22.0.0-1 \
- libarrow-dataset-dev=22.0.0-1 \
- libarrow-acero-dev=22.0.0-1 \
- libparquet-dev=22.0.0-1 \
- libarrow-compute-dev=22.0.0-1
+ sudo apt install -y libarrow-dev=24.0.0-1 \
+ libarrow-dataset-dev=24.0.0-1 \
+ libarrow-acero-dev=24.0.0-1 \
+ libparquet-dev=24.0.0-1 \
+ libarrow-compute-dev=24.0.0-1
sudo apt-get install -y libboost-graph-dev ccache
libcurl4-openssl-dev doxygen lcov cmake
- name: Spell Check
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 9936bcbb..e997dd35 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -311,7 +311,7 @@ if(BUILD_ARROW_FROM_SOURCE)
include(apache-arrow)
- if(ARROW_VERSION_TO_BUILD VERSION_GREATER_EQUAL "23.0.0")
+ if(ARROW_VERSION_TO_BUILD VERSION_GREATER_EQUAL "24.0.0")
set(GAR_CXX_STANDARD 20)
endif()
build_arrow()
@@ -349,7 +349,7 @@ else()
endif()
# Update C++ standard based on detected system Arrow version
- if(${Arrow_VERSION} VERSION_GREATER_EQUAL "23.0.0")
+ if(${Arrow_VERSION} VERSION_GREATER_EQUAL "24.0.0")
set(GAR_CXX_STANDARD 20)
message(STATUS "Arrow ${Arrow_VERSION} detected")
endif()
diff --git a/cpp/cmake/apache-arrow.cmake b/cpp/cmake/apache-arrow.cmake
index 9873230a..d35352ce 100644
--- a/cpp/cmake/apache-arrow.cmake
+++ b/cpp/cmake/apache-arrow.cmake
@@ -124,14 +124,14 @@ function(build_arrow)
find_package(Threads)
find_package(Arrow QUIET)
- # Set Arrow version: from env var or default to 23.0.0
+ # Set Arrow version: from env var or default to 24.0.0
if(DEFINED ENV{GRAPHAR_ARROW_VERSION})
set(ARROW_VERSION_TO_BUILD
"$ENV{GRAPHAR_ARROW_VERSION}"
CACHE INTERNAL "arrow version")
else()
set(ARROW_VERSION_TO_BUILD
- "23.0.0"
+ "24.0.0"
CACHE INTERNAL "arrow version")
endif()
diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt
index b82d38a7..6279edb7 100644
--- a/cpp/src/CMakeLists.txt
+++ b/cpp/src/CMakeLists.txt
@@ -129,6 +129,7 @@ else()
endif()
else()
if(USE_STATIC_ARROW)
+ find_package(LibXml2 REQUIRED)
target_link_libraries(graphar
PRIVATE -Wl,--exclude-libs,ALL
-Wl,--whole-archive
@@ -136,7 +137,8 @@ else()
Parquet::parquet_static
ArrowDataset::arrow_dataset_static
ArrowAcero::arrow_acero_static
- -Wl,--no-whole-archive)
+ -Wl,--no-whole-archive
+ LibXml2::LibXml2)
else()
target_link_libraries(graphar
PRIVATE -Wl,--exclude-libs,ALL
diff --git a/cpp/src/graphar/arrow/chunk_reader.cc
b/cpp/src/graphar/arrow/chunk_reader.cc
index 7499936f..71e2ffa9 100644
--- a/cpp/src/graphar/arrow/chunk_reader.cc
+++ b/cpp/src/graphar/arrow/chunk_reader.cc
@@ -73,6 +73,11 @@ Result<std::shared_ptr<arrow::Schema>> LabelToSchema(
Status GeneralCast(const std::shared_ptr<arrow::Array>& in,
const std::shared_ptr<arrow::DataType>& to_type,
std::shared_ptr<arrow::Array>* out) {
+ static bool initialized = false;
+ if (!initialized) {
+ RETURN_NOT_ARROW_OK(arrow::compute::Initialize());
+ initialized = true;
+ }
GAR_RETURN_ON_ARROW_ERROR_AND_ASSIGN(*out,
arrow::compute::Cast(*in, to_type));
return Status::OK();
@@ -110,6 +115,11 @@ Status CastStringToLargeString(const
std::shared_ptr<arrow::Array>& in,
Status CastTableWithSchema(const std::shared_ptr<arrow::Table>& table,
const std::shared_ptr<arrow::Schema>& schema,
std::shared_ptr<arrow::Table>* out_table) {
+ static bool initialized = false;
+ if (!initialized) {
+ RETURN_NOT_ARROW_OK(arrow::compute::Initialize());
+ initialized = true;
+ }
if (table->schema()->Equals(*schema)) {
*out_table = table;
}
diff --git a/cpp/src/graphar/arrow/chunk_writer.cc
b/cpp/src/graphar/arrow/chunk_writer.cc
index 0fd862b2..564ea614 100644
--- a/cpp/src/graphar/arrow/chunk_writer.cc
+++ b/cpp/src/graphar/arrow/chunk_writer.cc
@@ -1021,9 +1021,8 @@ Result<std::shared_ptr<arrow::Table>>
EdgeChunkWriter::getOffsetTable(
Result<std::shared_ptr<arrow::Table>> EdgeChunkWriter::sortTable(
const std::shared_ptr<arrow::Table>& input_table,
const std::string& column_name) {
-#if ARROW_VERSION >= 21000000
RETURN_NOT_ARROW_OK(arrow::compute::Initialize());
-#endif
+ arrow::dataset::internal::Initialize();
auto exec_context = arrow::compute::default_exec_context();
auto plan = arrow_acero_namespace::ExecPlan::Make(exec_context).ValueOrDie();
auto table_source_options =
diff --git a/cpp/src/graphar/expression.cc b/cpp/src/graphar/expression.cc
index 843e9637..18f22392 100644
--- a/cpp/src/graphar/expression.cc
+++ b/cpp/src/graphar/expression.cc
@@ -22,15 +22,27 @@
namespace graphar {
+Status EnsureComputeInitialized() {
+ static bool initialized = false;
+ if (!initialized) {
+ RETURN_NOT_ARROW_OK(arrow::compute::Initialize());
+ initialized = true;
+ }
+ return Status::OK();
+}
+
Result<ArrowExpression> ExpressionProperty::Evaluate() {
+ GAR_RETURN_NOT_OK(EnsureComputeInitialized());
return arrow::compute::field_ref(property_.name);
}
Result<ArrowExpression> ExpressionNot::Evaluate() {
+ GAR_RETURN_NOT_OK(EnsureComputeInitialized());
GAR_ASSIGN_OR_RAISE(auto expr, expr_->Evaluate());
return arrow::compute::not_(expr);
}
Result<ArrowExpression> ExpressionEqual::Evaluate() {
+ GAR_RETURN_NOT_OK(EnsureComputeInitialized());
GAR_RETURN_NOT_OK(CheckNullArgs(lhs_, rhs_));
GAR_ASSIGN_OR_RAISE(auto lexpr, lhs_->Evaluate());
GAR_ASSIGN_OR_RAISE(auto rexpr, rhs_->Evaluate());
@@ -38,6 +50,7 @@ Result<ArrowExpression> ExpressionEqual::Evaluate() {
}
Result<ArrowExpression> ExpressionNotEqual::Evaluate() {
+ GAR_RETURN_NOT_OK(EnsureComputeInitialized());
GAR_RETURN_NOT_OK(CheckNullArgs(lhs_, rhs_));
GAR_ASSIGN_OR_RAISE(auto lexpr, lhs_->Evaluate());
GAR_ASSIGN_OR_RAISE(auto rexpr, rhs_->Evaluate());
@@ -45,6 +58,7 @@ Result<ArrowExpression> ExpressionNotEqual::Evaluate() {
}
Result<ArrowExpression> ExpressionGreaterThan::Evaluate() {
+ GAR_RETURN_NOT_OK(EnsureComputeInitialized());
GAR_RETURN_NOT_OK(CheckNullArgs(lhs_, rhs_));
GAR_ASSIGN_OR_RAISE(auto lexpr, lhs_->Evaluate());
GAR_ASSIGN_OR_RAISE(auto rexpr, rhs_->Evaluate());
@@ -52,6 +66,7 @@ Result<ArrowExpression> ExpressionGreaterThan::Evaluate() {
}
Result<ArrowExpression> ExpressionGreaterEqual::Evaluate() {
+ GAR_RETURN_NOT_OK(EnsureComputeInitialized());
GAR_RETURN_NOT_OK(CheckNullArgs(lhs_, rhs_));
GAR_ASSIGN_OR_RAISE(auto lexpr, lhs_->Evaluate());
GAR_ASSIGN_OR_RAISE(auto rexpr, rhs_->Evaluate());
@@ -59,6 +74,7 @@ Result<ArrowExpression> ExpressionGreaterEqual::Evaluate() {
}
Result<ArrowExpression> ExpressionLessThan::Evaluate() {
+ GAR_RETURN_NOT_OK(EnsureComputeInitialized());
GAR_RETURN_NOT_OK(CheckNullArgs(lhs_, rhs_));
GAR_ASSIGN_OR_RAISE(auto lexpr, lhs_->Evaluate());
GAR_ASSIGN_OR_RAISE(auto rexpr, rhs_->Evaluate());
@@ -66,6 +82,7 @@ Result<ArrowExpression> ExpressionLessThan::Evaluate() {
}
Result<ArrowExpression> ExpressionLessEqual::Evaluate() {
+ GAR_RETURN_NOT_OK(EnsureComputeInitialized());
GAR_RETURN_NOT_OK(CheckNullArgs(lhs_, rhs_));
GAR_ASSIGN_OR_RAISE(auto lexpr, lhs_->Evaluate());
GAR_ASSIGN_OR_RAISE(auto rexpr, rhs_->Evaluate());
@@ -73,6 +90,7 @@ Result<ArrowExpression> ExpressionLessEqual::Evaluate() {
}
Result<ArrowExpression> ExpressionAnd::Evaluate() {
+ GAR_RETURN_NOT_OK(EnsureComputeInitialized());
GAR_RETURN_NOT_OK(CheckNullArgs(lhs_, rhs_));
GAR_ASSIGN_OR_RAISE(auto lexpr, lhs_->Evaluate());
GAR_ASSIGN_OR_RAISE(auto rexpr, rhs_->Evaluate());
@@ -80,6 +98,7 @@ Result<ArrowExpression> ExpressionAnd::Evaluate() {
}
Result<ArrowExpression> ExpressionOr::Evaluate() {
+ GAR_RETURN_NOT_OK(EnsureComputeInitialized());
GAR_RETURN_NOT_OK(CheckNullArgs(lhs_, rhs_));
GAR_ASSIGN_OR_RAISE(auto lexpr, lhs_->Evaluate());
GAR_ASSIGN_OR_RAISE(auto rexpr, rhs_->Evaluate());
diff --git a/cpp/src/graphar/expression.h b/cpp/src/graphar/expression.h
index a917e333..85633fa1 100644
--- a/cpp/src/graphar/expression.h
+++ b/cpp/src/graphar/expression.h
@@ -89,6 +89,8 @@ class ExpressionLiteral : public Expression {
~ExpressionLiteral() = default;
Result<ArrowExpression> Evaluate() override {
+ extern Status EnsureComputeInitialized();
+ GAR_RETURN_NOT_OK(EnsureComputeInitialized());
return arrow::compute::literal(value_);
}
diff --git a/cpp/src/graphar/filesystem.cc b/cpp/src/graphar/filesystem.cc
index e1a5ef56..69e27208 100644
--- a/cpp/src/graphar/filesystem.cc
+++ b/cpp/src/graphar/filesystem.cc
@@ -27,6 +27,7 @@
#include "arrow/api.h"
#include "arrow/csv/api.h"
#include "arrow/dataset/api.h"
+#include "arrow/dataset/plan.h"
#include "parquet/arrow/reader.h"
#if defined(ARROW_VERSION) && ARROW_VERSION <= 12000000
#include "arrow/dataset/file_json.h"
@@ -93,6 +94,23 @@ static Status CastToLargeOffsetArray(
namespace graphar {
namespace ds = arrow::dataset;
+namespace {
+Status EnsureDatasetScannerInitialized() {
+ static bool initialized = false;
+ static Status init_status = Status::OK();
+ if (!initialized) {
+ auto st = arrow::compute::Initialize();
+ if (!st.ok()) {
+ init_status = Status::ArrowError(st.ToString());
+ } else {
+ arrow::dataset::internal::Initialize();
+ }
+ initialized = true;
+ }
+ return init_status;
+}
+} // namespace
+
std::shared_ptr<ds::FileFormat> FileSystem::GetFileFormat(
const FileType type) const {
switch (type) {
@@ -114,6 +132,7 @@ std::shared_ptr<ds::FileFormat> FileSystem::GetFileFormat(
Result<std::shared_ptr<arrow::Table>> FileSystem::ReadFileToTable(
const std::string& path, FileType file_type,
const std::vector<int>& column_indices) const noexcept {
+ GAR_RETURN_NOT_OK(EnsureDatasetScannerInitialized());
parquet::arrow::FileReaderBuilder builder;
auto open_file_status = builder.OpenFile(path);
if (!open_file_status.ok()) {
@@ -122,23 +141,19 @@ Result<std::shared_ptr<arrow::Table>>
FileSystem::ReadFileToTable(
}
builder.memory_pool(arrow::default_memory_pool());
GAR_RETURN_ON_ARROW_ERROR_AND_ASSIGN(auto reader, builder.Build());
- std::shared_ptr<arrow::Table> table;
- arrow::Status read_status;
if (column_indices.empty()) {
- read_status = reader->ReadTable(&table);
- } else {
- read_status = reader->ReadTable(column_indices, &table);
- }
- if (!read_status.ok()) {
- return Status::Invalid("Failed to read table from file: ", path, " - ",
- read_status.ToString());
+ GAR_RETURN_ON_ARROW_ERROR_AND_ASSIGN(auto table, reader->ReadTable());
+ return table;
}
+ GAR_RETURN_ON_ARROW_ERROR_AND_ASSIGN(auto table,
+ reader->ReadTable(column_indices));
return table;
}
Result<std::shared_ptr<arrow::Table>> FileSystem::ReadFileToTable(
const std::string& path, FileType file_type,
const util::FilterOptions& options) const noexcept {
+ GAR_RETURN_NOT_OK(EnsureDatasetScannerInitialized());
std::shared_ptr<ds::FileFormat> format = GetFileFormat(file_type);
GAR_RETURN_ON_ARROW_ERROR_AND_ASSIGN(
auto factory, arrow::dataset::FileSystemDatasetFactory::Make(
@@ -146,9 +161,6 @@ Result<std::shared_ptr<arrow::Table>>
FileSystem::ReadFileToTable(
arrow::dataset::FileSystemFactoryOptions()));
GAR_RETURN_ON_ARROW_ERROR_AND_ASSIGN(auto dataset, factory->Finish());
GAR_RETURN_ON_ARROW_ERROR_AND_ASSIGN(auto scan_builder, dataset->NewScan());
-#if ARROW_VERSION >= 21000000
- RETURN_NOT_ARROW_OK(arrow::compute::Initialize());
-#endif
// Apply the row filter and select the specified columns
if (options.filter) {
GAR_ASSIGN_OR_RAISE(auto filter, options.filter->Evaluate());
diff --git a/cpp/src/graphar/high-level/graph_reader.cc
b/cpp/src/graphar/high-level/graph_reader.cc
index 284311eb..757f38dd 100644
--- a/cpp/src/graphar/high-level/graph_reader.cc
+++ b/cpp/src/graphar/high-level/graph_reader.cc
@@ -21,6 +21,8 @@
#include <algorithm>
#include <unordered_set>
#include "arrow/array.h"
+#include "arrow/dataset/dataset.h"
+#include "arrow/dataset/plan.h"
#include "graphar/api/arrow_reader.h"
#include "graphar/convert_to_arrow_type.h"
#include "graphar/label.h"
@@ -940,6 +942,12 @@ Result<std::shared_ptr<EdgesCollection>>
EdgesCollection::Make(
const std::string& edge_type, const std::string& dst_type,
AdjListType adj_list_type, const IdType vertex_chunk_begin,
const IdType vertex_chunk_end) noexcept {
+ static bool initialized = false;
+ if (!initialized) {
+ RETURN_NOT_ARROW_OK(arrow::compute::Initialize());
+ arrow::dataset::internal::Initialize();
+ initialized = true;
+ }
auto edge_info = graph_info->GetEdgeInfo(src_type, edge_type, dst_type);
if (!edge_info) {
return Status::KeyError("The edge ", src_type, " ", edge_type, " ",
diff --git a/cpp/src/graphar/reader_util.cc b/cpp/src/graphar/reader_util.cc
index 0048c2bc..df1f2791 100644
--- a/cpp/src/graphar/reader_util.cc
+++ b/cpp/src/graphar/reader_util.cc
@@ -45,6 +45,11 @@ namespace graphar::util {
Status CheckFilterOptions(
const FilterOptions& filter_options,
const std::shared_ptr<PropertyGroup>& property_group) noexcept {
+ static bool initialized = false;
+ if (!initialized) {
+ RETURN_NOT_ARROW_OK(arrow::compute::Initialize());
+ initialized = true;
+ }
if (filter_options.filter) {
GAR_ASSIGN_OR_RAISE(auto filter, filter_options.filter->Evaluate());
for (const auto& field : arrow::compute::FieldsInExpression(filter)) {
diff --git a/cpp/test/test_arrow_chunk_writer.cc
b/cpp/test/test_arrow_chunk_writer.cc
index d3926cf7..b3522705 100644
--- a/cpp/test/test_arrow_chunk_writer.cc
+++ b/cpp/test/test_arrow_chunk_writer.cc
@@ -145,8 +145,9 @@ TEST_CASE_METHOD(GlobalFixture, "TestVertexPropertyWriter")
{
st = graphar::util::OpenParquetArrowReader(path2, pool, &arrow_reader);
// Read entire file as a single Arrow table
- std::shared_ptr<arrow::Table> table2;
- st = arrow_reader->ReadTable(&table2);
+ auto maybe_table2 = arrow_reader->ReadTable();
+ REQUIRE(maybe_table2.ok());
+ auto table2 = maybe_table2.ValueOrDie();
REQUIRE(table1->GetColumnByName("firstName")->ToString() ==
table2->GetColumnByName("firstName")->ToString());
@@ -217,9 +218,9 @@ TEST_CASE_METHOD(GlobalFixture, "TestVertexPropertyWriter")
{
auto st = graphar::util::OpenParquetArrowReader(
parquet_file, arrow::default_memory_pool(), &parquet_reader);
REQUIRE(st.ok());
- std::shared_ptr<arrow::Table> parquet_table;
- st = parquet_reader->ReadTable(&parquet_table);
- REQUIRE(st.ok());
+ auto maybe_parquet_table = parquet_reader->ReadTable();
+ REQUIRE(maybe_parquet_table.ok());
+ auto parquet_table = maybe_parquet_table.ValueOrDie();
auto parquet_metadata = parquet_reader->parquet_reader()->metadata();
auto row_group_meta = parquet_metadata->RowGroup(0);
auto col_meta = row_group_meta->ColumnChunk(0);
@@ -280,12 +281,11 @@ TEST_CASE_METHOD(GlobalFixture, "TestEdgeChunkWriter") {
std::unique_ptr<parquet::arrow::FileReader> arrow_reader;
st = graphar::util::OpenParquetArrowReader(path, pool, &arrow_reader);
// Read entire file as a single Arrow table
- std::shared_ptr<arrow::Table> maybe_table;
- st = arrow_reader->ReadTable(&maybe_table);
- REQUIRE(st.ok());
+ auto maybe_table = arrow_reader->ReadTable();
+ REQUIRE(maybe_table.ok());
std::shared_ptr<arrow::Table> table =
- maybe_table
+ maybe_table.ValueOrDie()
->RenameColumns(
{GeneralParams::kSrcIndexCol, GeneralParams::kDstIndexCol})
.ValueOrDie();
@@ -334,7 +334,7 @@ TEST_CASE_METHOD(GlobalFixture, "TestEdgeChunkWriter") {
"/tmp/edge/person_knows_person/ordered_by_source/vertex_count")
.ValueOrDie();
auto vertex_num = input3->Read(sizeof(IdType)).ValueOrDie();
- const IdType* vertex_num_ptr =
+ const auto* vertex_num_ptr =
reinterpret_cast<const IdType*>(vertex_num->data());
REQUIRE((*vertex_num_ptr) == 903);
@@ -429,9 +429,9 @@ TEST_CASE_METHOD(GlobalFixture, "TestEdgeChunkWriter") {
auto st = graphar::util::OpenParquetArrowReader(
parquet_file, arrow::default_memory_pool(), &parquet_reader);
REQUIRE(st.ok());
- std::shared_ptr<arrow::Table> parquet_table;
- st = parquet_reader->ReadTable(&parquet_table);
- REQUIRE(st.ok());
+ auto maybe_parquet_table = parquet_reader->ReadTable();
+ REQUIRE(maybe_parquet_table.ok());
+ auto parquet_table = maybe_parquet_table.ValueOrDie();
auto parquet_metadata = parquet_reader->parquet_reader()->metadata();
auto row_group_meta = parquet_metadata->RowGroup(0);
auto col_meta = row_group_meta->ColumnChunk(0);
diff --git a/cpp/test/test_builder.cc b/cpp/test/test_builder.cc
index 404427c6..9a25b5af 100644
--- a/cpp/test/test_builder.cc
+++ b/cpp/test/test_builder.cc
@@ -38,6 +38,7 @@
#include "graphar/api/high_level_writer.h"
#include <catch2/catch_test_macros.hpp>
+
namespace graphar {
TEST_CASE_METHOD(GlobalFixture, "Test_vertices_builder") {
std::cout << "Test vertex builder" << std::endl;
@@ -137,8 +138,9 @@ TEST_CASE_METHOD(GlobalFixture, "Test_vertices_builder") {
REQUIRE(graphar::util::OpenParquetArrowReader(
parquet_file, arrow::default_memory_pool(), &parquet_reader)
.ok());
- std::shared_ptr<arrow::Table> parquet_table;
- REQUIRE(parquet_reader->ReadTable(&parquet_table).ok());
+ auto maybe_parquet_table = parquet_reader->ReadTable();
+ REQUIRE(maybe_parquet_table.ok());
+ auto parquet_table = maybe_parquet_table.ValueOrDie();
auto parquet_metadata = parquet_reader->parquet_reader()->metadata();
auto row_group_meta = parquet_metadata->RowGroup(0);
auto col_meta = row_group_meta->ColumnChunk(0);
@@ -242,8 +244,9 @@ TEST_CASE_METHOD(GlobalFixture, "test_edges_builder") {
REQUIRE(graphar::util::OpenParquetArrowReader(
parquet_file, arrow::default_memory_pool(), &parquet_reader)
.ok());
- std::shared_ptr<arrow::Table> parquet_table;
- REQUIRE(parquet_reader->ReadTable(&parquet_table).ok());
+ auto maybe_parquet_table = parquet_reader->ReadTable();
+ REQUIRE(maybe_parquet_table.ok());
+ auto parquet_table = maybe_parquet_table.ValueOrDie();
auto parquet_metadata = parquet_reader->parquet_reader()->metadata();
auto row_group_meta = parquet_metadata->RowGroup(0);
auto col_meta = row_group_meta->ColumnChunk(0);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
