This is an automated email from the ASF dual-hosted git repository. taiyang-li pushed a commit to branch fake_add_bolt_backend in repository https://gitbox.apache.org/repos/asf/gluten.git
commit abe3f09666d6b1e98e8febf18d29f925154e3126 Author: liyang.127 <[email protected]> AuthorDate: Tue Jun 30 11:19:43 2026 +0800 [GLUTEN][CORE] Extract shared VariantToVectorConverter and Jol filesystem registration into cpp/core Move the backend-agnostic parts of VariantToVectorConverter and the registerJolFileSystem declaration into cpp/core so future backend rebases don't conflict on these files. The Velox backend keeps its original entry names and only thin-wraps the shared implementation (backend-specific dispatch / unsupported macros are injected via GLUTEN_* aliases), so Velox behavior is unchanged. Generated-by: TraeCli openrouter-3o --- cpp/{velox => core}/jni/JniFileSystem.h | 9 +--- .../substrait/VariantToVectorConverter.cc | 13 +++-- .../substrait/VariantToVectorConverter.h | 6 +-- cpp/velox/jni/JniFileSystem.h | 7 +-- cpp/velox/substrait/VariantToVectorConverter.cc | 55 +++------------------- cpp/velox/substrait/VariantToVectorConverter.h | 8 +--- 6 files changed, 22 insertions(+), 76 deletions(-) diff --git a/cpp/velox/jni/JniFileSystem.h b/cpp/core/jni/JniFileSystem.h similarity index 86% copy from cpp/velox/jni/JniFileSystem.h copy to cpp/core/jni/JniFileSystem.h index 11c0f27b36..ada83b68a7 100644 --- a/cpp/velox/jni/JniFileSystem.h +++ b/cpp/core/jni/JniFileSystem.h @@ -17,10 +17,7 @@ #pragma once -#include <jni.h> - -#include "velox/common/file/File.h" -#include "velox/common/file/FileSystems.h" +#include <cstdint> namespace gluten { @@ -29,8 +26,4 @@ namespace gluten { // local fs directly. void registerJolFileSystem(uint64_t maxFileSize); -void initVeloxJniFileSystem(JNIEnv* env); - -void finalizeVeloxJniFileSystem(JNIEnv* env); - } // namespace gluten diff --git a/cpp/velox/substrait/VariantToVectorConverter.cc b/cpp/core/substrait/VariantToVectorConverter.cc similarity index 82% copy from cpp/velox/substrait/VariantToVectorConverter.cc copy to cpp/core/substrait/VariantToVectorConverter.cc index f33633b91e..86a80e8772 100644 --- a/cpp/velox/substrait/VariantToVectorConverter.cc +++ b/cpp/core/substrait/VariantToVectorConverter.cc @@ -16,7 +16,14 @@ */ #include "VariantToVectorConverter.h" -#include "velox/vector/FlatVector.h" + +#ifndef GLUTEN_DYNAMIC_SCALAR_TYPE_DISPATCH +#error "GLUTEN_DYNAMIC_SCALAR_TYPE_DISPATCH must be defined before including VariantToVectorConverter.cc" +#endif + +#ifndef GLUTEN_UNSUPPORTED +#error "GLUTEN_UNSUPPORTED must be defined before including VariantToVectorConverter.cc" +#endif namespace gluten { @@ -43,7 +50,7 @@ VectorPtr setVectorFromVariantsByKind<TypeKind::VARBINARY>( const std::vector<variant>& /* values */, const TypePtr& /*type*/, memory::MemoryPool* /* pool */) { - VELOX_UNSUPPORTED("Return of VARBINARY data is not supported"); + GLUTEN_UNSUPPORTED("Return of VARBINARY data is not supported"); } template <> @@ -65,7 +72,7 @@ VectorPtr setVectorFromVariantsByKind<TypeKind::VARCHAR>( } // namespace VectorPtr setVectorFromVariants(const TypePtr& type, const std::vector<variant>& values, memory::MemoryPool* pool) { - return VELOX_DYNAMIC_SCALAR_TYPE_DISPATCH(setVectorFromVariantsByKind, type->kind(), values, type, pool); + return GLUTEN_DYNAMIC_SCALAR_TYPE_DISPATCH(setVectorFromVariantsByKind, type->kind(), values, type, pool); } } // namespace gluten diff --git a/cpp/velox/substrait/VariantToVectorConverter.h b/cpp/core/substrait/VariantToVectorConverter.h similarity index 89% copy from cpp/velox/substrait/VariantToVectorConverter.h copy to cpp/core/substrait/VariantToVectorConverter.h index 5d034884c9..5c27e0ad52 100644 --- a/cpp/velox/substrait/VariantToVectorConverter.h +++ b/cpp/core/substrait/VariantToVectorConverter.h @@ -17,13 +17,11 @@ #pragma once -#include "velox/vector/BaseVector.h" - -using namespace facebook::velox; +#include <vector> namespace gluten { -/// Create Base Vector from velox variants. +/// Create Base Vector from backend variants. /// Only scalar types are supported except VARBINARY. VectorPtr setVectorFromVariants(const TypePtr& type, const std::vector<variant>& values, memory::MemoryPool* pool); diff --git a/cpp/velox/jni/JniFileSystem.h b/cpp/velox/jni/JniFileSystem.h index 11c0f27b36..c3a5f3351a 100644 --- a/cpp/velox/jni/JniFileSystem.h +++ b/cpp/velox/jni/JniFileSystem.h @@ -22,12 +22,9 @@ #include "velox/common/file/File.h" #include "velox/common/file/FileSystems.h" -namespace gluten { +#include "../../core/jni/JniFileSystem.h" -// Register JNI-or-local (or JVM-over-local, as long as it describes what happens here) file system. maxFileSize is -// necessary (!= 0) because we use this size to decide whether a new file can fit in JVM heap, otherwise we write it via -// local fs directly. -void registerJolFileSystem(uint64_t maxFileSize); +namespace gluten { void initVeloxJniFileSystem(JNIEnv* env); diff --git a/cpp/velox/substrait/VariantToVectorConverter.cc b/cpp/velox/substrait/VariantToVectorConverter.cc index f33633b91e..aa80237a44 100644 --- a/cpp/velox/substrait/VariantToVectorConverter.cc +++ b/cpp/velox/substrait/VariantToVectorConverter.cc @@ -16,56 +16,13 @@ */ #include "VariantToVectorConverter.h" -#include "velox/vector/FlatVector.h" - -namespace gluten { - -namespace { -template <TypeKind KIND> -VectorPtr -setVectorFromVariantsByKind(const std::vector<variant>& values, const TypePtr& type, memory::MemoryPool* pool) { - using T = typename TypeTraits<KIND>::NativeType; - - auto flatVector = BaseVector::create<FlatVector<T>>(type, values.size(), pool); - for (vector_size_t i = 0; i < values.size(); i++) { - if (values[i].isNull()) { - flatVector->setNull(i, true); - } else { - flatVector->set(i, values[i].value<T>()); - } - } - return flatVector; -} - -template <> -VectorPtr setVectorFromVariantsByKind<TypeKind::VARBINARY>( - const std::vector<variant>& /* values */, - const TypePtr& /*type*/, - memory::MemoryPool* /* pool */) { - VELOX_UNSUPPORTED("Return of VARBINARY data is not supported"); -} - -template <> -VectorPtr setVectorFromVariantsByKind<TypeKind::VARCHAR>( - const std::vector<variant>& values, - const TypePtr& type, - memory::MemoryPool* pool) { - auto flatVector = BaseVector::create<FlatVector<StringView>>(type, values.size(), pool); +#include "velox/vector/FlatVector.h" - for (vector_size_t i = 0; i < values.size(); i++) { - if (values[i].isNull()) { - flatVector->setNull(i, true); - } else { - flatVector->set(i, StringView(values[i].value<TypeKind::VARCHAR>())); - } - } - return flatVector; -} -} // namespace +#define GLUTEN_DYNAMIC_SCALAR_TYPE_DISPATCH VELOX_DYNAMIC_SCALAR_TYPE_DISPATCH +#define GLUTEN_UNSUPPORTED VELOX_UNSUPPORTED -VectorPtr setVectorFromVariants(const TypePtr& type, const std::vector<variant>& values, memory::MemoryPool* pool) { - return VELOX_DYNAMIC_SCALAR_TYPE_DISPATCH(setVectorFromVariantsByKind, type->kind(), values, type, pool); -} +#include "../../core/substrait/VariantToVectorConverter.cc" -} // namespace gluten +#undef GLUTEN_DYNAMIC_SCALAR_TYPE_DISPATCH +#undef GLUTEN_UNSUPPORTED diff --git a/cpp/velox/substrait/VariantToVectorConverter.h b/cpp/velox/substrait/VariantToVectorConverter.h index 5d034884c9..262dab080e 100644 --- a/cpp/velox/substrait/VariantToVectorConverter.h +++ b/cpp/velox/substrait/VariantToVectorConverter.h @@ -21,10 +21,4 @@ using namespace facebook::velox; -namespace gluten { - -/// Create Base Vector from velox variants. -/// Only scalar types are supported except VARBINARY. -VectorPtr setVectorFromVariants(const TypePtr& type, const std::vector<variant>& values, memory::MemoryPool* pool); - -} // namespace gluten +#include "../../core/substrait/VariantToVectorConverter.h" --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
