This is an automated email from the ASF dual-hosted git repository. taiyang-li pushed a commit to branch fake_add_bolt_backend in repository https://gitbox.apache.org/repos/asf/gluten.git
commit e04ac1d151ce26f78ff80a2e40abe5c9d41985ae Author: liyang.127 <[email protected]> AuthorDate: Mon Jun 29 21:49:18 2026 +0800 [GLUTEN][CORE] Extract more shared cpp helper headers into cpp/core Move another low-risk batch of identical Bolt/Velox helper definitions into cpp/core, covering UDF/UDAF ABI headers, function registration helpers, the shared datasource abstraction, common test helper headers, JSON-to-protobuf parsing helpers, cudf validator declaration and shared JNI UDF glue. Keep the Velox-side files as thin wrappers or aliases so existing include paths and backend-specific entry points stay unchanged while future Bolt rebases stop conflicting on the duplicated bodies. --- cpp/core/CMakeLists.txt | 1 + cpp/{velox => core}/cudf/CudfPlanValidator.h | 2 - cpp/{velox => core}/jni/JniUdf.cc | 16 +++-- cpp/{velox => core}/jni/JniUdf.h | 6 +- .../operators/functions/Arithmetic.h | 3 + .../operators/functions/RegistrationAllFunctions.h | 0 .../operators/writer/GlutenDataSource.h} | 10 ++- cpp/{velox => core}/tests/FilePathGenerator.h | 0 cpp/{velox => core}/tests/JsonToProtoConverter.cc | 18 ++--- .../utils => core/tests}/JsonToProtoConverter.h | 2 + .../tests/utils/TestAllocationListener.h | 3 +- cpp/{velox => core}/tests/utils/TestStreamReader.h | 3 + cpp/{velox => core}/tests/utils/TestUtils.h | 3 + cpp/{velox => core}/udf/Udaf.h | 0 cpp/{velox => core}/udf/Udf.h | 0 cpp/{velox => core}/utils/JsonToProtoConverter.h | 2 + cpp/velox/cudf/CudfPlanValidator.h | 11 +--- cpp/velox/jni/JniUdf.cc | 76 ++-------------------- cpp/velox/jni/JniUdf.h | 6 +- cpp/velox/operators/functions/Arithmetic.h | 51 +-------------- .../operators/functions/RegistrationAllFunctions.h | 6 +- cpp/velox/operators/writer/VeloxDataSource.h | 26 +------- cpp/velox/tests/FilePathGenerator.h | 7 +- cpp/velox/tests/JsonToProtoConverter.cc | 20 +----- cpp/velox/tests/JsonToProtoConverter.h | 8 +-- cpp/velox/tests/utils/TestAllocationListener.h | 48 +------------- cpp/velox/tests/utils/TestStreamReader.h | 19 +----- cpp/velox/tests/utils/TestUtils.h | 8 +-- cpp/velox/udf/Udaf.h | 25 +------ cpp/velox/udf/Udf.h | 24 +------ cpp/velox/utils/JsonToProtoConverter.h | 8 +-- 31 files changed, 67 insertions(+), 345 deletions(-) diff --git a/cpp/core/CMakeLists.txt b/cpp/core/CMakeLists.txt index d976e92bd0..936bccaa2e 100644 --- a/cpp/core/CMakeLists.txt +++ b/cpp/core/CMakeLists.txt @@ -124,6 +124,7 @@ set(SPARK_COLUMNAR_PLUGIN_SRCS substrait/SubstraitExtensionRegistry.cc config/GlutenConfig.cc jni/JniWrapper.cc + jni/JniUdf.cc memory/AllocationListener.cc memory/MemoryAllocator.cc memory/MemoryManager.cc diff --git a/cpp/velox/cudf/CudfPlanValidator.h b/cpp/core/cudf/CudfPlanValidator.h similarity index 88% copy from cpp/velox/cudf/CudfPlanValidator.h copy to cpp/core/cudf/CudfPlanValidator.h index de4fb150e0..25a30b60b9 100644 --- a/cpp/velox/cudf/CudfPlanValidator.h +++ b/cpp/core/cudf/CudfPlanValidator.h @@ -17,13 +17,11 @@ #pragma once -#include "memory/VeloxMemoryManager.h" #include "substrait/plan.pb.h" namespace gluten { class CudfPlanValidator { public: - // Validate if the plan contains cudf unsupported operator except TableScan and ValueStream. static bool validate(const ::substrait::Plan& substraitPlan); }; } // namespace gluten diff --git a/cpp/velox/jni/JniUdf.cc b/cpp/core/jni/JniUdf.cc similarity index 94% copy from cpp/velox/jni/JniUdf.cc copy to cpp/core/jni/JniUdf.cc index 32514e7ae7..ff9a8f3b31 100644 --- a/cpp/velox/jni/JniUdf.cc +++ b/cpp/core/jni/JniUdf.cc @@ -16,6 +16,9 @@ */ #include "JniUdf.h" + +#include <string> + #include "jni/JniCommon.h" #include "udf/UdfLoader.h" #include "utils/Exception.h" @@ -32,24 +35,23 @@ static jmethodID registerUDAFMethod; } // namespace -void gluten::initVeloxJniUDF(JNIEnv* env) { +namespace gluten { + +void initJniUDF(JNIEnv* env) { if (env->GetJavaVM(&vm) != JNI_OK) { throw gluten::GlutenException("Unable to get JavaVM instance"); } - // classes udfResolverClass = createGlobalClassReferenceOrError(env, kUdfResolverClassPath.c_str()); - - // methods registerUDFMethod = getMethodIdOrError(env, udfResolverClass, "registerUDF", "(Ljava/lang/String;[B[BZZ)V"); registerUDAFMethod = getMethodIdOrError(env, udfResolverClass, "registerUDAF", "(Ljava/lang/String;[B[B[BZZ)V"); } -void gluten::finalizeVeloxJniUDF(JNIEnv* env) { +void finalizeJniUDF(JNIEnv* env) { env->DeleteGlobalRef(udfResolverClass); } -void gluten::jniRegisterFunctionSignatures(JNIEnv* env) { +void jniRegisterFunctionSignatures(JNIEnv* env) { auto udfLoader = gluten::UdfLoader::getInstance(); const auto& signatures = udfLoader->getRegisteredUdfSignatures(); @@ -92,3 +94,5 @@ void gluten::jniRegisterFunctionSignatures(JNIEnv* env) { checkException(env); } } + +} // namespace gluten diff --git a/cpp/velox/jni/JniUdf.h b/cpp/core/jni/JniUdf.h similarity index 88% copy from cpp/velox/jni/JniUdf.h copy to cpp/core/jni/JniUdf.h index 568439d184..dbc72c4865 100644 --- a/cpp/velox/jni/JniUdf.h +++ b/cpp/core/jni/JniUdf.h @@ -18,14 +18,12 @@ #pragma once #include <jni.h> -#include <string> -#include <unordered_map> namespace gluten { -void initVeloxJniUDF(JNIEnv* env); +void initJniUDF(JNIEnv* env); -void finalizeVeloxJniUDF(JNIEnv* env); +void finalizeJniUDF(JNIEnv* env); void jniRegisterFunctionSignatures(JNIEnv* env); diff --git a/cpp/velox/operators/functions/Arithmetic.h b/cpp/core/operators/functions/Arithmetic.h similarity index 99% copy from cpp/velox/operators/functions/Arithmetic.h copy to cpp/core/operators/functions/Arithmetic.h index 9b2e22ac41..c9487c109b 100644 --- a/cpp/velox/operators/functions/Arithmetic.h +++ b/cpp/core/operators/functions/Arithmetic.h @@ -14,6 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +#pragma once + #include <folly/CPortability.h> #include <stdint.h> #include <cmath> diff --git a/cpp/velox/operators/functions/RegistrationAllFunctions.h b/cpp/core/operators/functions/RegistrationAllFunctions.h similarity index 100% copy from cpp/velox/operators/functions/RegistrationAllFunctions.h copy to cpp/core/operators/functions/RegistrationAllFunctions.h diff --git a/cpp/velox/operators/writer/VeloxDataSource.h b/cpp/core/operators/writer/GlutenDataSource.h similarity index 87% copy from cpp/velox/operators/writer/VeloxDataSource.h copy to cpp/core/operators/writer/GlutenDataSource.h index 043f1b8b17..74ad1f180d 100644 --- a/cpp/velox/operators/writer/VeloxDataSource.h +++ b/cpp/core/operators/writer/GlutenDataSource.h @@ -17,6 +17,10 @@ #pragma once +#include <memory> +#include <string> +#include <unordered_map> + #include <arrow/c/abi.h> #include <arrow/memory_pool.h> #include <arrow/record_batch.h> @@ -27,12 +31,12 @@ namespace gluten { -class VeloxDataSource { +class GlutenDataSource { public: - VeloxDataSource(const std::string& filePath, std::shared_ptr<arrow::Schema> schema) + GlutenDataSource(const std::string& filePath, std::shared_ptr<arrow::Schema> schema) : filePath_(filePath), schema_(schema) {} - virtual ~VeloxDataSource() = default; + virtual ~GlutenDataSource() = default; virtual void init(const std::unordered_map<std::string, std::string>& sparkConfs) {} virtual void inspectSchema(struct ArrowSchema* out) = 0; diff --git a/cpp/velox/tests/FilePathGenerator.h b/cpp/core/tests/FilePathGenerator.h similarity index 100% copy from cpp/velox/tests/FilePathGenerator.h copy to cpp/core/tests/FilePathGenerator.h diff --git a/cpp/velox/tests/JsonToProtoConverter.cc b/cpp/core/tests/JsonToProtoConverter.cc similarity index 75% copy from cpp/velox/tests/JsonToProtoConverter.cc copy to cpp/core/tests/JsonToProtoConverter.cc index 4ed1372d1f..a5498f755e 100644 --- a/cpp/velox/tests/JsonToProtoConverter.cc +++ b/cpp/core/tests/JsonToProtoConverter.cc @@ -15,22 +15,24 @@ * limitations under the License. */ -#include "utils//JsonToProtoConverter.h" +#include "../utils/JsonToProtoConverter.h" + +#include <cerrno> +#include <cstring> #include <fstream> #include <sstream> -#include "velox/common/base/Exceptions.h" + +#include "../utils/Exception.h" void JsonToProtoConverter::readFromFile(const std::string& msgPath, google::protobuf::Message& msg) { - // Read json file and resume the Substrait plan. std::ifstream msgJson(msgPath); - VELOX_CHECK(!msgJson.fail(), "Failed to open file: {}. {}", msgPath, strerror(errno)); + GLUTEN_CHECK(!msgJson.fail(), std::string("Failed to open file: ") + msgPath + ". " + strerror(errno)); std::stringstream buffer; buffer << msgJson.rdbuf(); std::string msgData = buffer.str(); auto status = google::protobuf::util::JsonStringToMessage(msgData, &msg); - VELOX_CHECK( + GLUTEN_CHECK( status.ok(), - "Failed to parse Substrait JSON: {} {}", - static_cast<int8_t>(status.code()), - status.message().ToString()); + "Failed to parse Substrait JSON: " + std::to_string(static_cast<int>(status.code())) + " " + + status.message().ToString()); } diff --git a/cpp/velox/utils/JsonToProtoConverter.h b/cpp/core/tests/JsonToProtoConverter.h similarity index 98% copy from cpp/velox/utils/JsonToProtoConverter.h copy to cpp/core/tests/JsonToProtoConverter.h index f9c41bb695..0ef9cd4dd8 100644 --- a/cpp/velox/utils/JsonToProtoConverter.h +++ b/cpp/core/tests/JsonToProtoConverter.h @@ -17,6 +17,8 @@ #pragma once +#include <string> + #include <google/protobuf/util/json_util.h> class JsonToProtoConverter { diff --git a/cpp/velox/tests/utils/TestAllocationListener.h b/cpp/core/tests/utils/TestAllocationListener.h similarity index 96% copy from cpp/velox/tests/utils/TestAllocationListener.h copy to cpp/core/tests/utils/TestAllocationListener.h index 9a94709aa3..3e51bf2aa4 100644 --- a/cpp/velox/tests/utils/TestAllocationListener.h +++ b/cpp/core/tests/utils/TestAllocationListener.h @@ -17,13 +17,14 @@ #pragma once +#include <limits> + #include "compute/ResultIterator.h" #include "memory/AllocationListener.h" #include "shuffle/ShuffleWriter.h" namespace gluten { -// instance with limited capacity, used by tests and benchmarks. class TestAllocationListener final : public AllocationListener { public: TestAllocationListener() = default; diff --git a/cpp/velox/tests/utils/TestStreamReader.h b/cpp/core/tests/utils/TestStreamReader.h similarity index 97% copy from cpp/velox/tests/utils/TestStreamReader.h copy to cpp/core/tests/utils/TestStreamReader.h index fd47574c6b..184d3ab671 100644 --- a/cpp/velox/tests/utils/TestStreamReader.h +++ b/cpp/core/tests/utils/TestStreamReader.h @@ -17,6 +17,9 @@ #pragma once +#include <memory> +#include <utility> + #include "shuffle/ShuffleReader.h" #include "shuffle/ShuffleWriter.h" diff --git a/cpp/velox/tests/utils/TestUtils.h b/cpp/core/tests/utils/TestUtils.h similarity index 98% copy from cpp/velox/tests/utils/TestUtils.h copy to cpp/core/tests/utils/TestUtils.h index 5de4b729f3..880e5eef71 100644 --- a/cpp/velox/tests/utils/TestUtils.h +++ b/cpp/core/tests/utils/TestUtils.h @@ -17,6 +17,9 @@ #pragma once +#include <stdexcept> + + #define ASSERT_NOT_OK(status) \ do { \ arrow::Status __s = (status); \ diff --git a/cpp/velox/udf/Udaf.h b/cpp/core/udf/Udaf.h similarity index 100% copy from cpp/velox/udf/Udaf.h copy to cpp/core/udf/Udaf.h diff --git a/cpp/velox/udf/Udf.h b/cpp/core/udf/Udf.h similarity index 100% copy from cpp/velox/udf/Udf.h copy to cpp/core/udf/Udf.h diff --git a/cpp/velox/utils/JsonToProtoConverter.h b/cpp/core/utils/JsonToProtoConverter.h similarity index 98% copy from cpp/velox/utils/JsonToProtoConverter.h copy to cpp/core/utils/JsonToProtoConverter.h index f9c41bb695..0ef9cd4dd8 100644 --- a/cpp/velox/utils/JsonToProtoConverter.h +++ b/cpp/core/utils/JsonToProtoConverter.h @@ -17,6 +17,8 @@ #pragma once +#include <string> + #include <google/protobuf/util/json_util.h> class JsonToProtoConverter { diff --git a/cpp/velox/cudf/CudfPlanValidator.h b/cpp/velox/cudf/CudfPlanValidator.h index de4fb150e0..7b08d1d57d 100644 --- a/cpp/velox/cudf/CudfPlanValidator.h +++ b/cpp/velox/cudf/CudfPlanValidator.h @@ -17,13 +17,4 @@ #pragma once -#include "memory/VeloxMemoryManager.h" -#include "substrait/plan.pb.h" - -namespace gluten { -class CudfPlanValidator { - public: - // Validate if the plan contains cudf unsupported operator except TableScan and ValueStream. - static bool validate(const ::substrait::Plan& substraitPlan); -}; -} // namespace gluten +#include "../../core/cudf/CudfPlanValidator.h" diff --git a/cpp/velox/jni/JniUdf.cc b/cpp/velox/jni/JniUdf.cc index 32514e7ae7..5b72be3e09 100644 --- a/cpp/velox/jni/JniUdf.cc +++ b/cpp/velox/jni/JniUdf.cc @@ -16,79 +16,15 @@ */ #include "JniUdf.h" -#include "jni/JniCommon.h" -#include "udf/UdfLoader.h" -#include "utils/Exception.h" -namespace { +namespace gluten { -static JavaVM* vm; - -const std::string kUdfResolverClassPath = "Lorg/apache/spark/sql/expression/UDFResolver$;"; - -static jclass udfResolverClass; -static jmethodID registerUDFMethod; -static jmethodID registerUDAFMethod; - -} // namespace - -void gluten::initVeloxJniUDF(JNIEnv* env) { - if (env->GetJavaVM(&vm) != JNI_OK) { - throw gluten::GlutenException("Unable to get JavaVM instance"); - } - - // classes - udfResolverClass = createGlobalClassReferenceOrError(env, kUdfResolverClassPath.c_str()); - - // methods - registerUDFMethod = getMethodIdOrError(env, udfResolverClass, "registerUDF", "(Ljava/lang/String;[B[BZZ)V"); - registerUDAFMethod = getMethodIdOrError(env, udfResolverClass, "registerUDAF", "(Ljava/lang/String;[B[B[BZZ)V"); +void initVeloxJniUDF(JNIEnv* env) { + initJniUDF(env); } -void gluten::finalizeVeloxJniUDF(JNIEnv* env) { - env->DeleteGlobalRef(udfResolverClass); +void finalizeVeloxJniUDF(JNIEnv* env) { + finalizeJniUDF(env); } -void gluten::jniRegisterFunctionSignatures(JNIEnv* env) { - auto udfLoader = gluten::UdfLoader::getInstance(); - - const auto& signatures = udfLoader->getRegisteredUdfSignatures(); - for (const auto& signature : signatures) { - jstring name = env->NewStringUTF(signature->name.c_str()); - jbyteArray returnType = env->NewByteArray(signature->returnType.length()); - env->SetByteArrayRegion( - returnType, 0, signature->returnType.length(), reinterpret_cast<const jbyte*>(signature->returnType.c_str())); - jbyteArray argTypes = env->NewByteArray(signature->argTypes.length()); - env->SetByteArrayRegion( - argTypes, 0, signature->argTypes.length(), reinterpret_cast<const jbyte*>(signature->argTypes.c_str())); - jobject instance = env->GetStaticObjectField( - udfResolverClass, env->GetStaticFieldID(udfResolverClass, "MODULE$", kUdfResolverClassPath.c_str())); - if (!signature->intermediateType.empty()) { - jbyteArray intermediateType = env->NewByteArray(signature->intermediateType.length()); - env->SetByteArrayRegion( - intermediateType, - 0, - signature->intermediateType.length(), - reinterpret_cast<const jbyte*>(signature->intermediateType.c_str())); - env->CallVoidMethod( - instance, - registerUDAFMethod, - name, - returnType, - argTypes, - intermediateType, - signature->variableArity, - signature->allowTypeConversion); - } else { - env->CallVoidMethod( - instance, - registerUDFMethod, - name, - returnType, - argTypes, - signature->variableArity, - signature->allowTypeConversion); - } - checkException(env); - } -} +} // namespace gluten diff --git a/cpp/velox/jni/JniUdf.h b/cpp/velox/jni/JniUdf.h index 568439d184..5c4ba2c996 100644 --- a/cpp/velox/jni/JniUdf.h +++ b/cpp/velox/jni/JniUdf.h @@ -18,8 +18,8 @@ #pragma once #include <jni.h> -#include <string> -#include <unordered_map> + +#include "../../core/jni/JniUdf.h" namespace gluten { @@ -27,6 +27,4 @@ void initVeloxJniUDF(JNIEnv* env); void finalizeVeloxJniUDF(JNIEnv* env); -void jniRegisterFunctionSignatures(JNIEnv* env); - } // namespace gluten diff --git a/cpp/velox/operators/functions/Arithmetic.h b/cpp/velox/operators/functions/Arithmetic.h index 9b2e22ac41..045bda7bcd 100644 --- a/cpp/velox/operators/functions/Arithmetic.h +++ b/cpp/velox/operators/functions/Arithmetic.h @@ -14,53 +14,4 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include <folly/CPortability.h> -#include <stdint.h> -#include <cmath> -#include <limits> -#include <type_traits> - -namespace gluten { -template <typename T> -/// Round function -/// When AlwaysRoundNegDec is true, spark semantics is followed which -/// rounds negative decimals for integrals and does not round it otherwise -/// Note that is is likely techinically impossible for this function to return -/// expected results in all cases as the loss of precision plagues it on both -/// paths: factor multiplication for large numbers and addition of truncated -/// number to the rounded fraction for small numbers. -/// We are trying to minimize the loss of precision by using the best path for -/// the number, but the journey is likely not over yet. -struct RoundFunction { - template <typename TNum, typename TDecimals, bool alwaysRoundNegDec = true> - FOLLY_ALWAYS_INLINE TNum round(const TNum& number, const TDecimals& decimals = 0) { - static_assert(!std::is_same_v<TNum, bool> && "round not supported for bool"); - - if constexpr (std::is_integral_v<TNum>) { - if constexpr (alwaysRoundNegDec) { - if (decimals >= 0) - return number; - } else { - return number; - } - } - if (!std::isfinite(number)) { - return number; - } - - // Using long double for high precision during intermediate calculations. - // TODO: Make this more efficient with Boost to support high arbitrary precision at runtime. - long double factor = std::pow(10.0L, static_cast<long double>(decimals)); - static const TNum kInf = std::numeric_limits<TNum>::infinity(); - - if (number < 0) { - return static_cast<TNum>((std::round(std::nextafter(number, -kInf) * factor * -1) / factor) * -1); - } - return static_cast<TNum>(std::round(std::nextafter(number, kInf) * factor) / factor); - } - template <typename TInput> - FOLLY_ALWAYS_INLINE void call(TInput& result, const TInput& a, const int32_t b = 0) { - result = round(a, b); - } -}; -} // namespace gluten +#include "../../../core/operators/functions/Arithmetic.h" diff --git a/cpp/velox/operators/functions/RegistrationAllFunctions.h b/cpp/velox/operators/functions/RegistrationAllFunctions.h index 9ba754d14f..319579b1ec 100644 --- a/cpp/velox/operators/functions/RegistrationAllFunctions.h +++ b/cpp/velox/operators/functions/RegistrationAllFunctions.h @@ -17,8 +17,4 @@ #pragma once -namespace gluten { - -void registerAllFunctions(); - -} // namespace gluten +#include "../../../core/operators/functions/RegistrationAllFunctions.h" diff --git a/cpp/velox/operators/writer/VeloxDataSource.h b/cpp/velox/operators/writer/VeloxDataSource.h index 043f1b8b17..7c4b8ae29f 100644 --- a/cpp/velox/operators/writer/VeloxDataSource.h +++ b/cpp/velox/operators/writer/VeloxDataSource.h @@ -17,32 +17,10 @@ #pragma once -#include <arrow/c/abi.h> -#include <arrow/memory_pool.h> -#include <arrow/record_batch.h> -#include <arrow/type.h> -#include <arrow/type_fwd.h> - -#include "memory/ColumnarBatch.h" +#include "../../../core/operators/writer/GlutenDataSource.h" namespace gluten { -class VeloxDataSource { - public: - VeloxDataSource(const std::string& filePath, std::shared_ptr<arrow::Schema> schema) - : filePath_(filePath), schema_(schema) {} - - virtual ~VeloxDataSource() = default; - - virtual void init(const std::unordered_map<std::string, std::string>& sparkConfs) {} - virtual void inspectSchema(struct ArrowSchema* out) = 0; - virtual void write(const std::shared_ptr<ColumnarBatch>& cb) {} - virtual void close() {} - virtual std::shared_ptr<arrow::Schema> getSchema() = 0; - - private: - std::string filePath_; - std::shared_ptr<arrow::Schema> schema_; -}; +using VeloxDataSource = GlutenDataSource; } // namespace gluten diff --git a/cpp/velox/tests/FilePathGenerator.h b/cpp/velox/tests/FilePathGenerator.h index 0858fcc2e0..331043dc90 100644 --- a/cpp/velox/tests/FilePathGenerator.h +++ b/cpp/velox/tests/FilePathGenerator.h @@ -17,9 +17,4 @@ #pragma once -#include <string> - -class FilePathGenerator { - public: - static const std::string getDataFilePath(const std::string& fileName); -}; +#include "../../core/tests/FilePathGenerator.h" diff --git a/cpp/velox/tests/JsonToProtoConverter.cc b/cpp/velox/tests/JsonToProtoConverter.cc index 4ed1372d1f..1c422d5740 100644 --- a/cpp/velox/tests/JsonToProtoConverter.cc +++ b/cpp/velox/tests/JsonToProtoConverter.cc @@ -15,22 +15,4 @@ * limitations under the License. */ -#include "utils//JsonToProtoConverter.h" -#include <fstream> -#include <sstream> -#include "velox/common/base/Exceptions.h" - -void JsonToProtoConverter::readFromFile(const std::string& msgPath, google::protobuf::Message& msg) { - // Read json file and resume the Substrait plan. - std::ifstream msgJson(msgPath); - VELOX_CHECK(!msgJson.fail(), "Failed to open file: {}. {}", msgPath, strerror(errno)); - std::stringstream buffer; - buffer << msgJson.rdbuf(); - std::string msgData = buffer.str(); - auto status = google::protobuf::util::JsonStringToMessage(msgData, &msg); - VELOX_CHECK( - status.ok(), - "Failed to parse Substrait JSON: {} {}", - static_cast<int8_t>(status.code()), - status.message().ToString()); -} +#include "../../core/tests/JsonToProtoConverter.cc" diff --git a/cpp/velox/tests/JsonToProtoConverter.h b/cpp/velox/tests/JsonToProtoConverter.h index f9c41bb695..f594ccc0a0 100644 --- a/cpp/velox/tests/JsonToProtoConverter.h +++ b/cpp/velox/tests/JsonToProtoConverter.h @@ -17,10 +17,4 @@ #pragma once -#include <google/protobuf/util/json_util.h> - -class JsonToProtoConverter { - public: - /// Reconstruct Protobuf message from Json file. - static void readFromFile(const std::string& msgPath, google::protobuf::Message& msg); -}; +#include "../../core/tests/JsonToProtoConverter.h" diff --git a/cpp/velox/tests/utils/TestAllocationListener.h b/cpp/velox/tests/utils/TestAllocationListener.h index 9a94709aa3..980c41ef53 100644 --- a/cpp/velox/tests/utils/TestAllocationListener.h +++ b/cpp/velox/tests/utils/TestAllocationListener.h @@ -17,50 +17,4 @@ #pragma once -#include "compute/ResultIterator.h" -#include "memory/AllocationListener.h" -#include "shuffle/ShuffleWriter.h" - -namespace gluten { - -// instance with limited capacity, used by tests and benchmarks. -class TestAllocationListener final : public AllocationListener { - public: - TestAllocationListener() = default; - - void setThrowIfOOM(bool throwIfOOM) { - throwIfOOM_ = throwIfOOM; - } - - void updateLimit(uint64_t limit) { - limit_ = limit; - } - - void setIterator(ResultIterator* iterator) { - iterator_ = iterator; - } - - void setShuffleWriter(ShuffleWriter* shuffleWriter) { - shuffleWriter_ = shuffleWriter; - } - - void allocationChanged(int64_t diff) override; - - int64_t currentBytes() override; - - int64_t reclaimedBytes() const; - - void reset(); - - private: - bool throwIfOOM_{false}; - - uint64_t usedBytes_{0L}; - uint64_t reclaimedBytes_{0L}; - - uint64_t limit_{std::numeric_limits<uint64_t>::max()}; - ResultIterator* iterator_{nullptr}; - ShuffleWriter* shuffleWriter_{nullptr}; -}; - -} // namespace gluten +#include "../../../core/tests/utils/TestAllocationListener.h" diff --git a/cpp/velox/tests/utils/TestStreamReader.h b/cpp/velox/tests/utils/TestStreamReader.h index fd47574c6b..0c14395b15 100644 --- a/cpp/velox/tests/utils/TestStreamReader.h +++ b/cpp/velox/tests/utils/TestStreamReader.h @@ -17,21 +17,4 @@ #pragma once -#include "shuffle/ShuffleReader.h" -#include "shuffle/ShuffleWriter.h" - -namespace gluten { - -class TestStreamReader : public StreamReader { - public: - explicit TestStreamReader(const std::shared_ptr<arrow::io::InputStream>& inputStream) : inputStream_(inputStream) {} - - std::shared_ptr<arrow::io::InputStream> readNextStream(arrow::MemoryPool*) override { - return std::move(inputStream_); - } - - private: - std::shared_ptr<arrow::io::InputStream> inputStream_; -}; - -} // namespace gluten +#include "../../../core/tests/utils/TestStreamReader.h" diff --git a/cpp/velox/tests/utils/TestUtils.h b/cpp/velox/tests/utils/TestUtils.h index 5de4b729f3..817ba2b552 100644 --- a/cpp/velox/tests/utils/TestUtils.h +++ b/cpp/velox/tests/utils/TestUtils.h @@ -17,10 +17,4 @@ #pragma once -#define ASSERT_NOT_OK(status) \ - do { \ - arrow::Status __s = (status); \ - if (!__s.ok()) { \ - throw std::runtime_error(__s.message()); \ - } \ - } while (false); +#include "../../../core/tests/utils/TestUtils.h" diff --git a/cpp/velox/udf/Udaf.h b/cpp/velox/udf/Udaf.h index f421ecbc31..31f364da79 100644 --- a/cpp/velox/udf/Udaf.h +++ b/cpp/velox/udf/Udaf.h @@ -17,27 +17,4 @@ #pragma once -namespace gluten { - -struct UdafEntry { - const char* name; - const char* dataType; - - int numArgs; - const char** argTypes; - - const char* intermediateType{nullptr}; - bool variableArity{false}; - bool allowTypeConversion{false}; -}; - -#define GLUTEN_GET_NUM_UDAF getNumUdaf -#define DEFINE_GET_NUM_UDAF extern "C" int GLUTEN_GET_NUM_UDAF() - -#define GLUTEN_GET_UDAF_ENTRIES getUdfEntries -#define DEFINE_GET_UDAF_ENTRIES extern "C" void GLUTEN_GET_UDAF_ENTRIES(gluten::UdafEntry* udafEntries) - -#define GLUTEN_REGISTER_UDAF registerUdf -#define DEFINE_REGISTER_UDAF extern "C" void GLUTEN_REGISTER_UDAF() - -} // namespace gluten +#include "../../core/udf/Udaf.h" diff --git a/cpp/velox/udf/Udf.h b/cpp/velox/udf/Udf.h index e0b3a70004..a31b50f182 100644 --- a/cpp/velox/udf/Udf.h +++ b/cpp/velox/udf/Udf.h @@ -17,26 +17,4 @@ #pragma once -namespace gluten { - -struct UdfEntry { - const char* name; - const char* dataType; - - int numArgs; - const char** argTypes; - - bool variableArity{false}; - bool allowTypeConversion{false}; -}; - -#define GLUTEN_GET_NUM_UDF getNumUdf -#define DEFINE_GET_NUM_UDF extern "C" int GLUTEN_GET_NUM_UDF() - -#define GLUTEN_GET_UDF_ENTRIES getUdfEntries -#define DEFINE_GET_UDF_ENTRIES extern "C" void GLUTEN_GET_UDF_ENTRIES(gluten::UdfEntry* udfEntries) - -#define GLUTEN_REGISTER_UDF registerUdf -#define DEFINE_REGISTER_UDF extern "C" void GLUTEN_REGISTER_UDF() - -} // namespace gluten +#include "../../core/udf/Udf.h" diff --git a/cpp/velox/utils/JsonToProtoConverter.h b/cpp/velox/utils/JsonToProtoConverter.h index f9c41bb695..137206312f 100644 --- a/cpp/velox/utils/JsonToProtoConverter.h +++ b/cpp/velox/utils/JsonToProtoConverter.h @@ -17,10 +17,4 @@ #pragma once -#include <google/protobuf/util/json_util.h> - -class JsonToProtoConverter { - public: - /// Reconstruct Protobuf message from Json file. - static void readFromFile(const std::string& msgPath, google::protobuf::Message& msg); -}; +#include "../../core/utils/JsonToProtoConverter.h" --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
