This is an automated email from the ASF dual-hosted git repository.
liuneng1994 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 3f4439f428 [GLUTEN-12998][BOLT] Enable build-only x86 CI for Bolt
(#13486)
3f4439f428 is described below
commit 3f4439f428bb7c7e399f60754b7ebe28263d0f6b
Author: Weixin-Xu <[email protected]>
AuthorDate: Thu Nov 12 20:40:08 2026 +0800
[GLUTEN-12998][BOLT] Enable build-only x86 CI for Bolt (#13486)
What changes are proposed in this pull request?
Refs #12998
Enable Bolt native test support and Spark test discovery on Apache Gluten
without adding CI orchestration to this repository.
Adapt benchmark/test utilities to the current Bolt interfaces and add a
local RSS test client.
Wire shared Bolt test settings and smoke/assertion suites into Spark 4.0,
4.1 and 4.2, with per-version exclusions and inventory checks.
Update Spark 3.5 settings and Delta/Hudi test compatibility.
CI scripts remain in Bolt. This change does not add Spark 3.4 support or
claim that all selected UTs pass.
How was this patch tested?
./dev/format-scala-code.sh check: passed.
./build/mvn -q spotless:check -Pbackends-bolt
-Pspark-3.5,spark-4.0,spark-4.1,spark-4.2 -Pspark-ut
-Piceberg,delta,hudi,paimon: passed.
clang-format 15.0.7 --dry-run --Werror on all changed C++ sources/headers:
passed.
License header check against Apache base
0447c1aa45f3a5c96cf921490bbea8d1f1dd1bfe: passed.
POM XML parsing and git diff --check: passed.
From Bolt commit 791113cd41e5be36c1eb8d9760e996db960e0018,
GLUTEN_TEST_SOURCE=/path/to/this/checkout python3 -m unittest discover -s
scripts/gluten-ci -q: 40 tests passed. These are helper/selection tests, not
native or Spark execution.
Independent full-diff review found no actionable introduced defects.
Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (GPT-6)
---
.../apache/gluten/execution/BoltDeltaSuite.scala | 3 +
.../gluten/execution/BoltTPCHHudiSuite.scala | 3 -
cpp/bolt/benchmarks/CMakeLists.txt | 3 +-
cpp/bolt/benchmarks/GenericBenchmark.cc | 230 +++++---
cpp/bolt/benchmarks/PlanValidatorUtil.cc | 2 +-
cpp/bolt/tests/utils/LocalRssClient.cc | 62 ++
cpp/bolt/tests/utils/LocalRssClient.h | 44 ++
cpp/bolt/tests/utils/TestAllocationListener.cc | 4 +
cpp/bolt/tests/utils/TestAllocationListener.h | 6 +-
gluten-ut/pom.xml | 62 ++
.../gluten/utils/bolt/BoltTestSettings.scala | 3 +-
gluten-ut/spark40/pom.xml | 81 +++
.../gluten/utils/bolt/BoltTestSettings.scala | 29 +
.../gluten/GlutenColumnarWriteTestSupport.scala | 27 +
.../gluten/utils/bolt/BoltSpark4TestSettings.scala | 630 +++++++++++++++++++++
.../GlutenLogicalPlanTagInSparkPlanSuite.scala | 3 +-
.../execution/GlutenWholeStageCodegenSuite.scala | 20 +-
gluten-ut/spark41/pom.xml | 80 +++
.../gluten/utils/bolt/BoltTestSettings.scala | 57 ++
.../GlutenLogicalPlanTagInSparkPlanSuite.scala | 3 +-
.../execution/GlutenWholeStageCodegenSuite.scala | 20 +-
gluten-ut/spark42/pom.xml | 80 +++
.../gluten/utils/bolt/BoltTestSettings.scala | 65 +++
.../GlutenLogicalPlanTagInSparkPlanSuite.scala | 3 +-
.../execution/GlutenWholeStageCodegenSuite.scala | 20 +-
25 files changed, 1407 insertions(+), 133 deletions(-)
diff --git
a/backends-bolt/src-delta/test/scala/org/apache/gluten/execution/BoltDeltaSuite.scala
b/backends-bolt/src-delta/test/scala/org/apache/gluten/execution/BoltDeltaSuite.scala
index dd4b1fa151..580f1ea1cf 100644
---
a/backends-bolt/src-delta/test/scala/org/apache/gluten/execution/BoltDeltaSuite.scala
+++
b/backends-bolt/src-delta/test/scala/org/apache/gluten/execution/BoltDeltaSuite.scala
@@ -49,7 +49,10 @@ class BoltDeltaSuite extends DeltaSuite {
}.head
var listings = 0
val snapshot = log.update()
+ val tableLogPath = log.logPath
val index = new TahoeFileIndex(spark, log, new Path(path)) {
+ // SnapshotDescriptor requires the log path on Delta 4.x.
+ def logPath: Path = tableLogPath
override def version: Long = snapshot.version
override def metadata = snapshot.metadata
override def protocol = snapshot.protocol
diff --git
a/backends-bolt/src-hudi/test/scala/org/apache/gluten/execution/BoltTPCHHudiSuite.scala
b/backends-bolt/src-hudi/test/scala/org/apache/gluten/execution/BoltTPCHHudiSuite.scala
index 74b6e2fc8c..e63a509a42 100644
---
a/backends-bolt/src-hudi/test/scala/org/apache/gluten/execution/BoltTPCHHudiSuite.scala
+++
b/backends-bolt/src-hudi/test/scala/org/apache/gluten/execution/BoltTPCHHudiSuite.scala
@@ -24,9 +24,6 @@ class BoltTPCHHudiSuite extends BoltTPCHSuite {
protected val tpchBasePath: String =
getClass.getResource("/").getPath + "../../../src/test/resources"
- override protected val resourcePath: String =
- getClass.getResource("/tpch-data-parquet").getFile
-
override protected val queriesResults: String =
new File(tpchBasePath, "queries-output").getCanonicalPath
diff --git a/cpp/bolt/benchmarks/CMakeLists.txt
b/cpp/bolt/benchmarks/CMakeLists.txt
index e2c5f6d8cf..5417b1b94a 100644
--- a/cpp/bolt/benchmarks/CMakeLists.txt
+++ b/cpp/bolt/benchmarks/CMakeLists.txt
@@ -16,7 +16,8 @@
find_package(Arrow CONFIG REQUIRED)
find_package(gfx-timsort REQUIRED CONFIG)
-set(BOLT_BENCHMARK_COMMON_SRCS common/BenchmarkUtils.cc)
+set(BOLT_BENCHMARK_COMMON_SRCS common/BenchmarkUtils.cc
+ ../tests/utils/LocalRssClient.cc)
add_library(bolt_benchmark_common STATIC ${BOLT_BENCHMARK_COMMON_SRCS})
target_include_directories(
bolt_benchmark_common PUBLIC ${CMAKE_SOURCE_DIR}/bolt
diff --git a/cpp/bolt/benchmarks/GenericBenchmark.cc
b/cpp/bolt/benchmarks/GenericBenchmark.cc
index 9b80d61b4e..933301cf0d 100644
--- a/cpp/bolt/benchmarks/GenericBenchmark.cc
+++ b/cpp/bolt/benchmarks/GenericBenchmark.cc
@@ -23,6 +23,7 @@
#include <benchmark/benchmark.h>
+#include <folly/ScopeGuard.h>
#include <gflags/gflags.h>
#include "benchmarks/common/BenchmarkUtils.h"
@@ -33,9 +34,8 @@
#include "config/GlutenConfig.h"
#include "operators/reader/FileReaderIterator.h"
#include "operators/writer/BoltColumnarBatchWriter.h"
-#include "shuffle/BoltShuffleWriter.h"
-#include "shuffle/LocalPartitionWriter.h"
-#include "shuffle/rss/RssPartitionWriter.h"
+#include "shuffle/BoltShuffleReaderWrapper.h"
+#include "shuffle/ShuffleWriterBase.h"
#include "tests/utils/LocalRssClient.h"
#include "tests/utils/TestAllocationListener.h"
#include "tests/utils/TestStreamReader.h"
@@ -53,11 +53,11 @@ DEFINE_string(save_output, "", "Path to parquet file for
saving the task output
DEFINE_bool(with_shuffle, false, "Add shuffle split at end.");
DEFINE_bool(run_shuffle, false, "Only run shuffle write.");
DEFINE_bool(run_shuffle_read, false, "Whether to run shuffle read when
run_shuffle is true.");
-DEFINE_string(shuffle_writer, "hash", "Shuffle writer type. Can be hash or
sort");
+DEFINE_string(shuffle_writer, "hash", "Shuffle writer type: hash, sort, or
rss_sort (requires --rss)");
DEFINE_string(
partitioning,
"rr",
- "Short partitioning name. Valid options are rr, hash, range, single,
random (only for test purpose)");
+ "Partitioning: rr, single, hash, or range. hash/range require an input
partition-id column");
DEFINE_bool(rss, false, "Mocking rss.");
DEFINE_string(compression, "lz4", "Specify the compression codec. Valid
options are none, lz4, zstd");
DEFINE_int32(shuffle_partitions, 200, "Number of shuffle split (reducer)
partitions");
@@ -123,7 +123,7 @@ std::string generateUniqueSubdir(const std::string& parent,
const std::string& p
std::error_code ec{};
while (!std::filesystem::create_directories(path, ec)) {
if (ec) {
- LOG(ERROR) << fmt::format("Failed to created spill directory: {}, error
code: {}", path, ec.message());
+ LOG(ERROR) << fmt::format("Failed to created spill directory: {}, error
code: {}", path.string(), ec.message());
std::exit(EXIT_FAILURE);
}
path = std::filesystem::path(parent) / (prefix + generateUuid());
@@ -162,88 +162,129 @@ void cleanupLocalDirs(const std::vector<std::string>&
localDirs) {
}
}
-void setCompressionTypeFromFlag(arrow::Compression::type& compressionType,
CodecBackend& codecBackend) {
- codecBackend = CodecBackend::NONE;
- if (FLAGS_compression == "none") {
- compressionType = arrow::Compression::UNCOMPRESSED;
- } else if (FLAGS_compression == "lz4") {
- compressionType = arrow::Compression::LZ4_FRAME;
- } else if (FLAGS_compression == "zstd") {
- compressionType = arrow::Compression::ZSTD;
- } else {
- throw GlutenException("Unrecognized compression type: " +
FLAGS_compression);
+std::string validateShuffleFlags() {
+ if (!FLAGS_run_shuffle && !FLAGS_with_shuffle) {
+ return "";
}
-}
-
-std::unique_ptr<arrow::util::Codec> createCodec() {
- // Configure compression.
- if (FLAGS_compression == "none") {
- return nullptr;
+ if (FLAGS_partitioning != "rr" && FLAGS_partitioning != "single" &&
FLAGS_partitioning != "hash" &&
+ FLAGS_partitioning != "range") {
+ return "Bolt shuffle partitioning must be rr, single, hash, or range";
}
-
- arrow::Compression::type compressionType;
- CodecBackend codecBackend;
-
- setCompressionTypeFromFlag(compressionType, codecBackend);
-
- return createCompressionCodec(compressionType, codecBackend);
+ if (FLAGS_shuffle_writer != "hash" && FLAGS_shuffle_writer != "sort" &&
FLAGS_shuffle_writer != "rss_sort") {
+ return "Bolt shuffle writer must be hash, sort, or rss_sort";
+ }
+ if (FLAGS_shuffle_writer == "rss_sort" && !FLAGS_rss) {
+ return "--shuffle_writer=rss_sort requires --rss";
+ }
+ if (FLAGS_run_shuffle && FLAGS_partitioning != "rr" && FLAGS_partitioning !=
"single") {
+ return "--run_shuffle supports only rr or single partitioning; it does not
generate partition IDs";
+ }
+ if (FLAGS_shuffle_writer != "hash" && FLAGS_partitioning != "hash" &&
FLAGS_partitioning != "range") {
+ return "Bolt sort shuffle requires hash or range partitioning with an
input partition-id column";
+ }
+ if (FLAGS_shuffle_dictionary) {
+ return "Bolt shuffle does not support dictionary encoding";
+ }
+ return "";
}
-std::shared_ptr<PartitionWriter>
-createPartitionWriter(Runtime* runtime, const std::string& dataFile, const
std::vector<std::string>& localDirs) {
- std::unique_ptr<PartitionWriter> partitionWriter;
- if (FLAGS_rss) {
- auto options = std::make_shared<RssPartitionWriterOptions>();
- auto rssClient = std::make_unique<LocalRssClient>(dataFile);
- return std::make_shared<RssPartitionWriter>(
- FLAGS_shuffle_partitions, createCodec(), runtime->memoryManager(),
options, std::move(rssClient));
+int32_t forcedShuffleWriterType() {
+ using bytedance::bolt::shuffle::sparksql::ShuffleWriterType;
+ if (FLAGS_shuffle_writer == "hash") {
+ return static_cast<int32_t>(ShuffleWriterType::V1);
}
-
- auto options = std::make_shared<LocalPartitionWriterOptions>();
- options->enableDictionary = FLAGS_shuffle_dictionary;
- return std::make_unique<LocalPartitionWriter>(
- FLAGS_shuffle_partitions, createCodec(), runtime->memoryManager(),
options, dataFile, localDirs);
+ if (FLAGS_shuffle_writer == "sort" || FLAGS_shuffle_writer == "rss_sort") {
+ // Bolt's sort writer requires a partition-id column.
+ GLUTEN_CHECK(
+ FLAGS_partitioning == "hash" || FLAGS_partitioning == "range",
+ "Bolt sort shuffle requires hash or range partitioning");
+ return static_cast<int32_t>(ShuffleWriterType::RowBased);
+ }
+ throw GlutenException("Unsupported shuffle writer type: " +
FLAGS_shuffle_writer);
}
-std::shared_ptr<ShuffleWriterOptions> createShuffleWriterOptions() {
- std::shared_ptr<ShuffleWriterOptions> options;
-
- switch (ShuffleWriter::stringToType(FLAGS_shuffle_writer)) {
- case ShuffleWriterType::kHashShuffle:
- options = std::make_shared<HashShuffleWriterOptions>();
- break;
- case ShuffleWriterType::kSortShuffle:
- options = std::make_shared<SortShuffleWriterOptions>();
- break;
- case ShuffleWriterType::kRssSortShuffle:
- options = std::make_shared<RssSortShuffleWriterOptions>();
- break;
- default:
- throw GlutenException("Unsupported shuffle writer type: " +
FLAGS_shuffle_writer);
- }
-
- options->partitioning = toPartitioning(FLAGS_partitioning);
- return options;
+std::string compressionCodec() {
+ if (FLAGS_compression == "none") {
+ return "uncompressed";
+ }
+ if (FLAGS_compression == "lz4") {
+ return "lz4";
+ }
+ if (FLAGS_compression == "zstd") {
+ return "zstd";
+ }
+ throw GlutenException("Unrecognized compression type: " + FLAGS_compression);
}
-std::shared_ptr<BoltShuffleWriter> createShuffleWriter(
- Runtime* runtime,
- std::shared_ptr<PartitionWriter> partitionWriter) {
- auto shuffleWriterOptions = createShuffleWriterOptions();
- auto shuffleWriter =
- runtime->createShuffleWriter(FLAGS_shuffle_partitions,
std::move(partitionWriter), shuffleWriterOptions);
- return std::dynamic_pointer_cast<BoltShuffleWriter>(shuffleWriter);
+std::shared_ptr<ShuffleWriterBase> createShuffleWriter(
+ BoltRuntime* runtime,
+ const std::string& dataFile,
+ const std::vector<std::string>& localDirs,
+ const std::shared_ptr<ColumnarBatch>& firstBatch) {
+ GLUTEN_CHECK(!FLAGS_shuffle_dictionary, "Bolt shuffle does not support
dictionary encoding");
+ const bytedance::bolt::shuffle::sparksql::ShuffleWriterOptions defaults{};
+ const auto& partitionDefaults = defaults.partitionWriterOptions;
+ ShuffleWriterInfo info;
+ info.set_partitioning_name(FLAGS_partitioning);
+ info.set_num_partitions(FLAGS_shuffle_partitions);
+ info.set_task_attempt_id(defaults.taskAttemptId);
+ info.set_buffer_size(defaults.bufferSize);
+ info.set_merge_buffer_size(partitionDefaults.mergeBufferSize);
+ info.set_merge_threshold(partitionDefaults.mergeThreshold);
+ info.set_compression_codec(compressionCodec());
+ info.set_compression_backend(partitionDefaults.codecBackend);
+ info.set_compression_level(partitionDefaults.compressionLevel);
+ info.set_compression_threshold(partitionDefaults.compressionThreshold);
+ info.set_compression_mode(partitionDefaults.compressionMode);
+ info.set_data_file(dataFile);
+ info.set_num_sub_dirs(partitionDefaults.numSubDirs);
+ std::string joinedDirs;
+ for (const auto& dir : localDirs) {
+ if (!joinedDirs.empty()) {
+ joinedDirs += ",";
+ }
+ joinedDirs += dir;
+ }
+ info.set_local_dirs(joinedDirs);
+ info.set_realloc_threshold(defaults.bufferReallocThreshold);
+ info.set_mem_limit(ShuffleWriter::kMaxMemLimit);
+ info.set_push_buffer_max_size(partitionDefaults.pushBufferMaxSize);
+ info.set_writer_type(FLAGS_rss ? "celeborn" : "local");
+ info.set_forced_writer_type(forcedShuffleWriterType());
+ info.set_use_v2_prealloc_threshold(defaults.useV2PreallocSizeThreshold);
+
info.set_row_compression_min_cols(defaults.rowvectorModeCompressionMinColumns);
+
info.set_row_compression_max_buffer(defaults.rowvectorModeCompressionMaxBufferSize);
+ info.set_enable_vector_combination(defaults.enableVectorCombination);
+ info.set_accumulate_batch_max_columns(defaults.accumulateBatchMaxColumns);
+ info.set_accumulate_batch_max_batches(defaults.accumulateBatchMaxBatches);
+ info.set_recommended_c2r_size(defaults.recommendedColumn2RowSize);
+ info.set_shuffle_batch_byte_size(partitionDefaults.shuffleBufferSize);
+ info.set_row_format("compact");
+ std::shared_ptr<RssClient> rssClient;
+ if (FLAGS_rss) {
+ rssClient = std::make_shared<LocalRssClient>(dataFile);
+ }
+ return runtime->createShuffleWriter(info, std::move(rssClient), firstBatch);
}
-std::shared_ptr<ShuffleReader> createShuffleReader(Runtime* runtime, const
std::shared_ptr<arrow::Schema>& schema) {
- auto readerOptions = ShuffleReaderOptions{};
- readerOptions.shuffleWriterType =
ShuffleWriter::stringToType(FLAGS_shuffle_writer),
- setCompressionTypeFromFlag(readerOptions.compressionType,
readerOptions.codecBackend);
- return runtime->createShuffleReader(schema, readerOptions);
+std::shared_ptr<BoltShuffleReaderWrapper> createShuffleReader(
+ BoltRuntime* runtime,
+ const std::shared_ptr<arrow::Schema>& schema) {
+ const bytedance::bolt::shuffle::sparksql::ShuffleReaderOptions defaults{};
+ ShuffleReaderInfo info;
+ info.set_compression_type(compressionCodec());
+ info.set_codec(defaults.codecBackend);
+ info.set_batch_size(defaults.batchSize);
+ info.set_shuffle_batch_byte_size(defaults.shuffleBatchByteSize);
+ info.set_num_partitions(FLAGS_shuffle_partitions);
+ info.set_partition_short_name(FLAGS_partitioning);
+ info.set_forced_writer_type(forcedShuffleWriterType());
+ info.set_row_format("compact");
+ return
std::dynamic_pointer_cast<BoltShuffleReaderWrapper>(runtime->createShuffleReader(schema,
info));
}
void populateWriterMetrics(
- const std::shared_ptr<BoltShuffleWriter>& shuffleWriter,
+ const std::shared_ptr<ShuffleWriterBase>& shuffleWriter,
int64_t totalTime,
WriterMetrics& metrics) {
metrics.compressTime += shuffleWriter->totalCompressTime();
@@ -270,7 +311,7 @@ void setCpu(::benchmark::State& state) {
}
void runShuffle(
- Runtime* runtime,
+ BoltRuntime* runtime,
TestAllocationListener* listener,
const std::shared_ptr<gluten::ResultIterator>& resultIter,
WriterMetrics& writerMetrics,
@@ -278,11 +319,12 @@ void runShuffle(
bool readAfterWrite,
const std::vector<std::string>& localDirs,
const std::string& dataFileDir) {
+ listener->setShuffleWriter(nullptr);
GLUTEN_ASSIGN_OR_THROW(auto dataFile,
gluten::createTempShuffleFile(dataFileDir));
- auto partitionWriter = createPartitionWriter(runtime, dataFile, localDirs);
- auto shuffleWriter = createShuffleWriter(runtime,
std::move(partitionWriter));
- listener->setShuffleWriter(shuffleWriter.get());
+ std::shared_ptr<ShuffleWriterBase> shuffleWriter;
+ // Clear the listener before the writer is destroyed, including on
exceptions.
+ auto clearShuffleWriter = folly::makeGuard([&] {
listener->setShuffleWriter(nullptr); });
int64_t totalTime = 0;
std::shared_ptr<ArrowSchema> cSchema;
@@ -290,14 +332,22 @@ void runShuffle(
gluten::ScopedTimer timer(&totalTime);
while (resultIter->hasNext()) {
auto cb = resultIter->next();
- if (!cSchema) {
+ if (!shuffleWriter) {
cSchema = cb->exportArrowSchema();
+ shuffleWriter = createShuffleWriter(runtime, dataFile, localDirs, cb);
+ listener->setShuffleWriter(shuffleWriter.get());
}
- GLUTEN_THROW_NOT_OK(shuffleWriter->write(cb, ShuffleWriter::kMaxMemLimit
- shuffleWriter->cachedPayloadSize()));
+ GLUTEN_THROW_NOT_OK(shuffleWriter->split(cb, ShuffleWriter::kMaxMemLimit
- shuffleWriter->cachedPayloadSize()));
+ }
+ if (shuffleWriter) {
+ GLUTEN_THROW_NOT_OK(shuffleWriter->stop());
}
- GLUTEN_THROW_NOT_OK(shuffleWriter->stop());
}
+ if (!shuffleWriter) {
+ std::filesystem::remove(dataFile);
+ return;
+ }
populateWriterMetrics(shuffleWriter, totalTime, writerMetrics);
if (readAfterWrite && cSchema != nullptr) {
@@ -307,7 +357,7 @@ void runShuffle(
GLUTEN_ASSIGN_OR_THROW(auto in, arrow::io::ReadableFile::Open(dataFile));
auto streamReader = std::make_shared<TestStreamReader>(std::move(in));
// Read all partitions.
- auto iter = reader->read(streamReader);
+ auto iter = reader->readStream(streamReader);
while (iter->hasNext()) {
// Read and discard.
auto cb = iter->next();
@@ -449,7 +499,7 @@ auto BM_Generic = [](::benchmark::State& state,
runtime->parseSplitInfo(reinterpret_cast<uint8_t*>(split.data()),
split.size(), i);
}
- auto resultIter = runtime->createResultIterator(boltSpillDir,
std::move(inputIters), runtime->getConfMap());
+ auto resultIter = runtime->createResultIterator(boltSpillDir,
std::move(inputIters));
listenerPtr->setIterator(resultIter.get());
if (FLAGS_with_shuffle) {
@@ -572,6 +622,12 @@ int main(int argc, char** argv) {
::benchmark::Initialize(&argc, argv);
+ if (const auto error = validateShuffleFlags(); !error.empty()) {
+ LOG(ERROR) << "Incorrect usage: " << error;
+ ::benchmark::Shutdown();
+ return EXIT_FAILURE;
+ }
+
// Init Bolt backend.
std::unordered_map<std::string, std::string> backendConf{};
std::unordered_map<std::string, std::string> sessionConf{};
@@ -641,9 +697,7 @@ int main(int argc, char** argv) {
if (FLAGS_run_shuffle) {
std::string errorMsg{};
if (FLAGS_data.empty()) {
- errorMsg = "Missing '--split' or '--data' option.";
- } else if (FLAGS_partitioning != "rr" && FLAGS_partitioning != "random") {
- errorMsg = "--run-shuffle only support round-robin partitioning and
random partitioning.";
+ errorMsg = "Missing '--data' option.";
}
if (errorMsg.empty()) {
try {
@@ -686,8 +740,8 @@ int main(int argc, char** argv) {
if (!errorMsg.empty()) {
LOG(ERROR) << "Incorrect usage: " << errorMsg << std::endl
- << "*** The Bolt benchmark takes the same options as the
Velox one. Please check"
- << " docs/backends/velox/developers/MicroBenchmarks.md for
the full usage. ***";
+ << "See --help for Bolt-supported options and"
+ << " docs/backends/velox/developers/MicroBenchmarks.md for
general usage.";
::benchmark::Shutdown();
std::exit(EXIT_FAILURE);
}
diff --git a/cpp/bolt/benchmarks/PlanValidatorUtil.cc
b/cpp/bolt/benchmarks/PlanValidatorUtil.cc
index fb28f96c6b..073e49cac7 100644
--- a/cpp/bolt/benchmarks/PlanValidatorUtil.cc
+++ b/cpp/bolt/benchmarks/PlanValidatorUtil.cc
@@ -45,7 +45,7 @@ int main(int argc, char** argv) {
conf.insert({kDebugModeEnabled, "true"});
initBoltBackend(conf);
auto pool = defaultLeafBoltMemoryPool().get();
- SubstraitToBoltPlanValidator planValidator(pool);
+ SubstraitToBoltPlanValidator planValidator(pool, conf);
::substrait::Plan subPlan;
parseProtobuf(reinterpret_cast<uint8_t*>(plan.data()), plan.size(),
&subPlan);
diff --git a/cpp/bolt/tests/utils/LocalRssClient.cc
b/cpp/bolt/tests/utils/LocalRssClient.cc
new file mode 100644
index 0000000000..5dfe0561cf
--- /dev/null
+++ b/cpp/bolt/tests/utils/LocalRssClient.cc
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "tests/utils/LocalRssClient.h"
+#include "utils/Common.h"
+
+#include <arrow/io/file.h>
+
+namespace gluten {
+
+int32_t LocalRssClient::pushPartitionData(int32_t partitionId, const char*
bytes, int64_t size) {
+ auto [iter, inserted] = partitionBufferMap_.try_emplace(partitionId,
buffers_.size());
+ int32_t idx = iter->second;
+
+ // Allocate new buffer if it's a new partition.
+ if (inserted) {
+ buffers_.emplace_back();
+ GLUTEN_ASSIGN_OR_THROW(buffers_.back(), arrow::AllocateResizableBuffer(0));
+ }
+
+ auto& buffer = buffers_[idx];
+ auto newSize = buffer->size() + size;
+
+ if (buffer->capacity() < newSize) {
+ GLUTEN_THROW_NOT_OK(buffer->Reserve(newSize));
+ }
+
+ fastCopy(buffer->mutable_data() + buffer->size(), bytes, size);
+ GLUTEN_THROW_NOT_OK(buffer->Resize(newSize));
+
+ return size;
+}
+
+void LocalRssClient::stop() {
+ std::shared_ptr<arrow::io::FileOutputStream> out;
+ GLUTEN_ASSIGN_OR_THROW(out, arrow::io::FileOutputStream::Open(dataFile_));
+
+ for (auto item : partitionBufferMap_) {
+ GLUTEN_THROW_NOT_OK(out->Write(buffers_[item.second]->data(),
buffers_[item.second]->size()));
+ GLUTEN_THROW_NOT_OK(out->Flush());
+ }
+ GLUTEN_THROW_NOT_OK(out->Close());
+
+ buffers_.clear();
+ partitionBufferMap_.clear();
+}
+
+} // namespace gluten
diff --git a/cpp/bolt/tests/utils/LocalRssClient.h
b/cpp/bolt/tests/utils/LocalRssClient.h
new file mode 100644
index 0000000000..e14c9fd01f
--- /dev/null
+++ b/cpp/bolt/tests/utils/LocalRssClient.h
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "shuffle/rss/RssClient.h"
+#include "utils/Common.h"
+#include "utils/Macros.h"
+
+#include <arrow/buffer.h>
+
+#include <map>
+
+namespace gluten {
+
+/// A local implementation of the RssClient interface for testing purposes.
+class LocalRssClient : public RssClient {
+ public:
+ LocalRssClient(std::string dataFile) : dataFile_(dataFile) {}
+
+ int32_t pushPartitionData(int32_t partitionId, const char* bytes, int64_t
size) override;
+
+ void stop() override;
+
+ private:
+ std::string dataFile_;
+ std::vector<std::unique_ptr<arrow::ResizableBuffer>> buffers_;
+ std::map<uint32_t, uint32_t> partitionBufferMap_;
+};
+} // namespace gluten
diff --git a/cpp/bolt/tests/utils/TestAllocationListener.cc
b/cpp/bolt/tests/utils/TestAllocationListener.cc
index 495e812d08..7d98a7e1b0 100644
--- a/cpp/bolt/tests/utils/TestAllocationListener.cc
+++ b/cpp/bolt/tests/utils/TestAllocationListener.cc
@@ -19,6 +19,10 @@
#include "bolt/common/base/SuccinctPrinter.h"
#include <fmt/format.h>
+#include <glog/logging.h>
+
+#include "utils/Exception.h"
+#include "utils/Macros.h"
namespace gluten {
diff --git a/cpp/bolt/tests/utils/TestAllocationListener.h
b/cpp/bolt/tests/utils/TestAllocationListener.h
index 3194d64def..6d3bdfffa7 100644
--- a/cpp/bolt/tests/utils/TestAllocationListener.h
+++ b/cpp/bolt/tests/utils/TestAllocationListener.h
@@ -19,7 +19,7 @@
#include "compute/ResultIterator.h"
#include "memory/AllocationListener.h"
-#include "shuffle/ShuffleWriter.h"
+#include "memory/Reclaimable.h"
namespace gluten {
@@ -40,7 +40,7 @@ class TestAllocationListener final : public
AllocationListener {
iterator_ = iterator;
}
- void setShuffleWriter(ShuffleWriter* shuffleWriter) {
+ void setShuffleWriter(Reclaimable* shuffleWriter) {
shuffleWriter_ = shuffleWriter;
}
@@ -60,7 +60,7 @@ class TestAllocationListener final : public
AllocationListener {
uint64_t limit_{std::numeric_limits<uint64_t>::max()};
ResultIterator* iterator_{nullptr};
- ShuffleWriter* shuffleWriter_{nullptr};
+ Reclaimable* shuffleWriter_{nullptr};
};
} // namespace gluten
diff --git a/gluten-ut/pom.xml b/gluten-ut/pom.xml
index 51fbeb55e9..9080ab62b1 100644
--- a/gluten-ut/pom.xml
+++ b/gluten-ut/pom.xml
@@ -200,6 +200,68 @@
</build>
<profiles>
+ <profile>
+ <id>bolt-ut-delta</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.gluten</groupId>
+ <artifactId>gluten-delta</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+ <profile>
+ <id>bolt-ut-hudi</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.gluten</groupId>
+ <artifactId>gluten-hudi</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hudi</groupId>
+
<artifactId>hudi-spark${sparkbundle.version}-bundle_${scala.binary.version}</artifactId>
+ <version>${hudi.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+ <profile>
+ <id>bolt-ut-iceberg</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.gluten</groupId>
+ <artifactId>gluten-iceberg</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.iceberg</groupId>
+
<artifactId>iceberg-spark-runtime-${sparkbundle.version}_${scala.binary.version}</artifactId>
+ <version>${iceberg.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+ <profile>
+ <id>bolt-ut-paimon</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.gluten</groupId>
+ <artifactId>gluten-paimon</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+
<artifactId>paimon-spark-${sparkbundle.version}_${scala.binary.version}</artifactId>
+ <version>${paimon.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ </profile>
<profile>
<id>spark-3.4</id>
<modules>
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/bolt/BoltTestSettings.scala
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/bolt/BoltTestSettings.scala
index 28bd84b1c0..e9bad0745b 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/bolt/BoltTestSettings.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/bolt/BoltTestSettings.scala
@@ -39,7 +39,7 @@ import org.apache.spark.sql.execution.joins._
import org.apache.spark.sql.execution.python._
import
org.apache.spark.sql.extension.{GlutenCollapseProjectExecTransformerSuite,
GlutenSessionExtensionSuite}
import org.apache.spark.sql.gluten.GlutenFallbackSuite
-import org.apache.spark.sql.hive.execution.GlutenHiveSQLQuerySuite
+import org.apache.spark.sql.hive.execution.{GlutenHiveExplainSuite,
GlutenHiveSQLQuerySuite}
import org.apache.spark.sql.sources._
// Some settings' line length exceeds 100
@@ -965,6 +965,7 @@ class BoltTestSettings extends BackendTestSettings {
.exclude("cases when literal is max")
enableSuite[GlutenXPathFunctionsSuite]
enableSuite[GlutenFallbackSuite]
+ enableSuite[GlutenHiveExplainSuite]
enableSuite[GlutenHiveSQLQuerySuite]
enableSuite[GlutenCollapseProjectExecTransformerSuite]
enableSuite[GlutenSparkSessionExtensionSuite]
diff --git a/gluten-ut/spark40/pom.xml b/gluten-ut/spark40/pom.xml
index 7c701cddcc..ee15d69a92 100644
--- a/gluten-ut/spark40/pom.xml
+++ b/gluten-ut/spark40/pom.xml
@@ -365,6 +365,87 @@
</build>
</profile>
+ <profile>
+ <id>backends-bolt</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter</artifactId>
+ <version>5.9.3</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>net.aichler</groupId>
+ <artifactId>jupiter-interface</artifactId>
+ <version>0.11.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.gluten</groupId>
+ <artifactId>backends-bolt</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <!-- log4j -->
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j2-impl</artifactId>
+ <version>${log4j.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.h2database</groupId>
+ <artifactId>h2</artifactId>
+ <version>2.2.220</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>add-sources</id>
+ <goals>
+ <goal>add-test-source</goal>
+ </goals>
+ <phase>generate-sources</phase>
+ <configuration>
+ <sources>
+ <source>src/test/backends-bolt</source>
+
<source>${project.basedir}/../spark40/src/test/bolt-shared</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>com.diffplug.spotless</groupId>
+ <artifactId>spotless-maven-plugin</artifactId>
+ <configuration>
+ <scala>
+ <includes combine.children="append">
+ <include>src/test/backends-bolt/**/*.scala</include>
+ <include>src/test/bolt-shared/**/*.scala</include>
+ </includes>
+ </scala>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
<profile>
<id>delta</id>
<dependencies>
diff --git
a/gluten-ut/spark40/src/test/backends-bolt/org/apache/gluten/utils/bolt/BoltTestSettings.scala
b/gluten-ut/spark40/src/test/backends-bolt/org/apache/gluten/utils/bolt/BoltTestSettings.scala
new file mode 100644
index 0000000000..5bad4f6d67
--- /dev/null
+++
b/gluten-ut/spark40/src/test/backends-bolt/org/apache/gluten/utils/bolt/BoltTestSettings.scala
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.utils.bolt
+
+// Suites added or renamed in this Spark version, in addition to the shared
regression inventory.
+class BoltTestSettings extends BoltSpark4TestSettings {
+ enableSuite[org.apache.spark.sql.GlutenDefaultANSIValueSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenCollationRegexpExpressionsSuite]
+ enableSuite[org.apache.spark.sql.execution.joins.GlutenHashedRelationSuite]
+ enableSuite[org.apache.spark.sql.execution.python.GlutenRowQueueSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenStreamingFullOuterJoinSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenStreamingInnerJoinSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenStreamingLeftSemiJoinSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenStreamingOuterJoinSuite]
+}
diff --git
a/gluten-ut/spark40/src/test/bolt-shared/org/apache/gluten/GlutenColumnarWriteTestSupport.scala
b/gluten-ut/spark40/src/test/bolt-shared/org/apache/gluten/GlutenColumnarWriteTestSupport.scala
new file mode 100644
index 0000000000..4258cd891a
--- /dev/null
+++
b/gluten-ut/spark40/src/test/bolt-shared/org/apache/gluten/GlutenColumnarWriteTestSupport.scala
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten
+
+import org.apache.spark.sql.execution.{ColumnarWriteFilesExec, SparkPlan}
+
+trait GlutenColumnarWriteTestSupport {
+
+ def checkWriteFilesAndGetChild(sparkPlan: SparkPlan): SparkPlan = {
+ assert(sparkPlan.isInstanceOf[ColumnarWriteFilesExec])
+ sparkPlan.asInstanceOf[ColumnarWriteFilesExec].child
+ }
+}
diff --git
a/gluten-ut/spark40/src/test/bolt-shared/org/apache/gluten/utils/bolt/BoltSpark4TestSettings.scala
b/gluten-ut/spark40/src/test/bolt-shared/org/apache/gluten/utils/bolt/BoltSpark4TestSettings.scala
new file mode 100644
index 0000000000..9116e747a9
--- /dev/null
+++
b/gluten-ut/spark40/src/test/bolt-shared/org/apache/gluten/utils/bolt/BoltSpark4TestSettings.scala
@@ -0,0 +1,630 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.utils.bolt
+
+import org.apache.gluten.utils.{BackendTestSettings, SQLQueryTestSettings}
+import org.apache.gluten.utils.velox.VeloxSQLQueryTestSettings
+
+import java.nio.file.Paths
+
+/**
+ * Spark regression inventory shared by 4.0, 4.1 and 4.2. Version-specific
suites are registered in
+ * each module's BoltTestSettings. Keep this aligned with the common Velox
suite inventory, but do
+ * not inherit Velox result exclusions or modified SQL inputs/goldens. Newly
exposed Bolt
+ * compatibility failures must be reported by nightly CI, not silently
filtered here.
+ */
+abstract class BoltSpark4TestSettings extends BackendTestSettings {
+ enableSuite[org.apache.spark.GlutenSortShuffleSuite]
+
enableSuite[org.apache.spark.sql.GlutenApproxCountDistinctForIntervalsQuerySuite]
+ enableSuite[org.apache.spark.sql.GlutenApproximatePercentileQuerySuite]
+ enableSuite[org.apache.spark.sql.GlutenBitmapExpressionsQuerySuite]
+ enableSuite[org.apache.spark.sql.GlutenBloomFilterAggregateQuerySuite]
+ enableSuite[org.apache.spark.sql.GlutenBloomFilterAggregateQuerySuiteCGOff]
+ enableSuite[org.apache.spark.sql.GlutenCTEHintSuite]
+ enableSuite[org.apache.spark.sql.GlutenCTEInlineSuiteAEOff]
+ enableSuite[org.apache.spark.sql.GlutenCTEInlineSuiteAEOn]
+ enableSuite[org.apache.spark.sql.GlutenCacheManagerSuite]
+ enableSuite[org.apache.spark.sql.GlutenCachedTableSuite]
+ enableSuite[org.apache.spark.sql.GlutenColumnExpressionSuite]
+ enableSuite[org.apache.spark.sql.GlutenConfigBehaviorSuite]
+ enableSuite[org.apache.spark.sql.GlutenCountMinSketchAggQuerySuite]
+ enableSuite[org.apache.spark.sql.GlutenCsvFunctionsSuite]
+ enableSuite[org.apache.spark.sql.GlutenDSV2CharVarcharTestSuite]
+ enableSuite[org.apache.spark.sql.GlutenDSV2SQLInsertTestSuite]
+ enableSuite[org.apache.spark.sql.GlutenDataFrameAggregateSuite]
+ enableSuite[org.apache.spark.sql.GlutenDataFrameAsOfJoinSuite]
+ enableSuite[org.apache.spark.sql.GlutenDataFrameComplexTypeSuite]
+ enableSuite[org.apache.spark.sql.GlutenDataFrameFunctionsSuite]
+ enableSuite[org.apache.spark.sql.GlutenDataFrameHintSuite]
+ enableSuite[org.apache.spark.sql.GlutenDataFrameImplicitsSuite]
+ enableSuite[org.apache.spark.sql.GlutenDataFrameJoinSuite]
+ enableSuite[org.apache.spark.sql.GlutenDataFrameNaFunctionsSuite]
+ enableSuite[org.apache.spark.sql.GlutenDataFramePivotSuite]
+ enableSuite[org.apache.spark.sql.GlutenDataFrameRangeSuite]
+ enableSuite[org.apache.spark.sql.GlutenDataFrameSelfJoinSuite]
+ enableSuite[org.apache.spark.sql.GlutenDataFrameSessionWindowingSuite]
+ enableSuite[org.apache.spark.sql.GlutenDataFrameSetOperationsSuite]
+ enableSuite[org.apache.spark.sql.GlutenDataFrameShowSuite]
+ enableSuite[org.apache.spark.sql.GlutenDataFrameStatSuite]
+ enableSuite[org.apache.spark.sql.GlutenDataFrameSubquerySuite]
+ enableSuite[org.apache.spark.sql.GlutenDataFrameSuite]
+ enableSuite[org.apache.spark.sql.GlutenDataFrameTableValuedFunctionsSuite]
+ enableSuite[org.apache.spark.sql.GlutenDataFrameTimeWindowingSuite]
+ enableSuite[org.apache.spark.sql.GlutenDataFrameToSchemaSuite]
+ enableSuite[org.apache.spark.sql.GlutenDataFrameTransposeSuite]
+ enableSuite[org.apache.spark.sql.GlutenDataFrameTungstenSuite]
+ enableSuite[org.apache.spark.sql.GlutenDataFrameWindowFramesSuite]
+ enableSuite[org.apache.spark.sql.GlutenDataFrameWindowFunctionsSuite]
+ enableSuite[org.apache.spark.sql.GlutenDataFrameWriterV2Suite]
+ enableSuite[org.apache.spark.sql.GlutenDatasetAggregatorSuite]
+ enableSuite[org.apache.spark.sql.GlutenDatasetCacheSuite]
+ enableSuite[org.apache.spark.sql.GlutenDatasetOptimizationSuite]
+ enableSuite[org.apache.spark.sql.GlutenDatasetPrimitiveSuite]
+ enableSuite[org.apache.spark.sql.GlutenDatasetSerializerRegistratorSuite]
+ enableSuite[org.apache.spark.sql.GlutenDatasetSuite]
+ enableSuite[org.apache.spark.sql.GlutenDatasetUnpivotSuite]
+ enableSuite[org.apache.spark.sql.GlutenDateFunctionsSuite]
+ enableSuite[org.apache.spark.sql.GlutenDeprecatedAPISuite]
+ enableSuite[org.apache.spark.sql.GlutenDeprecatedDatasetAggregatorSuite]
+ enableSuite[org.apache.spark.sql.GlutenDynamicPartitionPruningV1SuiteAEOff]
+
enableSuite[org.apache.spark.sql.GlutenDynamicPartitionPruningV1SuiteAEOffDisableScan]
+
enableSuite[org.apache.spark.sql.GlutenDynamicPartitionPruningV1SuiteAEOffWSCGOffDisableProject]
+
enableSuite[org.apache.spark.sql.GlutenDynamicPartitionPruningV1SuiteAEOffWSCGOnDisableProject]
+ enableSuite[org.apache.spark.sql.GlutenDynamicPartitionPruningV1SuiteAEOn]
+
enableSuite[org.apache.spark.sql.GlutenDynamicPartitionPruningV1SuiteAEOnDisableScan]
+ enableSuite[org.apache.spark.sql.GlutenDynamicPartitionPruningV2SuiteAEOff]
+
enableSuite[org.apache.spark.sql.GlutenDynamicPartitionPruningV2SuiteAEOffDisableScan]
+
enableSuite[org.apache.spark.sql.GlutenDynamicPartitionPruningV2SuiteAEOffWSCGOffDisableProject]
+
enableSuite[org.apache.spark.sql.GlutenDynamicPartitionPruningV2SuiteAEOffWSCGOnDisableProject]
+ enableSuite[org.apache.spark.sql.GlutenDynamicPartitionPruningV2SuiteAEOn]
+
enableSuite[org.apache.spark.sql.GlutenDynamicPartitionPruningV2SuiteAEOnDisableScan]
+ enableSuite[org.apache.spark.sql.GlutenEmptyInSuite]
+ enableSuite[org.apache.spark.sql.GlutenExpressionsSchemaSuite]
+ enableSuite[org.apache.spark.sql.GlutenExtraStrategiesSuite]
+ enableSuite[org.apache.spark.sql.GlutenFileBasedDataSourceSuite]
+ enableSuite[org.apache.spark.sql.GlutenFileScanSuite]
+ enableSuite[org.apache.spark.sql.GlutenFileSourceCharVarcharTestSuite]
+ enableSuite[org.apache.spark.sql.GlutenFileSourceSQLInsertTestSuite]
+ enableSuite[org.apache.spark.sql.GlutenGeneratorFunctionSuite]
+ enableSuite[org.apache.spark.sql.GlutenICUCollationsMapSuite]
+ enableSuite[org.apache.spark.sql.GlutenInjectRuntimeFilterSuite]
+ enableSuite[org.apache.spark.sql.GlutenInlineTableParsingImprovementsSuite]
+ enableSuite[org.apache.spark.sql.GlutenIntervalFunctionsSuite]
+ enableSuite[org.apache.spark.sql.GlutenJoinHintSuite]
+ enableSuite[org.apache.spark.sql.GlutenJoinSuite]
+ enableSuite[org.apache.spark.sql.GlutenJsonFunctionsSuite]
+ enableSuite[org.apache.spark.sql.GlutenLateralColumnAliasSuite]
+ enableSuite[org.apache.spark.sql.GlutenLogQuerySuite]
+ enableSuite[org.apache.spark.sql.GlutenMathFunctionsSuite]
+ enableSuite[org.apache.spark.sql.GlutenMetadataCacheSuite]
+ enableSuite[org.apache.spark.sql.GlutenMiscFunctionsSuite]
+ enableSuite[org.apache.spark.sql.GlutenNestedDataSourceV1Suite]
+ enableSuite[org.apache.spark.sql.GlutenNestedDataSourceV2Suite]
+ enableSuite[org.apache.spark.sql.GlutenParametersSuite]
+ enableSuite[org.apache.spark.sql.GlutenPercentileQuerySuite]
+ enableSuite[org.apache.spark.sql.GlutenProcessingTimeSuite]
+ enableSuite[org.apache.spark.sql.GlutenProductAggSuite]
+ enableSuite[org.apache.spark.sql.GlutenRandomDataGeneratorSuite]
+
enableSuite[org.apache.spark.sql.GlutenReplaceNullWithFalseInPredicateEndToEndSuite]
+ enableSuite[org.apache.spark.sql.GlutenResolveDefaultColumnsSuite]
+ enableSuite[org.apache.spark.sql.GlutenRowJsonSuite]
+ enableSuite[org.apache.spark.sql.GlutenRowSuite]
+ enableSuite[org.apache.spark.sql.GlutenRuntimeConfigSuite]
+ enableSuite[org.apache.spark.sql.GlutenRuntimeNullChecksV2Writes]
+ enableSuite[org.apache.spark.sql.GlutenSQLQuerySuite]
+ enableSuite[org.apache.spark.sql.GlutenSQLQueryTestSuite]
+ enableSuite[org.apache.spark.sql.GlutenSSBQuerySuite]
+ enableSuite[org.apache.spark.sql.GlutenScalaReflectionRelationSuite]
+ enableSuite[org.apache.spark.sql.GlutenSerializationSuite]
+ enableSuite[org.apache.spark.sql.GlutenSessionStateSuite]
+ enableSuite[org.apache.spark.sql.GlutenSetCommandSuite]
+ enableSuite[org.apache.spark.sql.GlutenSparkSessionBuilderSuite]
+ enableSuite[org.apache.spark.sql.GlutenSparkSessionExtensionSuite]
+
enableSuite[org.apache.spark.sql.GlutenSparkSessionJobTaggingAndCancellationSuite]
+ enableSuite[org.apache.spark.sql.GlutenStatisticsCollectionSuite]
+ enableSuite[org.apache.spark.sql.GlutenStringFunctionsSuite]
+ enableSuite[org.apache.spark.sql.GlutenSubqueryHintPropagationSuite]
+ enableSuite[org.apache.spark.sql.GlutenSubquerySuite]
+ enableSuite[org.apache.spark.sql.GlutenTPCDSCollationQueryTestSuite]
+ enableSuite[org.apache.spark.sql.GlutenTPCDSQueryANSISuite]
+ enableSuite[org.apache.spark.sql.GlutenTPCDSQuerySuite]
+ enableSuite[org.apache.spark.sql.GlutenTPCDSQueryTestSuite]
+ enableSuite[org.apache.spark.sql.GlutenTPCDSQueryWithStatsSuite]
+ enableSuite[org.apache.spark.sql.GlutenTPCHQuerySuite]
+ enableSuite[org.apache.spark.sql.GlutenTableOptionsConstantFoldingSuite]
+ enableSuite[org.apache.spark.sql.GlutenTypedImperativeAggregateSuite]
+ enableSuite[org.apache.spark.sql.GlutenUDFSuite]
+ enableSuite[org.apache.spark.sql.GlutenUDTRegistrationSuite]
+ enableSuite[org.apache.spark.sql.GlutenUnsafeRowSuite]
+ enableSuite[org.apache.spark.sql.GlutenUnwrapCastInComparisonEndToEndSuite]
+ enableSuite[org.apache.spark.sql.GlutenUrlFunctionsSuite]
+ enableSuite[org.apache.spark.sql.GlutenUserDefinedTypeSuite]
+ enableSuite[org.apache.spark.sql.GlutenVariantEndToEndSuite]
+ enableSuite[org.apache.spark.sql.GlutenVariantShreddingSuite]
+ enableSuite[org.apache.spark.sql.GlutenVariantSuite]
+ enableSuite[org.apache.spark.sql.GlutenVariantWriteShreddingSuite]
+ enableSuite[org.apache.spark.sql.GlutenXPathFunctionsSuite]
+ enableSuite[org.apache.spark.sql.GlutenXmlFunctionsSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenArithmeticExpressionSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenAttributeResolutionSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenAttributeSetSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenBitmapExpressionUtilsSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenBitwiseExpressionsSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenCallMethodViaReflectionSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenCanonicalizeSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenCastWithAnsiOffSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenCodeGenerationSuite]
+ enableSuite[
+
org.apache.spark.sql.catalyst.expressions.GlutenCodeGeneratorWithInterpretedFallbackSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenCollationExpressionSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenCollectionExpressionsSuite]
+ enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenComplexTypeSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenConditionalExpressionSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenCsvExpressionsSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenDateExpressionsSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenDecimalExpressionSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenDecimalPrecisionSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenDynamicPruningSubquerySuite]
+ enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenExprIdSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenExpressionImplUtilsSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenExpressionSQLBuilderSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenExpressionSetSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenExtractPredicatesWithinOutputSetSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenGeneratorExpressionSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenHashExpressionsSuite]
+ enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenHexSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenHigherOrderFunctionsSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenIntervalExpressionsSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenJsonExpressionsSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenLiteralExpressionSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenMathExpressionsSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenMiscExpressionsSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenMutableProjectionSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenNamedExpressionSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenNondeterministicSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenNullExpressionsSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenObjectExpressionsSuite]
+ enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenOrderingSuite]
+ enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenPredicateSuite]
+ enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenRandomSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenRegexpExpressionsSuite]
+ enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenScalaUDFSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenSchemaPruningSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenSelectedFieldSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenSortOrderExpressionsSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenStringExpressionsSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenSubexpressionEliminationSuite]
+ enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenTimeWindowSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenToPrettyStringSuite]
+ enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenTryCastSuite]
+ enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenTryEvalSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenUnsafeRowConverterSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenUnwrapUDTExpressionSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenV2ExpressionUtilsSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenValidateExternalTypeSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenXmlExpressionsSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.aggregate.GlutenAggregateExpressionSuite]
+ enableSuite[
+
org.apache.spark.sql.catalyst.expressions.aggregate.GlutenApproxCountDistinctForIntervalsSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.aggregate.GlutenApproximatePercentileSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.aggregate.GlutenCentralMomentAggSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.aggregate.GlutenCountMinSketchAggSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.aggregate.GlutenCovarianceAggSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.aggregate.GlutenDatasketchesHllSketchSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.aggregate.GlutenFirstLastTestSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.aggregate.GlutenHistogramNumericSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.aggregate.GlutenHyperLogLogPlusPlusSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.aggregate.GlutenProductAggSuite]
+
enableSuite[org.apache.spark.sql.connector.GlutenDataSourceV2DataFrameSessionCatalogSuite]
+ enableSuite[org.apache.spark.sql.connector.GlutenDataSourceV2DataFrameSuite]
+ enableSuite[org.apache.spark.sql.connector.GlutenDataSourceV2FunctionSuite]
+ enableSuite[org.apache.spark.sql.connector.GlutenDataSourceV2MetricsSuite]
+ enableSuite[org.apache.spark.sql.connector.GlutenDataSourceV2OptionSuite]
+
enableSuite[org.apache.spark.sql.connector.GlutenDataSourceV2SQLSessionCatalogSuite]
+
enableSuite[org.apache.spark.sql.connector.GlutenDataSourceV2SQLSuiteV1Filter]
+
enableSuite[org.apache.spark.sql.connector.GlutenDataSourceV2SQLSuiteV2Filter]
+ enableSuite[org.apache.spark.sql.connector.GlutenDataSourceV2Suite]
+ enableSuite[org.apache.spark.sql.connector.GlutenDataSourceV2UtilsSuite]
+ enableSuite[org.apache.spark.sql.connector.GlutenDeleteFromTableSuite]
+
enableSuite[org.apache.spark.sql.connector.GlutenDeltaBasedDeleteFromTableSuite]
+
enableSuite[org.apache.spark.sql.connector.GlutenDeltaBasedMergeIntoTableSuite]
+ enableSuite[
+
org.apache.spark.sql.connector.GlutenDeltaBasedMergeIntoTableUpdateAsDeleteAndInsertSuite]
+
enableSuite[org.apache.spark.sql.connector.GlutenDeltaBasedUpdateAsDeleteAndInsertTableSuite]
+ enableSuite[org.apache.spark.sql.connector.GlutenDeltaBasedUpdateTableSuite]
+
enableSuite[org.apache.spark.sql.connector.GlutenFileDataSourceV2FallBackSuite]
+
enableSuite[org.apache.spark.sql.connector.GlutenGroupBasedDeleteFromTableSuite]
+
enableSuite[org.apache.spark.sql.connector.GlutenGroupBasedMergeIntoTableSuite]
+ enableSuite[org.apache.spark.sql.connector.GlutenGroupBasedUpdateTableSuite]
+ enableSuite[org.apache.spark.sql.connector.GlutenKeyGroupedPartitioningSuite]
+ enableSuite[org.apache.spark.sql.connector.GlutenLocalScanSuite]
+ enableSuite[org.apache.spark.sql.connector.GlutenMergeIntoDataFrameSuite]
+ enableSuite[org.apache.spark.sql.connector.GlutenMetadataColumnSuite]
+ enableSuite[org.apache.spark.sql.connector.GlutenProcedureSuite]
+ enableSuite[org.apache.spark.sql.connector.GlutenPushablePredicateSuite]
+ enableSuite[org.apache.spark.sql.connector.GlutenSupportsCatalogOptionsSuite]
+ enableSuite[org.apache.spark.sql.connector.GlutenTableCapabilityCheckSuite]
+
enableSuite[org.apache.spark.sql.connector.GlutenV1ReadFallbackWithCatalogSuite]
+
enableSuite[org.apache.spark.sql.connector.GlutenV1ReadFallbackWithDataFrameReaderSuite]
+
enableSuite[org.apache.spark.sql.connector.GlutenV1WriteFallbackSessionCatalogSuite]
+ enableSuite[org.apache.spark.sql.connector.GlutenV1WriteFallbackSuite]
+
enableSuite[org.apache.spark.sql.connector.GlutenV2CommandsCaseSensitivitySuite]
+
enableSuite[org.apache.spark.sql.connector.GlutenWriteDistributionAndOrderingSuite]
+
enableSuite[org.apache.spark.sql.errors.GlutenQueryCompilationErrorsDSv2Suite]
+ enableSuite[org.apache.spark.sql.errors.GlutenQueryCompilationErrorsSuite]
+ enableSuite[org.apache.spark.sql.errors.GlutenQueryContextSuite]
+ enableSuite[org.apache.spark.sql.errors.GlutenQueryExecutionAnsiErrorsSuite]
+ enableSuite[org.apache.spark.sql.errors.GlutenQueryExecutionErrorsSuite]
+ enableSuite[org.apache.spark.sql.errors.GlutenQueryParsingErrorsSuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenAggregatingAccumulatorSuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenBroadcastExchangeSuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenCoGroupedIteratorSuite]
+
enableSuite[org.apache.spark.sql.execution.GlutenCoalesceShufflePartitionsSuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenColumnarRulesSuite]
+
enableSuite[org.apache.spark.sql.execution.GlutenDataSourceScanExecRedactionSuite]
+
enableSuite[org.apache.spark.sql.execution.GlutenDataSourceV2ScanExecRedactionSuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenExchangeSuite]
+
enableSuite[org.apache.spark.sql.execution.GlutenExecuteImmediateEndToEndSuite]
+
enableSuite[org.apache.spark.sql.execution.GlutenExternalAppendOnlyUnsafeRowArraySuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenGlobalTempViewSuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenGlobalTempViewTestSuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenGroupedIteratorSuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenHiveResultSuite]
+
enableSuite[org.apache.spark.sql.execution.GlutenInsertSortForLimitAndOffsetSuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenLocalBroadcastExchangeSuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenLocalTempViewTestSuite]
+
enableSuite[org.apache.spark.sql.execution.GlutenLogicalPlanTagInSparkPlanSuite]
+
enableSuite[org.apache.spark.sql.execution.GlutenOptimizeMetadataOnlyQuerySuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenPersistedViewTestSuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenQueryExecutionSuite]
+
enableSuite[org.apache.spark.sql.execution.GlutenQueryPlanningTrackerEndToEndSuite]
+
enableSuite[org.apache.spark.sql.execution.GlutenRemoveRedundantProjectsSuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenRemoveRedundantSortsSuite]
+
enableSuite[org.apache.spark.sql.execution.GlutenRemoveRedundantWindowGroupLimitsSuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenReplaceHashWithSortAggSuite]
+
enableSuite[org.apache.spark.sql.execution.GlutenReuseExchangeAndSubquerySuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenRowToColumnConverterSuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenSQLAggregateFunctionSuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenSQLCollectLimitExecSuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenSQLExecutionSuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenSQLFunctionSuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenSQLJsonProtocolSuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenSQLWindowFunctionSuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenSameResultSuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenShufflePartitionsUtilSuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenSimpleSQLViewSuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenSortSuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenSparkPlanSuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenSparkPlannerSuite]
+
enableSuite[org.apache.spark.sql.execution.GlutenSparkScriptTransformationSuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenSparkSqlParserSuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenTakeOrderedAndProjectSuite]
+
enableSuite[org.apache.spark.sql.execution.GlutenUnsafeFixedWidthAggregationMapSuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenUnsafeKVExternalSorterSuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenUnsafeRowSerializerSuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenWholeStageCodegenSuite]
+ // Rewrite with Gluten-aware native whole-stage plan assertions.
+ .exclude("range/filter should be combined")
+ .exclude("HashAggregate should be included in WholeStageCodegen")
+ .exclude("SortAggregate should be included in WholeStageCodegen")
+ .exclude("GenerateExec should be included in WholeStageCodegen
(whole-stage-codegen on)")
+ .exclude("HashAggregate with grouping keys should be included in
WholeStageCodegen")
+ .exclude("BroadcastHashJoin should be included in WholeStageCodegen")
+ .exclude("Inner ShuffledHashJoin should be included in WholeStageCodegen")
+ .exclude(
+ "Full Outer ShuffledHashJoin and SortMergeJoin should be included in
WholeStageCodegen")
+ .exclude("SPARK-44060 Code-gen for build side outer shuffled hash join")
+ .exclude("Left/Right Outer SortMergeJoin should be included in
WholeStageCodegen")
+ .exclude("Left Semi SortMergeJoin should be included in WholeStageCodegen")
+ .exclude("Left Anti SortMergeJoin should be included in WholeStageCodegen")
+ .exclude("Inner/Cross BroadcastNestedLoopJoinExec should be included in
WholeStageCodegen")
+ .exclude("Left/Right outer BroadcastNestedLoopJoinExec should be included
in WholeStageCodegen")
+ .exclude("Left semi/anti BroadcastNestedLoopJoinExec should be included in
WholeStageCodegen")
+ .exclude("Sort should be included in WholeStageCodegen")
+ .exclude("Control splitting consume function by operators with config")
+ .exclude("Skip splitting consume function when parameter number exceeds
JVM limit")
+ .exclude(
+ "including codegen stage ID in generated class name should not regress
codegen caching")
+ .exclude("SPARK-26572: evaluate non-deterministic expressions for
aggregate results")
+ .exclude("SPARK-28520: WholeStageCodegen does not work properly for
LocalTableScanExec")
+ .exclude("Give up splitting aggregate code if a parameter length goes over
the limit")
+ .exclude("Give up splitting subexpression code if a parameter length goes
over the limit")
+ .exclude("SPARK-47238: Test broadcast threshold for generated code")
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenBasicWriteJobStatsTrackerMetricSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenBasicWriteTaskStatsTrackerSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenBucketingUtilsSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenCSVReadSchemaSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenCustomWriteTaskStatsTrackerSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenDataSourceManagerSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenDataSourceResolverSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenDataSourceStrategySuite]
+ enableSuite[org.apache.spark.sql.execution.datasources.GlutenDataSourceSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenFileFormatWriterSuite]
+ enableSuite[org.apache.spark.sql.execution.datasources.GlutenFileIndexSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenFileMetadataStructSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenFileResolverSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenFileSourceCustomMetadataStructSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenFileSourceStrategySuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenHadoopFileLinesReaderSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenHeaderCSVReadSchemaSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenInMemoryTableMetricSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenJsonReadSchemaSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenMergedOrcReadSchemaSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenMergedParquetReadSchemaSuite]
+ enableSuite[org.apache.spark.sql.execution.datasources.GlutenOrcCodecSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenOrcReadSchemaSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenOrcV1AggregatePushDownSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenOrcV2AggregatePushDownSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenParquetCodecSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenParquetReadSchemaSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenParquetV1AggregatePushDownSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenParquetV2AggregatePushDownSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenPathFilterStrategySuite]
+ enableSuite[org.apache.spark.sql.execution.datasources.GlutenPathFilterSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenPruneFileSourcePartitionsSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenPushVariantIntoScanSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenRowDataSourceStrategySuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenSaveIntoDataSourceCommandSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenTableLocationSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenV1WriteCommandSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenVectorizedOrcReadSchemaSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.GlutenVectorizedParquetReadSchemaSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.binaryfile.GlutenBinaryFileFormatSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.csv.GlutenCSVLegacyTimeParserSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.csv.GlutenCSVParsingOptionsSuite]
+ enableSuite[org.apache.spark.sql.execution.datasources.csv.GlutenCSVv1Suite]
+ enableSuite[org.apache.spark.sql.execution.datasources.csv.GlutenCSVv2Suite]
+
enableSuite[org.apache.spark.sql.execution.datasources.json.GlutenJsonLegacyTimeParserSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.json.GlutenJsonParsingOptionsSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.json.GlutenJsonV1Suite]
+
enableSuite[org.apache.spark.sql.execution.datasources.json.GlutenJsonV2Suite]
+
enableSuite[org.apache.spark.sql.execution.datasources.orc.GlutenOrcColumnarBatchReaderSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.orc.GlutenOrcEncryptionSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.orc.GlutenOrcFilterSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.orc.GlutenOrcPartitionDiscoverySuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.orc.GlutenOrcSourceSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.orc.GlutenOrcV1FilterSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.orc.GlutenOrcV1PartitionDiscoverySuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.orc.GlutenOrcV1QuerySuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.orc.GlutenOrcV1SchemaPruningSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.orc.GlutenOrcV2QuerySuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.orc.GlutenOrcV2SchemaPruningSuite]
+ enableSuite[
+
org.apache.spark.sql.execution.datasources.parquet.GlutenParquetAvroCompatibilitySuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.parquet.GlutenParquetColumnIndexSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.parquet.GlutenParquetCommitterSuite]
+ enableSuite[
+
org.apache.spark.sql.execution.datasources.parquet.GlutenParquetCompressionCodecPrecedenceSuite]
+ enableSuite[
+
org.apache.spark.sql.execution.datasources.parquet.GlutenParquetDeltaByteArrayEncodingSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.parquet.GlutenParquetDeltaEncodingInteger]
+
enableSuite[org.apache.spark.sql.execution.datasources.parquet.GlutenParquetDeltaEncodingLong]
+
enableSuite[org.apache.spark.sql.execution.datasources.parquet.GlutenParquetDeltaLengthByteArrayEncodingSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.parquet.GlutenParquetEncodingSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.parquet.GlutenParquetFieldIdIOSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.parquet.GlutenParquetFieldIdSchemaSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.parquet.GlutenParquetFileFormatV1Suite]
+
enableSuite[org.apache.spark.sql.execution.datasources.parquet.GlutenParquetFileFormatV2Suite]
+ enableSuite[
+
org.apache.spark.sql.execution.datasources.parquet.GlutenParquetFileMetadataStructRowIndexSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.parquet.GlutenParquetIOSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.parquet.GlutenParquetInteroperabilitySuite]
+ enableSuite[
+
org.apache.spark.sql.execution.datasources.parquet.GlutenParquetProtobufCompatibilitySuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.parquet.GlutenParquetRebaseDatetimeV1Suite]
+
enableSuite[org.apache.spark.sql.execution.datasources.parquet.GlutenParquetRebaseDatetimeV2Suite]
+
enableSuite[org.apache.spark.sql.execution.datasources.parquet.GlutenParquetRowIndexSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.parquet.GlutenParquetSchemaInferenceSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.parquet.GlutenParquetSchemaSuite]
+ enableSuite[
+
org.apache.spark.sql.execution.datasources.parquet.GlutenParquetThriftCompatibilitySuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.parquet.GlutenParquetTypeWideningSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.parquet.GlutenParquetV1FilterSuite]
+ enableSuite[
+
org.apache.spark.sql.execution.datasources.parquet.GlutenParquetV1PartitionDiscoverySuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.parquet.GlutenParquetV1QuerySuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.parquet.GlutenParquetV1SchemaPruningSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.parquet.GlutenParquetV2FilterSuite]
+ enableSuite[
+
org.apache.spark.sql.execution.datasources.parquet.GlutenParquetV2PartitionDiscoverySuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.parquet.GlutenParquetV2QuerySuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.parquet.GlutenParquetV2SchemaPruningSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.parquet.GlutenParquetVariantShreddingSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.parquet.GlutenParquetVectorizedSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.text.GlutenTextV1Suite]
+
enableSuite[org.apache.spark.sql.execution.datasources.text.GlutenTextV2Suite]
+
enableSuite[org.apache.spark.sql.execution.datasources.text.GlutenWholeTextFileV1Suite]
+
enableSuite[org.apache.spark.sql.execution.datasources.text.GlutenWholeTextFileV2Suite]
+
enableSuite[org.apache.spark.sql.execution.datasources.v2.GlutenDataSourceV2StrategySuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.v2.GlutenFileTableSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.v2.GlutenFileWriterFactorySuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.v2.GlutenV2PredicateSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.v2.GlutenV2SessionCatalogNamespaceSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.v2.GlutenV2SessionCatalogTableSuite]
+
enableSuite[org.apache.spark.sql.execution.exchange.GlutenEnsureRequirementsSuite]
+
enableSuite[org.apache.spark.sql.execution.exchange.GlutenValidateRequirementsSuite]
+ enableSuite[org.apache.spark.sql.execution.joins.GlutenBroadcastJoinSuite]
+ enableSuite[org.apache.spark.sql.execution.joins.GlutenExistenceJoinSuite]
+
enableSuite[org.apache.spark.sql.execution.joins.GlutenInnerJoinSuiteForceShjOff]
+
enableSuite[org.apache.spark.sql.execution.joins.GlutenInnerJoinSuiteForceShjOn]
+
enableSuite[org.apache.spark.sql.execution.joins.GlutenOuterJoinSuiteForceShjOff]
+
enableSuite[org.apache.spark.sql.execution.joins.GlutenOuterJoinSuiteForceShjOn]
+ enableSuite[org.apache.spark.sql.execution.joins.GlutenSingleJoinSuite]
+ enableSuite[org.apache.spark.sql.execution.metric.GlutenCustomMetricsSuite]
+ enableSuite[org.apache.spark.sql.execution.metric.GlutenSQLMetricsSuite]
+
enableSuite[org.apache.spark.sql.execution.python.GlutenBatchEvalPythonExecSuite]
+
enableSuite[org.apache.spark.sql.execution.python.GlutenExtractPythonUDFsSuite]
+
enableSuite[org.apache.spark.sql.extension.GlutenCollapseProjectExecTransformerSuite]
+ enableSuite[org.apache.spark.sql.extension.GlutenSessionExtensionSuite]
+ enableSuite[org.apache.spark.sql.gluten.GlutenFallbackStrategiesSuite]
+ enableSuite[org.apache.spark.sql.gluten.GlutenFallbackSuite]
+
enableSuite[org.apache.spark.sql.hive.execution.GlutenHashAggregationQuerySuite]
+ enableSuite[
+
org.apache.spark.sql.hive.execution.GlutenHashAggregationQueryWithControlledFallbackSuite]
+ enableSuite[org.apache.spark.sql.hive.execution.GlutenHashUDAQuerySuite]
+
enableSuite[org.apache.spark.sql.hive.execution.GlutenHashUDAQueryWithControlledFallbackSuite]
+ enableSuite[org.apache.spark.sql.hive.execution.GlutenHiveCommandSuite]
+ enableSuite[org.apache.spark.sql.hive.execution.GlutenHiveDDLSuite]
+ enableSuite[org.apache.spark.sql.hive.execution.GlutenHiveExplainSuite]
+ enableSuite[org.apache.spark.sql.hive.execution.GlutenHivePlanTest]
+ enableSuite[org.apache.spark.sql.hive.execution.GlutenHiveQuerySuite]
+ enableSuite[org.apache.spark.sql.hive.execution.GlutenHiveResolutionSuite]
+ enableSuite[org.apache.spark.sql.hive.execution.GlutenHiveSQLQuerySuite]
+ enableSuite[org.apache.spark.sql.hive.execution.GlutenHiveSQLViewSuite]
+
enableSuite[org.apache.spark.sql.hive.execution.GlutenHiveScriptTransformationSuite]
+
enableSuite[org.apache.spark.sql.hive.execution.GlutenHiveSerDeReadWriteSuite]
+ enableSuite[org.apache.spark.sql.hive.execution.GlutenHiveSerDeSuite]
+ enableSuite[org.apache.spark.sql.hive.execution.GlutenHiveTableScanSuite]
+ enableSuite[org.apache.spark.sql.hive.execution.GlutenHiveTypeCoercionSuite]
+ enableSuite[org.apache.spark.sql.hive.execution.GlutenHiveUDAFSuite]
+ enableSuite[org.apache.spark.sql.hive.execution.GlutenHiveUDFSuite]
+
enableSuite[org.apache.spark.sql.hive.execution.GlutenObjectHashAggregateSuite]
+
enableSuite[org.apache.spark.sql.hive.execution.GlutenPruneHiveTablePartitionsSuite]
+ enableSuite[org.apache.spark.sql.hive.execution.GlutenPruningSuite]
+ enableSuite[org.apache.spark.sql.hive.execution.GlutenSQLMetricsSuite]
+ enableSuite[org.apache.spark.sql.hive.execution.GlutenSQLQuerySuite]
+ enableSuite[org.apache.spark.sql.hive.execution.GlutenSQLQuerySuiteAE]
+ enableSuite[org.apache.spark.sql.hive.execution.GlutenWindowQuerySuite]
+
enableSuite[org.apache.spark.sql.sources.GlutenBucketedReadWithoutHiveSupportSuite]
+
enableSuite[org.apache.spark.sql.sources.GlutenBucketedWriteWithoutHiveSupportSuite]
+ enableSuite[org.apache.spark.sql.sources.GlutenCreateTableAsSelectSuite]
+ enableSuite[org.apache.spark.sql.sources.GlutenDDLSourceLoadSuite]
+ enableSuite[org.apache.spark.sql.sources.GlutenDataSourceAnalysisSuite]
+ enableSuite[
+
org.apache.spark.sql.sources.GlutenDisableUnnecessaryBucketedScanWithoutHiveSupportSuiteAE]
+ enableSuite[org.apache.spark.sql.sources.GlutenExternalCommandRunnerSuite]
+ enableSuite[org.apache.spark.sql.sources.GlutenFilteredScanSuite]
+ enableSuite[org.apache.spark.sql.sources.GlutenFiltersSuite]
+ enableSuite[org.apache.spark.sql.sources.GlutenInsertSuite]
+ enableSuite[org.apache.spark.sql.sources.GlutenPartitionedWriteSuite]
+ enableSuite[org.apache.spark.sql.sources.GlutenPathOptionSuite]
+ enableSuite[org.apache.spark.sql.sources.GlutenPrunedScanSuite]
+ enableSuite[org.apache.spark.sql.sources.GlutenResolvedDataSourceSuite]
+ enableSuite[org.apache.spark.sql.sources.GlutenSaveLoadSuite]
+ enableSuite[org.apache.spark.sql.sources.GlutenTableScanSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenAcceptsLatestSeenOffsetSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenCommitLogSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenEventTimeWatermarkSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenFileStreamSinkV1Suite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenFileStreamSinkV2Suite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenFileStreamSourceStressTestSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenFileStreamSourceSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenFileStreamStressSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenFlatMapGroupsInPandasWithStateDistributionSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenFlatMapGroupsInPandasWithStateSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenFlatMapGroupsWithStateDistributionSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenFlatMapGroupsWithStateSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenFlatMapGroupsWithStateWithInitialStateSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenGroupStateSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenLongOffsetSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenMemorySourceStressSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenMultiStatefulOperatorsSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenReportSinkMetricsSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenRocksDBStateStoreFlatMapGroupsWithStateSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenRocksDBStateStoreStreamingAggregationSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenRocksDBStateStoreStreamingDeduplicationSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenStreamSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenStreamingAggregationDistributionSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenStreamingAggregationSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenStreamingDeduplicationDistributionSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenStreamingDeduplicationSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenStreamingDeduplicationWithinWatermarkSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenStreamingQueryHashPartitionVerifySuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenStreamingQueryListenerSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenStreamingQueryListenersConfSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenStreamingQueryManagerSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenStreamingQueryOptimizationCorrectnessSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenStreamingQueryStatusAndProgressSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenStreamingQuerySuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenStreamingSelfUnionSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenStreamingSessionWindowDistributionSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenStreamingSessionWindowSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenStreamingStateStoreFormatCompatibilitySuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenStreamingSymmetricHashJoinHelperSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenTransformWithListStateSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenTransformWithListStateTTLSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenTransformWithMapStateSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenTransformWithMapStateTTLSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenTransformWithStateAvroSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenTransformWithStateChainingSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenTransformWithStateClusterSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenTransformWithStateInitialStateSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenTransformWithStateUnsafeRowSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenTransformWithStateValidationSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenTransformWithValueStateTTLSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenTriggerAvailableNowSuite]
+
+ // Explicit implementation/harness boundaries; these are not native result
failures.
+ disableSuite[org.apache.spark.sql.GlutenExplainSuite](
+ "Validates Spark-specific physical plans and JVM codegen output, neither
of which applies to Gluten's native execution plan")
+ disableSuite[org.apache.spark.sql.GlutenTPCDSModifiedPlanStabilitySuite](
+ "Requires backend-specific approved plan files; Bolt goldens are not
available.")
+
disableSuite[org.apache.spark.sql.GlutenTPCDSModifiedPlanStabilityWithStatsSuite](
+ "Requires backend-specific approved plan files; Bolt goldens are not
available.")
+ disableSuite[org.apache.spark.sql.GlutenTPCDSV1_4_PlanStabilitySuite](
+ "Requires backend-specific approved plan files; Bolt goldens are not
available.")
+
disableSuite[org.apache.spark.sql.GlutenTPCDSV1_4_PlanStabilityWithStatsSuite](
+ "Requires backend-specific approved plan files; Bolt goldens are not
available.")
+ disableSuite[org.apache.spark.sql.GlutenTPCDSV2_7_PlanStabilitySuite](
+ "Requires backend-specific approved plan files; Bolt goldens are not
available.")
+
disableSuite[org.apache.spark.sql.GlutenTPCDSV2_7_PlanStabilityWithStatsSuite](
+ "Requires backend-specific approved plan files; Bolt goldens are not
available.")
+ disableSuite[org.apache.spark.sql.GlutenTPCHPlanStabilitySuite](
+ "Requires backend-specific approved plan files; Bolt goldens are not
available.")
+
disableSuite[org.apache.spark.sql.catalyst.expressions.GlutenExpressionEvalHelperSuite](
+ "Validates Spark's ExpressionEvalHelper contract, while Gluten overrides
checkEvaluation/checkExceptionInExpression")
+
disableSuite[org.apache.spark.sql.catalyst.expressions.GlutenSubExprEvaluationRuntimeSuite](
+ "Spark's test JAR uses unshaded Guava, while SubExprEvaluationRuntime uses
shaded Guava")
+ disableSuite[org.apache.spark.sql.execution.GlutenPlannerSuite](
+ "Validates Spark planner implementation details")
+
disableSuite[org.apache.spark.sql.execution.GlutenProjectedOrderingAndPartitioningSuite](
+ "Validates Spark planner output ordering and partitioning metadata")
+
disableSuite[org.apache.spark.sql.execution.GlutenWholeStageCodegenSparkSubmitSuite](
+ "The SparkSubmit test launches Spark's main class without the Gluten
plugin")
+ disableSuite(
+
"org.apache.spark.sql.execution.adaptive.velox.VeloxAdaptiveQueryExecSuite",
+ "Tests Velox-specific adaptive operators; no Spark 4 Bolt counterpart
exists."
+ )
+ disableSuite[
+
org.apache.spark.sql.sources.GlutenDisableUnnecessaryBucketedScanWithoutHiveSupportSuite](
+ "GLUTEN-4893: Vanilla UT checks scan operator by exactly matching the
class type")
+
+ override def getSQLQueryTestSettings: SQLQueryTestSettings =
BoltSQLQueryTestSettings
+}
+
+private object BoltSQLQueryTestSettings extends SQLQueryTestSettings {
+ // Reuse only each Spark version's case inventory, including cases Velox
rewrites. Execute
+ // Spark's original input/output pair: Velox-specific expected results are
not Bolt's contract.
+ override def getResourceFilePath: String = {
+ val home =
sys.props.get("spark.test.home").orElse(sys.env.get("SPARK_HOME")).getOrElse(".")
+ Paths.get(home, "sql/core/src/test/resources/sql-tests").toString
+ }
+ override def getSupportedSQLQueryTests: Set[String] = {
+ val inventory = VeloxSQLQueryTestSettings.getSupportedSQLQueryTests ++
+ VeloxSQLQueryTestSettings.getOverwriteSQLQueryTests ++
+ Set("bitwise.sql", "conditional-functions.sql", "count.sql",
"null-handling.sql")
+ inventory.map {
+ // These two Velox copies predate Spark 4's default ANSI mode. Use the
release originals.
+ case "ansi/conditional-functions.sql" => "conditional-functions.sql"
+ case "ansi/decimalArithmeticOperations.sql" =>
"decimalArithmeticOperations.sql"
+ // SQLQueryTestSuite registers UDF cases with a language suffix, not the
bare filename.
+ case name if name.startsWith("udf/") && !name.contains(" - ") => name +
" - Scala UDF"
+ case name => name
+ } - "ignored.sql" // Spark's deliberate ignore-functionality fixture has
no golden file.
+ }
+ override def getOverwriteSQLQueryTests: Set[String] = Set.empty
+}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenLogicalPlanTagInSparkPlanSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenLogicalPlanTagInSparkPlanSuite.scala
index 491961d7d0..bed7cd69f7 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenLogicalPlanTagInSparkPlanSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenLogicalPlanTagInSparkPlanSuite.scala
@@ -96,8 +96,7 @@ class GlutenLogicalPlanTagInSparkPlanSuite
assert(plan.getTagValue(SparkPlan.LOGICAL_PLAN_TAG).isEmpty)
// Gluten infrastructure nodes (no corresponding logical plan)
- case _: WholeStageTransformer | _: InputIteratorTransformer | _:
ColumnarInputAdapter |
- _: VeloxResizeBatchesExec =>
+ case _: WholeStageTransformer | _: InputIteratorTransformer | _:
ColumnarInputAdapter =>
// These are Gluten-specific wrapper nodes without logical plan links.
// Scan trees
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenWholeStageCodegenSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenWholeStageCodegenSuite.scala
index 142cd5700c..be53f48b44 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenWholeStageCodegenSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenWholeStageCodegenSuite.scala
@@ -492,7 +492,7 @@ class GlutenWholeStageCodegenSuite extends
WholeStageCodegenSuite with GlutenSQL
Seq(true, false).foreach {
codegenEnabled =>
withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key ->
codegenEnabled.toString) {
- assertWholeStageCount[VeloxBroadcastNestedLoopJoinExecTransformer](
+ assertWholeStageCount[BroadcastNestedLoopJoinExecTransformer](
df1.join(df2),
expectedCount = 1)
checkAnswer(
@@ -511,7 +511,7 @@ class GlutenWholeStageCodegenSuite extends
WholeStageCodegenSuite with GlutenSQL
Row(3, 1),
Row(3, 2)))
val conditionalJoin = df1.join(df2, $"k1" + 1 =!= $"k2")
- assertWholeStageCount[VeloxBroadcastNestedLoopJoinExecTransformer](
+ assertWholeStageCount[BroadcastNestedLoopJoinExecTransformer](
conditionalJoin,
expectedCount = 1)
checkAnswer(
@@ -528,7 +528,7 @@ class GlutenWholeStageCodegenSuite extends
WholeStageCodegenSuite with GlutenSQL
Row(3, 1),
Row(3, 2)))
val twoJoins = df1.join(df2, $"k1" < $"k2").crossJoin(df3)
- assertWholeStageCount[VeloxBroadcastNestedLoopJoinExecTransformer](
+ assertWholeStageCount[BroadcastNestedLoopJoinExecTransformer](
twoJoins,
expectedCount = 1)
checkAnswer(
@@ -556,7 +556,7 @@ class GlutenWholeStageCodegenSuite extends
WholeStageCodegenSuite with GlutenSQL
codegenEnabled =>
withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key ->
codegenEnabled.toString) {
val leftOuterJoin = df1.join(df2, $"k1" > $"k2", "left_outer")
- assertWholeStageCount[VeloxBroadcastNestedLoopJoinExecTransformer](
+ assertWholeStageCount[BroadcastNestedLoopJoinExecTransformer](
leftOuterJoin,
expectedCount = 1)
checkAnswer(
@@ -570,7 +570,7 @@ class GlutenWholeStageCodegenSuite extends
WholeStageCodegenSuite with GlutenSQL
Row(3, 1),
Row(3, 2)))
val rightOuterJoin = df1.join(df2, $"k1" < $"k2", "right_outer")
- assertWholeStageCount[VeloxBroadcastNestedLoopJoinExecTransformer](
+ assertWholeStageCount[BroadcastNestedLoopJoinExecTransformer](
rightOuterJoin,
expectedCount = 1)
checkAnswer(
@@ -579,14 +579,14 @@ class GlutenWholeStageCodegenSuite extends
WholeStageCodegenSuite with GlutenSQL
val twoJoins = df1
.join(df2, $"k1" > $"k2" + 1, "right_outer")
.join(df3, $"k1" <= $"k3", "left_outer")
- assertWholeStageCount[VeloxBroadcastNestedLoopJoinExecTransformer](
+ assertWholeStageCount[BroadcastNestedLoopJoinExecTransformer](
twoJoins,
expectedCount = 1)
checkAnswer(
twoJoins,
Seq(Row(2, 0, null), Row(3, 0, null), Row(3, 1, null), Row(null,
2, null)))
val emptyBuildSide = df3.join(empty, $"k3" > $"k4", "left_outer")
- assertWholeStageCount[VeloxBroadcastNestedLoopJoinExecTransformer](
+ assertWholeStageCount[BroadcastNestedLoopJoinExecTransformer](
emptyBuildSide,
expectedCount = 1)
checkAnswer(
@@ -607,13 +607,13 @@ class GlutenWholeStageCodegenSuite extends
WholeStageCodegenSuite with GlutenSQL
codegenEnabled =>
withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key ->
codegenEnabled.toString) {
val semiJoin = df1.join(df2, $"k1" + 1 <= $"k2", "left_semi")
- assertWholeStageCount[VeloxBroadcastNestedLoopJoinExecTransformer](
+ assertWholeStageCount[BroadcastNestedLoopJoinExecTransformer](
semiJoin,
expectedCount = 0)
checkAnswer(semiJoin, Seq(Row(0), Row(1)))
val antiJoin = df1.join(df2, $"k1" + 1 <= $"k2", "left_anti")
- assertWholeStageCount[VeloxBroadcastNestedLoopJoinExecTransformer](
+ assertWholeStageCount[BroadcastNestedLoopJoinExecTransformer](
antiJoin,
expectedCount = 0)
checkAnswer(antiJoin, Seq(Row(2), Row(3)))
@@ -621,7 +621,7 @@ class GlutenWholeStageCodegenSuite extends
WholeStageCodegenSuite with GlutenSQL
val twoJoins = df1
.join(df2, $"k1" < $"k2", "left_semi")
.join(df3, $"k1" > $"k3", "left_anti")
- assertWholeStageCount[VeloxBroadcastNestedLoopJoinExecTransformer](
+ assertWholeStageCount[BroadcastNestedLoopJoinExecTransformer](
twoJoins,
expectedCount = 0)
checkAnswer(twoJoins, Row(0))
diff --git a/gluten-ut/spark41/pom.xml b/gluten-ut/spark41/pom.xml
index 314f3fde9f..5e9b31f3cb 100644
--- a/gluten-ut/spark41/pom.xml
+++ b/gluten-ut/spark41/pom.xml
@@ -366,6 +366,86 @@
</build>
</profile>
+ <profile>
+ <id>backends-bolt</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter</artifactId>
+ <version>5.9.3</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>net.aichler</groupId>
+ <artifactId>jupiter-interface</artifactId>
+ <version>0.11.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.gluten</groupId>
+ <artifactId>backends-bolt</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <!-- log4j -->
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j2-impl</artifactId>
+ <version>${log4j.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.h2database</groupId>
+ <artifactId>h2</artifactId>
+ <version>2.2.220</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>add-sources</id>
+ <goals>
+ <goal>add-test-source</goal>
+ </goals>
+ <phase>generate-sources</phase>
+ <configuration>
+ <sources>
+ <source>src/test/backends-bolt</source>
+
<source>${project.basedir}/../spark40/src/test/bolt-shared</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>com.diffplug.spotless</groupId>
+ <artifactId>spotless-maven-plugin</artifactId>
+ <configuration>
+ <scala>
+ <includes combine.children="append">
+ <include>src/test/backends-bolt/**/*.scala</include>
+ </includes>
+ </scala>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
<profile>
<id>delta</id>
<dependencies>
diff --git
a/gluten-ut/spark41/src/test/backends-bolt/org/apache/gluten/utils/bolt/BoltTestSettings.scala
b/gluten-ut/spark41/src/test/backends-bolt/org/apache/gluten/utils/bolt/BoltTestSettings.scala
new file mode 100644
index 0000000000..df8f5cfd86
--- /dev/null
+++
b/gluten-ut/spark41/src/test/backends-bolt/org/apache/gluten/utils/bolt/BoltTestSettings.scala
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.utils.bolt
+
+// Suites added or renamed in this Spark version, in addition to the shared
regression inventory.
+class BoltTestSettings extends BoltSpark4TestSettings {
+ enableSuite[org.apache.spark.sql.GlutenAddMetadataColumnsSuite]
+ enableSuite[org.apache.spark.sql.GlutenAlwaysPersistedConfigsSuite]
+ enableSuite[org.apache.spark.sql.GlutenApproxTopKSuite]
+ enableSuite[org.apache.spark.sql.GlutenCacheTableInKryoSuite]
+ enableSuite[org.apache.spark.sql.GlutenGeographyDataFrameSuite]
+ enableSuite[org.apache.spark.sql.GlutenGeometryDataFrameSuite]
+ enableSuite[org.apache.spark.sql.GlutenLegacyParameterSubstitutionSuite]
+ enableSuite[org.apache.spark.sql.GlutenMapStatusEndToEndSuite]
+
enableSuite[org.apache.spark.sql.GlutenReplaceIntegerLiteralsWithOrdinalsDataframeSuite]
+
enableSuite[org.apache.spark.sql.GlutenReplaceIntegerLiteralsWithOrdinalsSqlSuite]
+ enableSuite[org.apache.spark.sql.GlutenSTExpressionsSuite]
+ enableSuite[org.apache.spark.sql.GlutenSTFunctionsSuite]
+ enableSuite[org.apache.spark.sql.GlutenStringLiteralCoalescingSuite]
+ enableSuite[org.apache.spark.sql.GlutenTimestampNtzAggregateSuite]
+ enableSuite[org.apache.spark.sql.GlutenTimestampNtzLiteralSuite]
+ enableSuite[org.apache.spark.sql.GlutenUnsafeRowChecksumSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenAttributeMapSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenBindReferencesSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenConstraintExpressionSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenTimeExpressionsSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.aggregate.GlutenApproxTopKSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.aggregate.GlutenThetasketchesAggSuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenShowNamespacesParserSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.parquet.GlutenVariantInferShreddingSuite]
+ enableSuite[
+
org.apache.spark.sql.execution.datasources.v2.GlutenDSV2JoinPushDownAliasGenerationSuite]
+ enableSuite[org.apache.spark.sql.execution.joins.GlutenHashedRelationSuite]
+
enableSuite[org.apache.spark.sql.execution.python.GlutenPythonWorkerLogsSuite]
+ enableSuite[org.apache.spark.sql.gluten.GlutenRowBasedChecksumSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenStreamRealTimeModeAllowlistSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenStreamRealTimeModeE2ESuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenStreamRealTimeModeSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenStreamingFullOuterJoinSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenStreamingInnerJoinSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenStreamingLeftSemiJoinSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenStreamingOuterJoinSuite]
+}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenLogicalPlanTagInSparkPlanSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenLogicalPlanTagInSparkPlanSuite.scala
index 491961d7d0..bed7cd69f7 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenLogicalPlanTagInSparkPlanSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenLogicalPlanTagInSparkPlanSuite.scala
@@ -96,8 +96,7 @@ class GlutenLogicalPlanTagInSparkPlanSuite
assert(plan.getTagValue(SparkPlan.LOGICAL_PLAN_TAG).isEmpty)
// Gluten infrastructure nodes (no corresponding logical plan)
- case _: WholeStageTransformer | _: InputIteratorTransformer | _:
ColumnarInputAdapter |
- _: VeloxResizeBatchesExec =>
+ case _: WholeStageTransformer | _: InputIteratorTransformer | _:
ColumnarInputAdapter =>
// These are Gluten-specific wrapper nodes without logical plan links.
// Scan trees
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenWholeStageCodegenSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenWholeStageCodegenSuite.scala
index 142cd5700c..be53f48b44 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenWholeStageCodegenSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenWholeStageCodegenSuite.scala
@@ -492,7 +492,7 @@ class GlutenWholeStageCodegenSuite extends
WholeStageCodegenSuite with GlutenSQL
Seq(true, false).foreach {
codegenEnabled =>
withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key ->
codegenEnabled.toString) {
- assertWholeStageCount[VeloxBroadcastNestedLoopJoinExecTransformer](
+ assertWholeStageCount[BroadcastNestedLoopJoinExecTransformer](
df1.join(df2),
expectedCount = 1)
checkAnswer(
@@ -511,7 +511,7 @@ class GlutenWholeStageCodegenSuite extends
WholeStageCodegenSuite with GlutenSQL
Row(3, 1),
Row(3, 2)))
val conditionalJoin = df1.join(df2, $"k1" + 1 =!= $"k2")
- assertWholeStageCount[VeloxBroadcastNestedLoopJoinExecTransformer](
+ assertWholeStageCount[BroadcastNestedLoopJoinExecTransformer](
conditionalJoin,
expectedCount = 1)
checkAnswer(
@@ -528,7 +528,7 @@ class GlutenWholeStageCodegenSuite extends
WholeStageCodegenSuite with GlutenSQL
Row(3, 1),
Row(3, 2)))
val twoJoins = df1.join(df2, $"k1" < $"k2").crossJoin(df3)
- assertWholeStageCount[VeloxBroadcastNestedLoopJoinExecTransformer](
+ assertWholeStageCount[BroadcastNestedLoopJoinExecTransformer](
twoJoins,
expectedCount = 1)
checkAnswer(
@@ -556,7 +556,7 @@ class GlutenWholeStageCodegenSuite extends
WholeStageCodegenSuite with GlutenSQL
codegenEnabled =>
withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key ->
codegenEnabled.toString) {
val leftOuterJoin = df1.join(df2, $"k1" > $"k2", "left_outer")
- assertWholeStageCount[VeloxBroadcastNestedLoopJoinExecTransformer](
+ assertWholeStageCount[BroadcastNestedLoopJoinExecTransformer](
leftOuterJoin,
expectedCount = 1)
checkAnswer(
@@ -570,7 +570,7 @@ class GlutenWholeStageCodegenSuite extends
WholeStageCodegenSuite with GlutenSQL
Row(3, 1),
Row(3, 2)))
val rightOuterJoin = df1.join(df2, $"k1" < $"k2", "right_outer")
- assertWholeStageCount[VeloxBroadcastNestedLoopJoinExecTransformer](
+ assertWholeStageCount[BroadcastNestedLoopJoinExecTransformer](
rightOuterJoin,
expectedCount = 1)
checkAnswer(
@@ -579,14 +579,14 @@ class GlutenWholeStageCodegenSuite extends
WholeStageCodegenSuite with GlutenSQL
val twoJoins = df1
.join(df2, $"k1" > $"k2" + 1, "right_outer")
.join(df3, $"k1" <= $"k3", "left_outer")
- assertWholeStageCount[VeloxBroadcastNestedLoopJoinExecTransformer](
+ assertWholeStageCount[BroadcastNestedLoopJoinExecTransformer](
twoJoins,
expectedCount = 1)
checkAnswer(
twoJoins,
Seq(Row(2, 0, null), Row(3, 0, null), Row(3, 1, null), Row(null,
2, null)))
val emptyBuildSide = df3.join(empty, $"k3" > $"k4", "left_outer")
- assertWholeStageCount[VeloxBroadcastNestedLoopJoinExecTransformer](
+ assertWholeStageCount[BroadcastNestedLoopJoinExecTransformer](
emptyBuildSide,
expectedCount = 1)
checkAnswer(
@@ -607,13 +607,13 @@ class GlutenWholeStageCodegenSuite extends
WholeStageCodegenSuite with GlutenSQL
codegenEnabled =>
withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key ->
codegenEnabled.toString) {
val semiJoin = df1.join(df2, $"k1" + 1 <= $"k2", "left_semi")
- assertWholeStageCount[VeloxBroadcastNestedLoopJoinExecTransformer](
+ assertWholeStageCount[BroadcastNestedLoopJoinExecTransformer](
semiJoin,
expectedCount = 0)
checkAnswer(semiJoin, Seq(Row(0), Row(1)))
val antiJoin = df1.join(df2, $"k1" + 1 <= $"k2", "left_anti")
- assertWholeStageCount[VeloxBroadcastNestedLoopJoinExecTransformer](
+ assertWholeStageCount[BroadcastNestedLoopJoinExecTransformer](
antiJoin,
expectedCount = 0)
checkAnswer(antiJoin, Seq(Row(2), Row(3)))
@@ -621,7 +621,7 @@ class GlutenWholeStageCodegenSuite extends
WholeStageCodegenSuite with GlutenSQL
val twoJoins = df1
.join(df2, $"k1" < $"k2", "left_semi")
.join(df3, $"k1" > $"k3", "left_anti")
- assertWholeStageCount[VeloxBroadcastNestedLoopJoinExecTransformer](
+ assertWholeStageCount[BroadcastNestedLoopJoinExecTransformer](
twoJoins,
expectedCount = 0)
checkAnswer(twoJoins, Row(0))
diff --git a/gluten-ut/spark42/pom.xml b/gluten-ut/spark42/pom.xml
index 66d346f347..563337071a 100644
--- a/gluten-ut/spark42/pom.xml
+++ b/gluten-ut/spark42/pom.xml
@@ -366,6 +366,86 @@
</build>
</profile>
+ <profile>
+ <id>backends-bolt</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter</artifactId>
+ <version>5.9.3</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>net.aichler</groupId>
+ <artifactId>jupiter-interface</artifactId>
+ <version>0.11.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.gluten</groupId>
+ <artifactId>backends-bolt</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <!-- log4j -->
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j2-impl</artifactId>
+ <version>${log4j.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.h2database</groupId>
+ <artifactId>h2</artifactId>
+ <version>2.2.220</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>add-sources</id>
+ <goals>
+ <goal>add-test-source</goal>
+ </goals>
+ <phase>generate-sources</phase>
+ <configuration>
+ <sources>
+ <source>src/test/backends-bolt</source>
+
<source>${project.basedir}/../spark40/src/test/bolt-shared</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>com.diffplug.spotless</groupId>
+ <artifactId>spotless-maven-plugin</artifactId>
+ <configuration>
+ <scala>
+ <includes combine.children="append">
+ <include>src/test/backends-bolt/**/*.scala</include>
+ </includes>
+ </scala>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
<profile>
<id>delta</id>
<dependencies>
diff --git
a/gluten-ut/spark42/src/test/backends-bolt/org/apache/gluten/utils/bolt/BoltTestSettings.scala
b/gluten-ut/spark42/src/test/backends-bolt/org/apache/gluten/utils/bolt/BoltTestSettings.scala
new file mode 100644
index 0000000000..a1e39e8cd5
--- /dev/null
+++
b/gluten-ut/spark42/src/test/backends-bolt/org/apache/gluten/utils/bolt/BoltTestSettings.scala
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.utils.bolt
+
+// Suites added or renamed in this Spark version, in addition to the shared
regression inventory.
+class BoltTestSettings extends BoltSpark4TestSettings {
+ enableSuite[org.apache.spark.sql.GlutenAddMetadataColumnsSuite]
+ enableSuite[org.apache.spark.sql.GlutenAlwaysPersistedConfigsSuite]
+ enableSuite[org.apache.spark.sql.GlutenApproxTopKSuite]
+ enableSuite[org.apache.spark.sql.GlutenCacheTableInKryoSuite]
+ enableSuite[org.apache.spark.sql.GlutenGeographyDataFrameSuite]
+ enableSuite[org.apache.spark.sql.GlutenGeometryDataFrameSuite]
+ enableSuite[org.apache.spark.sql.GlutenLegacyParameterSubstitutionSuite]
+ enableSuite[org.apache.spark.sql.GlutenMapStatusEndToEndSuite]
+
enableSuite[org.apache.spark.sql.GlutenReplaceIntegerLiteralsWithOrdinalsDataframeSuite]
+
enableSuite[org.apache.spark.sql.GlutenReplaceIntegerLiteralsWithOrdinalsSqlSuite]
+ enableSuite[org.apache.spark.sql.GlutenSTExpressionsSuite]
+ enableSuite[org.apache.spark.sql.GlutenSTFunctionsSuite]
+ enableSuite[org.apache.spark.sql.GlutenStringLiteralCoalescingSuite]
+ enableSuite[org.apache.spark.sql.GlutenTimestampNtzAggregateSuite]
+ enableSuite[org.apache.spark.sql.GlutenTimestampNtzLiteralSuite]
+ enableSuite[org.apache.spark.sql.GlutenUnsafeRowChecksumSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenAttributeMapSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenBindReferencesSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenConstraintExpressionSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.GlutenTimeExpressionsSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.aggregate.GlutenApproxTopKSuite]
+
enableSuite[org.apache.spark.sql.catalyst.expressions.aggregate.GlutenThetasketchesAggSuite]
+
enableSuite[org.apache.spark.sql.connector.GlutenDataSourceV2EnhancedDeleteFilterSuite]
+
enableSuite[org.apache.spark.sql.connector.GlutenDataSourceV2EnhancedPartitionFilterSuite]
+
enableSuite[org.apache.spark.sql.connector.GlutenDataSourceV2EnhancedRuntimePartitionFilterSuite]
+ enableSuite[org.apache.spark.sql.execution.GlutenShowNamespacesParserSuite]
+
enableSuite[org.apache.spark.sql.execution.datasources.parquet.GlutenVariantInferShreddingSuite]
+ enableSuite[
+
org.apache.spark.sql.execution.datasources.v2.GlutenDSV2JoinPushDownAliasGenerationSuite]
+
enableSuite[org.apache.spark.sql.execution.joins.GlutenHashedRelationOffHeapSuite]
+
enableSuite[org.apache.spark.sql.execution.joins.GlutenHashedRelationOnHeapSuite]
+
enableSuite[org.apache.spark.sql.execution.python.GlutenPythonWorkerLogsSuite]
+ enableSuite[org.apache.spark.sql.gluten.GlutenRowBasedChecksumSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenStreamRealTimeModeAllowlistSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenStreamRealTimeModeE2ESuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenStreamRealTimeModeSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenStreamingFullOuterWithVCFSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenStreamingFullOuterWithoutVCFSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenStreamingInnerWithVCFSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenStreamingInnerWithoutVCFSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenStreamingLeftSemiWithVCFSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenStreamingLeftSemiWithoutVCFSuite]
+ enableSuite[org.apache.spark.sql.streaming.GlutenStreamingOuterWithVCFSuite]
+
enableSuite[org.apache.spark.sql.streaming.GlutenStreamingOuterWithoutVCFSuite]
+}
diff --git
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/execution/GlutenLogicalPlanTagInSparkPlanSuite.scala
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/execution/GlutenLogicalPlanTagInSparkPlanSuite.scala
index 491961d7d0..bed7cd69f7 100644
---
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/execution/GlutenLogicalPlanTagInSparkPlanSuite.scala
+++
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/execution/GlutenLogicalPlanTagInSparkPlanSuite.scala
@@ -96,8 +96,7 @@ class GlutenLogicalPlanTagInSparkPlanSuite
assert(plan.getTagValue(SparkPlan.LOGICAL_PLAN_TAG).isEmpty)
// Gluten infrastructure nodes (no corresponding logical plan)
- case _: WholeStageTransformer | _: InputIteratorTransformer | _:
ColumnarInputAdapter |
- _: VeloxResizeBatchesExec =>
+ case _: WholeStageTransformer | _: InputIteratorTransformer | _:
ColumnarInputAdapter =>
// These are Gluten-specific wrapper nodes without logical plan links.
// Scan trees
diff --git
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/execution/GlutenWholeStageCodegenSuite.scala
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/execution/GlutenWholeStageCodegenSuite.scala
index 29329d5e60..5c5fd41540 100644
---
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/execution/GlutenWholeStageCodegenSuite.scala
+++
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/execution/GlutenWholeStageCodegenSuite.scala
@@ -494,7 +494,7 @@ class GlutenWholeStageCodegenSuite extends
WholeStageCodegenSuite with GlutenSQL
Seq(true, false).foreach {
codegenEnabled =>
withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key ->
codegenEnabled.toString) {
- assertWholeStageCount[VeloxBroadcastNestedLoopJoinExecTransformer](
+ assertWholeStageCount[BroadcastNestedLoopJoinExecTransformer](
df1.join(df2),
expectedCount = 1)
checkAnswer(
@@ -513,7 +513,7 @@ class GlutenWholeStageCodegenSuite extends
WholeStageCodegenSuite with GlutenSQL
Row(3, 1),
Row(3, 2)))
val conditionalJoin = df1.join(df2, $"k1" + 1 =!= $"k2")
- assertWholeStageCount[VeloxBroadcastNestedLoopJoinExecTransformer](
+ assertWholeStageCount[BroadcastNestedLoopJoinExecTransformer](
conditionalJoin,
expectedCount = 1)
checkAnswer(
@@ -530,7 +530,7 @@ class GlutenWholeStageCodegenSuite extends
WholeStageCodegenSuite with GlutenSQL
Row(3, 1),
Row(3, 2)))
val twoJoins = df1.join(df2, $"k1" < $"k2").crossJoin(df3)
- assertWholeStageCount[VeloxBroadcastNestedLoopJoinExecTransformer](
+ assertWholeStageCount[BroadcastNestedLoopJoinExecTransformer](
twoJoins,
expectedCount = 1)
checkAnswer(
@@ -558,7 +558,7 @@ class GlutenWholeStageCodegenSuite extends
WholeStageCodegenSuite with GlutenSQL
codegenEnabled =>
withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key ->
codegenEnabled.toString) {
val leftOuterJoin = df1.join(df2, $"k1" > $"k2", "left_outer")
- assertWholeStageCount[VeloxBroadcastNestedLoopJoinExecTransformer](
+ assertWholeStageCount[BroadcastNestedLoopJoinExecTransformer](
leftOuterJoin,
expectedCount = 1)
checkAnswer(
@@ -572,7 +572,7 @@ class GlutenWholeStageCodegenSuite extends
WholeStageCodegenSuite with GlutenSQL
Row(3, 1),
Row(3, 2)))
val rightOuterJoin = df1.join(df2, $"k1" < $"k2", "right_outer")
- assertWholeStageCount[VeloxBroadcastNestedLoopJoinExecTransformer](
+ assertWholeStageCount[BroadcastNestedLoopJoinExecTransformer](
rightOuterJoin,
expectedCount = 1)
checkAnswer(
@@ -581,14 +581,14 @@ class GlutenWholeStageCodegenSuite extends
WholeStageCodegenSuite with GlutenSQL
val twoJoins = df1
.join(df2, $"k1" > $"k2" + 1, "right_outer")
.join(df3, $"k1" <= $"k3", "left_outer")
- assertWholeStageCount[VeloxBroadcastNestedLoopJoinExecTransformer](
+ assertWholeStageCount[BroadcastNestedLoopJoinExecTransformer](
twoJoins,
expectedCount = 1)
checkAnswer(
twoJoins,
Seq(Row(2, 0, null), Row(3, 0, null), Row(3, 1, null), Row(null,
2, null)))
val emptyBuildSide = df3.join(empty, $"k3" > $"k4", "left_outer")
- assertWholeStageCount[VeloxBroadcastNestedLoopJoinExecTransformer](
+ assertWholeStageCount[BroadcastNestedLoopJoinExecTransformer](
emptyBuildSide,
expectedCount = 1)
checkAnswer(
@@ -609,13 +609,13 @@ class GlutenWholeStageCodegenSuite extends
WholeStageCodegenSuite with GlutenSQL
codegenEnabled =>
withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key ->
codegenEnabled.toString) {
val semiJoin = df1.join(df2, $"k1" + 1 <= $"k2", "left_semi")
- assertWholeStageCount[VeloxBroadcastNestedLoopJoinExecTransformer](
+ assertWholeStageCount[BroadcastNestedLoopJoinExecTransformer](
semiJoin,
expectedCount = 0)
checkAnswer(semiJoin, Seq(Row(0), Row(1)))
val antiJoin = df1.join(df2, $"k1" + 1 <= $"k2", "left_anti")
- assertWholeStageCount[VeloxBroadcastNestedLoopJoinExecTransformer](
+ assertWholeStageCount[BroadcastNestedLoopJoinExecTransformer](
antiJoin,
expectedCount = 0)
checkAnswer(antiJoin, Seq(Row(2), Row(3)))
@@ -623,7 +623,7 @@ class GlutenWholeStageCodegenSuite extends
WholeStageCodegenSuite with GlutenSQL
val twoJoins = df1
.join(df2, $"k1" < $"k2", "left_semi")
.join(df3, $"k1" > $"k3", "left_anti")
- assertWholeStageCount[VeloxBroadcastNestedLoopJoinExecTransformer](
+ assertWholeStageCount[BroadcastNestedLoopJoinExecTransformer](
twoJoins,
expectedCount = 0)
checkAnswer(twoJoins, Row(0))
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]