yikf commented on code in PR #8186:
URL: https://github.com/apache/incubator-gluten/pull/8186#discussion_r1875720584
##########
shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala:
##########
@@ -2102,7 +2102,7 @@ object GlutenConfig {
.internal()
.doc("Set the load quantum for velox file scan")
.bytesConf(ByteUnit.BYTE)
- .createWithDefaultString("256MB")
+ .createWithDefaultString("8MB")
Review Comment:
the default value 256MB will raise a error.
```log
Caused by: org.apache.gluten.exception.GlutenException: Exception:
VeloxRuntimeError
Error Source: RUNTIME
Error Code: INVALID_STATE
Reason: (268435456 vs. 8388608) Load quantum exceeded SSD cache entry size
limit.
Retriable: False
Expression: options_.loadQuantum() <= 1 << cache::SsdRun::kSizeBits
Context: Split [Hive:
file:///root/apache/gluten/spark-warehouse/metrics_t1/part-00000-647158a0-43d3-4064-be90-0bb0654bd461-c000.snappy.parquet
0 - 519] Task Gluten_Stage_1_TID_1_VTID_0
Additional Context: Operator: TableScan[0] 0
Function: checkLoadQuantum
File:
/root/apache/gluten/ep/build-velox/build/velox_ep/./velox/dwio/common/CachedBufferedInput.h
Line: 186
Stack trace:
# 0 facebook::velox::VeloxException::VeloxException(char const*, unsigned
long, char const*, std::basic_string_view<char, std::char_traits<char> >,
std::basic_string_view<char, std::char_traits<char> >,
std::basic_string_view<char, std::char_traits<char> >,
std::basic_string_view<char, std::char_traits<char> >, bool,
facebook::velox::VeloxException::Type, std::basic_string_view<char,
std::char_traits<char> >)
# 1 void
facebook::velox::detail::veloxCheckFail<facebook::velox::VeloxRuntimeError,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
const&>(facebook::velox::detail::VeloxCheckFailArgs const&,
std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >
const&)
# 2
facebook::velox::connector::hive::createBufferedInput(facebook::velox::FileHandle
const&, facebook::velox::dwio::common::ReaderOptions const&,
facebook::velox::connector::ConnectorQueryCtx const*,
std::shared_ptr<facebook::velox::io::IoStatistics>, folly::Executor*)
# 3
facebook::velox::connector::hive::SplitReader::createReader(std::shared_ptr<facebook::velox::common::MetadataFilter>)
# 4
facebook::velox::connector::hive::SplitReader::prepareSplit(std::shared_ptr<facebook::velox::common::MetadataFilter>,
facebook::velox::dwio::common::RuntimeStatistics&)
# 5
facebook::velox::connector::hive::HiveDataSource::addSplit(std::shared_ptr<facebook::velox::connector::ConnectorSplit>)
# 6 facebook::velox::exec::TableScan::getOutput()
# 7
facebook::velox::exec::Driver::runInternal(std::shared_ptr<facebook::velox::exec::Driver>&,
std::shared_ptr<facebook::velox::exec::BlockingState>&,
std::shared_ptr<facebook::velox::RowVector>&)::{lambda()#10}::operator()() const
# 8
facebook::velox::exec::Driver::runInternal(std::shared_ptr<facebook::velox::exec::Driver>&,
std::shared_ptr<facebook::velox::exec::BlockingState>&,
std::shared_ptr<facebook::velox::RowVector>&)
# 9 facebook::velox::exec::Driver::next(folly::SemiFuture<folly::Unit>*)
# 10 facebook::velox::exec::Task::next(folly::SemiFuture<folly::Unit>*)
# 11 gluten::WholeStageResultIterator::next()
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
