lgbo-ustc opened a new issue, #6583:
URL: https://github.com/apache/incubator-gluten/issues/6583
### Backend
CH (ClickHouse)
### Bug description
[Expected behavior] and [actual behavior].
```
Job aborted due to stage failure: Task 1 in stage 43.0 failed 4 times, most
recent failure: Lost task 1.3 in stage 43.0 (TID 2881)
(sg-dn2338.bigdata.bigo.inner executor 1223):
org.apache.gluten.exception.GlutenException:
org.apache.gluten.exception.GlutenException: Allocator: Cannot realloc from
4.00 KiB to 128.00 PiB: , errno: 22, strerror: Invalid argument: While
executing StreamingAggregatingTransform
0. Poco::Exception::Exception(String const&, int) @ 0x000000001306d1b9
1. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @
0x0000000006343a5b
2. DB::Exception::Exception(PreformattedMessage&&, int) @ 0x00000000061f77ac
3. DB::ErrnoException::ErrnoException<ReadableSize, ReadableSize>(int,
FormatStringHelperImpl<std::type_identity<ReadableSize>::type,
std::type_identity<ReadableSize>::type>, ReadableSize&&, ReadableSize&&) @
0x00000000063252ee
4. Allocator<false, false>::realloc(void*, unsigned long, unsigned long,
unsigned long) @ 0x000000000632523b
5. void DB::PODArrayBase<1ul, 4096ul, Allocator<false, false>, 63ul,
64ul>::realloc<>(unsigned long) @ 0x000000000620b36d
6. DB::ColumnString::deserializeAndInsertFromArena(char const*) @
0x000000000fb4c0ff
7. DB::AggregationMethodSerialized<HashMapTable<StringRef,
HashMapCellWithSavedHash<StringRef, char*, DefaultHash<StringRef>,
HashTableNoState>, DefaultHash<StringRef>,
HashTableGrowerWithPrecalculation<8ul>, Allocator<true, true>>, true,
false>::insertKeyIntoColumns(StringRef, std::vector<DB::IColumn*,
std::allocator<DB::IColumn*>>&, std::vector<unsigned long,
std::allocator<unsigned long>> const&) @ 0x000000000ed16ecf
8. auto std::variant<DB::Block, std::list<DB::Block,
std::allocator<DB::Block>>>
DB::Aggregator::convertToBlockImplNotFinal<DB::AggregationMethodSerialized<HashMapTable<StringRef,
HashMapCellWithSavedHash<StringRef, char*, DefaultHash<StringRef>,
HashTableNoState>, DefaultHash<StringRef>,
HashTableGrowerWithPrecalculation<8ul>, Allocator<true, true>>, true, false>,
HashMapTable<StringRef, HashMapCellWithSavedHash<StringRef, char*,
DefaultHash<StringRef>, HashTableNoState>, DefaultHash<StringRef>,
HashTableGrowerWithPrecalculation<8ul>, Allocator<true,
true>>>(DB::AggregationMethodSerialized<HashMapTable<StringRef,
HashMapCellWithSavedHash<StringRef, char*, DefaultHash<StringRef>,
HashTableNoState>, DefaultHash<StringRef>,
HashTableGrowerWithPrecalculation<8ul>, Allocator<true, true>>, true, false>&,
HashMapTable<StringRef, HashMapCellWithSavedHash<StringRef, char*,
DefaultHash<StringRef>, HashTableNoState>, DefaultHash<StringRef>,
HashTableGrowerWithPrecalculation<8ul>, Allocator<
true, true>>&, std::vector<std::shared_ptr<DB::Arena>,
std::allocator<std::shared_ptr<DB::Arena>>>&, unsigned long, bool)
const::'lambda'(DB::AggregationMethodSerialized<HashMapTable<StringRef,
HashMapCellWithSavedHash<StringRef, char*, DefaultHash<StringRef>,
HashTableNoState>, DefaultHash<StringRef>,
HashTableGrowerWithPrecalculation<8ul>, Allocator<true, true>>, true, false>
const&, HashMapTable<StringRef, HashMapCellWithSavedHash<StringRef, char*,
DefaultHash<StringRef>, HashTableNoState>, DefaultHash<StringRef>,
HashTableGrowerWithPrecalculation<8ul>, Allocator<true,
true>>&)::operator()<StringRef,
char*>(DB::AggregationMethodSerialized<HashMapTable<StringRef,
HashMapCellWithSavedHash<StringRef, char*, DefaultHash<StringRef>,
HashTableNoState>, DefaultHash<StringRef>,
HashTableGrowerWithPrecalculation<8ul>, Allocator<true, true>>, true, false>
const&, HashMapTable<StringRef, HashMapCellWithSavedHash<StringRef, char*,
DefaultHash<StringRef>, HashTableNoState>, DefaultHash<String
Ref>, HashTableGrowerWithPrecalculation<8ul>, Allocator<true, true>>&) const @
0x000000000ece7e98
9. std::variant<DB::Block, std::list<DB::Block, std::allocator<DB::Block>>>
DB::Aggregator::convertToBlockImplNotFinal<DB::AggregationMethodSerialized<HashMapTable<StringRef,
HashMapCellWithSavedHash<StringRef, char*, DefaultHash<StringRef>,
HashTableNoState>, DefaultHash<StringRef>,
HashTableGrowerWithPrecalculation<8ul>, Allocator<true, true>>, true, false>,
HashMapTable<StringRef, HashMapCellWithSavedHash<StringRef, char*,
DefaultHash<StringRef>, HashTableNoState>, DefaultHash<StringRef>,
HashTableGrowerWithPrecalculation<8ul>, Allocator<true,
true>>>(DB::AggregationMethodSerialized<HashMapTable<StringRef,
HashMapCellWithSavedHash<StringRef, char*, DefaultHash<StringRef>,
HashTableNoState>, DefaultHash<StringRef>,
HashTableGrowerWithPrecalculation<8ul>, Allocator<true, true>>, true, false>&,
HashMapTable<StringRef, HashMapCellWithSavedHash<StringRef, char*,
DefaultHash<StringRef>, HashTableNoState>, DefaultHash<StringRef>,
HashTableGrowerWithPrecalculation<8ul>, Allocator<true,
true>>&, std::vector<std::shared_ptr<DB::Arena>,
std::allocator<std::shared_ptr<DB::Arena>>>&, unsigned long, bool) const @
0x000000000ece6a15
10. std::variant<DB::Block, std::list<DB::Block, std::allocator<DB::Block>>>
DB::Aggregator::convertToBlockImpl<DB::AggregationMethodSerialized<HashMapTable<StringRef,
HashMapCellWithSavedHash<StringRef, char*, DefaultHash<StringRef>,
HashTableNoState>, DefaultHash<StringRef>,
HashTableGrowerWithPrecalculation<8ul>, Allocator<true, true>>, true, false>,
HashMapTable<StringRef, HashMapCellWithSavedHash<StringRef, char*,
DefaultHash<StringRef>, HashTableNoState>, DefaultHash<StringRef>,
HashTableGrowerWithPrecalculation<8ul>, Allocator<true,
true>>>(DB::AggregationMethodSerialized<HashMapTable<StringRef,
HashMapCellWithSavedHash<StringRef, char*, DefaultHash<StringRef>,
HashTableNoState>, DefaultHash<StringRef>,
HashTableGrowerWithPrecalculation<8ul>, Allocator<true, true>>, true, false>&,
HashMapTable<StringRef, HashMapCellWithSavedHash<StringRef, char*,
DefaultHash<StringRef>, HashTableNoState>, DefaultHash<StringRef>,
HashTableGrowerWithPrecalculation<8ul>, Allocator<true, true>>
&, DB::Arena*, std::vector<std::shared_ptr<DB::Arena>,
std::allocator<std::shared_ptr<DB::Arena>>>&, bool, unsigned long, bool) const
@ 0x000000000eca0b61
11. std::conditional<false, DB::Block, std::list<DB::Block,
std::allocator<DB::Block>>>::type
DB::Aggregator::prepareBlockAndFillSingleLevel<false>(DB::AggregatedDataVariants&,
bool) const @ 0x000000000eb0215b
12. DB::Aggregator::convertToBlocks(DB::AggregatedDataVariants&, bool,
unsigned long) const @ 0x000000000ea6cf18
13. local_engine::AggregateDataBlockConverter::hasNext() @ 0x0000000006877e0b
14. local_engine::StreamingAggregatingTransform::work()::$_0::operator()()
const @ 0x000000000687e944
15. local_engine::StreamingAggregatingTransform::work() @ 0x000000000687e76f
16. DB::ExecutionThreadContext::executeTask() @ 0x0000000010662f22
17. DB::PipelineExecutor::executeStepImpl(unsigned long, std::atomic<bool>*)
@ 0x0000000010658f30
18. DB::PipelineExecutor::executeStep(std::atomic<bool>*) @
0x0000000010658ac9
19. DB::PullingPipelineExecutor::pull(DB::Chunk&) @ 0x00000000106697c8
20. DB::PullingPipelineExecutor::pull(DB::Block&) @ 0x00000000106699b0
21. local_engine::LocalExecutor::hasNext() @ 0x0000000006720289
22. Java_org_apache_gluten_vectorized_BatchIterator_nativeHasNext @
0x00000000061dd397
at
org.apache.gluten.vectorized.GeneralOutIterator.hasNext(GeneralOutIterator.java:39)
at
org.apache.gluten.backendsapi.clickhouse.CollectMetricIterator.hasNext(CHIteratorApi.scala:306)
at
org.apache.gluten.vectorized.CloseableCHColumnBatchIterator.$anonfun$hasNext$1(CloseableCHColumnBatchIterator.scala:42)
at
scala.runtime.java8.JFunction0$mcZ$sp.apply(JFunction0$mcZ$sp.java:23)
at
org.apache.gluten.metrics.GlutenTimeMetric$.withNanoTime(GlutenTimeMetric.scala:41)
at
org.apache.gluten.vectorized.CloseableCHColumnBatchIterator.hasNext(CloseableCHColumnBatchIterator.scala:42)
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
at
org.apache.spark.shuffle.CHCelebornHashBasedColumnarShuffleWriter.internalWrite(CHCelebornHashBasedColumnarShuffleWriter.scala:65)
at
org.apache.spark.shuffle.CelebornHashBasedColumnarShuffleWriter.write(CelebornHashBasedColumnarShuffleWriter.scala:99)
at
org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59)
at
org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99)
at
org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52)
at org.apache.spark.scheduler.Task.run(Task.scala:136)
at
org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:548)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1504)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:551)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
Caused by: org.apache.gluten.exception.GlutenException: Allocator: Cannot
realloc from 4.00 KiB to 128.00 PiB: , errno: 22, strerror: Invalid argument:
While executing StreamingAggregatingTransform
0. Poco::Exception::Exception(String const&, int) @ 0x000000001306d1b9
1. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @
0x0000000006343a5b
2. DB::Exception::Exception(PreformattedMessage&&, int) @ 0x00000000061f77ac
3. DB::ErrnoException::ErrnoException<ReadableSize, ReadableSize>(int,
FormatStringHelperImpl<std::type_identity<ReadableSize>::type,
std::type_identity<ReadableSize>::type>, ReadableSize&&, ReadableSize&&) @
0x00000000063252ee
4. Allocator<false, false>::realloc(void*, unsigned long, unsigned long,
unsigned long) @ 0x000000000632523b
5. void DB::PODArrayBase<1ul, 4096ul, Allocator<false, false>, 63ul,
64ul>::realloc<>(unsigned long) @ 0x000000000620b36d
6. DB::ColumnString::deserializeAndInsertFromArena(char const*) @
0x000000000fb4c0ff
7. DB::AggregationMethodSerialized<HashMapTable<StringRef,
HashMapCellWithSavedHash<StringRef, char*, DefaultHash<StringRef>,
HashTableNoState>, DefaultHash<StringRef>,
HashTableGrowerWithPrecalculation<8ul>, Allocator<true, true>>, true,
false>::insertKeyIntoColumns(StringRef, std::vector<DB::IColumn*,
std::allocator<DB::IColumn*>>&, std::vector<unsigned long,
std::allocator<unsigned long>> const&) @ 0x000000000ed16ecf
8. auto std::variant<DB::Block, std::list<DB::Block,
std::allocator<DB::Block>>>
DB::Aggregator::convertToBlockImplNotFinal<DB::AggregationMethodSerialized<HashMapTable<StringRef,
HashMapCellWithSavedHash<StringRef, char*, DefaultHash<StringRef>,
HashTableNoState>, DefaultHash<StringRef>,
HashTableGrowerWithPrecalculation<8ul>, Allocator<true, true>>, true, false>,
HashMapTable<StringRef, HashMapCellWithSavedHash<StringRef, char*,
DefaultHash<StringRef>, HashTableNoState>, DefaultHash<StringRef>,
HashTableGrowerWithPrecalculation<8ul>, Allocator<true,
true>>>(DB::AggregationMethodSerialized<HashMapTable<StringRef,
HashMapCellWithSavedHash<StringRef, char*, DefaultHash<StringRef>,
HashTableNoState>, DefaultHash<StringRef>,
HashTableGrowerWithPrecalculation<8ul>, Allocator<true, true>>, true, false>&,
HashMapTable<StringRef, HashMapCellWithSavedHash<StringRef, char*,
DefaultHash<StringRef>, HashTableNoState>, DefaultHash<StringRef>,
HashTableGrowerWithPrecalculation<8ul>, Allocator<
true, true>>&, std::vector<std::shared_ptr<DB::Arena>,
std::allocator<std::shared_ptr<DB::Arena>>>&, unsigned long, bool)
const::'lambda'(DB::AggregationMethodSerialized<HashMapTable<StringRef,
HashMapCellWithSavedHash<StringRef, char*, DefaultHash<StringRef>,
HashTableNoState>, DefaultHash<StringRef>,
HashTableGrowerWithPrecalculation<8ul>, Allocator<true, true>>, true, false>
const&, HashMapTable<StringRef, HashMapCellWithSavedHash<StringRef, char*,
DefaultHash<StringRef>, HashTableNoState>, DefaultHash<StringRef>,
HashTableGrowerWithPrecalculation<8ul>, Allocator<true,
true>>&)::operator()<StringRef,
char*>(DB::AggregationMethodSerialized<HashMapTable<StringRef,
HashMapCellWithSavedHash<StringRef, char*, DefaultHash<StringRef>,
HashTableNoState>, DefaultHash<StringRef>,
HashTableGrowerWithPrecalculation<8ul>, Allocator<true, true>>, true, false>
const&, HashMapTable<StringRef, HashMapCellWithSavedHash<StringRef, char*,
DefaultHash<StringRef>, HashTableNoState>, DefaultHash<String
Ref>, HashTableGrowerWithPrecalculation<8ul>, Allocator<true, true>>&) const @
0x000000000ece7e98
9. std::variant<DB::Block, std::list<DB::Block, std::allocator<DB::Block>>>
DB::Aggregator::convertToBlockImplNotFinal<DB::AggregationMethodSerialized<HashMapTable<StringRef,
HashMapCellWithSavedHash<StringRef, char*, DefaultHash<StringRef>,
HashTableNoState>, DefaultHash<StringRef>,
HashTableGrowerWithPrecalculation<8ul>, Allocator<true, true>>, true, false>,
HashMapTable<StringRef, HashMapCellWithSavedHash<StringRef, char*,
DefaultHash<StringRef>, HashTableNoState>, DefaultHash<StringRef>,
HashTableGrowerWithPrecalculation<8ul>, Allocator<true,
true>>>(DB::AggregationMethodSerialized<HashMapTable<StringRef,
HashMapCellWithSavedHash<StringRef, char*, DefaultHash<StringRef>,
HashTableNoState>, DefaultHash<StringRef>,
HashTableGrowerWithPrecalculation<8ul>, Allocator<true, true>>, true, false>&,
HashMapTable<StringRef, HashMapCellWithSavedHash<StringRef, char*,
DefaultHash<StringRef>, HashTableNoState>, DefaultHash<StringRef>,
HashTableGrowerWithPrecalculation<8ul>, Allocator<true,
true>>&, std::vector<std::shared_ptr<DB::Arena>,
std::allocator<std::shared_ptr<DB::Arena>>>&, unsigned long, bool) const @
0x000000000ece6a15
10. std::variant<DB::Block, std::list<DB::Block, std::allocator<DB::Block>>>
DB::Aggregator::convertToBlockImpl<DB::AggregationMethodSerialized<HashMapTable<StringRef,
HashMapCellWithSavedHash<StringRef, char*, DefaultHash<StringRef>,
HashTableNoState>, DefaultHash<StringRef>,
HashTableGrowerWithPrecalculation<8ul>, Allocator<true, true>>, true, false>,
HashMapTable<StringRef, HashMapCellWithSavedHash<StringRef, char*,
DefaultHash<StringRef>, HashTableNoState>, DefaultHash<StringRef>,
HashTableGrowerWithPrecalculation<8ul>, Allocator<true,
true>>>(DB::AggregationMethodSerialized<HashMapTable<StringRef,
HashMapCellWithSavedHash<StringRef, char*, DefaultHash<StringRef>,
HashTableNoState>, DefaultHash<StringRef>,
HashTableGrowerWithPrecalculation<8ul>, Allocator<true, true>>, true, false>&,
HashMapTable<StringRef, HashMapCellWithSavedHash<StringRef, char*,
DefaultHash<StringRef>, HashTableNoState>, DefaultHash<StringRef>,
HashTableGrowerWithPrecalculation<8ul>, Allocator<true, true>>
&, DB::Arena*, std::vector<std::shared_ptr<DB::Arena>,
std::allocator<std::shared_ptr<DB::Arena>>>&, bool, unsigned long, bool) const
@ 0x000000000eca0b61
11. std::conditional<false, DB::Block, std::list<DB::Block,
std::allocator<DB::Block>>>::type
DB::Aggregator::prepareBlockAndFillSingleLevel<false>(DB::AggregatedDataVariants&,
bool) const @ 0x000000000eb0215b
12. DB::Aggregator::convertToBlocks(DB::AggregatedDataVariants&, bool,
unsigned long) const @ 0x000000000ea6cf18
13. local_engine::AggregateDataBlockConverter::hasNext() @ 0x0000000006877e0b
14. local_engine::StreamingAggregatingTransform::work()::$_0::operator()()
const @ 0x000000000687e944
15. local_engine::StreamingAggregatingTransform::work() @ 0x000000000687e76f
16. DB::ExecutionThreadContext::executeTask() @ 0x0000000010662f22
17. DB::PipelineExecutor::executeStepImpl(unsigned long, std::atomic<bool>*)
@ 0x0000000010658f30
18. DB::PipelineExecutor::executeStep(std::atomic<bool>*) @
0x0000000010658ac9
19. DB::PullingPipelineExecutor::pull(DB::Chunk&) @ 0x00000000106697c8
20. DB::PullingPipelineExecutor::pull(DB::Block&) @ 0x00000000106699b0
21. local_engine::LocalExecutor::hasNext() @ 0x0000000006720289
22. Java_org_apache_gluten_vectorized_BatchIterator_nativeHasNext @
0x00000000061dd397
at org.apache.gluten.vectorized.BatchIterator.nativeHasNext(Native
Method)
at
org.apache.gluten.vectorized.BatchIterator.hasNextInternal(BatchIterator.java:57)
at
org.apache.gluten.vectorized.GeneralOutIterator.hasNext(GeneralOutIterator.java:37)
... 18 more
```
### Spark version
None
### Spark configurations
_No response_
### System information
_No response_
### Relevant logs
_No response_
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
