Hello everyone, I'm suffering from a coredump issue when writing Arrow table to Parquet file. This issue is stable reproducible when compiling Arrow without Jemalloc. I made some initial investigations:
1. It happened on v4.0.1 & master branch 2. It happened on different distro & different gcc versions (Debian 8/9/11 & Ubuntu 20.04, gcc 4.9.2/6.3.0/10.2.1, clang 12.0.1) More details recorded in ARROW-13676 (including callstack, reproducing codes). Looking forward to any helping! Regards, Shuai Zhang ------ Code: https://github.com/hcoona/arrow/commit/8fa6cdb0c756c17ea3edc43b7b73c717823bda85 ------ Callstack #0 0x7f4478d3c62c in parquet::arrow::(anonymous namespace)::VarRangeSelector<int>::GetRange(long) const /data00/home/zhangshuai.ds/GitEnlistments/code.byted.org/cpp3rdlib/arrow/cpp/build/../src/parquet/arrow/path_internal.cc:438:41 #1 0x7f4478d3bdfa in parquet::arrow::(anonymous namespace)::ListPathNode<parquet::arrow::(anonymous namespace)::VarRangeSelector<int> >::Run(parquet::arrow::ElementRange*, parquet::arrow::ElementRange*, parquet::arrow::(anonymous namespace)::PathWriteContext*) /data00/home/zhangshuai.ds/GitEnlistments/code.byted.org/cpp3rdlib/arrow/cpp/build/../src/parquet/arrow/path_internal.cc:345:32 #2 0x7f4478d3b9e7 in parquet::arrow::(anonymous namespace)::WritePath(parquet::arrow::ElementRange, parquet::arrow::(anonymous namespace)::PathInfo*, parquet::ArrowWriteContext*, std::function<arrow::Status (parquet::arrow::MultipathLevelBuilderResult const&)>)::$_0::operator()(parquet::arrow::(anonymous namespace)::ListPathNode<parquet::arrow::(anonymous namespace)::VarRangeSelector<int> >*) /data00/home/zhangshuai.ds/GitEnlistments/code.byted.org/cpp3rdlib/arrow/cpp/build/../src/parquet/arrow/path_internal.cc:583:22 #3 0x7f4478d3aa32 in parquet::arrow::(anonymous namespace)::IterationResult arrow::util::detail::VariantImpl<arrow::util::Variant<parquet::arrow::(anonymous namespace)::NullableTerminalNode, parquet::arrow::(anonymous namespace)::ListPathNode<parquet::arrow::(anonymous namespace)::VarRangeSelector<int> >, parquet::arrow::(anonymous namespace)::ListPathNode<parquet::arrow::(anonymous namespace)::VarRangeSelector<long> >, parquet::arrow::(anonymous namespace)::ListPathNode<parquet::arrow::(anonymous namespace)::FixedSizedRangeSelector>, parquet::arrow::(anonymous namespace)::NullableNode, parquet::arrow::(anonymous namespace)::AllPresentTerminalNode, parquet::arrow::(anonymous namespace)::AllNullsTerminalNode>, parquet::arrow::(anonymous namespace)::ListPathNode<parquet::arrow::(anonymous namespace)::VarRangeSelector<int> >, parquet::arrow::(anonymous namespace)::ListPathNode<parquet::arrow::(anonymous namespace)::VarRangeSelector<long> >, parquet::arrow::(anonymous namespace)::ListPathNode<parquet::arrow::(anonymous namespace)::FixedSizedRangeSelector>, parquet::arrow::(anonymous namespace)::NullableNode, parquet::arrow::(anonymous namespace)::AllPresentTerminalNode, parquet::arrow::(anonymous namespace)::AllNullsTerminalNode>::visit_mutable<parquet::arrow::(anonymous namespace)::IterationResult, parquet::arrow::(anonymous namespace)::WritePath(parquet::arrow::ElementRange, parquet::arrow::(anonymous namespace)::PathInfo*, parquet::ArrowWriteContext*, std::function<arrow::Status (parquet::arrow::MultipathLevelBuilderResult const&)>)::$_0&>(parquet::arrow::(anonymous namespace)::WritePath(parquet::arrow::ElementRange, parquet::arrow::(anonymous namespace)::PathInfo*, parquet::ArrowWriteContext*, std::function<arrow::Status (parquet::arrow::MultipathLevelBuilderResult const&)>)::$_0&) /data00/home/zhangshuai.ds/GitEnlistments/code.byted.org/cpp3rdlib/arrow/cpp/build/../src/arrow/util/variant.h:236:14 #4 0x7f4478d3a8f7 in parquet::arrow::(anonymous namespace)::IterationResult arrow::util::detail::VariantImpl<arrow::util::Variant<parquet::arrow::(anonymous namespace)::NullableTerminalNode, parquet::arrow::(anonymous namespace)::ListPathNode<parquet::arrow::(anonymous namespace)::VarRangeSelector<int> >, parquet::arrow::(anonymous namespace)::ListPathNode<parquet::arrow::(anonymous namespace)::VarRangeSelector<long> >, parquet::arrow::(anonymous namespace)::ListPathNode<parquet::arrow::(anonymous namespace)::FixedSizedRangeSelector>, parquet::arrow::(anonymous namespace)::NullableNode, parquet::arrow::(anonymous namespace)::AllPresentTerminalNode, parquet::arrow::(anonymous namespace)::AllNullsTerminalNode>, parquet::arrow::(anonymous namespace)::NullableTerminalNode, parquet::arrow::(anonymous namespace)::ListPathNode<parquet::arrow::(anonymous namespace)::VarRangeSelector<int> >, parquet::arrow::(anonymous namespace)::ListPathNode<parquet::arrow::(anonymous namespace)::VarRangeSelector<long> >, parquet::arrow::(anonymous namespace)::ListPathNode<parquet::arrow::(anonymous namespace)::FixedSizedRangeSelector>, parquet::arrow::(anonymous namespace)::NullableNode, parquet::arrow::(anonymous namespace)::AllPresentTerminalNode, parquet::arrow::(anonymous namespace)::AllNullsTerminalNode>::visit_mutable<parquet::arrow::(anonymous namespace)::IterationResult, parquet::arrow::(anonymous namespace)::WritePath(parquet::arrow::ElementRange, parquet::arrow::(anonymous namespace)::PathInfo*, parquet::ArrowWriteContext*, std::function<arrow::Status (parquet::arrow::MultipathLevelBuilderResult const&)>)::$_0&>(parquet::arrow::(anonymous namespace)::WritePath(parquet::arrow::ElementRange, parquet::arrow::(anonymous namespace)::PathInfo*, parquet::ArrowWriteContext*, std::function<arrow::Status (parquet::arrow::MultipathLevelBuilderResult const&)>)::$_0&) /data00/home/zhangshuai.ds/GitEnlistments/code.byted.org/cpp3rdlib/arrow/cpp/build/../src/arrow/util/variant.h:238:27 #5 0x7f4478d3a7ec in parquet::arrow::(anonymous namespace)::IterationResult arrow::util::visit<parquet::arrow::(anonymous namespace)::WritePath(parquet::arrow::ElementRange, parquet::arrow::(anonymous namespace)::PathInfo*, parquet::ArrowWriteContext*, std::function<arrow::Status (parquet::arrow::MultipathLevelBuilderResult const&)>)::$_0&, parquet::arrow::(anonymous namespace)::NullableTerminalNode, parquet::arrow::(anonymous namespace)::ListPathNode<parquet::arrow::(anonymous namespace)::VarRangeSelector<int> >, parquet::arrow::(anonymous namespace)::ListPathNode<parquet::arrow::(anonymous namespace)::VarRangeSelector<long> >, parquet::arrow::(anonymous namespace)::ListPathNode<parquet::arrow::(anonymous namespace)::FixedSizedRangeSelector>, parquet::arrow::(anonymous namespace)::NullableNode, parquet::arrow::(anonymous namespace)::AllPresentTerminalNode, parquet::arrow::(anonymous namespace)::AllNullsTerminalNode, parquet::arrow::(anonymous namespace)::IterationResult>(parquet::arrow::(anonymous namespace)::WritePath(parquet::arrow::ElementRange, parquet::arrow::(anonymous namespace)::PathInfo*, parquet::ArrowWriteContext*, std::function<arrow::Status (parquet::arrow::MultipathLevelBuilderResult const&)>)::$_0&, arrow::util::Variant<parquet::arrow::(anonymous namespace)::NullableTerminalNode, parquet::arrow::(anonymous namespace)::ListPathNode<parquet::arrow::(anonymous namespace)::VarRangeSelector<int> >, parquet::arrow::(anonymous namespace)::ListPathNode<parquet::arrow::(anonymous namespace)::VarRangeSelector<long> >, parquet::arrow::(anonymous namespace)::ListPathNode<parquet::arrow::(anonymous namespace)::FixedSizedRangeSelector>, parquet::arrow::(anonymous namespace)::NullableNode, parquet::arrow::(anonymous namespace)::AllPresentTerminalNode, parquet::arrow::(anonymous namespace)::AllNullsTerminalNode>*) /data00/home/zhangshuai.ds/GitEnlistments/code.byted.org/cpp3rdlib/arrow/cpp/build/../src/arrow/util/variant.h:370:22 #6 0x7f4478d39bfc in parquet::arrow::(anonymous namespace)::WritePath(parquet::arrow::ElementRange, parquet::arrow::(anonymous namespace)::PathInfo*, parquet::ArrowWriteContext*, std::function<arrow::Status (parquet::arrow::MultipathLevelBuilderResult const&)>) /data00/home/zhangshuai.ds/GitEnlistments/code.byted.org/cpp3rdlib/arrow/cpp/build/../src/parquet/arrow/path_internal.cc:604:30 #7 0x7f4478d701a1 in parquet::arrow::MultipathLevelBuilderImpl::Write(int, parquet::ArrowWriteContext*, std::function<arrow::Status (parquet::arrow::MultipathLevelBuilderResult const&)>) /data00/home/zhangshuai.ds/GitEnlistments/code.byted.org/cpp3rdlib/arrow/cpp/build/../src/parquet/arrow/path_internal.cc:865:12 #8 0x7f4478eb50a6 in parquet::arrow::(anonymous namespace)::ArrowColumnWriterV2::Write(parquet::ArrowWriteContext*) /data00/home/zhangshuai.ds/GitEnlistments/code.byted.org/cpp3rdlib/arrow/cpp/build/../src/parquet/arrow/writer.cc:134:9 #9 0x7f4478ebb0ee in parquet::arrow::FileWriterImpl::WriteColumnChunk(std::shared_ptr<arrow::ChunkedArray> const&, long, long) /data00/home/zhangshuai.ds/GitEnlistments/code.byted.org/cpp3rdlib/arrow/cpp/build/../src/parquet/arrow/writer.cc:313:22 #10 0x7f4478ebc1c0 in parquet::arrow::FileWriterImpl::WriteTable(arrow::Table const&, long)::'lambda'(long, long)::operator()(long, long) const /data00/home/zhangshuai.ds/GitEnlistments/code.byted.org/cpp3rdlib/arrow/cpp/build/../src/parquet/arrow/writer.cc:340:9 #11 0x7f4478eba0ae in parquet::arrow::FileWriterImpl::WriteTable(arrow::Table const&, long) /data00/home/zhangshuai.ds/GitEnlistments/code.byted.org/cpp3rdlib/arrow/cpp/build/../src/parquet/arrow/writer.cc:353:7 #12 0x7f4478eb2d50 in parquet::arrow::WriteTable(arrow::Table const&, arrow::MemoryPool*, std::shared_ptr<arrow::io::OutputStream>, long, std::shared_ptr<parquet::WriterProperties>, std::shared_ptr<parquet::ArrowWriterProperties>) /data00/home/zhangshuai.ds/GitEnlistments/code.byted.org/cpp3rdlib/arrow/cpp/build/../src/parquet/arrow/writer.cc:479:3 ..... ------- <https://www.avast.com/sig-email?utm_medium=email&utm_source=link&utm_campaign=sig-email&utm_content=webmail> 无病毒。www.avast.com <https://www.avast.com/sig-email?utm_medium=email&utm_source=link&utm_campaign=sig-email&utm_content=webmail> <#DAB4FAD8-2DD7-40BB-A1B8-4E2AA1F9FDF2>
