This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push: new 9019a71770e [Fix](multi-catalog) Filter _temporary dir which is generated by spark. (#26194) 9019a71770e is described below commit 9019a71770ea158b035159bb523a4501aee87b55 Author: Xiangyu Wang <dut.xian...@gmail.com> AuthorDate: Fri Nov 3 13:42:29 2023 +0800 [Fix](multi-catalog) Filter _temporary dir which is generated by spark. (#26194) Sometimes there are some temporary files which are generated by some spark jobs, we need filter these files. ``` errCode = 2, detailMessage = (xxx.xxx.xxx.xxx.xxx) [CANCELLED][INTERNAL_ERROR]failed to init reader for file hdfs://xxx/user/hive/warehouse/xxx.db/xxx/_temporary/0/_temporary/attempt_202311010952379045223848154629697_0013_m_000442_3355/part-00442-3f97ac18-0583-4d2c-8995-8d21ceda56ec-c000.snappy.orc, err: [INTERNAL_ERROR]Init OrcReader failed. reason = Failed to parse the postscript from hdfs://xxx/user/hive/warehouse/xxx.db/xxx/_temporary/0/_temporary/attempt_202311010952379045223848154629697_0013_m_000442_3355/part-00442-3f97ac18-0583-4d2c-8995-8d21ceda56ec-c000.snappy.orc 0. @ 0x000000000bdbd89f doris::Status doris::Status::Error<6, true, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&>(std::basic_string_view<char, std::char_traits<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/unique_ptr.h:173 in /hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-36 [...] doris::vectorized::OrcReader::_create_file_reader() /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.h:187 in /hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-360-master.tar.gz/apache-doris-be-360-master-bin-x86_64/lib/doris_be 2. @ 0x000000000ecceff9 doris::vectorized::OrcReader::init_reader(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const*, std::unordered_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::variant<doris::ColumnValueRange<(doris::PrimitiveType)3>, doris::ColumnValueRange<(doris::PrimitiveType)4>, doris::ColumnValueRange<(doris: [...] doris::vectorized::VFileScanner::_get_next_reader() /root/doris-master/be/src/vec/exec/scan/vfile_scanner.cpp:800 in /hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-360-master.tar.gz/apache-doris-be-360-master-bin-x86_64/lib/doris_be 4. @ 0x000000000ff4256f doris::vectorized::VFileScanner::_get_block_impl(doris::RuntimeState*, doris::vectorized::Block*, bool*) /root/doris-master/be/src/common/status.h:448 in /hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-360-master.tar.gz/apache-doris-be-360-master-bin-x86_64/lib/doris_be 5. @ 0x000000000ffdb1aa doris::vectorized::VScanner::get_block(doris::RuntimeState*, doris::vectorized::Block*, bool*) /root/doris-master/be/src/vec/exec/scan/vscanner.cpp:0 in /hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-360-master.tar.gz/apache-doris-be-360-master-bin-x86_64/lib/doris_be 6. @ 0x000000000ff3b379 doris::vectorized::ScannerScheduler::_scanner_scan(doris::vectorized::ScannerScheduler*, doris::vectorized::ScannerContext*, std::shared_ptr<doris::vectorized::VScanner>) /root/doris-master/be/src/common/status.h:346 in /hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-360-master.tar.gz/apache-doris-be-360-master-bin-x86_64/lib/doris_be 7. @ 0x000000000ff3ca11 std::_Function_handler<void (), doris::vectorized::ScannerSch [...] doris::ThreadPool::dispatch_thread() /root/doris-master/be/src/util/threadpool.cpp:0 in /hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-360-master.tar.gz/apache-doris-be-360-master-bin-x86_64/lib/doris_be 9. @ 0x000000000c97c2ac doris::Thread::supervise_thread(void*) /var/local/ldb-toolchain/bin/../usr/include/pthread.h:562 in /hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-360-master.tar.gz/apache-doris-be-360-master-bin-x86_64/lib/doris_be 10. @ 0x0000000000007ea5 start_thread in /usr/lib64/libpthread-2.17.so 11. @ 0x00000000000feb0d __clone in /usr/lib64/libc-2.17.so 0. @ 0x000000000c647366 doris::Status doris::Status::Error<6, true, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >(std::basic_string_view<char, std::char_traits<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_6 [...] doris::vectorized::VFileScanner::_get_next_reader() /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.h:187 in /hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-360-master.tar.gz/apache-doris-be-360-master-bin-x86_64/lib/doris_be 2. @ 0x000000000ff4256f doris::vectorized::VFileScanner::_get_block_impl(doris::RuntimeState*, doris::vectorized::Block*, bool*) /root/doris-master/be/src/common/status.h:448 in /hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-360-master.tar.gz/apache-doris-be-360-master-bin-x86_64/lib/doris_be 3. @ 0x000000000ffdb1aa doris::vectorized::VScanner::get_block(doris::RuntimeState*, doris::vectorized::Block*, bool*) /root/doris-master/be/src/vec/exec/scan/vscanner.cpp:0 in /hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-360-master.tar.gz/apache-doris-be-360-master-bin-x86_64/lib/doris_be 4. @ 0x000000000ff3b379 doris::vectorized::ScannerScheduler::_scanner_scan(doris::vectorized::ScannerScheduler*, doris::vectorized::ScannerContext*, std::shared_ptr<doris::vectorized::VScanner>) /root/doris-master/be/src/common/status.h:346 in /hdpData10/yarn/localcache/usercache/hive/appcache/a… ``` Co-authored-by: wangxiangyu <wangxian...@360shuke.com> --- .../java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java index 480edd60f2c..bac891eb920 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java @@ -1058,9 +1058,14 @@ public class HiveMetaStoreCache { return false; } for (String name : path.toString().split("/")) { + // generated by hive if (name.startsWith(".hive-staging")) { return false; } + // generated by spark + if ("_temporary".equals(name)) { + return false; + } } return true; } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org