This is an automated email from the ASF dual-hosted git repository. huor pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hawq.git
The following commit(s) were added to refs/heads/master by this push: new b3f60b3 HAWQ-1769. Fix DirectoryIterator of libhdfs3 b3f60b3 is described below commit b3f60b333fbbdc43429fbb35dc33a35c4f415a52 Author: qinzuoyan <qinzuo...@xiaomi.com> AuthorDate: Tue Oct 20 23:25:55 2020 +0800 HAWQ-1769. Fix DirectoryIterator of libhdfs3 --- depends/libhdfs3/src/client/DirectoryIterator.cpp | 32 +++++++++++------------ depends/libhdfs3/src/client/DirectoryIterator.h | 1 + depends/libhdfs3/src/server/Namenode.h | 2 ++ 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/depends/libhdfs3/src/client/DirectoryIterator.cpp b/depends/libhdfs3/src/client/DirectoryIterator.cpp index 40b39a4..7cfb97f 100644 --- a/depends/libhdfs3/src/client/DirectoryIterator.cpp +++ b/depends/libhdfs3/src/client/DirectoryIterator.cpp @@ -28,17 +28,17 @@ namespace Hdfs { DirectoryIterator::DirectoryIterator() : - needLocations(false), filesystem(NULL), next(0) { + needLocations(false), hasMore(true), filesystem(NULL), next(0) { } DirectoryIterator::DirectoryIterator(Hdfs::Internal::FileSystemImpl * const fs, std::string path, bool needLocations) : - needLocations(needLocations), filesystem(fs), next(0), path(path) { + needLocations(needLocations), hasMore(true), filesystem(fs), next(0), path(path) { } DirectoryIterator::DirectoryIterator(const DirectoryIterator & it) : - needLocations(it.needLocations), filesystem(it.filesystem), next(it.next), path(it.path), startAfter( - it.startAfter), lists(it.lists) { + needLocations(it.needLocations), hasMore(it.hasMore), filesystem(it.filesystem), + next(it.next), path(it.path), startAfter(it.startAfter), lists(it.lists) { } DirectoryIterator & DirectoryIterator::operator =(const DirectoryIterator & it) { @@ -47,6 +47,7 @@ DirectoryIterator & DirectoryIterator::operator =(const DirectoryIterator & it) } needLocations = it.needLocations; + hasMore = it.hasMore; filesystem = it.filesystem; next = it.next; path = it.path; @@ -56,36 +57,33 @@ DirectoryIterator & DirectoryIterator::operator =(const DirectoryIterator & it) } bool DirectoryIterator::getListing() { - bool more; - - if (NULL == filesystem) { + if (NULL == filesystem || !hasMore) { return false; } next = 0; lists.clear(); - more = filesystem->getListing(path, startAfter, needLocations, lists); - if (!lists.empty()) { + hasMore = filesystem->getListing(path, startAfter, needLocations, lists); + + if (hasMore && !lists.empty()) { startAfter = lists.back().getPath(); } - return more || !lists.empty(); + return !lists.empty(); } bool DirectoryIterator::hasNext() { - if (next >= lists.size()) { - return getListing(); + if (next < lists.size()) { + return true; } - return true; + return getListing(); } Hdfs::FileStatus DirectoryIterator::getNext() { - if (next >= lists.size()) { - if (!getListing()) { - THROW(HdfsIOException, "End of the dir flow"); - } + if (!hasNext()) { + THROW(HdfsIOException, "End of the dir flow"); } return lists[next++]; diff --git a/depends/libhdfs3/src/client/DirectoryIterator.h b/depends/libhdfs3/src/client/DirectoryIterator.h index 932f086..513241f 100644 --- a/depends/libhdfs3/src/client/DirectoryIterator.h +++ b/depends/libhdfs3/src/client/DirectoryIterator.h @@ -45,6 +45,7 @@ private: private: bool needLocations; + bool hasMore; Hdfs::Internal::FileSystemImpl * filesystem; size_t next; std::string path; diff --git a/depends/libhdfs3/src/server/Namenode.h b/depends/libhdfs3/src/server/Namenode.h index ad213e8..0628788 100644 --- a/depends/libhdfs3/src/server/Namenode.h +++ b/depends/libhdfs3/src/server/Namenode.h @@ -514,6 +514,8 @@ public: * @param a partial listing starting after startAfter * @param dl append the returned directories. * + * @return true if there are more remaining entries. + * * @throw AccessControlException permission denied * @throw FileNotFoundException file <code>src</code> is not found * @throw UnresolvedLinkException If <code>src</code> contains a symlink