xiaonanyang-db commented on code in PR #56374:
URL: https://github.com/apache/spark/pull/56374#discussion_r3391588401
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala:
##########
@@ -182,8 +185,11 @@ object InMemoryFileIndex extends Logging {
}
-private class PathFilterWrapper(val filter: PathFilter) extends PathFilter
with Serializable {
+private class PathFilterWrapper(
+ val filter: PathFilter,
+ val listHiddenFiles: Boolean) extends PathFilter with Serializable {
override def accept(path: Path): Boolean = {
- (filter == null || filter.accept(path)) &&
!HadoopFSUtils.shouldFilterOutPathName(path.getName)
+ (filter == null || filter.accept(path)) &&
+ (listHiddenFiles || !HadoopFSUtils.shouldFilterOutPathName(path.getName))
Review Comment:
can we just pass `listHiddenFiles` into
`HadoopFSUtils.shouldFilterOutPathName`?
##########
core/src/test/scala/org/apache/spark/util/HadoopFSUtilsSuite.scala:
##########
@@ -17,9 +17,43 @@
package org.apache.spark.util
-import org.apache.spark.SparkFunSuite
+import java.io.File
+import java.nio.file.Files
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.{FileStatus, Path, PathFilter}
+
+import org.apache.spark.{SparkContext, SparkFunSuite}
+import org.apache.spark.LocalSparkContext.withSpark
class HadoopFSUtilsSuite extends SparkFunSuite {
+
+ // Accept everything; hidden-file filtering is exercised via the
listHiddenFiles flag.
+ private val acceptAllFilter: PathFilter = AcceptAllPathFilter
+
+ // Builds a tree with one regular file, hidden entries ('_'-, '.'-,
'._COPYING_'-named) and a
+ // hidden subdir with its own file. Returns (rootPath, regularFileName).
+ private def createHiddenFileTree(root: File): (Path, String) = {
Review Comment:
nit: add Scala doc to explain the returned values
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
