Github user maropu commented on a diff in the pull request:
https://github.com/apache/spark/pull/21608#discussion_r204654664
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningAwareFileIndex.scala
---
@@ -59,14 +59,15 @@ abstract class PartitioningAwareFileIndex(
override def listFiles(
partitionFilters: Seq[Expression], dataFilters: Seq[Expression]):
Seq[PartitionDirectory] = {
val selectedPartitions = if (partitionSpec().partitionColumns.isEmpty)
{
- PartitionDirectory(InternalRow.empty, allFiles().filter(f =>
isDataPath(f.getPath))) :: Nil
+ PartitionDirectory(InternalRow.empty,
+ allFiles().filter(f => DataSourceUtils.isDataPath(f.getPath))) ::
Nil
--- End diff --
super nit:
```
val files = allFiles().filter(f =>
DataSourceUtils.isDataPath(f.getPath))
PartitionDirectory(InternalRow.empty, files) :: Nil
```
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
