This is an automated email from the ASF dual-hosted git repository.
srowen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 46f9f44 [SPARK-27202][MINOR][SQL] Update comments to keep according
with code
46f9f44 is described below
commit 46f9f44918ba2589c6ffc938822cbdfac1c6f4d1
Author: wangguangxin.cn <[email protected]>
AuthorDate: Wed Mar 20 17:54:28 2019 -0500
[SPARK-27202][MINOR][SQL] Update comments to keep according with code
## What changes were proposed in this pull request?
Update comments in `InMemoryFileIndex.listLeafFiles` to keep according with
code.
## How was this patch tested?
existing test cases
Closes #24146 from WangGuangxin/SPARK-27202.
Authored-by: wangguangxin.cn <[email protected]>
Signed-off-by: Sean Owen <[email protected]>
---
.../apache/spark/sql/execution/datasources/InMemoryFileIndex.scala | 2 +-
.../spark/sql/execution/datasources/PartitioningAwareFileIndex.scala | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala
index fe418e6..db55a06 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala
@@ -308,7 +308,7 @@ object InMemoryFileIndex extends Logging {
// implementations don't actually issue RPC for this method.
//
// - Here we are calling `getFileBlockLocations` in a sequential manner,
but it should not
- // be a big deal since we always use to `listLeafFilesInParallel` when
the number of
+ // be a big deal since we always use to `bulkListLeafFiles` when the
number of
// paths exceeds threshold.
case f =>
// The other constructor of LocatedFileStatus will call
FileStatus.getPermission(),
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningAwareFileIndex.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningAwareFileIndex.scala
index b2e4155..f5ae095 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningAwareFileIndex.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningAwareFileIndex.scala
@@ -89,7 +89,7 @@ abstract class PartitioningAwareFileIndex(
if (partitionSpec().partitionColumns.isEmpty) {
// For each of the root input paths, get the list of files inside them
rootPaths.flatMap { path =>
- // Make the path qualified (consistent with listLeafFiles and
listLeafFilesInParallel).
+ // Make the path qualified (consistent with listLeafFiles and
bulkListLeafFiles).
val fs = path.getFileSystem(hadoopConf)
val qualifiedPathPre = fs.makeQualified(path)
val qualifiedPath: Path = if (qualifiedPathPre.isRoot &&
!qualifiedPathPre.isAbsolute) {
@@ -203,7 +203,7 @@ abstract class PartitioningAwareFileIndex(
case None =>
rootPaths.map { path =>
- // Make the path qualified (consistent with listLeafFiles and
listLeafFilesInParallel).
+ // Make the path qualified (consistent with listLeafFiles and
bulkListLeafFiles).
val qualifiedPath =
path.getFileSystem(hadoopConf).makeQualified(path)
if (leafFiles.contains(qualifiedPath)) qualifiedPath.getParent else
qualifiedPath }.toSet
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
