Github user jackylk commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2661#discussion_r213574978
--- Diff:
integration/spark-datasource/src/main/scala/org/apache/spark/sql/carbondata/execution/datasources/CarbonFileIndexReplaceRule.scala
---
@@ -82,4 +82,23 @@ class CarbonFileIndexReplaceRule extends
Rule[LogicalPlan] {
fileIndex
}
}
+
+ /**
+ * Get datafolders recursively
+ */
+ private def getDataFolders(carbonFile: CarbonFile): Seq[CarbonFile] = {
+ val files = carbonFile.listFiles()
+ var folders: Seq[CarbonFile] = Seq()
+ files.foreach { f =>
+ if (f.isDirectory) {
+ val files = f.listFiles()
+ if (files.nonEmpty && !files(0).isDirectory) {
+ folders = Seq(f) ++ folders
--- End diff --
Will this create too many immutable object? Would ArrayBuffer better?
---
