akashrn5 commented on a change in pull request #3719: [CARBONDATA-3754]avoid
listing index files during SI rebuild
URL: https://github.com/apache/carbondata/pull/3719#discussion_r410655316
##########
File path:
integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/util/SecondaryIndexUtil.scala
##########
@@ -280,6 +286,35 @@ object SecondaryIndexUtil {
}
}
+ /**
+ * This method deletes the old index files or merge index file after data
files merge
+ */
+ def deleteOldIndexOrMergeIndexFiles(factTimeStamp: Long,
+ validSegments: util.List[Segment],
+ indexCarbonTable: CarbonTable): Unit = {
+ // delete the index/merge index carbonFile of old data files
+ validSegments.asScala.foreach { segment =>
+ var indexFiles: util.Map[String, String] = null
+ // for old store scenario
+ if (segment.getSegmentFileName == null) {
+ val segmentPath =
CarbonTablePath.getSegmentPath(indexCarbonTable.getTablePath,
+ segment.getSegmentNo)
+ indexFiles = new
SegmentIndexFileStore().getMergeOrIndexFilesFromSegment(segmentPath)
+ } else {
+ val segmentFileStore = new
SegmentFileStore(indexCarbonTable.getTablePath,
+ segment.getSegmentFileName)
+ indexFiles = segmentFileStore.getIndexOrMergeFiles
+ }
+ val indexFilesToDelete = indexFiles.keySet().asScala.filter { indexFile
=>
+ DataFileUtil.getTimeStampFromFileName(indexFile).toLong >
+ factTimeStamp
+ }
+ indexFilesToDelete.foreach { indexFile =>
Review comment:
done
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
