Indhumathi27 commented on a change in pull request #3719:
[CARBONDATA-3754]avoid listing index files during SI rebuild
URL: https://github.com/apache/carbondata/pull/3719#discussion_r410099985
##########
File path:
integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/util/SecondaryIndexUtil.scala
##########
@@ -280,6 +284,40 @@ object SecondaryIndexUtil {
}
}
+ /**
+ * This method deletes the old index files or merge index file after data
files merge
+ */
+ def deleteOldIndexOrMergeIndexFiles(carbonLoadModel: CarbonLoadModel,
+ validSegments: util.List[Segment],
+ indexCarbonTable: CarbonTable): Unit = {
+ // delete the index/merge index carbonFile of old data files
+ val loadModel = SecondaryIndexUtil
+
.getCarbonLoadModel(carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable,
+ carbonLoadModel.getLoadMetadataDetails,
+ carbonLoadModel.getFactTimeStamp,
+ carbonLoadModel.getColumnCompressor)
+ var indexFiles: util.Map[String, String] = null
+ validSegments.asScala.foreach { segment =>
+ // for old store scenario
+ if (segment.getSegmentFileName == null) {
+ val segmentPath =
CarbonTablePath.getSegmentPath(indexCarbonTable.getTablePath,
+ segment.getSegmentNo)
+ indexFiles = new
SegmentIndexFileStore().getMergeOrIndexFilesFromSegment(segmentPath)
+ } else {
+ val segmentFileStore = new
SegmentFileStore(indexCarbonTable.getTablePath,
+ segment.getSegmentFileName)
+ indexFiles = segmentFileStore.getIndexOrMergeFiles
+ }
+ }
+ val indexFilesToDelete = indexFiles.keySet().asScala.filter { indexFile =>
Review comment:
I think for each segment we have to delete old indexFiles. This delete code
has to be moved inside foreach loop?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
