ravipesala commented on a change in pull request #3179: [CARBONDATA-3338]
Support Incremental DataLoad for MV Datamap[with single parent table]
URL: https://github.com/apache/carbondata/pull/3179#discussion_r278124174
##########
File path:
core/src/main/java/org/apache/carbondata/core/datamap/status/DataMapStatusManager.java
##########
@@ -127,4 +150,115 @@ private static DataMapSchema getDataMapSchema(String
dataMapName)
return DataMapStoreManager.getInstance().getDataMapSchema(dataMapName);
}
+ /**
+ * Returns valid segment list for a given RelationIdentifier
+ *
+ * @param relationIdentifier
+ * @return
+ * @throws IOException
+ */
+ public static List<String> getSegmentList(RelationIdentifier
relationIdentifier)
+ throws IOException {
+ List<String> segmentList = new ArrayList<>();
+ AbsoluteTableIdentifier absoluteTableIdentifier =
+ AbsoluteTableIdentifier.from(relationIdentifier.getTablePath());
+ List<Segment> validSegments =
+ new
SegmentStatusManager(absoluteTableIdentifier).getValidAndInvalidSegments()
+ .getValidSegments();
+ for (Segment segment : validSegments) {
+ segmentList.add(segment.getSegmentNo());
+ }
+ return segmentList;
+ }
+
+ /**
+ * This method will delete segment folders of the mv datamap table and
update the
+ * datamapSegmentStatus map in case of Insert-Overwrite/Update operation on
main table
+ */
+ public static void cleanMVdatamap(CarbonTable carbonTable) throws
IOException {
+ List<DataMapSchema> allDataMapSchemas =
+
DataMapStoreManager.getInstance().getDataMapSchemasOfTable(carbonTable);
+ for (DataMapSchema datamapschema : allDataMapSchemas) {
+ if (datamapschema.getProviderName()
+ .equalsIgnoreCase(DataMapClassProvider.MV.getShortName())) {
+ CarbonTable datamapTable = CarbonTable
+
.buildFromTablePath(datamapschema.getRelationIdentifier().getTableName(),
+ datamapschema.getRelationIdentifier().getDatabaseName(),
+ datamapschema.getRelationIdentifier().getTablePath(),
+ datamapschema.getRelationIdentifier().getTableId());
+ SegmentStatusManager segmentStatusManager =
+ new
SegmentStatusManager(datamapTable.getAbsoluteTableIdentifier());
+ ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();
+ try {
+ if (carbonLock.lockWithRetries()) {
+ LOGGER.info(
+ "Acquired lock for table" + datamapTable.getDatabaseName() +
"." + datamapTable
+ .getTableName() + " for table status updation");
+ LoadMetadataDetails[] loadMetadataDetails =
+
SegmentStatusManager.readLoadMetadata(datamapTable.getMetadataPath());
+ List<CarbonFile> staleFolders = new ArrayList<>();
+ for (LoadMetadataDetails entry : loadMetadataDetails) {
+ entry.setSegmentStatus(SegmentStatus.MARKED_FOR_DELETE);
+ // For insert overwrite, we will delete the old segment folder
immediately
+ // So collect the old segments here
+ String segmentPath = CarbonTablePath
+
.getSegmentPath(datamapschema.getRelationIdentifier().getTablePath(),
+ entry.getLoadName());
+ if (FileFactory.isFileExist(segmentPath,
FileFactory.getFileType(segmentPath))) {
+ staleFolders.add(FileFactory.getCarbonFile(segmentPath));
Review comment:
Please don't remove immediately as some queries might be running on it. Let
it remove during next time load
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
