ravipesala commented on a change in pull request #3179: [CARBONDATA-3338]
Support Incremental DataLoad for MV Datamap[with single parent table]
URL: https://github.com/apache/carbondata/pull/3179#discussion_r278130624
##########
File path:
core/src/main/java/org/apache/carbondata/core/datamap/status/DataMapStatusManager.java
##########
@@ -127,4 +150,115 @@ private static DataMapSchema getDataMapSchema(String
dataMapName)
return DataMapStoreManager.getInstance().getDataMapSchema(dataMapName);
}
+ /**
+ * Returns valid segment list for a given RelationIdentifier
+ *
+ * @param relationIdentifier
+ * @return
+ * @throws IOException
+ */
+ public static List<String> getSegmentList(RelationIdentifier
relationIdentifier)
+ throws IOException {
+ List<String> segmentList = new ArrayList<>();
+ AbsoluteTableIdentifier absoluteTableIdentifier =
+ AbsoluteTableIdentifier.from(relationIdentifier.getTablePath());
+ List<Segment> validSegments =
+ new
SegmentStatusManager(absoluteTableIdentifier).getValidAndInvalidSegments()
+ .getValidSegments();
+ for (Segment segment : validSegments) {
+ segmentList.add(segment.getSegmentNo());
+ }
+ return segmentList;
+ }
+
+ /**
+ * This method will delete segment folders of the mv datamap table and
update the
+ * datamapSegmentStatus map in case of Insert-Overwrite/Update operation on
main table
+ */
+ public static void cleanMVdatamap(CarbonTable carbonTable) throws
IOException {
+ List<DataMapSchema> allDataMapSchemas =
+
DataMapStoreManager.getInstance().getDataMapSchemasOfTable(carbonTable);
+ for (DataMapSchema datamapschema : allDataMapSchemas) {
+ if (datamapschema.getProviderName()
+ .equalsIgnoreCase(DataMapClassProvider.MV.getShortName())) {
+ CarbonTable datamapTable = CarbonTable
+
.buildFromTablePath(datamapschema.getRelationIdentifier().getTableName(),
+ datamapschema.getRelationIdentifier().getDatabaseName(),
+ datamapschema.getRelationIdentifier().getTablePath(),
+ datamapschema.getRelationIdentifier().getTableId());
+ SegmentStatusManager segmentStatusManager =
+ new
SegmentStatusManager(datamapTable.getAbsoluteTableIdentifier());
+ ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();
+ try {
+ if (carbonLock.lockWithRetries()) {
+ LOGGER.info(
+ "Acquired lock for table" + datamapTable.getDatabaseName() +
"." + datamapTable
+ .getTableName() + " for table status updation");
+ LoadMetadataDetails[] loadMetadataDetails =
+
SegmentStatusManager.readLoadMetadata(datamapTable.getMetadataPath());
+ List<CarbonFile> staleFolders = new ArrayList<>();
+ for (LoadMetadataDetails entry : loadMetadataDetails) {
+ entry.setSegmentStatus(SegmentStatus.MARKED_FOR_DELETE);
+ // For insert overwrite, we will delete the old segment folder
immediately
+ // So collect the old segments here
+ String segmentPath = CarbonTablePath
+
.getSegmentPath(datamapschema.getRelationIdentifier().getTablePath(),
+ entry.getLoadName());
+ if (FileFactory.isFileExist(segmentPath,
FileFactory.getFileType(segmentPath))) {
+ staleFolders.add(FileFactory.getCarbonFile(segmentPath));
+ }
+ }
+ SegmentStatusManager.writeLoadDetailsIntoFile(CarbonTablePath
+
.getTableStatusFilePath(datamapschema.getRelationIdentifier().getTablePath()),
+ loadMetadataDetails);
+ // Update datamapSegmentStatus map for mv
+ storageProvider.clearSegmentMapping(datamapschema);
+ // Delete all old stale segment folders
+ for (CarbonFile staleFolder : staleFolders) {
+ try {
+ CarbonUtil.deleteFoldersAndFiles(staleFolder);
+ } catch (IOException | InterruptedException e) {
+ LOGGER.error("Failed to delete stale folder: " +
e.getMessage(), e);
+ }
+ }
+ }
+ } finally {
+ if (carbonLock.unlock()) {
+ LOGGER.info("Table unlocked successfully after table status
updation" + datamapTable
+ .getDatabaseName() + "." + datamapTable.getTableName());
+ } else {
+ LOGGER.error(
+ "Unable to unlock Table lock for table" +
datamapTable.getDatabaseName() + "."
+ + datamapTable.getTableName() + " during table status
updation");
+ }
+ }
+ }
+ }
+ }
+
+ public static void updateDataMapSegmentStatusAfterCompaction(CarbonTable
carbonTable)
+ throws IOException, NoSuchDataMapException {
+ DataMapSchema dataMapSchema = getDataMapSchema(carbonTable.getTableName()
+ .substring(0,
carbonTable.getTableName().lastIndexOf(CarbonCommonConstants.UNDERSCORE)));
+ LoadMetadataDetails[] loadMetadataDetails =
+ SegmentStatusManager.readLoadMetadata(carbonTable.getMetadataPath());
+ storageProvider.updateMappingAfterCompaction(dataMapSchema,
loadMetadataDetails);
Review comment:
Please move the logic to here
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
