Github user jackylk commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1970#discussion_r167482170
--- Diff:
core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
---
@@ -708,4 +710,129 @@ public static Boolean
checkIfValidLoadInProgress(AbsoluteTableIdentifier absolut
}
}
+ private static boolean isLoadDeletionRequired(String metaDataLocation) {
+ LoadMetadataDetails[] details =
SegmentStatusManager.readLoadMetadata(metaDataLocation);
+ if (details != null && details.length > 0) {
+ for (LoadMetadataDetails oneRow : details) {
+ if ((SegmentStatus.MARKED_FOR_DELETE == oneRow.getSegmentStatus()
+ || SegmentStatus.COMPACTED == oneRow.getSegmentStatus()
+ || SegmentStatus.INSERT_IN_PROGRESS ==
oneRow.getSegmentStatus()
+ || SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS ==
oneRow.getSegmentStatus())
+ && oneRow.getVisibility().equalsIgnoreCase("true")) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * This will update the old table status details before clean files to
the latest table status.
+ * @param oldList
+ * @param newList
+ * @return
+ */
+ public static List<LoadMetadataDetails> updateLoadMetadataFromOldToNew(
+ LoadMetadataDetails[] oldList, LoadMetadataDetails[] newList) {
+
+ List<LoadMetadataDetails> newListMetadata =
+ new ArrayList<LoadMetadataDetails>(Arrays.asList(newList));
+ for (LoadMetadataDetails oldSegment : oldList) {
+ if ("false".equalsIgnoreCase(oldSegment.getVisibility())) {
+
newListMetadata.get(newListMetadata.indexOf(oldSegment)).setVisibility("false");
+ }
+ }
+ return newListMetadata;
+ }
+
+ private static void writeLoadMetadata(AbsoluteTableIdentifier identifier,
+ List<LoadMetadataDetails> listOfLoadFolderDetails) throws
IOException {
+ String dataLoadLocation =
CarbonTablePath.getTableStatusFilePath(identifier.getTablePath());
+
+ DataOutputStream dataOutputStream;
+ Gson gsonObjectToWrite = new Gson();
+ BufferedWriter brWriter = null;
+
+ AtomicFileOperations writeOperation =
+ new AtomicFileOperationsImpl(dataLoadLocation,
FileFactory.getFileType(dataLoadLocation));
+
+ try {
+
+ dataOutputStream =
writeOperation.openForWrite(FileWriteOperation.OVERWRITE);
+ brWriter = new BufferedWriter(new
OutputStreamWriter(dataOutputStream,
+ Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
+
+ String metadataInstance =
gsonObjectToWrite.toJson(listOfLoadFolderDetails.toArray());
+ brWriter.write(metadataInstance);
+ } finally {
+ try {
+ if (null != brWriter) {
+ brWriter.flush();
+ }
+ } catch (Exception e) {
+ LOG.error("error in flushing ");
+
+ }
+ CarbonUtil.closeStreams(brWriter);
+ writeOperation.close();
+ }
+ }
+
+ public static void deleteLoadsAndUpdateMetadata(
+ CarbonTable carbonTable,
+ boolean isForceDeletion) throws IOException {
+ if (isLoadDeletionRequired(carbonTable.getMetadataPath())) {
+ LoadMetadataDetails[] details =
+
SegmentStatusManager.readLoadMetadata(carbonTable.getMetadataPath());
+ AbsoluteTableIdentifier identifier =
carbonTable.getAbsoluteTableIdentifier();
+ ICarbonLock carbonTableStatusLock =
CarbonLockFactory.getCarbonLockObj(
+ identifier, LockUsage.TABLE_STATUS_LOCK);
+
+ // Delete marked loads
+ boolean isUpdationRequired =
DeleteLoadFolders.deleteLoadFoldersFromFileSystem(
+ identifier,
--- End diff --
fixed
---
