Repository: incubator-carbondata Updated Branches: refs/heads/master 7a6e3517d -> 0391d4223
in clean files it will read the table status first and it will delete the physical locations and update the table metadata and then take the lock for writing and after that it will write the old read list to the file. but this is wrong as in the time between the read and write the table status would have been changed. so before writing it should read the table status again and update the changes and then write. making false as first parameter. Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/816f8e36 Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/816f8e36 Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/816f8e36 Branch: refs/heads/master Commit: 816f8e3669440875066e41f664e0b5e44daa79ee Parents: 7a6e351 Author: ravikiran <ravikiran.sn...@gmail.com> Authored: Tue Sep 20 23:35:17 2016 +0530 Committer: Venkata Ramana G <ramana.gollam...@huawei.com> Committed: Wed Sep 21 00:30:18 2016 +0530 ---------------------------------------------------------------------- .../carbondata/spark/load/CarbonLoaderUtil.java | 21 ++++++++++++++++++++ .../spark/rdd/CarbonDataRDDFactory.scala | 12 ++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/816f8e36/integration/spark/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java b/integration/spark/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java index 2be1baf..88b8266 100644 --- a/integration/spark/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java +++ b/integration/spark/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java @@ -28,6 +28,7 @@ import java.net.UnknownHostException; import java.nio.charset.Charset; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.HashMap; @@ -993,4 +994,24 @@ public final class CarbonLoaderUtil { LOGGER.info("No.Of Blocks after Blocklet distribution: " + tableBlockInfos.size()); return tableBlockInfos; } + + /** + * This will update the old table status details before clean files to the latest table status. + * @param oldList + * @param newList + * @return + */ + public static List<LoadMetadataDetails> updateLoadMetadataFromOldToNew( + LoadMetadataDetails[] oldList, LoadMetadataDetails[] newList) { + + List<LoadMetadataDetails> newListMetadata = + new ArrayList<LoadMetadataDetails>(Arrays.asList(newList)); + for (LoadMetadataDetails oldSegment : oldList) { + if ("false".equalsIgnoreCase(oldSegment.getVisibility())) { + newListMetadata.get(newListMetadata.indexOf(oldSegment)).setVisibility("false"); + } + } + return newListMetadata; + } + } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/816f8e36/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala index d4382eb..b9289ce 100644 --- a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala +++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala @@ -1122,10 +1122,20 @@ object CarbonDataRDDFactory extends Logging { // Update load metadate file after cleaning deleted nodes if (carbonTableStatusLock.lockWithRetries()) { logger.info("Table status lock has been successfully acquired.") + + // read latest table status again. + val latestMetadata = segmentStatusManager + .readLoadMetadata(loadMetadataFilePath) + + // update the metadata details from old to new status. + + val latestStatus = CarbonLoaderUtil + .updateLoadMetadataFromOldToNew(details, latestMetadata) + CarbonLoaderUtil.writeLoadMetadata( carbonLoadModel.getCarbonDataLoadSchema, carbonLoadModel.getDatabaseName, - carbonLoadModel.getTableName, details.toList.asJava + carbonLoadModel.getTableName, latestStatus ) } else {
