Github user kumarvishal09 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/3029#discussion_r244713354
--- Diff:
processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
---
@@ -400,24 +417,53 @@ private static int
getDimensionDefaultCardinality(CarbonDimension dimension) {
* @param tableLastUpdatedTime
* @return
*/
- public static boolean checkIfAnyRestructuredBlockExists(Map<String,
TaskBlockInfo> segmentMapping,
- Map<String, List<DataFileFooter>> dataFileMetadataSegMapping, long
tableLastUpdatedTime) {
- boolean restructuredBlockExists = false;
- for (Map.Entry<String, TaskBlockInfo> taskMap :
segmentMapping.entrySet()) {
- String segmentId = taskMap.getKey();
+ public static boolean checkIfAnyRestructuredBlockExists(
+ Map<String, TaskBlockInfo> segmentMapping,
+ Map<String, List<DataFileFooter>> dataFileMetadataSegMapping,
+ long tableLastUpdatedTime) {
+
+ for (Map.Entry<String, TaskBlockInfo> segmentEntry :
segmentMapping.entrySet()) {
+ String segmentId = segmentEntry.getKey();
List<DataFileFooter> listMetadata =
dataFileMetadataSegMapping.get(segmentId);
- for (DataFileFooter dataFileFooter : listMetadata) {
- // if schema modified timestamp is greater than footer stored
schema timestamp,
- // it indicates it is a restructured block
- if (tableLastUpdatedTime >
dataFileFooter.getSchemaUpdatedTimeStamp()) {
- restructuredBlockExists = true;
- break;
- }
+
+ if (isRestructured(listMetadata, tableLastUpdatedTime)) {
+ return true;
}
- if (restructuredBlockExists) {
- break;
+ }
+
+ return false;
+ }
+
+ public static boolean isRestructured(List<DataFileFooter> listMetadata,
+ long tableLastUpdatedTime) {
+ /*
+ * TODO: only in case of add and drop this variable should be true
+ */
+ for (DataFileFooter dataFileFooter : listMetadata) {
+ // if schema modified timestamp is greater than footer stored schema
timestamp,
+ // it indicates it is a restructured block
+ if (tableLastUpdatedTime >
dataFileFooter.getSchemaUpdatedTimeStamp()) {
+ return true;
}
}
- return restructuredBlockExists;
+ return false;
}
+
+ public static boolean isSorted(TaskBlockInfo taskBlockInfo) throws
IOException {
--- End diff --
Use CarbonUtil.readMetadataFile(tableblockInfo) for reading file footer
...if carbondata file version is V1/V2 this code will not able to read the file
footer
---
