Repository: carbondata Updated Branches: refs/heads/master 26d9f3d8e -> 9336924de
[CARBONDATA-2804] fix the bug when bloom filter or preaggregate datamap tried to be created on older V1-V2 version stores This PR change read carbon file version from carbondata file header to carbonindex file header, because the version filed of carondata file header is not compatible with older v1/v2 version. This closes #2601 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/9336924d Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/9336924d Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/9336924d Branch: refs/heads/master Commit: 9336924de3f3103507335b5ffbf9c0145be4812f Parents: 26d9f3d Author: ndwangsen <[email protected]> Authored: Thu Aug 2 16:21:22 2018 +0800 Committer: xuchuanyin <[email protected]> Committed: Fri Aug 3 15:25:46 2018 +0800 ---------------------------------------------------------------------- .../apache/carbondata/core/util/CarbonUtil.java | 85 ++++++++++++++------ 1 file changed, 59 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/9336924d/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java index 3eb6aae..205d160 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java @@ -43,6 +43,7 @@ import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter; import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.exception.InvalidConfigurationException; import org.apache.carbondata.core.indexstore.BlockletDetailInfo; +import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore; import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor; import org.apache.carbondata.core.localdictionary.generator.ColumnLocalDictionaryGenerator; import org.apache.carbondata.core.localdictionary.generator.LocalDictionaryGenerator; @@ -88,7 +89,7 @@ import org.apache.carbondata.core.util.path.CarbonTablePath; import org.apache.carbondata.format.BlockletHeader; import org.apache.carbondata.format.DataChunk2; import org.apache.carbondata.format.DataChunk3; -import org.apache.carbondata.format.FileHeader; +import org.apache.carbondata.format.IndexHeader; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -3194,11 +3195,33 @@ public final class CarbonUtil { */ public static ColumnarFormatVersion getFormatVersion(CarbonTable carbonTable) throws IOException { - String storePath = null; - // if the carbontable is support flat folder + String segmentPath = null; boolean supportFlatFolder = carbonTable.isSupportFlatFolder(); + CarbonIndexFileReader indexReader = new CarbonIndexFileReader(); + ColumnarFormatVersion version = null; + SegmentIndexFileStore fileStore = new SegmentIndexFileStore(); + CarbonProperties carbonProperties = CarbonProperties.getInstance(); + // if the carbontable is support flat folder if (supportFlatFolder) { - storePath = carbonTable.getTablePath(); + segmentPath = carbonTable.getTablePath(); + FileFactory.FileType fileType = FileFactory.getFileType(segmentPath); + if (FileFactory.isFileExist(segmentPath, fileType)) { + fileStore.readAllIIndexOfSegment(segmentPath); + Map<String, byte[]> carbonIndexMap = fileStore.getCarbonIndexMap(); + if (carbonIndexMap.size() == 0) { + version = carbonProperties.getFormatVersion(); + } + for (byte[] fileData : carbonIndexMap.values()) { + try { + indexReader.openThriftReader(fileData); + IndexHeader indexHeader = indexReader.readIndexHeader(); + version = ColumnarFormatVersion.valueOf((short)indexHeader.getVersion()); + break; + } finally { + indexReader.closeThriftReader(); + } + } + } } else { // get the valid segments SegmentStatusManager segmentStatusManager = @@ -3206,34 +3229,44 @@ public final class CarbonUtil { SegmentStatusManager.ValidAndInvalidSegmentsInfo validAndInvalidSegmentsInfo = segmentStatusManager.getValidAndInvalidSegments(); List<Segment> validSegments = validAndInvalidSegmentsInfo.getValidSegments(); - CarbonProperties carbonProperties = CarbonProperties.getInstance(); if (validSegments.isEmpty()) { return carbonProperties.getFormatVersion(); } - storePath = carbonTable.getSegmentPath(validSegments.get(0).getSegmentNo()); - } - - CarbonFile[] carbonFiles = FileFactory - .getCarbonFile(storePath) - .listFiles(new CarbonFileFilter() { - @Override - public boolean accept(CarbonFile file) { - if (file == null) { - return false; + // get the carbon index file header from a valid segment + for (Segment segment : validSegments) { + segmentPath = carbonTable.getSegmentPath(segment.getSegmentNo()); + FileFactory.FileType fileType = FileFactory.getFileType(segmentPath); + if (FileFactory.isFileExist(segmentPath, fileType)) { + fileStore.readAllIIndexOfSegment(segmentPath); + Map<String, byte[]> carbonIndexMap = fileStore.getCarbonIndexMap(); + if (carbonIndexMap.size() == 0) { + LOGGER.warn("the valid segment path: " + segmentPath + + " does not exist in the system of table: " + carbonTable.getTableUniqueName()); + continue; + } + for (byte[] fileData : carbonIndexMap.values()) { + try { + indexReader.openThriftReader(fileData); + IndexHeader indexHeader = indexReader.readIndexHeader(); + version = ColumnarFormatVersion.valueOf((short)indexHeader.getVersion()); + break; + } finally { + indexReader.closeThriftReader(); } - return file.getName().endsWith("carbondata"); } - }); - if (carbonFiles == null || carbonFiles.length < 1) { - return CarbonProperties.getInstance().getFormatVersion(); + // if get the carbon file version from a valid segment, then end + if (version != null) { + break; + } + } + } + // if all valid segments path does not in the system, + // then the carbon file verion as default + if (version == null) { + version = CarbonProperties.getInstance().getFormatVersion(); + } } - - CarbonFile carbonFile = carbonFiles[0]; - // get the carbon file header - CarbonHeaderReader headerReader = new CarbonHeaderReader(carbonFile.getCanonicalPath()); - FileHeader fileHeader = headerReader.readHeader(); - int version = fileHeader.getVersion(); - return ColumnarFormatVersion.valueOf((short)version); + return version; } /**
