This is an automated email from the ASF dual-hosted git repository. ajantha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push: new 1954551 [CARBONDATA-3601] Show Segments displays wrong Index size for Partition table with Merge Index Enabled 1954551 is described below commit 1954551b0484421b025d3f3dfc8190673d6e48a7 Author: Indhumathi27 <indhumathi...@gmail.com> AuthorDate: Wed Dec 4 12:17:37 2019 +0530 [CARBONDATA-3601] Show Segments displays wrong Index size for Partition table with Merge Index Enabled Problem: After merging index for partition table, new LoadMetaData entry is not updated with merge index file data size. Solution: Update index size on new LoadMetaDataEntry after merge index on partitioned table This closes #3496 --- .../hadoop/api/CarbonOutputCommitter.java | 9 ++++++ .../StandardPartitionTableLoadingTestCase.scala | 33 ++++++++++++++++++++++ .../processing/util/CarbonLoaderUtil.java | 15 ++++++++++ 3 files changed, 57 insertions(+) diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java index 549ca7c..3eda548 100644 --- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java +++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java @@ -39,6 +39,7 @@ import org.apache.carbondata.core.mutate.CarbonUpdateUtil; import org.apache.carbondata.core.statusmanager.LoadMetadataDetails; import org.apache.carbondata.core.statusmanager.SegmentStatus; import org.apache.carbondata.core.statusmanager.SegmentStatusManager; +import org.apache.carbondata.core.util.CarbonProperties; import org.apache.carbondata.core.util.CarbonSessionInfo; import org.apache.carbondata.core.util.ThreadLocalSessionInfo; import org.apache.carbondata.core.util.path.CarbonTablePath; @@ -168,6 +169,14 @@ public class CarbonOutputCommitter extends FileOutputCommitter { throw new IOException(e); } } + // After merging index, update newMetaEntry with updated merge index size + boolean isMergeIndexEnabled = Boolean.parseBoolean(CarbonProperties.getInstance() + .getProperty(CarbonCommonConstants.CARBON_MERGE_INDEX_IN_SEGMENT, + CarbonCommonConstants.CARBON_MERGE_INDEX_IN_SEGMENT_DEFAULT)); + if (isMergeIndexEnabled) { + CarbonLoaderUtil + .addIndexSizeIntoMetaEntry(newMetaEntry, loadModel.getSegmentId(), carbonTable); + } String uniqueId = null; if (overwriteSet) { if (!loadModel.isCarbonTransactionalTable()) { diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala index bee118a..79f2cf9 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala @@ -29,7 +29,10 @@ import org.apache.spark.sql.test.util.QueryTest import org.apache.spark.sql.{AnalysisException, CarbonEnv, Row} import org.scalatest.BeforeAndAfterAll +import org.apache.carbondata.common.Strings import org.apache.carbondata.core.constants.CarbonCommonConstants +import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter} +import org.apache.carbondata.core.datastore.impl.FileFactory import org.apache.carbondata.core.metadata.{CarbonMetadata, SegmentFileStore} import org.apache.carbondata.core.statusmanager.SegmentStatusManager import org.apache.carbondata.core.util.CarbonProperties @@ -503,7 +506,37 @@ class StandardPartitionTableLoadingTestCase extends QueryTest with BeforeAndAfte assert(new File(s"$storeLocation/new_par/Metadata/segments/").listFiles().size == 1) } + test("test index and data size after merge index on partition table") { + sql("drop table if exists new_par") + sql("create table new_par(a int) partitioned by (b string) stored by 'carbondata'") + sql("insert into new_par select 1,'k'") + val result = sql("show segments for table new_par").collectAsList() + val dataAndIndexSize = getDataAndIndexSize(s"$storeLocation/new_par/b=k") + assert(result.get(0).get(6).equals(dataAndIndexSize._1)) + assert(result.get(0).get(7).equals(dataAndIndexSize._2)) + } + def getDataAndIndexSize(path: String): (String, String) = { + val mergeIndexFiles = FileFactory.getCarbonFile(path).listFiles(new CarbonFileFilter { + override def accept(file: CarbonFile): Boolean = { + file.getName.endsWith(CarbonTablePath.MERGE_INDEX_FILE_EXT) + } + }) + val dataFiles = FileFactory.getCarbonFile(path).listFiles(new CarbonFileFilter { + override def accept(file: CarbonFile): Boolean = { + file.getName.endsWith(CarbonTablePath.CARBON_DATA_EXT) + } + }) + var indexSize: Long = 0 + for (file <- mergeIndexFiles) { + indexSize += file.getSize + } + var dataSize: Long = 0 + for (file <- dataFiles) { + dataSize += file.getSize + } + (Strings.formatSize(dataSize.toFloat), Strings.formatSize(indexSize.toFloat)) + } def restoreData(dblocation: String, tableName: String) = { val destination = dblocation + CarbonCommonConstants.FILE_SEPARATOR + tableName diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java index 60c5abe..1a40ff4 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java +++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java @@ -47,6 +47,7 @@ import org.apache.carbondata.core.locks.CarbonLockUtil; import org.apache.carbondata.core.locks.ICarbonLock; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.metadata.ColumnIdentifier; +import org.apache.carbondata.core.metadata.SegmentFileStore; import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.schema.table.CarbonTable; import org.apache.carbondata.core.mutate.CarbonUpdateUtil; @@ -1170,6 +1171,20 @@ public final class CarbonLoaderUtil { return dataSize + indexSize; } + public static void addIndexSizeIntoMetaEntry(LoadMetadataDetails loadMetadataDetails, + String segmentId, CarbonTable carbonTable) throws IOException { + Segment segment = new Segment(segmentId, loadMetadataDetails.getSegmentFile()); + if (segment.getSegmentFileName() != null) { + SegmentFileStore fileStore = + new SegmentFileStore(carbonTable.getTablePath(), segment.getSegmentFileName()); + if (fileStore.getLocationMap() != null) { + fileStore.readIndexFiles(FileFactory.getConfiguration()); + long carbonIndexSize = CarbonUtil.getCarbonIndexSize(fileStore, fileStore.getLocationMap()); + loadMetadataDetails.setIndexSize(String.valueOf(carbonIndexSize)); + } + } + } + /** * Merge index files with in the segment of partitioned table *