This is an automated email from the ASF dual-hosted git repository.
ajantha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 1954551 [CARBONDATA-3601] Show Segments displays wrong Index size for
Partition table with Merge Index Enabled
1954551 is described below
commit 1954551b0484421b025d3f3dfc8190673d6e48a7
Author: Indhumathi27 <[email protected]>
AuthorDate: Wed Dec 4 12:17:37 2019 +0530
[CARBONDATA-3601] Show Segments displays wrong Index size for Partition
table with Merge Index Enabled
Problem:
After merging index for partition table, new LoadMetaData entry is not
updated with merge index file data size.
Solution:
Update index size on new LoadMetaDataEntry after merge index on
partitioned table
This closes #3496
---
.../hadoop/api/CarbonOutputCommitter.java | 9 ++++++
.../StandardPartitionTableLoadingTestCase.scala | 33 ++++++++++++++++++++++
.../processing/util/CarbonLoaderUtil.java | 15 ++++++++++
3 files changed, 57 insertions(+)
diff --git
a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
index 549ca7c..3eda548 100644
---
a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
+++
b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
@@ -39,6 +39,7 @@ import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
import org.apache.carbondata.core.statusmanager.SegmentStatus;
import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonSessionInfo;
import org.apache.carbondata.core.util.ThreadLocalSessionInfo;
import org.apache.carbondata.core.util.path.CarbonTablePath;
@@ -168,6 +169,14 @@ public class CarbonOutputCommitter extends
FileOutputCommitter {
throw new IOException(e);
}
}
+ // After merging index, update newMetaEntry with updated merge index size
+ boolean isMergeIndexEnabled =
Boolean.parseBoolean(CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.CARBON_MERGE_INDEX_IN_SEGMENT,
+ CarbonCommonConstants.CARBON_MERGE_INDEX_IN_SEGMENT_DEFAULT));
+ if (isMergeIndexEnabled) {
+ CarbonLoaderUtil
+ .addIndexSizeIntoMetaEntry(newMetaEntry, loadModel.getSegmentId(),
carbonTable);
+ }
String uniqueId = null;
if (overwriteSet) {
if (!loadModel.isCarbonTransactionalTable()) {
diff --git
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
index bee118a..79f2cf9 100644
---
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
+++
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
@@ -29,7 +29,10 @@ import org.apache.spark.sql.test.util.QueryTest
import org.apache.spark.sql.{AnalysisException, CarbonEnv, Row}
import org.scalatest.BeforeAndAfterAll
+import org.apache.carbondata.common.Strings
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.filesystem.{CarbonFile,
CarbonFileFilter}
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.metadata.{CarbonMetadata, SegmentFileStore}
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
import org.apache.carbondata.core.util.CarbonProperties
@@ -503,7 +506,37 @@ class StandardPartitionTableLoadingTestCase extends
QueryTest with BeforeAndAfte
assert(new
File(s"$storeLocation/new_par/Metadata/segments/").listFiles().size == 1)
}
+ test("test index and data size after merge index on partition table") {
+ sql("drop table if exists new_par")
+ sql("create table new_par(a int) partitioned by (b string) stored by
'carbondata'")
+ sql("insert into new_par select 1,'k'")
+ val result = sql("show segments for table new_par").collectAsList()
+ val dataAndIndexSize = getDataAndIndexSize(s"$storeLocation/new_par/b=k")
+ assert(result.get(0).get(6).equals(dataAndIndexSize._1))
+ assert(result.get(0).get(7).equals(dataAndIndexSize._2))
+ }
+ def getDataAndIndexSize(path: String): (String, String) = {
+ val mergeIndexFiles = FileFactory.getCarbonFile(path).listFiles(new
CarbonFileFilter {
+ override def accept(file: CarbonFile): Boolean = {
+ file.getName.endsWith(CarbonTablePath.MERGE_INDEX_FILE_EXT)
+ }
+ })
+ val dataFiles = FileFactory.getCarbonFile(path).listFiles(new
CarbonFileFilter {
+ override def accept(file: CarbonFile): Boolean = {
+ file.getName.endsWith(CarbonTablePath.CARBON_DATA_EXT)
+ }
+ })
+ var indexSize: Long = 0
+ for (file <- mergeIndexFiles) {
+ indexSize += file.getSize
+ }
+ var dataSize: Long = 0
+ for (file <- dataFiles) {
+ dataSize += file.getSize
+ }
+ (Strings.formatSize(dataSize.toFloat),
Strings.formatSize(indexSize.toFloat))
+ }
def restoreData(dblocation: String, tableName: String) = {
val destination = dblocation + CarbonCommonConstants.FILE_SEPARATOR +
tableName
diff --git
a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
index 60c5abe..1a40ff4 100644
---
a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
+++
b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
@@ -47,6 +47,7 @@ import org.apache.carbondata.core.locks.CarbonLockUtil;
import org.apache.carbondata.core.locks.ICarbonLock;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.ColumnIdentifier;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
@@ -1170,6 +1171,20 @@ public final class CarbonLoaderUtil {
return dataSize + indexSize;
}
+ public static void addIndexSizeIntoMetaEntry(LoadMetadataDetails
loadMetadataDetails,
+ String segmentId, CarbonTable carbonTable) throws IOException {
+ Segment segment = new Segment(segmentId,
loadMetadataDetails.getSegmentFile());
+ if (segment.getSegmentFileName() != null) {
+ SegmentFileStore fileStore =
+ new SegmentFileStore(carbonTable.getTablePath(),
segment.getSegmentFileName());
+ if (fileStore.getLocationMap() != null) {
+ fileStore.readIndexFiles(FileFactory.getConfiguration());
+ long carbonIndexSize = CarbonUtil.getCarbonIndexSize(fileStore,
fileStore.getLocationMap());
+ loadMetadataDetails.setIndexSize(String.valueOf(carbonIndexSize));
+ }
+ }
+ }
+
/**
* Merge index files with in the segment of partitioned table
*
