This is an automated email from the ASF dual-hosted git repository.
akashrn5 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new e643747 [CARBONDATA-3766] Fixed desc formatted and show segment data
size issues
e643747 is described below
commit e6437479fa11eef9a8fc03c2572905bee458c5ad
Author: kunal642 <[email protected]>
AuthorDate: Thu Mar 26 14:55:34 2020 +0530
[CARBONDATA-3766] Fixed desc formatted and show segment data size issues
Why is this PR needed?
Show segments shows the datasize as index size for non carbon segments in
show segments command.
What changes were proposed in this PR?
Take data size from indexsize variable and show in show segments as non
carbon segmnets do not have
index files. Index Size wil be shown as 0.
This closes #3680
---
.../carbondata/core/metadata/SegmentFileStore.java | 3 ++-
.../java/org/apache/carbondata/core/util/CarbonUtil.java | 16 ++++++++++++----
.../scala/org/apache/carbondata/api/CarbonStore.scala | 2 +-
.../spark/testsuite/addsegment/AddSegmentTestCase.scala | 2 ++
4 files changed, 17 insertions(+), 6 deletions(-)
diff --git
a/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
b/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
index afb76bd..7aa2843 100644
---
a/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
+++
b/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
@@ -527,10 +527,11 @@ public class SegmentFileStore {
for (LoadMetadataDetails detail : listOfLoadFolderDetailsArray) {
// if the segments is in the list of marked for delete then update
the status.
if (segmentId.equals(detail.getLoadName())) {
+ detail.setLoadEndTime(System.currentTimeMillis());
detail.setSegmentFile(segmentFile);
if (segmentStatus != null) {
HashMap<String, Long> dataSizeAndIndexSize =
- CarbonUtil.getDataSizeAndIndexSize(segmentFileStore);
+ CarbonUtil.getDataSizeAndIndexSize(segmentFileStore,
detail.isCarbonFormat());
detail.setDataSize(
dataSizeAndIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE)
.toString());
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index d00da15..7917ddd 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -2514,7 +2514,8 @@ public final class CarbonUtil {
}
// Get the total size of carbon data and the total size of carbon index
- public static HashMap<String, Long> getDataSizeAndIndexSize(SegmentFileStore
fileStore)
+ public static HashMap<String, Long> getDataSizeAndIndexSize(SegmentFileStore
fileStore,
+ boolean isCarbonSegment)
throws IOException {
long carbonDataSize = 0L;
long carbonIndexSize = 0L;
@@ -2532,8 +2533,15 @@ public final class CarbonUtil {
}
}
}
- dataAndIndexSize.put(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE,
carbonDataSize);
- dataAndIndexSize.put(CarbonCommonConstants.CARBON_TOTAL_INDEX_SIZE,
carbonIndexSize);
+ if (isCarbonSegment) {
+ dataAndIndexSize.put(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE,
carbonDataSize);
+ dataAndIndexSize.put(CarbonCommonConstants.CARBON_TOTAL_INDEX_SIZE,
carbonIndexSize);
+ } else {
+ // In case of non-carbon segments all the dataSize is collected in
carbonIndexSize because of
+ // listing behaviour of the SegmentFileStore.
+ dataAndIndexSize.put(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE,
carbonIndexSize);
+ dataAndIndexSize.put(CarbonCommonConstants.CARBON_TOTAL_INDEX_SIZE, 0L);
+ }
return dataAndIndexSize;
}
@@ -2582,7 +2590,7 @@ public final class CarbonUtil {
Segment segment) throws IOException {
if (segment.getSegmentFileName() != null) {
SegmentFileStore fileStore = new SegmentFileStore(tablePath,
segment.getSegmentFileName());
- return getDataSizeAndIndexSize(fileStore);
+ return getDataSizeAndIndexSize(fileStore, segment.isCarbonSegment());
} else {
return getDataSizeAndIndexSize(tablePath, segment.getSegmentNo());
}
diff --git
a/integration/spark/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
b/integration/spark/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
index 5571656..275d260 100644
---
a/integration/spark/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
+++
b/integration/spark/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
@@ -123,7 +123,7 @@ object CarbonStore {
// If the added segment is other than carbon segment then we can
only display the data
// size and not index size, we can get the data size from table
status file directly
if (!load.getFileFormat.isCarbonFormat) {
- (if (load.getIndexSize == null) -1L else
load.getIndexSize.toLong, -1L)
+ (if (load.getDataSize == null) -1L else load.getDataSize.toLong,
-1L)
} else {
(if (load.getDataSize == null) -1L else load.getDataSize.toLong,
if (load.getIndexSize == null) -1L else
load.getIndexSize.toLong)
diff --git
a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/addsegment/AddSegmentTestCase.scala
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/addsegment/AddSegmentTestCase.scala
index 13472cc..6c41abd 100644
---
a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/addsegment/AddSegmentTestCase.scala
+++
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/addsegment/AddSegmentTestCase.scala
@@ -230,7 +230,9 @@ class AddSegmentTestCase extends QueryTest with
BeforeAndAfterAll {
checkAnswer(sql("select count(*) from addsegment1"), Seq(Row(20)))
sql("show segments for table addsegment1").show(100, false)
val showSeg = sql("show segments for table addsegment1").collectAsList()
+ val descFormattedSize = sql("desc formatted
addsegment1").collect().filter(_.get(0).toString.startsWith("Table Data
Size")).head.get(1).toString
val size = getDataSize(newPath)
+ assert(descFormattedSize.split("KB")(0).toDouble > 0.0d)
assert(showSeg.get(0).get(6).toString.equalsIgnoreCase(size))
assert(showSeg.get(0).get(7).toString.equalsIgnoreCase("NA"))
FileFactory.deleteAllFilesOfDir(new File(newPath))
