This is an automated email from the ASF dual-hosted git repository.
ajantha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 9affb68 Added validation for compacting mixed format segments
9affb68 is described below
commit 9affb688836d41ac873a5c5a024b27bd8be28b82
Author: manishnalla1994 <[email protected]>
AuthorDate: Mon Nov 4 17:46:06 2019 +0530
Added validation for compacting mixed format segments
Custom compaction with ids containing non carbon formats gave error.
Custom compaction with segment ids containing mixed format segments
blocked.
Data size and Index size is not displayed correctly for other format
segments. Data-size is displayed as the size of files of the other
segments and Index-size displayed as NA during show segments for
external segments.
This closes #3432
---
.../core/constants/CarbonCommonConstants.java | 5 +++++
.../carbondata/core/statusmanager/FileFormat.java | 5 +++++
.../testsuite/addsegment/AddSegmentTestCase.scala | 21 +++++++++++++++++++++
.../org/apache/carbondata/api/CarbonStore.scala | 11 ++++++++---
.../CarbonAlterTableCompactionCommand.scala | 17 ++++++++++++++---
5 files changed, 53 insertions(+), 6 deletions(-)
diff --git
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 4a66fa8..da5586e 100644
---
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1701,6 +1701,11 @@ public final class CarbonCommonConstants {
public static final String FACT_FILE_EXT = ".carbondata";
/**
+ * PARQUET_FILE_EXT
+ */
+ public static final String PARQUET_FILE_EXT = ".parquet";
+
+ /**
* DELETE_DELTA_FILE_EXT
*/
public static final String DELETE_DELTA_FILE_EXT = ".deletedelta";
diff --git
a/core/src/main/java/org/apache/carbondata/core/statusmanager/FileFormat.java
b/core/src/main/java/org/apache/carbondata/core/statusmanager/FileFormat.java
index 6f098e2..29f2564 100644
---
a/core/src/main/java/org/apache/carbondata/core/statusmanager/FileFormat.java
+++
b/core/src/main/java/org/apache/carbondata/core/statusmanager/FileFormat.java
@@ -56,6 +56,11 @@ public class FileFormat implements Serializable {
return COLUMNAR_V3;
}
+ public boolean isCarbonFormat() {
+ return (this.format.equalsIgnoreCase("COLUMNAR_V3") ||
+ this.format.equalsIgnoreCase("ROW_V1"));
+ }
+
@Override public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
diff --git
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/addsegment/AddSegmentTestCase.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/addsegment/AddSegmentTestCase.scala
index 8da73e2..50407fc 100644
---
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/addsegment/AddSegmentTestCase.scala
+++
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/addsegment/AddSegmentTestCase.scala
@@ -35,6 +35,9 @@ import org.apache.carbondata.sdk.file.{CarbonReader,
CarbonWriter}
import org.junit.Assert
import scala.io.Source
+import org.apache.carbondata.common.Strings
+import org.apache.carbondata.core.datastore.filesystem.{CarbonFile,
CarbonFileFilter}
+
class AddSegmentTestCase extends QueryTest with BeforeAndAfterAll {
override def beforeAll {
@@ -300,6 +303,11 @@ class AddSegmentTestCase extends QueryTest with
BeforeAndAfterAll {
checkAnswer(sql("select empname from addsegment1 where empname='arvind'"),
Seq(Row("arvind"),Row("arvind")))
checkAnswer(sql("select count(empname) from addsegment1"), Seq(Row(20)))
checkAnswer(sql("select count(*) from addsegment1"), Seq(Row(20)))
+ sql("show segments for table addsegment1").show(100, false)
+ val showSeg = sql("show segments for table addsegment1").collectAsList()
+ val size = getDataSize(newPath)
+ assert(showSeg.get(0).get(6).toString.equalsIgnoreCase(size))
+ assert(showSeg.get(0).get(7).toString.equalsIgnoreCase("NA"))
FileFactory.deleteAllFilesOfDir(new File(newPath))
}
@@ -754,6 +762,19 @@ class AddSegmentTestCase extends QueryTest with
BeforeAndAfterAll {
}
}
+ def getDataSize(path: String): String = {
+ val allFiles = FileFactory.getCarbonFile(path).listFiles(new
CarbonFileFilter {
+ override def accept(file: CarbonFile): Boolean = {
+ file.getName.endsWith(CarbonCommonConstants.PARQUET_FILE_EXT)
+ }
+ })
+ var size: Long = 0
+ for (file <- allFiles) {
+ size += file.getSize
+ }
+ Strings.formatSize(size.toFloat)
+ }
+
override def afterAll = {
dropTable
diff --git
a/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
b/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
index ab8b91a..cac0b11 100644
---
a/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
+++
b/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
@@ -120,9 +120,14 @@ object CarbonStore {
(-1L, -1L)
}
} else {
- // for batch segment, we can get the data size from table status
file directly
- (if (load.getDataSize == null) -1L else load.getDataSize.toLong,
- if (load.getIndexSize == null) -1L else load.getIndexSize.toLong)
+ // If the added segment is other than carbon segment then we can
only display the data
+ // size and not index size, we can get the data size from table
status file directly
+ if (!load.getFileFormat.isCarbonFormat) {
+ (if (load.getIndexSize == null) -1L else
load.getIndexSize.toLong, -1L)
+ } else {
+ (if (load.getDataSize == null) -1L else load.getDataSize.toLong,
+ if (load.getIndexSize == null) -1L else
load.getIndexSize.toLong)
+ }
}
if (showHistory) {
diff --git
a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
index ab6b972..0a0ce19 100644
---
a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
+++
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
@@ -22,7 +22,7 @@ import java.util
import scala.collection.JavaConverters._
-import org.apache.spark.sql.{CarbonEnv, Row, SparkSession, SQLContext}
+import org.apache.spark.sql.{AnalysisException, CarbonEnv, Row, SparkSession,
SQLContext}
import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.execution.command.{AlterTableModel,
AtomicRunnableCommand, CompactionModel}
@@ -42,7 +42,7 @@ import org.apache.carbondata.core.locks.{CarbonLockFactory,
LockUsage}
import org.apache.carbondata.core.metadata.ColumnarFormatVersion
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable,
TableInfo}
import org.apache.carbondata.core.mutate.CarbonUpdateUtil
-import org.apache.carbondata.core.statusmanager.SegmentStatusManager
+import org.apache.carbondata.core.statusmanager.{FileFormat,
SegmentStatusManager}
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.core.util.CarbonUtil
import org.apache.carbondata.core.util.path.CarbonTablePath
@@ -276,7 +276,18 @@ case class CarbonAlterTableCompactionCommand(
val isCompactionTriggerByDDl = true
val segmentIds: Option[List[String]] = if (compactionType ==
CompactionType.CUSTOM &&
alterTableModel.customSegmentIds.isDefined) {
- alterTableModel.customSegmentIds
+ val ids = alterTableModel.customSegmentIds
+ ids match {
+ case Some(x) =>
+ val loadMetadataDetails =
carbonLoadModel.getLoadMetadataDetails.asScala
+ val otherLoadDetails = loadMetadataDetails
+ .exists(p => x.exists(a => a.equalsIgnoreCase(p.getLoadName)) &&
!p.isCarbonFormat)
+ if (otherLoadDetails) {
+ throw new AnalysisException(s"Custom compaction does not support
other format segment")
+ }
+ case None =>
+ }
+ ids
} else {
None
}
