This is an automated email from the ASF dual-hosted git repository.
indhumuthumurugesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new b8511b6279 [CARBONDATA-4345] update/delete operations failed when
other format segemnt deleted from carbon table
b8511b6279 is described below
commit b8511b6279fb86f000c70ad00b109aba2ea2501a
Author: Mahesh Raju Somalaraju <[email protected]>
AuthorDate: Tue Jun 28 17:03:39 2022 +0530
[CARBONDATA-4345] update/delete operations failed when other format segemnt
deleted from carbon table
Why is this PR needed?
Update/delete operations failed when other format segments deleted from
carbon table
Steps to reproduce:
1. create carbon table and load the data
2. create parquet/orc tables and load the data
3. add parquet/orc format segments in carbon table by alter add segment
command
4. perform update/delete operations in carbon table and they will fail as
table
contains mixed format segments. This is expected behaviour only.
5. delete the other format segments which is added in step3
6. try to perform update/delete operation in carbon data. They should not
fail
For update/delete operations we are checking if other format segments
present
in table path. If found then carbon data throwing exception by saying mixed
format segments exists even though the other format segments deleted from
table.
What changes were proposed in this PR?
When we are checking other format segment present in carbon table then it
should check only for SUCCESS/PARTIAL_SUCCESS segments.
Does this PR introduce any user interface change?
No
Is any new testcase added?
Yes
This closes #4285
---
.../execution/strategy/MixedFormatHandler.scala | 4 +-
.../testsuite/addsegment/AddSegmentTestCase.scala | 46 ++++++++++++++++++++++
2 files changed, 49 insertions(+), 1 deletion(-)
diff --git
a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/MixedFormatHandler.scala
b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/MixedFormatHandler.scala
index 330cbfc472..30233b8f33 100644
---
a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/MixedFormatHandler.scala
+++
b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/MixedFormatHandler.scala
@@ -399,6 +399,8 @@ object MixedFormatHandler {
*/
def otherFormatSegmentsExist(metadataPath: String, tableStatusVersion:
String): Boolean = {
val allSegments = SegmentStatusManager.readLoadMetadata(metadataPath,
tableStatusVersion)
- allSegments.exists(a => a.getFileFormat != null && !a.isCarbonFormat)
+ allSegments.exists(a => (a.getSegmentStatus == SegmentStatus.SUCCESS ||
+ a.getSegmentStatus ==
SegmentStatus.LOAD_PARTIAL_SUCCESS) &&
+ a.getFileFormat != null && !a.isCarbonFormat)
}
}
diff --git
a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/addsegment/AddSegmentTestCase.scala
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/addsegment/AddSegmentTestCase.scala
index 0fd798be18..f4ab61a605 100644
---
a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/addsegment/AddSegmentTestCase.scala
+++
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/addsegment/AddSegmentTestCase.scala
@@ -1190,6 +1190,52 @@ class AddSegmentTestCase extends QueryTest with
BeforeAndAfterAll {
assert(ex.getMessage.contains("PATH cannot be empty"))
}
+ test("Test delete/update on a carbon table after deleting all non-carbon
format segments") {
+ createCarbonTable()
+ createParquetTable()
+ createOrcTable()
+ // parquet files
+ val parquet_path =
SparkSQLUtil.sessionState(sqlContext.sparkSession).catalog
+ .getTableMetadata(TableIdentifier("addsegment2")).location
+ val parquet_newPath = copyseg("addsegment2", "parquet_addsegtest1")
+ checkAnswer(sql("select count(*) from addsegment2"), Seq(Row(10)))
+ FileFactory.deleteAllFilesOfDir(new File(parquet_newPath))
+ CarbonTestUtil.copy(parquet_path.toString, parquet_newPath)
+ // orc files
+ val orc_path = SparkSQLUtil.sessionState(sqlContext.sparkSession).catalog
+ .getTableMetadata(TableIdentifier("addsegment3")).location
+ val orc_newPath = copyseg("addsegment3", "orc_addsegtest1")
+ checkAnswer(sql("select count(*) from addsegment3"), Seq(Row(10)))
+ FileFactory.deleteAllFilesOfDir(new File(orc_newPath))
+ CarbonTestUtil.copy(orc_path.toString, orc_newPath)
+ // check data before performing alter
+ checkAnswer(sql("select count(empname) from addsegment1"), Seq(Row(10)))
+ checkAnswer(sql("select count(*) from addsegment1 where empname='ravi'"),
Seq(Row(0)))
+ sql("alter table addsegment1 add segment " +
+ s"options('path'='$parquet_newPath', 'format'='parquet')").collect()
+ sql(s"alter table addsegment1 add segment options('path'='$orc_newPath',
'format'='orc')")
+ .collect()
+ val exception1 = intercept[MalformedCarbonCommandException](sql(
+ """update addsegment1 d set (d.empname) = ('ravi') where d.empname =
'arvind'""").collect())
+ assertResult("Unsupported update operation on table containing mixed
format segments")(
+ exception1.getMessage())
+ val exception2 = intercept[MalformedCarbonCommandException](sql(
+ "delete from addsegment1 where deptno = 10"))
+ assertResult("Unsupported delete operation on table containing mixed
format segments")(
+ exception2.getMessage())
+ // update operation should be successful after deleting other formats
segments
+ sql("delete from table addsegment1 where segment.id in (1,2)")
+ sql(
+ """update addsegment1 d set (d.empname) = ('ravi') where d.empname =
'arvind'""").collect()
+ checkAnswer(sql("select count(empname) from addsegment1"), Seq(Row(10)))
+ checkAnswer(sql("select count(*) from addsegment1 where empname='ravi'"),
Seq(Row(1)))
+ sql("delete from addsegment1 where deptno = 11")
+ checkAnswer(sql("select count(*) from addsegment1 where deptno = 11"),
Seq(Row(0)))
+ checkAnswer(sql("select count(empname) from addsegment1"), Seq(Row(8)))
+ FileFactory.deleteAllFilesOfDir(new File(parquet_newPath))
+ FileFactory.deleteAllFilesOfDir(new File(orc_newPath))
+ }
+
def getDataSize(path: String): String = {
val allFiles = FileFactory.getCarbonFile(path).listFiles(new
CarbonFileFilter {
override def accept(file: CarbonFile): Boolean = {
