Repository: carbondata Updated Branches: refs/heads/master 878bbd8a4 -> b0aee53f5
[CARBONDATA-2740]segment file is not getting deleted if load fails and correct the error message for local dictionary validation Problem error message is wrong when complex column which does not have any string datatype column is given in local dictionary exclude column. when data load is failed, corresponding segment file is not getting deleted. Solution: error message is corrected when data load is failed, segment file will be deleted This closes #2514 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b0aee53f Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b0aee53f Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b0aee53f Branch: refs/heads/master Commit: b0aee53f52eb2ac2741834a94874e0ed4a0955e0 Parents: 878bbd8 Author: akashrn5 <[email protected]> Authored: Tue Jul 17 11:05:08 2018 +0530 Committer: kunal642 <[email protected]> Committed: Wed Jul 25 10:24:27 2018 +0530 ---------------------------------------------------------------------- .../LocalDictionarySupportCreateTableTest.scala | 6 +++--- .../org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala | 5 +++-- .../org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala | 5 +++++ .../processing/store/CarbonFactDataHandlerModel.java | 1 + 4 files changed, 12 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/b0aee53f/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala index d865b2a..eec1582 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala @@ -2165,7 +2165,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } assert(exception.getMessage .contains( - "None of the child columns specified in the complex dataType column(s) in " + + "None of the child columns of complex dataType column st specified in " + "local_dictionary_include are not of string dataType.")) } @@ -2183,7 +2183,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } assert(exception.getMessage .contains( - "None of the child columns specified in the complex dataType column(s) in " + + "None of the child columns of complex dataType column city specified in " + "local_dictionary_include are not of string dataType.")) } @@ -2322,7 +2322,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte } assert(exception.getMessage .contains( - "None of the child columns specified in the complex dataType column(s) in " + + "None of the child columns of complex dataType column city specified in " + "local_dictionary_include are not of string dataType.")) } http://git-wip-us.apache.org/repos/asf/carbondata/blob/b0aee53f/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala index 6f8ab63..2599b3f 100644 --- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala +++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala @@ -491,8 +491,9 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser { .exists(x => x.column.equalsIgnoreCase(dictColm) && x.children.isDefined && null != x.children.get && !validateChildColumnsRecursively(x))) { - val errMsg = "None of the child columns specified in the complex dataType column(s) in " + - "local_dictionary_include are not of string dataType." + val errMsg = + s"None of the child columns of complex dataType column $dictColm specified in " + + "local_dictionary_include are not of string dataType." throw new MalformedCarbonCommandException(errMsg) } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/b0aee53f/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala index 804193c..b723968 100644 --- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala +++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala @@ -17,6 +17,7 @@ package org.apache.carbondata.spark.rdd +import java.io.File import java.text.SimpleDateFormat import java.util import java.util.TimeZone @@ -558,6 +559,10 @@ object CarbonDataRDDFactory { if (carbonLoadModel.isCarbonTransactionalTable) { // delete segment is applicable for transactional table CarbonLoaderUtil.deleteSegment(carbonLoadModel, carbonLoadModel.getSegmentId.toInt) + // delete corresponding segment file from metadata + val segmentFile = CarbonTablePath.getSegmentFilesLocation(carbonLoadModel.getTablePath) + + File.separator + segmentFileName + FileFactory.deleteFile(segmentFile, FileFactory.getFileType(segmentFile)) clearDataMapFiles(carbonTable, carbonLoadModel.getSegmentId) } LOGGER.info("********clean up done**********") http://git-wip-us.apache.org/repos/asf/carbondata/blob/b0aee53f/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java index a0483fe..767b409 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java +++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java @@ -334,6 +334,7 @@ public class CarbonFactDataHandlerModel { carbonFactDataHandlerModel.setStoreLocation(tempStoreLocation); carbonFactDataHandlerModel.setDimLens(segmentProperties.getDimColumnsCardinality()); carbonFactDataHandlerModel.setSegmentProperties(segmentProperties); + carbonFactDataHandlerModel.setSegmentId(loadModel.getSegmentId()); carbonFactDataHandlerModel .setNoDictionaryCount(segmentProperties.getNumberOfNoDictionaryDimension()); carbonFactDataHandlerModel.setDimensionCount(
