Repository: carbondata Updated Branches: refs/heads/metadata 8d3c9bf1b -> 39d7a237f
[CARBONDATA-1311] Added carbon storelocation to spark warehouse. And extracted storelocation out of metastore Changed default storelocation to sparkwouse, ,so if user does not provide storelocation then it chooses sparkwarelocation as store location. Changed file metastore and avoid reading all schema files once keep it in memory, instead implemented cache based storage where it reads when request comes. Extracted store location out of metastore and refactored carbonmetastore. This closes #1176 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/39d7a237 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/39d7a237 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/39d7a237 Branch: refs/heads/metadata Commit: 39d7a237f066d223e75e4aaed03b41eba6c96cbd Parents: 8d3c9bf Author: Ravindra Pesala <[email protected]> Authored: Fri Jul 14 11:12:57 2017 +0530 Committer: Jacky Li <[email protected]> Committed: Wed Jul 19 16:41:59 2017 +0800 ---------------------------------------------------------------------- .../core/constants/CarbonCommonConstants.java | 2 +- .../core/metadata/AbsoluteTableIdentifier.java | 4 ++-- .../core/metadata/schema/BucketingInfo.java | 4 +++- .../core/metadata/schema/PartitionInfo.java | 13 ++++++++---- .../core/metadata/schema/SchemaEvolution.java | 4 +++- .../core/metadata/schema/table/TableSchema.java | 7 +++++-- .../carbondata/core/util/CarbonProperties.java | 4 ---- .../apache/carbondata/core/util/CarbonUtil.java | 9 -------- .../carbondata/spark/load/CarbonLoaderUtil.java | 6 ++++-- .../org/apache/carbondata/api/CarbonStore.scala | 3 ++- .../spark/thriftserver/CarbonThriftServer.scala | 4 ++-- .../org/apache/spark/sql/CarbonContext.scala | 12 ++++++++--- .../spark/thriftserver/CarbonThriftServer.scala | 4 ++-- .../org/apache/spark/sql/CarbonSession.scala | 13 ++++++++---- .../spark/sql/hive/CarbonFileMetastore.scala | 2 +- .../apache/spark/util/CarbonCommandSuite.scala | 22 ++++++++++---------- .../util/CarbonDataProcessorUtil.java | 6 ++++-- 17 files changed, 67 insertions(+), 52 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/39d7a237/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java index 55a292e..64958bd 100644 --- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java +++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java @@ -73,7 +73,7 @@ public final class CarbonCommonConstants { /** * default location of the carbon member, hierarchy and fact files */ - public static final String STORE_LOCATION_DEFAULT_VAL = "../carbon.store"; +// public static final String STORE_LOCATION_DEFAULT_VAL = "../carbon.store"; /** * CARDINALITY_INCREMENT_DEFAULT_VALUE */ http://git-wip-us.apache.org/repos/asf/carbondata/blob/39d7a237/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java b/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java index 3c39145..8ba341e 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java @@ -68,9 +68,9 @@ public class AbsoluteTableIdentifier implements Serializable { return carbonTableIdentifier; } - public static AbsoluteTableIdentifier from(String dbName, String tableName) { + public static AbsoluteTableIdentifier from(String storePath, String dbName, String tableName) { CarbonTableIdentifier identifier = new CarbonTableIdentifier(dbName, tableName, ""); - return new AbsoluteTableIdentifier(CarbonUtil.getCarbonStorePath(), identifier); + return new AbsoluteTableIdentifier(storePath, identifier); } /** http://git-wip-us.apache.org/repos/asf/carbondata/blob/39d7a237/core/src/main/java/org/apache/carbondata/core/metadata/schema/BucketingInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/BucketingInfo.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/BucketingInfo.java index b5080fe..a48002e 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/BucketingInfo.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/BucketingInfo.java @@ -18,6 +18,7 @@ package org.apache.carbondata.core.metadata.schema; import java.io.Serializable; +import java.util.ArrayList; import java.util.List; import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema; @@ -32,7 +33,8 @@ public class BucketingInfo implements Serializable { private int numberOfBuckets; public BucketingInfo(List<ColumnSchema> listOfColumns, int numberOfBuckets) { - this.listOfColumns = listOfColumns; + this.listOfColumns = new ArrayList<>(); + this.listOfColumns.addAll(listOfColumns); this.numberOfBuckets = numberOfBuckets; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/39d7a237/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java index 2b08536..7dfca9c 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java @@ -18,6 +18,7 @@ package org.apache.carbondata.core.metadata.schema; import java.io.Serializable; +import java.util.ArrayList; import java.util.List; import org.apache.carbondata.core.metadata.schema.partition.PartitionType; @@ -48,7 +49,8 @@ public class PartitionInfo implements Serializable { private int numPartitions; public PartitionInfo(List<ColumnSchema> columnSchemaList, PartitionType partitionType) { - this.columnSchemaList = columnSchemaList; + this.columnSchemaList = new ArrayList<>(); + this.columnSchemaList.addAll(columnSchemaList); this.partitionType = partitionType; } @@ -57,7 +59,8 @@ public class PartitionInfo implements Serializable { } public void setColumnSchemaList(List<ColumnSchema> columnSchemaList) { - this.columnSchemaList = columnSchemaList; + this.columnSchemaList = new ArrayList<>(); + this.columnSchemaList.addAll(columnSchemaList); } public PartitionType getPartitionType() { @@ -73,7 +76,8 @@ public class PartitionInfo implements Serializable { } public void setRangeInfo(List<String> rangeInfo) { - this.rangeInfo = rangeInfo; + this.rangeInfo = new ArrayList<>(); + this.rangeInfo.addAll(rangeInfo); } public List<String> getRangeInfo() { @@ -81,7 +85,8 @@ public class PartitionInfo implements Serializable { } public void setListInfo(List<List<String>> listInfo) { - this.listInfo = listInfo; + this.listInfo = new ArrayList<>(); + this.listInfo.addAll(listInfo); } public List<List<String>> getListInfo() { http://git-wip-us.apache.org/repos/asf/carbondata/blob/39d7a237/core/src/main/java/org/apache/carbondata/core/metadata/schema/SchemaEvolution.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/SchemaEvolution.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/SchemaEvolution.java index 6960736..ccfc595 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/SchemaEvolution.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/SchemaEvolution.java @@ -17,6 +17,7 @@ package org.apache.carbondata.core.metadata.schema; import java.io.Serializable; +import java.util.ArrayList; import java.util.List; /** @@ -45,7 +46,8 @@ public class SchemaEvolution implements Serializable { * @param schemaEvolutionEntryList the schemaEvolutionEntryList to set */ public void setSchemaEvolutionEntryList(List<SchemaEvolutionEntry> schemaEvolutionEntryList) { - this.schemaEvolutionEntryList = schemaEvolutionEntryList; + this.schemaEvolutionEntryList = new ArrayList<>(); + this.schemaEvolutionEntryList.addAll(schemaEvolutionEntryList); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/39d7a237/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java index a396d19..9bfc499 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java @@ -21,6 +21,7 @@ import java.io.DataOutput; import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -104,7 +105,8 @@ public class TableSchema implements Serializable, Writable { * @param listOfColumns the listOfColumns to set */ public void setListOfColumns(List<ColumnSchema> listOfColumns) { - this.listOfColumns = listOfColumns; + this.listOfColumns = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); + this.listOfColumns.addAll(listOfColumns); } /** @@ -183,7 +185,8 @@ public class TableSchema implements Serializable, Writable { * @param tableProperties */ public void setTableProperties(Map<String, String> tableProperties) { - this.tableProperties = tableProperties; + this.tableProperties = new HashMap<>(); + this.tableProperties.putAll(tableProperties); } public BucketingInfo getBucketingInfo() { http://git-wip-us.apache.org/repos/asf/carbondata/blob/39d7a237/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java index d14b7ab..2f5874b 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java @@ -81,10 +81,6 @@ public final class CarbonProperties { } catch (IllegalAccessException e) { LOGGER.error("Illelagal access to declared field" + e.getMessage()); } - if (null == carbonProperties.getProperty(CarbonCommonConstants.STORE_LOCATION)) { - carbonProperties.setProperty(CarbonCommonConstants.STORE_LOCATION, - CarbonCommonConstants.STORE_LOCATION_DEFAULT_VAL); - } validateBlockletSize(); validateNumCores(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/39d7a237/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java index b9c164a..e0bf710 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java @@ -730,15 +730,6 @@ public final class CarbonUtil { .startsWith("file://") || lowerPath.startsWith(ALLUXIO_PREFIX); } - public static String getCarbonStorePath() { - CarbonProperties prop = CarbonProperties.getInstance(); - if (null == prop) { - return null; - } - return prop.getProperty(CarbonCommonConstants.STORE_LOCATION, - CarbonCommonConstants.STORE_LOCATION_DEFAULT_VAL); - } - /** * This method will check the existence of a file at a given path */ http://git-wip-us.apache.org/repos/asf/carbondata/blob/39d7a237/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java index 5b603aa..34c9b96 100644 --- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java +++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java @@ -213,8 +213,10 @@ public final class CarbonLoaderUtil { String tempLocationKey = CarbonDataProcessorUtil .getTempStoreLocationKey(databaseName, tableName, loadModel.getTaskNo(), isCompactionFlow); // form local store location - final String localStoreLocation = CarbonProperties.getInstance() - .getProperty(tempLocationKey, CarbonCommonConstants.STORE_LOCATION_DEFAULT_VAL); + final String localStoreLocation = CarbonProperties.getInstance().getProperty(tempLocationKey); + if (localStoreLocation == null) { + throw new RuntimeException("Store location not set for the key "+tempLocationKey); + } // submit local folder clean up in another thread so that main thread execution is not blocked ExecutorService localFolderDeletionService = Executors.newFixedThreadPool(1); try { http://git-wip-us.apache.org/repos/asf/carbondata/blob/39d7a237/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala index 45719fc..dc37360 100644 --- a/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala +++ b/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala @@ -157,8 +157,9 @@ object CarbonStore { def isSegmentValid( dbName: String, tableName: String, + storePath: String, segmentId: String): Boolean = { - val identifier = AbsoluteTableIdentifier.from(dbName, tableName) + val identifier = AbsoluteTableIdentifier.from(storePath, dbName, tableName) val validAndInvalidSegments: SegmentStatusManager.ValidAndInvalidSegmentsInfo = new SegmentStatusManager( identifier).getValidAndInvalidSegments http://git-wip-us.apache.org/repos/asf/carbondata/blob/39d7a237/integration/spark/src/main/scala/org/apache/carbondata/spark/thriftserver/CarbonThriftServer.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/thriftserver/CarbonThriftServer.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/thriftserver/CarbonThriftServer.scala index b8ba9f7..f8275d1 100644 --- a/integration/spark/src/main/scala/org/apache/carbondata/spark/thriftserver/CarbonThriftServer.scala +++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/thriftserver/CarbonThriftServer.scala @@ -57,8 +57,8 @@ object CarbonThriftServer { "Using default Value and proceeding") Thread.sleep(30000) } - - val cc = new CarbonContext(sc, args.head) + val storePath = if (args.length > 0) args.head else null + val cc = new CarbonContext(sc, storePath) HiveThriftServer2.startWithContext(cc) } http://git-wip-us.apache.org/repos/asf/carbondata/blob/39d7a237/integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala index 1aeda95..bc3f84e 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala @@ -44,7 +44,7 @@ class CarbonContext( def this(sc: SparkContext) = { this(sc, - new File(CarbonCommonConstants.STORE_LOCATION_DEFAULT_VAL).getCanonicalPath, + null, new File(CarbonCommonConstants.METASTORE_LOCATION_DEFAULT_VAL).getCanonicalPath) } @@ -66,8 +66,14 @@ class CarbonContext( @transient override lazy val catalog = { - CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.STORE_LOCATION, storePath) + val carbonProperties = CarbonProperties.getInstance() + if (storePath != null) { + carbonProperties.addProperty(CarbonCommonConstants.STORE_LOCATION, storePath) + //In case if it is in carbon.properties for backward compatible + } else if(carbonProperties.getProperty(CarbonCommonConstants.STORE_LOCATION) == null) { + carbonProperties.addProperty(CarbonCommonConstants.STORE_LOCATION, + conf.getConfString("spark.sql.warehouse.dir")) + } new CarbonMetastore(this, storePath, metadataHive, queryId) with OverrideCatalog } http://git-wip-us.apache.org/repos/asf/carbondata/blob/39d7a237/integration/spark2/src/main/scala/org/apache/carbondata/spark/thriftserver/CarbonThriftServer.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/thriftserver/CarbonThriftServer.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/thriftserver/CarbonThriftServer.scala index aba6891..34ac940 100644 --- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/thriftserver/CarbonThriftServer.scala +++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/thriftserver/CarbonThriftServer.scala @@ -53,9 +53,9 @@ object CarbonThriftServer { System.setProperty("carbon.properties.filepath", sparkConf.get("carbon.properties.filepath")) } - CarbonProperties.getInstance().addProperty(CarbonCommonConstants.STORE_LOCATION, args.head) + val storePath = if (args.length > 0) args.head else null - val spark = builder.getOrCreateCarbonSession(args.head) + val spark = builder.getOrCreateCarbonSession(storePath) val warmUpTime = CarbonProperties.getInstance().getProperty("carbon.spark.warmUpTime", "5000") try { Thread.sleep(Integer.parseInt(warmUpTime)) http://git-wip-us.apache.org/repos/asf/carbondata/blob/39d7a237/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala index b436891..cf8d25b 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala @@ -65,7 +65,7 @@ object CarbonSession { def getOrCreateCarbonSession(): SparkSession = { getOrCreateCarbonSession( - new File(CarbonCommonConstants.STORE_LOCATION_DEFAULT_VAL).getCanonicalPath, + null, new File(CarbonCommonConstants.METASTORE_LOCATION_DEFAULT_VAL).getCanonicalPath) } @@ -145,9 +145,14 @@ object CarbonSession { } sc } - - CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.STORE_LOCATION, storePath) + val carbonProperties = CarbonProperties.getInstance() + if (storePath != null) { + carbonProperties.addProperty(CarbonCommonConstants.STORE_LOCATION, storePath) + //In case if it is in carbon.properties for backward compatible + } else if (carbonProperties.getProperty(CarbonCommonConstants.STORE_LOCATION) == null) { + carbonProperties.addProperty(CarbonCommonConstants.STORE_LOCATION, + sparkContext.conf.get("spark.sql.warehouse.dir")) + } session = new CarbonSession(sparkContext) options.foreach { case (k, v) => session.sessionState.conf.setConfString(k, v) } SparkSession.setDefaultSession(session) http://git-wip-us.apache.org/repos/asf/carbondata/blob/39d7a237/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala index 549841b..a63164f 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala @@ -62,7 +62,7 @@ case class DictionaryMap(dictionaryMap: Map[String, Boolean]) { } } -class CarbonFileMetastore(conf: RuntimeConfig, val storePath: String) extends CarbonMetaStore { +class CarbonFileMetastore(conf: RuntimeConfig) extends CarbonMetaStore { @transient val LOGGER = LogServiceFactory.getLogService("org.apache.spark.sql.CarbonMetastoreCatalog") http://git-wip-us.apache.org/repos/asf/carbondata/blob/39d7a237/integration/spark2/src/test/scala/org/apache/spark/util/CarbonCommandSuite.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/spark/util/CarbonCommandSuite.scala b/integration/spark2/src/test/scala/org/apache/spark/util/CarbonCommandSuite.scala index 25be4a0..2fb9d7f 100644 --- a/integration/spark2/src/test/scala/org/apache/spark/util/CarbonCommandSuite.scala +++ b/integration/spark2/src/test/scala/org/apache/spark/util/CarbonCommandSuite.scala @@ -43,31 +43,31 @@ class CarbonCommandSuite extends QueryTest with BeforeAndAfterAll { } test("show segment") { - ShowSegments.main(Array(s"${CarbonUtil.getCarbonStorePath}", "carbon_table")) + ShowSegments.main(Array(s"${TestQueryExecutor.storeLocation}", "carbon_table")) } test("delete segment by id") { - DeleteSegmentById.main(Array(s"${CarbonUtil.getCarbonStorePath}", "carbon_table", "0")) - assert(!CarbonStore.isSegmentValid("default", "carbon_table", "0")) + DeleteSegmentById.main(Array(s"${TestQueryExecutor.storeLocation}", "carbon_table", "0")) + assert(!CarbonStore.isSegmentValid("default", "carbon_table",TestQueryExecutor.storeLocation, "0")) } test("delete segment by date") { createAndLoadTestTable("carbon_table2", "csv_table") val time = new Timestamp(new Date().getTime) - DeleteSegmentByDate.main(Array(s"${CarbonUtil.getCarbonStorePath}", "carbon_table2", time.toString)) - assert(!CarbonStore.isSegmentValid("default", "carbon_table2", "0")) + DeleteSegmentByDate.main(Array(s"${TestQueryExecutor.storeLocation}", "carbon_table2", time.toString)) + assert(!CarbonStore.isSegmentValid("default", "carbon_table2", TestQueryExecutor.storeLocation, "0")) dropTable("carbon_table2") } test("clean files") { val table = "carbon_table3" createAndLoadTestTable(table, "csv_table") - ShowSegments.main(Array(s"${CarbonUtil.getCarbonStorePath}", table)) - DeleteSegmentById.main(Array(s"${CarbonUtil.getCarbonStorePath}", table, "0")) - ShowSegments.main(Array(s"${CarbonUtil.getCarbonStorePath}", table)) - CleanFiles.main(Array(s"${CarbonUtil.getCarbonStorePath}", table)) - ShowSegments.main(Array(s"${CarbonUtil.getCarbonStorePath}", table)) - val tablePath = s"${CarbonUtil.getCarbonStorePath}${File.separator}default${File.separator}$table" + ShowSegments.main(Array(s"${TestQueryExecutor.storeLocation}", table)) + DeleteSegmentById.main(Array(s"${TestQueryExecutor.storeLocation}", table, "0")) + ShowSegments.main(Array(s"${TestQueryExecutor.storeLocation}", table)) + CleanFiles.main(Array(s"${TestQueryExecutor.storeLocation}", table)) + ShowSegments.main(Array(s"${TestQueryExecutor.storeLocation}", table)) + val tablePath = s"${TestQueryExecutor.storeLocation}${File.separator}default${File.separator}$table" val f = new File(s"$tablePath/Fact/Part0") assert(f.isDirectory) http://git-wip-us.apache.org/repos/asf/carbondata/blob/39d7a237/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java index 62f13db..69db86f 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java +++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java @@ -173,8 +173,10 @@ public final class CarbonDataProcessorUtil { String taskId, String partitionId, String segmentId, boolean isCompactionFlow) { String tempLocationKey = getTempStoreLocationKey(databaseName, tableName, taskId, isCompactionFlow); - String baseStorePath = CarbonProperties.getInstance() - .getProperty(tempLocationKey, CarbonCommonConstants.STORE_LOCATION_DEFAULT_VAL); + String baseStorePath = CarbonProperties.getInstance().getProperty(tempLocationKey); + if (baseStorePath == null) { + throw new RuntimeException("Locatio not set for the key "+tempLocationKey); + } CarbonTable carbonTable = CarbonMetadata.getInstance() .getCarbonTable(databaseName + CarbonCommonConstants.UNDERSCORE + tableName); CarbonTablePath carbonTablePath =
