[CARBONDATA-1408]:Data loading with globalSort is failing in long run After 437 load, data loading getting failed. On analyzing the root cause it concluded that any new folder is not getting created in temp location. There's a certain limit based on OS in folder creation. Thus in long run data loading case the folder creation limit reaches its maximum due to which any new folder is not getting created in temp folder, and thus the data loading is getting failed. Solution:- Need to delete the created folder from temp folder once data loading is completed.
This closes #1330 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b269384d Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b269384d Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b269384d Branch: refs/heads/branch-1.2 Commit: b269384d4e88d3571ed964b04372f916c13b3f04 Parents: df95547 Author: kushalsaha <kushalsaha1...@gmail.com> Authored: Wed Sep 6 11:32:37 2017 +0530 Committer: Ravindra Pesala <ravi.pes...@gmail.com> Committed: Thu Sep 14 16:42:25 2017 +0530 ---------------------------------------------------------------------- .../carbondata/spark/load/DataLoadProcessorStepOnSpark.scala | 2 +- .../scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala | 2 +- .../org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/b269384d/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala index e43d204..aaf7926 100644 --- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala +++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala @@ -225,7 +225,7 @@ object DataLoadProcessorStepOnSpark { } else { storeLocation = System.getProperty("java.io.tmpdir") } - storeLocation = storeLocation + '/' + System.nanoTime() + '/' + index + storeLocation = storeLocation + '/' + System.nanoTime() + '_' + index storeLocation } http://git-wip-us.apache.org/repos/asf/carbondata/blob/b269384d/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala index 52f37ef..c8a55af 100644 --- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala +++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala @@ -101,7 +101,7 @@ class CarbonMergerRDD[K, V]( } else { storeLocation = System.getProperty("java.io.tmpdir") } - storeLocation = storeLocation + '/' + System.nanoTime() + '/' + theSplit.index + storeLocation = storeLocation + '/' + System.nanoTime() + '_' + theSplit.index var mergeStatus = false var mergeNumber = "" var exec: CarbonCompactionExecutor = null http://git-wip-us.apache.org/repos/asf/carbondata/blob/b269384d/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala index 7c6274b..67c71a0 100644 --- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala +++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala @@ -169,7 +169,7 @@ class SparkPartitionLoader(model: CarbonLoadModel, LOGGER.info("Temp location for loading data: " + storeLocation.mkString(",")) } - private def tmpLocationSuffix = File.separator + System.nanoTime() + File.separator + splitIndex + private def tmpLocationSuffix = File.separator + System.nanoTime() + "_" + splitIndex } /**
