Github user sraghunandan commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1841#discussion_r164371164
--- Diff:
integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
---
@@ -941,4 +943,40 @@ object CommonUtil {
}
}
+ def setTempStoreLocation(
+ index: Int,
+ carbonLoadModel: CarbonLoadModel,
+ isCompactionFlow: Boolean,
+ isAltPartitionFlow: Boolean) : Unit = {
+ var storeLocation: String = null
+
+ // this property is used to determine whether temp location for carbon
is inside
+ // container temp dir or is yarn application directory.
+ val carbonUseLocalDir = CarbonProperties.getInstance()
+ .getProperty("carbon.use.local.dir", "false")
+
+ if (carbonUseLocalDir.equalsIgnoreCase("true")) {
+
+ val storeLocations = Util.getConfiguredLocalDirs(SparkEnv.get.conf)
+ if (null != storeLocations && storeLocations.nonEmpty) {
+ storeLocation =
storeLocations(Random.nextInt(storeLocations.length))
+ }
+ if (storeLocation == null) {
+ storeLocation = System.getProperty("java.io.tmpdir")
+ }
+ } else {
+ storeLocation = System.getProperty("java.io.tmpdir")
+ }
+ storeLocation = storeLocation + CarbonCommonConstants.FILE_SEPARATOR +
"carbon" +
+ System.nanoTime() + CarbonCommonConstants.UNDERSCORE + index
+
+ val tempLocationKey = CarbonDataProcessorUtil
+ .getTempStoreLocationKey(carbonLoadModel.getDatabaseName,
+ carbonLoadModel.getTableName,
+ carbonLoadModel.getSegmentId,
+ carbonLoadModel.getTaskNo,
+ isCompactionFlow,
+ isAltPartitionFlow)
+ CarbonProperties.getInstance().addProperty(tempLocationKey,
storeLocation)
--- End diff --
CarbonProperty is the methodology used during data loading, compaction.
changing alone for streaming handoff will cause confusion and difficult to
maintain code due to difference in implementation. your suggested design need
to be modified for all flows and needs extensive testing under concurrent
scenarios also.In my opinion, that should not be the scope of this PR
---
