Github user gvramana commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2484#discussion_r203061344
--- Diff:
integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonRDD.scala
---
@@ -79,14 +66,11 @@ abstract class CarbonRDD[T: ClassTag](@transient sc:
SparkContext,
}
def getConf: Configuration = {
- val configuration = new Configuration(false)
- val bai = new
ByteArrayInputStream(CompressorFactory.getInstance().getCompressor
- .unCompressByte(confBytes))
- val ois = new ObjectInputStream(bai)
- configuration.readFields(ois)
- ois.close()
- configuration
+ val carbonConfiguration =
carbonSessionInfo.getThreadParams.getExtraInfo("carbonConf")
--- End diff --
Also need to handle InputFormat and OutputFormat cases where configuration
is explicitly passed using job interface.
---
