Repository: carbondata Updated Branches: refs/heads/master df8f06739 -> 359f6e6b2
Advertising
[CARBONDATA-2329] Non Serializable extra info in session is overwritten from stale thread Problem: 1. Non Serializable extra info is copied from thread which causes stale data from old session when the thread is reused by spark. 2. CarboSessionInfo clone is not copying Non serializable info to new object which can damage session level values if local query thread updates values. Solution: Remove logic to copy Non Serializable extra info and fix clone logic This closes #2154 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/359f6e6b Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/359f6e6b Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/359f6e6b Branch: refs/heads/master Commit: 359f6e6b22cca44a344545470720c02d07bbb4f4 Parents: df8f067 Author: KanakaKumar <kanaka.avv...@huawei.com> Authored: Tue Apr 10 19:42:30 2018 +0530 Committer: manishgupta88 <tomanishgupt...@gmail.com> Committed: Wed Apr 11 14:25:55 2018 +0530 ---------------------------------------------------------------------- .../org/apache/carbondata/core/util/CarbonSessionInfo.java | 6 ++---- .../spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala | 2 -- .../src/main/scala/org/apache/spark/sql/CarbonSession.scala | 4 ---- 3 files changed, 2 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/359f6e6b/core/src/main/java/org/apache/carbondata/core/util/CarbonSessionInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonSessionInfo.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonSessionInfo.java index 25d1c29..38f2d76 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonSessionInfo.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonSessionInfo.java @@ -60,11 +60,9 @@ public class CarbonSessionInfo implements Serializable, Cloneable { CarbonSessionInfo newObj = new CarbonSessionInfo(); newObj.setSessionParams(sessionParams.clone()); newObj.setThreadParams(threadParams.clone()); - Map<String, Object> nonSerializableExtraInfo = getNonSerializableExtraInfo(); - for (Map.Entry<String, Object> entry : nonSerializableExtraInfo.entrySet()) { - nonSerializableExtraInfo.put(entry.getKey(), entry.getValue()); + for (Map.Entry<String, Object> entry : getNonSerializableExtraInfo().entrySet()) { + newObj.getNonSerializableExtraInfo().put(entry.getKey(), entry.getValue()); } - newObj.setNonSerializableExtraInfo(nonSerializableExtraInfo); return newObj; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/359f6e6b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala index 881a4f5..593ecce 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala @@ -82,11 +82,9 @@ class CarbonEnv { // update carbon session parameters , preserve thread parameters val currentThreadSesssionInfo = ThreadLocalSessionInfo.getCarbonSessionInfo carbonSessionInfo = new CarbonSessionInfo() - sessionParams = carbonSessionInfo.getSessionParams if (currentThreadSesssionInfo != null) { carbonSessionInfo.setThreadParams(currentThreadSesssionInfo.getThreadParams) } - carbonSessionInfo.setSessionParams(sessionParams) ThreadLocalSessionInfo.setCarbonSessionInfo(carbonSessionInfo) val config = new CarbonSQLConf(sparkSession) if (sparkSession.conf.getOption(CarbonCommonConstants.ENABLE_UNSAFE_SORT).isEmpty) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/359f6e6b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala index 1038fcf..a380308 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala @@ -292,11 +292,7 @@ object CarbonSession { // copy all the thread parameters to apply to session parameters currentThreadSessionInfo.getThreadParams.getAll.asScala .foreach(entry => carbonSessionInfo.getSessionParams.addProperty(entry._1, entry._2)) - currentThreadSessionInfo.getNonSerializableExtraInfo.asScala - .foreach(entry => carbonSessionInfo.getNonSerializableExtraInfo.put(entry._1, entry._2)) carbonSessionInfo.setThreadParams(currentThreadSessionInfo.getThreadParams) - carbonSessionInfo - .setNonSerializableExtraInfo(currentThreadSessionInfo.getNonSerializableExtraInfo) } // preserve thread parameters across call ThreadLocalSessionInfo.setCarbonSessionInfo(carbonSessionInfo)
