This is an automated email from the ASF dual-hosted git repository. jackylk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push: new e40912c [HOTFIX] Remove spark dependency in Processing module e40912c is described below commit e40912caa4aad58d71f1e1195c68a27927ea9a57 Author: Indhumathi27 <indhumathi...@gmail.com> AuthorDate: Mon Mar 2 13:54:20 2020 +0530 [HOTFIX] Remove spark dependency in Processing module Why is this PR needed? Remove spark dependency in Processing module, as it is not required What changes were proposed in this PR? Removed spark dependency in Processing module Does this PR introduce any user interface change? No Is any new testcase added? No This closes #3650 --- .../org/apache/carbondata/streaming/StreamSinkFactory.scala | 3 +-- .../sql/execution/command/management/CommonLoadUtils.scala | 3 +-- .../sql/execution/streaming/CarbonAppendableStreamSink.scala | 3 +-- processing/pom.xml | 5 ----- .../carbondata/processing/loading/events/LoadEvents.java | 10 +--------- 5 files changed, 4 insertions(+), 20 deletions(-) diff --git a/integration/spark/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala b/integration/spark/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala index b22d4aa..152c6e7 100644 --- a/integration/spark/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala +++ b/integration/spark/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala @@ -102,8 +102,7 @@ object StreamSinkFactory { false, parameters.asJava, parameters.asJava, - false, - sparkSession + false ) OperationListenerBus.getInstance().fireEvent(loadTablePreExecutionEvent, operationContext) // prepare the stream segment diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CommonLoadUtils.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CommonLoadUtils.scala index 0aada09..9e01fb1 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CommonLoadUtils.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CommonLoadUtils.scala @@ -289,8 +289,7 @@ object CommonLoadUtils { isDataFrame, optionsFinal, options, - isOverwriteTable, - sparkSession) + isOverwriteTable) operationContext.setProperty("isOverwrite", isOverwriteTable) OperationListenerBus.getInstance.fireEvent(loadTablePreExecutionEvent, operationContext) // Add pre event listener for index datamap diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala index 272d107..039c946 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala @@ -133,8 +133,7 @@ class CarbonAppendableStreamSink( false, parameters.asJava, parameters.asJava, - false, - sparkSession + false ) OperationListenerBus.getInstance().fireEvent(loadTablePreExecutionEvent, operationContext) checkOrHandOffSegment() diff --git a/processing/pom.xml b/processing/pom.xml index 3c178dd..5a73405 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -41,11 +41,6 @@ <version>${project.version}</version> </dependency> <dependency> - <groupId>org.apache.spark</groupId> - <artifactId>spark-sql_${scala.binary.version}</artifactId> - <version>${spark.version}</version> - </dependency> - <dependency> <groupId>com.univocity</groupId> <artifactId>univocity-parsers</artifactId> <version>2.2.1</version> diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/events/LoadEvents.java b/processing/src/main/java/org/apache/carbondata/processing/loading/events/LoadEvents.java index 7f6f8fd..7759b02 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/loading/events/LoadEvents.java +++ b/processing/src/main/java/org/apache/carbondata/processing/loading/events/LoadEvents.java @@ -24,8 +24,6 @@ import org.apache.carbondata.core.metadata.schema.table.CarbonTable; import org.apache.carbondata.events.Event; import org.apache.carbondata.processing.loading.model.CarbonLoadModel; -import org.apache.spark.sql.SparkSession; - public class LoadEvents { /** * Class for handling operations before start of a load process. @@ -40,12 +38,11 @@ public class LoadEvents { // userProvidedOptions are needed if we need only the load options given by user private Map<String, String> userProvidedOptions; private boolean isOverWriteTable; - private SparkSession sparkSession; public LoadTablePreExecutionEvent(CarbonTableIdentifier carbonTableIdentifier, CarbonLoadModel carbonLoadModel, String factPath, boolean isDataFrameDefined, Map<String, String> optionsFinal, Map<String, String> userProvidedOptions, - boolean isOverWriteTable, SparkSession sparkSession) { + boolean isOverWriteTable) { this.carbonTableIdentifier = carbonTableIdentifier; this.carbonLoadModel = carbonLoadModel; this.factPath = factPath; @@ -53,7 +50,6 @@ public class LoadEvents { this.optionsFinal = optionsFinal; this.userProvidedOptions = userProvidedOptions; this.isOverWriteTable = isOverWriteTable; - this.sparkSession = sparkSession; } public LoadTablePreExecutionEvent(CarbonTableIdentifier carbonTableIdentifier, @@ -62,10 +58,6 @@ public class LoadEvents { this.carbonLoadModel = carbonLoadModel; } - public SparkSession getSparkSession() { - return sparkSession; - } - public CarbonTableIdentifier getCarbonTableIdentifier() { return carbonTableIdentifier; }