This is an automated email from the ASF dual-hosted git repository.
jackylk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new e40912c [HOTFIX] Remove spark dependency in Processing module
e40912c is described below
commit e40912caa4aad58d71f1e1195c68a27927ea9a57
Author: Indhumathi27 <indhumathi...@gmail.com>
AuthorDate: Mon Mar 2 13:54:20 2020 +0530
[HOTFIX] Remove spark dependency in Processing module
Why is this PR needed?
Remove spark dependency in Processing module, as it is not required
What changes were proposed in this PR?
Removed spark dependency in Processing module
Does this PR introduce any user interface change?
No
Is any new testcase added?
No
This closes #3650
---
.../org/apache/carbondata/streaming/StreamSinkFactory.scala | 3 +--
.../sql/execution/command/management/CommonLoadUtils.scala | 3 +--
.../sql/execution/streaming/CarbonAppendableStreamSink.scala | 3 +--
processing/pom.xml | 5 -----
.../carbondata/processing/loading/events/LoadEvents.java | 10 +---------
5 files changed, 4 insertions(+), 20 deletions(-)
diff --git
a/integration/spark/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
b/integration/spark/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
index b22d4aa..152c6e7 100644
---
a/integration/spark/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
+++
b/integration/spark/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
@@ -102,8 +102,7 @@ object StreamSinkFactory {
false,
parameters.asJava,
parameters.asJava,
- false,
- sparkSession
+ false
)
OperationListenerBus.getInstance().fireEvent(loadTablePreExecutionEvent,
operationContext)
// prepare the stream segment
diff --git
a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CommonLoadUtils.scala
b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CommonLoadUtils.scala
index 0aada09..9e01fb1 100644
---
a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CommonLoadUtils.scala
+++
b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CommonLoadUtils.scala
@@ -289,8 +289,7 @@ object CommonLoadUtils {
isDataFrame,
optionsFinal,
options,
- isOverwriteTable,
- sparkSession)
+ isOverwriteTable)
operationContext.setProperty("isOverwrite", isOverwriteTable)
OperationListenerBus.getInstance.fireEvent(loadTablePreExecutionEvent,
operationContext)
// Add pre event listener for index datamap
diff --git
a/integration/spark/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
b/integration/spark/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
index 272d107..039c946 100644
---
a/integration/spark/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
+++
b/integration/spark/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
@@ -133,8 +133,7 @@ class CarbonAppendableStreamSink(
false,
parameters.asJava,
parameters.asJava,
- false,
- sparkSession
+ false
)
OperationListenerBus.getInstance().fireEvent(loadTablePreExecutionEvent,
operationContext)
checkOrHandOffSegment()
diff --git a/processing/pom.xml b/processing/pom.xml
index 3c178dd..5a73405 100644
--- a/processing/pom.xml
+++ b/processing/pom.xml
@@ -41,11 +41,6 @@
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- </dependency>
- <dependency>
<groupId>com.univocity</groupId>
<artifactId>univocity-parsers</artifactId>
<version>2.2.1</version>
diff --git
a/processing/src/main/java/org/apache/carbondata/processing/loading/events/LoadEvents.java
b/processing/src/main/java/org/apache/carbondata/processing/loading/events/LoadEvents.java
index 7f6f8fd..7759b02 100644
---
a/processing/src/main/java/org/apache/carbondata/processing/loading/events/LoadEvents.java
+++
b/processing/src/main/java/org/apache/carbondata/processing/loading/events/LoadEvents.java
@@ -24,8 +24,6 @@ import
org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.events.Event;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
-import org.apache.spark.sql.SparkSession;
-
public class LoadEvents {
/**
* Class for handling operations before start of a load process.
@@ -40,12 +38,11 @@ public class LoadEvents {
// userProvidedOptions are needed if we need only the load options given
by user
private Map<String, String> userProvidedOptions;
private boolean isOverWriteTable;
- private SparkSession sparkSession;
public LoadTablePreExecutionEvent(CarbonTableIdentifier
carbonTableIdentifier,
CarbonLoadModel carbonLoadModel, String factPath, boolean
isDataFrameDefined,
Map<String, String> optionsFinal, Map<String, String>
userProvidedOptions,
- boolean isOverWriteTable, SparkSession sparkSession) {
+ boolean isOverWriteTable) {
this.carbonTableIdentifier = carbonTableIdentifier;
this.carbonLoadModel = carbonLoadModel;
this.factPath = factPath;
@@ -53,7 +50,6 @@ public class LoadEvents {
this.optionsFinal = optionsFinal;
this.userProvidedOptions = userProvidedOptions;
this.isOverWriteTable = isOverWriteTable;
- this.sparkSession = sparkSession;
}
public LoadTablePreExecutionEvent(CarbonTableIdentifier
carbonTableIdentifier,
@@ -62,10 +58,6 @@ public class LoadEvents {
this.carbonLoadModel = carbonLoadModel;
}
- public SparkSession getSparkSession() {
- return sparkSession;
- }
-
public CarbonTableIdentifier getCarbonTableIdentifier() {
return carbonTableIdentifier;
}
