Repository: carbondata Updated Branches: refs/heads/master 0bcd8677a -> 6e04fe65a
[CARBONDATA-3128]Fix the HiveExample exception This pull request fixes HiveExample exception with different spark versions. When through idea running HiveExample application occurs exception with spark-2.1 and spark-2.3 profile, this fix update pom.xml of carbondata modules for spark dependencies. Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/6e04fe65 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/6e04fe65 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/6e04fe65 Branch: refs/heads/master Commit: 6e04fe65ad5bfa775e4153ed53b0890c6da5b19e Parents: 0bcd867 Author: Nicholas Jiang <[email protected]> Authored: Tue Nov 27 14:51:56 2018 +0800 Committer: Zhang Zhichao <[email protected]> Committed: Fri Nov 30 13:16:22 2018 +0800 ---------------------------------------------------------------------- datamap/mv/core/pom.xml | 1 - datamap/mv/plan/pom.xml | 1 - examples/spark2/pom.xml | 3 -- integration/hive/pom.xml | 6 ++- .../carbondata/hiveexample/HiveExample.scala | 1 - .../hiveexampleCI/RunHiveExampleTest.scala | 43 ++++++++++++++++++++ integration/presto/pom.xml | 3 -- integration/spark-common-test/pom.xml | 1 - integration/spark2/pom.xml | 6 --- pom.xml | 29 ++++++++++++- streaming/pom.xml | 3 -- 11 files changed, 76 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/6e04fe65/datamap/mv/core/pom.xml ---------------------------------------------------------------------- diff --git a/datamap/mv/core/pom.xml b/datamap/mv/core/pom.xml index 96ffb3f..dbfb22e 100644 --- a/datamap/mv/core/pom.xml +++ b/datamap/mv/core/pom.xml @@ -53,7 +53,6 @@ <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-core_${scala.binary.version}</artifactId> - <version>${spark.version}</version> </dependency> </dependencies> http://git-wip-us.apache.org/repos/asf/carbondata/blob/6e04fe65/datamap/mv/plan/pom.xml ---------------------------------------------------------------------- diff --git a/datamap/mv/plan/pom.xml b/datamap/mv/plan/pom.xml index 9286c14..6c965e8 100644 --- a/datamap/mv/plan/pom.xml +++ b/datamap/mv/plan/pom.xml @@ -47,7 +47,6 @@ <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-core_${scala.binary.version}</artifactId> - <version>${spark.version}</version> </dependency> </dependencies> http://git-wip-us.apache.org/repos/asf/carbondata/blob/6e04fe65/examples/spark2/pom.xml ---------------------------------------------------------------------- diff --git a/examples/spark2/pom.xml b/examples/spark2/pom.xml index 6f7aba6..48728ac 100644 --- a/examples/spark2/pom.xml +++ b/examples/spark2/pom.xml @@ -74,8 +74,6 @@ <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-streaming_${scala.binary.version}</artifactId> - <version>${spark.version}</version> - <scope>${spark.deps.scope}</scope> </dependency> <dependency> <groupId>org.apache.httpcomponents</groupId> @@ -91,7 +89,6 @@ <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-core_${scala.binary.version}</artifactId> - <version>${spark.version}</version> </dependency> <dependency> <groupId>org.apache.carbondata</groupId> http://git-wip-us.apache.org/repos/asf/carbondata/blob/6e04fe65/integration/hive/pom.xml ---------------------------------------------------------------------- diff --git a/integration/hive/pom.xml b/integration/hive/pom.xml index 35c431f..b2256e7 100644 --- a/integration/hive/pom.xml +++ b/integration/hive/pom.xml @@ -82,7 +82,6 @@ <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-catalyst_${scala.binary.version}</artifactId> - <version>${spark.version}</version> </dependency> <dependency> <groupId>org.apache.httpcomponents</groupId> @@ -104,6 +103,11 @@ <artifactId>carbondata-hadoop</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>org.scalatest</groupId> + <artifactId>scalatest_${scala.binary.version}</artifactId> + <scope>test</scope> + </dependency> </dependencies> <build> http://git-wip-us.apache.org/repos/asf/carbondata/blob/6e04fe65/integration/hive/src/main/scala/org/apache/carbondata/hiveexample/HiveExample.scala ---------------------------------------------------------------------- diff --git a/integration/hive/src/main/scala/org/apache/carbondata/hiveexample/HiveExample.scala b/integration/hive/src/main/scala/org/apache/carbondata/hiveexample/HiveExample.scala index 11839c9..3638321 100644 --- a/integration/hive/src/main/scala/org/apache/carbondata/hiveexample/HiveExample.scala +++ b/integration/hive/src/main/scala/org/apache/carbondata/hiveexample/HiveExample.scala @@ -192,7 +192,6 @@ object HiveExample { outOfOrderColFetched = outOfOrderColFetched + 1 } hiveEmbeddedServer2.stop() - System.exit(0) } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/6e04fe65/integration/hive/src/test/scala/org/apache/carbondata/hiveexampleCI/RunHiveExampleTest.scala ---------------------------------------------------------------------- diff --git a/integration/hive/src/test/scala/org/apache/carbondata/hiveexampleCI/RunHiveExampleTest.scala b/integration/hive/src/test/scala/org/apache/carbondata/hiveexampleCI/RunHiveExampleTest.scala new file mode 100644 index 0000000..92f7374 --- /dev/null +++ b/integration/hive/src/test/scala/org/apache/carbondata/hiveexampleCI/RunHiveExampleTest.scala @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.hiveexampleCI + +import org.apache.spark.sql.test.util.QueryTest +import org.scalatest.BeforeAndAfterAll + +import org.apache.carbondata.core.constants.CarbonCommonConstants +import org.apache.carbondata.core.util.CarbonProperties +import org.apache.carbondata.hiveexample.HiveExample + +class RunHiveExampleTest extends QueryTest with BeforeAndAfterAll { + + private val spark = sqlContext.sparkSession + + override def beforeAll: Unit = { + CarbonProperties.getInstance().addProperty( + CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, + CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT) + CarbonProperties.getInstance().addProperty( + CarbonCommonConstants.CARBON_DATE_FORMAT, + CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT) + } + + test("HiveExample") { + HiveExample.main(null) + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata/blob/6e04fe65/integration/presto/pom.xml ---------------------------------------------------------------------- diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml index 5ac146c..31a4de6 100644 --- a/integration/presto/pom.xml +++ b/integration/presto/pom.xml @@ -437,13 +437,11 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-network-common_${scala.binary.version}</artifactId> <scope>test</scope> - <version>${spark.version}</version> </dependency> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-core_${scala.binary.version}</artifactId> <scope>test</scope> - <version>${spark.version}</version> <exclusions> <exclusion> <groupId>org.apache.hadoop</groupId> @@ -487,7 +485,6 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-catalyst_${scala.binary.version}</artifactId> <scope>test</scope> - <version>${spark.version}</version> </dependency> <dependency> <groupId>org.lz4</groupId> http://git-wip-us.apache.org/repos/asf/carbondata/blob/6e04fe65/integration/spark-common-test/pom.xml ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/pom.xml b/integration/spark-common-test/pom.xml index 5da0c67..fd96f88 100644 --- a/integration/spark-common-test/pom.xml +++ b/integration/spark-common-test/pom.xml @@ -135,7 +135,6 @@ will fail to execute --> <groupId>org.apache.spark</groupId> <artifactId>spark-core_${scala.binary.version}</artifactId> - <version>${spark.version}</version> <scope>test</scope> <exclusions> <!-- need to Exclude Avro jar from this project,spark core is using http://git-wip-us.apache.org/repos/asf/carbondata/blob/6e04fe65/integration/spark2/pom.xml ---------------------------------------------------------------------- diff --git a/integration/spark2/pom.xml b/integration/spark2/pom.xml index 69594eb..d066142 100644 --- a/integration/spark2/pom.xml +++ b/integration/spark2/pom.xml @@ -71,18 +71,14 @@ <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-streaming_${scala.binary.version}</artifactId> - <version>${spark.version}</version> - <scope>${spark.deps.scope}</scope> </dependency> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-core_${scala.binary.version}</artifactId> - <version>${spark.version}</version> </dependency> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-sql_${scala.binary.version}</artifactId> - <version>${spark.version}</version> <exclusions> <!-- from transitive dependency com.univocity:univocity-parsers:2.5.9 is added from the org.apache.spark:spark-sql_2.11,so need to remove @@ -96,8 +92,6 @@ <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-sql-kafka-0-10_${scala.binary.version}</artifactId> - <version>${spark.version}</version> - <scope>${spark.deps.scope}</scope> <exclusions> <exclusion> <groupId>net.jpountz.lz4</groupId> http://git-wip-us.apache.org/repos/asf/carbondata/blob/6e04fe65/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 8e67c26..df8ae42 100644 --- a/pom.xml +++ b/pom.xml @@ -100,7 +100,6 @@ <module>processing</module> <module>hadoop</module> <module>integration/spark-common</module> - <module>integration/spark-datasource</module> <module>integration/spark-common-test</module> <module>datamap/examples</module> <module>store/sdk</module> @@ -202,12 +201,30 @@ </dependency> <dependency> <groupId>org.apache.spark</groupId> + <artifactId>spark-core_${scala.binary.version}</artifactId> + <version>${spark.version}</version> + <scope>${spark.deps.scope}</scope> + </dependency> + <dependency> + <groupId>org.apache.spark</groupId> <artifactId>spark-sql_${scala.binary.version}</artifactId> <version>${spark.version}</version> <scope>${spark.deps.scope}</scope> </dependency> <dependency> <groupId>org.apache.spark</groupId> + <artifactId>spark-catalyst_${scala.binary.version}</artifactId> + <version>${spark.version}</version> + <scope>${spark.deps.scope}</scope> + </dependency> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-streaming_${scala.binary.version}</artifactId> + <version>${spark.version}</version> + <scope>${spark.deps.scope}</scope> + </dependency> + <dependency> + <groupId>org.apache.spark</groupId> <artifactId>spark-hive-thriftserver_${scala.binary.version}</artifactId> <version>${spark.version}</version> <scope>${spark.deps.scope}</scope> @@ -220,6 +237,12 @@ </dependency> <dependency> <groupId>org.apache.spark</groupId> + <artifactId>spark-network-common_${scala.binary.version}</artifactId> + <version>${spark.version}</version> + <scope>${spark.deps.scope}</scope> + </dependency> + <dependency> + <groupId>org.apache.spark</groupId> <artifactId>spark-sql-kafka-0-10_${scala.binary.version}</artifactId> <version>${spark.version}</version> <scope>${spark.deps.scope}</scope> @@ -461,6 +484,7 @@ <modules> <module>format</module> <module>integration/spark2</module> + <module>integration/spark-datasource</module> <module>examples/spark2</module> <module>integration/hive</module> <module>integration/presto</module> @@ -495,6 +519,7 @@ </properties> <modules> <module>integration/spark2</module> + <module>integration/spark-datasource</module> <module>integration/hive</module> <module>integration/presto</module> <module>streaming</module> @@ -555,6 +580,7 @@ </properties> <modules> <module>integration/spark2</module> + <module>integration/spark-datasource</module> <module>integration/hive</module> <module>integration/presto</module> <module>streaming</module> @@ -613,6 +639,7 @@ </properties> <modules> <module>integration/spark2</module> + <module>integration/spark-datasource</module> <module>integration/hive</module> <module>integration/presto</module> <module>streaming</module> http://git-wip-us.apache.org/repos/asf/carbondata/blob/6e04fe65/streaming/pom.xml ---------------------------------------------------------------------- diff --git a/streaming/pom.xml b/streaming/pom.xml index f8b9c22..4e8eb3b 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -28,13 +28,10 @@ <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-sql_${scala.binary.version}</artifactId> - <version>${spark.version}</version> </dependency> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-streaming_${scala.binary.version}</artifactId> - <version>${spark.version}</version> - <scope>${spark.deps.scope}</scope> </dependency> <dependency> <groupId>junit</groupId>
