This is an automated email from the ASF dual-hosted git repository. jackylk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push: new 0119007 [CARBONDATA-3784] Added spark binary version to related modules 0119007 is described below commit 0119007db9b312c848181dfb44162a027a660654 Author: kunal642 <kunalkapoor...@gmail.com> AuthorDate: Wed Apr 8 15:32:39 2020 +0530 [CARBONDATA-3784] Added spark binary version to related modules Why is this PR needed? For deploying multiple carbon jars version with different spark versions, the jar/module names should be different What changes were proposed in this PR? Add spark binary version to the related modules to distinguish the jars based on spark version Does this PR introduce any user interface change? No Is any new testcase added? No This closes #3700 --- .gitignore | 1 + assembly/pom.xml | 4 ++-- examples/flink/pom.xml | 2 +- examples/spark/pom.xml | 17 +++++++++++++- index/examples/pom.xml | 20 ++++++++++++++++- index/secondary-index/pom.xml | 17 +++++++++++++- integration/flink/pom.xml | 10 +++++++-- integration/spark/pom.xml | 20 +++++++---------- mv/plan/pom.xml | 8 ++++++- pom.xml | 52 +++++++++++++++++++++++++++++++++++++++++++ 10 files changed, 130 insertions(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index 0af3918..41d6f18 100644 --- a/.gitignore +++ b/.gitignore @@ -24,4 +24,5 @@ python/.idea/ */*/.cache-tests */*/*/.cache-main */*/*/.cache-tests +*.flattened-pom.xml python/pycarbon/.pylintrc diff --git a/assembly/pom.xml b/assembly/pom.xml index 3003611..6c0e30e 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -58,12 +58,12 @@ </dependency> <dependency> <groupId>org.apache.carbondata</groupId> - <artifactId>carbondata-spark</artifactId> + <artifactId>carbondata-spark_${spark.binary.version}</artifactId> <version>${project.version}</version> </dependency> <dependency> <groupId>org.apache.carbondata</groupId> - <artifactId>carbondata-mv-plan</artifactId> + <artifactId>carbondata-mv-plan_${spark.binary.version}</artifactId> <version>${project.version}</version> </dependency> </dependencies> diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml index f746060..9d3bb76 100644 --- a/examples/flink/pom.xml +++ b/examples/flink/pom.xml @@ -63,7 +63,7 @@ </dependency> <dependency> <groupId>org.apache.carbondata</groupId> - <artifactId>carbondata-spark</artifactId> + <artifactId>carbondata-spark_${spark.binary.version}</artifactId> <version>${project.version}</version> </dependency> <dependency> diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml index 5f10c06..a28875b 100644 --- a/examples/spark/pom.xml +++ b/examples/spark/pom.xml @@ -36,7 +36,7 @@ <dependencies> <dependency> <groupId>org.apache.carbondata</groupId> - <artifactId>carbondata-spark</artifactId> + <artifactId>carbondata-spark_${spark.binary.version}</artifactId> <version>${project.version}</version> </dependency> <dependency> @@ -202,5 +202,20 @@ <maven.test.skip>true</maven.test.skip> </properties> </profile> + <profile> + <id>spark-2.3</id> + <activation> + <activeByDefault>true</activeByDefault> + </activation> + <properties> + <spark.binary.version>2.3</spark.binary.version> + </properties> + </profile> + <profile> + <id>spark-2.4</id> + <properties> + <spark.binary.version>2.4</spark.binary.version> + </properties> + </profile> </profiles> </project> diff --git a/index/examples/pom.xml b/index/examples/pom.xml index 3fa440b..c634b99 100644 --- a/index/examples/pom.xml +++ b/index/examples/pom.xml @@ -38,7 +38,7 @@ <dependencies> <dependency> <groupId>org.apache.carbondata</groupId> - <artifactId>carbondata-spark</artifactId> + <artifactId>carbondata-spark_${spark.binary.version}</artifactId> <version>${project.version}</version> </dependency> <dependency> @@ -80,4 +80,22 @@ </plugins> </build> + <profiles> + <profile> + <id>spark-2.3</id> + <activation> + <activeByDefault>true</activeByDefault> + </activation> + <properties> + <spark.binary.version>2.3</spark.binary.version> + </properties> + </profile> + <profile> + <id>spark-2.4</id> + <properties> + <spark.binary.version>2.4</spark.binary.version> + </properties> + </profile> + </profiles> + </project> \ No newline at end of file diff --git a/index/secondary-index/pom.xml b/index/secondary-index/pom.xml index b3034f5..f6bd005 100644 --- a/index/secondary-index/pom.xml +++ b/index/secondary-index/pom.xml @@ -37,7 +37,7 @@ <dependencies> <dependency> <groupId>org.apache.carbondata</groupId> - <artifactId>carbondata-spark</artifactId> + <artifactId>carbondata-spark_${spark.binary.version}</artifactId> <version>${project.version}</version> <exclusions> <exclusion> @@ -205,6 +205,21 @@ <maven.test.skip>true</maven.test.skip> </properties> </profile> + <profile> + <id>spark-2.3</id> + <activation> + <activeByDefault>true</activeByDefault> + </activation> + <properties> + <spark.binary.version>2.3</spark.binary.version> + </properties> + </profile> + <profile> + <id>spark-2.4</id> + <properties> + <spark.binary.version>2.4</spark.binary.version> + </properties> + </profile> </profiles> </project> diff --git a/integration/flink/pom.xml b/integration/flink/pom.xml index de69f41..1b37465 100644 --- a/integration/flink/pom.xml +++ b/integration/flink/pom.xml @@ -198,10 +198,13 @@ <activation> <activeByDefault>true</activeByDefault> </activation> + <properties> + <spark.binary.version>2.3</spark.binary.version> + </properties> <dependencies> <dependency> <groupId>org.apache.carbondata</groupId> - <artifactId>carbondata-spark</artifactId> + <artifactId>carbondata-spark_${spark.binary.version}</artifactId> <version>${project.version}</version> <scope>test</scope> <exclusions> @@ -215,10 +218,13 @@ </profile> <profile> <id>spark-2.4</id> + <properties> + <spark.binary.version>2.4</spark.binary.version> + </properties> <dependencies> <dependency> <groupId>org.apache.carbondata</groupId> - <artifactId>carbondata-spark</artifactId> + <artifactId>carbondata-spark_${spark.binary.version}</artifactId> <version>${project.version}</version> <scope>test</scope> <exclusions> diff --git a/integration/spark/pom.xml b/integration/spark/pom.xml index 0b7a1a7..55b739a 100644 --- a/integration/spark/pom.xml +++ b/integration/spark/pom.xml @@ -26,7 +26,7 @@ <relativePath>../../pom.xml</relativePath> </parent> - <artifactId>carbondata-spark</artifactId> + <artifactId>carbondata-spark_${spark.binary.version}</artifactId> <name>Apache CarbonData :: Spark</name> <properties> @@ -39,8 +39,6 @@ <build.directory.projectHadoop>../../hadoop/target</build.directory.projectHadoop> <build.directory.projectFormat>../../format/target</build.directory.projectFormat> <build.directory.projectSpark>../../integration/spark/target</build.directory.projectSpark> - <!--<build.directory.projectHive>../../integration/hive/target</build.directory.projectHive>--> - <!--<build.directory.projectPresto>../../integration/presto/target</build.directory.projectPresto>--> <build.directory.projectStoreSdk>../../sdk/sdk/target</build.directory.projectStoreSdk> <build.directory.projectStreaming>../../streaming/target</build.directory.projectStreaming> <build.directory.projectBloom>../../index/bloom/target</build.directory.projectBloom> @@ -52,8 +50,6 @@ <classes.directory.projectHadoop>../../hadoop/target/classes</classes.directory.projectHadoop> <classes.directory.projectFormat>../../format/target/classes</classes.directory.projectFormat> <classes.directory.projectSpark>../../integration/spark/target/classes</classes.directory.projectSpark> - <!--<classes.directory.projectHive>../../integration/hive/target/classes</classes.directory.projectHive>--> - <!--<classes.directory.projectPresto>../../integration/presto/target/classes</classes.directory.projectPresto>--> <classes.directory.projectStoreSdk>../../sdk/sdk/target/classes</classes.directory.projectStoreSdk> <classes.directory.projectStreaming>../../streaming/target/classes</classes.directory.projectStreaming> <classes.directory.projectBloom>../../index/bloom/target/classes</classes.directory.projectBloom> @@ -66,10 +62,6 @@ <sources.directory.projectFormat>../../format/src/main/thrift</sources.directory.projectFormat> <sources.directory.projectSpark>../../integration/spark/src/main/scala</sources.directory.projectSpark> <sources.directory.projectSpark>../../integration/spark/src/main/java</sources.directory.projectSpark> - <!--<sources.directory.projectHive>../../integration/hive/src/main/java</sources.directory.projectHive>--> - <!--<sources.directory.projectHive>../../integration/hive/src/main/scala</sources.directory.projectHive>--> - <!--<sources.directory.projectPresto>../../integration/presto/src/main/java</sources.directory.projectPresto>--> - <!--<sources.directory.projectPresto>../../integration/presto/src/main/scala</sources.directory.projectPresto>--> <sources.directory.projectStoreSdk>../../sdk/sdk/src/main/java</sources.directory.projectStoreSdk> <sources.directory.projectStreaming>../../streaming/src/main/java</sources.directory.projectStreaming> <sources.directory.projectStreaming>../../streaming/src/main/scala</sources.directory.projectStreaming> @@ -82,8 +74,6 @@ <generated-sources.directory.projectHadoop>../../hadoop/target/generated-sources/annotations</generated-sources.directory.projectHadoop> <generated-sources.directory.projectFormat>../../format/target/generated-sources/annotations</generated-sources.directory.projectFormat> <generated-sources.directory.projectSpark>../../integration/spark/target/generated-sources/annotations</generated-sources.directory.projectSpark> - <!--<generated-sources.directory.projectHive>../../integration/hive/target/generated-sources/annotations</generated-sources.directory.projectHive>--> - <!--<generated-sources.directory.projectPresto>../../integration/presto/target/generated-sources/annotations</generated-sources.directory.projectPresto>--> <generated-sources.directory.projectStoreSdk>../../sdk/sdk/target/generated-sources/annotations</generated-sources.directory.projectStoreSdk> <generated-sources.directory.projectStreaming>../../streaming/target/generated-sources/annotations</generated-sources.directory.projectStreaming> <generated-sources.directory.projectBloom>../../index/bloom/target/generated-sources/annotations</generated-sources.directory.projectBloom> @@ -139,7 +129,7 @@ </dependency> <dependency> <groupId>org.apache.carbondata</groupId> - <artifactId>carbondata-mv-plan</artifactId> + <artifactId>carbondata-mv-plan_${spark.binary.version}</artifactId> <version>${project.version}</version> </dependency> <!-- spark --> @@ -495,6 +485,9 @@ <activation> <activeByDefault>true</activeByDefault> </activation> + <properties> + <spark.binary.version>2.3</spark.binary.version> + </properties> <build> <plugins> <plugin> @@ -530,6 +523,9 @@ </profile> <profile> <id>spark-2.4</id> + <properties> + <spark.binary.version>2.4</spark.binary.version> + </properties> <build> <plugins> <plugin> diff --git a/mv/plan/pom.xml b/mv/plan/pom.xml index da9080c..a26ce4c 100644 --- a/mv/plan/pom.xml +++ b/mv/plan/pom.xml @@ -26,7 +26,7 @@ <relativePath>../../pom.xml</relativePath> </parent> - <artifactId>carbondata-mv-plan</artifactId> + <artifactId>carbondata-mv-plan_${spark.binary.version}</artifactId> <name>Apache CarbonData :: Materialized View Plan</name> <properties> @@ -157,6 +157,9 @@ <activation> <activeByDefault>true</activeByDefault> </activation> + <properties> + <spark.binary.version>2.3</spark.binary.version> + </properties> <build> <plugins> <plugin> @@ -192,6 +195,9 @@ </profile> <profile> <id>spark-2.4</id> + <properties> + <spark.binary.version>2.4</spark.binary.version> + </properties> <build> <plugins> <plugin> diff --git a/pom.xml b/pom.xml index f603b23..0a9db3f 100644 --- a/pom.xml +++ b/pom.xml @@ -538,6 +538,7 @@ <activeByDefault>true</activeByDefault> </activation> <properties> + <spark.binary.version>2.3</spark.binary.version> <spark.version>2.3.4</spark.version> <scala.binary.version>2.11</scala.binary.version> <scala.version>2.11.8</scala.version> @@ -545,6 +546,31 @@ <build> <plugins> <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>flatten-maven-plugin</artifactId> + <!--<version>1.2.2</version>--> + <configuration> + </configuration> + <executions> + <!-- enable flattening --> + <execution> + <id>flatten</id> + <phase>process-resources</phase> + <goals> + <goal>flatten</goal> + </goals> + </execution> + <!-- ensure proper cleanup --> + <execution> + <id>flatten.clean</id> + <phase>clean</phase> + <goals> + <goal>clean</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> <groupId>org.eluder.coveralls</groupId> <artifactId>coveralls-maven-plugin</artifactId> <version>4.3.0</version> @@ -583,6 +609,7 @@ <profile> <id>spark-2.4</id> <properties> + <spark.binary.version>2.4</spark.binary.version> <spark.version>2.4.5</spark.version> <scala.binary.version>2.11</scala.binary.version> <scala.version>2.11.12</scala.version> @@ -590,6 +617,31 @@ <build> <plugins> <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>flatten-maven-plugin</artifactId> + <!--<version>1.2.2</version>--> + <configuration> + </configuration> + <executions> + <!-- enable flattening --> + <execution> + <id>flatten</id> + <phase>process-resources</phase> + <goals> + <goal>flatten</goal> + </goals> + </execution> + <!-- ensure proper cleanup --> + <execution> + <id>flatten.clean</id> + <phase>clean</phase> + <goals> + <goal>clean</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> <groupId>org.eluder.coveralls</groupId> <artifactId>coveralls-maven-plugin</artifactId> <version>4.3.0</version>