Repository: mahout Updated Branches: refs/heads/master fe4cb1a09 -> d9b32f308
MAHOUT-1999 Multi-artifact build closes apache/mahout#350 Project: http://git-wip-us.apache.org/repos/asf/mahout/repo Commit: http://git-wip-us.apache.org/repos/asf/mahout/commit/d9b32f30 Tree: http://git-wip-us.apache.org/repos/asf/mahout/tree/d9b32f30 Diff: http://git-wip-us.apache.org/repos/asf/mahout/diff/d9b32f30 Branch: refs/heads/master Commit: d9b32f308804859b382e855d5a323a62ba7153e1 Parents: fe4cb1a Author: Trevor a.k.a @rawkintrevo <[email protected]> Authored: Fri Nov 17 17:04:24 2017 -0600 Committer: Trevor a.k.a @rawkintrevo <[email protected]> Committed: Fri Nov 17 17:04:24 2017 -0600 ---------------------------------------------------------------------- .travis.yml | 19 +-- buildtools/build-all-release-jars.sh | 30 ----- buildtools/change-scala-version.sh | 75 +++++++++++ distribution/pom.xml | 173 +++++++++++++++++++++++++ flink/pom.xml | 2 +- h2o/pom.xml | 2 +- math-scala/pom.xml | 2 +- pom.xml | 13 +- spark/pom.xml | 2 +- viennacl-omp/pom.xml | 2 +- viennacl/pom.xml | 2 +- website/front/community/buildingmahout.md | 69 +++++++++- 12 files changed, 342 insertions(+), 49 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mahout/blob/d9b32f30/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index 09e49c5..8ad2458 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,39 +44,39 @@ matrix: include: # Build Spark 1.6.3 , Scala 2.10 - jdk: "openjdk7" - env: PROFILES="${PROFILES} -Pscala-2.10 -Pspark-1.6" SPARK_BIN=$SPARK_1_6 + env: PROFILES="${PROFILES} -Pscala-2.10 -Pspark-1.6" SPARK_BIN=$SPARK_1_6 SCALA_VERSION="2.10" # Build Spark 2.0.2 , Scala 2.11 - replace -D... with profiles when available - jdk: "openjdk7" - env: PROFILES="${PROFILES} -Pspark-2.0 -Pscala-2.11" SPARK_BIN=$SPARK_2_0 + env: PROFILES="${PROFILES} -Pspark-2.0 -Pscala-2.11" SPARK_BIN=$SPARK_2_0 SCALA_VERSION="2.11" # Build Spark 2.1.0 , Scala 2.11 - replace -D... with profiles when available - jdk: "openjdk7" - env: PROFILES="${PROFILES} -Pspark-2.1 -Pscala-2.11" SPARK_BIN=$SPARK_2_1 + env: PROFILES="${PROFILES} -Pspark-2.1 -Pscala-2.11" SPARK_BIN=$SPARK_2_1 SCALA_VERSION="2.11" # Build Spark 1.6.3 , Scala 2.10, ViennaCL - jdk: "openjdk7" - env: PROFILES="${PROFILES} -Pscala-2.10 -Pviennacl" SPARK_BIN=$SPARK_1_6 + env: PROFILES="${PROFILES} -Pscala-2.10 -Pviennacl" SPARK_BIN=$SPARK_1_6 SCALA_VERSION="2.10" # Build Spark 2.0.2 , Scala 2.11, ViennaCL - replace -D... with profiles when available - jdk: "openjdk7" - env: PROFILES="${PROFILES} -Pspark-2.0 -Pscala-2.11 -Pviennacl" SPARK_BIN=$SPARK_2_0 + env: PROFILES="${PROFILES} -Pspark-2.0 -Pscala-2.11 -Pviennacl" SPARK_BIN=$SPARK_2_0 SCALA_VERSION="2.11" # Build Spark 2.1.0 , Scala 2.11, ViennaCL - replace -D... with profiles when available - jdk: "openjdk7" - env: PROFILES="${PROFILES} -Pspark-2.1 -Pscala-2.11 -Pviennacl" SPARK_BIN=$SPARK_2_1 + env: PROFILES="${PROFILES} -Pspark-2.1 -Pscala-2.11 -Pviennacl" SPARK_BIN=$SPARK_2_1 SCALA_VERSION="2.11" # Build Spark 1.6.3 , Scala 2.10, ViennaCL-OMP - jdk: "openjdk7" - env: PROFILES="${PROFILES} -Pscala-2.10 -Pviennacl-omp" TEST_MODULES="${TEST_MODULES},viennacl-omp" SPARK_BIN=$SPARK_1_6 + env: PROFILES="${PROFILES} -Pscala-2.10 -Pviennacl-omp" TEST_MODULES="${TEST_MODULES},viennacl-omp" SPARK_BIN=$SPARK_1_6 SCALA_VERSION="2.10" # Build Spark 2.0.2 , Scala 2.11, ViennaCL-OMP - replace -D... with profiles when available - jdk: "openjdk7" - env: PROFILES="${PROFILES} -Pspark-2.0 -Pscala-2.11 -Pviennacl-omp" TEST_MODULES="${TEST_MODULES},viennacl-omp" SPARK_BIN=$SPARK_2_0 + env: PROFILES="${PROFILES} -Pspark-2.0 -Pscala-2.11 -Pviennacl-omp" TEST_MODULES="${TEST_MODULES},viennacl-omp" SPARK_BIN=$SPARK_2_0 SCALA_VERSION="2.11" # Build Spark 2.1.0 , Scala 2.11, ViennaCL-OMP - replace -D... with profiles when available - jdk: "openjdk7" - env: PROFILES="${PROFILES} -Pspark-2.1 -Pscala-2.11 -Pviennacl-omp" TEST_MODULES="${TEST_MODULES},viennacl-omp" SPARK_BIN=$SPARK_2_1 + env: PROFILES="${PROFILES} -Pspark-2.1 -Pscala-2.11 -Pviennacl-omp" TEST_MODULES="${TEST_MODULES},viennacl-omp" SPARK_BIN=$SPARK_2_1 SCALA_VERSION="2.11" git: depth: 10 @@ -107,6 +107,7 @@ before_install: script: + - buildtools/change-scala-version.sh $SCALA_VERSION # Build Mahout - mvn clean package $PROFILES $STANDARD_BUILD_OPTS -DskipTests # Start Spark http://git-wip-us.apache.org/repos/asf/mahout/blob/d9b32f30/buildtools/build-all-release-jars.sh ---------------------------------------------------------------------- diff --git a/buildtools/build-all-release-jars.sh b/buildtools/build-all-release-jars.sh deleted file mode 100755 index cdb87b6..0000000 --- a/buildtools/build-all-release-jars.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# This file compiles all varients of scala and spark required for -# release assemblies -# - -cd ../ -mvn clean -mvn -Pscala-2.10,spark-1.6,viennacl,viennacl-omp -DskipTests -mvn -Pscala-2.11,spark-2.0,viennacl,viennacl-omp -DskipTests -cd spark -mvn -Pscala-2.11,spark-2.1 -DskipTests \ No newline at end of file http://git-wip-us.apache.org/repos/asf/mahout/blob/d9b32f30/buildtools/change-scala-version.sh ---------------------------------------------------------------------- diff --git a/buildtools/change-scala-version.sh b/buildtools/change-scala-version.sh new file mode 100755 index 0000000..b61af80 --- /dev/null +++ b/buildtools/change-scala-version.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# A BIG Shoutout to the Bros and Bro-ettes at Apache Spark for this + +set -e + +VALID_VERSIONS=( 2.10 2.11 ) + +usage() { + echo "Usage: $(basename $0) [-h|--help] <version> +where : + -h| --help Display this help text + valid version values : ${VALID_VERSIONS[*]} +" 1>&2 + exit 1 +} + +if [[ ($# -ne 1) || ( $1 == "--help") || $1 == "-h" ]]; then + usage +fi + +TO_VERSION=$1 + +check_scala_version() { + for i in ${VALID_VERSIONS[*]}; do [ $i = "$1" ] && return 0; done + echo "Invalid Scala version: $1. Valid versions: ${VALID_VERSIONS[*]}" 1>&2 + exit 1 +} + +check_scala_version "$TO_VERSION" + +if [ $TO_VERSION = "2.11" ]; then + FROM_VERSION="2.10" +else + FROM_VERSION="2.11" +fi + +sed_i() { + sed -e "$1" "$2" > "$2.tmp" && mv "$2.tmp" "$2" +} + +export -f sed_i + +BASEDIR=$(dirname $0)/.. +find "$BASEDIR" -name 'pom.xml' -not -path '*target*' -print \ + -exec bash -c "sed_i 's/\(artifactId.*\)_'$FROM_VERSION'/\1_'$TO_VERSION'/g' {}" \; + +# Also update <scala.binary.version> in parent POM +# Match any scala binary version to ensure idempotency +sed_i '1,/<scala\.compat\.version>[0-9]*\.[0-9]*</s/<scala\.compat\.version>[0-9]*\.[0-9]*</<scala.compat.version>'$TO_VERSION'</' \ + "$BASEDIR/pom.xml" + +# +# Mahout doesn't need this until we get our website acting right. +# +## Update source of scaladocs +#echo "$BASEDIR/docs/_plugins/copy_api_dirs.rb" +#sed_i 's/scala\-'$FROM_VERSION'/scala\-'$TO_VERSION'/' "$BASEDIR/docs/_plugins/copy_api_dirs.rb" http://git-wip-us.apache.org/repos/asf/mahout/blob/d9b32f30/distribution/pom.xml ---------------------------------------------------------------------- diff --git a/distribution/pom.xml b/distribution/pom.xml index bbf6fa0..c7a3fc5 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -29,10 +29,125 @@ <packaging>pom</packaging> <properties> <mahout.skip.distribution>true</mahout.skip.distribution> + <scala.210.version>2.10.6</scala.210.version> + <scala.211.version>2.11.8</scala.211.version> + <lifecycle.target>package</lifecycle.target> </properties> <build> <defaultGoal>install</defaultGoal> <plugins> + + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <version>1.1.1</version> + <executions> + <execution> + <id>change-to-scala-2.11</id> + <phase>compile</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <workingDirectory>../buildtools</workingDirectory> + <executable>./change-scala-version.sh</executable> + <arguments> + <argument>2.11</argument> + </arguments> + </configuration> + </execution> + <execution> + <id>mahout-math-scala-2.11</id> + <phase>compile</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <workingDirectory>../math-scala</workingDirectory> + <executable>mvn</executable> + <arguments> + <argument>${lifecycle.target}</argument> + <argument>-Dscala.version=${scala.211.version}</argument> + <argument>-Dscala.compat.version=2.11</argument> + <argument>-DskipTests</argument> + </arguments> + </configuration> + </execution> + <execution> + <id>viennacl-2.11</id> + <phase>compile</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <workingDirectory>../viennacl</workingDirectory> + <executable>mvn</executable> + <arguments> + <argument>${lifecycle.target}</argument> + <argument>-Dscala.version=${scala.211.version}</argument> + <argument>-Dscala.compat.version=2.11</argument> + <argument>-DskipTests</argument> + </arguments> + </configuration> + </execution> + <execution> + <id>viennacl-omp-2.11</id> + <phase>compile</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <workingDirectory>../viennacl-omp</workingDirectory> + <executable>mvn</executable> + <arguments> + <argument>${lifecycle.target}</argument> + <argument>-Dscala.version=${scala.211.version}</argument> + <argument>-Dscala.compat.version=2.11</argument> + <argument>-DskipTests</argument> + </arguments> + </configuration> + </execution> + <execution> + <id>spark-2.0</id> + <phase>compile</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <workingDirectory>../spark</workingDirectory> + <executable>mvn</executable> + <arguments> + <argument>${lifecycle.target}</argument> + <argument>-Dspark.version=2.0.2</argument> + <argument>-Dspark.compat.version=2.0</argument> + <argument>-Dscala.version=${scala.211.version}</argument> + <argument>-Dscala.compat.version=2.11</argument> + <argument>-DskipTests</argument> + </arguments> + </configuration> + </execution> + <execution> + <id>spark-2.1</id> + <phase>compile</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <workingDirectory>../spark</workingDirectory> + <executable>mvn</executable> + <arguments> + <argument>${lifecycle.target}</argument> + <argument>-Dspark.version=2.1.1</argument> + <argument>-Dspark.compat.version=2.1</argument> + <argument>-Dscala.version=${scala.211.version}</argument> + <argument>-Dscala.compat.version=2.11</argument> + <argument>-DskipTests</argument> + </arguments> + </configuration> + </execution> + </executions> + </plugin> + <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-assembly-plugin</artifactId> @@ -100,6 +215,7 @@ </execution> </executions> </plugin> + </plugins> </build> <profiles> @@ -155,6 +271,34 @@ <version>${project.version}</version> </dependency> </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <version>1.1.1</version> + <executions> + <execution> + <id>viennacl-2.11</id> + <phase>compile</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <workingDirectory>../viennacl</workingDirectory> + <executable>mvn</executable> + <arguments> + <argument>${lifecycle.target}</argument> + <argument>-Dscala.version=${scala.211.version}</argument> + <argument>-Dscala.compat.version=2.11</argument> + <argument>-DskipTests</argument> + </arguments> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> </profile> <profile> @@ -196,6 +340,35 @@ <version>${project.version}</version> </dependency> </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <version>1.1.1</version> + <executions> + <execution> + <id>viennacl-omp-2.11</id> + <phase>compile</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <workingDirectory>../viennacl-omp</workingDirectory> + <executable>mvn</executable> + <arguments> + <argument>${lifecycle.target}</argument> + <argument>-Dscala.version=${scala.211.version}</argument> + <argument>-Dscala.compat.version=2.11</argument> + <argument>-DskipTests</argument> + </arguments> + </configuration> + </execution> + + </executions> + </plugin> + </plugins> + </build> </profile> </profiles> http://git-wip-us.apache.org/repos/asf/mahout/blob/d9b32f30/flink/pom.xml ---------------------------------------------------------------------- diff --git a/flink/pom.xml b/flink/pom.xml index 4d21166..bc2b2d9 100644 --- a/flink/pom.xml +++ b/flink/pom.xml @@ -27,7 +27,7 @@ <relativePath>../pom.xml</relativePath> </parent> - <artifactId>mahout-flink_${scala.compat.version}</artifactId> + <artifactId>mahout-flink_2.10</artifactId> <name>Mahout Flink bindings</name> <description> Mahout Bindings for Apache Flink http://git-wip-us.apache.org/repos/asf/mahout/blob/d9b32f30/h2o/pom.xml ---------------------------------------------------------------------- diff --git a/h2o/pom.xml b/h2o/pom.xml index 22aa1a3..d911f5b 100644 --- a/h2o/pom.xml +++ b/h2o/pom.xml @@ -27,7 +27,7 @@ <relativePath>../pom.xml</relativePath> </parent> - <artifactId>mahout-h2o_${scala.compat.version}</artifactId> + <artifactId>mahout-h2o_2.10</artifactId> <name>Mahout H2O backend</name> <description> H2O Backend for Mahout DSL http://git-wip-us.apache.org/repos/asf/mahout/blob/d9b32f30/math-scala/pom.xml ---------------------------------------------------------------------- diff --git a/math-scala/pom.xml b/math-scala/pom.xml index 6b5f85d..0d74e32 100644 --- a/math-scala/pom.xml +++ b/math-scala/pom.xml @@ -27,7 +27,7 @@ <relativePath>../pom.xml</relativePath> </parent> - <artifactId>mahout-math-scala_${scala.compat.version}</artifactId> + <artifactId>mahout-math-scala_2.10</artifactId> <name>Mahout Math Scala bindings</name> <description>High performance scientific and technical computing data structures and methods, mostly based on CERN's http://git-wip-us.apache.org/repos/asf/mahout/blob/d9b32f30/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 419d290..a8a29ff 100644 --- a/pom.xml +++ b/pom.xml @@ -586,7 +586,7 @@ <version>2.5.2</version> <configuration> <useReleaseProfile>true</useReleaseProfile> - <releaseProfiles>mahout-release,apache-release,hadoop2</releaseProfiles> + <releaseProfiles>viennacl,viennacl-omp,distribution,mahout-release,apache-release</releaseProfiles> <goals>deploy</goals> <autoVersionSubmodules>true</autoVersionSubmodules> <arguments>${arguments}</arguments> @@ -842,13 +842,22 @@ <module>mr</module> <module>integration</module> <module>examples</module> - <module>distribution</module> + <module>math-scala</module> <module>spark</module> <module>h2o</module> </modules> <profiles> + <profile> + <id>distribution</id> + <activation> + <activeByDefault>false</activeByDefault> + </activation> + <modules> + <module>distribution</module> + </modules> + </profile> <profile> <id>flink</id> http://git-wip-us.apache.org/repos/asf/mahout/blob/d9b32f30/spark/pom.xml ---------------------------------------------------------------------- diff --git a/spark/pom.xml b/spark/pom.xml index dc26f6a..eba7ee8 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -27,7 +27,7 @@ <relativePath>../pom.xml</relativePath> </parent> - <artifactId>mahout-spark_${scala.compat.version}</artifactId> + <artifactId>mahout-spark_2.10</artifactId> <name>Mahout Spark bindings</name> <description> Mahout Bindings for Apache Spark http://git-wip-us.apache.org/repos/asf/mahout/blob/d9b32f30/viennacl-omp/pom.xml ---------------------------------------------------------------------- diff --git a/viennacl-omp/pom.xml b/viennacl-omp/pom.xml index 5fe5bd6..4f66cb9 100644 --- a/viennacl-omp/pom.xml +++ b/viennacl-omp/pom.xml @@ -27,7 +27,7 @@ <relativePath>../pom.xml</relativePath> </parent> - <artifactId>mahout-native-viennacl-omp_${scala.compat.version}</artifactId> + <artifactId>mahout-native-viennacl-omp_2.10</artifactId> <name>Mahout Native ViennaCL OpenMP Bindings</name> <description>Native Structures and interfaces to be used from Mahout math-scala. http://git-wip-us.apache.org/repos/asf/mahout/blob/d9b32f30/viennacl/pom.xml ---------------------------------------------------------------------- diff --git a/viennacl/pom.xml b/viennacl/pom.xml index 9c8d302..629c524 100644 --- a/viennacl/pom.xml +++ b/viennacl/pom.xml @@ -27,7 +27,7 @@ <relativePath>../pom.xml</relativePath> </parent> - <artifactId>mahout-native-viennacl_${scala.compat.version}</artifactId> + <artifactId>mahout-native-viennacl_2.10</artifactId> <name>Mahout Native ViennaCL OpenCL Bindings</name> <description>Native Structures and interfaces to be used from Mahout math-scala. http://git-wip-us.apache.org/repos/asf/mahout/blob/d9b32f30/website/front/community/buildingmahout.md ---------------------------------------------------------------------- diff --git a/website/front/community/buildingmahout.md b/website/front/community/buildingmahout.md index f6e33d2..40b509b 100644 --- a/website/front/community/buildingmahout.md +++ b/website/front/community/buildingmahout.md @@ -110,7 +110,7 @@ These should be added to the your ~/.bashrc file. ###### Building Mahout with Apache Maven -Currently Mahout has 3 builds. From the $MAHOUT_HOME directory we may issue the commands to build each using mvn profiles. +From the $MAHOUT_HOME directory we may issue the commands to build each using mvn profiles. JVM only: ``` @@ -125,4 +125,69 @@ JVM with native OpenMP and OpenCL for Level 2 and level 3 matrix/vector Multipli ``` mvn clean install -Pviennacl -Phadoop2 -DskipTests ``` - \ No newline at end of file + +### Changing Scala Version + +To change the Scala version used it is possible to use profiles, however the resulting artifacts seem to have trouble being resolved with SBT. + +```bash +mvn clean install -Pscala-2.11 +``` + +Maven is able to resolve the resulting artifacts effectively, this will also work if the goal is simply to use the Mahout-Shell. However if the goal is to build with SBT, the following tool should be used + +```bash +cd $MAHOUT_HOME/buildtools +./change-scala-version.sh 2.11 +``` + +Now go back to `$MAHOUT_HOME` and execute + +```bash +mvn clean install -Pscala-2.11 +``` + +**NOTE:** you still need to pass the `-Pscala-2.11` profile, as this determines and propegates the minor scala version (e.g. 2.11.8) + + +### The Distribution Profile + +The distribution profile, among other things, will produce the same artifact for multiple Scala and Spark versions. + +Specifically, in addition to creating all of the + +Default Targets: +- Spark 1.6 Bindings, Scala-2.10 +- Mahout-Math Scala-2.10 +- ViennaCL Scala-2.10* +- ViennaCL-OMP Scala-2.10* +- H2O Scala-2.10 + +It will also create: +- Spark 2.0 Bindings, Scala-2.11 +- Spark 2.1 Bindings, Scala-2.11 +- Mahout-Math Scala-2.11 +- ViennaCL Scala-2.11* +- ViennaCL-OMP Scala-2.11* +- H2O Scala-2.11 + +Note: * ViennaCLs are only created if the `viennacl` or `viennacl-omp` profiles are activated. + +By default, this phase will execute the `package` lifecycle goal on all built "extra" varients. + +E.g. if you were to run + +```bash +mvn clean install -Pdistribution +``` + +You will `install` all of the "Default Targets" but only `package` the "Also created". + +If you wish to `install` all of the above, you can set the `lifecycle.target` switch as follows: + +```bash +mvn clean install -Pdistribution -Dlifecycle.target=install +``` + + +
