Repository: incubator-hivemall Updated Branches: refs/heads/master cb16a3944 -> bba252ac1 (forced update)
Close #72: [HIVEMALL-86] Updated Hadoop version dependencies from cdh3 to v2.4.0 Project: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/commit/bba252ac Tree: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/tree/bba252ac Diff: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/diff/bba252ac Branch: refs/heads/master Commit: bba252ac10fccda022b630e3137460dd8d2f9302 Parents: 8aae974 Author: myui <yuin...@gmail.com> Authored: Tue Apr 18 18:56:54 2017 +0900 Committer: myui <yuin...@gmail.com> Committed: Tue Apr 18 18:56:54 2017 +0900 ---------------------------------------------------------------------- bin/spark-shell | 2 +- core/pom.xml | 16 +++++++++---- docs/gitbook/getting_started/installation.md | 16 +++++++++++-- mixserv/pom.xml | 16 +++++++++---- nlp/pom.xml | 16 +++++++++---- pom.xml | 28 ++++++++++------------- spark/spark-2.0/pom.xml | 2 +- spark/spark-2.1/pom.xml | 2 +- spark/spark-common/pom.xml | 10 ++++++-- xgboost/pom.xml | 10 ++++++-- 10 files changed, 78 insertions(+), 40 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/bba252ac/bin/spark-shell ---------------------------------------------------------------------- diff --git a/bin/spark-shell b/bin/spark-shell index 5dcd5d5..199e001 100755 --- a/bin/spark-shell +++ b/bin/spark-shell @@ -71,7 +71,7 @@ install_app() { # install Spark under the bin/ folder if needed. install_spark() { local SPARK_VERSION=`grep "<spark.version>" "${_DIR}/../pom.xml" | head -n1 | awk -F '[<>]' '{print $3}'` - local HADOOP_VERSION=`grep "<hadoop.version>" "${_DIR}/../pom.xml" | head -n1 | awk -F '[<>]' '{print $3}'` + local HADOOP_VERSION=`grep "<hadoop.version>" "${_DIR}/../pom.xml" | head -n1 | awk -F '[<>]' '{print $3}' | cut -d '.' -f1-2` local SPARK_DIR="${_DIR}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}" local APACHE_MIRROR=${APACHE_MIRROR:-'http://d3kbcqa49mib13.cloudfront.net'} http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/bba252ac/core/pom.xml ---------------------------------------------------------------------- diff --git a/core/pom.xml b/core/pom.xml index d7655f4..9368993 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -39,8 +39,14 @@ <!-- provided scope --> <dependency> <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-core</artifactId> - <version>0.20.2-cdh3u6</version> + <artifactId>hadoop-common</artifactId> + <version>${hadoop.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-core</artifactId> + <version>${hadoop.version}</version> <scope>provided</scope> </dependency> <dependency> @@ -92,9 +98,9 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>org.apache.hadoop.thirdparty.guava</groupId> + <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> - <version>r09-jarjar</version> + <version>${guava.version}</version> <scope>provided</scope> </dependency> @@ -141,7 +147,7 @@ <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> - <version>4.12</version> + <version>${junit.version}</version> <scope>test</scope> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/bba252ac/docs/gitbook/getting_started/installation.md ---------------------------------------------------------------------- diff --git a/docs/gitbook/getting_started/installation.md b/docs/gitbook/getting_started/installation.md index 3a3c97f..896d247 100644 --- a/docs/gitbook/getting_started/installation.md +++ b/docs/gitbook/getting_started/installation.md @@ -20,7 +20,8 @@ Prerequisites ============ -* Hive v0.12 or later +* Hadoop v2.4.0 or later +* Hive v0.13 or later * Java 7 or later * [hivemall-core-xxx-with-dependencies.jar](https://github.com/myui/hivemall/releases) * [define-all.hive](https://github.com/myui/hivemall/releases) @@ -41,4 +42,15 @@ This automatically loads all Hivemall functions every time you start a Hive sess $ hive add jar /tmp/hivemall-core-xxx-with-dependencies.jar; source /tmp/define-all.hive; -``` \ No newline at end of file +``` + +Build from Source +================== + +```sh +$ git clone https://github.com/apache/incubator-hivemall.git +$ cd incubator-hivemall +$ bin/build.sh +``` + +Then, you can find hivemall jars in `./target`. http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/bba252ac/mixserv/pom.xml ---------------------------------------------------------------------- diff --git a/mixserv/pom.xml b/mixserv/pom.xml index 41ba401..0e0e83c 100644 --- a/mixserv/pom.xml +++ b/mixserv/pom.xml @@ -39,8 +39,14 @@ <!-- provided scope --> <dependency> <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-core</artifactId> - <version>0.20.2-cdh3u6</version> + <artifactId>hadoop-common</artifactId> + <version>${hadoop.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-core</artifactId> + <version>${hadoop.version}</version> <scope>provided</scope> </dependency> <dependency> @@ -74,9 +80,9 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>org.apache.hadoop.thirdparty.guava</groupId> + <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> - <version>r09-jarjar</version> + <version>${guava.version}</version> <scope>provided</scope> </dependency> @@ -124,7 +130,7 @@ <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> - <version>4.12</version> + <version>${junit.version}</version> <scope>test</scope> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/bba252ac/nlp/pom.xml ---------------------------------------------------------------------- diff --git a/nlp/pom.xml b/nlp/pom.xml index c7d2cef..b6ea409 100644 --- a/nlp/pom.xml +++ b/nlp/pom.xml @@ -39,8 +39,14 @@ <!-- provided scope --> <dependency> <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-core</artifactId> - <version>0.20.2-cdh3u6</version> + <artifactId>hadoop-common</artifactId> + <version>${hadoop.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-core</artifactId> + <version>${hadoop.version}</version> <scope>provided</scope> </dependency> <dependency> @@ -92,9 +98,9 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>org.apache.hadoop.thirdparty.guava</groupId> + <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> - <version>r09-jarjar</version> + <version>${guava.version}</version> <scope>provided</scope> </dependency> <dependency> @@ -116,7 +122,7 @@ <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> - <version>4.12</version> + <version>${junit.version}</version> <scope>test</scope> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/bba252ac/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 7743d5a..63abe87 100644 --- a/pom.xml +++ b/pom.xml @@ -25,7 +25,7 @@ <version>0.4.2-rc.2</version> <name>Apache Hivemall</name> - <description>Scalable Machine Learning Library for Apache Hive</description> + <description>Scalable Machine Learning Library for Apache Hive, Apache Spark, and Apache Pig</description> <url>http://hivemall.incubator.apache.org/</url> <inceptionYear>2013</inceptionYear> <organization> @@ -158,8 +158,8 @@ <name>Takuya Kitazawa</name> <email>takuti[at]apache.org</email> <url>https://github.com/takuti</url> - <organization>The University of Tokyo</organization> - <organizationUrl>http://www.u-tokyo.ac.jp/</organizationUrl> + <organization>Treasure Data, Inc.</organization> + <organizationUrl>https://www.treasuredata.com/</organizationUrl> <roles> <role>Committer</role> </roles> @@ -241,25 +241,23 @@ </modules> <properties> + <java.source.version>1.7</java.source.version> + <java.target.version>1.7</java.target.version> + <scala.version>2.11.8</scala.version> + <scala.binary.version>2.11</scala.binary.version> <maven.build.timestamp.format>yyyy</maven.build.timestamp.format> <build.year>${maven.build.timestamp}</build.year> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <protobuf.version>2.5.0</protobuf.version> <protoc.path>${env.PROTOC_PATH}</protoc.path> + <hadoop.version>2.4.0</hadoop.version> <hive.version>0.13.0</hive.version> - <scala.version>2.11.8</scala.version> + <guava.version>11.0.2</guava.version> + <junit.version>4.12</junit.version> <dependency.locations.enabled>false</dependency.locations.enabled> - <scala.binary.version>2.11</scala.binary.version> <main.basedir>${project.basedir}</main.basedir> </properties> - <repositories> - <repository> - <id>cloudera</id> - <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url> - </repository> - </repositories> - <distributionManagement> <snapshotRepository> <id>ossrh</id> @@ -275,7 +273,6 @@ <module>spark/spark-common</module> </modules> <properties> - <hadoop.version>2.7</hadoop.version> <spark.version>2.1.0</spark.version> <spark.binary.version>2.1</spark.binary.version> </properties> @@ -287,7 +284,6 @@ <module>spark/spark-common</module> </modules> <properties> - <hadoop.version>2.7</hadoop.version> <spark.version>2.0.2</spark.version> <spark.binary.version>2.0</spark.binary.version> </properties> @@ -480,8 +476,8 @@ <artifactId>maven-compiler-plugin</artifactId> <version>3.1</version> <configuration> - <source>1.7</source> - <target>1.7</target> + <source>${java.source.version}</source> + <target>${java.target.version}</target> <debug>true</debug> <debuglevel>lines,vars,source</debuglevel> <encoding>UTF-8</encoding> http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/bba252ac/spark/spark-2.0/pom.xml ---------------------------------------------------------------------- diff --git a/spark/spark-2.0/pom.xml b/spark/spark-2.0/pom.xml index f0937d7..123c424 100644 --- a/spark/spark-2.0/pom.xml +++ b/spark/spark-2.0/pom.xml @@ -234,7 +234,7 @@ <junitxml>.</junitxml> <filereports>SparkTestSuite.txt</filereports> <argLine>-ea -Xmx2g -XX:MaxPermSize=${MaxPermGen} -XX:ReservedCodeCacheSize=${CodeCacheSize}</argLine> - <stderr/> + <stderr /> <environmentVariables> <SPARK_PREPEND_CLASSES>1</SPARK_PREPEND_CLASSES> <SPARK_SCALA_VERSION>${scala.binary.version}</SPARK_SCALA_VERSION> http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/bba252ac/spark/spark-2.1/pom.xml ---------------------------------------------------------------------- diff --git a/spark/spark-2.1/pom.xml b/spark/spark-2.1/pom.xml index a0f380f..22d3e12 100644 --- a/spark/spark-2.1/pom.xml +++ b/spark/spark-2.1/pom.xml @@ -234,7 +234,7 @@ <junitxml>.</junitxml> <filereports>SparkTestSuite.txt</filereports> <argLine>-ea -Xmx2g -XX:MaxPermSize=${MaxPermGen} -XX:ReservedCodeCacheSize=${CodeCacheSize}</argLine> - <stderr/> + <stderr /> <environmentVariables> <SPARK_PREPEND_CLASSES>1</SPARK_PREPEND_CLASSES> <SPARK_SCALA_VERSION>${scala.binary.version}</SPARK_SCALA_VERSION> http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/bba252ac/spark/spark-common/pom.xml ---------------------------------------------------------------------- diff --git a/spark/spark-common/pom.xml b/spark/spark-common/pom.xml index 7d76ea4..e8e8ff4 100644 --- a/spark/spark-common/pom.xml +++ b/spark/spark-common/pom.xml @@ -69,8 +69,14 @@ <dependency> <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-core</artifactId> - <version>0.20.2-cdh3u6</version> + <artifactId>hadoop-common</artifactId> + <version>${hadoop.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-core</artifactId> + <version>${hadoop.version}</version> <scope>provided</scope> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/bba252ac/xgboost/pom.xml ---------------------------------------------------------------------- diff --git a/xgboost/pom.xml b/xgboost/pom.xml index cd5dc0d..853edb9 100644 --- a/xgboost/pom.xml +++ b/xgboost/pom.xml @@ -41,8 +41,14 @@ <!-- provided scope --> <dependency> <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-core</artifactId> - <version>0.20.2-cdh3u6</version> + <artifactId>hadoop-common</artifactId> + <version>${hadoop.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-core</artifactId> + <version>${hadoop.version}</version> <scope>provided</scope> </dependency> <dependency>
