bigtop git commit: BIGTOP-1738: Update HBase because javadoc broken

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master d07cf3369 - a61c0312f


BIGTOP-1738: Update HBase because javadoc broken

Signed-off-by: YoungWoo Kim yw...@apache.org


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/a61c0312
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/a61c0312
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/a61c0312

Branch: refs/heads/master
Commit: a61c0312f78e08753a5a940eacf3fe43b9b2a813
Parents: d07cf33
Author: Olaf Flebbe o...@oflebbe.de
Authored: Sun Mar 8 17:36:35 2015 +0100
Committer: YoungWoo Kim yw...@apache.org
Committed: Tue Mar 17 19:54:18 2015 +0900

--
 bigtop-packages/src/deb/hbase/compat|  2 +-
 bigtop-packages/src/deb/hbase/control   |  2 +-
 .../src/deb/hbase/install_init_scripts.sh   | 10 +---
 .../src/deb/hbase/service-postinst.tpl  | 53 
 .../src/deb/hbase/service-postrm.tpl| 53 
 bigtop.mk   |  2 +-
 6 files changed, 5 insertions(+), 117 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/a61c0312/bigtop-packages/src/deb/hbase/compat
--
diff --git a/bigtop-packages/src/deb/hbase/compat 
b/bigtop-packages/src/deb/hbase/compat
index 7f8f011..ec63514 100644
--- a/bigtop-packages/src/deb/hbase/compat
+++ b/bigtop-packages/src/deb/hbase/compat
@@ -1 +1 @@
-7
+9

http://git-wip-us.apache.org/repos/asf/bigtop/blob/a61c0312/bigtop-packages/src/deb/hbase/control
--
diff --git a/bigtop-packages/src/deb/hbase/control 
b/bigtop-packages/src/deb/hbase/control
index 850047f..bfa4218 100644
--- a/bigtop-packages/src/deb/hbase/control
+++ b/bigtop-packages/src/deb/hbase/control
@@ -17,7 +17,7 @@ Section: misc
 Priority: extra
 Maintainer: Bigtop d...@bigtop.apache.org
 Build-Depends: debhelper (= 7.0.50~)
-Standards-Version: 3.8.0
+Standards-Version: 3.9.4
 Homepage: http://hbase.apache.org/
 
 Package: hbase

http://git-wip-us.apache.org/repos/asf/bigtop/blob/a61c0312/bigtop-packages/src/deb/hbase/install_init_scripts.sh
--
diff --git a/bigtop-packages/src/deb/hbase/install_init_scripts.sh 
b/bigtop-packages/src/deb/hbase/install_init_scripts.sh
index 9f0f52a..ad15193 100755
--- a/bigtop-packages/src/deb/hbase/install_init_scripts.sh
+++ b/bigtop-packages/src/deb/hbase/install_init_scripts.sh
@@ -18,23 +18,17 @@
 SRC_PKG=hbase
 for node in master regionserver rest thrift thrift2; do
 service_pkgdir=debian/$SRC_PKG-$node
-debdir=$service_pkgdir/DEBIAN
-mkdir -p $service_pkgdir/etc/init.d/ $debdir
 if [ $node == regionserver ] ; then
 # Region servers start from a different template that allows
 # them to run multiple concurrent instances of the daemon
 template=debian/regionserver-init.d.tpl
 sed -i -e s|@INIT_DEFAULT_START@|2 3 4 5| $template
 sed -i -e s|@INIT_DEFAULT_STOP@|0 1 6| $template
-sed -e s|@HBASE_DAEMON@|$node| -e s|@CHKCONFIG@|2345 87 13| 
$template  $service_pkgdir/etc/init.d/$SRC_PKG-$node
+sed -e s|@HBASE_DAEMON@|$node| -e s|@CHKCONFIG@|2345 87 13| 
$template  debian/hbase-$node.init
 else
 sed -e s|@HBASE_DAEMON@|$node| debian/hbase.svc  
debian/hbase-$node.svc
-bash debian/init.d.tmpl debian/hbase-$node.svc deb 
$service_pkgdir/etc/init.d/$SRC_PKG-$node
+   bash debian/init.d.tmpl debian/hbase-$node.svc deb 
debian/hbase-$node.init 
 fi
-sed -e s|@HBASE_DAEMON@|$node| debian/service-postinst.tpl  
$debdir/postinst
-sed -e s|@HBASE_DAEMON@|$node| debian/service-postrm.tpl  $debdir/postrm
-echo /etc/init.d/$SRC_PKG-$node  $debdir/conffiles
-chmod 755 $debdir/postinst $debdir/postrm $service_pkgdir/etc/init.d*
 
 mkdir -p $service_pkgdir/usr/share/lintian/overrides
 echo $SRC_PKG-$node: new-package-should-close-itp-bug  
$service_pkgdir/usr/share/lintian/overrides/$SRC_PKG-$node

http://git-wip-us.apache.org/repos/asf/bigtop/blob/a61c0312/bigtop-packages/src/deb/hbase/service-postinst.tpl
--
diff --git a/bigtop-packages/src/deb/hbase/service-postinst.tpl 
b/bigtop-packages/src/deb/hbase/service-postinst.tpl
deleted file mode 100644
index b15ef66..000
--- a/bigtop-packages/src/deb/hbase/service-postinst.tpl
+++ /dev/null
@@ -1,53 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You 

bigtop git commit: BIGTOP-1755. Upgrade to Spark 1.3.0

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master e55414e9a - 0d2ac058d


BIGTOP-1755. Upgrade to Spark 1.3.0

Signed-off-by: YoungWoo Kim yw...@apache.org


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/0d2ac058
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/0d2ac058
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/0d2ac058

Branch: refs/heads/master
Commit: 0d2ac058d88fec3b8a82e732c8cd7fa8780f15d1
Parents: e55414e
Author: Jonathan Kelly jonat...@amazon.com
Authored: Wed Mar 11 10:52:50 2015 -0700
Committer: YoungWoo Kim yw...@apache.org
Committed: Fri Mar 20 11:16:48 2015 +0900

--
 bigtop-bigpetstore/bigpetstore-spark/build.gradle | 2 +-
 bigtop.mk | 4 ++--
 pom.xml   | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/0d2ac058/bigtop-bigpetstore/bigpetstore-spark/build.gradle
--
diff --git a/bigtop-bigpetstore/bigpetstore-spark/build.gradle 
b/bigtop-bigpetstore/bigpetstore-spark/build.gradle
index 2f82031..6d5a957 100644
--- a/bigtop-bigpetstore/bigpetstore-spark/build.gradle
+++ b/bigtop-bigpetstore/bigpetstore-spark/build.gradle
@@ -54,7 +54,7 @@ targetCompatibility = 1.7
 
 // Specify any additional project properties.
 ext {
-sparkVersion = 1.2.1
+sparkVersion = 1.3.0
 scalaVersion = 2.10
 }
 

http://git-wip-us.apache.org/repos/asf/bigtop/blob/0d2ac058/bigtop.mk
--
diff --git a/bigtop.mk b/bigtop.mk
index 4cb9526..706942f 100644
--- a/bigtop.mk
+++ b/bigtop.mk
@@ -265,8 +265,8 @@ $(eval $(call PACKAGE,crunch,CRUNCH))
 SPARK_NAME=spark
 SPARK_RELNOTES_NAME=Spark
 SPARK_PKG_NAME=spark-core
-SPARK_BASE_VERSION=1.2.1
-SPARK_PKG_VERSION=1.2.1
+SPARK_BASE_VERSION=1.3.0
+SPARK_PKG_VERSION=1.3.0
 SPARK_RELEASE_VERSION=1
 SPARK_TARBALL_DST=spark-$(SPARK_BASE_VERSION).tar.gz
 SPARK_TARBALL_SRC=spark-$(SPARK_BASE_VERSION).tgz

http://git-wip-us.apache.org/repos/asf/bigtop/blob/0d2ac058/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 906f605..ccc30f5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -49,7 +49,7 @@
 zookeeper.version3.4.5/zookeeper.version
 giraph.version1.0.0/giraph.version
 solr.version4.6.0/solr.version
-spark.version1.2.1/spark.version
+spark.version1.3.0/spark.version
 kafka.version0.8.1.1/kafka.version
 phoenix.version4.2.2/phoenix.version
 spark-smoke.version${project.version}/spark-smoke.version

bigtop git commit: BIGTOP-1742: Add ywkim as a maintainer for Apache Hive, Spark, Sqoop* and Phoenix

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 05a18b235 - 4fd59b243


BIGTOP-1742: Add ywkim as a maintainer for Apache Hive, Spark, Sqoop* and 
Phoenix


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/4fd59b24
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/4fd59b24
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/4fd59b24

Branch: refs/heads/master
Commit: 4fd59b2432e1b2d7eb55f925a7b62d2324304372
Parents: 05a18b2
Author: YoungWoo Kim yw...@apache.org
Authored: Mon Mar 9 17:14:18 2015 +0900
Committer: YoungWoo Kim yw...@apache.org
Committed: Tue Mar 10 13:39:13 2015 +0900

--
 MAINTAINERS.txt | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/4fd59b24/MAINTAINERS.txt
--
diff --git a/MAINTAINERS.txt b/MAINTAINERS.txt
index 4b85206..75e43b4 100644
--- a/MAINTAINERS.txt
+++ b/MAINTAINERS.txt
@@ -4,17 +4,17 @@ gradle / build system: cos, rvs
 gridgain-hadoop: cos, Sergey Vladykin sergi.vlady...@gmail.com
 hadoop: mark grover, cos, rvs
 hbase: andrew purtell, rvs
-hive: mark grover
+hive: mark grover, youngwoo kim
 hue: oflebbe
 itest: cos, rvs
 mvn publishing/packaging: rvs
 oozie evans ye, rvs
-phoenix: andrew purtell
+phoenix: andrew purtell, youngwoo kim
 pig: evans ye, daniel dai
 puppet recipes: jay vyas, cos, evans ye, rvs
 smoke-tests: jay vyas, david capwell
-spark: jay vyas,
-sqoop: sean mackrory
+spark: jay vyas, youngwoo kim
+sqoop: sean mackrory, youngwoo kim
 tachyon: david capwell, jay vyas, huamin chen
 test-artifacts and test-execution repos:
 tez: evans ye, oflebbe

svn commit: r943174 - /websites/staging/bigtop/trunk/content/

[ywkim]

Author: ywkim
Date: Tue Mar 10 14:04:55 2015
New Revision: 943174

Log:
Site checkin for project Bigtop

Modified:
websites/staging/bigtop/trunk/content/dependencies.html
websites/staging/bigtop/trunk/content/dependency-info.html
websites/staging/bigtop/trunk/content/dependency-management.html
websites/staging/bigtop/trunk/content/distribution-management.html
websites/staging/bigtop/trunk/content/integration.html
websites/staging/bigtop/trunk/content/license.html
websites/staging/bigtop/trunk/content/plugin-management.html
websites/staging/bigtop/trunk/content/plugins.html
websites/staging/bigtop/trunk/content/project-info.html
websites/staging/bigtop/trunk/content/project-summary.html
websites/staging/bigtop/trunk/content/source-repository.html
websites/staging/bigtop/trunk/content/team-list.html

Modified: websites/staging/bigtop/trunk/content/dependencies.html
==
--- websites/staging/bigtop/trunk/content/dependencies.html (original)
+++ websites/staging/bigtop/trunk/content/dependencies.html Tue Mar 10 14:04:55 
2015
@@ -1,13 +1,13 @@
 !DOCTYPE html
 !--
- | Generated by Apache Maven Doxia at 2015-03-08
+ | Generated by Apache Maven Doxia at 2015-03-10
  | Rendered using Apache Maven Fluido Skin 1.3.0
 --
 html xmlns=http://www.w3.org/1999/xhtml; xml:lang=en lang=en
   head
 meta charset=UTF-8 /
 meta name=viewport content=width=device-width, initial-scale=1.0 /
-meta name=Date-Revision-mmdd content=20150308 /
+meta name=Date-Revision-mmdd content=20150310 /
 meta http-equiv=Content-Language content=en /
 titleBigtop - Project Dependencies/title
 link rel=stylesheet href=./css/apache-maven-fluido-1.3.0.min.css /
@@ -157,7 +157,7 @@
 
 
 
-  li id=publishDate class=pull-rightLast Published: 
2015-03-08/li li class=divider pull-right|/li
+  li id=publishDate class=pull-rightLast Published: 
2015-03-10/li li class=divider pull-right|/li
   li id=projectVersion class=pull-rightVersion: 
0.9.0-SNAPSHOT/li
 
 /ul

Modified: websites/staging/bigtop/trunk/content/dependency-info.html
==
--- websites/staging/bigtop/trunk/content/dependency-info.html (original)
+++ websites/staging/bigtop/trunk/content/dependency-info.html Tue Mar 10 
14:04:55 2015
@@ -1,13 +1,13 @@
 !DOCTYPE html
 !--
- | Generated by Apache Maven Doxia at 2015-03-08
+ | Generated by Apache Maven Doxia at 2015-03-10
  | Rendered using Apache Maven Fluido Skin 1.3.0
 --
 html xmlns=http://www.w3.org/1999/xhtml; xml:lang=en lang=en
   head
 meta charset=UTF-8 /
 meta name=viewport content=width=device-width, initial-scale=1.0 /
-meta name=Date-Revision-mmdd content=20150308 /
+meta name=Date-Revision-mmdd content=20150310 /
 meta http-equiv=Content-Language content=en /
 titleBigtop - Dependency Information/title
 link rel=stylesheet href=./css/apache-maven-fluido-1.3.0.min.css /
@@ -157,7 +157,7 @@
 
 
 
-  li id=publishDate class=pull-rightLast Published: 
2015-03-08/li li class=divider pull-right|/li
+  li id=publishDate class=pull-rightLast Published: 
2015-03-10/li li class=divider pull-right|/li
   li id=projectVersion class=pull-rightVersion: 
0.9.0-SNAPSHOT/li
 
 /ul

Modified: websites/staging/bigtop/trunk/content/dependency-management.html
==
--- websites/staging/bigtop/trunk/content/dependency-management.html (original)
+++ websites/staging/bigtop/trunk/content/dependency-management.html Tue Mar 10 
14:04:55 2015
@@ -1,13 +1,13 @@
 !DOCTYPE html
 !--
- | Generated by Apache Maven Doxia at 2015-03-08
+ | Generated by Apache Maven Doxia at 2015-03-10
  | Rendered using Apache Maven Fluido Skin 1.3.0
 --
 html xmlns=http://www.w3.org/1999/xhtml; xml:lang=en lang=en
   head
 meta charset=UTF-8 /
 meta name=viewport content=width=device-width, initial-scale=1.0 /
-meta name=Date-Revision-mmdd content=20150308 /
+meta name=Date-Revision-mmdd content=20150310 /
 meta http-equiv=Content-Language content=en /
 titleBigtop - Project Dependency Management/title
 link rel=stylesheet href=./css/apache-maven-fluido-1.3.0.min.css /
@@ -157,7 +157,7 @@
 
 
 
-  li id=publishDate class=pull-rightLast Published: 
2015-03-08/li li class=divider pull-right|/li
+  li id=publishDate class=pull-rightLast Published: 
2015-03-10/li li class=divider pull-right|/li
   li id=projectVersion class=pull-rightVersion: 
0.9.0-SNAPSHOT/li

svn commit: r942855 - /websites/production/bigtop/content/

[ywkim]

Author: ywkim
Date: Sun Mar  8 13:43:05 2015
New Revision: 942855

Log:
Publishing svnmucc operation to bigtop site by ywkim

Added:
websites/production/bigtop/content/
  - copied from r942854, websites/staging/bigtop/trunk/content/

bigtop git commit: Add YoungWoo Kim to team list

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master bcce3c9a2 - e665d55d9


Add YoungWoo Kim to team list


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/e665d55d
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/e665d55d
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/e665d55d

Branch: refs/heads/master
Commit: e665d55d9520378131d78701e7f5a231b45fcbbf
Parents: bcce3c9
Author: YoungWoo Kim yw...@apache.org
Authored: Sun Mar 8 21:54:50 2015 +0900
Committer: YoungWoo Kim yw...@apache.org
Committed: Sun Mar 8 21:57:38 2015 +0900

--
 src/site/xdoc/team-list.xml | 8 +++-
 1 file changed, 7 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/e665d55d/src/site/xdoc/team-list.xml
--
diff --git a/src/site/xdoc/team-list.xml b/src/site/xdoc/team-list.xml
index 50450d1..9ae6c5b 100644
--- a/src/site/xdoc/team-list.xml
+++ b/src/site/xdoc/team-list.xml
@@ -202,7 +202,13 @@
 td align=centerscience+computing 
ag/td
 td align=centercommitter/td
 /tr
-   /table
+tr valign=top
+   td align=centerywkim/td
+   td align=centerYoungWoo Kim/td
+   td align=centerKT NexR/td
+td align=centercommitter/td
+/tr
+   /table
 
subsection name=Contributors/subsection
/section

bigtop git commit: BIGTOP-1701: Upgrade to Hive 1.1.0

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 0d2ac058d - c8235426b


BIGTOP-1701: Upgrade to Hive 1.1.0


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/c8235426
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/c8235426
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/c8235426

Branch: refs/heads/master
Commit: c8235426bb00d1af2f0e8ce1e484f1954c041e1d
Parents: 0d2ac05
Author: YoungWoo Kim yw...@apache.org
Authored: Mon Mar 16 14:04:39 2015 +0900
Committer: YoungWoo Kim yw...@apache.org
Committed: Mon Mar 23 13:51:47 2015 +0900

--
 bigtop.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/c8235426/bigtop.mk
--
diff --git a/bigtop.mk b/bigtop.mk
index 706942f..4517c92 100644
--- a/bigtop.mk
+++ b/bigtop.mk
@@ -97,7 +97,7 @@ $(eval $(call PACKAGE,pig,PIG))
 HIVE_NAME=hive
 HIVE_RELNOTES_NAME=Apache Hive
 HIVE_PKG_NAME=hive
-HIVE_BASE_VERSION=1.0.0
+HIVE_BASE_VERSION=1.1.0
 HIVE_PKG_VERSION=$(HIVE_BASE_VERSION)
 HIVE_RELEASE_VERSION=1
 HIVE_TARBALL_DST=apache-hive-$(HIVE_BASE_VERSION)-src.tar.gz

bigtop git commit: BIGTOP-1768: Use openjdk in ubuntu14, Remove references to jdk6

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master c8235426b - 785902193


BIGTOP-1768: Use openjdk in ubuntu14, Remove references to jdk6

Signed-off-by: YoungWoo Kim yw...@apache.org


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/78590219
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/78590219
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/78590219

Branch: refs/heads/master
Commit: 78590219352f7c661d03c68c2330398ce86ed40c
Parents: c823542
Author: Olaf Flebbe o...@oflebbe.de
Authored: Sun Mar 22 19:45:33 2015 +0100
Committer: YoungWoo Kim yw...@apache.org
Committed: Mon Mar 23 16:43:11 2015 +0900

--
 bigtop_toolchain/files/jenkins.sh.ubu | 26 --
 bigtop_toolchain/manifests/env.pp | 11 +---
 bigtop_toolchain/manifests/jdk.pp | 18 +--
 bigtop_toolchain/manifests/jdk6.pp| 81 --
 4 files changed, 2 insertions(+), 134 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/78590219/bigtop_toolchain/files/jenkins.sh.ubu
--
diff --git a/bigtop_toolchain/files/jenkins.sh.ubu 
b/bigtop_toolchain/files/jenkins.sh.ubu
deleted file mode 100644
index 99b5d8e..000
--- a/bigtop_toolchain/files/jenkins.sh.ubu
+++ /dev/null
@@ -1,26 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the License); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an AS IS BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-export MAVEN_HOME=/usr/local/maven
-export JAVA6_HOME=/usr/lib/jvm/java-6-oracle
-export JAVA7_HOME=/usr/lib/jdk7-latest
-export JAVA_HOME=$JAVA7_HOME
-export ANT_HOME=/usr/local/ant
-export FORREST_HOME=/usr/local/apache-forrest
-export SCALA_HOME=/usr/share/java
-export GRADLE_HOME=/usr/local/gradle
-export 
PATH=$JAVA_HOME/bin:$MAVEN_HOME/bin:$ANT_HOME/bin:$FORREST_HOME/bin:$GRADLE_HOME/bin:$PATH
-
-export GRADLE_OPTS=-Dorg.gradle.daemon=true

http://git-wip-us.apache.org/repos/asf/bigtop/blob/78590219/bigtop_toolchain/manifests/env.pp
--
diff --git a/bigtop_toolchain/manifests/env.pp 
b/bigtop_toolchain/manifests/env.pp
index 656560f..e66962d 100644
--- a/bigtop_toolchain/manifests/env.pp
+++ b/bigtop_toolchain/manifests/env.pp
@@ -15,16 +15,7 @@
 
 class bigtop_toolchain::env {
   case $operatingsystem{
-/Ubuntu/: {
-  file { '/etc/profile.d/bigtop.sh':
-source = 'puppet:///modules/bigtop_toolchain/jenkins.sh.ubu',
-ensure = present,
-owner  = root,
-group  = root,
-mode   = 644,
-  }
-}
-/Debian/: {
+Ubuntu,Debian: {
   file { '/etc/profile.d/bigtop.sh':
 source = 'puppet:///modules/bigtop_toolchain/jenkins.sh.debian',
 ensure = present,

http://git-wip-us.apache.org/repos/asf/bigtop/blob/78590219/bigtop_toolchain/manifests/jdk.pp
--
diff --git a/bigtop_toolchain/manifests/jdk.pp 
b/bigtop_toolchain/manifests/jdk.pp
index c086cad..1a57fdb 100644
--- a/bigtop_toolchain/manifests/jdk.pp
+++ b/bigtop_toolchain/manifests/jdk.pp
@@ -16,7 +16,7 @@
 class bigtop_toolchain::jdk {
 
   case $operatingsystem{
-Debian: {
+Debian,Ubuntu: {
   package { 'openjdk-7-jdk' :
 ensure = present
   }
@@ -26,21 +26,5 @@ class bigtop_toolchain::jdk {
 ensure = present
   }
 }
-
-default : {
-  file { '/tmp/jdk-7u60-linux-x64.gz':
-source = 'puppet:///modules/bigtop_toolchain/jdk-7u60-linux-x64.gz',
-ensure = present,
-owner  = root,
-group  = root,
-mode   = 755
-  }
-
-  exec {'/bin/tar -xzvf /tmp/jdk-7u60-linux-x64.gz; ln -s jdk1.7.0_60 
jdk7-latest':
-cwd = '/usr/lib',
-refreshonly = true,
-subscribe   = File[/tmp/jdk-7u60-linux-x64.gz],
-  }
-}
   }
 }

http://git-wip-us.apache.org/repos/asf/bigtop/blob/78590219/bigtop_toolchain/manifests/jdk6.pp
--
diff --git a/bigtop_toolchain/manifests/jdk6.pp 

bigtop git commit: BIGTOP-1467: Fix oozie build - add correct hadoop-auth lib version

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master b58c80ea6 - 4b9cb14e0


BIGTOP-1467: Fix oozie build - add correct hadoop-auth lib version

Signed-off-by: YoungWoo Kim yw...@apache.org


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/4b9cb14e
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/4b9cb14e
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/4b9cb14e

Branch: refs/heads/master
Commit: 4b9cb14e0ff29360ec80770fa7ff679cac207e47
Parents: b58c80e
Author: Amir Sanjar amir.san...@canonical.com
Authored: Thu Apr 2 14:40:11 2015 -0500
Committer: YoungWoo Kim yw...@apache.org
Committed: Fri Apr 3 10:33:07 2015 +0900

--
 bigtop-packages/src/common/oozie/do-component-build | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/4b9cb14e/bigtop-packages/src/common/oozie/do-component-build
--
diff --git a/bigtop-packages/src/common/oozie/do-component-build 
b/bigtop-packages/src/common/oozie/do-component-build
index 2ecd660..f468928 100755
--- a/bigtop-packages/src/common/oozie/do-component-build
+++ b/bigtop-packages/src/common/oozie/do-component-build
@@ -92,7 +92,7 @@ done
 
 export MAVEN_OPTS=-Xmx512m -XX:MaxPermSize=128m
 # Invoke Oozie build script
-${WORKDIR}/bin/mkdistro.sh -DjavaVersion=${JDK_VERSION} -DskipTests 
-Dhadoop.version=${HADOOP_VERSION} ${EXTRA_GOALS} 
-Dmaven.repo.local=${HOME}/.m2/repository $@
+${WORKDIR}/bin/mkdistro.sh -DjavaVersion=${JDK_VERSION} -DskipTests 
-Dhadoop.version=${HADOOP_VERSION} -Dhadoop.auth.version=${HADOOP_VERSION} 
${EXTRA_GOALS} -Dmaven.repo.local=${HOME}/.m2/repository $@
 MKDISTRO_RESULT=$?
 if [ ${MKDISTRO_RESULT} != 0 ]; then
   echo ERROR: mkdistro.sh failed with error ${MKDISTRO_RESULT}

bigtop git commit: BIGTOP-1841. toolchain can not set java correctly in fedora

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master c82359bb3 - 66b84e547


BIGTOP-1841. toolchain can not set java correctly in fedora

Signed-off-by: YoungWoo Kim yw...@apache.org


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/66b84e54
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/66b84e54
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/66b84e54

Branch: refs/heads/master
Commit: 66b84e54759676f8da70222f41704d11253f24b7
Parents: c82359b
Author: Evans Ye evan...@apache.org
Authored: Tue Apr 21 17:55:33 2015 +
Committer: YoungWoo Kim yw...@apache.org
Committed: Wed Apr 22 10:18:04 2015 +0900

--
 bigtop_toolchain/files/jenkins.sh.fedora | 23 +++
 bigtop_toolchain/manifests/env.pp|  8 
 bigtop_toolchain/manifests/jdk.pp|  2 +-
 3 files changed, 32 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/66b84e54/bigtop_toolchain/files/jenkins.sh.fedora
--
diff --git a/bigtop_toolchain/files/jenkins.sh.fedora 
b/bigtop_toolchain/files/jenkins.sh.fedora
new file mode 100644
index 000..2dc8e14
--- /dev/null
+++ b/bigtop_toolchain/files/jenkins.sh.fedora
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the License); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an AS IS BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export MAVEN_HOME=/usr/local/maven
+export JAVA_HOME=/usr/lib/jvm/java-1.7.0-openjdk
+export ANT_HOME=/usr/local/ant
+export SCALA_HOME=/usr/share/java
+export GRADLE_HOME=/usr/local/gradle
+export 
PATH=$MAVEN_HOME/bin:$ANT_HOME/bin:$FORREST_HOME/bin:$GRADLE_HOME/bin:$PATH
+
+export GRADLE_OPTS=-Dorg.gradle.daemon=true

http://git-wip-us.apache.org/repos/asf/bigtop/blob/66b84e54/bigtop_toolchain/manifests/env.pp
--
diff --git a/bigtop_toolchain/manifests/env.pp 
b/bigtop_toolchain/manifests/env.pp
index e66962d..c85dd4b 100644
--- a/bigtop_toolchain/manifests/env.pp
+++ b/bigtop_toolchain/manifests/env.pp
@@ -24,6 +24,14 @@ class bigtop_toolchain::env {
 mode   = 644,
   }
 }
+Fedora: {
+  file {'/etc/profile.d/bigtop.sh':
+source = 'puppet:///modules/bigtop_toolchain/jenkins.sh.fedora',
+ensure = present,
+owner  = root,
+group  = root,
+mode   = 644,
+  }
 default: {
   file {'/etc/profile.d/bigtop.sh':
 source = 'puppet:///modules/bigtop_toolchain/jenkins.sh.centos',

http://git-wip-us.apache.org/repos/asf/bigtop/blob/66b84e54/bigtop_toolchain/manifests/jdk.pp
--
diff --git a/bigtop_toolchain/manifests/jdk.pp 
b/bigtop_toolchain/manifests/jdk.pp
index 1a57fdb..c3dcc30 100644
--- a/bigtop_toolchain/manifests/jdk.pp
+++ b/bigtop_toolchain/manifests/jdk.pp
@@ -21,7 +21,7 @@ class bigtop_toolchain::jdk {
 ensure = present
   }
 }
-/(CentOS|Amazon)/: {
+/(CentOS|Fedora|Amazon)/: {
   package { 'java-1.7.0-openjdk-devel' :
 ensure = present
   }

bigtop git commit: BIGTOP-1834. Upgrade to Spark 1.3.1

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 66b84e547 - 3fb8ae588


BIGTOP-1834. Upgrade to Spark 1.3.1

Signed-off-by: YoungWoo Kim yw...@apache.org


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/3fb8ae58
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/3fb8ae58
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/3fb8ae58

Branch: refs/heads/master
Commit: 3fb8ae588834e718cb3b6615ebc5e8e5f38439ad
Parents: 66b84e5
Author: Jonathan Kelly jonat...@amazon.com
Authored: Thu Apr 16 11:51:21 2015 -0500
Committer: YoungWoo Kim yw...@apache.org
Committed: Wed Apr 22 14:45:55 2015 +0900

--
 bigtop.mk | 4 ++--
 pom.xml   | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/3fb8ae58/bigtop.mk
--
diff --git a/bigtop.mk b/bigtop.mk
index 1a80adf..6071af0 100644
--- a/bigtop.mk
+++ b/bigtop.mk
@@ -262,8 +262,8 @@ $(eval $(call PACKAGE,crunch,CRUNCH))
 SPARK_NAME=spark
 SPARK_RELNOTES_NAME=Spark
 SPARK_PKG_NAME=spark-core
-SPARK_BASE_VERSION=1.3.0
-SPARK_PKG_VERSION=1.3.0
+SPARK_BASE_VERSION=1.3.1
+SPARK_PKG_VERSION=1.3.1
 SPARK_RELEASE_VERSION=1
 SPARK_TARBALL_DST=spark-$(SPARK_BASE_VERSION).tar.gz
 SPARK_TARBALL_SRC=spark-$(SPARK_BASE_VERSION).tgz

http://git-wip-us.apache.org/repos/asf/bigtop/blob/3fb8ae58/pom.xml
--
diff --git a/pom.xml b/pom.xml
index ccc30f5..35894ae 100644
--- a/pom.xml
+++ b/pom.xml
@@ -49,7 +49,7 @@
 zookeeper.version3.4.5/zookeeper.version
 giraph.version1.0.0/giraph.version
 solr.version4.6.0/solr.version
-spark.version1.3.0/spark.version
+spark.version1.3.1/spark.version
 kafka.version0.8.1.1/kafka.version
 phoenix.version4.2.2/phoenix.version
 spark-smoke.version${project.version}/spark-smoke.version

bigtop git commit: BIGTOP-1831: Upgrade Mahout to 0.10

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master e4ff5ff6e - 4a10e473d


BIGTOP-1831: Upgrade Mahout to 0.10


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/4a10e473
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/4a10e473
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/4a10e473

Branch: refs/heads/master
Commit: 4a10e473d75eb4ace28630c4bbd424b712b88740
Parents: e4ff5ff
Author: YoungWoo Kim yw...@apache.org
Authored: Thu Mar 12 20:12:24 2015 +0900
Committer: YoungWoo Kim yw...@apache.org
Committed: Tue Apr 28 13:41:57 2015 +0900

--
 bigtop-packages/src/common/mahout/do-component-build | 7 ++-
 bigtop-packages/src/common/mahout/install_mahout.sh  | 4 
 bigtop-packages/src/deb/mahout/control   | 2 +-
 bigtop-packages/src/rpm/mahout/SPECS/mahout.spec | 3 ++-
 bigtop.mk| 4 ++--
 5 files changed, 15 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/4a10e473/bigtop-packages/src/common/mahout/do-component-build
--
diff --git a/bigtop-packages/src/common/mahout/do-component-build 
b/bigtop-packages/src/common/mahout/do-component-build
index e2db59b..06b6380 100644
--- a/bigtop-packages/src/common/mahout/do-component-build
+++ b/bigtop-packages/src/common/mahout/do-component-build
@@ -18,7 +18,12 @@ set -ex
 
 . `dirname $0`/bigtop.bom
 
-mvn clean install -Dmahout.skip.distribution=false -DskipTests 
-Dhadoop2.version=$HADOOP_VERSION $@
+mvn clean install -DskipTests -Dmahout.skip.distribution=false \
+  -Phadoop2 \
+  -Dhadoop.version=$HADOOP_VERSION \
+  -Dhbase.version=$HBASE_VERSION-hadoop2 \
+  $@
+
 mkdir build
 for i in distribution/target/mahout*.tar.gz ; do
   tar -C build --strip-components=1 -xzf $i

http://git-wip-us.apache.org/repos/asf/bigtop/blob/4a10e473/bigtop-packages/src/common/mahout/install_mahout.sh
--
diff --git a/bigtop-packages/src/common/mahout/install_mahout.sh 
b/bigtop-packages/src/common/mahout/install_mahout.sh
index 3bc5c1a..af8906f 100644
--- a/bigtop-packages/src/common/mahout/install_mahout.sh
+++ b/bigtop-packages/src/common/mahout/install_mahout.sh
@@ -109,6 +109,7 @@ cp -ra ${BUILD_DIR}/lib/* $PREFIX/${LIB_DIR}/lib/
 cp ${BUILD_DIR}/mahout*.jar $PREFIX/$LIB_DIR
 cp -a ${BUILD_DIR}/*.txt $PREFIX/$DOC_DIR
 cp -a ${BUILD_DIR}/bin/* $PREFIX/${LIB_DIR}/bin
+rm -rf $PREFIX/${LIB_DIR}/bin/*.cmd
 
 # Copy in the configuration files
 install -d -m 0755 $PREFIX/$CONF_DIR
@@ -138,3 +139,6 @@ exec $INSTALLED_LIB_DIR/bin/mahout \$@
 EOF
 chmod 755 $PREFIX/$BIN_DIR/mahout
 
+rm -f ${PREFIX}/${LIB_DIR}/lib/zookeeper-*.jar
+ln -sf /usr/lib/zookeeper/zookeeper.jar ${PREFIX}/${LIB_DIR}/lib/
+

http://git-wip-us.apache.org/repos/asf/bigtop/blob/4a10e473/bigtop-packages/src/deb/mahout/control
--
diff --git a/bigtop-packages/src/deb/mahout/control 
b/bigtop-packages/src/deb/mahout/control
index b860c62..84b63ff 100644
--- a/bigtop-packages/src/deb/mahout/control
+++ b/bigtop-packages/src/deb/mahout/control
@@ -23,7 +23,7 @@ Homepage: http://mahout.apache.org
 
 Package:  mahout
 Architecture: all
-Depends: hadoop-client, bigtop-utils (= 0.7)
+Depends: hadoop-client, bigtop-utils (= 0.7), zookeeper
 Description: A set of Java libraries for scalable machine learning.
  Mahout's goal is to build scalable machine learning libraries. 
  With scalable we mean:

http://git-wip-us.apache.org/repos/asf/bigtop/blob/4a10e473/bigtop-packages/src/rpm/mahout/SPECS/mahout.spec
--
diff --git a/bigtop-packages/src/rpm/mahout/SPECS/mahout.spec 
b/bigtop-packages/src/rpm/mahout/SPECS/mahout.spec
index a36e72c..3635b21 100644
--- a/bigtop-packages/src/rpm/mahout/SPECS/mahout.spec
+++ b/bigtop-packages/src/rpm/mahout/SPECS/mahout.spec
@@ -44,7 +44,8 @@ License: ASL 2.0
 Source0: %{name}-distribution-%{mahout_base_version}-src.tar.gz
 Source1: do-component-build 
 Source2: install_%{name}.sh
-Requires: hadoop-client, bigtop-utils = 0.7
+Source3: bigtop.bom
+Requires: hadoop-client, bigtop-utils = 0.7, zookeeper
 
 
 %description 

http://git-wip-us.apache.org/repos/asf/bigtop/blob/4a10e473/bigtop.mk
--
diff --git a/bigtop.mk b/bigtop.mk
index b04..10d640b 100644
--- a/bigtop.mk
+++ b/bigtop.mk
@@ -166,8 +166,8 @@ $(eval $(call PACKAGE,oozie,OOZIE))
 MAHOUT_NAME=mahout
 MAHOUT_RELNOTES_NAME=Apache Mahout
 MAHOUT_PKG_NAME=mahout
-MAHOUT_BASE_VERSION=0.9
-MAHOUT_PKG_VERSION=0.9
+MAHOUT_BASE_VERSION=0.10.0
+MAHOUT_PKG_VERSION=0.10.0
 

bigtop git commit: BIGTOP-1874. HBase build failed due to Codehaus repository is out of service

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master acf86f8a2 - 3ee4257de


BIGTOP-1874. HBase build failed due to Codehaus repository is out of service

Signed-off-by: YoungWoo Kim yw...@apache.org


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/3ee4257d
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/3ee4257d
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/3ee4257d

Branch: refs/heads/master
Commit: 3ee4257de33a93ea499703ca56d864eadb8ac6ba
Parents: acf86f8
Author: Evans Ye evan...@apache.org
Authored: Sun May 17 12:57:36 2015 +
Committer: YoungWoo Kim yw...@apache.org
Committed: Mon May 18 14:38:17 2015 +0900

--
 bigtop-packages/src/common/hbase/do-component-build | 3 +++
 1 file changed, 3 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ee4257d/bigtop-packages/src/common/hbase/do-component-build
--
diff --git a/bigtop-packages/src/common/hbase/do-component-build 
b/bigtop-packages/src/common/hbase/do-component-build
index e12b656..049d4c8 100644
--- a/bigtop-packages/src/common/hbase/do-component-build
+++ b/bigtop-packages/src/common/hbase/do-component-build
@@ -18,6 +18,9 @@ set -ex
 
 . `dirname $0`/bigtop.bom
 
+#A hack to disable Codehaus repository since it's out of service
+sed --in-place -e 
'/http:\/\/repository.codehaus.org/,/\/repository/s/true/false/' pom.xml
+
 #An awful hack for BIGTOP-1429 to satisfy ASF CI build deps. resolution
 bash -x ./dev-support/generate-hadoopX-poms.sh $HBASE_VERSION 
$HBASE_VERSION-hadoop2
 

bigtop git commit: BIGTOP-1956 Multi RS HBase requires unique hbase.tmp.dir to be set for each RS on a node

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 1f11c766d - f40c9abd1


BIGTOP-1956 Multi RS HBase requires unique hbase.tmp.dir to be set for each RS 
on a node

Signed-off-by: YoungWoo Kim yw...@apache.org


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/f40c9abd
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/f40c9abd
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/f40c9abd

Branch: refs/heads/master
Commit: f40c9abd1cdd91575a7dbba4040cf01bac686645
Parents: 1f11c76
Author: Biju Nair gs.b...@gmail.com
Authored: Wed Aug 19 15:19:25 2015 -0400
Committer: YoungWoo Kim yw...@apache.org
Committed: Thu Aug 20 09:34:53 2015 +0900

--
 bigtop-packages/src/common/hbase/hbase.default   |  6 ++
 .../src/common/hbase/regionserver-init.d.tpl | 15 ---
 2 files changed, 18 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/f40c9abd/bigtop-packages/src/common/hbase/hbase.default
--
diff --git a/bigtop-packages/src/common/hbase/hbase.default 
b/bigtop-packages/src/common/hbase/hbase.default
index cc5fa26..b537157 100644
--- a/bigtop-packages/src/common/hbase/hbase.default
+++ b/bigtop-packages/src/common/hbase/hbase.default
@@ -33,3 +33,9 @@ export HBASE_IDENT_STRING=hbase
 # which will be equal to starting JMX port + offset
 # 
 # export JMXPORT=10103
+
+#
+# Set a directory which will be used for Java temp directory. For multi RS the
+# directory will be appended with offset to make it unique for each JVM process
+#
+# export JAVA_TMP_DIR=/tmp/java_tmp_dir

http://git-wip-us.apache.org/repos/asf/bigtop/blob/f40c9abd/bigtop-packages/src/common/hbase/regionserver-init.d.tpl
--
diff --git a/bigtop-packages/src/common/hbase/regionserver-init.d.tpl 
b/bigtop-packages/src/common/hbase/regionserver-init.d.tpl
index df9a06f..d2de686 100644
--- a/bigtop-packages/src/common/hbase/regionserver-init.d.tpl
+++ b/bigtop-packages/src/common/hbase/regionserver-init.d.tpl
@@ -194,18 +194,27 @@ multi_hbase_daemon() {
 exit 1
 fi
 
-export HBASE_${UPPERCASE_HBASE_DAEMON}_OPTS= 
-
 for OFFSET in ${OFFSETS} ; do
+
+export HBASE_${UPPERCASE_HBASE_DAEMON}_OPTS= 
+
 echo -n $ACTION @HBASE_DAEMON@ daemon $OFFSET: 
 export HBASE_IDENT_STRING=hbase-${OFFSET}
 
LOG_FILE=$HBASE_LOG_DIR/hbase-$HBASE_IDENT_STRING-@HBASE_DAEMON@-$HOSTNAME.pid
 PID_FILE=$HBASE_PID_DIR/hbase-$HBASE_IDENT_STRING-@HBASE_DAEMON@.pid
 HBASE_MULTI_ARGS=-D hbase.regionserver.port=`expr ${FIRST_PORT} + 
$OFFSET` \
   -D hbase.regionserver.info.port=`expr 
${FIRST_INFO_PORT} + ${OFFSET}`
+
+if [ x$JAVA_TMP_DIR == x ] ; then
+JAVA_TMP_DIR=/tmp/java_tmp_dir
+fi
+HBASE_TMP_DIR= -Djava.io.tmpdir=${JAVA_TMP_DIR}/${OFFSET}
+
 if [ x$JMXPORT != x ] ; then
-HBASE_MULTI_ARGS=${HBASE_MULTI_ARGS} 
-Dcom.sun.management.jmxremote.port=`expr ${JMXPORT} + ${OFFSET}`
+HBASE_JMX_PORT= -Dcom.sun.management.jmxremote.port=`expr 
${JMXPORT} + ${OFFSET}`
 fi
+export HBASE_${UPPERCASE_HBASE_DAEMON}_OPTS=`eval 
'$'HBASE_${UPPERCASE_HBASE_DAEMON}_OPTS`$HBASE_TMP_DIR$HBASE_JMX_PORT
+
 hbase_check_pidfile $PID_FILE
 STATUS=$?
 if [[ $STATUS == 0  $COMMAND == start ]] ; then

bigtop git commit: BIGTOP-1126. Add Hama to Bigtop

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 7e5a9225f - 1f11c766d


BIGTOP-1126. Add Hama to Bigtop

Signed-off-by: YoungWoo Kim yw...@apache.org


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/1f11c766
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/1f11c766
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/1f11c766

Branch: refs/heads/master
Commit: 1f11c766df34d1b878350fe86c569262dfefcc58
Parents: 7e5a922
Author: Minho Kim minwise@samsung.com
Authored: Wed Aug 12 09:55:09 2015 +0900
Committer: YoungWoo Kim yw...@apache.org
Committed: Wed Aug 19 09:05:17 2015 +0900

--
 MAINTAINERS.txt |   1 +
 .../src/common/hama/conf.pseudo/hama-site.xml   |  58 
 .../src/common/hama/do-component-build  |  21 ++
 .../src/common/hama/hama-bspmaster.init | 158 +++
 bigtop-packages/src/common/hama/hama-groom.init | 159 +++
 bigtop-packages/src/common/hama/hama.default|  17 ++
 bigtop-packages/src/common/hama/install_hama.sh | 171 
 bigtop-packages/src/deb/hama/changelog  |   1 +
 bigtop-packages/src/deb/hama/compat |   1 +
 bigtop-packages/src/deb/hama/control|  67 +
 bigtop-packages/src/deb/hama/copyright  |  16 ++
 .../src/deb/hama/hama-bspmaster.postinst|  25 ++
 .../src/deb/hama/hama-conf-pseudo.dirs  |   1 +
 .../src/deb/hama/hama-conf-pseudo.install   |   1 +
 .../src/deb/hama/hama-conf-pseudo.postinst  |  61 
 .../src/deb/hama/hama-conf-pseudo.prerm |  57 
 bigtop-packages/src/deb/hama/hama-doc.dirs  |   1 +
 bigtop-packages/src/deb/hama/hama-doc.install   |   1 +
 .../src/deb/hama/hama-groom.postinst|  25 ++
 bigtop-packages/src/deb/hama/hama.dirs  |   4 +
 bigtop-packages/src/deb/hama/hama.install   |   4 +
 bigtop-packages/src/deb/hama/hama.postinst  |  36 +++
 bigtop-packages/src/deb/hama/hama.preinst   |  62 +
 bigtop-packages/src/deb/hama/hama.prerm |  57 
 bigtop-packages/src/deb/hama/rules  | 103 +++
 bigtop-packages/src/rpm/hama/BUILD/.gitignore   |   0
 bigtop-packages/src/rpm/hama/RPMS/.gitignore|   0
 bigtop-packages/src/rpm/hama/SOURCES/.gitignore |   0
 bigtop-packages/src/rpm/hama/SPECS/hama.spec| 276 +++
 bigtop-packages/src/rpm/hama/SRPMS/.gitignore   |   0
 bigtop.mk   |  14 +
 31 files changed, 1398 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/1f11c766/MAINTAINERS.txt
--
diff --git a/MAINTAINERS.txt b/MAINTAINERS.txt
index fb22c4b..4b0cbf3 100644
--- a/MAINTAINERS.txt
+++ b/MAINTAINERS.txt
@@ -3,6 +3,7 @@ flume: bmahe
 gradle / build system: cos, rvs
 ignite-hadoop: cos, Sergey Vladykin sergi.vlady...@gmail.com
 hadoop: mark grover, cos, rvs
+hama: minho kim mi...@apache.org, edward j. yoon
 hbase: andrew purtell, rvs
 hive: mark grover, youngwoo kim
 hue: oflebbe

http://git-wip-us.apache.org/repos/asf/bigtop/blob/1f11c766/bigtop-packages/src/common/hama/conf.pseudo/hama-site.xml
--
diff --git a/bigtop-packages/src/common/hama/conf.pseudo/hama-site.xml 
b/bigtop-packages/src/common/hama/conf.pseudo/hama-site.xml
new file mode 100644
index 000..759cf9e
--- /dev/null
+++ b/bigtop-packages/src/common/hama/conf.pseudo/hama-site.xml
@@ -0,0 +1,58 @@
+?xml version=1.0?
+?xml-stylesheet type=text/xsl href=configuration.xsl?
+!--
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+--
+configuration
+   property
+   namebsp.master.address/name
+   valuelocalhost:4/value
+   descriptionThe address of the bsp master server. Either the
+   literal string local or a host:port for distributed 
mode
+   /description
+   /property
+

bigtop git commit: BIGTOP-1975. Smoke tests for Spark SQL

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 570c81cc0 - ba9136c57


BIGTOP-1975. Smoke tests for Spark SQL


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/ba9136c5
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/ba9136c5
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/ba9136c5

Branch: refs/heads/master
Commit: ba9136c572485c896d41dceb6fb6c1f99a9f4599
Parents: 570c81c
Author: YoungWoo Kim yw...@apache.org
Authored: Thu Aug 20 16:45:24 2015 +0900
Committer: YoungWoo Kim yw...@apache.org
Committed: Mon Aug 24 11:38:03 2015 +0900

--
 bigtop-tests/smoke-tests/README |  3 +-
 bigtop-tests/smoke-tests/spark/TestSpark.groovy | 63 
 bigtop-tests/smoke-tests/spark/build.gradle | 53 
 3 files changed, 118 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/ba9136c5/bigtop-tests/smoke-tests/README
--
diff --git a/bigtop-tests/smoke-tests/README b/bigtop-tests/smoke-tests/README
index f8adb3b..1ca4674 100644
--- a/bigtop-tests/smoke-tests/README
+++ b/bigtop-tests/smoke-tests/README
@@ -27,6 +27,7 @@ export HIVE_CONF_DIR=/etc/hive/conf/
 export JAVA_HOME=/usr/lib/jvm/java-openjdk/
 export MAHOUT_HOME=/usr/lib/mahout
 export KITE_HOME=/usr/lib/kite
+export SPARK_HOME=/usr/lib/spark
 export ITEST=0.7.0
 ```
 
@@ -36,7 +37,7 @@ Each directory is a gradle subproject mapping to an 
ecosystem component.
 If you specify, for example, flume, then the flume test runs.  To test hive 
as well,
 you would specify flume,hive, and so on.   An example is below:
 ```
-./gradlew compileGroovy test -Dsmoke.tests=flume,hive --info
+./gradlew clean compileGroovy test -Dsmoke.tests=flume,hive --info
 ```
 
 The --info option is a nice feature to have here:  You can easily debug your 
tests this way,

http://git-wip-us.apache.org/repos/asf/bigtop/blob/ba9136c5/bigtop-tests/smoke-tests/spark/TestSpark.groovy
--
diff --git a/bigtop-tests/smoke-tests/spark/TestSpark.groovy 
b/bigtop-tests/smoke-tests/spark/TestSpark.groovy
new file mode 100644
index 000..b8bd26a
--- /dev/null
+++ b/bigtop-tests/smoke-tests/spark/TestSpark.groovy
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * p/
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * p/
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.bigtop.itest.spark
+
+import org.junit.BeforeClass
+import org.junit.AfterClass
+import static org.junit.Assert.assertNotNull
+import org.apache.bigtop.itest.shell.Shell
+import static org.junit.Assert.assertTrue
+import org.junit.Test
+import org.apache.bigtop.itest.JarContent
+import org.apache.bigtop.itest.TestUtils
+import org.apache.commons.logging.LogFactory
+import org.apache.commons.logging.Log
+
+import static org.apache.bigtop.itest.LogErrorsUtils.logError
+
+class TestSpark {
+  static private Log LOG = LogFactory.getLog(Object.class)
+
+  static Shell sh = new Shell(/bin/bash -s)
+  static final String SPARK_HOME = System.getenv(SPARK_HOME)
+  static final String SPARK_SHELL = SPARK_HOME + /bin/spark-shell --master 
yarn-client
+  static final String TEST_SPARKSQL_LOG = /tmp/TestSpark_testSparkSQL.log
+
+  @BeforeClass
+  static void setUp() {
+sh.exec(rm -f  + TEST_SPARKSQL_LOG)
+sh.exec(hdfs dfs -put  + SPARK_HOME + /examples examples)
+logError(sh)
+  }
+
+  @AfterClass
+  public static void tearDown() {
+sh.exec(hdfs dfs -ls)
+logError(sh)
+sh.exec(hdfs dfs -rmr people* examples)
+logError(sh)
+  }
+
+  @Test
+  void testSparkSQL() {
+sh.exec(SPARK_SHELL +  --class org.apache.spark.examples.sql.JavaSparkSQL 
 +  --jars  + SPARK_HOME + /lib/spark-examples*.jar   + TEST_SPARKSQL_LOG 
+  21)
+logError(sh)
+assertTrue(Failed ..., sh.getRet() == 0);
+  }
+}

http://git-wip-us.apache.org/repos/asf/bigtop/blob/ba9136c5/bigtop-tests/smoke-tests/spark/build.gradle
--
diff --git 

bigtop git commit: BIGTOP-1941. Upgrade Phoenix to 4.4.0

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master b5ad721be - db0c8e3c8


BIGTOP-1941. Upgrade Phoenix to 4.4.0


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/db0c8e3c
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/db0c8e3c
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/db0c8e3c

Branch: refs/heads/master
Commit: db0c8e3c85df18988f594c8f65308a28c059ebe0
Parents: b5ad721
Author: YoungWoo Kim yw...@apache.org
Authored: Tue Jul 28 15:11:25 2015 +0900
Committer: YoungWoo Kim yw...@apache.org
Committed: Mon Aug 3 09:02:52 2015 +0900

--
 .../src/common/phoenix/do-component-build   |  5 +-
 .../src/common/phoenix/install_phoenix.sh   | 17 +++--
 .../common/phoenix/phoenix-queryserver.default  | 16 
 .../src/common/phoenix/phoenix-queryserver.svc  | 79 
 bigtop-packages/src/deb/phoenix/compat  |  2 +-
 bigtop-packages/src/deb/phoenix/control | 11 ++-
 bigtop-packages/src/deb/phoenix/phoenix.preinst | 52 +
 bigtop-packages/src/deb/phoenix/rules   |  7 +-
 .../src/rpm/phoenix/SPECS/phoenix.spec  | 50 -
 .../smoke-tests/phoenix/STOCK_SYMBOL.csv|  9 +++
 .../smoke-tests/phoenix/STOCK_SYMBOL.sql| 24 ++
 .../phoenix/TestPhoenixQueryServer.groovy   | 57 ++
 bigtop-tests/smoke-tests/phoenix/build.gradle   | 53 +
 .../smoke-tests/phoenix/log4j.properties| 24 ++
 .../smoke-tests/phoenix/smoke-test-teardown.sql |  1 +
 bigtop-tests/smoke-tests/phoenix/smoke-test.sql |  1 +
 bigtop.mk   |  7 +-
 17 files changed, 398 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/db0c8e3c/bigtop-packages/src/common/phoenix/do-component-build
--
diff --git a/bigtop-packages/src/common/phoenix/do-component-build 
b/bigtop-packages/src/common/phoenix/do-component-build
index 49fd2ef..81d0fb4 100644
--- a/bigtop-packages/src/common/phoenix/do-component-build
+++ b/bigtop-packages/src/common/phoenix/do-component-build
@@ -18,12 +18,15 @@ set -ex
 
 . `dirname $0`/bigtop.bom
 
+PHOENIX_HBASE_VERSION=HBase-0.98
+
 mvn -DskipTests \
 -Dhadoop-two.version=$HADOOP_VERSION  \
 -Dhbase.version=$HBASE_VERSION-hadoop2  \
 -Dpig.version=$PIG_VERSION  \
 -Dflume.version=$FLUME_VERSION  \
+-Dspark.version=${SPARK_VERSION} \
 clean install package $@
 rm -rf build
 mkdir build
-tar -C build --strip-components=1 -xzf 
phoenix-assembly/target/phoenix-$PHOENIX_VERSION.tar.gz
+tar -C build --strip-components=1 -xzf 
phoenix-assembly/target/phoenix-$PHOENIX_VERSION-${PHOENIX_HBASE_VERSION}.tar.gz

http://git-wip-us.apache.org/repos/asf/bigtop/blob/db0c8e3c/bigtop-packages/src/common/phoenix/install_phoenix.sh
--
diff --git a/bigtop-packages/src/common/phoenix/install_phoenix.sh 
b/bigtop-packages/src/common/phoenix/install_phoenix.sh
index b45053f..dca0066 100755
--- a/bigtop-packages/src/common/phoenix/install_phoenix.sh
+++ b/bigtop-packages/src/common/phoenix/install_phoenix.sh
@@ -111,13 +111,13 @@ install -d -m 0755 $PREFIX/$MAN_DIR
 install -d -m 0755 $PREFIX/$ETC_DIR
 install -d -m 0755 $PREFIX/$CONF_DIR
 
-cp -ra $BUILD_DIR/lib/* $PREFIX/$LIB_DIR/lib
-cp -a $BUILD_DIR/phoenix*.jar $PREFIX/$LIB_DIR
-cp -a $BUILD_DIR/examples $PREFIX/$DOC_DIR
-cp -a  $BUILD_DIR/bin/*.txt $PREFIX/$DOC_DIR
+install -d -m 0755 $PREFIX/var/lib/phoenix
+install -d -m 0755 $PREFIX/var/log/phoenix
 
-cp -a $BUILD_DIR/bin/*.py $PREFIX/$BIN_DIR
-cp -a $BUILD_DIR/bin/*.properties $PREFIX/$BIN_DIR
+cp -ra $BUILD_DIR/{bin,lib} $PREFIX/$LIB_DIR/
+chmod 755 $PREFIX/$BIN_DIR/*.py
+cp -a $BUILD_DIR/*.jar $PREFIX/$LIB_DIR/
+cp -a $BUILD_DIR/examples $PREFIX/$DOC_DIR
 
 # Remove sources jar
 rm $PREFIX/$LIB_DIR/lib/phoenix-*-sources.jar
@@ -128,7 +128,10 @@ find $PREFIX/$LIB_DIR -name '*.jar' -exec chmod a-x {} \;
 # Create version independent symlinks
 
 # phoenix-client for clients like sqlline
-ln -s `cd $PREFIX/$LIB_DIR ; ls phoenix*-client.jar` 
$PREFIX/$LIB_DIR/phoenix-client.jar
+ln -s `cd $PREFIX/$LIB_DIR ; ls phoenix*-client.jar | grep -v thin` 
$PREFIX/$LIB_DIR/phoenix-client.jar
+
+# phoenix-thin-client
+ln -s `cd $PREFIX/$LIB_DIR ; ls phoenix*-thin-client.jar` 
$PREFIX/$LIB_DIR/phoenix-thin-client.jar
 
 # phoenix-server for placing on the HBase regionserver classpath
 ln -s `cd $PREFIX/$LIB_DIR ; ls phoenix*-server.jar` 
$PREFIX/$LIB_DIR/phoenix-server.jar

http://git-wip-us.apache.org/repos/asf/bigtop/blob/db0c8e3c/bigtop-packages/src/common/phoenix/phoenix-queryserver.default
--
diff --git 

bigtop git commit: BIGTOP-1921. Puppet recipe for YCSB

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 4055b7869 - 8effb9062


BIGTOP-1921. Puppet recipe for YCSB


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/8effb906
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/8effb906
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/8effb906

Branch: refs/heads/master
Commit: 8effb9062f6a44fffa6d220bf7906ac87ab968d4
Parents: 4055b78
Author: YoungWoo Kim yw...@apache.org
Authored: Thu Jul 9 14:58:01 2015 +0900
Committer: YoungWoo Kim yw...@apache.org
Committed: Thu Jul 30 14:36:56 2015 +0900

--
 .../puppet/hieradata/bigtop/cluster.yaml|  2 +-
 bigtop-deploy/puppet/hieradata/site.yaml|  1 +
 bigtop-deploy/puppet/manifests/cluster.pp   |  3 +++
 .../puppet/modules/ycsb/manifests/init.pp   | 22 
 bigtop-deploy/puppet/modules/ycsb/tests/init.pp | 17 +++
 5 files changed, 44 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/8effb906/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
--
diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml 
b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
index aaa4a06..85583d1 100644
--- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
+++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
@@ -17,7 +17,7 @@
 # be installed as usual. Otherwise only a specified list will be set
 # Possible elements:
 # 
hadoop,yarn,hbase,tachyon,flume,solrcloud,spark,oozie,hcat,sqoop,sqoop2,httpfs,
-# hue,mahout,giraph,crunch,pig,hive,zookeeper
+# hue,mahout,giraph,crunch,pig,hive,zookeeper, ycsb
 # Example (to deploy only HDFS and YARN server and gateway parts)
 # This can be a comma-separated list or an array.
 #hadoop_cluster_node::cluster_components:

http://git-wip-us.apache.org/repos/asf/bigtop/blob/8effb906/bigtop-deploy/puppet/hieradata/site.yaml
--
diff --git a/bigtop-deploy/puppet/hieradata/site.yaml 
b/bigtop-deploy/puppet/hieradata/site.yaml
index 826d748..f8c1b7b 100644
--- a/bigtop-deploy/puppet/hieradata/site.yaml
+++ b/bigtop-deploy/puppet/hieradata/site.yaml
@@ -33,6 +33,7 @@ hadoop::hadoop_storage_dirs:
 #  - tez
 #  - yarn
 #  - zookeeper
+#  - ycsb
 
 #bigtop::jdk_package_name: openjdk-7-jre-headless
 #bigtop::bigtop_repo_uri: http://mirror.example.com/path/to/mirror/;

http://git-wip-us.apache.org/repos/asf/bigtop/blob/8effb906/bigtop-deploy/puppet/manifests/cluster.pp
--
diff --git a/bigtop-deploy/puppet/manifests/cluster.pp 
b/bigtop-deploy/puppet/manifests/cluster.pp
index 77cbbcf..19cfc65 100644
--- a/bigtop-deploy/puppet/manifests/cluster.pp
+++ b/bigtop-deploy/puppet/manifests/cluster.pp
@@ -214,4 +214,7 @@ class hadoop_gateway_node inherits hadoop_cluster_node {
   if ($all or tez in $components) {
 include tez::client
   }
+  if ($all or ycsb in $components) {
+include ycsb::client
+  }
 }

http://git-wip-us.apache.org/repos/asf/bigtop/blob/8effb906/bigtop-deploy/puppet/modules/ycsb/manifests/init.pp
--
diff --git a/bigtop-deploy/puppet/modules/ycsb/manifests/init.pp 
b/bigtop-deploy/puppet/modules/ycsb/manifests/init.pp
new file mode 100644
index 000..8e7e7d5
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/ycsb/manifests/init.pp
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the License); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an AS IS BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class ycsb {
+  class client {
+package { [ycsb]:
+  ensure = latest,
+}
+  }
+}

http://git-wip-us.apache.org/repos/asf/bigtop/blob/8effb906/bigtop-deploy/puppet/modules/ycsb/tests/init.pp
--
diff --git a/bigtop-deploy/puppet/modules/ycsb/tests/init.pp 
b/bigtop-deploy/puppet/modules/ycsb/tests/init.pp
new file mode 100644
index 000..da06706
--- /dev/null
+++ 

bigtop git commit: BIGTOP-1149: Package Kite

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 8effb9062 - a04eacebf


BIGTOP-1149: Package Kite


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/a04eaceb
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/a04eaceb
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/a04eaceb

Branch: refs/heads/master
Commit: a04eacebfeb8692bf05b193cef7fef6493fa281e
Parents: 8effb90
Author: Youngwoo Kim warwit...@gmail.com
Authored: Thu Jan 22 16:47:29 2015 +0900
Committer: YoungWoo Kim yw...@apache.org
Committed: Thu Jul 30 14:59:28 2015 +0900

--
 MAINTAINERS.txt |   1 +
 .../src/common/kite/do-component-build  |  48 +
 bigtop-packages/src/common/kite/install_kite.sh | 106 +++
 bigtop-packages/src/deb/kite/changelog  |   1 +
 bigtop-packages/src/deb/kite/compat |   1 +
 bigtop-packages/src/deb/kite/control|  28 +
 bigtop-packages/src/deb/kite/copyright  |  15 +++
 bigtop-packages/src/deb/kite/kite.dirs  |   2 +
 bigtop-packages/src/deb/kite/kite.install   |   2 +
 bigtop-packages/src/deb/kite/kite.postinst  |  32 ++
 bigtop-packages/src/deb/kite/kite.prerm |  37 +++
 bigtop-packages/src/deb/kite/rules  |  49 +
 bigtop-packages/src/deb/kite/source/format  |   1 +
 bigtop-packages/src/rpm/kite/SPECS/kite.spec|  77 ++
 bigtop-tests/smoke-tests/README |   1 +
 .../smoke-tests/kite/TestKiteDataset.groovy |  76 +
 bigtop-tests/smoke-tests/kite/build.gradle  |  54 ++
 bigtop-tests/smoke-tests/kite/log4j.properties  |  24 +
 bigtop-tests/smoke-tests/kite/sandwiches.csv|   3 +
 bigtop.mk   |  13 +++
 pom.xml |   1 +
 21 files changed, 572 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/a04eaceb/MAINTAINERS.txt
--
diff --git a/MAINTAINERS.txt b/MAINTAINERS.txt
index bf09d6d..559c0fb 100644
--- a/MAINTAINERS.txt
+++ b/MAINTAINERS.txt
@@ -7,6 +7,7 @@ hbase: andrew purtell, rvs
 hive: mark grover, youngwoo kim
 hue: oflebbe
 itest: cos, rvs
+kite: youngwoo kim
 mvn publishing/packaging: rvs
 oozie evans ye, rvs
 phoenix: andrew purtell, youngwoo kim

http://git-wip-us.apache.org/repos/asf/bigtop/blob/a04eaceb/bigtop-packages/src/common/kite/do-component-build
--
diff --git a/bigtop-packages/src/common/kite/do-component-build 
b/bigtop-packages/src/common/kite/do-component-build
new file mode 100644
index 000..e747d14
--- /dev/null
+++ b/bigtop-packages/src/common/kite/do-component-build
@@ -0,0 +1,48 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the License); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an AS IS BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+. `dirname $0`/bigtop.bom
+
+mvn clean install \
+  -DjavaVersion=${JDK_VERSION} \
+  -Dvers.hadoop2=${HADOOP_VERSION} \
+  -Dvers.hbase2=${HBASE_VERSION}-hadoop2 \
+  -Dvers.flume=${FLUME_VERSION} \
+  -Dvers.spark=${SPARK_VERSION} \
+  -Dvers.crunch=${CRUNCH_VERSION} \
+  -DskipTests -Dmaven.repo.local=${HOME}/.m2/repository $@
+
+rm -rf build
+
+mkdir -p build/kite-${KITE_VERSION}/{bin,lib}
+
+# Kite Data
+cp kite-data/kite-*/target/kite-*.jar build/kite-${KITE_VERSION}/lib
+# Kite Tools
+cp kite-tools-parent/kite-tools/target/kite-dataset 
build/kite-${KITE_VERSION}/bin/
+cp kite-tools-parent/kite-tools/target/kite-*.jar 
build/kite-${KITE_VERSION}/lib/
+
+# Kite Morphlines
+cp kite-morphlines/kite-morphlines-all/target/lib/* 
build/kite-${KITE_VERSION}/lib/
+
+# Kite Maven plugin
+cp kite-maven-plugin/target/kite-*.jar build/kite-${KITE_VERSION}/lib/
+
+
+
+

http://git-wip-us.apache.org/repos/asf/bigtop/blob/a04eaceb/bigtop-packages/src/common/kite/install_kite.sh
--
diff --git a/bigtop-packages/src/common/kite/install_kite.sh 

bigtop git commit: BIGTOP-1920. Include YCSB in Bigtop

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 4cc64586f - c33343d93


BIGTOP-1920. Include YCSB in Bigtop


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/c33343d9
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/c33343d9
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/c33343d9

Branch: refs/heads/master
Commit: c33343d930f5c9a648c789eca1b1ff06d490ceae
Parents: 4cc6458
Author: YoungWoo Kim yw...@apache.org
Authored: Wed Jul 8 16:37:58 2015 +0900
Committer: YoungWoo Kim yw...@apache.org
Committed: Thu Jul 30 14:31:43 2015 +0900

--
 .../src/common/ycsb/do-component-build  | 27 +++
 bigtop-packages/src/common/ycsb/install_ycsb.sh | 79 
 .../src/common/ycsb/patch0-0.1.4-hbase98.diff   | 39 ++
 bigtop-packages/src/deb/ycsb/changelog  |  1 +
 bigtop-packages/src/deb/ycsb/compat |  1 +
 bigtop-packages/src/deb/ycsb/control| 30 
 bigtop-packages/src/deb/ycsb/copyright  | 15 
 bigtop-packages/src/deb/ycsb/rules  | 35 +
 bigtop-packages/src/deb/ycsb/source/format  |  1 +
 bigtop-packages/src/deb/ycsb/ycsb.install   |  1 +
 bigtop-packages/src/rpm/ycsb/BUILD/.gitignore   |  0
 bigtop-packages/src/rpm/ycsb/RPMS/.gitignore|  0
 bigtop-packages/src/rpm/ycsb/SOURCES/.gitignore |  0
 bigtop-packages/src/rpm/ycsb/SPECS/ycsb.spec| 58 ++
 bigtop-packages/src/rpm/ycsb/SRPMS/.gitignore   |  0
 bigtop.mk   | 13 
 16 files changed, 300 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/c33343d9/bigtop-packages/src/common/ycsb/do-component-build
--
diff --git a/bigtop-packages/src/common/ycsb/do-component-build 
b/bigtop-packages/src/common/ycsb/do-component-build
new file mode 100644
index 000..f13545e
--- /dev/null
+++ b/bigtop-packages/src/common/ycsb/do-component-build
@@ -0,0 +1,27 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the License); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an AS IS BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+. `dirname $0`/bigtop.bom
+
+mvn clean package -DskipTests -Dhbase.version=$HBASE_VERSION-hadoop2 $@
+
+rm -rf build/dist
+mkdir -p build/dist
+tar -C build/dist --strip-components=1 -xzf 
distribution/target/ycsb-$YCSB_VERSION.tar.gz
+
+

http://git-wip-us.apache.org/repos/asf/bigtop/blob/c33343d9/bigtop-packages/src/common/ycsb/install_ycsb.sh
--
diff --git a/bigtop-packages/src/common/ycsb/install_ycsb.sh 
b/bigtop-packages/src/common/ycsb/install_ycsb.sh
new file mode 100755
index 000..acf54fa
--- /dev/null
+++ b/bigtop-packages/src/common/ycsb/install_ycsb.sh
@@ -0,0 +1,79 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the License); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an AS IS BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+usage() {
+  echo 
+usage: $0 options
+  Required not-so-options:
+ --build-dir=DIR path to pig dist.dir
+ --prefix=PREFIX path to install into
+
+  Optional options:
+ --lib-dir=DIR   path to install pig home [/usr/lib/pig]
+ --build-dir=DIR path to pig dist dir
+ ... [ see source for more similar options ]
+  
+  exit 1
+}
+
+OPTS=$(getopt \
+  -n $0 \
+  -o '' \
+  -l 'prefix:' \
+  -l 'lib-dir:' \
+  -l 'build-dir:' -- 

bigtop git commit: BIGTOP-2102. Upgrade YCSB to 0.4.0

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 1afdffe91 -> 29e68a2ee


BIGTOP-2102. Upgrade YCSB to 0.4.0


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/29e68a2e
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/29e68a2e
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/29e68a2e

Branch: refs/heads/master
Commit: 29e68a2ee592d19364cb7a97a8ec8fc1d4db8224
Parents: 1afdffe
Author: YoungWoo Kim 
Authored: Wed Oct 28 19:46:08 2015 +0900
Committer: YoungWoo Kim 
Committed: Tue Nov 10 20:09:15 2015 +0900

--
 .../src/common/ycsb/do-component-build  |  2 +-
 .../src/common/ycsb/patch0-0.1.4-hbase98.diff   | 39 
 bigtop-packages/src/rpm/ycsb/SPECS/ycsb.spec|  3 --
 bigtop.bom  |  2 +-
 4 files changed, 2 insertions(+), 44 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/29e68a2e/bigtop-packages/src/common/ycsb/do-component-build
--
diff --git a/bigtop-packages/src/common/ycsb/do-component-build 
b/bigtop-packages/src/common/ycsb/do-component-build
index f13545e..ab1b107 100644
--- a/bigtop-packages/src/common/ycsb/do-component-build
+++ b/bigtop-packages/src/common/ycsb/do-component-build
@@ -18,7 +18,7 @@ set -ex
 
 . `dirname $0`/bigtop.bom
 
-mvn clean package -DskipTests -Dhbase.version=$HBASE_VERSION-hadoop2 "$@"
+mvn clean package -DskipTests -Dhbase098.version=$HBASE_VERSION-hadoop2 "$@"
 
 rm -rf build/dist
 mkdir -p build/dist

http://git-wip-us.apache.org/repos/asf/bigtop/blob/29e68a2e/bigtop-packages/src/common/ycsb/patch0-0.1.4-hbase98.diff
--
diff --git a/bigtop-packages/src/common/ycsb/patch0-0.1.4-hbase98.diff 
b/bigtop-packages/src/common/ycsb/patch0-0.1.4-hbase98.diff
deleted file mode 100644
index bd73a53..000
--- a/bigtop-packages/src/common/ycsb/patch0-0.1.4-hbase98.diff
+++ /dev/null
@@ -1,39 +0,0 @@
-diff --git a/hbase/pom.xml b/hbase/pom.xml
-index 55d243c..bcfe015 100644
 a/hbase/pom.xml
-+++ b/hbase/pom.xml
-@@ -11,10 +11,20 @@
-   HBase DB Binding
- 
-   
--
-+
-+
-+  org.apache.hbase
-+  hbase-client
-+  ${hbase.version}
-+
-+
-+  org.apache.hbase
-+  hbase-common
-+  ${hbase.version}
- 
- 
-   com.yahoo.ycsb
-diff --git a/pom.xml b/pom.xml
-index bc181f5..f85055a 100644
 a/pom.xml
-+++ b/pom.xml
-@@ -63,7 +63,7 @@
- gemfire
- infinispan
- jdbc
--mapkeeper
-+
- mongodb
- 
- redis

http://git-wip-us.apache.org/repos/asf/bigtop/blob/29e68a2e/bigtop-packages/src/rpm/ycsb/SPECS/ycsb.spec
--
diff --git a/bigtop-packages/src/rpm/ycsb/SPECS/ycsb.spec 
b/bigtop-packages/src/rpm/ycsb/SPECS/ycsb.spec
index 6c43437..b208c36 100644
--- a/bigtop-packages/src/rpm/ycsb/SPECS/ycsb.spec
+++ b/bigtop-packages/src/rpm/ycsb/SPECS/ycsb.spec
@@ -32,7 +32,6 @@ Source0: %{ycsb_name}-%{ycsb_base_version}.tar.gz
 Source1: do-component-build 
 Source2: install_%{ycsb_name}.sh
 Source3: bigtop.bom
-#BIGTOP_PATCH_FILES
 Requires: python
 
 %description 
@@ -44,8 +43,6 @@ performance of NoSQL database management systems.
 %prep
 %setup -n YCSB-%{ycsb_base_version}
 
-#BIGTOP_PATCH_COMMANDS
-
 %build
 bash $RPM_SOURCE_DIR/do-component-build
 

http://git-wip-us.apache.org/repos/asf/bigtop/blob/29e68a2e/bigtop.bom
--
diff --git a/bigtop.bom b/bigtop.bom
index 04290bc..7b08952 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -365,7 +365,7 @@ bigtop {
 'ycsb' {
   name= 'ycsb'
   relNotes = 'Yahoo! Cloud Serving Benchmark'
-  version { base = '0.1.4'; pkg = base; release = 1 }
+  version { base = '0.4.0'; pkg = base; release = 1 }
   tarball { destination = "$name-${version.base}.tar.gz"
 source  = "${version.base}.tar.gz" }
   url { site = "https://github.com/brianfrankcooper/YCSB/archive;

bigtop git commit: BIGTOP-1942. Upgrade Phoenix to 4.6.0

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master b2d94ebec -> 7ae4a0876


BIGTOP-1942. Upgrade Phoenix to 4.6.0

Signed-off-by: YoungWoo Kim 


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/7ae4a087
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/7ae4a087
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/7ae4a087

Branch: refs/heads/master
Commit: 7ae4a08767021c58e04fe4c591bfb9cf1f902ab3
Parents: b2d94eb
Author: Jonathan Kelly 
Authored: Mon Nov 2 14:09:45 2015 -0800
Committer: YoungWoo Kim 
Committed: Sun Nov 8 00:08:02 2015 +0900

--
 bigtop.bom | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/7ae4a087/bigtop.bom
--
diff --git a/bigtop.bom b/bigtop.bom
index 1388008..ec27685 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -298,7 +298,7 @@ bigtop {
* HBase version in the BOM.
*/
   phoenix.hbase ='HBase-0.98'
-  version { base = "4.4.0-${phoenix.hbase}"; pkg = '4.4.0'; release = 1 }
+  version { base = "4.6.0-${phoenix.hbase}"; pkg = '4.6.0'; release = 1 }
   tarball { destination = "$name-${version.base}-src.tar.gz"
 source  = destination }
   url { download_path = "/$name/$name-${version.base}/src"

[3/3] bigtop git commit: BIGTOP-2009: added new tests for chgrp, cp, ls, mv, du, put, get, mkdir, stat and touchz

[ywkim]

BIGTOP-2009: added new tests for chgrp, cp, ls, mv, du, put, get, mkdir, stat 
and touchz

Signed-off-by: YoungWoo Kim 


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/96ecf29a
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/96ecf29a
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/96ecf29a

Branch: refs/heads/master
Commit: 96ecf29a1aa4f6109639411cd9287245e2ed0387
Parents: 49705da
Author: srinivas-altiscale 
Authored: Sun Sep 6 01:18:34 2015 +0530
Committer: YoungWoo Kim 
Committed: Sun Sep 6 14:44:06 2015 +0900

--
 .../itest/hadoop/hdfs/CommonFunctions.groovy|   37 +
 .../bigtop/itest/hadoop/hdfs/TestChgrp.groovy   |  249 ++
 .../bigtop/itest/hadoop/hdfs/TestCp.groovy  |  300 ++
 .../bigtop/itest/hadoop/hdfs/TestDu.groovy  |  323 ++
 .../bigtop/itest/hadoop/hdfs/TestGet.groovy |  245 ++
 .../bigtop/itest/hadoop/hdfs/TestLs.groovy  |  240 ++
 .../bigtop/itest/hadoop/hdfs/TestMkdir.groovy   |  170 +
 .../bigtop/itest/hadoop/hdfs/TestMv.groovy  |  288 ++
 .../bigtop/itest/hadoop/hdfs/TestPut.groovy |  192 +
 .../bigtop/itest/hadoop/hdfs/TestStat.groovy|  222 ++
 .../bigtop/itest/hadoop/hdfs/TestTouchz.groovy  |  173 +
 .../src/main/resources/test_data/test.zip   |  Bin 0 -> 346 bytes
 .../src/main/resources/test_data/test_1.txt |4 +
 .../src/main/resources/test_data/test_2.txt |4 +
 .../hadoop/src/main/resources/test_data/test_3  | 3321 ++
 15 files changed, 5768 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/96ecf29a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/hdfs/CommonFunctions.groovy
--
diff --git 
a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/hdfs/CommonFunctions.groovy
 
b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/hdfs/CommonFunctions.groovy
new file mode 100644
index 000..1844778
--- /dev/null
+++ 
b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/hdfs/CommonFunctions.groovy
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.bigtop.itest.hadoop.hdfs;
+import java.util.ArrayList;
+import java.util.List;
+
+public class CommonFunctions {
+  /**
+   * lookForGivenString check the given string is present in the list data
+   */
+  private boolean lookForGivenString(List data,
+ String searchString) {
+boolean result = false;
+for( String output_String : data) {
+  if(output_String.contains(searchString)) {
+result = true;
+break;
+  }
+}
+return result;
+  }
+}

http://git-wip-us.apache.org/repos/asf/bigtop/blob/96ecf29a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/hdfs/TestChgrp.groovy
--
diff --git 
a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/hdfs/TestChgrp.groovy
 
b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/hdfs/TestChgrp.groovy
new file mode 100644
index 000..1569e0a
--- /dev/null
+++ 
b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/hdfs/TestChgrp.groovy
@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, 

[1/3] bigtop git commit: BIGTOP-2009: added new tests for chgrp, cp, ls, mv, du, put, get, mkdir, stat and touchz

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 49705dad1 -> 96ecf29a1


http://git-wip-us.apache.org/repos/asf/bigtop/blob/96ecf29a/bigtop-tests/test-artifacts/hadoop/src/main/resources/test_data/test_3
--
diff --git 
a/bigtop-tests/test-artifacts/hadoop/src/main/resources/test_data/test_3 
b/bigtop-tests/test-artifacts/hadoop/src/main/resources/test_data/test_3
new file mode 100644
index 000..c59d21a
--- /dev/null
+++ b/bigtop-tests/test-artifacts/hadoop/src/main/resources/test_data/test_3
@@ -0,0 +1,3321 @@
+%PDF-1.3
+%����
+4 0 obj
+<< /Type /Info
+/Producer (FOP 0.20.5) >>
+endobj
+5 0 obj
+<< /Length 1127 /Filter [ /ASCII85Decode /FlateDecode ]
+ >>
+stream

[2/3] bigtop git commit: BIGTOP-2009: added new tests for chgrp, cp, ls, mv, du, put, get, mkdir, stat and touchz

[ywkim]

http://git-wip-us.apache.org/repos/asf/bigtop/blob/96ecf29a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/hdfs/TestStat.groovy
--
diff --git 
a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/hdfs/TestStat.groovy
 
b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/hdfs/TestStat.groovy
new file mode 100644
index 000..1e6f89a
--- /dev/null
+++ 
b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/hdfs/TestStat.groovy
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.bigtop.itest.hadoop.hdfs;
+
+import static org.junit.Assert.assertTrue;
+import org.junit.AfterClass;
+import org.junit.*;
+import org.junit.Test;
+import org.apache.bigtop.itest.shell.Shell;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.bigtop.itest.JarContent;
+import static org.apache.bigtop.itest.LogErrorsUtils.logError
+import java.util.ArrayList;
+import java.util.List;
+
+public class TestStat {
+
+  private static Shell sh = new Shell("/bin/bash -s");
+  //extracting user identity for ls absolute path
+  private static final String USERNAME = System.getProperty("user.name");
+  private static String date = sh.exec("date").getOut().get(0).
+   replaceAll("\\s","").replaceAll(":","");
+  private static String namenode = "";
+  private static String testStatInputDir = "testStatInputDir" + date;
+  private static String testStatInputs = "test_data_TestStat";
+  private static String testStatOut = "testStatOut" + date;
+  private static String testStatOutCmp= "testStatOutCmp" + date;
+  private static int repfactor = 2;
+  private static String TESTDIR  = "/user/$USERNAME/$testStatInputDir";
+  private CommonFunctions scripts = new CommonFunctions();
+  static boolean result = false;
+
+  @BeforeClass
+  public static void setUp() {
+// unpack resource
+JarContent.unpackJarContainer(TestStat.class, "." , null);
+
+sh.exec("cp -r test_data test_data_TestStat");
+assertTrue("Could not copy data into test_data_TestStat",
+   sh.getRet() == 0);
+
+// get namenode hostname from core-site.xml
+Configuration conf = new Configuration();
+namenode = conf.get("fs.defaultFS");
+if (namenode == null) {
+  namenode = conf.get("fs.default.name");
+}
+assertTrue("Could not find namenode", namenode != null);
+
+sh.exec("hdfs dfs -test -d $TESTDIR");
+if (sh.getRet() == 0) {
+  sh.exec("hdfs dfs -rm -r -skipTrash $TESTDIR");
+  assertTrue("Could not remove input directory", sh.getRet() == 0);
+}
+
+sh.exec("hdfs dfs -mkdir -p $TESTDIR");
+assertTrue("Could not create input directory on HDFS",
+   sh.getRet() == 0);
+
+// copy input directory to hdfs
+sh.exec("hdfs dfs -put $testStatInputs $TESTDIR");
+assertTrue("Could not copy files to HDFS", sh.getRet() == 0);
+
+// set the replication if file exists
+sh.exec("hdfs dfs -test -f $TESTDIR/$testStatInputs/test_2.txt");
+assertTrue("Could not find files on HDFS", sh.getRet() == 0);
+sh.exec("hdfs dfs -setrep $repfactor " +
+"$TESTDIR/$testStatInputs/test_3");
+assertTrue("Could not set replication factor", sh.getRet() == 0);
+
+println("Running stat:");
+  }
+
+  @AfterClass
+  public static void tearDown() {
+sh.exec("hdfs dfs -test -d $TESTDIR");
+if (sh.getRet() == 0) {
+  sh.exec("hdfs dfs -rm -r -skipTrash $TESTDIR");
+  assertTrue("Could not remove input directory", sh.getRet() == 0);
+}
+sh.exec("hdfs dfs -test -d $testStatOut");
+if (sh.getRet() == 0) {
+  sh.exec("hdfs dfs -rm -r -skipTrash $testStatOut");
+  assertTrue("Could not remove input directory", sh.getRet() == 0);
+}
+sh.exec("hdfs dfs -test -d $testStatOutCmp");
+if (sh.getRet() == 0) {
+  sh.exec("hdfs dfs -rm -r -skipTrash $testStatOutCmp");
+  assertTrue("Could not remove input directory", sh.getRet() == 0);
+}
+
+sh.exec("test -d $testStatInputs");
+if (sh.getRet() == 0) {
+  

bigtop git commit: BIGTOP-1909. Include compiled .mo files for HUE i18n

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master d4ada6133 -> 5efdbfbfa


BIGTOP-1909. Include compiled .mo files for HUE i18n


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/5efdbfbf
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/5efdbfbf
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/5efdbfbf

Branch: refs/heads/master
Commit: 5efdbfbfae25b09f6a706f511abdf04943bb8057
Parents: d4ada61
Author: YoungWoo Kim 
Authored: Thu Jun 25 13:34:37 2015 +0900
Committer: YoungWoo Kim 
Committed: Sun Sep 6 23:59:16 2015 +0900

--
 bigtop-packages/src/common/hue/do-component-build | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/5efdbfbf/bigtop-packages/src/common/hue/do-component-build
--
diff --git a/bigtop-packages/src/common/hue/do-component-build 
b/bigtop-packages/src/common/hue/do-component-build
index f3df5b1..1efd0bb 100644
--- a/bigtop-packages/src/common/hue/do-component-build
+++ b/bigtop-packages/src/common/hue/do-component-build
@@ -20,7 +20,7 @@ set -ex
 #FIXME: this needs to be fixed upstream
 sed -i -e "s#${FULL_VERSION}-SNAPSHOT#${FULL_VERSION}#g" `grep -lR 
${FULL_VERSION}-SNAPSHOT .`
 
-make MAVEN_VERSION=${HUE_VERSION} apps docs
+make MAVEN_VERSION=${HUE_VERSION} apps docs locales
 
 # Make the entire tree relocatable
 bash tools/relocatable.sh

bigtop git commit: Update affiliation for ywkim

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 4244b0ca1 -> cd0a66afe


Update affiliation for ywkim


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/cd0a66af
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/cd0a66af
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/cd0a66af

Branch: refs/heads/master
Commit: cd0a66afe3e9ad67192eb4f86edfcf60787df8dd
Parents: 4244b0c
Author: YoungWoo Kim <yw...@apache.org>
Authored: Wed Sep 2 23:17:16 2015 +0900
Committer: YoungWoo Kim <yw...@apache.org>
Committed: Wed Sep 2 23:17:16 2015 +0900

--
 src/site/xdoc/team-list.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/cd0a66af/src/site/xdoc/team-list.xml
--
diff --git a/src/site/xdoc/team-list.xml b/src/site/xdoc/team-list.xml
index 2d4e0f9..9fb04b9 100644
--- a/src/site/xdoc/team-list.xml
+++ b/src/site/xdoc/team-list.xml
@@ -205,7 +205,7 @@

    ywkim
YoungWoo Kim
-   KT NexR
+   SK telecom
committer

svn commit: r964053 [2/2] - in /websites/staging/bigtop/trunk/content: ./ book/ book/apache-bigtop-user-guide/

[ywkim]

Modified: websites/staging/bigtop/trunk/content/distribution-management.html
==
--- websites/staging/bigtop/trunk/content/distribution-management.html 
(original)
+++ websites/staging/bigtop/trunk/content/distribution-management.html Wed Sep  
2 14:26:42 2015
@@ -1,13 +1,13 @@
 
 
 http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
   
 
 
-
+
 
 Bigtop - Project Distribution Management
 
@@ -157,7 +157,7 @@
 
 
 
-  Last Published: 
2015-09-01 |
+  Last Published: 
2015-09-02 |
   Version: 
1.1.0-SNAPSHOT
 
 
@@ -176,7 +176,7 @@
 Snapshot Repository - apache.snapshots.httpshttps://repository.apache.org/content/repositories/snapshots;>https://repository.apache.org/content/repositories/snapshots
 
 Site - apache.website
-scm:svn:https://ofle...@svn.apache.org/repos/infra/websites/staging/bigtop/trunk/content
+scm:svn:https://yw...@svn.apache.org/repos/infra/websites/staging/bigtop/trunk/content
   
   
 

Modified: websites/staging/bigtop/trunk/content/index.html
==
--- websites/staging/bigtop/trunk/content/index.html (original)
+++ websites/staging/bigtop/trunk/content/index.html Wed Sep  2 14:26:42 2015
@@ -1,13 +1,13 @@
 
 
 http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
   
 
 
-
+
 
 Bigtop - 
Apache Bigtop
@@ -159,7 +159,7 @@
 
 
 
-  Last Published: 
2015-09-01 |
+  Last Published: 
2015-09-02 |
   Version: 
1.1.0-SNAPSHOT
 
 

Modified: websites/staging/bigtop/trunk/content/integration.html
==
--- websites/staging/bigtop/trunk/content/integration.html (original)
+++ websites/staging/bigtop/trunk/content/integration.html Wed Sep  2 14:26:42 
2015
@@ -1,13 +1,13 @@
 
 
 http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
   
 
 
-
+
 
 Bigtop - Continuous Integration
 
@@ -157,7 +157,7 @@
 
 
 
-  Last Published: 
2015-09-01 |
+  Last Published: 
2015-09-02 |
   Version: 
1.1.0-SNAPSHOT
 
 

Modified: websites/staging/bigtop/trunk/content/irc-channel.html
==
--- websites/staging/bigtop/trunk/content/irc-channel.html (original)
+++ websites/staging/bigtop/trunk/content/irc-channel.html Wed Sep  2 14:26:42 
2015
@@ -1,13 +1,13 @@
 
 
 http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
   
 
 
-
+
 
 Bigtop - 
IRC Channel
@@ -159,7 +159,7 @@
 
 
 
-  Last Published: 
2015-09-01 |
+  Last Published: 
2015-09-02 |
   Version: 
1.1.0-SNAPSHOT
 
 

Modified: websites/staging/bigtop/trunk/content/issue-tracking.html
==
--- websites/staging/bigtop/trunk/content/issue-tracking.html (original)
+++ websites/staging/bigtop/trunk/content/issue-tracking.html Wed Sep  2 
14:26:42 2015
@@ -1,13 +1,13 @@
 
 
 http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
   
 
 
-
+
 
 Bigtop - 
Issue Tracking
@@ -159,7 +159,7 @@
 
 
 
-  Last Published: 
2015-09-01 |
+  Last Published: 
2015-09-02 |
   Version: 
1.1.0-SNAPSHOT
 
 

Modified: websites/staging/bigtop/trunk/content/license.html
==
--- websites/staging/bigtop/trunk/content/license.html (original)
+++ websites/staging/bigtop/trunk/content/license.html Wed Sep  2 14:26:42 2015
@@ -1,13 +1,13 @@
 
 
 http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
   
 
 
-
+
 
 Bigtop - Project License
 
@@ -157,7 +157,7 @@
 
 
 
-  Last Published: 
2015-09-01 |
+  Last Published: 
2015-09-02 |
   Version: 
1.1.0-SNAPSHOT
 
 

Modified: websites/staging/bigtop/trunk/content/mail-lists.html
==
--- websites/staging/bigtop/trunk/content/mail-lists.html (original)
+++ websites/staging/bigtop/trunk/content/mail-lists.html Wed Sep  2 14:26:42 
2015
@@ -1,13 +1,13 

svn commit: r964055 - /websites/production/bigtop/content/

[ywkim]

Author: ywkim
Date: Wed Sep  2 14:27:24 2015
New Revision: 964055

Log:
Publishing svnmucc operation to bigtop site by ywkim

Added:
websites/production/bigtop/content/
  - copied from r964054, websites/staging/bigtop/trunk/content/

svn commit: r964053 [1/2] - in /websites/staging/bigtop/trunk/content: ./ book/ book/apache-bigtop-user-guide/

[ywkim]

Author: ywkim
Date: Wed Sep  2 14:26:42 2015
New Revision: 964053

Log:
Site checkin for project Bigtop

Modified:
websites/staging/bigtop/trunk/content/book/apache-bigtop-user-guide.fo
websites/staging/bigtop/trunk/content/book/apache-bigtop-user-guide.pdf

websites/staging/bigtop/trunk/content/book/apache-bigtop-user-guide/apache-bigtop-user-guide.html

websites/staging/bigtop/trunk/content/book/apache-bigtop-user-guide/book_index.html

websites/staging/bigtop/trunk/content/book/apache-bigtop-user-guide/introduction.html

websites/staging/bigtop/trunk/content/book/apache-bigtop-user-guide/what-s-new.html
websites/staging/bigtop/trunk/content/dependencies.html
websites/staging/bigtop/trunk/content/dependency-info.html
websites/staging/bigtop/trunk/content/dependency-management.html
websites/staging/bigtop/trunk/content/distribution-management.html
websites/staging/bigtop/trunk/content/index.html
websites/staging/bigtop/trunk/content/integration.html
websites/staging/bigtop/trunk/content/irc-channel.html
websites/staging/bigtop/trunk/content/issue-tracking.html
websites/staging/bigtop/trunk/content/license.html
websites/staging/bigtop/trunk/content/mail-lists.html
websites/staging/bigtop/trunk/content/plugin-management.html
websites/staging/bigtop/trunk/content/plugins.html
websites/staging/bigtop/trunk/content/project-info.html
websites/staging/bigtop/trunk/content/project-summary.html
websites/staging/bigtop/trunk/content/release-notes.html
websites/staging/bigtop/trunk/content/source-repository.html
websites/staging/bigtop/trunk/content/team-list.html

Modified: websites/staging/bigtop/trunk/content/book/apache-bigtop-user-guide.fo
==
--- websites/staging/bigtop/trunk/content/book/apache-bigtop-user-guide.fo 
(original)
+++ websites/staging/bigtop/trunk/content/book/apache-bigtop-user-guide.fo Wed 
Sep  2 14:26:42 2015
@@ -6,7 +6,7 @@


http://bigtop.apache.org)">
-   
+   
 [http://bigtop.apache.org)">http://bigtop.apache.org]

http://bigtop.apache.org)">
@@ -16,7 +16,7 @@
 [http://bigtop.apache.org)">http://bigtop.apache.org]
: 
http://bigtop.apache.org)">
-   
+   
 [http://bigtop.apache.org)">http://bigtop.apache.org]
Copyright 
© 2013 
Apache Software Foundation.
@@ -45,5 +45,5 @@
Apache Bigtop™
User Guide

-   Table of 
ContentsPreface  1. Introduction  2. What is new in Apache Bigtop 
1.1.0-SNAPSHOT  Index  http://www.antennahouse.com/names/XSL/Extensions; hyphenate="true" 
master-reference="front" language="en" format="i" initial-page-number="auto" 
force-page-count="no-force" hyphenation-character="-" 
hyphenation-push-character-count="2" 
hyphenation-remain-character-count="2">PrefacePreface
 text-align="center" display-align="after" 
 >relative-align="baseline"> text-align="end" display-align="after" 
 >relative-align="baseline"> flow-name="xsl-region-after-blank">font-family="serif,Symbol,ZapfDingbats" margin-left="0pt">table-layout="fixed" width="100%" border-top-width="0.5pt" 
 >border-top-style="solid" border-top-color="black">column-number="1" 
 >column-width="proportional-column-width(1)"/>column-number="2" 
 >column-width="proportional-column-width(1)"/>column-number="3" 
 >column-width="proportional-column-width(1)"/>block-progression-dimension.minimum="14pt">di
 splay-align="after" 
relative-align="baseline">Preface
-   http://www.antennahouse.com/names/XSL/Extensions; hyphenate="true" 
master-reference="body" language="en" format="1" initial-page-number="1" 
force-page-count="no-force" hyphenation-character="-" 
hyphenation-push-character-count="2" 
hyphenation-remain-character-count="2"><
 fo:block/>IntroductionIntroductionChapter 1. 
Introductionhttp://www.antennahouse.com/names/XSL/Extensions; hyphenate="true" 
master-reference="body" language="en" format="1" initial-page-number="

bigtop git commit: BIGTOP-1987. Recover resources/kmeans_data.txt for Spark smokes

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 0e2443bcf -> 68389448a


BIGTOP-1987. Recover resources/kmeans_data.txt for Spark smokes


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/68389448
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/68389448
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/68389448

Branch: refs/heads/master
Commit: 68389448a53158fd808cff6b4343137eab54e5e1
Parents: 0e2443b
Author: YoungWoo Kim 
Authored: Tue Aug 25 17:52:16 2015 +0900
Committer: Youngwoo Kim 
Committed: Wed Sep 16 09:02:14 2015 +0900

--
 .../test-artifacts/spark/src/main/resources/kmeans_data.txt| 6 ++
 1 file changed, 6 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/68389448/bigtop-tests/test-artifacts/spark/src/main/resources/kmeans_data.txt
--
diff --git 
a/bigtop-tests/test-artifacts/spark/src/main/resources/kmeans_data.txt 
b/bigtop-tests/test-artifacts/spark/src/main/resources/kmeans_data.txt
new file mode 100644
index 000..338664f
--- /dev/null
+++ b/bigtop-tests/test-artifacts/spark/src/main/resources/kmeans_data.txt
@@ -0,0 +1,6 @@
+0.0 0.0 0.0
+0.1 0.1 0.1
+0.2 0.2 0.2
+9.0 9.0 9.0
+9.1 9.1 9.1
+9.2 9.2 9.2

bigtop git commit: BIGTOP-2027. Bump gradle version, the wrapper to 2.7

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 8eef76a73 -> be796c983


BIGTOP-2027. Bump gradle version, the wrapper to 2.7

Signed-off-by: Youngwoo Kim 


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/be796c98
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/be796c98
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/be796c98

Branch: refs/heads/master
Commit: be796c98323573c7a3dff84a0a5115490c966a0f
Parents: 8eef76a
Author: Konstantin Boudnik 
Authored: Mon Sep 14 12:50:19 2015 -0700
Committer: Youngwoo Kim 
Committed: Tue Sep 15 09:25:22 2015 +0900

--
 bigtop_toolchain/manifests/deps.pp   |  6 
 bigtop_toolchain/manifests/gradle.pp | 16 ++
 gradle/wrapper/gradle-wrapper.properties |  4 ++-
 gradlew  | 43 ---
 4 files changed, 47 insertions(+), 22 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/be796c98/bigtop_toolchain/manifests/deps.pp
--
diff --git a/bigtop_toolchain/manifests/deps.pp 
b/bigtop_toolchain/manifests/deps.pp
index c52d8ef..9ec2c20 100644
--- a/bigtop_toolchain/manifests/deps.pp
+++ b/bigtop_toolchain/manifests/deps.pp
@@ -29,10 +29,4 @@ class bigtop_toolchain::deps {
 require => Package[$packages::pkgs],
 unless  => "/usr/bin/test -f /usr/src/apache-maven-3.3.3-bin.tar.gz",
   }
-
-  exec {"/usr/bin/wget 
http://services.gradle.org/distributions/gradle-2.4-bin.zip":
-cwd => "/usr/src",
-require => Package[$packages::pkgs],
-unless  => "/usr/bin/test -f /usr/src/gradle-2.4-bin.zip",
-  }
 }

http://git-wip-us.apache.org/repos/asf/bigtop/blob/be796c98/bigtop_toolchain/manifests/gradle.pp
--
diff --git a/bigtop_toolchain/manifests/gradle.pp 
b/bigtop_toolchain/manifests/gradle.pp
index 2c0081f..70be36e 100644
--- a/bigtop_toolchain/manifests/gradle.pp
+++ b/bigtop_toolchain/manifests/gradle.pp
@@ -17,16 +17,22 @@ class bigtop_toolchain::gradle {
 
   include bigtop_toolchain::deps
 
-  exec {'/usr/bin/unzip -x -o /usr/src/gradle-2.4-bin.zip':
+  exec {"/usr/bin/wget 
http://services.gradle.org/distributions/gradle-2.7-bin.zip":
+cwd => "/usr/src",
+require => Package[$packages::pkgs],
+unless  => "/usr/bin/test -f /usr/src/gradle-2.7-bin.zip",
+  }
+
+  exec {'/usr/bin/unzip -x -o /usr/src/gradle-2.7-bin.zip':
 cwd => '/usr/local',
 refreshonly => true,
-subscribe   => Exec["/usr/bin/wget 
http://services.gradle.org/distributions/gradle-2.4-bin.zip;],
-require => Exec["/usr/bin/wget 
http://services.gradle.org/distributions/gradle-2.4-bin.zip;],
+subscribe   => Exec["/usr/bin/wget 
http://services.gradle.org/distributions/gradle-2.7-bin.zip;],
+require => Exec["/usr/bin/wget 
http://services.gradle.org/distributions/gradle-2.7-bin.zip;],
   }
 
   file {'/usr/local/gradle':
 ensure  => link,
-target  => '/usr/local/gradle-2.4',
-require => Exec['/usr/bin/unzip -x -o /usr/src/gradle-2.4-bin.zip'],
+target  => '/usr/local/gradle-2.7',
+require => Exec['/usr/bin/unzip -x -o /usr/src/gradle-2.7-bin.zip'],
   }
 }

http://git-wip-us.apache.org/repos/asf/bigtop/blob/be796c98/gradle/wrapper/gradle-wrapper.properties
--
diff --git a/gradle/wrapper/gradle-wrapper.properties 
b/gradle/wrapper/gradle-wrapper.properties
index 8b98035..c7ae288 100644
--- a/gradle/wrapper/gradle-wrapper.properties
+++ b/gradle/wrapper/gradle-wrapper.properties
@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
+zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.4-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-2.7-bin.zip

http://git-wip-us.apache.org/repos/asf/bigtop/blob/be796c98/gradlew
--
diff --git a/gradlew b/gradlew
index 2ad3dae..48f44a9 100755
--- a/gradlew
+++ b/gradlew
@@ -15,7 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-## 
+##
 ## Tries to recreate Gradle's gradlew command in pure bash.
 ## This way you don't have to worry about binaries in your build.
 ##
@@ -26,10 +26,12 @@
 set -e
 set -o pipefail
 
-
 APP_NAME="Gradle"
 APP_BASE_NAME=`basename "$0"`
 
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD="maximum"
+
 # 

bigtop git commit: BIGTOP-2164. Phoenix Queryserver should write it's logs into /var/log/phoenix/

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 152bebe1d -> 0876d636e


BIGTOP-2164. Phoenix Queryserver should write it's logs into /var/log/phoenix/


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/0876d636
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/0876d636
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/0876d636

Branch: refs/heads/master
Commit: 0876d636e410a181e6345019add4a1ba8cc39e40
Parents: 152bebe
Author: Youngwoo Kim 
Authored: Thu Dec 3 14:10:35 2015 +0900
Committer: Youngwoo Kim 
Committed: Mon Dec 7 10:16:44 2015 +0900

--
 bigtop-packages/src/common/phoenix/phoenix-queryserver.default | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/0876d636/bigtop-packages/src/common/phoenix/phoenix-queryserver.default
--
diff --git a/bigtop-packages/src/common/phoenix/phoenix-queryserver.default 
b/bigtop-packages/src/common/phoenix/phoenix-queryserver.default
index 6c7d85a..0381220 100644
--- a/bigtop-packages/src/common/phoenix/phoenix-queryserver.default
+++ b/bigtop-packages/src/common/phoenix/phoenix-queryserver.default
@@ -14,3 +14,4 @@
 # limitations under the License.
 
 export HBASE_CONF_DIR=/etc/hbase/conf/
+export PHOENIX_QUERYSERVER_OPTS="-Dpsql.log.dir=/var/log/phoenix 
-Dpsql.log.file=phoenix-queryserver.log"

bigtop git commit: BIGTOP-2163. Add a phoenix-pherf sub-package for Phoenix

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 4ac6f736d -> db414adc3


BIGTOP-2163. Add a phoenix-pherf sub-package for Phoenix


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/db414adc
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/db414adc
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/db414adc

Branch: refs/heads/master
Commit: db414adc33fc7103b659d975b651ee46dcc57909
Parents: 4ac6f73
Author: Youngwoo Kim 
Authored: Thu Dec 3 14:59:58 2015 +0900
Committer: Youngwoo Kim 
Committed: Mon Dec 7 10:25:40 2015 +0900

--
 .../src/common/phoenix/install_phoenix.sh   | 49 +++-
 bigtop-packages/src/deb/phoenix/control |  6 +++
 .../src/deb/phoenix/phoenix-pherf.install   |  1 +
 .../src/rpm/phoenix/SPECS/phoenix.spec  | 12 +
 4 files changed, 67 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/db414adc/bigtop-packages/src/common/phoenix/install_phoenix.sh
--
diff --git a/bigtop-packages/src/common/phoenix/install_phoenix.sh 
b/bigtop-packages/src/common/phoenix/install_phoenix.sh
index 0a85f74..a6ab9f7 100755
--- a/bigtop-packages/src/common/phoenix/install_phoenix.sh
+++ b/bigtop-packages/src/common/phoenix/install_phoenix.sh
@@ -100,6 +100,7 @@ MAN_DIR=${MAN_DIR:-/usr/share/man/man1}
 DOC_DIR=${DOC_DIR:-/usr/share/doc/phoenix}
 LIB_DIR=${LIB_DIR:-/usr/lib/phoenix}
 BIN_DIR=${BIN_DIR:-/usr/lib/phoenix/bin}
+PHERF_DIR=${PHERF_DIR:-/usr/lib/phoenix/pherf}
 ETC_DIR=${ETC_DIR:-/etc/phoenix}
 CONF_DIR=${CONF_DIR:-${ETC_DIR}/conf.dist}
 
@@ -110,7 +111,7 @@ install -d -m 0755 $PREFIX/$DOC_DIR
 install -d -m 0755 $PREFIX/$MAN_DIR
 install -d -m 0755 $PREFIX/$ETC_DIR
 install -d -m 0755 $PREFIX/$CONF_DIR
-
+install -d -m 0755 $PREFIX/$PHERF_DIR
 install -d -m 0755 $PREFIX/var/lib/phoenix
 install -d -m 0755 $PREFIX/var/log/phoenix
 
@@ -135,3 +136,49 @@ ln -s `cd $PREFIX/$LIB_DIR ; ls phoenix*-thin-client.jar` 
$PREFIX/$LIB_DIR/phoen
 
 # phoenix-server for placing on the HBase regionserver classpath
 ln -s `cd $PREFIX/$LIB_DIR ; ls phoenix*-server.jar` 
$PREFIX/$LIB_DIR/phoenix-server.jar
+
+# Pherf
+unzip $BUILD_DIR/phoenix-pherf/phoenix-pherf-*-cluster.zip -d 
$BUILD_DIR/phoenix-pherf/
+mv $BUILD_DIR/phoenix-pherf/phoenix-pherf-*-cluster/* $PREFIX/$PHERF_DIR/
+rm -rf $PREFIX/$PHERF_DIR/config/env.sh
+
+cat >>$PREFIX/$PHERF_DIR/config/env.sh 

bigtop git commit: BIGTOP-2456 Update Apache Apex version to 3.4.0

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 9dda04be8 -> 85a2bdd9b


BIGTOP-2456 Update Apache Apex version to 3.4.0

Signed-off-by: Youngwoo Kim 


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/85a2bdd9
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/85a2bdd9
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/85a2bdd9

Branch: refs/heads/master
Commit: 85a2bdd9bfce49a904b45a4d0d015434d1a89216
Parents: 9dda04b
Author: Chinmay Kolhatkar 
Authored: Thu Jun 9 16:49:37 2016 -0700
Committer: Youngwoo Kim 
Committed: Tue Jun 14 19:54:26 2016 +0900

--
 bigtop-packages/src/common/apex/apex.1  | 2 +-
 bigtop-packages/src/common/apex/install_apex.sh | 7 +++
 bigtop-packages/src/deb/apex/control| 2 +-
 bigtop-packages/src/deb/apex/copyright  | 4 ++--
 bigtop-packages/src/rpm/apex/SPECS/apex.spec| 5 ++---
 bigtop.bom  | 8 
 6 files changed, 13 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/85a2bdd9/bigtop-packages/src/common/apex/apex.1
--
diff --git a/bigtop-packages/src/common/apex/apex.1 
b/bigtop-packages/src/common/apex/apex.1
index 38a639d..dcdb070 100644
--- a/bigtop-packages/src/common/apex/apex.1
+++ b/bigtop-packages/src/common/apex/apex.1
@@ -46,7 +46,7 @@ Dynamically change the application logical plan
 Create custom macros
 
 .RS 0
-For more information about Apache Apex, see http://apex.incubator.apache.org/
+For more information about Apache Apex, see http://apex.apache.org/
 
 
 .SH OPTIONS

http://git-wip-us.apache.org/repos/asf/bigtop/blob/85a2bdd9/bigtop-packages/src/common/apex/install_apex.sh
--
diff --git a/bigtop-packages/src/common/apex/install_apex.sh 
b/bigtop-packages/src/common/apex/install_apex.sh
index fed8f30..3993c52 100644
--- a/bigtop-packages/src/common/apex/install_apex.sh
+++ b/bigtop-packages/src/common/apex/install_apex.sh
@@ -94,13 +94,13 @@ install -d -m 0755 $BIN_DIR
 install -d -m 0755 $DOC_DIR
 install -d -m 0755 $MAN_DIR
 
-# Install dtcli
+# Install apex cli
 BINARY_FILE=$APEX_DIR/bin/apex
-cp engine/src/main/scripts/dtcli $APEX_DIR/bin/apex
+cp engine/src/main/scripts/apex $APEX_DIR/bin/apex
 chmod 755 $BINARY_FILE
 
 # Make bin wrappers
-for file in $BIN_DIR/apex $BIN_DIR/dtcli
+for file in $BIN_DIR/apex
 do
   cat > $file < $MAN_DIR/apex.1.gz
 # Install license here.
 cp LICENSE $DOC_DIR
 cp *-LICENSE.txt $DOC_DIR
-cp DISCLAIMER $DOC_DIR
 cp NOTICE $DOC_DIR

http://git-wip-us.apache.org/repos/asf/bigtop/blob/85a2bdd9/bigtop-packages/src/deb/apex/control
--
diff --git a/bigtop-packages/src/deb/apex/control 
b/bigtop-packages/src/deb/apex/control
index 8bed3cc..6145c73 100644
--- a/bigtop-packages/src/deb/apex/control
+++ b/bigtop-packages/src/deb/apex/control
@@ -19,7 +19,7 @@ Priority: extra
 Maintainer: Bigtop 
 Build-Depends: debhelper (>= 9)
 Standards-Version: 3.9.4
-Homepage: http://apex.incubator.apache.org
+Homepage: http://apex.apache.org
 
 Package:  apex
 Architecture: all

http://git-wip-us.apache.org/repos/asf/bigtop/blob/85a2bdd9/bigtop-packages/src/deb/apex/copyright
--
diff --git a/bigtop-packages/src/deb/apex/copyright 
b/bigtop-packages/src/deb/apex/copyright
index ab45999..48354d1 100644
--- a/bigtop-packages/src/deb/apex/copyright
+++ b/bigtop-packages/src/deb/apex/copyright
@@ -1,6 +1,6 @@
 Format: http://dep.debian.net/deps/dep5
-Source: http://apex.incubator.apache.org/
-Upstream-Name: Apache Apex (incubating)
+Source: http://apex.apache.org/
+Upstream-Name: Apache Apex
 
 Files: *
 Copyright: 2015, The Apache Software Foundation

http://git-wip-us.apache.org/repos/asf/bigtop/blob/85a2bdd9/bigtop-packages/src/rpm/apex/SPECS/apex.spec
--
diff --git a/bigtop-packages/src/rpm/apex/SPECS/apex.spec 
b/bigtop-packages/src/rpm/apex/SPECS/apex.spec
index 5f0291a..65cd9bb 100644
--- a/bigtop-packages/src/rpm/apex/SPECS/apex.spec
+++ b/bigtop-packages/src/rpm/apex/SPECS/apex.spec
@@ -35,7 +35,7 @@ Version: %{apex_version}
 Release: %{apex_release}
 Summary: Apache Apex is an enterprise Grade YARN-native platform for unified 
stream and batch processing
 License: ASL 2.0
-URL: http://apex.incubator.apache.org
+URL: http://apex.apache.org
 Group: Development/Libraries
 Buildroot: %{_topdir}/INSTALL/%{name}-%{version}
 BuildArch: noarch
@@ -57,7 +57,7 @@ Apache Apex includes following key 

bigtop git commit: BIGTOP-2314 Added deb and rpm package of Apache Apex to bigtop.

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 3ff441726 -> 510ac7b5e


BIGTOP-2314 Added deb and rpm package of Apache Apex to bigtop.

Signed-off-by: Youngwoo Kim 


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/510ac7b5
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/510ac7b5
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/510ac7b5

Branch: refs/heads/master
Commit: 510ac7b5e0c0be81450d6886adfa377fe4838adf
Parents: 3ff4417
Author: chinmaykolhatkar 
Authored: Tue Feb 9 21:47:34 2016 +0530
Committer: Youngwoo Kim 
Committed: Thu Feb 25 22:44:49 2016 +0900

--
 MAINTAINERS.txt |   1 +
 bigtop-packages/src/common/apex/apex.1  | 499 +++
 .../src/common/apex/do-component-build  |  81 +++
 bigtop-packages/src/common/apex/install_apex.sh | 125 +
 bigtop-packages/src/deb/apex/apex.dirs  |   5 +
 bigtop-packages/src/deb/apex/apex.install   |   6 +
 .../src/deb/apex/apex.lintian-overrides |   3 +
 bigtop-packages/src/deb/apex/changelog  |   1 +
 bigtop-packages/src/deb/apex/compat |   1 +
 bigtop-packages/src/deb/apex/control|  35 ++
 bigtop-packages/src/deb/apex/copyright  |  15 +
 bigtop-packages/src/deb/apex/rules  |  35 ++
 bigtop-packages/src/deb/apex/source/format  |   1 +
 bigtop-packages/src/rpm/apex/SPECS/apex.spec|  77 +++
 bigtop.bom  |  12 +-
 15 files changed, 896 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/510ac7b5/MAINTAINERS.txt
--
diff --git a/MAINTAINERS.txt b/MAINTAINERS.txt
index 220f3d9..4954801 100644
--- a/MAINTAINERS.txt
+++ b/MAINTAINERS.txt
@@ -1,3 +1,4 @@
+apex: chinmay , aniruddha 
 bigpetstore: jay vyas, rj nowling
 CI infra: rvs
 data generators: rj nowling, jay vyas

http://git-wip-us.apache.org/repos/asf/bigtop/blob/510ac7b5/bigtop-packages/src/common/apex/apex.1
--
diff --git a/bigtop-packages/src/common/apex/apex.1 
b/bigtop-packages/src/common/apex/apex.1
new file mode 100644
index 000..38a639d
--- /dev/null
+++ b/bigtop-packages/src/common/apex/apex.1
@@ -0,0 +1,499 @@
+.\" Licensed to the Apache Software Foundation (ASF) under one or more
+.\" contributor license agreements.  See the NOTICE file distributed with
+.\" this work for additional information regarding copyright ownership.
+.\" The ASF licenses this file to You under the Apache License, Version 2.0
+.\" (the "License"); you may not use this file except in compliance with
+.\" the License.  You may obtain a copy of the License at
+.\"
+.\" http://www.apache.org/licenses/LICENSE-2.0
+.\"
+.\" Unless required by applicable law or agreed to in writing, software
+.\" distributed under the License is distributed on an "AS IS" BASIS,
+.\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.\" See the License for the specific language governing permissions and
+.\" limitations under the License.
+.\"
+.\" Process this file with
+.\" groff -man -Tascii apex.1
+.\"
+.TH apex 1 "August 2015 " Linux "User Manuals"
+
+.SH NAME
+Apex \- Unified streaming and batch processing engine
+
+.SH SYNOPSIS
+
+.B apex 
+[options] [-efhpr] [-kp] [-kt] [-v|vv|vvv|]
+
+.SH DESCRIPTION
+
+Apache Apex command line interface can be used to launch, monitor, and manage 
Apache Apex applications. It enables easy integration with existing enterprise 
toolsets for automated application monitoring and management. Currently the 
following high level tasks are supported:
+
+.IP \[bu] 2
+Launch or kill applications
+.IP \[bu] 2
+View system metrics including load, throughput, latency, etc.
+.IP \[bu] 2
+Start or stop tuple recording
+.IP \[bu] 2
+Read operator, stream, port properties and attributes
+.IP \[bu] 2
+Write to operator properties
+.IP \[bu] 2
+Dynamically change the application logical plan
+.IP \[bu] 2
+Create custom macros
+
+.RS 0
+For more information about Apache Apex, see http://apex.incubator.apache.org/
+
+
+.SH OPTIONS
+
+.TP
+.BI \-e " arg"
+Commands are read from the argument
+.TP
+.BI \-f " arg"
+Use the specified prompt at all time
+.TP
+.BI \-h
+Print this help
+.TP
+.BI \-kp " arg"
+Use the specified kerberos principal
+.TP
+.BI \-kt " arg"
+Use the specified kerberos keytab
+.TP
+.BI \-p " arg"
+JSONP padding function
+.TP
+.BI \-r
+JSON Raw mode
+.TP
+.BI \-v[v|vv|vvv]
+Verbose mode level 1-4
+
+
+
+.SH COMMANDS
+.SS Global command except when changing logical plan:
+
+.TP
+.BI alias " alias-name 

bigtop git commit: BIGTOP-2220 : flume-agent.init incorrectly handles flume.conf

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 1e9784144 -> 03a3f3494


BIGTOP-2220 : flume-agent.init incorrectly handles flume.conf

Signed-off-by: Youngwoo Kim 


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/03a3f349
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/03a3f349
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/03a3f349

Branch: refs/heads/master
Commit: 03a3f34942cd419f0332e93b376feaf576db42f7
Parents: 1e97841
Author: zenmyo 
Authored: Thu Jan 7 14:21:11 2016 +0900
Committer: Youngwoo Kim 
Committed: Fri Feb 26 10:21:37 2016 +0900

--
 bigtop-packages/src/common/flume/flume-agent.init | 8 ++--
 1 file changed, 6 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/03a3f349/bigtop-packages/src/common/flume/flume-agent.init
--
diff --git a/bigtop-packages/src/common/flume/flume-agent.init 
b/bigtop-packages/src/common/flume/flume-agent.init
index 1d041a2..ec179c0 100644
--- a/bigtop-packages/src/common/flume/flume-agent.init
+++ b/bigtop-packages/src/common/flume/flume-agent.init
@@ -197,10 +197,14 @@ checkallstatus() {
 # Common function to perform user action on all flume conf files
 #
 run_functions_on_conf() {
+  if [ -f ${FLUME_CONF_DIR}/flume.conf ]; then
+echo $FLUME_AGENT_NAME
+$1
+  fi
+  agent_conf_pattern="${FLUME_CONF_DIR}/flume-.*\.conf"
   for f in ${FLUME_CONF_DIR}/*
   do
-file_ext=${f##*.}
-if [ "$file_ext" = "conf" ]; then
+if [ $(echo $f | grep -e ${agent_conf_pattern}) ]; then
   conf_file=${f%.*}
   file_name=${conf_file##*/}
   agent_name=${file_name#*-}

bigtop git commit: BIGTOP-2179: Add packaging scripts for Apache Tajo

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master abc43b807 -> 22caf2ad8


BIGTOP-2179: Add packaging scripts for Apache Tajo

Signed-off-by: Youngwoo Kim 


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/22caf2ad
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/22caf2ad
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/22caf2ad

Branch: refs/heads/master
Commit: 22caf2ad8d253edf6da83bcdabda8038cbbdd5e8
Parents: abc43b8
Author: yeongeon 
Authored: Mon Feb 15 10:56:01 2016 +0900
Committer: Youngwoo Kim 
Committed: Mon Feb 15 20:22:04 2016 +0900

--
 MAINTAINERS.txt |   1 +
 .../src/common/tajo/do-component-build  |  28 +++
 bigtop-packages/src/common/tajo/install_tajo.sh | 156 +++
 .../src/common/tajo/tajo-env.default|  97 +
 bigtop-packages/src/common/tajo/tajo-master.svc |  76 +++
 bigtop-packages/src/common/tajo/tajo-worker.svc |  76 +++
 bigtop-packages/src/deb/tajo/compat |   1 +
 bigtop-packages/src/deb/tajo/control|  43 
 bigtop-packages/src/deb/tajo/copyright  |  15 ++
 bigtop-packages/src/deb/tajo/rules  |  41 
 bigtop-packages/src/deb/tajo/source/format  |   1 +
 bigtop-packages/src/deb/tajo/tajo.postinst  |  35 
 bigtop-packages/src/deb/tajo/tajo.preinst   |  62 ++
 bigtop-packages/src/deb/tajo/tajo.prerm |  38 
 bigtop-packages/src/rpm/tajo/SPECS/tajo.spec| 199 +++
 bigtop.bom  |  13 +-
 16 files changed, 881 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/22caf2ad/MAINTAINERS.txt
--
diff --git a/MAINTAINERS.txt b/MAINTAINERS.txt
index 151c3aa..220f3d9 100644
--- a/MAINTAINERS.txt
+++ b/MAINTAINERS.txt
@@ -20,6 +20,7 @@ smoke-tests: jay vyas, david capwell
 spark: jay vyas, youngwoo kim
 sqoop: sean mackrory, youngwoo kim
 tachyon: jay vyas, huamin chen
+tajo: yeongeon kim
 test-artifacts and test-execution repos:
 tez: evans ye, oflebbe
 vagrant recipes: jay vyas, evans ye

http://git-wip-us.apache.org/repos/asf/bigtop/blob/22caf2ad/bigtop-packages/src/common/tajo/do-component-build
--
diff --git a/bigtop-packages/src/common/tajo/do-component-build 
b/bigtop-packages/src/common/tajo/do-component-build
new file mode 100644
index 000..579a326
--- /dev/null
+++ b/bigtop-packages/src/common/tajo/do-component-build
@@ -0,0 +1,28 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# recognize for TAJO_VERSION, HADOOP_VERSION
+. `dirname $0`/bigtop.bom
+
+rm -rf build
+mkdir build
+
+mvn clean package -DskipTests -Pdist -Dtar -Dhadoop.version=$HADOOP_VERSION 
"$@"
+
+(cd build; tar --strip-components=1 -xzvf  
../tajo-dist/target/tajo-${TAJO_VERSION}.tar.gz)
+

http://git-wip-us.apache.org/repos/asf/bigtop/blob/22caf2ad/bigtop-packages/src/common/tajo/install_tajo.sh
--
diff --git a/bigtop-packages/src/common/tajo/install_tajo.sh 
b/bigtop-packages/src/common/tajo/install_tajo.sh
new file mode 100644
index 000..1aa7422
--- /dev/null
+++ b/bigtop-packages/src/common/tajo/install_tajo.sh
@@ -0,0 +1,156 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed 

bigtop git commit: BIGTOP-2337. Fix script to make deb in Tajo

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 3565e09d8 -> 530985d83


BIGTOP-2337. Fix script to make deb in Tajo

Signed-off-by: Youngwoo Kim 


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/530985d8
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/530985d8
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/530985d8

Branch: refs/heads/master
Commit: 530985d83258229c3c178d37599ae3d1b1e47e7a
Parents: 3565e09
Author: yeongeon 
Authored: Wed Feb 17 16:12:29 2016 +0900
Committer: Youngwoo Kim 
Committed: Wed Feb 17 21:07:26 2016 +0900

--
 bigtop-packages/src/deb/tajo/tajo.install | 13 +
 1 file changed, 13 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/530985d8/bigtop-packages/src/deb/tajo/tajo.install
--
diff --git a/bigtop-packages/src/deb/tajo/tajo.install 
b/bigtop-packages/src/deb/tajo/tajo.install
new file mode 100644
index 000..2a01b3b
--- /dev/null
+++ b/bigtop-packages/src/deb/tajo/tajo.install
@@ -0,0 +1,13 @@
+/etc/tajo
+/usr/bin/tsql
+/usr/bin/tajo
+/usr/lib/tajo/*.jar
+/usr/lib/tajo/bin/*
+/usr/lib/tajo/extlib
+/usr/lib/tajo/conf
+/usr/lib/tajo/hive/*.jar
+/usr/lib/tajo/lib/*.jar
+/usr/lib/tajo/share/jdbc-dist/*.jar
+/var/lib/tajo
+/var/log/tajo
+/var/run/tajo
\ No newline at end of file

bigtop git commit: BIGTOP-2332. Upgrade Tajo version to 0.11.1

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 530985d83 -> 0e3c72192


BIGTOP-2332. Upgrade Tajo version to 0.11.1

Signed-off-by: Youngwoo Kim 


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/0e3c7219
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/0e3c7219
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/0e3c7219

Branch: refs/heads/master
Commit: 0e3c72192e4f3648cfd85b13941656545062d314
Parents: 530985d
Author: yeongeon 
Authored: Wed Feb 17 16:29:36 2016 +0900
Committer: Youngwoo Kim 
Committed: Wed Feb 17 21:31:28 2016 +0900

--
 bigtop.bom | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/0e3c7219/bigtop.bom
--
diff --git a/bigtop.bom b/bigtop.bom
index b6cd6d1..2f1abc9 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -405,7 +405,7 @@ bigtop {
 'tajo' {
   name= 'tajo'
   relNotes = 'Apache Tajo'
-  version { base = '0.11.0'; pkg = base; release = 1 }
+  version { base = '0.11.1'; pkg = base; release = 1 }
   tarball { source  = "$name-${version.base}-src.tar.gz"
 destination = "$name-${version.base}.tar.gz" }
   url { download_path = "/$name/$name-${version.base}"

bigtop git commit: BIGTOP-2316. Adding puppet recipes for apache apex.

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 374cf9b74 -> bc1312d64


BIGTOP-2316. Adding puppet recipes for apache apex.

Signed-off-by: Youngwoo Kim 


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/bc1312d6
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/bc1312d6
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/bc1312d6

Branch: refs/heads/master
Commit: bc1312d64e68988086ec08a59fe662a3b89a28ab
Parents: 374cf9b
Author: chinmaykolhatkar 
Authored: Mon Mar 14 11:30:05 2016 +0530
Committer: Youngwoo Kim 
Committed: Fri Mar 18 14:23:07 2016 +0900

--
 bigtop-deploy/puppet/hieradata/site.yaml|  1 +
 bigtop-deploy/puppet/manifests/cluster.pp   |  4 +++
 .../puppet/modules/apex/manifests/init.pp   | 30 
 bigtop-deploy/puppet/modules/apex/tests/init.pp | 17 +++
 4 files changed, 52 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/bc1312d6/bigtop-deploy/puppet/hieradata/site.yaml
--
diff --git a/bigtop-deploy/puppet/hieradata/site.yaml 
b/bigtop-deploy/puppet/hieradata/site.yaml
index e44cb50..ca429e6 100644
--- a/bigtop-deploy/puppet/hieradata/site.yaml
+++ b/bigtop-deploy/puppet/hieradata/site.yaml
@@ -12,6 +12,7 @@ hadoop::hadoop_storage_dirs:
   - /data/4
 
 #hadoop_cluster_node::cluster_components:
+#  - apex
 #  - crunch
 #  - flume
 #  - giraph

http://git-wip-us.apache.org/repos/asf/bigtop/blob/bc1312d6/bigtop-deploy/puppet/manifests/cluster.pp
--
diff --git a/bigtop-deploy/puppet/manifests/cluster.pp 
b/bigtop-deploy/puppet/manifests/cluster.pp
index a0be567..00042a9 100644
--- a/bigtop-deploy/puppet/manifests/cluster.pp
+++ b/bigtop-deploy/puppet/manifests/cluster.pp
@@ -14,6 +14,9 @@
 # limitations under the License.
 
 $roles_map = {
+  apex => {
+client => ["apex-client"],
+  },
   hdfs-non-ha => {
 master => ["namenode"],
 worker => ["datanode"],
@@ -150,6 +153,7 @@ class node_with_roles ($roles = hiera("bigtop::roles")) 
inherits hadoop_cluster_
   }
 
   $modules = [
+"apex",
 "crunch",
 "giraph",
 "hadoop",

http://git-wip-us.apache.org/repos/asf/bigtop/blob/bc1312d6/bigtop-deploy/puppet/modules/apex/manifests/init.pp
--
diff --git a/bigtop-deploy/puppet/modules/apex/manifests/init.pp 
b/bigtop-deploy/puppet/modules/apex/manifests/init.pp
new file mode 100644
index 000..c4b91c9
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/apex/manifests/init.pp
@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class apex {
+
+  class deploy ($roles) {
+if ("apex-client" in $roles) {
+  include apex::client
+}
+  }
+
+  class client {
+package { "apex":
+  ensure => latest,
+  require => Package["hadoop"],
+}
+  }
+}

http://git-wip-us.apache.org/repos/asf/bigtop/blob/bc1312d6/bigtop-deploy/puppet/modules/apex/tests/init.pp
--
diff --git a/bigtop-deploy/puppet/modules/apex/tests/init.pp 
b/bigtop-deploy/puppet/modules/apex/tests/init.pp
new file mode 100644
index 000..0885cbf
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/apex/tests/init.pp
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# 

bigtop git commit: BIGTOP-2327. Bump up Phoenix to 4.7.0

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 2805e17e0 -> 374cf9b74


BIGTOP-2327. Bump up Phoenix to 4.7.0


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/374cf9b7
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/374cf9b7
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/374cf9b7

Branch: refs/heads/master
Commit: 374cf9b7431b2268e429d406e422586bea93ae7c
Parents: 2805e17
Author: Youngwoo Kim 
Authored: Sun Mar 13 23:18:24 2016 +0900
Committer: Youngwoo Kim 
Committed: Mon Mar 14 20:44:12 2016 +0900

--
 .../src/common/phoenix/do-component-build   |  4 +-
 .../src/common/phoenix/install_phoenix.sh   | 56 +++-
 bigtop-packages/src/deb/phoenix/control |  5 --
 .../src/deb/phoenix/phoenix-pherf.install   |  1 -
 .../src/rpm/phoenix/SPECS/phoenix.spec  | 13 -
 bigtop.bom  |  2 +-
 pom.xml |  2 +-
 7 files changed, 12 insertions(+), 71 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/374cf9b7/bigtop-packages/src/common/phoenix/do-component-build
--
diff --git a/bigtop-packages/src/common/phoenix/do-component-build 
b/bigtop-packages/src/common/phoenix/do-component-build
index 9eaf3bd..1f71864 100644
--- a/bigtop-packages/src/common/phoenix/do-component-build
+++ b/bigtop-packages/src/common/phoenix/do-component-build
@@ -24,7 +24,9 @@ mvn -DskipTests \
 -Dpig.version=$PIG_VERSION  \
 -Dflume.version=$FLUME_VERSION  \
 -Dspark.version=${SPARK_VERSION} \
-clean install package "$@"
+clean install "$@"
 rm -rf build
 mkdir build
 tar -C build --strip-components=1 -xzf 
phoenix-assembly/target/phoenix-$PHOENIX_VERSION.tar.gz
+cp -ra dev/release_files/* build/
+cp -ra phoenix-pherf/config build/bin/

http://git-wip-us.apache.org/repos/asf/bigtop/blob/374cf9b7/bigtop-packages/src/common/phoenix/install_phoenix.sh
--
diff --git a/bigtop-packages/src/common/phoenix/install_phoenix.sh 
b/bigtop-packages/src/common/phoenix/install_phoenix.sh
index a6ab9f7..b1b25b7 100755
--- a/bigtop-packages/src/common/phoenix/install_phoenix.sh
+++ b/bigtop-packages/src/common/phoenix/install_phoenix.sh
@@ -100,7 +100,6 @@ MAN_DIR=${MAN_DIR:-/usr/share/man/man1}
 DOC_DIR=${DOC_DIR:-/usr/share/doc/phoenix}
 LIB_DIR=${LIB_DIR:-/usr/lib/phoenix}
 BIN_DIR=${BIN_DIR:-/usr/lib/phoenix/bin}
-PHERF_DIR=${PHERF_DIR:-/usr/lib/phoenix/pherf}
 ETC_DIR=${ETC_DIR:-/etc/phoenix}
 CONF_DIR=${CONF_DIR:-${ETC_DIR}/conf.dist}
 
@@ -111,18 +110,22 @@ install -d -m 0755 $PREFIX/$DOC_DIR
 install -d -m 0755 $PREFIX/$MAN_DIR
 install -d -m 0755 $PREFIX/$ETC_DIR
 install -d -m 0755 $PREFIX/$CONF_DIR
-install -d -m 0755 $PREFIX/$PHERF_DIR
 install -d -m 0755 $PREFIX/var/lib/phoenix
 install -d -m 0755 $PREFIX/var/log/phoenix
 
 cp -ra $BUILD_DIR/{bin,lib} $PREFIX/$LIB_DIR/
 rm $PREFIX/$LIB_DIR/lib/phoenix-*.jar
-chmod 755 $PREFIX/$BIN_DIR/*.py
 cp -a $BUILD_DIR/*.jar $PREFIX/$LIB_DIR/
 cp -a $BUILD_DIR/lib/phoenix-*.jar $PREFIX/$LIB_DIR/
+
+chmod 755 $PREFIX/$BIN_DIR/*.py
+chmod 755 $PREFIX/$BIN_DIR/*.sh
+
 # Remove sources jar
 rm $PREFIX/$LIB_DIR/phoenix-*-sources.jar
-cp -a $BUILD_DIR/examples $PREFIX/$DOC_DIR
+
+cp -a $BUILD_DIR/{LICENSE,NOTICE} $PREFIX/$DOC_DIR/
+cp -ra $BUILD_DIR/examples $PREFIX/$DOC_DIR
 
 # Remove the executable bit from jars to avoid lintian warnings
 find $PREFIX/$LIB_DIR -name '*.jar' -exec chmod a-x {} \;
@@ -137,48 +140,3 @@ ln -s `cd $PREFIX/$LIB_DIR ; ls phoenix*-thin-client.jar` 
$PREFIX/$LIB_DIR/phoen
 # phoenix-server for placing on the HBase regionserver classpath
 ln -s `cd $PREFIX/$LIB_DIR ; ls phoenix*-server.jar` 
$PREFIX/$LIB_DIR/phoenix-server.jar
 
-# Pherf
-unzip $BUILD_DIR/phoenix-pherf/phoenix-pherf-*-cluster.zip -d 
$BUILD_DIR/phoenix-pherf/
-mv $BUILD_DIR/phoenix-pherf/phoenix-pherf-*-cluster/* $PREFIX/$PHERF_DIR/
-rm -rf $PREFIX/$PHERF_DIR/config/env.sh
-
-cat >>$PREFIX/$PHERF_DIR/config/env.sh 

bigtop git commit: BIGTOP-2359. Add .DS_Store to .gitignore

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master bc1312d64 -> bba8493fa


BIGTOP-2359. Add .DS_Store to .gitignore


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/bba8493f
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/bba8493f
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/bba8493f

Branch: refs/heads/master
Commit: bba8493faed56e71066642638881c11eff3f3535
Parents: bc1312d
Author: Youngwoo Kim 
Authored: Fri Mar 18 15:34:54 2016 +0900
Committer: Youngwoo Kim 
Committed: Fri Mar 18 15:34:54 2016 +0900

--
 .gitignore | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/bba8493f/.gitignore
--
diff --git a/.gitignore b/.gitignore
index 8e6be6d..dd81dfa 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,3 +12,4 @@ target
 .gradle
 gradle*
 .vagrant
+.DS_Store

bigtop git commit: BIGTOP-2315. Added smoke test for apex.

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 64c262fca -> 66de460b8


BIGTOP-2315. Added smoke test for apex.

Signed-off-by: Youngwoo Kim 


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/66de460b
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/66de460b
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/66de460b

Branch: refs/heads/master
Commit: 66de460b80777d812cfc5f87af81d9df77ed1d40
Parents: 64c262f
Author: chinmaykolhatkar 
Authored: Thu Mar 3 23:08:38 2016 +0530
Committer: Youngwoo Kim 
Committed: Thu Apr 7 22:58:33 2016 +0900

--
 .../smoke-tests/apex/TestApexSmoke.groovy   | 117 
 bigtop-tests/smoke-tests/apex/build.gradle  |  34 
 bigtop-tests/test-artifacts/apex/pom.xml| 191 +++
 .../apex/src/assemble/appPackage.xml|  59 ++
 .../java/com/example/myapexapp/Application.java |  41 
 .../myapexapp/ConsoleOutputOperator.java|  79 
 .../myapexapp/RandomNumberGenerator.java|  61 ++
 .../src/main/resources/META-INF/properties.xml  |  28 +++
 8 files changed, 610 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/66de460b/bigtop-tests/smoke-tests/apex/TestApexSmoke.groovy
--
diff --git a/bigtop-tests/smoke-tests/apex/TestApexSmoke.groovy 
b/bigtop-tests/smoke-tests/apex/TestApexSmoke.groovy
new file mode 100644
index 000..dd712c2
--- /dev/null
+++ b/bigtop-tests/smoke-tests/apex/TestApexSmoke.groovy
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.bigtop.itest.apex
+
+import org.junit.BeforeClass
+import org.junit.AfterClass
+import org.junit.FixMethodOrder
+import org.junit.runners.MethodSorters
+import org.junit.Test
+import static org.junit.Assert.assertTrue
+
+import org.apache.bigtop.itest.shell.Shell
+import org.apache.bigtop.itest.TestUtils
+import org.apache.commons.logging.LogFactory
+import org.apache.commons.logging.Log
+
+import org.junit.runner.RunWith
+
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
+class TestApexSmoke {
+  static private Log LOG = LogFactory.getLog(Object.class);
+
+  static def testDir = "build/resources/test/"
+  static Shell sh = new Shell("/bin/bash -s");
+  static appId = null;
+
+  @AfterClass
+  public static void tearDown() {
+sh.exec('hadoop fs -rm -r -skipTrash datatorrent');
+sh.exec('rm -rf ' + testDir);
+  }
+
+  @BeforeClass
+  static void setUp() {
+sh.exec('pushd ' + testDir,
+'mvn clean package -DskipTests',
+'popd');
+LOG.info('Setup Done. Package ready to launch.');
+  }
+
+  @Test
+  void test1_Start() {
+String apaPath = testDir + "target/myapexapp-1.0-SNAPSHOT.apa";
+LOG.info('Starting apex application...');
+sh.exec('echo "launch ' + apaPath + '" | apex');
+
+String out = sh.getOut()[1].trim();
+LOG.info("ApplicationID: " + out);
+assertTrue(out.contains('{"appId": "application_'));
+out = out.replace('{"appId": "', '');
+appId = out.replace('"}', '');
+LOG.info('Sleeping for 20 sec for application to start...');
+sh.exec('sleep 20');
+sh.exec("yarn application -status " + appId);
+out = sh.getOut();
+for (String item : out) {
+  if (item.contains('Application-Id :')) {
+assertTrue(item.contains(appId));
+  }
+  if (item.contains('Application-Name :')) {
+assertTrue(item.contains('MyFirstApplication'));
+  }
+  if (item.contains('Application-Type :')) {
+assertTrue(item.contains('DataTorrent'));
+  }
+  else if (item.contains('State :')) {
+assertTrue(item.contains('RUNNING'));
+  }
+  else if (item.contains('Final-State :')) {
+assertTrue(item.contains('UNDEFINED'));
+  }
+}
+  }
+
+  @Test
+  void test2_Stop() {
+LOG.info('Stopping apex application..');
+sh.exec('echo "shutdown-app ' + 

bigtop git commit: BIGTOP-1843: Upgrade Sqoop to 1.4.6

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 03a3f3494 -> 027037e63


BIGTOP-1843: Upgrade Sqoop to 1.4.6

Signed-off-by: Youngwoo Kim 


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/027037e6
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/027037e6
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/027037e6

Branch: refs/heads/master
Commit: 027037e6330c8269ca20d1424ff3d02967218c03
Parents: 03a3f34
Author: Craig Foster 
Authored: Wed Feb 10 10:32:27 2016 -0800
Committer: Youngwoo Kim 
Committed: Wed Mar 2 20:45:23 2016 +0900

--
 bigtop.bom | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/027037e6/bigtop.bom
--
diff --git a/bigtop.bom b/bigtop.bom
index 363b3fc..64e6687 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -185,7 +185,7 @@ bigtop {
 'sqoop' {
   name= 'sqoop'
   relNotes = 'Apache Sqoop v1'
-  version { base = '1.4.5'; pkg = base; release = 1 }
+  version { base = '1.4.6'; pkg = base; release = 1 }
   tarball { destination = "${name}-${version.base}.tar.gz"
 source  = destination }
   url { download_path = "/$name/${version.base}/"

bigtop git commit: BIGTOP-2425: crunch needs JDK8 javadoc

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master c6033e96c -> e5480da42


BIGTOP-2425: crunch needs JDK8 javadoc

Signed-off-by: Youngwoo Kim 


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/e5480da4
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/e5480da4
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/e5480da4

Branch: refs/heads/master
Commit: e5480da42b9fe4db2cd26c578cc599cab81e06c6
Parents: c6033e9
Author: Olaf Flebbe 
Authored: Mon May 9 21:58:57 2016 +0200
Committer: Youngwoo Kim 
Committed: Thu May 12 20:11:34 2016 +0900

--
 .../src/common/crunch/patch1-remove-Xdoclintnone.diff | 10 ++
 bigtop-packages/src/rpm/crunch/SPECS/crunch.spec  |  3 +++
 2 files changed, 13 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/e5480da4/bigtop-packages/src/common/crunch/patch1-remove-Xdoclintnone.diff
--
diff --git a/bigtop-packages/src/common/crunch/patch1-remove-Xdoclintnone.diff 
b/bigtop-packages/src/common/crunch/patch1-remove-Xdoclintnone.diff
new file mode 100644
index 000..8f0fc5b
--- /dev/null
+++ b/bigtop-packages/src/common/crunch/patch1-remove-Xdoclintnone.diff
@@ -0,0 +1,10 @@
+--- apache-crunch-0.13.0-src/pom.xml.orig  2016-05-09 20:10:32.634858687 
+0200
 apache-crunch-0.13.0-src/pom.xml   2016-05-09 20:11:04.566858115 +0200
+@@ -610,7 +610,6 @@
+   
+ 
+ 
+--Xdoclint:none
+   
+   
+ 

http://git-wip-us.apache.org/repos/asf/bigtop/blob/e5480da4/bigtop-packages/src/rpm/crunch/SPECS/crunch.spec
--
diff --git a/bigtop-packages/src/rpm/crunch/SPECS/crunch.spec 
b/bigtop-packages/src/rpm/crunch/SPECS/crunch.spec
index c8e0975..871fbe9 100644
--- a/bigtop-packages/src/rpm/crunch/SPECS/crunch.spec
+++ b/bigtop-packages/src/rpm/crunch/SPECS/crunch.spec
@@ -42,6 +42,7 @@ Source0: %{crunch_folder}.tar.gz
 Source1: do-component-build 
 Source2: install_%{crunch_name}.sh
 Source3: bigtop.bom
+#BIGTOP_PATCH_FILES
 Requires: hadoop-client, bigtop-utils >= 0.7, hbase
 
 %description 
@@ -59,6 +60,8 @@ Apache Crunch documentation
 %prep
 %setup -n %{crunch_folder}
 
+#BIGTOP_PATCH_COMMANDS
+
 %build
 bash $RPM_SOURCE_DIR/do-component-build
 

bigtop git commit: BIGTOP-2415. readme spelling error

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 9375e3e1a -> d9b12bbc9


BIGTOP-2415. readme spelling error

Signed-off-by: Youngwoo Kim 


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/d9b12bbc
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/d9b12bbc
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/d9b12bbc

Branch: refs/heads/master
Commit: d9b12bbc98197d9feb14224f12cefe5344cb44e9
Parents: 9375e3e
Author: Artem Ervits 
Authored: Sun May 1 21:59:07 2016 -0400
Committer: Youngwoo Kim 
Committed: Mon May 2 16:29:30 2016 +0900

--
 bigtop-deploy/vm/vagrant-puppet-docker/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/d9b12bbc/bigtop-deploy/vm/vagrant-puppet-docker/README.md
--
diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/README.md 
b/bigtop-deploy/vm/vagrant-puppet-docker/README.md
index 79e50f7..602b10b 100644
--- a/bigtop-deploy/vm/vagrant-puppet-docker/README.md
+++ b/bigtop-deploy/vm/vagrant-puppet-docker/README.md
@@ -157,4 +157,4 @@ See `bigtop-deploy/puppet/config/site.csv.example` for more 
details.
 
 ##Notes
 
-* Users currently using vagrant 1.6+ is strongly recommanded to upgrade to 
1.6.4+, otherwise you will encounter the 
[issue](https://github.com/mitchellh/vagrant/issues/3769) when installing 
plguins
+* Users currently using vagrant 1.6+ is strongly recommanded to upgrade to 
1.6.4+, otherwise you will encounter the 
[issue](https://github.com/mitchellh/vagrant/issues/3769) when installing 
plugins

bigtop git commit: BIGTOP-2500. Remove hadoop jars from TEZ_HOME/lib and create appropriate symlinks

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 47d1f6970 -> 19c442c66


BIGTOP-2500. Remove hadoop jars from TEZ_HOME/lib and create appropriate 
symlinks


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/19c442c6
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/19c442c6
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/19c442c6

Branch: refs/heads/master
Commit: 19c442c666f5c179d1b4ab17af7074eeaa1b887d
Parents: 47d1f69
Author: Youngwoo Kim 
Authored: Tue Jul 19 10:25:47 2016 +0900
Committer: Youngwoo Kim 
Committed: Mon Aug 22 15:59:35 2016 +0900

--
 bigtop-packages/src/deb/tez/rules  | 7 +++
 bigtop-packages/src/rpm/tez/SPECS/tez.spec | 6 ++
 2 files changed, 13 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/19c442c6/bigtop-packages/src/deb/tez/rules
--
diff --git a/bigtop-packages/src/deb/tez/rules 
b/bigtop-packages/src/deb/tez/rules
index 467de0a..a330120 100644
--- a/bigtop-packages/src/deb/tez/rules
+++ b/bigtop-packages/src/deb/tez/rules
@@ -25,6 +25,8 @@ export DH_VERBOSE=1
 export DH_OPTIONS
 
 tez_pkg_name=tez
+tez_home=/usr/lib/tez
+lib_tez=/usr/lib/tez/lib
 
 %:
dh $@
@@ -40,3 +42,8 @@ override_dh_auto_install:
--build-dir=. \
--doc-dir=usr/share/doc/${tez_pkg_name}-doc \
--prefix=debian/tmp
+   ln -sf /usr/lib/hadoop/hadoop-annotations.jar 
debian/tmp/${lib_tez}/hadoop-annotations.jar
+   ln -sf /usr/lib/hadoop/hadoop-auth.jar 
debian/tmp/${lib_tez}/hadoop-auth.jar
+   ln -sf /usr/lib/hadoop-mapreduce/hadoop-mapreduce-client-common.jar 
debian/tmp/${lib_tez}/hadoop-mapreduce-client-common.jar
+   ln -sf /usr/lib/hadoop-mapreduce/hadoop-mapreduce-client-core.jar 
debian/tmp/${lib_tez}/hhadoop-mapreduce-client-core.jar
+   ln -sf /usr/lib/hadoop-yarn/hadoop-yarn-server-web-proxy.jar 
debian/tmp/${lib_tez}/hadoop-yarn-server-web-proxy.jar

http://git-wip-us.apache.org/repos/asf/bigtop/blob/19c442c6/bigtop-packages/src/rpm/tez/SPECS/tez.spec
--
diff --git a/bigtop-packages/src/rpm/tez/SPECS/tez.spec 
b/bigtop-packages/src/rpm/tez/SPECS/tez.spec
index bf74337..b7697c3 100644
--- a/bigtop-packages/src/rpm/tez/SPECS/tez.spec
+++ b/bigtop-packages/src/rpm/tez/SPECS/tez.spec
@@ -93,6 +93,12 @@ sh %{SOURCE2} \
 --libexec-dir=%{libexec_tez} \
--prefix=$RPM_BUILD_ROOT
 
+%__ln_s -f /usr/lib/hadoop/hadoop-annotations.jar 
$RPM_BUILD_ROOT/%{lib_tez}/hadoop-annotations.jar
+%__ln_s -f /usr/lib/hadoop/hadoop-auth.jar 
$RPM_BUILD_ROOT/%{lib_tez}/hadoop-auth.jar
+%__ln_s -f /usr/lib/hadoop-mapreduce/hadoop-mapreduce-client-common.jar 
$RPM_BUILD_ROOT/%{lib_tez}/hadoop-mapreduce-client-common.jar
+%__ln_s -f /usr/lib/hadoop-mapreduce/hadoop-mapreduce-client-core.jar 
$RPM_BUILD_ROOT/%{lib_tez}/hhadoop-mapreduce-client-core.jar
+%__ln_s -f /usr/lib/hadoop-yarn/hadoop-yarn-server-web-proxy.jar 
$RPM_BUILD_ROOT/%{lib_tez}/hadoop-yarn-server-web-proxy.jar
+
 %pre
 
 %post

bigtop git commit: BIGTOP-2567. Upgrade Flume version to 1.7.0

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 5f65c5e6d -> 34f94010b


BIGTOP-2567. Upgrade Flume version to 1.7.0


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/34f94010
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/34f94010
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/34f94010

Branch: refs/heads/master
Commit: 34f94010bafdac50b4de3190945ad6511fa19733
Parents: 5f65c5e
Author: YoungWoo Kim 
Authored: Mon Nov 7 00:53:06 2016 +
Committer: Youngwoo Kim 
Committed: Wed Nov 9 16:44:41 2016 +0900

--
 bigtop-packages/src/common/flume/do-component-build | 1 +
 bigtop.bom  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/34f94010/bigtop-packages/src/common/flume/do-component-build
--
diff --git a/bigtop-packages/src/common/flume/do-component-build 
b/bigtop-packages/src/common/flume/do-component-build
index 10a97e3..bbf7656 100755
--- a/bigtop-packages/src/common/flume/do-component-build
+++ b/bigtop-packages/src/common/flume/do-component-build
@@ -32,4 +32,5 @@ mvn -DskipTests  \
   -Dhbase.version=${HBASE_VERSION} \
   -Dkite.version=${KITE_VERSION} \
   -Dhive.version=${HIVE_VERSION} \
+  -Dkafka.version=${KAFKA_VERSION} \
   clean install "$@"

http://git-wip-us.apache.org/repos/asf/bigtop/blob/34f94010/bigtop.bom
--
diff --git a/bigtop.bom b/bigtop.bom
index a905370..e2c6673 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -228,7 +228,7 @@ bigtop {
 'flume' {
   name= 'flume'
   relNotes = 'Apache Flume'
-  version { base = '1.6.0'; pkg = base; release = 1 }
+  version { base = '1.7.0'; pkg = base; release = 1 }
   tarball { destination = "apache-$name-${version.base}-src.tar.gz"
 source  = destination }
   url { download_path = "/$name/${version.base}/"

bigtop git commit: BIGTOP-2497: Upgrade Apache Phoenix to 4.8.1

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 00611578d -> 702b02951


BIGTOP-2497: Upgrade Apache Phoenix to 4.8.1

Signed-off-by: Youngwoo Kim 


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/702b0295
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/702b0295
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/702b0295

Branch: refs/heads/master
Commit: 702b02951f962a3c233fe3d28fa14c7ec478ad11
Parents: 0061157
Author: Kengo Seki 
Authored: Tue Oct 25 15:08:36 2016 +
Committer: Youngwoo Kim 
Committed: Tue Nov 1 11:54:08 2016 +0900

--
 bigtop-packages/src/common/phoenix/do-component-build | 3 +--
 bigtop-packages/src/rpm/phoenix/SPECS/phoenix.spec| 2 +-
 bigtop.bom| 6 +++---
 3 files changed, 5 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/702b0295/bigtop-packages/src/common/phoenix/do-component-build
--
diff --git a/bigtop-packages/src/common/phoenix/do-component-build 
b/bigtop-packages/src/common/phoenix/do-component-build
index 8471639..0ca 100644
--- a/bigtop-packages/src/common/phoenix/do-component-build
+++ b/bigtop-packages/src/common/phoenix/do-component-build
@@ -32,8 +32,7 @@ mkdir build
 # Make binary tar
 rm -rf $(find . -type d -name archive-tmp);
 
-phoenix_jars=$(find -iname phoenix-*.jar)
-cp $phoenix_jars build/;
+find -iname phoenix-*.jar -exec cp -n {} build/ \;
 
 cp -R bin build/;
 

http://git-wip-us.apache.org/repos/asf/bigtop/blob/702b0295/bigtop-packages/src/rpm/phoenix/SPECS/phoenix.spec
--
diff --git a/bigtop-packages/src/rpm/phoenix/SPECS/phoenix.spec 
b/bigtop-packages/src/rpm/phoenix/SPECS/phoenix.spec
index 04a0d80..3f33574 100644
--- a/bigtop-packages/src/rpm/phoenix/SPECS/phoenix.spec
+++ b/bigtop-packages/src/rpm/phoenix/SPECS/phoenix.spec
@@ -116,7 +116,7 @@ with Phoenix and HBase. Soon this will enable access from 
environments
 other than the JVM.
 
 %prep
-%setup -n %{name}-%{phoenix_base_version}-src
+%setup -n apache-%{name}-%{phoenix_base_version}-src
 
 %build
 bash %{SOURCE1}

http://git-wip-us.apache.org/repos/asf/bigtop/blob/702b0295/bigtop.bom
--
diff --git a/bigtop.bom b/bigtop.bom
index 272ba5e..ff0d7fd 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -317,10 +317,10 @@ bigtop {
* HBase version in the BOM.
*/
   phoenix.hbase ='HBase-1.1'
-  version { base = "4.7.0-${phoenix.hbase}"; pkg = '4.7.0'; release = 1 }
+  version { base = "4.8.1-${phoenix.hbase}"; pkg = '4.8.1'; release = 1 }
   tarball { destination = "$name-${version.base}-src.tar.gz"
-source  = destination }
-  url { download_path = "/$name/$name-${version.base}/src"
+source  = "apache-$name-${version.base}-src.tar.gz" }
+  url { download_path = "/$name/apache-$name-${version.base}/src"
 site = "${apache.APACHE_MIRROR}/${download_path}"
 archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
 }

bigtop git commit: BIGTOP-2605: Addendum for BIGTOP-2514

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 3466b5f6e -> 5bee2caee


BIGTOP-2605: Addendum for BIGTOP-2514


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/5bee2cae
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/5bee2cae
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/5bee2cae

Branch: refs/heads/master
Commit: 5bee2caeea2a13ff58fb08a04dceb358864825bc
Parents: 3466b5f
Author: Youngwoo Kim 
Authored: Thu Nov 24 11:01:39 2016 +0900
Committer: Youngwoo Kim 
Committed: Wed Dec 14 16:47:06 2016 +0900

--
 .../src/common/zeppelin/install_zeppelin.sh |  2 +
 .../src/common/zeppelin/zeppelin-env.sh | 72 +++-
 2 files changed, 71 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/5bee2cae/bigtop-packages/src/common/zeppelin/install_zeppelin.sh
--
diff --git a/bigtop-packages/src/common/zeppelin/install_zeppelin.sh 
b/bigtop-packages/src/common/zeppelin/install_zeppelin.sh
index 29e3e30..0e305c6 100644
--- a/bigtop-packages/src/common/zeppelin/install_zeppelin.sh
+++ b/bigtop-packages/src/common/zeppelin/install_zeppelin.sh
@@ -122,10 +122,12 @@ tar --wildcards --strip-components=1 -C $PREFIX/$LIB_DIR 
-zxf ${BUILD_DIR}/zeppe
 tar --wildcards --strip-components=1 -C $PREFIX/$LIB_DIR -zxf 
${BUILD_DIR}/zeppelin-distribution/target/zeppelin-*.tar.gz \*zeppelin-web\*.war
 tar --wildcards --strip-components=1 -C $PREFIX/var/lib/zeppelin -zxf 
${BUILD_DIR}/zeppelin-distribution/target/zeppelin-*.tar.gz \*/notebook/\*
 
+rm -f $PREFIX/$LIB_DIR/bin/*.cmd
 chmod 755 $PREFIX/$LIB_DIR/bin/*
 
 cp -a ${BUILD_DIR}/{LICENSE,README.md} $PREFIX/$LIB_DIR
 
 cp -a ${BUILD_DIR}/conf/* $PREFIX/$CONF_DIR
+rm -f $PREFIX/$CONF_DIR/*.cmd.*
 cp -a ${SOURCE_DIR}/zeppelin-env.sh $PREFIX/$CONF_DIR
 ln -s /etc/zeppelin/conf $PREFIX/$LIB_DIR/conf

http://git-wip-us.apache.org/repos/asf/bigtop/blob/5bee2cae/bigtop-packages/src/common/zeppelin/zeppelin-env.sh
--
diff --git a/bigtop-packages/src/common/zeppelin/zeppelin-env.sh 
b/bigtop-packages/src/common/zeppelin/zeppelin-env.sh
index 038d13e..abc5f33 100644
--- a/bigtop-packages/src/common/zeppelin/zeppelin-env.sh
+++ b/bigtop-packages/src/common/zeppelin/zeppelin-env.sh
@@ -13,13 +13,79 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-export 
ZEPPELIN_INTERPRETERS="org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter"
-export ZEPPELIN_PORT=8080
+# export JAVA_HOME=
+# export MASTER=   # Spark master url. eg. 
spark://master_addr:7077. Leave empty if you want to use local mode.
+# export ZEPPELIN_JAVA_OPTS# Additional jvm options. for 
example, export ZEPPELIN_JAVA_OPTS="-Dspark.executor.memory=8g 
-Dspark.cores.max=16"
+# export ZEPPELIN_MEM  # Zeppelin jvm mem options 
Default -Xms1024m -Xmx1024m -XX:MaxPermSize=512m
+# export ZEPPELIN_INTP_MEM # zeppelin interpreter process 
jvm mem options. Default -Xms1024m -Xmx1024m -XX:MaxPermSize=512m
+# export ZEPPELIN_INTP_JAVA_OPTS   # zeppelin interpreter process 
jvm options.
+
+# export ZEPPELIN_LOG_DIR  # Where log files are stored.  
PWD by default.
+# export ZEPPELIN_PID_DIR  # The pid files are stored. 
${ZEPPELIN_HOME}/run by default.
+# export ZEPPELIN_WAR_TEMPDIR  # The location of jetty 
temporary directory.
+# export ZEPPELIN_NOTEBOOK_DIR # Where notebook saved
+# export ZEPPELIN_NOTEBOOK_HOMESCREEN  # Id of notebook to be 
displayed in homescreen. ex) 2A94M5J1Z
+# export ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE # hide homescreen notebook from 
list when this value set to "true". default "false"
+# export ZEPPELIN_NOTEBOOK_S3_BUCKET# Bucket where notebook saved
+# export ZEPPELIN_NOTEBOOK_S3_ENDPOINT  # Endpoint of the bucket
+# export ZEPPELIN_NOTEBOOK_S3_USER  # User in bucket where notebook 
saved. For example bucket/user/notebook/2A94M5J1Z/note.json
+# export ZEPPELIN_IDENT_STRING # A string representing this 
instance of zeppelin. $USER by default.
+# export ZEPPELIN_NICENESS # The scheduling priority for 
daemons. Defaults to 0.
+# export ZEPPELIN_INTERPRETER_LOCALREPO # Local 

bigtop git commit: BIGTOP-2689: Upgrade Zeppelin to version 0.7

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 286b31ca3 -> 8e9b02303


BIGTOP-2689: Upgrade Zeppelin to version 0.7


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/8e9b0230
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/8e9b0230
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/8e9b0230

Branch: refs/heads/master
Commit: 8e9b0230347785df080d4e8ceabd1074d3ff88b2
Parents: 286b31c
Author: Youngwoo Kim 
Authored: Tue Feb 7 12:10:28 2017 +0900
Committer: Youngwoo Kim 
Committed: Fri Mar 10 10:10:10 2017 +0900

--
 bigtop-packages/src/common/zeppelin/do-component-build |  7 ++-
 .../src/common/zeppelin/install_zeppelin.sh| 13 -
 bigtop-packages/src/rpm/zeppelin/SPECS/zeppelin.spec   |  3 ---
 bigtop.bom |  2 +-
 4 files changed, 11 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/8e9b0230/bigtop-packages/src/common/zeppelin/do-component-build
--
diff --git a/bigtop-packages/src/common/zeppelin/do-component-build 
b/bigtop-packages/src/common/zeppelin/do-component-build
index 755627e..2939b2a 100644
--- a/bigtop-packages/src/common/zeppelin/do-component-build
+++ b/bigtop-packages/src/common/zeppelin/do-component-build
@@ -40,8 +40,10 @@ t="spark.version"
 find . -name pom.xml -exec sed -i "s#<$t>.*#<$t>${SPARK_VERSION}#" 
{} \;
 t="hadoop.version"
 find . -name pom.xml -exec sed -i "s#<$t>.*#<$t>${HADOOP_VERSION}#" 
{} \;
-t="hbase.version"
+t="hbase.hbase.version"
 find . -name pom.xml -exec sed -i "s#<$t>.*#<$t>${HBASE_VERSION}#" 
{} \;
+t="hbase.hadoop.version"
+find . -name pom.xml -exec sed -i "s#<$t>.*#<$t>${HADOOP_VERSION}#" 
{} \;
 t="ignite.version"
 find . -name pom.xml -exec sed -i 
"s#<$t>.*#<$t>${IGNITE_HADOOP_VERSION}#" {} \;
 
@@ -60,3 +62,6 @@ fi
 export MAVEN_OPTS="-Xmx1500m -Xms1500m -XX:MaxPermSize=256m"
 
 mvn $BUILD_OPTS clean package
+
+mkdir -p build/dist
+tar -C build/dist --strip-components=1 -xzf 
zeppelin-distribution/target/zeppelin-*.tar.gz

http://git-wip-us.apache.org/repos/asf/bigtop/blob/8e9b0230/bigtop-packages/src/common/zeppelin/install_zeppelin.sh
--
diff --git a/bigtop-packages/src/common/zeppelin/install_zeppelin.sh 
b/bigtop-packages/src/common/zeppelin/install_zeppelin.sh
index 0e305c6..b7e31df 100644
--- a/bigtop-packages/src/common/zeppelin/install_zeppelin.sh
+++ b/bigtop-packages/src/common/zeppelin/install_zeppelin.sh
@@ -115,19 +115,14 @@ install -d -m 0755 $PREFIX/var/log/zeppelin/
 install -d -m 0755 $PREFIX/var/run/zeppelin/
 install -d -m 0755 $PREFIX/var/run/zeppelin/webapps
 
-tar --wildcards --strip-components=1 -C $PREFIX/$LIB_DIR -zxf 
${BUILD_DIR}/zeppelin-distribution/target/zeppelin-*.tar.gz \*/bin/\*
-tar --wildcards --strip-components=1 -C $PREFIX/$LIB_DIR -zxf 
${BUILD_DIR}/zeppelin-distribution/target/zeppelin-*.tar.gz \*/interpreter/\*
-tar --wildcards --strip-components=1 -C $PREFIX/$LIB_DIR -zxf 
${BUILD_DIR}/zeppelin-distribution/target/zeppelin-*.tar.gz \*/lib/\*
-tar --wildcards --strip-components=1 -C $PREFIX/$LIB_DIR -zxf 
${BUILD_DIR}/zeppelin-distribution/target/zeppelin-*.tar.gz 
\*zeppelin-server\*.jar
-tar --wildcards --strip-components=1 -C $PREFIX/$LIB_DIR -zxf 
${BUILD_DIR}/zeppelin-distribution/target/zeppelin-*.tar.gz \*zeppelin-web\*.war
-tar --wildcards --strip-components=1 -C $PREFIX/var/lib/zeppelin -zxf 
${BUILD_DIR}/zeppelin-distribution/target/zeppelin-*.tar.gz \*/notebook/\*
+cp -a 
${BUILD_DIR}/build/dist/{bin,interpreter,lib,zeppelin-web-${ZEPPELIN_VERSION}.war}
 $PREFIX/$LIB_DIR/
+cp -a ${BUILD_DIR}/build/dist/{LICENSE,NOTICE,README.md,licenses} 
$PREFIX/$DOC_DIR
+cp -a ${BUILD_DIR}/build/dist/conf/* $PREFIX/$CONF_DIR
+cp -a ${BUILD_DIR}/build/dist/notebook $PREFIX/var/lib/zeppelin/
 
 rm -f $PREFIX/$LIB_DIR/bin/*.cmd
 chmod 755 $PREFIX/$LIB_DIR/bin/*
 
-cp -a ${BUILD_DIR}/{LICENSE,README.md} $PREFIX/$LIB_DIR
-
-cp -a ${BUILD_DIR}/conf/* $PREFIX/$CONF_DIR
 rm -f $PREFIX/$CONF_DIR/*.cmd.*
 cp -a ${SOURCE_DIR}/zeppelin-env.sh $PREFIX/$CONF_DIR
 ln -s /etc/zeppelin/conf $PREFIX/$LIB_DIR/conf

http://git-wip-us.apache.org/repos/asf/bigtop/blob/8e9b0230/bigtop-packages/src/rpm/zeppelin/SPECS/zeppelin.spec
--
diff --git a/bigtop-packages/src/rpm/zeppelin/SPECS/zeppelin.spec 
b/bigtop-packages/src/rpm/zeppelin/SPECS/zeppelin.spec
index 73b7b3f..cf59e98 100644
--- a/bigtop-packages/src/rpm/zeppelin/SPECS/zeppelin.spec
+++ b/bigtop-packages/src/rpm/zeppelin/SPECS/zeppelin.spec
@@ -120,9 +120,6 @@ chkconfig --del %{name}
 %defattr(-,root,root,755)
 

bigtop git commit: BIGTOP-2793. BIGTOP-2790 broke Ambari build on rpm-based system

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master a41c485df -> d1c9c9d86


BIGTOP-2793. BIGTOP-2790 broke Ambari build on rpm-based system

Signed-off-by: Youngwoo Kim 


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/d1c9c9d8
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/d1c9c9d8
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/d1c9c9d8

Branch: refs/heads/master
Commit: d1c9c9d8663e620a0c043fb98b8c275ac5021573
Parents: a41c485
Author: Kengo Seki 
Authored: Wed May 31 10:33:07 2017 +0200
Committer: Youngwoo Kim 
Committed: Thu Jun 1 09:13:47 2017 +0900

--
 bigtop-packages/src/rpm/ambari/SPECS/ambari.spec | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/d1c9c9d8/bigtop-packages/src/rpm/ambari/SPECS/ambari.spec
--
diff --git a/bigtop-packages/src/rpm/ambari/SPECS/ambari.spec 
b/bigtop-packages/src/rpm/ambari/SPECS/ambari.spec
index e32f84b..2433497 100644
--- a/bigtop-packages/src/rpm/ambari/SPECS/ambari.spec
+++ b/bigtop-packages/src/rpm/ambari/SPECS/ambari.spec
@@ -37,7 +37,7 @@ Group: Development
 BuildArch: noarch
 Buildroot: %(mktemp -ud 
%{_tmppath}/apache-%{ambari_name}-%{version}-%{release}-XX)
 License: ASL 2.0 
-Source0: apache-%{ambari_name}-%{ambari_version}-src.tar.gz
+Source0: apache-%{ambari_name}-%{ambari_base_version}-src.tar.gz
 Source1: do-component-build 
 Source2: install_%{ambari_name}.sh
 Source3: bigtop.bom

bigtop git commit: BIGTOP-2807: Upgrade Spark to 2.1.1

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master c6b60a81e -> 5b8bba96a


BIGTOP-2807: Upgrade Spark to 2.1.1


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/5b8bba96
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/5b8bba96
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/5b8bba96

Branch: refs/heads/master
Commit: 5b8bba96a20d026194a042f46489041d5525d17d
Parents: c6b60a8
Author: Youngwoo Kim 
Authored: Mon Jun 12 13:21:29 2017 +0900
Committer: Youngwoo Kim 
Committed: Sat Jun 17 16:02:06 2017 +0900

--
 .../src/common/spark/do-component-build |  2 +-
 bigtop-packages/src/deb/spark/control   |  2 +-
 bigtop-packages/src/deb/spark/rules | 21 
 bigtop-packages/src/rpm/spark/SPECS/spark.spec  | 21 +++-
 bigtop.bom  |  2 +-
 5 files changed, 44 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/5b8bba96/bigtop-packages/src/common/spark/do-component-build
--
diff --git a/bigtop-packages/src/common/spark/do-component-build 
b/bigtop-packages/src/common/spark/do-component-build
index 21dffbe..3b79b0d 100644
--- a/bigtop-packages/src/common/spark/do-component-build
+++ b/bigtop-packages/src/common/spark/do-component-build
@@ -28,7 +28,7 @@ BUILD_OPTS="-Divy.home=${HOME}/.ivy2 
-Dsbt.ivy.home=${HOME}/.ivy2 -Duser.home=${
 -Dyarn.version=$HADOOP_VERSION \
 -Dprotobuf.version=2.5.0 \
 -DrecompileMode=all \
--Pyarn -Phadoop-2.6 \
+-Pyarn -Phadoop-2.7 \
 -Phive -Phive-thriftserver \
 $SPARK_BUILD_OPTS"
 

http://git-wip-us.apache.org/repos/asf/bigtop/blob/5b8bba96/bigtop-packages/src/deb/spark/control
--
diff --git a/bigtop-packages/src/deb/spark/control 
b/bigtop-packages/src/deb/spark/control
index 4b7f2ff..01a393d 100644
--- a/bigtop-packages/src/deb/spark/control
+++ b/bigtop-packages/src/deb/spark/control
@@ -23,7 +23,7 @@ Homepage: http://spark.apache.org/
 
 Package: spark-core
 Architecture: all
-Depends: adduser, bigtop-utils (>= 0.7), hadoop-client
+Depends: adduser, bigtop-utils (>= 0.7), hadoop-client, hadoop-yarn
 Description: Lightning-Fast Cluster Computing
  Spark is a MapReduce-like cluster computing framework designed to support
  low-latency iterative jobs and interactive use from an interpreter. It is

http://git-wip-us.apache.org/repos/asf/bigtop/blob/5b8bba96/bigtop-packages/src/deb/spark/rules
--
diff --git a/bigtop-packages/src/deb/spark/rules 
b/bigtop-packages/src/deb/spark/rules
index 0bc7fc7..d5fdb6d 100644
--- a/bigtop-packages/src/deb/spark/rules
+++ b/bigtop-packages/src/deb/spark/rules
@@ -22,6 +22,10 @@ export DH_VERBOSE=1
 # This has to be exported to make some magic below work.
 export DH_OPTIONS
 
+lib_spark=/usr/lib/spark
+lib_hadoop_client=/usr/lib/hadoop/client
+lib_hadoop_yarn=/usr/lib/hadoop-yarn/
+
 %:
dh $@
 
@@ -45,3 +49,20 @@ override_dh_install:
dh_install
# BIGTOP-2588 drop datanucleus jars from spark-core
rm -Rf debian/spark-core/usr/lib/spark/jars/datanucleus*
+
+   rm -f debian/tmp/${lib_spark}/jars/hadoop-*.jar
+   ln -s  ${lib_hadoop_client}/hadoop-annotations.jar 
debian/tmp/${lib_spark}/jars/
+   ln -s  ${lib_hadoop_client}/hadoop-auth.jar 
debian/tmp/${lib_spark}/jars/
+   ln -s  ${lib_hadoop_client}/hadoop-client.jar 
debian/tmp/${lib_spark}/jars/
+   ln -s  ${lib_hadoop_client}/hadoop-common.jar 
debian/tmp/${lib_spark}/jars/
+   ln -s  ${lib_hadoop_client}/hadoop-hdfs.jar 
debian/tmp/${lib_spark}/jars/
+   ln -s  ${lib_hadoop_client}/hadoop-mapreduce-client-app.jar 
debian/tmp/${lib_spark}/jars/
+   ln -s  ${lib_hadoop_client}/hadoop-mapreduce-client-common.jar 
debian/tmp/${lib_spark}/jars/
+   ln -s  ${lib_hadoop_client}/hadoop-mapreduce-client-core.jar 
debian/tmp/${lib_spark}/jars/
+   ln -s  ${lib_hadoop_client}/hadoop-mapreduce-client-jobclient.jar 
debian/tmp/${lib_spark}/jars/
+   ln -s  ${lib_hadoop_client}/hadoop-mapreduce-client-shuffle.jar 
debian/tmp/${lib_spark}/jars/
+   ln -s  ${lib_hadoop_yarn}/hadoop-yarn-api.jar 
debian/tmp/${lib_spark}/jars/
+   ln -s  ${lib_hadoop_yarn}/hadoop-yarn-client.jar 
debian/tmp/${lib_spark}/jars/
+   ln -s  ${lib_hadoop_yarn}/hadoop-yarn-common.jar 
debian/tmp/${lib_spark}/jars/
+   ln -s  ${lib_hadoop_yarn}/hadoop-yarn-server-common.jar 
debian/tmp/${lib_spark}/jars/
+   ln -s  ${lib_hadoop_yarn}/hadoop-yarn-server-web-proxy.jar 

bigtop git commit: BIGTOP-2936: Add RedHat default Java install location to detection script

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 8d0cdf5cd -> 7567b884c


BIGTOP-2936: Add RedHat default Java install location to detection script

Signed-off-by: Youngwoo Kim 


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/7567b884
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/7567b884
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/7567b884

Branch: refs/heads/master
Commit: 7567b884cd114f9be3416b26088acd3a6a495f78
Parents: 8d0cdf5
Author: Lars Francke 
Authored: Tue Nov 21 11:23:16 2017 +0100
Committer: Youngwoo Kim 
Committed: Wed Nov 22 16:06:45 2017 +0900

--
 bigtop-packages/src/common/bigtop-utils/bigtop-detect-javahome | 3 +++
 1 file changed, 3 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/7567b884/bigtop-packages/src/common/bigtop-utils/bigtop-detect-javahome
--
diff --git a/bigtop-packages/src/common/bigtop-utils/bigtop-detect-javahome 
b/bigtop-packages/src/common/bigtop-utils/bigtop-detect-javahome
index f3e57c2..c56f7ba 100644
--- a/bigtop-packages/src/common/bigtop-utils/bigtop-detect-javahome
+++ b/bigtop-packages/src/common/bigtop-utils/bigtop-detect-javahome
@@ -27,6 +27,8 @@ JAVA8_HOME_CANDIDATES=(
 '/usr/lib/jvm/j2sdk1.8-oracle/jre'
 '/usr/lib/jvm/java-8-oracle'
 '/usr/lib/jdk8-latest'
+'/usr/lib/jvm/java-1.8.0'
+'/usr/lib/jvm/java-1.8.0-oracle'
 )
 
 OPENJAVA8_HOME__CANDIDATES=(
@@ -44,6 +46,7 @@ MISCJAVA_HOME_CANDIDATES=(
 '/usr/lib/jvm/default-java'
 '/usr/lib/jvm/java-openjdk'
 '/usr/lib/jvm/jre-openjdk'
+'/usr/lib/jvm/java-oracle'
 )
 
 case ${BIGTOP_JAVA_MAJOR} in

bigtop git commit: BIGTOP-2893 Bump Hadoop to 2.8.1

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 2e1795361 -> fb26cdadd


BIGTOP-2893 Bump Hadoop to 2.8.1

Patches:
===
HADOOP-12366 was ported to 2.8.1
HADOOP-11628 was removed because it was merged to 2.8

Build time issues fixed:

Fuse_dfs target dir has been changed in 2.8
---
cp: cannot stat 
'hadoop-hdfs-project/hadoop-hdfs/target/native/main/native/fuse-dfs/fuse_dfs': 
No such file or directory

add package goal to mvn
--
Without that goal tar is incomplete:

for dir in '${HADOOP_DIR}/{lib,}' '${HDFS_DIR}/{lib,}' '${YARN_DIR}/{lib,}' 
'${MAPREDUCE_DIR}/{lib,}'
'[' -e 
/code/build/hadoop/rpm/BUILDROOT/hadoop-2.8.1-243.el7.centos.x86_64/usr/lib/hadoop-mapreduce//hadoop-common-2.8.1.jar
 ']'
exit 1
error: Bad exit status from /var/tmp/rpm-tmp.5Etrah (%install)
Bad exit status from /var/tmp/rpm-tmp.5Etrah (%install)

Signed-off-by: Youngwoo Kim 


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/fb26cdad
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/fb26cdad
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/fb26cdad

Branch: refs/heads/master
Commit: fb26cdadd11d6b0164ac25bca7a5fb8f4d1f97cf
Parents: 2e17953
Author: Anton Chevychalov 
Authored: Wed Sep 27 14:48:38 2017 +0300
Committer: Youngwoo Kim 
Committed: Tue Dec 19 15:23:34 2017 +0900

--
 .../src/common/hadoop/do-component-build|  4 +--
 ...0-HADOOP-12366.-expose-calculated-paths.diff | 38 ++--
 .../src/common/hadoop/patch1-HADOOP-11628.diff  | 22 
 bigtop.bom  |  2 +-
 4 files changed, 22 insertions(+), 44 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/fb26cdad/bigtop-packages/src/common/hadoop/do-component-build
--
diff --git a/bigtop-packages/src/common/hadoop/do-component-build 
b/bigtop-packages/src/common/hadoop/do-component-build
index 32429c2..dcd802f 100644
--- a/bigtop-packages/src/common/hadoop/do-component-build
+++ b/bigtop-packages/src/common/hadoop/do-component-build
@@ -56,7 +56,7 @@ mkdir build/src
 # Build artifacts
 MAVEN_OPTS="-Dzookeeper.version=$ZOOKEEPER_VERSION"
 MAVEN_OPTS="$MAVEN_OPTS -DskipTests -DskipTest -DskipITs"
-mvn $ANT_OPTS $BUNDLE_SNAPPY -Pdist -Pnative -Psrc -Dtar ${MAVEN_OPTS} 
-Dtomcat.version=${BIGTOP_TOMCAT_VERSION} 
-Dtomcat.download.url="http://archive.apache.org/dist/tomcat/tomcat-6/v${BIGTOP_TOMCAT_VERSION}/bin/apache-tomcat-${BIGTOP_TOMCAT_VERSION}.tar.gz;
 install "$@"
+mvn $ANT_OPTS $BUNDLE_SNAPPY -Pdist -Pnative -Psrc -Dtar ${MAVEN_OPTS} 
-Dtomcat.version=${BIGTOP_TOMCAT_VERSION} 
-Dtomcat.download.url="http://archive.apache.org/dist/tomcat/tomcat-6/v${BIGTOP_TOMCAT_VERSION}/bin/apache-tomcat-${BIGTOP_TOMCAT_VERSION}.tar.gz;
 install package "$@"
 mvn site site:stage ${MAVEN_OPTS} $@
 
 (cd build ; tar --strip-components=1 -xzvf  
../hadoop-dist/target/hadoop-${HADOOP_VERSION}.tar.gz)
@@ -70,4 +70,4 @@ cp -r target/staging/hadoop-project build/share/doc
 (cd hadoop-client/target/hadoop-client-*/share/hadoop/client/lib ; ls) > 
build/hadoop-client.list
 
 # Copy fuse output to the build directory
-cp hadoop-hdfs-project/hadoop-hdfs/target/native/main/native/fuse-dfs/fuse_dfs 
build/bin/
+cp 
hadoop-hdfs-project/hadoop-hdfs-native-client/target/main/native/fuse-dfs/fuse_dfs
 build/bin

http://git-wip-us.apache.org/repos/asf/bigtop/blob/fb26cdad/bigtop-packages/src/common/hadoop/patch0-HADOOP-12366.-expose-calculated-paths.diff
--
diff --git 
a/bigtop-packages/src/common/hadoop/patch0-HADOOP-12366.-expose-calculated-paths.diff
 
b/bigtop-packages/src/common/hadoop/patch0-HADOOP-12366.-expose-calculated-paths.diff
index b0c831c..026c01f 100644
--- 
a/bigtop-packages/src/common/hadoop/patch0-HADOOP-12366.-expose-calculated-paths.diff
+++ 
b/bigtop-packages/src/common/hadoop/patch0-HADOOP-12366.-expose-calculated-paths.diff
@@ -1,9 +1,9 @@
-From 3131744ed6e1e874d7cd0666208adc424fec3224 Mon Sep 17 00:00:00 2001
-From: Konstantin Boudnik 
-Date: Mon, 15 Feb 2016 10:42:16 +0300
-Subject: [PATCH] HADOOP-12366. expose calculated paths
+From f10ec9f9bec56bcfa5e2015ea87a8a5054d096c5 Mon Sep 17 00:00:00 2001
+From: Anton Chevychalov 
+Date: Tue, 5 Sep 2017 15:14:19 +0300
+Subject: [PATCH] [PATCH] HADOOP-12366. expose calculated paths
 
-Adapted the patch to apply cleanly on branch-2.7.1
+Adapted the patch to apply cleanly on branch-2.8.1
 ---
  hadoop-common-project/hadoop-common/src/main/bin/hadoop | 10 ++
  hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs   |  9 

bigtop git commit: BIGTOP-2895 Bump Pig to 0.17.0 with Hadoop-2.8

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master fb26cdadd -> 62e9c3ad9


BIGTOP-2895 Bump Pig to 0.17.0 with Hadoop-2.8

Bump Pig to 0.17.0.
Remove useless patches PIG-4676 and PIG-4728 it is already in 0.17 release.
Add patch PIG-5304 to build pig with Hadoop 2.8

Signed-off-by: Youngwoo Kim 


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/62e9c3ad
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/62e9c3ad
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/62e9c3ad

Branch: refs/heads/master
Commit: 62e9c3ad9c7dffda839c90bbb71bc8e7f0ee6557
Parents: fb26cda
Author: Anton Chevychalov 
Authored: Thu Sep 14 15:01:06 2017 +0300
Committer: Youngwoo Kim 
Committed: Sat Dec 23 16:39:18 2017 +0900

--
 ...IG-5304-Fix-build-issue-with-Hadoop-2.8.diff | 30 +++
 .../src/common/pig/patch1-PIG-4676.diff | 93 
 .../src/common/pig/patch2-PIG-4728.diff | 36 
 bigtop.bom  |  2 +-
 4 files changed, 31 insertions(+), 130 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/62e9c3ad/bigtop-packages/src/common/pig/patch0-PIG-5304-Fix-build-issue-with-Hadoop-2.8.diff
--
diff --git 
a/bigtop-packages/src/common/pig/patch0-PIG-5304-Fix-build-issue-with-Hadoop-2.8.diff
 
b/bigtop-packages/src/common/pig/patch0-PIG-5304-Fix-build-issue-with-Hadoop-2.8.diff
new file mode 100644
index 000..044e112
--- /dev/null
+++ 
b/bigtop-packages/src/common/pig/patch0-PIG-5304-Fix-build-issue-with-Hadoop-2.8.diff
@@ -0,0 +1,30 @@
+From 65b931966108f374210b83368b8356a79058226d Mon Sep 17 00:00:00 2001
+From: Anton Chevychalov 
+Date: Thu, 14 Sep 2017 11:58:21 +0300
+Subject: [PATCH] PIG-5304 Fix build issue with Hadoop 2.8
+
+There is a regression in build after HDFS-8740.
+We have to add hadoop-hdfs-client artifact as dependency now.
+---
+ ivy.xml | 5 +
+ 1 file changed, 5 insertions(+)
+
+diff --git a/ivy.xml b/ivy.xml
+index db722a5..297 100644
+--- a/ivy.xml
 b/ivy.xml
+@@ -129,6 +129,11 @@
+   
+   
+ 
++
++  
++  
++
+ 
+ http://git-wip-us.apache.org/repos/asf/bigtop/blob/62e9c3ad/bigtop-packages/src/common/pig/patch1-PIG-4676.diff
--
diff --git a/bigtop-packages/src/common/pig/patch1-PIG-4676.diff 
b/bigtop-packages/src/common/pig/patch1-PIG-4676.diff
deleted file mode 100644
index 2844116..000
--- a/bigtop-packages/src/common/pig/patch1-PIG-4676.diff
+++ /dev/null
@@ -1,93 +0,0 @@
-diff -ur pig-0.15.0-src_/ivy/libraries.properties 
pig-0.15.0-src/ivy/libraries.properties
 pig-0.15.0-src_/ivy/libraries.properties   2015-06-01 20:32:30.0 
+0200
-+++ pig-0.15.0-src/ivy/libraries.properties2015-09-18 11:32:06.063454643 
+0200
-@@ -47,7 +47,7 @@
- hbase94.version=0.94.1
- hbase95.version=0.98.12-${hbase.hadoop.version}
- hsqldb.version=1.8.0.10
--hive.version=0.14.0
-+hive.version=1.2.1
- httpcomponents.version=4.1
- jackson.version=1.8.8
- jackson-pig-3039-test.version=1.9.9
-@@ -98,3 +98,4 @@
- leveldbjni.version=1.8
- curator.version=2.6.0
- htrace.version=3.1.0-incubating
-+commons-lang3.version=3.1
-diff -ur pig-0.15.0-src_/ivy.xml pig-0.15.0-src/ivy.xml
 pig-0.15.0-src_/ivy.xml2015-06-01 20:32:30.0 +0200
-+++ pig-0.15.0-src/ivy.xml 2015-09-18 11:32:06.067454643 +0200
-@@ -417,8 +417,6 @@
-   conf="compile->master" />
- 
--
- 
- 
- 
-+
- 
- 
-diff -ur pig-0.15.0-src_/src/org/apache/pig/builtin/HiveUDFBase.java 
pig-0.15.0-src/src/org/apache/pig/builtin/HiveUDFBase.java
 pig-0.15.0-src_/src/org/apache/pig/builtin/HiveUDFBase.java
2015-06-01 20:32:27.0 +0200
-+++ pig-0.15.0-src/src/org/apache/pig/builtin/HiveUDFBase.java 2015-09-18 
11:32:06.067454643 +0200
-@@ -28,6 +28,7 @@
- import org.apache.hadoop.hive.ql.exec.MapredContext;
- import org.apache.hadoop.hive.ql.exec.UDAF;
- import org.apache.hadoop.hive.ql.exec.UDF;
-+import org.apache.hadoop.hive.ql.parse.SemanticException;
- import org.apache.hadoop.hive.ql.udf.generic.Collector;
- import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFResolver;
- import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
-@@ -103,7 +104,12 @@
- String className = funcName;
- Class udfClass;
- if (FunctionRegistry.getFunctionNames().contains(funcName)) {
--FunctionInfo func = FunctionRegistry.getFunctionInfo(funcName);
-+FunctionInfo func;
-+try {
-+func = FunctionRegistry.getFunctionInfo(funcName);
-+} catch (SemanticException e) {
-+

bigtop git commit: BIGTOP-2898 Remove h2 pig classifier in Phoenix

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 932bfb44f -> 8cf130ccf


BIGTOP-2898 Remove h2 pig classifier in Phoenix

There is a regression in Pig relase 0.17
After PIG-4923 they drop h2 classifer.
There is only one jar now and it suports Hadoop 2

That commit also adds BIGTOP_PATCH macro to Phoenix

Signed-off-by: Youngwoo Kim 


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/8cf130cc
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/8cf130cc
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/8cf130cc

Branch: refs/heads/master
Commit: 8cf130ccfbe7bcffe8ea83ba9465cdedf89f130f
Parents: 932bfb4
Author: Anton Chevychalov 
Authored: Mon Oct 2 18:55:12 2017 +0300
Committer: Youngwoo Kim 
Committed: Mon Dec 25 15:00:00 2017 +0900

--
 ...emove-h2-pig-classifier-in-Phoenix-deps.diff | 40 
 .../src/rpm/phoenix/SPECS/phoenix.spec  |  2 +
 2 files changed, 42 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/8cf130cc/bigtop-packages/src/common/phoenix/patch0-ADH-207-Remove-h2-pig-classifier-in-Phoenix-deps.diff
--
diff --git 
a/bigtop-packages/src/common/phoenix/patch0-ADH-207-Remove-h2-pig-classifier-in-Phoenix-deps.diff
 
b/bigtop-packages/src/common/phoenix/patch0-ADH-207-Remove-h2-pig-classifier-in-Phoenix-deps.diff
new file mode 100644
index 000..a7c69a2
--- /dev/null
+++ 
b/bigtop-packages/src/common/phoenix/patch0-ADH-207-Remove-h2-pig-classifier-in-Phoenix-deps.diff
@@ -0,0 +1,40 @@
+From 401934c00d7184a05c308bc219a4597ba4e3dce2 Mon Sep 17 00:00:00 2001
+From: Anton Chevychalov 
+Date: Thu, 28 Sep 2017 19:06:07 +0300
+Subject: [PATCH] ADH-207 Remove h2 pig classifier in Phoenix deps
+
+There is a regression in Pig relase 0.17
+After PIG-4923 they drop h2 classifer.
+There is only one jar now and it suports Hadoop 2
+---
+ phoenix-pig/pom.xml | 1 -
+ pom.xml | 1 -
+ 2 files changed, 2 deletions(-)
+
+diff --git a/phoenix-pig/pom.xml b/phoenix-pig/pom.xml
+index aed4a9d..77d2e23 100644
+--- a/phoenix-pig/pom.xml
 b/phoenix-pig/pom.xml
+@@ -54,7 +54,6 @@
+ 
+   org.apache.pig
+   pig
+-  h2
+ 
+ 
+   org.apache.hbase
+diff --git a/pom.xml b/pom.xml
+index 0f93146..35dc64e 100644
+--- a/pom.xml
 b/pom.xml
+@@ -748,7 +748,6 @@
+ org.apache.pig
+ pig
+ ${pig.version}
+-h2
+ 
+   
+ org.xerial.snappy
+-- 
+2.7.4
+

http://git-wip-us.apache.org/repos/asf/bigtop/blob/8cf130cc/bigtop-packages/src/rpm/phoenix/SPECS/phoenix.spec
--
diff --git a/bigtop-packages/src/rpm/phoenix/SPECS/phoenix.spec 
b/bigtop-packages/src/rpm/phoenix/SPECS/phoenix.spec
index 3f33574..a99282e 100644
--- a/bigtop-packages/src/rpm/phoenix/SPECS/phoenix.spec
+++ b/bigtop-packages/src/rpm/phoenix/SPECS/phoenix.spec
@@ -25,6 +25,7 @@
 %define hadoop_yarn_home /usr/lib/hadoop-yarn
 %define hadoop_hdfs_home /usr/lib/hadoop-hdfs
 %define hbase_home /usr/lib/hbase
+#BIGTOP_PATCH_FILES
 
 %if  %{?suse_version:1}0
 
@@ -117,6 +118,7 @@ other than the JVM.
 
 %prep
 %setup -n apache-%{name}-%{phoenix_base_version}-src
+#BIGTOP_PATCH_COMMANDS
 
 %build
 bash %{SOURCE1}

bigtop git commit: BIGTOP-2899 remove h2 pig classifier in Oozie code

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 62e9c3ad9 -> 932bfb44f


BIGTOP-2899 remove h2 pig classifier in Oozie code

There is a regression in Pig relase 0.17
After PIG-4923 they drop h2 classifer.
There is only one jar now and it suports Hadoop 2

Signed-off-by: Youngwoo Kim 


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/932bfb44
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/932bfb44
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/932bfb44

Branch: refs/heads/master
Commit: 932bfb44fe5b5a86d05c83800dd137feb7f97517
Parents: 62e9c3a
Author: Anton Chevychalov 
Authored: Fri Sep 29 19:09:10 2017 +0300
Committer: Youngwoo Kim 
Committed: Mon Dec 25 14:47:24 2017 +0900

--
 ...-remove-h2-pig-classifier-in-Oozie-code.diff | 27 
 1 file changed, 27 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/932bfb44/bigtop-packages/src/common/oozie/patch0-ADH-204-remove-h2-pig-classifier-in-Oozie-code.diff
--
diff --git 
a/bigtop-packages/src/common/oozie/patch0-ADH-204-remove-h2-pig-classifier-in-Oozie-code.diff
 
b/bigtop-packages/src/common/oozie/patch0-ADH-204-remove-h2-pig-classifier-in-Oozie-code.diff
new file mode 100644
index 000..5d9c0ec
--- /dev/null
+++ 
b/bigtop-packages/src/common/oozie/patch0-ADH-204-remove-h2-pig-classifier-in-Oozie-code.diff
@@ -0,0 +1,27 @@
+From a9591e083a6c5a63a38e46792fd65687764bdf64 Mon Sep 17 00:00:00 2001
+From: Anton Chevychalov 
+Date: Fri, 29 Sep 2017 19:06:01 +0300
+Subject: [PATCH] ADH-204 remove h2 pig classifier in Oozie code
+
+There is a regression in Pig relase 0.17
+After PIG-4923 they drop h2 classifer.
+There is only one jar now and it suports Hadoop 2
+---
+ pom.xml | 1 -
+ 1 file changed, 1 deletion(-)
+
+diff --git a/pom.xml b/pom.xml
+index 9daf6b1..5d92d77 100644
+--- a/pom.xml
 b/pom.xml
+@@ -1902,7 +1902,6 @@
+ 
+ 2.4.0
+ 2
+-h2
+ hadoop200
+ 1.9.13
+ 
+-- 
+2.7.4
+

[2/2] bigtop git commit: BIGTOP-2990: Upgrade Phoenix version to 4.13.1-HBase-1.3

[ywkim]

BIGTOP-2990: Upgrade Phoenix version to 4.13.1-HBase-1.3


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/943ea913
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/943ea913
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/943ea913

Branch: refs/heads/master
Commit: 943ea913de5ab7b6cbc4772d12e4347cd914950f
Parents: eafac8e
Author: Youngwoo Kim 
Authored: Sat Feb 3 12:53:09 2018 +0900
Committer: Youngwoo Kim 
Committed: Sat Feb 3 12:53:09 2018 +0900

--
 .../src/common/phoenix/patch1-PHOENIX-4423.diff | 433 +++
 bigtop.bom  |   4 +-
 2 files changed, 435 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/943ea913/bigtop-packages/src/common/phoenix/patch1-PHOENIX-4423.diff
--
diff --git a/bigtop-packages/src/common/phoenix/patch1-PHOENIX-4423.diff 
b/bigtop-packages/src/common/phoenix/patch1-PHOENIX-4423.diff
new file mode 100644
index 000..ca76713
--- /dev/null
+++ b/bigtop-packages/src/common/phoenix/patch1-PHOENIX-4423.diff
@@ -0,0 +1,433 @@
+diff --git a/phoenix-hive/pom.xml b/phoenix-hive/pom.xml
+index f75009e32..cb28c76b4 100644
+--- a/phoenix-hive/pom.xml
 b/phoenix-hive/pom.xml
+@@ -141,6 +141,33 @@
+   ${mockito-all.version}
+   test
+ 
++
++  org.apache.calcite.avatica
++  avatica
++  
++  1.8.0
++  test
++  
++  
++
++  org.hsqldb
++  hsqldb
++
++
++  com.fasterxml.jackson.core
++  jackson-databind
++
++
++  com.fasterxml.jackson.core
++  jackson-annotations
++
++
++  com.fasterxml.jackson.core
++  jackson-core
++
++  
++
+   
+ 
+   
+diff --git 
a/phoenix-hive/src/it/java/org/apache/phoenix/hive/BaseHivePhoenixStoreIT.java 
b/phoenix-hive/src/it/java/org/apache/phoenix/hive/BaseHivePhoenixStoreIT.java
+index c705e2db7..3210409ce 100644
+--- 
a/phoenix-hive/src/it/java/org/apache/phoenix/hive/BaseHivePhoenixStoreIT.java
 
b/phoenix-hive/src/it/java/org/apache/phoenix/hive/BaseHivePhoenixStoreIT.java
+@@ -84,7 +84,7 @@ public class BaseHivePhoenixStoreIT {
+ try {
+ qt = new HiveTestUtil(hiveOutputDir, hiveLogDir, clusterType, 
null);
+ } catch (Exception e) {
+-LOG.error("Unexpected exception in setup", e);
++LOG.error("Unexpected exception in setup: " + e.getMessage(), e);
+ fail("Unexpected exception in setup");
+ }
+ 
+@@ -143,14 +143,6 @@ public class BaseHivePhoenixStoreIT {
+ 
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+-if (qt != null) {
+-try {
+-qt.shutdown();
+-} catch (Exception e) {
+-LOG.error("Unexpected exception in setup", e);
+-fail("Unexpected exception in tearDown");
+-}
+-}
+ try {
+ conn.close();
+ } finally {
+@@ -164,5 +156,14 @@ public class BaseHivePhoenixStoreIT {
+ }
+ }
+ }
++// Shutdowns down the filesystem -- do this after stopping HBase.
++if (qt != null) {
++  try {
++  qt.shutdown();
++  } catch (Exception e) {
++  LOG.error("Unexpected exception in setup", e);
++  fail("Unexpected exception in tearDown");
++  }
++  }
+ }
+ }
+diff --git 
a/phoenix-hive/src/it/java/org/apache/phoenix/hive/HiveTestUtil.java 
b/phoenix-hive/src/it/java/org/apache/phoenix/hive/HiveTestUtil.java
+index b4c4e4624..295e8b491 100644
+--- a/phoenix-hive/src/it/java/org/apache/phoenix/hive/HiveTestUtil.java
 b/phoenix-hive/src/it/java/org/apache/phoenix/hive/HiveTestUtil.java
+@@ -23,6 +23,8 @@ import org.apache.commons.io.FileUtils;
+ import org.apache.commons.io.IOUtils;
+ import org.apache.commons.logging.Log;
+ import org.apache.commons.logging.LogFactory;
++import org.apache.hadoop.conf.Configuration;
++import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+ import org.apache.hadoop.fs.FileStatus;
+ import org.apache.hadoop.fs.FileSystem;
+ import org.apache.hadoop.fs.Path;
+@@ -36,8 +38,10 @@ import org.apache.hadoop.hive.common.io.SortPrintStream;
+ import org.apache.hadoop.hive.conf.HiveConf;
+ import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+ import org.apache.hadoop.hive.metastore.api.Index;
++import org.apache.hadoop.hive.ql.QueryState;
+ import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
+ import org.apache.hadoop.hive.ql.exec.Utilities;
++import org.apache.hadoop.hive.ql.exec.tez.TezSessionState;
+ import 

[1/2] bigtop git commit: BIGTOP-2894 Bump hbase to 1.3.1

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 3f723b7d0 -> 943ea913d


BIGTOP-2894 Bump hbase to 1.3.1

There are a number of issues with licenses check when hbase
building with Hadoop 2.8.x

That patch bump hbase to latest release and add couple patches.

Signed-off-by: Youngwoo Kim 


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/eafac8e4
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/eafac8e4
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/eafac8e4

Branch: refs/heads/master
Commit: eafac8e4aba13d69a5dcf16e7fd51670cb291744
Parents: 3f723b7
Author: Anton Chevychalov 
Authored: Fri Sep 8 19:52:14 2017 +0300
Committer: Youngwoo Kim 
Committed: Sat Feb 3 10:48:25 2018 +0900

--
 ...1-Partial-backport-HBASE-16712-to-1.3.1.diff |  60 
 ...patch2-0002-Backport-HBASE-17893-to-1.3.diff | 278 +++
 bigtop.bom  |   2 +-
 3 files changed, 339 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/eafac8e4/bigtop-packages/src/common/hbase/patch1-0001-Partial-backport-HBASE-16712-to-1.3.1.diff
--
diff --git 
a/bigtop-packages/src/common/hbase/patch1-0001-Partial-backport-HBASE-16712-to-1.3.1.diff
 
b/bigtop-packages/src/common/hbase/patch1-0001-Partial-backport-HBASE-16712-to-1.3.1.diff
new file mode 100644
index 000..ed79b35
--- /dev/null
+++ 
b/bigtop-packages/src/common/hbase/patch1-0001-Partial-backport-HBASE-16712-to-1.3.1.diff
@@ -0,0 +1,60 @@
+From 7e3b1d7f830b6e6abd5c4ee6a775f4063b648b60 Mon Sep 17 00:00:00 2001
+From: Anton Chevychalov 
+Date: Fri, 8 Sep 2017 19:09:23 +0300
+Subject: [PATCH 1/2] Partial backport HBASE-16712 to 1.3.1
+
+That partial backport should fix net:jcip package licence trouble
+when we are building hbase with Hadoop 2.8.x
+
+https://issues.apache.org/jira/browse/HBASE-17893?focusedCommentId=15963617
+https://issues.apache.org/jira/browse/HBASE-16712
+---
+ .../src/main/resources/META-INF/LICENSE.vm |  4 +++-
+ .../src/main/resources/supplemental-models.xml | 18 ++
+ 2 files changed, 21 insertions(+), 1 deletion(-)
+
+diff --git a/hbase-resource-bundle/src/main/resources/META-INF/LICENSE.vm 
b/hbase-resource-bundle/src/main/resources/META-INF/LICENSE.vm
+index f403c89..0a4a23e 100644
+--- a/hbase-resource-bundle/src/main/resources/META-INF/LICENSE.vm
 b/hbase-resource-bundle/src/main/resources/META-INF/LICENSE.vm
+@@ -1561,7 +1561,9 @@ You can redistribute it and/or modify it under either 
the terms of the GPL
+ ## Whitelist of licenses that it's safe to not aggregate as above.
+ ## Note that this doesn't include ALv2 or the aforementioned aggregate
+ ## license mentions.
+-#set($non_aggregate_fine = [ 'Public Domain', 'New BSD license', 'BSD 
license', 'Mozilla Public License Version 2.0' ])
++##
++## See this FAQ link for justifications: 
https://www.apache.org/legal/resolved.html
++#set($non_aggregate_fine = [ 'Public Domain', 'New BSD license', 'BSD 
license', 'Mozilla Public License Version 2.0', 'Creative Commons Attribution 
License, Version 2.5', 'MPL 1.1'])
+ ## include LICENSE sections for anything not under ASL2.0
+ #foreach( ${dep} in ${projects} )
+ #if(${debug-print-included-work-info.equalsIgnoreCase("true")})
+diff --git a/hbase-resource-bundle/src/main/resources/supplemental-models.xml 
b/hbase-resource-bundle/src/main/resources/supplemental-models.xml
+index d6237d0..d5495cb 100644
+--- a/hbase-resource-bundle/src/main/resources/supplemental-models.xml
 b/hbase-resource-bundle/src/main/resources/supplemental-models.xml
+@@ -2047,4 +2047,22 @@ Copyright (c) 2007-2011 The JRuby project
+   
+ 
+   
++  
++
++  net.jcip
++  jcip-annotations
++  1.0
++  
++Brian Goetz and Tim Peierls
++http://www.jcip.net
++  
++  
++
++  Creative Commons Attribution License, Version 2.5
++  http://creativecommons.org/licenses/by/2.5
++  repo
++
++  
++
++  
+ 
+-- 
+2.7.4
+

http://git-wip-us.apache.org/repos/asf/bigtop/blob/eafac8e4/bigtop-packages/src/common/hbase/patch2-0002-Backport-HBASE-17893-to-1.3.diff
--
diff --git 
a/bigtop-packages/src/common/hbase/patch2-0002-Backport-HBASE-17893-to-1.3.diff 
b/bigtop-packages/src/common/hbase/patch2-0002-Backport-HBASE-17893-to-1.3.diff
new file mode 100644
index 000..8bfaec0
--- /dev/null
+++ 
b/bigtop-packages/src/common/hbase/patch2-0002-Backport-HBASE-17893-to-1.3.diff
@@ -0,0 +1,278 @@
+From 1a58e47067d64866917b8834eabc37e4c2887ebb Mon Sep 17 00:00:00 2001
+From: 

bigtop git commit: BIGTOP-2811: Add R to toolchain for Spark and Zeppelin

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 8cf130ccf -> c716b1272


BIGTOP-2811: Add R to toolchain for Spark and Zeppelin


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/c716b127
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/c716b127
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/c716b127

Branch: refs/heads/master
Commit: c716b12729c602ea9814670051f237161b9579b6
Parents: 8cf130c
Author: Youngwoo Kim 
Authored: Wed Dec 13 17:24:24 2017 +0900
Committer: Youngwoo Kim 
Committed: Tue Dec 26 11:02:01 2017 +0900

--
 bigtop_toolchain/manifests/installer.pp |  1 +
 bigtop_toolchain/manifests/packages.pp  |  3 +-
 bigtop_toolchain/manifests/renv.pp  | 50 
 3 files changed, 53 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/c716b127/bigtop_toolchain/manifests/installer.pp
--
diff --git a/bigtop_toolchain/manifests/installer.pp 
b/bigtop_toolchain/manifests/installer.pp
index 5bc99b6..813d203 100644
--- a/bigtop_toolchain/manifests/installer.pp
+++ b/bigtop_toolchain/manifests/installer.pp
@@ -23,6 +23,7 @@ class bigtop_toolchain::installer {
   include bigtop_toolchain::packages
   include bigtop_toolchain::env
   include bigtop_toolchain::user
+  include bigtop_toolchain::renv
 
   stage { 'last':
 require => Stage['main'],

http://git-wip-us.apache.org/repos/asf/bigtop/blob/c716b127/bigtop_toolchain/manifests/packages.pp
--
diff --git a/bigtop_toolchain/manifests/packages.pp 
b/bigtop_toolchain/manifests/packages.pp
index 1514827..715a606 100644
--- a/bigtop_toolchain/manifests/packages.pp
+++ b/bigtop_toolchain/manifests/packages.pp
@@ -62,7 +62,8 @@ class bigtop_toolchain::packages {
 "libevent-devel",
 "apr-devel",
 "bison",
-"libffi-devel"
+"libffi-devel",
+"epel-release"
   ]
 }
 /(?i:(SLES|opensuse))/: { $pkgs = [

http://git-wip-us.apache.org/repos/asf/bigtop/blob/c716b127/bigtop_toolchain/manifests/renv.pp
--
diff --git a/bigtop_toolchain/manifests/renv.pp 
b/bigtop_toolchain/manifests/renv.pp
new file mode 100644
index 000..776340a
--- /dev/null
+++ b/bigtop_toolchain/manifests/renv.pp
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class bigtop_toolchain::renv {
+  case $operatingsystem{
+/(?i:(centos|fedora|Amazon))/: {
+  $pkgs = [
+"R",
+"R-devel",
+"pandoc"
+  ]
+}
+/(?i:(SLES|opensuse))/: { 
+  $pkgs = [
+"R-base",
+"R-base-devel",
+"pandoc"
+  ]
+}
+/(Ubuntu|Debian)/: {
+  $pkgs = [
+"r-base",
+"r-base-dev",
+"pandoc"
+  ]
+}
+  }
+  package { $pkgs:
+ensure => installed,
+before => [Exec['install_r_packages']] 
+  }
+
+  # Install required R packages
+  exec { 'install_r_packages':
+cwd => "/usr/bin",
+command => "/usr/bin/R -e \"install.packages(c('devtools', 'evaluate', 
'rmarkdown', 'knitr', 'roxygen2', 'testthat', 'e1071'), repos = 
'http://cran.us.r-project.org')\"",
+  }
+}

bigtop git commit: BIGTOP-2809: Support R integration for Spark

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 44aa7bb0f -> 9767c2b77


BIGTOP-2809: Support R integration for Spark


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/9767c2b7
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/9767c2b7
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/9767c2b7

Branch: refs/heads/master
Commit: 9767c2b77e07021e20da4eaadecb6ec20a9806e6
Parents: 44aa7bb
Author: Youngwoo Kim 
Authored: Thu Jun 15 10:59:21 2017 +0900
Committer: Youngwoo Kim 
Committed: Tue Dec 26 23:00:14 2017 +0900

--
 bigtop-packages/src/common/spark/do-component-build |  3 ++-
 bigtop-packages/src/common/spark/install_spark.sh   |  2 +-
 bigtop-packages/src/deb/spark/control   |  5 +
 bigtop-packages/src/deb/spark/spark-sparkr.install  |  3 +++
 bigtop-packages/src/rpm/spark/SPECS/spark.spec  | 16 
 5 files changed, 27 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/9767c2b7/bigtop-packages/src/common/spark/do-component-build
--
diff --git a/bigtop-packages/src/common/spark/do-component-build 
b/bigtop-packages/src/common/spark/do-component-build
index 3b79b0d..c2a2322 100644
--- a/bigtop-packages/src/common/spark/do-component-build
+++ b/bigtop-packages/src/common/spark/do-component-build
@@ -30,9 +30,10 @@ BUILD_OPTS="-Divy.home=${HOME}/.ivy2 
-Dsbt.ivy.home=${HOME}/.ivy2 -Duser.home=${
 -DrecompileMode=all \
 -Pyarn -Phadoop-2.7 \
 -Phive -Phive-thriftserver \
+-Psparkr \
 $SPARK_BUILD_OPTS"
 
-./dev/make-distribution.sh $BUILD_OPTS -DskipTests
+./dev/make-distribution.sh --r $BUILD_OPTS -DskipTests
 
 SPARK_SKIP_TESTS=$([ "$SPARK_RUN_TESTS" = "true" ] && echo false || echo true)
 

http://git-wip-us.apache.org/repos/asf/bigtop/blob/9767c2b7/bigtop-packages/src/common/spark/install_spark.sh
--
diff --git a/bigtop-packages/src/common/spark/install_spark.sh 
b/bigtop-packages/src/common/spark/install_spark.sh
index 50c3f2b..2a7e1ae 100644
--- a/bigtop-packages/src/common/spark/install_spark.sh
+++ b/bigtop-packages/src/common/spark/install_spark.sh
@@ -141,7 +141,7 @@ ln -s /etc/spark/conf $PREFIX/$LIB_DIR/conf
 
 # Copy in the wrappers
 install -d -m 0755 $PREFIX/$BIN_DIR
-for wrap in bin/spark-class bin/spark-shell bin/spark-sql bin/spark-submit 
bin/find-spark-home; do
+for wrap in bin/spark-class bin/spark-shell bin/spark-sql bin/spark-submit 
bin/find-spark-home bin/sparkR; do
   cat > $PREFIX/$BIN_DIR/$(basename $wrap) 

bigtop git commit: BIGTOP-2973: Hama packaging is broken with Hadoop 2.8+

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 9e78a0365 -> badd85ebf


BIGTOP-2973: Hama packaging is broken with Hadoop 2.8+


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/badd85eb
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/badd85eb
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/badd85eb

Branch: refs/heads/master
Commit: badd85ebf366a0c2a7b3c9537915cf655d06660c
Parents: 9e78a03
Author: Youngwoo Kim 
Authored: Wed Dec 27 11:44:51 2017 +0900
Committer: Youngwoo Kim 
Committed: Wed Dec 27 11:44:51 2017 +0900

--
 .../src/common/hama/patch0-HAMA-1002.diff| 15 +++
 bigtop-packages/src/rpm/hama/SPECS/hama.spec |  3 +++
 2 files changed, 18 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/badd85eb/bigtop-packages/src/common/hama/patch0-HAMA-1002.diff
--
diff --git a/bigtop-packages/src/common/hama/patch0-HAMA-1002.diff 
b/bigtop-packages/src/common/hama/patch0-HAMA-1002.diff
new file mode 100644
index 000..bba4924
--- /dev/null
+++ b/bigtop-packages/src/common/hama/patch0-HAMA-1002.diff
@@ -0,0 +1,15 @@
+diff --git a/commons/pom.xml b/commons/pom.xml
+index 327733630..4166523ca 100644
+--- a/commons/pom.xml
 b/commons/pom.xml
+@@ -36,6 +36,10 @@
+   guava
+   13.0.1
+ 
++
++  junit
++  junit
++
+   
+   
+   

http://git-wip-us.apache.org/repos/asf/bigtop/blob/badd85eb/bigtop-packages/src/rpm/hama/SPECS/hama.spec
--
diff --git a/bigtop-packages/src/rpm/hama/SPECS/hama.spec 
b/bigtop-packages/src/rpm/hama/SPECS/hama.spec
index fc24d90..3b9bb10 100644
--- a/bigtop-packages/src/rpm/hama/SPECS/hama.spec
+++ b/bigtop-packages/src/rpm/hama/SPECS/hama.spec
@@ -50,6 +50,7 @@ Source3: %{name}-bspmaster.init
 Source4: %{name}-groom.init
 Source5: %{name}.default
 Source6: bigtop.bom
+#BIGTOP_PATCH_FILES
 Requires: hadoop, bigtop-utils, zookeeper
 
 # Disable automatic Requires generation
@@ -161,6 +162,8 @@ Documentation for Apache Hama
 %prep
 %setup -n %{name}-%{hama_base_version}
 
+#BIGTOP_PATCH_COMMANDS
+
 %build
 bash $RPM_SOURCE_DIR/do-component-build
 

bigtop git commit: BIGTOP-2959: Add SPARK_DIST_CLASSPATH to include libraries for HDFS and YARN

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 9dd5f5a1d -> 9e78a0365


BIGTOP-2959: Add SPARK_DIST_CLASSPATH to include libraries for HDFS and YARN


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/9e78a036
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/9e78a036
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/9e78a036

Branch: refs/heads/master
Commit: 9e78a03655ee68b8a60258c2a5efd9dbe6dfac30
Parents: 9dd5f5a
Author: Youngwoo Kim 
Authored: Tue Dec 26 15:17:19 2017 +0900
Committer: Youngwoo Kim 
Committed: Wed Dec 27 11:24:13 2017 +0900

--
 .../src/common/spark/do-component-build|  2 +-
 bigtop-packages/src/common/spark/spark-env.sh  |  3 +--
 bigtop-packages/src/deb/spark/rules| 17 -
 bigtop-packages/src/rpm/spark/SPECS/spark.spec | 17 -
 4 files changed, 2 insertions(+), 37 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/9e78a036/bigtop-packages/src/common/spark/do-component-build
--
diff --git a/bigtop-packages/src/common/spark/do-component-build 
b/bigtop-packages/src/common/spark/do-component-build
index c2a2322..084f9f2 100644
--- a/bigtop-packages/src/common/spark/do-component-build
+++ b/bigtop-packages/src/common/spark/do-component-build
@@ -28,7 +28,7 @@ BUILD_OPTS="-Divy.home=${HOME}/.ivy2 
-Dsbt.ivy.home=${HOME}/.ivy2 -Duser.home=${
 -Dyarn.version=$HADOOP_VERSION \
 -Dprotobuf.version=2.5.0 \
 -DrecompileMode=all \
--Pyarn -Phadoop-2.7 \
+-Pyarn -Phadoop-2.7 -Phadoop-provided \
 -Phive -Phive-thriftserver \
 -Psparkr \
 $SPARK_BUILD_OPTS"

http://git-wip-us.apache.org/repos/asf/bigtop/blob/9e78a036/bigtop-packages/src/common/spark/spark-env.sh
--
diff --git a/bigtop-packages/src/common/spark/spark-env.sh 
b/bigtop-packages/src/common/spark/spark-env.sh
index f772ef1..1806119 100755
--- a/bigtop-packages/src/common/spark/spark-env.sh
+++ b/bigtop-packages/src/common/spark/spark-env.sh
@@ -28,8 +28,7 @@ export SPARK_LAUNCH_WITH_SCALA=0
 export SPARK_LIBRARY_PATH=${SPARK_LIBRARY_PATH:-${SPARK_HOME}/lib}
 export SCALA_LIBRARY_PATH=${SCALA_LIBRARY_PATH:-${SPARK_HOME}/lib}
 
-# Let's make sure that all needed hadoop libs are added properly
-export 
CLASSPATH="$CLASSPATH:$HADOOP_HOME/*:$HADOOP_HDFS_HOME/*:$HADOOP_YARN_HOME/*:$HADOOP_MAPRED_HOME/*"
+export SPARK_DIST_CLASSPATH=$(hadoop classpath)
 export SPARK_LIBRARY_PATH=$SPARK_LIBRARY_PATH:${HADOOP_HOME}/lib/native
 
 export STANDALONE_SPARK_MASTER_HOST=`hostname -f`

http://git-wip-us.apache.org/repos/asf/bigtop/blob/9e78a036/bigtop-packages/src/deb/spark/rules
--
diff --git a/bigtop-packages/src/deb/spark/rules 
b/bigtop-packages/src/deb/spark/rules
index d5fdb6d..5bd9833 100644
--- a/bigtop-packages/src/deb/spark/rules
+++ b/bigtop-packages/src/deb/spark/rules
@@ -49,20 +49,3 @@ override_dh_install:
dh_install
# BIGTOP-2588 drop datanucleus jars from spark-core
rm -Rf debian/spark-core/usr/lib/spark/jars/datanucleus*
-
-   rm -f debian/tmp/${lib_spark}/jars/hadoop-*.jar
-   ln -s  ${lib_hadoop_client}/hadoop-annotations.jar 
debian/tmp/${lib_spark}/jars/
-   ln -s  ${lib_hadoop_client}/hadoop-auth.jar 
debian/tmp/${lib_spark}/jars/
-   ln -s  ${lib_hadoop_client}/hadoop-client.jar 
debian/tmp/${lib_spark}/jars/
-   ln -s  ${lib_hadoop_client}/hadoop-common.jar 
debian/tmp/${lib_spark}/jars/
-   ln -s  ${lib_hadoop_client}/hadoop-hdfs.jar 
debian/tmp/${lib_spark}/jars/
-   ln -s  ${lib_hadoop_client}/hadoop-mapreduce-client-app.jar 
debian/tmp/${lib_spark}/jars/
-   ln -s  ${lib_hadoop_client}/hadoop-mapreduce-client-common.jar 
debian/tmp/${lib_spark}/jars/
-   ln -s  ${lib_hadoop_client}/hadoop-mapreduce-client-core.jar 
debian/tmp/${lib_spark}/jars/
-   ln -s  ${lib_hadoop_client}/hadoop-mapreduce-client-jobclient.jar 
debian/tmp/${lib_spark}/jars/
-   ln -s  ${lib_hadoop_client}/hadoop-mapreduce-client-shuffle.jar 
debian/tmp/${lib_spark}/jars/
-   ln -s  ${lib_hadoop_yarn}/hadoop-yarn-api.jar 
debian/tmp/${lib_spark}/jars/
-   ln -s  ${lib_hadoop_yarn}/hadoop-yarn-client.jar 
debian/tmp/${lib_spark}/jars/
-   ln -s  ${lib_hadoop_yarn}/hadoop-yarn-common.jar 
debian/tmp/${lib_spark}/jars/
-   ln -s  ${lib_hadoop_yarn}/hadoop-yarn-server-common.jar 
debian/tmp/${lib_spark}/jars/
-   ln -s  ${lib_hadoop_yarn}/hadoop-yarn-server-web-proxy.jar 
debian/tmp/${lib_spark}/jars/

bigtop git commit: BIGTOP-2970: Bump Zeppelin version to 0.7.3

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 2ebb17d5e -> 9dd5f5a1d


BIGTOP-2970: Bump Zeppelin version to 0.7.3


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/9dd5f5a1
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/9dd5f5a1
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/9dd5f5a1

Branch: refs/heads/master
Commit: 9dd5f5a1ddd8cc391f71495c55c09fa5663059a9
Parents: 2ebb17d
Author: Youngwoo Kim 
Authored: Wed Dec 27 01:45:40 2017 +0900
Committer: Youngwoo Kim 
Committed: Wed Dec 27 11:14:51 2017 +0900

--
 bigtop-packages/src/common/zeppelin/do-component-build | 2 +-
 bigtop.bom | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/9dd5f5a1/bigtop-packages/src/common/zeppelin/do-component-build
--
diff --git a/bigtop-packages/src/common/zeppelin/do-component-build 
b/bigtop-packages/src/common/zeppelin/do-component-build
index 1f7eae5..37e516c 100644
--- a/bigtop-packages/src/common/zeppelin/do-component-build
+++ b/bigtop-packages/src/common/zeppelin/do-component-build
@@ -30,7 +30,7 @@ BUILD_OPTS="-Dscala.version=${SCALA_VERSION} \
 -Dignite.version=${IGNITE_HADOOP_VERSION} \
 -Dflink.version=${FLINK_VERSION} \
 -Pspark-${SPARK_VERSION%.*} \
--Phadoop-${HADOOP_VERSION%.*} \
+-Phadoop-2.7 \
 -Pscala-${SCALA_VERSION%.*} \
 -Ppyspark \
 -Psparkr \

http://git-wip-us.apache.org/repos/asf/bigtop/blob/9dd5f5a1/bigtop.bom
--
diff --git a/bigtop.bom b/bigtop.bom
index 53805a8..e7144c7 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -433,7 +433,7 @@ bigtop {
 'zeppelin' {
   name= 'zeppelin'
   relNotes = 'Apache Zeppelin'
-  version { base = '0.7.2'; pkg = base; release = 1 }
+  version { base = '0.7.3'; pkg = base; release = 1 }
   tarball { source  = "$name-${version.base}.tgz"
 destination = "$name-${version.base}.tar.gz" }
   url { download_path = "/$name/$name-${version.base}/"

bigtop git commit: BIGTOP-2969: Bump up Spark version to 2.2.0

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master faa65e935 -> 2ebb17d5e


BIGTOP-2969: Bump up Spark version to 2.2.0


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/2ebb17d5
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/2ebb17d5
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/2ebb17d5

Branch: refs/heads/master
Commit: 2ebb17d5e6fb0664fd0207aff63ed70741b9d35c
Parents: faa65e9
Author: Youngwoo Kim 
Authored: Tue Dec 26 23:08:32 2017 +0900
Committer: Youngwoo Kim 
Committed: Wed Dec 27 11:13:34 2017 +0900

--
 bigtop.bom | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/2ebb17d5/bigtop.bom
--
diff --git a/bigtop.bom b/bigtop.bom
index 81837d2..53805a8 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -319,7 +319,7 @@ bigtop {
   name= 'spark'
   pkg = 'spark-core'
   relNotes = 'Apache Spark'
-  version { base = '2.1.1'; pkg = base; release = 1 }
+  version { base = '2.2.0'; pkg = base; release = 1 }
   tarball { destination = "$name-${version.base}.tar.gz"
 source  = "$name-${version.base}.tgz" }
   url { download_path = "/$name/$name-${version.base}"

bigtop git commit: BIGTOP-2981: Packaging SparkR is broken due to BIGTOP-2959

[ywkim]

Repository: bigtop
Updated Branches:
  refs/heads/master 7007c0265 -> 51e751f78


BIGTOP-2981: Packaging SparkR is broken due to BIGTOP-2959


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/51e751f7
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/51e751f7
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/51e751f7

Branch: refs/heads/master
Commit: 51e751f78db4a157a2dfcf893a4616e5a31f190b
Parents: 7007c02
Author: Youngwoo Kim 
Authored: Thu Jan 4 10:28:41 2018 +0900
Committer: Youngwoo Kim 
Committed: Thu Jan 4 10:28:41 2018 +0900

--
 bigtop-packages/src/common/spark/do-component-build | 2 +-
 bigtop-packages/src/deb/spark/rules | 2 ++
 bigtop-packages/src/rpm/spark/SPECS/spark.spec  | 2 ++
 3 files changed, 5 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/bigtop/blob/51e751f7/bigtop-packages/src/common/spark/do-component-build
--
diff --git a/bigtop-packages/src/common/spark/do-component-build 
b/bigtop-packages/src/common/spark/do-component-build
index 084f9f2..c2a2322 100644
--- a/bigtop-packages/src/common/spark/do-component-build
+++ b/bigtop-packages/src/common/spark/do-component-build
@@ -28,7 +28,7 @@ BUILD_OPTS="-Divy.home=${HOME}/.ivy2 
-Dsbt.ivy.home=${HOME}/.ivy2 -Duser.home=${
 -Dyarn.version=$HADOOP_VERSION \
 -Dprotobuf.version=2.5.0 \
 -DrecompileMode=all \
--Pyarn -Phadoop-2.7 -Phadoop-provided \
+-Pyarn -Phadoop-2.7 \
 -Phive -Phive-thriftserver \
 -Psparkr \
 $SPARK_BUILD_OPTS"

http://git-wip-us.apache.org/repos/asf/bigtop/blob/51e751f7/bigtop-packages/src/deb/spark/rules
--
diff --git a/bigtop-packages/src/deb/spark/rules 
b/bigtop-packages/src/deb/spark/rules
index 5bd9833..501963f 100644
--- a/bigtop-packages/src/deb/spark/rules
+++ b/bigtop-packages/src/deb/spark/rules
@@ -49,3 +49,5 @@ override_dh_install:
dh_install
# BIGTOP-2588 drop datanucleus jars from spark-core
rm -Rf debian/spark-core/usr/lib/spark/jars/datanucleus*
+
+   rm -f debian/tmp/${lib_spark}/jars/hadoop-*.jar

http://git-wip-us.apache.org/repos/asf/bigtop/blob/51e751f7/bigtop-packages/src/rpm/spark/SPECS/spark.spec
--
diff --git a/bigtop-packages/src/rpm/spark/SPECS/spark.spec 
b/bigtop-packages/src/rpm/spark/SPECS/spark.spec
index 8e73cd8..95fbc88 100644
--- a/bigtop-packages/src/rpm/spark/SPECS/spark.spec
+++ b/bigtop-packages/src/rpm/spark/SPECS/spark.spec
@@ -166,6 +166,8 @@ bash $RPM_SOURCE_DIR/install_spark.sh \
   --prefix=$RPM_BUILD_ROOT  \
   --doc-dir=%{doc_spark}
 
+%__rm -f $RPM_BUILD_ROOT/%{lib_spark}/jars/hadoop-*.jar
+
 for service in %{spark_services}
 do
 # Install init script

[bigtop] branch cnb updated: BIGTOP-3267: Standalone Minio as default

[ywkim]

This is an automated email from the ASF dual-hosted git repository.

ywkim pushed a commit to branch cnb
in repository https://gitbox.apache.org/repos/asf/bigtop.git


The following commit(s) were added to refs/heads/cnb by this push:
 new 0e47463  BIGTOP-3267: Standalone Minio as default
0e47463 is described below

commit 0e474633d53e916bb3c29e216894deed9cf61d68
Author: Youngwoo Kim 
AuthorDate: Wed Oct 30 17:06:37 2019 +0900

BIGTOP-3267: Standalone Minio as default
---
 README.md  |  13 +++-
 storage/local-path/README.md   |  14 
 storage/local-path/local-path-storage.yaml | 105 +
 storage/minio/README.md|   5 ++
 storage/minio/values.yaml  |  16 ++---
 storage/rook/minio/object-store.yaml   |   5 +-
 6 files changed, 146 insertions(+), 12 deletions(-)

diff --git a/README.md b/README.md
index 6b980a4..f4b93fc 100755
--- a/README.md
+++ b/README.md
@@ -100,7 +100,7 @@ Minikube is the easiest tool to run a single-node 
Kubernetes cluster.
 
 ```
 $ cd $BIGTOP_HOME
-$ minikube start --cpus 8 --memory 8196 --container-runtime=cri-o 
+$ minikube start --cpus 8 --memory 8196 --disk-size=80g 
--container-runtime=cri-o 
 $ kubectl cluster-info
 
 ```
@@ -252,21 +252,30 @@ $ kubectl create -f 
dl/rook-1.1.2/cluster/examples/kubernetes/minio/operator.yam
 $ kubectl -n rook-minio-system get pod
 ```
 
-Create object store:
+Create standalone Minio object store:
 ```
 $ kubectl create -f storage/rook/minio/object-store.yaml
 $ kubectl -n rook-minio get objectstores.minio.rook.io
 $ kubectl -n rook-minio get pod -l app=minio,objectstore=bigtop-rook-minio
 ```
+If you want to deploy Distributed Minio cluster, Increase the ```nodeCount``` 
in object-store.yaml file. e.g., ```nodeCount: 4``` 
 
 ### Minio
 
+Standalone Minio:
 ```
 $ cd $BIGTOP_HOME
 $ helm install --name bigtop-minio --namespace bigtop -f 
storage/minio/values.yaml stable/minio
 
 ```
 
+Distributed Minio:
+```
+$ cd $BIGTOP_HOME
+$ helm install --name bigtop-minio --namespace bigtop --set 
mode=distributed,replicas=4 -f storage/minio/values.yaml stable/minio
+
+```
+
 ```
 Minio can be accessed via port 9000 on the following DNS name from within your 
cluster:
 bigtop-minio.bigtop.svc.cluster.local
diff --git a/storage/local-path/README.md b/storage/local-path/README.md
new file mode 100644
index 000..ce5707c
--- /dev/null
+++ b/storage/local-path/README.md
@@ -0,0 +1,14 @@
+## Use HostPath for persistent local storage with Kubernetes
+
+https://github.com/rancher/local-path-provisioner
+
+```
+$ kubectl create -f $BIGTOP_HOME/storage/local-path/local-path-storage.yaml
+$ kubectl get sc
+
+```
+
+Mark ```local-path``` StorageClass as default:
+```
+kubectl patch storageclass local-path -p '{"metadata": 
{"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
+```
diff --git a/storage/local-path/local-path-storage.yaml 
b/storage/local-path/local-path-storage.yaml
new file mode 100644
index 000..3022c0a
--- /dev/null
+++ b/storage/local-path/local-path-storage.yaml
@@ -0,0 +1,105 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: local-path-storage
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: local-path-provisioner-service-account
+  namespace: local-path-storage
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: local-path-provisioner-role
+rules:
+- apiGroups: [""]
+  resources: ["nodes", "persistentvolumeclaims"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: [""]
+  resources: ["endpoints", "persistentvolumes", "pods"]
+  verbs: ["*"]
+- apiGroups: [""]
+  resources: ["events"]
+  verbs: ["create", "patch"]
+- apiGroups: ["storage.k8s.io"]
+  resources: ["storageclasses"]
+  verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: local-path-provisioner-bind
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: local-path-provisioner-role
+subjects:
+- kind: ServiceAccount
+  name: local-path-provisioner-service-account
+  namespace: local-path-storage
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: local-path-provisioner
+  namespace: local-path-storage
+spec:
+  replicas: 1
+  selector:
+matchLabels:
+  app: local-path-provisioner
+  template:
+metadata:
+  labels:
+app: local-path-provisioner
+spec:
+  serviceAccountName: local-path-provisioner-service-account
+  containers:
+  - name: local-path-provisioner
+image: rancher/local-path-provisioner:v0.0.11
+imagePullPolicy: IfNotPresent
+command:
+- local-p

[bigtop] branch cnb updated: BIGTOP-3248: Smoke test for Minio on Kubernetes

[ywkim]

This is an automated email from the ASF dual-hosted git repository.

ywkim pushed a commit to branch cnb
in repository https://gitbox.apache.org/repos/asf/bigtop.git


The following commit(s) were added to refs/heads/cnb by this push:
 new 2215c51  BIGTOP-3248: Smoke test for Minio on Kubernetes
2215c51 is described below

commit 2215c511652103896f37e47bd1033ce5467dbf82
Author: Youngwoo Kim 
AuthorDate: Thu Oct 31 14:23:42 2019 +0900

BIGTOP-3248: Smoke test for Minio on Kubernetes
---
 README.md  | 28 +++-
 .../smoke-tests/minio/TestMinioSmoke.groovy| 75 ++
 bigtop-tests/smoke-tests/minio/build.gradle| 35 ++
 storage/mc/minio-client.yaml   | 34 ++
 storage/minio/values.yaml  |  4 +-
 5 files changed, 172 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index f4b93fc..1f31664 100755
--- a/README.md
+++ b/README.md
@@ -262,14 +262,14 @@ If you want to deploy Distributed Minio cluster, Increase 
the ```nodeCount``` in
 
 ### Minio
 
-Standalone Minio:
+Deploy a Standalone Minio:
 ```
 $ cd $BIGTOP_HOME
 $ helm install --name bigtop-minio --namespace bigtop -f 
storage/minio/values.yaml stable/minio
 
 ```
 
-Distributed Minio:
+Deploy a Distributed Minio:
 ```
 $ cd $BIGTOP_HOME
 $ helm install --name bigtop-minio --namespace bigtop --set 
mode=distributed,replicas=4 -f storage/minio/values.yaml stable/minio
@@ -300,6 +300,30 @@ Alternately, you can use your browser or the Minio SDK to 
access the server - ht
 
 ```
 
+### mc (Minio Client)
+
+Deploy mc pod:
+```
+$ cd $BIGTOP_HOME
+$ kubectl create -n bigtop -f storage/mc/minio-client.yaml
+```
+```
+$ kubectl exec -n bigtop minio-client mc admin info server bigtop-minio
+```
+
+Make a bucket and remove the bucket:
+```
+$ kubectl exec -n bigtop minio-client mc mb bigtop-minio/testbucket1
+Bucket created successfully `bigtop-minio/testbucket1`.
+
+$ kubectl exec -n bigtop minio-client mc ls bigtop-minio/
+[2019-10-31 04:46:44 UTC]  0B testbucket1/
+
+$ kubectl exec -n bigtop minio-client mc rb bigtop-minio/testbucket1
+Removing `bigtop-minio/testbucket1`.
+
+```
+
 ## Zookeeper
 Deploy Zookeeper cluster on Kubernetes cluster via Helm chart:
 ```
diff --git a/bigtop-tests/smoke-tests/minio/TestMinioSmoke.groovy 
b/bigtop-tests/smoke-tests/minio/TestMinioSmoke.groovy
new file mode 100644
index 000..98bb8fa
--- /dev/null
+++ b/bigtop-tests/smoke-tests/minio/TestMinioSmoke.groovy
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.bigtop.itest.minio
+
+import org.junit.BeforeClass
+import org.junit.AfterClass
+import org.apache.bigtop.itest.shell.Shell
+import static org.junit.Assert.assertNotNull
+import static org.junit.Assert.assertTrue
+import org.junit.Test
+import org.apache.bigtop.itest.JarContent
+import org.apache.bigtop.itest.TestUtils
+import org.junit.runner.RunWith
+
+import org.apache.commons.logging.LogFactory
+import org.apache.commons.logging.Log
+
+import static org.apache.bigtop.itest.LogErrorsUtils.logError
+
+class TestKafkaSmoke {
+  static private Log LOG = LogFactory.getLog(Object.class)
+  
+  static Shell sh = new Shell("/bin/bash -s");
+
+  static String TEST_BUCKET = "testbucket123";
+  static String NS = "bigtop";
+
+  @BeforeClass
+  static void minioSetUp() {
+  }
+
+  @AfterClass
+  public static void deleteMinioTestBucket() {
+sh.exec("kubectl exec -n " + NS + " minio-client mc rb bigtop-minio/" + 
TEST_BUCKET);
+  }
+
+  @Test
+  public void testServerInfo() {
+sh.exec("kubectl exec -n " + NS + " minio-client mc admin info server 
bigtop-minio");
+String out = sh.getOut().toString();
+LOG.info(out);
+assertTrue(
+  out.contains("Uptime") ||
+  out.contains("Version") ||
+  out.contains("Storage")
+);
+  }
+
+  @Test
+  public void testCreateTopics() {
+sh.exec("kubectl exec -n " + NS + " minio-client mc mb bigtop-minio/" + 
TEST_BUCKET);
+String out = sh.getOut().toString();

[bigtop] branch cnb updated: BIGTOP-3273: Rename Presto component

[ywkim]

This is an automated email from the ASF dual-hosted git repository.

ywkim pushed a commit to branch cnb
in repository https://gitbox.apache.org/repos/asf/bigtop.git


The following commit(s) were added to refs/heads/cnb by this push:
 new 210bd9a  BIGTOP-3273: Rename Presto component
210bd9a is described below

commit 210bd9a17302166b6117b3e3bae9128aa848c76a
Author: Youngwoo Kim 
AuthorDate: Fri Nov 15 10:14:15 2019 +0900

BIGTOP-3273: Rename Presto component
---
 {prestosql => presto}/README.md| 0
 {prestosql => presto}/docker/Makefile  | 4 ++--
 presto/docker/README.md| 6 ++
 {prestosql => presto}/docker/coordinator/Dockerfile| 0
 presto/docker/coordinator/README.md| 7 +++
 {prestosql => presto}/docker/coordinator/config.properties | 0
 {prestosql/docker/worker => presto/docker/coordinator}/jvm.config  | 0
 .../docker/worker => presto/docker/coordinator}/minio.properties   | 0
 .../docker/worker => presto/docker/coordinator}/node.properties| 0
 {prestosql => presto}/docker/worker/Dockerfile | 0
 {prestosql => presto}/docker/worker/README.md  | 0
 {prestosql => presto}/docker/worker/config.properties  | 0
 {prestosql/docker/coordinator => presto/docker/worker}/jvm.config  | 0
 .../docker/coordinator => presto/docker/worker}/minio.properties   | 0
 .../docker/coordinator => presto/docker/worker}/node.properties| 0
 {prestosql => presto}/presto-deployment-master.yaml| 2 +-
 {prestosql => presto}/presto-deployment-worker.yaml| 2 +-
 {prestosql => presto}/presto-deployment.yaml   | 2 +-
 {prestosql => presto}/service.yaml | 0
 prestosql/docker/README.md | 6 --
 prestosql/docker/coordinator/README.md | 7 ---
 21 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/prestosql/README.md b/presto/README.md
similarity index 100%
rename from prestosql/README.md
rename to presto/README.md
diff --git a/prestosql/docker/Makefile b/presto/docker/Makefile
similarity index 53%
rename from prestosql/docker/Makefile
rename to presto/docker/Makefile
index aa5c1ac..184ef98 100644
--- a/prestosql/docker/Makefile
+++ b/presto/docker/Makefile
@@ -3,6 +3,6 @@ PRESTO_VERSION := latest
 .PHONY: build 
 
 build:
-   docker build --squash --build-arg 
PRESTO_IMAGE=prestosql/presto:${PRESTO_VERSION} -t 
bigtop/prestosql-coordinator:${PRESTO_VERSION} coordinator
-   docker build --squash --build-arg 
PRESTO_IMAGE=prestosql/presto:${PRESTO_VERSION} -t 
bigtop/prestosql-worker:${PRESTO_VERSION} worker
+   docker build --squash --build-arg 
PRESTO_IMAGE=prestosql/presto:${PRESTO_VERSION} -t 
bigtop/presto-coordinator:${PRESTO_VERSION} coordinator
+   docker build --squash --build-arg 
PRESTO_IMAGE=prestosql/presto:${PRESTO_VERSION} -t 
bigtop/presto-worker:${PRESTO_VERSION} worker
 
diff --git a/presto/docker/README.md b/presto/docker/README.md
new file mode 100644
index 000..b38e6ea
--- /dev/null
+++ b/presto/docker/README.md
@@ -0,0 +1,6 @@
+Buil Docker images for Presto 
+
+```
+$ make build
+
+```
diff --git a/prestosql/docker/coordinator/Dockerfile 
b/presto/docker/coordinator/Dockerfile
similarity index 100%
rename from prestosql/docker/coordinator/Dockerfile
rename to presto/docker/coordinator/Dockerfile
diff --git a/presto/docker/coordinator/README.md 
b/presto/docker/coordinator/README.md
new file mode 100755
index 000..b7a9625
--- /dev/null
+++ b/presto/docker/coordinator/README.md
@@ -0,0 +1,7 @@
+# Use squash !
+
+Building this image, make sure you use squash so you can gut the 12 GB file :) 
+
+docker build --squash -t bigtop/presto-coordinator:320 --build-arg 
PRESTO_IMAGE=prestosql/presto:320 .
+
+
diff --git a/prestosql/docker/coordinator/config.properties 
b/presto/docker/coordinator/config.properties
similarity index 100%
rename from prestosql/docker/coordinator/config.properties
rename to presto/docker/coordinator/config.properties
diff --git a/prestosql/docker/worker/jvm.config 
b/presto/docker/coordinator/jvm.config
similarity index 100%
rename from prestosql/docker/worker/jvm.config
rename to presto/docker/coordinator/jvm.config
diff --git a/prestosql/docker/worker/minio.properties 
b/presto/docker/coordinator/minio.properties
similarity index 100%
rename from prestosql/docker/worker/minio.properties
rename to presto/docker/coordinator/minio.properties
diff --git a/prestosql/docker/worker/node.properties 
b/presto/docker/coordinator/node.properties
similarity index 100%
rename from prestosql/docker/worker/node.properties
rename to presto/docker/coordinator/node.properties
diff --git a/prestosql/docker/worker/Dockerfile 
b/presto/docker/

[bigtop] branch cnb updated: BIGTOP-3255: Add Spark operator into distribution

[ywkim]

This is an automated email from the ASF dual-hosted git repository.

ywkim pushed a commit to branch cnb
in repository https://gitbox.apache.org/repos/asf/bigtop.git


The following commit(s) were added to refs/heads/cnb by this push:
 new 472c525  BIGTOP-3255: Add Spark operator into distribution
472c525 is described below

commit 472c525c701a53c815d5cff1240eee21a9f5e897
Author: Youngwoo Kim 
AuthorDate: Fri Oct 18 11:59:38 2019 +0900

BIGTOP-3255: Add Spark operator into distribution
---
 README.md   |  4 ++
 bigtop-tests/smoke-tests/spark/TestSpark.groovy | 53 +++---
 bigtop-tests/smoke-tests/spark/build.gradle |  2 +-
 bigtop.bom  | 13 +-
 kubespray/vagrant/Vagrantfile   |  2 +-
 spark/README.md | 50 +
 spark/examples/spark-pi-batch.yaml  | 40 +
 spark/examples/spark-pi.yaml| 39 +
 spark/spark-rbac.yaml   | 32 ++
 spark/values.yaml   | 58 +
 10 files changed, 253 insertions(+), 40 deletions(-)

diff --git a/README.md b/README.md
index 53c288b..02fba6e 100755
--- a/README.md
+++ b/README.md
@@ -234,6 +234,10 @@ incubator/schema-registry
 
 ```
 
+## Spark
+
+See spark/README.md
+
 # Getting Started
 
 Below are some recipes for getting started with using Apache Bigtop. As Apache 
Bigtop has different subprojects, these recipes will continue to evolve.
diff --git a/bigtop-tests/smoke-tests/spark/TestSpark.groovy 
b/bigtop-tests/smoke-tests/spark/TestSpark.groovy
index 6d13ebf..29e0957 100644
--- a/bigtop-tests/smoke-tests/spark/TestSpark.groovy
+++ b/bigtop-tests/smoke-tests/spark/TestSpark.groovy
@@ -39,54 +39,33 @@ class TestSpark {
   static private Log LOG = LogFactory.getLog(Object.class)
 
   static Shell sh = new Shell("/bin/bash -s")
-  static final String SPARK_HOME = System.getenv("SPARK_HOME")
-  static final String SPARK_MASTER_IP = System.getenv("SPARK_MASTER_IP")
-  static final String SPARK_MASTER_PORT = System.getenv("SPARK_MASTER_PORT")
-  static final String TEST_SPARKSQL_LOG = "/tmp/TestSpark_testSparkSQL.log"
+  static final String BIGTOP_HOME = System.getenv("BIGTOP_HOME")
+  static final String BIGTOP_K8S_NS = "bigtop";
 
   @BeforeClass
   static void setUp() {
-sh.exec("rm -f " + TEST_SPARKSQL_LOG)
-// create HDFS examples/src/main/resources
-sh.exec("hdfs dfs -mkdir -p examples/src/main/resources")
-// extract people.txt file into it
-String examplesJar = JarContent.getJarName("$SPARK_HOME/examples/jars", 
'spark-examples.*jar')
-assertNotNull(examplesJar, "spark-examples.jar file wasn't found")
-ZipInputStream zipInputStream = new ZipInputStream(new 
FileInputStream("$SPARK_HOME/examples/jars/$examplesJar"))
-File examplesDir = new File('examples')
-examplesDir.mkdirs()
-zipInputStream.unzip(examplesDir.getName(), 'people')
-sh.exec("hdfs dfs -put examples/* examples/src/main/resources")
-logError(sh)
   }
 
   @AfterClass
   public static void tearDown() {
-sh.exec("hdfs dfs -ls")
-logError(sh)
-sh.exec("hdfs dfs -rm -r examples")
+sh.exec("kubectl delete sparkapplication " + "-n " + BIGTOP_K8S_NS + " 
spark-pi")
 logError(sh)
   }
 
   @Test
-  void testSparkSQL() {
-// Let's figure out the proper mode for the submission
-// If SPARK_MASTER_IP nor SPARK_MASTER_PORT are set, we'll assume
-// 'yarn-client' mode
-String masterMode = 'yarn-client'
-if (SPARK_MASTER_IP != null && SPARK_MASTER_PORT != null)
-  masterMode = "spark://$SPARK_MASTER_IP:$SPARK_MASTER_PORT"
-else
-  println("SPARK_MASTER isn't set. yarn-client submission will be used. " +
-  "Refer to smoke-tests/README If this isn't what you you expect.")
-
-final String SPARK_SHELL = SPARK_HOME + "/bin/spark-shell --master 
$masterMode"
-// Let's use time, 'cause the test has one job
-sh.exec("timeout 120 " + SPARK_SHELL +
-" --class org.apache.spark.examples.sql.JavaSparkSQLExample " +
-" --jars " + SPARK_HOME + "/examples/jars/spark-examples*.jar > " +
-TEST_SPARKSQL_LOG + " 2>&1")
-logError(sh)
+  void testSparkPi() {
+sh.exec("kubectl apply -f " + BIGTOP_HOME + 
"/spark/examples/spark-pi.yaml");
+logError(sh);
 assertTrue("Failed ...", sh.getRet() == 0);
+
+// sleep 20s
+sleep(2);
+
+sh.exec("kubectl logs " + "-n " + BIGTOP_K8S_NS + " spark-pi-driver");
+logError(sh);
+String

[bigtop] branch cnb updated: BIGTOP-3258: Deploy distributed Minio object storage

[ywkim]

This is an automated email from the ASF dual-hosted git repository.

ywkim pushed a commit to branch cnb
in repository https://gitbox.apache.org/repos/asf/bigtop.git


The following commit(s) were added to refs/heads/cnb by this push:
 new a46354f  BIGTOP-3258: Deploy distributed Minio object storage
a46354f is described below

commit a46354f2b9a229de148838684acf5ba04ca500c8
Author: Youngwoo Kim 
AuthorDate: Fri Oct 18 16:12:35 2019 +0900

BIGTOP-3258: Deploy distributed Minio object storage
---
 README.md|  43 +-
 kubectl/plugin/kubectl-bigtop|  37 +
 storage/minio/values.yaml| 278 +++
 storage/rook/minio/object-store.yaml |  16 +-
 4 files changed, 364 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index 02fba6e..79b9e3a 100755
--- a/README.md
+++ b/README.md
@@ -130,6 +130,8 @@ sudo apt-get install -y lvm2
 ```
 Refer to https://rook.io/docs/rook/v1.1/k8s-pre-reqs.html for prerequisites on 
Rook
 
+### Rook Ceph
+
 Run ```download``` task to get Rook binary:
 ```
 $ ./gradlew rook-clean rook-download && cd dl/ && tar xvfz rook-1.1.2.tar.gz
@@ -169,10 +171,12 @@ Refer to https://rook.io/docs/rook/v1.1/ceph-toolbox.html 
for more details.
 Create a StorageClass for Ceph RBD:
 ```
 $ kubectl create -f 
dl/rook-1.1.2/cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml
-kubectl get storageclass
+
+$ kubectl get storageclass
 rook-ceph-block
 ```
 
+### Rook Minio
 Create Minio operator:
 ```
 $ kubectl create -f 
dl/rook-1.1.2/cluster/examples/kubernetes/minio/operator.yaml
@@ -180,9 +184,44 @@ $ kubectl create -f 
dl/rook-1.1.2/cluster/examples/kubernetes/minio/operator.yam
 #
 $ kubectl -n rook-minio-system get pod
 ```
+
+Create object store:
 ```
 $ kubectl create -f storage/rook/minio/object-store.yaml
-$ kubectl -n rook-minio get pod -l app=minio,objectstore=my-store
+$ kubectl -n rook-minio get objectstores.minio.rook.io
+$ kubectl -n rook-minio get pod -l app=minio,objectstore=bigtop-rook-minio
+```
+
+### Minio
+
+```
+$ cd $BIGTOP_HOME
+$ helm install --name bigtop-minio --namespace bigtop -f 
storage/minio/values.yaml stable/minio
+
+```
+
+```
+Minio can be accessed via port 9000 on the following DNS name from within your 
cluster:
+bigtop-minio.bigtop.svc.cluster.local
+
+To access Minio from localhost, run the below commands:
+
+  1. export POD_NAME=$(kubectl get pods --namespace bigtop -l 
"release=bigtop-minio" -o jsonpath="{.items[0].metadata.name}")
+
+  2. kubectl port-forward $POD_NAME 9000 --namespace bigtop
+
+Read more about port forwarding here: 
http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/
+
+You can now access Minio server on http://localhost:9000. Follow the below 
steps to connect to Minio server with mc client:
+
+  1. Download the Minio mc client - 
https://docs.minio.io/docs/minio-client-quickstart-guide
+
+  2. mc config host add bigtop-minio-local http://localhost:9000 minio 
minio123 S3v4
+
+  3. mc ls bigtop-minio-local
+
+Alternately, you can use your browser or the Minio SDK to access the server - 
https://docs.minio.io/categories/17
+
 ```
 
 ## Zookeeper
diff --git a/kubectl/plugin/kubectl-bigtop b/kubectl/plugin/kubectl-bigtop
index 89abf3c..1291376 100755
--- a/kubectl/plugin/kubectl-bigtop
+++ b/kubectl/plugin/kubectl-bigtop
@@ -61,3 +61,40 @@ if [[ "$1" == "helm-install" ]]; then
 helm version
 exit 0
 fi
+
+# Install Rook-Ceph
+if [[ "$1" == "rook-ceph-install" ]]; then
+
+kubectl create -f 
$BIGTOP_HOME/dl/rook-1.1.2/cluster/examples/kubernetes/ceph/common.yaml
+kubectl create -f 
$BIGTOP_HOME/dl/rook-1.1.2/cluster/examples/kubernetes/ceph/operator.yaml
+kubectl -n rook-ceph get pod
+
+# Ceph cluster
+# test
+kubectl create -f $BIGTOP_HOME/storage/rook/ceph/cluster-test.yaml
+# production
+# kubectl create -f $BIGTOP_HOME/storage/rook/ceph/cluster.yaml
+
+sleep 10s
+
+# Ceph toolbox
+kubectl create -f 
$BIGTOP_HOME/dl/rook-1.1.2/cluster/examples/kubernetes/ceph/toolbox.yaml
+
+# StorageClass: Ceph RBD
+kubectl create -f 
$BIGTOP_HOME/dl/rook-1.1.2/cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml
+kubectl get storageclass
+exit 0
+fi
+
+# Rook Ceph toolbox
+if [[ "$1" == "rook-ceph-toolbox" ]]; then
+kubectl -n rook-ceph exec -it $(kubectl -n rook-ceph get pod -l 
"app=rook-ceph-tools" -o jsonpath='{.items[0].metadata.name}') bash
+exit 0
+fi
+
+# Minio
+if [[ "$1" == "minio-install" ]]; then
+helm install --name bigtop-minio --namespace bigtop -f 
storage/minio/values.yaml stable/minio
+exit 0
+fi
+
diff --git a/storage/minio/values.yaml b/storage/minio/values.yaml
new file mode 100644
index 000..272e64a
--- /dev/null
+++ b/storage/minio/values.yaml
@@ -0,0 +1,278 @@
+## set kubernetes cluste

[bigtop] branch cnb updated: BIGTOP-3255: Add Spark operator into distribution (addendum)

[ywkim]

This is an automated email from the ASF dual-hosted git repository.

ywkim pushed a commit to branch cnb
in repository https://gitbox.apache.org/repos/asf/bigtop.git


The following commit(s) were added to refs/heads/cnb by this push:
 new 564d7f2  BIGTOP-3255: Add Spark operator into distribution (addendum)
564d7f2 is described below

commit 564d7f276de1c8fbb4a60b3dba8ca715870bbbf3
Author: Youngwoo Kim 
AuthorDate: Fri Oct 18 13:11:44 2019 +0900

BIGTOP-3255: Add Spark operator into distribution (addendum)
---
 kubespray/vagrant/Vagrantfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/kubespray/vagrant/Vagrantfile b/kubespray/vagrant/Vagrantfile
index dff6145..0e5b75b 100644
--- a/kubespray/vagrant/Vagrantfile
+++ b/kubespray/vagrant/Vagrantfile
@@ -32,8 +32,8 @@ SUPPORTED_OS = {
 $num_instances = 3
 $instance_name_prefix = "k8s"
 $vm_gui = false
-$vm_memory = 2048
-$vm_cpus = 1
+$vm_memory = 3096
+$vm_cpus = 3
 $shared_folders = {}
 $forwarded_ports = {}
 $subnet = "172.17.8"

[bigtop] branch master updated: BIGTOP-3259. Update the default stack version for docker-provisioner to 1.4.0

[ywkim]

This is an automated email from the ASF dual-hosted git repository.

ywkim pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git


The following commit(s) were added to refs/heads/master by this push:
 new 906fe70  BIGTOP-3259. Update the default stack version for 
docker-provisioner to 1.4.0
906fe70 is described below

commit 906fe70646688708742efe4778844925b894d9dc
Author: Kengo Seki 
AuthorDate: Sun Oct 20 20:13:24 2019 +0900

BIGTOP-3259. Update the default stack version for docker-provisioner to 
1.4.0

Signed-off-by: Youngwoo Kim 
---
 provisioner/docker/config_centos-7.yaml | 2 +-
 provisioner/docker/config_debian-9.yaml | 2 +-
 provisioner/docker/config_fedora-26.yaml| 2 +-
 provisioner/docker/config_ubuntu-16.04.yaml | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/provisioner/docker/config_centos-7.yaml 
b/provisioner/docker/config_centos-7.yaml
index 49f86ae..ab0e9d8 100644
--- a/provisioner/docker/config_centos-7.yaml
+++ b/provisioner/docker/config_centos-7.yaml
@@ -17,7 +17,7 @@ docker:
 memory_limit: "4g"
 image: "bigtop/puppet:trunk-centos-7"
 
-repo: "http://repos.bigtop.apache.org/releases/1.3.0/centos/7/$basearch;
+repo: "http://repos.bigtop.apache.org/releases/1.4.0/centos/7/$basearch;
 distro: centos
 components: [hdfs, yarn, mapreduce]
 enable_local_repo: false
diff --git a/provisioner/docker/config_debian-9.yaml 
b/provisioner/docker/config_debian-9.yaml
index 1818f2d..9b07254 100644
--- a/provisioner/docker/config_debian-9.yaml
+++ b/provisioner/docker/config_debian-9.yaml
@@ -17,7 +17,7 @@ docker:
 memory_limit: "4g"
 image: "bigtop/puppet:trunk-debian-9"
 
-repo: "http://repos.bigtop.apache.org/releases/1.3.0/debian/9/$(ARCH)"
+repo: "http://repos.bigtop.apache.org/releases/1.4.0/debian/9/$(ARCH)"
 distro: debian
 components: [hdfs, yarn, mapreduce]
 enable_local_repo: false
diff --git a/provisioner/docker/config_fedora-26.yaml 
b/provisioner/docker/config_fedora-26.yaml
index 8881c14..7510cb1 100644
--- a/provisioner/docker/config_fedora-26.yaml
+++ b/provisioner/docker/config_fedora-26.yaml
@@ -17,7 +17,7 @@ docker:
 memory_limit: "4g"
 image: "bigtop/puppet:trunk-fedora-26"
 
-repo: "http://repos.bigtop.apache.org/releases/1.3.0/fedora/26/$basearch;
+repo: "http://repos.bigtop.apache.org/releases/1.4.0/fedora/26/$basearch;
 distro: centos
 components: [hdfs, yarn, mapreduce]
 enable_local_repo: false
diff --git a/provisioner/docker/config_ubuntu-16.04.yaml 
b/provisioner/docker/config_ubuntu-16.04.yaml
index 64cace9..e087a78 100644
--- a/provisioner/docker/config_ubuntu-16.04.yaml
+++ b/provisioner/docker/config_ubuntu-16.04.yaml
@@ -17,7 +17,7 @@ docker:
 memory_limit: "4g"
 image:  "bigtop/puppet:trunk-ubuntu-16.04"
 
-repo: "http://repos.bigtop.apache.org/releases/1.3.0/ubuntu/16.04/$(ARCH)"
+repo: "http://repos.bigtop.apache.org/releases/1.4.0/ubuntu/16.04/$(ARCH)"
 distro: debian
 components: [hdfs, yarn, mapreduce]
 enable_local_repo: false

[bigtop] branch cnb updated: BIGTOP-3240: Smoke test for Rook-Ceph on Kubernetes

[ywkim]

This is an automated email from the ASF dual-hosted git repository.

ywkim pushed a commit to branch cnb
in repository https://gitbox.apache.org/repos/asf/bigtop.git


The following commit(s) were added to refs/heads/cnb by this push:
 new 5fda286  BIGTOP-3240: Smoke test for Rook-Ceph on Kubernetes
5fda286 is described below

commit 5fda286cbf8f4b61477b0d334a26cb7dae1ecf9a
Author: Youngwoo Kim 
AuthorDate: Tue Oct 15 13:24:34 2019 +0900

BIGTOP-3240: Smoke test for Rook-Ceph on Kubernetes
---
 .../smoke-tests/rook-ceph/TestRookCeph.groovy  | 62 ++
 bigtop-tests/smoke-tests/rook-ceph/build.gradle| 32 +++
 2 files changed, 94 insertions(+)

diff --git a/bigtop-tests/smoke-tests/rook-ceph/TestRookCeph.groovy 
b/bigtop-tests/smoke-tests/rook-ceph/TestRookCeph.groovy
new file mode 100644
index 000..e4c64ce
--- /dev/null
+++ b/bigtop-tests/smoke-tests/rook-ceph/TestRookCeph.groovy
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.junit.BeforeClass
+import org.junit.AfterClass
+
+import org.apache.bigtop.itest.shell.Shell
+import static org.junit.Assert.assertTrue
+import org.junit.Test
+import org.apache.commons.logging.LogFactory
+import org.apache.commons.logging.Log
+
+import static org.apache.bigtop.itest.LogErrorsUtils.logError
+
+class TestRookCeph {
+  static private Log LOG = LogFactory.getLog(Object.class)
+
+  static Shell sh = new Shell("/bin/bash -s")
+
+  @BeforeClass
+  static void setUp() {
+// noop for now.
+  }
+
+  @AfterClass
+  public static void tearDown() {
+// noop for now
+  }
+
+  @Test
+  void testRookCephStatus() {
+// Basic test to verify that the server is running, and is in a
+// state that we expect.
+LOG.info("Running ceph status");
+// $ kubectl -n rook-ceph exec -it $(kubectl -n rook-ceph get pod -l 
"app=rook-ceph-tools" -o jsonpath='{.items[0].metadata.name}') ceph status
+LOG.info("How to get Cepth status from Kubernetes: " + "kubectl -n 
rook-ceph exec -it \$(kubectl -n rook-ceph get pod -l \"app=rook-ceph-tools\" 
-o jsonpath='{.items[0].metadata.name}') ceph status");
+sh.exec("kubectl -n rook-ceph exec -it \$(kubectl -n rook-ceph get pod -l 
\"app=rook-ceph-tools\" -o jsonpath='{.items[0].metadata.name}') ceph status");
+logError(sh);
+assertTrue("Failed ...", sh.getRet() == 0);
+
+String out = sh.getOut().toString();
+LOG.info("Output from command:\n" + out + "\n");
+assertTrue(out.contains("health:"));
+assertTrue(out.contains("HEALTH_OK"));
+  }
+}
+
diff --git a/bigtop-tests/smoke-tests/rook-ceph/build.gradle 
b/bigtop-tests/smoke-tests/rook-ceph/build.gradle
new file mode 100644
index 000..88510d4
--- /dev/null
+++ b/bigtop-tests/smoke-tests/rook-ceph/build.gradle
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+def tests_to_include() {
+  return ["TestRookCeph.groovy"];
+}
+
+sourceSets {
+  test {
+groovy {
+  srcDirs = ["./"]
+  exclude { FileTreeElement elem -> (doExclude(elem.getName())) }
+}
+  }
+}
+
+test.doFirst {
+}

[bigtop] branch cnb updated: BIGTOP-3241: Smoke test for Zookeeper on Kubernetes

[ywkim]

This is an automated email from the ASF dual-hosted git repository.

ywkim pushed a commit to branch cnb
in repository https://gitbox.apache.org/repos/asf/bigtop.git


The following commit(s) were added to refs/heads/cnb by this push:
 new e1cd59d  BIGTOP-3241: Smoke test for Zookeeper on Kubernetes
e1cd59d is described below

commit e1cd59de8619dc225d3054c26d7da40e23ceb0de
Author: Youngwoo Kim 
AuthorDate: Tue Oct 15 13:08:34 2019 +0900

BIGTOP-3241: Smoke test for Zookeeper on Kubernetes
---
 README.md  | 13 +
 bigtop-tests/smoke-tests/README| 31 +-
 .../smoke-tests/zookeeper/TestZookeeper.groovy |  9 ---
 3 files changed, 26 insertions(+), 27 deletions(-)

diff --git a/README.md b/README.md
index 77b24c6..53f07d8 100755
--- a/README.md
+++ b/README.md
@@ -129,6 +129,19 @@ $ kubectl exec -n $NS zookeeper-0 -- bin/zkServer.sh status
 ```
 Refer to https://github.com/helm/charts/tree/master/incubator/zookeeper for 
more configurations.
 
+Getting Started
+===
+
+Below are some recipes for getting started with using Apache Bigtop. As Apache 
Bigtop has different subprojects, these recipes will continue to evolve.
+For specific questions it's always a good idea to ping the mailing list at 
dev-subscr...@bigtop.apache.org to get some immediate feedback, or [open a 
JIRA](https://issues.apache.org/jira/browse/BIGTOP).
+
+For Users: Running the smoke tests
+---
+
+The simplest way to test bigtop is described in 
bigtop-tests/smoke-tests/README file
+
+For integration (API level) testing with maven, read on.
+
 # Cloud Native Bigtop
 This is the content for the talk given by jay vyas and sid mani @ apachecon 
2019 in Las Vegas,  you can watch it here  
https://www.youtube.com/watch?v=LUCE63q !
 
diff --git a/bigtop-tests/smoke-tests/README b/bigtop-tests/smoke-tests/README
index 777948f..e4c9ae2 100644
--- a/bigtop-tests/smoke-tests/README
+++ b/bigtop-tests/smoke-tests/README
@@ -1,9 +1,9 @@
-# This is the new smoke testing module for bigtop
+# This is the new smoke testing module for Bigtop
 
 ## Goals
 
 This gradle based test framework aims at making it extremely simple to
-customize and test Apache BigTop's hadoop distribution.
+customize and test Apache BigTop's distribution.
 
  - Easily add new tests to bigtop without adding new directories / xml files.
  - Debug tests easily and make logging extremely transparent.
@@ -17,27 +17,10 @@ So overall, this framework makes testing with apache bigtop 
easy as well as fun!
 
 You need to export several environment variables.   For example,
 ```
-export HADOOP_CONF_DIR=/etc/hadoop/conf/
-export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce/
-export HIVE_HOME=/usr/lib/hive/
-export PIG_HOME=/usr/lib/pig/
-export FLUME_HOME=/usr/lib/flume/
-export HIVE_CONF_DIR=/etc/hive/conf/
 export JAVA_HOME="/usr/lib/jvm/java-openjdk/"
-export MAHOUT_HOME="/usr/lib/mahout"
-export SPARK_HOME="/usr/lib/spark"
-export TAJO_HOME="/usr/lib/tajo"
-export OOZIE_TAR_HOME="/usr/lib/oozie/doc"
-```
 
-Fallback mode for shark-shell submission is 'yarn-client'. For a good reason,
-we aren't using YARN to provision cluster services. Hence, in order to execute
-the test against a standalone Spark cluster, like we deploy would require two
-more environment variables:
-`
-export SPARK_MASTER_IP=bigtop1.docker
-export SPARK_MASTER_PORT=7077
-`
+```
+If you are running smoke tests on Kubernetes, you need ```kubectl``` in your 
PATH variable.
 
 Then, simply invoke the tests you want to run, as described below.
 
@@ -46,17 +29,17 @@ Invoking test task of flume subproject will execute flume 
tests. To test hive as
 you would add the invocation of hive:test, and so on. To run the tests from 
the top-level
 Bigtop folder, follow the example below:
 ```
-./gradlew bigtop-tests:smoke-tests:flume:test 
bigtop-tests:smoke-tests:hive:test -Psmoke.tests --info
+./gradlew bigtop-tests:smoke-tests:zookeeper:test 
bigtop-tests:smoke-tests:rook-ceph:test -Psmoke.tests --info
 ```
 You also can run smoke tests from bigtop-tests/smoke-tests directory,
 or from any subproject of it. E.g.
 ```
 cd bigtop-tests/smoke-tests
-../../gradlew hive:test -Psmoke.tests
+../../gradlew zookeeper:test -Psmoke.tests
 ```
 or
 ```
-cd bigtop-tests/smoke-tests/hive
+cd bigtop-tests/smoke-tests/zookeeper
 ../../../gradlew test -Psmoke.tests
 ```
 Instead of using top-level Gradle wrapper one can also use system-wide
diff --git a/bigtop-tests/smoke-tests/zookeeper/TestZookeeper.groovy 
b/bigtop-tests/smoke-tests/zookeeper/TestZookeeper.groovy
index 28be70a..0c95241 100644
--- a/bigtop-tests/smoke-tests/zookeeper/TestZookeeper.groovy
+++ b/bigtop-tests/smoke-tests/zookeeper/TestZookeeper.groovy
@@ -31,6 +31,8 @@ class TestZookeeper {
   static private Log LOG = LogFactory.getLog(Object.class)
 
   sta

[bigtop] branch cnb updated: BIGTOP-3260: Add hostpath provisioner and NFS exports

[ywkim]

This is an automated email from the ASF dual-hosted git repository.

ywkim pushed a commit to branch cnb
in repository https://gitbox.apache.org/repos/asf/bigtop.git


The following commit(s) were added to refs/heads/cnb by this push:
 new 248b1cb  BIGTOP-3260: Add hostpath provisioner and NFS exports
248b1cb is described below

commit 248b1cbd7b6064200d9c1905137061b44187270e
Author: Youngwoo Kim 
AuthorDate: Tue Oct 22 15:32:57 2019 +0900

BIGTOP-3260: Add hostpath provisioner and NFS exports
---
 README.md  | 37 +++---
 kubectl/plugin/kubectl-bigtop  | 21 ---
 storage/hostpath/README.md | 16 +++
 storage/hostpath/hostpath-pvc.yaml | 11 ++
 storage/rook/nfs/nfs-ceph.yaml | 41 ++
 storage/rook/nfs/nfs-hostpath.yaml | 35 
 storage/rook/nfs/nfs.yaml  | 37 ++
 storage/rook/nfs/storageclass.yaml | 13 
 8 files changed, 205 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 79b9e3a..114029c 100755
--- a/README.md
+++ b/README.md
@@ -115,7 +115,7 @@ $ vagrant ssh k8s-1
 ```
 ```
 k8s-1$ kubectl plugin list
-k8s-1$ kubectl bigtop kubectl-config && kubectl bigtop helm-install
+k8s-1$ kubectl bigtop kubectl-config && kubectl bigtop helm-deploy
 
 ```
 
@@ -176,8 +176,39 @@ $ kubectl get storageclass
 rook-ceph-block
 ```
 
+Mark ```rook-ceph-block``` StorageClass as default:
+```
+kubectl patch storageclass rook-ceph-block -p '{"metadata": 
{"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
+
+```
+
+### NFS
+
+```
+$ kubectl create -f dl/rook-1.1.2/cluster/examples/kubernetes/nfs/operator.yaml
+$ kubectl -n rook-nfs-system get pod
+
+# NFS by the default storageClass
+$ kubectl create -f storage/rook/nfs/nfs.yaml
+
+# NFS by the Ceph RBD volumes
+$ kubectl create -f storage/rook/nfs/nfs-ceph.yaml
+
+# NFS by the ```hostpath``` storageClass
+$ kubectl create -f storage/rook/nfs/nfs-hostpath.yaml
+
+# storageClass by NFS export
+$ kubectl create -f storage/rook/nfs/storageclass.yaml
+```
+
+If you want to mark ```rook-nfs-share1``` storageClass as default:
+```
+$ kubectl patch storageclass rook-nfs-share1 -p '{"metadata": 
{"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
+$ kubectl get storageclass
+```
+
 ### Rook Minio
-Create Minio operator:
+Deploy Minio operator:
 ```
 $ kubectl create -f 
dl/rook-1.1.2/cluster/examples/kubernetes/minio/operator.yaml
 
@@ -265,7 +296,7 @@ $ kubectl -n bigtop exec kafka-client -- kafka-topics \
 ```
 
 ### Schema Registry
-Optionally, You can create schema registry service for Kafka:
+Optionally, You can deploy schema registry service for Kafka:
 ```
 helm install --name kafka-schema-registry --namespace bigtop -f 
kafka/schema-registry/values.yaml \
 --set 
kafkaStore.overrideBootstrapServers="kafka-0.kafka-headless:9092\,kafka-1.kafka-headless:9092\,kafka-2.kafka-headless:9092"
 \
diff --git a/kubectl/plugin/kubectl-bigtop b/kubectl/plugin/kubectl-bigtop
index 1291376..a59beb3 100755
--- a/kubectl/plugin/kubectl-bigtop
+++ b/kubectl/plugin/kubectl-bigtop
@@ -49,7 +49,7 @@ if [[ "$1" == "cluster-info" ]]; then
 fi
 
 # Install Helm
-if [[ "$1" == "helm-install" ]]; then
+if [[ "$1" == "helm-deploy" ]]; then
 curl -L https://git.io/get_helm.sh | bash
 helm version -c
 kubectl --namespace kube-system create serviceaccount tiller
@@ -63,12 +63,14 @@ if [[ "$1" == "helm-install" ]]; then
 fi
 
 # Install Rook-Ceph
-if [[ "$1" == "rook-ceph-install" ]]; then
+if [[ "$1" == "rook-ceph-deploy" ]]; then
 
 kubectl create -f 
$BIGTOP_HOME/dl/rook-1.1.2/cluster/examples/kubernetes/ceph/common.yaml
 kubectl create -f 
$BIGTOP_HOME/dl/rook-1.1.2/cluster/examples/kubernetes/ceph/operator.yaml
 kubectl -n rook-ceph get pod
 
+sleep 10s
+
 # Ceph cluster
 # test
 kubectl create -f $BIGTOP_HOME/storage/rook/ceph/cluster-test.yaml
@@ -82,6 +84,8 @@ if [[ "$1" == "rook-ceph-install" ]]; then
 
 # StorageClass: Ceph RBD
 kubectl create -f 
$BIGTOP_HOME/dl/rook-1.1.2/cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml
+# Make the storageClass as default
+kubectl patch storageclass rook-ceph-block -p '{"metadata": 
{"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
 kubectl get storageclass
 exit 0
 fi
@@ -92,8 +96,19 @@ if [[ "$1" == "rook-ceph-toolbox" ]]; then
 exit 0
 fi
 
+# Rook NFS
+if [[ "$1" == "rook-nfs-deploy" ]]; then
+kubectl create -f 
$BIGTOP_HOME/dl/rook-1.1.2/cluste

[bigtop] branch cnb updated: BIGTOP-3262: Use default storageClass (addendum)

[ywkim]

This is an automated email from the ASF dual-hosted git repository.

ywkim pushed a commit to branch cnb
in repository https://gitbox.apache.org/repos/asf/bigtop.git


The following commit(s) were added to refs/heads/cnb by this push:
 new 57bdb21  BIGTOP-3262: Use default storageClass (addendum)
57bdb21 is described below

commit 57bdb219390081e2f5ef2f45d1fed4418cb782dc
Author: Youngwoo Kim 
AuthorDate: Thu Oct 24 13:39:21 2019 +0900

BIGTOP-3262: Use default storageClass (addendum)
---
 storage/minio/values.yaml| 1 -
 storage/rook/minio/object-store.yaml | 1 -
 2 files changed, 2 deletions(-)

diff --git a/storage/minio/values.yaml b/storage/minio/values.yaml
index 272e64a..a463a66 100644
--- a/storage/minio/values.yaml
+++ b/storage/minio/values.yaml
@@ -84,7 +84,6 @@ persistence:
   ## If the PV uses a different storage class, specify that here.
   # storageClass: standard
   # VolumeName: ""
-  storageClass: "rook-ceph-block"
 
   accessMode: ReadWriteOnce
   size: 10Gi
diff --git a/storage/rook/minio/object-store.yaml 
b/storage/rook/minio/object-store.yaml
index e40cf24..3bc6710 100644
--- a/storage/rook/minio/object-store.yaml
+++ b/storage/rook/minio/object-store.yaml
@@ -32,7 +32,6 @@ spec:
 accessModes: [ "ReadWriteOnce" ]
 # Set the storage class that will be used, otherwise Kubernetes' 
default storage class will be used.
 #storageClassName: "my-storage-class"
-storageClassName: "rook-ceph-block"
 resources:
   requests:
 storage: "8Gi"

[bigtop] branch cnb updated: BIGTOP-3263: Log aggregation for Kubernetes cluster

[ywkim]

This is an automated email from the ASF dual-hosted git repository.

ywkim pushed a commit to branch cnb
in repository https://gitbox.apache.org/repos/asf/bigtop.git


The following commit(s) were added to refs/heads/cnb by this push:
 new d35e545  BIGTOP-3263: Log aggregation for Kubernetes cluster
d35e545 is described below

commit d35e545088c2003cfd4d7de3d378ec240c026106
Author: Youngwoo Kim 
AuthorDate: Thu Oct 24 17:21:05 2019 +0900

BIGTOP-3263: Log aggregation for Kubernetes cluster
---
 README.md  |   2 +
 kubectl/plugin/kubectl-bigtop  |  11 +++
 logs/loki/README.md|  22 +
 logs/loki/loki-values.yaml | 208 +
 logs/loki/promtail-values.yaml | 136 +++
 5 files changed, 379 insertions(+)

diff --git a/README.md b/README.md
index a865f6f..6b980a4 100755
--- a/README.md
+++ b/README.md
@@ -348,6 +348,8 @@ See spark/README.md
 
 See prometheus/README.md
 
+And also, You can aggregate logs from Kubernetes cluster (Pods) in using Loki 
stack (Loki and promtail). See logs/loki/README.md for deploying Liki and 
querying logs from Loki on Grafana.
+
 # Getting Started
 
 Below are some recipes for getting started with using Apache Bigtop. As Apache 
Bigtop has different subprojects, these recipes will continue to evolve.
diff --git a/kubectl/plugin/kubectl-bigtop b/kubectl/plugin/kubectl-bigtop
index e2c921e..3387d11 100755
--- a/kubectl/plugin/kubectl-bigtop
+++ b/kubectl/plugin/kubectl-bigtop
@@ -123,3 +123,14 @@ if [[ "$1" == "minio-deploy" ]]; then
 exit 0
 fi
 
+# Loki
+if [[ "$1" == "loki-deploy" ]]; then
+helm repo add loki https://grafana.github.io/loki/charts
+helm repo update
+helm upgrade --install --namespace default loki -f 
logs/loki/loki-values.yaml loki/loki
+sleep 10s
+helm upgrade --install --namespace default promtail -f 
logs/loki/proimtail-values.yaml loki/promtail --set "loki.serviceName=loki"
+# http url for loki server: http://loki:3100
+helm list && kubectl get po -A
+exit 0
+fi
diff --git a/logs/loki/README.md b/logs/loki/README.md
new file mode 100644
index 000..e17933b
--- /dev/null
+++ b/logs/loki/README.md
@@ -0,0 +1,22 @@
+# Log aggregation for Kubernetes cluster
+
+Deploy Loki stack:
+```
+$ helm repo add loki https://grafana.github.io/loki/charts
+$ helm repo update
+$ helm upgrade --install --namespace default loki -f 
logs/loki/loki-values.yaml loki/loki
+$ helm upgrade --install --namespace default promtail -f 
logs/loki/promtail-values.yaml loki/promtail --set "loki.serviceName=loki"
+```
+
+## Loki in Grafana
+You should add the Loki data source to Grafana configuration.
+
+1. In Grafana, go to Configuration > Data Sources
+2. Go to ```Add data source```
+3. Choose Loki --> The http URL field ```http://loki:3100```
+
+## Querying logs from Loki on Grafana
+
+1. Explore > Choose 'Loki' data source
+2. Enter a Liki query. E.g., ```{app="zookeeper",namespace="bigtop"}```
+3. You can see the log streams from Zookeeper Pods on Kubernetes cluster 
diff --git a/logs/loki/loki-values.yaml b/logs/loki/loki-values.yaml
new file mode 100644
index 000..623a11f
--- /dev/null
+++ b/logs/loki/loki-values.yaml
@@ -0,0 +1,208 @@
+## Affinity for pod assignment
+## ref: 
https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
+affinity: {}
+# podAntiAffinity:
+#   requiredDuringSchedulingIgnoredDuringExecution:
+#   - labelSelector:
+#   matchExpressions:
+#   - key: app
+# operator: In
+# values:
+# - loki
+# topologyKey: "kubernetes.io/hostname"
+
+## StatefulSet annotations
+annotations: {}
+
+# enable tracing for debug, need install jaeger and specify right 
jaeger_agent_host
+tracing:
+  jaegerAgentHost:
+
+config:
+  auth_enabled: false
+  ingester:
+chunk_idle_period: 3m
+chunk_block_size: 262144
+chunk_retain_period: 1m
+lifecycler:
+  ring:
+kvstore:
+  store: inmemory
+replication_factor: 1
+
+  ## Different ring configs can be used. E.g. Consul
+  # ring:
+  #   store: consul
+  #   replication_factor: 1
+  #   consul:
+  # host: "consul:8500"
+  # prefix: ""
+  # httpclienttimeout: "20s"
+  # consistentreads: true
+  limits_config:
+enforce_metric_name: false
+reject_old_samples: true
+reject_old_samples_max_age: 168h
+  schema_config:
+configs:
+- from: 2018-04-15
+  store: boltdb
+  object_store: filesystem
+  schema: v9
+  index:
+prefix: index_
+period: 168h
+  server:
+http_listen_port: 3100
+  storage_config:
+boltdb:
+  directory: /data/loki/index
+filesystem:
+  directory: /data/loki/chunks
+  chunk_store_config:
+max_look_back_period

[bigtop] branch cnb updated: BIGTOP-3250: Upgrade Presto docker image

[ywkim]

This is an automated email from the ASF dual-hosted git repository.

ywkim pushed a commit to branch cnb
in repository https://gitbox.apache.org/repos/asf/bigtop.git


The following commit(s) were added to refs/heads/cnb by this push:
 new 9a40032  BIGTOP-3250: Upgrade Presto docker image
9a40032 is described below

commit 9a40032bb65443db6e0e696f67802386d75b8c1c
Author: Youngwoo Kim 
AuthorDate: Wed Oct 16 17:04:29 2019 +0900

BIGTOP-3250: Upgrade Presto docker image
---
 presto3-minio/coordinator/Dockerfile  |  6 --
 presto3-minio/coordinator/README.md   |  8 
 presto3-minio/coordinator/config.properties   |  8 
 presto3-minio/coordinator/json|  2 --
 presto3-minio/test.sh | 19 ---
 presto3-minio/worker/Dockerfile   |  5 -
 presto3-minio/worker/README.md|  6 --
 presto3-minio/worker/json |  2 --
 {presto3-minio => prestosql}/README.md|  0
 prestosql/docker/Makefile |  8 
 prestosql/docker/README.md|  6 ++
 prestosql/docker/coordinator/Dockerfile   | 14 ++
 prestosql/docker/coordinator/README.md|  7 +++
 prestosql/docker/coordinator/config.properties|  5 +
 prestosql/docker/coordinator/jvm.config   | 12 
 .../docker/coordinator}/minio.properties  |  0
 prestosql/docker/coordinator/node.properties  |  3 +++
 prestosql/docker/worker/Dockerfile| 12 
 prestosql/docker/worker/README.md |  6 ++
 .../docker}/worker/config.properties  |  3 ---
 prestosql/docker/worker/jvm.config| 12 
 .../docker/worker}/minio.properties   |  0
 prestosql/docker/worker/node.properties   |  3 +++
 .../presto-deployment-master.yaml |  2 +-
 .../presto-deployment-worker.yaml |  4 ++--
 {presto3-minio => prestosql}/presto-deployment.yaml   |  2 +-
 {presto3-minio => prestosql}/service.yaml |  0
 27 files changed, 92 insertions(+), 63 deletions(-)

diff --git a/presto3-minio/coordinator/Dockerfile 
b/presto3-minio/coordinator/Dockerfile
deleted file mode 100755
index ac9c71b..000
--- a/presto3-minio/coordinator/Dockerfile
+++ /dev/null
@@ -1,6 +0,0 @@
-FROM wlan0/presto-benchmark-environment:coordinator 
-COPY ./json /resources/nyc_taxi_data_3M.json
-COPY ./minio.properties 
/resources/presto-server-302-e.11/etc/catalog/minio.properties
-
-COPY config.properties /resources/presto-server-302-e.11/etc/config.properties 
-CMD cat   /resources/presto-server-302-e.11/etc/config.properties ;  
/resources/presto-server-302-e.11/bin/launcher start && cat 
/resources/presto-server-302-e.11/etc/config.properties && echo "Hardcoded to 
s3://minio:9000 , using 'minio' and 'minio123' as the access credentials..." && 
sleep 60 && cat /resources/presto-server-302-e.11/etc/config.properties &&  
tail -f ./resources/presto-server-302-e.11/var/log/server.log
diff --git a/presto3-minio/coordinator/README.md 
b/presto3-minio/coordinator/README.md
deleted file mode 100755
index 3423f65..000
--- a/presto3-minio/coordinator/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-# Use squash !
-
-Building this image, make sure you use squash so you can gut the 12 GB file :) 
-
-
-docker build --squash -t jayunit100/wlan0-presto:latest ./
-
-
diff --git a/presto3-minio/coordinator/config.properties 
b/presto3-minio/coordinator/config.properties
deleted file mode 100755
index 6a214ae..000
--- a/presto3-minio/coordinator/config.properties
+++ /dev/null
@@ -1,8 +0,0 @@
-coordinator=true
-node-scheduler.include-coordinator=true
-http-server.http.port=8080
-query.max-memory=1GB
-query.max-memory-per-node=1GB
-query.max-total-memory-per-node=1GB
-discovery-server.enabled=true
-discovery.uri=http://presto:8080
diff --git a/presto3-minio/coordinator/json b/presto3-minio/coordinator/json
deleted file mode 100755
index 6906dcd..000
--- a/presto3-minio/coordinator/json
+++ /dev/null
@@ -1,2 +0,0 @@
-{"id":"3418178","vendor_id":"1","pickup_datetime":"2014-03-13 
20:17:16","dropoff_datetime":"2014-03-13 
20:22:29","store_and_fwd_flag":"N","rate_code_id":"1","pickup_longitude":"-73.9873046875","pickup_latitude":"40.691890716552734","dropoff_longitude":"-73.975677490234375","dropoff_latitude":"40.696006774902344","passenger_count":"1","trip_distance":"0.90","fare_amount":"5.5",

[bigtop] branch cnb updated: BIGTOP-3254: Prometheus for gathering metrics

[ywkim]

This is an automated email from the ASF dual-hosted git repository.

ywkim pushed a commit to branch cnb
in repository https://gitbox.apache.org/repos/asf/bigtop.git


The following commit(s) were added to refs/heads/cnb by this push:
 new 7d8c270  BIGTOP-3254: Prometheus for gathering metrics
7d8c270 is described below

commit 7d8c2702f1f1a99f189cf2b6ac6128f2eaa89e3f
Author: Youngwoo Kim 
AuthorDate: Wed Oct 23 13:24:12 2019 +0900

BIGTOP-3254: Prometheus for gathering metrics
---
 README.md  |4 +
 prometheus/README.md   |   36 +
 prometheus/values.yaml | 1808 
 3 files changed, 1848 insertions(+)

diff --git a/README.md b/README.md
index 114029c..b89e7e4 100755
--- a/README.md
+++ b/README.md
@@ -308,6 +308,10 @@ incubator/schema-registry
 
 See spark/README.md
 
+## Monitoring
+
+See prometheus/README.md
+
 # Getting Started
 
 Below are some recipes for getting started with using Apache Bigtop. As Apache 
Bigtop has different subprojects, these recipes will continue to evolve.
diff --git a/prometheus/README.md b/prometheus/README.md
new file mode 100644
index 000..6dd5b83
--- /dev/null
+++ b/prometheus/README.md
@@ -0,0 +1,36 @@
+# Prometheus Operator
+
+https://github.com/helm/charts/tree/master/stable/prometheus-operator
+
+
+Prerequisites:
+- Kubernetes
+- Helm
+
+Deploy Prometheus operator:
+```
+$ cd $BIGTOP_HOME
+$ helm install --namespace default --name prometheus -f prometheus/values.yaml 
stable/prometheus-operator
+ 
+```
+
+Port forward for Grafana:
+```
+$ kubectl port-forward $(kubectl get pods --selector=app=grafana 
--output=jsonpath="{.items..metadata.name}") 3000
+```
+
+Get Grafana usernamd and password:
+```
+$ kubectl get secrets --all-namespaces | grep grafana | grep Opaque
+$ kubectl get secret --namespace default  -o 
jsonpath="{.data.admin-password}" | base64 --decode ; echo
+```
+
+Browse http://localhost:3000 for Grafana.
+
+Port forward for Prometheus:
+```
+$ kubectl port-forward --namespace default  9090
+
+```
+
+Browse http://localhost:9090 for Prometheus web UI.
diff --git a/prometheus/values.yaml b/prometheus/values.yaml
new file mode 100644
index 000..a5cf728
--- /dev/null
+++ b/prometheus/values.yaml
@@ -0,0 +1,1808 @@
+# Default values for prometheus-operator.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+## Provide a name in place of prometheus-operator for `app:` labels
+##
+nameOverride: ""
+
+## Provide a name to substitute for the full names of resources
+##
+fullnameOverride: ""
+
+## Labels to apply to all resources
+##
+commonLabels: {}
+# scmhash: abc123
+# myLabel: aakkmd
+
+## Create default rules for monitoring the cluster
+##
+defaultRules:
+  create: true
+  rules:
+alertmanager: true
+etcd: true
+general: true
+k8s: true
+kubeApiserver: true
+kubePrometheusNodeAlerting: true
+kubePrometheusNodeRecording: true
+kubernetesAbsent: true
+kubernetesApps: true
+kubernetesResources: true
+kubernetesStorage: true
+kubernetesSystem: true
+kubeScheduler: true
+network: true
+node: true
+prometheus: true
+prometheusOperator: true
+time: true
+
+  ## Labels for default rules
+  labels: {}
+  ## Annotations for default rules
+  annotations: {}
+
+## Provide custom recording or alerting rules to be deployed into the cluster.
+##
+additionalPrometheusRules: []
+#  - name: my-rule-file
+#groups:
+#  - name: my_group
+#rules:
+#- record: my_record
+#  expr: 100 * my_record
+
+##
+global:
+  rbac:
+create: true
+pspEnabled: true
+
+  ## Reference to one or more secrets to be used when pulling images
+  ## ref: 
https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+  ##
+  imagePullSecrets: []
+  # - name: "image-pull-secret"
+
+## Configuration for alertmanager
+## ref: https://prometheus.io/docs/alerting/alertmanager/
+##
+alertmanager:
+
+  ## Deploy alertmanager
+  ##
+  enabled: true
+
+  ## Service account for Alertmanager to use.
+  ## ref: 
https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/
+  ##
+  serviceAccount:
+create: true
+name: ""
+
+  ## Configure pod disruption budgets for Alertmanager
+  ## ref: 
https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget
+  ## This configuration is immutable once created and will require the PDB to 
be deleted to be changed
+  ## https://github.com/kubernetes/kubernetes/issues/45398
+  ##
+  podDisruptionBudget:
+enabled: false
+minAvailable: 1
+maxUnavailable: ""
+
+  ## Alertmanager configuration directives
+  ## ref: https://prometheus.io/docs/alerting/configuration/#configuration-file
+  ##  https://prometheus.io/webtools/alerting/routing-tree-editor/
+  ##
+  config:
+   

[bigtop] branch cnb updated: BIGTOP-3262: Use default storageClass

[ywkim]

This is an automated email from the ASF dual-hosted git repository.

ywkim pushed a commit to branch cnb
in repository https://gitbox.apache.org/repos/asf/bigtop.git


The following commit(s) were added to refs/heads/cnb by this push:
 new 2c15c08  BIGTOP-3262: Use default storageClass
2c15c08 is described below

commit 2c15c082986f4380bcaab5238aa85d66906d27ee
Author: Youngwoo Kim 
AuthorDate: Thu Oct 24 12:05:53 2019 +0900

BIGTOP-3262: Use default storageClass
---
 README.md | 42 +++---
 README_STORAGE.md | 14 --
 kafka/values.yaml |  1 -
 kubectl/plugin/kubectl-bigtop | 10 ++
 prometheus/values.yaml| 14 +++---
 zookeeper/values.yaml |  2 +-
 6 files changed, 57 insertions(+), 26 deletions(-)

diff --git a/README.md b/README.md
index b89e7e4..a865f6f 100755
--- a/README.md
+++ b/README.md
@@ -96,11 +96,22 @@ For example, the stable helm charts don't properly 
configure zepplin, allow for
 
 # Immediately Get Started with Deployment and Smoke Testing of Cloud Native 
BigTop
 
+Minikube is the easiest tool to run a single-node Kubernetes cluster.
+
+```
+$ cd $BIGTOP_HOME
+$ minikube start --cpus 8 --memory 8196 --container-runtime=cri-o 
+$ kubectl cluster-info
+
+```
+
+## Set up 3-Node Kubernetes cluster via Kubespray on local machine
+
 Prerequisites:
 - Vagrant
 - Java
 
-## Set up 3-Node Kubernetes cluster via Kubespray on local machine
+If you want a multi-node cluster on local machine, you can create the cluster 
using Kubespray:
 ```
 $ cd $BIGTOP_HOME
 $ ./gradlew kubespray-clean kubespray-download && cd dl/ && tar xvfz 
kubespray-2.11.0.tar.gz
@@ -120,6 +131,33 @@ k8s-1$ kubectl bigtop kubectl-config && kubectl bigtop 
helm-deploy
 ```
 
 ## Storage
+
+The easiest way to get simple persistent volumes with dynamic volume 
provisiong on a one node cluster:
+```
+$ helm repo add rimusz https://charts.rimusz.net
+$ helm repo update
+$ helm upgrade --install hostpath-provisioner \
+--namespace kube-system \
+--set storageClass.defaultClass=true \
+rimusz/hostpath-provisioner
+
+$ kubectl get storageclass
+```
+
+Mark ```hostpath``` StorageClass as default:
+```
+$ kubectl patch storageclass hostpath -p '{"metadata": 
{"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
+$ kubectl get storageclass
+```
+
+On Minikube, there is 'standard' storage class as default storage class. you 
can make 'hostpath' storage class as default:
+```
+$ kubectl patch storageclass standard -p '{"metadata": 
{"annotations":{"storageclass.kubernetes.io/is-default-class":"false"}}}'
+$ kubectl get storageclass
+```
+
+### Rook
+
 You need to install ```lvm2``` package for Rook-Ceph:
 ```
 # Centos
@@ -130,8 +168,6 @@ sudo apt-get install -y lvm2
 ```
 Refer to https://rook.io/docs/rook/v1.1/k8s-pre-reqs.html for prerequisites on 
Rook
 
-### Rook Ceph
-
 Run ```download``` task to get Rook binary:
 ```
 $ ./gradlew rook-clean rook-download && cd dl/ && tar xvfz rook-1.1.2.tar.gz
diff --git a/README_STORAGE.md b/README_STORAGE.md
deleted file mode 100755
index 431261a..000
--- a/README_STORAGE.md
+++ /dev/null
@@ -1,14 +0,0 @@
-This inlcudes the various storage recipes curated for
-use in a bigdata distro that would run on a cloud native platform.
-
-
-
-- Minio: Global object store to support spark/kafka/etc
-  You can install it from the yamls in this repo, or else,
-  `helm install --name minio stable/minio --namespace=bigdata` directly.
-
-- Hbase: For use by tools like PredictionIO.
-  For installation,
-   - git clone https://github.com/warp-poke/hbase-helm
-   - cd to hbase-helm
-   - modify configmap to use nifi-zookeeper as the zk.quorum field.
diff --git a/kafka/values.yaml b/kafka/values.yaml
index bd4b789..4d2e3e6 100644
--- a/kafka/values.yaml
+++ b/kafka/values.yaml
@@ -231,7 +231,6 @@ persistence:
   ##   GKE, AWS & OpenStack)
   ##
   # storageClass:
-  storageClass: "rook-ceph-block"
 
 jmx:
   ## Rules to apply to the Prometheus JMX Exporter.  Note while lots of stats 
have been cleaned and exposed,
diff --git a/kubectl/plugin/kubectl-bigtop b/kubectl/plugin/kubectl-bigtop
index a59beb3..e2c921e 100755
--- a/kubectl/plugin/kubectl-bigtop
+++ b/kubectl/plugin/kubectl-bigtop
@@ -62,6 +62,16 @@ if [[ "$1" == "helm-deploy" ]]; then
 exit 0
 fi
 
+# Hostpah storageClass
+if [[ "$1" == "hostpath-deploy" ]]; then
+helm repo add rimusz https://charts.rimusz.net
+helm repo update
+helm upgrade --install hostpath-provisioner --namespace kube-system --set 
storageClass.defaultClass=true rimusz/hostpath-provisioner
+sleep 5s ; kubectl get storageclass
+
+exit 0
+fi
+
 # Install Rook-Ceph
 if [[ "$1" == "rook-ceph-deploy" ]]; then
 
diff --git

[bigtop] branch cnb updated: BIGTOP-3256: Better initial setup via vagrant and kubectl plugin

[ywkim]

This is an automated email from the ASF dual-hosted git repository.

ywkim pushed a commit to branch cnb
in repository https://gitbox.apache.org/repos/asf/bigtop.git


The following commit(s) were added to refs/heads/cnb by this push:
 new ff4ae5b  BIGTOP-3256: Better initial setup via vagrant and kubectl 
plugin
ff4ae5b is described below

commit ff4ae5b0f2a3033503c6140b704bd0822c56ef12
Author: Youngwoo Kim 
AuthorDate: Thu Oct 17 13:35:18 2019 +0900

BIGTOP-3256: Better initial setup via vagrant and kubectl plugin
---
 README.md| 13 ++---
 {kubernetes/kubectl => kubectl}/plugin/README.md |  0
 kubectl/plugin/kubectl-bigtop| 63 
 kubernetes/kubectl/plugin/kubectl-bigtop | 17 ---
 kubespray/vagrant/Vagrantfile|  5 +-
 5 files changed, 71 insertions(+), 27 deletions(-)

diff --git a/README.md b/README.md
index abc870f..53c288b 100755
--- a/README.md
+++ b/README.md
@@ -112,16 +112,11 @@ $ vagrant up
 ```
 $ vagrant ssh k8s-1
 
-k8s-1$ mkdir -p ~/.kube
-k8s-1$ sudo cp /etc/kubernetes/admin.conf .kube/config
-k8s-1$ sudo chown -R vagrant:vagrant .kube
-k8s-1$ kubectl cluster-info
-
-Kubernetes master is running at https://172.17.8.101:6443
-coredns is running at 
https://172.17.8.101:6443/api/v1/namespaces/kube-system/services/coredns:dns/proxy
-kubernetes-dashboard is running at 
https://172.17.8.101:6443/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy
+```
+```
+k8s-1$ kubectl plugin list
+k8s-1$ kubectl bigtop kubectl-config && kubectl bigtop helm-install
 
-To further debug and diagnose cluster problems, use 'kubectl cluster-info 
dump'.
 ```
 
 ## Storage
diff --git a/kubernetes/kubectl/plugin/README.md b/kubectl/plugin/README.md
similarity index 100%
rename from kubernetes/kubectl/plugin/README.md
rename to kubectl/plugin/README.md
diff --git a/kubectl/plugin/kubectl-bigtop b/kubectl/plugin/kubectl-bigtop
new file mode 100755
index 000..89abf3c
--- /dev/null
+++ b/kubectl/plugin/kubectl-bigtop
@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+
+set -ex
+
+echo "Kubectl plugin for Apache Bigtop"
+
+BIGTOP_HOME=${BIGTOP_HOME:-/bigtop}
+
+
+detect_java() {
+  # detect JAVA_HOME
+  source 
$BIGTOP_HOME/bigtop-packages/src/common/bigtop-utils/bigtop-detect-javahome
+
+  if [ -x "$JAVA_HOME/bin/java" ]; then
+  JAVA="$JAVA_HOME/bin/java"
+  else
+  JAVA=`which java`
+  fi
+
+  if [ -z "$JAVA" ]; then
+  echo "could not find java; set JAVA_HOME or ensure java is in PATH"
+  exit 1
+  fi
+}
+
+# version of Bigtop plugin
+if [[ "$1" == "version" ]]; then
+# FIXME
+echo "9.9.9"
+exit 0
+fi
+
+# Configure kubectl
+if [[ "$1" == "kubectl-config" ]]; then
+# FIXME
+if [[ ! -f "$HOME/.kube/config" ]]; then
+echo "configuring kubectl..."
+sudo mkdir -p ~/.kube
+sudo cp /etc/kubernetes/admin.conf ~/.kube/config
+sudo chown -R $USER:$USER ~/.kube
+kubectl config view
+fi
+fi
+
+# Cluster info
+if [[ "$1" == "cluster-info" ]]; then
+kubectl cluster-info
+exit 0
+fi
+
+# Install Helm
+if [[ "$1" == "helm-install" ]]; then
+curl -L https://git.io/get_helm.sh | bash
+helm version -c
+kubectl --namespace kube-system create serviceaccount tiller
+kubectl create clusterrolebinding tiller-cluster-rule 
--clusterrole=cluster-admin --serviceaccount=kube-system:tiller
+helm init --history-max 200
+sleep 10s
+kubectl --namespace kube-system patch deploy tiller-deploy -p 
'{"spec":{"template":{"spec":{"serviceAccount":"tiller"'
+sleep 10s
+helm version
+exit 0
+fi
diff --git a/kubernetes/kubectl/plugin/kubectl-bigtop 
b/kubernetes/kubectl/plugin/kubectl-bigtop
deleted file mode 100755
index 6dd2040..000
--- a/kubernetes/kubectl/plugin/kubectl-bigtop
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-
-echo "Kubectl plugin for Apache Bigtop"
-
-# version of Bigtop plugin
-if [[ "$1" == "version" ]]
-then
-echo "2.0.0"
-exit 0
-fi
-
-if [[ "$1" == "cluster-info" ]]
-then
-kubectl cluster-info
-exit 0
-fi
-
diff --git a/kubespray/vagrant/Vagrantfile b/kubespray/vagrant/Vagrantfile
index 9702059..99131d5 100644
--- a/kubespray/vagrant/Vagrantfile
+++ b/kubespray/vagrant/Vagrantfile
@@ -178,6 +178,9 @@ Vagrant.configure("2") do |config|
   # Disable swap for each vm
   node.vm.provision "shell", inline: "swapoff -a"
 
+  node.vm.provision "shell", inline: "yum install -y lvm2"
+  node.vm.provision "shell", inline: "sudo cp 
/bigtop/kubectl/plugin/kubectl-bigtop /usr/local/bin/

[bigtop] branch master updated: BIGTOP-3261 Fix broken BPS Dockerfile

[ywkim]

This is an automated email from the ASF dual-hosted git repository.

ywkim pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git


The following commit(s) were added to refs/heads/master by this push:
 new 30d3e08  BIGTOP-3261 Fix broken BPS Dockerfile
30d3e08 is described below

commit 30d3e0839a660084793c910cace6a61c98934845
Author: Ben 
AuthorDate: Wed May 29 20:09:23 2019 -0400

BIGTOP-3261 Fix broken BPS Dockerfile

Update dockerfile to remove dependency on gradle and use only docker for 
the build.

Signed-off-by: Youngwoo Kim 
---
 .../bigpetstore-transaction-queue/.dockerignore|  3 +++
 .../bigpetstore-transaction-queue/Dockerfile   | 26 +-
 .../bigpetstore-transaction-queue/README.md|  5 ++---
 .../bigpetstore-transaction-queue/settings.gradle  | 16 +
 4 files changed, 32 insertions(+), 18 deletions(-)

diff --git a/bigtop-bigpetstore/bigpetstore-transaction-queue/.dockerignore 
b/bigtop-bigpetstore/bigpetstore-transaction-queue/.dockerignore
new file mode 100644
index 000..eea7a5a
--- /dev/null
+++ b/bigtop-bigpetstore/bigpetstore-transaction-queue/.dockerignore
@@ -0,0 +1,3 @@
+.gradle
+.settings
+build
\ No newline at end of file
diff --git a/bigtop-bigpetstore/bigpetstore-transaction-queue/Dockerfile 
b/bigtop-bigpetstore/bigpetstore-transaction-queue/Dockerfile
index 3f43aa6..e173f56 100644
--- a/bigtop-bigpetstore/bigpetstore-transaction-queue/Dockerfile
+++ b/bigtop-bigpetstore/bigpetstore-transaction-queue/Dockerfile
@@ -12,22 +12,18 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-FROM centos:7
-MAINTAINER j...@apache.org
-RUN yum update -y
-RUN yum install -y java-1.8.0-openjdk unzip wget
-RUN yum install -y java-1.8.0-openjdk-devel
+FROM gradle:5.6.3-jdk12 as build
+
+ADD . /bigpetstore-transaction-queue
+WORKDIR /bigpetstore-transaction-queue
+
+RUN [ "gradle", "fatJar" ]
+
+FROM openjdk:14-ea-12-jdk-alpine3.10
+LABEL maintainer="j...@apache.org"
 
 WORKDIR /opt/
 
-# Get Bigtop
-# This comes with a gradlew wrapper we can use.
-RUN wget 
http://www.apache.org/dist/bigtop/bigtop-1.0.0/bigtop-1.0.0-project.tar.gz
-RUN  tar -xvf bigtop-1.0.0-project.tar.gz
+COPY --from=build 
/bigpetstore-transaction-queue/build/libs/bigpetstore-transaction-queue-all-1.0.jar
 /opt
 
-# Install bigpetstore transaction queue
-WORKDIR /opt/bigtop-1.0.0/bigtop-bigpetstore/bigpetstore-transaction-queue
-RUN /opt/bigtop-1.0.0/gradlew distZip
-RUN unzip build/distributions/bigpetstore-transaction-queue-1.0.zip
-RUN mv ./bigpetstore-transaction-queue-1.0 
/opt/bigpetstore-transaction-queue-1.0/
-CMD 
/opt/bigpetstore-transaction-queue-1.0/bigpetstore-transaction-queue-1.0/bin/bigpetstore-transaction-queue
+ENTRYPOINT ["java", "-jar", "bigpetstore-transaction-queue-all-1.0.jar"]
diff --git a/bigtop-bigpetstore/bigpetstore-transaction-queue/README.md 
b/bigtop-bigpetstore/bigpetstore-transaction-queue/README.md
index 7782e1c..e60d106 100644
--- a/bigtop-bigpetstore/bigpetstore-transaction-queue/README.md
+++ b/bigtop-bigpetstore/bigpetstore-transaction-queue/README.md
@@ -73,7 +73,7 @@ The command line arguments are:
 From docker
 
 ```
-docker run -t -i jayunit100/bigpetstore-load-generator  
/opt/bigpetstore-transaction-queue-1.0/bin/bigpetstore-transaction-queue /tmp/ 
1 1 3.0
+docker run -t -i jayunit100/bigpetstore-load-generator /tmp/ 1 1 3.0
 ```
 or Java
 ```
@@ -87,8 +87,7 @@ OR Replace the file path with a REST API root (it will 
jsonify the transactions,
 From docker
 
 ```
-docker run -t -i jayunit100/bigpetstore-load-generator 
/opt/bigpetstore-transaction-queue-1.0/bin/bigpetstore-transaction-queue
- http://localhost:3000/restapi/rpush/ 1 5 1 123
+docker run -t -i jayunit100/bigpetstore-load-generator 
http://localhost:3000/restapi/rpush/ 1 5 1 123
 ```
 Or from java
 ```
diff --git a/bigtop-bigpetstore/bigpetstore-transaction-queue/settings.gradle 
b/bigtop-bigpetstore/bigpetstore-transaction-queue/settings.gradle
new file mode 100644
index 000..ed284e5
--- /dev/null
+++ b/bigtop-bigpetstore/bigpetstore-transaction-queue/settings.gradle
@@ -0,0 +1,16 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BA

[bigtop] branch cnb created (now 8aca335)

[ywkim]

This is an automated email from the ASF dual-hosted git repository.

ywkim pushed a change to branch cnb
in repository https://gitbox.apache.org/repos/asf/bigtop.git.


  at 8aca335  BIGTOP-3168: Kafka smoke tests implementation (#537)

No new revisions were added by this update.

[bigtop] branch cnb updated: BIGTOP-3238: Local muti-node cluster via vagrant and kubespray

[ywkim]

This is an automated email from the ASF dual-hosted git repository.

ywkim pushed a commit to branch cnb
in repository https://gitbox.apache.org/repos/asf/bigtop.git


The following commit(s) were added to refs/heads/cnb by this push:
 new dccbc42  BIGTOP-3238: Local muti-node cluster via vagrant and kubespray
dccbc42 is described below

commit dccbc42270058fe29af5223bf8e6b642320be469
Author: Youngwoo Kim 
AuthorDate: Mon Oct 14 10:29:58 2019 +0900

BIGTOP-3238: Local muti-node cluster via vagrant and kubespray
---
 README.md |  38 +++
 bigtop.bom|  12 +++
 build.gradle  |   2 +-
 kubespray/vagrant/Vagrantfile | 227 ++
 4 files changed, 278 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 5bec287..0db5be6 100755
--- a/README.md
+++ b/README.md
@@ -1,3 +1,41 @@
+[![Travis 
CI](https://img.shields.io/travis/apache/bigtop.svg?branch=master)](https://travis-ci.org/apache/bigtop)
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
+[Apache Bigtop](http://bigtop.apache.org/)
+==
+
+TBD
+
+# Get Started with Deployment and Smoke Testing of Cloud Native BigTop
+
+Prerequisites:
+- Vagrant
+- Java 
+
+## Set up 3-Node Kubernetes cluster via Kubespray on local machine
+```
+$ cd $BIGTOP_HOME
+$ ./gradlew kubespray-clean kubespray-download && cd dl/ && tar xvfz 
kubespray-2.11.0.tar.gz
+$ cd dl/kubespray-2.11.0/ && cp ../../kubespray/vagrant/Vagrantfile .
+$ vagrant up
+```
+
+# Cloud Native Bigtop
 This is the content for the talk given by jay vyas and sid mani @ apachecon 
2019 in Las Vegas,  you can watch it here  
https://www.youtube.com/watch?v=LUCE63q !
 
 # TLDR, heres how you create an analytics distro on K8s...
diff --git a/bigtop.bom b/bigtop.bom
index bafca95..99ab5c8 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -122,6 +122,18 @@ bigtop {
   ]
 
   components {
+'kubespray' {
+  name= "kubespray"
+  pkg = "kubespray"
+  relNotes = ""
+  website = "https://github.com/kubernetes-sigs/kubespray;
+  version { base = '2.11.0'; pkg = base; release = 1 }
+  tarball { destination = "$name-${version.base}.tar.gz"
+source  = "v${version.base}.tar.gz" }
+  url { site = "https://github.com/kubernetes-sigs/kubespray/archive/;
+archive = site }
+}
+
 'zookeeper' {
   name= 'zookeeper'
   pkg = name
diff --git a/build.gradle b/build.gradle
index 8dc9095..a9065fd 100644
--- a/build.gradle
+++ b/build.gradle
@@ -23,7 +23,7 @@ buildscript {
 }
 
 plugins {
-  id "de.undercouch.download" version "3.2.0"
+  id "de.undercouch.download" version "4.0.0"
   id "org.nosphere.apache.rat" version "0.2.0"
 }
 
diff --git a/kubespray/vagrant/Vagrantfile b/kubespray/vagrant/Vagrantfile
new file mode 100644
index 000..9702059
--- /dev/null
+++ b/kubespray/vagrant/Vagrantfile
@@ -0,0 +1,227 @@
+# -*- mode: ruby -*-
+# # vi: set ft=ruby :
+
+# For help on using kubespray with vagrant, check out docs/vagrant.md
+
+require 'fileutils'
+
+Vagrant.require_version ">= 2.0.0"
+
+CONFIG = File.join(File.dirname(__FILE__), "vagrant/config.rb")
+
+COREOS_URL_TEMPLATE = 
"https://storage.googleapis.com/%s.release.core-os.net/amd64-usr/current/coreos_production_vagrant.json;
+
+# Uniq disk UUID for libvirt
+DISK_UUID = Time.now.utc.to_i
+
+SUPPORTED_OS = {
+  "coreos-stable"   => {box: "coreos-stable",  user: "core", box_url: 
COREOS_URL_TEMPLATE % ["stable"]},
+  "coreos-alpha"=> {box: "coreos-alpha",   user: "core", box_url: 
COREOS_URL_TEMPLATE % ["alpha"]},
+  "coreos-beta" => {box: "coreos-beta",user: "core", box_url: 
COREOS_URL_TEMPLATE % ["beta"]},
+  "ubuntu1604"  => {box: "generic/ubuntu1604", user: "vagrant"},
+  "ubuntu1804"  => {box: &

[bigtop] branch cnb updated: BIGTOP-3230: Import the groundwork for CNB

[ywkim]

This is an automated email from the ASF dual-hosted git repository.

ywkim pushed a commit to branch cnb
in repository https://gitbox.apache.org/repos/asf/bigtop.git


The following commit(s) were added to refs/heads/cnb by this push:
 new e76ae87  BIGTOP-3230: Import the groundwork for CNB
e76ae87 is described below

commit e76ae872de0c51f6ed405d34dcff7db1a7411339
Author: Youngwoo Kim 
AuthorDate: Mon Oct 7 12:09:43 2019 +0900

BIGTOP-3230: Import the groundwork for CNB
---
 POSTINSTALL_EXAMPLE.md |  26 ++
 README.md  | 354 +++--
 README.md => README.md.BAK |   0
 README_STORAGE.md  |  14 +
 data/a.txt |   9 +
 data/b.txt |  15 +
 get_helm.sh| 245 ++
 hbase-helm/.helmignore |  21 ++
 hbase-helm/Chart.yaml  |  13 +
 hbase-helm/README.md   |   1 +
 hbase-helm/templates/NOTES.txt |  26 ++
 hbase-helm/templates/_helpers.yaml |  14 +
 hbase-helm/templates/hadoop-configmap.yaml | 296 +
 hbase-helm/templates/hbase-configmap.yaml  |  67 
 hbase-helm/templates/hbase-master-pdb.yaml |  17 +
 hbase-helm/templates/hbase-master-statefulset.yaml |  59 
 hbase-helm/templates/hbase-master-svc.yaml |  26 ++
 hbase-helm/templates/hbase-rs-pdb.yaml |  17 +
 hbase-helm/templates/hbase-rs-statefulset.yaml |  59 
 hbase-helm/templates/hbase-rs-svc.yaml |  22 ++
 hbase-helm/templates/hdfs-dn-pdb.yaml  |  17 +
 hbase-helm/templates/hdfs-dn-pvc.yaml  |  25 ++
 hbase-helm/templates/hdfs-dn-statefulset.yaml  |  82 +
 hbase-helm/templates/hdfs-dn-svc.yaml  |  23 ++
 hbase-helm/templates/hdfs-nn-pdb.yaml  |  18 ++
 hbase-helm/templates/hdfs-nn-pvc.yaml  |  25 ++
 hbase-helm/templates/hdfs-nn-statefulset.yaml  |  82 +
 hbase-helm/templates/hdfs-nn-svc.yaml  |  23 ++
 hbase-helm/tools/calc_resources.sh |  75 +
 hbase-helm/values.yaml |  55 
 kafka/README.md|  25 ++
 kafka/kafka.yaml   |   7 +
 minio/object-store.yaml|  63 
 minio/operator.yaml|  99 ++
 ms-spark/README.md |  21 ++
 ms-spark/core-site.xml |  49 +++
 ms-spark/log4j.properties  |  12 +
 ms-spark/spark-defaults.conf   |   1 +
 ms-spark/spark-deployment.yaml |  94 ++
 ms-spark/spark-env.sh  |   1 +
 nifi/README.md |  21 ++
 nifi/volume.yaml   |  31 ++
 presto3-minio/README.md|  32 ++
 presto3-minio/coordinator/Dockerfile   |   6 +
 presto3-minio/coordinator/README.md|   8 +
 presto3-minio/coordinator/config.properties|   8 +
 presto3-minio/coordinator/json |   2 +
 presto3-minio/coordinator/minio.properties |  14 +
 presto3-minio/presto-deployment-master.yaml|  26 ++
 presto3-minio/presto-deployment-worker.yaml|  26 ++
 presto3-minio/presto-deployment.yaml   |  23 ++
 presto3-minio/service.yaml |   9 +
 presto3-minio/test.sh  |  19 ++
 presto3-minio/worker/Dockerfile|   5 +
 presto3-minio/worker/README.md |   6 +
 presto3-minio/worker/config.properties |   7 +
 presto3-minio/worker/json  |   2 +
 presto3-minio/worker/minio.properties  |  14 +
 storage-provisioner/README.md  |  28 ++
 59 files changed, 2151 insertions(+), 234 deletions(-)

diff --git a/POSTINSTALL_EXAMPLE.md b/POSTINSTALL_EXAMPLE.md
new file mode 100755
index 000..67b7823
--- /dev/null
+++ b/POSTINSTALL_EXAMPLE.md
@@ -0,0 +1,26 @@
+
+
+# Example of what needs to be done...
+
+Minio can be accessed via port 9000 on the following DNS name from within your 
cluster:
+minio.bigdata.svc.cluster.local
+
+To access Minio from localhost, run the below commands:
+
+  1. export POD_NAME=$(kubectl get pods --namespace bigdata -l "release=minio" 
-o jsonpath="{.items[0].metadata.name}")
+
+  2. kubectl port-forward $POD_NAME 9000 --namespace bigdata
+
+Read more about port forwarding here: 
http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/
+
+You can now access Minio server on http://localhost:9000. Follow the below 
steps to 

[bigtop] branch cnb updated: BIGTOP-3242: Add instructions on configuring kubectl for local k8s cluster

[ywkim]

This is an automated email from the ASF dual-hosted git repository.

ywkim pushed a commit to branch cnb
in repository https://gitbox.apache.org/repos/asf/bigtop.git


The following commit(s) were added to refs/heads/cnb by this push:
 new 1511c90  BIGTOP-3242: Add instructions on configuring kubectl for 
local k8s cluster
1511c90 is described below

commit 1511c907d039895d7d1ed25fc9e7b47a261dc727
Author: Youngwoo Kim 
AuthorDate: Mon Oct 14 12:15:44 2019 +0900

BIGTOP-3242: Add instructions on configuring kubectl for local k8s cluster
---
 README.md | 16 
 1 file changed, 16 insertions(+)

diff --git a/README.md b/README.md
index 0db5be6..5952ea0 100755
--- a/README.md
+++ b/README.md
@@ -35,6 +35,22 @@ $ cd dl/kubespray-2.11.0/ && cp 
../../kubespray/vagrant/Vagrantfile .
 $ vagrant up
 ```
 
+## Configuring ```kubectl``` for local cluster
+```
+$ vagrant ssh k8s-1
+
+k8s-1$ mkdir -p ~/.kube
+k8s-1$ sudo cp /etc/kubernetes/admin.conf .kube/config
+k8s-1$ sudo chown -R vagrant:vagrant .kube
+k8s-1$ kubectl cluster-info
+
+Kubernetes master is running at https://172.17.8.101:6443
+coredns is running at 
https://172.17.8.101:6443/api/v1/namespaces/kube-system/services/coredns:dns/proxy
+kubernetes-dashboard is running at 
https://172.17.8.101:6443/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy
+
+To further debug and diagnose cluster problems, use 'kubectl cluster-info 
dump'.
+```
+
 # Cloud Native Bigtop
 This is the content for the talk given by jay vyas and sid mani @ apachecon 
2019 in Las Vegas,  you can watch it here  
https://www.youtube.com/watch?v=LUCE63q !
 

[bigtop] branch cnb updated: BIGTOP-3239: Provisioning the storage on kubernetes via Rook

[ywkim]

This is an automated email from the ASF dual-hosted git repository.

ywkim pushed a commit to branch cnb
in repository https://gitbox.apache.org/repos/asf/bigtop.git


The following commit(s) were added to refs/heads/cnb by this push:
 new 898259d  BIGTOP-3239: Provisioning the storage on kubernetes via Rook
898259d is described below

commit 898259d1d41663e1d25ccd36fee094bf102d8629
Author: Youngwoo Kim 
AuthorDate: Mon Oct 14 15:35:23 2019 +0900

BIGTOP-3239: Provisioning the storage on kubernetes via Rook
---
 README.md|  82 +++--
 bigtop.bom   |  12 ++-
 storage/rook/ceph/cluster-test.yaml  |  57 
 storage/rook/ceph/cluster.yaml   | 170 +++
 storage/rook/minio/object-store.yaml |  72 +++
 5 files changed, 383 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index 5952ea0..a254467 100755
--- a/README.md
+++ b/README.md
@@ -25,7 +25,7 @@ TBD
 
 Prerequisites:
 - Vagrant
-- Java 
+- Java
 
 ## Set up 3-Node Kubernetes cluster via Kubespray on local machine
 ```
@@ -51,6 +51,72 @@ kubernetes-dashboard is running at 
https://172.17.8.101:6443/api/v1/namespaces/k
 To further debug and diagnose cluster problems, use 'kubectl cluster-info 
dump'.
 ```
 
+## Storage
+You need to install ```lvm2``` package for Rook-Ceph:
+```
+# Centos
+sudo yum install -y lvm2
+
+# Ubuntu
+sudo apt-get install -y lvm2
+```
+Refer to https://rook.io/docs/rook/v1.1/k8s-pre-reqs.html for prerequisites on 
Rook
+
+Run ```download``` task to get Rook binary:
+```
+$ ./gradlew rook-clean rook-download && cd dl/ && tar xvfz rook-1.1.2.tar.gz
+```
+
+Create Rook operator:
+```
+$ kubectl create -f dl/rook-1.1.2/cluster/examples/kubernetes/ceph/common.yaml
+$ kubectl create -f 
dl/rook-1.1.2/cluster/examples/kubernetes/ceph/operator.yaml
+$ kubectl -n rook-ceph get pod
+```
+
+Create Ceph cluster:
+```
+# test
+$ kubectl create -f storage/rook/ceph/cluster-test.yaml
+
+# production
+$ kubectl create -f storage/rook/ceph/cluster.yaml
+
+$ kubectl get pod -n rook-ceph
+```
+
+Deploy Ceph Toolbox:
+```
+$ kubectl create -f dl/rook-1.1.2/cluster/examples/kubernetes/ceph/toolbox.yaml
+$ kubectl -n rook-ceph get pod -l "app=rook-ceph-tools"
+$ kubectl -n rook-ceph exec -it $(kubectl -n rook-ceph get pod -l 
"app=rook-ceph-tools" -o jsonpath='{.items[0].metadata.name}') bash
+
+# ceph status &&
+ceph osd status &&
+ceph df &&
+rados df &&
+```
+Refer to https://rook.io/docs/rook/v1.1/ceph-toolbox.html for more details.
+
+Create a StorageClass for Ceph RBD:
+```
+$ kubectl create -f 
dl/rook-1.1.2/cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml
+kubectl get storageclass
+rook-ceph-block
+```
+
+Create Minio operator:
+```
+$ kubectl create -f 
dl/rook-1.1.2/cluster/examples/kubernetes/minio/operator.yaml
+
+#
+$ kubectl -n rook-minio-system get pod
+```
+```
+$ kubectl create -f storage/rook/minio/object-store.yaml
+$ kubectl -n rook-minio get pod -l app=minio,objectstore=my-store
+```
+
 # Cloud Native Bigtop
 This is the content for the talk given by jay vyas and sid mani @ apachecon 
2019 in Las Vegas,  you can watch it here  
https://www.youtube.com/watch?v=LUCE63q !
 
@@ -61,7 +127,7 @@ helm install stable/nfs-server-provisioner ; kubectl patch 
storageclass nfs -p '
 Minio:  kubectl -n minio create secret generic my-minio-secret 
--from-literal=accesskey=minio --from-literal=secretkey=minio123
 helm install --set existingSecret=my-minio-secret stable/minio 
--namespace=minio --name=minio
 Nifi: helm repo add cetic https://cetic.github.io/helm-charts ; helm install 
nifi --namespace=minio
-Kafka:  helm repo add incubator 
http://storage.googleapis.com/kubernetes-charts-incubator $ helm install --name 
my-kafka incubator/kafka , kubectl edit statefulset kafka 
+Kafka:  helm repo add incubator 
http://storage.googleapis.com/kubernetes-charts-incubator $ helm install --name 
my-kafka incubator/kafka , kubectl edit statefulset kafka
  envFrom:
 - configMapRef:
 name: kafka-cm
@@ -177,25 +243,23 @@ In particular, this repo modifies stock helm charts in a 
variety of ways to make
 1. We don't use stable/spark because its *old*.  Instead we use microsofts 
spark, which comes integrated
 with zepplin properly.
 2. We use configmaps for configuration of *spark*.  For spark, this allows us 
to inject
-different types of configuration stuff from the kuberentes level, rather then 
baking them into the image (note that 
+different types of configuration stuff from the kuberentes level, rather then 
baking them into the image (note that
 you cant just inject a single file from a config map, b/c it overwrites the 
whole directory).  This allows us
 to inject minio access properties into spark itself, while also injecting 
other config.
-3. For Kafka, we config map the environment variables so that we c

[bigtop] branch cnb updated (898259d -> 0449b74)

[ywkim]

This is an automated email from the ASF dual-hosted git repository.

ywkim pushed a change to branch cnb
in repository https://gitbox.apache.org/repos/asf/bigtop.git.


from 898259d  BIGTOP-3239: Provisioning the storage on kubernetes via Rook
 new 28b8315  BIGTOP-3243: Add instructions for Helm
 new 0449b74  BIGTOP-3244: Remove Minio operator and hostpath provisioner

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 get_helm.sh   | 245 --
 helm/README.md|  25 +
 minio/object-store.yaml   |  63 ---
 minio/operator.yaml   |  99 -
 storage-provisioner/README.md |  28 -
 5 files changed, 25 insertions(+), 435 deletions(-)
 delete mode 100755 get_helm.sh
 create mode 100644 helm/README.md
 delete mode 100755 minio/object-store.yaml
 delete mode 100755 minio/operator.yaml
 delete mode 100755 storage-provisioner/README.md

[bigtop] 01/02: BIGTOP-3243: Add instructions for Helm

[ywkim]

This is an automated email from the ASF dual-hosted git repository.

ywkim pushed a commit to branch cnb
in repository https://gitbox.apache.org/repos/asf/bigtop.git

commit 28b8315db2edb921d2a53c3c039ddbf737e5841b
Author: Youngwoo Kim 
AuthorDate: Mon Oct 14 15:57:41 2019 +0900

BIGTOP-3243: Add instructions for Helm
---
 get_helm.sh| 245 -
 helm/README.md |  25 ++
 2 files changed, 25 insertions(+), 245 deletions(-)

diff --git a/get_helm.sh b/get_helm.sh
deleted file mode 100755
index ea2056c..000
--- a/get_helm.sh
+++ /dev/null
@@ -1,245 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright The Helm Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# The install script is based off of the MIT-licensed script from glide,
-# the package manager for Go: 
https://github.com/Masterminds/glide.sh/blob/master/get
-
-PROJECT_NAME="helm"
-TILLER_NAME="tiller"
-
-: ${USE_SUDO:="true"}
-: ${HELM_INSTALL_DIR:="/usr/local/bin"}
-
-# initArch discovers the architecture for this system.
-initArch() {
-  ARCH=$(uname -m)
-  case $ARCH in
-armv5*) ARCH="armv5";;
-armv6*) ARCH="armv6";;
-armv7*) ARCH="arm";;
-aarch64) ARCH="arm64";;
-x86) ARCH="386";;
-x86_64) ARCH="amd64";;
-i686) ARCH="386";;
-i386) ARCH="386";;
-  esac
-}
-
-# initOS discovers the operating system for this system.
-initOS() {
-  OS=$(echo `uname`|tr '[:upper:]' '[:lower:]')
-
-  case "$OS" in
-# Minimalist GNU for Windows
-mingw*) OS='windows';;
-  esac
-}
-
-# runs the given command as root (detects if we are root already)
-runAsRoot() {
-  local CMD="$*"
-
-  if [ $EUID -ne 0 -a $USE_SUDO = "true" ]; then
-CMD="sudo $CMD"
-  fi
-
-  $CMD
-}
-
-# verifySupported checks that the os/arch combination is supported for
-# binary builds.
-verifySupported() {
-  local 
supported="darwin-386\ndarwin-amd64\nlinux-386\nlinux-amd64\nlinux-arm\nlinux-arm64\nlinux-ppc64le\nwindows-386\nwindows-amd64"
-  if ! echo "${supported}" | grep -q "${OS}-${ARCH}"; then
-echo "No prebuilt binary for ${OS}-${ARCH}."
-echo "To build from source, go to https://github.com/helm/helm;
-exit 1
-  fi
-
-  if ! type "curl" > /dev/null && ! type "wget" > /dev/null; then
-echo "Either curl or wget is required"
-exit 1
-  fi
-}
-
-# checkDesiredVersion checks if the desired version is available.
-checkDesiredVersion() {
-  if [ "x$DESIRED_VERSION" == "x" ]; then
-# Get tag from release URL
-local latest_release_url="https://github.com/helm/helm/releases/latest;
-if type "curl" > /dev/null; then
-  TAG=$(curl -Ls -o /dev/null -w %{url_effective} $latest_release_url | 
grep -oE "[^/]+$" )
-elif type "wget" > /dev/null; then
-  TAG=$(wget $latest_release_url --server-response -O /dev/null 2>&1 | awk 
'/^  Location: /{DEST=$2} END{ print DEST}' | grep -oE "[^/]+$")
-fi
-  else
-TAG=$DESIRED_VERSION
-  fi
-}
-
-# checkHelmInstalledVersion checks which version of helm is installed and
-# if it needs to be changed.
-checkHelmInstalledVersion() {
-  if [[ -f "${HELM_INSTALL_DIR}/${PROJECT_NAME}" ]]; then
-local version=$(helm version -c | grep '^Client' | cut -d'"' -f2)
-if [[ "$version" == "$TAG" ]]; then
-  echo "Helm ${version} is already ${DESIRED_VERSION:-latest}"
-  return 0
-else
-  echo "Helm ${TAG} is available. Changing from version ${version}."
-  return 1
-fi
-  else
-return 1
-  fi
-}
-
-# downloadFile downloads the latest binary package and also the checksum
-# for that binary.
-downloadFile() {
-  HELM_DIST="helm-$TAG-$OS-$ARCH.tar.gz"
-  DOWNLOAD_URL="https://get.helm.sh/$HELM_DIST;
-  CHECKSUM_URL="$DOWNLOAD_URL.sha256"
-  HELM_TMP_ROOT="$(mktemp -dt helm-installer-XX)"
-  HELM_TMP_FILE="$HELM_TMP_ROOT/$HELM_DIST"
-  HELM_SUM_FILE="$HELM_TMP_ROOT/$HELM_DIST.sha256"
-  echo "Downloading $DOWNLOAD_URL"
-  if type "curl" > /dev/null; then
-curl -SsL "$CHECKSUM_URL" -o "$HELM_SUM_

[bigtop] 02/02: BIGTOP-3244: Remove Minio operator and hostpath provisioner

[ywkim]

This is an automated email from the ASF dual-hosted git repository.

ywkim pushed a commit to branch cnb
in repository https://gitbox.apache.org/repos/asf/bigtop.git

commit 0449b74054f6ff67d6db81bc186ac3138762f370
Author: Youngwoo Kim 
AuthorDate: Mon Oct 14 16:00:22 2019 +0900

BIGTOP-3244: Remove Minio operator and hostpath provisioner
---
 minio/object-store.yaml   | 63 ---
 minio/operator.yaml   | 99 ---
 storage-provisioner/README.md | 28 
 3 files changed, 190 deletions(-)

diff --git a/minio/object-store.yaml b/minio/object-store.yaml
deleted file mode 100755
index 2afc9ce..000
--- a/minio/object-store.yaml
+++ /dev/null
@@ -1,63 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: minio-my-store-access-keys
-type: Opaque
-data:
-  # Base64 encoded string: "TEMP_DEMO_ACCESS_KEY"
-  username: bWluaW8K
-  # Base64 encoded string: "TEMP_DEMO_SECRET_KEY"
-  password: bWluaW8xMjMK

-apiVersion: minio.rook.io/v1alpha1
-kind: ObjectStore
-metadata:
-  name: my-store
-spec:
-  scope:
-nodeCount: 4
-# You can have multiple PersistentVolumeClaims in the volumeClaimTemplates 
list.
-# Be aware though that all PersistentVolumeClaim Templates will be used 
for each intance (see nodeCount).
-volumeClaimTemplates:
-- metadata:
-name: rook-minio-data1
-  spec:
-accessModes: [ "ReadWriteOnce" ]
-# Set the storage class that will be used, otherwise Kubernetes' 
default storage class will be used.
-#storageClassName: "my-storage-class"
-resources:
-  requests:
-storage: "1M"
-#- metadata:
-#name: rook-minio-data2
-#  spec:
-#accessModes: [ "ReadWriteOnce" ]
-## Uncomment and specify your StorageClass, otherwise
-## the cluster admin defined default StorageClass will be used.
-##storageClassName: "your-cluster-storageclass"
-#resources:
-#  requests:
-#storage: "8Gi"
-  # A key value list of annotations
-  annotations:
-  #  key: value
-  placement:
-tolerations:
-nodeAffinity:
-podAffinity:
-podAnyAffinity:
-  credentials:
-name: minio-my-store-access-keys
-namespace: rook-minio
-  clusterDomain:

-apiVersion: v1
-kind: Service
-metadata:
-  name: minio-my-store
-spec:
-  type: NodePort
-  ports:
-- port: 9000
-  selector:
-app: minio
diff --git a/minio/operator.yaml b/minio/operator.yaml
deleted file mode 100755
index 3b347c5..000
--- a/minio/operator.yaml
+++ /dev/null
@@ -1,99 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: objectstores.minio.rook.io
-spec:
-  group: minio.rook.io
-  names:
-kind: ObjectStore
-listKind: ObjectStoreList
-plural: objectstores
-singular: objectstore
-  scope: Namespaced
-  version: v1alpha1

-apiVersion: rbac.authorization.k8s.io/v1beta1
-kind: ClusterRole
-metadata:
-  name: rook-minio-operator
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - namespaces
-  - secrets
-  - pods
-  - services
-  verbs:
-  - get
-  - watch
-  - create
-  - update
-- apiGroups:
-  - apps
-  resources:
-  - statefulsets
-  verbs:
-  - get
-  - create
-  - update
-- apiGroups:
-  - minio.rook.io
-  resources:
-  - "*"
-  verbs:
-  - "*"
-- apiGroups:
-  - rook.io
-  resources:
-  - "*"
-  verbs:
-  - "*"

-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: rook-minio-operator

-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1beta1
-metadata:
-  name: rook-minio-operator
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: rook-minio-operator
-subjects:
-- kind: ServiceAccount
-  name: rook-minio-operator

-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: rook-minio-operator
-  labels:
-app: rook-minio-operator
-spec:
-  selector:
-matchLabels:
-  app: rook-minio-operator
-  replicas: 1
-  template:
-metadata:
-  labels:
-app: rook-minio-operator
-spec:
-  serviceAccountName: rook-minio-operator
-  containers:
-  - name: rook-minio-operator
-image: rook/minio:v1.0.5
-args: ["minio", "operator"]
-env:
-- name: POD_NAME
-  valueFrom:
-fieldRef:
-  fieldPath: metadata.name
-- name: POD_NAMESPACE
-  valueFrom:
-fieldRef:
-  fieldPath: metadata.namespace
diff --git a/storage-provisioner/README.md b/storage-provisioner/README.md
deleted file mode 100755
index 09abdd1..000
--- a/storage-provisioner/README.md
+++ /dev/null
@@ -1,28 +0,0 @@
-1. The easiest way to get simple PVCs working on a one node cluster:
-
-```
-helm upgrade --install hostpath-provisioner --namespace ku

[bigtop] branch master updated: BIGTOP-3297. Bump HBase to 1.5.0.

[ywkim]

This is an automated email from the ASF dual-hosted git repository.

ywkim pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git


The following commit(s) were added to refs/heads/master by this push:
 new 4d2b50b  BIGTOP-3297. Bump HBase to 1.5.0.
4d2b50b is described below

commit 4d2b50b4635e755694f4e46ea0ee67de3df344c6
Author: Masatake Iwasaki 
AuthorDate: Mon Feb 3 01:47:47 2020 +

BIGTOP-3297. Bump HBase to 1.5.0.

Signed-off-by: Youngwoo Kim 
---
 .../src/common/hbase/do-component-build|   2 +-
 .../src/common/hbase/patch1-HBASE-19663.diff   |  29 +++
 .../patch2-0002-Backport-HBASE-17893-to-1.3.diff   | 278 -
 bigtop.bom |   2 +-
 4 files changed, 31 insertions(+), 280 deletions(-)

diff --git a/bigtop-packages/src/common/hbase/do-component-build 
b/bigtop-packages/src/common/hbase/do-component-build
index 70b83cf..114f2e5 100644
--- a/bigtop-packages/src/common/hbase/do-component-build
+++ b/bigtop-packages/src/common/hbase/do-component-build
@@ -36,7 +36,7 @@ if [ $HOSTTYPE = "aarch64" ] ; then
sed -i '/1.5.0-alpha.6<\/version>/,+1d' ./pom.xml
sed -i 
'/asciidoctorj-pdf<\/artifactId>/a\1.5.0-alpha.6<\/version>\n\n\norg.jruby<\/groupId>\njruby-complete<\/artifactId>\n9.1.8.0<\/version>\n<\/dependency>'
 ./pom.xml
 fi
-mvn -DskipTests -Dslf4j.version=1.6.1 \
+mvn -DskipTests -Dslf4j.version=1.7.25\
 -Dhadoop-two.version=$HADOOP_VERSION  \
 -Dzookeeper.version=$ZOOKEEPER_VERSION\
 -Dcheckstyle.skip=true\
diff --git a/bigtop-packages/src/common/hbase/patch1-HBASE-19663.diff 
b/bigtop-packages/src/common/hbase/patch1-HBASE-19663.diff
new file mode 100644
index 000..d035305
--- /dev/null
+++ b/bigtop-packages/src/common/hbase/patch1-HBASE-19663.diff
@@ -0,0 +1,29 @@
+From 2d0d7973f97424a16047cbab2b8ced43f90b4e7a Mon Sep 17 00:00:00 2001
+From: Sean Busbey 
+Date: Wed, 16 Oct 2019 10:31:47 -0500
+Subject: [PATCH] HBASE-19663 javadoc creation needs jsr305.
+
+---
+ pom.xml | 6 ++
+ 1 file changed, 6 insertions(+)
+
+diff --git a/pom.xml b/pom.xml
+index 5461457ee0..0cc1f28d6d 100644
+--- a/pom.xml
 b/pom.xml
+@@ -2908,6 +2908,12 @@
+   hamcrest-core
+   ${hamcrest.version}
+ 
++
++
++  com.google.code.findbugs
++  jsr305
++  3.0.2
++
+   
+ 
+ 
+-- 
+2.16.1
+
diff --git 
a/bigtop-packages/src/common/hbase/patch2-0002-Backport-HBASE-17893-to-1.3.diff 
b/bigtop-packages/src/common/hbase/patch2-0002-Backport-HBASE-17893-to-1.3.diff
deleted file mode 100644
index 8bfaec0..000
--- 
a/bigtop-packages/src/common/hbase/patch2-0002-Backport-HBASE-17893-to-1.3.diff
+++ /dev/null
@@ -1,278 +0,0 @@
-From 1a58e47067d64866917b8834eabc37e4c2887ebb Mon Sep 17 00:00:00 2001
-From: Anton Chevychalov 
-Date: Fri, 8 Sep 2017 19:20:48 +0300
-Subject: [PATCH 2/2] Backport HBASE-17893 to 1.3
-

- .../src/main/resources/supplemental-models.xml | 222 +
- 1 file changed, 222 insertions(+)
-
-diff --git a/hbase-resource-bundle/src/main/resources/supplemental-models.xml 
b/hbase-resource-bundle/src/main/resources/supplemental-models.xml
-index d5495cb..e8e6bbb 100644
 a/hbase-resource-bundle/src/main/resources/supplemental-models.xml
-+++ b/hbase-resource-bundle/src/main/resources/supplemental-models.xml
-@@ -644,6 +644,21 @@ under the License.
- 
-   
-   
-+ 
-+  org.apache.commons
-+  commons-csv
-+  1.0
-+  
-+
-+  Apache License, Version 2.0
-+  http://www.apache.org/licenses/LICENSE-2.0.txt
-+  repo
-+
-+  
-+
-+  
-+
-+  
- 
-   org.apache.commons
-   commons-math
-@@ -714,6 +729,21 @@ under the License.
- 
-   
-   
-+   
-+  org.apache.curator
-+  curator-test
-+  2.7.1
-+
-+  
-+
-+  Apache License, Version 2.0
-+  http://www.apache.org/licenses/LICENSE-2.0.txt
-+  repo
-+
-+  
-+
-+  
-+  
- 
-   org.apache.directory.api
-   api-asn1-api
-@@ -783,6 +813,21 @@ under the License.
-   
- 
-   
-+  
-+   
-+  org.apache.htrace
-+  htrace-core4
-+  4.0.1-incubating
-+  
-+
-+  Apache License, Version 2.0
-+  http://www.apache.org/licenses/LICENSE-2.0.txt
-+  repo
-+
-+  
-+
-+  
-+
-   
-   
- 
-@@ -980,6 +1025,7 @@ under the License.
-   
- 
-   
-+
- 
-   
- 
-@@ -1082,6 +1128,143 @@ Copyright 2006 Envoi Solutions LLC
-   
- 
-   
-+  
-+   
-+  com.codahale.metrics
-+  metrics-core
-+  3.0.1
-+  
-+
-+  Apache License, Version 2.0
-+  http://www.