Repository: bigtop Updated Branches: refs/heads/master 8effb9062 -> a04eacebf
BIGTOP-1149: Package Kite Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/a04eaceb Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/a04eaceb Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/a04eaceb Branch: refs/heads/master Commit: a04eacebfeb8692bf05b193cef7fef6493fa281e Parents: 8effb90 Author: Youngwoo Kim <[email protected]> Authored: Thu Jan 22 16:47:29 2015 +0900 Committer: YoungWoo Kim <[email protected]> Committed: Thu Jul 30 14:59:28 2015 +0900 ---------------------------------------------------------------------- MAINTAINERS.txt | 1 + .../src/common/kite/do-component-build | 48 +++++++++ bigtop-packages/src/common/kite/install_kite.sh | 106 +++++++++++++++++++ bigtop-packages/src/deb/kite/changelog | 1 + bigtop-packages/src/deb/kite/compat | 1 + bigtop-packages/src/deb/kite/control | 28 +++++ bigtop-packages/src/deb/kite/copyright | 15 +++ bigtop-packages/src/deb/kite/kite.dirs | 2 + bigtop-packages/src/deb/kite/kite.install | 2 + bigtop-packages/src/deb/kite/kite.postinst | 32 ++++++ bigtop-packages/src/deb/kite/kite.prerm | 37 +++++++ bigtop-packages/src/deb/kite/rules | 49 +++++++++ bigtop-packages/src/deb/kite/source/format | 1 + bigtop-packages/src/rpm/kite/SPECS/kite.spec | 77 ++++++++++++++ bigtop-tests/smoke-tests/README | 1 + .../smoke-tests/kite/TestKiteDataset.groovy | 76 +++++++++++++ bigtop-tests/smoke-tests/kite/build.gradle | 54 ++++++++++ bigtop-tests/smoke-tests/kite/log4j.properties | 24 +++++ bigtop-tests/smoke-tests/kite/sandwiches.csv | 3 + bigtop.mk | 13 +++ pom.xml | 1 + 21 files changed, 572 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/a04eaceb/MAINTAINERS.txt ---------------------------------------------------------------------- diff --git a/MAINTAINERS.txt b/MAINTAINERS.txt index bf09d6d..559c0fb 100644 --- a/MAINTAINERS.txt +++ b/MAINTAINERS.txt @@ -7,6 +7,7 @@ hbase: andrew purtell, rvs hive: mark grover, youngwoo kim hue: oflebbe itest: cos, rvs +kite: youngwoo kim mvn publishing/packaging: rvs oozie evans ye, rvs phoenix: andrew purtell, youngwoo kim http://git-wip-us.apache.org/repos/asf/bigtop/blob/a04eaceb/bigtop-packages/src/common/kite/do-component-build ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/kite/do-component-build b/bigtop-packages/src/common/kite/do-component-build new file mode 100644 index 0000000..e747d14 --- /dev/null +++ b/bigtop-packages/src/common/kite/do-component-build @@ -0,0 +1,48 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +. `dirname $0`/bigtop.bom + +mvn clean install \ + -DjavaVersion=${JDK_VERSION} \ + -Dvers.hadoop2=${HADOOP_VERSION} \ + -Dvers.hbase2=${HBASE_VERSION}-hadoop2 \ + -Dvers.flume=${FLUME_VERSION} \ + -Dvers.spark=${SPARK_VERSION} \ + -Dvers.crunch=${CRUNCH_VERSION} \ + -DskipTests -Dmaven.repo.local=${HOME}/.m2/repository "$@" + +rm -rf build + +mkdir -p build/kite-${KITE_VERSION}/{bin,lib} + +# Kite Data +cp kite-data/kite-*/target/kite-*.jar build/kite-${KITE_VERSION}/lib +# Kite Tools +cp kite-tools-parent/kite-tools/target/kite-dataset build/kite-${KITE_VERSION}/bin/ +cp kite-tools-parent/kite-tools/target/kite-*.jar build/kite-${KITE_VERSION}/lib/ + +# Kite Morphlines +cp kite-morphlines/kite-morphlines-all/target/lib/* build/kite-${KITE_VERSION}/lib/ + +# Kite Maven plugin +cp kite-maven-plugin/target/kite-*.jar build/kite-${KITE_VERSION}/lib/ + + + + http://git-wip-us.apache.org/repos/asf/bigtop/blob/a04eaceb/bigtop-packages/src/common/kite/install_kite.sh ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/kite/install_kite.sh b/bigtop-packages/src/common/kite/install_kite.sh new file mode 100644 index 0000000..efdc500 --- /dev/null +++ b/bigtop-packages/src/common/kite/install_kite.sh @@ -0,0 +1,106 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +usage() { + echo " +usage: $0 <options> + Required not-so-options: + --build-dir=DIR path to dist.dir + --prefix=PREFIX path to install into + + Optional options: + --lib-dir=DIR path to install home [/usr/lib/kite] + --build-dir=DIR path to dist dir + ... [ see source for more similar options ] + " + exit 1 +} + +OPTS=$(getopt \ + -n $0 \ + -o '' \ + -l 'prefix:' \ + -l 'lib-dir:' \ + -l 'build-dir:' -- "$@") + +if [ $? != 0 ] ; then + usage +fi + +eval set -- "$OPTS" +while true ; do + case "$1" in + --prefix) + PREFIX=$2 ; shift 2 + ;; + --build-dir) + BUILD_DIR=$2 ; shift 2 + ;; + --lib-dir) + LIB_DIR=$2 ; shift 2 + ;; + --) + shift ; break + ;; + *) + echo "Unknown option: $1" + usage + exit 1 + ;; + esac +done + +for var in PREFIX BUILD_DIR ; do + if [ -z "$(eval "echo \$$var")" ]; then + echo Missing param: $var + usage + fi +done + +LIB_DIR=${LIB_DIR:-/usr/lib/kite} +BIN_DIR=${BIN_DIR:-/usr/bin} + +# First we'll move everything into lib +install -d -m 0755 $PREFIX/$LIB_DIR +cp -r $BUILD_DIR/* $PREFIX/$LIB_DIR + +install -d -m 0755 $PREFIX/$BIN_DIR + +# Wrapper for kite-dataset +cat > $PREFIX/$BIN_DIR/kite-dataset <<EOF +#!/bin/bash + +# Autodetect JAVA_HOME if not defined +. /usr/lib/bigtop-utils/bigtop-detect-javahome + +exec ${LIB_DIR}/bin/kite-dataset "\$@" +EOF +chmod 755 $PREFIX/$BIN_DIR/kite-dataset + +rm $PREFIX/$LIB_DIR/lib/kite*-{sources,javadoc,tests}.jar + +# Version-less symlinks +for jar in $PREFIX/$LIB_DIR/lib/kite-*jar ; do + jar=$(basename $jar) + versionless_jar=`echo $jar | sed "s|-$KITE_VERSION||"` + (cd $PREFIX/$LIB_DIR; mv lib/$jar ./; ln -s $jar $versionless_jar) +done + + + http://git-wip-us.apache.org/repos/asf/bigtop/blob/a04eaceb/bigtop-packages/src/deb/kite/changelog ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/kite/changelog b/bigtop-packages/src/deb/kite/changelog new file mode 100644 index 0000000..d4858cd --- /dev/null +++ b/bigtop-packages/src/deb/kite/changelog @@ -0,0 +1 @@ +--- This is auto-generated http://git-wip-us.apache.org/repos/asf/bigtop/blob/a04eaceb/bigtop-packages/src/deb/kite/compat ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/kite/compat b/bigtop-packages/src/deb/kite/compat new file mode 100644 index 0000000..ec63514 --- /dev/null +++ b/bigtop-packages/src/deb/kite/compat @@ -0,0 +1 @@ +9 http://git-wip-us.apache.org/repos/asf/bigtop/blob/a04eaceb/bigtop-packages/src/deb/kite/control ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/kite/control b/bigtop-packages/src/deb/kite/control new file mode 100644 index 0000000..18b6eb0 --- /dev/null +++ b/bigtop-packages/src/deb/kite/control @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Source: kite +Section: misc +Priority: extra +Maintainer: Bigtop <[email protected]> +Build-Depends: debhelper (>= 9) +Standards-Version: 3.9.4 +Homepage: http://kitesdk.org + +Package: kite +Architecture: all +Depends: hadoop-client, hbase, hive, zookeeper, bigtop-utils (>= 0.7) +Description: The Kite Software Development Kit is a set of libraries, tools, examples, and + documentation focused on making it easier to build systems on top of the Hadoop ecosystem. http://git-wip-us.apache.org/repos/asf/bigtop/blob/a04eaceb/bigtop-packages/src/deb/kite/copyright ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/kite/copyright b/bigtop-packages/src/deb/kite/copyright new file mode 100644 index 0000000..022f5e4 --- /dev/null +++ b/bigtop-packages/src/deb/kite/copyright @@ -0,0 +1,15 @@ +Format: http://dep.debian.net/deps/dep5 +Source: http://kitesdk.org/ +Upstream-Name: Kite Software Development Kit + +Files: * +Copyright: 2013, Cloudera, Inc +License: Apache-2.0 + +Files debian/* +Copyright: 2011, The Apache Software Foundation +License: Apache-2.0 + +License: Apache-2.0 + On Debian systems, the complete text of the Apache 2.0 license + can be found in "/usr/share/common-licenses/Apache-2.0". http://git-wip-us.apache.org/repos/asf/bigtop/blob/a04eaceb/bigtop-packages/src/deb/kite/kite.dirs ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/kite/kite.dirs b/bigtop-packages/src/deb/kite/kite.dirs new file mode 100644 index 0000000..9b98872 --- /dev/null +++ b/bigtop-packages/src/deb/kite/kite.dirs @@ -0,0 +1,2 @@ +/usr/lib/kite +/usr/bin http://git-wip-us.apache.org/repos/asf/bigtop/blob/a04eaceb/bigtop-packages/src/deb/kite/kite.install ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/kite/kite.install b/bigtop-packages/src/deb/kite/kite.install new file mode 100644 index 0000000..99e2775 --- /dev/null +++ b/bigtop-packages/src/deb/kite/kite.install @@ -0,0 +1,2 @@ +/usr/lib/kite +/usr/bin/kite-dataset http://git-wip-us.apache.org/repos/asf/bigtop/blob/a04eaceb/bigtop-packages/src/deb/kite/kite.postinst ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/kite/kite.postinst b/bigtop-packages/src/deb/kite/kite.postinst new file mode 100644 index 0000000..a2078db --- /dev/null +++ b/bigtop-packages/src/deb/kite/kite.postinst @@ -0,0 +1,32 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +case "$1" in + configure) + ;; + + abort-upgrade|abort-remove|abort-deconfigure) + ;; + + *) + echo "postinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +#DEBHELPER# http://git-wip-us.apache.org/repos/asf/bigtop/blob/a04eaceb/bigtop-packages/src/deb/kite/kite.prerm ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/kite/kite.prerm b/bigtop-packages/src/deb/kite/kite.prerm new file mode 100644 index 0000000..4ba1a94 --- /dev/null +++ b/bigtop-packages/src/deb/kite/kite.prerm @@ -0,0 +1,37 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +case "$1" in + remove|upgrade|deconfigure) + ;; + + failed-upgrade) + ;; + + *) + echo "prerm called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 http://git-wip-us.apache.org/repos/asf/bigtop/blob/a04eaceb/bigtop-packages/src/deb/kite/rules ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/kite/rules b/bigtop-packages/src/deb/kite/rules new file mode 100755 index 0000000..9bce602 --- /dev/null +++ b/bigtop-packages/src/deb/kite/rules @@ -0,0 +1,49 @@ +#!/usr/bin/make -f +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# -*- makefile -*- + +# Uncomment this to turn on verbose mode. +export DH_VERBOSE=1 + +lib_kite=/usr/lib/kite +lib_zookeeper=/usr/lib/zookeeper +lib_hadoop=/usr/lib/hadoop + +# This has to be exported to make some magic below work. +export DH_OPTIONS + +%: + dh $@ + +override_dh_auto_build: + # we'll just use the build from the tarball. + bash debian/do-component-build + +override_dh_auto_install: + bash -x debian/install_kite.sh \ + --prefix=debian/tmp \ + --build-dir=build/kite-${KITE_BASE_VERSION} + + rm -f debian/tmp/${lib_kite}/lib/zookeeper*.jar + rm -f debian/tmp/${lib_kite}/lib/hadoop*.jar + + ln -s ${lib_zookeeper}/zookeeper.jar debian/tmp/${lib_kite}/lib + ln -s ${lib_hadoop}/client/hadoop-annotations.jar debian/tmp/${lib_kite}/lib + ln -s ${lib_hadoop}/client/hadoop-auth.jar debian/tmp/${lib_kite}/lib + ln -s ${lib_hadoop}/client/hadoop-common.jar debian/tmp/${lib_kite}/lib + ln -s ${lib_hadoop}/client/hadoop-hdfs.jar debian/tmp/${lib_kite}/lib http://git-wip-us.apache.org/repos/asf/bigtop/blob/a04eaceb/bigtop-packages/src/deb/kite/source/format ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/kite/source/format b/bigtop-packages/src/deb/kite/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/bigtop-packages/src/deb/kite/source/format @@ -0,0 +1 @@ +3.0 (quilt) http://git-wip-us.apache.org/repos/asf/bigtop/blob/a04eaceb/bigtop-packages/src/rpm/kite/SPECS/kite.spec ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/rpm/kite/SPECS/kite.spec b/bigtop-packages/src/rpm/kite/SPECS/kite.spec new file mode 100644 index 0000000..5ccaede --- /dev/null +++ b/bigtop-packages/src/rpm/kite/SPECS/kite.spec @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +%define kite_name kite +%define lib_kite /usr/lib/kite +%define lib_zookeeper /usr/lib/zookeeper +%define lib_hadoop /usr/lib/hadoop + +%if %{?suse_version:1}0 +%define doc_kite %{_docdir}/kite-doc +%else +%define doc_kite %{_docdir}/kite-doc-%{kite_version} +%endif + +# disable repacking jars +%define __os_install_post %{nil} + +Name: kite +Version: %{kite_version} +Release: %{kite_release} +Summary: Kite Software Development Kit. +URL: http://kitesdk.org +Group: Development/Libraries +BuildArch: noarch +Buildroot: %(mktemp -ud %{_tmppath}/%{kite_name}-%{version}-%{release}-XXXXXX) +License: ASL 2.0 +Source0: %{kite_name}-%{kite_version}.tar.gz +Source1: do-component-build +Source2: install_%{kite_name}.sh +Source3: bigtop.bom +Requires: hadoop-client, hbase, hive, zookeeper, bigtop-utils >= 0.7 + +%description +The Kite Software Development Kit is a set of libraries, tools, examples, and +documentation focused on making it easier to build systems on top of the +Hadoop ecosystem. + +%prep +%setup -n %{kite_name}-release-%{kite_base_version} + +%build +bash $RPM_SOURCE_DIR/do-component-build + +%install +%__rm -rf $RPM_BUILD_ROOT +KITE_VERSION=%{kite_version} bash $RPM_SOURCE_DIR/install_kite.sh \ + --build-dir=build/kite-%{kite_base_version} \ + --prefix=$RPM_BUILD_ROOT + +%__rm -f $RPM_BUILD_ROOT/%{lib_kite}/lib/zookeeper*.jar +%__rm -f $RPM_BUILD_ROOT/%{lib_kite}/lib/hadoop*.jar + +%__ln_s %{lib_zookeeper}/zookeeper.jar $RPM_BUILD_ROOT/%{lib_kite}/lib/ +%__ln_s %{lib_hadoop}/client/hadoop-annotations.jar $RPM_BUILD_ROOT/%{lib_kite}/lib/ +%__ln_s %{lib_hadoop}/client/hadoop-auth.jar $RPM_BUILD_ROOT/%{lib_kite}/lib/ +%__ln_s %{lib_hadoop}/client/hadoop-common.jar $RPM_BUILD_ROOT/%{lib_kite}/lib/ +%__ln_s %{lib_hadoop}/client/hadoop-hdfs.jar $RPM_BUILD_ROOT/%{lib_kite}/lib/ + +%files +%defattr(-,root,root,755) +/usr/bin/kite-dataset +%{lib_kite} +%{lib_kite}/bin +%{lib_kite}/lib + http://git-wip-us.apache.org/repos/asf/bigtop/blob/a04eaceb/bigtop-tests/smoke-tests/README ---------------------------------------------------------------------- diff --git a/bigtop-tests/smoke-tests/README b/bigtop-tests/smoke-tests/README index c480c32..f8adb3b 100644 --- a/bigtop-tests/smoke-tests/README +++ b/bigtop-tests/smoke-tests/README @@ -26,6 +26,7 @@ export FLUME_HOME=/usr/lib/flume/ export HIVE_CONF_DIR=/etc/hive/conf/ export JAVA_HOME="/usr/lib/jvm/java-openjdk/" export MAHOUT_HOME="/usr/lib/mahout" +export KITE_HOME="/usr/lib/kite" export ITEST="0.7.0" ``` http://git-wip-us.apache.org/repos/asf/bigtop/blob/a04eaceb/bigtop-tests/smoke-tests/kite/TestKiteDataset.groovy ---------------------------------------------------------------------- diff --git a/bigtop-tests/smoke-tests/kite/TestKiteDataset.groovy b/bigtop-tests/smoke-tests/kite/TestKiteDataset.groovy new file mode 100644 index 0000000..a7a11b8 --- /dev/null +++ b/bigtop-tests/smoke-tests/kite/TestKiteDataset.groovy @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.bigtop.itest.kite + +import static org.junit.Assert.assertTrue + +import org.junit.Assert +import org.junit.BeforeClass +import org.junit.AfterClass +import org.junit.Test +import org.apache.bigtop.itest.JarContent +import org.apache.bigtop.itest.TestUtils +import org.apache.commons.logging.LogFactory +import org.apache.commons.logging.Log +import java.lang.reflect.Constructor; +import junit.framework.TestCase; +import junit.framework.TestResult; +import org.junit.runner.RunWith + + +class TestKiteDataset { + static private Log LOG = LogFactory.getLog(Object.class); + + static Shell sh = new Shell("/bin/bash -s"); + + + @AfterClass + public static void tearDown() { + sh.exec("hadoop fs -rmr -skipTrash /tmp/kite"); + sh.exec("rm -rf *.avsc"); + + } + + @BeforeClass + static void setUp() { + + } + + @Test + void test() { + // Infer the Schema + sh.exec("kite-dataset csv-schema sandwiches.csv --class Sandwich -o sandwich.avsc"); + + // kite-dataset create sandwiches -s sandwich.avsc + sh.exec("kite-dataset create dataset:hdfs:/tmp/kite/sandwiches -s sandwich.avsc"); + + // Import the CSV Data + sh.exec("kite-dataset csv-import sandwiches.csv dataset:hdfs:/tmp/kite/sandwiches"); + + // Show the Results + sh.exec("kite-dataset show dataset:hdfs:/tmp/kite/sandwiches"); + sh.exec("kite-dataset show dataset:hdfs:/tmp/kite/sandwiches -n 1"); + + String r = sh.out; + LOG.info(r); + assertTrue("Incorrect record", r.contains("Reuben")); + + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/a04eaceb/bigtop-tests/smoke-tests/kite/build.gradle ---------------------------------------------------------------------- diff --git a/bigtop-tests/smoke-tests/kite/build.gradle b/bigtop-tests/smoke-tests/kite/build.gradle new file mode 100644 index 0000000..890c0d6 --- /dev/null +++ b/bigtop-tests/smoke-tests/kite/build.gradle @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +apply plugin: 'groovy' + +repositories { + mavenCentral() +} + +dependencies { + //needed to avoid groovy not on classpath error. + testCompile module('org.codehaus.groovy:groovy:1.8.0') + testCompile group: 'org.apache.bigtop.itest', name: 'itest-common', version: itestVersion, transitive: 'true' +} + + +def tests_to_include() { + return [ + "TestKiteDataset.groovy" + ]; +} + +sourceSets { + test { + resources { + srcDirs = [ + 'conf/' + ] + } + groovy { + srcDirs = ["./"] + exclude { FileTreeElement elem -> (doExclude(elem.getName())) } + } + } +} + +test.doFirst { + checkEnv(["KITE_HOME"]) +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/a04eaceb/bigtop-tests/smoke-tests/kite/log4j.properties ---------------------------------------------------------------------- diff --git a/bigtop-tests/smoke-tests/kite/log4j.properties b/bigtop-tests/smoke-tests/kite/log4j.properties new file mode 100644 index 0000000..db5aacf --- /dev/null +++ b/bigtop-tests/smoke-tests/kite/log4j.properties @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Set root logger level to DEBUG and its only appender to A1. +log4j.rootLogger=TRACE, A1 + +# A1 is set to be a ConsoleAppender. +log4j.appender.A1=org.apache.log4j.ConsoleAppender + +# A1 uses PatternLayout. +log4j.appender.A1.layout=org.apache.log4j.PatternLayout +log4j.appender.A1.layout.ConversionPattern=%-4r [BIGTOP-TEST-LOG %t] %-5p %c %x --- %m%n http://git-wip-us.apache.org/repos/asf/bigtop/blob/a04eaceb/bigtop-tests/smoke-tests/kite/sandwiches.csv ---------------------------------------------------------------------- diff --git a/bigtop-tests/smoke-tests/kite/sandwiches.csv b/bigtop-tests/smoke-tests/kite/sandwiches.csv new file mode 100644 index 0000000..7a8fc2b --- /dev/null +++ b/bigtop-tests/smoke-tests/kite/sandwiches.csv @@ -0,0 +1,3 @@ +name,description +Reuben,Pastrami and sauerkraut on toasted rye with Russian dressing. +PBJ,Peanut butter and grape jelly on white bread. http://git-wip-us.apache.org/repos/asf/bigtop/blob/a04eaceb/bigtop.mk ---------------------------------------------------------------------- diff --git a/bigtop.mk b/bigtop.mk index 880acfa..ff19b7d 100644 --- a/bigtop.mk +++ b/bigtop.mk @@ -374,3 +374,16 @@ YCSB_TARBALL_SRC=$(YCSB_BASE_VERSION).tar.gz YCSB_SITE=https://github.com/brianfrankcooper/YCSB/archive YCSB_ARCHIVE=$(YCSB_SITE) $(eval $(call PACKAGE,ycsb,YCSB)) + +# Kite +KITE_NAME=kite +KITE_RELNOTES_NAME=Kite Software Development Kit +KITE_PKG_NAME=kite +KITE_BASE_VERSION=1.1.0 +KITE_PKG_VERSION=$(KITE_BASE_VERSION) +KITE_RELEASE_VERSION=1 +KITE_TARBALL_SRC=release-$(KITE_BASE_VERSION).tar.gz +KITE_TARBALL_DST=kite-$(KITE_BASE_VERSION).tar.gz +KITE_SITE=https://github.com/kite-sdk/kite/archive +KITE_ARCHIVE=$(KITE_SITE) +$(eval $(call PACKAGE,kite,KITE)) http://git-wip-us.apache.org/repos/asf/bigtop/blob/a04eaceb/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 217b79d..74cc3c5 100644 --- a/pom.xml +++ b/pom.xml @@ -52,6 +52,7 @@ <spark.version>1.3.1</spark.version> <kafka.version>0.8.1.1</kafka.version> <phoenix.version>4.3.1</phoenix.version> + <kite.version>1.1.0</kite.version> <spark-smoke.version>${project.version}</spark-smoke.version> <itest-common.version>${project.version}</itest-common.version>
