Repository: bigtop Updated Branches: refs/heads/master 1883e81c3 -> c7e814b51
BIGTOP-1616. Upgrade to Sqoop 1.99.4 Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/c7e814b5 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/c7e814b5 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/c7e814b5 Branch: refs/heads/master Commit: c7e814b514e7ba8c6f43a6d6e5484a902c3dfb5e Parents: 1883e81 Author: Sean Mackrory <[email protected]> Authored: Thu Jan 22 16:19:16 2015 -0700 Committer: Sean Mackrory <[email protected]> Committed: Thu Feb 5 14:59:16 2015 -0700 ---------------------------------------------------------------------- .../src/common/sqoop/install_sqoop.sh | 19 +++- bigtop-packages/src/common/sqoop/sqoop-tool.sh | 36 +++++++ .../src/common/sqoop/sqoop.properties | 101 ------------------- .../src/deb/sqoop/sqoop-client.install | 4 + bigtop-packages/src/deb/sqoop/sqoop.install | 3 + bigtop-packages/src/rpm/sqoop/SPECS/sqoop.spec | 19 ++-- bigtop.mk | 4 +- 7 files changed, 72 insertions(+), 114 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/c7e814b5/bigtop-packages/src/common/sqoop/install_sqoop.sh ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/sqoop/install_sqoop.sh b/bigtop-packages/src/common/sqoop/install_sqoop.sh index 683d1ec..5d42a2c 100755 --- a/bigtop-packages/src/common/sqoop/install_sqoop.sh +++ b/bigtop-packages/src/common/sqoop/install_sqoop.sh @@ -129,11 +129,15 @@ install -d -m 0755 ${PREFIX}/${CONF_DIR} install -d -m 0755 ${PREFIX}/etc/default install -d -m 0755 ${PREFIX}/var/lib/sqoop -install -m 0644 ${DIST_DIR}/client/lib/*.jar ${PREFIX}/${LIB_DIR}/client-lib/ +install -m 0644 ${DIST_DIR}/shell/lib/*.jar ${PREFIX}/${LIB_DIR}/client-lib/ install -m 0755 ${DIST_DIR}/bin/sqoop.sh ${PREFIX}/${BIN_DIR}/ +install -m 0755 ${DIST_DIR}/bin/sqoop-sys.sh ${PREFIX}/${BIN_DIR}/ + +install -m 0644 ${DIST_DIR}/server/conf/sqoop.properties ${PREFIX}/${CONF_DIR}/sqoop.properties +sed -i 's#@LOGDIR@#/var/log/sqoop#' ${PREFIX}/${CONF_DIR}/sqoop.properties +sed -i 's#@BASEDIR@#/var/lib/sqoop#' ${PREFIX}/${CONF_DIR}/sqoop.properties install -m 0644 ${DIST_DIR}/server/conf/sqoop_bootstrap.properties ${PREFIX}/${CONF_DIR} -install -m 0644 ${EXTRA_DIR}/sqoop.properties ${PREFIX}/${CONF_DIR} install -m 0644 ${EXTRA_DIR}/sqoop.default ${PREFIX}/etc/default/sqoop-server rm ${EXTRA_DIR}/sqoop.default # Otherwise debhelper will re-install this @@ -148,7 +152,6 @@ unzip -d $SQOOP_WEBAPPS/sqoop $SQOOP_WEBAPPS/sqoop.war install -m 0755 ${EXTRA_DIR}/tomcat-deployment.sh ${PREFIX}/${LIB_DIR}/tomcat-deployment.sh -# Create MR2 configuration install -d -m 0755 ${PREFIX}/${TOMCAT_CONF_DIR}.dist/conf for conf in web.xml tomcat-users.xml server.xml logging.properties context.xml catalina.policy do @@ -158,12 +161,20 @@ sed -i -e "s|<Host |<Host workDir=\"/var/tmp/sqoop\" |" ${PREFIX}/${TOMCAT_CONF_ sed -i -e "s|\${catalina\.base}/logs|/var/log/sqoop|" ${PREFIX}/${TOMCAT_CONF_DIR}.dist/conf/logging.properties cp -f ${EXTRA_DIR}/catalina.properties ${PREFIX}/${TOMCAT_CONF_DIR}.dist/conf/catalina.properties install -d -m 0755 ${PREFIX}/${TOMCAT_CONF_DIR}.dist/WEB-INF -mv ${SQOOP_WEBAPPS}/sqoop/WEB-INF/*.xml ${PREFIX}/${TOMCAT_CONF_DIR}.dist/WEB-INF +mv $SQOOP_WEBAPPS/sqoop/WEB-INF/*.xml ${PREFIX}/${TOMCAT_CONF_DIR}.dist/WEB-INF # Create wrapper scripts for the client and server client_wrapper=$PREFIX/usr/bin/sqoop server_wrapper=$PREFIX/usr/bin/sqoop-server +tool_wrapper=$PREFIX/usr/bin/sqoop-tool mkdir -p $PREFIX/usr/bin install -m 0755 $EXTRA_DIR/sqoop.sh $client_wrapper install -m 0755 $EXTRA_DIR/sqoop-server.sh $server_wrapper +install -m 0755 $EXTRA_DIR/sqoop-tool.sh $tool_wrapper + +CATALINA_HOME=/usr/lib/bigtop-tomcat +install -d ${PREFIX}/${CATALINA_HOME}/lib +install -m 0644 ${DIST_DIR}/server/lib/sqoop-tomcat*.jar ${PREFIX}/${CATALINA_HOME}/lib/ + +cp ${DIST_DIR}/{LICENSE,NOTICE}.txt ${PREFIX}/${LIB_DIR}/ http://git-wip-us.apache.org/repos/asf/bigtop/blob/c7e814b5/bigtop-packages/src/common/sqoop/sqoop-tool.sh ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/sqoop/sqoop-tool.sh b/bigtop-packages/src/common/sqoop/sqoop-tool.sh new file mode 100644 index 0000000..c008665 --- /dev/null +++ b/bigtop-packages/src/common/sqoop/sqoop-tool.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Autodetect JAVA_HOME if not defined + +. /usr/lib/bigtop-utils/bigtop-detect-javahome + +export TOMCAT_DEPLOYMENT=/var/lib/sqoop/tool-tomcat-deployment +. /usr/lib/sqoop/tomcat-deployment.sh + +LIB_DIR=/usr/lib/sqoop +BIN_DIR=${LIB_DIR}/bin + +export CLASSPATH=$CLASSPATH:$(echo "$LIB_DIR"/client-lib/*.jar | tr ' ' ':') +export CATALINA_HOME=/usr/lib/bigtop-tomcat +export CATALINA_BIN=${CATALINA_HOME}/bin +export CATALINA_BASE=/var/lib/sqoop/tool-tomcat-deployment +export JAVA_OPTS="$JAVA_OPTS -Dsqoop.config.dir=/etc/sqoop/conf" + +COMMAND="cd ~/ && ${BIN_DIR}/sqoop.sh tool $@" +su -s /bin/bash -c "${COMMAND}" sqoop + http://git-wip-us.apache.org/repos/asf/bigtop/blob/c7e814b5/bigtop-packages/src/common/sqoop/sqoop.properties ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/sqoop/sqoop.properties b/bigtop-packages/src/common/sqoop/sqoop.properties deleted file mode 100644 index a98ab0a..0000000 --- a/bigtop-packages/src/common/sqoop/sqoop.properties +++ /dev/null @@ -1,101 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# Sqoop configuration file used by the built in configuration -# provider: org.apache.sqoop.core.PropertiesConfigurationProvider. -# This file must reside in the system configuration directory -# which is specified by the system property "sqoop.config.dir" -# and must be called sqoop.properties. -# - -# -# Logging Configuration -# Any property that starts with the prefix -# org.apache.sqoop.log4j is parsed out by the configuration -# system and passed to the log4j subsystem. This allows you -# to specify log4j configuration properties from within the -# Sqoop configuration. -# -org.apache.sqoop.log4j.appender.file=org.apache.log4j.RollingFileAppender -org.apache.sqoop.log4j.appender.file.File=/var/log/sqoop/sqoop.log -org.apache.sqoop.log4j.appender.file.MaxFileSize=25MB -org.apache.sqoop.log4j.appender.file.MaxBackupIndex=5 -org.apache.sqoop.log4j.appender.file.layout=org.apache.log4j.PatternLayout -org.apache.sqoop.log4j.appender.file.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} [%l] %m%n -org.apache.sqoop.log4j.debug=true -org.apache.sqoop.log4j.rootCategory=WARN, file -org.apache.sqoop.log4j.category.org.apache.sqoop=DEBUG -org.apache.sqoop.log4j.category.org.apache.derby=INFO - -# -# Repository configuration -# The Repository subsystem provides the special prefix which -# is "org.apache.sqoop.repository.sysprop". Any property that -# is specified with this prefix is parsed out and set as a -# system property. For example, if the built in Derby repository -# is being used, the sysprop prefixed proeprties can be used -# to affect Derby configuration at startup time by setting -# the appropriate system properties. -# - -# Repository provider -org.apache.sqoop.repository.provider=org.apache.sqoop.repository.JdbcRepositoryProvider - -# JDBC repository provider configuration -org.apache.sqoop.repository.jdbc.handler=org.apache.sqoop.repository.derby.DerbyRepositoryHandler -org.apache.sqoop.repository.jdbc.transaction.isolation=READ_COMMITTED -org.apache.sqoop.repository.jdbc.maximum.connections=10 -org.apache.sqoop.repository.jdbc.url=jdbc:derby:/var/lib/sqoop/repository/db;create=true -org.apache.sqoop.repository.jdbc.create.schema=true -org.apache.sqoop.repository.jdbc.driver=org.apache.derby.jdbc.EmbeddedDriver -org.apache.sqoop.repository.jdbc.user=sa -org.apache.sqoop.repository.jdbc.password= - -# System properties for embedded Derby configuration -org.apache.sqoop.repository.sysprop.derby.stream.error.file=/var/log/sqoop/derbyrepo.log - -# Sleeping period for reloading configuration file (once a minute) -org.apache.sqoop.core.configuration.provider.properties.sleep=60000 - -# -# Submission engine configuration -# - -# Submission engine class -org.apache.sqoop.submission.engine=org.apache.sqoop.submission.mapreduce.MapreduceSubmissionEngine - -# Number of milliseconds, submissions created before this limit will be removed, default is one day -#org.apache.sqoop.submission.purge.threshold= - -# Number of milliseconds for purge thread to sleep, by default one day -#org.apache.sqoop.submission.purge.sleep= - -# Number of milliseconds for update thread to sleep, by default 5 minutes -#org.apache.sqoop.submission.update.sleep= - -# -# Configuration for Mapreduce submission engine (applicable if it's configured) -# - -# Hadoop configuration directory -org.apache.sqoop.submission.engine.mapreduce.configuration.directory=/etc/hadoop/conf/ - -# -# Execution engine configuration -# -org.apache.sqoop.execution.engine=org.apache.sqoop.execution.mapreduce.MapreduceExecutionEngine http://git-wip-us.apache.org/repos/asf/bigtop/blob/c7e814b5/bigtop-packages/src/deb/sqoop/sqoop-client.install ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/sqoop/sqoop-client.install b/bigtop-packages/src/deb/sqoop/sqoop-client.install index 669a206..8421bd5 100644 --- a/bigtop-packages/src/deb/sqoop/sqoop-client.install +++ b/bigtop-packages/src/deb/sqoop/sqoop-client.install @@ -1,3 +1,7 @@ /usr/bin/sqoop +/usr/bin/sqoop-tool /usr/lib/sqoop/bin/sqoop.sh /usr/lib/sqoop/client-lib +/usr/lib/sqoop/LICENSE.txt +/usr/lib/sqoop/NOTICE.txt + http://git-wip-us.apache.org/repos/asf/bigtop/blob/c7e814b5/bigtop-packages/src/deb/sqoop/sqoop.install ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/sqoop/sqoop.install b/bigtop-packages/src/deb/sqoop/sqoop.install index 935887f..a5011a9 100644 --- a/bigtop-packages/src/deb/sqoop/sqoop.install +++ b/bigtop-packages/src/deb/sqoop/sqoop.install @@ -2,6 +2,9 @@ /etc/sqoop /etc/default/sqoop-server /usr/lib/sqoop/bin/setenv.sh +/usr/lib/sqoop/bin/sqoop-sys.sh /usr/lib/sqoop/webapps /usr/lib/sqoop/tomcat-deployment.sh /var/lib/sqoop +/usr/lib/bigtop-tomcat/lib/sqoop-tomcat*.jar + http://git-wip-us.apache.org/repos/asf/bigtop/blob/c7e814b5/bigtop-packages/src/rpm/sqoop/SPECS/sqoop.spec ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/rpm/sqoop/SPECS/sqoop.spec b/bigtop-packages/src/rpm/sqoop/SPECS/sqoop.spec index 875acbd..944bc8d 100644 --- a/bigtop-packages/src/rpm/sqoop/SPECS/sqoop.spec +++ b/bigtop-packages/src/rpm/sqoop/SPECS/sqoop.spec @@ -61,13 +61,13 @@ Source0: %{name}-%{sqoop_base_version}.tar.gz Source1: do-component-build Source2: install_%{name}.sh Source3: sqoop.sh -Source4: sqoop.properties -Source5: catalina.properties -Source7: sqoop.default -Source8: init.d.tmpl -Source9: sqoop-server.svc -Source10: sqoop-server.sh -Source11: tomcat-deployment.sh +Source4: catalina.properties +Source5: sqoop.default +Source6: init.d.tmpl +Source7: sqoop-server.svc +Source8: sqoop-server.sh +Source9: tomcat-deployment.sh +Source10: sqoop-tool.sh Buildarch: noarch Requires: hadoop-client, bigtop-utils >= 0.7, bigtop-tomcat, %{name}-client = %{version}-%{release} @@ -177,15 +177,20 @@ fi %config(noreplace) /etc/default/sqoop-server %{lib_sqoop}/webapps %{lib_sqoop}/bin/setenv.sh +%{lib_sqoop}/bin/sqoop-sys.sh %{lib_sqoop}/tomcat-deployment.sh +/usr/lib/bigtop-tomcat/lib/sqoop-tomcat-*.jar %defattr(0755,sqoop,sqoop) /var/lib/sqoop %files client %attr(0755,root,root) /usr/bin/sqoop +/usr/bin/sqoop-tool %{lib_sqoop}/bin/sqoop.sh %{lib_sqoop}/client-lib +%{lib_sqoop}/LICENSE.txt +%{lib_sqoop}/NOTICE.txt %files server %attr(0755,root,root) %{initd_dir}/sqoop-server http://git-wip-us.apache.org/repos/asf/bigtop/blob/c7e814b5/bigtop.mk ---------------------------------------------------------------------- diff --git a/bigtop.mk b/bigtop.mk index e56386a..af9045b 100644 --- a/bigtop.mk +++ b/bigtop.mk @@ -107,8 +107,8 @@ $(eval $(call PACKAGE,hive,HIVE)) SQOOP_NAME=sqoop SQOOP_RELNOTES_NAME=Sqoop SQOOP_PKG_NAME=sqoop -SQOOP_BASE_VERSION=1.99.2 -SQOOP_PKG_VERSION=1.99.2 +SQOOP_BASE_VERSION=1.99.4 +SQOOP_PKG_VERSION=$(SQOOP_BASE_VERSION) SQOOP_RELEASE_VERSION=1 SQOOP_TARBALL_DST=$(SQOOP_NAME)-$(SQOOP_BASE_VERSION).tar.gz SQOOP_TARBALL_SRC=$(SQOOP_TARBALL_DST)
