Repository: incubator-hawq Updated Branches: refs/heads/HAWQ-1297 [created] 2c6f4ed55
HAWQ-1297. Update PXF build and service scripts to make it installable from source code Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/2c6f4ed5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/2c6f4ed5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/2c6f4ed5 Branch: refs/heads/HAWQ-1297 Commit: 2c6f4ed5598a49ecac03341230cdcb299c43db3a Parents: 8a5e65b Author: Shivram Mani <[email protected]> Authored: Wed Jan 25 17:13:29 2017 -0800 Committer: Shivram Mani <[email protected]> Committed: Wed Jan 25 17:13:29 2017 -0800 ---------------------------------------------------------------------- pxf/Makefile | 22 +++- pxf/build.gradle | 29 ++++- .../src/main/resources/pxf-private.classpath | 67 ++++++++++ pxf/pxf-service/src/scripts/pxf-env.sh | 6 + pxf/pxf-service/src/scripts/pxf-service | 125 ++++++++++++++----- 5 files changed, 209 insertions(+), 40 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2c6f4ed5/pxf/Makefile ---------------------------------------------------------------------- diff --git a/pxf/Makefile b/pxf/Makefile index 7669772..c8afe92 100644 --- a/pxf/Makefile +++ b/pxf/Makefile @@ -19,14 +19,23 @@ default: all ifneq "$(HD)" "" -BUILD_PARAMS= -Dhd=$(HD) + BUILD_PARAMS= -Dhd=$(HD) +else + ifneq "$(PXF_HOME)" "" + BUILD_PARAMS= -DdeployPath=$(PXF_HOME) + else ifneq "$(GPHOME)" "" + BUILD_PARAMS= -DdeployPath="$(GPHOME)/pxf" + else + @echo "Cannot invoke install without either PXF_HOME or GPHOME configured" + endif endif ifneq "$(LICENSE)" "" -BUILD_PARAMS+= -Plicense="$(LICENSE)" + BUILD_PARAMS+= -Plicense="$(LICENSE)" endif + ifneq "$(VENDOR)" "" -BUILD_PARAMS+= -Pvendor="$(VENDOR)" + BUILD_PARAMS+= -Pvendor="$(VENDOR)" endif help: @@ -39,6 +48,7 @@ help: @echo " - - VENDOR=<vendor name> - add vendor name to created RPMs" @echo " - tomcat - builds tomcat rpm from downloaded tarball" @echo " - - LICENSE and VENDOR parameters can be used as well" + @echo " - deploy - setup PXF along with tomcat in the configured deployPath" @echo " - doc - creates aggregate javadoc under docs" all: @@ -65,3 +75,9 @@ doc: .PHONY: tomcat tomcat: ./gradlew tomcatRpm $(BUILD_PARAMS) + +install: + ./gradlew install $(BUILD_PARAMS) + + + http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2c6f4ed5/pxf/build.gradle ---------------------------------------------------------------------- diff --git a/pxf/build.gradle b/pxf/build.gradle index 385bf08..3c6d591 100644 --- a/pxf/build.gradle +++ b/pxf/build.gradle @@ -473,8 +473,7 @@ task rpm(type: Copy, dependsOn: [subprojects.build, distSubprojects.buildRpm]) { // tomcat def tomcatName = "apache-tomcat-${tomcatVersion}" -def tomcatTargetDir = "tomcat/build/" - +def tomcatTargetDir = "tomcat/build" task tomcatGet << { @@ -506,12 +505,12 @@ task tomcatGet << { apply plugin: 'os-package' task tomcatRpm(type: Rpm) { - buildDir = 'tomcat/build/' + buildDir = "${tomcatTargetDir}" // clean should not delete the downloaded tarball // and RPM, so this is a bogus directory to delete instead. clean { - delete = 'tomcat/build/something' + delete = "${tomcatTargetDir}/something" } ospackage { @@ -544,6 +543,26 @@ task tomcatRpm(type: Rpm) { tomcatRpm.dependsOn tomcatGet +def pxfTargetDir = System.properties['deployPath'] ?: "build/" + +task install(type: Copy, dependsOn: [subprojects.build, tomcatGet]) { + into "${pxfTargetDir}" + subprojects { subProject -> + from("${project.name}/build/libs") { into 'lib' } + } + from("pxf-service/src/scripts/pxf-service") { + into 'bin' + fileMode 0755 + rename('pxf-service', 'pxf') + } + + from("${tomcatTargetDir}/${tomcatName}") { into 'apache-tomcat' } + from("pxf-service/src/main/resources") { into 'conf' } + from("pxf-service/src/configs/pxf-site.xml") { into 'conf' } + from("pxf-service/src/scripts/pxf-env.sh") { into 'conf' } + from("pxf-service/src/configs/tomcat") { into 'tomcat-templates' } +} + buildDir = '.' -apply plugin: 'nebula-aggregate-javadocs' \ No newline at end of file +apply plugin: 'nebula-aggregate-javadocs' http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2c6f4ed5/pxf/pxf-service/src/main/resources/pxf-private.classpath ---------------------------------------------------------------------- diff --git a/pxf/pxf-service/src/main/resources/pxf-private.classpath b/pxf/pxf-service/src/main/resources/pxf-private.classpath new file mode 100644 index 0000000..48ac2f9 --- /dev/null +++ b/pxf/pxf-service/src/main/resources/pxf-private.classpath @@ -0,0 +1,67 @@ +################################################################## +# This file contains the internal classpaths required to run PXF. +# Edit to set the base paths according to your specific package layout +# Adding new resources should be done using pxf-public.classpath file. +################################################################## + +# PXF Configuration +pxf/conf + +# Hadoop Configuration +hadoop/etc/hadoop + +# Hive Configuration +hive/conf + +# Hbase Configuration +hbase/conf + +# PXF Libraries +pxf/pxf-hbase-*[0-9].jar +pxf/pxf-hdfs-*[0-9].jar +pxf/pxf-hive-*[0-9].jar +pxf/pxf-json-*[0-9].jar + +# Hadoop Libraries +hadoop/share/hadoop/hdfs/hadoop-hdfs-*[0-9].jar +hadoop/share/hadoop/mapreduce/hadoop-mapreduce-client-core-*[0-9].jar +hadoop/share/hadoop/common/lib/hadoop-auth-*[0-9].jar +hadoop/share/hadoop/common/hadoop-common-*[0-9].jar +hadoop/share/hadoop/common/lib/asm-*[0-9].jar +hadoop/share/hadoop/common/lib/avro-*[0-9].jar +hadoop/share/hadoop/common/lib/commons-cli-*[0-9].jar +hadoop/share/hadoop/common/lib/commons-codec-*[0-9].jar +hadoop/share/hadoop/common/lib/commons-collections-*[0-9].jar +hadoop/share/hadoop/common/lib/commons-configuration-*[0-9].jar +hadoop/share/hadoop/common/lib/commons-io-*[0-9].jar +hadoop/share/hadoop/common/lib/commons-lang-*[0-9].jar +hadoop/share/hadoop/common/lib/commons-logging-*[0-9].jar +hadoop/share/hadoop/common/lib/guava-*[0-9].jar +hadoop/share/hadoop/common/lib/htrace-core-*[0-9]*.jar +hadoop/share/hadoop/common/lib/jetty-*.jar +hadoop/share/hadoop/common/lib/jackson-core-asl-*[0-9].jar +hadoop/share/hadoop/common/lib/jackson-mapper-asl-*[0-9].jar +hadoop/share/hadoop/common/lib/jersey-core-*[0-9].jar +hadoop/share/hadoop/common/lib/jersey-server-*[0-9].jar +hadoop/share/hadoop/common/lib/log4j-*[0-9].jar +hadoop/share/hadoop/common/lib/protobuf-java-*[0-9].jar +hadoop/share/hadoop/common/lib/slf4j-api-*[0-9].jar + +# Hive Libraries +hive/lib/antlr-runtime-*[0-9].jar +hive/lib/datanucleus-api-jdo-*[0-9].jar +hive/lib/datanucleus-core-*[0-9].jar +hive/lib/hive-exec-*[0-9].jar +hive/lib/hive-metastore-*[0-9].jar +hive/lib/jdo-api-*[0-9].jar +hive/lib/libfb303-*[0-9].jar +# when running on OSx, 1.0.5 or higher version is required +hive/lib/snappy-java-*[0-9].jar + +# HBase Libraries +hbase/lib/hbase-client-*[0-9].jar +hbase/lib/hbase-common-*[0-9].jar +hbase/lib/hbase-protocol-*[0-9].jar +hbase/lib/htrace-core-*[0-9]*.jar +hbase/lib/netty-*[0-9].Final.jar +hbase/lib/zookeeper-*[0-9].jar \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2c6f4ed5/pxf/pxf-service/src/scripts/pxf-env.sh ---------------------------------------------------------------------- diff --git a/pxf/pxf-service/src/scripts/pxf-env.sh b/pxf/pxf-service/src/scripts/pxf-env.sh index 2ac80a9..ba9b368 100644 --- a/pxf/pxf-service/src/scripts/pxf-env.sh +++ b/pxf/pxf-service/src/scripts/pxf-env.sh @@ -28,3 +28,9 @@ export CATALINA_OUT=${PXF_LOGDIR}/catalina.out # Path to Run directory export PXF_RUNDIR=/var/run/pxf + +# Configured user +export PXF_USER=pxf + +# Port +export PXF_PORT=51200 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2c6f4ed5/pxf/pxf-service/src/scripts/pxf-service ---------------------------------------------------------------------- diff --git a/pxf/pxf-service/src/scripts/pxf-service b/pxf/pxf-service/src/scripts/pxf-service index 7f62504..6809ca6 100644 --- a/pxf/pxf-service/src/scripts/pxf-service +++ b/pxf/pxf-service/src/scripts/pxf-service @@ -20,18 +20,12 @@ # pxf-service start/stop/initialize/status the PXF instance # -pxf_root=/usr/lib/pxf -env_script=/etc/pxf/conf/pxf-env.sh -tomcat_root=/opt/apache-tomcat -tomcat_templates=/opt/pxf/tomcat-templates -instance_root=/var/pxf -pxf_user=pxf -instance_name=pxf-service -instance_port=51200 -instance_owner=pxf:pxf - -curl=`which curl` +if [ -z $PXF_HOME ]; then + env_script=/etc/pxf/conf/pxf-env.sh +else + env_script=$PXF_HOME/conf/pxf-env.sh +fi # load pxf-env.sh script if [ ! -f $env_script ]; then @@ -40,6 +34,29 @@ else source $env_script fi +pxf_user=$PXF_USER +instance_port=$PXF_PORT +instance_name=pxf-service + +if [ -z $PXF_HOME ]; then + # RPM based setup + pxf_root=/usr/lib/pxf + tomcat_root=/opt/apache-tomcat + tomcat_templates=/opt/pxf/tomcat-templates + instance_root=/var/pxf + instance_owner=$pxf_user:$pxf_user +else + # OSS/Source code based setup + pxf_root=$PXF_HOME/lib + tomcat_root=$PXF_HOME/apache-tomcat + tomcat_templates=$PXF_HOME/tomcat-templates + instance_root=$PXF_HOME + instance_owner=$pxf_user +fi + +curl=`which curl` + + # validate JAVA_HOME if [ ! -x $JAVA_HOME/bin/java ]; then echo ERROR: \$JAVA_HOME is invalid @@ -61,7 +78,7 @@ function createInstance() return 1 fi - chown $instance_owner -R $instance_root + chown -R $instance_owner $instance_root chmod 700 $instance_root/$instance_name return 0 @@ -97,7 +114,7 @@ function configureInstance() # set pid catalinaEnv=$instance_root/$instance_name/bin/setenv.sh cat $catalinaEnv | \ - sed "s|^[[:blank:]]*CATALINA_PID=.*$|CATALINA_PID=$PXF_RUNDIR/catalina.pid|g" \ + sed -e "s|^[[:blank:]]*CATALINA_PID=.*$|CATALINA_PID=$PXF_RUNDIR/catalina.pid|g" \ > ${catalinaEnv}.tmp rm $catalinaEnv mv ${catalinaEnv}.tmp $catalinaEnv @@ -176,7 +193,7 @@ function checkWebapp() # non zero otherwise function instanceExists() { - if [ ! -d $instance_root ]; then + if [ ! -d "$instance_root/$instance_name" ]; then return 1 fi @@ -193,39 +210,83 @@ function doInit() return 0 fi - createInstance || return 1 + createInstance || return 1 configureInstance || return 1 deployWebapp || return 1 } +# +# patchWebapp patches the webapp config files +# patch applied only if PXF_HOME is defined +# +function patchWebapp() +{ + if [ -z $PXF_HOME ]; then + # webapp doesn't require patch + return 0 + fi + pushd $instance_root/$instance_name/webapps || return 1 + rm -rf pxf + mkdir pxf + cd pxf + unzip -q ../pxf.war + popd + + context_file=$instance_root/$instance_name/webapps/pxf/META-INF/context.xml + cat $context_file | \ + sed -e "s:classpathFiles=\"[a-zA-Z0-9\/\;.-]*\":classpathFiles=\"$PXF_HOME\/conf\/pxf-private.classpath\":" \ + -e "s:secondaryClasspathFiles=\"[a-zA-Z0-9\/\;.-]*\":secondaryClasspathFiles=\"$PXF_HOME\/conf\/pxf-public.classpath\":" > context.xml.tmp + mv context.xml.tmp $context_file + + web_file=$instance_root/$instance_name/webapps/pxf/WEB-INF/web.xml + cat $web_file | \ + sed "s:<param-value>.*pxf-log4j.properties<\/param-value>:<param-value>$PXF_HOME\/conf\/pxf-log4j.properties<\/param-value>:" > web.xml.tmp + mv web.xml.tmp $web_file +} + +function commandWebapp() +{ + command=$1 + pushd $instance_root + su $pxf_user -c "$instance_root/$instance_name/bin/catalina.sh $command" + if [ $? -ne 0 ]; then + return 1 + fi + popd +} + # -# doStartStop handles start/stop commands -# commands are executed as the user $pxf_user +# doStart handles start command +# command is executed as the user $pxf_user # # after start, uses checkWebapp to verify the PXF webapp was loaded # successfully # -function doStartStop() +function doStart() { - command=$1 - instanceExists if [ $? -ne 0 ]; then echo ERROR: cant find PXF instance, maybe call init? return 1 fi + patchWebapp || return 1 + commandWebapp start || return 1 + checkWebapp 300 || return 1 +} - pushd $instance_root - su $pxf_user -c "$instance_root/$instance_name/bin/catalina.sh $command" +# +# doStart handles stop command +# command is executed as the user $pxf_user +# +# +function doStop() +{ + instanceExists if [ $? -ne 0 ]; then + echo ERROR: cant find PXF instance, maybe call init? return 1 - fi - popd - - if [ "$command" = "start" ]; then - # try to connect for 5 minutes - checkWebapp 300 || return 1 fi + commandWebapp stop || return 1 } function doStatus() @@ -240,15 +301,15 @@ case "$command" in doInit ;; "start" ) - doStartStop $command + doStart ;; "stop" ) - doStartStop $command + doStop ;; "restart" ) - doStartStop stop + doStart sleep 1s - doStartStop start + doStop ;; "status" ) doStatus
