Repository: incubator-trafodion Updated Branches: refs/heads/master b2fe5fd7a -> 6eec3fbb1
[TRAFODION-1522] Consolidate HBase build dependencies Centralized a few variables in sqenvcom.sh for version number of dependencies (Hive, Thrift, HBase). These are items pulled in as shared libraries and also as jars (via maven). We want to make sure these are consistent. Each flavor of hbase-trx depends on distro-specific HBase for compatibility. Other components (SQL, TM, hbase_utilities) have general dependency, so we choose one hbase and hbase-trx to build against. We specify this in single location (sqenvcom.sh) for consistency. Also removed two pom.xml files no longer used. Minor changes to setupdir target to clarify what is happening. Tested both methods of building: with and without install_local_hadoop prior to build. Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/e49325c8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/e49325c8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/e49325c8 Branch: refs/heads/master Commit: e49325c83008bf39f3cbd10ab6b2029835489f1d Parents: f712ccb Author: Steve Varnau <[email protected]> Authored: Thu Feb 25 17:57:26 2016 +0000 Committer: Steve Varnau <[email protected]> Committed: Fri Feb 26 21:11:33 2016 +0000 ---------------------------------------------------------------------- core/sqf/Makefile | 32 +- core/sqf/hbase_utilities/pom.xml | 2 +- core/sqf/makelink | 33 -- core/sqf/sqenvcom.sh | 43 +-- core/sqf/sql/scripts/install_local_hadoop | 6 +- core/sqf/src/seatrans/hbase-trx/Makefile | 16 +- core/sqf/src/seatrans/hbase-trx/genvers | 3 +- .../seatrans/hbase-trx/hbase-protocol/pom.xml | 191 ----------- core/sqf/src/seatrans/hbase-trx/pom.xml | 342 ------------------- core/sqf/src/seatrans/tm/hbasetmlib2/pom.xml | 6 +- core/sql/nskgmake/Makerules.linux | 19 +- core/sql/pom.xml | 10 +- 12 files changed, 61 insertions(+), 642 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e49325c8/core/sqf/Makefile ---------------------------------------------------------------------- diff --git a/core/sqf/Makefile b/core/sqf/Makefile index 873da82..6c3cae9 100644 --- a/core/sqf/Makefile +++ b/core/sqf/Makefile @@ -176,34 +176,34 @@ setupdir: mkdir -p $(EXPORTLIB_DIR) mkdir -p $(EXPORTINCSQL_DIR) mkdir -p $(EXPORTINCEVL_DIR) + @# side-effect directory creation in these targets -cd monitor/linux; $(MAKE) setup -cd src/win; $(MAKE) setup -cd src/seabed; $(MAKE) setup -cd src/tm; $(MAKE) setup - cd sql; $(MAKE) WROOT=$(SQL_W) setup -cd sqevlog; $(MAKE) setup - -@./makelink ../../mondump $(EXPORTBIN_DIR) - -@./makelink ../../sqid $(EXPORTBIN_DIR) - -@./makelink ../../sqinfo $(EXPORTBIN_DIR) - -@./makelink ../../sqvers $(EXPORTBIN_DIR) - @# SQL .mdf files - + @# builds/copies 3rd party dependencies as necessary + cd sql; $(MAKE) WROOT=$(SQL_W) setup + @# commands to export + -ln -sf ../../mondump $(EXPORTBIN_DIR) + -ln -sf ../../sqid $(EXPORTBIN_DIR) + -ln -sf ../../sqinfo $(EXPORTBIN_DIR) + -ln -sf ../../sqvers $(EXPORTBIN_DIR) @# SQL Messages .txt file - -@./makelink $(SQL_W)/bin/SqlciErrors.txt $(EXPORTINCSQL_DIR) - + -ln -sf $(SQL_W)/bin/SqlciErrors.txt $(EXPORTINCSQL_DIR) @# copy some header files -ln -sf $(SQL_W)/cli/sqlcli.h $(EXPORTINCSQL_DIR) -ln -sf $(SQL_W)/sqludr/sqludr.h $(EXPORTINCSQL_DIR) @# - -mkdir -p export/include/nsk - -cp -p inc/cextdecs/cextdecs.h export/include/nsk + mkdir -p export/include/nsk + -ln -sf inc/cextdecs/cextdecs.h export/include/nsk @# USTAT -mkdir -p export/lib/mx_ustat - -cp -f $(SQL_W)/ustat/USAS.sh export/lib/mx_ustat - -cp -f $(SQL_W)/ustat/FillAutoTbl.sh export/lib/mx_ustat - -cp -f $(SQL_W)/ustat/RunLogUstats.sh export/lib/mx_ustat - -cp -f $(SQL_W)/ustat/StopAutoStats.sh export/lib/mx_ustat - -cp -f $(SQL_W)/ustat/stats_profile export/lib/mx_ustat + -ln -sf $(SQL_W)/ustat/USAS.sh export/lib/mx_ustat + -ln -sf $(SQL_W)/ustat/FillAutoTbl.sh export/lib/mx_ustat + -ln -sf $(SQL_W)/ustat/RunLogUstats.sh export/lib/mx_ustat + -ln -sf $(SQL_W)/ustat/StopAutoStats.sh export/lib/mx_ustat + -ln -sf $(SQL_W)/ustat/stats_profile export/lib/mx_ustat # "setupdir" target clean up setupdir_clean: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e49325c8/core/sqf/hbase_utilities/pom.xml ---------------------------------------------------------------------- diff --git a/core/sqf/hbase_utilities/pom.xml b/core/sqf/hbase_utilities/pom.xml index c1d0d5d..4e77b68 100644 --- a/core/sqf/hbase_utilities/pom.xml +++ b/core/sqf/hbase_utilities/pom.xml @@ -29,7 +29,7 @@ <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <hadoop.version>2.5.0</hadoop.version> - <hbase.version>0.98.6-cdh5.3.0</hbase.version> + <hbase.version>${env.HBASE_DEP_VER}</hbase.version> <java.version>1.7</java.version> </properties> http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e49325c8/core/sqf/makelink ---------------------------------------------------------------------- diff --git a/core/sqf/makelink b/core/sqf/makelink deleted file mode 100755 index bcd36d1..0000000 --- a/core/sqf/makelink +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -# @@@ START COPYRIGHT @@@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# @@@ END COPYRIGHT @@@ - -if [ $# -ne 2 ]; then - echo "makelink <tgt> <link>" - exit 1 -fi - -name=`basename $1` -if [ ! -L $2/$name ]; then - echo ln -sf $1 $2 - ln -sf $1 $2 -fi - http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e49325c8/core/sqf/sqenvcom.sh ---------------------------------------------------------------------- diff --git a/core/sqf/sqenvcom.sh b/core/sqf/sqenvcom.sh index f40ee6f..72a8549 100644 --- a/core/sqf/sqenvcom.sh +++ b/core/sqf/sqenvcom.sh @@ -140,12 +140,18 @@ fi export MY_SQROOT=$PWD export SQ_HOME=$PWD +# general Hadoop & TRX dependencies - not distro specific, choose one to build against export HBASE_TRXDIR=$MY_SQROOT/export/lib -export HBASE_TRX_JAR=hbase-trx-cdh5_3-${TRAFODION_VER}.jar +export HBASE_TRX_ID=hbase-trx-cdh5_3 +export HBASE_DEP_VER=0.98.6-cdh5.3.0 +export HBASE_TRX_JAR=${HBASE_TRX_ID}-${TRAFODION_VER}.jar export UTIL_JAR=trafodion-utility-${TRAFODION_VER}.jar if [[ "$SQ_HBASE_DISTRO" = "HDP" ]]; then export HBASE_TRX_JAR=hbase-trx-hdp2_2-${TRAFODION_VER}.jar fi +# set common version to be consistent between shared lib and maven dependencies +export THRIFT_DEP_VER=0.9.0 +export HIVE_DEP_VER=0.13.1 # check for workstation env # want to make sure SQ_VIRTUAL_NODES is set in the shell running sqstart @@ -236,12 +242,6 @@ if [[ -e $MY_SQROOT/sql/scripts/sw_env.sh ]]; then # native library directories and include directories export HADOOP_LIB_DIR=$YARN_HOME/lib/native export HADOOP_INC_DIR=$YARN_HOME/include - if [ -z $THRIFT_LIB_DIR ]; then - export THRIFT_LIB_DIR=$TOOLSDIR/thrift-0.9.0/lib - fi - if [ -z $THRIFT_INC_DIR ]; then - export THRIFT_INC_DIR=$TOOLSDIR/thrift-0.9.0/include - fi export CURL_INC_DIR=/usr/include export CURL_LIB_DIR=/usr/lib64 # directories with jar files and list of jar files @@ -276,10 +276,6 @@ elif [[ -d /opt/cloudera/parcels/CDH ]]; then export HADOOP_LIB_DIR=/opt/cloudera/parcels/CDH/lib/hadoop/lib/native export HADOOP_INC_DIR=/opt/cloudera/parcels/CDH/include - ### Thrift not supported on Cloudera yet (so use TOOLSDIR download) - export THRIFT_LIB_DIR=$TOOLSDIR/thrift-0.9.0/lib - export THRIFT_INC_DIR=$TOOLSDIR/thrift-0.9.0/include - export CURL_INC_DIR=/usr/include export CURL_LIB_DIR=/usr/lib64 @@ -322,10 +318,6 @@ elif [[ -n "$(ls /usr/lib/hadoop/hadoop-*cdh*.jar 2>/dev/null)" ]]; then export HADOOP_LIB_DIR=/usr/lib/hadoop/lib/native export HADOOP_INC_DIR=/usr/include - ### Thrift not supported on Cloudera yet (so use TOOLSDIR download) - export THRIFT_LIB_DIR=$TOOLSDIR/thrift-0.9.0/lib - export THRIFT_INC_DIR=$TOOLSDIR/thrift-0.9.0/include - export CURL_INC_DIR=/usr/include export CURL_LIB_DIR=/usr/lib64 @@ -369,9 +361,6 @@ elif [[ -n "$(ls /etc/init.d/ambari* 2>/dev/null)" ]]; then # The supported HDP version, HDP 1.3 uses Hadoop 1 export USE_HADOOP_1=1 - ### Thrift not supported on Hortonworks yet (so use TOOLSDIR download) - export THRIFT_LIB_DIR=$TOOLSDIR/thrift-0.9.0/lib - export THRIFT_INC_DIR=$TOOLSDIR/thrift-0.9.0/include export CURL_INC_DIR=/usr/include export CURL_LIB_DIR=/usr/lib64 @@ -432,9 +421,6 @@ elif [[ -d /opt/mapr ]]; then MAPR_HIVEDIR=$(echo /opt/mapr/hive/hive-*) fi - ### Thrift not supported on MapR (so use TOOLSDIR download) - export THRIFT_LIB_DIR=$TOOLSDIR/thrift-0.9.0/lib - export THRIFT_INC_DIR=$TOOLSDIR/thrift-0.9.0/include export CURL_INC_DIR=/usr/include export CURL_LIB_DIR=/usr/lib64 @@ -553,13 +539,6 @@ EOF echo "**** ERROR: Unable to determine location of HBase lib directory" fi - if [[ -d $TOOLSDIR/thrift-0.9.0 ]]; then - # this is mostly for a build environment, where we need - # thrift from TOOLSDIR - export THRIFT_LIB_DIR=$TOOLSDIR/thrift-0.9.0/lib - export THRIFT_INC_DIR=$TOOLSDIR/thrift-0.9.0/include - fi - if [ -n "$HBASE_CNF_DIR" -a -n "$HADOOP_CNF_DIR" -a \ -d $APACHE_HADOOP_HOME/lib -a -d $APACHE_HBASE_HOME/lib ]; then # We are on a system with Apache HBase, probably without a distro @@ -615,6 +594,14 @@ EOF fi fi +# Common for all distros +if [[ -d $TOOLSDIR/thrift-${THRIFT_DEP_VER} ]]; then + # this is for a build environment, where we need + # thrift from TOOLSDIR + export THRIFT_LIB_DIR=$TOOLSDIR/thrift-${THRIFT_DEP_VER}/lib + export THRIFT_INC_DIR=$TOOLSDIR/thrift-${THRIFT_DEP_VER}/include +fi + # ---+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+- # end of customization variables http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e49325c8/core/sqf/sql/scripts/install_local_hadoop ---------------------------------------------------------------------- diff --git a/core/sqf/sql/scripts/install_local_hadoop b/core/sqf/sql/scripts/install_local_hadoop index 5115e40..5d01264 100755 --- a/core/sqf/sql/scripts/install_local_hadoop +++ b/core/sqf/sql/scripts/install_local_hadoop @@ -602,13 +602,13 @@ MYSQL_TAR=mysql-5.6.10-linux-glibc2.5-x86_64.tar.gz MYSQL_JDBC_URL=http://cdn.mysql.com/archives/mysql-connector-java-5.1 MYSQL_JDBC_TAR=mysql-connector-java-5.1.23.tar.gz -HIVE_MIRROR_URL=https://archive.apache.org/dist/hive/hive-0.13.1 -HIVE_PREFIX=apache-hive-0.13.1-bin +HIVE_MIRROR_URL=https://archive.apache.org/dist/hive/hive-${HIVE_DEP_VER} +HIVE_PREFIX=apache-hive-${HIVE_DEP_VER}-bin HIVE_TAR=${HIVE_PREFIX}.tar.gz HBASE_MIRROR_URL=http://archive.cloudera.com/cdh5/cdh/5 -HBASE_TAR=hbase-0.98.6-cdh5.3.0.tar.gz +HBASE_TAR=hbase-${HBASE_DEP_VER}.tar.gz if [[ "$SQ_HBASE_DISTRO" = "HDP" ]]; then HBASE_TAR=hbase-0.98.4.2.2.0.0-2041-hadoop2.tar.gz fi http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e49325c8/core/sqf/src/seatrans/hbase-trx/Makefile ---------------------------------------------------------------------- diff --git a/core/sqf/src/seatrans/hbase-trx/Makefile b/core/sqf/src/seatrans/hbase-trx/Makefile index 5cab6dd..51ca957 100644 --- a/core/sqf/src/seatrans/hbase-trx/Makefile +++ b/core/sqf/src/seatrans/hbase-trx/Makefile @@ -34,29 +34,29 @@ GENVERS =./genvers all: build_all -jdk_1_7_cdh: - $(MAKE) build_chk_cdh +jdk_1_7_cdh: build_chk_cdh set -o pipefail && $(MAVEN) -f pom.xml.cdh package install -DskipTests | tee -a build_trx.log + mkdir -p $(MY_SQROOT)/export/lib cp -pf target/$(BLD_HBASE_CDH_TRX_JARNAME) $(MY_SQROOT)/export/lib $(RM) $(VFILE) -jdk_1_7_mapr: - $(MAKE) build_chk_mapr +jdk_1_7_mapr: build_chk_mapr set -o pipefail && $(MAVEN) -f pom.xml.mapr package -DskipTests | tee -a build_trx.log + mkdir -p $(MY_SQROOT)/export/lib cp -pf target/$(BLD_HBASE_MAPR_TRX_JARNAME) $(MY_SQROOT)/export/lib $(RM) $(VFILE) -jdk_1_7_hdp: - $(MAKE) build_chk_hdp +jdk_1_7_hdp: build_chk_hdp set -o pipefail && $(MAVEN) -f pom.xml.hdp package -DskipTests | tee -a build_trx.log + mkdir -p $(MY_SQROOT)/export/lib cp -pf target/$(BLD_HBASE_HDP_TRX_JARNAME) $(MY_SQROOT)/export/lib $(RM) $(VFILE) -jdk_1_7_apache: - $(MAKE) build_chk_apache +jdk_1_7_apache: build_chk_apache echo "$(MAVEN) package -DskipTests" echo "### For full Maven output, see file build_trx.log" set -o pipefail && $(MAVEN) -f pom.xml.apache package -DskipTests | tee build_trx.log + mkdir -p $(MY_SQROOT)/export/lib cp -pf target/$(BLD_HBASE_APACHE_TRX_JARNAME) $(MY_SQROOT)/export/lib $(RM) $(VFILE) http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e49325c8/core/sqf/src/seatrans/hbase-trx/genvers ---------------------------------------------------------------------- diff --git a/core/sqf/src/seatrans/hbase-trx/genvers b/core/sqf/src/seatrans/hbase-trx/genvers index ccc880c..49c9bae 100755 --- a/core/sqf/src/seatrans/hbase-trx/genvers +++ b/core/sqf/src/seatrans/hbase-trx/genvers @@ -1,3 +1,5 @@ +#!/bin/sh + # # @@@ START COPYRIGHT @@@ # @@ -21,6 +23,5 @@ # @@@ END COPYRIGHT @@@ # -#!/bin/sh CV_VERS=1.0.1 $MY_SQROOT/export/include/SCMBuildJava.sh $CV_VERS http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e49325c8/core/sqf/src/seatrans/hbase-trx/hbase-protocol/pom.xml ---------------------------------------------------------------------- diff --git a/core/sqf/src/seatrans/hbase-trx/hbase-protocol/pom.xml b/core/sqf/src/seatrans/hbase-trx/hbase-protocol/pom.xml deleted file mode 100755 index ee35457..0000000 --- a/core/sqf/src/seatrans/hbase-trx/hbase-protocol/pom.xml +++ /dev/null @@ -1,191 +0,0 @@ -<?xml version="1.0"?> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <!-- - /** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - --> - <modelVersion>4.0.0</modelVersion> - <parent> - <artifactId>hbase</artifactId> - <groupId>org.apache.hbase</groupId> - <version>0.98.3</version> - <relativePath>..</relativePath> - </parent> - - <artifactId>hbase-protocol</artifactId> - <name>HBase - Protocol</name> - <description>Protobuf protocol classes used by HBase to communicate.</description> - - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-site-plugin</artifactId> - <configuration> - <skip>true</skip> - </configuration> - </plugin> - <!-- Make a jar and put the sources in the jar --> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-source-plugin</artifactId> - </plugin> - <plugin> - <!--Make it so assembly:single does nothing in here--> - <artifactId>maven-assembly-plugin</artifactId> - <version>${maven.assembly.version}</version> - <configuration> - <skipAssembly>true</skipAssembly> - </configuration> - </plugin> - <plugin> - <artifactId>maven-surefire-plugin</artifactId> - <!-- Always skip the second part executions, since we only run simple unit tests in this module --> - <executions> - <execution> - <id>secondPartTestsExecution</id> - <phase>test</phase> - <goals> - <goal>test</goal> - </goals> - <configuration> - <skip>true</skip> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - <pluginManagement> - <plugins> - <!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself.--> - <plugin> - <groupId>org.eclipse.m2e</groupId> - <artifactId>lifecycle-mapping</artifactId> - <version>1.0.0</version> - <configuration> - <lifecycleMappingMetadata> - <pluginExecutions> - <pluginExecution> - <pluginExecutionFilter> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-maven-plugins</artifactId> - <versionRange>[2.0.5-alpha,)</versionRange> - <goals> - <goal>protoc</goal> - </goals> - </pluginExecutionFilter> - <action> - <ignore></ignore> - </action> - </pluginExecution> - </pluginExecutions> - </lifecycleMappingMetadata> - </configuration> - </plugin> - </plugins> - </pluginManagement> - </build> - - <dependencies> - <!-- Intra-project dependencies --> - <!-- General dependencies --> - <dependency> - <groupId>com.google.protobuf</groupId> - <artifactId>protobuf-java</artifactId> - </dependency> - </dependencies> - - <profiles> - <!-- Skip the tests in this module --> - <profile> - <id>skip-rpc-tests</id> - <activation> - <property> - <name>skip-rpc-tests</name> - </property> - </activation> - <properties> - <surefire.skipFirstPart>true</surefire.skipFirstPart> - </properties> - </profile> - <profile> - <id>compile-protobuf</id> - <activation> - <property> - <name>compile-protobuf</name> - </property> - </activation> - <build> - <plugins> - <plugin> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-maven-plugins</artifactId> - <executions> - <execution> - <id>compile-protoc</id> - <phase>generate-sources</phase> - <goals> - <goal>protoc</goal> - </goals> - <configuration> - <imports> - <param>${basedir}/src/main/protobuf</param> - </imports> - <source> - <directory>${basedir}/src/main/protobuf</directory> - <includes> - <include>AccessControl.proto</include> - <include>Admin.proto</include> - <include>Aggregate.proto</include> - <include>Authentication.proto</include> - <include>Cell.proto</include> - <include>Client.proto</include> - <include>ClusterId.proto</include> - <include>ClusterStatus.proto</include> - <include>Comparator.proto</include> - <include>Encryption.proto</include> - <include>ErrorHandling.proto</include> - <include>Filter.proto</include> - <include>FS.proto</include> - <include>HBase.proto</include> - <include>HFile.proto</include> - <include>LoadBalancer.proto</include> - <include>MapReduce.proto</include> - <include>Master.proto</include> - <include>MultiRowMutation.proto</include> - <include>RegionServerStatus.proto</include> - <include>RowProcessor.proto</include> - <include>RPC.proto</include> - <include>SecureBulkLoad.proto</include> - <include>Tracing.proto</include> - <include>VisibilityLabels.proto</include> - <include>WAL.proto</include> - <include>ZooKeeper.proto</include> - </includes> - </source> - <!--<output>${project.build.directory}/generated-sources/java</output>--> - <output>${basedir}/src/main/java/</output> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - </profile> - </profiles> -</project> http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e49325c8/core/sqf/src/seatrans/hbase-trx/pom.xml ---------------------------------------------------------------------- diff --git a/core/sqf/src/seatrans/hbase-trx/pom.xml b/core/sqf/src/seatrans/hbase-trx/pom.xml deleted file mode 100755 index 4a9ba48..0000000 --- a/core/sqf/src/seatrans/hbase-trx/pom.xml +++ /dev/null @@ -1,342 +0,0 @@ -<?xml version="1.0"?> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <!-- -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ ---> -<!-- - <parent> - <artifactId>hbase</artifactId> - <groupId>org.apache.hbase</groupId> - <version>0.98.3</version> - <relativePath>..</relativePath> - </parent> ---> - <repositories> - <repository> - <id>cloudera</id> - <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url> - </repository> - </repositories> - - <properties> - <hadoop.version>2.5.0</hadoop.version> - <hbase.version>0.98.1-cdh5.1.0</hbase.version> - <protobuf.version>2.5.0</protobuf.version> - <protocVersion>2.5.0</protocVersion> - <!--<hbase.version>0.98.3-hadoop2</hbase.version>--> - <!--<hbase.version>0.98.3-hadoop1</hbase.version>--> - <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <java.version>1.7</java.version> - <trx-suffix></trx-suffix> - </properties> - - - <groupId>org.apache</groupId> - <modelVersion>4.0.0</modelVersion> - <artifactId>hbase-trx</artifactId> - <version>${env.TRAFODION_VER}</version> - <name>HBase - Trx</name> - <description>Trx of HBase usage</description> - - - <build> - <defaultGoal>package</defaultGoal> - <!-- allow for different build targets with separate classes directories and - output jars, used for Java 6 compile for now --> - <outputDirectory>${project.build.directory}/classes${trx-suffix}</outputDirectory> - <finalName>${project.artifactId}-${project.version}${trx-suffix}</finalName> - <!-- Some plugins (javadoc for example) can be used in the normal build- and the site phase. - These plugins inherit their options from the <reporting> section below. These settings - can be overwritten here. --> - <plugins> - <plugin> - <!--Make it so assembly:single does nothing in here--> - <artifactId>maven-assembly-plugin</artifactId> - <version>2.2-beta-5</version> - <configuration> - <skipAssembly>true</skipAssembly> - </configuration> - </plugin> - <plugin> - <artifactId>maven-surefire-plugin</artifactId> - <version>2.5</version> - <configuration> - <!-- Have to set the groups here because we only do - split tests in this package, so groups on live in this module --> - <groups>${surefire.firstPartGroups}</groups> - </configuration> - </plugin> - <!-- Make a jar and put the sources in the jar --> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-source-plugin</artifactId> - <version>2.1.1</version> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-jar-plugin</artifactId> - <configuration> - <archive> - <manifestFile>hbase-trx.jar.versions</manifestFile> - </archive> - </configuration> - </plugin> - </plugins> - </build> - - - - <dependencies> - <dependency> - <groupId>org.apache.hbase</groupId> - <artifactId>hbase-common</artifactId> - <version>${hbase.version}</version> - </dependency> - <dependency> - <groupId>org.apache.hbase</groupId> - <artifactId>hbase-protocol</artifactId> - <version>${hbase.version}</version> - </dependency> - <dependency> - <groupId>org.apache.hbase</groupId> - <artifactId>hbase-client</artifactId> - <version>${hbase.version}</version> - </dependency> - <dependency> - <groupId>org.apache.hbase</groupId> - <artifactId>hbase-server</artifactId> - <version>${hbase.version}</version> - </dependency> - <dependency> - <groupId>org.apache.hbase</groupId> - <artifactId>hbase-thrift</artifactId> - <version>${hbase.version}</version> - </dependency> - <dependency> - <groupId>org.apache.hbase</groupId> - <artifactId>hbase-testing-util</artifactId> - <scope>test</scope> - <version>${hbase.version}</version> - </dependency> - - <dependency> - <groupId>org.apache.thrift</groupId> - <artifactId>libthrift</artifactId> - <version>0.9.1</version> - </dependency> - <dependency> - <groupId>commons-logging</groupId> - <artifactId>commons-logging</artifactId> - <version>1.1.3</version> - </dependency> - <dependency> - <groupId>org.apache.zookeeper</groupId> - <artifactId>zookeeper</artifactId> - <version>3.4.5</version> - </dependency> - <dependency> - <groupId>com.google.protobuf</groupId> - <artifactId>protobuf-java</artifactId> - <version>2.5.0</version> - </dependency> - </dependencies> - <profiles> - <!-- Skip the tests in this module --> - <profile> - <id>skipTrxTests</id> - <activation> - <property> - <name>skipTrxTests</name> - </property> - </activation> - <properties> - <surefire.skipFirstPart>true</surefire.skipFirstPart> - <surefire.skipSecondPart>true</surefire.skipSecondPart> - </properties> - </profile> - - <!-- Profiles for building against different hadoop versions --> - <!-- There are a lot of common dependencies used here, should investigate -if we can combine these profiles somehow --> - <!-- profile against Hadoop 1.1.x: This is the default. It has to have the same - activation property as the parent Hadoop 1.1.x profile to make sure it gets run at - the same time. --> - <profile> - <id>hadoop-1.1</id> - <activation> - <property> - <!--Below formatting for dev-support/generate-hadoopX-poms.sh--> - <!--h1--><name>hadoop.profile</name><value>1.1</value> - </property> - </activation> - <dependencies> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-core</artifactId> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-test</artifactId> - </dependency> - </dependencies> - </profile> - <!-- - profile for building against Hadoop 2.0.0-alpha. Activate using: - mvn -Dhadoop.profile=2.0 - --> - <profile> - <id>hadoop-2.0</id> - <activation> - <property> - <!--Below formatting for dev-support/generate-hadoopX-poms.sh--> - <!--h2--><name>!hadoop.profile</name> - </property> - </activation> - <dependencies> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-mapreduce-client-core</artifactId> - <version>${hadoop.version}</version> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - <version>${hadoop.version}</version> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <artifactId>maven-dependency-plugin</artifactId> - <executions> - <execution> - <id>create-mrapp-generated-classpath</id> - <phase>generate-test-resources</phase> - <goals> - <goal>build-classpath</goal> - </goals> - <configuration> - <!-- needed to run the unit test for DS to generate - the required classpath that is required in the env - of the launch container in the mini mr/yarn cluster - --> - <outputFile>${project.build.directory}/test-classes/mrapp-generated-classpath</outputFile> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - </profile> - <!-- - profile for building against Hadoop 3.0.x. Activate using: - mvn -Dhadoop.profile=3.0 - --> - <profile> - <id>hadoop-3.0</id> - <activation> - <property> - <name>hadoop.profile</name> - <value>3.0</value> - </property> - </activation> - <properties> - <hadoop.version>3.0-SNAPSHOT</hadoop.version> - </properties> - <dependencies> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-annotations</artifactId> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-minicluster</artifactId> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <artifactId>maven-dependency-plugin</artifactId> - <executions> - <execution> - <id>create-mrapp-generated-classpath</id> - <phase>generate-test-resources</phase> - <goals> - <goal>build-classpath</goal> - </goals> - <configuration> - <!-- needed to run the unit test for DS to generate - the required classpath that is required in the env - of the launch container in the mini mr/yarn cluster - --> - <outputFile>${project.build.directory}/test-classes/mrapp-generated-classpath</outputFile> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - </profile> - <profile> - <id>compile-protobuf</id> - <activation> - <property> - <name>compile-protobuf</name> - </property> - </activation> - <build> - <plugins> - <plugin> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-maven-plugins</artifactId> - <executions> - <execution> - <id>compile-protoc</id> - <phase>generate-sources</phase> - <goals> - <goal>protoc</goal> - </goals> - <configuration> - <protobuf.version>2.5.0</protobuf.version> - <protocVersion>2.5.0</protocVersion> - <imports> - <param>${basedir}/src/main/protobuf</param> - <param>${basedir}/hbase-protocol/src/main/protobuf</param> - </imports> - <source> - <directory>${basedir}/src/main/protobuf</directory> - <includes> - <include>TrxRegion.proto</include> - <include>SsccRegion.proto</include> - </includes> - </source> - <output>${basedir}/src/main/java/</output> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - </profile> - </profiles> -</project> http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e49325c8/core/sqf/src/seatrans/tm/hbasetmlib2/pom.xml ---------------------------------------------------------------------- diff --git a/core/sqf/src/seatrans/tm/hbasetmlib2/pom.xml b/core/sqf/src/seatrans/tm/hbasetmlib2/pom.xml index d13e43e..d833f45 100644 --- a/core/sqf/src/seatrans/tm/hbasetmlib2/pom.xml +++ b/core/sqf/src/seatrans/tm/hbasetmlib2/pom.xml @@ -28,8 +28,8 @@ <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <hbase.version>0.98.1-cdh5.1.0</hbase.version> - <hbase-trx.id>hbase-trx-cdh5_3</hbase-trx.id> + <hbase.version>${env.HBASE_DEP_VER}</hbase.version> + <hbase-trx.id>${env.HBASE_TRX_ID}</hbase-trx.id> <java.version>1.7</java.version> </properties> @@ -49,7 +49,7 @@ <artifactId>${hbase-trx.id}</artifactId> <version>${env.TRAFODION_VER}</version> <scope>system</scope> - <systemPath>${env.MY_SQROOT}/export/lib/${hbase-trx.id}-${env.TRAFODION_VER}.jar</systemPath> + <systemPath>${env.HBASE_TRXDIR}/${env.HBASE_TRX_JAR}</systemPath> </dependency> <dependency> <groupId>org.slf4j</groupId> http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e49325c8/core/sql/nskgmake/Makerules.linux ---------------------------------------------------------------------- diff --git a/core/sql/nskgmake/Makerules.linux b/core/sql/nskgmake/Makerules.linux index 1fce092..a707a5e 100755 --- a/core/sql/nskgmake/Makerules.linux +++ b/core/sql/nskgmake/Makerules.linux @@ -418,18 +418,15 @@ linuxcleandebug linuxdebugclean linuxcleanrelease linuxreleaseclean : clean # need extra command to get the link (without extra libs) copytoolslibs: mkdir -p $(LIBROOT) - -(for i in $(HADOOP_LIB_DIR)/$(LIBHDFS_SO) \ - $(HADOOP_LIB_DIR)/$(LIBHADOOP_SO) \ - $(THRIFT_LIB_DIR)/$(THRIFT_SO) \ - $(LOG4CXX_LIB_DIR)/$(LOG4CXX_SO); \ - do\ - cp -Pf $$i* $(LIBROOT); \ - done\ - ) - -cp -Pf $(THRIFT_LIB_DIR)/libthrift.so $(LIBROOT) + # these must be found + cp -Pf $(LOG4CXX_LIB_DIR)/$(LOG4CXX_SO)* $(LIBROOT) + cp -Pf $(THRIFT_LIB_DIR)/$(THRIFT_SO)* $(LIBROOT) + cp -Pf $(THRIFT_LIB_DIR)/libthrift.so $(LIBROOT) + # if these are not found, then... + -cp -Pf $(HADOOP_LIB_DIR)/$(LIBHDFS_SO)* $(LIBROOT) + -cp -Pf $(HADOOP_LIB_DIR)/$(LIBHADOOP_SO)* $(LIBROOT) -cp -Pf $(HADOOP_INC_DIR)/hdfs.h $(MY_SQROOT)/export/include - # if the above did not copy the files yet, - # download Hadoop and make the hdfs library + # download Hadoop source and build the hdfs library get_libhdfs_files --verbose linuxmklinksdebug linuxmklinksrelease: copytoolslibs http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e49325c8/core/sql/pom.xml ---------------------------------------------------------------------- diff --git a/core/sql/pom.xml b/core/sql/pom.xml index 50616e4..83472fa 100755 --- a/core/sql/pom.xml +++ b/core/sql/pom.xml @@ -28,10 +28,10 @@ <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <hbase.version>0.98.1-cdh5.1.0</hbase.version> - <hbase-trx.id>hbase-trx-cdh5_3</hbase-trx.id> - <hive.version>0.13.1</hive.version> - <thrift.version>0.9.0</thrift.version> + <hbase.version>${env.HBASE_DEP_VER}</hbase.version> + <hbase-trx.id>${env.HBASE_TRX_ID}</hbase-trx.id> + <hive.version>${env.HIVE_DEP_VER}</hive.version> + <thrift.version>${env.THRIFT_DEP_VER}</thrift.version> <java.version>1.7</java.version> </properties> @@ -51,7 +51,7 @@ <artifactId>${hbase-trx.id}</artifactId> <version>${env.TRAFODION_VER}</version> <scope>system</scope> - <systemPath>${env.MY_SQROOT}/export/lib/${hbase-trx.id}-${env.TRAFODION_VER}.jar</systemPath> + <systemPath>${env.HBASE_TRXDIR}/${env.HBASE_TRX_JAR}</systemPath> </dependency> <dependency> <groupId>org.apache.hive</groupId>
