update
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/f1c7f3a7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/f1c7f3a7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/f1c7f3a7 Branch: refs/heads/master Commit: f1c7f3a7a896031fe37c261d26af52bd8eda611c Parents: 73c164b e24ea17 Author: mashengchen <[email protected]> Authored: Sun Mar 6 07:34:45 2016 +0000 Committer: mashengchen <[email protected]> Committed: Sun Mar 6 07:34:45 2016 +0000 ---------------------------------------------------------------------- core/.gitignore | 1 + core/Makefile | 5 +- core/conn/trafci/manualtests/README | 50 + core/conn/trafci/manualtests/statcheck.sql | 47 + .../src/org/trafodion/ci/DatabaseQuery.java | 3 +- .../odbc/odbcclient/unixcli/makefile.lnx | 2 +- .../odbc/odbcclient/unixcli/package/LICENSE | 1 - .../odbc/odbcclient/unixcli/package/license.txt | 18 - .../odbc/odbcclient/unixcli/package/mklnxpkg.sh | 6 +- .../unixcli/package/trafodbclnx_install.sh | 22 +- core/sqf/LocalSettingsTemplate.sh | 4 +- core/sqf/Makefile | 32 +- core/sqf/hbase_utilities/pom.xml | 2 +- core/sqf/makelink | 33 - core/sqf/sqenvcom.sh | 48 +- core/sqf/sql/scripts/install_local_hadoop | 5 +- core/sqf/src/seatrans/hbase-trx/Makefile | 13 +- core/sqf/src/seatrans/hbase-trx/genvers | 3 +- .../seatrans/hbase-trx/hbase-protocol/pom.xml | 191 -- core/sqf/src/seatrans/hbase-trx/pom.xml | 342 --- .../transactional/TransactionManager.java | 22 +- .../TransactionalScanner.java.tmpl | 14 +- core/sqf/src/seatrans/tm/hbasetmlib2/pom.xml | 6 +- .../java/org/trafodion/dtm/HBaseTxClient.java | 14 +- .../java/org/trafodion/dtm/TmAuditTlog.java | 23 +- core/sql/SqlCompilerDebugger/mk.sh | 3 + core/sql/arkcmp/CmpContext.cpp | 12 +- core/sql/arkcmp/CmpContext.h | 18 + core/sql/arkcmp/CmpStatement.cpp | 19 +- core/sql/arkcmp/CmpStatement.h | 1 + core/sql/bin/SqlciErrors.txt | 1 + core/sql/cli/Context.cpp | 106 +- core/sql/cli/Context.h | 13 +- core/sql/cli/ExSqlComp.cpp | 5 + core/sql/cli/Statement.cpp | 36 +- core/sql/comexe/CmpMessage.h | 23 + core/sql/comexe/ComTdbHbaseAccess.cpp | 3 + core/sql/comexe/ComTdbHbaseAccess.h | 4 +- core/sql/common/ComSmallDefs.h | 11 + core/sql/common/DateTimeType.cpp | 3 +- core/sql/executor/ExExeUtil.h | 1 + core/sql/executor/ExExeUtilGet.cpp | 55 +- core/sql/executor/ExFirstN.cpp | 45 +- core/sql/executor/ExFirstN.h | 4 +- core/sql/executor/ExHbaseAccess.cpp | 22 +- core/sql/executor/ExHbaseAccess.h | 1 + core/sql/executor/ExHbaseIUD.cpp | 11 +- core/sql/executor/ExStats.cpp | 13 +- core/sql/executor/HBaseClient_JNI.cpp | 41 +- core/sql/executor/HBaseClient_JNI.h | 7 +- core/sql/executor/ex_ddl.cpp | 2 + core/sql/executor/ex_transaction.cpp | 278 ++- core/sql/executor/ex_transaction.h | 2 +- core/sql/exp/ExpHbaseInterface.cpp | 34 +- core/sql/exp/ExpHbaseInterface.h | 16 +- core/sql/generator/GenPreCode.cpp | 13 - core/sql/generator/GenRelExeUtil.cpp | 31 +- core/sql/generator/GenRelMisc.cpp | 13 +- core/sql/generator/GenRelUpdate.cpp | 112 +- core/sql/generator/Generator.cpp | 5 + core/sql/lib_mgmt/.gitignore | 7 + core/sql/lib_mgmt/Makefile | 35 + core/sql/lib_mgmt/README.rst | 63 + core/sql/lib_mgmt/pom.xml | 108 + .../java/org/trafodion/libmgmt/FileMgmt.java | 645 ++++++ .../java/org/trafodion/libmgmt/Vproc.java.tmpl | 35 + .../lib_mgmt/src/main/resources/init_libmgmt.sh | 206 ++ .../com/trafodion/libmgmt/JarFileMgmtTest.java | 27 + core/sql/nskgmake/Makerules.linux | 19 +- core/sql/nskgmake/SqlCompilerDebugger/Makefile | 15 +- core/sql/optimizer/BindRelExpr.cpp | 86 +- core/sql/optimizer/ItemOther.h | 7 + core/sql/optimizer/NAColumn.h | 43 +- core/sql/optimizer/NARoutine.cpp | 2 +- core/sql/optimizer/NARoutineDB.h | 5 +- core/sql/optimizer/NATable.cpp | 58 +- core/sql/optimizer/NATable.h | 10 +- core/sql/optimizer/RelExeUtil.cpp | 155 +- core/sql/optimizer/RelExeUtil.h | 30 +- core/sql/optimizer/RelExpr.cpp | 2 +- core/sql/optimizer/RelUpdate.h | 2 + core/sql/parser/SqlParserAux.cpp | 2 +- core/sql/parser/StmtDDLAlter.cpp | 11 + .../sql/parser/StmtDDLAlterTableAlterColumn.cpp | 20 +- core/sql/parser/StmtDDLAlterTableAlterColumn.h | 17 +- core/sql/parser/StmtDDLNode.cpp | 14 + core/sql/parser/StmtDDLNode.h | 15 +- core/sql/parser/sqlparser.y | 6 +- core/sql/parser/ulexer.h | 21 + core/sql/pom.xml | 10 +- core/sql/regress/charsets/DIFF012.KNOWN.SB | 2 + core/sql/regress/compGeneral/EXPECTED071 | 8 +- core/sql/regress/core/DIFF116.KNOWN.SB | 21 + core/sql/regress/core/EXPECTED056.SB | 75 +- core/sql/regress/core/EXPECTED116 | 542 +++-- core/sql/regress/core/TEST029 | 2 +- core/sql/regress/core/TEST056 | 7 +- core/sql/regress/core/TEST116 | 123 +- core/sql/regress/executor/EXPECTED013.SB | 39 +- core/sql/regress/executor/EXPECTED140 | 40 +- core/sql/regress/fullstack2/EXPECTED062 | 3 + core/sql/regress/fullstack2/EXPECTED062.RELEASE | 3 + core/sql/regress/privs2/EXPECTED135 | 329 --- core/sql/regress/privs2/EXPECTED138 | 2 +- core/sql/regress/privs2/EXPECTED142 | 760 +++++++ core/sql/regress/privs2/TEST135 | 43 - core/sql/regress/privs2/TEST142 | 280 +++ core/sql/regress/seabase/EXPECTED010 | 460 ++-- core/sql/regress/seabase/EXPECTED011 | 102 +- core/sql/regress/seabase/EXPECTED016 | 35 +- core/sql/regress/seabase/EXPECTED020 | 233 ++ core/sql/regress/seabase/EXPECTED027 | 433 +++- core/sql/regress/seabase/TEST011 | 16 - core/sql/regress/seabase/TEST020 | 54 + core/sql/regress/seabase/TEST027 | 58 + core/sql/regress/tools/runregr_catman1.ksh | 3 +- core/sql/regress/tools/runregr_charsets.ksh | 3 +- core/sql/regress/tools/runregr_compGeneral.ksh | 3 +- core/sql/regress/tools/runregr_core.ksh | 3 +- core/sql/regress/tools/runregr_executor.ksh | 3 +- core/sql/regress/tools/runregr_fullstack2.ksh | 3 +- core/sql/regress/tools/runregr_hive.ksh | 4 +- core/sql/regress/tools/runregr_privs1.ksh | 3 +- core/sql/regress/tools/runregr_privs2.ksh | 3 +- core/sql/regress/tools/runregr_qat.ksh | 3 +- core/sql/regress/tools/runregr_seabase.ksh | 4 +- core/sql/regress/tools/runregr_udr.ksh | 9 +- core/sql/regress/tools/sbdefs | 1 + core/sql/sqlcomp/CmpDescribe.cpp | 186 +- core/sql/sqlcomp/CmpSeabaseDDL.h | 104 +- core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp | 42 +- core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp | 490 +++-- core/sql/sqlcomp/CmpSeabaseDDLindex.cpp | 98 +- core/sql/sqlcomp/CmpSeabaseDDLrepos.cpp | 2 +- core/sql/sqlcomp/CmpSeabaseDDLroutine.cpp | 8 +- core/sql/sqlcomp/CmpSeabaseDDLschema.cpp | 26 +- core/sql/sqlcomp/CmpSeabaseDDLtable.cpp | 1990 ++++++++++++------ core/sql/sqlcomp/CmpSeabaseDDLupgrade.cpp | 14 +- core/sql/sqlcomp/CmpSeabaseDDLupgrade.h | 6 +- core/sql/sqlcomp/CmpSeabaseDDLview.cpp | 30 +- core/sql/sqlcomp/DefaultConstants.h | 8 + core/sql/sqlcomp/PrivMgrDefs.h | 4 +- core/sql/sqlcomp/PrivMgrMD.cpp | 171 +- core/sql/sqlcomp/PrivMgrMD.h | 132 +- core/sql/sqlcomp/PrivMgrPrivileges.cpp | 793 ++++--- core/sql/sqlcomp/PrivMgrPrivileges.h | 48 +- core/sql/sqlcomp/nadefaults.cpp | 9 +- .../java/org/trafodion/sql/HBaseClient.java | 70 +- core/sql/ustat/hs_globals.cpp | 20 - dcs/bin/stop-dcs.sh | 2 +- .../dcs/master/listener/ConnectReply.java | 60 +- .../src/asciidoc/_chapters/about.adoc | 1 + .../src/asciidoc/_chapters/odb.adoc | 14 +- docs/client_install/src/asciidoc/index.adoc | 3 +- docs/command_interface/pom.xml | 2 + .../src/asciidoc/_chapters/about.adoc | 1 + docs/command_interface/src/asciidoc/index.adoc | 5 +- docs/cqd_reference/pom.xml | 291 +++ .../src/asciidoc/_chapters/about.adoc | 179 ++ .../src/asciidoc/_chapters/histograms.adoc | 215 ++ .../src/asciidoc/_chapters/introduction.adoc | 111 + .../_chapters/operational_controls.adoc | 110 + .../src/asciidoc/_chapters/query_plans.adoc | 515 +++++ .../asciidoc/_chapters/runtime_controls.adoc | 83 + .../src/asciidoc/_chapters/schema_controls.adoc | 49 + .../src/asciidoc/_chapters/transactions.adoc | 94 + .../src/asciidoc/_chapters/update_stats.adoc | 92 + docs/cqd_reference/src/asciidoc/index.adoc | 69 + docs/load_transform/pom.xml | 289 +++ .../src/asciidoc/_chapters/about.adoc | 186 ++ .../src/asciidoc/_chapters/bulk_load.adoc | 291 +++ .../src/asciidoc/_chapters/bulk_unload.adoc | 73 + .../src/asciidoc/_chapters/introduction.adoc | 117 + .../src/asciidoc/_chapters/monitor.adoc | 77 + .../src/asciidoc/_chapters/tables_indexes.adoc | 208 ++ .../src/asciidoc/_chapters/trickle_load.adoc | 496 +++++ .../src/asciidoc/_chapters/troubleshoot.adoc | 159 ++ docs/load_transform/src/asciidoc/index.adoc | 68 + docs/messages_guide/pom.xml | 2 + .../src/asciidoc/_chapters/about.adoc | 3 + docs/messages_guide/src/asciidoc/index.adoc | 5 +- docs/odb_user/pom.xml | 2 + docs/odb_user/src/asciidoc/_chapters/about.adoc | 3 + docs/odb_user/src/asciidoc/_chapters/load.adoc | 48 +- docs/odb_user/src/asciidoc/index.adoc | 5 +- .../src/asciidoc/_chapters/about.adoc | 2 + .../src/asciidoc/_chapters/prepare.adoc | 31 +- docs/provisioning_guide/src/asciidoc/index.adoc | 3 +- docs/shared/trafodion-logo.jpg | Bin 0 -> 28850 bytes docs/sql_reference/pom.xml | 2 + .../src/asciidoc/_chapters/about.adoc | 2 + .../src/asciidoc/_chapters/cqds.adoc | 20 - .../src/asciidoc/_chapters/sql_statements.adoc | 8 +- .../src/asciidoc/_chapters/sql_utilities.adoc | 117 +- docs/sql_reference/src/asciidoc/index.adoc | 5 +- docs/src/site/markdown/download.md | 134 +- docs/src/site/markdown/index.md | 12 +- docs/src/site/markdown/logo.md | 56 + docs/src/site/markdown/quickstart.md | 95 +- docs/src/site/resources/css/site.css | 18 +- .../site/resources/images/carousel/scale.png | Bin 93783 -> 127571 bytes .../site/resources/images/carousel/stack.png | Bin 102472 -> 170554 bytes .../resources/images/logos/pb-Trafodion.png | Bin 0 -> 49089 bytes .../resources/images/logos/pb-trafodion.jpg | Bin 0 -> 80904 bytes .../images/logos/trafodion-dragon-large.jpg | Bin 0 -> 50000 bytes .../images/logos/trafodion-dragon-large.png | Bin 0 -> 20595 bytes .../resources/images/logos/trafodion-dragon.png | Bin 0 -> 12786 bytes .../resources/images/logos/trafodion-logo.jpg | Bin 0 -> 58953 bytes docs/src/site/site.xml | 113 +- install/traf_tools_setup.sh | 6 +- licenses/lic-clients-bin | 206 +- licenses/lic-server-bin | 15 - .../phoenix/end2end/AlterTableTest.java | 22 + .../phoenix/end2end/ArithmeticQueryTest.java | 22 + .../phoenix/end2end/AutoCommitTest.java | 22 + .../org/trafodion/phoenix/end2end/BaseTest.java | 22 + .../phoenix/end2end/BinaryRowKeyTest.java | 22 + .../phoenix/end2end/CoalesceFunctionTest.java | 22 + .../end2end/CompareDecimalToLongTest.java | 22 + .../phoenix/end2end/CreateTableTest.java | 22 + .../phoenix/end2end/CustomEntityDataTest.java | 22 + .../phoenix/end2end/DeleteRangeTest.java | 22 + .../end2end/DescColumnSortOrderTest.java | 24 +- .../phoenix/end2end/DistinctCountTest.java | 22 + .../phoenix/end2end/ExecuteStatementsTest.java | 22 + .../phoenix/end2end/ExtendedQueryExecTest.java | 22 + .../phoenix/end2end/FunkyNamesTest.java | 22 + .../phoenix/end2end/GroupByCaseTest.java | 22 + .../trafodion/phoenix/end2end/IndexTest.java | 22 + .../trafodion/phoenix/end2end/IsNullTest.java | 22 + .../trafodion/phoenix/end2end/KeyOnlyTest.java | 22 + .../phoenix/end2end/MultiCfQueryExecTest.java | 22 + .../trafodion/phoenix/end2end/OrderByTest.java | 22 + .../phoenix/end2end/ProductMetricsTest.java | 24 +- .../phoenix/end2end/QueryExecTest.java | 22 + .../end2end/QueryExecWithoutSCNTest.java | 22 + .../phoenix/end2end/QueryPlanTest.java | 22 + .../phoenix/end2end/ReadIsolationLevelTest.java | 22 + .../phoenix/end2end/SaltedTableTest.java | 22 + .../end2end/SaltedTableUpsertSelectTest.java | 22 + .../end2end/SaltedTableVarLengthRowKeyTest.java | 22 + .../phoenix/end2end/ServerExceptionTest.java | 22 + .../phoenix/end2end/SkipScanQueryTest.java | 22 + .../phoenix/end2end/StatementHintsTest.java | 22 + .../trafodion/phoenix/end2end/StddevTest.java | 22 + .../phoenix/end2end/ToCharFunctionTest.java | 22 + .../phoenix/end2end/ToNumberFunctionTest.java | 22 + .../org/trafodion/phoenix/end2end/TopNTest.java | 22 + .../phoenix/end2end/UpsertBigValuesTest.java | 22 + .../end2end/UpsertSelectAutoCommitTest.java | 22 + .../phoenix/end2end/UpsertSelectTest.java | 22 + .../phoenix/end2end/UpsertValuesTest.java | 22 + .../phoenix/end2end/VariableLengthPKTest.java | 22 + 253 files changed, 13242 insertions(+), 3950 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f1c7f3a7/core/sqf/hbase_utilities/pom.xml ---------------------------------------------------------------------- diff --cc core/sqf/hbase_utilities/pom.xml index 1a996e3,4e77b68..c7081e4 --- a/core/sqf/hbase_utilities/pom.xml +++ b/core/sqf/hbase_utilities/pom.xml @@@ -28,8 -28,8 +28,8 @@@ <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <hadoop.version>2.5.0</hadoop.version> + <hadoop.version>2.6.0</hadoop.version> - <hbase.version>1.0.0-cdh5.4.4</hbase.version> + <hbase.version>${env.HBASE_DEP_VER}</hbase.version> <java.version>1.7</java.version> </properties> http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f1c7f3a7/core/sqf/sqenvcom.sh ---------------------------------------------------------------------- diff --cc core/sqf/sqenvcom.sh index 5555bc6,c2335d4..34d4f7a --- a/core/sqf/sqenvcom.sh +++ b/core/sqf/sqenvcom.sh @@@ -139,23 -140,18 +139,29 @@@ f export MY_SQROOT=$PWD export SQ_HOME=$PWD + # general Hadoop & TRX dependencies - not distro specific, choose one to build against export HBASE_TRXDIR=$MY_SQROOT/export/lib - export HBASE_TRX_JAR=hbase-trx-cdh5_4-${TRAFODION_VER}.jar -export HBASE_TRX_ID=hbase-trx-cdh5_3 -export HBASE_DEP_VER=0.98.6-cdh5.3.0 ++export HBASE_TRX_ID=hbase-trx-cdh5_4 ++export HBASE_DEP_VER=hbase-1.0.0-cdh5.4.4.tar.gz + export HBASE_TRX_JAR=${HBASE_TRX_ID}-${TRAFODION_VER}.jar +export DTM_COMMON_JAR=trafodion-dtm-${TRAFODION_VER}.jar +export SQL_JAR=trafodion-sql-${TRAFODION_VER}.jar export UTIL_JAR=trafodion-utility-${TRAFODION_VER}.jar -if [[ "$SQ_HBASE_DISTRO" = "HDP" ]]; then - export HBASE_TRX_JAR=hbase-trx-hdp2_2-${TRAFODION_VER}.jar +if [[ "$HBASE_DISTRO" = "HDP" ]]; then + export HBASE_VERSION_ID=hdp2_3 + export HBASE_TRX_JAR=hbase-trx-${HBASE_VERSION_ID}-${TRAFODION_VER}.jar + export DTM_COMMON_JAR=trafodion-dtm-${HBASE_VERSION_ID}-${TRAFODION_VER}.jar + export SQL_JAR=trafodion-sql-${HBASE_VERSION_ID}-${TRAFODION_VER}.jar +fi +if [[ "$HBASE_DISTRO" = "APACHE" ]]; then + export HBASE_VERSION_ID=apache1_0_2 + export HBASE_TRX_JAR=hbase-trx-${HBASE_VERSION_ID}-${TRAFODION_VER}.jar + export DTM_COMMON_JAR=trafodion-dtm-${HBASE_VERSION_ID}-${TRAFODION_VER}.jar + export SQL_JAR=trafodion-sql-${HBASE_VERSION_ID}-${TRAFODION_VER}.jar fi + # set common version to be consistent between shared lib and maven dependencies + export THRIFT_DEP_VER=0.9.0 + export HIVE_DEP_VER=0.13.1 # check for workstation env # want to make sure SQ_VIRTUAL_NODES is set in the shell running sqstart http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f1c7f3a7/core/sqf/sql/scripts/install_local_hadoop ---------------------------------------------------------------------- diff --cc core/sqf/sql/scripts/install_local_hadoop index 2c10839,5d01264..e0081c2 --- a/core/sqf/sql/scripts/install_local_hadoop +++ b/core/sqf/sql/scripts/install_local_hadoop @@@ -600,28 -602,15 +600,29 @@@ MYSQL_TAR=mysql-5.6.10-linux-glibc2.5-x MYSQL_JDBC_URL=http://cdn.mysql.com/archives/mysql-connector-java-5.1 MYSQL_JDBC_TAR=mysql-connector-java-5.1.23.tar.gz -HIVE_MIRROR_URL=https://archive.apache.org/dist/hive/hive-${HIVE_DEP_VER} -HIVE_PREFIX=apache-hive-${HIVE_DEP_VER}-bin +HIVE_MIRROR_URL=http://archive.cloudera.com/cdh5/cdh/5 +HIVE_PREFIX=hive-1.1.0-cdh5.4.4 HIVE_TAR=${HIVE_PREFIX}.tar.gz +if [[ "$HBASE_DISTRO" = "APACHE" ]]; then - HIVE_PREFIX=apache-hive-1.2.0-bin ++ HIVE_MIRROR_URL=https://archive.apache.org/dist/hive/hive-${HIVE_DEP_VER} ++ HIVE_PREFIX=apache-hive-${HIVE_DEP_VER}-bin + HIVE_TAR=${HIVE_PREFIX}.tar.gz +fi + +if [[ "$HBASE_DISTRO" = "HDP" ]]; then + HIVE_PREFIX=apache-hive-1.2.1.2.3.2.0-2950-bin + HIVE_TAR=${HIVE_PREFIX}.tar.gz +fi + HBASE_MIRROR_URL=http://archive.cloudera.com/cdh5/cdh/5 - HBASE_TAR=hbase-1.0.0-cdh5.4.4.tar.gz ++HBASE_TARR=hbase-${HBASE_DEP_VER}.tar.gz -HBASE_TAR=hbase-${HBASE_DEP_VER}.tar.gz -if [[ "$SQ_HBASE_DISTRO" = "HDP" ]]; then - HBASE_TAR=hbase-0.98.4.2.2.0.0-2041-hadoop2.tar.gz +if [[ "$HBASE_DISTRO" = "HDP" ]]; then + HBASE_TAR=hbase-1.1.2.2.3.2.0-2950.tar.gz +fi +if [[ "$HBASE_DISTRO" = "APACHE" ]]; then + HBASE_TAR=hbase-1.0.2-bin.tar.gz fi echo "Checking for existing Hadoop processes..." http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f1c7f3a7/core/sqf/src/seatrans/hbase-trx/Makefile ---------------------------------------------------------------------- diff --cc core/sqf/src/seatrans/hbase-trx/Makefile index 82ce21d,51ca957..65a4c6c --- a/core/sqf/src/seatrans/hbase-trx/Makefile +++ b/core/sqf/src/seatrans/hbase-trx/Makefile @@@ -36,11 -34,9 +36,12 @@@ REINSTATE_ORIG =./reinstate_ori all: build_all - jdk_1_7_cdh: + jdk_1_7_cdh: build_chk_cdh + $(REINSTATE_ORIG) + $(UNCOMMENT_STRING) CDH1.0 + $(MAKE) build_chk_cdh set -o pipefail && $(MAVEN) -f pom.xml.cdh package install -DskipTests | tee -a build_trx.log + mkdir -p $(MY_SQROOT)/export/lib cp -pf target/$(BLD_HBASE_CDH_TRX_JARNAME) $(MY_SQROOT)/export/lib $(RM) $(VFILE) @@@ -50,26 -46,22 +51,28 @@@ jdk_1_7_mapr: build_chk_map cp -pf target/$(BLD_HBASE_MAPR_TRX_JARNAME) $(MY_SQROOT)/export/lib $(RM) $(VFILE) - jdk_1_7_hdp: + jdk_1_7_hdp: build_chk_hdp + $(REINSTATE_ORIG) + $(UNCOMMENT_STRING) HDP2.3 + $(MAKE) build_chk_hdp set -o pipefail && $(MAVEN) -f pom.xml.hdp package -DskipTests | tee -a build_trx.log + mkdir -p $(MY_SQROOT)/export/lib cp -pf target/$(BLD_HBASE_HDP_TRX_JARNAME) $(MY_SQROOT)/export/lib $(RM) $(VFILE) - jdk_1_7_apache: + jdk_1_7_apache: build_chk_apache + $(REINSTATE_ORIG) + $(UNCOMMENT_STRING) APACHE + $(MAKE) build_chk_apache echo "$(MAVEN) package -DskipTests" echo "### For full Maven output, see file build_trx.log" set -o pipefail && $(MAVEN) -f pom.xml.apache package -DskipTests | tee build_trx.log + mkdir -p $(MY_SQROOT)/export/lib cp -pf target/$(BLD_HBASE_APACHE_TRX_JARNAME) $(MY_SQROOT)/export/lib $(RM) $(VFILE) + $(REINSTATE_ORIG) -build_all: jdk_1_7_apache jdk_1_7_cdh jdk_1_7_hdp -#build_all: jdk_1_7_apache jdk_1_7_cdh +build_all: jdk_1_7_apache jdk_1_7_hdp jdk_1_7_cdh build_chk_apache: $(GENVERS) > $(VFILE) http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f1c7f3a7/core/sqf/src/seatrans/hbase-trx/src/main/java/org/apache/hadoop/hbase/client/transactional/TransactionManager.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f1c7f3a7/core/sqf/src/seatrans/hbase-trx/src/main/java/org/apache/hadoop/hbase/client/transactional/TransactionalScanner.java.tmpl ---------------------------------------------------------------------- diff --cc core/sqf/src/seatrans/hbase-trx/src/main/java/org/apache/hadoop/hbase/client/transactional/TransactionalScanner.java.tmpl index 50d088d,0000000..40bbeeb mode 100755,000000..100755 --- a/core/sqf/src/seatrans/hbase-trx/src/main/java/org/apache/hadoop/hbase/client/transactional/TransactionalScanner.java.tmpl +++ b/core/sqf/src/seatrans/hbase-trx/src/main/java/org/apache/hadoop/hbase/client/transactional/TransactionalScanner.java.tmpl @@@ -1,323 -1,0 +1,325 @@@ +/** +* @@@ START COPYRIGHT @@@ +* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +* @@@ END COPYRIGHT @@@ +**/ + + +package org.apache.hadoop.hbase.client.transactional; + +import java.io.IOException; +import java.util.LinkedList; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.client.AbstractClientScanner; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.coprocessor.Batch; +import org.apache.hadoop.hbase.coprocessor.transactional.generated.TrxRegionProtos; +import org.apache.hadoop.hbase.coprocessor.transactional.generated.TrxRegionProtos.CloseScannerRequest; +import org.apache.hadoop.hbase.coprocessor.transactional.generated.TrxRegionProtos.CloseScannerResponse; +import org.apache.hadoop.hbase.coprocessor.transactional.generated.TrxRegionProtos.OpenScannerRequest; +import org.apache.hadoop.hbase.coprocessor.transactional.generated.TrxRegionProtos.PerformScanRequest; +import org.apache.hadoop.hbase.coprocessor.transactional.generated.TrxRegionProtos.PerformScanResponse; +import org.apache.hadoop.hbase.coprocessor.transactional.generated.TrxRegionProtos.TrxRegionService; +import org.apache.hadoop.hbase.ipc.BlockingRpcCallback; +import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel; +import org.apache.hadoop.hbase.ipc.ServerRpcController; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.util.Bytes; + +import com.google.protobuf.ServiceException; +import com.google.protobuf.ByteString; - ++import org.apache.commons.codec.binary.Hex; + +/* + * Transaction Scanner + */ +public class TransactionalScanner extends AbstractClientScanner { + private final Log LOG = LogFactory.getLog(this.getClass()); + public Scan scan; + public Long scannerID; + public TransactionState ts; + public TransactionalTable ttable; + protected boolean closed = false; + // Experiment with this parameter, may be faster without having to send the final close() + protected boolean doNotCloseOnLast = true; + protected int nbRows = 100; + protected long nextCallSeq = 0; + private boolean hasMore = true; + private boolean moreScanners = true; + private boolean interrupted = false; + public HRegionInfo currentRegion; + public byte[] currentBeginKey; + public byte[] currentEndKey; + protected final LinkedList<Result> cache = new LinkedList<Result>(); + + public TransactionalScanner(final TransactionalTable ttable, final TransactionState ts, final Scan scan, final Long scannerID) { + super(); + this.scan = scan; + this.scannerID = scannerID; + this.ts = ts; + this.ttable = ttable; + this.nbRows = scan.getCaching(); + if (nbRows <= 0) + nbRows = 100; + try { + nextScanner(false); + }catch (IOException e) { + LOG.error("nextScanner error"); + } + } + + protected boolean checkScanStopRow(final byte [] endKey) { + if (this.scan.getStopRow().length > 0) { + byte [] stopRow = scan.getStopRow(); + int cmp = Bytes.compareTo(stopRow, 0, stopRow.length, + endKey, 0, endKey.length); + if (cmp <= 0) { + return true; + } + } + return false; + } + + @Override + public void close() { + if(LOG.isTraceEnabled()) LOG.trace("close() -- ENTRY txID: " + ts.getTransactionId()); + if(closed) { + if(LOG.isTraceEnabled()) LOG.trace("close() already closed -- EXIT txID: " + ts.getTransactionId()); + return; + } + this.closed = true; + if(this.interrupted) { + if(LOG.isDebugEnabled()) LOG.debug("close() resetting connection, txID: " + ts.getTransactionId()); + try { + ttable.resetConnection(); + } catch(IOException e) { + if(LOG.isErrorEnabled()) LOG.error("close() unable to reset connection, txID: " + ts.getTransactionId()); + return; + } + this.interrupted = false; + } + TrxRegionProtos.CloseScannerRequest.Builder requestBuilder = CloseScannerRequest.newBuilder(); + requestBuilder.setTransactionId(ts.getTransactionId()); + requestBuilder.setRegionName(ByteString.copyFromUtf8(currentRegion.getRegionNameAsString())); + requestBuilder.setScannerId(scannerID); + TrxRegionProtos.CloseScannerRequest closeRequest = requestBuilder.build(); + try { + CoprocessorRpcChannel channel = ttable.coprocessorService(this.currentBeginKey); + TrxRegionService.BlockingInterface trxService = TrxRegionService.newBlockingStub(channel); + TrxRegionProtos.CloseScannerResponse response = trxService.closeScanner(null, closeRequest); + String exception = response.getException(); + if(response.getHasException()) { + String errMsg = "closeScanner encountered Exception txID: " + + ts.getTransactionId() + " Exception: " + exception; + LOG.error(errMsg); + } + } + catch(ServiceException se) { + this.interrupted = true; + this.closed = false; + } + + catch (Throwable e) { + String errMsg = "CloseScanner error on coprocessor call, scannerID: " + this.scannerID + " " + e; + LOG.error(errMsg); + } + + if(LOG.isTraceEnabled()) LOG.trace("close() -- EXIT txID: " + ts.getTransactionId()); + } + + protected boolean nextScanner(final boolean done) throws IOException{ + if(LOG.isTraceEnabled()) LOG.trace("nextScanner() -- ENTRY txID: " + ts.getTransactionId()); + if(this.interrupted) { + if(LOG.isDebugEnabled()) LOG.debug("nextScanner() resetting connection, txID: " + ts.getTransactionId()); + ttable.resetConnection(); + this.interrupted = false; + } + if(this.currentBeginKey != null) { + if(LOG.isTraceEnabled()) LOG.trace("nextScanner() currentBeginKey != null txID: " + ts.getTransactionId()); + if (doNotCloseOnLast) + close(); + if((this.currentEndKey == HConstants.EMPTY_END_ROW) || + Bytes.equals(this.currentEndKey, HConstants.EMPTY_BYTE_ARRAY) || + checkScanStopRow(this.currentEndKey) || + done) { + if(LOG.isTraceEnabled()) LOG.trace("endKey: " + Bytes.toString(this.currentEndKey)); + if(LOG.isTraceEnabled()) LOG.trace("nextScanner() -- EXIT -- returning false txID: " + ts.getTransactionId()); + this.moreScanners = false; + return false; + } + else - this.currentBeginKey = TransactionManager.binaryIncrementPos(this.currentEndKey,1); ++ //this.currentBeginKey = TransactionManager.binaryIncrementPos(this.currentEndKey,1); ++ this.currentBeginKey = this.currentEndKey; + } + else { + // First call to nextScanner + this.currentBeginKey = this.scan.getStartRow(); + } + - this.currentRegion = ttable.getRegionLocation(this.currentBeginKey).getRegionInfo(); ++ this.currentRegion = ttable.getRegionLocation(this.currentBeginKey, false).getRegionInfo(); + this.currentEndKey = this.currentRegion.getEndKey(); + - if(LOG.isTraceEnabled()) LOG.trace("Region Info: " + currentRegion.getRegionNameAsString()); - if(this.currentEndKey != HConstants.EMPTY_END_ROW) - this.currentEndKey = TransactionManager.binaryIncrementPos(currentRegion.getEndKey(), -1); ++ if(LOG.isTraceEnabled()) LOG.trace("nextScanner() txID: " + ts.getTransactionId() + " Region Info: " + currentRegion.getRegionNameAsString()+ ++ "currentBeginKey: " + Hex.encodeHexString(this.currentBeginKey) + "currentEndKey: " + Hex.encodeHexString(this.currentEndKey)); ++ //if(this.currentEndKey != HConstants.EMPTY_END_ROW) ++ // this.currentEndKey = TransactionManager.binaryIncrementPos(currentRegion.getEndKey(), -1); + + this.closed = false; + + TrxRegionProtos.OpenScannerRequest.Builder requestBuilder = OpenScannerRequest.newBuilder(); + requestBuilder.setTransactionId(ts.getTransactionId()); + requestBuilder.setRegionName(ByteString.copyFromUtf8(currentRegion.getRegionNameAsString())); + requestBuilder.setScan(ProtobufUtil.toScan(scan)); + TrxRegionProtos.OpenScannerRequest openRequest = requestBuilder.build(); + try { + CoprocessorRpcChannel channel = ttable.coprocessorService(this.currentBeginKey); + TrxRegionService.BlockingInterface trxService = TrxRegionService.newBlockingStub(channel); + TrxRegionProtos.OpenScannerResponse response = trxService.openScanner(null, openRequest); + String exception = response.getException(); + if(response.getHasException()) { + String errMsg = "nextScanner encountered Exception txID: " + + ts.getTransactionId() + " Exception: " + exception; + LOG.error(errMsg); + throw new IOException(errMsg); + } + this.scannerID = response.getScannerId(); + } + catch(ServiceException se) { + this.interrupted = true; + String errMsg = "OpenScanner error encountered Service Exception, scannerID: " + this.scannerID + " " + se; + LOG.error(errMsg); + throw new IOException(errMsg); + } + catch (Throwable e) { + String errMsg = "OpenScanner error on coprocessor call, scannerID: " + this.scannerID + " " + e; + LOG.error(errMsg); + throw new IOException(errMsg); + } + this.nextCallSeq = 0; + if(LOG.isTraceEnabled()) LOG.trace("nextScanner() -- EXIT -- returning true txID: " + ts.getTransactionId()); + return true; + } + + @Override + public Result next() throws IOException { + // if (LOG.isTraceEnabled()) LOG.trace("next -- ENTRY txID: " + ts.getTransactionId() + " cache size: " + cache.size()); + if(cache.size() == 0) { + if (LOG.isTraceEnabled()) LOG.trace("next -- cache.size() == 0 txID: " + ts.getTransactionId()); + if(this.hasMore) { + if (LOG.isTraceEnabled()) + LOG.trace("next() before coprocessor PerformScan call txID: " + ts.getTransactionId()); + final long nextCallSeqInput = this.nextCallSeq; + TrxRegionProtos.PerformScanRequest.Builder requestBuilder = PerformScanRequest.newBuilder(); + requestBuilder.setTransactionId(ts.getTransactionId()); + requestBuilder.setRegionName(ByteString.copyFromUtf8(currentRegion.getRegionNameAsString())); + requestBuilder.setScannerId(scannerID); + requestBuilder.setNumberOfRows(nbRows); + if (doNotCloseOnLast) + requestBuilder.setCloseScanner(false); + else + requestBuilder.setCloseScanner(true); + requestBuilder.setNextCallSeq(nextCallSeqInput); + TrxRegionProtos.PerformScanRequest perfScanRequest = requestBuilder.build(); + TrxRegionProtos.PerformScanResponse response; + try { + CoprocessorRpcChannel channel = ttable.coprocessorService(this.currentBeginKey); + TrxRegionService.BlockingInterface trxService = TrxRegionService.newBlockingStub(channel); + response = trxService.performScan(null, perfScanRequest); + String exception = response.getException(); + if(response.getHasException()) { + String errMsg = "performScan encountered Exception txID: " + + ts.getTransactionId() + " Exception: " + exception; + LOG.error(errMsg); + throw new IOException(errMsg); + } + } + catch (ServiceException se) { + this.interrupted = true; + if(LOG.isDebugEnabled()) LOG.debug("PerformScan encountered Service Exception, scannerID: " + this.scannerID + " " + se); + return null; + } + catch (Throwable e) { + String errMsg = "PerformScan error on coprocessor call, scannerID: " + this.scannerID + " " + e; + if(LOG.isErrorEnabled()) LOG.error(errMsg); + throw new IOException(errMsg); + } + int count; + org.apache.hadoop.hbase.protobuf.generated.ClientProtos.Result result; + + this.nextCallSeq = response.getNextCallSeq(); + count = response.getResultCount(); + if (LOG.isTraceEnabled()) LOG.trace("next() nextCallSeq: " + this.nextCallSeq + + " count: " + count); + if(count == 0) { + this.hasMore = false; + } + else { + for (int i = 0; i < count; i++) { + result = response.getResult(i); + if (result != null) { + cache.add(ProtobufUtil.toResult(result)); + } + this.hasMore = response.getHasMore(); + if (LOG.isTraceEnabled()) + LOG.trace(" PerformScan response count " + count + ", hasMore is " + hasMore + ", result " + result); + } + + } + } + else { + if(LOG.isTraceEnabled()) LOG.trace("hasMore is false"); + if(nextScanner(false)){ + if(LOG.isTraceEnabled()) LOG.trace("next(), nextScanner == true nextCallSeq: " + this.nextCallSeq); + this.hasMore = true; + return next(); + } + else { + if(LOG.isTraceEnabled()) LOG.trace("next(), nextScanner == false"); + this.moreScanners = false; + return null; + } + } + } + if (cache.size() > 0) { + // if(LOG.isTraceEnabled()) LOG.trace("next() returning cache.poll()"); + return cache.poll(); + } + else if(this.moreScanners){ + if(LOG.isTraceEnabled()) LOG.trace("next() more scanners"); + return next(); + } + else { + if(LOG.isTraceEnabled()) LOG.trace("next() returning null"); + return null; + } + } +#ifdef HDP2.3 + @Override +#endif + public boolean renewLease() { + return true; + } +} http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f1c7f3a7/core/sqf/src/seatrans/tm/hbasetmlib2/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f1c7f3a7/core/sql/pom.xml ---------------------------------------------------------------------- diff --cc core/sql/pom.xml index 1c3ba4f,83472fa..f96c38f mode 100644,100755..100644 --- a/core/sql/pom.xml +++ b/core/sql/pom.xml @@@ -28,11 -28,10 +28,11 @@@ <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <hadoop.version>2.6.0-cdh5.4.4</hadoop.version> - <hbase.version>1.0.0-cdh5.4.4</hbase.version> - <hbase-trx.id>hbase-trx-cdh5_4</hbase-trx.id> - <hive.version>1.1.0-cdh5.4.4</hive.version> - <thrift.version>0.9.0</thrift.version> + <hbase.version>${env.HBASE_DEP_VER}</hbase.version> + <hbase-trx.id>${env.HBASE_TRX_ID}</hbase-trx.id> + <hive.version>${env.HIVE_DEP_VER}</hive.version> + <thrift.version>${env.THRIFT_DEP_VER}</thrift.version> <java.version>1.7</java.version> </properties> http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f1c7f3a7/core/sql/src/main/java/org/trafodion/sql/HBaseClient.java ----------------------------------------------------------------------
