update
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/681cad66 Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/681cad66 Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/681cad66 Branch: refs/heads/master Commit: 681cad66935981b7e0f6bcb9b6c72fe18e2f69f6 Parents: f9f3748 Author: mashengchen <[email protected]> Authored: Tue Mar 22 05:40:23 2016 +0000 Committer: mashengchen <[email protected]> Committed: Tue Mar 22 05:40:23 2016 +0000 ---------------------------------------------------------------------- .rat-excludes | 1 + core/conn/odbc/src/odbc/nsksrvr/Makefile | 2 + .../odbc/src/odbc/nsksrvrcore/csrvrstmt.cpp | 7 +- core/dbsecurity/auth/Makefile | 2 +- core/rest/pom.xml | 2 +- core/sqf/Makefile | 2 +- .../sqf/conf/log4cxx.trafodion.masterexe.config | 1 + core/sqf/hbase_utilities/pom.xml | 2 +- core/sqf/monitor/linux/clio.cxx | 7 + core/sqf/monitor/linux/cluster.cxx | 11 + core/sqf/monitor/linux/cmsh.cxx | 29 +- core/sqf/monitor/linux/config.cxx | 1 + core/sqf/monitor/linux/internal.h | 9 + core/sqf/monitor/linux/makefile | 5 +- core/sqf/monitor/linux/mlio.cxx | 6 +- core/sqf/monitor/linux/monitor.cxx | 9 + core/sqf/monitor/linux/msgdef.h | 17 + core/sqf/monitor/linux/phnode.h | 2 + core/sqf/monitor/linux/pnode.cxx | 10 + core/sqf/monitor/linux/pnode.h | 1 + core/sqf/monitor/linux/pnodeconfig.cxx | 6 + core/sqf/monitor/linux/pnodeconfig.h | 2 +- core/sqf/monitor/linux/replicate.cxx | 56 +- core/sqf/monitor/linux/replicate.h | 14 + core/sqf/monitor/linux/reqnodename.cxx | 114 + core/sqf/monitor/linux/reqqueue.cxx | 60 +- core/sqf/monitor/linux/reqqueue.h | 32 +- core/sqf/monitor/linux/shell.cxx | 195 +- core/sqf/monitor/linux/testpoint.h | 2 +- core/sqf/sqenvcom.sh | 8 + core/sqf/sql/scripts/gensq.pl | 54 +- core/sqf/sql/scripts/genstats.sh | 43 + core/sqf/sql/scripts/get_libhdfs_files | 203 +- core/sqf/sql/scripts/sqnodeipcrm | 2 +- core/sqf/sql/scripts/sqstart | 3 +- core/sqf/sql/scripts/sqstop | 2 +- core/sqf/src/seabed/src/msmon.cpp | 6 + core/sqf/src/seabed/src/sautil.cpp | 5 - core/sqf/src/seabed/src/stream.cpp | 2 - .../transactional/SplitBalanceHelper.java | 6 +- core/sqf/src/tm/Makefile | 3 + core/sqf/src/tm/tm.cpp | 33 +- core/sqf/src/tm/tmaudit.cpp | 18 +- core/sqf/src/tm/tmauditobj.cpp | 6 +- core/sqf/src/tm/tmddlrequests.cpp | 12 +- core/sqf/src/tm/tmlib.cpp | 13 +- core/sqf/src/tm/tmpoolelement.h | 2 +- core/sqf/src/tm/tmrecov.cpp | 3 +- core/sqf/src/tm/tmregisterregion.cpp | 4 +- core/sqf/src/tm/tmrmhbase.cpp | 3 - core/sqf/src/tm/tmrmtse.cpp | 3 +- core/sqf/src/tm/tmtime.h | 3 +- core/sqf/src/tm/tmxarmmain.cpp | 2 +- core/sqf/src/tm/tmxatxn.cpp | 4 +- core/sqf/src/tm/tools/Makefile | 3 + core/sqf/src/tm/tools/dtmci.cpp | 14 +- core/sqf/src/tm/xarmapi.cpp | 5 +- core/sqf/src/tm/xatmapi.cpp | 7 +- core/sql/SqlCompilerDebugger/mk.sh | 5 + core/sql/bin/SqlciErrors.txt | 7 +- core/sql/cli/Cli.cpp | 445 ++- core/sql/cli/Cli.h | 15 +- core/sql/cli/CliExpExchange.cpp | 18 +- core/sql/cli/CliExtern.cpp | 54 +- core/sql/cli/Descriptor.cpp | 6 +- core/sql/comexe/ComQueue.h | 2 +- core/sql/comexe/ComTdb.h | 1 + core/sql/comexe/ComTdbHbaseAccess.cpp | 2 +- core/sql/comexe/ComTdbHbaseAccess.h | 2 +- core/sql/common/CharType.cpp | 3 +- core/sql/common/CharType.h | 2 +- core/sql/common/ComMisc.cpp | 13 + core/sql/common/ComMisc.h | 8 + core/sql/common/ComSmallDefs.h | 14 +- core/sql/common/ComUser.cpp | 54 + core/sql/common/ComUser.h | 6 + core/sql/common/NAString.cpp | 23 + core/sql/common/NAString.h | 3 + core/sql/common/NAUserId.h | 32 +- core/sql/common/OperTypeEnum.h | 1 + core/sql/executor/ExExeUtilLoad.cpp | 3 +- core/sql/executor/ExHbaseAccess.h | 3 + core/sql/executor/ExHbaseIUD.cpp | 27 +- core/sql/executor/hiveHook.cpp | 2 +- core/sql/exp/ExpHbaseInterface.h | 1 - core/sql/exp/ExpLOB.cpp | 158 +- core/sql/exp/ExpLOB.h | 41 +- core/sql/exp/ExpLOBaccess.cpp | 2905 +++++++++--------- core/sql/exp/ExpLOBaccess.h | 288 +- core/sql/exp/ExpLOBenums.h | 5 +- core/sql/exp/ExpLOBexternal.h | 20 +- core/sql/exp/ExpLOBinterface.cpp | 291 +- core/sql/exp/ExpLOBinterface.h | 24 +- core/sql/exp/ExpLOBprocess.cpp | 494 +-- core/sql/exp/ExpLOBprocess.h | 2 +- core/sql/exp/exp_conv.cpp | 1 + core/sql/exp/exp_datetime.cpp | 1239 ++++---- core/sql/exp/exp_datetime.h | 187 +- core/sql/exp/exp_function.cpp | 28 +- core/sql/generator/GenExpGenerator.cpp | 3 +- core/sql/generator/GenItemFunc.cpp | 151 +- core/sql/generator/GenPreCode.cpp | 36 +- core/sql/generator/GenRelUpdate.cpp | 47 +- core/sql/generator/Generator.cpp | 10 +- core/sql/nskgmake/Makerules.linux | 8 +- core/sql/nskgmake/SqlCompilerDebugger/Makefile | 5 +- core/sql/optimizer/BindItemExpr.cpp | 598 ++-- core/sql/optimizer/BindRI.cpp | 45 +- core/sql/optimizer/BindRelExpr.cpp | 40 +- core/sql/optimizer/ItemCache.cpp | 5 +- core/sql/optimizer/ItemExpr.cpp | 52 +- core/sql/optimizer/ItemFunc.h | 70 +- core/sql/optimizer/NAFileSet.cpp | 2 + core/sql/optimizer/NAFileSet.h | 7 + core/sql/optimizer/NATable.cpp | 29 +- core/sql/optimizer/NATable.h | 3 +- core/sql/optimizer/QRDescGenerator.cpp | 19 +- core/sql/optimizer/RelExeUtil.cpp | 12 +- core/sql/optimizer/RelExeUtil.h | 17 +- core/sql/optimizer/RelFastTransport.cpp | 2 +- core/sql/optimizer/RelUpdate.h | 3 +- core/sql/optimizer/Stats.h | 8 +- core/sql/optimizer/SynthType.cpp | 154 +- core/sql/optimizer/ValueDesc.cpp | 5 +- core/sql/optimizer/ValueDesc.h | 6 +- core/sql/parser/ElemDDLNode.cpp | 6 + core/sql/parser/ElemDDLNode.h | 3 + core/sql/parser/ParKeyWords.cpp | 7 +- .../sql/parser/StmtDDLAlterTableAlterColumn.cpp | 39 +- core/sql/parser/StmtDDLAlterTableAlterColumn.h | 40 +- core/sql/parser/sqlparser.y | 177 +- core/sql/regress/charsets/DIFF012.KNOWN.SB | 2 +- core/sql/regress/charsets/EXPECTED002 | 13 +- core/sql/regress/charsets/EXPECTED003 | 11 +- core/sql/regress/charsets/EXPECTED012 | 19 +- core/sql/regress/charsets/EXPECTED310 | 198 +- core/sql/regress/charsets/EXPECTED312 | 198 +- core/sql/regress/charsets/EXPECTED313 | 286 +- core/sql/regress/compGeneral/DIFF006.KNOWN.SB | 4 - core/sql/regress/compGeneral/EXPECTED005 | 10 +- core/sql/regress/compGeneral/EXPECTED006 | 1235 -------- core/sql/regress/compGeneral/EXPECTED006.SB | 33 +- core/sql/regress/compGeneral/EXPECTED013.SB | 48 + core/sql/regress/compGeneral/EXPECTED042 | 8 +- core/sql/regress/compGeneral/TEST006 | 12 +- core/sql/regress/compGeneral/TEST013 | 28 + core/sql/regress/core/EXPECTED001.SB | 48 +- core/sql/regress/core/EXPECTED037.SB | 14 +- core/sql/regress/core/EXPECTED056.SB | 6 - core/sql/regress/core/EXPECTED061.SB | 13 +- core/sql/regress/core/EXPECTED116 | 149 +- core/sql/regress/core/TEST001 | 2 + core/sql/regress/core/TEST029 | 1 - core/sql/regress/core/TEST056 | 2 - core/sql/regress/core/TEST116 | 11 +- core/sql/regress/executor/EXPECTED001 | 80 +- core/sql/regress/executor/EXPECTED013.SB | 59 +- core/sql/regress/executor/EXPECTED014.SB | 73 +- core/sql/regress/executor/EXPECTED016.SB | 10 +- core/sql/regress/executor/EXPECTED122 | 20 +- core/sql/regress/executor/EXPECTED130 | 106 +- core/sql/regress/executor/TEST130 | 32 + core/sql/regress/hive/EXPECTED009 | 7 + core/sql/regress/hive/EXPECTED017 | 3 + core/sql/regress/privs1/EXPECTED132 | 96 +- core/sql/regress/privs2/EXPECTED135 | 2 + core/sql/regress/seabase/EXPECTED010 | 394 +-- core/sql/regress/seabase/EXPECTED011 | 16 +- core/sql/regress/seabase/EXPECTED012 | 5 +- core/sql/regress/seabase/EXPECTED020 | 52 +- core/sql/regress/seabase/EXPECTED022 | 12 +- core/sql/regress/seabase/EXPECTED025 | 7 +- core/sql/regress/seabase/EXPECTED027 | 499 ++- core/sql/regress/seabase/EXPECTED030 | 599 ++++ core/sql/regress/seabase/EXPECTED031 | 214 ++ core/sql/regress/seabase/TEST020 | 15 +- core/sql/regress/seabase/TEST027 | 85 +- core/sql/regress/seabase/TEST030 | 117 + core/sql/regress/seabase/TEST031 | 108 + core/sql/regress/tools/dll-compile.ksh | 4 +- core/sql/regress/tools/runregr_udr.ksh | 73 +- core/sql/regress/udr/EXPECTED102 | 421 +++ core/sql/regress/udr/TEST102 | 119 + core/sql/sqlcat/desc.h | 1 + core/sql/sqlcomp/CmpDescribe.cpp | 30 +- core/sql/sqlcomp/CmpDescribe.h | 2 - core/sql/sqlcomp/CmpSeabaseDDL.h | 79 +- core/sql/sqlcomp/CmpSeabaseDDLauth.cpp | 110 +- core/sql/sqlcomp/CmpSeabaseDDLauth.h | 2 +- core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp | 7 +- core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp | 128 +- core/sql/sqlcomp/CmpSeabaseDDLindex.cpp | 4 +- core/sql/sqlcomp/CmpSeabaseDDLroutine.cpp | 171 ++ core/sql/sqlcomp/CmpSeabaseDDLroutine.h | 280 ++ core/sql/sqlcomp/CmpSeabaseDDLschema.cpp | 114 +- core/sql/sqlcomp/CmpSeabaseDDLtable.cpp | 1761 ++++++++--- core/sql/sqlcomp/DefaultConstants.h | 9 +- core/sql/sqlcomp/PrivMgrComponentOperations.cpp | 24 +- core/sql/sqlcomp/PrivMgrComponentOperations.h | 3 +- core/sql/sqlcomp/PrivMgrComponentPrivileges.cpp | 15 +- core/sql/sqlcomp/PrivMgrComponentPrivileges.h | 3 +- core/sql/sqlcomp/PrivMgrMD.cpp | 129 +- core/sql/sqlcomp/PrivMgrMD.h | 3 +- core/sql/sqlcomp/PrivMgrRoles.cpp | 65 +- core/sql/sqlcomp/PrivMgrRoles.h | 4 +- core/sql/sqlcomp/nadefaults.cpp | 40 +- core/sql/ustat/hs_cli.cpp | 22 +- core/sql/ustat/hs_globals.cpp | 2 +- dcs/Makefile | 2 + dcs/bin/dcs | 20 - dcs/bin/dcs-config.sh | 21 - dcs/bin/dcs-daemon.sh | 21 - dcs/bin/dcs-daemons.sh | 20 - dcs/pom.xml | 157 +- dcs/src/assembly/all.xml | 11 +- .../org/trafodion/dcs/master/DcsMaster.java | 2 - .../dcs/master/MasterStatusServlet.java | 22 +- .../java/org/trafodion/dcs/rest/DcsRest.java | 215 -- .../trafodion/dcs/rest/GetStatusResponse.java | 50 - .../org/trafodion/dcs/rest/RESTServlet.java | 172 -- .../org/trafodion/dcs/rest/ResourceBase.java | 34 - .../org/trafodion/dcs/rest/ResourceConfig.java | 31 - .../org/trafodion/dcs/rest/RestConstants.java | 63 - .../org/trafodion/dcs/rest/RootResource.java | 102 - .../org/trafodion/dcs/rest/ServerConnector.java | 79 - .../org/trafodion/dcs/rest/ServerResource.java | 158 - .../org/trafodion/dcs/rest/VersionResource.java | 122 - .../trafodion/dcs/rest/WorkloadResource.java | 140 - .../org/trafodion/dcs/rest/client/Client.java | 504 --- .../org/trafodion/dcs/rest/client/Cluster.java | 102 - .../org/trafodion/dcs/rest/client/Response.java | 129 - .../trafodion/dcs/rest/model/ServerModel.java | 530 ---- .../trafodion/dcs/rest/model/VersionModel.java | 193 -- .../dcs/rest/model/WorkloadListModel.java | 225 -- .../trafodion/dcs/rest/model/WorkloadModel.java | 139 - .../dcs/rest/provider/JAXBContextResolver.java | 84 - .../producer/PlainTextMessageBodyProducer.java | 92 - .../dcs-webapps/master/aggr_querystats.html | 88 - .../dcs-webapps/master/aggr_querystats.jsp | 85 - .../resources/dcs-webapps/master/explain.html | 62 - .../resources/dcs-webapps/master/queryPlan.jsp | 77 - .../dcs-webapps/master/querystats.html | 89 - .../resources/dcs-webapps/master/querystats.jsp | 89 - .../resources/dcs-webapps/master/repository.jsp | 114 - .../resources/dcs-webapps/master/sessions.html | 87 - .../resources/dcs-webapps/master/sessions.jsp | 89 - .../src/asciidoc/_chapters/odbc_windows.adoc | 10 +- docs/cqd_reference/pom.xml | 8 +- .../src/asciidoc/_chapters/debugging.adoc | 95 + .../src/asciidoc/_chapters/histograms.adoc | 38 +- .../src/asciidoc/_chapters/introduction.adoc | 13 +- .../_chapters/operational_controls.adoc | 138 +- .../src/asciidoc/_chapters/query_execution.adoc | 239 ++ .../src/asciidoc/_chapters/query_plans.adoc | 330 +- .../asciidoc/_chapters/runtime_controls.adoc | 4 +- .../src/asciidoc/_chapters/schema_controls.adoc | 24 +- .../asciidoc/_chapters/table_definition.adoc | 142 + .../src/asciidoc/_chapters/transactions.adoc | 4 +- .../src/asciidoc/_chapters/update_stats.adoc | 31 +- docs/cqd_reference/src/asciidoc/index.adoc | 3 + docs/load_transform/pom.xml | 8 +- .../src/asciidoc/_chapters/about.adoc | 8 +- .../src/asciidoc/_chapters/bulk_load.adoc | 37 +- .../src/asciidoc/_chapters/introduction.adoc | 14 +- .../src/asciidoc/_chapters/tables_indexes.adoc | 18 +- .../src/asciidoc/_chapters/trickle_load.adoc | 12 +- .../src/asciidoc/_chapters/troubleshoot.adoc | 4 +- .../src/asciidoc/_chapters/about.adoc | 4 +- .../src/asciidoc/_chapters/cqds.adoc | 329 -- .../src/asciidoc/_chapters/introduction.adoc | 11 +- .../src/asciidoc/_chapters/sql_statements.adoc | 6 +- docs/sql_reference/src/asciidoc/index.adoc | 1 - docs/src/site/markdown/documentation.md | 66 +- docs/src/site/markdown/logo.md | 2 +- .../resources/images/logos/pb-Trafodion.png | Bin 49089 -> 0 bytes .../resources/images/logos/pb-trafodion.png | Bin 0 -> 49089 bytes docs/src/site/site.xml | 1 + install/installer/checkHBase | 37 + install/installer/dcs_installer | 8 +- install/installer/parseHBaseSite.py | 56 + install/installer/traf_add_sudoAccess | 6 +- install/installer/traf_cloudera_mods98 | 40 +- install/installer/traf_config_check | 56 +- install/installer/traf_config_setup | 3 +- install/installer/traf_getHadoopNodes | 21 +- install/installer/traf_hortonworks_mods98 | 42 +- install/installer/traf_package_setup | 2 +- install/installer/traf_sqgen | 2 +- install/installer/trafodion_install | 86 +- install/traf_tools_setup.sh | 56 +- licenses/lic-clients-bin | 33 +- licenses/lic-dcs-bin | 651 +++- licenses/lic-winodbc-bin | 50 + licenses/not-dcs-bin | 20 + pom.xml | 2 + tools/docker/Dockerfile | 45 + tools/docker/build-base-docker.sh | 29 + tools/docker/start-compile-docker.sh | 67 + wms/pom.xml | 2 +- 299 files changed, 12964 insertions(+), 11880 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/.rat-excludes ---------------------------------------------------------------------- diff --git a/.rat-excludes b/.rat-excludes index 1073071..1c0642d 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -59,6 +59,7 @@ license*.txt* LICENSE.rtf* lic-.*-src lic-.*-bin +not-.*-bin # win generated SetCertificateDirReg_os.vcxproj* SetCertificateDirReg_os.vcxproj.* http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/conn/odbc/src/odbc/nsksrvr/Makefile ---------------------------------------------------------------------- diff --git a/core/conn/odbc/src/odbc/nsksrvr/Makefile b/core/conn/odbc/src/odbc/nsksrvr/Makefile index 2952f9d..36404e6 100644 --- a/core/conn/odbc/src/odbc/nsksrvr/Makefile +++ b/core/conn/odbc/src/odbc/nsksrvr/Makefile @@ -130,6 +130,8 @@ LOC_JSIG=$(JAVA_HOME)/jre/lib/i386/server endif +#gcc 4.8 need explicit enable this flag, otherwise ld must explicitly link with all so using '-l' +GCCMODE += -Xlinker --copy-dt-needed-entries # Produce mxosrvr $(BINEXPDIR)/mxosrvr: $(OBJS) $(LIBEXPDIR)/libzookeeper_mt.so $(CXX) $(GCCMODE) -o $@ $(DBG_FLGS) -L$(LIBEXPDIR) -D_SQ64 -L$(LOC_JSIG) $(EARLY_LIBS) $(COMMON_LIBS) $(DBT_LIBS) -lzookeeper_mt $(HADOOP_LIBS) $(OBJS) -lz http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/conn/odbc/src/odbc/nsksrvrcore/csrvrstmt.cpp ---------------------------------------------------------------------- diff --git a/core/conn/odbc/src/odbc/nsksrvrcore/csrvrstmt.cpp b/core/conn/odbc/src/odbc/nsksrvrcore/csrvrstmt.cpp index ed5ce72..e04682e 100644 --- a/core/conn/odbc/src/odbc/nsksrvrcore/csrvrstmt.cpp +++ b/core/conn/odbc/src/odbc/nsksrvrcore/csrvrstmt.cpp @@ -129,7 +129,12 @@ SRVR_STMT_HDL::SRVR_STMT_HDL() IRD = NULL; outputDataValue._length = 0; outputDataValue._buffer = NULL; - outputDataValue.pad_to_offset_8_ = {'\0', '\0', '\0', '\0'}; + //outputDataValue.pad_to_offset_8_ = {'\0', '\0', '\0', '\0'}; + //use trafodional way to initialze array, above is c++11 in gcc 4.8 + //for gcc 4.4 backward compatibilty, Trafodion now is using c++0x , gcc 4.4 cannot support c++11 mode + //once tested with -std=c++11, may recover this coding style + memset( outputDataValue.pad_to_offset_8_ , 0, sizeof(outputDataValue.pad_to_offset_8_) ); + inputDataValue._length = 0; inputDataValue._buffer = NULL; delayedOutputDataValue._length = 0; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/dbsecurity/auth/Makefile ---------------------------------------------------------------------- diff --git a/core/dbsecurity/auth/Makefile b/core/dbsecurity/auth/Makefile index 1e3ea4c..f916a94 100644 --- a/core/dbsecurity/auth/Makefile +++ b/core/dbsecurity/auth/Makefile @@ -73,7 +73,7 @@ INCLUDES = -I. -I./inc -I ../shared/inc \ -I ../../sql/common -LINK_OPTIONS = -L$(LIBEXPDIR) -lldap -lssl +LINK_OPTIONS = -L$(LIBEXPDIR) -lldap -lssl -llber LINK_OPTIONS += $(LNK_FLGS) COMMON_LIBS = -ltdm_sqlcli -larkcmp_dll http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/rest/pom.xml ---------------------------------------------------------------------- diff --git a/core/rest/pom.xml b/core/rest/pom.xml index 540660a..be74105 100644 --- a/core/rest/pom.xml +++ b/core/rest/pom.xml @@ -417,7 +417,7 @@ <compileSource>1.6</compileSource> <!-- Dependencies --> - <hadoop.version>2.4.0</hadoop.version> + <hadoop.version>${env.HADOOP_DEP_VER}</hadoop.version> <commons-cli.version>1.2</commons-cli.version> <commons-codec.version>1.4</commons-codec.version> <commons-io.version>2.1</commons-io.version> http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/Makefile ---------------------------------------------------------------------- diff --git a/core/sqf/Makefile b/core/sqf/Makefile index 6c3cae9..1a13269 100644 --- a/core/sqf/Makefile +++ b/core/sqf/Makefile @@ -196,7 +196,7 @@ setupdir: -ln -sf $(SQL_W)/sqludr/sqludr.h $(EXPORTINCSQL_DIR) @# mkdir -p export/include/nsk - -ln -sf inc/cextdecs/cextdecs.h export/include/nsk + -ln -sf $(MY_SQROOT)/inc/cextdecs/cextdecs.h export/include/nsk @# USTAT -mkdir -p export/lib/mx_ustat -ln -sf $(SQL_W)/ustat/USAS.sh export/lib/mx_ustat http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/conf/log4cxx.trafodion.masterexe.config ---------------------------------------------------------------------- diff --git a/core/sqf/conf/log4cxx.trafodion.masterexe.config b/core/sqf/conf/log4cxx.trafodion.masterexe.config index 5a8e96d..5d037ce 100644 --- a/core/sqf/conf/log4cxx.trafodion.masterexe.config +++ b/core/sqf/conf/log4cxx.trafodion.masterexe.config @@ -47,6 +47,7 @@ log4j.appender.mxoAppender.layout.ConversionPattern=%d, %p, %c, %m%n log4j.logger.MXOSRVR=ERROR, mxoAppender # SQL +#log4j.logger.SQL.EXE=DEBUG log4j.logger.SQL=INFO log4j.logger.SQL.COMP=ERROR log4j.logger.SQL.EXE.Memory=INFO http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/hbase_utilities/pom.xml ---------------------------------------------------------------------- diff --git a/core/sqf/hbase_utilities/pom.xml b/core/sqf/hbase_utilities/pom.xml index e0c7a74..7d7063f 100644 --- a/core/sqf/hbase_utilities/pom.xml +++ b/core/sqf/hbase_utilities/pom.xml @@ -28,7 +28,7 @@ <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <hadoop.version>2.6.0</hadoop.version> + <hadoop.version>${env.HADOOP_DEP_VER}</hadoop.version> <hbase.version>${env.HBASE_DEP_VER_CDH}</hbase.version> <java.version>1.7</java.version> </properties> http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/monitor/linux/clio.cxx ---------------------------------------------------------------------- diff --git a/core/sqf/monitor/linux/clio.cxx b/core/sqf/monitor/linux/clio.cxx index d181039..672c543 100644 --- a/core/sqf/monitor/linux/clio.cxx +++ b/core/sqf/monitor/linux/clio.cxx @@ -370,6 +370,13 @@ Local_IO_To_Monitor::Local_IO_To_Monitor(int pv_pid) la_node_name[0] = '\0'; } + char *tmpptr = la_node_name; + while ( *tmpptr ) + { + *tmpptr = (char)tolower( *tmpptr); + tmpptr++; + } + char *lp_nodes = getenv("SQ_VIRTUAL_NODES"); if (lp_nodes != NULL) { http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/monitor/linux/cluster.cxx ---------------------------------------------------------------------- diff --git a/core/sqf/monitor/linux/cluster.cxx b/core/sqf/monitor/linux/cluster.cxx index 0373bfd..8190b49 100644 --- a/core/sqf/monitor/linux/cluster.cxx +++ b/core/sqf/monitor/linux/cluster.cxx @@ -1888,7 +1888,13 @@ void CCluster::HandleOtherNodeMsg (struct internal_msg_def *recv_msg, // Queue the node down request for processing by a worker thread. ReqQueue.enqueueDownReq( recv_msg->u.down.pnid ); break; + case InternalType_NodeName: + if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS)) + trace_printf("%s@%d - Internal node name request (%s to %s)\n", method_name, __LINE__, recv_msg->u.nodename.current_name, recv_msg->u.nodename.new_name); + // Queue the node down request for processing by a worker thread. + ReqQueue.enqueueNodeNameReq( recv_msg->u.nodename.current_name, recv_msg->u.nodename.new_name ); + break; case InternalType_SoftNodeDown: if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS)) trace_printf("%s@%d - Internal soft node down request for pnid=%d\n", method_name, __LINE__, recv_msg->u.down.pnid); @@ -2394,6 +2400,11 @@ void CCluster::HandleMyNodeMsg (struct internal_msg_def *recv_msg, trace_printf("%s@%d - Internal down node request for pnid=%d\n", method_name, __LINE__, recv_msg->u.down.pnid); break; + case InternalType_NodeName: + if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS)) + trace_printf("%s@%d - Internal node name request (%s to %s)\n", method_name, __LINE__, recv_msg->u.nodename.current_name, recv_msg->u.nodename.new_name); + break; + case InternalType_SoftNodeDown: if (trace_settings & (TRACE_SYNC | TRACE_REQUEST | TRACE_PROCESS)) trace_printf("%s@%d - Internal soft down node request for pnid=%d\n", method_name, __LINE__, recv_msg->u.down.pnid); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/monitor/linux/cmsh.cxx ---------------------------------------------------------------------- diff --git a/core/sqf/monitor/linux/cmsh.cxx b/core/sqf/monitor/linux/cmsh.cxx index 1fd9af4..c8c4975 100755 --- a/core/sqf/monitor/linux/cmsh.cxx +++ b/core/sqf/monitor/linux/cmsh.cxx @@ -127,19 +127,32 @@ int CCmsh::GetClusterState( PhysicalNodeNameMap_t &physicalNodeMap ) it = physicalNodeMap.find( nodeName.c_str() ); if (it != physicalNodeMap.end()) { - // TEST_POINT: to force state down on node name - const char *downNodeName = getenv( TP001_NODE_DOWN ); - if ( downNodeName != NULL && - !strcmp( downNodeName, nodeName.c_str() ) ) - { - nodeState = StateDown ; - } + // TEST_POINT and Exclude List : to force state down on node name + const char *downNodeName = getenv( TP001_NODE_DOWN ); + const char *downNodeList = getenv( TRAF_EXCLUDE_LIST ); + string downNodeString = " "; + if (downNodeList) + { + downNodeString += downNodeList; + downNodeString += " "; + } + string downNodeToFind = " "; + downNodeToFind += nodeName.c_str(); + downNodeToFind += " "; + if (((downNodeList != NULL) && + strstr(downNodeString.c_str(),downNodeToFind.c_str())) || + ( (downNodeName != NULL) && + !strcmp( downNodeName, nodeName.c_str()) )) + { + nodeState = StateDown; + } + // Set physical node state physicalNode = it->second; physicalNode->SetState( nodeState ); } } - } + } TRACE_EXIT; return( rc ); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/monitor/linux/config.cxx ---------------------------------------------------------------------- diff --git a/core/sqf/monitor/linux/config.cxx b/core/sqf/monitor/linux/config.cxx index aaf4e0b..dc78ba2 100755 --- a/core/sqf/monitor/linux/config.cxx +++ b/core/sqf/monitor/linux/config.cxx @@ -26,6 +26,7 @@ #include <stdlib.h> #include <string.h> #include <iostream> +#include <unistd.h> using namespace std; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/monitor/linux/internal.h ---------------------------------------------------------------------- diff --git a/core/sqf/monitor/linux/internal.h b/core/sqf/monitor/linux/internal.h index 1edec1a..5766bd4 100644 --- a/core/sqf/monitor/linux/internal.h +++ b/core/sqf/monitor/linux/internal.h @@ -39,6 +39,7 @@ enum InternalType InternalType_Exit, // Delete process for monitor InternalType_IoData, // Stdin/Stdout data for a process InternalType_Kill, // Kill monitored process + InternalType_NodeName, // Node Name Change InternalType_Notify, // Register for monitoring of process death InternalType_Process, // Add process to monitor InternalType_ProcessInit, // Process fork completed @@ -140,6 +141,13 @@ struct down_def int pnid; // Physical node id }; +struct nodename_def +{ + char current_name[MAX_PROCESS_NAME]; + char new_name[MAX_PROCESS_NAME]; +}; + + struct shutdown_def { int level; // shutdown level @@ -362,6 +370,7 @@ struct internal_msg_def struct event_def event; ioData_t iodata; struct kill_def kill; + struct nodename_def nodename; struct notify_def notify; struct process_def process; struct process_init_def processInit; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/monitor/linux/makefile ---------------------------------------------------------------------- diff --git a/core/sqf/monitor/linux/makefile b/core/sqf/monitor/linux/makefile index 6a9efac..ef9316b 100755 --- a/core/sqf/monitor/linux/makefile +++ b/core/sqf/monitor/linux/makefile @@ -48,7 +48,7 @@ FLAGS+= -UNDEBUG FLAGS+= -Wall -Wextra # Flags for use in compiling Seabed trace modules -SBFLAGS = -Wall -Wextra -pedantic -Werror -Wno-long-long $(DBG_FLGS) $(OPTIM_FLGS) +SBFLAGS = -Wall -Wextra -pedantic -Werror -Wno-long-long $(DBG_FLGS) $(OPTIM_FLGS) -Xlinker --copy-dt-needed-entries SBDIR = ../../src/seabed/src COMMONLOGGERDIR = ../../commonLogger @@ -84,6 +84,7 @@ INCLUDES+= -I$(LOG4CXX_INC_DIR)/log4cxx LIBS+= -lsqlite3 LIBS+= -llog4cxx +LIBS+= -ldl ifeq ($(USE_TESTPOINTS),1) FLAGS+= -DUSE_TESTPOINTS @@ -130,6 +131,7 @@ MONITORSRC += reqmount.cxx MONITORSRC += reqnewproc.cxx MONITORSRC += reqnodedown.cxx MONITORSRC += reqnodeinfo.cxx +MONITORSRC += reqnodename.cxx MONITORSRC += reqnodeup.cxx MONITORSRC += reqnotify.cxx MONITORSRC += reqopen.cxx @@ -189,6 +191,7 @@ MONITOROBJS += $(OUTDIR)/reqmount.o MONITOROBJS += $(OUTDIR)/reqnewproc.o MONITOROBJS += $(OUTDIR)/reqnodedown.o MONITOROBJS += $(OUTDIR)/reqnodeinfo.o +MONITOROBJS += $(OUTDIR)/reqnodename.o MONITOROBJS += $(OUTDIR)/reqpnodeinfo.o MONITOROBJS += $(OUTDIR)/reqnodeup.o MONITOROBJS += $(OUTDIR)/reqnotify.o http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/monitor/linux/mlio.cxx ---------------------------------------------------------------------- diff --git a/core/sqf/monitor/linux/mlio.cxx b/core/sqf/monitor/linux/mlio.cxx index 20c4c02..47deba3 100644 --- a/core/sqf/monitor/linux/mlio.cxx +++ b/core/sqf/monitor/linux/mlio.cxx @@ -113,7 +113,8 @@ const int SQ_LocalIOToClient::serviceRequestSize[] = { sizeof(REQTYPE) + sizeof( TmSync_def ), // ReqType_TmSync sizeof(REQTYPE) + sizeof( TransInfo_def ), // ReqType_TransInfo sizeof(REQTYPE) + sizeof( MonStats_def ), // ReqType_MonStats - sizeof(REQTYPE) + sizeof( ZoneInfo_def ) // ReqType_ZoneInfo + sizeof(REQTYPE) + sizeof( ZoneInfo_def ), // ReqType_ZoneInfo + sizeof(REQTYPE) + sizeof( NodeName_def ) // ReqType_NodeName }; // The serviceReplySize array holds the size of the request messages for @@ -136,7 +137,8 @@ const int SQ_LocalIOToClient::serviceReplySize[] = { sizeof(REPLYTYPE) + sizeof( Startup_reply_def ), // ReplyType_Startup sizeof(REPLYTYPE) + sizeof( Mount_reply_def ), // ReplyType_Mount sizeof(REPLYTYPE) + sizeof( MonStats_reply_def ), // ReplyType_MonStats - sizeof(REPLYTYPE) + sizeof( ZoneInfo_reply_def ) // ReplyType_ZoneInfo + sizeof(REPLYTYPE) + sizeof( ZoneInfo_reply_def ), // ReplyType_ZoneInfo + sizeof(REPLYTYPE) + sizeof( NodeName_reply_def ) // ReplyType_NodeName }; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/monitor/linux/monitor.cxx ---------------------------------------------------------------------- diff --git a/core/sqf/monitor/linux/monitor.cxx b/core/sqf/monitor/linux/monitor.cxx index b1115fc..31f7928 100755 --- a/core/sqf/monitor/linux/monitor.cxx +++ b/core/sqf/monitor/linux/monitor.cxx @@ -35,6 +35,9 @@ using namespace std; #include <unistd.h> #include <malloc.h> #include <sys/ipc.h> +#include <sys/time.h> +#include <sys/resource.h> //add for getrlimit, strange centos6/gcc4.4 don't need this + //#include <sys/stat.h> #include "monlogging.h" #include "monprof.h" @@ -927,6 +930,12 @@ int main (int argc, char *argv[]) SnmpLog->setPNid( MyPNID ); gethostname(Node_name, MPI_MAX_PROCESSOR_NAME); + char *tmpptr = Node_name; + while ( *tmpptr ) + { + *tmpptr = (char)tolower( *tmpptr ); + tmpptr++; + } #ifdef MULTI_TRACE_FILES setThreadVariable( (char *)"mainThread" ); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/monitor/linux/msgdef.h ---------------------------------------------------------------------- diff --git a/core/sqf/monitor/linux/msgdef.h b/core/sqf/monitor/linux/msgdef.h index 5941950..2fc18b1 100755 --- a/core/sqf/monitor/linux/msgdef.h +++ b/core/sqf/monitor/linux/msgdef.h @@ -257,6 +257,7 @@ typedef enum { ReqType_TransInfo, // request transaction enlistment information ReqType_MonStats, // get monitor statistics ReqType_ZoneInfo, // zone information request + ReqType_NodeName, // change node name request ReqType_Invalid // marks the end of the request // types, add any new request types @@ -285,6 +286,7 @@ typedef enum { ReplyType_Mount, // reply with mount info ReplyType_MonStats, // reply with monitor statistics ReplyType_ZoneInfo, // reply with info on list of zones + ReplyType_NodeName, // reply with results ReplyType_Invalid // marks the end of the reply types, @@ -605,6 +607,19 @@ struct NodeInfo_reply_def bool continuation; // true if continuation of earlier request }; +struct NodeName_def +{ + int nid; // node id of requesting process + int pid; // process id of requesting process + char new_name[MPI_MAX_PROCESSOR_NAME]; // get information on node id (-1 for all) + char current_name[MPI_MAX_PROCESSOR_NAME]; // current name of node (validation) +}; + +struct NodeName_reply_def +{ + int return_code; // error returned to sender +}; + struct NodeJoining_def { int pnid; @@ -1059,6 +1074,7 @@ struct request_def struct PNodeInfo_def pnode_info; struct SpareUp_def spare_up; struct NodeReInt_def reintegrate; + struct NodeName_def nodename; } u; }; @@ -1088,6 +1104,7 @@ struct reply_def struct Close_reply_def close; struct MonStats_reply_def mon_info; struct ZoneInfo_reply_def zone_info; + struct NodeName_reply_def nodename; } u; }; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/monitor/linux/phnode.h ---------------------------------------------------------------------- diff --git a/core/sqf/monitor/linux/phnode.h b/core/sqf/monitor/linux/phnode.h index 937939f..b029ada 100755 --- a/core/sqf/monitor/linux/phnode.h +++ b/core/sqf/monitor/linux/phnode.h @@ -55,6 +55,8 @@ public: inline const char *GetName( void ) { return( name_.c_str() ); } inline NodeState_t GetState( void ) { return( state_ ); } inline void SetState( NodeState_t state ) { state_ = state; } + inline void SetName( const char * newName ) { if (newName) name_ = newName; } + private: string name_; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/monitor/linux/pnode.cxx ---------------------------------------------------------------------- diff --git a/core/sqf/monitor/linux/pnode.cxx b/core/sqf/monitor/linux/pnode.cxx index ab81b2d..dda44f2 100755 --- a/core/sqf/monitor/linux/pnode.cxx +++ b/core/sqf/monitor/linux/pnode.cxx @@ -1528,6 +1528,14 @@ void CNodeContainer::AddNodes( ) int pnid; int rank; int *sparePNids = NULL; + + // only relevant on a workstation acting as a single node + const char* envVar = getenv("SQ_MAX_RANK"); + int maxNode; + if (envVar != NULL) + maxNode = atoi (envVar); + else + maxNode = MAX_NODES; CPNodeConfig *pnodeConfig = clusterConfig_->GetFirstPNodeConfig(); for ( ; pnodeConfig; pnodeConfig = pnodeConfig->GetNext() ) @@ -1561,6 +1569,8 @@ void CNodeContainer::AddNodes( ) } else { + if (pnid >= maxNode) // only for workstation acting as single node + rank = -1; node = new CNode( (char *)pnodeConfig->GetName(), pnid, rank ); assert( node != NULL ); } http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/monitor/linux/pnode.h ---------------------------------------------------------------------- diff --git a/core/sqf/monitor/linux/pnode.h b/core/sqf/monitor/linux/pnode.h index 9bfe655..0cc6fd3 100755 --- a/core/sqf/monitor/linux/pnode.h +++ b/core/sqf/monitor/linux/pnode.h @@ -285,6 +285,7 @@ public: inline void SetTmSyncNid( int nid ) { tmSyncNid_ = nid; } inline void SetTmSyncState( SyncState syncState ) { tmSyncState_ = syncState; } inline void SetZone( int zid ) { zid_ = zid; } + inline void SetName( char *newName ) { if (newName) strcpy (name_, newName); } void StartPStartDProcess( void ); void StartPStartDPersistent( void ); void StartPStartDPersistentDTM( int nid ); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/monitor/linux/pnodeconfig.cxx ---------------------------------------------------------------------- diff --git a/core/sqf/monitor/linux/pnodeconfig.cxx b/core/sqf/monitor/linux/pnodeconfig.cxx index 3c20ef6..6647601 100755 --- a/core/sqf/monitor/linux/pnodeconfig.cxx +++ b/core/sqf/monitor/linux/pnodeconfig.cxx @@ -292,6 +292,12 @@ int CPNodeConfigContainer::GetPNid( char *nodename ) return( pnid ); } +void CPNodeConfig::SetName( char *newName ) +{ + if (newName) + strcpy(name_, newName); +} + CPNodeConfig *CPNodeConfigContainer::GetPNodeConfig( int pnid ) { CPNodeConfig *config; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/monitor/linux/pnodeconfig.h ---------------------------------------------------------------------- diff --git a/core/sqf/monitor/linux/pnodeconfig.h b/core/sqf/monitor/linux/pnodeconfig.h index 939c898..f4c7fb1 100755 --- a/core/sqf/monitor/linux/pnodeconfig.h +++ b/core/sqf/monitor/linux/pnodeconfig.h @@ -77,7 +77,7 @@ public: inline const char *GetName( void ) { return ( name_ ); } inline CPNodeConfig *GetNext( void ) { return ( next_ ); } inline int GetPNid( void ) { return ( pnid_ ); } - + void SetName( char *newName ); int GetSpareList( int sparePNid[] ); inline int GetSparesCount( void ) { return ( sparePNidsCount_ ); } http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/monitor/linux/replicate.cxx ---------------------------------------------------------------------- diff --git a/core/sqf/monitor/linux/replicate.cxx b/core/sqf/monitor/linux/replicate.cxx index 895cf94..fb2565f 100755 --- a/core/sqf/monitor/linux/replicate.cxx +++ b/core/sqf/monitor/linux/replicate.cxx @@ -68,7 +68,7 @@ void CReplObj::validateObj() // Determine the maximum size of a replication object (excluding CReplEvent) int CReplObj::calcAllocSize() { - return max(max(max(max(max(max(max(max(max(max(max(max(max(max(max(max(max(max(sizeof(CReplSoftNodeUp), + return max(max(max(max(max(max(max(max(max(max(max(max(max(max(max(max(max(max(max(sizeof(CReplSoftNodeUp), sizeof(CReplSoftNodeDown)), sizeof(CReplSchedData)), sizeof(CReplActivateSpare)), @@ -81,6 +81,7 @@ int CReplObj::calcAllocSize() sizeof(CReplKill)), sizeof(CReplDevice)), sizeof(CReplNodeDown)), + sizeof(CReplNodeName)), sizeof(CReplNodeUp)), sizeof(CReplDump)), sizeof(CReplDumpComplete)), @@ -1208,6 +1209,59 @@ bool CReplShutdown::replicate(struct internal_msg_def *&msg) return true; } +CReplNodeName::CReplNodeName(const char *current_name, const char* new_name) +{ + // Add eyecatcher sequence as a debugging aid + memcpy(&eyecatcher_, "RPLZ", 4); + + // Compute message size (adjust if needed to conform to + // internal_msg_def structure alignment). + replSize_ = (MSG_HDR_SIZE + sizeof ( nodename_def ) + msgAlignment_) + & ~msgAlignment_; + + if (trace_settings & (TRACE_SYNC_DETAIL | TRACE_PROCESS_DETAIL)) + { + const char method_name[] = "CReplNodeName::CReplNodeName"; + trace_printf("%s@%d - Changing node name, old name =%s, new name = %s\n", + method_name, __LINE__, current_name, new_name); + } + + current_name_ = current_name; + new_name_ = new_name; +} + +CReplNodeName::~CReplNodeName() +{ + const char method_name[] = "CReplNodeName::~CReplNodeName"; + + if (trace_settings & (TRACE_SYNC_DETAIL | TRACE_PROCESS_DETAIL)) + trace_printf("%s@%d - Node name change replication\n", method_name, __LINE__ ); + + // Alter eyecatcher sequence as a debugging aid to identify deleted object + memcpy(&eyecatcher_, "rplz", 4); +} + +bool CReplNodeName::replicate(struct internal_msg_def *&msg) +{ + const char method_name[] = "CReplNodeName::replicate"; + TRACE_ENTRY; + + if (trace_settings & (TRACE_SYNC | TRACE_PROCESS)) + trace_printf("%s@%d" " - Changing node name (%s to %s)\n", method_name, __LINE__, current_name_.c_str(), new_name_.c_str()); + + // build message to replicate this process kill to other nodes + msg->type = InternalType_NodeName; + strcpy (msg->u.nodename.new_name, new_name_.c_str()); + strcpy (msg->u.nodename.current_name, current_name_.c_str()); + + // Advance sync buffer pointer + Nodes->AddMsg( msg, replSize() ); + + TRACE_EXIT; + + return true; +} + CReplNodeDown::CReplNodeDown(int pnid) : pnid_(pnid) { // Add eyecatcher sequence as a debugging aid http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/monitor/linux/replicate.h ---------------------------------------------------------------------- diff --git a/core/sqf/monitor/linux/replicate.h b/core/sqf/monitor/linux/replicate.h index 1b7e6ac..92bf985 100755 --- a/core/sqf/monitor/linux/replicate.h +++ b/core/sqf/monitor/linux/replicate.h @@ -284,6 +284,20 @@ private: int pnid_; }; +class CReplNodeName: public CReplObj +{ +public: + CReplNodeName(const char *current_name, const char *new_name); + virtual ~CReplNodeName(); + + bool replicate(struct internal_msg_def *& msg); + +private: + string current_name_; + string new_name_; +}; + + class CReplNodeUp: public CReplObj { public: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/monitor/linux/reqnodename.cxx ---------------------------------------------------------------------- diff --git a/core/sqf/monitor/linux/reqnodename.cxx b/core/sqf/monitor/linux/reqnodename.cxx new file mode 100644 index 0000000..6c55e23 --- /dev/null +++ b/core/sqf/monitor/linux/reqnodename.cxx @@ -0,0 +1,114 @@ +/////////////////////////////////////////////////////////////////////////////// +// +// @@@ START COPYRIGHT @@@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// @@@ END COPYRIGHT @@@ +// +/////////////////////////////////////////////////////////////////////////////// + +#include <stdio.h> +#include "reqqueue.h" +#include "montrace.h" +#include "monsonar.h" +#include "monlogging.h" +#include "replicate.h" +#include "mlio.h" + +extern CMonitor *Monitor; +extern CMonStats *MonStats; +extern CNodeContainer *Nodes; +extern CReplicate Replicator; +extern CNode *MyNode; + + +CExtNodeNameReq::CExtNodeNameReq (reqQueueMsg_t msgType, int pid, + struct message_def *msg ) + : CExternalReq(msgType, pid, msg) +{ + // Add eyecatcher sequence as a debugging aid + memcpy(&eyecatcher_, "RQEZ", 4); +} + +CExtNodeNameReq::~CExtNodeNameReq() +{ + // Alter eyecatcher sequence as a debugging aid to identify deleted object + memcpy(&eyecatcher_, "rqez", 4); +} + +void CExtNodeNameReq::populateRequestString( void ) +{ + char strBuf[MON_STRING_BUF_SIZE/2] = { 0 }; + + snprintf( strBuf, sizeof(strBuf), + "ExtReq(%s) req #=%ld requester(pid=%d)" + , CReqQueue::svcReqType[reqType_], getId(), pid_); + requestString_.assign( strBuf ); +} + +void CExtNodeNameReq::performRequest() +{ + const char method_name[] = "CExtNodeNameReq::performRequest"; + TRACE_ENTRY; + + CNode *node = NULL; + + // Trace info about request + node = Nodes->GetNode(msg_->u.request.u.nodename.current_name); + if (node) + { + if (trace_settings & (TRACE_REQUEST | TRACE_PROCESS)) + { + trace_printf("[%s], Requested node name change from %s to %s\n" , method_name, msg_->u.request.u.nodename.current_name, msg_->u.request.u.nodename.new_name); + } + + node->SetName(msg_->u.request.u.nodename.new_name); + + CReplNodeName *repl = new CReplNodeName(msg_->u.request.u.nodename.current_name, + msg_->u.request.u.nodename.new_name); + Replicator.addItem(repl); + if (!msg_->noreply) // client needs a reply + { + msg_->u.reply.type = ReplyType_NodeInfo; + msg_->u.reply.u.generic.return_code = MPI_SUCCESS; + + // Send reply to requester + lioreply(msg_, pid_); + } + } else + { + if (trace_settings & (TRACE_REQUEST | TRACE_PROCESS)) + { + trace_printf("[%s], Internal Error - Requested node name change from %s to %s\n" , method_name, msg_->u.request.u.nodename.current_name, msg_->u.request.u.nodename.new_name); + } + + if (!msg_->noreply) // client needs a reply + { + msg_->u.reply.type = ReplyType_NodeInfo; + msg_->u.reply.u.generic.return_code = MPI_ERR_UNKNOWN; + + // Send reply to requester + lioreply(msg_, pid_); + } + + } + + TRACE_EXIT; + +} http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/monitor/linux/reqqueue.cxx ---------------------------------------------------------------------- diff --git a/core/sqf/monitor/linux/reqqueue.cxx b/core/sqf/monitor/linux/reqqueue.cxx index 5fb8e38..fcf7620 100644 --- a/core/sqf/monitor/linux/reqqueue.cxx +++ b/core/sqf/monitor/linux/reqqueue.cxx @@ -1623,6 +1623,48 @@ void CIntShutdownReq::performRequest() TRACE_EXIT; } +CIntNodeNameReq::CIntNodeNameReq( const char *current_name, const char *new_name ) + : CInternalReq() +{ + // Add eyecatcher sequence as a debugging aid + memcpy(&eyecatcher_, "RQIZ", 4); + new_name_ = new_name; + current_name_=current_name; +} + +CIntNodeNameReq::~CIntNodeNameReq() +{ + // Alter eyecatcher sequence as a debugging aid to identify deleted object + memcpy(&eyecatcher_, "rqiz", 4); +} + +void CIntNodeNameReq::populateRequestString( void ) +{ + char strBuf[MON_STRING_BUF_SIZE/2]; + sprintf( strBuf, "IntReq(%s) req #=%ld" + , CReqQueue::intReqType[InternalType_NodeName] + , getId() ); + requestString_.assign( strBuf ); +} + +void CIntNodeNameReq::performRequest() +{ + const char method_name[] = "CIntNodeNameReq::performRequest"; + TRACE_ENTRY; + char current_n[MAX_PROCESS_NAME]; + char new_n[MAX_PROCESS_NAME]; + + strcpy (current_n, current_name_.c_str()); + strcpy (new_n, new_name_.c_str()); + + CNode *node = Nodes->GetNode(current_n); + if (node) + node->SetName(new_n); + + TRACE_EXIT; +} + + CIntDownReq::CIntDownReq( int pnid ) : CInternalReq(), pnid_ ( pnid ) @@ -2591,6 +2633,11 @@ CExternalReq *CReqQueue::prepExternalReq(CExternalReq::reqQueueMsg_t msgType, request->setConcurrent(reqConcurrent[msg->u.request.type]); break; + case ReqType_NodeName: + request = new CExtNodeNameReq(msgType, pid, msg); + request->setConcurrent(reqConcurrent[msg->u.request.type]); + break; + case ReqType_ProcessInfo: request = new CExtProcInfoReq(msgType, pid, msg); request->setConcurrent(reqConcurrent[msg->u.request.type]); @@ -2916,6 +2963,15 @@ void CReqQueue::enqueueDownReq( int pnid ) enqueueReq ( request ); } +void CReqQueue::enqueueNodeNameReq( char *current_name, char *new_name) +{ + CInternalReq * request; + + request = new CIntNodeNameReq ( current_name, new_name ); + + enqueueReq ( request ); +} + void CReqQueue::enqueueSoftNodeDownReq( int pnid ) { CInternalReq * request; @@ -3565,7 +3621,8 @@ const bool CReqQueue::reqConcurrent[] = { false, // ReqType_TmSync false, // ReqType_TransInfo true, // ReqType_MonStats - true // ReqType_ZoneInfo + true, // ReqType_ZoneInfo + false // ReqType_NodeName }; #endif @@ -3602,6 +3659,7 @@ const bool CReqQueue::reqConcurrent[] = { false, // ReqType_TransInfo false, // ReqType_MonStats false, // ReqType_ZoneInfo + false, // ReqType_NodeName false // ReqType_Invalid }; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/monitor/linux/reqqueue.h ---------------------------------------------------------------------- diff --git a/core/sqf/monitor/linux/reqqueue.h b/core/sqf/monitor/linux/reqqueue.h index 43e008e..4f5a21b 100755 --- a/core/sqf/monitor/linux/reqqueue.h +++ b/core/sqf/monitor/linux/reqqueue.h @@ -220,6 +220,20 @@ private: void populateRequestString( void ); }; +class CExtNodeNameReq: public CExternalReq +{ +public: + CExtNodeNameReq (reqQueueMsg_t msgType, int pid, + struct message_def *msg ); + virtual ~CExtNodeNameReq(); + + void performRequest(); + +private: + void populateRequestString( void ); +}; + + class CExtEventReq: public CExternalReq { public: @@ -849,6 +863,21 @@ private: int level_; }; +class CIntNodeNameReq: public CInternalReq +{ +public: + CIntNodeNameReq( const char *current_name, const char *new_name ); + virtual ~CIntNodeNameReq(); + + void performRequest(); + +private: + void populateRequestString( void ); + + string current_name_; + string new_name_; +}; + class CIntDownReq: public CInternalReq { public: @@ -1027,6 +1056,7 @@ class CReqQueue void enqueueChildDeathReq ( pid_t pid ); void enqueueAttachedDeathReq ( pid_t pid ); void enqueueDownReq( int pnid ); + void enqueueNodeNameReq( char *current_name, char *new_name); void enqueueSoftNodeDownReq( int pnid ); void enqueueSoftNodeUpReq( int pnid ); void enqueueShutdownReq( int level ); @@ -1157,7 +1187,7 @@ private: RQEW CExtZoneInfoReq RQEX (unused) RQEY (unused) - RQEZ (unused) + RQEZ CExtNodeNameReq RQE_ CExtNullReq */ http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/monitor/linux/shell.cxx ---------------------------------------------------------------------- diff --git a/core/sqf/monitor/linux/shell.cxx b/core/sqf/monitor/linux/shell.cxx index 7727c80..c30f1da 100755 --- a/core/sqf/monitor/linux/shell.cxx +++ b/core/sqf/monitor/linux/shell.cxx @@ -183,14 +183,26 @@ bool init_pnode_map( void ) pnodeConfig = ClusterConfig.GetFirstPNodeConfig(); for ( ; pnodeConfig; pnodeConfig = pnodeConfig->GetNext() ) { - // TEST_POINT: to force state down on node name - NodeState_t nodeState = StateUp; - const char *downNodeName = getenv( TP001_NODE_DOWN ); - if ( downNodeName != NULL && - !strcmp( downNodeName, pnodeConfig->GetName() ) ) - { - nodeState = StateDown; - } + // TEST_POINT and exclude list : to force state down on node name + NodeState_t nodeState = StateUp; + const char *downNodeName = getenv( TP001_NODE_DOWN ); + const char *downNodeList = getenv( TRAF_EXCLUDE_LIST ); + string downNodeString = " "; + if (downNodeList) + { + downNodeString += downNodeList; + downNodeString += " "; + } + string downNodeToFind = " "; + downNodeToFind += pnodeConfig->GetName(); + downNodeToFind += " "; + if (((downNodeList != NULL) && + (strstr(downNodeString.c_str(),downNodeToFind.c_str()))) || + ( downNodeName != NULL && + !strcmp( downNodeName, pnodeConfig->GetName() ) )) + { + nodeState = StateDown; + } // effectively remove spare nodes on startup only if ( SpareNodeColdStandby && pnodeConfig->IsSpareNode() ) @@ -2822,7 +2834,7 @@ void help_cmd (void) printf ("[%s] -- ls [{[detail]}] [<path>]\n", MyName); printf ("[%s] -- measure | measure_cpu\n", MyName); printf ("[%s] -- monstats\n", MyName); - printf ("[%s] -- node [info [<nid>]]\n", MyName); + printf ("[%s] -- node [[info [<nid>]] | [name <old name> <new name>]]\n", MyName); printf ("[%s] -- path [<directory>[,<directory>]...]\n", MyName); printf ("[%s] -- ps [{ASE|TSE|DTM|AMP|BO|VR|CS}] [<process_name>|<nid,pid>]\n", MyName); printf ("[%s] -- pwd\n", MyName); @@ -3069,6 +3081,103 @@ const char *zone_type_string( ZoneType type ) return( str ); } +void changeNodeName (char *current_name, char *new_name) +{ + + if ((current_name == NULL) || (new_name == NULL)) + { + printf( "[%s] Error: Invalid node name while attempting to change node name.\n", MyName ); + return; + } + int count; + MPI_Status status; + CPhysicalNode *physicalNode; + PhysicalNodeNameMap_t::iterator it; + pair<PhysicalNodeNameMap_t::iterator, bool> pnmit; + + // Look up name + it = PhysicalNodeMap.find( current_name ); + + if (it != PhysicalNodeMap.end()) + { + physicalNode = it->second; + if (physicalNode) + { + CPhysicalNode *newPhysicalNode = new CPhysicalNode( new_name, physicalNode->GetState() ); + if (newPhysicalNode == NULL) + { + printf( "[%s] Error: Internal error with configuration while changing node name.\n", MyName ); + return; + } + //remove and read + PhysicalNodeMap.erase(current_name); + pnmit = PhysicalNodeMap.insert( PhysicalNodeNameMap_t::value_type + ( newPhysicalNode->GetName(), newPhysicalNode )); + if (pnmit.second == false) + { // Already had an entry with the given key value. + printf( "[%s] Error: Internal error while changing node name. Node name exists, node name=%s\n", MyName, new_name ); + return; + } + } + else + { + printf( "[%s] Error: Internal error while changing node name. Node name=%s\n", MyName,new_name ); + return; + } + } + else + { + printf( "[%s] Error: Internal error while changing node name. Node name=%s\n", MyName, new_name ); + return; + } + + // change in another local location + int pnid = ClusterConfig.GetPNid (current_name); + CPNodeConfig *pConfig = ClusterConfig.GetPNodeConfig (pnid); + if (pConfig != NULL) + pConfig->SetName(new_name); + + //and..... in another local location + for( int i=0; i<NumNodes; i++) + { + if ( strcmp (PNode[i], current_name) == 0) + strcpy(PNode[i],new_name); + } + + // now change it in the monitors + if ( gp_local_mon_io->acquire_msg( &msg ) != 0 ) + { // Could not acquire a message buffer + printf ("[%s] Unable to acquire message buffer.\n", MyName); + return; + } + + msg->type = MsgType_Service; + msg->noreply = false; + msg->reply_tag = REPLY_TAG; + msg->u.request.type = ReqType_NodeName; + msg->u.request.u.nodename.nid = MyNid; + msg->u.request.u.nodename.pid = MyPid; + strcpy (msg->u.request.u.nodename.new_name, new_name); + strcpy (msg->u.request.u.nodename.current_name, current_name); + + gp_local_mon_io->send_recv( msg ); + count = sizeof( *msg ); + status.MPI_TAG = msg->reply_tag; + + if ((status.MPI_TAG == REPLY_TAG) && + (count == sizeof (struct message_def))) + { + if ((msg->type == MsgType_Service) && + (msg->u.reply.type == ReplyType_NodeName)) + { + if (msg->u.reply.u.node_info.return_code != MPI_SUCCESS) + printf ("[%s] Unable to change node name in monitors.\n", MyName); + } + } + else + printf ("[%s] Invalid Message/Reply type for Node Name Change request.\n", MyName); +} + void listNodeInfo( int nid ) { int i; @@ -3083,6 +3192,14 @@ void listNodeInfo( int nid ) return; } + const char *downNodeList = getenv( TRAF_EXCLUDE_LIST ); + string downNodeString = " "; + if (downNodeList) + { + downNodeString += downNodeList; + downNodeString += " "; + } + bool needBanner = true; bool getMoreInfo = false; do @@ -3128,6 +3245,20 @@ void listNodeInfo( int nid ) for (i=0; i < msg->u.reply.u.node_info.num_returned; i++) { + CPNodeConfig *pConfig = ClusterConfig.GetPNodeConfig (msg->u.reply.u.node_info.node[i].pnid); + + if (pConfig != NULL) + { + string downNodeToFind = " "; + downNodeToFind += pConfig->GetName(); + downNodeToFind += " "; + + if ((downNodeList != NULL) && strstr(downNodeString.c_str(),downNodeToFind.c_str())) + { + continue; // We do not want to consider this node since it is in our exclude list + } + } + if ( last_nid != -1 ) { if ( (msg->u.reply.u.node_info.node[i].pnid != @@ -3484,6 +3615,14 @@ void node_cmd (char *cmd_tail) char token[MAX_TOKEN]; char delimiter; char *ptr; + const char *downNodeList = getenv( TRAF_EXCLUDE_LIST ); + string downNodeString = " "; + + if (downNodeList) + { + downNodeString += downNodeList; + downNodeString += " "; + } if (*cmd_tail == '\0') { @@ -3505,9 +3644,14 @@ void node_cmd (char *cmd_tail) } pnodeConfig = lnodeConfig->GetPNodeConfig(); - if ( lnodeConfig ) + if ( pnodeConfig ) { - printf( "[%s] Node[%d]=%s, %s, %s\n" + string downNodeToFind = " "; + downNodeToFind += pnodeConfig->GetName(); + downNodeToFind += " "; + + if (((!downNodeList) || ((downNodeList) && !(strstr(downNodeString.c_str(),downNodeToFind.c_str()))))) + printf( "[%s] Node[%d]=%s, %s, %s\n" , MyName , i , pnodeConfig->GetName() @@ -3521,7 +3665,16 @@ void node_cmd (char *cmd_tail) else { ptr = get_token (cmd_tail, token, &delimiter); - if (strcmp (token, "info") == 0) + if (strcmp (token, "name") == 0) + { + char* ptr2 = get_token (ptr, token, &delimiter); + if (ptr2 && *ptr2) + changeNodeName(token, ptr2); + else + printf ("[%s] Invalid node name\n", MyName); + } + else if (strcmp (token, "info") == 0) + { if (Started) { @@ -4569,6 +4722,18 @@ bool start_monitor( char *cmd_tail, bool warmstart, bool reintegrate ) // Ensure that we are on a node that is part of the configuration char mynode[MPI_MAX_PROCESSOR_NAME]; gethostname(mynode, MPI_MAX_PROCESSOR_NAME); + //JIRA: TRAFODION-1854, hostname contains upper case letters + //change all char in mynode to lowercase + //since cluster config strings will all in lowercase , see CTokenizer::NormalizeCase() + if(!VirtualNodes) // for VirtualNodes, it use same gethostname, so do not tolower + { + char *tmpptr = mynode; + while ( *tmpptr ) + { + *tmpptr = tolower( *tmpptr ); + tmpptr++; + } + } bool nodeInConfig = false; for ( i = 0; i < NumNodes; i++ ) { @@ -6137,6 +6302,12 @@ int main (int argc, char *argv[]) else { gethostname(MyNode, MPI_MAX_PROCESSOR_NAME); + char *tmpptr = MyNode; + while ( *tmpptr ) + { + *tmpptr = (char)tolower( *tmpptr ); + tmpptr++; + } if ( !load_nodes() ) { exit (1); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/monitor/linux/testpoint.h ---------------------------------------------------------------------- diff --git a/core/sqf/monitor/linux/testpoint.h b/core/sqf/monitor/linux/testpoint.h index 4125b7b..058a36f 100755 --- a/core/sqf/monitor/linux/testpoint.h +++ b/core/sqf/monitor/linux/testpoint.h @@ -32,7 +32,7 @@ // Startup in real cluster with down node simulation #define TP001_NODE_DOWN "MON_TP_NODE_DOWN" // down node name at startup - +#define TRAF_EXCLUDE_LIST "TRAF_EXCLUDE_LIST" // exclude node // Node re-integration failure testpoints // Roles: // new - new monitor created by shell http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/sqenvcom.sh ---------------------------------------------------------------------- diff --git a/core/sqf/sqenvcom.sh b/core/sqf/sqenvcom.sh index 4364d2e..bfec256 100644 --- a/core/sqf/sqenvcom.sh +++ b/core/sqf/sqenvcom.sh @@ -150,6 +150,13 @@ export THRIFT_DEP_VER=0.9.0 export HBASE_TRX_ID_CDH=hbase-trx-cdh5_4 export HBASE_TRX_ID_APACHE=hbase-trx-apache1_0_2 export HBASE_TRX_ID_HDP=hbase-trx-hdp2_3 +export HIVE_DEP_VER=0.13.1 +export HADOOP_DEP_VER=2.6.0 + + +# staged build-time dependencies +export HADOOP_BLD_LIB=${TOOLSDIR}/hadoop-${HADOOP_DEP_VER}/lib/native +export HADOOP_BLD_INC=${TOOLSDIR}/hadoop-${HADOOP_DEP_VER}/include # general Hadoop & TRX dependencies - not distro specific, choose one to build against export HBASE_TRXDIR=$MY_SQROOT/export/lib @@ -172,6 +179,7 @@ if [[ "$HBASE_DISTRO" = "APACHE" ]]; then export SQL_JAR=trafodion-sql-${HBVER}-${TRAFODION_VER}.jar fi + # check for workstation env # want to make sure SQ_VIRTUAL_NODES is set in the shell running sqstart # so we can determine if we are on a workstation or not http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/sql/scripts/gensq.pl ---------------------------------------------------------------------- diff --git a/core/sqf/sql/scripts/gensq.pl b/core/sqf/sql/scripts/gensq.pl index e826c99..58ccd06 100755 --- a/core/sqf/sql/scripts/gensq.pl +++ b/core/sqf/sql/scripts/gensq.pl @@ -21,7 +21,6 @@ # # @@@ END COPYRIGHT @@@ # -require "ctime.pl"; use sqnodes; use POSIX; use DBI; @@ -145,6 +144,10 @@ my $DBH = 0; # instance type my $instanceType = ""; +sub getTime { + return strftime("%a %b %d %H:%M:%S %Y\n", localtime(time)); +} + sub printScript { ($dWhich, @rest) = @_; @@ -199,7 +202,7 @@ sub printRMSCheckScript{ } sub printTime { - printScript(1, "# Trafodion Startup script generated @ ",&ctime(time),"\n"); + printScript(1, "# Trafodion Startup script generated @ ",getTime(),"\n"); } sub validate_config_script { @@ -479,24 +482,7 @@ sub genComponentWait { printScript(1, "fi\n"); } -sub genLOBConfig { - # Generate sqconfig.db config for LOB. - # This allows the process startup daemon (pstartd) - # to start it up after a node failure. - for ($i=0; $i < $gdNumNodes; $i++) { - my $l_progname="mxlobsrvr"; - my $l_procargs=""; - my $l_procname="\$ZLOBSRV$i"; - my $l_procname_config = sprintf('$ZLOBSRV%d', $i); - my $l_stdout="stdout_\$ZLOBSRV_$i"; - addDbProcData($l_procname_config, "PERSIST_RETRIES", "10,60"); - addDbProcData($l_procname_config, "PERSIST_ZONES", $i); - addDbPersistProc($l_procname_config, $i, 1); - addDbProcDef( $ProcessType_Generic, $l_procname_config, $i, $l_progname, $l_stdout, $l_procargs); - } - -} sub genIdTmSrv { if ($SQ_IDTMSRV > 0) { @@ -575,6 +561,24 @@ sub genDTM { printScript(1, "done\n"); printScript(1, "echo \"The Transaction Service is Ready.\"\n"); } +sub genLOBConfig { + + # Generate sqconfig.db config for LOB. + # This allows the process startup daemon (pstartd) + # to start it up after a node failure. + for ($i=0; $i < $gdNumNodes; $i++) { + my $l_progname="mxlobsrvr"; + my $l_procargs=""; + my $l_procname="\$ZLOBSRV$i"; + my $l_procname_config = sprintf('$ZLOBSRV%d', $i); + my $l_stdout="stdout_\$ZLOBSRV_$i"; + addDbProcData($l_procname_config, "PERSIST_RETRIES", "10,60"); + addDbProcData($l_procname_config, "PERSIST_ZONES", $i); + addDbPersistProc($l_procname_config, $i, 1); + addDbProcDef( $ProcessType_Generic, $l_procname_config, $i, $l_progname, $l_stdout, $l_procargs); + } + +} sub genSSMPCommand { @@ -660,30 +664,30 @@ sub generateRMS { #generate rmsstart and rmsstop scripts printRMSScript(0, "#!/bin/sh\n"); - printRMSScript(0, "# SQ config/utility file generated @ ",&ctime(time),"\n"); + printRMSScript(0, "# SQ config/utility file generated @ ",getTime(),"\n"); printRMSScript(0, "\n# Start the RMS processes\n"); printRMSScript(0, "sscpstart\n"); printRMSScript(0, "ssmpstart\n"); printRMSStopScript(0, "#!/bin/sh\n"); - printRMSStopScript(0, "# SQ config/utility file generated @ ",&ctime(time),"\n"); + printRMSStopScript(0, "# SQ config/utility file generated @ ",getTime(),"\n"); printRMSStopScript(0, "\n# Stop the RMS processes\n"); printRMSStopScript(0, "ssmpstop\n"); printRMSStopScript(0, "sscpstop\n"); #generate ssmpstart, ssmpstop, sscpstart, sscpstop scripts printRMSScript(1, "#!/bin/sh\n"); - printRMSScript(1, "# SQ config/utility file generated @ ",&ctime(time),"\n"); + printRMSScript(1, "# SQ config/utility file generated @ ",getTime(),"\n"); printRMSScript(2, "\n# Start the SSMP processes\n"); printRMSScript(3, "\n# Start the SSCP processes\n"); printRMSStopScript(1, "#!/bin/sh\n"); - printRMSStopScript(1, "# SQ config/utility file generated @ ",&ctime(time),"\n"); + printRMSStopScript(1, "# SQ config/utility file generated @ ",getTime(),"\n"); printRMSStopScript(1, "sqshell -a << eof\n"); printRMSStopScript(2, "\n!Stop the SSMP processes\n"); printRMSStopScript(3, "\n!Stop the SSCP processes\n"); - printRMSCheckScript(1, "-- SQ config/utility file generated @ ",&ctime(time),"\n"); + printRMSCheckScript(1, "-- SQ config/utility file generated @ ",getTime(),"\n"); printRMSCheckScript(1, "prepare rms_check from select current_timestamp, \n"); printRMSCheckScript(1, "cast('Node' as varchar(5)), \n"); printRMSCheckScript(1, "cast(tokenstr('nodeId:', variable_info) as varchar(3)) node, \n"); @@ -1028,7 +1032,7 @@ sub printInitLinesAuxFiles { my $file_ptr = @_[0]; print $file_ptr "#!/bin/sh\n"; - print $file_ptr "# SQ config/utility file generated @ ", &ctime(time), "\n"; + print $file_ptr "# SQ config/utility file generated @ ", getTime(), "\n"; } sub openFiles { http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/sql/scripts/genstats.sh ---------------------------------------------------------------------- diff --git a/core/sqf/sql/scripts/genstats.sh b/core/sqf/sql/scripts/genstats.sh new file mode 100755 index 0000000..7d7a7f0 --- /dev/null +++ b/core/sqf/sql/scripts/genstats.sh @@ -0,0 +1,43 @@ +#!/bin/sh + +#-- @@@ START COPYRIGHT @@@ +#-- +#-- Licensed to the Apache Software Foundation (ASF) under one +#-- or more contributor license agreements. See the NOTICE file +#-- distributed with this work for additional information +#-- regarding copyright ownership. The ASF licenses this file +#-- to you under the Apache License, Version 2.0 (the +#-- "License"); you may not use this file except in compliance +#-- with the License. You may obtain a copy of the License at +#-- +#-- http://www.apache.org/licenses/LICENSE-2.0 +#-- +#-- Unless required by applicable law or agreed to in writing, +#-- software distributed under the License is distributed on an +#-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +#-- KIND, either express or implied. See the License for the +#-- specific language governing permissions and limitations +#-- under the License. +#-- +#-- @@@ END COPYRIGHT @@@ +# +# This script generates the run-time stats for all the queries currently running. +# The statistics outputs are channeled to stdout. +# +# If no argument is given, the per-operator run-time stats are generated. If arguments +# are given, they are passed to the SQL command "get statistics for qid" without +# modification. A common argument is 'default', which generates the per-table stats. + +lso=$MY_SQROOT/export/limited-support-tools/LSO + +qids=`$lso/offender -s active -t 10 | grep -o 'MXID[^ ]*'` + +for qid in $qids +do + if [ $# = 0 ]; then + echo "get statistics for qid $qid ;" | sqlci + else + echo "get statistics for qid $qid $*;" | sqlci + fi +done + http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/sql/scripts/get_libhdfs_files ---------------------------------------------------------------------- diff --git a/core/sqf/sql/scripts/get_libhdfs_files b/core/sqf/sql/scripts/get_libhdfs_files index abe92d2..55d7197 100755 --- a/core/sqf/sql/scripts/get_libhdfs_files +++ b/core/sqf/sql/scripts/get_libhdfs_files @@ -23,6 +23,11 @@ # This script downloads and/or makes the required libhdfs files # to be able to build Trafodion, which acts as a libhdfs client. # +# This script is now redundant, given that install/traf_tools_setup.sh +# puts the libhdfs dependency in TOOLSDIR. This is now just a +# transitional script until build environment is updated to include +# these files as prerequisite. +# # Basically, what we need are three files: # # hdfs.h (copied to $TGT_INC_DIR) @@ -37,10 +42,11 @@ fi LOGFILE=${LIBHDFS_TEMP_DIR}/build.log # Hadoop source tar file to build libhdfs from -HADOOP_SRC_MIRROR_URL=https://archive.apache.org/dist/hadoop/common/hadoop-2.6.0 -HADOOP_ID=hadoop-2.6.0 +HADOOP_ID=hadoop-${HADOOP_DEP_VER} +HADOOP_MIRROR_URL=https://archive.apache.org/dist/hadoop/common/${HADOOP_ID} HADOOP_SRC_ID=${HADOOP_ID}-src HADOOP_SRC_TAR=${HADOOP_SRC_ID}.tar.gz +HADOOP_BIN_TAR=${HADOOP_ID}.tar.gz # files to build required version of Google Protocol Buffers PROTOBUF_MIRROR_URL=https://github.com/google/protobuf/releases/download/v2.5.0 @@ -56,11 +62,13 @@ TGT_INC_DIR=$MY_SQROOT/export/include TGT_LIB_DIR=$MY_SQROOT/export/lib${SQ_MBTYPE} FORCE_BUILD=false +SOURCE_BUILD=false VERBOSE=false usage() { echo "Usage: $0" echo " [ -f | --force ]" + echo " [ -s | --source ]" echo " [ -v | --verbose ]" echo " [ -d <temp dir> | --tempDir <temp dir> ]" } @@ -74,6 +82,10 @@ do FORCE_BUILD=true ;; + -s|--source) + SOURCE_BUILD=true + ;; + -v|--verbose) VERBOSE=true ;; @@ -99,8 +111,9 @@ do done -if [[ $FORCE_BUILD == true || \ - ! -e ${TGT_INC_DIR}/hdfs.h || \ +if [[ $FORCE_BUILD == true || + ! -e ${TGT_INC_DIR}/hdfs.h || + ! -e ${TGT_LIB_DIR}/libhadoop.so || ! -e ${TGT_LIB_DIR}/libhdfs.so ]]; then if [[ ! -d $LIBHDFS_TEMP_DIR ]]; then @@ -110,113 +123,141 @@ if [[ $FORCE_BUILD == true || \ exit 1 fi fi - cd $LIBHDFS_TEMP_DIR - PROTOBUF_VER=`protoc --version 2>/dev/null | cut -f 2 -d ' '` + if [[ $SOURCE_BUILD != true ]]; then + echo "Downloading Hadoop-common binary distro..." | tee -a ${LOGFILE} + wget ${HADOOP_MIRROR_URL}/${HADOOP_BIN_TAR} 2>&1 >>${LOGFILE} + tar -xzf $HADOOP_BIN_TAR \ + $HADOOP_ID/lib/native/libhadoop\*so\* \ + $HADOOP_ID/lib/native/libhdfs\*so\* \ + $HADOOP_ID/include/hdfs.h + + cp -f ${HADOOP_ID}/include/hdfs.h ${TGT_INC_DIR} + cp -Pf ${HADOOP_ID}/lib/native/libhdfs*.so* ${TGT_LIB_DIR} + cp -Pf ${HADOOP_ID}/lib/native/libhadoop*.so* ${TGT_LIB_DIR} + + else + + PROTOBUF_VER=`protoc --version 2>/dev/null | cut -f 2 -d ' '` + + # download and build protoc v2.5.0 if not already in the path + if [[ "$PROTOBUF_VER" != "${PROTOBUF_VERSION}" ]]; then + if [[ ! -f ${PROTOBUF_TAR} ]]; then + echo "Downloading Google Protocol Buffers..." | tee -a ${LOGFILE} + wget ${PROTOBUF_MIRROR_URL}/${PROTOBUF_TAR} >${LOGFILE} + fi + + if [[ $FORCE_BUILD == true ]]; then + rm -rf ${LIBHDFS_TEMP_DIR}/${PROTOBUF_ID} + rm -rf ${LIBHDFS_TEMP_DIR}/${PROTOBUF_TGT_ID} + fi - # download and build protoc v2.5.0 if not already in the path - if [[ "$PROTOBUF_VER" != "${PROTOBUF_VERSION}" ]]; then - if [[ ! -f ${PROTOBUF_TAR} ]]; then - echo "Downloading Google Protocol Buffers..." | tee -a ${LOGFILE} - wget ${PROTOBUF_MIRROR_URL}/${PROTOBUF_TAR} >${LOGFILE} + if [[ ! -d ${PROTOBUF_ID} ]]; then + echo "Unpacking Google Protocol Buffer tar file..." | tee -a ${LOGFILE} + rm -rf ${LIBHDFS_TEMP_DIR}/${PROTOBUF_TGT_ID} + tar -xzf ${PROTOBUF_TAR} >>${LOGFILE} + fi + + if [[ ! -d $PROTOBUF_TGT_ID ]]; then + cd ${PROTOBUF_ID} + echo "Building Google Protocol Buffers, this could take a while..." | tee -a ${LOGFILE} + if [[ $VERBOSE == true ]]; then + ./configure --prefix=${LIBHDFS_TEMP_DIR}/${PROTOBUF_TGT_ID} 2>&1 | tee -a ${LOGFILE} + else + ./configure --prefix=${LIBHDFS_TEMP_DIR}/${PROTOBUF_TGT_ID} 2>&1 >>${LOGFILE} + fi + if [[ $? != 0 ]]; then + echo "Error during configure step, exiting" | tee -a ${LOGFILE} + exit 1 + fi + make 2>&1 >>${LOGFILE} + if [[ $? != 0 ]]; then + echo "Error during make step, exiting" | tee -a ${LOGFILE} + exit 1 + fi + # skip the tests + # make check 2>&1 >>${LOGFILE} + # if [[ $? != 0 ]]; then + # echo "Error during check step, exiting" | tee -a ${LOGFILE} + # exit 1 + # fi + make install 2>&1 >>${LOGFILE} + if [[ $? != 0 ]]; then + echo "Error during install step, exiting" | tee -a ${LOGFILE} + # remove partial results, if any + rm -rf ${LIBHDFS_TEMP_DIR}/${PROTOBUF_TGT_ID} + exit 1 + fi + fi + + # Tell the Hadoop build to use our custom-built protoc + export HADOOP_PROTOC_PATH=${LIBHDFS_TEMP_DIR}/${PROTOBUF_TGT_ID}/bin/protoc + fi + + cd $LIBHDFS_TEMP_DIR + + if [[ ! -f ${HADOOP_SRC_TAR} ]]; then + echo "Downloading Hadoop tar file ${HADOOP_SRC_TAR}..." | tee -a ${LOGFILE} + wget ${HADOOP_MIRROR_URL}/${HADOOP_SRC_TAR} 2>&1 >>${LOGFILE} fi if [[ $FORCE_BUILD == true ]]; then - rm -rf ${LIBHDFS_TEMP_DIR}/${PROTOBUF_ID} - rm -rf ${LIBHDFS_TEMP_DIR}/${PROTOBUF_TGT_ID} + rm -rf ${LIBHDFS_TEMP_DIR}/${HADOOP_SRC_ID} fi - if [[ ! -d ${PROTOBUF_ID} ]]; then - echo "Unpacking Google Protocol Buffer tar file..." | tee -a ${LOGFILE} - rm -rf ${LIBHDFS_TEMP_DIR}/${PROTOBUF_TGT_ID} - tar -xzf ${PROTOBUF_TAR} >>${LOGFILE} + if [[ ! -d ${HADOOP_SRC_ID} ]]; then + echo "Unpacking Hadoop tar file..." | tee -a ${LOGFILE} + tar -xzf ${HADOOP_SRC_TAR} fi - if [[ ! -d $PROTOBUF_TGT_ID ]]; then - cd ${PROTOBUF_ID} - echo "Building Google Protocol Buffers, this could take a while..." | tee -a ${LOGFILE} + if [[ ! -d ${LIBHDFS_TEMP_DIR}/${HADOOP_SRC_ID}/hadoop-dist/target ]]; then + cd ${HADOOP_SRC_ID} + echo "Building native library, this will take several minutes..." | tee -a ${LOGFILE} if [[ $VERBOSE == true ]]; then - ./configure --prefix=${LIBHDFS_TEMP_DIR}/${PROTOBUF_TGT_ID} 2>&1 | tee -a ${LOGFILE} + mvn package -Pdist,native -Dmaven.javadoc.skip=true -DskipTests -Dtar 2>&1 | tee -a ${LOGFILE} else - ./configure --prefix=${LIBHDFS_TEMP_DIR}/${PROTOBUF_TGT_ID} 2>&1 >>${LOGFILE} - fi - if [[ $? != 0 ]]; then - echo "Error during configure step, exiting" | tee -a ${LOGFILE} - exit 1 + mvn package -Pdist,native -Dmaven.javadoc.skip=true -DskipTests -Dtar 2>&1 >>${LOGFILE} fi - make 2>&1 >>${LOGFILE} if [[ $? != 0 ]]; then - echo "Error during make step, exiting" | tee -a ${LOGFILE} - exit 1 - fi - # skip the tests - # make check 2>&1 >>${LOGFILE} - # if [[ $? != 0 ]]; then - # echo "Error during check step, exiting" | tee -a ${LOGFILE} - # exit 1 - # fi - make install 2>&1 >>${LOGFILE} - if [[ $? != 0 ]]; then - echo "Error during install step, exiting" | tee -a ${LOGFILE} - # remove partial results, if any - rm -rf ${LIBHDFS_TEMP_DIR}/${PROTOBUF_TGT_ID} + echo "Error during Maven build step for libhdfs, exiting" | tee -a ${LOGFILE} exit 1 fi fi - # Tell the Hadoop build to use our custom-built protoc - export HADOOP_PROTOC_PATH=${LIBHDFS_TEMP_DIR}/${PROTOBUF_TGT_ID}/bin/protoc - fi - - cd $LIBHDFS_TEMP_DIR - - if [[ ! -f ${HADOOP_SRC_TAR} ]]; then - echo "Downloading Hadoop tar file ${HADOOP_SRC_TAR}..." | tee -a ${LOGFILE} - wget ${HADOOP_SRC_MIRROR_URL}/${HADOOP_SRC_TAR} 2>&1 >>${LOGFILE} - fi - - if [[ $FORCE_BUILD == true ]]; then - rm -rf ${LIBHDFS_TEMP_DIR}/${HADOOP_SRC_ID} - fi - - if [[ ! -d ${HADOOP_SRC_ID} ]]; then - echo "Unpacking Hadoop tar file..." | tee -a ${LOGFILE} - tar -xzf ${HADOOP_SRC_TAR} - fi - - if [[ ! -d ${LIBHDFS_TEMP_DIR}/${HADOOP_SRC_ID}/hadoop-dist/target ]]; then - cd ${HADOOP_SRC_ID} - echo "Building native library, this will take several minutes..." | tee -a ${LOGFILE} + echo "Copying include file and built libraries to Trafodion export dir..." | tee -a ${LOGFILE} if [[ $VERBOSE == true ]]; then - mvn package -Pdist,native -Dmaven.javadoc.skip=true -DskipTests -Dtar 2>&1 | tee -a ${LOGFILE} - else - mvn package -Pdist,native -Dmaven.javadoc.skip=true -DskipTests -Dtar 2>&1 >>${LOGFILE} - fi - if [[ $? != 0 ]]; then - echo "Error during Maven build step for libhdfs, exiting" | tee -a ${LOGFILE} - exit 1 + set -x fi - fi - - echo "Copying include file and built libraries to Trafodion export dir..." | tee -a ${LOGFILE} - if [[ $VERBOSE == true ]]; then - set -x - fi - cp -f ${LIBHDFS_TEMP_DIR}/${HADOOP_SRC_ID}/hadoop-dist/target/${HADOOP_ID}/include/hdfs.h ${TGT_INC_DIR} - cp -Pf ${LIBHDFS_TEMP_DIR}/${HADOOP_SRC_ID}/hadoop-dist/target/${HADOOP_ID}/lib/native/libhdfs*.so* ${TGT_LIB_DIR} - cp -Pf ${LIBHDFS_TEMP_DIR}/${HADOOP_SRC_ID}/hadoop-dist/target/${HADOOP_ID}/lib/native/libhadoop*.so* ${TGT_LIB_DIR} + cp -f ${LIBHDFS_TEMP_DIR}/${HADOOP_SRC_ID}/hadoop-dist/target/${HADOOP_ID}/include/hdfs.h ${TGT_INC_DIR} + cp -Pf ${LIBHDFS_TEMP_DIR}/${HADOOP_SRC_ID}/hadoop-dist/target/${HADOOP_ID}/lib/native/libhdfs*.so* ${TGT_LIB_DIR} + cp -Pf ${LIBHDFS_TEMP_DIR}/${HADOOP_SRC_ID}/hadoop-dist/target/${HADOOP_ID}/lib/native/libhadoop*.so* ${TGT_LIB_DIR} + fi # source build ls -l ${TGT_INC_DIR}/hdfs.h >> ${LOGFILE} ls -l ${TGT_LIB_DIR}/libhdfs.so >> ${LOGFILE} ls -l ${TGT_LIB_DIR}/libhadoop.so >> ${LOGFILE} # Final check whether all the needed files are there - if [[ ! -r ${TGT_INC_DIR}/hdfs.h || \ + if [[ ! -r ${TGT_INC_DIR}/hdfs.h || + ! -r ${TGT_LIB_DIR}/libhadoop.so || ! -r ${TGT_LIB_DIR}/libhdfs.so ]]; then echo "Error, not all files were created" | tee -a ${LOGFILE} ls -l ${TGT_INC_DIR}/hdfs.h ls -l ${TGT_LIB_DIR}/libhdfs.so + ls -l ${TGT_LIB_DIR}/libhadoop.so exit 1 fi + # check that we have 64=-bit libs + libcheck=0 + file -L ${TGT_LIB_DIR}/libhdfs.so | grep -q 'ELF 64-bit' || libcheck=1 + file -L ${TGT_LIB_DIR}/libhadoop.so | grep -q 'ELF 64-bit' || libcheck=1 + if [[ $libcheck == 1 ]]; then + echo "Error, libraries are not 'ELF 64-bit'" | tee -a ${LOGFILE} + file -L ${TGT_LIB_DIR}/libhdfs.so + file -L ${TGT_LIB_DIR}/libhadoop + exit 1 + fi + fi +exit 0 http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/sql/scripts/sqnodeipcrm ---------------------------------------------------------------------- diff --git a/core/sqf/sql/scripts/sqnodeipcrm b/core/sqf/sql/scripts/sqnodeipcrm index 1f5ec13..00e9153 100755 --- a/core/sqf/sql/scripts/sqnodeipcrm +++ b/core/sqf/sql/scripts/sqnodeipcrm @@ -25,7 +25,7 @@ function rm_shm { lv_port_file=$1 echo "Using Portfile $lv_port_file" declare -i lv_virtual_node - let lv_virtual_node=`echo $lv_port_file | cut -d '.' -f 3` + let lv_virtual_node=`echo $lv_port_file | awk -F/ '{print $NF}' | cut -d '.' -f 3` lv_port_no=`cat $lv_port_file | cut -d ':' -f 2` #echo "lv_virtual_node=$lv_virtual_node" #echo "lv_port_no=$lv_port_no" http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/sql/scripts/sqstart ---------------------------------------------------------------------- diff --git a/core/sqf/sql/scripts/sqstart b/core/sqf/sql/scripts/sqstart index 7c63eb1..93e49bd 100755 --- a/core/sqf/sql/scripts/sqstart +++ b/core/sqf/sql/scripts/sqstart @@ -542,7 +542,8 @@ dcsstart sleep 10 sqcheck | tee -a $SQMON_LOG # Start the LOB Service -lobstart +#comment out since it's unused right now +#lobstart # Start the REST Service reststart http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/sql/scripts/sqstop ---------------------------------------------------------------------- diff --git a/core/sqf/sql/scripts/sqstop b/core/sqf/sql/scripts/sqstop index 55cfa84..e4ecbb1 100755 --- a/core/sqf/sql/scripts/sqstop +++ b/core/sqf/sql/scripts/sqstop @@ -92,7 +92,7 @@ reststop dcsstop # Stop the lob Service -lobstop +#lobstop # Issue the shutdown request echoLog "Shutting down ($shutdowntype) the SQ environment!" http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/src/seabed/src/msmon.cpp ---------------------------------------------------------------------- diff --git a/core/sqf/src/seabed/src/msmon.cpp b/core/sqf/src/seabed/src/msmon.cpp index 42f9600..2f41fb5 100644 --- a/core/sqf/src/seabed/src/msmon.cpp +++ b/core/sqf/src/seabed/src/msmon.cpp @@ -3818,6 +3818,12 @@ int msg_mon_init_attach(const char *pp_where, } gethostname(la_host_name, sizeof(la_host_name)); lp_nodes = getenv(gp_ms_env_sq_vnodes); + char *tmpptr = la_host_name; + while ( *tmpptr ) + { + *tmpptr = (char)tolower( *tmpptr ); + tmpptr++; + } lp_nid = getenv(gp_ms_env_sq_vnid); if ((lp_nid != NULL) && *lp_nid) lv_nid = atoi(lp_nid); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/src/seabed/src/sautil.cpp ---------------------------------------------------------------------- diff --git a/core/sqf/src/seabed/src/sautil.cpp b/core/sqf/src/seabed/src/sautil.cpp index 30b60da..683bc0e 100644 --- a/core/sqf/src/seabed/src/sautil.cpp +++ b/core/sqf/src/seabed/src/sautil.cpp @@ -95,11 +95,6 @@ void SB_SA_Util_assert_fun_com(const char *pp_exp, const char *pp_file, unsigned pv_line, const char *pp_fun) { - pid_t lv_pid; - pid_t lv_tid; - - lv_pid = getpid(); - lv_tid = sb_sa_util_gettid(); gv_sb_sa_util_save_assert.iv_errno = errno; gv_sb_sa_util_save_assert.iv_line = pv_line; gv_sb_sa_util_save_assert.iv_lhs = pv_lhs; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/681cad66/core/sqf/src/seabed/src/stream.cpp ---------------------------------------------------------------------- diff --git a/core/sqf/src/seabed/src/stream.cpp b/core/sqf/src/seabed/src/stream.cpp index 90c5e9c..fccbccd 100644 --- a/core/sqf/src/seabed/src/stream.cpp +++ b/core/sqf/src/seabed/src/stream.cpp @@ -353,11 +353,9 @@ void SB_Trans::Trans_Stream::checksum_recv_check(Rd_Type *pp_rd, int lv_chk; int lv_ctrl_len; int lv_data_len; - int lv_data_off; int lv_inx; lv_chk = -1; - lv_data_off = 0; pv_client = pv_client; // touch lp_ctrl = pp_rd->ip_ctrl; lv_ctrl_len = pp_rd->iv_ctrl_len;
