Fixees to support stroring Udr libraries s Blobs.
Project: http://git-wip-us.apache.org/repos/asf/trafodion/repo Commit: http://git-wip-us.apache.org/repos/asf/trafodion/commit/9528f8c0 Tree: http://git-wip-us.apache.org/repos/asf/trafodion/tree/9528f8c0 Diff: http://git-wip-us.apache.org/repos/asf/trafodion/diff/9528f8c0 Branch: refs/heads/master Commit: 9528f8c0d2c75bb7f783c4b1589d11b1d4b64869 Parents: 5f230e2 Author: Sandhya Sundaresan <[email protected]> Authored: Fri Sep 28 19:44:56 2018 +0000 Committer: Sandhya Sundaresan <[email protected]> Committed: Fri Sep 28 19:44:56 2018 +0000 ---------------------------------------------------------------------- core/sqf/sqenvcom.sh | 8 + core/sqf/sql/scripts/createhooks | 2 + core/sqf/sql/scripts/genms | 2 + core/sqf/sql/scripts/install_local_hadoop | 5 + core/sql/bin/SqlciErrors.txt | 4 +- core/sql/cli/Cli.cpp | 16 +- core/sql/cli/sqlcli.h | 2 +- core/sql/comexe/ComTdb.h | 10 +- core/sql/comexe/ComTdbUdr.cpp | 36 +- core/sql/comexe/ComTdbUdr.h | 11 +- core/sql/common/ComMisc.cpp | 117 ++ core/sql/common/ComMisc.h | 5 + core/sql/executor/ExExeUtil.h | 1 + core/sql/executor/ExExeUtilLoad.cpp | 39 + core/sql/executor/ExUdr.cpp | 92 +- core/sql/executor/ExUdr.h | 14 +- core/sql/generator/GenUdr.cpp | 31 +- core/sql/generator/Generator.cpp | 8 + core/sql/optimizer/NARoutine.cpp | 26 + core/sql/optimizer/NARoutine.h | 16 +- core/sql/optimizer/UdfDllInteraction.cpp | 65 +- core/sql/regress/executor/EXPECTED130 | 82 +- core/sql/regress/tools/rgrCleanup | 4 + core/sql/regress/tools/runregr | 2 + core/sql/regress/udr/EXPECTED001 | 24 +- core/sql/regress/udr/EXPECTED103 | 10 +- core/sql/regress/udr/EXPECTED108 | 4 +- core/sql/regress/udr/FILTER001 | 2 +- core/sql/regress/udr/FILTER108 | 2 +- core/sql/regress/udr/TEST001 | 4 +- core/sql/sqlcat/TrafDDLdesc.cpp | 4 +- core/sql/sqlcat/TrafDDLdesc.h | 6 +- core/sql/sqlcomp/CmpDDLCatErrorCodes.h | 2 +- core/sql/sqlcomp/CmpSeabaseDDL.h | 33 +- core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp | 89 +- core/sql/sqlcomp/CmpSeabaseDDLincludes.h | 2 +- core/sql/sqlcomp/CmpSeabaseDDLinitraf.cpp | 36 + core/sql/sqlcomp/CmpSeabaseDDLmd.h | 12 +- core/sql/sqlcomp/CmpSeabaseDDLrepos.h | 29 +- core/sql/sqlcomp/CmpSeabaseDDLroutine.cpp | 1800 +++++++++++++++++++++--- core/sql/sqlcomp/CmpSeabaseDDLroutine.h | 77 + core/sql/sqlcomp/CmpSeabaseDDLtable.cpp | 149 +- core/sql/sqlcomp/CmpSeabaseDDLupgrade.cpp | 184 ++- core/sql/sqlcomp/CmpSeabaseDDLupgrade.h | 185 ++- core/sql/sqlcomp/DefaultConstants.h | 2 + core/sql/sqlcomp/nadefaults.cpp | 9 +- 46 files changed, 2878 insertions(+), 385 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/trafodion/blob/9528f8c0/core/sqf/sqenvcom.sh ---------------------------------------------------------------------- diff --git a/core/sqf/sqenvcom.sh b/core/sqf/sqenvcom.sh index 1010a44..804e03f 100644 --- a/core/sqf/sqenvcom.sh +++ b/core/sqf/sqenvcom.sh @@ -61,6 +61,13 @@ fi export SQ_IC=${SQ_IC:-TCP} export MPI_IC_ORDER=$SQ_IC +export ARCH=`arch` +if [ "${ARCH:0:3}" == "ppc" ]; then + export JRE_LIB_DIR=${ARCH} +else + export JRE_LIB_DIR="amd64" +fi + # use sock #export SQ_TRANS_SOCK=1 @@ -200,6 +207,7 @@ export DTM_COMMON_JAR=trafodion-dtm-cdh-${TRAFODION_VER}.jar export SQL_JAR=trafodion-sql-cdh-${TRAFODION_VER}.jar export UTIL_JAR=trafodion-utility-${TRAFODION_VER}.jar export JDBCT4_JAR=jdbcT4-${TRAFODION_VER}.jar +export MY_UDR_CACHE_LIBDIR=cached_libs if [[ "$HBASE_DISTRO" == "HDP" ]]; then http://git-wip-us.apache.org/repos/asf/trafodion/blob/9528f8c0/core/sqf/sql/scripts/createhooks ---------------------------------------------------------------------- diff --git a/core/sqf/sql/scripts/createhooks b/core/sqf/sql/scripts/createhooks index e83b4c3..eb2b483 100755 --- a/core/sqf/sql/scripts/createhooks +++ b/core/sqf/sql/scripts/createhooks @@ -41,6 +41,8 @@ touch revive.hook touch snmppa.hook touch spp.hook touch sqlci.hook +touch tdm_arkcmp.hook +touch tdm_arkesp.hook touch tdm_arkqvp.hook touch tm.hook touch TPA.hook http://git-wip-us.apache.org/repos/asf/trafodion/blob/9528f8c0/core/sqf/sql/scripts/genms ---------------------------------------------------------------------- diff --git a/core/sqf/sql/scripts/genms b/core/sqf/sql/scripts/genms index 29ac957..794e2e2 100755 --- a/core/sqf/sql/scripts/genms +++ b/core/sqf/sql/scripts/genms @@ -183,6 +183,8 @@ else echo "MY_UDR_ROOT=\$TRAF_HOME/udr" fi +echo "MY_UDR_CACHE_LIBDIR=cached_libs" + echo "" echo "#creating sqllogs folder for capturing osim data" sqllogdir=$HOME/sqllogs http://git-wip-us.apache.org/repos/asf/trafodion/blob/9528f8c0/core/sqf/sql/scripts/install_local_hadoop ---------------------------------------------------------------------- diff --git a/core/sqf/sql/scripts/install_local_hadoop b/core/sqf/sql/scripts/install_local_hadoop index ca7789a..807ba74 100755 --- a/core/sqf/sql/scripts/install_local_hadoop +++ b/core/sqf/sql/scripts/install_local_hadoop @@ -1373,6 +1373,11 @@ echo "$MY_LOCAL_SW_DIST/${MYSQL_JDBC_TAR}" fi # end of MySQL JDBC setup +#udr cached lib setup +echo "Creating udr cached lib : ${MY_UDR_CACHE_LIBDIR}" +mkdir -p $MY_PUBLIC_UDR_CACHE_LIBDIR +#end of udr cached lib setup + cd $MY_SW_ROOT if [ -d hive/bin ]; then http://git-wip-us.apache.org/repos/asf/trafodion/blob/9528f8c0/core/sql/bin/SqlciErrors.txt ---------------------------------------------------------------------- diff --git a/core/sql/bin/SqlciErrors.txt b/core/sql/bin/SqlciErrors.txt index 7032bf7..6f88b9f 100644 --- a/core/sql/bin/SqlciErrors.txt +++ b/core/sql/bin/SqlciErrors.txt @@ -37,7 +37,7 @@ 1035 ZZZZZ 99999 BEGINNER MAJOR DBADMIN Catalog $0~CatalogName already exists. 1036 ZZZZZ 99999 BEGINNER MAJOR DBADMIN Authorization ID $0~String0 cannot grant to authorization ID $1~String1 because it could create a circular dependency. 1037 ZZZZZ 99999 BEGINNER MAJOR DBADMIN Revoke failed because of a dependent grant between authorization ID $0~string0 and authorization ID $1~string1. -1038 ZZZZZ 99999 BEGINNER MAJOR DBADMIN --- unused --- +1038 ZZZZZ 99999 BEGINNER MAJOR DBADMIN Unable to upgrade library management 1039 ZZZZZ 99999 BEGINNER MAJOR DBADMIN No privileges were revoked. You lack the grant option for the specified privilege(s). 1040 ZZZZZ 99999 BEGINNER MAJOR DBADMIN The use of ALTER on metadata tables is not permitted. 1041 ZZZZZ 99999 BEGINNER MINOR DBADMIN The primary key has already been defined. @@ -1327,7 +1327,7 @@ $1~String1 -------------------------------- 4313 ZZZZZ 99999 BEGINNER MAJOR DBADMIN Function rand() is not supported. 4314 ZZZZZ 99999 BEGINNER MAJOR DBADMIN An error was detected while creating an audit row image for $0~TableName. AUDIT_IMAGE function is only supported on index tables. 4315 ZZZZZ 99999 BEGINNER MAJOR DBADMIN An error was detected while creating an audit row image. The columns defined for the index $0~TableName do not match what was requested for the audit row image. -4316 ZZZZZ 99999 BEGINNER MAJOR DBADMIN An error was detected while creating an audit row image. The data type defined for the index $0~TableName and $0~ColumnName do not match what was requested for the audit row image. +4316 ZZZZZ 99999 BEGINNER MAJOR DBADMIN An error was detected while extracting a udr routine library to local cache : $0~string0 . 4320 ZZZZZ 99999 BEGINNER MAJOR DBADMIN Stream access is not allowed on multi-partitioned table or index, when flag ATTEMPT_ASYNCHRONOUS_ACCESS is set to OFF. Object in scope: $0~TableName. 4321 ZZZZZ 99999 BEGINNER MAJOR DBADMIN An embedded update/delete is not allowed on a partitioned table, when flag ATTEMPT_ASYNCHRONOUS_ACCESS is set to OFF. Object in scope: $0~TableName. 4322 0A000 99999 BEGINNER MAJOR DBADMIN A column with BLOB datatype cannot be used in this clause or function. http://git-wip-us.apache.org/repos/asf/trafodion/blob/9528f8c0/core/sql/cli/Cli.cpp ---------------------------------------------------------------------- diff --git a/core/sql/cli/Cli.cpp b/core/sql/cli/Cli.cpp index 9e14307..9fdc051 100644 --- a/core/sql/cli/Cli.cpp +++ b/core/sql/cli/Cli.cpp @@ -2203,6 +2203,14 @@ Lng32 SQLCLI_ProcessRetryQuery( } } } + + // if this is a udr/call statement and not a standalone statement, return an error + //The caller will need to reprepare the query so we have an updated tdb to execute. + // If it's a standalone statement, the statement will get reprepared and executed. + if ((rootTdb->getUdrCount() > 0) && !stmt->isStandaloneQ()) + { + return retcode; + } } AQRStatementInfo * aqrSI = NULL; @@ -9541,7 +9549,7 @@ Lng32 SQLCLI_LOBddlInterface exLobGlob = ExpLOBoper::initLOBglobal(currContext.exHeap(), &currContext, useLibHdfs); if (exLobGlob == NULL) { - cliRC = 0; + cliRC = -1; ComDiagsArea * da = &diags; ExRaiseSqlError(currContext.exHeap(), &da, (ExeErrorCode)(8442), NULL, &cliRC , @@ -9560,7 +9568,7 @@ Lng32 SQLCLI_LOBddlInterface if (rc) { - cliRC = 0; + cliRC = -1; ComDiagsArea * da = &diags; ExRaiseSqlError(currContext.exHeap(), &da, (ExeErrorCode)(8442), NULL, &cliRC , @@ -9656,7 +9664,7 @@ Lng32 SQLCLI_LOBddlInterface exLobGlob = ExpLOBoper::initLOBglobal(currContext.exHeap(), &currContext, useLibHdfs); if (exLobGlob == NULL) { - cliRC = 0; + cliRC = -1; ComDiagsArea * da = &diags; ExRaiseSqlError(currContext.exHeap(), &da, (ExeErrorCode)(8442), NULL, &cliRC , @@ -9705,7 +9713,7 @@ Lng32 SQLCLI_LOBddlInterface exLobGlob = ExpLOBoper::initLOBglobal(currContext.exHeap(), &currContext, useLibHdfs); if (exLobGlob == NULL) { - cliRC = 0; + cliRC = -1; ComDiagsArea * da = &diags; ExRaiseSqlError(currContext.exHeap(), &da, (ExeErrorCode)(8442), NULL, &cliRC , http://git-wip-us.apache.org/repos/asf/trafodion/blob/9528f8c0/core/sql/cli/sqlcli.h ---------------------------------------------------------------------- diff --git a/core/sql/cli/sqlcli.h b/core/sql/cli/sqlcli.h index 00ae7b2..d01048e 100644 --- a/core/sql/cli/sqlcli.h +++ b/core/sql/cli/sqlcli.h @@ -1,4 +1,4 @@ -/********************************************************************** +/*********************************************************************y // @@@ START COPYRIGHT @@@ // // Licensed to the Apache Software Foundation (ASF) under one http://git-wip-us.apache.org/repos/asf/trafodion/blob/9528f8c0/core/sql/comexe/ComTdb.h ---------------------------------------------------------------------- diff --git a/core/sql/comexe/ComTdb.h b/core/sql/comexe/ComTdb.h index e00728f..5271784 100644 --- a/core/sql/comexe/ComTdb.h +++ b/core/sql/comexe/ComTdb.h @@ -44,6 +44,7 @@ #include "sqlcli.h" #include "ComSmallDefs.h" #include "PrivMgrDesc.h" // Privilege descriptors +#include "ExpLOBenums.h" // ----------------------------------------------------------------------- // Classes defined in this file @@ -1013,11 +1014,12 @@ class ComTdbVirtTableRoutineInfo : public ComTdbVirtTableBase Int16 d, const char * sa, Int16 con, Int16 i, const char * ps, const char * ta, Int32 mr, Int32 sas, const char * en, const char * p, const char * uv, const char * es, const char * em, - const char * lf, Int32 lv, const char * s, const char *ls) + const char * lf, Int32 lv, const char * s, const char *ls, Int64 luid + ) : ComTdbVirtTableBase(), routine_name(rn), deterministic(d), call_on_null(con), isolate(i), max_results(mr), state_area_size(sas), external_name(en), library_filename(lf), library_version(lv), - signature(s), library_sqlname(ls) + signature(s), library_sqlname(ls),lib_obj_uid(luid) { strcpy(UDR_type, ut); strcpy(language_type, lt); @@ -1061,6 +1063,10 @@ class ComTdbVirtTableRoutineInfo : public ComTdbVirtTableBase Int64 object_uid; Int32 object_owner_id; Int32 schema_owner_id; + Int64 lib_redef_time; + char *lib_blob_handle; + char *lib_sch_name; + Int64 lib_obj_uid; }; class ComTdbVirtTableSequenceInfo : public ComTdbVirtTableBase http://git-wip-us.apache.org/repos/asf/trafodion/blob/9528f8c0/core/sql/comexe/ComTdbUdr.cpp ---------------------------------------------------------------------- diff --git a/core/sql/comexe/ComTdbUdr.cpp b/core/sql/comexe/ComTdbUdr.cpp index ab29c3a..08c0ba3 100644 --- a/core/sql/comexe/ComTdbUdr.cpp +++ b/core/sql/comexe/ComTdbUdr.cpp @@ -45,7 +45,10 @@ ComTdbUdr::ComTdbUdr(char *sqlName, char *containerName, char *externalPath, char *librarySqlName, - + Int64 libraryRedefTime, + char *libraryBlobHandle, + char *librarySchName, + Int32 libraryVersion, char *runtimeOptions, char *runtimeOptionDelimiters, @@ -111,7 +114,10 @@ ComTdbUdr::ComTdbUdr(char *sqlName, containerName_(containerName), externalPath_(externalPath), librarySqlName_(librarySqlName), - + libraryRedefTime_(libraryRedefTime), + libraryBlobHandle_(libraryBlobHandle), + librarySchName_(librarySchName), + libraryVersion_(libraryVersion), runtimeOptions_(runtimeOptions), runtimeOptionDelimiters_(runtimeOptionDelimiters), @@ -206,6 +212,8 @@ Long ComTdbUdr::pack(void *space) optionalData_.pack(space); udrSerInvocationInfo_.pack(space); udrSerPlanInfo_.pack(space); + libraryBlobHandle_.pack(space); + librarySchName_.pack(space); udrChildTableDescInfo_.pack(space,(Lng32)numChildTableInputs_); childInputExprs_.pack(space,(Lng32)numChildTableInputs_); childTdbs_.pack(space,(Lng32)numChildTableInputs_); @@ -228,7 +236,7 @@ Lng32 ComTdbUdr::unpack(void *base, void *reallocator) if (outputExpr_.unpack(base, reallocator)) return -1; if (scanExpr_.unpack(base, reallocator)) return -1; if (projExpr_.unpack(base, reallocator)) return -1; - + // // The NAVersionedObject array templates use long values to index // into the array, so we cast numParams_ to long here. This is assumed @@ -251,7 +259,8 @@ Lng32 ComTdbUdr::unpack(void *base, void *reallocator) return -1; if (udrSerPlanInfo_.unpack(base)) return -1; - + if (libraryBlobHandle_.unpack(base)) return -1; + if (librarySchName_.unpack(base)) return -1; return ComTdb::unpack(base, reallocator); } @@ -306,7 +315,24 @@ void ComTdbUdr::displayContents(Space *space, ULng32 flag) str_sprintf(buf, "librarySqlName = %s", s); space->allocateAndCopyToAlignedSpace(buf, str_len(buf), sz); } - + if (libraryBlobHandle_) + { + char *s = libraryBlobHandle_; + str_sprintf(buf, "libraryBlobHandle = %s", s); + space->allocateAndCopyToAlignedSpace(buf, str_len(buf), sz); + } + if (librarySchName_) + { + char *s = librarySchName_; + str_sprintf(buf, "librarySchName = %s", s); + space->allocateAndCopyToAlignedSpace(buf, str_len(buf), sz); + } + str_sprintf(buf, "\nlibrayRedefTimestamp = %ld", + libraryRedefTime_); + space->allocateAndCopyToAlignedSpace(buf, str_len(buf), sz); + str_sprintf(buf, "\nlibrayVersion = %d", + libraryVersion_); + space->allocateAndCopyToAlignedSpace(buf, str_len(buf), sz); // Some strings come from the user and there is no limit on the // maximum length. For these strings we will print two lines, the // first a header line and the second the actual string. For http://git-wip-us.apache.org/repos/asf/trafodion/blob/9528f8c0/core/sql/comexe/ComTdbUdr.h ---------------------------------------------------------------------- diff --git a/core/sql/comexe/ComTdbUdr.h b/core/sql/comexe/ComTdbUdr.h index 4bb4a88..d9f9752 100644 --- a/core/sql/comexe/ComTdbUdr.h +++ b/core/sql/comexe/ComTdbUdr.h @@ -68,7 +68,10 @@ public: char *containerName, char *externalPath, char *librarySqlName, - + Int64 libraryRedefTime, + char *libraryBlobHandle, + char *librarySchName, + Int32 libraryVersion, char *javaOptions, char *javaOptionDelimiters, @@ -322,8 +325,12 @@ protected: Int32 javaDebugPort_; // 236-239 Int32 javaDebugTimeout_; // 240-243 + Int64 libraryRedefTime_; // 244-251 + NABasicPtr libraryBlobHandle_; // 252-259 + NABasicPtr librarySchName_; // 260-267 + Int32 libraryVersion_; // 268-271 // Make sure class size is a multiple of 8 - char fillerComTdbUdr2_[28]; // 244-271 + char fillerComTdbUdr2_[24]; // 272-295 }; http://git-wip-us.apache.org/repos/asf/trafodion/blob/9528f8c0/core/sql/common/ComMisc.cpp ---------------------------------------------------------------------- diff --git a/core/sql/common/ComMisc.cpp b/core/sql/common/ComMisc.cpp index 387b309..baa3a00 100644 --- a/core/sql/common/ComMisc.cpp +++ b/core/sql/common/ComMisc.cpp @@ -41,6 +41,7 @@ #include "ComMisc.h" #include "ComDistribution.h" // enumToLiteral, literalToEnum, literalAndEnumStruct #include "CmpSeabaseDDL.h" +#include <sys/stat.h> // define the enum-to-literal function #define ComDefXLateE2L(E2L,eType,array) void E2L (const eType e, NAString &l) \ @@ -357,3 +358,119 @@ NABoolean ComTrafReservedColName( return FALSE; } + + +Int32 ComGenerateUdrCachedLibName(NAString libname,Int64 redeftime, NAString schemaName, NAString userid, NAString &cachedLibName, NAString &cachedLibPath) +{ + NAString libPrefix, libSuffix; + struct stat statbuf; + NAString redefTimeString = Int64ToNAString(redeftime); + size_t lastDot = libname.last('.'); + if (lastDot != NA_NPOS) + { + libSuffix = libname(lastDot,libname.length()-lastDot); + libPrefix = libname(0,lastDot); + } + + //when isolated user support is added + if (userid.length()!=0) + { + + cachedLibPath = getenv("TRAF_HOME") ; + cachedLibPath += "/udr"; + if ( stat(cachedLibPath, &statbuf) != 0) + { + if (mkdir(cachedLibPath,S_IRWXU|S_IRWXG|S_IRWXO)) + { + return -1; + } + + } + cachedLibPath += "/"; + cachedLibPath += getenv("MY_UDR_CACHE_LIBDIR"); + if ( stat(cachedLibPath, &statbuf) != 0) + { + if (mkdir(cachedLibPath,S_IRWXU|S_IRWXG|S_IRWXO)) + { + return -1; + } + + } + cachedLibPath += "/"+ userid ; + if (stat(cachedLibPath, &statbuf) != 0) + { + if (mkdir(cachedLibPath,S_IRUSR|S_IWUSR|S_IXUSR))//Only this user has + //permission to read/write/execute in this directory and below. + { + return -1; + } + + } + cachedLibPath += "/" + schemaName; + if ( stat(cachedLibPath, &statbuf) != 0) + { + if (mkdir(cachedLibPath,S_IRWXU|S_IRWXG|S_IRWXO)) + { + return -1; + } + + } + + + } + else + { + cachedLibPath = getenv("TRAF_HOME") ; + cachedLibPath += "/udr"; + if ( stat(cachedLibPath, &statbuf) != 0) + { + if (mkdir(cachedLibPath,S_IRWXU|S_IRWXG|S_IRWXO)) + { + return -1; + } + + } + cachedLibPath += "/"; + cachedLibPath += getenv("MY_UDR_CACHE_LIBDIR"); + if ( stat(cachedLibPath, &statbuf) != 0) + { + if (mkdir(cachedLibPath,S_IRWXU|S_IRWXG|S_IRWXO)) + { + return -1; + } + + } + cachedLibPath += "/"+ NAString("DB__ROOT") ; + if (stat(cachedLibPath, &statbuf) != 0) + { + if (mkdir(cachedLibPath,S_IRWXU|S_IRWXG|S_IRWXO)) // these permissions + //need to change when we have isolated user support so only DB_ROOT + //can access this directory. Right now we allow all to access this directory + { + return -1; + } + + } + cachedLibPath += "/" + schemaName; + if ( stat(cachedLibPath, &statbuf) != 0) + { + if (mkdir(cachedLibPath,S_IRWXU|S_IRWXG|S_IRWXO)) + { + return -1; + } + + } + + } + + + + cachedLibName += libPrefix + "_" ; + cachedLibName += redefTimeString; + cachedLibName += libSuffix ; + + return 0; + +} + + http://git-wip-us.apache.org/repos/asf/trafodion/blob/9528f8c0/core/sql/common/ComMisc.h ---------------------------------------------------------------------- diff --git a/core/sql/common/ComMisc.h b/core/sql/common/ComMisc.h index 1581489..7b1cebb 100644 --- a/core/sql/common/ComMisc.h +++ b/core/sql/common/ComMisc.h @@ -112,4 +112,9 @@ NAString ComConvertTrafHiveNameToNativeHiveName( // with underscore(_) NABoolean ComTrafReservedColName(const NAString &colName); +// Converts a library name like myfile.jar or myfile.so to this format +// $TRAF_HOME/$MY_UDR_CACHE_LIBDIR/<user>|public/myfile_<redeftime>.jar|so +Int32 ComGenerateUdrCachedLibName(NAString libname,Int64 redeftime,NAString schemaName, NAString user, NAString&cachedLibName, NAString &cachedPathName ); + + #endif // COMMISC_H http://git-wip-us.apache.org/repos/asf/trafodion/blob/9528f8c0/core/sql/executor/ExExeUtil.h ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExExeUtil.h b/core/sql/executor/ExExeUtil.h index 435dfb5..49acbad 100644 --- a/core/sql/executor/ExExeUtil.h +++ b/core/sql/executor/ExExeUtil.h @@ -4183,6 +4183,7 @@ public: protected: }; +short ExExeUtilLobExtractLibrary(ExeCliInterface *cliInterface,char *libHandle, char *cachedLibName,ComDiagsArea *toDiags); #endif http://git-wip-us.apache.org/repos/asf/trafodion/blob/9528f8c0/core/sql/executor/ExExeUtilLoad.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExExeUtilLoad.cpp b/core/sql/executor/ExExeUtilLoad.cpp index 7c051c4..ff4564b 100644 --- a/core/sql/executor/ExExeUtilLoad.cpp +++ b/core/sql/executor/ExExeUtilLoad.cpp @@ -3288,6 +3288,45 @@ short ExExeUtilLobExtractTcb::work() return 0; } + +short ExExeUtilLobExtractLibrary(ExeCliInterface *cliInterface,char *libHandle, char *cachedLibName,ComDiagsArea *toDiags) +{ + char buf[1000]; + Int32 cliRC =0; + str_sprintf(buf, "extract lobtofile(LOB '%s','%s');",libHandle,cachedLibName); + + + cliRC = cliInterface->fetchRowsPrologue(buf, TRUE/*no exec*/); + if (cliRC < 0) + { + cliInterface->retrieveSQLDiagnostics(toDiags); + return cliRC; + } + + cliRC = cliInterface->clearExecFetchClose(NULL, 0); + if (cliRC < 0) + { + cliInterface->retrieveSQLDiagnostics(toDiags); + //Ignore error if the target file exists. This could be because the cached + // file already got created by another process at the same time. So we can ignore + // the error and use the already cached file. + ComCondition *cond = NULL; + Int32 entryNumber; + cond = toDiags->findCondition(-EXE_ERROR_FROM_LOB_INTERFACE, &entryNumber); + if (cond) + { + if (cond->getOptionalInteger(0) == LOB_TARGET_FILE_EXISTS_ERROR) + { + toDiags->deleteError(entryNumber); + return 0; + } + } + + return cliRC; + } + return cliRC; +} + ExExeUtilFileExtractTcb::ExExeUtilFileExtractTcb ( const ComTdbExeUtilLobExtract & exe_util_tdb, http://git-wip-us.apache.org/repos/asf/trafodion/blob/9528f8c0/core/sql/executor/ExUdr.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExUdr.cpp b/core/sql/executor/ExUdr.cpp index 230aa42..fc2686a 100644 --- a/core/sql/executor/ExUdr.cpp +++ b/core/sql/executor/ExUdr.cpp @@ -62,7 +62,8 @@ #include "Statement.h" #include "ExRsInfo.h" #include "Descriptor.h" - +#include "ExExeUtil.h" +#include <sys/stat.h> #define TF_STRING(x) ((x) ? ("TRUE") : ("FALSE")) #define YN_STRING(x) ((x) ? ("YES") : ("NO")) @@ -631,7 +632,10 @@ Int32 ExUdrTcb::fixup() UdrDebug1("[BEGIN TCB FIXUP] %p", this); setUdrTcbState(FIXUP); - + // CliGlobals *cliGlobals = getGlobals()->castToExExeStmtGlobals()-> + // castToExMasterStmtGlobals()->getCliGlobals(); + CliGlobals *cliGlobals = GetCliGlobals(); + ExeCliInterface cliInterface(getHeap(), 0, cliGlobals->currContext()); // // Non-zero return value indicates an error // @@ -792,9 +796,57 @@ Int32 ExUdrTcb::fixup() this, udrServer_->getUdrControlConnection()); } } - + + NAString cachedLibName, cachedLibPath; + if ((myTdb().getLibraryRedefTime() != -1) && (myTdb().getLibraryRedefTime() != 0)) + { + // Cache library locally. + NAString dummyUser; + NAString libOrJarName; + + if (myTdb().getLanguage() == COM_LANGUAGE_JAVA) + libOrJarName = myTdb().getPathName(); + else + libOrJarName = myTdb().getContainerName(); + if(ComGenerateUdrCachedLibName(libOrJarName.data(), + myTdb().getLibraryRedefTime(), + myTdb().getLibrarySchName(), + dummyUser, + cachedLibName, cachedLibPath)) + { + NAString cachedFullName = cachedLibPath+"/"+cachedLibName; + *getOrCreateStmtDiags() << DgSqlCode(-4316) + << DgString0(( char *)cachedFullName.data());; + return FIXUP_ERROR; + } + NAString cachedFullName = cachedLibPath+"/"+cachedLibName; + //If the local copy already exists, don't bother extracting. + struct stat statbuf; + if (stat(cachedFullName, &statbuf) != 0) + { + ComDiagsArea *returnedDiags = ComDiagsArea::allocate(getHeap()); + if (ExExeUtilLobExtractLibrary(&cliInterface, + (char *)myTdb().getLibraryBlobHandle(), + ( char *)cachedFullName.data(),returnedDiags)) + { + *returnedDiags << DgSqlCode(-4316) + << DgString0(( char *)cachedFullName.data()); + getOrCreateStmtDiags()->mergeAfter(*returnedDiags); + returnedDiags->decrRefCount(); + returnedDiags = NULL; + return FIXUP_ERROR; + } + + returnedDiags->decrRefCount(); + returnedDiags = NULL; + } + + } + if (sendControlMessage(isResultSet ? UDR_MSG_RS_LOAD : UDR_MSG_LOAD, - TRUE)) + TRUE, + cachedLibName.length()? (char *)cachedLibName.data():NULL, + cachedLibPath.length()? (char *)cachedLibPath.data():NULL)) { if (verifyUdrServerProcessId()) { @@ -1816,7 +1868,7 @@ void ExUdrTcb::releaseServerResources() // - RS CLOSE is sent by the RS operator when down queue entries are // cancelled. NABoolean ExUdrTcb::sendControlMessage(UdrIpcObjectType t, - NABoolean callbackRequired) + NABoolean callbackRequired, char *cachedLibName, char *cachedLibPath) { // Determine the transid we should send in the message. We // use the statement transid if all the following are true @@ -1837,7 +1889,7 @@ NABoolean ExUdrTcb::sendControlMessage(UdrIpcObjectType t, { transIdToSend = stmtTransId; } - + #ifdef UDR_DEBUG { char stmtTx[256]; @@ -1848,7 +1900,8 @@ NABoolean ExUdrTcb::sendControlMessage(UdrIpcObjectType t, UdrDebug2(" stmt tx %s, msg tx %s", stmtTx, msgTx); } #endif - + + ex_assert(outstandingControlStream_ == NULL, "Sending a UDR control message while one is already outstanding"); @@ -1882,17 +1935,38 @@ NABoolean ExUdrTcb::sendControlMessage(UdrIpcObjectType t, if (numInstances == 0) numInstances = 1; // I compute, therefore I am. + // Steps to create a LOAD message: // - First create a UdrLoadMsg instance on the IPC heap and initialize // it with a UDR handle and UDR metadata // - Next put metadata for each formal parameter into the object + + //In the case of java, the path name contains the fully qualified jar file location + NAString pathName, containerName; + if (udrTdb.getLanguage() == COM_LANGUAGE_JAVA) + { + pathName = cachedLibPath? cachedLibPath :udrTdb.getPathName(); + if (cachedLibName) + { + pathName+= "/"; + pathName+=cachedLibName; + } + containerName = udrTdb.getContainerName(); + } + else + //In the case of C/C++ the DLL name and the path names need to be sent + //separately so langman can process them properly + { + pathName = cachedLibPath? cachedLibPath :udrTdb.getPathName(); + containerName = cachedLibName?cachedLibName: udrTdb.getContainerName(); + } UdrLoadMsg *loadMsg = new (h) UdrLoadMsg( h, udrTdb.getSqlName(), udrTdb.getRoutineName(), udrTdb.getSignature(), - udrTdb.getContainerName(), - udrTdb.getPathName(), + containerName, + pathName, udrTdb.getLibrarySqlName(), udrTdb.getTransactionAttrs(), udrTdb.getSqlAccessMode(), http://git-wip-us.apache.org/repos/asf/trafodion/blob/9528f8c0/core/sql/executor/ExUdr.h ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExUdr.h b/core/sql/executor/ExUdr.h index 3ba2838..ba6822b 100644 --- a/core/sql/executor/ExUdr.h +++ b/core/sql/executor/ExUdr.h @@ -122,6 +122,18 @@ public: { return externalPath_; } + inline const Int64 getLibraryRedefTime() const + { + return libraryRedefTime_; + } + inline const char *getLibraryBlobHandle() const + { + return libraryBlobHandle_; + } + inline const char *getLibrarySchName() const + { + return librarySchName_; + } inline const char *getLibrarySqlName() const { return librarySqlName_; @@ -512,7 +524,7 @@ protected: // Helper function to send control messages // NABoolean sendControlMessage(UdrIpcObjectType t, - NABoolean callbackRequired); + NABoolean callbackRequired, char *cachedLibName=NULL, char *cachedLibPath = NULL); // --------------------------------------------------------------------- // Helper functions called by the work method. See comments in the http://git-wip-us.apache.org/repos/asf/trafodion/blob/9528f8c0/core/sql/generator/GenUdr.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenUdr.cpp b/core/sql/generator/GenUdr.cpp index 96767b3..a31af77 100644 --- a/core/sql/generator/GenUdr.cpp +++ b/core/sql/generator/GenUdr.cpp @@ -876,6 +876,11 @@ static short udr_codegen(Generator *generator, char *container = NULL; char *path = NULL; char *librarySqlName = NULL; + Int64 libraryRedefTime = NULL; + char * libraryBlobHandle; + char *librarySchName = NULL; + Int32 libraryVersion = 0; + Int64 libraryObjUID = 0; char *runtimeOptions = NULL; char *runtimeOptionDelimiters = NULL; ComRoutineType rtype = COM_UNKNOWN_ROUTINE_TYPE; @@ -941,7 +946,11 @@ static short udr_codegen(Generator *generator, const ComObjectName &libName = metadata->getLibrarySqlName(); librarySqlName = AllocStringInSpace(*space, libName.getExternalName()); - + libraryRedefTime = metadata->getLibRedefTime(); + libraryBlobHandle = AllocStringInSpace (*space,metadata->getLibBlobHandle()); + librarySchName = AllocStringInSpace(*space,metadata->getLibSchName()); + libraryVersion = metadata->getLibVersion(); + libraryObjUID = metadata->getLibObjUID(); rtype = metadata->getRoutineType(); sqlmode = metadata->getSqlAccess(); @@ -1055,7 +1064,10 @@ static short udr_codegen(Generator *generator, container, path, librarySqlName, - + libraryRedefTime, + libraryBlobHandle, + librarySchName, + libraryVersion, runtimeOptions, runtimeOptionDelimiters, @@ -1567,7 +1579,10 @@ IsolatedScalarUDF::codeGen(Generator *generator) NULL); // TMUDF only if (effectiveMetadata.getRoutineID() > 0) - generator->objectUids().insert(effectiveMetadata.getRoutineID()); + { + generator->objectUids().insert(effectiveMetadata.getRoutineID()); + generator->objectUids().insert(metadata.getLibObjUID()); + } return result; } @@ -1664,7 +1679,10 @@ short CallSP::codeGen(Generator *generator) NULL); // TMUDF only if (metadata.getRoutineID() > 0) - generator->objectUids().insert(metadata.getRoutineID()); + { + generator->objectUids().insert(metadata.getRoutineID()); + generator->objectUids().insert(metadata.getLibObjUID()); + } return result; @@ -1899,7 +1917,10 @@ PhysicalTableMappingUDF::codeGen(Generator *generator) } if (metadata.getRoutineID() > 0) - generator->objectUids().insert(metadata.getRoutineID()); + { + generator->objectUids().insert(metadata.getRoutineID()); + generator->objectUids().insert(metadata.getLibObjUID()); + } return result; } http://git-wip-us.apache.org/repos/asf/trafodion/blob/9528f8c0/core/sql/generator/Generator.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/Generator.cpp b/core/sql/generator/Generator.cpp index d9e736a..5e43266 100644 --- a/core/sql/generator/Generator.cpp +++ b/core/sql/generator/Generator.cpp @@ -2352,6 +2352,14 @@ TrafDesc *Generator::createVirtualRoutineDesc( strcpy(routine_desc->routineDesc()->signature, routineInfo->signature); routine_desc->routineDesc()->librarySqlName = new GENHEAP(space) char[strlen(routineInfo->library_sqlname)+1]; strcpy(routine_desc->routineDesc()->librarySqlName, routineInfo->library_sqlname); + routine_desc->routineDesc()->libRedefTime = routineInfo->lib_redef_time; + routine_desc->routineDesc()->libBlobHandle = routineInfo->lib_blob_handle; + + routine_desc->routineDesc()->libVersion = routineInfo->library_version; + routine_desc->routineDesc()->libObjUID = routineInfo->lib_obj_uid; + //routine_desc->routineDesc()->libSchName = new GENHEAP(space) char[strlen(routineInfo->lib_sch_name)+1]; + //strcpy(routine_desc->routineDesc()->libSchName ,routineInfo->lib_sch_name); + routine_desc->routineDesc()->libSchName = routineInfo->lib_sch_name; routine_desc->routineDesc()->language = CmGetComRoutineLanguageAsRoutineLanguage(routineInfo->language_type); routine_desc->routineDesc()->UDRType = http://git-wip-us.apache.org/repos/asf/trafodion/blob/9528f8c0/core/sql/optimizer/NARoutine.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/NARoutine.cpp b/core/sql/optimizer/NARoutine.cpp index 905e374..bfd831b 100644 --- a/core/sql/optimizer/NARoutine.cpp +++ b/core/sql/optimizer/NARoutine.cpp @@ -114,6 +114,11 @@ NARoutine::NARoutine (CollHeap *heap) , dataSource_ ("", heap) , fileSuffix_ ("", heap) , schemaVersionOfRoutine_ (COM_VERS_UNKNOWN) + , libRedefTime_(0) + , libBlobHandle_("",heap) + , libSchName_("",heap) + , libVersion_(1) + , libObjUID_(0) , objectOwner_ (0) , schemaOwner_ (0) , privInfo_ (NULL) @@ -143,6 +148,11 @@ NARoutine::NARoutine ( const QualifiedName &name , externalPath_ ("", heap) , externalName_ ("", heap) , librarySqlName_ (NULL) + , libRedefTime_(-1) + , libBlobHandle_("",heap) + , libSchName_("",heap) + , libVersion_(1) + , libObjUID_(0) , signature_ ("", heap) , paramStyle_ (COM_STYLE_SQLROW) , paramStyleVersion_ (COM_ROUTINE_PARAM_STYLE_VERSION_1) @@ -275,6 +285,11 @@ NARoutine::NARoutine (const NARoutine &old, CollHeap *h) , externalName_ (old.externalName_, h) , signature_ (old.signature_, h) , librarySqlName_ (old.librarySqlName_, h) + , libRedefTime_ (old.libRedefTime_) + , libBlobHandle_ (old.libBlobHandle_,h) + , libSchName_ (old.libSchName_,h) + , libVersion_ (old.libVersion_) + , libObjUID_ (old.libObjUID_) , paramStyle_ (old.paramStyle_) , paramStyleVersion_ (old.paramStyleVersion_) , isDeterministic_ (old.isDeterministic_) @@ -357,6 +372,9 @@ NARoutine::NARoutine(const QualifiedName &name, , externalPath_ (routine_desc->routineDesc()->libraryFileName, heap) , externalName_ ("", heap) , librarySqlName_ (routine_desc->routineDesc()->librarySqlName, COM_UNKNOWN_NAME, FALSE, heap) //TODO + , libRedefTime_ (routine_desc->routineDesc()->libRedefTime) + , libVersion_ (routine_desc->routineDesc()->libVersion) + , libObjUID_ (routine_desc->routineDesc()->libObjUID) , signature_ (routine_desc->routineDesc()->signature, heap) , paramStyle_ (routine_desc->routineDesc()->paramStyle) , paramStyleVersion_ (COM_ROUTINE_PARAM_STYLE_VERSION_1) @@ -393,6 +411,14 @@ NARoutine::NARoutine(const QualifiedName &name, char parallelism[5]; CmGetComRoutineParallelismAsLit(routine_desc->routineDesc()->parallelism, parallelism); comRoutineParallelism_ = ((char *)parallelism); + if (routine_desc->routineDesc()->libBlobHandle) + libBlobHandle_ = NAString(routine_desc->routineDesc()->libBlobHandle,heap); + else + libBlobHandle_ = NAString(); + if (routine_desc->routineDesc()->libSchName) + libSchName_ = NAString(routine_desc->routineDesc()->libSchName,heap); + else + libSchName_ = NAString(); if (paramStyle_ == COM_STYLE_JAVA_CALL) { http://git-wip-us.apache.org/repos/asf/trafodion/blob/9528f8c0/core/sql/optimizer/NARoutine.h ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/NARoutine.h b/core/sql/optimizer/NARoutine.h index 65aa473..9b536d4 100644 --- a/core/sql/optimizer/NARoutine.h +++ b/core/sql/optimizer/NARoutine.h @@ -176,11 +176,20 @@ public: inline PrivMgrUserPrivs * getPrivInfo() const { return privInfo_; } inline Int32 getObjectOwner() const { return objectOwner_; } inline Int32 getSchemaOwner() const { return schemaOwner_; } - + inline Int64 getLibRedefTime() const {return libRedefTime_;} + inline const NAString &getLibBlobHandle() const {return libBlobHandle_;} + inline const NAString &getLibSchName() const {return libSchName_;} + inline Int32 getLibVersion() const {return libVersion_;} + inline Int64 getLibObjUID() const {return libObjUID_;} inline void setudfFanOut (Int32 fanOut) { udfFanOut_ = fanOut; } inline void setExternalPath (ComString path) { externalPath_ = path; } inline void setFile (ComString file) { externalFile_ = file; } inline void setExternalName (ComString fname) { externalName_ = fname; } + inline void setLibRedefTime (Int64 rtime) { libRedefTime_ = rtime;} + inline void setLibBlobHandle(NAString lobHandle) {libBlobHandle_ = lobHandle;} + inline void setLibVersion(Int32 version) { libVersion_ = version;} + inline void setLibObjUID(Int64 libobjuid) { libObjUID_ = libobjuid;} + inline void setLibSchName(NAString schName) {libSchName_ = schName;} inline void setLibrarySqlName (ComObjectName lib) { librarySqlName_ = lib; } inline void setLanguage (ComRoutineLanguage lang) { language_ = lang; } inline void setRoutineType (ComRoutineType typ) { UDRType_ = typ; } @@ -247,6 +256,11 @@ private: ComString externalPath_; // URL ComString externalFile_; ComString externalName_; // Java method name + Int64 libRedefTime_; + NAString libBlobHandle_; + NAString libSchName_; + Int32 libVersion_; + Int64 libObjUID_; ComString signature_; ComObjectName librarySqlName_; // ANSI name of JAR/DLL ComRoutineParamStyle paramStyle_; http://git-wip-us.apache.org/repos/asf/trafodion/blob/9528f8c0/core/sql/optimizer/UdfDllInteraction.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/UdfDllInteraction.cpp b/core/sql/optimizer/UdfDllInteraction.cpp index 79df4ae..951833d 100644 --- a/core/sql/optimizer/UdfDllInteraction.cpp +++ b/core/sql/optimizer/UdfDllInteraction.cpp @@ -46,8 +46,9 @@ #include "exp_attrs.h" #include "LmError.h" #include "ComUser.h" +#include "sys/stat.h" - +short ExExeUtilLobExtractLibrary(ExeCliInterface *cliInterface,char *libHandle, char *cachedLibName,ComDiagsArea *toDiags); // ----------------------------------------------------------------------- // methods for class TMUDFDllInteraction // ----------------------------------------------------------------------- @@ -136,6 +137,64 @@ NABoolean TMUDFDllInteraction::describeParamsAndMaxOutputs( bindWA->setErrStatus(); return FALSE; } + NAString externalPath, container; + + // If the library is old style (no blob) and it's not a predfined udf with no entry in metadata + // i.e redeftime of library is not -1 + if( routine->getLibRedefTime() !=-1) + { + // Cache library locally. + NAString dummyUser; + NAString libOrJarName; + NAString cachedLibName,cachedLibPath; + if (routine->getLanguage() == COM_LANGUAGE_JAVA) + libOrJarName = routine->getExternalPath(); + else + libOrJarName = routine->getContainerName(); + if(ComGenerateUdrCachedLibName(libOrJarName.data(), + routine->getLibRedefTime(), + routine->getLibSchName(), + dummyUser, + cachedLibName, cachedLibPath)) + { + NAString cachedFullName = cachedLibPath+"/"+cachedLibName; + *CmpCommon::diags() << DgSqlCode(-4316) + << DgString0(( char *)cachedFullName.data()); + bindWA->setErrStatus(); + return FALSE; + } + + NAString cachedFullName = cachedLibPath+"/"+cachedLibName; + //If the local copy already exists, don't bother extracting. + struct stat statbuf; + if (stat(cachedFullName, &statbuf) != 0) + { + //ComDiagsArea *returnedDiags = ComDiagsArea::allocate(CmpCommon::statementHeap()); + if (ExExeUtilLobExtractLibrary(&cliInterface_,(char *)routine->getLibBlobHandle().data(), + ( char *)cachedFullName.data(),CmpCommon::diags())) + { + *CmpCommon::diags() << DgSqlCode(-4316) + << DgString0(( char *)cachedFullName.data()); + bindWA->setErrStatus(); + return FALSE; + } + } + if (routine->getLanguage() == COM_LANGUAGE_JAVA) + { + externalPath = cachedFullName; + container = routine->getContainerName(); + } + else + { + externalPath = cachedLibPath; + container = cachedLibName; + } + } + else + { + externalPath = routine->getExternalPath(); + container = routine->getContainerName(); + } Int32 cliRC = cliInterface_.getRoutine( serializedUDRInvocationInfo, @@ -147,8 +206,8 @@ NABoolean TMUDFDllInteraction::describeParamsAndMaxOutputs( routine->getMethodName(), // for C/C++ the container that gets loaded is the library file // name, for Java it's the class name - routine->getContainerName(), - routine->getExternalPath(), + container, + externalPath, routine->getLibrarySqlName().getExternalName(), &routineHandle, CmpCommon::diags());
