Support for external lobs. Data files are stored externally in HDFS and only filehandles and file locations are stored in internal Trafodion tables.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/5a3f2024 Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/5a3f2024 Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/5a3f2024 Branch: refs/heads/master Commit: 5a3f20249cabb16b3afe64ab0d21cdf2decafeea Parents: 17c0530 Author: Sandhya Sundaresan <[email protected]> Authored: Fri May 27 04:41:13 2016 +0000 Committer: Sandhya Sundaresan <[email protected]> Committed: Fri May 27 04:41:13 2016 +0000 ---------------------------------------------------------------------- core/sql/bin/SqlciErrors.txt | 1 + core/sql/cli/Cli.cpp | 49 +++- core/sql/comexe/ComTdbExeUtil.cpp | 25 ++ core/sql/comexe/ComTdbExeUtil.h | 34 +-- core/sql/common/CharType.cpp | 4 + core/sql/common/CharType.h | 1 + core/sql/executor/ExExeUtil.h | 6 +- core/sql/executor/ExExeUtilCli.cpp | 5 +- core/sql/executor/ExExeUtilGet.cpp | 153 ++++++++----- core/sql/executor/ExHdfsScan.cpp | 1 + core/sql/exp/ExpLOB.cpp | 35 +-- core/sql/exp/ExpLOB.h | 14 +- core/sql/exp/ExpLOBaccess.cpp | 133 ++++++++--- core/sql/exp/ExpLOBenums.h | 3 +- core/sql/exp/ExpLOBinterface.cpp | 8 +- core/sql/generator/GenPreCode.cpp | 35 ++- core/sql/generator/GenRelExeUtil.cpp | 15 +- core/sql/optimizer/BindRelExpr.cpp | 2 +- core/sql/optimizer/ItemFunc.h | 5 + core/sql/optimizer/NAColumn.h | 2 +- core/sql/parser/ElemDDLCol.cpp | 2 +- core/sql/parser/sqlparser.y | 33 ++- core/sql/regress/executor/EXPECTED130 | 293 +++++++++++++++++++++--- core/sql/regress/executor/TEST130 | 50 +++- core/sql/regress/executor/TEST130_argfile2 | 2 +- core/sql/sqlcomp/CmpSeabaseDDLtable.cpp | 9 +- 26 files changed, 717 insertions(+), 203 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/5a3f2024/core/sql/bin/SqlciErrors.txt ---------------------------------------------------------------------- diff --git a/core/sql/bin/SqlciErrors.txt b/core/sql/bin/SqlciErrors.txt index 198a5d9..6fbdae8 100644 --- a/core/sql/bin/SqlciErrors.txt +++ b/core/sql/bin/SqlciErrors.txt @@ -409,6 +409,7 @@ 1429 ZZZZZ 99999 BEGINNER MINOR DBADMIN Inserts into _ROW_ format external hbase tables can only use the VALUES clause and must use the column_create function to create values. 1430 3F000 99999 BEGINNER MAJOR DBADMIN A schema name that starts and ends with an "_"(underscore) is reserved for internal usage. It cannot be used to create a user schema. 1431 ZZZZZ 99999 BEGINNER MINOR DBADMIN Object $0~String0 exists in HBase. This could be due to a concurrent transactional ddl operation in progress on this table. +1432 ZZZZZ 99999 BEGINNER MINOR DBADMIN Input LOB type $0~Int0 does not match column's storage type : $1~Int1 Column name : $0~String0 . 1500 ZZZZZ 99999 ADVANCED CRTCL DIALOUT The CATSYS - CAT_REFERENCES system schema relationship for catalog $0~CatalogName might be corrupt. 1501 ZZZZZ 99999 UUUUUUUU UUUUU UUUUUUU --- unused as of 5/7/12 --- 1502 ZZZZZ 99999 ADVANCED CRTCL DIALOUT The OBJECTS - REPLICAS definition schema relationship for $0~String0 $1~TableName might be corrupt. http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/5a3f2024/core/sql/cli/Cli.cpp ---------------------------------------------------------------------- diff --git a/core/sql/cli/Cli.cpp b/core/sql/cli/Cli.cpp index 09f03c5..ecfe072 100644 --- a/core/sql/cli/Cli.cpp +++ b/core/sql/cli/Cli.cpp @@ -9677,6 +9677,7 @@ Lng32 SQLCLI_LOBcliInterface // insert into lob descriptor chunks table if (blackBox && (blackBoxLen && (*blackBoxLen > 0))) { + //blackBox points to external file name str_sprintf(query, "insert into table(ghost table %s) values (%Ld, %Ld, 1, %Ld, %Ld, '%s')", lobDescChunksName, descPartnKey, descSyskey, (dataLen ? *dataLen : 0), @@ -9731,6 +9732,13 @@ Lng32 SQLCLI_LOBcliInterface case LOB_CLI_INSERT_APPEND: { + if (lobType == Lob_External_HDFS_File) + { + // Not allowed. For external Lobs there shoudl be only one + // chunk associated with the contents of the external file. + cliRC = -LOB_DESC_APPEND_ERROR; + goto error_return; + } str_sprintf(query, "update table(ghost table %s) set numChunks = numChunks + 1, lobLen = lobLen + %Ld where descPartnKey = %Ld and syskey = %Ld", lobDescHandleName, (dataLen ? *dataLen : 0), @@ -9860,6 +9868,13 @@ Lng32 SQLCLI_LOBcliInterface goto error_return; } + if ((lobType != Lob_External_HDFS_File) && blackBox) + { + //Error . Cannot update an LOB column that has a non-external LOB + // with an external LOB. + cliRC = -LOB_DESC_UPDATE_ERROR; + goto error_return; + } // insert the new chunk into lob descriptor chunks table if (blackBox && (blackBoxLen && (*blackBoxLen > 0))) { @@ -9973,7 +9988,7 @@ Lng32 SQLCLI_LOBcliInterface goto error_return; } - cliRC = cliInterface->fetch(); + cliRC = cliInterface->fetch(); if (cliRC < 0) { cliInterface->retrieveSQLDiagnostics(myDiags); @@ -9997,6 +10012,13 @@ Lng32 SQLCLI_LOBcliInterface cliInterface->fetchRowsEpilogue(0); goto error_return; } + else if ((numChunks > 1 ) && (lobType == Lob_External_HDFS_File)) + { + // Should not happen. For external Lobs there should be only one + // chunk associated with the contents of the external file. + cliRC = -LOB_DATA_READ_ERROR; + goto error_return; + } else { cliRC = cliInterface->fetchRowsEpilogue(0); @@ -10008,9 +10030,11 @@ Lng32 SQLCLI_LOBcliInterface } // This lob has only one chunk. Read and return the single descriptor. - str_sprintf(query, "select c.chunkLen, c.dataOffset from table(ghost table %s) h, table(ghost table %s) c where h.descPartnKey = c.descPartnKey and h.syskey = c.descSyskey and h.descPartnKey = %Ld and h.syskey = %Ld and c.chunkNum = h.numChunks for read committed access", + + str_sprintf(query, "select c.chunkLen, c.dataOffset ,c.stringParam from table(ghost table %s) h, table(ghost table %s) c where h.descPartnKey = c.descPartnKey and h.syskey = c.descSyskey and h.descPartnKey = %Ld and h.syskey = %Ld and c.chunkNum = h.numChunks for read committed access", lobDescHandleName, lobDescChunksName, descPartnKey, inDescSyskey); + lobDebugInfo(query,0,__LINE__,lobTrace); // set parserflags to allow ghost table currContext.setSqlParserFlags(0x1); @@ -10041,15 +10065,20 @@ Lng32 SQLCLI_LOBcliInterface char * ptr; Lng32 len; - cliInterface->getPtrAndLen(1, ptr, len); - + cliInterface->getPtrAndLen(1, ptr, len); if (dataLen) str_cpy_all((char*)dataLen, ptr, len); - cliInterface->getPtrAndLen(2, ptr, len); - + cliInterface->getPtrAndLen(2, ptr, len); if (dataOffset) str_cpy_all((char*)dataOffset, ptr, len); + + cliInterface->getPtrAndLen(3, ptr, len); + if (blackBox) + { + str_cpy_all((char*)blackBox, ptr, len); + *blackBoxLen = len; + } if (inoutDescPartnKey) *inoutDescPartnKey = descPartnKey; @@ -10076,7 +10105,7 @@ Lng32 SQLCLI_LOBcliInterface case LOB_CLI_SELECT_CURSOR: { - str_sprintf(query, "select dataOffset, chunkLen from table(ghost table %s) where descPartnKey = %Ld and descSyskey = %Ld order by chunkNum for read committed access", + str_sprintf(query, "select dataOffset, chunkLen, stringParam from table(ghost table %s) where descPartnKey = %Ld and descSyskey = %Ld order by chunkNum for read committed access", lobDescChunksName, descPartnKey, inDescSyskey); lobDebugInfo(query,0,__LINE__,lobTrace); // set parserflags to allow ghost table @@ -10129,6 +10158,12 @@ Lng32 SQLCLI_LOBcliInterface if (dataLen) str_cpy_all((char*)dataLen, ptr, len); + cliInterface->getPtrAndLen(3, ptr, len); + if (blackBox) + { + str_cpy_all((char*)blackBox, ptr, len); + *blackBoxLen = len; + } cliRC = 0; } else http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/5a3f2024/core/sql/comexe/ComTdbExeUtil.cpp ---------------------------------------------------------------------- diff --git a/core/sql/comexe/ComTdbExeUtil.cpp b/core/sql/comexe/ComTdbExeUtil.cpp index 3442714..1244e1b 100644 --- a/core/sql/comexe/ComTdbExeUtil.cpp +++ b/core/sql/comexe/ComTdbExeUtil.cpp @@ -2329,6 +2329,7 @@ ComTdbExeUtilLobShowddl::ComTdbExeUtilLobShowddl Lng32 numLOBs, char * lobNumArray, char * lobLocArray, + char * lobTypeArray, short maxLocLen, short sdOptions, ex_cri_desc * given_cri_desc, @@ -2352,6 +2353,7 @@ ComTdbExeUtilLobShowddl::ComTdbExeUtilLobShowddl numLOBs_(numLOBs), lobNumArray_(lobNumArray), lobLocArray_(lobLocArray), + lobTypeArray_(lobTypeArray), maxLocLen_(maxLocLen), sdOptions_(sdOptions), schName_(schName), @@ -2371,6 +2373,9 @@ Long ComTdbExeUtilLobShowddl::pack(void * space) if (lobLocArray_) lobLocArray_.pack(space); + if (lobTypeArray_) + lobTypeArray_.pack(space); + return ComTdbExeUtil::pack(space); } @@ -2385,6 +2390,9 @@ Lng32 ComTdbExeUtilLobShowddl::unpack(void * base, void * reallocator) if(lobLocArray_.unpack(base)) return -1; + if(lobTypeArray_.unpack(base)) + return -1; + return ComTdbExeUtil::unpack(base, reallocator); } @@ -2417,6 +2425,15 @@ short ComTdbExeUtilLobShowddl::getLOBnum(short i) return lobNum; } +NABoolean ComTdbExeUtilLobShowddl::getIsExternalLobCol(short i) +{ + if ((i > numLOBs_) || (i <= 0)) + return -1; + + NABoolean isExternal = (*((Int32*)&getLOBtypeArray()[4*(i-1)]) == Lob_External_HDFS_File); + + return isExternal; +} char * ComTdbExeUtilLobShowddl::getLOBloc(short i) { if ((i > numLOBs_) || (i <= 0)) @@ -2812,6 +2829,7 @@ ComTdbExeUtilLobInfo::ComTdbExeUtilLobInfo char *lobColArray, char * lobNumArray, char * lobLocArray, + char *lobTypeArray, Int32 hdfsPort, char *hdfsServer, NABoolean tableFormat, @@ -2839,6 +2857,7 @@ ComTdbExeUtilLobInfo::ComTdbExeUtilLobInfo lobColArray_(lobColArray), lobNumArray_(lobNumArray), lobLocArray_(lobLocArray), + lobTypeArray_(lobTypeArray), hdfsPort_(0), hdfsServer_(hdfsServer), tableFormat_(tableFormat) @@ -2857,6 +2876,9 @@ Long ComTdbExeUtilLobInfo::pack(void * space) if (lobLocArray_) lobLocArray_.pack(space); + if(lobTypeArray_) + lobTypeArray_.pack(space); + if (hdfsServer_) hdfsServer_.pack(space); return ComTdbExeUtil::pack(space); @@ -2873,6 +2895,9 @@ Lng32 ComTdbExeUtilLobInfo::unpack(void * base, void * reallocator) if(lobLocArray_.unpack(base)) return -1; + if(lobTypeArray_.unpack(base)) + return -1; + if(hdfsServer_.unpack(base)) return -1; return ComTdbExeUtil::unpack(base, reallocator); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/5a3f2024/core/sql/comexe/ComTdbExeUtil.h ---------------------------------------------------------------------- diff --git a/core/sql/comexe/ComTdbExeUtil.h b/core/sql/comexe/ComTdbExeUtil.h index ea3ef69..a4cc558 100644 --- a/core/sql/comexe/ComTdbExeUtil.h +++ b/core/sql/comexe/ComTdbExeUtil.h @@ -2815,6 +2815,7 @@ public: Lng32 numLOBs, char * lobNumArray, char * lobLocArray, + char * lobTypeArray, short maxLocLen, short sdOptions, ex_cri_desc * given_cri_desc, @@ -2845,35 +2846,36 @@ public: short getLOBnum(short i); char * getLOBloc(short i); - + NABoolean getIsExternalLobCol(short i); UInt16 numLOBs() { return numLOBs_; } char * getLOBnumArray() { return lobNumArray_; } char * getLOBlocArray() { return lobLocArray_; } - + char * getLOBtypeArray() { return lobTypeArray_;} + char * schName() { return schName_; }; private: - UInt32 flags_; // 00-03 + UInt32 flags_; - UInt16 numLOBs_; // 04-05 + UInt16 numLOBs_; short maxLocLen_; // array of shorts. numLOBs entries. // Each entry is the lobNum. - NABasicPtr lobNumArray_; // 08-15 + NABasicPtr lobNumArray_; // array of string, null terminated. numLOBs entries. // Each entry is the storage location of lob data file. - NABasicPtr lobLocArray_; // 16-23 - - Int64 objectUID_; // 24-31 + NABasicPtr lobLocArray_; + //array of int32. numLOBs entries + NABasicPtr lobTypeArray_; + Int64 objectUID_; - NABasicPtr schName_; // 32-39 - short schNameLen_; // 40-41 + NABasicPtr schName_; + short schNameLen_; - short sdOptions_; // 42-43 - - char fillersComTdbExeUtilLobShowddl_[4]; // 44-47 + short sdOptions_; + char fillersComTdbExeUtilLobShowddl_[4]; }; @@ -3625,6 +3627,7 @@ public: char *getLobColList(){return lobColArray_;} char *getLobNumList() {return lobNumArray_;} char *getLobLocList() {return lobLocArray_;} + char *getLobTypeList(){return lobTypeArray_;} Int64 getObjectUID() { return objectUID_;} Int32 getHdfsPort() { return hdfsPort_;} char *getHdfsServer() {return hdfsServer_;} @@ -3636,6 +3639,7 @@ public: char* lobColArray, char* lobNumArray, char* lobLocArray, + char *lobTypeArray, Int32 hdfsPort, char *hdfsServer, NABoolean tableFormat, @@ -3690,7 +3694,9 @@ private: NABasicPtr lobColArray_; // array of shorts. numLOBs entries. // Each entry is the lobNum. - NABasicPtr lobNumArray_; + NABasicPtr lobNumArray_; + //array of int32. numLOBS entries + NABasicPtr lobTypeArray_; // array of string, null terminated. numLOBs entries. // Each entry is the storage location of lob data file. http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/5a3f2024/core/sql/common/CharType.cpp ---------------------------------------------------------------------- diff --git a/core/sql/common/CharType.cpp b/core/sql/common/CharType.cpp index dc08028..29d57cc 100644 --- a/core/sql/common/CharType.cpp +++ b/core/sql/common/CharType.cpp @@ -1496,6 +1496,10 @@ SQLlob::SQLlob( lobStorage_(ls), lobLength_(lobLength) { + if (externalFormat_) + lobStorage_ = Lob_External_HDFS_File; + else + lobStorage_ = Lob_HDFS_File; } // --------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/5a3f2024/core/sql/common/CharType.h ---------------------------------------------------------------------- diff --git a/core/sql/common/CharType.h b/core/sql/common/CharType.h index 11369b6..eefc953 100644 --- a/core/sql/common/CharType.h +++ b/core/sql/common/CharType.h @@ -749,6 +749,7 @@ public: charSet_ = cs; } LobsStorage getLobStorage() {return lobStorage_;} + NABoolean isExternal() { return externalFormat_;} private: Int64 lobLength_; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/5a3f2024/core/sql/executor/ExExeUtil.h ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExExeUtil.h b/core/sql/executor/ExExeUtil.h index 6ed29c1..c3de823 100755 --- a/core/sql/executor/ExExeUtil.h +++ b/core/sql/executor/ExExeUtil.h @@ -3079,11 +3079,11 @@ public: { return (ExExeUtilLobShowddlTdb &) tdb; }; - + private: short fetchRows(char * query, short &rc); short returnRows(short &rc); - + enum Step { INITIAL_, @@ -3108,7 +3108,7 @@ public: char * lobMDName_; Lng32 currLobNum_; - + char sdOptionStr_[100]; }; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/5a3f2024/core/sql/executor/ExExeUtilCli.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExExeUtilCli.cpp b/core/sql/executor/ExExeUtilCli.cpp index 451d884..53b7db5 100644 --- a/core/sql/executor/ExExeUtilCli.cpp +++ b/core/sql/executor/ExExeUtilCli.cpp @@ -1011,9 +1011,8 @@ Lng32 ExeCliInterface::executeImmediateExec(const char * stmtStr, { char * ptr; Lng32 len; - getPtrAndLen(1, ptr, len); - - str_cpy_all(outputBuf, ptr, len); + getPtrAndLen(1, ptr, len); + str_cpy_all(outputBuf, ptr, len); if (nullTerminate) outputBuf[len] = 0; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/5a3f2024/core/sql/executor/ExExeUtilGet.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExExeUtilGet.cpp b/core/sql/executor/ExExeUtilGet.cpp index a5c9b51..b88bfd8 100644 --- a/core/sql/executor/ExExeUtilGet.cpp +++ b/core/sql/executor/ExExeUtilGet.cpp @@ -4478,6 +4478,12 @@ ExExeUtilLobShowddlTcb::ExExeUtilLobShowddlTcb : ExExeUtilTcb(exe_util_tdb, NULL, glob), step_(INITIAL_) { + strcpy(lobMDNameBuf_,""); + lobMDNameLen_=0; + lobMDName_ = NULL; + + Lng32 currLobNum_ = 0; + strcpy(sdOptionStr_,""); } short ExExeUtilLobShowddlTcb::fetchRows(char * query, short &rc) @@ -4698,12 +4704,18 @@ short ExExeUtilLobShowddlTcb::work() ExpLOBoper::ExpGetLOBname (lobTdb().objectUID_, lobTdb().getLOBnum(currLobNum_), tgtLobNameBuf, 100); - - str_sprintf(query_, "Location: %s", + + if (lobTdb().getIsExternalLobCol(currLobNum_)) + str_sprintf(query_, "<External HDFS location>"); + else + str_sprintf(query_, "Location: %s", lobTdb().getLOBloc(currLobNum_)); moveRowToUpQueue(query_); - str_sprintf(query_, "DataFile: %s", tgtLobName); + if (lobTdb().getIsExternalLobCol(currLobNum_)) + str_sprintf(query_, "<External HDFS file>"); + else + str_sprintf(query_, "DataFile: %s", tgtLobName); moveRowToUpQueue(query_); step_ = FETCH_LOB_DESC_HANDLE_SHOWDDL_; @@ -6222,21 +6234,40 @@ short ExExeUtilLobInfoTcb::collectAndReturnLobInfo(char * tableName,Int32 currLo return rc; //lob location - strcpy(lobLocation, &((getLItdb().getLobLocList())[offset])); + + strcpy(lobLocation, &((getLItdb().getLobLocList())[offset])); removeTrailingBlanks(lobLocation, LOBINFO_MAX_FILE_LEN); +if (getLItdb().getLobTypeList()[(currLobNum-1)*sizeof(Int32)] == Lob_External_HDFS_File) + str_sprintf(buf, " Lob Location : External HDFS Location"); +else str_sprintf(buf, " Lob Location : %s", lobLocation); if (moveRowToUpQueue(buf, strlen(buf), &rc)) return rc; - + + char lobDescChunkFileBuf[LOBINFO_MAX_FILE_LEN*2]; + //Get the descriptor chunks table name + char *lobDescChunksFile = + ExpLOBoper::ExpGetLOBDescChunksName(strlen(schName),schName, + getLItdb().objectUID_, currLobNum, + lobDescChunkFileBuf, LOBINFO_MAX_FILE_LEN*2); + + char *query = new(getGlobals()->getDefaultHeap()) char[4096]; // lobDataFile char tgtLobNameBuf[LOBINFO_MAX_FILE_LEN]; - char *lobDataFile = - ExpLOBoper::ExpGetLOBname - (getLItdb().objectUID_, currLobNum, - tgtLobNameBuf, LOBINFO_MAX_FILE_LEN); + + + char *lobDataFile = + ExpLOBoper::ExpGetLOBname + (getLItdb().objectUID_, currLobNum, + tgtLobNameBuf, LOBINFO_MAX_FILE_LEN); + + removeTrailingBlanks(lobDataFile, LOBINFO_MAX_FILE_LEN); - str_sprintf(buf, " LOB Data File: %s", lobDataFile); + if (getLItdb().getLobTypeList()[(currLobNum-1)*sizeof(Int32)] == Lob_External_HDFS_File) + str_sprintf(buf, " LOB Data File: External HDFS File"); + else + str_sprintf(buf, " LOB Data File: %s", lobDataFile); if (moveRowToUpQueue(buf, strlen(buf), &rc)) return rc; @@ -6245,6 +6276,7 @@ short ExExeUtilLobInfoTcb::collectAndReturnLobInfo(char * tableName,Int32 currLo if (fs == NULL) return LOB_DATA_FILE_OPEN_ERROR; + snprintf(lobDataFilePath, LOBINFO_MAX_FILE_LEN, "%s/%s", lobLocation, lobDataFile); hdfsFile fdData = hdfsOpenFile(fs, lobDataFilePath,O_RDONLY,0,0,0); if (!fdData) @@ -6256,7 +6288,8 @@ short ExExeUtilLobInfoTcb::collectAndReturnLobInfo(char * tableName,Int32 currLo hdfsFileInfo *fInfo = hdfsGetPathInfo(fs, lobDataFilePath); if (fInfo) lobEOD = fInfo->mSize; - + else + lobEOD = 0; str_sprintf(buf, " LOB EOD : %Ld", lobEOD); if (moveRowToUpQueue(buf, strlen(buf), &rc)) @@ -6264,23 +6297,17 @@ short ExExeUtilLobInfoTcb::collectAndReturnLobInfo(char * tableName,Int32 currLo // Sum of all the lobDescChunks for used space - char lobDescChunkFileBuf[LOBINFO_MAX_FILE_LEN*2]; - //Get the descriptor chunks table name - char *lobDescChunksFile = - ExpLOBoper::ExpGetLOBDescChunksName(strlen(schName),schName, - getLItdb().objectUID_, currLobNum, - lobDescChunkFileBuf, LOBINFO_MAX_FILE_LEN*2); - - char *query = new(getGlobals()->getDefaultHeap()) char[4096]; + str_sprintf (query, "select sum(chunklen) from %s ", lobDescChunksFile); - // set parserflags to allow ghost table - currContext->setSqlParserFlags(0x1); - + currContext->setSqlParserFlags(0x1); Int64 outlen = 0;Lng32 len = 0; Int32 cliRC = cliInterface()->executeImmediate(query,(char *)&outlen, &len, FALSE); + if ((len ==0) ||(getLItdb().getLobTypeList()[(currLobNum-1)*sizeof(Int32)] == Lob_External_HDFS_File)) + outlen = 0; NADELETEBASIC(query, getGlobals()->getDefaultHeap()); + currContext->resetSqlParserFlags(0x1); if (cliRC <0 ) { @@ -6296,7 +6323,7 @@ short ExExeUtilLobInfoTcb::work() { short retcode = 0; Lng32 cliRC = 0; - const char *parentQid = NULL; + const char *parentQid = NULL; char buf[1000]; short rc = 0; // if no parent request, return @@ -6410,6 +6437,7 @@ short ExExeUtilLobInfoTcb::work() case HANDLE_ERROR_: { + retcode = handleError(); if (retcode == 1) return WORK_OK; @@ -6511,18 +6539,39 @@ short ExExeUtilLobInfoTableTcb::collectLobInfo(char * tableName,Int32 currLobNum offset = (currLobNum-1)*LOBINFO_MAX_FILE_LEN; str_cpy_all(lobInfo_->columnName, &((getLItdb().getLobColList())[offset]), strlen(&((getLItdb().getLobColList())[offset]))); - - //lob location - char *lobLocation = &((getLItdb().getLobLocList())[offset]); - str_cpy_all(lobInfo_->lobLocation, lobLocation, strlen(lobLocation)); + + char *lobLocation = new(getGlobals()->getDefaultHeap()) char[LOBINFO_MAX_FILE_LEN] ; + + + lobLocation = &((getLItdb().getLobLocList())[offset]); + if (getLItdb().getLobTypeList()[(currLobNum-1)*sizeof(Int32)] == Lob_External_HDFS_File) + str_cpy_all(lobInfo_->lobLocation, "External HDFS Location", strlen("External HDFS Location")); + else + str_cpy_all(lobInfo_->lobLocation, (char *)&lobLocation[0], strlen(lobLocation)); // lobDataFile char tgtLobNameBuf[LOBINFO_MAX_FILE_LEN]; - char *lobDataFile = + char query[4096]; + char lobDescChunkFileBuf[LOBINFO_MAX_FILE_LEN*2]; + + //Get the descriptor chunks table name + char *lobDescChunksFile = + ExpLOBoper::ExpGetLOBDescChunksName(strlen(schName),schName, + getLItdb().objectUID_, currLobNum, + lobDescChunkFileBuf, LOBINFO_MAX_FILE_LEN*2); + char *lobDataFile = ExpLOBoper::ExpGetLOBname (getLItdb().objectUID_, currLobNum, tgtLobNameBuf, LOBINFO_MAX_FILE_LEN); - str_cpy_all(lobInfo_->lobDataFile, lobDataFile,strlen(lobDataFile)); + + if (getLItdb().getLobTypeList()[(currLobNum-1)*sizeof(Int32)] == Lob_External_HDFS_File) + { + str_cpy_all(lobInfo_->lobDataFile, "External HDFS File" ,strlen("External HDFS File")); + } + else + { + str_cpy_all(lobInfo_->lobDataFile, lobDataFile,strlen(lobDataFile)); + } //EOD of LOB data file hdfsFS fs = hdfsConnect(getLItdb().getHdfsServer(),getLItdb().getHdfsPort()); if (fs == NULL) @@ -6530,41 +6579,39 @@ short ExExeUtilLobInfoTableTcb::collectLobInfo(char * tableName,Int32 currLobNum snprintf(lobDataFilePath, LOBINFO_MAX_FILE_LEN, "%s/%s", lobLocation, lobDataFile); hdfsFile fdData = hdfsOpenFile(fs, lobDataFilePath,O_RDONLY,0,0,0); - if (!fdData) + if (!fdData) { hdfsCloseFile(fs,fdData); fdData = NULL; return LOB_DATA_FILE_OPEN_ERROR; } - hdfsFileInfo *fInfo = hdfsGetPathInfo(fs, lobDataFilePath); - if (fInfo) - lobEOD = fInfo->mSize; - lobInfo_->lobDataFileSizeEod=lobEOD; + hdfsFileInfo *fInfo = hdfsGetPathInfo(fs, lobDataFilePath); + if (fInfo) + lobEOD = fInfo->mSize; + else + lobEOD = 0; + lobInfo_->lobDataFileSizeEod=lobEOD; // Sum of all the lobDescChunks for used space - char lobDescChunkFileBuf[LOBINFO_MAX_FILE_LEN*2]; - //Get the descriptor chunks table name - char *lobDescChunksFile = - ExpLOBoper::ExpGetLOBDescChunksName(strlen(schName),schName, - getLItdb().objectUID_, currLobNum, - lobDescChunkFileBuf, LOBINFO_MAX_FILE_LEN*2); - char query[4096]; - str_sprintf (query, "select sum(chunklen) from %s ", lobDescChunksFile); + str_sprintf (query, "select sum(chunklen) from %s ", lobDescChunksFile); - // set parserflags to allow ghost table - currContext->setSqlParserFlags(0x1); + // set parserflags to allow ghost table + currContext->setSqlParserFlags(0x1); + Int64 outlen = 0;Lng32 len = 0; + Int32 cliRC = cliInterface()->executeImmediate(query,(char *)&outlen, &len, FALSE); + if ((len == 0) || (getLItdb().getLobTypeList()[(currLobNum-1)*sizeof(Int32)] == Lob_External_HDFS_File)) + outlen = 0; + lobInfo_->lobDataFileSizeUsed = outlen; + currContext->resetSqlParserFlags(0x1); + - Int64 outlen = 0;Lng32 len = 0; - Int32 cliRC = cliInterface()->executeImmediate(query,(char *)&outlen, &len, FALSE); - lobInfo_->lobDataFileSizeUsed = outlen; - currContext->resetSqlParserFlags(0x1); - if (cliRC <0 ) - { - return cliRC; - } + if (cliRC <0 ) + { + return cliRC; + } - return 0; + return 0; } short ExExeUtilLobInfoTableTcb::work() { http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/5a3f2024/core/sql/executor/ExHdfsScan.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExHdfsScan.cpp b/core/sql/executor/ExHdfsScan.cpp index 1278c3a..3bad231 100644 --- a/core/sql/executor/ExHdfsScan.cpp +++ b/core/sql/executor/ExHdfsScan.cpp @@ -253,6 +253,7 @@ void ExHdfsScanTcb::freeResources() ExpLOBinterfaceCleanup (lobGlob_, getGlobals()->getDefaultHeap()); + lobGlob_= NULL; } NABoolean ExHdfsScanTcb::needStatsEntry() { http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/5a3f2024/core/sql/exp/ExpLOB.cpp ---------------------------------------------------------------------- diff --git a/core/sql/exp/ExpLOB.cpp b/core/sql/exp/ExpLOB.cpp index f1442b6..e4f024c 100644 --- a/core/sql/exp/ExpLOB.cpp +++ b/core/sql/exp/ExpLOB.cpp @@ -390,7 +390,7 @@ Lng32 findNumDigits(Int64 val) // <--4--><--4---><----8----><---8---><--8---><-----2----><--vc---> void ExpLOBoper::genLOBhandle(Int64 uid, Lng32 lobNum, - short lobType, + Int32 lobType, Int64 descKey, Int64 descTS, Lng32 flags, @@ -401,8 +401,7 @@ void ExpLOBoper::genLOBhandle(Int64 uid, { LOBHandle * lobHandle = (LOBHandle*)ptr; lobHandle->flags_ = flags; - lobHandle->lobType_ = (char)lobType; - lobHandle->filler1_ = 0; + lobHandle->lobType_ = lobType; lobHandle->lobNum_ = lobNum; lobHandle->objUID_ = uid; lobHandle->descSyskey_ = descKey; @@ -422,14 +421,12 @@ void ExpLOBoper::genLOBhandle(Int64 uid, } void ExpLOBoper::updLOBhandle(Int64 descSyskey, - Lng32 flags, + Lng32 flags, char * ptr) { LOBHandle * lobHandle = (LOBHandle*)ptr; - lobHandle->flags_ = flags; - // lobHandle->lobLen_ = lobLen; + lobHandle->flags_ = flags; lobHandle->descSyskey_ = descSyskey; - // lobHandle->numChunks_ = numChunks; } // Extracts values from the LOB handle stored at ptr @@ -479,7 +476,7 @@ void ExpLOBoper::createLOBhandleString(Int16 flags, char * schName, char * lobHandleBuf) { - str_sprintf(lobHandleBuf, "LOBH%04d%02d%04d%020Ld%02d%Ld%02d%Ld%03d%s", + str_sprintf(lobHandleBuf, "LOBH%04d%04d%04d%020Ld%02d%Ld%02d%Ld%03d%s", flags, lobType, lobNum, uid, findNumDigits(descKey), descKey, findNumDigits(descTS), descTS, @@ -523,7 +520,7 @@ Lng32 ExpLOBoper::extractFromLOBstring(Int64 &uid, curPos += 4; lobType = (Lng32)str_atoi(&handle[curPos], 2); - curPos += 2; + curPos += 4; lobNum = (Lng32)str_atoi(&handle[curPos], 4); curPos += 4; @@ -751,6 +748,8 @@ ex_expr::exp_return_type ExpLOBiud::insertDesc(char *op_data[], so = Lob_Foreign_Lob; else if (fromBuffer()) so = Lob_Buffer; + else if (fromExternal()) + so = Lob_External; @@ -780,7 +779,7 @@ ex_expr::exp_return_type ExpLOBiud::insertDesc(char *op_data[], char * lobData = NULL; lobData= new(h) char[lobLen]; //send lobData only if it's a lob_file operation - if (so == Lob_File) + if ((so == Lob_File) || (so == Lob_External)) { str_cpy_and_null(lobData,op_data[1],lobLen,'\0',' ',TRUE); @@ -846,7 +845,8 @@ ex_expr::exp_return_type ExpLOBiud::insertDesc(char *op_data[], // extract and update lob handle with the returned values if (outHandleLen_ > 0) { - ExpLOBoper::extractFromLOBhandle(NULL, NULL, NULL, NULL, &descSyskey, + Int32 lobType = 0; + ExpLOBoper::extractFromLOBhandle(NULL, &lobType, NULL, NULL, &descSyskey, NULL, NULL, NULL, outLobHandle_); ExpLOBoper::updLOBhandle(descSyskey, 0, lobHandle); @@ -902,6 +902,11 @@ ex_expr::exp_return_type ExpLOBiud::insertData(Lng32 handleLen, lobLen = getOperand(1)->getLength(); char * lobData = NULL; + if (fromExternal()) + { + //no need to insert any data. All data resides in the external file + return ex_expr::EXPR_OK; + } if(fromFile()) { lobData = new (h) char[lobLen]; @@ -932,6 +937,8 @@ ex_expr::exp_return_type ExpLOBiud::insertData(Lng32 handleLen, so = Lob_Foreign_Lob; else if(fromBuffer()) so = Lob_Buffer; + else if (fromExternal()) + so = Lob_External; Lng32 waitedOp = 0; @@ -1021,9 +1028,6 @@ ex_expr::exp_return_type ExpLOBinsert::eval(char *op_data[], if (err == ex_expr::EXPR_ERROR) return err; - if (fromExternal()) - return err; - #ifndef __EID char * handle = op_data[0]; Lng32 handleLen = getOperand(0)->getLength(); @@ -1246,6 +1250,7 @@ ex_expr::exp_return_type ExpLOBupdate::eval(char *op_data[], &sSchNameLen, sSchName, lobHandle); //op_data[2]); + // get the lob name where data need to be updated char tgtLobNameBuf[100]; char * tgtLobName = ExpGetLOBname(sUid, sLobNum, tgtLobNameBuf, 100); @@ -1285,6 +1290,8 @@ ex_expr::exp_return_type ExpLOBupdate::eval(char *op_data[], so = Lob_Foreign_Lob; else if (fromBuffer()) so= Lob_Buffer; + else if (fromExternal()) + so = Lob_External; Int64 lobMaxSize = 0; if (getLobSize() > 0) { http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/5a3f2024/core/sql/exp/ExpLOB.h ---------------------------------------------------------------------- diff --git a/core/sql/exp/ExpLOB.h b/core/sql/exp/ExpLOB.h index 015fcac..cbbeecd 100644 --- a/core/sql/exp/ExpLOB.h +++ b/core/sql/exp/ExpLOB.h @@ -191,6 +191,7 @@ public: virtual unsigned char getClassVersionID() { return 1; + } virtual void populateImageVersionIDArray() @@ -264,11 +265,11 @@ public: // Generates LOB handle that is stored in the SQL row. // LOB handle max len: 512 bytes - // <flags><LOBnum><objectUid><LOBlen><descKey><descTS><chunkNum><schNameLen><schName> - // <--4--><--4---><----8----><---8--><---8---><--8---><----2---><---2------><--vc---> + // <flags><LOBType><LOBnum><objectUid><LOBlen><descKey><descTS><chunkNum><schNameLen><schName> + // <--4--><--4----><--4---><----8----><---8--><---8---><--8---><----2---><---2------><--vc---> static void genLOBhandle(Int64 uid, Lng32 lobNum, - short lobType, + Int32 lobType, Int64 descKey, Int64 descTS, Lng32 flags, @@ -278,7 +279,7 @@ public: char * ptr); static void updLOBhandle(Int64 descSyskey, - Lng32 flags, + Lng32 flags, char * ptr); static Lng32 genLOBhandleFromHandleString(char * lobHandleString, @@ -344,9 +345,8 @@ public: struct LOBHandle { - short flags_; - char lobType_; - char filler1_; + Int32 flags_; + Int32 lobType_; Lng32 lobNum_; Int64 objUID_; Int64 descSyskey_; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/5a3f2024/core/sql/exp/ExpLOBaccess.cpp ---------------------------------------------------------------------- diff --git a/core/sql/exp/ExpLOBaccess.cpp b/core/sql/exp/ExpLOBaccess.cpp index 929db63..70426e1 100644 --- a/core/sql/exp/ExpLOBaccess.cpp +++ b/core/sql/exp/ExpLOBaccess.cpp @@ -163,6 +163,16 @@ Ex_Lob_Error ExLob::initialize(char *lobFile, Ex_Lob_Mode mode, return LOB_HDFS_CONNECT_ERROR; lobGlobals->setHdfsFs(fs_); } + //check validity of fs_ + else if (hdfsGetUsed(fs_) == -1 ) + { + hdfsDisconnect(fs_); //ignore errors since fs_ may be corrupt. + fs_ = hdfsConnect(hdfsServer_, hdfsPort_); + if (fs_ == NULL) + return LOB_HDFS_CONNECT_ERROR; + lobGlobals->setHdfsFs(fs_); + } + else { fs_ = lobGlobals->getHdfsFs(); @@ -214,7 +224,8 @@ Ex_Lob_Error ExLob::fetchCursor(char *handleIn, Int32 handleLenIn, Int64 &outOff lobCursors_it it = lobCursors_.find(string(handleIn, handleLenIn)); char logBuf[4096]; lobDebugInfo("In ExLob::fetchCursor",0,__LINE__,lobTrace_); - + char *blackBox = new(getLobGlobalHeap()) char[MAX_LOB_FILE_NAME_LEN+6]; + Int32 blackBoxLen = 0; if (it == lobCursors_.end()) { @@ -225,8 +236,8 @@ Ex_Lob_Error ExLob::fetchCursor(char *handleIn, Int32 handleLenIn, Int64 &outOff cliErr = SQL_EXEC_LOBcliInterface(handleIn, handleLenIn, - 0, 0, - (char *)&dummyParam, (Lng32 *)&dummyParam, + blackBox, &blackBoxLen, + (char *)&dummyParam, (Lng32 *)&dummyParam, LOB_CLI_SELECT_FETCH, LOB_CLI_ExecImmed, &offset, &size, &dummyParam, &dummyParam, @@ -262,6 +273,13 @@ Ex_Lob_Error ExLob::fetchCursor(char *handleIn, Int32 handleLenIn, Int64 &outOff } else { + if (blackBox && blackBoxLen >0 ) + { + // we have received the external data file name from the descriptor table + // replace the contents of the lobDataFile with this name + str_cpy_and_null(lobDataFile_, blackBox, blackBoxLen,'\0','0',TRUE); + + } outOffset = offset; outSize = size; } @@ -669,9 +687,10 @@ Ex_Lob_Error ExLob::writeDesc(Int64 &sourceLen, char *source, LobsSubOper subOpe Int64 outDescSyskey = 0; Int32 clierr = 0; char logBuf[4096]; + lobDebugInfo("In ExLob::writeDesc",0,__LINE__,lobTrace_); // Calculate sourceLen for each subOper. - if (subOper == Lob_File) + if ((subOper == Lob_File)||(subOper == Lob_External)) { err = statSourceFile(source, sourceLen); if (err != LOB_OPER_OK) @@ -682,16 +701,21 @@ Ex_Lob_Error ExLob::writeDesc(Int64 &sourceLen, char *source, LobsSubOper subOpe return LOB_MAX_LIMIT_ERROR; //exceeded the size of the max lob size //TBD trigger compaction } - lobDebugInfo("Calling ExLob::allocateDesc",0,__LINE__,lobTrace_); - err = allocateDesc((unsigned int)sourceLen, descNumOut, dataOffset, lobMaxSize, lobMaxChunkMemSize,handleIn, handleInLen, lobGCLimit,lobGlobals); - + if (subOper != Lob_External) + { + lobDebugInfo("Calling ExLob::allocateDesc",0,__LINE__,lobTrace_); + err = allocateDesc((unsigned int)sourceLen, descNumOut, dataOffset, lobMaxSize, lobMaxChunkMemSize,handleIn, handleInLen, lobGCLimit,lobGlobals); + } + operLen = 0; if (err != LOB_OPER_OK) return err; lobDebugInfo("Calling cli LOB_CLI_INSERT",0,__LINE__,lobTrace_); + clierr = SQL_EXEC_LOBcliInterface(handleIn, handleInLen, - NULL, blackBoxLen, + blackBox, + blackBoxLen, handleOut, &handleOutLen, LOB_CLI_INSERT, LOB_CLI_ExecImmed, &dataOffset, &sourceLen, @@ -747,7 +771,7 @@ Ex_Lob_Error ExLob::insertDesc(Int64 offset, Int64 size, char *handleIn, Int32 Ex_Lob_Error ExLob::writeLobData(char *source, Int64 sourceLen, LobsSubOper subOperation, Int64 tgtOffset,Int64 &operLen, Int64 lobMaxChunkMemSize) { - Ex_Lob_Error err; + Ex_Lob_Error err=LOB_OPER_OK; char logBuf[4096]; lobDebugInfo("In ExLob::writeLobData",0,__LINE__,lobTrace_); char *inputAddr = source; @@ -755,6 +779,8 @@ Ex_Lob_Error ExLob::writeLobData(char *source, Int64 sourceLen, LobsSubOper subO Int32 allocMemSize = 0; Int64 inputSize = sourceLen; Int64 writeOffset = tgtOffset; + if (subOperation == Lob_External) + return LOB_OPER_OK; while(inputSize > 0) { allocMemSize = MINOF(lobMaxChunkMemSize, inputSize); @@ -823,7 +849,14 @@ Ex_Lob_Error ExLob::readToMem(char *memAddr, Int64 size, Int64 &operLen,char * return err; } sizeToRead = MINOF(size,desc.getSize()); - + if (blackBox) + { + + // we have received the external data file name from the descriptor table + // replace the contents of the lobDataFile with this name + str_cpy_and_null(lobDataFile_, blackBox, blackBoxLen,'\0','0',TRUE); + + } if (blackBoxLen == -1) { lobDebugInfo("Reading multiple chunks",0,__LINE__,lobTrace_); @@ -892,6 +925,13 @@ Ex_Lob_Error ExLob::readToFile(char *tgtFileName, Int64 tgtLength, Int64 &operLe srcOffset = desc.getOffset(); } + if (blackBox) + { + // we have received the external data file name from the descriptor table + // replace the contents of the lobDataFile with this name + str_cpy_and_null(lobDataFile_, blackBox, blackBoxLen,'\0','0',TRUE); + + } if (tgtType == HDFS_FILE) { err = readDataToHdfsFile(tgtFileName, srcOffset , tgtLength,operLen, lobMaxChunkMemLen, fileflags,handleIn,handleInLen,multipleChunks,transId); @@ -926,24 +966,35 @@ Ex_Lob_Error ExLob::append(char *data, Int64 size, LobsSubOper so, Int64 headDes Int32 chunkNum = 0; Int64 outDescPartnKey, outDescSyskey = 0; char logBuf[4096]; + char *blackBox = NULL; + Int32 blackBoxLen = 0; + + if (so ==Lob_External) + { + blackBox = data; + blackBoxLen = (Int32)size; + } lobDebugInfo("In ExLob::append",0,__LINE__,lobTrace_); - if (so == Lob_File) + if ((so == Lob_File)) { err = statSourceFile(data, sourceLen); if (err != LOB_OPER_OK) return err; } + + if (sourceLen <= 0 || sourceLen > lobMaxSize) { - return LOB_MAX_LIMIT_ERROR; //exceeded the size of the max lob size + return LOB_MAX_LIMIT_ERROR; //exceeded the size of the max lob size } err = allocateDesc((unsigned int)sourceLen, dummyParam, dataOffset, lobMaxSize,lobMaxChunkMemSize,handleIn, handleInLen,lobGCLimit,lobGlobals); if (err != LOB_OPER_OK) return err; + lobDebugInfo("Calling cli LOB_CLI_INSERT_APPEND",0,__LINE__,lobTrace_); clierr = SQL_EXEC_LOBcliInterface(handleIn, handleInLen, - 0, &chunkNum, + blackBox, &blackBoxLen, handleOut, &handleOutLen, LOB_CLI_INSERT_APPEND, LOB_CLI_ExecImmed, &dataOffset, &sourceLen, @@ -1024,8 +1075,16 @@ Ex_Lob_Error ExLob::update(char *data, Int64 size, LobsSubOper so,Int64 headDesc Int64 outDescPartnKey,outDescSyskey = 0; Int32 chunkNum = 0; char logBuf[4096]; + char *blackBox = NULL; + Int32 blackBoxLen = 0; + + if (so == Lob_External) + { + blackBox = data; + blackBoxLen = (Int32)size; + } lobDebugInfo("In ExLob::update",0,__LINE__,lobTrace_); - if (so == Lob_File) + if ((so == Lob_File) || (so == Lob_External)) { str_sprintf(logBuf,"Calling statSourceFile: source:%s, sourceLen: %Ld", data,sourceLen); @@ -1034,18 +1093,21 @@ Ex_Lob_Error ExLob::update(char *data, Int64 size, LobsSubOper so,Int64 headDesc if (err != LOB_OPER_OK) return err; } - if (sourceLen <= 0 || sourceLen > lobMaxSize) + if(so != Lob_External) { - return LOB_MAX_LIMIT_ERROR; //exceeded the size of the max lob size + if (sourceLen <= 0 || sourceLen > lobMaxSize) + { + return LOB_MAX_LIMIT_ERROR; //exceeded the size of the max lob size + } + lobDebugInfo("Calling allocateDesc",0,__LINE__,lobTrace_); + err = allocateDesc((unsigned int)sourceLen, dummyParam, dataOffset, lobMaxSize, lobMaxChunkMemSize, handleIn, handleInLen, lobGCLimit,lobGlobals); + if (err != LOB_OPER_OK) + return err; } - lobDebugInfo("Calling allocateDesc",0,__LINE__,lobTrace_); - err = allocateDesc((unsigned int)sourceLen, dummyParam, dataOffset, lobMaxSize, lobMaxChunkMemSize, handleIn, handleInLen, lobGCLimit,lobGlobals); - if (err != LOB_OPER_OK) - return err; lobDebugInfo("Calling CLI LOB_CLI_UPDATE_UNIQUE",0,__LINE__,lobTrace_); clierr = SQL_EXEC_LOBcliInterface(handleIn, handleInLen, - 0, &chunkNum, + blackBox, &blackBoxLen, handleOut, &handleOutLen, LOB_CLI_UPDATE_UNIQUE, LOB_CLI_ExecImmed, &dataOffset, &sourceLen, @@ -1119,10 +1181,11 @@ Ex_Lob_Error ExLob::openCursor(char *handleIn, Int32 handleInLen,Int64 transId) Int64 dummyParam = 0; void *cliInterface = NULL; char logBuf[4096]; + lobDebugInfo("In ExLob::openCursor",0,__LINE__,lobTrace_); clierr = SQL_EXEC_LOBcliInterface(handleIn, handleInLen, - 0, 0, + 0,0, (char *)&dummyParam, (Lng32 *)&dummyParam, LOB_CLI_SELECT_CURSOR, LOB_CLI_ExecImmed, &dummyParam, &dummyParam, @@ -1135,7 +1198,7 @@ Ex_Lob_Error ExLob::openCursor(char *handleIn, Int32 handleInLen,Int64 transId) return LOB_DESC_READ_ERROR; } - + cursor.bytesRead_ = -1; cursor.descOffset_ = -1; cursor.descSize_ = -1; @@ -2080,7 +2143,8 @@ Ex_Lob_Error ExLobsOper ( { if (operation == Lob_Init) { - globPtr = (void *) new ExLobGlobals(); + NAHeap *passedInHeap=(NAHeap *)blackBox; + globPtr = (void *) new (passedInHeap) ExLobGlobals(); if (globPtr == NULL) return LOB_INIT_ERROR; @@ -2172,6 +2236,11 @@ Ex_Lob_Error ExLobsOper ( break; case Lob_Read: + if (storage == Lob_External_HDFS_File) + //Allocate storage to read the lob external file name from the + //descriptor tables to get the data from. + blackBox = new(lobGlobals->getHeap()) char[MAX_LOB_FILE_NAME_LEN+6]; + if (subOperation == Lob_Memory) { err = lobPtr->readToMem(source,sourceLen,retOperLen,handleIn,handleInLen,(char *)blackBox, blackBoxLen,handleOut,handleOutLen,transId); @@ -2190,6 +2259,8 @@ Ex_Lob_Error ExLobsOper ( } else err = LOB_SUBOPER_ERROR; + if (blackBox) + (lobGlobals->getHeap())->deallocateMemory((char*) blackBox); break; case Lob_ReadDesc: // read desc only. Needed for pass thru. @@ -2250,7 +2321,7 @@ Ex_Lob_Error ExLobsOper ( break; case Lob_Append: - if ((subOperation == Lob_Memory) ||(subOperation == Lob_Buffer)) + if ((subOperation == Lob_Memory) ||(subOperation == Lob_Buffer) || (subOperation ==Lob_External)) { err = lobPtr->append(source, sourceLen, subOperation, descNumIn, retOperLen,lobMaxSize, lobMaxChunkMemSize,lobGCLimit,handleIn,handleInLen,handleOut,handleOutLen,lobGlobals); if (err != LOB_OPER_OK) @@ -2271,7 +2342,7 @@ Ex_Lob_Error ExLobsOper ( break; case Lob_Update: - if ((subOperation == Lob_Memory)||(subOperation == Lob_Buffer)) + if ((subOperation == Lob_Memory)||(subOperation == Lob_Buffer)||(subOperation ==Lob_External)) { err = lobPtr->update(source, sourceLen, subOperation, descNumIn, retOperLen, lobMaxSize, lobMaxChunkMemSize,lobGCLimit,handleIn,handleInLen,handleOut,handleOutLen,lobGlobals); if (err != LOB_OPER_OK) @@ -2328,8 +2399,9 @@ Ex_Lob_Error ExLobsOper ( break; case Lob_Cleanup: - delete lobGlobals; - break; + delete lobGlobals; + break; + case Lob_PerformGC: err = lobPtr->compactLobDataFile((ExLobInMemoryDescChunksEntry *)source,sourceLen); if (err != LOB_OPER_OK) @@ -2863,6 +2935,9 @@ ExLobGlobals::~ExLobGlobals() if (threadTraceFile_) fclose(threadTraceFile_); threadTraceFile_ = NULL; + + hdfsDisconnect(fs_); + fs_ = NULL; } @@ -2872,7 +2947,7 @@ Ex_Lob_Error ExLobGlobals::initialize() { Ex_Lob_Error err = LOB_OPER_OK; - lobMap_ = (lobMap_t *) new (getHeap())lobMap_t; // Leaving this allocated from system heap. Since this class contains hdfsFS unable to derive from LOB heap + lobMap_ = (lobMap_t *) new (getHeap())lobMap_t; if (lobMap_ == NULL) return LOB_INIT_ERROR; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/5a3f2024/core/sql/exp/ExpLOBenums.h ---------------------------------------------------------------------- diff --git a/core/sql/exp/ExpLOBenums.h b/core/sql/exp/ExpLOBenums.h index 99e072d..1159d1d 100644 --- a/core/sql/exp/ExpLOBenums.h +++ b/core/sql/exp/ExpLOBenums.h @@ -174,7 +174,8 @@ typedef enum { Lob_File, Lob_Memory, Lob_Buffer, - Lob_Foreign_Lob + Lob_Foreign_Lob, + Lob_External } LobsSubOper; typedef enum { http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/5a3f2024/core/sql/exp/ExpLOBinterface.cpp ---------------------------------------------------------------------- diff --git a/core/sql/exp/ExpLOBinterface.cpp b/core/sql/exp/ExpLOBinterface.cpp index 23a2083..f6f5529 100644 --- a/core/sql/exp/ExpLOBinterface.cpp +++ b/core/sql/exp/ExpLOBinterface.cpp @@ -57,7 +57,7 @@ Lng32 ExpLOBinterfaceInit(void *& lobGlob, void * lobHeap,NABoolean isHive, Int6 1, // waited op lobGlob, 0, - NULL, 0, + lobGlob, 0, lobMaxSize); if (lobGlob) { @@ -178,7 +178,7 @@ Lng32 ExpLOBinterfaceCleanup(void *& lobGlob, void * lobHeap) 1, // waited op lobGlob, 0, - NULL, 0 + lobHeap, 0 ); if (err != LOB_OPER_OK) return -1; @@ -555,7 +555,7 @@ Lng32 ExpLOBInterfaceUpdateAppend(void * lobGlob, Ex_Lob_Error status; Int64 cliError = -1; Int64 srcLen = 0; - if(so == Lob_Memory) + if((so == Lob_Memory) || (so== Lob_External)) srcLen = strlen(srcLobData); else if (so == Lob_Buffer) srcLen = tgtLobLen; @@ -625,7 +625,7 @@ Lng32 ExpLOBInterfaceUpdate(void * lobGlob, Ex_Lob_Error status; Int64 cliError = -1; Int64 sourceLen = 0; - if(so == Lob_Memory) + if((so == Lob_Memory) || (so == Lob_External)) sourceLen = strlen(srcLobData); else if (so == Lob_Buffer) sourceLen = tgtLobLen; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/5a3f2024/core/sql/generator/GenPreCode.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenPreCode.cpp b/core/sql/generator/GenPreCode.cpp index 811c0ed..300dddf 100644 --- a/core/sql/generator/GenPreCode.cpp +++ b/core/sql/generator/GenPreCode.cpp @@ -5418,7 +5418,15 @@ RelExpr * HbaseUpdate::preCodeGen(Generator * generator, lu->updatedTableSchemaName() += "\""; lu->lobSize() = col->getType()->getPrecision(); lu->lobNum() = col->lobNum(); - lu->lobStorageType() = col->lobStorageType(); + // lu->lobStorageType() = col->lobStorageType(); + if (lu->lobStorageType() != col->lobStorageType()) + { + *CmpCommon::diags() << DgSqlCode(-1432) + << DgInt0((Int32)lu->lobStorageType()) + << DgInt1((Int32)col->lobStorageType()) + << DgString0(col->getColName()); + GenExit(); + } lu->lobStorageLocation() = col->lobStorageLocation(); } } // for @@ -5528,13 +5536,21 @@ RelExpr * HbaseInsert::preCodeGen(Generator * generator, getTableName().getSchemaName()); li->insertedTableSchemaName() += "\""; - // li->lobNum() = col->getPosition(); - // li->lobSize() = srcValueId.getType().getPrecision(); + li->lobSize() = tgtValueId.getType().getPrecision(); li->lobFsType() = tgtValueId.getType().getFSDatatype(); li->lobNum() = col->lobNum(); - li->lobStorageType() = col->lobStorageType(); + if ((child1Expr->getOperatorType() == ITM_CONSTANT) && + !(((ConstValue *)child1Expr)->isNull())) + if (li->lobStorageType() != col->lobStorageType()) + { + *CmpCommon::diags() << DgSqlCode(-1432) + << DgInt0((Int32)li->lobStorageType()) + << DgInt1((Int32)col->lobStorageType()) + << DgString0(col->getColName()); + GenExit(); + } li->lobStorageLocation() = col->lobStorageLocation(); li->bindNode(generator->getBindWA()); @@ -5561,7 +5577,14 @@ RelExpr * HbaseInsert::preCodeGen(Generator * generator, li->insertedTableSchemaName() += "\""; li->lobNum() = col->lobNum(); - li->lobStorageType() = col->lobStorageType(); + if (li->lobStorageType() != col->lobStorageType()) + { + *CmpCommon::diags() << DgSqlCode(-1432) + << DgInt0((Int32)li->lobStorageType()) + << DgInt1((Int32)col->lobStorageType()) + << DgString0(col->getColName()); + GenExit(); + } li->lobStorageLocation() = col->lobStorageLocation(); li->lobSize() = tgtValueId.getType().getPrecision(); @@ -5593,7 +5616,7 @@ RelExpr * HbaseInsert::preCodeGen(Generator * generator, li->lobFsType() = tgtValueId.getType().getFSDatatype(); li->lobNum() = col->lobNum(); - li->lobStorageType() = col->lobStorageType(); + li->lobStorageLocation() = col->lobStorageLocation(); li->bindNode(generator->getBindWA()); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/5a3f2024/core/sql/generator/GenRelExeUtil.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenRelExeUtil.cpp b/core/sql/generator/GenRelExeUtil.cpp index 55eba37..fb773e1 100644 --- a/core/sql/generator/GenRelExeUtil.cpp +++ b/core/sql/generator/GenRelExeUtil.cpp @@ -3474,7 +3474,7 @@ short ExeUtilLobInfo::codeGen(Generator * generator) char *lobColArray = NULL; char * lobNumArray = NULL; char * lobLocArray = NULL; - + char * lobTypeArray = NULL; const NATable * naTable = getUtilTableDesc()->getNATable(); Lng32 numLOBs = 0; @@ -3496,6 +3496,7 @@ short ExeUtilLobInfo::codeGen(Generator * generator) lobColArray = space->allocateAlignedSpace(numLOBs*LOBINFO_MAX_FILE_LEN); lobNumArray = space->allocateAlignedSpace(numLOBs*2); lobLocArray = space->allocateAlignedSpace(numLOBs * LOBINFO_MAX_FILE_LEN); + lobTypeArray = space->allocateAlignedSpace(numLOBs * 4); const NATable * naTable = getUtilTableDesc()->getNATable(); CollIndex j = 0; @@ -3510,6 +3511,7 @@ short ExeUtilLobInfo::codeGen(Generator * generator) *(short*)(&lobNumArray[2*j]) = col->lobNum(); strcpy(&lobLocArray[j*LOBINFO_MAX_FILE_LEN], col->lobStorageLocation()); + *(Int32 *)(&lobTypeArray[4*j]) = col->lobStorageType(); j++; } @@ -3529,6 +3531,7 @@ short ExeUtilLobInfo::codeGen(Generator * generator) lobColArray, lobNumArray, lobLocArray, + lobTypeArray, hdfsPort, hdfsServer, tableFormat_, @@ -4101,6 +4104,7 @@ short ExeUtilLobShowddl::codeGen(Generator * generator) char * lobNumArray = NULL; char * lobLocArray = NULL; + char * lobTypeArray = NULL; const NATable * naTable = getUtilTableDesc()->getNATable(); Lng32 numLOBs = 0; @@ -4117,6 +4121,7 @@ short ExeUtilLobShowddl::codeGen(Generator * generator) maxLocLen = MAXOF(maxLocLen, (short)strlen(col->lobStorageLocation()) + 1); + } // if } // for } @@ -4125,7 +4130,7 @@ short ExeUtilLobShowddl::codeGen(Generator * generator) { lobNumArray = space->allocateAlignedSpace(numLOBs*2); lobLocArray = space->allocateAlignedSpace(numLOBs * maxLocLen); - + lobTypeArray = space->allocateAlignedSpace(numLOBs * 4); const NATable * naTable = getUtilTableDesc()->getNATable(); CollIndex j = 0; @@ -4136,9 +4141,8 @@ short ExeUtilLobShowddl::codeGen(Generator * generator) if (col->getType()->isLob()) { *(short*)(&lobNumArray[2*j]) = col->lobNum(); - strcpy(&lobLocArray[j*maxLocLen], col->lobStorageLocation()); - + *(Int32 *)(&lobTypeArray[4*j]) = col->lobStorageType(); j++; } } @@ -4154,6 +4158,7 @@ short ExeUtilLobShowddl::codeGen(Generator * generator) numLOBs, lobNumArray, lobLocArray, + lobTypeArray, maxLocLen, (short)sdOptions_, givenDesc, @@ -4164,7 +4169,7 @@ short ExeUtilLobShowddl::codeGen(Generator * generator) 2, 32000); #pragma warn(1506) // warning elimination - + generator->initTdbFields(exe_util_tdb); if(!generator->explainDisabled()) { http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/5a3f2024/core/sql/optimizer/BindRelExpr.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/BindRelExpr.cpp b/core/sql/optimizer/BindRelExpr.cpp index 0fa2288..c4cc73e 100644 --- a/core/sql/optimizer/BindRelExpr.cpp +++ b/core/sql/optimizer/BindRelExpr.cpp @@ -7813,7 +7813,7 @@ static RelExpr *checkTupleElementsAreAllScalar(BindWA *bindWA, RelExpr *re) *CmpCommon::diags() << DgSqlCode(-4125); bindWA->setErrStatus(); return NULL; - } + } else if (cols.entries() == 1) { // if cols.entries() > 1 && subq->getDegree() > 1 // we do not want to make the transformation velow. We want to keep the // values clause, so that it cann be attached by a tsj to the subquery http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/5a3f2024/core/sql/optimizer/ItemFunc.h ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/ItemFunc.h b/core/sql/optimizer/ItemFunc.h index f57d1d8..41e5c92 100644 --- a/core/sql/optimizer/ItemFunc.h +++ b/core/sql/optimizer/ItemFunc.h @@ -2749,6 +2749,11 @@ public: hdfsPort_((Lng32)CmpCommon::getDefaultNumeric(LOB_HDFS_PORT)), hdfsServer_( CmpCommon::getDefaultString(LOB_HDFS_SERVER)) { + if ((obj == STRING_) || (obj == BUFFER_) || (obj == FILE_)) + lobStorageType_ = Lob_HDFS_File; + else if (obj == EXTERNAL_) + lobStorageType_ = Lob_External_HDFS_File; + } // copyTopNode method http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/5a3f2024/core/sql/optimizer/NAColumn.h ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/NAColumn.h b/core/sql/optimizer/NAColumn.h index ec58fe9..af3fb84 100644 --- a/core/sql/optimizer/NAColumn.h +++ b/core/sql/optimizer/NAColumn.h @@ -134,7 +134,7 @@ public: isSaltColumn_(isSaltColumn), isDivisioningColumn_(isDivisioningColumn), lobNum_(-1), - lobStorageType_(Lob_Invalid_Storage), + lobStorageType_(Lob_HDFS_File), lobStorageLocation_(NULL), hbaseColFlags_(0) { http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/5a3f2024/core/sql/parser/ElemDDLCol.cpp ---------------------------------------------------------------------- diff --git a/core/sql/parser/ElemDDLCol.cpp b/core/sql/parser/ElemDDLCol.cpp index 33ee187..a18c317 100644 --- a/core/sql/parser/ElemDDLCol.cpp +++ b/core/sql/parser/ElemDDLCol.cpp @@ -85,7 +85,7 @@ ElemDDLColDef::ElemDDLColDef( isDivisionColumn_(FALSE), divisionColumnSeqNum_(-1), isLobAttrsSpec_(FALSE), - lobStorage_(Lob_Invalid_Storage), + lobStorage_(Lob_HDFS_File), isSeabaseSerializedSpec_(FALSE), seabaseSerialized_(FALSE), isColDefaultSpec_(FALSE) http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/5a3f2024/core/sql/parser/sqlparser.y ---------------------------------------------------------------------- diff --git a/core/sql/parser/sqlparser.y b/core/sql/parser/sqlparser.y index 36a8e5a..6466bdd 100755 --- a/core/sql/parser/sqlparser.y +++ b/core/sql/parser/sqlparser.y @@ -11440,6 +11440,7 @@ blob_type : TOK_BLOB blob_optional_left_len_right $$ = new (PARSERHEAP()) SQLClob ( $2 ); } } + blob_optional_left_len_right: '(' NUMERIC_LITERAL_EXACT_NO_SCALE optional_blob_unit ')' { @@ -11454,7 +11455,7 @@ blob_optional_left_len_right: '(' NUMERIC_LITERAL_EXACT_NO_SCALE optional_blob_u longIntVal = longIntVal * $3; - $$ = (Lng32)longIntVal; + $$ = (Int64)longIntVal; delete $2; } @@ -11464,12 +11465,12 @@ blob_optional_left_len_right: '(' NUMERIC_LITERAL_EXACT_NO_SCALE optional_blob_u if (CmpCommon::getDefault(TRAF_BLOB_AS_VARCHAR) == DF_ON) { - $$ = (Lng32)100000; + $$ = (Int64)100000; } else { - $$ = (Lng32)CmpCommon::getDefaultNumeric(LOB_MAX_SIZE)*1024*1024; + $$ = (Int64)CmpCommon::getDefaultNumeric(LOB_MAX_SIZE)*1024*1024; } } @@ -12710,12 +12711,12 @@ insert_obj_to_lob_function : } | TOK_EXTERNALTOLOB '(' literal ')' { - YYERROR; + $$ = new (PARSERHEAP()) LOBinsert( $3, NULL, LOBoper::EXTERNAL_); } | TOK_EXTERNALTOLOB '(' literal ',' literal ')' { - YYERROR; + $$ = new (PARSERHEAP()) LOBinsert( $3, $5, LOBoper::EXTERNAL_); } @@ -12759,8 +12760,13 @@ update_obj_to_lob_function : | TOK_EXTERNALTOLOB '(' literal ')' { - YYERROR; + $$ = new (PARSERHEAP()) LOBupdate( $3, NULL, NULL,LOBoper::EXTERNAL_, FALSE); + } + | TOK_EXTERNALTOLOB '(' literal ',' TOK_APPEND ')' + { + YYERROR; + $$ = new (PARSERHEAP()) LOBupdate( $3, NULL, NULL,LOBoper::EXTERNAL_, TRUE); } | TOK_EXTERNALTOLOB '(' literal ',' literal ')' { @@ -25258,21 +25264,14 @@ optional_loggable : TOK_LOGGABLE } // pElemDDLL -optional_lobattrs : TOK_LOB TOK_STORAGE TOK_TYPE QUOTED_STRING +optional_lobattrs : TOK_STORAGE QUOTED_STRING { LobsStorage ls; - if (*$4 == "HDFS") - ls = Lob_HDFS_File; - else if (*$4 == "HBASE") - ls = Lob_HBASE_Table; - else if (*$4 == "External HDFS") + + if (*$2 == "external") ls = Lob_External_HDFS_File; - else if (*$4 == "External HBASE") - ls = Lob_External_HBASE_Table; - else if (*$4 == "External Local") - ls = Lob_External_Local_File; else - ls = Lob_Local_File; + ls = Lob_HDFS_File; $$ = new (PARSERHEAP()) ElemDDLLobAttrs(ls); }
