Merge PR 1417 [TRAFODION-2917] Refactor Trafodion implementation of hdfs scan for text formatted hive tables
Project: http://git-wip-us.apache.org/repos/asf/trafodion/repo Commit: http://git-wip-us.apache.org/repos/asf/trafodion/commit/de835767 Tree: http://git-wip-us.apache.org/repos/asf/trafodion/tree/de835767 Diff: http://git-wip-us.apache.org/repos/asf/trafodion/diff/de835767 Branch: refs/heads/master Commit: de8357677276e3987f24b550da5b7d0fc2b8200c Parents: 087af70 cc43432 Author: selvaganesang <[email protected]> Authored: Fri Feb 16 19:59:39 2018 +0000 Committer: selvaganesang <[email protected]> Committed: Fri Feb 16 19:59:39 2018 +0000 ---------------------------------------------------------------------- core/sql/comexe/ComTdbHdfsScan.h | 10 +- core/sql/executor/ExExeUtil.h | 3 +- core/sql/executor/ExExeUtilCli.cpp | 1 - core/sql/executor/ExExeUtilGet.cpp | 16 +- core/sql/executor/ExExeUtilLoad.cpp | 53 +- core/sql/executor/ExExeUtilMisc.cpp | 1 - core/sql/executor/ExFastTransport.cpp | 100 ++- core/sql/executor/ExFastTransport.h | 5 +- core/sql/executor/ExHbaseAccess.cpp | 53 +- core/sql/executor/ExHbaseAccess.h | 32 +- core/sql/executor/ExHbaseIUD.cpp | 58 +- core/sql/executor/ExHdfsScan.cpp | 378 ++++++--- core/sql/executor/ExHdfsScan.h | 69 +- core/sql/executor/HBaseClient_JNI.cpp | 147 +--- core/sql/executor/HBaseClient_JNI.h | 23 +- core/sql/executor/HdfsClient_JNI.cpp | 763 +++++++++++++++++++ core/sql/executor/HdfsClient_JNI.h | 192 +++++ core/sql/executor/JavaObjectInterface.cpp | 24 +- core/sql/executor/JavaObjectInterface.h | 24 +- core/sql/executor/OrcFileReader.cpp | 3 +- core/sql/executor/SequenceFileReader.cpp | 304 +------- core/sql/executor/SequenceFileReader.h | 30 - core/sql/exp/ExpErrorEnums.h | 1 + core/sql/exp/ExpHbaseInterface.cpp | 93 +-- core/sql/exp/ExpHbaseInterface.h | 18 +- core/sql/exp/ExpLOBinterface.h | 13 - core/sql/generator/GenRelScan.cpp | 3 + core/sql/nskgmake/executor/Makefile | 1 + core/sql/qmscommon/QRLogger.cpp | 1 + core/sql/qmscommon/QRLogger.h | 1 + core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp | 1 - core/sql/sqlcomp/DefaultConstants.h | 3 + core/sql/sqlcomp/nadefaults.cpp | 2 + .../main/java/org/trafodion/sql/HDFSClient.java | 394 ++++++++++ .../main/java/org/trafodion/sql/HdfsScan.java | 289 +++++++ .../org/trafodion/sql/SequenceFileWriter.java | 154 ---- .../org/trafodion/sql/TrafConfiguration.java | 1 + 37 files changed, 2244 insertions(+), 1020 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/trafodion/blob/de835767/core/sql/executor/ExExeUtil.h ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/trafodion/blob/de835767/core/sql/executor/ExExeUtilLoad.cpp ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/trafodion/blob/de835767/core/sql/executor/ExExeUtilMisc.cpp ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/trafodion/blob/de835767/core/sql/executor/ExFastTransport.cpp ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/trafodion/blob/de835767/core/sql/executor/ExHdfsScan.cpp ---------------------------------------------------------------------- diff --cc core/sql/executor/ExHdfsScan.cpp index 7dd933b,c36270e..2b73feb --- a/core/sql/executor/ExHdfsScan.cpp +++ b/core/sql/executor/ExHdfsScan.cpp @@@ -280,9 -307,15 +307,15 @@@ void ExHdfsScanTcb::freeResources( } deallocateRuntimeRanges(); if (lobGlob_) { - ExpLOBinterfaceCleanup(lobGlob_, (NAHeap *)getGlobals()->getDefaultHeap()); + ExpLOBinterfaceCleanup(lobGlob_); lobGlob_ = NULL; } + if (hdfsClient_ != NULL) + NADELETE(hdfsClient_, HdfsClient, getHeap()); + if (logFileHdfsClient_ != NULL) + NADELETE(logFileHdfsClient_, HdfsClient, getHeap()); + if (hdfsScan_ != NULL) + NADELETE(hdfsScan_, HdfsScan, getHeap()); } NABoolean ExHdfsScanTcb::needStatsEntry() http://git-wip-us.apache.org/repos/asf/trafodion/blob/de835767/core/sql/exp/ExpLOBinterface.h ---------------------------------------------------------------------- diff --cc core/sql/exp/ExpLOBinterface.h index c8a8591,b98d2b4..8194859 --- a/core/sql/exp/ExpLOBinterface.h +++ b/core/sql/exp/ExpLOBinterface.h @@@ -342,42 -342,8 +342,29 @@@ Lng32 ExpLOBInterfaceGetLobLength(ExLob Int64 &outLobLen ); +Lng32 ExpLOBInterfaceGetFileName(ExLobGlobals * exLobGlob, + char * lobName, + char * lobLoc, + Lng32 lobType, + char * lobHdfsServer, + Lng32 lobHdfsPort, + Int32 handleLen, + char * lobHandle, + char * outFileName, + Int32 &outFileLen); +Lng32 ExpLOBInterfaceGetOffset(ExLobGlobals * exLobGlob, + char * lobName, + char * lobLoc, + Lng32 lobType, + char * lobHdfsServer, + Lng32 lobHdfsPort, + Int32 handleLen, + char * lobHandle, + Int64 &outLobOffset + + ); - /* - class HdfsFileInfo - { - public: - char * fileName() { return fileName_; } - Int64 getStartOffset() { return startOffset_; } - Int64 getBytesToRead() { return bytesToRead_; } - Lng32 entryNum_; // 0 based, first entry is entry num 0. - NABasicPtr fileName_; - Int64 startOffset_; - Int64 bytesToRead_; - }; - */ #endif http://git-wip-us.apache.org/repos/asf/trafodion/blob/de835767/core/sql/qmscommon/QRLogger.cpp ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/trafodion/blob/de835767/core/sql/qmscommon/QRLogger.h ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/trafodion/blob/de835767/core/sql/sqlcomp/DefaultConstants.h ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/trafodion/blob/de835767/core/sql/sqlcomp/nadefaults.cpp ----------------------------------------------------------------------
