JIRA TRAFODION-2013 Hive null values are now handled correctly.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/1a549e0e Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/1a549e0e Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/1a549e0e Branch: refs/heads/master Commit: 1a549e0e73811c85ab3438fc6d527aec8519c446 Parents: 7862d94 Author: Anoop Sharma <[email protected]> Authored: Fri May 27 17:38:08 2016 +0000 Committer: Anoop Sharma <[email protected]> Committed: Fri May 27 17:38:08 2016 +0000 ---------------------------------------------------------------------- core/sql/comexe/ComTdbFastTransport.h | 4 +- core/sql/comexe/ComTdbHdfsScan.cpp | 13 +- core/sql/comexe/ComTdbHdfsScan.h | 47 +- core/sql/common/ComSmallDefs.h | 3 + core/sql/executor/ExFastTransport.cpp | 18 +- core/sql/executor/ExHdfsScan.cpp | 38 +- core/sql/executor/hiveHook.cpp | 27 +- core/sql/generator/GenFastTransport.cpp | 5 +- core/sql/generator/GenRelScan.cpp | 32 +- core/sql/optimizer/BindRelExpr.cpp | 30 +- core/sql/optimizer/HDFSHook.cpp | 1 + core/sql/optimizer/HDFSHook.h | 4 + core/sql/optimizer/RelFastTransport.cpp | 1 + core/sql/optimizer/RelFastTransport.h | 18 +- core/sql/optimizer/hiveHook.h | 25 +- core/sql/regress/core/EXPECTED162 | 22 +- core/sql/regress/hive/EXPECTED001 | 22 +- core/sql/regress/hive/EXPECTED003 | 244 ++++---- core/sql/regress/hive/EXPECTED004 | 240 ++++---- core/sql/regress/hive/EXPECTED005 | 56 +- core/sql/regress/hive/EXPECTED006 | 16 +- core/sql/regress/hive/EXPECTED015 | 64 +-- core/sql/regress/hive/EXPECTED018 | 570 +++++++++++-------- core/sql/regress/hive/TEST018 | 48 +- .../hive/TEST018_create_hive_tables.hive | 16 +- core/sql/regress/tools/runregr_hive.ksh | 39 +- core/sql/sqlcomp/DefaultConstants.h | 2 +- core/sql/sqlcomp/nadefaults.cpp | 1 - 28 files changed, 932 insertions(+), 674 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1a549e0e/core/sql/comexe/ComTdbFastTransport.h ---------------------------------------------------------------------- diff --git a/core/sql/comexe/ComTdbFastTransport.h b/core/sql/comexe/ComTdbFastTransport.h index b92e8f1..1ae7625 100644 --- a/core/sql/comexe/ComTdbFastTransport.h +++ b/core/sql/comexe/ComTdbFastTransport.h @@ -427,9 +427,9 @@ protected: UInt16 ioTimeout_; // 128 - 129 UInt16 filler_; // 130 - 131 UInt32 childDataRowLen_; // 132 - 135 - + // Make sure class size is a multiple of 8 - char fillerComTdbFastTransport_[8]; // 136 -143 + char fillerComTdbFastTransport_[8]; // 136 - 143 }; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1a549e0e/core/sql/comexe/ComTdbHdfsScan.cpp ---------------------------------------------------------------------- diff --git a/core/sql/comexe/ComTdbHdfsScan.cpp b/core/sql/comexe/ComTdbHdfsScan.cpp index 0f42446..93704cb 100755 --- a/core/sql/comexe/ComTdbHdfsScan.cpp +++ b/core/sql/comexe/ComTdbHdfsScan.cpp @@ -47,6 +47,7 @@ ComTdbHdfsScan::ComTdbHdfsScan( Queue * hdfsFileRangeNumList, char recordDelimiter, char columnDelimiter, + char * nullFormat, Int64 hdfsBufSize, UInt32 rangeTailIOSize, Int64 hdfsSqlMaxRecLen, @@ -57,6 +58,7 @@ ComTdbHdfsScan::ComTdbHdfsScan( const unsigned short asciiTuppIndex, const unsigned short workAtpIndex, const unsigned short moveColsTuppIndex, + const unsigned short origTuppIndex, ex_cri_desc * work_cri_desc, ex_cri_desc * given_cri_desc, ex_cri_desc * returned_cri_desc, @@ -93,12 +95,14 @@ ComTdbHdfsScan::ComTdbHdfsScan( hdfsFileRangeNumList_(hdfsFileRangeNumList), recordDelimiter_(recordDelimiter), columnDelimiter_(columnDelimiter), + nullFormat_(nullFormat), hdfsBufSize_(hdfsBufSize), rangeTailIOSize_(rangeTailIOSize), hdfsSqlMaxRecLen_(hdfsSqlMaxRecLen), outputRowLength_(outputRowLength), asciiRowLen_(asciiRowLen), moveExprColsRowLength_(moveColsRowLen), + origTuppIndex_(origTuppIndex), tuppIndex_(tuppIndex), asciiTuppIndex_(asciiTuppIndex), workAtpIndex_(workAtpIndex), @@ -139,6 +143,9 @@ Long ComTdbHdfsScan::pack(void * space) hdfsFileInfoList_.pack(space); hdfsFileRangeBeginList_.pack(space); hdfsFileRangeNumList_.pack(space); + + nullFormat_.pack(space); + errCountTable_.pack(space); loggingLocation_.pack(space); errCountRowId_.pack(space); @@ -170,6 +177,8 @@ Lng32 ComTdbHdfsScan::unpack(void * base, void * reallocator) if (hdfsFileRangeBeginList_.unpack(base, reallocator)) return -1; if (hdfsFileRangeNumList_.unpack(base, reallocator)) return -1; + if (nullFormat_.unpack(base)) return -1; + if (errCountTable_.unpack(base)) return -1; if (loggingLocation_.unpack(base)) return -1; if (errCountRowId_.unpack(base)) return -1; @@ -474,13 +483,13 @@ ComTdbOrcFastAggr::ComTdbOrcFastAggr( hdfsFileInfoList, hdfsFileRangeBeginList, hdfsFileRangeNumList, - 0, 0, 0, 0, 0, + 0, 0, NULL, 0, 0, 0, projRowLen, 0, 0, returnedTuppIndex, 0, projTuppIndex, - 0, + 0, 0, work_cri_desc, given_cri_desc, returned_cri_desc, http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1a549e0e/core/sql/comexe/ComTdbHdfsScan.h ---------------------------------------------------------------------- diff --git a/core/sql/comexe/ComTdbHdfsScan.h b/core/sql/comexe/ComTdbHdfsScan.h index 0842c19..70573e1 100755 --- a/core/sql/comexe/ComTdbHdfsScan.h +++ b/core/sql/comexe/ComTdbHdfsScan.h @@ -131,7 +131,10 @@ class ComTdbHdfsScan : public ComTdb NABasicPtr loggingLocation_; // 168 - 175 NABasicPtr errCountRowId_; // 176 - 183 UInt32 hiveScanMode_; // 184 - 187 - char fillersComTdbHdfsScan1_[12]; // 188 - 199 + UInt16 origTuppIndex_; // 188 - 189 + char fillersComTdbHdfsScan1_[2]; // 190 - 191 + NABasicPtr nullFormat_; // 192 - 199 + char fillersComTdbHdfsScan2_[8]; // 200 - 207 public: enum HDFSFileType @@ -161,6 +164,7 @@ public: Queue * hdfsFileRangeNumList, char recordDelimiter, char columnDelimiter, + char * nullFormat, Int64 hdfsBufSize, UInt32 rangeTailIOSize, Int64 hdfsSqlMaxRecLen, @@ -171,6 +175,7 @@ public: const unsigned short asciiTuppIndex, const unsigned short workAtpIndex, const unsigned short moveColsTuppIndex, + const unsigned short origTuppIndex, ex_cri_desc * work_cri_desc, ex_cri_desc * given_cri_desc, ex_cri_desc * returned_cri_desc, @@ -225,6 +230,8 @@ public: Queue* getHdfsFileRangeBeginList() {return hdfsFileRangeBeginList_;} Queue* getHdfsFileRangeNumList() {return hdfsFileRangeNumList_;} + char * getNullFormat() { return nullFormat_; } + const NABoolean isTextFile() const { return (type_ == TEXT_);} const NABoolean isSequenceFile() const { return (type_ == SEQUENCE_);} const NABoolean isOrcFile() const { return (type_ == ORC_);} @@ -242,23 +249,24 @@ public: NABoolean hdfsPrefetch() { return (flags_ & HDFS_PREFETCH) != 0; }; void setUseCif(NABoolean v) - {(v ? flags_ |= USE_CIF : flags_ &= ~USE_CIF); }; - NABoolean useCif() { return (flags_ & USE_CIF) != 0; }; - - void setUseCifDefrag(NABoolean v) - {(v ? flags_ |= USE_CIF_DEFRAG : flags_ &= ~USE_CIF_DEFRAG); }; - NABoolean useCifDefrag() { return (flags_ & USE_CIF_DEFRAG) != 0; }; - void setContinueOnError(NABoolean v) - {(v ? flags_ |= CONTINUE_ON_ERROR : flags_ &= ~CONTINUE_ON_ERROR); }; - NABoolean continueOnError() { return (flags_ & CONTINUE_ON_ERROR) != 0; }; - - void setLogErrorRows(NABoolean v) - {(v ? flags_ |= LOG_ERROR_ROWS : flags_ &= ~LOG_ERROR_ROWS); }; - NABoolean getLogErrorRows() { return (flags_ & LOG_ERROR_ROWS) != 0; }; - - UInt32 getMaxErrorRows() const { return maxErrorRows_;} - void setMaxErrorRows(UInt32 v ) { maxErrorRows_= v; } + {(v ? flags_ |= USE_CIF : flags_ &= ~USE_CIF); }; + NABoolean useCif() { return (flags_ & USE_CIF) != 0; }; + + void setUseCifDefrag(NABoolean v) + {(v ? flags_ |= USE_CIF_DEFRAG : flags_ &= ~USE_CIF_DEFRAG); }; + NABoolean useCifDefrag() { return (flags_ & USE_CIF_DEFRAG) != 0; }; + void setContinueOnError(NABoolean v) + {(v ? flags_ |= CONTINUE_ON_ERROR : flags_ &= ~CONTINUE_ON_ERROR); }; + NABoolean continueOnError() { return (flags_ & CONTINUE_ON_ERROR) != 0; }; + + void setLogErrorRows(NABoolean v) + {(v ? flags_ |= LOG_ERROR_ROWS : flags_ &= ~LOG_ERROR_ROWS); }; + NABoolean getLogErrorRows() { return (flags_ & LOG_ERROR_ROWS) != 0; }; + + UInt32 getMaxErrorRows() const { return maxErrorRows_;} + void setMaxErrorRows(UInt32 v ) { maxErrorRows_= v; } + // --------------------------------------------------------------------- // Used by the internal SHOWPLAN command to get attributes of a TDB. // --------------------------------------------------------------------- @@ -309,6 +317,11 @@ public: return workCriDesc_->getTupleDescriptor(asciiTuppIndex_); } + ExpTupleDesc *getHdfsOrigRowDesc() const + { + return workCriDesc_->getTupleDescriptor(origTuppIndex_); + } + ExpTupleDesc *getMoveExprColsRowDesc() const { return workCriDesc_->getTupleDescriptor(moveExprColsTuppIndex_); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1a549e0e/core/sql/common/ComSmallDefs.h ---------------------------------------------------------------------- diff --git a/core/sql/common/ComSmallDefs.h b/core/sql/common/ComSmallDefs.h index f83e199..8055e22 100644 --- a/core/sql/common/ComSmallDefs.h +++ b/core/sql/common/ComSmallDefs.h @@ -117,6 +117,9 @@ typedef NABoolean ComBoolean; #define HBASE_STATS_CATALOG "TRAFODION" #define HBASE_STATS_SCHEMA "\"_HBASESTATS_\"" +// default null format for data in hive files. +#define HIVE_DEFAULT_NULL_STRING "\\N" + #define TRAFODION_SYSCAT_LIT "TRAFODION" #define SEABASE_SYSTEM_SCHEMA "SEABASE" #define SEABASE_OLD_PRIVMGR_SCHEMA "PRIVMGR_MD" http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1a549e0e/core/sql/executor/ExFastTransport.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExFastTransport.cpp b/core/sql/executor/ExFastTransport.cpp index f42832d..b881b14 100644 --- a/core/sql/executor/ExFastTransport.cpp +++ b/core/sql/executor/ExFastTransport.cpp @@ -52,6 +52,7 @@ #include "SequenceFileReader.h" #endif #include "cli_stdh.h" +#include "ComSmallDefs.h" //---------------------------------------------------------------------- @@ -587,11 +588,15 @@ void ExHdfsFastExtractTcb::convertSQRowToString(ULng32 nullLen, if (attr->getNullFlag() && ExpAlignedFormat::isNullValue(childRow + attr->getNullIndOffset(), attr->getNullBitIndex())) { - if ( !getEmptyNullString()) // includes hive null which is empty string - { - memcpy(targetData, myTdb().getNullString(), nullLen); - targetData += nullLen; - } + // source is a null value. + + nullLen = 0; + if (myTdb().getNullString()) { + nullLen = strlen(myTdb().getNullString()); + memcpy(targetData, myTdb().getNullString(), nullLen); + } + + targetData += nullLen; currBuffer_->bytesLeft_ -= nullLen; } else { switch ((conv_case_index) sourceFieldsConvIndex_[i]) { @@ -649,7 +654,8 @@ ExWorkProcRetcode ExHdfsFastExtractTcb::work() SFW_RetCode sfwRetCode = SFW_OK; ULng32 recSepLen = strlen(myTdb().getRecordSeparator()); ULng32 delimLen = strlen(myTdb().getDelimiter()); - ULng32 nullLen = strlen(myTdb().getNullString()); + ULng32 nullLen = + (myTdb().getNullString() ? strlen(myTdb().getNullString()) : 0); if (myTdb().getIsHiveInsert()) { recSepLen = 1; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1a549e0e/core/sql/executor/ExHdfsScan.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExHdfsScan.cpp b/core/sql/executor/ExHdfsScan.cpp index 1278c3a..bbdb0c6 100644 --- a/core/sql/executor/ExHdfsScan.cpp +++ b/core/sql/executor/ExHdfsScan.cpp @@ -44,7 +44,7 @@ //#include "hdfs.h" #include "ExpORCinterface.h" - +#include "ComSmallDefs.h" ex_tcb * ExHdfsScanTdb::build(ex_globals * glob) { @@ -1391,6 +1391,9 @@ char * ExHdfsScanTcb::extractAndTransformAsciiSourceToSqlRow(int &err, ExpTupleDesc * asciiSourceTD = hdfsScanTdb().workCriDesc_->getTupleDescriptor(hdfsScanTdb().asciiTuppIndex_); + ExpTupleDesc * origSourceTD = + hdfsScanTdb().workCriDesc_->getTupleDescriptor(hdfsScanTdb().origTuppIndex_); + const char cd = hdfsScanTdb().columnDelimiter_; const char rd = hdfsScanTdb().recordDelimiter_; const char *sourceDataEnd = hdfsScanBuffer_+trailingPrevRead_+ bytesRead_; @@ -1415,6 +1418,7 @@ char * ExHdfsScanTcb::extractAndTransformAsciiSourceToSqlRow(int &err, Lng32 neededColIndex = 0; Attributes * attr = NULL; + Attributes * tgtAttr = NULL; NABoolean rdSeen = FALSE; for (Lng32 i = 0; i < hdfsScanTdb().convertSkipListSize_; i++) @@ -1424,9 +1428,11 @@ char * ExHdfsScanTcb::extractAndTransformAsciiSourceToSqlRow(int &err, if (neededColIndex == asciiSourceTD->numAttrs()) continue; + tgtAttr = NULL; if (hdfsScanTdb().convertSkipList_[i] > 0) { attr = asciiSourceTD->getAttr(neededColIndex); + tgtAttr = origSourceTD->getAttr(neededColIndex); neededColIndex++; } else @@ -1465,13 +1471,27 @@ char * ExHdfsScanTcb::extractAndTransformAsciiSourceToSqlRow(int &err, if (attr->getNullFlag()) { - if (len == 0) - *(short *)&hdfsAsciiSourceData_[attr->getNullIndOffset()] = -1; - else if (memcmp(sourceData, "\\N", len) == 0) - *(short *)&hdfsAsciiSourceData_[attr->getNullIndOffset()] = -1; - else - *(short *)&hdfsAsciiSourceData_[attr->getNullIndOffset()] = 0; - } + *(short *)&hdfsAsciiSourceData_[attr->getNullIndOffset()] = 0; + if (hdfsScanTdb().getNullFormat()) // null format specified by user + { + if (((len == 0) && (strlen(hdfsScanTdb().getNullFormat()) == 0)) || + ((len > 0) && (memcmp(sourceData, hdfsScanTdb().getNullFormat(), len) == 0))) + { + *(short *)&hdfsAsciiSourceData_[attr->getNullIndOffset()] = -1; + } + } // if + else // null format not specified by user + { + // Use default null format. + // for non-varchar, length of zero indicates a null value. + // For all datatypes, HIVE_DEFAULT_NULL_STRING('\N') indicates a null value. + if (((len == 0) && (tgtAttr && (NOT DFS2REC::isSQLVarChar(tgtAttr->getDatatype())))) || + ((len > 0) && (memcmp(sourceData, HIVE_DEFAULT_NULL_STRING, len) == 0))) + { + *(short *)&hdfsAsciiSourceData_[attr->getNullIndOffset()] = -1; + } + } // else + } // if nullable attr if (len > 0) { @@ -1697,7 +1717,7 @@ short ExOrcScanTcb::extractAndTransformOrcSourceToSqlRow( { if (currColLen == 0) *(short *)&hdfsAsciiSourceData_[attr->getNullIndOffset()] = -1; - else if (memcmp(sourceData, "\\N", currColLen) == 0) + else if (memcmp(sourceData, HIVE_DEFAULT_NULL_STRING, currColLen) == 0) *(short *)&hdfsAsciiSourceData_[attr->getNullIndOffset()] = -1; else *(short *)&hdfsAsciiSourceData_[attr->getNullIndOffset()] = 0; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1a549e0e/core/sql/executor/hiveHook.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/hiveHook.cpp b/core/sql/executor/hiveHook.cpp index a83ead4..7e05763 100644 --- a/core/sql/executor/hiveHook.cpp +++ b/core/sql/executor/hiveHook.cpp @@ -45,8 +45,9 @@ struct hive_skey_desc* populateSortCols(HiveMetaData *md, Int32 sdID, struct hive_bkey_desc* populateBucketingCols(HiveMetaData *md, Int32 sdID, NAText* tblStr, size_t& pos); NABoolean populateSerDeParams(HiveMetaData *md, Int32 serdeID, - char& fieldSep, char& recordSep, - NAText* tblStr, size_t& pos); + char& fieldSep, char& recordSep, + NABoolean &nullFormatSpec, NAString &nullFormat, + NAText* tblStr, size_t& pos); NABoolean findAToken (HiveMetaData *md, NAText* tblStr, size_t& pos, const char* tok, const char* errStr, @@ -288,8 +289,12 @@ struct hive_sd_desc* populateSD(HiveMetaData *md, Int32 mainSdID, return NULL; Int32 numBuckets = atoi(numBucketsStr.c_str()); + NABoolean nullFormatSpec = FALSE; + NAString nullFormat; NABoolean success = populateSerDeParams(md, 0, fieldTerminator, - recordTerminator, tblStr, pos); + recordTerminator, + nullFormatSpec, nullFormat, + tblStr, pos); if (!success) return NULL; @@ -306,6 +311,7 @@ struct hive_sd_desc* populateSD(HiveMetaData *md, Int32 mainSdID, numBuckets, inputStr.c_str(), outputStr.c_str(), + (nullFormatSpec ? nullFormat.data() : NULL), hive_sd_desc::TABLE_SD, // TODO : no support for hive_sd_desc::PARTN_SD newColumns, @@ -527,6 +533,7 @@ static int getAsciiDecimalValue(const char * valPtr) NABoolean populateSerDeParams(HiveMetaData *md, Int32 serdeID, char& fieldTerminator, char& recordTerminator, + NABoolean &nullFormatSpec, NAString &nullFormat, NAText* tblStr, size_t& pos) { @@ -543,10 +550,20 @@ NABoolean populateSerDeParams(HiveMetaData *md, Int32 serdeID, "populateSerDeParams::serDeInfo:)},###")) return NULL; - + const char * nullStr = "serialization.null.format="; const char * fieldStr = "field.delim" ; const char * lineStr = "line.delim" ; + nullFormatSpec = FALSE; + foundB = tblStr->find(nullStr,pos); + if ((foundB != std::string::npos) && (foundB < foundE)) + { + nullFormatSpec = TRUE; + std::size_t foundNB = foundB + strlen(nullStr); + std::size_t foundNE = tblStr->find(", ", foundNB); + nullFormat = NAString(tblStr->substr(foundNB, (foundNE-foundNB))); + } + foundB = tblStr->find(fieldStr,pos); if ((foundB != std::string::npos) && (foundB < foundE)) fieldTerminator = tblStr->at(foundB+strlen(fieldStr)+1); @@ -661,7 +678,7 @@ struct hive_tbl_desc* HiveMetaData::getFakedTableDesc(const char* tblName) hive_bkey_desc* bk1 = new (h) hive_bkey_desc("C2", 1); - hive_sd_desc* sd1 = new (h)hive_sd_desc(1, "loc", 0, 1, "ift", "oft", + hive_sd_desc* sd1 = new (h)hive_sd_desc(1, "loc", 0, 1, "ift", "oft", NULL, hive_sd_desc::TABLE_SD, c1, sk1, bk1, '\010', '\n'); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1a549e0e/core/sql/generator/GenFastTransport.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenFastTransport.cpp b/core/sql/generator/GenFastTransport.cpp index 08da2bc..953176b 100644 --- a/core/sql/generator/GenFastTransport.cpp +++ b/core/sql/generator/GenFastTransport.cpp @@ -45,7 +45,7 @@ #include "ComQueue.h" //#include "UdfDllInteraction.h" #include "RelFastTransport.h" - +#include "HDFSHook.h" // Helper function to allocate a string in the plan @@ -127,6 +127,7 @@ int CreateAllCharsExpr(const NAType &formalType, NAType *typ = NULL; Lng32 maxLength = GetDisplayLength(formalType); + maxLength = MAXOF(maxLength, 1); if (formalType.getTypeQualifier() != NA_CHARACTER_TYPE ) { @@ -566,8 +567,8 @@ PhysicalFastExtract::codeGen(Generator *generator) hiveTableName = AllocStringInSpace(*space, (char *)getHiveTableName().data()); delimiter = AllocStringInSpace(*space, newDelimiter); header = AllocStringInSpace(*space, (char *)getHeader().data()); - nullString = AllocStringInSpace(*space, (char *)getNullString().data()); recordSeparator = AllocStringInSpace(*space, newRecordSep); + nullString = AllocStringInSpace(*space, (char *)getNullString().data()); result = ft_codegen(generator, *this, // RelExpr &relExpr http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1a549e0e/core/sql/generator/GenRelScan.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenRelScan.cpp b/core/sql/generator/GenRelScan.cpp index 827ed90..9c17fad 100644 --- a/core/sql/generator/GenRelScan.cpp +++ b/core/sql/generator/GenRelScan.cpp @@ -820,8 +820,11 @@ short FileScan::codeGenForHive(Generator * generator) ULng32 asciiRowLen; ExpTupleDesc * asciiTupleDesc = 0; + const Int32 origTuppIndex = 5; + ExpTupleDesc * origTupleDesc = 0; + ex_cri_desc * work_cri_desc = NULL; - work_cri_desc = new(space) ex_cri_desc(5, space); + work_cri_desc = new(space) ex_cri_desc(6, space); returned_desc = new(space) ex_cri_desc(given_desc->noTuples() + 1, space); ExpTupleDesc::TupleDataFormat asciiRowFormat = ExpTupleDesc::SQLARK_EXPLODED_FORMAT; @@ -847,6 +850,7 @@ short FileScan::codeGenForHive(Generator * generator) // by making sure that the output ValueIds created during // binding refer to the outputs of the move expression + ValueIdList origExprVids; for (int ii = 0; ii < (int)hdfsVals.entries();ii++) { if (convertSkipList[ii] == 0) @@ -878,6 +882,8 @@ short FileScan::codeGenForHive(Generator * generator) else projectExprOnlyCastVids.insert(castValue->getValueId()); + origExprVids.insert(hdfsVals[ii]); + orcRowLen += sizeof(Lng32); orcRowLen += givenType.getDisplayLength(); @@ -904,6 +910,19 @@ short FileScan::codeGenForHive(Generator * generator) // Add the tuple descriptor for reply values to the work ATP work_cri_desc->setTupleDescriptor(asciiTuppIndex, asciiTupleDesc); + ULng32 origRowLen; + exp_gen->processValIdList( + origExprVids, // [IN] ValueIdList + asciiRowFormat, // [IN] tuple data format + origRowLen, // [OUT] tuple length + work_atp, // [IN] atp number + origTuppIndex, // [IN] index into atp + &origTupleDesc, // [optional OUT] tuple desc + ExpTupleDesc::LONG_FORMAT); // [optional IN] desc format + + // Add the tuple descriptor for reply values to the work ATP + work_cri_desc->setTupleDescriptor(origTuppIndex, origTupleDesc); + ExpTupleDesc * tuple_desc = 0; ExpTupleDesc * hdfs_desc = 0; ULng32 executorPredColsRecLength; @@ -1159,6 +1178,15 @@ if (hTabStats->isOrcFile()) char * tablename = space->AllocateAndCopyToAlignedSpace(GenGetQualifiedName(getIndexDesc()->getNAFileSet()->getFileSetName()), 0); + char * nullFormat = NULL; + if (hTabStats->getNullFormat()) + { + nullFormat = + space->allocateAndCopyToAlignedSpace(hTabStats->getNullFormat(), + strlen(hTabStats->getNullFormat()), + 0); + } + // create hdfsscan_tdb ComTdbHdfsScan *hdfsscan_tdb = new(space) ComTdbHdfsScan( @@ -1177,6 +1205,7 @@ if (hTabStats->isOrcFile()) hdfsFileRangeNumList, hTabStats->getRecordTerminator(), // recordDelimiter hTabStats->getFieldTerminator(), // columnDelimiter, + nullFormat, hdfsBufSize, rangeTailIOSize, executorPredColsRecLength, @@ -1187,6 +1216,7 @@ if (hTabStats->isOrcFile()) asciiTuppIndex, executorPredTuppIndex, projectOnlyTuppIndex, + origTuppIndex, work_cri_desc, given_desc, returned_desc, http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1a549e0e/core/sql/optimizer/BindRelExpr.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/BindRelExpr.cpp b/core/sql/optimizer/BindRelExpr.cpp index 59d0b9b..b81fdcf 100644 --- a/core/sql/optimizer/BindRelExpr.cpp +++ b/core/sql/optimizer/BindRelExpr.cpp @@ -16657,10 +16657,32 @@ RelExpr * FastExtract::bindNode(BindWA *bindWA) { delimiter_ = ActiveSchemaDB()->getDefaults().getValue(TRAF_UNLOAD_DEF_DELIMITER); } - if (getNullString().length() == 0) - { - nullString_ = ActiveSchemaDB()->getDefaults().getValue(TRAF_UNLOAD_DEF_NULL_STRING); - } + + // if inserting into a hive table and an explicit null string was + // not specified in the unload command, and the target table has a user + // specified null format string, then use it. + if ((isHiveInsert()) && + (hiveTableDesc_ && hiveTableDesc_->getNATable() && + hiveTableDesc_->getNATable()->getClusteringIndex()) && + (NOT nullStringSpec_)) + { + const HHDFSTableStats* hTabStats = + hiveTableDesc_->getNATable()->getClusteringIndex()->getHHDFSTableStats(); + + if (hTabStats->getNullFormat()) + { + nullString_ = hTabStats->getNullFormat(); + nullStringSpec_ = TRUE; + } + } + + // if an explicit or user specified null format was not used, then + // use the default null string. + if (NOT nullStringSpec_) + { + nullString_ = HIVE_DEFAULT_NULL_STRING; + } + if (getRecordSeparator().length() == 0) { recordSeparator_ = ActiveSchemaDB()->getDefaults().getValue(TRAF_UNLOAD_DEF_RECORD_SEPARATOR); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1a549e0e/core/sql/optimizer/HDFSHook.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/HDFSHook.cpp b/core/sql/optimizer/HDFSHook.cpp index 7a6e86d..b4719d2 100644 --- a/core/sql/optimizer/HDFSHook.cpp +++ b/core/sql/optimizer/HDFSHook.cpp @@ -825,6 +825,7 @@ NABoolean HHDFSTableStats::populate(struct hive_tbl_desc *htd) numOfPartCols_ = htd->getNumOfPartCols(); recordTerminator_ = hsd->getRecordTerminator(); fieldTerminator_ = hsd->getFieldTerminator() ; + nullFormat_ = hsd->getNullFormat(); NAString hdfsHost; Int32 hdfsPort = -1; NAString tableDir; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1a549e0e/core/sql/optimizer/HDFSHook.h ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/HDFSHook.h b/core/sql/optimizer/HDFSHook.h index cbe634c..b9b8294 100644 --- a/core/sql/optimizer/HDFSHook.h +++ b/core/sql/optimizer/HDFSHook.h @@ -263,6 +263,7 @@ public: totalNumPartitions_(0), recordTerminator_(0), fieldTerminator_(0), + nullFormat_(NULL), validationJTimestamp_(-1), listPartitionStatsList_(heap), hiveStatsSize_(0), @@ -297,6 +298,7 @@ public: char getRecordTerminator() const {return recordTerminator_;} char getFieldTerminator() const {return fieldTerminator_;} + char *getNullFormat() const { return nullFormat_; } Int32 getNumPartitions() const {return totalNumPartitions_;} @@ -355,6 +357,8 @@ private: char recordTerminator_ ; char fieldTerminator_ ; + char *nullFormat_; + Int64 validationJTimestamp_; // heap size used by the hive stats Int32 hiveStatsSize_; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1a549e0e/core/sql/optimizer/RelFastTransport.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelFastTransport.cpp b/core/sql/optimizer/RelFastTransport.cpp index 8daa6c2..45d4168 100644 --- a/core/sql/optimizer/RelFastTransport.cpp +++ b/core/sql/optimizer/RelFastTransport.cpp @@ -226,6 +226,7 @@ short FastExtract::setOptions(NAList<UnloadOption*> * *da << DgSqlCode(-4376) << DgString0("NULL_STRING"); return 1; } + nullStringSpec_ = TRUE; } break; case UnloadOption::RECORD_SEP_: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1a549e0e/core/sql/optimizer/RelFastTransport.h ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelFastTransport.h b/core/sql/optimizer/RelFastTransport.h index f0088e1..29be039 100644 --- a/core/sql/optimizer/RelFastTransport.h +++ b/core/sql/optimizer/RelFastTransport.h @@ -114,8 +114,10 @@ public : nullString_(*nullString, oHeap), recordSeparator_(*recordSep, oHeap), overwriteHiveTable_(FALSE), - isSequenceFile_(FALSE) - { }; + isSequenceFile_(FALSE), + nullStringSpec_((nullString ? TRUE : FALSE)) + { + }; FastExtract(RelExpr* child, NAString* targName, @@ -135,7 +137,8 @@ public : nullString_(oHeap), recordSeparator_(oHeap), overwriteHiveTable_(FALSE), - isSequenceFile_(FALSE) + isSequenceFile_(FALSE), + nullStringSpec_(FALSE) { }; @@ -155,7 +158,8 @@ public : nullString_(oHeap), recordSeparator_(oHeap), overwriteHiveTable_(FALSE), - isSequenceFile_(FALSE) + isSequenceFile_(FALSE), + nullStringSpec_(FALSE) { }; FastExtract(RelExpr* child, @@ -181,8 +185,10 @@ public : nullString_(oHeap), recordSeparator_(oHeap), overwriteHiveTable_(FALSE), - isSequenceFile_(FALSE) + isSequenceFile_(FALSE), + nullStringSpec_(FALSE) { }; + //! FastExtract Copy Constructor FastExtract(const FastExtract &other); @@ -270,6 +276,7 @@ public : Int32 getHdfsPort() const {return hdfsPort_;} const NAString& getHiveTableName() const {return hiveTableName_;} NABoolean isHiveInsert() const {return (hiveTableDesc_ != NULL);} + const TableDesc* getHiveTableDesc() const { return hiveTableDesc_; } const NAString& getDelimiter() const {return delimiter_;} NABoolean isAppend() const {return isAppend_;} NABoolean includeHeader() const {return includeHeader_ ;} @@ -319,6 +326,7 @@ private: NAString header_; CompressionType cType_; NAString nullString_; + NABoolean nullStringSpec_; // if null format string is specified NAString recordSeparator_; NABoolean isAppend_; TableDesc *hiveTableDesc_; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1a549e0e/core/sql/optimizer/hiveHook.h ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/hiveHook.h b/core/sql/optimizer/hiveHook.h index e877aec..442b164 100644 --- a/core/sql/optimizer/hiveHook.h +++ b/core/sql/optimizer/hiveHook.h @@ -137,11 +137,14 @@ struct hive_sd_desc char fieldTerminator_; char recordTerminator_; + char* nullFormat_; struct hive_sd_desc* next_; hive_sd_desc(Int32 sdID, const char* loc, Int64 creationTS, Int32 buckets, - const char* ift, const char* of, char knd, + const char* ift, const char* of, + const char* nf, + char knd, struct hive_column_desc* column, struct hive_skey_desc* skey, struct hive_bkey_desc* bkey, @@ -149,20 +152,22 @@ struct hive_sd_desc ) : sdID_(sdID), buckets_(buckets), kind_(knd), column_(column), - skey_(skey), bkey_(bkey), - fieldTerminator_(fieldTerminator), - recordTerminator_(recordTerminator), - next_(NULL) - { - location_ = strduph(loc, CmpCommon::contextHeap()); - inputFormat_ = strduph(ift, CmpCommon::contextHeap()); - outputFormat_= strduph(of, CmpCommon::contextHeap()); - } + skey_(skey), bkey_(bkey), + fieldTerminator_(fieldTerminator), + recordTerminator_(recordTerminator), + next_(NULL) + { + location_ = strduph(loc, CmpCommon::contextHeap()); + inputFormat_ = strduph(ift, CmpCommon::contextHeap()); + outputFormat_= strduph(of, CmpCommon::contextHeap()); + nullFormat_ = (nf ? strduph(nf, CmpCommon::contextHeap()) : NULL); + } ~hive_sd_desc(); char getFieldTerminator() const { return fieldTerminator_; } char getRecordTerminator() const { return recordTerminator_;} + char *getNullFormat() const {return nullFormat_; } NABoolean isSequenceFile() const; NABoolean isOrcFile() const; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1a549e0e/core/sql/regress/core/EXPECTED162 ---------------------------------------------------------------------- diff --git a/core/sql/regress/core/EXPECTED162 b/core/sql/regress/core/EXPECTED162 index c715ed1..9eae25a 100644 --- a/core/sql/regress/core/EXPECTED162 +++ b/core/sql/regress/core/EXPECTED162 @@ -753,11 +753,11 @@ P_PROMO_SK P_PROMO_ID P_START_DATE_SK P_END_DATE_SK P_ITEM_SK 8 AAAAAAAAIAAAAAAA 2450350 2450409 304 1.0000000E+003 1 eing Y N N N N N N N Offences feel only o Unknown N 9 AAAAAAAAJAAAAAAA 2450192 2450248 3439 1.0000000E+003 1 n st Y N N N N N N N External forces shal Unknown N 10 AAAAAAAAKAAAAAAA 2450324 2450365 3314 1.0000000E+003 1 bar N N N N N N N N Only local achieveme Unknown N - 11 AAAAAAAALAAAAAAA ? ? ? 1.0000000E+003 ? ought ? ? ? ? N ? N N Teachers shall not m Unknown N + 11 AAAAAAAALAAAAAAA ? ? ? 1.0000000E+003 ? ought N N N Teachers shall not m Unknown N 12 AAAAAAAAMAAAAAAA 2450153 2450169 6688 1.0000000E+003 1 able N N N N N N N N Also only times woul Unknown N 13 AAAAAAAANAAAAAAA 2450316 2450365 11354 1.0000000E+003 1 pri N N N N N N N N Years shall not go l Unknown N 14 AAAAAAAAOAAAAAAA 2450236 2450282 14480 1.0000000E+003 1 ese N N N N N N N N Adults would not del Unknown N - 15 AAAAAAAAPAAAAAAA ? ? ? 1.0000000E+003 ? anti Y ? N N ? N N N Old elements would s Unknown N + 15 AAAAAAAAPAAAAAAA ? ? ? 1.0000000E+003 ? anti Y N N N N N Old elements would s Unknown N 16 AAAAAAAAABAAAAAA 2450342 2450351 11899 1.0000000E+003 1 cally Y N N N N N N N Sudden, wooden theor Unknown N 17 AAAAAAAABBAAAAAA 2450360 2450406 15529 1.0000000E+003 1 ation Y N N N N N N N Almost old churches Unknown N 18 AAAAAAAACBAAAAAA 2450581 2450592 8599 1.0000000E+003 1 eing Y N N N N N N N National communities Unknown N @@ -770,7 +770,7 @@ P_PROMO_SK P_PROMO_ID P_START_DATE_SK P_END_DATE_SK P_ITEM_SK 25 AAAAAAAAJBAAAAAA 2450757 2450780 16328 1.0000000E+003 1 anti Y N N N N N N N Scenes might make hi Unknown N 26 AAAAAAAAKBAAAAAA 2450528 2450545 8600 1.0000000E+003 1 cally N N N N N N N N Legal discussions mu Unknown N 27 AAAAAAAALBAAAAAA 2450849 2450879 15421 1.0000000E+003 1 ation Y N N N N N N N Mental men go britis Unknown N - 28 AAAAAAAAMBAAAAAA 2450816 2450856 1885 ? 1 ? N N N N N ? N N Successful opportuni Unknown ? + 28 AAAAAAAAMBAAAAAA 2450816 2450856 1885 ? 1 N N N N N N N Successful opportuni Unknown 29 AAAAAAAANBAAAAAA 2450696 2450732 14660 1.0000000E+003 1 n st Y N N N N N N N Residential, necessa Unknown N 30 AAAAAAAAOBAAAAAA 2450718 2450762 10987 1.0000000E+003 1 bar N N N N N N N N Flowers used to stop Unknown N 31 AAAAAAAAPBAAAAAA 2450848 2450880 11990 1.0000000E+003 1 ought Y N N N N N N N Broad times might sa Unknown N @@ -800,7 +800,7 @@ P_PROMO_SK P_PROMO_ID P_START_DATE_SK P_END_DATE_SK P_ITEM_SK 55 AAAAAAAAHDAAAAAA 2450622 2450677 12104 1.0000000E+003 1 anti N N N N N N N N Recent, southern lis Unknown N 56 AAAAAAAAIDAAAAAA 2450838 2450888 17563 1.0000000E+003 1 cally Y N N N N N N N Procedures can call Unknown N 57 AAAAAAAAJDAAAAAA 2450834 2450842 12247 1.0000000E+003 1 ation Y N N N N N N N Hours discriminate t Unknown N - 58 AAAAAAAAKDAAAAAA ? 2450797 12404 1.0000000E+003 ? eing Y N N N ? N ? ? ? ? ? + 58 AAAAAAAAKDAAAAAA ? 2450797 12404 1.0000000E+003 ? eing Y N N N N 59 AAAAAAAALDAAAAAA 2450273 2450322 5102 1.0000000E+003 1 n st N N N N N N N N Strong samples parti Unknown N 60 AAAAAAAAMDAAAAAA 2450646 2450676 12076 1.0000000E+003 1 bar Y N N N N N N N Yards write too only Unknown N 61 AAAAAAAANDAAAAAA 2450619 2450667 14864 1.0000000E+003 1 ought Y N N N N N N N Necessary men shall Unknown N @@ -813,7 +813,7 @@ P_PROMO_SK P_PROMO_ID P_START_DATE_SK P_END_DATE_SK P_ITEM_SK 68 AAAAAAAAEEAAAAAA 2450851 2450893 1210 1.0000000E+003 1 eing Y N N N N N N N Reasonably additiona Unknown N 69 AAAAAAAAFEAAAAAA 2450202 2450219 7471 1.0000000E+003 1 n st N N N N N N N N All right able inter Unknown N 70 AAAAAAAAGEAAAAAA 2450679 2450735 3571 1.0000000E+003 1 bar Y N N N N N N N Commercial, cool cel Unknown N - 71 AAAAAAAAHEAAAAAA 2450347 ? ? ? ? ? Y ? N ? N N N N Impossible programme ? ? + 71 AAAAAAAAHEAAAAAA 2450347 ? ? ? ? Y N N N N N Impossible programme 72 AAAAAAAAIEAAAAAA 2450347 2450373 6662 1.0000000E+003 1 able N N N N N N N N Beautiful, british t Unknown N 73 AAAAAAAAJEAAAAAA 2450578 2450637 9032 1.0000000E+003 1 pri Y N N N N N N N Tremendous, social t Unknown N 74 AAAAAAAAKEAAAAAA 2450425 2450456 13904 1.0000000E+003 1 ese N N N N N N N N Decent bodies grant Unknown N @@ -843,7 +843,7 @@ P_PROMO_SK P_PROMO_ID P_START_DATE_SK P_END_DATE_SK P_ITEM_SK 98 AAAAAAAACGAAAAAA 2450384 2450387 10414 1.0000000E+003 1 eing Y N N N N N N N Periods shall not ne Unknown N 99 AAAAAAAADGAAAAAA 2450742 2450799 7544 1.0000000E+003 1 n st Y N N N N N N N White, easy shoulder Unknown N 100 AAAAAAAAEGAAAAAA 2450508 2450562 10645 1.0000000E+003 1 bar N N N N N N N N Villages go more. Mi Unknown N - 101 AAAAAAAAFGAAAAAA ? 2450811 ? ? ? ? ? N N ? ? ? N N Reports would make a Unknown N + 101 AAAAAAAAFGAAAAAA ? 2450811 ? ? ? N N N N Reports would make a Unknown N 102 AAAAAAAAGGAAAAAA 2450608 2450614 9614 1.0000000E+003 1 able Y N N N N N N N Medieval fires may r Unknown N 103 AAAAAAAAHGAAAAAA 2450499 2450525 12688 1.0000000E+003 1 pri N N N N N N N N Still ready eyes wil Unknown N 104 AAAAAAAAIGAAAAAA 2450276 2450293 2461 1.0000000E+003 1 ese N N N N N N N N Years will sum alway Unknown N @@ -938,8 +938,8 @@ P_PROMO_SK P_PROMO_ID P_START_DATE_SK P_END_DATE_SK P_ITEM_SK 193 AAAAAAAABMAAAAAA 2450259 2450309 8780 1.0000000E+003 1 pri Y N N N N N N N Terms should not kno Unknown N 194 AAAAAAAACMAAAAAA 2450664 2450723 289 1.0000000E+003 1 ese Y N N N N N N N Cultural deaths coul Unknown N 195 AAAAAAAADMAAAAAA 2450543 2450601 5167 1.0000000E+003 1 anti N N N N N N N N Conventions make imm Unknown N - 196 AAAAAAAAEMAAAAAA 2450315 ? ? ? 1 ? Y ? ? N N ? ? N Able patients will l ? ? - 197 AAAAAAAAFMAAAAAA 2450299 ? 10550 ? 1 ? ? N N N N ? ? ? ? Unknown ? + 196 AAAAAAAAEMAAAAAA 2450315 ? ? ? 1 Y N N N Able patients will l + 197 AAAAAAAAFMAAAAAA 2450299 ? 10550 ? 1 N N N N Unknown 198 AAAAAAAAGMAAAAAA 2450617 2450670 15439 1.0000000E+003 1 eing Y N N N N N N N German, crude expect Unknown N 199 AAAAAAAAHMAAAAAA 2450772 2450825 8587 1.0000000E+003 1 n st N N N N N N N N Powerful, considerab Unknown N 200 AAAAAAAAIMAAAAAA 2450215 2450237 12784 1.0000000E+003 1 bar N N N N N N N N Arts attend. Further Unknown N @@ -956,7 +956,7 @@ P_PROMO_SK P_PROMO_ID P_START_DATE_SK P_END_DATE_SK P_ITEM_SK 211 AAAAAAAADNAAAAAA 2450646 2450649 7508 1.0000000E+003 1 ought Y N N N N N N N Public, running ladi Unknown N 212 AAAAAAAAENAAAAAA 2450781 2450834 10768 1.0000000E+003 1 able N N N N N N N N Good decades take. P Unknown N 213 AAAAAAAAFNAAAAAA 2450403 2450415 6350 1.0000000E+003 1 pri Y N N N N N N N However powerful cri Unknown N - 214 AAAAAAAAGNAAAAAA ? 2450779 ? 1.0000000E+003 ? ? N ? ? ? ? ? N N Pupils will meet mil Unknown N + 214 AAAAAAAAGNAAAAAA ? 2450779 ? 1.0000000E+003 ? N N N Pupils will meet mil Unknown N 215 AAAAAAAAHNAAAAAA 2450235 2450252 5840 1.0000000E+003 1 anti Y N N N N N N N Apparent, experience Unknown N 216 AAAAAAAAINAAAAAA 2450885 2450898 13504 1.0000000E+003 1 cally N N N N N N N N Large, necessary car Unknown N 217 AAAAAAAAJNAAAAAA 2450265 2450316 466 1.0000000E+003 1 ation N N N N N N N N Also splendid terms Unknown N @@ -967,7 +967,7 @@ P_PROMO_SK P_PROMO_ID P_START_DATE_SK P_END_DATE_SK P_ITEM_SK 222 AAAAAAAAONAAAAAA 2450336 2450378 122 1.0000000E+003 1 able Y N N N N N N N Ways help. Everywher Unknown N 223 AAAAAAAAPNAAAAAA 2450671 2450721 11452 1.0000000E+003 1 pri N N N N N N N N Earlier traditional Unknown N 224 AAAAAAAAAOAAAAAA 2450135 2450140 13534 1.0000000E+003 1 ese Y N N N N N N N Then general prefere Unknown N - 225 AAAAAAAABOAAAAAA ? 2450434 6514 1.0000000E+003 ? anti ? N ? ? ? N ? ? Actual, very words u ? ? + 225 AAAAAAAABOAAAAAA ? 2450434 6514 1.0000000E+003 ? anti N N Actual, very words u 226 AAAAAAAACOAAAAAA 2450436 2450468 2848 1.0000000E+003 1 cally N N N N N N N N Signs must drink cer Unknown N 227 AAAAAAAADOAAAAAA 2450751 2450794 3697 1.0000000E+003 1 ation N N N N N N N N Arms can find social Unknown N 228 AAAAAAAAEOAAAAAA 2450337 2450354 11678 1.0000000E+003 1 eing Y N N N N N N N Most basic walls use Unknown N @@ -992,7 +992,7 @@ P_PROMO_SK P_PROMO_ID P_START_DATE_SK P_END_DATE_SK P_ITEM_SK 247 AAAAAAAAHPAAAAAA 2450100 2450145 12976 1.0000000E+003 1 ation Y N N N N N N N Therefore distinguis Unknown N 248 AAAAAAAAIPAAAAAA 2450824 2450853 12451 1.0000000E+003 1 eing N N N N N N N N Other facts ensure l Unknown N 249 AAAAAAAAJPAAAAAA 2450467 2450523 4195 1.0000000E+003 1 n st Y N N N N N N N Wrong materials shou Unknown N - 250 AAAAAAAAKPAAAAAA 2450606 ? ? ? 1 ? ? N N ? ? ? N N Private, political m Unknown ? + 250 AAAAAAAAKPAAAAAA 2450606 ? ? ? 1 N N N N Private, political m Unknown 251 AAAAAAAALPAAAAAA 2450150 2450181 9187 1.0000000E+003 1 ought N N N N N N N N Other quarters used Unknown N 252 AAAAAAAAMPAAAAAA 2450290 2450319 1292 1.0000000E+003 1 able N N N N N N N N Small characters cou Unknown N 253 AAAAAAAANPAAAAAA 2450337 2450364 13351 1.0000000E+003 1 pri Y N N N N N N N Similar, true friend Unknown N
