Repository: incubator-trafodion
Updated Branches:
refs/heads/master 13a4e372c -> 79552f0f1
TRAFODION-1940 better NATable statistics plus OID, cleanhb fixes
TRAFODION-1939 Remove OID from the list of reserved words
TRAFODION-1940 Improve NATable statistics
- Add the context of the NATable cache
to the natablecache() UDF
- Add the entry size to the
natablecacheentries() UDF
Also changed the ilh_cleanhb script to print a help text
on how to use it and require a parameter 0, 1 or 2.
Still leaving the old, misspelled directory for zookeeper
in the script until everyone has moved to a later version
that puts the zookeeper data under local_hadoop/data/zookeeper.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/985cecc4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/985cecc4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/985cecc4
Branch: refs/heads/master
Commit: 985cecc4675c0d2df9de4a04f95e8a34c9fbbd16
Parents: 119d471
Author: Hans Zeller <[email protected]>
Authored: Tue Apr 19 20:20:03 2016 +0000
Committer: Hans Zeller <[email protected]>
Committed: Tue Apr 19 20:20:03 2016 +0000
----------------------------------------------------------------------
core/sqf/sql/scripts/ilh_cleanhb | 31 +++++++++++++++++++++++--------
core/sql/arkcmp/CmpContext.h | 1 +
core/sql/arkcmp/NATableSt.cpp | 30 +++++++++++++++++++++---------
core/sql/common/ComResWords.cpp | 1 -
core/sql/optimizer/NATable.cpp | 23 ++++++++++++++++++++++-
core/sql/optimizer/NATable.h | 4 ++++
core/sql/parser/ParKeyWords.cpp | 1 -
core/sql/regress/core/EXPECTED037.SB | 9 +--------
core/sql/regress/hive/EXPECTED009 | 2 +-
core/sql/regress/hive/TEST009 | 2 +-
10 files changed, 74 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/985cecc4/core/sqf/sql/scripts/ilh_cleanhb
----------------------------------------------------------------------
diff --git a/core/sqf/sql/scripts/ilh_cleanhb b/core/sqf/sql/scripts/ilh_cleanhb
index 76c2617..42d5e20 100755
--- a/core/sqf/sql/scripts/ilh_cleanhb
+++ b/core/sqf/sql/scripts/ilh_cleanhb
@@ -61,11 +61,19 @@ function killMyHMaster {
}
-lv_clean_hb_data=0
-if [[ ! -z "$1" ]]; then
- lv_clean_hb_data=$1
+if [[ $# -eq 0 || $1 == "-h" ]]; then
+ echo " "
+ echo "Usage: $0 [ 0 | 1 | 2 ]"
+ echo " "
+ echo " 0: Clean HBase log files"
+ echo " 1: Clean HBase log files + zookeeper data"
+ echo " 2: Clean HBase log files + zookeeper data + /hbase in HDFS"
+ echo " "
+ exit 1
fi
+lv_clean_hb_data=$1
+
lv_nnpid=`getMyNameNode`
if [[ -z ${lv_nnpid} ]]; then
echo "NameNode is not up - please run swstarthadoop to start it or check
the Hadoop logs - exitting..."
@@ -85,11 +93,18 @@ killMyHMaster
# delete the zookeeper info
if [[ ${lv_clean_hb_data} -gt 0 ]]; then
- lv_zk_dir="${MY_SW_ROOT}/hdfs:"
- echo "Deleting the zookeeper directory: ${lv_zk_dir}"
- rm -rf ${lv_zk_dir}
- echo "Deleting the zookeeper directory: ${MY_ZOOKEEPER_DATA_DIR}"
- rm -rf ${MY_ZOOKEEPER_DATA_DIR}
+ # old value, which was actually a typo, remove this later, once
+ # everyone has switched over to the new value
+ old_lv_zk_dir="${MY_SW_ROOT}/hdfs:"
+ if [[ -d ${old_lv_zk_dir} ]]; then
+ echo "Deleting the zookeeper directory: ${old_lv_zk_dir}"
+ rm -rf ${old_lv_zk_dir}
+ elif [[ -d "${MY_ZOOKEEPER_DATA_DIR}" ]]; then
+ echo "Deleting the zookeeper directory: ${MY_ZOOKEEPER_DATA_DIR}"
+ rm -rf ${MY_ZOOKEEPER_DATA_DIR}
+ else
+ echo "Could not find zookeeper directory: ${MY_ZOOKEEPER_DATA_DIR}"
+ fi
fi
#delete the logs
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/985cecc4/core/sql/arkcmp/CmpContext.h
----------------------------------------------------------------------
diff --git a/core/sql/arkcmp/CmpContext.h b/core/sql/arkcmp/CmpContext.h
index f53648f..eaed39a 100644
--- a/core/sql/arkcmp/CmpContext.h
+++ b/core/sql/arkcmp/CmpContext.h
@@ -148,6 +148,7 @@ public :
{
return !(strncmp(name_, name, CMPCONTEXT_CLASS_NAME_LEN));
}
+ const char *getName() const { return name_; }
private :
char name_[CMPCONTEXT_CLASS_NAME_LEN]; // care upto CMPCONTEXT_CLASS_NAME_LEN
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/985cecc4/core/sql/arkcmp/NATableSt.cpp
----------------------------------------------------------------------
diff --git a/core/sql/arkcmp/NATableSt.cpp b/core/sql/arkcmp/NATableSt.cpp
index 5d3907d..f87ff96 100644
--- a/core/sql/arkcmp/NATableSt.cpp
+++ b/core/sql/arkcmp/NATableSt.cpp
@@ -68,7 +68,7 @@ SP_STATUS
NATableCacheStatStoredProcedure::sp_InputFormat(SP_FIELDDESC_STRUCT *i
return SP_SUCCESS;
}
-const Lng32 NUM_OF_OUTPUT = 6;
+const Lng32 NUM_OF_OUTPUT = 7;
SP_STATUS NATableCacheStatStoredProcedure::sp_NumOutputFields(
Lng32 *numFields,
@@ -88,6 +88,7 @@ SP_STATUS NATableCacheStatStoredProcedure::sp_OutputFormat(
SP_HANDLE spObj,
SP_ERROR_STRUCT *error)
{
+ strcpy(&((format++)->COLUMN_DEF[0]), "Context CHAR(8) CHARACTER SET
ISO88591");
strcpy(&((format++)->COLUMN_DEF[0]), "Num_lookups INT UNSIGNED");
strcpy(&((format++)->COLUMN_DEF[0]), "Num_cache_hits INT UNSIGNED");
strcpy(&((format++)->COLUMN_DEF[0]), "Num_entries INT UNSIGNED");
@@ -131,12 +132,13 @@ SP_STATUS NATableCacheStatStoredProcedure::sp_Process(
if(!it->getNext(stats))
return SP_SUCCESS;
- fFunc(0, outputData, sizeof(ULng32), &(stats.numLookups), 0);
- fFunc(1, outputData, sizeof(ULng32), &(stats.numCacheHits), 0);
- fFunc(2, outputData, sizeof(ULng32), &(stats.numEntries), 0);
- fFunc(3, outputData, sizeof(ULng32), &(stats.currentCacheSize), 0);
- fFunc(4, outputData, sizeof(ULng32), &(stats.highWaterMark), 0);
- fFunc(5, outputData, sizeof(ULng32), &(stats.maxCacheSize), 0);
+ fFunc(0, outputData, sizeof(stats.contextType), &(stats.contextType), 0);
+ fFunc(1, outputData, sizeof(ULng32), &(stats.numLookups), 0);
+ fFunc(2, outputData, sizeof(ULng32), &(stats.numCacheHits), 0);
+ fFunc(3, outputData, sizeof(ULng32), &(stats.numEntries), 0);
+ fFunc(4, outputData, sizeof(ULng32), &(stats.currentCacheSize), 0);
+ fFunc(5, outputData, sizeof(ULng32), &(stats.highWaterMark), 0);
+ fFunc(6, outputData, sizeof(ULng32), &(stats.maxCacheSize), 0);
return SP_MOREDATA;
}
@@ -192,7 +194,7 @@ SP_STATUS
SP_HANDLE spObj,
SP_ERROR_STRUCT *error)
{
- *numFields = 4;
+ *numFields = 5;
return SP_SUCCESS;
}
@@ -209,6 +211,7 @@ SP_STATUS
NATableCacheEntriesStoredProcedure::sp_OutputFormat(
strcpy(&((format++)->COLUMN_DEF[0]), "Catalog_name VARCHAR(128)
character set UTF8");
strcpy(&((format++)->COLUMN_DEF[0]), "Schema_name VARCHAR(128)
character set UTF8");
strcpy(&((format++)->COLUMN_DEF[0]), "Object_name VARCHAR(128)
character set UTF8");
+ strcpy(&((format++)->COLUMN_DEF[0]), "Entry_size INT");
return SP_SUCCESS;
}
@@ -251,6 +254,7 @@ SP_STATUS
NATableCacheEntriesStoredProcedure::sp_Process(SP_PROCESS_ACTION actio
fFunc(1,outputData,(Lng32)strlen(details.catalog),(void*)(details.catalog),1);
fFunc(2,outputData,(Lng32)strlen(details.schema),(void*)(details.schema),1);
fFunc(3,outputData,(Lng32)strlen(details.object),(void*)(details.object),1);
+ fFunc(4,outputData,(Lng32)sizeof(details.size), (void*)(&details.size),0);
return SP_MOREDATA;
}
@@ -351,7 +355,11 @@ NABoolean
NATableCacheStatsISPIterator::getNext(NATableCacheStats & stats)
//Only for remote tdm_arkcmp with 0 context
if(currCacheIndex_ == -1)
{
+ const char *cName = "ARKCMP";
+
ActiveSchemaDB()->getNATableDB()->getCacheStats(stats);
+ memset(stats.contextType, ' ', sizeof(stats.contextType));
+ memcpy(stats.contextType, cName, MINOF(sizeof(stats.contextType),
strlen(cName)));
currCacheIndex_ = -2;
return TRUE;
}
@@ -365,7 +373,11 @@ NABoolean
NATableCacheStatsISPIterator::getNext(NATableCacheStats & stats)
currCacheIndex_++;
return getNext(stats);
}
-
ctxInfos_[currCacheIndex_++]->getCmpContext()->getSchemaDB()->getNATableDB()->getCacheStats(stats);
+
ctxInfos_[currCacheIndex_]->getCmpContext()->getSchemaDB()->getNATableDB()->getCacheStats(stats);
+ const char *cName = ctxInfos_[currCacheIndex_++]->getName();
+
+ memset(stats.contextType, ' ', sizeof(stats.contextType));
+ memcpy(stats.contextType, cName, MINOF(sizeof(stats.contextType),
strlen(cName)));
return TRUE;
}
//all entries of all caches are fetched, we are done!
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/985cecc4/core/sql/common/ComResWords.cpp
----------------------------------------------------------------------
diff --git a/core/sql/common/ComResWords.cpp b/core/sql/common/ComResWords.cpp
index d54f38e..31cf2bd 100644
--- a/core/sql/common/ComResWords.cpp
+++ b/core/sql/common/ComResWords.cpp
@@ -255,7 +255,6 @@ const ComResWord ComResWords::resWords_[] = {
ComResWord("OCTET_LENGTH", ANS_|RESWORD_),
ComResWord("OF", ANS_|RESWORD_|MPWORD_),
ComResWord("OFF", POTANS_|RESWORD_),
- ComResWord("OID", POTANS_|RESWORD_),
ComResWord("OLD", POTANS_|RESWORD_|ALLOWOLDNEW_),
ComResWord("ON", ANS_|RESWORD_|MPWORD_),
ComResWord("ONLY", ANS_|RESWORD_),
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/985cecc4/core/sql/optimizer/NATable.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/NATable.cpp b/core/sql/optimizer/NATable.cpp
index 8727e2b..2d9e9e6 100644
--- a/core/sql/optimizer/NATable.cpp
+++ b/core/sql/optimizer/NATable.cpp
@@ -4924,6 +4924,8 @@ NATable::NATable(BindWA *bindWA,
resetAfterStatement_(FALSE),
hitCount_(0),
replacementCounter_(2),
+ sizeInCache_(0),
+ recentlyUsed_(TRUE),
tableConstructionHadWarnings_(FALSE),
isAnMPTableWithAnsiName_(FALSE),
isUMDTable_(FALSE),
@@ -5649,6 +5651,8 @@ NATable::NATable(BindWA *bindWA,
resetAfterStatement_(FALSE),
hitCount_(0),
replacementCounter_(2),
+ sizeInCache_(0),
+ recentlyUsed_(TRUE),
tableConstructionHadWarnings_(FALSE),
isAnMPTableWithAnsiName_(FALSE),
isUMDTable_(FALSE),
@@ -7811,6 +7815,8 @@ void NATableDB::getEntryDetails(
partLen = QNO.getObjectName().length();
strncpy(details.object, (char *)(QNO.getObjectName().data()), partLen );
details.object[partLen] = '\0';
+
+ details.size = object->sizeInCache_;
}
}
@@ -7944,6 +7950,7 @@ NATable * NATableDB::get(CorrName& corrName, BindWA *
bindWA,
//Heap used by the NATable object
NAMemory * naTableHeap = CmpCommon::statementHeap();
+ size_t allocSizeBefore = 0;
//if NATable caching is on check if this table is not already
//in the NATable cache. If it is in the cache create this NATable
@@ -7957,6 +7964,7 @@ NATable * NATableDB::get(CorrName& corrName, BindWA *
bindWA,
if (((NOT table) && cacheMetaData_ && useCache_) &&
corrName.isCacheable()){
naTableHeap = getHeap();
+ allocSizeBefore = naTableHeap->getAllocSize();
}
//if table is in cache tableInCache will be non-NULL
@@ -7968,6 +7976,9 @@ NATable * NATableDB::get(CorrName& corrName, BindWA *
bindWA,
(!isSQUtiDisplayExplain(corrName)) &&
(!isSQInternalStoredProcedure(corrName))
) {
+ // ------------------------------------------------------------------
+ // Create an NATable object for a Trafodion/HBase table
+ // ------------------------------------------------------------------
CmpSeabaseDDL cmpSBD((NAHeap *)CmpCommon::statementHeap());
@@ -8106,6 +8117,9 @@ NATable * NATableDB::get(CorrName& corrName, BindWA *
bindWA,
(!corrName.isSpecialTable()) &&
(!isSQInternalStoredProcedure(corrName))
) {
+ // ------------------------------------------------------------------
+ // Create an NATable object for a Hive table
+ // ------------------------------------------------------------------
if ( hiveMetaDB_ == NULL ) {
if (CmpCommon::getDefault(HIVE_USE_FAKE_TABLE_DESC) != DF_ON)
{
@@ -8179,6 +8193,9 @@ NATable * NATableDB::get(CorrName& corrName, BindWA *
bindWA,
}
} else
+ // ------------------------------------------------------------------
+ // Neither Trafodion nor Hive (probably dead code below)
+ // ------------------------------------------------------------------
table = new (naTableHeap)
NATable(bindWA, corrName, naTableHeap, inTableDescStruct);
@@ -8235,7 +8252,10 @@ NATable * NATableDB::get(CorrName& corrName, BindWA *
bindWA,
//if metadata caching is ON then adjust the size of the cache
//since we are adding an entry to the cache
if(cacheMetaData_)
- currentCacheSize_ = heap_->getAllocSize();
+ {
+ currentCacheSize_ = heap_->getAllocSize();
+ table->sizeInCache_ = currentCacheSize_ - allocSizeBefore;
+ }
//update the high watermark for caching statistics
if (currentCacheSize_ > highWatermarkCache_)
@@ -8831,6 +8851,7 @@ NATableDB::unmark_entries_marked_for_removal()
void NATableDB::getCacheStats(NATableCacheStats & stats)
{
+ memset(stats.contextType, ' ', sizeof(stats.contextType));
stats.numLookups = totalLookupsCount_;
stats.numCacheHits = totalCacheHits_;
stats.currentCacheSize = currentCacheSize_;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/985cecc4/core/sql/optimizer/NATable.h
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/NATable.h b/core/sql/optimizer/NATable.h
index e496242..5321a50 100644
--- a/core/sql/optimizer/NATable.h
+++ b/core/sql/optimizer/NATable.h
@@ -345,6 +345,7 @@ struct NATableEntryDetails {
char catalog[ComMAX_1_PART_INTERNAL_UTF8_NAME_LEN_IN_BYTES + 1]; // +1
for NULL byte
char schema[ComMAX_1_PART_INTERNAL_UTF8_NAME_LEN_IN_BYTES + 1];
char object[ComMAX_1_PART_INTERNAL_UTF8_NAME_LEN_IN_BYTES + 1];
+ int size;
};
// ***********************************************************************
@@ -1105,6 +1106,8 @@ private:
// Caching stats
UInt32 hitCount_;
UInt32 replacementCounter_;
+ Int64 sizeInCache_;
+ NABoolean recentlyUsed_;
COM_VERSION osv_;
COM_VERSION ofv_;
@@ -1168,6 +1171,7 @@ private:
#pragma warn(1506) // warning elimination
struct NATableCacheStats {
+ char contextType[8];
ULng32 numLookups;
ULng32 numCacheHits;
ULng32 currentCacheSize;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/985cecc4/core/sql/parser/ParKeyWords.cpp
----------------------------------------------------------------------
diff --git a/core/sql/parser/ParKeyWords.cpp b/core/sql/parser/ParKeyWords.cpp
index 9e52923..20c2ed7 100644
--- a/core/sql/parser/ParKeyWords.cpp
+++ b/core/sql/parser/ParKeyWords.cpp
@@ -722,7 +722,6 @@ ParKeyWord ParKeyWords::keyWords_[] = {
ParKeyWord("OFF", TOK_OFF, POTANS_|RESWORD_),
ParKeyWord("OFFLINE", TOK_OFFLINE, NONRESTOKEN_),
ParKeyWord("OFFSET", TOK_OFFSET, NONRESTOKEN_),
- ParKeyWord("OID", IDENTIFIER, POTANS_|RESWORD_),
ParKeyWord("OJ", TOK_OJ, NONRESTOKEN_),
ParKeyWord("OLD", TOK_OLD,
ANS_|RESWORD_|NONRESTOKEN_|ALLOWOLDNEW_),
ParKeyWord("ON", TOK_ON,
FIRST_|ANS_|RESWORD_|MPWORD_),
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/985cecc4/core/sql/regress/core/EXPECTED037.SB
----------------------------------------------------------------------
diff --git a/core/sql/regress/core/EXPECTED037.SB
b/core/sql/regress/core/EXPECTED037.SB
index 809df01..be117f4 100755
--- a/core/sql/regress/core/EXPECTED037.SB
+++ b/core/sql/regress/core/EXPECTED037.SB
@@ -786,14 +786,7 @@ SELECT OBJECT OBJECT from (values(0)) OBJECT(OBJECT);
>>-- Expect error [3128]
>>prepare s1 from SELECT OID OID from (values(0)) OID(OID);
-*** ERROR[3128] OID is a reserved word. It must be delimited by double-quotes
to be used as an identifier.
-
-*** ERROR[15001] A syntax error occurred at or before:
-SELECT OID OID from (values(0)) OID(OID);
- ^ (14 characters from start of SQL statement)
-
-*** ERROR[8822] The statement was not prepared.
-
+--- SQL command prepared.
>>
>>-- Expect error [3128]
>>prepare s1 from SELECT OLD OLD from (values(0)) OLD(OLD);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/985cecc4/core/sql/regress/hive/EXPECTED009
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/EXPECTED009
b/core/sql/regress/hive/EXPECTED009
index df03c8e..8d88288 100644
--- a/core/sql/regress/hive/EXPECTED009
+++ b/core/sql/regress/hive/EXPECTED009
@@ -471,7 +471,7 @@ T009T1
T009T2
--- SQL operation complete.
->>select * from table(natablecacheentries('user','local')) order by 2,3,4;
+>>select row_id, catalog_name, schema_name, object_name from
table(natablecacheentries('user','local')) order by 2,3,4;
ROW_ID CATALOG_NAME
SCHEMA_NAME
OBJECT_NAME
----------
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-----------------------------------------
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/985cecc4/core/sql/regress/hive/TEST009
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/TEST009 b/core/sql/regress/hive/TEST009
index 80bef86..1fe9641 100755
--- a/core/sql/regress/hive/TEST009
+++ b/core/sql/regress/hive/TEST009
@@ -170,7 +170,7 @@ sh sqlci -i "TEST009(select_stmts)";
select catalog_name, schema_name, object_name from
table(natablecacheentries('user','local')) order by 1,2,3;
drop external table t009t1 for hive.sch_t009.t009t1;
get tables in schema "_HV_SCH_T009_";
-select * from table(natablecacheentries('user','local')) order by 2,3,4;
+select row_id, catalog_name, schema_name, object_name from
table(natablecacheentries('user','local')) order by 2,3,4;
create external table t009t1 for hive.sch_t009.t009t1;
-- select should now succeed
