Repository: incubator-hawq Updated Branches: refs/heads/master 00f6074bd -> b6dcecda2
HAWQ-742. Add hawq config test common library. Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/b6dcecda Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/b6dcecda Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/b6dcecda Branch: refs/heads/master Commit: b6dcecda2483ade6c9394a449455f6b17213eec9 Parents: 00f6074 Author: hzhang2 <[email protected]> Authored: Tue May 10 15:14:45 2016 +0800 Committer: hzhang2 <[email protected]> Committed: Wed May 11 09:11:23 2016 +0800 ---------------------------------------------------------------------- src/test/feature/lib/hawq-config.cpp | 170 +++++++++++++++++++ src/test/feature/lib/hawq-config.h | 44 +++++ src/test/feature/lib/psql.h | 5 +- src/test/feature/lib/string-util.cpp | 95 +++++++++++ src/test/feature/lib/string-util.h | 35 ++++ src/test/feature/lib/xml-parser.cpp | 266 ++++++++++++++++++++++++++++++ src/test/feature/lib/xml-parser.h | 63 +++++++ 7 files changed, 676 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b6dcecda/src/test/feature/lib/hawq-config.cpp ---------------------------------------------------------------------- diff --git a/src/test/feature/lib/hawq-config.cpp b/src/test/feature/lib/hawq-config.cpp new file mode 100644 index 0000000..d973328 --- /dev/null +++ b/src/test/feature/lib/hawq-config.cpp @@ -0,0 +1,170 @@ +#include "hawq-config.h" + +#include <fstream> +#include <iostream> +#include <set> +#include <string> +#include <vector> + +#include "command.h" +#include "psql.h" +#include "string-util.h" +#include "xml-parser.h" + +bool HawqConfig::LoadFromConfigFile() { + const char *env = getenv("GPHOME"); + std::string confPath = env ? env : ""; + if (!confPath.empty()) { + confPath.append("/etc/hawq-site.xml"); + } else { + return false; + } + + xmlconf.reset(new XmlConfig(confPath.c_str())); + xmlconf->parse(); + return true; +} + +bool HawqConfig::getMaster(std::string &hostname, int &port) { + bool ret = LoadFromConfigFile(); + if(!ret){ + return false; + } + hostname = xmlconf->getString("hawq_master_address_host"); + port = xmlconf->getInt32("hawq_master_address_port"); + return true; +} + +void HawqConfig::getStandbyMaster(std::string &hostname, int &port) { + PSQLQueryResult result = psql.getQueryResult( + "select hostname, port from gp_segment_configuration where role ='s'"); + std::vector<std::vector<std::string> > table = result.getRows(); + if (table.size() > 0) { + hostname = table[0][0]; + std::string portStr = table[0][1]; + port = std::stoi(portStr); + } +} + +void HawqConfig::getTotalSegments(std::vector<std::string> &hostname, + std::vector<int> &port) { + PSQLQueryResult result = psql.getQueryResult( + "select hostname, port from gp_segment_configuration where role ='p'"); + std::vector<std::vector<std::string> > table = result.getRows(); + for (int i = 0; i < table.size(); i++) { + hostname.push_back(table[i][0]); + std::string portStr = table[i][1]; + port.push_back(std::stoi(portStr)); + } +} + +void HawqConfig::getSlaves(std::vector<std::string> &hostname) { + + std::ifstream inFile; + char* GPHOME = getenv("GPHOME"); + if (GPHOME == nullptr) { + return; + } + std::string slaveFile(GPHOME); + slaveFile.append("/etc/slaves"); + inFile.open(slaveFile.c_str()); + std::string line; + while (std::getline(inFile, line)) { + hostname.push_back(line); + } + inFile.close(); +} + +void HawqConfig::getUpSegments(std::vector<std::string> &hostname, + std::vector<int> &port) { + PSQLQueryResult result = + psql.getQueryResult( + "select hostname, port from gp_segment_configuration where role = 'p' and status = 'u'"); + std::vector<std::vector<std::string> > table = result.getRows(); + + if (table.size() > 0) { + hostname.push_back(table[0][0]); + std::string portStr = table[0][1]; + port.push_back(std::stoi(portStr)); + } +} + +void HawqConfig::getDownSegments(std::vector<std::string> &hostname, + std::vector<int> &port) { + PSQLQueryResult result = + psql.getQueryResult( + "select hostname, port from gp_segment_configuration where role = 'p' and status != 'u'"); + std::vector<std::vector<std::string> > table = result.getRows(); + + if (table.size() > 0) { + hostname.push_back(table[0][0]); + std::string portStr = table[0][1]; + port.push_back(std::stoi(portStr)); + } +} + +std::string HawqConfig::getGucValue(std::string gucName) { + std::string cmd = "hawq config -s "; + cmd.append(gucName); + Command c(cmd); + std::string result = c.run().getResultOutput(); + std::string gucValue = ""; + std::vector<std::string> lines = StringUtil::split(result, '\n'); + // second line is value output. + if (lines.size() >= 2) { + std::string valueLine = lines[1]; + int pos = valueLine.find_first_of(':'); + std::string value = valueLine.substr(pos + 1); + gucValue = StringUtil::trim(value); + } + return gucValue; +} + +std::string HawqConfig::setGucValue(std::string gucName, std::string gucValue) { + std::string cmd = "hawq config -c "; + cmd.append(gucName); + cmd.append(" -v "); + cmd.append(gucValue); + Command c(cmd); + std::string ret = c.run().getResultOutput(); + return ret; +} + +bool HawqConfig::isMasterMirrorSynchronized() { + PSQLQueryResult result = psql.getQueryResult( + "select summary_state from gp_master_mirroring"); + if (result.getRows().size() > 0) { + std::string syncInfo = result.getData(0, 0); + syncInfo = StringUtil::trim(syncInfo); + if (syncInfo == "Synchronized") { + return true; + } else { + return false; + } + } + return false; +} + +bool HawqConfig::isMultinodeMode() { + PSQLQueryResult result = psql.getQueryResult( + "select hostname from gp_segment_configuration"); + std::vector<std::vector<std::string> > table = result.getRows(); + + std::set<std::string> hostnameMap; + for (int i = 0; i < table.size(); i++) { + std::string hostname2 = table[i][0]; + if (hostname2 == "localhost") { + char hostnamestr[256]; + gethostname(hostnamestr, 255); + hostname2.assign(hostnamestr); + } + if (hostnameMap.find(hostname2) == hostnameMap.end()) { + hostnameMap.insert(hostname2); + } + } + if (hostnameMap.size() <= 1) { + return false; + } else { + return true; + } +} http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b6dcecda/src/test/feature/lib/hawq-config.h ---------------------------------------------------------------------- diff --git a/src/test/feature/lib/hawq-config.h b/src/test/feature/lib/hawq-config.h new file mode 100644 index 0000000..a32cc0c --- /dev/null +++ b/src/test/feature/lib/hawq-config.h @@ -0,0 +1,44 @@ +#ifndef SRC_TEST_FEATURE_LIB_HAWQ_CONFIG_H_ +#define SRC_TEST_FEATURE_LIB_HAWQ_CONFIG_H_ + +#include "psql.h" +#include "xml-parser.h" + +class HawqConfig { + public: + HawqConfig(const std::string& user = "gpadmin", + const std::string& password = "", const std::string& db = "postgres", + const std::string& host = "localhost", const std::string& port = "5432") : + psql(db, host, port, user, password) { + std::string masterHostname = ""; + int masterPort = 0; + bool ret = getMaster(masterHostname, masterPort); + if (ret) { + std::string masterPortStr = std::to_string(masterPort); + psql.setHost(masterHostname); + psql.setPort(masterPortStr); + } + } + ~HawqConfig() { + } + + bool LoadFromConfigFile(); + bool getMaster(std::string &hostname, int &port); + void getStandbyMaster(std::string &hostname, int &port); + void getTotalSegments(std::vector<std::string> &hostname, + std::vector<int> &port); + void getSlaves(std::vector<std::string> &hostname); + void getUpSegments(std::vector<std::string> &hostname, + std::vector<int> &port); + void getDownSegments(std::vector<std::string> &hostname, + std::vector<int> &port); + std::string getGucValue(std::string gucName); + std::string setGucValue(std::string gucName, std::string gucValue); + bool isMasterMirrorSynchronized(); + bool isMultinodeMode(); + private: + std::unique_ptr<XmlConfig> xmlconf; + PSQL psql; +}; + +#endif /* SRC_TEST_FEATURE_LIB_HAWQ_CONFIG_H_ */ http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b6dcecda/src/test/feature/lib/psql.h ---------------------------------------------------------------------- diff --git a/src/test/feature/lib/psql.h b/src/test/feature/lib/psql.h index fb6d64c..75bee61 100644 --- a/src/test/feature/lib/psql.h +++ b/src/test/feature/lib/psql.h @@ -1,10 +1,11 @@ #ifndef __PSQL_H__ #define __PSQL_H__ -#include "libpq-fe.h" + #include <vector> -#include "command.h" +#include "command.h" +#include "libpq-fe.h" class PSQLQueryResult { public: http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b6dcecda/src/test/feature/lib/string-util.cpp ---------------------------------------------------------------------- diff --git a/src/test/feature/lib/string-util.cpp b/src/test/feature/lib/string-util.cpp new file mode 100644 index 0000000..4474c80 --- /dev/null +++ b/src/test/feature/lib/string-util.cpp @@ -0,0 +1,95 @@ +#include "string-util.h" + +#include <algorithm> +#include <cassert> +#include <regex> +#include <string> + +bool StringUtil::iequals(const std::string &str1, const std::string &str2) { + if (str1.size() != str2.size()) { + return false; + } + for (std::string::const_iterator c1 = str1.begin(), c2 = str2.begin(); + c1 != str1.end(); ++c1, ++c2) { + if (tolower(*c1) != tolower(*c2)) { + return false; + } + } + return true; +} + +void StringUtil::replace(std::string *subject, const std::string &search, + const std::string &replace) { + size_t pos = 0; + while ((pos = subject->find(search, pos)) != std::string::npos) { + subject->replace(pos, search.length(), replace); + pos += replace.length(); + } +} + +void StringUtil::toLower(std::string *str) { + assert(str != nullptr); + + std::transform(str->begin(), str->end(), str->begin(), ::tolower); +} + +std::string StringUtil::lower(const std::string &str) { + std::string result; + + for (std::string::const_iterator iter = str.begin(); iter != str.end(); + iter++) { + char c = tolower(*iter); + result.append(&c, sizeof(char)); + } + + return std::move(result); +} + +std::string &StringUtil::trim(std::string &s) { // NOLINT + if (s.empty()) { + return s; + } + s.erase(0, s.find_first_not_of(" ")); + s.erase(s.find_last_not_of(" ") + 1); + return s; +} + +std::string &StringUtil::trimNewLine(std::string &s) { // NOLINT + s.erase(std::remove(s.begin(), s.end(), '\n'), s.end()); + return s; +} + +std::vector<std::string> StringUtil::split(const std::string &s, + char delimiter) { + std::vector<std::string> v; + + std::string::size_type i = 0; + std::string::size_type j = s.find(delimiter); + if (j == std::string::npos) { + v.push_back(s); + } + while (j != std::string::npos) { + v.push_back(s.substr(i, j - i)); + i = ++j; + j = s.find(delimiter, j); + + if (j == std::string::npos) v.push_back(s.substr(i, s.length())); + } + return v; +} + +std::string StringUtil::regexReplace(std::string *subject, + const std::string &pattern, + const std::string &replace) { + const std::regex regPattern(pattern); + return std::regex_replace(*subject, regPattern, replace); +} + +bool StringUtil::StartWith(const std::string &str, + const std::string &strStart) { + if (str.empty() || strStart.empty()) { + return false; + } + return str.compare(0, strStart.size(), strStart) == 0 ? true : false; +} + http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b6dcecda/src/test/feature/lib/string-util.h ---------------------------------------------------------------------- diff --git a/src/test/feature/lib/string-util.h b/src/test/feature/lib/string-util.h new file mode 100644 index 0000000..f593a8c --- /dev/null +++ b/src/test/feature/lib/string-util.h @@ -0,0 +1,35 @@ +#ifndef SRC_TEST_FEATURE_LIB_STRING_UTIL_H_ +#define SRC_TEST_FEATURE_LIB_STRING_UTIL_H_ + + +#include <iomanip> +#include <sstream> +#include <string> +#include <vector> + +class StringUtil { + public: + StringUtil() {} + ~StringUtil() {} + + static bool iequals(const std::string &str1, const std::string &str2); + static void replace(std::string *subject, const std::string &search, + const std::string &replace); + static std::string regexReplace(std::string *subject, + const std::string &pattern, + const std::string &replace); + static void toLower(std::string *str); + static std::string lower(const std::string &str); + static std::string &trim(std::string &s); // NOLINT + static std::string &trimNewLine(std::string &s); // NOLINT + static std::vector<std::string> split(const std::string &s, char delimiter); + static bool StartWith(const std::string &str, const std::string &strStart); + + template <typename T> + static std::string toStringWithPrecision(const T value, const int n) { + std::ostringstream out; + out << std::setiosflags(std::ios::fixed) << std::setprecision(n) << value; + return out.str(); + } +}; +#endif /* SRC_TEST_FEATURE_LIB_STRING_UTIL_H_ */ http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b6dcecda/src/test/feature/lib/xml-parser.cpp ---------------------------------------------------------------------- diff --git a/src/test/feature/lib/xml-parser.cpp b/src/test/feature/lib/xml-parser.cpp new file mode 100644 index 0000000..7d19721 --- /dev/null +++ b/src/test/feature/lib/xml-parser.cpp @@ -0,0 +1,266 @@ +#include "xml-parser.h" + +#include <limits> + +XmlConfig::XmlConfig(const char *p) : + path(p) { + parse(); +} + +void XmlConfig::parse() { + // the result document tree + xmlDocPtr doc; + LIBXML_TEST_VERSION kv + .clear(); + + if (access(path.c_str(), R_OK)) { + return; + } + + // parse the file + doc = xmlReadFile(path.c_str(), nullptr, 0); + if (doc == nullptr) { + return; + } + try { + // printf("ffff3\n"); + readConfigItems(doc); + // printf("ffff4\n"); + xmlFreeDoc(doc); + } catch (...) { + xmlFreeDoc(doc); + // LOG_ERROR(ERRCODE_INTERNAL_ERROR, "libxml internal error"); + } +} + +void XmlConfig::readConfigItems(xmlDocPtr doc) { + xmlNodePtr root, curNode; + root = xmlDocGetRootElement(doc); + if (root == nullptr || strcmp((const char *) root->name, "configuration")) { + return ; + } + + // for each property + + for (curNode = root->children; curNode != nullptr; curNode = curNode->next) { + if (curNode->type != XML_ELEMENT_NODE) { + continue; + } + + if (strcmp((const char *) curNode->name, "property")) { + return; + } + + readConfigItem(curNode->children); + } +} + +void XmlConfig::readConfigItem(xmlNodePtr root) { + std::string key, value, scope; + xmlNodePtr curNode; + bool hasName = false, hasValue = false, hasScope = false; + for (curNode = root; curNode != nullptr; curNode = curNode->next) { + if (curNode->type != XML_ELEMENT_NODE) { + continue; + } + + if (!hasName && !strcmp((const char *) curNode->name, "name")) { + if (curNode->children != nullptr + && XML_TEXT_NODE == curNode->children->type) { + key = (const char *) curNode->children->content; + hasName = true; + } + } else if (!hasValue && !strcmp((const char *) curNode->name, "value")) { + if (curNode->children != nullptr + && XML_TEXT_NODE == curNode->children->type) { + value = (const char *) curNode->children->content; + hasValue = true; + } + } else { + continue; + } + } + + if (hasName && hasValue) { + kv[key] = value; + return; + } else if (hasName && hasValue) { + return; + } else if (hasName && hasScope) { + return; + } else if (hasName) { + return; + } +} + +const char *XmlConfig::getString(const char *key) { + XmlConfigMapIterator it = kv.find(key); + + if (kv.end() == it) { + return ""; + } + + return it->second.c_str(); +} + +const char *XmlConfig::getString(const char *key, const char *def) { + XmlConfigMapIterator it = kv.find(key); + + if (kv.end() == it) { + return def; + } else { + return it->second.c_str(); + } +} + +const char *XmlConfig::getString(const std::string &key) { + return getString(key.c_str()); +} + +const char *XmlConfig::getString(const std::string &key, + const std::string &def) { + return getString(key.c_str(), def.c_str()); +} + +int64_t XmlConfig::getInt64(const char *key) { + int64_t retval; + XmlConfigMapIterator it = kv.find(key); + + if (kv.end() == it) { + return 0; + } + + retval = strToInt64(it->second.c_str()); + + return retval; +} + +int64_t XmlConfig::getInt64(const char *key, int64_t def) { + int64_t retval; + XmlConfigMapIterator it = kv.find(key); + + if (kv.end() == it) { + return def; + } + + retval = strToInt64(it->second.c_str()); + + return retval; +} + +int32_t XmlConfig::getInt32(const char *key) { + int32_t retval; + XmlConfigMapIterator it = kv.find(key); + + if (kv.end() == it) { + return 0; + } + + retval = strToInt32(it->second.c_str()); + + return retval; +} + +int32_t XmlConfig::getInt32(const char *key, int32_t def) { + int32_t retval; + XmlConfigMapIterator it = kv.find(key); + + if (kv.end() == it) { + return def; + } + + retval = strToInt32(it->second.c_str()); + + return retval; +} + +double XmlConfig::getDouble(const char *key) { + double retval; + XmlConfigMapIterator it = kv.find(key); + + if (kv.end() == it) { + return 0.0; + } + + retval = strToDouble(it->second.c_str()); + + return retval; +} + +double XmlConfig::getDouble(const char *key, double def) { + double retval; + XmlConfigMapIterator it = kv.find(key); + + if (kv.end() == it) { + return def; + } + + retval = strToDouble(it->second.c_str()); + + return retval; +} + +bool XmlConfig::getBool(const char *key) { + bool retval; + XmlConfigMapIterator it = kv.find(key); + + if (kv.end() == it) { + return false; + } + + retval = strToBool(it->second.c_str()); + + return retval; +} + +bool XmlConfig::getBool(const char *key, bool def) { + bool retval; + XmlConfigMapIterator it = kv.find(key); + + if (kv.end() == it) { + return def; + } + + retval = strToBool(it->second.c_str()); + + return retval; +} + +int64_t XmlConfig::strToInt64(const char *str) { + int64_t retval; + char *end = nullptr; + + retval = strtoll(str, &end, 0); + + return retval; +} + +int32_t XmlConfig::strToInt32(const char *str) { + int32_t retval; + char *end = nullptr; + retval = strtoll(str, &end, 0); + + return retval; +} + +bool XmlConfig::strToBool(const char *str) { + bool retval = false; + + if (!strcasecmp(str, "true") || !strcmp(str, "1")) { + retval = true; + } else if (!strcasecmp(str, "false") || !strcmp(str, "0")) { + retval = false; + } else { + return false; + } + + return retval; +} + +double XmlConfig::strToDouble(const char *str) { + double retval; + char *end = nullptr; + retval = strtod(str, &end); + + return retval; +} http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b6dcecda/src/test/feature/lib/xml-parser.h ---------------------------------------------------------------------- diff --git a/src/test/feature/lib/xml-parser.h b/src/test/feature/lib/xml-parser.h new file mode 100644 index 0000000..b2e3497 --- /dev/null +++ b/src/test/feature/lib/xml-parser.h @@ -0,0 +1,63 @@ +#ifndef SRC_TEST_FEATURE_LIB_XML_PARSER_H_ +#define SRC_TEST_FEATURE_LIB_XML_PARSER_H_ + + +#include <string> +#include <unordered_map> + +#include "libxml/parser.h" + +typedef std::unordered_map<std::string, std::string> XmlConfigMap; +typedef std::unordered_map<std::string, std::string>::const_iterator + XmlConfigMapIterator; + +class XmlConfig { + public: + explicit XmlConfig(const char *p); + + // parse the configuration file + void parse(); + + // @param key The key of the configuration item + // @ def The default value + // @ return The value of configuration item + const char *getString(const char *key); + + const char *getString(const char *key, const char *def); + + const char *getString(const std::string &key); + + const char *getString(const std::string &key, const std::string &def); + + int64_t getInt64(const char *key); + + int64_t getInt64(const char *key, int64_t def); + + int32_t getInt32(const char *key); + + int32_t getInt32(const char *key, int32_t def); + + double getDouble(const char *key); + + double getDouble(const char *key, double def); + + bool getBool(const char *key); + + bool getBool(const char *key, bool def); + + XmlConfigMap *getConfigMap() { return &kv; } + + private: + std::string path; + XmlConfigMap kv; // key2Value + + void readConfigItems(xmlDocPtr doc); + void readConfigItem(xmlNodePtr root); + int64_t strToInt64(const char *str); + int32_t strToInt32(const char *str); + bool strToBool(const char *str); + double strToDouble(const char *str); +}; + + +#endif /* SRC_TEST_FEATURE_LIB_XML_PARSER_H_ */
