Repository: incubator-hawq Updated Branches: refs/heads/master 09cf75bbf -> 820d97404
HAWQ-1510. Add TDE-related functionality into hawq init command Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/820d9740 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/820d9740 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/820d9740 Branch: refs/heads/master Commit: 820d9740449be212fed7d5d0397554c3660efc07 Parents: 09cf75b Author: interma <[email protected]> Authored: Mon Aug 14 16:30:45 2017 +0800 Committer: Wen Lin <[email protected]> Committed: Wed Aug 16 16:20:06 2017 +0800 ---------------------------------------------------------------------- src/bin/gpcheckhdfs/gpcheckhdfs.c | 38 ++++++++++++++++++++++++++-------- tools/bin/hawq_ctl | 19 +++++++++++++++-- tools/bin/hawqpylib/HAWQ_HELP.py | 1 + 3 files changed, 47 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/820d9740/src/bin/gpcheckhdfs/gpcheckhdfs.c ---------------------------------------------------------------------- diff --git a/src/bin/gpcheckhdfs/gpcheckhdfs.c b/src/bin/gpcheckhdfs/gpcheckhdfs.c index 685e79c..bf477a9 100644 --- a/src/bin/gpcheckhdfs/gpcheckhdfs.c +++ b/src/bin/gpcheckhdfs/gpcheckhdfs.c @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -69,8 +69,9 @@ int testHdfsConnect(hdfsFS * fs, const char * host, int port, /* * test whether the filepath which dfs_url defined in hdfs is existed or not. + * Note: if tde_keyname is not NULL, create an encryption zone on filepath by this key. */ -int testHdfsExisted(hdfsFS fs, const char * filepath, const char * dfscompleteurl); +int testHdfsExisted(hdfsFS fs, const char * filepath, const char * dfscompleteurl, const char * tde_keyname); /* * test whether basic file operation in hdfs is worked well or not @@ -85,21 +86,24 @@ int main(int argc, char * argv[]) { * argv[3]:krb status * argv[4]:krb service name * argv[5]:krb keytab file + * argv[6]:--with-tde + * argv[7]:tde encryption zone key name */ - if (argc < 3 || argc > 6 + if (argc < 3 || argc > 8 || ((argc == 4 || argc == 5) && 0 != strcasecmp(argv[3], "off") && 0 != strcasecmp(argv[3], "false"))) { fprintf(stderr, "ERROR: gpcheckhdfs parameter error, Please check your config file\n" - "\tDFS_NAME and DFS_URL are required, KERBEROS_SERVICENAME, KERBEROS_KEYFILE and " - "ENABLE_SECURE_FILESYSTEM are optional\n"); + "\tDFS_NAME and DFS_URL are required, KERBEROS_SERVICENAME, KERBEROS_KEYFILE " + "ENABLE_SECURE_FILESYSTEM and TDE_KeyName are optional\n"); return GPCHKHDFS_ERR; - } + } char * dfs_name = argv[1]; char * dfs_url = argv[2]; char * krbstatus = NULL; char * krb_srvname = NULL; char * krb_keytabfile = NULL; + char * tde_keyname = NULL; if (argc >= 4) { krbstatus = argv[3]; @@ -110,6 +114,13 @@ int main(int argc, char * argv[]) { krb_keytabfile = argv[5]; } + //get tde key name param + //to avoid the empty param's influences before it, use loop find + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "--with-tde") == 0 && i+1 < argc) + tde_keyname = argv[i+1]; + } + char * host = (char *)malloc(255 * sizeof(char)); char * port = (char *)malloc(5 * sizeof(char)); getHostAndPort(dfs_url, host, port); @@ -141,7 +152,7 @@ int main(int argc, char * argv[]) { * check hdfs's directory configured in dfs_url * such as sdw2:8020/gpsql/,the filepath is /gpsql * */ - int checkdirErrCode = testHdfsExisted(fs, filepath, dfscompleteurl); + int checkdirErrCode = testHdfsExisted(fs, filepath, dfscompleteurl, tde_keyname); if (checkdirErrCode) { return checkdirErrCode; @@ -260,7 +271,7 @@ int testHdfsConnect(hdfsFS * fsptr, const char * host, int iPort, return 0; } -int testHdfsExisted(hdfsFS fs, const char * filepath, const char * dfscompleteurl) { +int testHdfsExisted(hdfsFS fs, const char * filepath, const char * dfscompleteurl, const char * tde_keyname) { int notExisted = hdfsExists(fs, filepath); if (notExisted) { @@ -282,6 +293,15 @@ int testHdfsExisted(hdfsFS fs, const char * filepath, const char * dfscompleteur } } + if (tde_keyname != NULL && strlen(tde_keyname) > 0) { + int ret = hdfsCreateEncryptionZone(fs, filepath, tde_keyname); + if (ret != 0) { + fprintf(stderr, "ERROR: create encryption zone on directory %s failed, key_name:%s.\n", + dfscompleteurl, tde_keyname); + return DFSDIR_ERR; + } + } + return 0; } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/820d9740/tools/bin/hawq_ctl ---------------------------------------------------------------------- diff --git a/tools/bin/hawq_ctl b/tools/bin/hawq_ctl index f47b6fe..d2f05c3 100755 --- a/tools/bin/hawq_ctl +++ b/tools/bin/hawq_ctl @@ -68,6 +68,7 @@ class HawqInit: self.max_connections = opts.max_connections self.shared_buffers = opts.shared_buffers self.default_hash_table_bucket_number = opts.default_hash_table_bucket_number + self.tde_keyname= opts.tde_keyname self.lock = threading.Lock() self.ignore_bad_hosts = opts.ignore_bad_hosts self._get_config() @@ -170,11 +171,21 @@ class HawqInit: local_ssh(cmd, logger) def check_hdfs_path(self): - cmd = "%s/bin/gpcheckhdfs hdfs %s %s %s %s" % \ - (self.GPHOME, self.dfs_url, self.enable_secure_filesystem, self.krb_srvname, self.krb_server_keyfile) + if self.tde_keyname: + cmd = "%s/bin/gpcheckhdfs hdfs %s %s %s %s --with-tde %s" % \ + (self.GPHOME, self.dfs_url, self.enable_secure_filesystem, \ + self.krb_srvname, self.krb_server_keyfile, self.tde_keyname) + else: + cmd = "%s/bin/gpcheckhdfs hdfs %s %s %s %s" % \ + (self.GPHOME, self.dfs_url, self.enable_secure_filesystem, \ + self.krb_srvname, self.krb_server_keyfile) + logger.info("Check if hdfs path is available") logger.debug("Check hdfs: %s" % cmd) check_return_code(local_ssh(cmd, logger, warning = True), logger, "Check hdfs failed, please verify your hdfs settings") + if self.tde_keyname: + logger.info("Create encryption zone successfully, key_name:%s" % (self.tde_keyname)) + def set_new_standby_host(self): if self.new_standby_hostname == self.master_host_name: @@ -1429,6 +1440,10 @@ def create_parser(): type="int", dest="default_hash_table_bucket_number", help="Sets maximum number of virtual segments per node") + parser.add_option("--tde_keyname", + dest="tde_keyname", + default="", + help="Sets the encryption zone key(EZK) name for the hawq directory(hawq_dfs_url)") parser.add_option("--locale", dest="hawq_locale", default="en_US.utf8", http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/820d9740/tools/bin/hawqpylib/HAWQ_HELP.py ---------------------------------------------------------------------- diff --git a/tools/bin/hawqpylib/HAWQ_HELP.py b/tools/bin/hawqpylib/HAWQ_HELP.py index b544629..d427254 100755 --- a/tools/bin/hawqpylib/HAWQ_HELP.py +++ b/tools/bin/hawqpylib/HAWQ_HELP.py @@ -112,6 +112,7 @@ The "options" are: --max_connections Sets the max_connections for formatting hawq database. --shared_buffers Sets the shared_buffers for initializing hawq. --bucket_number Sets the GUC value of default_hash_table_bucket_number. + --tde_keyname Sets the encryption zone key(EZK) name for the hawq directory(hawq_dfs_url). See 'hawq --help' for more information on other commands. """
