Repository: incubator-hawq Updated Branches: refs/heads/master a16030888 -> 030491b89
HAWQ-845. Parameterize kerberos principal name for HAWQ Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/030491b8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/030491b8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/030491b8 Branch: refs/heads/master Commit: 030491b896875e574a2563d77c92fbfd1503d5bf Parents: a160308 Author: Wen Lin <[email protected]> Authored: Wed Aug 31 14:10:51 2016 +0800 Committer: Wen Lin <[email protected]> Committed: Wed Aug 31 14:10:51 2016 +0800 ---------------------------------------------------------------------- .../resourcebroker_LIBYARN_proc.c | 18 +++++++----- src/bin/gpcheckhdfs/gpcheckhdfs.c | 30 +++++++++++++------- tools/bin/gpcheck | 3 +- tools/bin/hawq_ctl | 9 ++++-- tools/bin/hawqfilespace | 7 ++++- 5 files changed, 45 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/030491b8/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN_proc.c ---------------------------------------------------------------------- diff --git a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN_proc.c b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN_proc.c index ee85d2c..9ea8510 100644 --- a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN_proc.c +++ b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN_proc.c @@ -488,6 +488,10 @@ int loadParameters(void) /* If kerberos is enable, fetch the principal from ticket cache file. */ if (enable_secure_filesystem) { + if (!login()) + { + elog(WARNING, "Resource broker failed to refresh kerberos ticket."); + } YARNUser = ExtractPrincipalFromTicketCache(krb5_ccname); YARNUserShouldFree = true; } @@ -503,13 +507,13 @@ int loadParameters(void) "Scheduler server %s:%s " "Queue %s Application name %s, " "by user:%s", - YARNServer.Str, - YARNPort.Str, - YARNSchedulerServer.Str, - YARNSchedulerPort.Str, - YARNQueueName.Str, - YARNAppName.Str, - YARNUser); + YARNServer.Str, + YARNPort.Str, + YARNSchedulerServer.Str, + YARNSchedulerPort.Str, + YARNQueueName.Str, + YARNAppName.Str, + YARNUser); exit: if ( res != FUNC_RETURN_OK ) { elog(WARNING, "YARN mode resource broker failed to load YARN connection arguments."); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/030491b8/src/bin/gpcheckhdfs/gpcheckhdfs.c ---------------------------------------------------------------------- diff --git a/src/bin/gpcheckhdfs/gpcheckhdfs.c b/src/bin/gpcheckhdfs/gpcheckhdfs.c index 935b076..685e79c 100644 --- a/src/bin/gpcheckhdfs/gpcheckhdfs.c +++ b/src/bin/gpcheckhdfs/gpcheckhdfs.c @@ -63,7 +63,9 @@ void getHostAndPort(const char * dfs_url, char * host, char * port); /* * test whether hdfs can be connected while kerberos is on or not */ -int testHdfsConnect(hdfsFS * fs, const char * host, int port, const char * krbstatus, const char * krb_keytabfile); +int testHdfsConnect(hdfsFS * fs, const char * host, int port, + const char * krbstatus, const char * krb_srvname, + const char * krb_keytabfile); /* * test whether the filepath which dfs_url defined in hdfs is existed or not. @@ -81,25 +83,31 @@ int main(int argc, char * argv[]) { * argv[1]:dfs_name * argv[2]:dfs_url * argv[3]:krb status - * argv[4]:krb keytab file + * argv[4]:krb service name + * argv[5]:krb keytab file */ - if (argc < 3 || argc > 5 || (argc == 4 && 0 != strcasecmp(argv[3],"off") && 0 != strcasecmp(argv[3],"false"))) { - fprintf(stderr, "ERROR: gpcheckhdfs parameter error, Please check your config file\n" - "\tDFS_NAME and DFS_URL are required, KERBEROS_KEYFILE and ENABLE_SECURE_FILESYSTEM are optional\n"); + if (argc < 3 || argc > 6 + || ((argc == 4 || argc == 5) && 0 != strcasecmp(argv[3], "off") && 0 != strcasecmp(argv[3], "false"))) { + fprintf(stderr, + "ERROR: gpcheckhdfs parameter error, Please check your config file\n" + "\tDFS_NAME and DFS_URL are required, KERBEROS_SERVICENAME, KERBEROS_KEYFILE and " + "ENABLE_SECURE_FILESYSTEM are optional\n"); return GPCHKHDFS_ERR; } char * dfs_name = argv[1]; char * dfs_url = argv[2]; char * krbstatus = NULL; + char * krb_srvname = NULL; char * krb_keytabfile = NULL; if (argc >= 4) { krbstatus = argv[3]; } - if (argc >= 5) { - krb_keytabfile = argv[4]; + if (argc >= 6) { + krb_srvname = argv[4]; + krb_keytabfile = argv[5]; } char * host = (char *)malloc(255 * sizeof(char)); @@ -113,7 +121,8 @@ int main(int argc, char * argv[]) { } hdfsFS fs; - int connErrCode = testHdfsConnect(&fs, host, iPort, krbstatus, krb_keytabfile); + int connErrCode = testHdfsConnect(&fs, host, iPort, krbstatus, krb_srvname, + krb_keytabfile); if (connErrCode) { return connErrCode; @@ -189,7 +198,9 @@ int testHdfsOperateFile(hdfsFS fs, const char * filepath, const char * dfscomple return 0; } -int testHdfsConnect(hdfsFS * fsptr, const char * host, int iPort, const char * krbstatus, const char * krb_keytabfile) { +int testHdfsConnect(hdfsFS * fsptr, const char * host, int iPort, + const char * krbstatus, const char * krb_srvname, + const char * krb_keytabfile) { struct hdfsBuilder * builder = hdfsNewBuilder(); hdfsBuilderSetNameNode(builder, host); @@ -198,7 +209,6 @@ int testHdfsConnect(hdfsFS * fsptr, const char * host, int iPort, const char * k if (NULL != krbstatus && NULL != krb_keytabfile && (!strcasecmp(krbstatus, "on") || !strcasecmp(krbstatus, "true"))) { //Kerberos if On - char * krb_srvname = "postgres"; char * krb5_ccname = "/tmp/postgres.ccname"; char cmd[1024]; snprintf(cmd, sizeof(cmd), "kinit -k -t %s -c %s %s", http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/030491b8/tools/bin/gpcheck ---------------------------------------------------------------------- diff --git a/tools/bin/gpcheck b/tools/bin/gpcheck index e9da85a..3d45f18 100755 --- a/tools/bin/gpcheck +++ b/tools/bin/gpcheck @@ -976,9 +976,8 @@ def testHDFSConfig(host): checkFailed(host.hostname, "YARN configuration: %s not defined" % item) # Check yarn kerberos properties - #yarn_kerberos_check_list = ['hadoop.proxyuser.yarn.groups', 'hadoop.proxyuser.yarn.hosts', 'hadoop.proxyuser.postgres.hosts', 'hadoop.proxyuser.postgres.groups'] if yarn_enabled and options.kerberos: - yarn_kerberos_check_list = ['yarn.nodemanager.keytab', 'yarn.nodemanager.principal','hadoop.proxyuser.postgres.groups', \ + yarn_kerberos_check_list = ['yarn.nodemanager.keytab', 'yarn.nodemanager.principal', \ 'yarn.resourcemanager.keytab', 'yarn.resourcemanager.principal'] for item in yarn_kerberos_check_list: if item in actual_config: http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/030491b8/tools/bin/hawq_ctl ---------------------------------------------------------------------- diff --git a/tools/bin/hawq_ctl b/tools/bin/hawq_ctl index bcfadc7..81296d7 100755 --- a/tools/bin/hawq_ctl +++ b/tools/bin/hawq_ctl @@ -128,6 +128,11 @@ class HawqInit: else: self.krb_server_keyfile = '' + if 'krb_srvname' in self.hawq_dict: + self.krb_srvname = self.hawq_dict['krb_srvname'] + else: + self.krb_srvname = 'postgres' + def _write_config(self): configFile = "%s/etc/_mgmt_config" % self.GPHOME # Clean configFile while the first write in. @@ -165,8 +170,8 @@ class HawqInit: local_ssh(cmd, logger) def check_hdfs_path(self): - cmd = "%s/bin/gpcheckhdfs hdfs %s %s %s" % \ - (self.GPHOME, self.dfs_url, self.enable_secure_filesystem, self.krb_server_keyfile) + cmd = "%s/bin/gpcheckhdfs hdfs %s %s %s %s" % \ + (self.GPHOME, self.dfs_url, self.enable_secure_filesystem, self.krb_srvname, self.krb_server_keyfile) logger.info("Check if hdfs path is available") logger.debug("Check hdfs: %s" % cmd) check_return_code(local_ssh(cmd, logger, warning = True), logger, "Check hdfs failed, please verify your hdfs settings") http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/030491b8/tools/bin/hawqfilespace ---------------------------------------------------------------------- diff --git a/tools/bin/hawqfilespace b/tools/bin/hawqfilespace index dd85aff..61dd423 100755 --- a/tools/bin/hawqfilespace +++ b/tools/bin/hawqfilespace @@ -261,6 +261,11 @@ def getdir(prompt, hosts=[], primary=None, shared=False, fsysn=None, db=None): rows = cursor.fetchall() krbstatus = rows[0][0] + cursor = dbconn.execSQL(db, "show krb_srvname") + db.commit() + rows = cursor.fetchall(); + krb_srvname = rows[0][0] + cursor = dbconn.execSQL(db, "show krb_server_keyfile") db.commit() rows = cursor.fetchall(); @@ -272,7 +277,7 @@ def getdir(prompt, hosts=[], primary=None, shared=False, fsysn=None, db=None): dfs_name = fsysn dfs_url = value - command = "gpcheckhdfs %s %s %s %s" % (dfs_name, dfs_url, krbstatus, krb_keyfile) + command = "gpcheckhdfs %s %s %s %s %s" % (dfs_name, dfs_url, krbstatus, krb_srvname, krb_keyfile) (status,returnCode) = commands.getstatusoutput(command) if status != 0: print command
