This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch hadoop-3.4.2 in repository https://gitbox.apache.org/repos/asf/doris-thirdparty.git
commit 97688a49f982c0dcad573deb9a4981d379a09641 Author: slothever <[email protected]> AuthorDate: Fri Jan 12 13:47:31 2024 +0800 [fix](tgt)hadoop kerberos support renew TGT by keytab (#173) --- .../main/java/org/apache/hadoop/fs/FileSystem.java | 13 +++ .../src/main/native/libhdfs/hdfs.c | 100 ++++++++++++++------- .../src/main/native/libhdfs/include/hdfs/hdfs.h | 9 ++ 3 files changed, 92 insertions(+), 30 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index 38ec6114517..ee6ac611e80 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -582,6 +582,19 @@ public abstract class FileSystem extends Configured }); } + public synchronized static FileSystem newInstanceFromKeytab(final URI uri, final Configuration conf, + final String principal, final String keytabPath) + throws IOException, InterruptedException { + UserGroupInformation.setConfiguration(conf); + UserGroupInformation.loginUserFromKeytab(principal, keytabPath); + return UserGroupInformation.getLoginUser().doAs(new PrivilegedExceptionAction<FileSystem>() { + @Override + public FileSystem run() throws IOException { + return newInstance(uri, conf); + } + }); + } + /** * Returns the FileSystem for this URI's scheme and authority. * The entire URI is passed to the FileSystem instance's initialize method. diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c index ea00393cfc7..b3ba27f8674 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c @@ -36,6 +36,7 @@ #define JMETHOD1(X, R) "(" X ")" R #define JMETHOD2(X, Y, R) "(" X Y ")" R #define JMETHOD3(X, Y, Z, R) "(" X Y Z")" R +#define JMETHOD4(X, Y, Z, A, R) "(" X Y Z A")" R #define KERBEROS_TICKET_CACHE_PATH "hadoop.security.kerberos.ticket.cache.path" @@ -514,6 +515,7 @@ struct hdfsBuilder { const char *kerbTicketCachePath; const char *kerb5ConfPath; const char *keyTabFile; + const char *kerbPrincipal; const char *userName; struct hdfsBuilderConfOpt *opts; struct hdfsBuilderConfFileOpt *fileOpts; @@ -634,6 +636,11 @@ void hdfsBuilderSetUserName(struct hdfsBuilder *bld, const char *userName) bld->userName = userName; } +void hdfsBuilderSetPrincipal(struct hdfsBuilder *bld, const char *kerbPrincipal) +{ + bld->kerbPrincipal = kerbPrincipal; +} + void hdfsBuilderSetKerbTicketCachePath(struct hdfsBuilder *bld, const char *kerbTicketCachePath) { @@ -771,7 +778,7 @@ hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld) { JNIEnv *env = 0; jobject jConfiguration = NULL, jFS = NULL, jURI = NULL, jCachePath = NULL; - jstring jURIString = NULL, jUserString = NULL, jKeyTabString = NULL; + jstring jURIString = NULL, jUserString = NULL, jPrincipalString = NULL, jKeyTabString = NULL; jvalue jVal; jthrowable jthr = NULL; char *cURI = 0, buf[512]; @@ -838,7 +845,7 @@ hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld) // fs = FileSytem#getLocal(conf); jthr = invokeMethod(env, &jVal, STATIC, NULL, JC_FILE_SYSTEM, "getLocal", - JMETHOD1(JPARAM(HADOOP_CONF), JPARAM(HADOOP_LOCALFS)), + JMETHOD1(JPARAM(HADOOP_CONF) , JPARAM(HADOOP_LOCALFS)), jConfiguration); if (jthr) { ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, @@ -893,23 +900,7 @@ hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld) hdfsBuilderToStr(bld, buf, sizeof(buf))); goto done; } - if (bld->kerb5ConfPath && bld->keyTabFile) { - jthr = invokeMethod(env, NULL, STATIC, NULL, JC_SECURITY_CONFIGURATION, "setConfiguration", JMETHOD1(JPARAM(HADOOP_CONF),JAVA_VOID), jConfiguration); - if (jthr) { - ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,"hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf))); - goto done; - } - jthr = newJavaStr(env, bld->keyTabFile, &jKeyTabString); - if (jthr) { - ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,"hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf))); - goto done; - } - jthr = invokeMethod(env, NULL, STATIC, NULL, JC_SECURITY_CONFIGURATION, "loginUserFromKeytab", JMETHOD2(JPARAM(JAVA_STRING), JPARAM(JAVA_STRING), JAVA_VOID), jUserString, jKeyTabString); - if (jthr) { - ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,"hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf))); - goto done; - } - } else if (bld->kerbTicketCachePath) { + if (bld->kerbTicketCachePath) { jthr = hadoopConfSetStr(env, jConfiguration, KERBEROS_TICKET_CACHE_PATH, bld->kerbTicketCachePath); if (jthr) { @@ -922,20 +913,69 @@ hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld) JMETHOD1(JPARAM(HADOOP_CONF),JAVA_VOID), jConfiguration); } if (bld->forceNewInstance) { - jthr = invokeMethod(env, &jVal, STATIC, NULL, - JC_FILE_SYSTEM, "newInstance", - JMETHOD3(JPARAM(JAVA_NET_URI), JPARAM(HADOOP_CONF), - JPARAM(JAVA_STRING), JPARAM(HADOOP_FS)), jURI, - jConfiguration, jUserString); - if (jthr) { - ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, - "hdfsBuilderConnect(%s)", - hdfsBuilderToStr(bld, buf, sizeof(buf))); - goto done; + // need kerb5ConfPath to enable kerberos authentication + if (bld->kerb5ConfPath && bld->kerbPrincipal && bld->keyTabFile) { + jthr = newJavaStr(env, bld->kerbPrincipal, &jPrincipalString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jthr = newJavaStr(env, bld->keyTabFile, &jKeyTabString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,"hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, + JC_FILE_SYSTEM, "newInstanceFromKeytab", + JMETHOD4(JPARAM(JAVA_NET_URI), JPARAM(HADOOP_CONF), + JPARAM(JAVA_STRING), JPARAM(JAVA_STRING), JPARAM(HADOOP_FS)), jURI, + jConfiguration, jPrincipalString, jKeyTabString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + } else { + jthr = invokeMethod(env, &jVal, STATIC, NULL, + JC_FILE_SYSTEM, "newInstance", + JMETHOD3(JPARAM(JAVA_NET_URI), JPARAM(HADOOP_CONF), + JPARAM(JAVA_STRING), JPARAM(HADOOP_FS)), jURI, + jConfiguration, jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } } jFS = jVal.l; } else { - if (bld->keyTabFile && bld->kerb5ConfPath) { + if (bld->keyTabFile && bld->kerb5ConfPath && bld->kerbPrincipal) { + jthr = newJavaStr(env, bld->kerbPrincipal, &jPrincipalString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jthr = invokeMethod(env, NULL, STATIC, NULL, JC_SECURITY_CONFIGURATION, "setConfiguration", JMETHOD1(JPARAM(HADOOP_CONF),JAVA_VOID), jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,"hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jthr = newJavaStr(env, bld->keyTabFile, &jKeyTabString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,"hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jthr = invokeMethod(env, NULL, STATIC, NULL, JC_SECURITY_CONFIGURATION, "loginUserFromKeytab", JMETHOD2(JPARAM(JAVA_STRING), JPARAM(JAVA_STRING), JAVA_VOID), jPrincipalString, jKeyTabString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,"hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } jthr = invokeMethod(env, &jVal, STATIC, NULL, JC_FILE_SYSTEM, "get", JMETHOD1(JPARAM(HADOOP_CONF), JPARAM(HADOOP_FS)), jConfiguration); } else { diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h index ac2de2d7467..f88545f3ba3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h @@ -347,6 +347,15 @@ extern "C" { */ LIBHDFS_EXTERNAL void hdfsBuilderSetUserName(struct hdfsBuilder *bld, const char *userName); + + /** + * Set the principal to use when connecting to the HDFS cluster. + * + * @param bld The HDFS builder + * @param kerbPrincipal The principal. The string will be shallow-copied. + */ + LIBHDFS_EXTERNAL + void hdfsBuilderSetPrincipal(struct hdfsBuilder *bld, const char *kerbPrincipal); /** * Set the path to the Kerberos ticket cache to use when connecting to --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
