This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch hadoop-3.3.4
in repository https://gitbox.apache.org/repos/asf/doris-thirdparty.git
The following commit(s) were added to refs/heads/hadoop-3.3.4 by this push:
new a16b0479 [fix](tgt)hadoop kerberos support renew TGT by keytab (#173)
a16b0479 is described below
commit a16b0479dbdbe624c4be6860430c4957f9ce9017
Author: slothever <[email protected]>
AuthorDate: Fri Jan 12 13:47:31 2024 +0800
[fix](tgt)hadoop kerberos support renew TGT by keytab (#173)
---
.../main/java/org/apache/hadoop/fs/FileSystem.java | 13 +++
.../src/main/native/libhdfs/hdfs.c | 100 ++++++++++++++-------
.../src/main/native/libhdfs/include/hdfs/hdfs.h | 9 ++
3 files changed, 92 insertions(+), 30 deletions(-)
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index f0c55f54..4cb2519d 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -565,6 +565,19 @@ public abstract class FileSystem extends Configured
});
}
+ public synchronized static FileSystem newInstanceFromKeytab(final URI uri,
final Configuration conf,
+ final String
principal, final String keytabPath)
+ throws IOException, InterruptedException {
+ UserGroupInformation.setConfiguration(conf);
+ UserGroupInformation.loginUserFromKeytab(principal, keytabPath);
+ return UserGroupInformation.getLoginUser().doAs(new
PrivilegedExceptionAction<FileSystem>() {
+ @Override
+ public FileSystem run() throws IOException {
+ return newInstance(uri, conf);
+ }
+ });
+ }
+
/**
* Returns the FileSystem for this URI's scheme and authority.
* The entire URI is passed to the FileSystem instance's initialize method.
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
index 85f0c15c..5fe67bb8 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
+++
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
@@ -36,6 +36,7 @@
#define JMETHOD1(X, R) "(" X ")" R
#define JMETHOD2(X, Y, R) "(" X Y ")" R
#define JMETHOD3(X, Y, Z, R) "(" X Y Z")" R
+#define JMETHOD4(X, Y, Z, A, R) "(" X Y Z A")" R
#define KERBEROS_TICKET_CACHE_PATH "hadoop.security.kerberos.ticket.cache.path"
@@ -510,6 +511,7 @@ struct hdfsBuilder {
const char *kerbTicketCachePath;
const char *kerb5ConfPath;
const char *keyTabFile;
+ const char *kerbPrincipal;
const char *userName;
struct hdfsBuilderConfOpt *opts;
struct hdfsBuilderConfFileOpt *fileOpts;
@@ -630,6 +632,11 @@ void hdfsBuilderSetUserName(struct hdfsBuilder *bld, const
char *userName)
bld->userName = userName;
}
+void hdfsBuilderSetPrincipal(struct hdfsBuilder *bld, const char
*kerbPrincipal)
+{
+ bld->kerbPrincipal = kerbPrincipal;
+}
+
void hdfsBuilderSetKerbTicketCachePath(struct hdfsBuilder *bld,
const char *kerbTicketCachePath)
{
@@ -767,7 +774,7 @@ hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld)
{
JNIEnv *env = 0;
jobject jConfiguration = NULL, jFS = NULL, jURI = NULL, jCachePath = NULL;
- jstring jURIString = NULL, jUserString = NULL, jKeyTabString = NULL;
+ jstring jURIString = NULL, jUserString = NULL, jPrincipalString = NULL,
jKeyTabString = NULL;
jvalue jVal;
jthrowable jthr = NULL;
char *cURI = 0, buf[512];
@@ -834,7 +841,7 @@ hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld)
// fs = FileSytem#getLocal(conf);
jthr = invokeMethod(env, &jVal, STATIC, NULL,
JC_FILE_SYSTEM, "getLocal",
- JMETHOD1(JPARAM(HADOOP_CONF), JPARAM(HADOOP_LOCALFS)),
+ JMETHOD1(JPARAM(HADOOP_CONF) , JPARAM(HADOOP_LOCALFS)),
jConfiguration);
if (jthr) {
ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
@@ -889,23 +896,7 @@ hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld)
hdfsBuilderToStr(bld, buf,
sizeof(buf)));
goto done;
}
- if (bld->kerb5ConfPath && bld->keyTabFile) {
- jthr = invokeMethod(env, NULL, STATIC, NULL,
JC_SECURITY_CONFIGURATION, "setConfiguration",
JMETHOD1(JPARAM(HADOOP_CONF),JAVA_VOID), jConfiguration);
- if (jthr) {
- ret = printExceptionAndFree(env, jthr,
PRINT_EXC_ALL,"hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf,
sizeof(buf)));
- goto done;
- }
- jthr = newJavaStr(env, bld->keyTabFile, &jKeyTabString);
- if (jthr) {
- ret = printExceptionAndFree(env, jthr,
PRINT_EXC_ALL,"hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf,
sizeof(buf)));
- goto done;
- }
- jthr = invokeMethod(env, NULL, STATIC, NULL,
JC_SECURITY_CONFIGURATION, "loginUserFromKeytab", JMETHOD2(JPARAM(JAVA_STRING),
JPARAM(JAVA_STRING), JAVA_VOID), jUserString, jKeyTabString);
- if (jthr) {
- ret = printExceptionAndFree(env, jthr,
PRINT_EXC_ALL,"hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf,
sizeof(buf)));
- goto done;
- }
- } else if (bld->kerbTicketCachePath) {
+ if (bld->kerbTicketCachePath) {
jthr = hadoopConfSetStr(env, jConfiguration,
KERBEROS_TICKET_CACHE_PATH, bld->kerbTicketCachePath);
if (jthr) {
@@ -918,20 +909,69 @@ hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld)
JMETHOD1(JPARAM(HADOOP_CONF),JAVA_VOID), jConfiguration);
}
if (bld->forceNewInstance) {
- jthr = invokeMethod(env, &jVal, STATIC, NULL,
- JC_FILE_SYSTEM, "newInstance",
- JMETHOD3(JPARAM(JAVA_NET_URI), JPARAM(HADOOP_CONF),
- JPARAM(JAVA_STRING), JPARAM(HADOOP_FS)), jURI,
- jConfiguration, jUserString);
- if (jthr) {
- ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
- "hdfsBuilderConnect(%s)",
- hdfsBuilderToStr(bld, buf, sizeof(buf)));
- goto done;
+ // need kerb5ConfPath to enable kerberos authentication
+ if (bld->kerb5ConfPath && bld->kerbPrincipal && bld->keyTabFile) {
+ jthr = newJavaStr(env, bld->kerbPrincipal, &jPrincipalString);
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "hdfsBuilderConnect(%s)",
+ hdfsBuilderToStr(bld, buf,
sizeof(buf)));
+ goto done;
+ }
+ jthr = newJavaStr(env, bld->keyTabFile, &jKeyTabString);
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr,
PRINT_EXC_ALL,"hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf,
sizeof(buf)));
+ goto done;
+ }
+ jthr = invokeMethod(env, &jVal, STATIC, NULL,
+ JC_FILE_SYSTEM, "newInstanceFromKeytab",
+ JMETHOD4(JPARAM(JAVA_NET_URI),
JPARAM(HADOOP_CONF),
+ JPARAM(JAVA_STRING),
JPARAM(JAVA_STRING), JPARAM(HADOOP_FS)), jURI,
+ jConfiguration, jPrincipalString,
jKeyTabString);
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "hdfsBuilderConnect(%s)",
+ hdfsBuilderToStr(bld, buf,
sizeof(buf)));
+ goto done;
+ }
+ } else {
+ jthr = invokeMethod(env, &jVal, STATIC, NULL,
+ JC_FILE_SYSTEM, "newInstance",
+ JMETHOD3(JPARAM(JAVA_NET_URI),
JPARAM(HADOOP_CONF),
+ JPARAM(JAVA_STRING),
JPARAM(HADOOP_FS)), jURI,
+ jConfiguration, jUserString);
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "hdfsBuilderConnect(%s)",
+ hdfsBuilderToStr(bld, buf,
sizeof(buf)));
+ goto done;
+ }
}
jFS = jVal.l;
} else {
- if (bld->keyTabFile && bld->kerb5ConfPath) {
+ if (bld->keyTabFile && bld->kerb5ConfPath && bld->kerbPrincipal) {
+ jthr = newJavaStr(env, bld->kerbPrincipal, &jPrincipalString);
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "hdfsBuilderConnect(%s)",
+ hdfsBuilderToStr(bld, buf,
sizeof(buf)));
+ goto done;
+ }
+ jthr = invokeMethod(env, NULL, STATIC, NULL,
JC_SECURITY_CONFIGURATION, "setConfiguration",
JMETHOD1(JPARAM(HADOOP_CONF),JAVA_VOID), jConfiguration);
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr,
PRINT_EXC_ALL,"hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf,
sizeof(buf)));
+ goto done;
+ }
+ jthr = newJavaStr(env, bld->keyTabFile, &jKeyTabString);
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr,
PRINT_EXC_ALL,"hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf,
sizeof(buf)));
+ goto done;
+ }
+ jthr = invokeMethod(env, NULL, STATIC, NULL,
JC_SECURITY_CONFIGURATION, "loginUserFromKeytab", JMETHOD2(JPARAM(JAVA_STRING),
JPARAM(JAVA_STRING), JAVA_VOID), jPrincipalString, jKeyTabString);
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr,
PRINT_EXC_ALL,"hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf,
sizeof(buf)));
+ goto done;
+ }
jthr = invokeMethod(env, &jVal, STATIC, NULL, JC_FILE_SYSTEM,
"get", JMETHOD1(JPARAM(HADOOP_CONF),
JPARAM(HADOOP_FS)), jConfiguration);
} else {
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h
index ed681ab6..28a5f91e 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h
+++
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h
@@ -324,6 +324,15 @@ extern "C" {
*/
LIBHDFS_EXTERNAL
void hdfsBuilderSetUserName(struct hdfsBuilder *bld, const char *userName);
+
+ /**
+ * Set the principal to use when connecting to the HDFS cluster.
+ *
+ * @param bld The HDFS builder
+ * @param kerbPrincipal The principal. The string will be shallow-copied.
+ */
+ LIBHDFS_EXTERNAL
+ void hdfsBuilderSetPrincipal(struct hdfsBuilder *bld, const char
*kerbPrincipal);
/**
* Set the path to the Kerberos ticket cache to use when connecting to
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
