This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch hadoop-3.3.4
in repository https://gitbox.apache.org/repos/asf/doris-thirdparty.git
The following commit(s) were added to refs/heads/hadoop-3.3.4 by this push:
new 91149c3c [feature] c api for use principal and keytab to login
kerberos (#69)
91149c3c is described below
commit 91149c3c377288523f485afdb313892d4c9b7529
Author: Long Zhao <[email protected]>
AuthorDate: Thu May 18 11:18:55 2023 +0800
[feature] c api for use principal and keytab to login kerberos (#69)
---
.../src/main/native/libhdfs/exception.c | 10 ++
.../src/main/native/libhdfs/hdfs.c | 143 +++++++++++++++++++--
.../src/main/native/libhdfs/include/hdfs/hdfs.h | 9 ++
.../src/main/native/libhdfs/jclasses.c | 4 +
.../src/main/native/libhdfs/jclasses.h | 4 +
.../src/main/native/libhdfs/jni_helper.c | 18 ++-
6 files changed, 168 insertions(+), 20 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/exception.c
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/exception.c
index fec9a103..ed61b77f 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/exception.c
+++
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/exception.c
@@ -90,6 +90,16 @@ static const struct ExceptionInfo gExceptionInfo[] = {
0,
ESTALE,
},
+ {
+ "org.apache.hadoop.security.KerberosAuthException",
+ NOPRINT_EXC_ACCESS_CONTROL,
+ ETIME
+ },
+ {
+ "java.net.ConnectException",
+ 0,
+ EFAULT
+ },
};
void getExceptionInfo(const char *excName, int noPrintFlags,
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
index 60f2826c..85f0c15c 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
+++
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
@@ -25,6 +25,7 @@
#include <fcntl.h>
#include <inttypes.h>
#include <stdio.h>
+#include <stdlib.h>
#include <string.h>
#define JAVA_VOID "V"
@@ -491,6 +492,11 @@ done:
return ret;
}
+struct hdfsBuilderConfFileOpt {
+ struct hdfsBuilderConfFileOpt *next;
+ const char *currentPath;
+};
+
struct hdfsBuilderConfOpt {
struct hdfsBuilderConfOpt *next;
const char *key;
@@ -502,10 +508,26 @@ struct hdfsBuilder {
const char *nn;
tPort port;
const char *kerbTicketCachePath;
+ const char *kerb5ConfPath;
+ const char *keyTabFile;
const char *userName;
struct hdfsBuilderConfOpt *opts;
+ struct hdfsBuilderConfFileOpt *fileOpts;
};
+void hdfsBuilderSetKerb5Conf(struct hdfsBuilder *bld, const char
*kerb5ConfPath) {
+ bld->kerb5ConfPath = kerb5ConfPath;
+ if (bld->kerb5ConfPath) {
+ systemPropertySetStr("java.security.krb5.conf", bld->kerb5ConfPath);
+ hdfsBuilderConfSetStr(bld, "hadoop.security.authorization", "true");
+ hdfsBuilderConfSetStr(bld, "hadoop.security.authentication",
"kerberos");
+ }
+}
+
+void hdfsBuilderSetKeyTabFile(struct hdfsBuilder *bld, const char *keyTabFile)
{
+ bld->keyTabFile = keyTabFile;
+}
+
struct hdfsBuilder *hdfsNewBuilder(void)
{
struct hdfsBuilder *bld = calloc(1, sizeof(struct hdfsBuilder));
@@ -516,6 +538,42 @@ struct hdfsBuilder *hdfsNewBuilder(void)
return bld;
}
+int systemPropertySetStr(const char *key, const char *val) {
+ JNIEnv *env = 0;
+ jobject jRet = NULL;
+ jvalue jVal;
+ jthrowable jthr;
+ int ret = EINTERNAL;
+ jstring jkey = NULL, jvalue0 = NULL;
+ env = getJNIEnv();
+ if (env == NULL) {
+ ret = EINTERNAL;
+ return ret;
+ }
+ jthr = newJavaStr(env, key, &jkey);
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr,
PRINT_EXC_ALL,"System::setProperty::new String");
+ goto done;
+ }
+ jthr = newJavaStr(env, val, &jvalue0);
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr,
PRINT_EXC_ALL,"System::setProperty::new String");
+ goto done;
+ }
+ jthr = invokeMethod(env, &jVal, STATIC, NULL, JC_SYSTEM, "setProperty",
"(Ljava/lang/String;Ljava/lang/String;)Ljava/lang/String;", jkey, jvalue0);
+ jRet = jVal.l;
+ ret = 0;
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr,
PRINT_EXC_ALL,"System::setProperty");
+ goto done;
+ }
+ done:
+ destroyLocalReference(env, jRet);
+ destroyLocalReference(env, jkey);
+ destroyLocalReference(env, jvalue0);
+ return ret;
+}
+
int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key,
const char *val)
{
@@ -542,6 +600,13 @@ void hdfsFreeBuilder(struct hdfsBuilder *bld)
free(cur);
cur = next;
}
+ struct hdfsBuilderConfFileOpt *cur0, *next0;
+ cur0 = bld->fileOpts;
+ for (cur0 = bld->fileOpts; cur0;) {
+ next0 = cur0->next;
+ free(cur0);
+ cur0 = next0;
+ }
free(bld);
}
@@ -684,17 +749,32 @@ static const char *hdfsBuilderToStr(const struct
hdfsBuilder *bld,
return buf;
}
+jthrowable hadoopConfFileAdd(JNIEnv *env, jobject jConfiguration, const char
*path) {
+ /* Create an object of org.apache.hadoop.fs.Path */
+ jobject jPath = NULL;
+ jthrowable jthr;
+ jthr = constructNewObjectOfPath(env, path, &jPath);
+ if (jthr) {
+ goto done;
+ }
+ jthr = invokeMethod(env, NULL, INSTANCE, jConfiguration, JC_CONFIGURATION,
"addResource", JMETHOD1(JPARAM(HADOOP_PATH), JAVA_VOID), jPath);
+ done:
+ destroyLocalReference(env, jPath);
+ return jthr;
+}
+
hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld)
{
JNIEnv *env = 0;
jobject jConfiguration = NULL, jFS = NULL, jURI = NULL, jCachePath = NULL;
- jstring jURIString = NULL, jUserString = NULL;
+ jstring jURIString = NULL, jUserString = NULL, jKeyTabString = NULL;
jvalue jVal;
jthrowable jthr = NULL;
char *cURI = 0, buf[512];
int ret;
jobject jRet = NULL;
struct hdfsBuilderConfOpt *opt;
+ struct hdfsBuilderConfFileOpt *fileOpt;
//Get the JNIEnv* corresponding to current thread
env = getJNIEnv();
@@ -711,6 +791,18 @@ hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld)
"hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf)));
goto done;
}
+
+ for (fileOpt = bld->fileOpts; fileOpt; fileOpt = fileOpt->next) {
+ // conf.addResource(new Path(fileOpt->path))
+ jthr = hadoopConfFileAdd(env, jConfiguration, fileOpt->currentPath);
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "hdfsBuilderConnect(%s): error adding
conffile '%s'",
+ hdfsBuilderToStr(bld, buf,
sizeof(buf)), fileOpt->currentPath);
+ goto done;
+ }
+ }
+
// set configuration values
for (opt = bld->opts; opt; opt = opt->next) {
jthr = hadoopConfSetStr(env, jConfiguration, opt->key, opt->val);
@@ -790,7 +882,30 @@ hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld)
jURI = jVal.l;
}
- if (bld->kerbTicketCachePath) {
+ jthr = newJavaStr(env, bld->userName, &jUserString);
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "hdfsBuilderConnect(%s)",
+ hdfsBuilderToStr(bld, buf,
sizeof(buf)));
+ goto done;
+ }
+ if (bld->kerb5ConfPath && bld->keyTabFile) {
+ jthr = invokeMethod(env, NULL, STATIC, NULL,
JC_SECURITY_CONFIGURATION, "setConfiguration",
JMETHOD1(JPARAM(HADOOP_CONF),JAVA_VOID), jConfiguration);
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr,
PRINT_EXC_ALL,"hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf,
sizeof(buf)));
+ goto done;
+ }
+ jthr = newJavaStr(env, bld->keyTabFile, &jKeyTabString);
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr,
PRINT_EXC_ALL,"hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf,
sizeof(buf)));
+ goto done;
+ }
+ jthr = invokeMethod(env, NULL, STATIC, NULL,
JC_SECURITY_CONFIGURATION, "loginUserFromKeytab", JMETHOD2(JPARAM(JAVA_STRING),
JPARAM(JAVA_STRING), JAVA_VOID), jUserString, jKeyTabString);
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr,
PRINT_EXC_ALL,"hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf,
sizeof(buf)));
+ goto done;
+ }
+ } else if (bld->kerbTicketCachePath) {
jthr = hadoopConfSetStr(env, jConfiguration,
KERBEROS_TICKET_CACHE_PATH, bld->kerbTicketCachePath);
if (jthr) {
@@ -799,13 +914,8 @@ hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld)
hdfsBuilderToStr(bld, buf, sizeof(buf)));
goto done;
}
- }
- jthr = newJavaStr(env, bld->userName, &jUserString);
- if (jthr) {
- ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
- "hdfsBuilderConnect(%s)",
- hdfsBuilderToStr(bld, buf, sizeof(buf)));
- goto done;
+ invokeMethod(env, &jVal, STATIC, NULL, JC_SECURITY_CONFIGURATION,
"setConfiguration",
+ JMETHOD1(JPARAM(HADOOP_CONF),JAVA_VOID), jConfiguration);
}
if (bld->forceNewInstance) {
jthr = invokeMethod(env, &jVal, STATIC, NULL,
@@ -821,11 +931,15 @@ hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld)
}
jFS = jVal.l;
} else {
- jthr = invokeMethod(env, &jVal, STATIC, NULL,
- JC_FILE_SYSTEM, "get",
- JMETHOD3(JPARAM(JAVA_NET_URI), JPARAM(HADOOP_CONF),
- JPARAM(JAVA_STRING), JPARAM(HADOOP_FS)), jURI,
- jConfiguration, jUserString);
+ if (bld->keyTabFile && bld->kerb5ConfPath) {
+ jthr = invokeMethod(env, &jVal, STATIC, NULL, JC_FILE_SYSTEM,
"get", JMETHOD1(JPARAM(HADOOP_CONF),
+ JPARAM(HADOOP_FS)), jConfiguration);
+ } else {
+ jthr = invokeMethod(env, &jVal, STATIC, NULL, JC_FILE_SYSTEM,
"get",
+ JMETHOD3(JPARAM(JAVA_NET_URI),
JPARAM(HADOOP_CONF),
+ JPARAM(JAVA_STRING),
JPARAM(HADOOP_FS)),
+ jURI, jConfiguration, jUserString);
+ }
if (jthr) {
ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
"hdfsBuilderConnect(%s)",
@@ -852,6 +966,7 @@ done:
destroyLocalReference(env, jCachePath);
destroyLocalReference(env, jURIString);
destroyLocalReference(env, jUserString);
+ destroyLocalReference(env, jKeyTabString);
free(cURI);
hdfsFreeBuilder(bld);
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h
index e58a6232..ed681ab6 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h
+++
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h
@@ -271,6 +271,9 @@ extern "C" {
LIBHDFS_EXTERNAL
struct hdfsBuilder *hdfsNewBuilder(void);
+ LIBHDFS_EXTERNAL
+ int systemPropertySetStr(const char *key, const char *val);
+
/**
* Force the builder to always create a new instance of the FileSystem,
* rather than possibly finding one in the cache.
@@ -334,6 +337,12 @@ extern "C" {
void hdfsBuilderSetKerbTicketCachePath(struct hdfsBuilder *bld,
const char *kerbTicketCachePath);
+ LIBHDFS_EXTERNAL
+ void hdfsBuilderSetKerb5Conf(struct hdfsBuilder *bld, const char
*kerb5ConfPath);
+
+ LIBHDFS_EXTERNAL
+ void hdfsBuilderSetKeyTabFile(struct hdfsBuilder *bld, const char
*keyTabFile);
+
/**
* Free an HDFS builder.
*
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jclasses.c
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jclasses.c
index cf880e91..321b9f43 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jclasses.c
+++
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jclasses.c
@@ -108,6 +108,10 @@ jthrowable initCachedClasses(JNIEnv* env) {
"java/util/EnumSet";
cachedJavaClasses[JC_EXCEPTION_UTILS].className =
"org/apache/commons/lang3/exception/ExceptionUtils";
+ cachedJavaClasses[JC_SYSTEM].className =
+ "java/lang/System";
+ cachedJavaClasses[JC_SECURITY_CONFIGURATION].className =
+ "org/apache/hadoop/security/UserGroupInformation";
// Create and set the jclass objects based on the class names set above
jthrowable jthr;
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jclasses.h
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jclasses.h
index 92cdd542..11523368 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jclasses.h
+++
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jclasses.h
@@ -59,6 +59,8 @@ typedef enum {
JC_BYTE_BUFFER,
JC_ENUM_SET,
JC_EXCEPTION_UTILS,
+ JC_SYSTEM,
+ JC_SECURITY_CONFIGURATION,
// A special marker enum that counts the number of cached jclasses
NUM_CACHED_CLASSES
} CachedJavaClass;
@@ -97,6 +99,7 @@ const char *getClassName(CachedJavaClass cachedJavaClass);
#define HADOOP_HDISTRM "org/apache/hadoop/hdfs/client/HdfsDataInputStream"
#define HADOOP_RO "org/apache/hadoop/fs/ReadOption"
#define HADOOP_DS "org/apache/hadoop/net/unix/DomainSocket"
+#define HADOOP_USER_INFORMATION
"org/apache/hadoop/security/UserGroupInformation"
/* Some frequently used Java class names */
#define JAVA_NET_ISA "java/net/InetSocketAddress"
@@ -104,6 +107,7 @@ const char *getClassName(CachedJavaClass cachedJavaClass);
#define JAVA_BYTEBUFFER "java/nio/ByteBuffer"
#define JAVA_STRING "java/lang/String"
#define JAVA_ENUMSET "java/util/EnumSet"
+#define JAVA_SYSTEM "java/lang/System"
/* Some frequently used third-party class names */
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jni_helper.c
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jni_helper.c
index 4efb3b61..98b2917a 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jni_helper.c
+++
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jni_helper.c
@@ -655,7 +655,7 @@ static char* getClassPath()
static JNIEnv* getGlobalJNIEnv(void)
{
JavaVM* vmBuf[VM_BUF_LENGTH];
- JNIEnv *env;
+ JNIEnv *env = NULL;
jint rv = 0;
jint noVMs = 0;
jthrowable jthr;
@@ -815,12 +815,9 @@ JNIEnv* getJNIEnv(void)
if (!state) {
mutexUnlock(&jvmMutex);
fprintf(stderr, "getJNIEnv: Unable to create ThreadLocalState\n");
- return NULL;
- }
- if (threadLocalStorageSet(state)) {
- mutexUnlock(&jvmMutex);
goto fail;
}
+ state->env = NULL;
THREAD_LOCAL_STORAGE_SET_QUICK(state);
state->env = getGlobalJNIEnv();
@@ -830,6 +827,12 @@ JNIEnv* getJNIEnv(void)
goto fail;
}
+ // when the env is created, the state can be set.
+ if (threadLocalStorageSet(state)) {
+ mutexUnlock(&jvmMutex);
+ goto fail;
+ }
+
jthrowable jthr = NULL;
jthr = initCachedClasses(state->env);
if (jthr) {
@@ -841,7 +844,10 @@ JNIEnv* getJNIEnv(void)
fail:
fprintf(stderr, "getJNIEnv: getGlobalJNIEnv failed\n");
- hdfsThreadDestructor(state);
+ THREAD_LOCAL_STORAGE_SET_QUICK(NULL);
+ if (state) {
+ hdfsThreadDestructor(state);
+ }
return NULL;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
