This is an automated email from the ASF dual-hosted git repository.
chengshiwen pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new f13e7a95ae [Improvement-11013][dolphinscheduler-common]
YarnHAAdminUtils#getActiveRMName function Add HTTPS Hadoop environment support
(#11017)
f13e7a95ae is described below
commit f13e7a95ae0c396e943ed048eacda54f9c7d68a1
Author: liyangyang <[email protected]>
AuthorDate: Mon Jul 25 11:07:40 2022 +0800
[Improvement-11013][dolphinscheduler-common]
YarnHAAdminUtils#getActiveRMName function Add HTTPS Hadoop environment support
(#11017)
* [Improvement-11013][dolphinscheduler-common]
YarnHAAdminUtils#getActiveRMName function Add HTTPS Hadoop environment support
* [Improvement-11013][dolphinscheduler-common] add test code
* [Improvement-11013][dolphinscheduler-common] code format
* [Improvement-11013][dolphinscheduler-common] add hadooputils test code
* [Improvement-11013][dolphinscheduler-common] remove extra blank lines
* [Improvement-11013][dolphinscheduler-common] adjusts test code
* [Improvement-11013][dolphinscheduler-common] remove useless import
* [Improvement-11013][dolphinscheduler-common] add test code
* [Improvement-11013][dolphinscheduler-common] shortened string
* [Improvement-11013][dolphinscheduler-common] format test code style
Co-authored-by: liyangyang <[email protected]>
---
.../dolphinscheduler/common/utils/HadoopUtils.java | 26 ++++++----
.../common/utils/CommonUtilsTest.java | 5 +-
.../{CommonUtilsTest.java => HadoopUtilsTest.java} | 58 ++++++++--------------
3 files changed, 38 insertions(+), 51 deletions(-)
diff --git
a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
index 21f6b9d110..b1ad942ac2 100644
---
a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
+++
b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
@@ -130,8 +130,8 @@ public class HadoopUtils implements Closeable,
StorageOperate {
String defaultFS = configuration.get(Constants.FS_DEFAULT_FS);
- if (StringUtils.isBlank(defaultFS)){
- defaultFS= PropertyUtils.getString(Constants.FS_DEFAULT_FS);
+ if (StringUtils.isBlank(defaultFS)) {
+ defaultFS = PropertyUtils.getString(Constants.FS_DEFAULT_FS);
}
//first get key from core-site.xml hdfs-site.xml ,if null ,then
try to get from properties file
@@ -615,12 +615,6 @@ public class HadoopUtils implements Closeable,
StorageOperate {
*/
public static String getAppAddress(String appAddress, String rmHa) {
- //get active ResourceManager
- String activeRM = YarnHAAdminUtils.getActiveRMName(rmHa);
-
- if (StringUtils.isEmpty(activeRM)) {
- return null;
- }
String[] split1 = appAddress.split(Constants.DOUBLE_SLASH);
@@ -637,6 +631,13 @@ public class HadoopUtils implements Closeable,
StorageOperate {
String end = Constants.COLON + split2[1];
+ //get active ResourceManager
+ String activeRM = YarnHAAdminUtils.getActiveRMName(start, rmHa);
+
+ if (StringUtils.isEmpty(activeRM)) {
+ return null;
+ }
+
return start + activeRM + end;
}
@@ -658,13 +659,16 @@ public class HadoopUtils implements Closeable,
StorageOperate {
private static final class YarnHAAdminUtils {
/**
- * get active resourcemanager
+ * get active resourcemanager node
+ * @param protocol http protocol
+ * @param rmIds yarn ha ids
+ * @return yarn active node
*/
- public static String getActiveRMName(String rmIds) {
+ public static String getActiveRMName(String protocol, String rmIds) {
String[] rmIdArr = rmIds.split(Constants.COMMA);
- String yarnUrl = "http://%s:"; +
HADOOP_RESOURCE_MANAGER_HTTP_ADDRESS_PORT_VALUE + "/ws/v1/cluster/info";
+ String yarnUrl = protocol + "%s:" +
HADOOP_RESOURCE_MANAGER_HTTP_ADDRESS_PORT_VALUE + "/ws/v1/cluster/info";
try {
diff --git
a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java
b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java
index 4facb999dd..8123da3690 100644
---
a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java
+++
b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java
@@ -36,9 +36,10 @@ import org.slf4j.LoggerFactory;
* configuration test
*/
@RunWith(PowerMockRunner.class)
-@PrepareForTest(value = { PropertyUtils.class, UserGroupInformation.class})
+@PrepareForTest(value = {PropertyUtils.class, UserGroupInformation.class})
public class CommonUtilsTest {
private static final Logger logger =
LoggerFactory.getLogger(CommonUtilsTest.class);
+
@Test
public void getSystemEnvPath() {
String envPath;
@@ -48,7 +49,7 @@ public class CommonUtilsTest {
@Test
public void isDevelopMode() {
- logger.info("develop mode: {}",CommonUtils.isDevelopMode());
+ logger.info("develop mode: {}", CommonUtils.isDevelopMode());
Assert.assertTrue(true);
}
diff --git
a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java
b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java
similarity index 52%
copy from
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java
copy to
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java
index 4facb999dd..d2b29254b9 100644
---
a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java
+++
b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java
@@ -17,74 +17,56 @@
package org.apache.dolphinscheduler.common.utils;
-import org.apache.dolphinscheduler.spi.utils.PropertyUtils;
-
-import org.apache.hadoop.security.UserGroupInformation;
-
-import java.net.InetAddress;
-import java.net.UnknownHostException;
+import org.apache.dolphinscheduler.spi.enums.ResourceType;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
+import
org.powermock.core.classloader.annotations.SuppressStaticInitializationFor;
import org.powermock.modules.junit4.PowerMockRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * configuration test
+ * hadoop utils test
*/
@RunWith(PowerMockRunner.class)
-@PrepareForTest(value = { PropertyUtils.class, UserGroupInformation.class})
-public class CommonUtilsTest {
- private static final Logger logger =
LoggerFactory.getLogger(CommonUtilsTest.class);
- @Test
- public void getSystemEnvPath() {
- String envPath;
- envPath = CommonUtils.getSystemEnvPath();
- Assert.assertEquals("/etc/profile", envPath);
- }
-
- @Test
- public void isDevelopMode() {
- logger.info("develop mode: {}",CommonUtils.isDevelopMode());
- Assert.assertTrue(true);
- }
+@PrepareForTest(value = {HadoopUtils.class})
+@SuppressStaticInitializationFor("org.apache.dolphinscheduler.common.utils.HttpUtils")
+public class HadoopUtilsTest {
+ private static final Logger logger =
LoggerFactory.getLogger(HadoopUtilsTest.class);
@Test
- public void getHdfsDataBasePath() {
- logger.info(HadoopUtils.getHdfsDataBasePath());
+ public void getHdfsTenantDir() {
+ logger.info(HadoopUtils.getHdfsTenantDir("1234"));
Assert.assertTrue(true);
}
@Test
- public void getDownloadFilename() {
- logger.info(FileUtils.getDownloadFilename("a.txt"));
+ public void getHdfsUdfFileName() {
+ logger.info(HadoopUtils.getHdfsUdfFileName("admin", "file_name"));
Assert.assertTrue(true);
}
@Test
- public void getUploadFilename() {
- logger.info(FileUtils.getUploadFilename("1234", "a.txt"));
+ public void getHdfsResourceFileName() {
+ logger.info(HadoopUtils.getHdfsResourceFileName("admin", "file_name"));
Assert.assertTrue(true);
}
@Test
- public void getHdfsDir() {
- logger.info(HadoopUtils.getHdfsResDir("1234"));
+ public void getHdfsFileName() {
+ logger.info(HadoopUtils.getHdfsFileName(ResourceType.FILE, "admin",
"file_name"));
Assert.assertTrue(true);
}
@Test
- public void test() {
- InetAddress ip;
- try {
- ip = InetAddress.getLocalHost();
- logger.info(ip.getHostAddress());
- } catch (UnknownHostException e) {
- e.printStackTrace();
- }
+ public void getAppAddress() {
+ PowerMockito.mockStatic(HttpUtils.class);
+
PowerMockito.when(HttpUtils.get("http://ds1:8088/ws/v1/cluster/info";)).thenReturn("{\"clusterInfo\":{\"state\":\"STARTED\",\"haState\":\"ACTIVE\"}}");
+
logger.info(HadoopUtils.getAppAddress("http://ds1:8088/ws/v1/cluster/apps/%s";,
"ds1,ds2"));
Assert.assertTrue(true);
}
