This is an automated email from the ASF dual-hosted git repository. kirs pushed a commit to branch revert-5860-1.3.7-prepare#5256 in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit a511b1c6ca2096e707671885893a249aa4f5a94e Author: Kirs <[email protected]> AuthorDate: Tue Jul 20 14:28:40 2021 +0800 Revert "[1.3.7-prepare#5256][Improvement][common]Task status error (#5860)" This reverts commit 8e2c312d4e8e4d9644fda4be6113ca1dd98884d4. --- .../apache/dolphinscheduler/common/Constants.java | 9 ------ .../dolphinscheduler/common/utils/HadoopUtils.java | 37 ++++++++++++---------- .../src/main/resources/common.properties | 4 +-- 3 files changed, 21 insertions(+), 29 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index e8447fa..fe83fa1 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -104,11 +104,6 @@ public final class Constants { public static final String YARN_APPLICATION_STATUS_ADDRESS = "yarn.application.status.address"; /** - * yarn.job.history.status.address - */ - public static final String YARN_JOB_HISTORY_STATUS_ADDRESS = "yarn.job.history.status.address"; - - /** * hdfs configuration * hdfs.root.user */ @@ -797,10 +792,6 @@ public final class Constants { */ public static final String HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE = "hadoop.security.authentication.startup.state"; - /** - * com.amazonaws.services.s3.enableV4 - */ - public static final String AWS_S3_V4 = "com.amazonaws.services.s3.enableV4"; /** * loginUserFromKeytab user diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index 667838d..ffead0b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -70,7 +70,6 @@ public class HadoopUtils implements Closeable { public static final String resourceUploadPath = PropertyUtils.getString(RESOURCE_UPLOAD_PATH, "/dolphinscheduler"); public static final String rmHaIds = PropertyUtils.getString(Constants.YARN_RESOURCEMANAGER_HA_RM_IDS); public static final String appAddress = PropertyUtils.getString(Constants.YARN_APPLICATION_STATUS_ADDRESS); - public static final String jobHistoryAddress = PropertyUtils.getString(Constants.YARN_JOB_HISTORY_STATUS_ADDRESS); private static final String HADOOP_UTILS_KEY = "HADOOP_UTILS_KEY"; @@ -123,7 +122,7 @@ public class HadoopUtils implements Closeable { try { configuration = new HdfsConfiguration(); - String resourceStorageType = PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE); + String resourceStorageType = PropertyUtils.getString(Constants.RESOURCE_STORAGE_TYPE); ResUploadType resUploadType = ResUploadType.valueOf(resourceStorageType); if (resUploadType == ResUploadType.HDFS){ @@ -156,18 +155,22 @@ public class HadoopUtils implements Closeable { logger.info("get property:{} -> {}, from core-site.xml hdfs-site.xml ", Constants.FS_DEFAULTFS, defaultFS); } + if (fs == null) { if (StringUtils.isNotEmpty(hdfsUser)) { UserGroupInformation ugi = UserGroupInformation.createRemoteUser(hdfsUser); - ugi.doAs((PrivilegedExceptionAction<Boolean>) () -> { + ugi.doAs(new PrivilegedExceptionAction<Boolean>() { + @Override + public Boolean run() throws Exception { fs = FileSystem.get(configuration); return true; + } }); } else { logger.warn("hdfs.root.user is not set value!"); fs = FileSystem.get(configuration); } + } } else if (resUploadType == ResUploadType.S3) { - System.setProperty(Constants.AWS_S3_V4, Constants.STRING_TRUE); configuration.set(Constants.FS_DEFAULTFS, PropertyUtils.getString(Constants.FS_DEFAULTFS)); configuration.set(Constants.FS_S3A_ENDPOINT, PropertyUtils.getString(Constants.FS_S3A_ENDPOINT)); configuration.set(Constants.FS_S3A_ACCESS_KEY, PropertyUtils.getString(Constants.FS_S3A_ACCESS_KEY)); @@ -201,23 +204,23 @@ public class HadoopUtils implements Closeable { * if rmHaIds is empty, single resourcemanager enabled * if rmHaIds not empty: resourcemanager HA enabled */ + String appUrl = ""; + if (StringUtils.isEmpty(rmHaIds)) { + //single resourcemanager enabled + appUrl = appAddress; yarnEnabled = true; - String appUrl = StringUtils.isEmpty(rmHaIds) ? appAddress : getAppAddress(appAddress, rmHaIds); - if (StringUtils.isBlank(appUrl)) { - throw new Exception("yarn application url generation failed"); - } - if (logger.isDebugEnabled()) { - logger.debug("yarn application url:{}, applicationId:{}", appUrl, applicationId); + } else { + //resourcemanager HA enabled + appUrl = getAppAddress(appAddress, rmHaIds); + yarnEnabled = true; + logger.info("application url : {}", appUrl); } - String activeResourceManagerPort = String.valueOf(PropertyUtils.getInt(Constants.HADOOP_RESOURCE_MANAGER_HTTPADDRESS_PORT, 8088)); - return String.format(appUrl, activeResourceManagerPort, applicationId); - } - public String getJobHistoryUrl(String applicationId) { - //eg:application_1587475402360_712719 -> job_1587475402360_712719 - String jobId = applicationId.replace("application", "job"); - return String.format(jobHistoryAddress, jobId); + if (StringUtils.isBlank(appUrl)) { + throw new Exception("application url is blank"); + } + return String.format(appUrl, applicationId); } /** diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index b658029..5bb7c39 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -61,9 +61,7 @@ fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx # if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname -yarn.application.status.address=http://ds1:%s/ws/v1/cluster/apps/%s -# if custom you resourcemanager port ,you need to replace 8088 else default value. -resource.manager.httpaddress.port=8088 +yarn.application.status.address=http://ds1:8088/ws/v1/cluster/apps/%s # system env path #dolphinscheduler.env.path=env/dolphinscheduler_env.sh
