This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new c8e31ad [Improvement-4884][Common] fix the subsequent processing of
getting the YARN ResourceManager address to be empty (#4885)
c8e31ad is described below
commit c8e31ad58e15936174e773172d12337a8cf26d69
Author: zhuangchong <[email protected]>
AuthorDate: Thu Mar 11 22:14:52 2021 +0800
[Improvement-4884][Common] fix the subsequent processing of getting the
YARN ResourceManager address to be empty (#4885)
* fix yarn application url generate empty.
* removing Excess Processing
* update hadoop utils test class .
* add base exception class.
* update hadoop utils class code style.
---
.../common/exception/BaseException.java | 43 ++++++++++++++++++++++
.../dolphinscheduler/common/utils/HadoopUtils.java | 32 ++++++----------
.../common/utils/HadoopUtilsTest.java | 2 +-
3 files changed, 56 insertions(+), 21 deletions(-)
diff --git
a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/exception/BaseException.java
b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/exception/BaseException.java
new file mode 100644
index 0000000..096bd8c
--- /dev/null
+++
b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/exception/BaseException.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.common.exception;
+
+/**
+ * Base Exception class for DolphinScheduler
+ */
+public class BaseException extends Exception {
+
+ public BaseException() {
+ }
+
+ public BaseException(String message) {
+ super(message);
+ }
+
+ public BaseException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public BaseException(Throwable cause) {
+ super(cause);
+ }
+
+ public BaseException(String message, Throwable cause, boolean
enableSuppression, boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+}
diff --git
a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
index ef89a30..6a53c00 100644
---
a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
+++
b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
@@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.enums.ResUploadType;
import org.apache.dolphinscheduler.common.enums.ResourceType;
+import org.apache.dolphinscheduler.common.exception.BaseException;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
@@ -204,21 +205,13 @@ public class HadoopUtils implements Closeable {
* if rmHaIds is empty, single resourcemanager enabled
* if rmHaIds not empty: resourcemanager HA enabled
*/
- String appUrl = "";
-
- if (StringUtils.isEmpty(rmHaIds)) {
- //single resourcemanager enabled
- appUrl = appAddress;
- yarnEnabled = true;
- } else {
- //resourcemanager HA enabled
- appUrl = getAppAddress(appAddress, rmHaIds);
- yarnEnabled = true;
- logger.info("application url : {}", appUrl);
- }
-
+ yarnEnabled = true;
+ String appUrl = StringUtils.isEmpty(rmHaIds) ? appAddress :
getAppAddress(appAddress, rmHaIds);
if (StringUtils.isBlank(appUrl)) {
- throw new Exception("application url is blank");
+ throw new BaseException("yarn application url generation failed");
+ }
+ if (logger.isDebugEnabled()) {
+ logger.debug("yarn application url:{}, applicationId:{}", appUrl,
applicationId);
}
return String.format(appUrl, applicationId);
}
@@ -597,6 +590,10 @@ public class HadoopUtils implements Closeable {
//get active ResourceManager
String activeRM = YarnHAAdminUtils.getAcitveRMName(rmHa);
+ if (StringUtils.isEmpty(activeRM)) {
+ return null;
+ }
+
String[] split1 = appAddress.split(Constants.DOUBLE_SLASH);
if (split1.length != 2) {
@@ -660,12 +657,7 @@ public class HadoopUtils implements Closeable {
}
} catch (Exception e) {
- for (int i = 1; i < rmIdArr.length; i++) {
- String state = getRMState(String.format(yarnUrl,
rmIdArr[i]));
- if (Constants.HADOOP_RM_STATE_ACTIVE.equals(state)) {
- return rmIdArr[i];
- }
- }
+ logger.error("yarn ha application url generation failed,
message:{}", e.getMessage());
}
return null;
}
diff --git
a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java
b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java
index f6efb1f..de7f9a4 100644
---
a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java
+++
b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java
@@ -184,7 +184,7 @@ public class HadoopUtilsTest {
}
}
- @Test
+ @Test(expected = Exception.class)
public void getApplicationUrl() throws Exception {
String application_1516778421218_0042 =
hadoopUtils.getApplicationUrl("application_1529051418016_0167");
logger.info(application_1516778421218_0042);
