This is an automated email from the ASF dual-hosted git repository.
rohithsharmaks pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 5718389 YARN-9149. yarn container -status misses logUrl when
integrated with ATSv2. Contributed by Abhishek Modi.
5718389 is described below
commit 571838920d2faca703e3c3417d64123927dfc4ad
Author: Rohith Sharma K S <[email protected]>
AuthorDate: Tue Feb 5 12:39:10 2019 +0530
YARN-9149. yarn container -status misses logUrl when integrated with ATSv2.
Contributed by Abhishek Modi.
---
.../apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java | 12 ++++++++++--
.../hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java | 9 +++++++++
.../hadoop/yarn/util/timeline/TimelineEntityV2Converter.java | 9 ++++++++-
3 files changed, 27 insertions(+), 3 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
index e797c28..19a7546 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.client.api.AHSClient;
import org.apache.hadoop.yarn.client.api.TimelineReaderClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.timeline.TimelineEntityV2Converter;
@@ -48,6 +49,7 @@ import java.util.Map;
@InterfaceStability.Unstable
public class AHSv2ClientImpl extends AHSClient {
private TimelineReaderClient readerClient;
+ private String logServerUrl;
public AHSv2ClientImpl() {
super(AHSv2ClientImpl.class.getName());
@@ -55,6 +57,8 @@ public class AHSv2ClientImpl extends AHSClient {
@Override
public void serviceInit(Configuration conf) {
+ logServerUrl = conf.get(
+ YarnConfiguration.YARN_LOG_SERVER_URL);
readerClient = TimelineReaderClient.createTimelineReaderClient();
readerClient.init(conf);
}
@@ -119,15 +123,19 @@ public class AHSv2ClientImpl extends AHSClient {
@Override
public ContainerReport getContainerReport(ContainerId containerId)
throws YarnException, IOException {
+ ApplicationReport appReport = getApplicationReport(
+ containerId.getApplicationAttemptId().getApplicationId());
TimelineEntity entity = readerClient.getContainerEntity(containerId,
"ALL", null);
- return TimelineEntityV2Converter.convertToContainerReport(entity);
+ return TimelineEntityV2Converter.convertToContainerReport(
+ entity, logServerUrl, appReport.getUser());
}
@Override
public List<ContainerReport> getContainers(ApplicationAttemptId
applicationAttemptId) throws YarnException, IOException {
ApplicationId appId = applicationAttemptId.getApplicationId();
+ ApplicationReport appReport = getApplicationReport(appId);
Map<String, String> filters = new HashMap<>();
filters.put("infofilters", "SYSTEM_INFO_PARENT_ENTITY eq {\"id\":\"" +
applicationAttemptId.toString() +
@@ -140,7 +148,7 @@ public class AHSv2ClientImpl extends AHSClient {
for (TimelineEntity entity : entities) {
ContainerReport container =
TimelineEntityV2Converter.convertToContainerReport(
- entity);
+ entity, logServerUrl, appReport.getUser());
containers.add(container);
}
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java
index 99473a3..1e0a1d6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java
@@ -61,7 +61,10 @@ public class TestAHSv2ClientImpl {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
+ conf.set(YarnConfiguration.YARN_LOG_SERVER_URL,
+ "https://localhost:8188/ahs";);
client = new AHSv2ClientImpl();
+ client.init(conf);
spyTimelineReaderClient = mock(TimelineReaderClient.class);
client.setReaderClient(spyTimelineReaderClient);
}
@@ -74,11 +77,17 @@ public class TestAHSv2ClientImpl {
final ContainerId containerId = ContainerId.newContainerId(appAttemptId,
1);
when(spyTimelineReaderClient.getContainerEntity(containerId, "ALL", null))
.thenReturn(createContainerEntity(containerId));
+ when(spyTimelineReaderClient.getApplicationEntity(appId, "ALL", null))
+ .thenReturn(createApplicationTimelineEntity(appId, true, false));
ContainerReport report = client.getContainerReport(containerId);
Assert.assertEquals(report.getContainerId(), containerId);
Assert.assertEquals(report.getAssignedNode().getHost(), "test host");
Assert.assertEquals(report.getAssignedNode().getPort(), 100);
Assert.assertEquals(report.getAllocatedResource().getVirtualCores(), 8);
+ Assert.assertEquals(report.getCreationTime(), 123456);
+ Assert.assertEquals(report.getLogUrl(),
+ "https://localhost:8188/ahs/logs/test host:100/"
+ + "container_0_0001_01_000001/container_0_0001_01_000001/user1");
}
@Test
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineEntityV2Converter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineEntityV2Converter.java
index 8d27e3b..98e1e7a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineEntityV2Converter.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineEntityV2Converter.java
@@ -49,6 +49,8 @@ import java.util.Map;
import java.util.NavigableSet;
import java.util.Set;
+import static org.apache.hadoop.yarn.util.StringHelper.PATH_JOINER;
+
/**
* Utility class to generate reports from timeline entities.
*/
@@ -57,7 +59,7 @@ public final class TimelineEntityV2Converter {
}
public static ContainerReport convertToContainerReport(
- TimelineEntity entity) {
+ TimelineEntity entity, String serverAddress, String user) {
int allocatedMem = 0;
int allocatedVcore = 0;
String allocatedHost = null;
@@ -142,8 +144,13 @@ public final class TimelineEntityV2Converter {
}
String logUrl = null;
NodeId allocatedNode = null;
+ String containerId = entity.getId();
if (allocatedHost != null) {
allocatedNode = NodeId.newInstance(allocatedHost, allocatedPort);
+ if (serverAddress != null && user != null) {
+ logUrl = PATH_JOINER.join(serverAddress,
+ "logs", allocatedNode, containerId, containerId, user);
+ }
}
ContainerReport container = ContainerReport.newInstance(
ContainerId.fromString(entity.getId()),
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
