This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tez.git
The following commit(s) were added to refs/heads/master by this push:
new 5766cdabf TEZ-4688 TEZ-4648: Tez upgrade to Hadoop 3.5.0 and jersey
2.x (#474) (Mahesh Raju Somalaraju reviewed by Laszlo Bodor)
5766cdabf is described below
commit 5766cdabfd7a184f4a00254a1c3ce0ee961341db
Author: Mahesh Raju Somalaraju <[email protected]>
AuthorDate: Wed Jun 17 10:45:57 2026 +0530
TEZ-4688 TEZ-4648: Tez upgrade to Hadoop 3.5.0 and jersey 2.x (#474)
(Mahesh Raju Somalaraju reviewed by Laszlo Bodor)
---
pom.xml | 180 ++++++++++++++++++---
tez-api/pom.xml | 6 +-
.../java/org/apache/tez/common/ATSConstants.java | 13 +-
.../tez/dag/api/client/DAGClientTimelineImpl.java | 35 ++--
.../tez/dag/api/client/TimelineReaderFactory.java | 41 ++---
.../tez/dag/api/client/TestATSHttpClient.java | 8 +-
.../dag/api/client/TestTimelineReaderFactory.java | 4 +-
tez-dag/pom.xml | 9 +-
.../logging/impl/HistoryEventJsonConversion.java | 74 ++++-----
.../impl/TestHistoryEventJsonConversion.java | 2 +-
tez-ext-service-tests/pom.xml | 6 +-
tez-plugins/tez-aux-services/pom.xml | 5 +
tez-plugins/tez-history-parser/pom.xml | 15 +-
.../java/org/apache/tez/history/ATSImportTool.java | 78 ++++-----
.../tez/history/parser/SimpleHistoryParser.java | 10 +-
.../tez/history/parser/datamodel/DagInfo.java | 2 +-
.../history/parser/datamodel/TaskAttemptInfo.java | 2 +-
.../tez/history/parser/datamodel/TaskInfo.java | 2 +-
.../tez/history/parser/datamodel/VertexInfo.java | 2 +-
.../proto/HistoryEventProtoJsonConversion.java | 74 ++++-----
.../tez-yarn-timeline-history-with-acls/pom.xml | 14 +-
.../history/ats/acls/TestATSHistoryWithACLs.java | 80 +++++----
.../tez-yarn-timeline-history-with-fs/pom.xml | 9 +-
tez-plugins/tez-yarn-timeline-history/pom.xml | 9 +-
.../logging/ats/TestATSHistoryWithMiniCluster.java | 22 +--
tez-tests/pom.xml | 5 +
tez-tools/analyzers/job-analyzer/pom.xml | 15 +-
.../java/org/apache/tez/analyzer/utils/Utils.java | 4 +-
tez-ui/src/main/webapp/app/serializers/ahs-app.js | 2 +-
tez-ui/src/main/webapp/app/serializers/app.js | 12 +-
tez-ui/src/main/webapp/app/serializers/attempt.js | 16 +-
tez-ui/src/main/webapp/app/serializers/dag-info.js | 12 +-
tez-ui/src/main/webapp/app/serializers/dag.js | 34 ++--
.../src/main/webapp/app/serializers/hive-query.js | 70 ++++----
tez-ui/src/main/webapp/app/serializers/task.js | 10 +-
tez-ui/src/main/webapp/app/serializers/timeline.js | 18 ++-
tez-ui/src/main/webapp/app/serializers/vertex.js | 44 ++---
.../src/main/webapp/app/utils/download-dag-zip.js | 2 +-
.../webapp/tests/unit/serializers/attempt-test.js | 2 +-
.../webapp/tests/unit/serializers/dag-info-test.js | 8 +-
.../main/webapp/tests/unit/serializers/dag-test.js | 44 ++---
.../tests/unit/serializers/hive-query-test.js | 28 ++--
.../webapp/tests/unit/serializers/timeline-test.js | 1 +
.../webapp/tests/unit/serializers/vertex-test.js | 4 +-
44 files changed, 614 insertions(+), 419 deletions(-)
diff --git a/pom.xml b/pom.xml
index ba08baeea..38f2fec78 100644
--- a/pom.xml
+++ b/pom.xml
@@ -81,11 +81,17 @@
<plexus-velocity.version>2.3.0</plexus-velocity.version>
<frontend-maven-plugin.version>1.15.0</frontend-maven-plugin.version>
<guava.version>32.0.1-jre</guava.version>
- <hadoop.version>3.4.2</hadoop.version>
- <jersey.version>1.19.4</jersey.version>
+ <hadoop.version>3.5.0</hadoop.version>
+ <!-- Align all Jackson modules with the version shipped by Hadoop 3.5.0
(2.18.6).
+ Importing jackson-bom in dependencyManagement ensures any transitive
dependency
+ (e.g. avro) that pulls an older jackson-core is automatically
overridden.
+ This is the Jackson project's recommended approach for version
alignment. -->
+ <jackson.version>2.18.6</jackson.version>
+ <jersey.version>2.46</jersey.version>
<jettison.version>1.5.4</jettison.version>
<jsr305.version>3.0.0</jsr305.version>
<junit.version>4.13.2</junit.version>
+ <junit.jupiter.version>5.9.3</junit.jupiter.version>
<leveldbjni-all.version>1.8</leveldbjni-all.version>
<lifecycle-mapping.version>1.0.0</lifecycle-mapping.version>
<metrics-core.version>3.1.0</metrics-core.version>
@@ -98,7 +104,7 @@
<restrict-imports.enforcer.version>3.0.0</restrict-imports.enforcer.version>
<roaringbitmap.version>1.2.1</roaringbitmap.version>
<scm.url>scm:git:https://gitbox.apache.org/repos/asf/tez.git</scm.url>
- <servlet-api.version>3.1.0</servlet-api.version>
+ <servlet-api.version>4.0.4</servlet-api.version>
<slf4j.version>1.7.36</slf4j.version>
<snappy-java.version>1.1.10.4</snappy-java.version>
<spotless-maven-plugin.version>3.1.0</spotless-maven-plugin.version>
@@ -153,6 +159,18 @@
<dependencyManagement>
<dependencies>
+ <!-- Import jackson-bom to enforce a consistent Jackson version across
all
+ transitive sources. avro 1.11.5 (via
hadoop-common/mapreduce-client-common)
+ pulls in jackson-core 2.14.3 which lacks
BufferRecycler.releaseToPool()
+ required by Jersey 2.46 in Hadoop 3.5.0+. The BOM overrides ALL
transitive
+ jackson versions without needing per-library exclusions -->
+ <dependency>
+ <groupId>com.fasterxml.jackson</groupId>
+ <artifactId>jackson-bom</artifactId>
+ <version>${jackson.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
<dependency>
<groupId>org.apache.tez</groupId>
<artifactId>hadoop-shim</artifactId>
@@ -257,6 +275,16 @@
<groupId>org.apache.pig</groupId>
<artifactId>pig</artifactId>
<version>${pig.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>io.dropwizard.metrics</groupId>
@@ -306,8 +334,8 @@
<version>${netty.version}</version>
</dependency>
<dependency>
- <groupId>javax.servlet</groupId>
- <artifactId>javax.servlet-api</artifactId>
+ <groupId>jakarta.servlet</groupId>
+ <artifactId>jakarta.servlet-api</artifactId>
<version>${servlet-api.version}</version>
</dependency>
<dependency>
@@ -325,16 +353,12 @@
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-core</artifactId>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
</exclusion>
<exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-server</artifactId>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jdt</groupId>
@@ -392,6 +416,14 @@
<artifactId>hadoop-auth</artifactId>
<version>${hadoop.version}</version>
<exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
@@ -410,16 +442,12 @@
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-core</artifactId>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
</exclusion>
<exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-server</artifactId>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging-api</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
@@ -499,6 +527,14 @@
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
@@ -514,6 +550,14 @@
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-util</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -533,6 +577,14 @@
<groupId>org.eclipse.jetty</groupId>
<artifactId>*</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -551,6 +603,14 @@
<groupId>org.eclipse.jetty</groupId>
<artifactId>*</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -602,6 +662,14 @@
<groupId>com.google.inject.extensions</groupId>
<artifactId>guice-servlet</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
@@ -617,6 +685,14 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-common</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
@@ -637,6 +713,14 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-shuffle</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
@@ -666,6 +750,14 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-common</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
@@ -679,6 +771,14 @@
<type>test-jar</type>
<version>${hadoop.version}</version>
<exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
@@ -702,6 +802,14 @@
<groupId>org.eclipse.jetty</groupId>
<artifactId>*</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
@@ -714,6 +822,10 @@
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
<exclusion>
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
@@ -722,6 +834,14 @@
<groupId>tomcat</groupId>
<artifactId>jasper-runtime</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
@@ -773,6 +893,18 @@
<version>${junit.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ <version>${junit.jupiter.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.vintage</groupId>
+ <artifactId>junit-vintage-engine</artifactId>
+ <version>${junit.jupiter.version}</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
@@ -789,13 +921,13 @@
<version>${jettison.version}</version>
</dependency>
<dependency>
- <groupId>com.sun.jersey</groupId>
+ <groupId>org.glassfish.jersey.core</groupId>
<artifactId>jersey-client</artifactId>
<version>${jersey.version}</version>
</dependency>
<dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-json</artifactId>
+ <groupId>org.glassfish.jersey.media</groupId>
+ <artifactId>jersey-media-json-jackson</artifactId>
<version>${jersey.version}</version>
</dependency>
<!-- BouncyCastle should be in default scope to make their way to
tez.tar.gz since Hadoop 3.4.1 -->
diff --git a/tez-api/pom.xml b/tez-api/pom.xml
index 58da12ae2..187e87d99 100644
--- a/tez-api/pom.xml
+++ b/tez-api/pom.xml
@@ -94,12 +94,12 @@
<artifactId>junit</artifactId>
</dependency>
<dependency>
- <groupId>com.sun.jersey</groupId>
+ <groupId>org.glassfish.jersey.core</groupId>
<artifactId>jersey-client</artifactId>
</dependency>
<dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-json</artifactId>
+ <groupId>org.glassfish.jersey.media</groupId>
+ <artifactId>jersey-media-json-jackson</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
diff --git a/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java
b/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java
index 501cc62ba..54f50ac5f 100644
--- a/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java
+++ b/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java
@@ -27,15 +27,16 @@ public class ATSConstants {
/* Top level keys */
public static final String ENTITIES = "entities";
- public static final String ENTITY = "entity";
- public static final String ENTITY_TYPE = "entitytype";
+ public static final String ENTITY_ID = "entityId";
+ public static final String ENTITY_TYPE = "entityType";
public static final String EVENTS = "events";
- public static final String EVENT_TYPE = "eventtype";
+ public static final String EVENT_TYPE = "eventType";
public static final String TIMESTAMP = "ts";
- public static final String EVENT_INFO = "eventinfo";
+ public static final String EVENT_INFO = "eventInfo";
public static final String RELATED_ENTITIES = "relatedEntities";
- public static final String PRIMARY_FILTERS = "primaryfilters";
- public static final String OTHER_INFO = "otherinfo";
+ public static final String PRIMARY_FILTERS = "primaryFilters";
+ public static final String OTHER_INFO = "otherInfo";
+ public static final String DOMAIN_ID = "domainId";
/* Section for related entities */
public static final String APPLICATION_ID = "applicationId";
diff --git
a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java
b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java
index 9e0a8d0df..290490c03 100644
---
a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java
+++
b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java
@@ -29,6 +29,8 @@ import java.util.Map;
import java.util.Set;
import javax.annotation.Nullable;
+import javax.ws.rs.client.Client;
+import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.MediaType;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -52,11 +54,6 @@ import
org.apache.tez.dag.api.records.DAGProtos.VertexStatusStateProto;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.ClientHandlerException;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.UniformInterfaceException;
-import com.sun.jersey.api.client.WebResource;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
@@ -212,7 +209,7 @@ public class DAGClientTimelineImpl extends
DAGClientInternal {
@Override
public void close() throws IOException {
if (httpClient != null) {
- httpClient.destroy();
+ httpClient.close();
httpClient = null;
}
if (timelineReaderStrategy != null) {
@@ -426,26 +423,14 @@ public class DAGClientTimelineImpl extends
DAGClientInternal {
@VisibleForTesting
protected JSONObject getJsonRootEntity(String url) throws TezException {
try {
- WebResource wr = getCachedHttpClient().resource(url);
- ClientResponse response = wr.accept(MediaType.APPLICATION_JSON_TYPE)
- .type(MediaType.APPLICATION_JSON_TYPE)
- .get(ClientResponse.class);
-
- final ClientResponse.Status clientResponseStatus =
response.getClientResponseStatus();
- if (clientResponseStatus != ClientResponse.Status.OK) {
- throw new TezException("Failed to get response from YARN Timeline:" +
- " errorCode:" + clientResponseStatus + ", url:" + url);
- }
-
- return response.getEntity(JSONObject.class);
- } catch (ClientHandlerException e) {
+ Client client = getCachedHttpClient();
+ WebTarget target = client.target(url);
+ String json = target.request(MediaType.APPLICATION_JSON_TYPE)
+ .accept(MediaType.APPLICATION_JSON_TYPE)
+ .get(String.class);
+ return new JSONObject(json);
+ } catch (Exception e) {
throw new TezException("Error processing response from YARN Timeline",
e);
- } catch (UniformInterfaceException e) {
- throw new TezException("Error accessing content from YARN Timeline -
unexpected response", e);
- } catch (IllegalArgumentException e) {
- throw new TezException("Error accessing content from YARN Timeline -
invalid url", e);
- } catch (IOException e) {
- throw new TezException("Error failed to get http client", e);
}
}
diff --git
a/tez-api/src/main/java/org/apache/tez/dag/api/client/TimelineReaderFactory.java
b/tez-api/src/main/java/org/apache/tez/dag/api/client/TimelineReaderFactory.java
index 17c84928f..2e9a3caf6 100644
---
a/tez-api/src/main/java/org/apache/tez/dag/api/client/TimelineReaderFactory.java
+++
b/tez-api/src/main/java/org/apache/tez/dag/api/client/TimelineReaderFactory.java
@@ -31,11 +31,12 @@ import java.security.GeneralSecurityException;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLSocketFactory;
+import javax.ws.rs.client.Client;
+import javax.ws.rs.client.ClientBuilder;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
import org.apache.hadoop.security.authentication.client.Authenticator;
import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
import org.apache.hadoop.security.ssl.SSLFactory;
@@ -43,13 +44,10 @@ import org.apache.tez.common.ReflectionUtils;
import org.apache.tez.dag.api.TezException;
import com.google.common.annotations.VisibleForTesting;
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.config.ClientConfig;
-import com.sun.jersey.api.client.config.DefaultClientConfig;
-import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
-import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
-import com.sun.jersey.json.impl.provider.entity.JSONRootElementProvider;
+import org.glassfish.jersey.client.ClientConfig;
+import org.glassfish.jersey.client.HttpUrlConnectorProvider;
+import org.glassfish.jersey.jackson.JacksonFeature;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -165,7 +163,7 @@ public final class TimelineReaderFactory {
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
UserGroupInformation realUgi = ugi.getRealUser();
String doAsUser;
- ClientConfig clientConfig = new
DefaultClientConfig(JSONRootElementProvider.App.class);
+ ClientConfig clientConfig = new
ClientConfig().register(JacksonFeature.class);
ConnectionConfigurator connectionConfigurator =
getNewConnectionConf(useHttps,
connTimeout, sslFactory);
@@ -182,14 +180,16 @@ public final class TimelineReaderFactory {
doAsUser = null;
}
- HttpURLConnectionFactory connectionFactory;
+ HttpUrlConnectorProvider.ConnectionFactory connectionFactory;
try {
connectionFactory = new
TokenAuthenticatedURLConnectionFactory(connectionConfigurator, authenticator,
doAsUser);
} catch (TezException e) {
throw new IOException("Fail to create
TokenAuthenticatedURLConnectionFactory", e);
}
- return new Client(new URLConnectionClientHandler(connectionFactory),
clientConfig);
+
+ clientConfig.connectorProvider(new
HttpUrlConnectorProvider().connectionFactory(connectionFactory));
+ return ClientBuilder.newClient(clientConfig);
}
private static Authenticator getTokenAuthenticator() throws TezException {
@@ -204,12 +204,12 @@ public final class TimelineReaderFactory {
return ReflectionUtils.createClazzInstance(authenticatorClazzName);
}
- private static class TokenAuthenticatedURLConnectionFactory implements
HttpURLConnectionFactory {
+ private static class TokenAuthenticatedURLConnectionFactory implements
HttpUrlConnectorProvider.ConnectionFactory {
private final Authenticator authenticator;
private final ConnectionConfigurator connConfigurator;
private final String doAsUser;
- private final AuthenticatedURL.Token token;
+ private final Object token;
public TokenAuthenticatedURLConnectionFactory(ConnectionConfigurator
connConfigurator,
Authenticator
authenticator,
@@ -222,9 +222,9 @@ public final class TimelineReaderFactory {
}
@Override
- public HttpURLConnection getHttpURLConnection(URL url) throws
IOException {
+ public HttpURLConnection getConnection(URL url) throws IOException {
try {
- AuthenticatedURL authenticatedURL=
ReflectionUtils.createClazzInstance(
+ HttpURLConnection authenticatedURL =
ReflectionUtils.createClazzInstance(
DELEGATION_TOKEN_AUTHENTICATED_URL_CLAZZ_NAME, new Class[] {
delegationTokenAuthenticatorClazz,
ConnectionConfigurator.class
@@ -266,13 +266,16 @@ public final class TimelineReaderFactory {
@Override
public Client getHttpClient() {
- ClientConfig config = new
DefaultClientConfig(JSONRootElementProvider.App.class);
- HttpURLConnectionFactory urlFactory = new
PseudoAuthenticatedURLConnectionFactory(connectionConf);
- return new Client(new URLConnectionClientHandler(urlFactory), config);
+ ClientConfig config = new ClientConfig().register(JacksonFeature.class);
+ HttpUrlConnectorProvider.ConnectionFactory urlFactory =
+ new PseudoAuthenticatedURLConnectionFactory(connectionConf);
+ config.connectorProvider(new
HttpUrlConnectorProvider().connectionFactory(urlFactory));
+ return ClientBuilder.newClient(config);
}
@VisibleForTesting
- protected static class PseudoAuthenticatedURLConnectionFactory implements
HttpURLConnectionFactory {
+ protected static class PseudoAuthenticatedURLConnectionFactory
+ implements HttpUrlConnectorProvider.ConnectionFactory {
private final ConnectionConfigurator connectionConf;
public PseudoAuthenticatedURLConnectionFactory(ConnectionConfigurator
connectionConf) {
@@ -280,7 +283,7 @@ public final class TimelineReaderFactory {
}
@Override
- public HttpURLConnection getHttpURLConnection(URL url) throws
IOException {
+ public HttpURLConnection getConnection(URL url) throws IOException {
String tokenString = (url.getQuery() == null ? "?" : "&") +
"user.name=" +
URLEncoder.encode(UserGroupInformation.getCurrentUser().getShortUserName(),
"UTF8");
diff --git
a/tez-api/src/test/java/org/apache/tez/dag/api/client/TestATSHttpClient.java
b/tez-api/src/test/java/org/apache/tez/dag/api/client/TestATSHttpClient.java
index a4cab8f1f..175c136a1 100644
--- a/tez-api/src/test/java/org/apache/tez/dag/api/client/TestATSHttpClient.java
+++ b/tez-api/src/test/java/org/apache/tez/dag/api/client/TestATSHttpClient.java
@@ -95,7 +95,7 @@ public class TestATSHttpClient {
final String jsonDagData =
"{ " +
- " otherinfo: { " +
+ " otherInfo: { " +
" status: 'SUCCEEDED'," +
" diagnostics: 'SAMPLE_DIAGNOSTICS'," +
" counters: { counterGroups: [ " +
@@ -108,9 +108,9 @@ public class TestATSHttpClient {
"}";
final String jsonVertexData = "{entities:[ " +
- "{otherinfo: {vertexName:'v1',
numTasks:5,numFailedTasks:1,numSucceededTasks:2," +
+ "{otherInfo: {vertexName:'v1',
numTasks:5,numFailedTasks:1,numSucceededTasks:2," +
"numKilledTasks:3,numCompletedTasks:3}}," +
- "{otherinfo:
{vertexName:'v2',numTasks:10,numFailedTasks:1,numSucceededTasks:5," +
+ "{otherInfo:
{vertexName:'v2',numTasks:10,numFailedTasks:1,numSucceededTasks:5," +
"numKilledTasks:3,numCompletedTasks:4}}" +
"]}";
@@ -151,7 +151,7 @@ public class TestATSHttpClient {
Set<StatusGetOpts> statusOptions = new HashSet<StatusGetOpts>(1);
statusOptions.add(StatusGetOpts.GET_COUNTERS);
- final String jsonData = "{entities:[
{otherinfo:{numFailedTasks:1,numSucceededTasks:2," +
+ final String jsonData = "{entities:[
{otherInfo:{numFailedTasks:1,numSucceededTasks:2," +
"status:'SUCCEEDED', vertexName:'vertex1name', numTasks:4,
numKilledTasks: 3, " +
"numCompletedTasks: 4, diagnostics: 'diagnostics1', " +
"counters: { counterGroups: [ " +
diff --git
a/tez-api/src/test/java/org/apache/tez/dag/api/client/TestTimelineReaderFactory.java
b/tez-api/src/test/java/org/apache/tez/dag/api/client/TestTimelineReaderFactory.java
index 2f514badf..e79950b0b 100644
---
a/tez-api/src/test/java/org/apache/tez/dag/api/client/TestTimelineReaderFactory.java
+++
b/tez-api/src/test/java/org/apache/tez/dag/api/client/TestTimelineReaderFactory.java
@@ -52,10 +52,10 @@ public class TestTimelineReaderFactory {
ConnectionConfigurator connConf = mock(ConnectionConfigurator.class);
TimelineReaderPseudoAuthenticatedStrategy.PseudoAuthenticatedURLConnectionFactory
connectionFactory = new TimelineReaderPseudoAuthenticatedStrategy
- .PseudoAuthenticatedURLConnectionFactory(connConf);
+ .PseudoAuthenticatedURLConnectionFactory(connConf);
String inputUrl = "http://host:8080/path";;
String expectedUrl = inputUrl + "?user.name=" +
UserGroupInformation.getCurrentUser().getShortUserName();
- HttpURLConnection httpURLConnection =
connectionFactory.getHttpURLConnection(URI.create(inputUrl).toURL());
+ HttpURLConnection httpURLConnection =
connectionFactory.getConnection(URI.create(inputUrl).toURL());
Assert.assertEquals(expectedUrl, httpURLConnection.getURL().toString());
}
diff --git a/tez-dag/pom.xml b/tez-dag/pom.xml
index a02611421..6f05d1b7e 100644
--- a/tez-dag/pom.xml
+++ b/tez-dag/pom.xml
@@ -146,8 +146,8 @@
<artifactId>jettison</artifactId>
</dependency>
<dependency>
- <groupId>javax.servlet</groupId>
- <artifactId>javax.servlet-api</artifactId>
+ <groupId>jakarta.servlet</groupId>
+ <artifactId>jakarta.servlet-api</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
@@ -161,6 +161,11 @@
<artifactId>curator-test</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
diff --git
a/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/HistoryEventJsonConversion.java
b/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/HistoryEventJsonConversion.java
index c32f847e5..34f90fb65 100644
---
a/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/HistoryEventJsonConversion.java
+++
b/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/HistoryEventJsonConversion.java
@@ -133,7 +133,7 @@ public final class HistoryEventJsonConversion {
private static JSONObject convertDAGRecoveredEvent(DAGRecoveredEvent event)
throws JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY,
+ jsonObject.put(ATSConstants.ENTITY_ID,
event.getDagID().toString());
jsonObject.put(ATSConstants.ENTITY_TYPE,
EntityTypes.TEZ_DAG_ID.name());
@@ -168,7 +168,7 @@ public final class HistoryEventJsonConversion {
private static JSONObject convertAppLaunchedEvent(AppLaunchedEvent event)
throws JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY,
+ jsonObject.put(ATSConstants.ENTITY_ID,
"tez_" + event.getApplicationId().toString());
jsonObject.put(ATSConstants.ENTITY_TYPE,
EntityTypes.TEZ_APPLICATION.name());
@@ -186,7 +186,7 @@ public final class HistoryEventJsonConversion {
private static JSONObject convertAMLaunchedEvent(AMLaunchedEvent event)
throws JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY,
+ jsonObject.put(ATSConstants.ENTITY_ID,
"tez_" + event.getApplicationAttemptId().toString());
jsonObject.put(ATSConstants.ENTITY_TYPE,
EntityTypes.TEZ_APPLICATION_ATTEMPT.name());
@@ -194,12 +194,12 @@ public final class HistoryEventJsonConversion {
// Related Entities
JSONArray relatedEntities = new JSONArray();
JSONObject appEntity = new JSONObject();
- appEntity.put(ATSConstants.ENTITY,
+ appEntity.put(ATSConstants.ENTITY_ID,
event.getApplicationAttemptId().getApplicationId().toString());
appEntity.put(ATSConstants.ENTITY_TYPE,
ATSConstants.APPLICATION_ID);
JSONObject appAttemptEntity = new JSONObject();
- appAttemptEntity.put(ATSConstants.ENTITY,
+ appAttemptEntity.put(ATSConstants.ENTITY_ID,
event.getApplicationAttemptId().toString());
appAttemptEntity.put(ATSConstants.ENTITY_TYPE,
ATSConstants.APPLICATION_ATTEMPT_ID);
@@ -227,7 +227,7 @@ public final class HistoryEventJsonConversion {
private static JSONObject convertAMStartedEvent(AMStartedEvent event) throws
JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY,
+ jsonObject.put(ATSConstants.ENTITY_ID,
"tez_" + event.getApplicationAttemptId().toString());
jsonObject.put(ATSConstants.ENTITY_TYPE,
EntityTypes.TEZ_APPLICATION_ATTEMPT.name());
@@ -235,12 +235,12 @@ public final class HistoryEventJsonConversion {
// Related Entities
JSONArray relatedEntities = new JSONArray();
JSONObject appEntity = new JSONObject();
- appEntity.put(ATSConstants.ENTITY,
+ appEntity.put(ATSConstants.ENTITY_ID,
event.getApplicationAttemptId().getApplicationId().toString());
appEntity.put(ATSConstants.ENTITY_TYPE,
ATSConstants.APPLICATION_ID);
JSONObject appAttemptEntity = new JSONObject();
- appAttemptEntity.put(ATSConstants.ENTITY,
+ appAttemptEntity.put(ATSConstants.ENTITY_ID,
event.getApplicationAttemptId().toString());
appAttemptEntity.put(ATSConstants.ENTITY_TYPE,
ATSConstants.APPLICATION_ATTEMPT_ID);
@@ -262,20 +262,20 @@ public final class HistoryEventJsonConversion {
private static JSONObject
convertContainerLaunchedEvent(ContainerLaunchedEvent event) throws
JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY,
+ jsonObject.put(ATSConstants.ENTITY_ID,
"tez_" + event.getContainerId().toString());
jsonObject.put(ATSConstants.ENTITY_TYPE,
EntityTypes.TEZ_CONTAINER_ID.name());
JSONArray relatedEntities = new JSONArray();
JSONObject appAttemptEntity = new JSONObject();
- appAttemptEntity.put(ATSConstants.ENTITY,
+ appAttemptEntity.put(ATSConstants.ENTITY_ID,
event.getApplicationAttemptId().toString());
appAttemptEntity.put(ATSConstants.ENTITY_TYPE,
EntityTypes.TEZ_APPLICATION_ATTEMPT.name());
JSONObject containerEntity = new JSONObject();
- containerEntity.put(ATSConstants.ENTITY,
event.getContainerId().toString());
+ containerEntity.put(ATSConstants.ENTITY_ID,
event.getContainerId().toString());
containerEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.CONTAINER_ID);
relatedEntities.put(appAttemptEntity);
@@ -301,20 +301,20 @@ public final class HistoryEventJsonConversion {
private static JSONObject convertContainerStoppedEvent(ContainerStoppedEvent
event) throws JSONException {
// structure is identical to ContainerLaunchedEvent
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY,
+ jsonObject.put(ATSConstants.ENTITY_ID,
"tez_" + event.getContainerId().toString());
jsonObject.put(ATSConstants.ENTITY_TYPE,
EntityTypes.TEZ_CONTAINER_ID.name());
JSONArray relatedEntities = new JSONArray();
JSONObject appAttemptEntity = new JSONObject();
- appAttemptEntity.put(ATSConstants.ENTITY,
+ appAttemptEntity.put(ATSConstants.ENTITY_ID,
event.getApplicationAttemptId().toString());
appAttemptEntity.put(ATSConstants.ENTITY_TYPE,
EntityTypes.TEZ_APPLICATION_ATTEMPT.name());
JSONObject containerEntity = new JSONObject();
- containerEntity.put(ATSConstants.ENTITY,
event.getContainerId().toString());
+ containerEntity.put(ATSConstants.ENTITY_ID,
event.getContainerId().toString());
containerEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.CONTAINER_ID);
relatedEntities.put(appAttemptEntity);
@@ -343,7 +343,7 @@ public final class HistoryEventJsonConversion {
private static JSONObject convertDAGFinishedEvent(DAGFinishedEvent event)
throws JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY,
+ jsonObject.put(ATSConstants.ENTITY_ID,
event.getDAGID().toString());
jsonObject.put(ATSConstants.ENTITY_TYPE,
EntityTypes.TEZ_DAG_ID.name());
@@ -386,7 +386,7 @@ public final class HistoryEventJsonConversion {
private static JSONObject convertDAGInitializedEvent(DAGInitializedEvent
event) throws JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY,
+ jsonObject.put(ATSConstants.ENTITY_ID,
event.getDAGID().toString());
jsonObject.put(ATSConstants.ENTITY_TYPE,
EntityTypes.TEZ_DAG_ID.name());
@@ -418,7 +418,7 @@ public final class HistoryEventJsonConversion {
private static JSONObject convertDAGStartedEvent(DAGStartedEvent event)
throws JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY,
+ jsonObject.put(ATSConstants.ENTITY_ID,
event.getDAGID().toString());
jsonObject.put(ATSConstants.ENTITY_TYPE,
EntityTypes.TEZ_DAG_ID.name());
@@ -441,7 +441,7 @@ public final class HistoryEventJsonConversion {
private static JSONObject convertDAGSubmittedEvent(DAGSubmittedEvent event)
throws JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY,
+ jsonObject.put(ATSConstants.ENTITY_ID,
event.getDAGID().toString());
jsonObject.put(ATSConstants.ENTITY_TYPE,
EntityTypes.TEZ_DAG_ID.name());
@@ -449,27 +449,27 @@ public final class HistoryEventJsonConversion {
// Related Entities
JSONArray relatedEntities = new JSONArray();
JSONObject tezAppEntity = new JSONObject();
- tezAppEntity.put(ATSConstants.ENTITY,
+ tezAppEntity.put(ATSConstants.ENTITY_ID,
"tez_" +
event.getApplicationAttemptId().getApplicationId().toString());
tezAppEntity.put(ATSConstants.ENTITY_TYPE,
EntityTypes.TEZ_APPLICATION.name());
JSONObject tezAppAttemptEntity = new JSONObject();
- tezAppAttemptEntity.put(ATSConstants.ENTITY,
+ tezAppAttemptEntity.put(ATSConstants.ENTITY_ID,
"tez_" + event.getApplicationAttemptId().toString());
tezAppAttemptEntity.put(ATSConstants.ENTITY_TYPE,
EntityTypes.TEZ_APPLICATION_ATTEMPT.name());
JSONObject appEntity = new JSONObject();
- appEntity.put(ATSConstants.ENTITY,
+ appEntity.put(ATSConstants.ENTITY_ID,
event.getApplicationAttemptId().getApplicationId().toString());
appEntity.put(ATSConstants.ENTITY_TYPE,
ATSConstants.APPLICATION_ID);
JSONObject appAttemptEntity = new JSONObject();
- appAttemptEntity.put(ATSConstants.ENTITY,
+ appAttemptEntity.put(ATSConstants.ENTITY_ID,
event.getApplicationAttemptId().toString());
appAttemptEntity.put(ATSConstants.ENTITY_TYPE,
ATSConstants.APPLICATION_ATTEMPT_ID);
JSONObject userEntity = new JSONObject();
- userEntity.put(ATSConstants.ENTITY,
+ userEntity.put(ATSConstants.ENTITY_ID,
event.getUser());
userEntity.put(ATSConstants.ENTITY_TYPE,
ATSConstants.USER);
@@ -531,7 +531,7 @@ public final class HistoryEventJsonConversion {
private static JSONObject
convertTaskAttemptFinishedEvent(TaskAttemptFinishedEvent event) throws
JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY, event.getTaskAttemptID().toString());
+ jsonObject.put(ATSConstants.ENTITY_ID,
event.getTaskAttemptID().toString());
jsonObject.put(ATSConstants.ENTITY_TYPE,
EntityTypes.TEZ_TASK_ATTEMPT_ID.name());
@@ -590,22 +590,22 @@ public final class HistoryEventJsonConversion {
private static JSONObject
convertTaskAttemptStartedEvent(TaskAttemptStartedEvent event) throws
JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY, event.getTaskAttemptID().toString());
+ jsonObject.put(ATSConstants.ENTITY_ID,
event.getTaskAttemptID().toString());
jsonObject.put(ATSConstants.ENTITY_TYPE,
EntityTypes.TEZ_TASK_ATTEMPT_ID.name());
// Related entities
JSONArray relatedEntities = new JSONArray();
JSONObject nodeEntity = new JSONObject();
- nodeEntity.put(ATSConstants.ENTITY, event.getNodeId().toString());
+ nodeEntity.put(ATSConstants.ENTITY_ID, event.getNodeId().toString());
nodeEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.NODE_ID);
JSONObject containerEntity = new JSONObject();
- containerEntity.put(ATSConstants.ENTITY,
event.getContainerId().toString());
+ containerEntity.put(ATSConstants.ENTITY_ID,
event.getContainerId().toString());
containerEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.CONTAINER_ID);
JSONObject taskEntity = new JSONObject();
- taskEntity.put(ATSConstants.ENTITY, event.getTaskID().toString());
+ taskEntity.put(ATSConstants.ENTITY_ID, event.getTaskID().toString());
taskEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_TASK_ID.name());
relatedEntities.put(nodeEntity);
@@ -633,7 +633,7 @@ public final class HistoryEventJsonConversion {
private static JSONObject convertTaskFinishedEvent(TaskFinishedEvent event)
throws JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY, event.getTaskID().toString());
+ jsonObject.put(ATSConstants.ENTITY_ID, event.getTaskID().toString());
jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_TASK_ID.name());
// Events
@@ -664,13 +664,13 @@ public final class HistoryEventJsonConversion {
private static JSONObject convertTaskStartedEvent(TaskStartedEvent event)
throws JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY, event.getTaskID().toString());
+ jsonObject.put(ATSConstants.ENTITY_ID, event.getTaskID().toString());
jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_TASK_ID.name());
// Related entities
JSONArray relatedEntities = new JSONArray();
JSONObject vertexEntity = new JSONObject();
- vertexEntity.put(ATSConstants.ENTITY, event.getVertexID().toString());
+ vertexEntity.put(ATSConstants.ENTITY_ID, event.getVertexID().toString());
vertexEntity.put(ATSConstants.ENTITY_TYPE,
EntityTypes.TEZ_VERTEX_ID.name());
relatedEntities.put(vertexEntity);
jsonObject.put(ATSConstants.RELATED_ENTITIES, relatedEntities);
@@ -696,7 +696,7 @@ public final class HistoryEventJsonConversion {
private static JSONObject convertVertexFinishedEvent(VertexFinishedEvent
event) throws JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY, event.getVertexID().toString());
+ jsonObject.put(ATSConstants.ENTITY_ID, event.getVertexID().toString());
jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_VERTEX_ID.name());
// Events
@@ -737,7 +737,7 @@ public final class HistoryEventJsonConversion {
private static JSONObject
convertVertexReconfigureDoneEvent(VertexConfigurationDoneEvent event) throws
JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY, event.getVertexID().toString());
+ jsonObject.put(ATSConstants.ENTITY_ID, event.getVertexID().toString());
jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_VERTEX_ID.name());
// Events
@@ -772,13 +772,13 @@ public final class HistoryEventJsonConversion {
private static JSONObject
convertVertexInitializedEvent(VertexInitializedEvent event) throws
JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY, event.getVertexID().toString());
+ jsonObject.put(ATSConstants.ENTITY_ID, event.getVertexID().toString());
jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_VERTEX_ID.name());
// Related entities
JSONArray relatedEntities = new JSONArray();
JSONObject vertexEntity = new JSONObject();
- vertexEntity.put(ATSConstants.ENTITY, event.getDAGID().toString());
+ vertexEntity.put(ATSConstants.ENTITY_ID, event.getDAGID().toString());
vertexEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name());
relatedEntities.put(vertexEntity);
jsonObject.put(ATSConstants.RELATED_ENTITIES, relatedEntities);
@@ -812,13 +812,13 @@ public final class HistoryEventJsonConversion {
private static JSONObject convertVertexStartedEvent(VertexStartedEvent event)
throws JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY, event.getVertexID().toString());
+ jsonObject.put(ATSConstants.ENTITY_ID, event.getVertexID().toString());
jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_VERTEX_ID.name());
// Related entities
JSONArray relatedEntities = new JSONArray();
JSONObject vertexEntity = new JSONObject();
- vertexEntity.put(ATSConstants.ENTITY, event.getDAGID().toString());
+ vertexEntity.put(ATSConstants.ENTITY_ID, event.getDAGID().toString());
vertexEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name());
relatedEntities.put(vertexEntity);
jsonObject.put(ATSConstants.RELATED_ENTITIES, relatedEntities);
diff --git
a/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java
b/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java
index fa3d734ba..0d37a137e 100644
---
a/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java
+++
b/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java
@@ -231,7 +231,7 @@ public class TestHistoryEventJsonConversion {
JSONObject jsonObject = HistoryEventJsonConversion.convertToJson(event);
Assert.assertNotNull(jsonObject);
- Assert.assertEquals(vId.toString(),
jsonObject.getString(ATSConstants.ENTITY));
+ Assert.assertEquals(vId.toString(),
jsonObject.getString(ATSConstants.ENTITY_ID));
Assert.assertEquals(ATSConstants.TEZ_VERTEX_ID,
jsonObject.get(ATSConstants.ENTITY_TYPE));
JSONArray events = jsonObject.getJSONArray(ATSConstants.EVENTS);
diff --git a/tez-ext-service-tests/pom.xml b/tez-ext-service-tests/pom.xml
index d5a6bfc08..d39bd1334 100644
--- a/tez-ext-service-tests/pom.xml
+++ b/tez-ext-service-tests/pom.xml
@@ -112,7 +112,11 @@
<artifactId>hadoop-mapreduce-client-shuffle</artifactId>
<scope>test</scope>
</dependency>
-
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
diff --git a/tez-plugins/tez-aux-services/pom.xml
b/tez-plugins/tez-aux-services/pom.xml
index 61535adc5..efa6fb107 100644
--- a/tez-plugins/tez-aux-services/pom.xml
+++ b/tez-plugins/tez-aux-services/pom.xml
@@ -126,6 +126,11 @@
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
diff --git a/tez-plugins/tez-history-parser/pom.xml
b/tez-plugins/tez-history-parser/pom.xml
index d50a16dab..d299f88e4 100644
--- a/tez-plugins/tez-history-parser/pom.xml
+++ b/tez-plugins/tez-history-parser/pom.xml
@@ -148,13 +148,24 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-json</artifactId>
+ <groupId>org.glassfish.jersey.media</groupId>
+ <artifactId>jersey-media-json-jackson</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-shuffle</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.vintage</groupId>
+ <artifactId>junit-vintage-engine</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
diff --git
a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/ATSImportTool.java
b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/ATSImportTool.java
index 9357d5011..574d2f137 100644
---
a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/ATSImportTool.java
+++
b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/ATSImportTool.java
@@ -28,12 +28,17 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
import java.util.Iterator;
import java.util.Objects;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
+import javax.ws.rs.client.Client;
+import javax.ws.rs.client.ClientBuilder;
+import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.DefaultParser;
@@ -42,7 +47,6 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.io.LineIterator;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.http.HttpConfig;
@@ -60,20 +64,12 @@ import org.apache.tez.history.parser.utils.Utils;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.google.common.base.Strings;
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.ClientHandlerException;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.UniformInterfaceException;
-import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.api.client.config.ClientConfig;
-import com.sun.jersey.api.client.config.DefaultClientConfig;
-import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
-import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
-import com.sun.jersey.json.impl.provider.entity.JSONRootElementProvider;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
+import org.glassfish.jersey.client.ClientConfig;
+import org.glassfish.jersey.client.HttpUrlConnectorProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -169,7 +165,7 @@ public class ATSImportTool extends Configured implements
Tool {
throw e;
} finally {
if (httpClient != null) {
- httpClient.destroy();
+ httpClient.close();
}
IOUtils.closeQuietly(fos);
}
@@ -279,72 +275,64 @@ public class ATSImportTool extends Configured implements
Tool {
//Set the last item in entities as the fromId
url = baseUrl + "&fromId="
- + entities.getJSONObject(entities.length() -
1).getString(Constants.ENTITY);
+ + entities.getJSONObject(entities.length() -
1).getString(Constants.ENTITY_ID);
- String firstItem = entities.getJSONObject(0).getString(Constants.ENTITY);
- String lastItem = entities.getJSONObject(entities.length() -
1).getString(Constants.ENTITY);
+ String firstItem =
entities.getJSONObject(0).getString(Constants.ENTITY_ID);
+ String lastItem = entities.getJSONObject(entities.length() -
1).getString(Constants.ENTITY_ID);
LOG.info("Downloaded={}, First item={}, LastItem={}, new url={}",
downloadedCount,
firstItem, lastItem, url);
}
}
- private void logErrorMessage(ClientResponse response) throws IOException {
- LOG.error("Response status={}",
response.getClientResponseStatus().toString());
- LineIterator it = null;
+ private void logErrorMessage(Response response) {
+ LOG.info("Response status={}", Integer.toString(response.getStatus()));
try {
- it = IOUtils.lineIterator(response.getEntityInputStream(), UTF8);
- while (it.hasNext()) {
- String line = it.nextLine();
- LOG.error(line);
- }
- } finally {
- if (it != null) {
- it.close();
+ String entity = response.readEntity(String.class);
+ if (entity != null) {
+ LOG.info("Response entity={}", entity);
}
+ } catch (Exception ignore) {
+ LOG.debug("Failed to read error response entity", ignore);
}
}
//For secure cluster, this should work as long as valid ticket is available
in the node.
private JSONObject getJsonRootEntity(String url) throws TezException,
IOException {
try {
- WebResource wr = getHttpClient().resource(url);
- ClientResponse response = wr.accept(MediaType.APPLICATION_JSON_TYPE)
- .type(MediaType.APPLICATION_JSON_TYPE)
- .get(ClientResponse.class);
+ WebTarget target = getHttpClient().target(url);
+ Response response = target.request(MediaType.APPLICATION_JSON_TYPE)
+ .accept(MediaType.APPLICATION_JSON_TYPE)
+ .get();
- if (response.getClientResponseStatus() != ClientResponse.Status.OK) {
+ if (response.getStatus() != Response.Status.OK.getStatusCode()) {
// In the case of secure cluster, if there is any auth exception it
sends the data back as
// a html page and JSON parsing could throw exceptions. Instead, get
the stream contents
// completely and log it in case of error.
logErrorMessage(response);
throw new TezException("Failed to get response from YARN Timeline:
url: " + url);
}
- return response.getEntity(JSONObject.class);
- } catch (ClientHandlerException e) {
+ String json = response.readEntity(String.class);
+ return new JSONObject(json);
+ } catch (Exception e) {
throw new TezException("Error processing response from YARN Timeline.
URL=" + url, e);
- } catch (UniformInterfaceException e) {
- throw new TezException("Error accessing content from YARN Timeline -
unexpected response. "
- + "URL=" + url, e);
- } catch (IllegalArgumentException e) {
- throw new TezException("Error accessing content from YARN Timeline -
invalid url. URL=" + url,
- e);
}
}
private Client getHttpClient() {
if (httpClient == null) {
- ClientConfig config = new
DefaultClientConfig(JSONRootElementProvider.App.class);
- HttpURLConnectionFactory urlFactory = new
PseudoAuthenticatedURLConnectionFactory();
- return new Client(new URLConnectionClientHandler(urlFactory), config);
+ ClientConfig config = new ClientConfig();
+ config.connectorProvider(new HttpUrlConnectorProvider()
+ .connectionFactory(new PseudoAuthenticatedURLConnectionFactory()));
+ return ClientBuilder.newClient(config);
}
return httpClient;
}
- static class PseudoAuthenticatedURLConnectionFactory implements
HttpURLConnectionFactory {
+ static class PseudoAuthenticatedURLConnectionFactory implements
HttpUrlConnectorProvider.ConnectionFactory {
@Override
- public HttpURLConnection getHttpURLConnection(URL url) throws IOException {
+ public HttpURLConnection getConnection(URL url) throws IOException {
String tokenString = (url.getQuery() == null ? "?" : "&") + "user.name="
+
-
URLEncoder.encode(UserGroupInformation.getCurrentUser().getShortUserName(),
"UTF8");
+
URLEncoder.encode(UserGroupInformation.getCurrentUser().getShortUserName(),
StandardCharsets.UTF_8);
return (HttpURLConnection) URI.create(url.toString() +
tokenString).toURL().openConnection();
}
}
diff --git
a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/SimpleHistoryParser.java
b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/SimpleHistoryParser.java
index b26184ad0..8279c6974 100644
---
a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/SimpleHistoryParser.java
+++
b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/SimpleHistoryParser.java
@@ -180,7 +180,7 @@ public class SimpleHistoryParser extends BaseParser {
if (relatedEntities == null) {
//This can happen when CONTAINER_EXITED abruptly. (e.g Container
failed, exitCode=1)
LOG.debug("entity {} did not have related entities",
- jsonObject.optJSONObject(Constants.ENTITY));
+ jsonObject.optJSONObject(Constants.ENTITY_ID));
} else {
JSONObject subJsonObject = relatedEntities.optJSONObject(0);
if (subJsonObject != null) {
@@ -188,7 +188,7 @@ public class SimpleHistoryParser extends BaseParser {
if (!Strings.isNullOrEmpty(nodeId) &&
nodeId.equalsIgnoreCase(Constants.NODE_ID)) {
//populate it in otherInfo
JSONObject otherInfo =
jsonObject.optJSONObject(Constants.OTHER_INFO);
- String nodeIdVal = subJsonObject.optString(Constants.ENTITY);
+ String nodeIdVal = subJsonObject.optString(Constants.ENTITY_ID);
if (otherInfo != null && nodeIdVal != null) {
otherInfo.put(Constants.NODE_ID, nodeIdVal);
}
@@ -202,7 +202,7 @@ public class SimpleHistoryParser extends BaseParser {
.equalsIgnoreCase(Constants.CONTAINER_ID)) {
//populate it in otherInfo
JSONObject otherInfo =
jsonObject.optJSONObject(Constants.OTHER_INFO);
- String containerIdVal = subJsonObject.optString(Constants.ENTITY);
+ String containerIdVal =
subJsonObject.optString(Constants.ENTITY_ID);
if (otherInfo != null && containerIdVal != null) {
otherInfo.put(Constants.CONTAINER_ID, containerIdVal);
}
@@ -225,7 +225,7 @@ public class SimpleHistoryParser extends BaseParser {
while (source.hasNext()) {
JSONObject jsonObject = source.next();
- String entity = jsonObject.getString(Constants.ENTITY);
+ String entity = jsonObject.getString(Constants.ENTITY_ID);
String entityType = jsonObject.getString(Constants.ENTITY_TYPE);
switch (entityType) {
case Constants.TEZ_DAG_ID:
@@ -255,7 +255,7 @@ public class SimpleHistoryParser extends BaseParser {
JSONObject subEntity = relatedEntities.getJSONObject(i);
String subEntityType = subEntity.optString(Constants.ENTITY_TYPE);
if (subEntityType != null && subEntityType.equals(Constants.USER))
{
- userName = subEntity.getString(Constants.ENTITY);
+ userName = subEntity.getString(Constants.ENTITY_ID);
break;
}
}
diff --git
a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java
b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java
index 802b189cf..13e04e323 100644
---
a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java
+++
b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java
@@ -102,7 +102,7 @@ public class DagInfo extends BaseInfo {
Preconditions.checkArgument(jsonObject.getString(Constants.ENTITY_TYPE).equalsIgnoreCase
(Constants.TEZ_DAG_ID));
- dagId = StringInterner.intern(jsonObject.getString(Constants.ENTITY));
+ dagId = StringInterner.intern(jsonObject.getString(Constants.ENTITY_ID));
//Parse additional Info
JSONObject otherInfoNode = jsonObject.getJSONObject(Constants.OTHER_INFO);
diff --git
a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskAttemptInfo.java
b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskAttemptInfo.java
index f88918a1e..b2e958321 100644
---
a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskAttemptInfo.java
+++
b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskAttemptInfo.java
@@ -98,7 +98,7 @@ public class TaskAttemptInfo extends BaseInfo {
jsonObject.getString(Constants.ENTITY_TYPE).equalsIgnoreCase
(Constants.TEZ_TASK_ATTEMPT_ID));
- taskAttemptId =
StringInterner.intern(jsonObject.optString(Constants.ENTITY));
+ taskAttemptId =
StringInterner.intern(jsonObject.optString(Constants.ENTITY_ID));
//Parse additional Info
final JSONObject otherInfoNode =
jsonObject.getJSONObject(Constants.OTHER_INFO);
diff --git
a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskInfo.java
b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskInfo.java
index 7810bd61f..819d6908c 100644
---
a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskInfo.java
+++
b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskInfo.java
@@ -73,7 +73,7 @@ public class TaskInfo extends BaseInfo {
jsonObject.getString(Constants.ENTITY_TYPE).equalsIgnoreCase
(Constants.TEZ_TASK_ID));
- taskId = StringInterner.intern(jsonObject.optString(Constants.ENTITY));
+ taskId = StringInterner.intern(jsonObject.optString(Constants.ENTITY_ID));
//Parse additional Info
final JSONObject otherInfoNode =
jsonObject.getJSONObject(Constants.OTHER_INFO);
diff --git
a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/VertexInfo.java
b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/VertexInfo.java
index bde42d17d..b061d8183 100644
---
a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/VertexInfo.java
+++
b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/VertexInfo.java
@@ -94,7 +94,7 @@ public class VertexInfo extends BaseInfo {
jsonObject.getString(Constants.ENTITY_TYPE).equalsIgnoreCase
(Constants.TEZ_VERTEX_ID));
- vertexId = StringInterner.intern(jsonObject.optString(Constants.ENTITY));
+ vertexId =
StringInterner.intern(jsonObject.optString(Constants.ENTITY_ID));
taskInfoMap = Maps.newHashMap();
inEdgeList = Lists.newLinkedList();
diff --git
a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/HistoryEventProtoJsonConversion.java
b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/HistoryEventProtoJsonConversion.java
index 2172447f8..f2bb209fd 100644
---
a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/HistoryEventProtoJsonConversion.java
+++
b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/HistoryEventProtoJsonConversion.java
@@ -115,7 +115,7 @@ public final class HistoryEventProtoJsonConversion {
private static JSONObject convertDAGRecoveredEvent(HistoryEventProto event)
throws JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY, event.getDagId());
+ jsonObject.put(ATSConstants.ENTITY_ID, event.getDagId());
jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name());
// Related Entities not needed as should have been done in
@@ -142,7 +142,7 @@ public final class HistoryEventProtoJsonConversion {
private static JSONObject convertAppLaunchedEvent(HistoryEventProto event)
throws JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY, "tez_" + event.getAppId().toString());
+ jsonObject.put(ATSConstants.ENTITY_ID, "tez_" +
event.getAppId().toString());
jsonObject.put(ATSConstants.ENTITY_TYPE,
EntityTypes.TEZ_APPLICATION.name());
// Other info to tag with Tez App
@@ -157,16 +157,16 @@ public final class HistoryEventProtoJsonConversion {
private static JSONObject convertAMLaunchedEvent(HistoryEventProto event)
throws JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY, "tez_" +
event.getAppAttemptId().toString());
+ jsonObject.put(ATSConstants.ENTITY_ID, "tez_" +
event.getAppAttemptId().toString());
jsonObject.put(ATSConstants.ENTITY_TYPE,
EntityTypes.TEZ_APPLICATION_ATTEMPT.name());
// Related Entities
JSONArray relatedEntities = new JSONArray();
JSONObject appEntity = new JSONObject();
- appEntity.put(ATSConstants.ENTITY, event.getAppId().toString());
+ appEntity.put(ATSConstants.ENTITY_ID, event.getAppId().toString());
appEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.APPLICATION_ID);
JSONObject appAttemptEntity = new JSONObject();
- appAttemptEntity.put(ATSConstants.ENTITY,
event.getAppAttemptId().toString());
+ appAttemptEntity.put(ATSConstants.ENTITY_ID,
event.getAppAttemptId().toString());
appAttemptEntity.put(ATSConstants.ENTITY_TYPE,
ATSConstants.APPLICATION_ATTEMPT_ID);
relatedEntities.put(appEntity);
relatedEntities.put(appAttemptEntity);
@@ -192,16 +192,16 @@ public final class HistoryEventProtoJsonConversion {
private static JSONObject convertAMStartedEvent(HistoryEventProto event)
throws JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY, "tez_" +
event.getAppAttemptId().toString());
+ jsonObject.put(ATSConstants.ENTITY_ID, "tez_" +
event.getAppAttemptId().toString());
jsonObject.put(ATSConstants.ENTITY_TYPE,
EntityTypes.TEZ_APPLICATION_ATTEMPT.name());
// Related Entities
JSONArray relatedEntities = new JSONArray();
JSONObject appEntity = new JSONObject();
- appEntity.put(ATSConstants.ENTITY, event.getAppId().toString());
+ appEntity.put(ATSConstants.ENTITY_ID, event.getAppId().toString());
appEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.APPLICATION_ID);
JSONObject appAttemptEntity = new JSONObject();
- appAttemptEntity.put(ATSConstants.ENTITY,
event.getAppAttemptId().toString());
+ appAttemptEntity.put(ATSConstants.ENTITY_ID,
event.getAppAttemptId().toString());
appAttemptEntity.put(ATSConstants.ENTITY_TYPE,
ATSConstants.APPLICATION_ATTEMPT_ID);
relatedEntities.put(appEntity);
relatedEntities.put(appAttemptEntity);
@@ -222,17 +222,17 @@ public final class HistoryEventProtoJsonConversion {
private static JSONObject convertContainerLaunchedEvent(HistoryEventProto
event)
throws JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY,
+ jsonObject.put(ATSConstants.ENTITY_ID,
"tez_" + getDataValueByKey(event, ATSConstants.CONTAINER_ID));
jsonObject.put(ATSConstants.ENTITY_TYPE,
EntityTypes.TEZ_CONTAINER_ID.name());
JSONArray relatedEntities = new JSONArray();
JSONObject appAttemptEntity = new JSONObject();
- appAttemptEntity.put(ATSConstants.ENTITY,
event.getAppAttemptId().toString());
+ appAttemptEntity.put(ATSConstants.ENTITY_ID,
event.getAppAttemptId().toString());
appAttemptEntity.put(ATSConstants.ENTITY_TYPE,
EntityTypes.TEZ_APPLICATION_ATTEMPT.name());
JSONObject containerEntity = new JSONObject();
- containerEntity.put(ATSConstants.ENTITY, getDataValueByKey(event,
ATSConstants.CONTAINER_ID));
+ containerEntity.put(ATSConstants.ENTITY_ID, getDataValueByKey(event,
ATSConstants.CONTAINER_ID));
containerEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.CONTAINER_ID);
relatedEntities.put(appAttemptEntity);
@@ -258,17 +258,17 @@ public final class HistoryEventProtoJsonConversion {
throws JSONException {
// structure is identical to ContainerLaunchedEvent
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY,
+ jsonObject.put(ATSConstants.ENTITY_ID,
"tez_" + getDataValueByKey(event, ATSConstants.CONTAINER_ID));
jsonObject.put(ATSConstants.ENTITY_TYPE,
EntityTypes.TEZ_CONTAINER_ID.name());
JSONArray relatedEntities = new JSONArray();
JSONObject appAttemptEntity = new JSONObject();
- appAttemptEntity.put(ATSConstants.ENTITY,
event.getAppAttemptId().toString());
+ appAttemptEntity.put(ATSConstants.ENTITY_ID,
event.getAppAttemptId().toString());
appAttemptEntity.put(ATSConstants.ENTITY_TYPE,
EntityTypes.TEZ_APPLICATION_ATTEMPT.name());
JSONObject containerEntity = new JSONObject();
- containerEntity.put(ATSConstants.ENTITY, getDataValueByKey(event,
ATSConstants.CONTAINER_ID));
+ containerEntity.put(ATSConstants.ENTITY_ID, getDataValueByKey(event,
ATSConstants.CONTAINER_ID));
containerEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.CONTAINER_ID);
relatedEntities.put(appAttemptEntity);
@@ -297,7 +297,7 @@ public final class HistoryEventProtoJsonConversion {
private static JSONObject convertDAGFinishedEvent(HistoryEventProto event)
throws JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY, event.getDagId());
+ jsonObject.put(ATSConstants.ENTITY_ID, event.getDagId());
jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name());
// Related Entities not needed as should have been done in
@@ -341,7 +341,7 @@ public final class HistoryEventProtoJsonConversion {
private static JSONObject convertDAGInitializedEvent(HistoryEventProto event)
throws JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY, event.getDagId());
+ jsonObject.put(ATSConstants.ENTITY_ID, event.getDagId());
jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name());
// Related Entities not needed as should have been done in
@@ -364,7 +364,7 @@ public final class HistoryEventProtoJsonConversion {
private static JSONObject convertDAGStartedEvent(HistoryEventProto event)
throws JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY, event.getDagId());
+ jsonObject.put(ATSConstants.ENTITY_ID, event.getDagId());
jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name());
// Related Entities not needed as should have been done in
@@ -384,25 +384,25 @@ public final class HistoryEventProtoJsonConversion {
private static JSONObject convertDAGSubmittedEvent(HistoryEventProto event)
throws JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY, event.getDagId());
+ jsonObject.put(ATSConstants.ENTITY_ID, event.getDagId());
jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name());
// Related Entities
JSONArray relatedEntities = new JSONArray();
JSONObject tezAppEntity = new JSONObject();
- tezAppEntity.put(ATSConstants.ENTITY, "tez_" +
event.getAppId().toString());
+ tezAppEntity.put(ATSConstants.ENTITY_ID, "tez_" +
event.getAppId().toString());
tezAppEntity.put(ATSConstants.ENTITY_TYPE,
EntityTypes.TEZ_APPLICATION.name());
JSONObject tezAppAttemptEntity = new JSONObject();
- tezAppAttemptEntity.put(ATSConstants.ENTITY, "tez_" +
event.getAppAttemptId().toString());
+ tezAppAttemptEntity.put(ATSConstants.ENTITY_ID, "tez_" +
event.getAppAttemptId().toString());
tezAppAttemptEntity.put(ATSConstants.ENTITY_TYPE,
EntityTypes.TEZ_APPLICATION_ATTEMPT.name());
JSONObject appEntity = new JSONObject();
- appEntity.put(ATSConstants.ENTITY, event.getAppId().toString());
+ appEntity.put(ATSConstants.ENTITY_ID, event.getAppId().toString());
appEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.APPLICATION_ID);
JSONObject appAttemptEntity = new JSONObject();
- appAttemptEntity.put(ATSConstants.ENTITY,
event.getAppAttemptId().toString());
+ appAttemptEntity.put(ATSConstants.ENTITY_ID,
event.getAppAttemptId().toString());
appAttemptEntity.put(ATSConstants.ENTITY_TYPE,
ATSConstants.APPLICATION_ATTEMPT_ID);
JSONObject userEntity = new JSONObject();
- userEntity.put(ATSConstants.ENTITY, event.getUser());
+ userEntity.put(ATSConstants.ENTITY_ID, event.getUser());
userEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.USER);
relatedEntities.put(tezAppEntity);
@@ -452,7 +452,7 @@ public final class HistoryEventProtoJsonConversion {
private static JSONObject convertTaskAttemptFinishedEvent(HistoryEventProto
event)
throws JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY, event.getTaskAttemptId());
+ jsonObject.put(ATSConstants.ENTITY_ID, event.getTaskAttemptId());
jsonObject.put(ATSConstants.ENTITY_TYPE,
EntityTypes.TEZ_TASK_ATTEMPT_ID.name());
// Events
@@ -503,21 +503,21 @@ public final class HistoryEventProtoJsonConversion {
private static JSONObject convertTaskAttemptStartedEvent(HistoryEventProto
event)
throws JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY, event.getTaskAttemptId());
+ jsonObject.put(ATSConstants.ENTITY_ID, event.getTaskAttemptId());
jsonObject.put(ATSConstants.ENTITY_TYPE,
EntityTypes.TEZ_TASK_ATTEMPT_ID.name());
// Related entities
JSONArray relatedEntities = new JSONArray();
JSONObject nodeEntity = new JSONObject();
- nodeEntity.put(ATSConstants.ENTITY, getDataValueByKey(event,
ATSConstants.NODE_ID));
+ nodeEntity.put(ATSConstants.ENTITY_ID, getDataValueByKey(event,
ATSConstants.NODE_ID));
nodeEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.NODE_ID);
JSONObject containerEntity = new JSONObject();
- containerEntity.put(ATSConstants.ENTITY, getDataValueByKey(event,
ATSConstants.CONTAINER_ID));
+ containerEntity.put(ATSConstants.ENTITY_ID, getDataValueByKey(event,
ATSConstants.CONTAINER_ID));
containerEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.CONTAINER_ID);
JSONObject taskEntity = new JSONObject();
- taskEntity.put(ATSConstants.ENTITY, event.getTaskAttemptId());
+ taskEntity.put(ATSConstants.ENTITY_ID, event.getTaskAttemptId());
taskEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_TASK_ID.name());
relatedEntities.put(nodeEntity);
@@ -546,7 +546,7 @@ public final class HistoryEventProtoJsonConversion {
private static JSONObject convertTaskFinishedEvent(HistoryEventProto event)
throws JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY, event.getTaskId());
+ jsonObject.put(ATSConstants.ENTITY_ID, event.getTaskId());
jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_TASK_ID.name());
// Events
@@ -577,13 +577,13 @@ public final class HistoryEventProtoJsonConversion {
private static JSONObject convertTaskStartedEvent(HistoryEventProto event)
throws JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY, event.getTaskId());
+ jsonObject.put(ATSConstants.ENTITY_ID, event.getTaskId());
jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_TASK_ID.name());
// Related entities
JSONArray relatedEntities = new JSONArray();
JSONObject vertexEntity = new JSONObject();
- vertexEntity.put(ATSConstants.ENTITY, event.getVertexId());
+ vertexEntity.put(ATSConstants.ENTITY_ID, event.getVertexId());
vertexEntity.put(ATSConstants.ENTITY_TYPE,
EntityTypes.TEZ_VERTEX_ID.name());
relatedEntities.put(vertexEntity);
jsonObject.put(ATSConstants.RELATED_ENTITIES, relatedEntities);
@@ -610,7 +610,7 @@ public final class HistoryEventProtoJsonConversion {
private static JSONObject convertVertexFinishedEvent(HistoryEventProto event)
throws JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY, event.getVertexId());
+ jsonObject.put(ATSConstants.ENTITY_ID, event.getVertexId());
jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_VERTEX_ID.name());
// Events
@@ -653,7 +653,7 @@ public final class HistoryEventProtoJsonConversion {
private static JSONObject
convertVertexReconfigureDoneEvent(HistoryEventProto event)
throws JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY, event.getVertexId());
+ jsonObject.put(ATSConstants.ENTITY_ID, event.getVertexId());
jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_VERTEX_ID.name());
// Events
@@ -681,13 +681,13 @@ public final class HistoryEventProtoJsonConversion {
private static JSONObject convertVertexInitializedEvent(HistoryEventProto
event)
throws JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY, event.getVertexId());
+ jsonObject.put(ATSConstants.ENTITY_ID, event.getVertexId());
jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_VERTEX_ID.name());
// Related entities
JSONArray relatedEntities = new JSONArray();
JSONObject vertexEntity = new JSONObject();
- vertexEntity.put(ATSConstants.ENTITY, event.getDagId());
+ vertexEntity.put(ATSConstants.ENTITY_ID, event.getDagId());
vertexEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name());
relatedEntities.put(vertexEntity);
jsonObject.put(ATSConstants.RELATED_ENTITIES, relatedEntities);
@@ -720,13 +720,13 @@ public final class HistoryEventProtoJsonConversion {
private static JSONObject convertVertexStartedEvent(HistoryEventProto event)
throws JSONException {
JSONObject jsonObject = new JSONObject();
- jsonObject.put(ATSConstants.ENTITY, event.getVertexId());
+ jsonObject.put(ATSConstants.ENTITY_ID, event.getVertexId());
jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_VERTEX_ID.name());
// Related entities
JSONArray relatedEntities = new JSONArray();
JSONObject vertexEntity = new JSONObject();
- vertexEntity.put(ATSConstants.ENTITY, event.getDagId());
+ vertexEntity.put(ATSConstants.ENTITY_ID, event.getDagId());
vertexEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name());
relatedEntities.put(vertexEntity);
jsonObject.put(ATSConstants.RELATED_ENTITIES, relatedEntities);
diff --git a/tez-plugins/tez-yarn-timeline-history-with-acls/pom.xml
b/tez-plugins/tez-yarn-timeline-history-with-acls/pom.xml
index 7505e71c8..15724e441 100644
--- a/tez-plugins/tez-yarn-timeline-history-with-acls/pom.xml
+++ b/tez-plugins/tez-yarn-timeline-history-with-acls/pom.xml
@@ -140,14 +140,24 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-json</artifactId>
+ <groupId>org.glassfish.jersey.media</groupId>
+ <artifactId>jersey-media-json-jackson</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-shuffle</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.vintage</groupId>
+ <artifactId>junit-vintage-engine</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
diff --git
a/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java
b/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java
index f701b6eaa..1a70f7d8e 100644
---
a/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java
+++
b/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java
@@ -31,7 +31,11 @@ import java.util.Collections;
import java.util.List;
import java.util.Random;
+import javax.ws.rs.client.Client;
+import javax.ws.rs.client.ClientBuilder;
+import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -46,6 +50,7 @@ import
org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.tez.client.TezClient;
+import org.apache.tez.common.ATSConstants;
import org.apache.tez.common.ReflectionUtils;
import org.apache.tez.common.security.DAGAccessControls;
import org.apache.tez.dag.api.DAG;
@@ -66,9 +71,6 @@ import
org.apache.tez.runtime.library.processor.SleepProcessor.SleepProcessorCon
import org.apache.tez.tests.MiniTezClusterWithTimeline;
import com.google.common.collect.Sets;
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
@@ -148,23 +150,38 @@ public class TestATSHistoryWithACLs {
// To be replaced after Timeline has java APIs for domains
private <K> K getTimelineData(String url, Class<K> clazz) {
- Client client = new Client();
- WebResource resource = client.resource(url);
-
- ClientResponse response = resource.accept(MediaType.APPLICATION_JSON)
- .get(ClientResponse.class);
+ Client client = ClientBuilder.newClient();
+ WebTarget target = client.target(url);
+ Response response = target.request(MediaType.APPLICATION_JSON).get();
assertEquals(200, response.getStatus());
-
assertTrue(MediaType.APPLICATION_JSON_TYPE.isCompatible(response.getType()));
-
- JSONObject entity = response.getEntity(JSONObject.class);
- K converted = null;
+
assertTrue(MediaType.APPLICATION_JSON_TYPE.isCompatible(response.getMediaType()));
+ String entityStr = response.readEntity(String.class);
try {
- converted = convertJSONObjectToTimelineObject(entity, clazz);
+ JSONObject jsonObject = new JSONObject(entityStr);
+ JSONObject effectiveJson;
+ // Handle the nesting introduced by Jersey 2/Jackson
+ if (clazz == TimelineDomain.class) {
+ // TimelineDomain is annotated @XmlRootElement(name="domain"), Jersey
2/Jackson wraps it
+ effectiveJson = jsonObject.has("domain")
+ ? jsonObject.getJSONObject("domain")
+ : jsonObject;
+ } else if (clazz == TimelineEntity.class) {
+ // TimelineEntity is annotated @XmlRootElement(name="entity"), Jersey
2/Jackson wraps it
+ effectiveJson = jsonObject.has("entity")
+ ? jsonObject.getJSONObject("entity")
+ : jsonObject;
+ } else {
+ throw new RuntimeException("Unsupported class for JSON conversion: " +
clazz);
+ }
+ K converted = convertJSONObjectToTimelineObject(effectiveJson, clazz);
+ assertNotNull(converted);
+ return converted;
} catch (JSONException e) {
throw new RuntimeException(e);
+ } finally {
+ response.close();
+ client.close();
}
- assertNotNull(converted);
- return converted;
}
private <K> K convertJSONObjectToTimelineObject(JSONObject jsonObj, Class<K>
clazz) throws JSONException {
@@ -178,9 +195,9 @@ public class TestATSHistoryWithACLs {
return (K) domain;
} else if (clazz == TimelineEntity.class) {
TimelineEntity entity = new TimelineEntity();
- entity.setEntityId(jsonObj.getString("entity"));
- entity.setEntityType(jsonObj.getString("entitytype"));
- entity.setDomainId(jsonObj.getString("domain"));
+ entity.setEntityId(jsonObj.getString(ATSConstants.ENTITY_ID));
+ entity.setEntityType(jsonObj.getString(ATSConstants.ENTITY_TYPE));
+ entity.setDomainId(jsonObj.getString(ATSConstants.DOMAIN_ID));
entity.setEvents(getEventsFromJSON(jsonObj));
return (K) entity;
} else {
@@ -194,7 +211,7 @@ public class TestATSHistoryWithACLs {
JSONArray arrEvents = jsonObj.getJSONArray("events");
for (int i = 0; i < arrEvents.length(); i++) {
TimelineEvent event = new TimelineEvent();
- event.setEventType(((JSONObject)
arrEvents.get(i)).getString("eventtype"));
+ event.setEventType(((JSONObject)
arrEvents.get(i)).getString(ATSConstants.EVENT_TYPE));
events.add(event);
}
return events;
@@ -453,12 +470,12 @@ public class TestATSHistoryWithACLs {
historyLoggingService.handle(new DAGHistoryEvent(tezDAGID,
submittedEvent));
Thread.sleep(1000l);
String url = "http://"; + timelineAddress +
"/ws/v1/timeline/TEZ_DAG_ID/"+event.getDAGID();
- Client client = new Client();
- WebResource resource = client.resource(url);
-
- ClientResponse response = resource.accept(MediaType.APPLICATION_JSON)
- .get(ClientResponse.class);
+ Client client = ClientBuilder.newClient();
+ WebTarget target = client.target(url);
+ Response response = target.request(MediaType.APPLICATION_JSON).get();
assertEquals(404, response.getStatus());
+ response.close();
+ client.close();
}
/**
@@ -498,17 +515,18 @@ public class TestATSHistoryWithACLs {
historyLoggingService.handle(new DAGHistoryEvent(tezDAGID,
submittedEvent));
Thread.sleep(1000l);
String url = "http://"; + timelineAddress +
"/ws/v1/timeline/TEZ_DAG_ID/"+event.getDAGID();
- Client client = new Client();
- WebResource resource = client.resource(url);
-
- ClientResponse response = resource.accept(MediaType.APPLICATION_JSON)
- .get(ClientResponse.class);
+ Client client = ClientBuilder.newClient();
+ WebTarget target = client.target(url);
+ Response response = target.request(MediaType.APPLICATION_JSON).get();
assertEquals(200, response.getStatus());
-
assertTrue(MediaType.APPLICATION_JSON_TYPE.isCompatible(response.getType()));
- JSONObject entityJson = response.getEntity(JSONObject.class);
+
assertTrue(MediaType.APPLICATION_JSON_TYPE.isCompatible(response.getMediaType()));
+ String entityStr = response.readEntity(String.class);
+ JSONObject entityJson = new JSONObject(entityStr);
TimelineEntity entity = convertJSONObjectToTimelineObject(entityJson,
TimelineEntity.class);
assertEquals(entity.getEntityType(), "TEZ_DAG_ID");
assertEquals(entity.getEvents().get(0).getEventType(),
HistoryEventType.DAG_SUBMITTED.toString());
+ response.close();
+ client.close();
}
private static final String atsHistoryACLManagerClassName =
diff --git a/tez-plugins/tez-yarn-timeline-history-with-fs/pom.xml
b/tez-plugins/tez-yarn-timeline-history-with-fs/pom.xml
index 55ed04bbc..5962138fb 100644
--- a/tez-plugins/tez-yarn-timeline-history-with-fs/pom.xml
+++ b/tez-plugins/tez-yarn-timeline-history-with-fs/pom.xml
@@ -148,8 +148,13 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-json</artifactId>
+ <groupId>org.glassfish.jersey.media</groupId>
+ <artifactId>jersey-media-json-jackson</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
diff --git a/tez-plugins/tez-yarn-timeline-history/pom.xml
b/tez-plugins/tez-yarn-timeline-history/pom.xml
index 840650011..5cd65b35b 100644
--- a/tez-plugins/tez-yarn-timeline-history/pom.xml
+++ b/tez-plugins/tez-yarn-timeline-history/pom.xml
@@ -134,8 +134,13 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-json</artifactId>
+ <groupId>org.glassfish.jersey.media</groupId>
+ <artifactId>jersey-media-json-jackson</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
diff --git
a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryWithMiniCluster.java
b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryWithMiniCluster.java
index 1ff3bfd8d..27b0da059 100644
---
a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryWithMiniCluster.java
+++
b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryWithMiniCluster.java
@@ -21,7 +21,11 @@ package org.apache.tez.dag.history.logging.ats;
import java.io.IOException;
import java.util.Random;
+import javax.ws.rs.client.Client;
+import javax.ws.rs.client.ClientBuilder;
+import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -40,10 +44,6 @@ import
org.apache.tez.runtime.library.processor.SleepProcessor;
import
org.apache.tez.runtime.library.processor.SleepProcessor.SleepProcessorConfig;
import org.apache.tez.tests.MiniTezClusterWithTimeline;
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
-
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -115,16 +115,18 @@ public class TestATSHistoryWithMiniCluster {
// To be replaced after Timeline has java APIs for domains
private <K> K getTimelineData(String url, Class<K> clazz) {
- Client client = new Client();
- WebResource resource = client.resource(url);
+ Client client = ClientBuilder.newClient();
+ WebTarget target = client.target(url);
- ClientResponse response = resource.accept(MediaType.APPLICATION_JSON)
- .get(ClientResponse.class);
+ Response response = target.request(MediaType.APPLICATION_JSON)
+ .get();
Assert.assertEquals(200, response.getStatus());
- Assert.assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+
Assert.assertTrue(MediaType.APPLICATION_JSON_TYPE.isCompatible(response.getMediaType()));
- K entity = response.getEntity(clazz);
+ K entity = response.readEntity(clazz);
Assert.assertNotNull(entity);
+ response.close();
+ client.close();
return entity;
}
diff --git a/tez-tests/pom.xml b/tez-tests/pom.xml
index 833a9978f..e35e0d35c 100644
--- a/tez-tests/pom.xml
+++ b/tez-tests/pom.xml
@@ -134,6 +134,11 @@
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
diff --git a/tez-tools/analyzers/job-analyzer/pom.xml
b/tez-tools/analyzers/job-analyzer/pom.xml
index 34f10b375..a837b94dd 100644
--- a/tez-tools/analyzers/job-analyzer/pom.xml
+++ b/tez-tools/analyzers/job-analyzer/pom.xml
@@ -157,13 +157,24 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-json</artifactId>
+ <groupId>org.glassfish.jersey.media</groupId>
+ <artifactId>jersey-media-json-jackson</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-shuffle</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.vintage</groupId>
+ <artifactId>junit-vintage-engine</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
diff --git
a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/utils/Utils.java
b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/utils/Utils.java
index c95049544..ab667133c 100644
---
a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/utils/Utils.java
+++
b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/utils/Utils.java
@@ -23,14 +23,14 @@ import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import javax.annotation.Nullable;
+
import org.apache.tez.dag.utils.Graph;
import org.apache.tez.history.parser.datamodel.AdditionalInputOutputDetails;
import org.apache.tez.history.parser.datamodel.DagInfo;
import org.apache.tez.history.parser.datamodel.EdgeInfo;
import org.apache.tez.history.parser.datamodel.VertexInfo;
-import com.sun.istack.Nullable;
-
public final class Utils {
private static Pattern sanitizeLabelPattern = Pattern.compile("[:\\-\\W]+");
diff --git a/tez-ui/src/main/webapp/app/serializers/ahs-app.js
b/tez-ui/src/main/webapp/app/serializers/ahs-app.js
index 0c35d54e8..5af385613 100644
--- a/tez-ui/src/main/webapp/app/serializers/ahs-app.js
+++ b/tez-ui/src/main/webapp/app/serializers/ahs-app.js
@@ -44,6 +44,6 @@ export default LoaderSerializer.extend({
startTime: 'startedTime',
endTime: 'finishedTime',
- diagnostics: 'otherinfo.diagnostics',
+ diagnostics: 'diagnostics',
}
});
diff --git a/tez-ui/src/main/webapp/app/serializers/app.js
b/tez-ui/src/main/webapp/app/serializers/app.js
index 0d05c6cc7..344a8baac 100644
--- a/tez-ui/src/main/webapp/app/serializers/app.js
+++ b/tez-ui/src/main/webapp/app/serializers/app.js
@@ -20,13 +20,13 @@ import TimelineSerializer from './timeline';
export default TimelineSerializer.extend({
maps: {
- domain: 'domain',
- user: 'otherinfo.user',
+ domain: 'domainId',
+ user: 'otherInfo.user',
- buildTime: 'otherinfo.tezVersion.buildTime',
- tezRevision: 'otherinfo.tezVersion.revision',
- tezVersion: 'otherinfo.tezVersion.version',
+ buildTime: 'otherInfo.tezVersion.buildTime',
+ tezRevision: 'otherInfo.tezVersion.revision',
+ tezVersion: 'otherInfo.tezVersion.version',
- configs: 'otherinfo.config',
+ configs: 'otherInfo.config',
}
});
diff --git a/tez-ui/src/main/webapp/app/serializers/attempt.js
b/tez-ui/src/main/webapp/app/serializers/attempt.js
index dd54135a4..b56f5e900 100644
--- a/tez-ui/src/main/webapp/app/serializers/attempt.js
+++ b/tez-ui/src/main/webapp/app/serializers/attempt.js
@@ -21,7 +21,7 @@ import Ember from 'ember';
import TimelineSerializer from './timeline';
function createContainerLogURL(source) {
- var logURL = Ember.get(source, 'otherinfo.inProgressLogsURL'),
+ var logURL = Ember.get(source, 'otherInfo.inProgressLogsURL'),
yarnProtocol = this.get('env.app.yarnProtocol');
if(logURL && logURL.indexOf("://") === -1) {
@@ -31,15 +31,15 @@ function createContainerLogURL(source) {
export default TimelineSerializer.extend({
maps: {
- taskID: 'primaryfilters.TEZ_TASK_ID.0',
- vertexID: 'primaryfilters.TEZ_VERTEX_ID.0',
- dagID: 'primaryfilters.TEZ_DAG_ID.0',
+ taskID: 'primaryFilters.TEZ_TASK_ID.0',
+ vertexID: 'primaryFilters.TEZ_VERTEX_ID.0',
+ dagID: 'primaryFilters.TEZ_DAG_ID.0',
- containerID: 'otherinfo.containerId',
- nodeID: 'otherinfo.nodeId',
+ containerID: 'otherInfo.containerId',
+ nodeID: 'otherInfo.nodeId',
- inProgressLogsURL: "otherinfo.inProgressLogsURL",
- completedLogsURL: "otherinfo.completedLogsURL",
+ inProgressLogsURL: "otherInfo.inProgressLogsURL",
+ completedLogsURL: "otherInfo.completedLogsURL",
containerLogURL: createContainerLogURL
}
diff --git a/tez-ui/src/main/webapp/app/serializers/dag-info.js
b/tez-ui/src/main/webapp/app/serializers/dag-info.js
index bba1d1b5c..8136bb271 100644
--- a/tez-ui/src/main/webapp/app/serializers/dag-info.js
+++ b/tez-ui/src/main/webapp/app/serializers/dag-info.js
@@ -22,20 +22,20 @@ import TimelineSerializer from './timeline';
export default TimelineSerializer.extend({
maps: {
- dagPlan: 'otherinfo.dagPlan',
+ dagPlan: 'otherInfo.dagPlan',
callerData: 'callerData',
},
normalizeResourceHash: function (resourceHash) {
var data = resourceHash.data,
callerData = {},
- dagInfo = Ember.get(data, "otherinfo.dagPlan.dagInfo"), // New style,
from TEZ-2851
- dagContext = Ember.get(data, "otherinfo.dagPlan.dagContext"); // Old
style
+ dagInfo = Ember.get(data, "otherInfo.dagPlan.dagInfo"), // New style,
from TEZ-2851
+ dagContext = Ember.get(data, "otherInfo.dagPlan.dagContext"); // Old
style
if(dagContext) {
callerData.callerContext = Ember.String.classify((Ember.get(dagContext,
"context")||"").toLowerCase());
callerData.callerDescription = Ember.get(dagContext, "description");
- callerData.callerType = Ember.get(dagContext, "callerType") ||
Ember.get(data, "otherinfo.callerType");
+ callerData.callerType = Ember.get(dagContext, "callerType") ||
Ember.get(data, "otherInfo.callerType");
}
else if(dagInfo) {
let infoObj = {};
@@ -45,7 +45,7 @@ export default TimelineSerializer.extend({
infoObj = dagInfo;
}
- callerData.callerContext = Ember.get(infoObj, "context") ||
Ember.get(data, "otherinfo.callerContext");
+ callerData.callerContext = Ember.get(infoObj, "context") ||
Ember.get(data, "otherInfo.callerContext");
callerData.callerDescription = Ember.get(infoObj, "description") ||
Ember.get(dagInfo, "blob") || dagInfo;
}
@@ -57,4 +57,4 @@ export default TimelineSerializer.extend({
extractAttributes: function (modelClass, resourceHash) {
return this._super(modelClass, this.normalizeResourceHash(resourceHash));
},
-});
\ No newline at end of file
+});
diff --git a/tez-ui/src/main/webapp/app/serializers/dag.js
b/tez-ui/src/main/webapp/app/serializers/dag.js
index 701c1d2af..adc8594f8 100644
--- a/tez-ui/src/main/webapp/app/serializers/dag.js
+++ b/tez-ui/src/main/webapp/app/serializers/dag.js
@@ -24,11 +24,11 @@ import DAGInfoSerializer from './dag-info';
var MoreObject = more.Object;
function getStatus(source) {
- var status = Ember.get(source, 'otherinfo.status') || Ember.get(source,
'primaryfilters.status.0'),
+ var status = Ember.get(source, 'otherInfo.status') || Ember.get(source,
'primaryFilters.status.0'),
event = source.events;
if(!status && event) {
- if(event.findBy('eventtype', 'DAG_STARTED')) {
+ if(event.findBy('eventType', 'DAG_STARTED')) {
status = 'RUNNING';
}
}
@@ -37,11 +37,11 @@ function getStatus(source) {
}
function getStartTime(source) {
- var time = Ember.get(source, 'otherinfo.startTime'),
+ var time = Ember.get(source, 'otherInfo.startTime'),
event = source.events;
if(!time && event) {
- event = event.findBy('eventtype', 'DAG_STARTED');
+ event = event.findBy('eventType', 'DAG_STARTED');
if(event) {
time = event.timestamp;
}
@@ -51,11 +51,11 @@ function getStartTime(source) {
}
function getEndTime(source) {
- var time = Ember.get(source, 'otherinfo.endTime'),
+ var time = Ember.get(source, 'otherInfo.endTime'),
event = source.events;
if(!time && event) {
- event = event.findBy('eventtype', 'DAG_FINISHED');
+ event = event.findBy('eventType', 'DAG_FINISHED');
if(event) {
time = event.timestamp;
}
@@ -66,7 +66,7 @@ function getEndTime(source) {
function getContainerLogs(source) {
var containerLogs = [],
- otherinfo = Ember.get(source, 'otherinfo');
+ otherinfo = Ember.get(source, 'otherInfo');
if(!otherinfo) {
return undefined;
@@ -74,7 +74,7 @@ function getContainerLogs(source) {
for (var key in otherinfo) {
if (key.indexOf('inProgressLogsURL_') === 0) {
- let logs = Ember.get(source, 'otherinfo.' + key);
+ let logs = Ember.get(source, 'otherInfo.' + key);
if (logs.indexOf("://") === -1) {
let yarnProtocol = this.get('env.app.yarnProtocol');
logs = yarnProtocol + '://' + logs;
@@ -91,7 +91,7 @@ function getContainerLogs(source) {
}
function getIdNameMap(source) {
- var nameIdMap = Ember.get(source, 'otherinfo.vertexNameIdMapping'),
+ var nameIdMap = Ember.get(source, 'otherInfo.vertexNameIdMapping'),
idNameMap = {};
if(nameIdMap) {
@@ -105,11 +105,11 @@ function getIdNameMap(source) {
export default DAGInfoSerializer.extend({
maps: {
- name: 'primaryfilters.dagName.0',
+ name: 'primaryFilters.dagName.0',
- submitter: 'primaryfilters.user.0',
+ submitter: 'primaryFilters.user.0',
- callerID: 'primaryfilters.callerId.0',
+ callerID: 'primaryFilters.callerId.0',
atsStatus: getStatus,
// progress
@@ -119,16 +119,16 @@ export default DAGInfoSerializer.extend({
// duration
// appID
- domain: 'domain',
+ domain: 'domainId',
- queueName: 'otherinfo.queueName',
+ queueName: 'otherInfo.queueName',
containerLogs: getContainerLogs,
vertexIdNameMap: getIdNameMap,
- vertexNameIdMap: 'otherinfo.vertexNameIdMapping',
+ vertexNameIdMap: 'otherInfo.vertexNameIdMapping',
- amWsVersion: 'otherinfo.amWebServiceVersion',
- failedTaskAttempts: 'otherinfo.numFailedTaskAttempts',
+ amWsVersion: 'otherInfo.amWebServiceVersion',
+ failedTaskAttempts: 'otherInfo.numFailedTaskAttempts',
}
});
diff --git a/tez-ui/src/main/webapp/app/serializers/hive-query.js
b/tez-ui/src/main/webapp/app/serializers/hive-query.js
index 604d5ba13..891370ea8 100644
--- a/tez-ui/src/main/webapp/app/serializers/hive-query.js
+++ b/tez-ui/src/main/webapp/app/serializers/hive-query.js
@@ -21,11 +21,11 @@ import Ember from 'ember';
import TimelineSerializer from './timeline';
function getEndTime(source) {
- var time = Ember.get(source, 'otherinfo.endTime'),
+ var time = Ember.get(source, 'otherInfo.endTime'),
event = source.events;
if(!time && event) {
- event = event.findBy('eventtype', 'QUERY_COMPLETED');
+ event = event.findBy('eventType', 'QUERY_COMPLETED');
if(event) {
time = event.timestamp;
}
@@ -35,7 +35,7 @@ function getEndTime(source) {
}
function getStatus(source) {
- var status = Ember.get(source, 'otherinfo.STATUS');
+ var status = Ember.get(source, 'otherInfo.STATUS');
switch(status) {
case true:
@@ -49,70 +49,70 @@ function getStatus(source) {
export default TimelineSerializer.extend({
maps: {
- queryText: 'otherinfo.QUERY.queryText',
+ queryText: 'otherInfo.QUERY.queryText',
- dagID: 'primaryfilters.DAG_ID',
- appID: 'primaryfilters.APP_ID',
- sessionID: 'otherinfo.INVOKER_INFO',
- operationID: 'primaryfilters.operationid.0',
- llapAppID: 'otherinfo.LLAP_APP_ID',
+ dagID: 'primaryFilters.DAG_ID',
+ appID: 'primaryFilters.APP_ID',
+ sessionID: 'otherInfo.INVOKER_INFO',
+ operationID: 'primaryFilters.operationid.0',
+ llapAppID: 'otherInfo.LLAP_APP_ID',
- instanceType: 'otherinfo.HIVE_INSTANCE_TYPE',
- executionMode: 'primaryfilters.executionmode.0',
+ instanceType: 'otherInfo.HIVE_INSTANCE_TYPE',
+ executionMode: 'primaryFilters.executionmode.0',
- domain: 'domain',
- threadName: 'otherinfo.THREAD_NAME',
- queue: 'primaryfilters.queue.0',
- version: 'otherinfo.VERSION',
+ domain: 'domainId',
+ threadName: 'otherInfo.THREAD_NAME',
+ queue: 'primaryFilters.queue.0',
+ version: 'otherInfo.VERSION',
- hiveAddress: 'otherinfo.HIVE_ADDRESS',
- clientAddress: 'otherinfo.CLIENT_IP_ADDRESS',
+ hiveAddress: 'otherInfo.HIVE_ADDRESS',
+ clientAddress: 'otherInfo.CLIENT_IP_ADDRESS',
- user: 'primaryfilters.user.0',
- requestUser: 'primaryfilters.requestuser.0',
+ user: 'primaryFilters.user.0',
+ requestUser: 'primaryFilters.requestuser.0',
- tablesRead: 'primaryfilters.tablesread',
- tablesWritten: 'primaryfilters.tableswritten',
+ tablesRead: 'primaryFilters.tablesread',
+ tablesWritten: 'primaryFilters.tableswritten',
status: getStatus,
- configsJSON: "otherinfo.CONF",
+ configsJSON: "otherInfo.CONF",
- startTime: 'starttime',
+ startTime: 'startTime',
endTime: getEndTime,
- perf: "otherinfo.PERF"
+ perf: "otherInfo.PERF"
},
extractAttributes: function (modelClass, resourceHash) {
var data = resourceHash.data,
- query = Ember.get(resourceHash, "data.otherinfo.QUERY"),
- perf = Ember.get(resourceHash, "data.otherinfo.PERF");
+ query = Ember.get(resourceHash, "data.otherInfo.QUERY"),
+ perf = Ember.get(resourceHash, "data.otherInfo.PERF");
if(query) {
try{
- data.otherinfo.QUERY = JSON.parse(query);
+ data.otherInfo.QUERY = JSON.parse(query);
}catch(e){}
}
- if(!data.otherinfo.CLIENT_IP_ADDRESS) {
- data.otherinfo.CLIENT_IP_ADDRESS = data.otherinfo.HIVE_ADDRESS;
+ if(!data.otherInfo.CLIENT_IP_ADDRESS) {
+ data.otherInfo.CLIENT_IP_ADDRESS = data.otherInfo.HIVE_ADDRESS;
}
if(perf) {
try{
let PERF = JSON.parse(perf);
PERF["PostATSHook"] =
PERF["PostHook.org.apache.hadoop.hive.ql.hooks.ATSHook"];
- data.otherinfo.PERF = PERF;
+ data.otherInfo.PERF = PERF;
}catch(e){}
}
- data.primaryfilters = data.primaryfilters || {};
- if(!Ember.get(data, "primaryfilters.tablesread.length")) {
- data.primaryfilters.tablesread = new Error("None");
+ data.primaryFilters = data.primaryFilters || {};
+ if(!Ember.get(data, "primaryFilters.tablesread.length")) {
+ data.primaryFilters.tablesread = new Error("None");
}
- if(!Ember.get(data, "primaryfilters.tableswritten.length")) {
- data.primaryfilters.tableswritten = new Error("None");
+ if(!Ember.get(data, "primaryFilters.tableswritten.length")) {
+ data.primaryFilters.tableswritten = new Error("None");
}
return this._super(modelClass, resourceHash);
diff --git a/tez-ui/src/main/webapp/app/serializers/task.js
b/tez-ui/src/main/webapp/app/serializers/task.js
index 65b8c9f0c..c41b8473d 100644
--- a/tez-ui/src/main/webapp/app/serializers/task.js
+++ b/tez-ui/src/main/webapp/app/serializers/task.js
@@ -20,12 +20,12 @@ import TimelineSerializer from './timeline';
export default TimelineSerializer.extend({
maps: {
- vertexID: 'primaryfilters.TEZ_VERTEX_ID.0',
- dagID: 'primaryfilters.TEZ_DAG_ID.0',
+ vertexID: 'primaryFilters.TEZ_VERTEX_ID.0',
+ dagID: 'primaryFilters.TEZ_DAG_ID.0',
- failedTaskAttempts: 'otherinfo.numFailedTaskAttempts',
+ failedTaskAttempts: 'otherInfo.numFailedTaskAttempts',
- successfulAttemptID: 'otherinfo.successfulAttemptId',
- attemptIDs: 'relatedentities.TEZ_TASK_ATTEMPT_ID',
+ successfulAttemptID: 'otherInfo.successfulAttemptId',
+ attemptIDs: 'relatedEntities.TEZ_TASK_ATTEMPT_ID',
}
});
diff --git a/tez-ui/src/main/webapp/app/serializers/timeline.js
b/tez-ui/src/main/webapp/app/serializers/timeline.js
index ba5614e1b..52bba51be 100644
--- a/tez-ui/src/main/webapp/app/serializers/timeline.js
+++ b/tez-ui/src/main/webapp/app/serializers/timeline.js
@@ -21,7 +21,7 @@ import Ember from 'ember';
import LoaderSerializer from './loader';
function getDiagnostics(source) {
- var diagnostics = Ember.get(source, 'otherinfo.diagnostics') || "";
+ var diagnostics = Ember.get(source, 'otherInfo.diagnostics') || "";
diagnostics = diagnostics.replace(/\t/g, "  ");
diagnostics = diagnostics.replace(/\[/g, "<div>» ");
@@ -31,24 +31,28 @@ function getDiagnostics(source) {
}
export default LoaderSerializer.extend({
- primaryKey: 'entity',
+ // Hadoop 3.5.0 ATS v1 REST API uses camelCase field names:
+ // entityId (was: entity), entityType (was: entitytype),
+ // otherInfo (was: otherinfo), primaryFilters (was: primaryfilters),
+ // relatedEntities (was: relatedentities), eventType (was: eventtype)
+ primaryKey: 'entityId',
extractArrayPayload: function (payload) {
return payload.entities;
},
maps: {
- entityID: 'entity',
+ entityID: 'entityId',
- atsStatus: 'otherinfo.status',
+ atsStatus: 'otherInfo.status',
- startTime: 'otherinfo.startTime',
- endTime: 'otherinfo.endTime',
+ startTime: 'otherInfo.startTime',
+ endTime: 'otherInfo.endTime',
diagnostics: getDiagnostics,
events: 'events',
- _counterGroups: 'otherinfo.counters.counterGroups'
+ _counterGroups: 'otherInfo.counters.counterGroups'
}
});
diff --git a/tez-ui/src/main/webapp/app/serializers/vertex.js
b/tez-ui/src/main/webapp/app/serializers/vertex.js
index a1d0d52d5..6d3a2b083 100644
--- a/tez-ui/src/main/webapp/app/serializers/vertex.js
+++ b/tez-ui/src/main/webapp/app/serializers/vertex.js
@@ -21,41 +21,41 @@ import Ember from 'ember';
import TimelineSerializer from './timeline';
function getProcessorClass(source) {
- var name = Ember.get(source, 'otherinfo.processorClassName') || "";
+ var name = Ember.get(source, 'otherInfo.processorClassName') || "";
return name.substr(name.lastIndexOf('.') + 1);
}
export default TimelineSerializer.extend({
maps: {
- name: 'otherinfo.vertexName',
+ name: 'otherInfo.vertexName',
- _initTime: 'otherinfo.initTime',
- _startTime: 'otherinfo.startTime',
- _endTime: 'otherinfo.endTime',
- _firstTaskStartTime: 'otherinfo.stats.firstTaskStartTime',
- _lastTaskFinishTime: 'otherinfo.stats.lastTaskFinishTime',
+ _initTime: 'otherInfo.initTime',
+ _startTime: 'otherInfo.startTime',
+ _endTime: 'otherInfo.endTime',
+ _firstTaskStartTime: 'otherInfo.stats.firstTaskStartTime',
+ _lastTaskFinishTime: 'otherInfo.stats.lastTaskFinishTime',
- totalTasks: 'otherinfo.numTasks',
- _failedTasks: 'otherinfo.numFailedTasks',
- _succeededTasks: 'otherinfo.numSucceededTasks',
- _killedTasks: 'otherinfo.numKilledTasks',
+ totalTasks: 'otherInfo.numTasks',
+ _failedTasks: 'otherInfo.numFailedTasks',
+ _succeededTasks: 'otherInfo.numSucceededTasks',
+ _killedTasks: 'otherInfo.numKilledTasks',
- _failedTaskAttempts: 'otherinfo.numFailedTaskAttempts',
- _killedTaskAttempts: 'otherinfo.numKilledTaskAttempts',
+ _failedTaskAttempts: 'otherInfo.numFailedTaskAttempts',
+ _killedTaskAttempts: 'otherInfo.numKilledTaskAttempts',
- minDuration: 'otherinfo.stats.minTaskDuration',
- maxDuration: 'otherinfo.stats.maxTaskDuration',
- avgDuration: 'otherinfo.stats.avgTaskDuration',
+ minDuration: 'otherInfo.stats.minTaskDuration',
+ maxDuration: 'otherInfo.stats.maxTaskDuration',
+ avgDuration: 'otherInfo.stats.avgTaskDuration',
- firstTasksToStart: 'otherinfo.stats.firstTasksToStart',
- lastTasksToFinish: 'otherinfo.stats.lastTasksToFinish',
- shortestDurationTasks: 'otherinfo.stats.shortestDurationTasks',
- longestDurationTasks: 'otherinfo.stats.longestDurationTasks',
+ firstTasksToStart: 'otherInfo.stats.firstTasksToStart',
+ lastTasksToFinish: 'otherInfo.stats.lastTasksToFinish',
+ shortestDurationTasks: 'otherInfo.stats.shortestDurationTasks',
+ longestDurationTasks: 'otherInfo.stats.longestDurationTasks',
processorClassName: getProcessorClass,
- dagID: 'primaryfilters.TEZ_DAG_ID.0',
+ dagID: 'primaryFilters.TEZ_DAG_ID.0',
- servicePlugin: 'otherinfo.servicePlugin',
+ servicePlugin: 'otherInfo.servicePlugin',
}
});
diff --git a/tez-ui/src/main/webapp/app/utils/download-dag-zip.js
b/tez-ui/src/main/webapp/app/utils/download-dag-zip.js
index 0af83aec1..46d54ed90 100644
--- a/tez-ui/src/main/webapp/app/utils/download-dag-zip.js
+++ b/tez-ui/src/main/webapp/app/utils/download-dag-zip.js
@@ -454,7 +454,7 @@ export default function downloadDagZip(dag, options) {
// need to handle no more entries , zero entries
if (data.entities.length > batchSize) {
- nextBatchStart = data.entities.pop().entity;
+ nextBatchStart = data.entities.pop().entityId;
}
obj[context.name] = data.entities;
diff --git a/tez-ui/src/main/webapp/tests/unit/serializers/attempt-test.js
b/tez-ui/src/main/webapp/tests/unit/serializers/attempt-test.js
index 4715acd5d..2b5a9da60 100644
--- a/tez-ui/src/main/webapp/tests/unit/serializers/attempt-test.js
+++ b/tez-ui/src/main/webapp/tests/unit/serializers/attempt-test.js
@@ -40,7 +40,7 @@ test('containerLogURL test', function(assert) {
});
assert.equal(serializer.maps.containerLogURL.call(serializer, {
- otherinfo: {
+ otherInfo: {
inProgressLogsURL: "abc.com/test/link",
}
}), "ptcl://abc.com/test/link");
diff --git a/tez-ui/src/main/webapp/tests/unit/serializers/dag-info-test.js
b/tez-ui/src/main/webapp/tests/unit/serializers/dag-info-test.js
index b5c46f04c..d8101f341 100644
--- a/tez-ui/src/main/webapp/tests/unit/serializers/dag-info-test.js
+++ b/tez-ui/src/main/webapp/tests/unit/serializers/dag-info-test.js
@@ -51,7 +51,7 @@ test('normalizeResourceHash test', function(assert) {
// dagContext test
data = serializer.normalizeResourceHash({
data: {
- otherinfo: {
+ otherInfo: {
dagPlan: {
dagContext: callerInfo
}
@@ -66,7 +66,7 @@ test('normalizeResourceHash test', function(assert) {
// dagInfo test
data = serializer.normalizeResourceHash({
data: {
- otherinfo: {
+ otherInfo: {
dagPlan: {
dagInfo: `{"context": "${callerInfo.context}", "description":
"${callerInfo.description}"}`
}
@@ -81,7 +81,7 @@ test('normalizeResourceHash test', function(assert) {
// dagInfo.blob test
data = serializer.normalizeResourceHash({
data: {
- otherinfo: {
+ otherInfo: {
dagPlan: {
dagInfo: {
context: callerInfo.context,
@@ -99,7 +99,7 @@ test('normalizeResourceHash test', function(assert) {
// dagContext have presidence over dagInfo
data = serializer.normalizeResourceHash({
data: {
- otherinfo: {
+ otherInfo: {
dagPlan: {
dagContext: callerInfo,
dagInfo: `{"context": "RandomContext", "description": "RandomDesc"}`
diff --git a/tez-ui/src/main/webapp/tests/unit/serializers/dag-test.js
b/tez-ui/src/main/webapp/tests/unit/serializers/dag-test.js
index 80f41b6bb..39d7281a1 100644
--- a/tez-ui/src/main/webapp/tests/unit/serializers/dag-test.js
+++ b/tez-ui/src/main/webapp/tests/unit/serializers/dag-test.js
@@ -44,22 +44,22 @@ test('atsStatus test', function(assert) {
mapper = serializer.maps.atsStatus;
assert.equal(mapper({
- events: [{eventtype: "SOME_EVENT"}]
+ events: [{eventType: "SOME_EVENT"}]
}), undefined);
assert.equal(mapper({
- events: [{eventtype: "DAG_STARTED"}]
+ events: [{eventType: "DAG_STARTED"}]
}), "RUNNING");
assert.equal(mapper({
- otherinfo: {status: "STATUS1"},
- primaryfilters: {status: ["STATUS2"]},
- events: [{eventtype: "DAG_STARTED"}]
+ otherInfo: {status: "STATUS1"},
+ primaryFilters: {status: ["STATUS2"]},
+ events: [{eventType: "DAG_STARTED"}]
}), "STATUS1");
assert.equal(mapper({
- primaryfilters: {status: ["STATUS2"]},
- events: [{eventtype: "DAG_STARTED"}]
+ primaryFilters: {status: ["STATUS2"]},
+ events: [{eventType: "DAG_STARTED"}]
}), "STATUS2");
});
@@ -69,16 +69,16 @@ test('startTime test', function(assert) {
testTimestamp = Date.now();
assert.equal(mapper({
- events: [{eventtype: "SOME_EVENT"}]
+ events: [{eventType: "SOME_EVENT"}]
}), undefined);
assert.equal(mapper({
- events: [{eventtype: "DAG_STARTED", timestamp: testTimestamp}]
+ events: [{eventType: "DAG_STARTED", timestamp: testTimestamp}]
}), testTimestamp);
assert.equal(mapper({
- otherinfo: {startTime: testTimestamp},
- events: [{eventtype: "DAG_STARTED"}]
+ otherInfo: {startTime: testTimestamp},
+ events: [{eventType: "DAG_STARTED"}]
}), testTimestamp);
});
@@ -88,16 +88,16 @@ test('endTime test', function(assert) {
testTimestamp = Date.now();
assert.equal(mapper({
- events: [{eventtype: "SOME_EVENT"}]
+ events: [{eventType: "SOME_EVENT"}]
}), undefined);
assert.equal(mapper({
- events: [{eventtype: "DAG_FINISHED", timestamp: testTimestamp}]
+ events: [{eventType: "DAG_FINISHED", timestamp: testTimestamp}]
}), testTimestamp);
assert.equal(mapper({
- otherinfo: {endTime: testTimestamp},
- events: [{eventtype: "DAG_FINISHED"}]
+ otherInfo: {endTime: testTimestamp},
+ events: [{eventType: "DAG_FINISHED"}]
}), testTimestamp);
});
@@ -106,11 +106,11 @@ test('containerLogs test', function(assert) {
mapper = serializer.maps.containerLogs;
assert.deepEqual(mapper({
- otherinfo: {},
+ otherInfo: {},
}), [], "No logs");
assert.deepEqual(mapper({
- otherinfo: {inProgressLogsURL_1: "http://foo";, inProgressLogsURL_2:
"https://bar"},
+ otherInfo: {inProgressLogsURL_1: "http://foo";, inProgressLogsURL_2:
"https://bar"},
}), [{text: "1", href: "http://foo"}, {text: "2", href: "https://bar"}], "2
logs");
});
@@ -119,7 +119,7 @@ test('vertexIdNameMap test', function(assert) {
mapper = serializer.maps.vertexIdNameMap;
let nameIdMap = {
- otherinfo: {
+ otherInfo: {
vertexNameIdMapping: {
name1: "ID1",
name2: "ID2",
@@ -150,7 +150,7 @@ test('normalizeResourceHash test', function(assert) {
// dagContext test
data = serializer.normalizeResourceHash({
data: {
- otherinfo: {
+ otherInfo: {
dagPlan: {
dagContext: callerInfo
}
@@ -165,7 +165,7 @@ test('normalizeResourceHash test', function(assert) {
// dagInfo test
data = serializer.normalizeResourceHash({
data: {
- otherinfo: {
+ otherInfo: {
dagPlan: {
dagInfo: `{"context": "${callerInfo.context}", "description":
"${callerInfo.description}"}`
}
@@ -180,7 +180,7 @@ test('normalizeResourceHash test', function(assert) {
// dagInfo.blob test
data = serializer.normalizeResourceHash({
data: {
- otherinfo: {
+ otherInfo: {
dagPlan: {
dagInfo: {
context: callerInfo.context,
@@ -198,7 +198,7 @@ test('normalizeResourceHash test', function(assert) {
// dagContext have presidence over dagInfo
data = serializer.normalizeResourceHash({
data: {
- otherinfo: {
+ otherInfo: {
dagPlan: {
dagContext: callerInfo,
dagInfo: `{"context": "RandomContext", "description": "RandomDesc"}`
diff --git a/tez-ui/src/main/webapp/tests/unit/serializers/hive-query-test.js
b/tez-ui/src/main/webapp/tests/unit/serializers/hive-query-test.js
index c1525894f..699726413 100644
--- a/tez-ui/src/main/webapp/tests/unit/serializers/hive-query-test.js
+++ b/tez-ui/src/main/webapp/tests/unit/serializers/hive-query-test.js
@@ -36,12 +36,12 @@ test('getStatus test', function(assert) {
assert.equal(getStatus({}), "RUNNING");
assert.equal(getStatus({
- otherinfo: {
+ otherInfo: {
STATUS: true
}
}), "SUCCEEDED");
assert.equal(getStatus({
- otherinfo: {
+ otherInfo: {
STATUS: false
}
}), "FAILED");
@@ -55,19 +55,19 @@ test('getEndTime test', function(assert) {
assert.equal(getEndTime({}), undefined);
assert.equal(getEndTime({
- otherinfo: {
+ otherInfo: {
endTime: endTime
}
}), endTime);
assert.equal(getEndTime({
events: [{
- eventtype: 'X',
+ eventType: 'X',
}, {
- eventtype: 'QUERY_COMPLETED',
+ eventType: 'QUERY_COMPLETED',
timestamp: endTime
}, {
- eventtype: 'Y',
+ eventType: 'Y',
}]
}), endTime);
});
@@ -80,7 +80,7 @@ test('extractAttributes test', function(assert) {
},
testHiveAddress = "1.2.3.4",
testData = {
- otherinfo: {
+ otherInfo: {
QUERY: JSON.stringify(testQuery),
HIVE_ADDRESS: testHiveAddress
}
@@ -91,15 +91,15 @@ test('extractAttributes test', function(assert) {
}), {
data: testData
});
- assert.deepEqual(testData.otherinfo.QUERY, testQuery);
+ assert.deepEqual(testData.otherInfo.QUERY, testQuery);
//CLIENT_IP_ADDRESS set
- assert.equal(testHiveAddress, testData.otherinfo.CLIENT_IP_ADDRESS);
+ assert.equal(testHiveAddress, testData.otherInfo.CLIENT_IP_ADDRESS);
// Tables read & tables written
- assert.ok(testData.primaryfilters);
- assert.ok(testData.primaryfilters.tablesread instanceof Error);
- assert.ok(testData.primaryfilters.tableswritten instanceof Error);
- assert.equal(testData.primaryfilters.tablesread.message, "None");
- assert.equal(testData.primaryfilters.tableswritten.message, "None");
+ assert.ok(testData.primaryFilters);
+ assert.ok(testData.primaryFilters.tablesread instanceof Error);
+ assert.ok(testData.primaryFilters.tableswritten instanceof Error);
+ assert.equal(testData.primaryFilters.tablesread.message, "None");
+ assert.equal(testData.primaryFilters.tableswritten.message, "None");
});
diff --git a/tez-ui/src/main/webapp/tests/unit/serializers/timeline-test.js
b/tez-ui/src/main/webapp/tests/unit/serializers/timeline-test.js
index 8f39df797..1aaabac2f 100644
--- a/tez-ui/src/main/webapp/tests/unit/serializers/timeline-test.js
+++ b/tez-ui/src/main/webapp/tests/unit/serializers/timeline-test.js
@@ -31,6 +31,7 @@ test('Basic creation test', function(assert) {
assert.ok(serializer.maps);
assert.equal(Object.keys(serializer.get("maps")).length, 7);
+ assert.equal(serializer.primaryKey, 'entityId');
});
test('extractArrayPayload test', function(assert) {
diff --git a/tez-ui/src/main/webapp/tests/unit/serializers/vertex-test.js
b/tez-ui/src/main/webapp/tests/unit/serializers/vertex-test.js
index 7dfb5da95..5a09fb318 100644
--- a/tez-ui/src/main/webapp/tests/unit/serializers/vertex-test.js
+++ b/tez-ui/src/main/webapp/tests/unit/serializers/vertex-test.js
@@ -37,12 +37,12 @@ test('processorClassName test', function(assert) {
assert.equal(processorClassName({}), "");
assert.equal(processorClassName({
- otherinfo: {
+ otherInfo: {
processorClassName: "foo"
}
}), "foo");
assert.equal(processorClassName({
- otherinfo: {
+ otherInfo: {
processorClassName: "a.b.foo"
}
}), "foo");
