Gopal V created TEZ-2889:
----------------------------
Summary: ATSHistoryACLPolicyManager makes DAG Submission wait on
ATS requests
Key: TEZ-2889
URL: https://issues.apache.org/jira/browse/TEZ-2889
Project: Apache Tez
Issue Type: Bug
Affects Versions: 0.7.0, 0.8.2
Reporter: Gopal V
On a cluster with a broken ATS install, the ATS domain creation is in-thread &
bottlenecks all Tez queries, reducing the cluster throughput by ~3x for very
small queries.
{code}
java.net.SocketInputStream.socketRead0(FileDescriptor, byte[], int, int, int)
SocketInputStream.java (native)
java.net.SocketInputStream.socketRead(FileDescriptor, byte[], int, int, int)
SocketInputStream.java:116
java.net.SocketInputStream.read(byte[], int, int, int)
SocketInputStream.java:170
java.net.SocketInputStream.read(byte[], int, int) SocketInputStream.java:141
java.io.BufferedInputStream.fill() BufferedInputStream.java:246
java.io.BufferedInputStream.read1(byte[], int, int) BufferedInputStream.java:286
java.io.BufferedInputStream.read(byte[], int, int) BufferedInputStream.java:345
sun.net.www.http.HttpClient.parseHTTPHeader(MessageHeader, ProgressSource,
HttpURLConnection) HttpClient.java:704
sun.net.www.http.HttpClient.parseHTTP(MessageHeader, ProgressSource,
HttpURLConnection) HttpClient.java:647
sun.net.www.protocol.http.HttpURLConnection.getInputStream0()
HttpURLConnection.java:1535
sun.net.www.protocol.http.HttpURLConnection.getInputStream()
HttpURLConnection.java:1440
java.net.HttpURLConnection.getResponseCode() HttpURLConnection.java:480
org.apache.hadoop.security.authentication.client.AuthenticatedURL.extractToken(HttpURLConnection,
AuthenticatedURL$Token) AuthenticatedURL.java:252
org.apache.hadoop.security.authentication.client.PseudoAuthenticator.authenticate(URL,
AuthenticatedURL$Token) PseudoAuthenticator.java:77
org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.authenticate(URL,
AuthenticatedURL$Token) DelegationTokenAuthenticator.java:128
org.apache.hadoop.security.authentication.client.AuthenticatedURL.openConnection(URL,
AuthenticatedURL$Token) AuthenticatedURL.java:215
org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.openConnection(URL,
DelegationTokenAuthenticatedURL$Token, String)
DelegationTokenAuthenticatedURL.java:322
org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl$TimelineURLConnectionFactory.getHttpURLConnection(URL)
TimelineClientImpl.java:493
com.sun.jersey.client.urlconnection.URLConnectionClientHandler._invoke(ClientRequest)
URLConnectionClientHandler.java:159
com.sun.jersey.client.urlconnection.URLConnectionClientHandler.handle(ClientRequest)
URLConnectionClientHandler.java:147
org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl$TimelineJerseyRetryFilter$1.run()
TimelineClientImpl.java:234
org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl$TimelineClientConnectionRetry.retryOn(TimelineClientImpl$TimelineClientRetryOp)
TimelineClientImpl.java:183
org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl$TimelineJerseyRetryFilter.handle(ClientRequest)
TimelineClientImpl.java:245
com.sun.jersey.api.client.Client.handle(ClientRequest) Client.java:648
com.sun.jersey.api.client.WebResource.handle(Class, ClientRequest)
WebResource.java:670
com.sun.jersey.api.client.WebResource.access$200(WebResource, Class,
ClientRequest) WebResource.java:74
com.sun.jersey.api.client.WebResource$Builder.put(Class, Object)
WebResource.java:533
org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl.doPostingObject(Object,
String) TimelineClientImpl.java:478
org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl$1.run()
TimelineClientImpl.java:323
org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl$1.run()
TimelineClientImpl.java:320
java.security.AccessController.doPrivileged(PrivilegedExceptionAction,
AccessControlContext) AccessController.java (native)
javax.security.auth.Subject.doAs(Subject, PrivilegedExceptionAction)
Subject.java:422
org.apache.hadoop.security.UserGroupInformation.doAs(PrivilegedExceptionAction)
UserGroupInformation.java:1657
org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl.doPosting(Object,
String) TimelineClientImpl.java:320
org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl.putDomain(TimelineDomain)
TimelineClientImpl.java:313
org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager.createTimelineDomain(String,
Configuration, DAGAccessControls) ATSHistoryACLPolicyManager.java:129
org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager.createDAGDomain(Configuration,
ApplicationId, String, DAGAccessControls) ATSHistoryACLPolicyManager.java:207
org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager.setupSessionDAGACLs(Configuration,
ApplicationId, String, DAGAccessControls) ATSHistoryACLPolicyManager.java:240
org.apache.tez.client.TezClient.submitDAGSession(DAG) TezClient.java:491
org.apache.tez.client.TezClient.submitDAG(DAG) TezClient.java:452
org.apache.hadoop.hive.ql.exec.tez.TezTask.submit(JobConf, DAG, Path,
LocalResource, TezSessionState, List, String[], Map) TezTask.java:427
org.apache.hadoop.hive.ql.exec.tez.TezTask.execute(DriverContext)
TezTask.java:170
org.apache.hadoop.hive.ql.exec.Task.executeTask() Task.java:160
org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential() TaskRunner.java:89
org.apache.hadoop.hive.ql.Driver.launchTask(Task, String, boolean, String, int,
DriverContext) Driver.java:1758
org.apache.hadoop.hive.ql.Driver.execute() Driver.java:1517
org.apache.hadoop.hive.ql.Driver.runInternal(String, boolean) Driver.java:1275
org.apache.hadoop.hive.ql.Driver.run(String, boolean) Driver.java:1091
org.apache.hadoop.hive.ql.Driver.run(String) Driver.java:1081
org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(String, CommandProcessor,
CliSessionState) CliDriver.java:225
org.apache.hadoop.hive.cli.CliDriver.processCmd(String) CliDriver.java:177
org.apache.hadoop.hive.cli.CliDriver.processLine(String, boolean)
CliDriver.java:388
org.apache.hadoop.hive.cli.CliDriver.executeDriver(CliSessionState, HiveConf,
OptionsProcessor) CliDriver.java:759
org.apache.hadoop.hive.cli.CliDriver.run(String[]) CliDriver.java:704
org.apache.hadoop.hive.cli.CliDriver.main(String[]) CliDriver.java:633
sun.reflect.NativeMethodAccessorImpl.invoke0(Method, Object, Object[])
NativeMethodAccessorImpl.java (native)
sun.reflect.NativeMethodAccessorImpl.invoke(Object, Object[])
NativeMethodAccessorImpl.java:62
sun.reflect.DelegatingMethodAccessorImpl.invoke(Object, Object[])
DelegatingMethodAccessorImpl.java:43
java.lang.reflect.Method.invoke(Object, Object[]) Method.java:497
org.apache.hadoop.util.RunJar.run(String[]) RunJar.java:221
org.apache.hadoop.util.RunJar.main(String[]) RunJar.java:136
{code}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
