[
https://issues.apache.org/jira/browse/TEZ-2889?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15135476#comment-15135476
]
Hitesh Shah edited comment on TEZ-2889 at 2/6/16 2:25 AM:
----------------------------------------------------------
\cc [~sseth] [~bikassaha] [~zjffdu] in case they have any other points to
suggest.
was (Author: hitesh):
\cc [~sseth] in case he has any other points to suggest.
> ATSHistoryACLPolicyManager makes DAG Submission wait on ATS requests
> --------------------------------------------------------------------
>
> Key: TEZ-2889
> URL: https://issues.apache.org/jira/browse/TEZ-2889
> Project: Apache Tez
> Issue Type: Bug
> Affects Versions: 0.7.0, 0.8.2
> Reporter: Gopal V
> Assignee: Hitesh Shah
> Labels: ATS, Performance
>
> On a cluster with a broken ATS install, the ATS domain creation is in-thread
> & bottlenecks all Tez queries, reducing the cluster throughput by ~3x for
> very small queries.
> {code}
> java.net.SocketInputStream.socketRead0(FileDescriptor, byte[], int, int, int)
> SocketInputStream.java (native)
> java.net.SocketInputStream.socketRead(FileDescriptor, byte[], int, int, int)
> SocketInputStream.java:116
> java.net.SocketInputStream.read(byte[], int, int, int)
> SocketInputStream.java:170
> java.net.SocketInputStream.read(byte[], int, int) SocketInputStream.java:141
> java.io.BufferedInputStream.fill() BufferedInputStream.java:246
> java.io.BufferedInputStream.read1(byte[], int, int)
> BufferedInputStream.java:286
> java.io.BufferedInputStream.read(byte[], int, int)
> BufferedInputStream.java:345
> sun.net.www.http.HttpClient.parseHTTPHeader(MessageHeader, ProgressSource,
> HttpURLConnection) HttpClient.java:704
> sun.net.www.http.HttpClient.parseHTTP(MessageHeader, ProgressSource,
> HttpURLConnection) HttpClient.java:647
> sun.net.www.protocol.http.HttpURLConnection.getInputStream0()
> HttpURLConnection.java:1535
> sun.net.www.protocol.http.HttpURLConnection.getInputStream()
> HttpURLConnection.java:1440
> java.net.HttpURLConnection.getResponseCode() HttpURLConnection.java:480
> org.apache.hadoop.security.authentication.client.AuthenticatedURL.extractToken(HttpURLConnection,
> AuthenticatedURL$Token) AuthenticatedURL.java:252
> org.apache.hadoop.security.authentication.client.PseudoAuthenticator.authenticate(URL,
> AuthenticatedURL$Token) PseudoAuthenticator.java:77
> org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.authenticate(URL,
> AuthenticatedURL$Token) DelegationTokenAuthenticator.java:128
> org.apache.hadoop.security.authentication.client.AuthenticatedURL.openConnection(URL,
> AuthenticatedURL$Token) AuthenticatedURL.java:215
> org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.openConnection(URL,
> DelegationTokenAuthenticatedURL$Token, String)
> DelegationTokenAuthenticatedURL.java:322
> org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl$TimelineURLConnectionFactory.getHttpURLConnection(URL)
> TimelineClientImpl.java:493
> com.sun.jersey.client.urlconnection.URLConnectionClientHandler._invoke(ClientRequest)
> URLConnectionClientHandler.java:159
> com.sun.jersey.client.urlconnection.URLConnectionClientHandler.handle(ClientRequest)
> URLConnectionClientHandler.java:147
> org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl$TimelineJerseyRetryFilter$1.run()
> TimelineClientImpl.java:234
> org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl$TimelineClientConnectionRetry.retryOn(TimelineClientImpl$TimelineClientRetryOp)
> TimelineClientImpl.java:183
> org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl$TimelineJerseyRetryFilter.handle(ClientRequest)
> TimelineClientImpl.java:245
> com.sun.jersey.api.client.Client.handle(ClientRequest) Client.java:648
> com.sun.jersey.api.client.WebResource.handle(Class, ClientRequest)
> WebResource.java:670
> com.sun.jersey.api.client.WebResource.access$200(WebResource, Class,
> ClientRequest) WebResource.java:74
> com.sun.jersey.api.client.WebResource$Builder.put(Class, Object)
> WebResource.java:533
> org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl.doPostingObject(Object,
> String) TimelineClientImpl.java:478
> org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl$1.run()
> TimelineClientImpl.java:323
> org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl$1.run()
> TimelineClientImpl.java:320
> java.security.AccessController.doPrivileged(PrivilegedExceptionAction,
> AccessControlContext) AccessController.java (native)
> javax.security.auth.Subject.doAs(Subject, PrivilegedExceptionAction)
> Subject.java:422
> org.apache.hadoop.security.UserGroupInformation.doAs(PrivilegedExceptionAction)
> UserGroupInformation.java:1657
> org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl.doPosting(Object,
> String) TimelineClientImpl.java:320
> org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl.putDomain(TimelineDomain)
> TimelineClientImpl.java:313
> org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager.createTimelineDomain(String,
> Configuration, DAGAccessControls) ATSHistoryACLPolicyManager.java:129
> org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager.createDAGDomain(Configuration,
> ApplicationId, String, DAGAccessControls) ATSHistoryACLPolicyManager.java:207
> org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager.setupSessionDAGACLs(Configuration,
> ApplicationId, String, DAGAccessControls) ATSHistoryACLPolicyManager.java:240
> org.apache.tez.client.TezClient.submitDAGSession(DAG) TezClient.java:491
> org.apache.tez.client.TezClient.submitDAG(DAG) TezClient.java:452
> org.apache.hadoop.hive.ql.exec.tez.TezTask.submit(JobConf, DAG, Path,
> LocalResource, TezSessionState, List, String[], Map) TezTask.java:427
> org.apache.hadoop.hive.ql.exec.tez.TezTask.execute(DriverContext)
> TezTask.java:170
> org.apache.hadoop.hive.ql.exec.Task.executeTask() Task.java:160
> org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential() TaskRunner.java:89
> org.apache.hadoop.hive.ql.Driver.launchTask(Task, String, boolean, String,
> int, DriverContext) Driver.java:1758
> org.apache.hadoop.hive.ql.Driver.execute() Driver.java:1517
> org.apache.hadoop.hive.ql.Driver.runInternal(String, boolean) Driver.java:1275
> org.apache.hadoop.hive.ql.Driver.run(String, boolean) Driver.java:1091
> org.apache.hadoop.hive.ql.Driver.run(String) Driver.java:1081
> org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(String,
> CommandProcessor, CliSessionState) CliDriver.java:225
> org.apache.hadoop.hive.cli.CliDriver.processCmd(String) CliDriver.java:177
> org.apache.hadoop.hive.cli.CliDriver.processLine(String, boolean)
> CliDriver.java:388
> org.apache.hadoop.hive.cli.CliDriver.executeDriver(CliSessionState, HiveConf,
> OptionsProcessor) CliDriver.java:759
> org.apache.hadoop.hive.cli.CliDriver.run(String[]) CliDriver.java:704
> org.apache.hadoop.hive.cli.CliDriver.main(String[]) CliDriver.java:633
> sun.reflect.NativeMethodAccessorImpl.invoke0(Method, Object, Object[])
> NativeMethodAccessorImpl.java (native)
> sun.reflect.NativeMethodAccessorImpl.invoke(Object, Object[])
> NativeMethodAccessorImpl.java:62
> sun.reflect.DelegatingMethodAccessorImpl.invoke(Object, Object[])
> DelegatingMethodAccessorImpl.java:43
> java.lang.reflect.Method.invoke(Object, Object[]) Method.java:497
> org.apache.hadoop.util.RunJar.run(String[]) RunJar.java:221
> org.apache.hadoop.util.RunJar.main(String[]) RunJar.java:136
> {code}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
