Hi, Not sure if anyone has encountered this. We have a Hadoop Cluster that is secured behind firewalls and the cluster is kerberized and we would like to use Knox to allow access to HiveServer2 using the httpthrift service. We have Tibco Spotfire setup to allow kerberos delegation to occur to HS2 so that it makes the call with the users kerberos context to Knox (using HadoopAuth) mechanism which is proxying the request to HS2 (this fails). When we allow Tibco Spotfire setup to allow kerberos delegation to occur to HS2 directly without Knox this works. Is this a bug in Knox 0.9 or something that has not been supported. I’ve attached the config files which are scrubbed of identifying info. Let me know thoughts on this. Have performed lots of debug and basically the failing request to knox makes it all the way to HS2 but Knox is terminating the requests and causing Hive to fail.
Error from Hive JDBC driver on SpotFire side this does not occur when going
directly to HS2 with httpthrift only when going at Knox using Knox’s HadoopAuth
plugin:
ERROR 2017-02-16T23:59:42,571-0500 [EXAMPLE-CORP\GSS2002, #39, #473]
api.common.InformationModelServiceCommon: Error retrieving metadata:
org.apache.http.client.ClientProtocolException
com.spotfire.ws.api.common.InformationModelWebServiceException: Error
retrieving metadata: org.apache.http.client.ClientProtocolException
at
com.spotfire.ws.api.common.InformationModelServiceCommon.wrapException(InformationModelServiceCommon.java:135)
at
com.spotfire.ws.api.common.InformationModelServiceCommon.wrapException(InformationModelServiceCommon.java:69)
at
com.spotfire.ws.api.element.ElementManagerService.listDataSourceElements(ElementManagerService.java:397)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:181)
at
org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:97)
at
org.apache.cxf.jaxws.AbstractJAXWSMethodInvoker.invoke(AbstractJAXWSMethodInvoker.java:232)
at
org.apache.cxf.jaxws.JAXWSMethodInvoker.invoke(JAXWSMethodInvoker.java:69)
at
org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:75)
at
org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
org.apache.cxf.interceptor.ServiceInvokerInterceptor$2.run(ServiceInvokerInterceptor.java:126)
at
org.apache.cxf.workqueue.SynchronousExecutor.execute(SynchronousExecutor.java:37)
at
org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:131)
at
org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:307)
at
org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121)
at
org.apache.cxf.transport.http.AbstractHTTPDestination.invoke(AbstractHTTPDestination.java:254)
at
org.apache.cxf.transport.servlet.ServletController.invokeDestination(ServletController.java:234)
at
org.apache.cxf.transport.servlet.ServletController.invoke(ServletController.java:208)
at
org.apache.cxf.transport.servlet.ServletController.invoke(ServletController.java:160)
at
org.apache.cxf.transport.servlet.CXFNonSpringServlet.invoke(CXFNonSpringServlet.java:180)
at
org.apache.cxf.transport.servlet.AbstractHTTPServlet.handleRequest(AbstractHTTPServlet.java:298)
at
org.apache.cxf.transport.servlet.AbstractHTTPServlet.doPost(AbstractHTTPServlet.java:217)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:648)
at
org.apache.cxf.transport.servlet.AbstractHTTPServlet.service(AbstractHTTPServlet.java:273)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:292)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207)
at
org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:240)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207)
at
com.spotfire.server.security.SecurityFilter.doFilter(SecurityFilter.java:318)
at
com.spotfire.server.security.AbstractFilter.doFilter(AbstractFilter.java:125)
at
org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:346)
at
org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:262)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:240)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207)
at
com.spotfire.server.security.CustomAuthFilterWrapper.doFilter(CustomAuthFilterWrapper.java:82)
at
com.spotfire.server.security.AbstractFilter.doFilter(AbstractFilter.java:125)
at
org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:346)
at
org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:262)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:240)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207)
at com.spotfire.server.security.CsrfFilter.doFilter(CsrfFilter.java:79)
at
com.spotfire.server.security.AbstractFilter.doFilter(AbstractFilter.java:125)
at
org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:346)
at
org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:262)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:240)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207)
at
com.spotfire.server.security.HttpMethodsFilter.doFilter(HttpMethodsFilter.java:189)
at
com.spotfire.server.security.AbstractFilter.doFilter(AbstractFilter.java:125)
at
org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:346)
at
org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:262)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:240)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207)
at
com.spotfire.server.security.headers.HeadersFilter.doFilter(HeadersFilter.java:192)
at
com.spotfire.server.security.AbstractFilter.doFilter(AbstractFilter.java:125)
at
org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:346)
at
org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:262)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:240)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207)
at
com.spotfire.server.security.AccessLogFilter.doFilter(AccessLogFilter.java:78)
at
com.spotfire.server.security.AbstractFilter.doFilter(AbstractFilter.java:125)
at
org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:346)
at
org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:262)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:240)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207)
at
com.spotfire.server.security.RequestContextFilter.doFilter(RequestContextFilter.java:114)
at
com.spotfire.server.security.RequestContextFilter.doFilter(RequestContextFilter.java:80)
at
com.spotfire.server.security.AbstractFilter.doFilter(AbstractFilter.java:125)
at
org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:346)
at
org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:262)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:240)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207)
at
org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:212)
at
org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:106)
at
org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:502)
at
org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:141)
at
org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:79)
at
org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:88)
at
org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:509)
at
org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1104)
at
org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:684)
at
org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1520)
at
org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.run(NioEndpoint.java:1476)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at
org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.lang.Thread.run(Thread.java:745)
Caused by: com.spotfire.ws.im.IMException: Error retrieving metadata:
org.apache.http.client.ClientProtocolException
at
com.spotfire.ws.im.ds.sql.JDBCDataSourceManager$MetaDataCache.getMetadata(JDBCDataSourceManager.java:1852)
at
com.spotfire.ws.im.ds.sql.JDBCDataSourceManager.getMetadata(JDBCDataSourceManager.java:254)
at
com.spotfire.ws.api.element.ElementManagerService.listDataSourceElements(ElementManagerService.java:393)
... 89 more
Caused by: java.sql.SQLException:
org.apache.http.client.ClientProtocolException
at
org.apache.hive.jdbc.HiveDatabaseMetaData.getTables(HiveDatabaseMetaData.java:656)
at
com.spotfire.server.util.sql.WrappedDatabaseMetaData.getTables(WrappedDatabaseMetaData.java:410)
at
com.spotfire.ws.im.ds.sql.BasicJDBCMetadataProvider.getSchemas(BasicJDBCMetadataProvider.java:318)
at
com.spotfire.ws.im.ds.sql.BasicJDBCMetadataProvider.getMetadata(BasicJDBCMetadataProvider.java:121)
at
com.spotfire.ws.im.ds.sql.JDBCDataSourceManager$MetaDataCache.getMetadata(JDBCDataSourceManager.java:1842)
... 91 more
Caused by: org.apache.thrift.transport.TTransportException:
org.apache.http.client.ClientProtocolException
at
org.apache.thrift.transport.THttpClient.flushUsingHttpClient(THttpClient.java:297)
at org.apache.thrift.transport.THttpClient.flush(THttpClient.java:313)
at org.apache.thrift.TServiceClient.sendBase(TServiceClient.java:73)
at org.apache.thrift.TServiceClient.sendBase(TServiceClient.java:62)
at
org.apache.hive.service.cli.thrift.TCLIService$Client.send_GetTables(TCLIService.java:315)
at
org.apache.hive.service.cli.thrift.TCLIService$Client.GetTables(TCLIService.java:307)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.apache.hive.jdbc.HiveConnection$SynchronizedHandler.invoke(HiveConnection.java:1388)
at com.sun.proxy.$Proxy146.GetTables(Unknown Source)
at
org.apache.hive.jdbc.HiveDatabaseMetaData.getTables(HiveDatabaseMetaData.java:654)
... 95 more
Caused by: org.apache.http.client.ClientProtocolException
at
org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:186)
at
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:117)
at
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
at
org.apache.thrift.transport.THttpClient.flushUsingHttpClient(THttpClient.java:251)
... 107 more
Caused by: org.apache.http.HttpException: The Subject is not set
at
org.apache.hive.jdbc.HttpRequestInterceptorBase.process(HttpRequestInterceptorBase.java:94)
at
org.apache.http.protocol.ImmutableHttpProcessor.process(ImmutableHttpProcessor.java:132)
at
org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:182)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
at
org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
at
org.apache.http.impl.execchain.ServiceUnavailableRetryExec.execute(ServiceUnavailableRetryExec.java:84)
at
org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
... 110 more
Caused by: org.apache.http.HttpException: The Subject is not set
at
org.apache.hive.jdbc.HttpKerberosRequestInterceptor.addHttpAuthHeader(HttpKerberosRequestInterceptor.java:73)
at
org.apache.hive.jdbc.HttpRequestInterceptorBase.process(HttpRequestInterceptorBase.java:78)
... 116 more
Caused by: java.lang.Exception: The Subject is not set
at
org.apache.hive.service.auth.HttpAuthUtils.getKerberosServiceTicket(HttpAuthUtils.java:118)
at
org.apache.hive.jdbc.HttpKerberosRequestInterceptor.addHttpAuthHeader(HttpKerberosRequestInterceptor.java:67)
... 117 more <topology>
<gateway>
<provider>
<role>authentication</role>
<name>HadoopAuth</name>
<enabled>true</enabled>
<param>
<name>config.prefix</name>
<value>hadoop.auth.config</value>
</param>
<param>
<name>hadoop.auth.config.signature.secret</name>
<value>knox-signature-secret</value>
</param>
<param>
<name>hadoop.auth.config.type</name>
<value>kerberos</value>
</param>
<param>
<name>hadoop.auth.config.simple.anonymous.allowed</name>
<value>false</value>
</param>
<param>
<name>hadoop.auth.config.token.validity</name>
<value>36000</value>
</param>
<param>
<name>hadoop.auth.config.cookie.domain</name>
<value>.tech.hdp.newyorklife.com</value>
</param>
<param>
<name>hadoop.auth.config.cookie.path</name>
<value>/gateway/default</value>
</param>
<param>
<name>hadoop.auth.config.kerberos.principal</name>
<value>HTTP/[email protected]</value>
</param>
<param>
<name>hadoop.auth.config.kerberos.keytab</name>
<value>/etc/security/keytabs/spnego.service.keytab</value>
</param>
<param>
<name>hadoop.auth.config.kerberos.name.rules</name>
<value>DEFAULT</value>
</param>
</provider>
<provider>
<role>identity-assertion</role>
<name>Default</name>
<enabled>true</enabled>
</provider>
<provider>
<role>authorization</role>
<name>AclsAuthz</name>
<enabled>true</enabled>
</provider>
<provider>
<role>ha</role>
<name>HaProvider</name>
<enabled>true</enabled>
<param>
<name>WEBHDFS</name>
<value>maxFailoverAttempts=3;failoverSleep=1000;maxRetryAttempts=300;retrySleep=1000;enabled=true</value>
</param>
</provider>
</gateway>
<service>
<role>NAMENODE</role>
<url>hdfs://tech</url>
</service>
<service>
<role>JOBTRACKER</role>
<url>rpc://ha21t52mn.tech.hdp.example.com:8050</url>
</service>
<service>
<role>WEBHDFS</role>
<url>http://ha21t51nn.tech.hdp.example.com:50070/webhdfs</url>
<url>http://ha21t52nn.tech.hdp.example.com:50070/webhdfs</url>
</service>
<service>
<role>WEBHCAT</role>
<url>http://ha21t54mn.tech.hdp.example.com:50111/templeton</url>
</service>
<service>
<role>OOZIE</role>
<url>http://ha21t51mn.tech.hdp.example.com:11000/oozie</url>
</service>
<service>
<role>WEBHBASE</role>
<url>http://ha21t54mn.tech.hdp.example.com:8080</url>
</service>
<service>
<role>HIVE</role>
<url>http://ha21t55mn.tech.hdp.example.com:10001/cliservice</url>
</service>
<service>
<role>RESOURCEMANAGER</role>
<url>http://ha21t52mn.tech.hdp.example.com:8088/ws</url>
</service>
</topology>com.sun.security.jgss.initiate {
com.sun.security.auth.module.Krb5LoginModule required
renewTGT=true
doNotPrompt=true
useKeyTab=true
keyTab="/etc/security/keytabs/knox.service.keytab"
principal="knox/[email protected]"
isInitiator=true
storeKey=true
useTicketCache=true
client=true;
};
gateway-site.xml
Description: XML document
