And on Ganglia server in /var/log/messages abrt: detected unhandled Python exception in '/var/www/cgi-bin/rrd.py' abrtd: New client connected abrt-server[1566]: Not saving repeating crash in '/var/www/cgi-bin/rrd.py'
From: Artem Ervits [mailto:[email protected]] Sent: Friday, October 11, 2013 2:36 PM To: [email protected] Subject: RE: Ganglia gmetad conf Sumit, After running the script I am getting the following errors on the Ambari server and dashboard is not loading: 14:34:59,363 ERROR ReadHandler:83 - Caught a runtime exception executing a query java.lang.NullPointerException at org.apache.ambari.server.controller.ganglia.GangliaPropertyProvider$RRDRequest.populateResources(GangliaPropertyProvider.java:426) at org.apache.ambari.server.controller.ganglia.GangliaPropertyProvider.populateResources(GangliaPropertyProvider.java:116) at org.apache.ambari.server.controller.internal.ClusterControllerImpl.populateResources(ClusterControllerImpl.java:350) at org.apache.ambari.server.controller.internal.ClusterControllerImpl.getResources(ClusterControllerImpl.java:120) at org.apache.ambari.server.controller.internal.ClusterControllerImpl.getResources(ClusterControllerImpl.java:94) at org.apache.ambari.server.api.query.QueryImpl.execute(QueryImpl.java:149) at org.apache.ambari.server.api.query.QueryImpl.execute(QueryImpl.java:169) at org.apache.ambari.server.api.query.QueryImpl.execute(QueryImpl.java:169) at org.apache.ambari.server.api.handlers.ReadHandler.handleRequest(ReadHandler.java:61) at org.apache.ambari.server.api.services.BaseRequest.process(BaseRequest.java:107) at org.apache.ambari.server.api.services.BaseService.handleRequest(BaseService.java:80) at org.apache.ambari.server.api.services.ServiceService.getServices(ServiceService.java:77) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at com.sun.jersey.spi.container.JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60) at com.sun.jersey.server.impl.model.method.dispatch.AbstractResourceMethodDispatchProvider$ResponseOutInvoker._dispatch(AbstractResourceMethodDispatchProvider.java:205) at com.sun.jersey.server.impl.model.method.dispatch.ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.java:75) at com.sun.jersey.server.impl.uri.rules.HttpMethodRule.accept(HttpMethodRule.java:288) at com.sun.jersey.server.impl.uri.rules.SubLocatorRule.accept(SubLocatorRule.java:134) at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) at com.sun.jersey.server.impl.uri.rules.ResourceClassRule.accept(ResourceClassRule.java:108) at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) at com.sun.jersey.server.impl.uri.rules.RootResourceClassesRule.accept(RootResourceClassesRule.java:84) at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1469) at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1400) at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1349) at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1339) at com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:416) at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:537) at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:708) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:652) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1329) at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:330) at org.springframework.security.web.access.intercept.FilterSecurityInterceptor.invoke(FilterSecurityInterceptor.java:118) at org.springframework.security.web.access.intercept.FilterSecurityInterceptor.doFilter(FilterSecurityInterceptor.java:84) at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) at org.springframework.security.web.access.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:113) at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) at org.springframework.security.web.session.SessionManagementFilter.doFilter(SessionManagementFilter.java:103) at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) at org.springframework.security.web.authentication.AnonymousAuthenticationFilter.doFilter(AnonymousAuthenticationFilter.java:113) at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) at org.springframework.security.web.servletapi.SecurityContextHolderAwareRequestFilter.doFilter(SecurityContextHolderAwareRequestFilter.java:54) at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) at org.springframework.security.web.savedrequest.RequestCacheAwareFilter.doFilter(RequestCacheAwareFilter.java:45) at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) at org.springframework.security.web.authentication.www.BasicAuthenticationFilter.doFilter(BasicAuthenticationFilter.java:150) at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) at org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilter(SecurityContextPersistenceFilter.java:87) at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) at org.springframework.security.web.FilterChainProxy.doFilterInternal(FilterChainProxy.java:192) at org.springframework.security.web.FilterChainProxy.doFilter(FilterChainProxy.java:160) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:237) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:167) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1300) at org.apache.ambari.server.api.AmbariPersistFilter.doFilter(AmbariPersistFilter.java:48) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1300) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:445) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:559) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:227) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1038) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:374) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:189) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:972) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116) at org.eclipse.jetty.server.Server.handle(Server.java:363) at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:483) at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:920) at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:982) at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:635) at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235) at org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82) at org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:627) at org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:51) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:662) From: Sumit Mohanty [mailto:[email protected]]<mailto:[mailto:[email protected]]> Sent: Friday, October 11, 2013 11:45 AM To: [email protected]<mailto:[email protected]> Subject: Re: Ganglia gmetad conf The following procedure can be used: Use the following script to read and update the config parameter that specifies the target folder for ganglia rrd files. https://git-wip-us.apache.org/repos/asf/incubator-ambari/repo?p=incubator-ambari.git;a=blob;f=ambari-server/src/main/resources/scripts/configs.sh;h=9ac2d9245b9401ee86014175fc24739eb0bd8218;hb=trunk<https://git-wip-us.apache.org/repos/asf/incubator-ambari/repo?p=incubator-ambari.git;a=blob;f=ambari-server/src/main/resources/scripts/configs.sh;h=2aa5250bde1d916e79fc08ecc2a6b00f292bc558;hb=HEAD> Script name: configs.sh Procedure: * Decide on the new location for the rrd files (e.g. /newlocation/ganglia/rrds) * Stop Ganglia service and wait for it to stop completely * Get the existing folder path for Ganglia rrd files using configs.sh o E.g. ./configs.sh get localhost c1 global | grep "rrdcached_base_dir" (example assume localhost for ambari-server host, c1 for the name of the cluster) * Backup this folder and then remove it * Use configs.sh to update the config property with the new location for the rrd files o E.g. ./configs.sh -u admin -p admin set localhost c1 global "rrdcached_base_dir" "/newlocation/ganglia/rrds" (example assume localhost for ambari-server host, c1 for the name of the cluster and admin/admin as username/password) * Start Ganglia service * You may need to wait for 5-10 minutes to see the metrics getting populated Artem, in addition we can also modify the gmetad config template to reduce number of data points stored for each metrics. Let me know if you need that information. Thanks Sumit From: Artem Ervits <[email protected]<mailto:[email protected]>> Reply-To: <[email protected]<mailto:[email protected]>> Date: Friday, October 11, 2013 8:25 AM To: "[email protected]<mailto:[email protected]>" <[email protected]<mailto:[email protected]>> Subject: Ganglia gmetad conf Hello all, How can I change settings in Ganglia gmetad using Ambari? Do I need to use the API? I need to change the location of the /var/lib/ganglia/rrds database due to low space Artem Ervits New York Presbyterian Hospital This electronic message is intended to be for the use only of the named recipient, and may contain information that is confidential or privileged. If you are not the intended recipient, you are hereby notified that any disclosure, copying, distribution or use of the contents of this message is strictly prohibited. If you have received this message in error or are not the named recipient, please notify us immediately by contacting the sender at the electronic mail address noted above, and delete and destroy all copies of this message. Thank you. CONFIDENTIALITY NOTICE NOTICE: This message is intended for the use of the individual or entity to which it is addressed and may contain information that is confidential, privileged and exempt from disclosure under applicable law. If the reader of this message is not the intended recipient, you are hereby notified that any printing, copying, dissemination, distribution, disclosure or forwarding of this communication is strictly prohibited. If you have received this communication in error, please contact the sender immediately and delete it from your system. Thank You. This electronic message is intended to be for the use only of the named recipient, and may contain information that is confidential or privileged. If you are not the intended recipient, you are hereby notified that any disclosure, copying, distribution or use of the contents of this message is strictly prohibited. If you have received this message in error or are not the named recipient, please notify us immediately by contacting the sender at the electronic mail address noted above, and delete and destroy all copies of this message. Thank you. ________________________________ Confidential Information subject to NYP's (and its affiliates') information management and security policies (http://infonet.nyp.org/QA/HospitalManual). This electronic message is intended to be for the use only of the named recipient, and may contain information that is confidential or privileged. If you are not the intended recipient, you are hereby notified that any disclosure, copying, distribution or use of the contents of this message is strictly prohibited. If you have received this message in error or are not the named recipient, please notify us immediately by contacting the sender at the electronic mail address noted above, and delete and destroy all copies of this message. Thank you.
