http://git-wip-us.apache.org/repos/asf/eagle/blob/d6fed677/_site/docs/hdfs-data-activity-monitoring.html ---------------------------------------------------------------------- diff --git a/_site/docs/hdfs-data-activity-monitoring.html b/_site/docs/hdfs-data-activity-monitoring.html new file mode 100644 index 0000000..b0a694e --- /dev/null +++ b/_site/docs/hdfs-data-activity-monitoring.html @@ -0,0 +1,327 @@ +<!DOCTYPE html> +<head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + + <title>Eagle - HDFS Data Activity Monitoring Quick Start</title> + <meta name="description" content="Eagle - Analyze Big Data Platforms for Security and Performance"> + + <meta name="keywords" content="Eagle, Hadoop, Security, Real Time"> + <meta name="author" content="eBay Inc."> + + <meta charset="utf-8"> + <meta name="viewport" content="initial-scale=1"> + + <link rel="stylesheet" href="/css/animate.css"> + <link rel="stylesheet" href="/css/bootstrap.min.css"> + + <link rel="stylesheet" href="/css/font-awesome.min.css"> + + <link rel="stylesheet" href="/css/misc.css"> + <link rel="stylesheet" href="/css/style.css"> + <link rel="stylesheet" href="/css/styles.css"> + <link rel="stylesheet" href="/css/main.css"> + <link rel="alternate" type="application/rss+xml" title="Eagle" href="http://goeagle.io/feed.xml"; /> + <link rel="shortcut icon" href="/images/favicon.png"> + + <!-- Baidu Analytics Tracking--> + <script> + var _hmt = _hmt || []; + (function() { + var hm = document.createElement("script"); + hm.src = "//hm.baidu.com/hm.js?fedc55df2ea52777a679192e8f849ece"; + var s = document.getElementsByTagName("script")[0]; + s.parentNode.insertBefore(hm, s); + })(); + </script> + + <!-- Google Analytics Tracking --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + ga('create', 'UA-68929805-1', 'auto'); + ga('send', 'pageview'); + </script> +</head> +<body> +<!-- header start --> +<div id="home_page"> + <div class="topbar"> + <div class="container"> + <div class="row" > + <nav class="navbar navbar-default"> + <div class="container-fluid"> + <!-- Brand and toggle get grouped for better mobile display --> + <div class="navbar-header"> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> <span class="sr-only">Toggle navigation</span> <span class="icon-bar"></span> <span class="icon-bar"></span> <span class="icon-bar"></span> </button> + <a class="navbar-brand" href="/"><img src="/images/logo2.png" height="44px" style="margin-top:-7px"></a> </div> + + <!-- Collect the nav links, forms, and other content for toggling --> + <!-- <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> + <ul class="nav navbar-nav navbar-right" id="top-menu"> + <li><a class="menu" href="/#home_page">HOME</a></li> + <li><a class="menu" href="/docs/">DOCS</a></li> + <li><a class="menu" href="/#about_page">ABOUT</a></li> + <li><a class="menu" href="/#diagram_page">ARCHITECTURE</a></li> + <li><a class="menu" href="/#modules_page">MODULES</a></li> + <li><a class="menu" href="/#usecase_page">USE CASES</a></li> + <li> + </li> + </ul> --> + </div> + </div> + <!-- /.container-fluid --> + </nav> + </div> + </div> + </div> + <div class="headerimage"> + <div class="flexslider"> + <ul class="slides"> + <li><img src="/images/slider/4.jpg" alt="Slide 1"></li> + </ul> + </div> + </div> + <div class="particles" style="height:40%"> </div><!---particles--> + <div class="slider-caption" style="top:80px;"> + <div class="homewrapper"> + <div class="hometitle"> + <a href="/"> + <img src="/images/feather.png" height="80px"> + </a> + </div> + <div class="hometext"> + <h2>Analyze Big Data Platforms For Security and Performance</h2> + <div class="social-buttons"> + <a href="https://github.com/apache/eagle";><i class="fa fa-github"></i></a> + <a href="http://twitter.com/TheApacheEagle";><i class="fa fa-twitter"></i></a> + <a href="https://www.facebook.com/TheApacheEagle/";><i class="fa fa-facebook"></i></a> + <a href="#"><i class="fa fa-weixin"></i></a> + <!-- <a href="https://www.weibo.com/ApacheEagle/";><i class="fa fa-weibo"></i></a> --> + </div> + </div> + </div> + </div> +</div> +<!-- header end --> +<div class="container-fluid page-content"> + <div class="row"> + <div class="col-md-10 col-md-offset-1"> + <!-- sidebar --> + <div class="col-xs-6 col-sm-3" id="sidebar" role="navigation"> + <ul class="nav" id="adminnav"> + + <li class="heading">Getting Started</li> + + <li class="sidenavli "><a href="/docs/index.html" data-permalink="/docs/hdfs-data-activity-monitoring.html" id="">Introduction</a></li> + + <li class="sidenavli "><a href="/docs/usecases.html" data-permalink="/docs/hdfs-data-activity-monitoring.html" id="">Use Cases</a></li> + + <li class="sidenavli "><a href="/docs/terminology.html" data-permalink="/docs/hdfs-data-activity-monitoring.html" id="">Terminology</a></li> + + <li class="sidenavli "><a href="/docs/ecosystem.html" data-permalink="/docs/hdfs-data-activity-monitoring.html" id="">Ecosystem</a></li> + + <li class="sidenavli "><a href="/docs/community.html" data-permalink="/docs/hdfs-data-activity-monitoring.html" id="">Community</a></li> + + <li class="sidenavli "><a href="/docs/FAQ.html" data-permalink="/docs/hdfs-data-activity-monitoring.html" id="">FAQ</a></li> + + <li class="divider"></li> + + <li class="heading">Documentations</li> + + <li class="sidenavli "><a href="/docs/latest/" data-permalink="/docs/hdfs-data-activity-monitoring.html" id="">Latest version (v0.5.0)</a></li> + + <li class="divider"></li> + + <li class="heading">Download</li> + + <li class="sidenavli "><a href="/docs/download-latest.html" data-permalink="/docs/hdfs-data-activity-monitoring.html" id="">Latest version (v0.5.0)</a></li> + + <li class="sidenavli "><a href="/docs/download.html" data-permalink="/docs/hdfs-data-activity-monitoring.html" id="">Archived</a></li> + + <li class="divider"></li> + + <li class="heading">Supplement</li> + + <li class="sidenavli "><a href="/docs/security.html" data-permalink="/docs/hdfs-data-activity-monitoring.html" id="">Security</a></li> + + <li class="divider"></li> + + <li class="sidenavli"> + <a href="mailto:d...@eagle.apache.org"; target="_blank">Need Help?</a> + </li> + </ul> + </div> + <div class="col-xs-6 col-sm-9 page-main-content" style="margin-left: -15px" id="loadcontent"> + <h1 class="page-header" style="margin-top: 0px">HDFS Data Activity Monitoring Quick Start</h1> + <p><em>Since Apache Eagle 0.3.0-incubating. Apache Eagle will be called Eagle in the following.</em></p> + +<p>This Guide describes the steps to enable data activity monitoring of âHDFS File Systemâ.</p> + +<ul> + <li>Prerequisite</li> + <li>Stream HDFS audit logs into Kafka<sup id="fnref:KAFKA"><a href="#fn:KAFKA" class="footnote">1</a></sup></li> + <li>Demos âHDFS Data Activity Monitoringâ +<br /><br /></li> +</ul> + +<h3 id="prerequisite"><strong>Prerequisite</strong></h3> +<ul> + <li>Complete the setup from <a href="/docs/quick-start.html">Quick Start(Eagle In Sandbox)</a> +<br /><br /></li> +</ul> + +<h3 id="stream-hdfs-audit-logs-into-kafka"><strong>Stream HDFS audit logs into Kafka</strong></h3> + +<p>Note: This section describes how to configure Kafka log4j to stream audit logs into Eagle platform. For another option to stream HDFS audit logs into Kafka using Logstash <a href="/docs/import-hdfs-auditLog.html">Click Here</a></p> + +<ul> + <li> + <p><strong>Step 1</strong>: Configure Advanced hdfs-log4j via <a href="http://localhost:8080/#/main/services/HDFS/configs"; target="_blank">Ambari UI</a><sup id="fnref:AMBARI"><a href="#fn:AMBARI" class="footnote">2</a></sup>, by adding below âKAFKA_HDFS_AUDITâ log4j appender to hdfs audit logging.</p> + + <pre><code> log4j.appender.KAFKA_HDFS_AUDIT=org.apache.eagle.log4j.kafka.KafkaLog4jAppender + log4j.appender.KAFKA_HDFS_AUDIT.Topic=sandbox_hdfs_audit_log + log4j.appender.KAFKA_HDFS_AUDIT.BrokerList=sandbox.hortonworks.com:6667 + log4j.appender.KAFKA_HDFS_AUDIT.KeyClass=org.apache.eagle.log4j.kafka.hadoop.AuditLogKeyer + log4j.appender.KAFKA_HDFS_AUDIT.Layout=org.apache.log4j.PatternLayout + log4j.appender.KAFKA_HDFS_AUDIT.Layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n + log4j.appender.KAFKA_HDFS_AUDIT.ProducerType=async +</code></pre> + + <p><img src="/images/docs/hdfs-log4j-conf.png" alt="HDFS LOG4J Configuration" title="hdfslog4jconf" /></p> + </li> + <li> + <p><strong>Step 2</strong>: Edit Advanced hadoop-env via <a href="http://localhost:8080/#/main/services/HDFS/configs"; target="_blank">Ambari UI</a>, and add the reference to KAFKA_HDFS_AUDIT to HADOOP_NAMENODE_OPTS.</p> + + <pre><code>-Dhdfs.audit.logger=INFO,DRFAAUDIT,KAFKA_HDFS_AUDIT +</code></pre> + + <p><img src="/images/docs/hdfs-env-conf.png" alt="HDFS Environment Configuration" title="hdfsenvconf" /></p> + </li> + <li> + <p><strong>Step 3</strong>: Edit Advanced hadoop-env via <a href="http://localhost:8080/#/main/services/HDFS/configs"; target="_blank">Ambari UI</a>, and append the following command to it.</p> + + <pre><code>export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:/usr/hdp/current/eagle/lib/log4jkafka/lib/* +</code></pre> + + <p><img src="/images/docs/hdfs-env-conf2.png" alt="HDFS Environment Configuration" title="hdfsenvconf2" /></p> + </li> + <li> + <p><strong>Step 4</strong>: save the changes</p> + </li> + <li> + <p><strong>Step 5</strong>: âRestart Allâ Storm<sup id="fnref:STORM"><a href="#fn:STORM" class="footnote">3</a></sup> & Kafka from Ambari.</p> + </li> + <li> + <p><strong>Step 6</strong>: Restart name node</p> + </li> +</ul> + +<p><img src="/images/docs/nn-restart.png" alt="Restart Services" title="Services" /></p> + +<ul> + <li> + <p><strong>Step 7</strong>: Check whether logs from â/var/log/hadoop/hdfs/hdfs-audit.logâ are flowing into topic <code>sandbox_hdfs_audit_log</code></p> + + <pre><code> $ /usr/hdp/2.2.4.2-2/kafka/bin/kafka-console-consumer.sh --zookeeper sandbox.hortonworks.com:2181 --topic sandbox_hdfs_audit_log +</code></pre> + </li> +</ul> + +<p><br /></p> + +<h3 id="demos"><strong>Demos</strong></h3> +<ul> + <li>Login to Eagle UI <a href="http://localhost:9099/eagle-service/";>http://localhost:9099/eagle-service/</a> using username and password as âadminâ and âsecretâ</li> + <li><strong>HDFS</strong>: + <ol> + <li>Click on menu âDAMâ and select âHDFSâ to view HDFS policy</li> + <li>You should see policy with name âviewPrivateâ. This Policy generates alert when any user reads HDFS file name âprivateâ under âtmpâ folder.</li> + <li> + <p>In sandbox read restricted HDFS file â/tmp/privateâ by using command</p> + + <blockquote> + <p>hadoop fs -cat /tmp/private</p> + </blockquote> + </li> + </ol> + + <p>From UI click on alert tab and you should see alert for the attempt to read restricted file.<br /> +<br /></p> + </li> +</ul> + +<hr /> + +<h4 id="footnotes"><em>Footnotes</em></h4> + +<div class="footnotes"> + <ol> + <li id="fn:KAFKA"> + <p><em>All mentions of âkafkaâ on this page represent Apache Kafka.</em> <a href="#fnref:KAFKA" class="reversefootnote">↩</a></p> + </li> + <li id="fn:AMBARI"> + <p><em>All mentions of âambariâ on this page represent Apache Ambari.</em> <a href="#fnref:AMBARI" class="reversefootnote">↩</a></p> + </li> + <li id="fn:STORM"> + <p><em>Apache Storm.</em> <a href="#fnref:STORM" class="reversefootnote">↩</a></p> + </li> + </ol> +</div> + + </div><!--end of loadcontent--> + </div> + <!--end of centered content--> + </div> +</div> +<!--end of container--> + +<!-- footer start --> +<div class="footerwrapper"> + <div class="container"> + <div class="row"> + <div class="col-md-12"><div style="margin-left:auto; margin-right:auto; text-align:center;font-size: 12px"> +<div> +</div> +<div> +<a href="http://www.apache.org";> +<img id="asf-logo" alt="Apache Software Foundation" src="/images/apache-logo-small.gif"> +</a> +</div> +<div> +Copyright © 2015 <a href="http://www.apache.org";>The Apache Software Foundation</a>, Licensed under the <a href="http://www.apache.org/licenses/LICENSE-2.0";>Apache License, Version 2.0</a>. +</div> +<div> +Apache Eagle, Eagle, Apache Hadoop, Hadoop, Apache HBase, HBase, Apache Hive, Hive, Apache Ambari, Ambari, Apache Spark, Spark, Apache Kafka, Kafka, Apache Storm, Storm, Apache Maven, Maven, Apache Tomcat, Tomcat, Apache Derby, Derby, Apache Cassandra, Cassandra, Apache ZooKeeper, ZooKeeper, Apache, the Apache feather logo, and the Apache project logo are trademarks of The Apache Software Foundation. +</div> + </div></div> + </div> + </div> +</div> +<!-- footer end --> + +<!-- JavaScripts --> +<script src="/js/jquery-1.11.1.min.js"></script> +<script src="/js/jquery.singlePageNav.js"></script> +<script src="/js/jquery.flexslider.js"></script> +<script src="/js/modernizr.min.js"></script> +<script src="/js/svg.js"></script> +<script> + /************** FlexSlider *********************/ + $('.flexslider').flexslider({ + animation: "fade", + directionNav: false + }); +</script> + +<script> + /************** FlexSlider *********************/ + $('.flexslider').flexslider({ + animation: "fade", + directionNav: false + }); +</script> +</body> +</html>
http://git-wip-us.apache.org/repos/asf/eagle/blob/d6fed677/_site/docs/hive-query-activity-monitoring.html ---------------------------------------------------------------------- diff --git a/_site/docs/hive-query-activity-monitoring.html b/_site/docs/hive-query-activity-monitoring.html new file mode 100644 index 0000000..d128562 --- /dev/null +++ b/_site/docs/hive-query-activity-monitoring.html @@ -0,0 +1,264 @@ +<!DOCTYPE html> +<head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + + <title>Eagle - Hive Query Activity Monitoring Quick Start</title> + <meta name="description" content="Eagle - Analyze Big Data Platforms for Security and Performance"> + + <meta name="keywords" content="Eagle, Hadoop, Security, Real Time"> + <meta name="author" content="eBay Inc."> + + <meta charset="utf-8"> + <meta name="viewport" content="initial-scale=1"> + + <link rel="stylesheet" href="/css/animate.css"> + <link rel="stylesheet" href="/css/bootstrap.min.css"> + + <link rel="stylesheet" href="/css/font-awesome.min.css"> + + <link rel="stylesheet" href="/css/misc.css"> + <link rel="stylesheet" href="/css/style.css"> + <link rel="stylesheet" href="/css/styles.css"> + <link rel="stylesheet" href="/css/main.css"> + <link rel="alternate" type="application/rss+xml" title="Eagle" href="http://goeagle.io/feed.xml"; /> + <link rel="shortcut icon" href="/images/favicon.png"> + + <!-- Baidu Analytics Tracking--> + <script> + var _hmt = _hmt || []; + (function() { + var hm = document.createElement("script"); + hm.src = "//hm.baidu.com/hm.js?fedc55df2ea52777a679192e8f849ece"; + var s = document.getElementsByTagName("script")[0]; + s.parentNode.insertBefore(hm, s); + })(); + </script> + + <!-- Google Analytics Tracking --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + ga('create', 'UA-68929805-1', 'auto'); + ga('send', 'pageview'); + </script> +</head> +<body> +<!-- header start --> +<div id="home_page"> + <div class="topbar"> + <div class="container"> + <div class="row" > + <nav class="navbar navbar-default"> + <div class="container-fluid"> + <!-- Brand and toggle get grouped for better mobile display --> + <div class="navbar-header"> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> <span class="sr-only">Toggle navigation</span> <span class="icon-bar"></span> <span class="icon-bar"></span> <span class="icon-bar"></span> </button> + <a class="navbar-brand" href="/"><img src="/images/logo2.png" height="44px" style="margin-top:-7px"></a> </div> + + <!-- Collect the nav links, forms, and other content for toggling --> + <!-- <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> + <ul class="nav navbar-nav navbar-right" id="top-menu"> + <li><a class="menu" href="/#home_page">HOME</a></li> + <li><a class="menu" href="/docs/">DOCS</a></li> + <li><a class="menu" href="/#about_page">ABOUT</a></li> + <li><a class="menu" href="/#diagram_page">ARCHITECTURE</a></li> + <li><a class="menu" href="/#modules_page">MODULES</a></li> + <li><a class="menu" href="/#usecase_page">USE CASES</a></li> + <li> + </li> + </ul> --> + </div> + </div> + <!-- /.container-fluid --> + </nav> + </div> + </div> + </div> + <div class="headerimage"> + <div class="flexslider"> + <ul class="slides"> + <li><img src="/images/slider/4.jpg" alt="Slide 1"></li> + </ul> + </div> + </div> + <div class="particles" style="height:40%"> </div><!---particles--> + <div class="slider-caption" style="top:80px;"> + <div class="homewrapper"> + <div class="hometitle"> + <a href="/"> + <img src="/images/feather.png" height="80px"> + </a> + </div> + <div class="hometext"> + <h2>Analyze Big Data Platforms For Security and Performance</h2> + <div class="social-buttons"> + <a href="https://github.com/apache/eagle";><i class="fa fa-github"></i></a> + <a href="http://twitter.com/TheApacheEagle";><i class="fa fa-twitter"></i></a> + <a href="https://www.facebook.com/TheApacheEagle/";><i class="fa fa-facebook"></i></a> + <a href="#"><i class="fa fa-weixin"></i></a> + <!-- <a href="https://www.weibo.com/ApacheEagle/";><i class="fa fa-weibo"></i></a> --> + </div> + </div> + </div> + </div> +</div> +<!-- header end --> +<div class="container-fluid page-content"> + <div class="row"> + <div class="col-md-10 col-md-offset-1"> + <!-- sidebar --> + <div class="col-xs-6 col-sm-3" id="sidebar" role="navigation"> + <ul class="nav" id="adminnav"> + + <li class="heading">Getting Started</li> + + <li class="sidenavli "><a href="/docs/index.html" data-permalink="/docs/hive-query-activity-monitoring.html" id="">Introduction</a></li> + + <li class="sidenavli "><a href="/docs/usecases.html" data-permalink="/docs/hive-query-activity-monitoring.html" id="">Use Cases</a></li> + + <li class="sidenavli "><a href="/docs/terminology.html" data-permalink="/docs/hive-query-activity-monitoring.html" id="">Terminology</a></li> + + <li class="sidenavli "><a href="/docs/ecosystem.html" data-permalink="/docs/hive-query-activity-monitoring.html" id="">Ecosystem</a></li> + + <li class="sidenavli "><a href="/docs/community.html" data-permalink="/docs/hive-query-activity-monitoring.html" id="">Community</a></li> + + <li class="sidenavli "><a href="/docs/FAQ.html" data-permalink="/docs/hive-query-activity-monitoring.html" id="">FAQ</a></li> + + <li class="divider"></li> + + <li class="heading">Documentations</li> + + <li class="sidenavli "><a href="/docs/latest/" data-permalink="/docs/hive-query-activity-monitoring.html" id="">Latest version (v0.5.0)</a></li> + + <li class="divider"></li> + + <li class="heading">Download</li> + + <li class="sidenavli "><a href="/docs/download-latest.html" data-permalink="/docs/hive-query-activity-monitoring.html" id="">Latest version (v0.5.0)</a></li> + + <li class="sidenavli "><a href="/docs/download.html" data-permalink="/docs/hive-query-activity-monitoring.html" id="">Archived</a></li> + + <li class="divider"></li> + + <li class="heading">Supplement</li> + + <li class="sidenavli "><a href="/docs/security.html" data-permalink="/docs/hive-query-activity-monitoring.html" id="">Security</a></li> + + <li class="divider"></li> + + <li class="sidenavli"> + <a href="mailto:d...@eagle.apache.org"; target="_blank">Need Help?</a> + </li> + </ul> + </div> + <div class="col-xs-6 col-sm-9 page-main-content" style="margin-left: -15px" id="loadcontent"> + <h1 class="page-header" style="margin-top: 0px">Hive Query Activity Monitoring Quick Start</h1> + <p><em>Since Apache Eagle 0.3.0-incubating. Apache Eagle will be called Eagle in the following.</em></p> + +<p>This Guide describes the steps to enable HIVE<sup id="fnref:HIVE"><a href="#fn:HIVE" class="footnote">1</a></sup> query activity monitoring.</p> + +<ul> + <li>Prerequisite</li> + <li>Stream HIVE query logs into Eagle platform</li> + <li>Demos âHive Query Activity Monitoringâ +<br /><br /></li> +</ul> + +<h3 id="prerequisite"><strong>Prerequisite</strong></h3> +<ul> + <li>Complete the setup from <a href="/docs/quick-start.html">Quick Start(Eagle In Sandbox)</a> +<br /><br /></li> +</ul> + +<h3 id="stream-hive-query-logs-into-eagle-platform"><strong>Stream HIVE query logs into Eagle platform</strong></h3> +<p>There are a couple of methods to capture HIVE query logs. As of 0.4.0, Eagle uses YARN API to periodically poll running HIVE jobs and in realtime parse query expressions. So here Eagle assumes resource manager is installed in Hadoop[^HADOOP] cluster.</p> + +<h3 id="demos"><strong>Demos</strong></h3> +<ul> + <li><strong>Hive</strong>: + <ol> + <li>Click on menu âDAMâ and select âHiveâ to view Hive policy</li> + <li>You should see policy with name âqueryPhoneNumberâ. This Policy generates alert when hive table with sensitivity(Phone_Number) information is queried.</li> + <li>In sandbox read restricted sensitive HIVE column. ( To learn more about data sensitivity settings click <a href="/docs/tutorial/classification.html">Data Classification Tutorial</a>)</li> + </ol> + </li> +</ul> + +<pre><code>$ su hive +$ hive +$ set hive.execution.engine=mr; +$ use xademo; +$ select a.phone_number from customer_details a, call_detail_records b where a.phone_number=b.phone_number; +</code></pre> + +<p>From UI click on alert tab and you should see alert for your attempt to read restricted column.</p> + +<hr /> + +<h4 id="footnotes"><em>Footnotes</em></h4> + +<div class="footnotes"> + <ol> + <li id="fn:HIVE"> + <p><em>All mentions of âhiveâ on this page represent Apache Hive.</em> <a href="#fnref:HIVE" class="reversefootnote">↩</a></p> + </li> + </ol> +</div> + + </div><!--end of loadcontent--> + </div> + <!--end of centered content--> + </div> +</div> +<!--end of container--> + +<!-- footer start --> +<div class="footerwrapper"> + <div class="container"> + <div class="row"> + <div class="col-md-12"><div style="margin-left:auto; margin-right:auto; text-align:center;font-size: 12px"> +<div> +</div> +<div> +<a href="http://www.apache.org";> +<img id="asf-logo" alt="Apache Software Foundation" src="/images/apache-logo-small.gif"> +</a> +</div> +<div> +Copyright © 2015 <a href="http://www.apache.org";>The Apache Software Foundation</a>, Licensed under the <a href="http://www.apache.org/licenses/LICENSE-2.0";>Apache License, Version 2.0</a>. +</div> +<div> +Apache Eagle, Eagle, Apache Hadoop, Hadoop, Apache HBase, HBase, Apache Hive, Hive, Apache Ambari, Ambari, Apache Spark, Spark, Apache Kafka, Kafka, Apache Storm, Storm, Apache Maven, Maven, Apache Tomcat, Tomcat, Apache Derby, Derby, Apache Cassandra, Cassandra, Apache ZooKeeper, ZooKeeper, Apache, the Apache feather logo, and the Apache project logo are trademarks of The Apache Software Foundation. +</div> + </div></div> + </div> + </div> +</div> +<!-- footer end --> + +<!-- JavaScripts --> +<script src="/js/jquery-1.11.1.min.js"></script> +<script src="/js/jquery.singlePageNav.js"></script> +<script src="/js/jquery.flexslider.js"></script> +<script src="/js/modernizr.min.js"></script> +<script src="/js/svg.js"></script> +<script> + /************** FlexSlider *********************/ + $('.flexslider').flexslider({ + animation: "fade", + directionNav: false + }); +</script> + +<script> + /************** FlexSlider *********************/ + $('.flexslider').flexslider({ + animation: "fade", + directionNav: false + }); +</script> +</body> +</html> http://git-wip-us.apache.org/repos/asf/eagle/blob/d6fed677/_site/docs/import-hdfs-auditLog.html ---------------------------------------------------------------------- diff --git a/_site/docs/import-hdfs-auditLog.html b/_site/docs/import-hdfs-auditLog.html new file mode 100644 index 0000000..5a18853 --- /dev/null +++ b/_site/docs/import-hdfs-auditLog.html @@ -0,0 +1,370 @@ +<!DOCTYPE html> +<head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + + <title>Eagle - How to stream hdfs log data into Kafka</title> + <meta name="description" content="Eagle - Analyze Big Data Platforms for Security and Performance"> + + <meta name="keywords" content="Eagle, Hadoop, Security, Real Time"> + <meta name="author" content="eBay Inc."> + + <meta charset="utf-8"> + <meta name="viewport" content="initial-scale=1"> + + <link rel="stylesheet" href="/css/animate.css"> + <link rel="stylesheet" href="/css/bootstrap.min.css"> + + <link rel="stylesheet" href="/css/font-awesome.min.css"> + + <link rel="stylesheet" href="/css/misc.css"> + <link rel="stylesheet" href="/css/style.css"> + <link rel="stylesheet" href="/css/styles.css"> + <link rel="stylesheet" href="/css/main.css"> + <link rel="alternate" type="application/rss+xml" title="Eagle" href="http://goeagle.io/feed.xml"; /> + <link rel="shortcut icon" href="/images/favicon.png"> + + <!-- Baidu Analytics Tracking--> + <script> + var _hmt = _hmt || []; + (function() { + var hm = document.createElement("script"); + hm.src = "//hm.baidu.com/hm.js?fedc55df2ea52777a679192e8f849ece"; + var s = document.getElementsByTagName("script")[0]; + s.parentNode.insertBefore(hm, s); + })(); + </script> + + <!-- Google Analytics Tracking --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + ga('create', 'UA-68929805-1', 'auto'); + ga('send', 'pageview'); + </script> +</head> +<body> +<!-- header start --> +<div id="home_page"> + <div class="topbar"> + <div class="container"> + <div class="row" > + <nav class="navbar navbar-default"> + <div class="container-fluid"> + <!-- Brand and toggle get grouped for better mobile display --> + <div class="navbar-header"> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> <span class="sr-only">Toggle navigation</span> <span class="icon-bar"></span> <span class="icon-bar"></span> <span class="icon-bar"></span> </button> + <a class="navbar-brand" href="/"><img src="/images/logo2.png" height="44px" style="margin-top:-7px"></a> </div> + + <!-- Collect the nav links, forms, and other content for toggling --> + <!-- <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> + <ul class="nav navbar-nav navbar-right" id="top-menu"> + <li><a class="menu" href="/#home_page">HOME</a></li> + <li><a class="menu" href="/docs/">DOCS</a></li> + <li><a class="menu" href="/#about_page">ABOUT</a></li> + <li><a class="menu" href="/#diagram_page">ARCHITECTURE</a></li> + <li><a class="menu" href="/#modules_page">MODULES</a></li> + <li><a class="menu" href="/#usecase_page">USE CASES</a></li> + <li> + </li> + </ul> --> + </div> + </div> + <!-- /.container-fluid --> + </nav> + </div> + </div> + </div> + <div class="headerimage"> + <div class="flexslider"> + <ul class="slides"> + <li><img src="/images/slider/4.jpg" alt="Slide 1"></li> + </ul> + </div> + </div> + <div class="particles" style="height:40%"> </div><!---particles--> + <div class="slider-caption" style="top:80px;"> + <div class="homewrapper"> + <div class="hometitle"> + <a href="/"> + <img src="/images/feather.png" height="80px"> + </a> + </div> + <div class="hometext"> + <h2>Analyze Big Data Platforms For Security and Performance</h2> + <div class="social-buttons"> + <a href="https://github.com/apache/eagle";><i class="fa fa-github"></i></a> + <a href="http://twitter.com/TheApacheEagle";><i class="fa fa-twitter"></i></a> + <a href="https://www.facebook.com/TheApacheEagle/";><i class="fa fa-facebook"></i></a> + <a href="#"><i class="fa fa-weixin"></i></a> + <!-- <a href="https://www.weibo.com/ApacheEagle/";><i class="fa fa-weibo"></i></a> --> + </div> + </div> + </div> + </div> +</div> +<!-- header end --> +<div class="container-fluid page-content"> + <div class="row"> + <div class="col-md-10 col-md-offset-1"> + <!-- sidebar --> + <div class="col-xs-6 col-sm-3" id="sidebar" role="navigation"> + <ul class="nav" id="adminnav"> + + <li class="heading">Getting Started</li> + + <li class="sidenavli "><a href="/docs/index.html" data-permalink="/docs/import-hdfs-auditLog.html" id="">Introduction</a></li> + + <li class="sidenavli "><a href="/docs/usecases.html" data-permalink="/docs/import-hdfs-auditLog.html" id="">Use Cases</a></li> + + <li class="sidenavli "><a href="/docs/terminology.html" data-permalink="/docs/import-hdfs-auditLog.html" id="">Terminology</a></li> + + <li class="sidenavli "><a href="/docs/ecosystem.html" data-permalink="/docs/import-hdfs-auditLog.html" id="">Ecosystem</a></li> + + <li class="sidenavli "><a href="/docs/community.html" data-permalink="/docs/import-hdfs-auditLog.html" id="">Community</a></li> + + <li class="sidenavli "><a href="/docs/FAQ.html" data-permalink="/docs/import-hdfs-auditLog.html" id="">FAQ</a></li> + + <li class="divider"></li> + + <li class="heading">Documentations</li> + + <li class="sidenavli "><a href="/docs/latest/" data-permalink="/docs/import-hdfs-auditLog.html" id="">Latest version (v0.5.0)</a></li> + + <li class="divider"></li> + + <li class="heading">Download</li> + + <li class="sidenavli "><a href="/docs/download-latest.html" data-permalink="/docs/import-hdfs-auditLog.html" id="">Latest version (v0.5.0)</a></li> + + <li class="sidenavli "><a href="/docs/download.html" data-permalink="/docs/import-hdfs-auditLog.html" id="">Archived</a></li> + + <li class="divider"></li> + + <li class="heading">Supplement</li> + + <li class="sidenavli "><a href="/docs/security.html" data-permalink="/docs/import-hdfs-auditLog.html" id="">Security</a></li> + + <li class="divider"></li> + + <li class="sidenavli"> + <a href="mailto:d...@eagle.apache.org"; target="_blank">Need Help?</a> + </li> + </ul> + </div> + <div class="col-xs-6 col-sm-9 page-main-content" style="margin-left: -15px" id="loadcontent"> + <h1 class="page-header" style="margin-top: 0px">How to stream hdfs log data into Kafka</h1> + <p>As Apache Eagle consumes the data via Kafka<sup id="fnref:KAFKA"><a href="#fn:KAFKA" class="footnote">1</a></sup> topics in some topologies, such as HDFS audit log. To enable the full function of monitoring, a user needs to stream its data into a Kafka topic.</p> + +<p>There are two ways to do that. The first one is <strong>Logstash</strong>, which naturally supports Kafka as the output plugin; the second one is to +install a <strong>namenode log4j Kafka appender</strong>.</p> + +<h3 id="logstash-kafka">Logstash-kafka</h3> + +<ul> + <li> + <p><strong>Step 1</strong>: Create a Kafka topic as the streaming input.</p> + + <p>Here is an sample Kafka command to create topic âsandbox_hdfs_audit_logâ</p> + + <pre><code>cd <kafka-home> +bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic sandbox_hdfs_audit_log +</code></pre> + </li> + <li> + <p><strong>Step 2</strong>: Install Logstash-kafka plugin</p> + + <ul> + <li> + <p>For Logstash 1.5.x, logstash-kafka has been intergrated into <a href="https://github.com/logstash-plugins/logstash-input-kafka";>logstash-input-kafka</a> and <a href="https://github.com/logstash-plugins/logstash-output-kafka";>logstash-output-kafka</a>, + and released with the 1.5 version of Logstash. So you can directly use it.</p> + </li> + <li> + <p>For Logstash 1.4.x, a user should install <a href="https://github.com/joekiller/logstash-kafka";>logstash-kafka</a> firstly. Notice that this version <strong>does not support partition_key_format</strong>.</p> + </li> + </ul> + </li> + <li> + <p><strong>Step 3</strong>: Create a Logstash configuration file under ${LOGSTASH_HOME}/conf. Here is a sample.</p> + + <pre><code> input { + file { + type => "hdp-nn-audit" + path => "/path/to/audit.log" + start_position => end + sincedb_path => "/var/log/logstash/" + } + } + + filter{ + if [type] == "hdp-nn-audit" { + grok { + match => ["message", "ugi=(?<user>([\w\d\-]+))@|ugi=(?<user>([\w\d\-]+))/[\w\d\-.]+@|ugi=(?<user>([\w\d.\-_]+))[\s(]+"] + } + } + } + + output { + if [type] == "hdp-nn-audit" { + kafka { + codec => plain { + format => "%{message}" + } + broker_list => "localhost:9092" + topic_id => "sandbox_hdfs_audit_log" + request_required_acks => 0 + request_timeout_ms => 10000 + producer_type => "async" + message_send_max_retries => 3 + retry_backoff_ms => 100 + queue_buffering_max_ms => 5000 + queue_enqueue_timeout_ms => 5000 + batch_num_messages => 200 + send_buffer_bytes => 102400 + client_id => "hdp-nn-audit" + partition_key_format => "%{user}" + } + # stdout { codec => rubydebug } + } + } +</code></pre> + </li> + <li> + <p><strong>Step 4</strong>: Start Logstash</p> + + <pre><code>bin/logstash -f conf/sample.conf +</code></pre> + </li> + <li> + <p><strong>Step 5</strong>: Check whether logs are flowing into the kafka topic specified by <code>topic_id</code></p> + </li> +</ul> + +<h3 id="log4j-kafka-appender">Log4j Kafka Appender</h3> + +<blockquote> + <p>Notice that if you use Ambari<sup id="fnref:AMBARI"><a href="#fn:AMBARI" class="footnote">2</a></sup>, such as in sandbox, you <strong>must</strong> follow below steps via Ambari UI. In addition, restarting namenode is required.</p> +</blockquote> + +<ul> + <li> + <p><strong>Step 1</strong>: Create a Kafka topic. Here is a example Kafka command for creating topic âsandbox_hdfs_audit_logâ</p> + + <pre><code>cd <kafka-home> +bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic sandbox_hdfs_audit_log +</code></pre> + </li> + <li> + <p><strong>Step 2</strong>: Configure $HADOOP_CONF_DIR/log4j.properties, and add a log4j appender âKAFKA_HDFS_AUDITâ to hdfs audit logging</p> + + <pre><code>log4j.appender.KAFKA_HDFS_AUDIT=org.apache.eagle.log4j.kafka.KafkaLog4jAppender +log4j.appender.KAFKA_HDFS_AUDIT.Topic=sandbox_hdfs_audit_log +log4j.appender.KAFKA_HDFS_AUDIT.BrokerList=sandbox.hortonworks.com:6667 +log4j.appender.KAFKA_HDFS_AUDIT.KeyClass=org.apache.eagle.log4j.kafka.hadoop.AuditLogKeyer +log4j.appender.KAFKA_HDFS_AUDIT.Layout=org.apache.log4j.PatternLayout +log4j.appender.KAFKA_HDFS_AUDIT.Layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n +log4j.appender.KAFKA_HDFS_AUDIT.ProducerType=async +#log4j.appender.KAFKA_HDFS_AUDIT.BatchSize=1 +#log4j.appender.KAFKA_HDFS_AUDIT.QueueSize=1 +</code></pre> + + <p><img src="/images/docs/hdfs-log4j-conf.png" alt="HDFS LOG4J Configuration" title="hdfslog4jconf" /></p> + </li> + <li> + <p><strong>Step 3</strong>: Edit $HADOOP_CONF_DIR/hadoop-env.sh, and add the reference to KAFKA_HDFS_AUDIT to HADOOP_NAMENODE_OPTS.</p> + + <pre><code>-Dhdfs.audit.logger=INFO,DRFAAUDIT,KAFKA_HDFS_AUDIT +</code></pre> + + <p><img src="/images/docs/hdfs-env-conf.png" alt="HDFS Environment Configuration" title="hdfsenvconf" /></p> + </li> + <li> + <p><strong>Step 4</strong>: Edit $HADOOP_CONF_DIR/hadoop-env.sh, and append the following command to it.</p> + + <pre><code>export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:/path/to/eagle/lib/log4jkafka/lib/* +</code></pre> + + <p><img src="/images/docs/hdfs-env-conf2.png" alt="HDFS Environment Configuration" title="hdfsenvconf2" /></p> + </li> + <li> + <p><strong>Step 5</strong>: save the changes and restart the namenode.</p> + </li> + <li> + <p><strong>Step 6</strong>: Check whether logs are flowing into Topic sandbox_hdfs_audit_log</p> + + <pre><code>$ /usr/hdp/current/kafka-broker/bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic sandbox_hdfs_audit_log +</code></pre> + </li> +</ul> + +<hr /> + +<h4 id="footnotes"><em>Footnotes</em></h4> + +<div class="footnotes"> + <ol> + <li id="fn:KAFKA"> + <p><em>All mentions of âkafkaâ on this page represent Apache Kafka.</em> <a href="#fnref:KAFKA" class="reversefootnote">↩</a></p> + </li> + <li id="fn:AMBARI"> + <p><em>all mentions of âambariâ on this page represent Apache Ambari.</em> <a href="#fnref:AMBARI" class="reversefootnote">↩</a></p> + </li> + </ol> +</div> + + </div><!--end of loadcontent--> + </div> + <!--end of centered content--> + </div> +</div> +<!--end of container--> + +<!-- footer start --> +<div class="footerwrapper"> + <div class="container"> + <div class="row"> + <div class="col-md-12"><div style="margin-left:auto; margin-right:auto; text-align:center;font-size: 12px"> +<div> +</div> +<div> +<a href="http://www.apache.org";> +<img id="asf-logo" alt="Apache Software Foundation" src="/images/apache-logo-small.gif"> +</a> +</div> +<div> +Copyright © 2015 <a href="http://www.apache.org";>The Apache Software Foundation</a>, Licensed under the <a href="http://www.apache.org/licenses/LICENSE-2.0";>Apache License, Version 2.0</a>. +</div> +<div> +Apache Eagle, Eagle, Apache Hadoop, Hadoop, Apache HBase, HBase, Apache Hive, Hive, Apache Ambari, Ambari, Apache Spark, Spark, Apache Kafka, Kafka, Apache Storm, Storm, Apache Maven, Maven, Apache Tomcat, Tomcat, Apache Derby, Derby, Apache Cassandra, Cassandra, Apache ZooKeeper, ZooKeeper, Apache, the Apache feather logo, and the Apache project logo are trademarks of The Apache Software Foundation. +</div> + </div></div> + </div> + </div> +</div> +<!-- footer end --> + +<!-- JavaScripts --> +<script src="/js/jquery-1.11.1.min.js"></script> +<script src="/js/jquery.singlePageNav.js"></script> +<script src="/js/jquery.flexslider.js"></script> +<script src="/js/modernizr.min.js"></script> +<script src="/js/svg.js"></script> +<script> + /************** FlexSlider *********************/ + $('.flexslider').flexslider({ + animation: "fade", + directionNav: false + }); +</script> + +<script> + /************** FlexSlider *********************/ + $('.flexslider').flexslider({ + animation: "fade", + directionNav: false + }); +</script> +</body> +</html> http://git-wip-us.apache.org/repos/asf/eagle/blob/d6fed677/_site/docs/index.html ---------------------------------------------------------------------- diff --git a/_site/docs/index.html b/_site/docs/index.html new file mode 100644 index 0000000..7e607b4 --- /dev/null +++ b/_site/docs/index.html @@ -0,0 +1,271 @@ +<!DOCTYPE html> +<head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + + <title>Eagle - Introduction</title> + <meta name="description" content="Eagle - Analyze Big Data Platforms for Security and Performance"> + + <meta name="keywords" content="Eagle, Hadoop, Security, Real Time"> + <meta name="author" content="eBay Inc."> + + <meta charset="utf-8"> + <meta name="viewport" content="initial-scale=1"> + + <link rel="stylesheet" href="/css/animate.css"> + <link rel="stylesheet" href="/css/bootstrap.min.css"> + + <link rel="stylesheet" href="/css/font-awesome.min.css"> + + <link rel="stylesheet" href="/css/misc.css"> + <link rel="stylesheet" href="/css/style.css"> + <link rel="stylesheet" href="/css/styles.css"> + <link rel="stylesheet" href="/css/main.css"> + <link rel="alternate" type="application/rss+xml" title="Eagle" href="http://goeagle.io/feed.xml"; /> + <link rel="shortcut icon" href="/images/favicon.png"> + + <!-- Baidu Analytics Tracking--> + <script> + var _hmt = _hmt || []; + (function() { + var hm = document.createElement("script"); + hm.src = "//hm.baidu.com/hm.js?fedc55df2ea52777a679192e8f849ece"; + var s = document.getElementsByTagName("script")[0]; + s.parentNode.insertBefore(hm, s); + })(); + </script> + + <!-- Google Analytics Tracking --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + ga('create', 'UA-68929805-1', 'auto'); + ga('send', 'pageview'); + </script> +</head> +<body> +<!-- header start --> +<div id="home_page"> + <div class="topbar"> + <div class="container"> + <div class="row" > + <nav class="navbar navbar-default"> + <div class="container-fluid"> + <!-- Brand and toggle get grouped for better mobile display --> + <div class="navbar-header"> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> <span class="sr-only">Toggle navigation</span> <span class="icon-bar"></span> <span class="icon-bar"></span> <span class="icon-bar"></span> </button> + <a class="navbar-brand" href="/"><img src="/images/logo2.png" height="44px" style="margin-top:-7px"></a> </div> + + <!-- Collect the nav links, forms, and other content for toggling --> + <!-- <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> + <ul class="nav navbar-nav navbar-right" id="top-menu"> + <li><a class="menu" href="/#home_page">HOME</a></li> + <li><a class="menu" href="/docs/">DOCS</a></li> + <li><a class="menu" href="/#about_page">ABOUT</a></li> + <li><a class="menu" href="/#diagram_page">ARCHITECTURE</a></li> + <li><a class="menu" href="/#modules_page">MODULES</a></li> + <li><a class="menu" href="/#usecase_page">USE CASES</a></li> + <li> + </li> + </ul> --> + </div> + </div> + <!-- /.container-fluid --> + </nav> + </div> + </div> + </div> + <div class="headerimage"> + <div class="flexslider"> + <ul class="slides"> + <li><img src="/images/slider/4.jpg" alt="Slide 1"></li> + </ul> + </div> + </div> + <div class="particles" style="height:40%"> </div><!---particles--> + <div class="slider-caption" style="top:80px;"> + <div class="homewrapper"> + <div class="hometitle"> + <a href="/"> + <img src="/images/feather.png" height="80px"> + </a> + </div> + <div class="hometext"> + <h2>Analyze Big Data Platforms For Security and Performance</h2> + <div class="social-buttons"> + <a href="https://github.com/apache/eagle";><i class="fa fa-github"></i></a> + <a href="http://twitter.com/TheApacheEagle";><i class="fa fa-twitter"></i></a> + <a href="https://www.facebook.com/TheApacheEagle/";><i class="fa fa-facebook"></i></a> + <a href="#"><i class="fa fa-weixin"></i></a> + <!-- <a href="https://www.weibo.com/ApacheEagle/";><i class="fa fa-weibo"></i></a> --> + </div> + </div> + </div> + </div> +</div> +<!-- header end --> +<div class="container-fluid page-content"> + <div class="row"> + <div class="col-md-10 col-md-offset-1"> + <!-- sidebar --> + <div class="col-xs-6 col-sm-3" id="sidebar" role="navigation"> + <ul class="nav" id="adminnav"> + + <li class="heading">Getting Started</li> + + <li class="sidenavli current"><a href="/docs/index.html" data-permalink="/docs/index.html" id="">Introduction</a></li> + + <li class="sidenavli "><a href="/docs/usecases.html" data-permalink="/docs/index.html" id="">Use Cases</a></li> + + <li class="sidenavli "><a href="/docs/terminology.html" data-permalink="/docs/index.html" id="">Terminology</a></li> + + <li class="sidenavli "><a href="/docs/ecosystem.html" data-permalink="/docs/index.html" id="">Ecosystem</a></li> + + <li class="sidenavli "><a href="/docs/community.html" data-permalink="/docs/index.html" id="">Community</a></li> + + <li class="sidenavli "><a href="/docs/FAQ.html" data-permalink="/docs/index.html" id="">FAQ</a></li> + + <li class="divider"></li> + + <li class="heading">Documentations</li> + + <li class="sidenavli "><a href="/docs/latest/" data-permalink="/docs/index.html" id="">Latest version (v0.5.0)</a></li> + + <li class="divider"></li> + + <li class="heading">Download</li> + + <li class="sidenavli "><a href="/docs/download-latest.html" data-permalink="/docs/index.html" id="">Latest version (v0.5.0)</a></li> + + <li class="sidenavli "><a href="/docs/download.html" data-permalink="/docs/index.html" id="">Archived</a></li> + + <li class="divider"></li> + + <li class="heading">Supplement</li> + + <li class="sidenavli "><a href="/docs/security.html" data-permalink="/docs/index.html" id="">Security</a></li> + + <li class="divider"></li> + + <li class="sidenavli"> + <a href="mailto:d...@eagle.apache.org"; target="_blank">Need Help?</a> + </li> + </ul> + </div> + <div class="col-xs-6 col-sm-9 page-main-content" style="margin-left: -15px" id="loadcontent"> + <h1 class="page-header" style="margin-top: 0px">Introduction</h1> + <h3 id="welcome-to-apache-eagle">Welcome to Apache Eagle</h3> + +<blockquote> + <p>Apache Eagle (called Eagle in the following) is an open source analytics solution for identifying security and performance issues instantly on big data platforms e.g. Hadoop<sup id="fnref:HADOOP"><a href="#fn:HADOOP" class="footnote">1</a></sup>, Spark<sup id="fnref:SPARK"><a href="#fn:SPARK" class="footnote">2</a></sup>, NoSQL etc. It analyzes data activities, yarn applications, jmx metrics, and daemon logs etc., provides state-of-the-art alert engine to identify security breach, performance issues and shows insights.</p> +</blockquote> + +<h3 id="basic-analytics-and-monitoring-flow">Basic Analytics and Monitoring Flow</h3> + +<p>Normally 3 basic steps are involved in Eagle platform: Data Integration, Alerting Engine and Insights.</p> + +<ul> + <li> + <p>Data Integration normally is streaming process application, for example Apache Storm topology or Spark streaming application. During data integration, raw data can be transformed, normalized, analyzed etc. The output of data integration will be streamed into Alerting Engine and Insights.</p> + </li> + <li> + <p>Alerting Engine is highly scalable rule execution engine, which is embedded into Data Integration application as of Eagle 0.4.0. With alerting engine, user can create/update comprehensive policies on the fly.</p> + </li> + <li> + <p>Insights is for correlating alerts with raw data for user to explore root causes.</p> + </li> +</ul> + +<p>Depending on data source, data integrations may use pull or push methods to stream data into Eagle platform, for example HDFS audit log can be pushed by logstash, but Hive<sup id="fnref:HIVE"><a href="#fn:HIVE" class="footnote">3</a></sup> query log can be pulled by Hive Integration application.</p> + +<h3 id="key-qualities">Key Qualities</h3> + +<ul> + <li> + <p><strong>Real Time</strong>: We understand the importance of timing and acting fast in case of a security breach or performance issue. So we designed Eagle to make sure the alerts are generated immediately even for very high volume streaming data. This is achieved by running Eagle alert engine on top of streaming framework, e.g. Apache Storm with high performance CEP engine e.g. WSO2 Siddhi etc.</p> + </li> + <li> + <p><strong>Scalability</strong>: At eBay we operate one of worldâs largest big data platforms Hadoop, Spark etc. The volume of operational logs and user activities generated by big data platforms is very huge. Eagle alert engine is well designed to evaluate polices against data in motion at cloud scale.This is achieved by dynamically partitioning data and policies based on policy semantics.</p> + </li> + <li> + <p><strong>Metadata Driven</strong>: Eagle understands what policies are evaluated for what data source in what physical boxes. This thanks to Eagle metadata design and master/slave distributed computing architecture. Eagle alert engineâs coordinator listens to metadata change, calculates snapshot for mapping policy to resource and then dynamically deploys snapshot onto storm spouts and bolts.</p> + </li> + <li> + <p><strong>Extensibility</strong>: Eagle is designed with extensibility in mind. You can integrate many different data sources into Eagle platform with a few clicks.</p> + </li> +</ul> + +<hr /> + +<h4 id="footnotes"><em>Footnotes</em></h4> + +<div class="footnotes"> + <ol> + <li id="fn:HADOOP"> + <p><em>All mentions of âhadoopâ on this page represent Apache Hadoop.</em> <a href="#fnref:HADOOP" class="reversefootnote">↩</a></p> + </li> + <li id="fn:SPARK"> + <p><em>All mentions of âsparkâ on this page represent Apache Spark.</em> <a href="#fnref:SPARK" class="reversefootnote">↩</a></p> + </li> + <li id="fn:HIVE"> + <p><em>All mentions of âhiveâ on this page represent Apache HIVE.</em> <a href="#fnref:HIVE" class="reversefootnote">↩</a></p> + </li> + </ol> +</div> + + </div><!--end of loadcontent--> + </div> + <!--end of centered content--> + </div> +</div> +<!--end of container--> + +<!-- footer start --> +<div class="footerwrapper"> + <div class="container"> + <div class="row"> + <div class="col-md-12"><div style="margin-left:auto; margin-right:auto; text-align:center;font-size: 12px"> +<div> +</div> +<div> +<a href="http://www.apache.org";> +<img id="asf-logo" alt="Apache Software Foundation" src="/images/apache-logo-small.gif"> +</a> +</div> +<div> +Copyright © 2015 <a href="http://www.apache.org";>The Apache Software Foundation</a>, Licensed under the <a href="http://www.apache.org/licenses/LICENSE-2.0";>Apache License, Version 2.0</a>. +</div> +<div> +Apache Eagle, Eagle, Apache Hadoop, Hadoop, Apache HBase, HBase, Apache Hive, Hive, Apache Ambari, Ambari, Apache Spark, Spark, Apache Kafka, Kafka, Apache Storm, Storm, Apache Maven, Maven, Apache Tomcat, Tomcat, Apache Derby, Derby, Apache Cassandra, Cassandra, Apache ZooKeeper, ZooKeeper, Apache, the Apache feather logo, and the Apache project logo are trademarks of The Apache Software Foundation. +</div> + </div></div> + </div> + </div> +</div> +<!-- footer end --> + +<!-- JavaScripts --> +<script src="/js/jquery-1.11.1.min.js"></script> +<script src="/js/jquery.singlePageNav.js"></script> +<script src="/js/jquery.flexslider.js"></script> +<script src="/js/modernizr.min.js"></script> +<script src="/js/svg.js"></script> +<script> + /************** FlexSlider *********************/ + $('.flexslider').flexslider({ + animation: "fade", + directionNav: false + }); +</script> + +<script> + /************** FlexSlider *********************/ + $('.flexslider').flexslider({ + animation: "fade", + directionNav: false + }); +</script> +</body> +</html> http://git-wip-us.apache.org/repos/asf/eagle/blob/d6fed677/_site/docs/installation.html ---------------------------------------------------------------------- diff --git a/_site/docs/installation.html b/_site/docs/installation.html new file mode 100644 index 0000000..adf5c4c --- /dev/null +++ b/_site/docs/installation.html @@ -0,0 +1,324 @@ +<!DOCTYPE html> +<head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + + <title>Eagle - Install Apache Eagle</title> + <meta name="description" content="Eagle - Analyze Big Data Platforms for Security and Performance"> + + <meta name="keywords" content="Eagle, Hadoop, Security, Real Time"> + <meta name="author" content="eBay Inc."> + + <meta charset="utf-8"> + <meta name="viewport" content="initial-scale=1"> + + <link rel="stylesheet" href="/css/animate.css"> + <link rel="stylesheet" href="/css/bootstrap.min.css"> + + <link rel="stylesheet" href="/css/font-awesome.min.css"> + + <link rel="stylesheet" href="/css/misc.css"> + <link rel="stylesheet" href="/css/style.css"> + <link rel="stylesheet" href="/css/styles.css"> + <link rel="stylesheet" href="/css/main.css"> + <link rel="alternate" type="application/rss+xml" title="Eagle" href="http://goeagle.io/feed.xml"; /> + <link rel="shortcut icon" href="/images/favicon.png"> + + <!-- Baidu Analytics Tracking--> + <script> + var _hmt = _hmt || []; + (function() { + var hm = document.createElement("script"); + hm.src = "//hm.baidu.com/hm.js?fedc55df2ea52777a679192e8f849ece"; + var s = document.getElementsByTagName("script")[0]; + s.parentNode.insertBefore(hm, s); + })(); + </script> + + <!-- Google Analytics Tracking --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + ga('create', 'UA-68929805-1', 'auto'); + ga('send', 'pageview'); + </script> +</head> +<body> +<!-- header start --> +<div id="home_page"> + <div class="topbar"> + <div class="container"> + <div class="row" > + <nav class="navbar navbar-default"> + <div class="container-fluid"> + <!-- Brand and toggle get grouped for better mobile display --> + <div class="navbar-header"> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> <span class="sr-only">Toggle navigation</span> <span class="icon-bar"></span> <span class="icon-bar"></span> <span class="icon-bar"></span> </button> + <a class="navbar-brand" href="/"><img src="/images/logo2.png" height="44px" style="margin-top:-7px"></a> </div> + + <!-- Collect the nav links, forms, and other content for toggling --> + <!-- <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> + <ul class="nav navbar-nav navbar-right" id="top-menu"> + <li><a class="menu" href="/#home_page">HOME</a></li> + <li><a class="menu" href="/docs/">DOCS</a></li> + <li><a class="menu" href="/#about_page">ABOUT</a></li> + <li><a class="menu" href="/#diagram_page">ARCHITECTURE</a></li> + <li><a class="menu" href="/#modules_page">MODULES</a></li> + <li><a class="menu" href="/#usecase_page">USE CASES</a></li> + <li> + </li> + </ul> --> + </div> + </div> + <!-- /.container-fluid --> + </nav> + </div> + </div> + </div> + <div class="headerimage"> + <div class="flexslider"> + <ul class="slides"> + <li><img src="/images/slider/4.jpg" alt="Slide 1"></li> + </ul> + </div> + </div> + <div class="particles" style="height:40%"> </div><!---particles--> + <div class="slider-caption" style="top:80px;"> + <div class="homewrapper"> + <div class="hometitle"> + <a href="/"> + <img src="/images/feather.png" height="80px"> + </a> + </div> + <div class="hometext"> + <h2>Analyze Big Data Platforms For Security and Performance</h2> + <div class="social-buttons"> + <a href="https://github.com/apache/eagle";><i class="fa fa-github"></i></a> + <a href="http://twitter.com/TheApacheEagle";><i class="fa fa-twitter"></i></a> + <a href="https://www.facebook.com/TheApacheEagle/";><i class="fa fa-facebook"></i></a> + <a href="#"><i class="fa fa-weixin"></i></a> + <!-- <a href="https://www.weibo.com/ApacheEagle/";><i class="fa fa-weibo"></i></a> --> + </div> + </div> + </div> + </div> +</div> +<!-- header end --> +<div class="container-fluid page-content"> + <div class="row"> + <div class="col-md-10 col-md-offset-1"> + <!-- sidebar --> + <div class="col-xs-6 col-sm-3" id="sidebar" role="navigation"> + <ul class="nav" id="adminnav"> + + <li class="heading">Getting Started</li> + + <li class="sidenavli "><a href="/docs/index.html" data-permalink="/docs/installation.html" id="">Introduction</a></li> + + <li class="sidenavli "><a href="/docs/usecases.html" data-permalink="/docs/installation.html" id="">Use Cases</a></li> + + <li class="sidenavli "><a href="/docs/terminology.html" data-permalink="/docs/installation.html" id="">Terminology</a></li> + + <li class="sidenavli "><a href="/docs/ecosystem.html" data-permalink="/docs/installation.html" id="">Ecosystem</a></li> + + <li class="sidenavli "><a href="/docs/community.html" data-permalink="/docs/installation.html" id="">Community</a></li> + + <li class="sidenavli "><a href="/docs/FAQ.html" data-permalink="/docs/installation.html" id="">FAQ</a></li> + + <li class="divider"></li> + + <li class="heading">Documentations</li> + + <li class="sidenavli "><a href="/docs/latest/" data-permalink="/docs/installation.html" id="">Latest version (v0.5.0)</a></li> + + <li class="divider"></li> + + <li class="heading">Download</li> + + <li class="sidenavli "><a href="/docs/download-latest.html" data-permalink="/docs/installation.html" id="">Latest version (v0.5.0)</a></li> + + <li class="sidenavli "><a href="/docs/download.html" data-permalink="/docs/installation.html" id="">Archived</a></li> + + <li class="divider"></li> + + <li class="heading">Supplement</li> + + <li class="sidenavli "><a href="/docs/security.html" data-permalink="/docs/installation.html" id="">Security</a></li> + + <li class="divider"></li> + + <li class="sidenavli"> + <a href="mailto:d...@eagle.apache.org"; target="_blank">Need Help?</a> + </li> + </ul> + </div> + <div class="col-xs-6 col-sm-9 page-main-content" style="margin-left: -15px" id="loadcontent"> + <h1 class="page-header" style="margin-top: 0px">Install Apache Eagle</h1> + <h3 id="install-apache-eagle-called-eagle-in-the-following-to-sandbox">Install Apache Eagle (called Eagle in the following) to Sandbox</h3> + +<h4 id="pre-requisites">Pre-requisites</h4> + +<blockquote> + <p>To insall eagle on a sandbox you need to have orcale virtual box and HDP sandbox image.</p> +</blockquote> + +<ol> + <li><a href="https://www.virtualbox.org/wiki/Downloads";>Oracle VirtualBox</a>.</li> + <li><a href="http://hortonworks.com/products/hortonworks-sandbox/#install";>Hortonworks Sandbox</a> v 2.2.4 or later.</li> +</ol> + +<h4 id="register-hdp-sandbox">Register HDP sandbox</h4> + +<ol> + <li><a href="http://127.0.0.1:8888/";>Register</a> Hortonworks sandbox.</li> + <li><a href="http://127.0.0.1:8000/";>Enable Ambari</a>. Click on Enable Button.</li> + <li><a href="http://127.0.0.1:8080";>Login</a> as admin/admin.</li> +</ol> + +<h4 id="install-eagle">Install Eagle</h4> + +<ul> + <li> + <p><strong>Step 1</strong>: Clone stable version from <a href="https://github.com/apache/eagle/releases/tag/v0.4.0-incubating";>eagle github</a> +> Build project mvn clean install -DskipTests=true</p> + </li> + <li> + <p><strong>Step 2</strong>: Download eagle-bin-0.1.0.tar.gz package from successful build into your HDP sandbox.</p> + + <ul> + <li> + <p>Option 1: <code>scp -P 2222 eagle/eagle-assembly/target/eagle-0.1.0-bin.tar.gz root@127.0.0.1:/usr/hdp/current/</code></p> + </li> + <li> + <p>Option 2: Create shared directory between host and Sandbox, and restart Sandbox. Then you can find the shared directory under /media in Sandbox.</p> + </li> + </ul> + </li> + <li> + <p><strong>Step 3</strong>: Extract eagle tarball package</p> + + <pre><code>$ cd /usr/hdp/current +$ tar -zxvf eagle-0.1.0-bin.tar.gz +$ mv eagle-0.1.0 eagle +</code></pre> + </li> + <li> + <p><strong>Step 4</strong>: Add root as a HBase<sup id="fnref:HBASE"><a href="#fn:HBASE" class="footnote">1</a></sup> superuser via <a href="http://127.0.0.1:8080/#/main/services/HBASE/configs";>Ambari</a> (Optional, a user can operate HBase by sudo su hbase, as an alternative).</p> + </li> + <li> + <p><strong>Step 5</strong>: Install Eagle Ambari<sup id="fnref:AMBARI"><a href="#fn:AMBARI" class="footnote">2</a></sup> service +> + /usr/hdp/current/eagle/bin/eagle-ambari.sh install.</p> + </li> + <li> + <p><strong>Step 6</strong>: Restart <a href="http://127.0.0.1:8000/";>Ambari</a> click on disable and enable Ambari back.</p> + </li> + <li> + <p><strong>Step 7</strong>: Start HBase & Storm<sup id="fnref:STORM"><a href="#fn:STORM" class="footnote">3</a></sup> & Kafka<sup id="fnref:KAFKA"><a href="#fn:KAFKA" class="footnote">4</a></sup> +From Ambari UI, restart any suggested components(âRestart button on topâ) & Start Storm (Start âNimbusâ ,âSupervisorâ & âStorm UI Serverâ), Kafka (Start âKafka Brokerâ) , HBase (Start âRegionServerâ and â HBase Masterâ) +> +<img src="/images/docs/Services.png" alt="Restart Services" title="Services" /></p> + </li> + <li> + <p><strong>Step 8</strong>: Add Eagle Service To Ambari. (Click For Video)</p> + + <ul> + <li> + <p>Click on âAdd Serviceâ under Actions button on Ambari Main page</p> + + <p><img src="/images/docs/add-service.png" alt="AddService" title="AddService" /></p> + </li> + <li> + <p>Select âEagleâ in list of services and proceed to install all eagle services. +EagleServiceSuccess</p> + + <p><img src="/images/docs/eagle-service-success.png" alt="Eagle Services" title="Eagle Services" /></p> + </li> + </ul> + </li> + <li> + <p><strong>Step 9</strong>: Add Policies and meta data required by running below script.</p> + + <pre><code>$ /usr/hdp/current/eagle/examples/sample-sensitivity-resource-create.sh +$ /usr/hdp/current/eagle/examples/sample-policy-create.sh +</code></pre> + </li> +</ul> + +<hr /> + +<h4 id="footnotes"><em>Footnotes</em></h4> + +<div class="footnotes"> + <ol> + <li id="fn:HBASE"> + <p><em>All mentions of âhbaseâ on this page represent Apache HBase.</em> <a href="#fnref:HBASE" class="reversefootnote">↩</a></p> + </li> + <li id="fn:AMBARI"> + <p><em>All mentions of âambariâ on this page represent Apache Ambari.</em> <a href="#fnref:AMBARI" class="reversefootnote">↩</a></p> + </li> + <li id="fn:STORM"> + <p><em>All mentions of âstormâ on this page represent Apache Storm.</em> <a href="#fnref:STORM" class="reversefootnote">↩</a></p> + </li> + <li id="fn:KAFKA"> + <p><em>All mentions of âkafkaâ on this page represent Apache Kafka.</em> <a href="#fnref:KAFKA" class="reversefootnote">↩</a></p> + </li> + </ol> +</div> + + </div><!--end of loadcontent--> + </div> + <!--end of centered content--> + </div> +</div> +<!--end of container--> + +<!-- footer start --> +<div class="footerwrapper"> + <div class="container"> + <div class="row"> + <div class="col-md-12"><div style="margin-left:auto; margin-right:auto; text-align:center;font-size: 12px"> +<div> +</div> +<div> +<a href="http://www.apache.org";> +<img id="asf-logo" alt="Apache Software Foundation" src="/images/apache-logo-small.gif"> +</a> +</div> +<div> +Copyright © 2015 <a href="http://www.apache.org";>The Apache Software Foundation</a>, Licensed under the <a href="http://www.apache.org/licenses/LICENSE-2.0";>Apache License, Version 2.0</a>. +</div> +<div> +Apache Eagle, Eagle, Apache Hadoop, Hadoop, Apache HBase, HBase, Apache Hive, Hive, Apache Ambari, Ambari, Apache Spark, Spark, Apache Kafka, Kafka, Apache Storm, Storm, Apache Maven, Maven, Apache Tomcat, Tomcat, Apache Derby, Derby, Apache Cassandra, Cassandra, Apache ZooKeeper, ZooKeeper, Apache, the Apache feather logo, and the Apache project logo are trademarks of The Apache Software Foundation. +</div> + </div></div> + </div> + </div> +</div> +<!-- footer end --> + +<!-- JavaScripts --> +<script src="/js/jquery-1.11.1.min.js"></script> +<script src="/js/jquery.singlePageNav.js"></script> +<script src="/js/jquery.flexslider.js"></script> +<script src="/js/modernizr.min.js"></script> +<script src="/js/svg.js"></script> +<script> + /************** FlexSlider *********************/ + $('.flexslider').flexslider({ + animation: "fade", + directionNav: false + }); +</script> + +<script> + /************** FlexSlider *********************/ + $('.flexslider').flexslider({ + animation: "fade", + directionNav: false + }); +</script> +</body> +</html> http://git-wip-us.apache.org/repos/asf/eagle/blob/d6fed677/_site/docs/jmx-metric-monitoring.html ---------------------------------------------------------------------- diff --git a/_site/docs/jmx-metric-monitoring.html b/_site/docs/jmx-metric-monitoring.html new file mode 100644 index 0000000..bd6b2d8 --- /dev/null +++ b/_site/docs/jmx-metric-monitoring.html @@ -0,0 +1,287 @@ +<!DOCTYPE html> +<head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + + <title>Eagle - JMX Metric Monitoring</title> + <meta name="description" content="Eagle - Analyze Big Data Platforms for Security and Performance"> + + <meta name="keywords" content="Eagle, Hadoop, Security, Real Time"> + <meta name="author" content="eBay Inc."> + + <meta charset="utf-8"> + <meta name="viewport" content="initial-scale=1"> + + <link rel="stylesheet" href="/css/animate.css"> + <link rel="stylesheet" href="/css/bootstrap.min.css"> + + <link rel="stylesheet" href="/css/font-awesome.min.css"> + + <link rel="stylesheet" href="/css/misc.css"> + <link rel="stylesheet" href="/css/style.css"> + <link rel="stylesheet" href="/css/styles.css"> + <link rel="stylesheet" href="/css/main.css"> + <link rel="alternate" type="application/rss+xml" title="Eagle" href="http://goeagle.io/feed.xml"; /> + <link rel="shortcut icon" href="/images/favicon.png"> + + <!-- Baidu Analytics Tracking--> + <script> + var _hmt = _hmt || []; + (function() { + var hm = document.createElement("script"); + hm.src = "//hm.baidu.com/hm.js?fedc55df2ea52777a679192e8f849ece"; + var s = document.getElementsByTagName("script")[0]; + s.parentNode.insertBefore(hm, s); + })(); + </script> + + <!-- Google Analytics Tracking --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + ga('create', 'UA-68929805-1', 'auto'); + ga('send', 'pageview'); + </script> +</head> +<body> +<!-- header start --> +<div id="home_page"> + <div class="topbar"> + <div class="container"> + <div class="row" > + <nav class="navbar navbar-default"> + <div class="container-fluid"> + <!-- Brand and toggle get grouped for better mobile display --> + <div class="navbar-header"> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> <span class="sr-only">Toggle navigation</span> <span class="icon-bar"></span> <span class="icon-bar"></span> <span class="icon-bar"></span> </button> + <a class="navbar-brand" href="/"><img src="/images/logo2.png" height="44px" style="margin-top:-7px"></a> </div> + + <!-- Collect the nav links, forms, and other content for toggling --> + <!-- <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> + <ul class="nav navbar-nav navbar-right" id="top-menu"> + <li><a class="menu" href="/#home_page">HOME</a></li> + <li><a class="menu" href="/docs/">DOCS</a></li> + <li><a class="menu" href="/#about_page">ABOUT</a></li> + <li><a class="menu" href="/#diagram_page">ARCHITECTURE</a></li> + <li><a class="menu" href="/#modules_page">MODULES</a></li> + <li><a class="menu" href="/#usecase_page">USE CASES</a></li> + <li> + </li> + </ul> --> + </div> + </div> + <!-- /.container-fluid --> + </nav> + </div> + </div> + </div> + <div class="headerimage"> + <div class="flexslider"> + <ul class="slides"> + <li><img src="/images/slider/4.jpg" alt="Slide 1"></li> + </ul> + </div> + </div> + <div class="particles" style="height:40%"> </div><!---particles--> + <div class="slider-caption" style="top:80px;"> + <div class="homewrapper"> + <div class="hometitle"> + <a href="/"> + <img src="/images/feather.png" height="80px"> + </a> + </div> + <div class="hometext"> + <h2>Analyze Big Data Platforms For Security and Performance</h2> + <div class="social-buttons"> + <a href="https://github.com/apache/eagle";><i class="fa fa-github"></i></a> + <a href="http://twitter.com/TheApacheEagle";><i class="fa fa-twitter"></i></a> + <a href="https://www.facebook.com/TheApacheEagle/";><i class="fa fa-facebook"></i></a> + <a href="#"><i class="fa fa-weixin"></i></a> + <!-- <a href="https://www.weibo.com/ApacheEagle/";><i class="fa fa-weibo"></i></a> --> + </div> + </div> + </div> + </div> +</div> +<!-- header end --> +<div class="container-fluid page-content"> + <div class="row"> + <div class="col-md-10 col-md-offset-1"> + <!-- sidebar --> + <div class="col-xs-6 col-sm-3" id="sidebar" role="navigation"> + <ul class="nav" id="adminnav"> + + <li class="heading">Getting Started</li> + + <li class="sidenavli "><a href="/docs/index.html" data-permalink="/docs/jmx-metric-monitoring.html" id="">Introduction</a></li> + + <li class="sidenavli "><a href="/docs/usecases.html" data-permalink="/docs/jmx-metric-monitoring.html" id="">Use Cases</a></li> + + <li class="sidenavli "><a href="/docs/terminology.html" data-permalink="/docs/jmx-metric-monitoring.html" id="">Terminology</a></li> + + <li class="sidenavli "><a href="/docs/ecosystem.html" data-permalink="/docs/jmx-metric-monitoring.html" id="">Ecosystem</a></li> + + <li class="sidenavli "><a href="/docs/community.html" data-permalink="/docs/jmx-metric-monitoring.html" id="">Community</a></li> + + <li class="sidenavli "><a href="/docs/FAQ.html" data-permalink="/docs/jmx-metric-monitoring.html" id="">FAQ</a></li> + + <li class="divider"></li> + + <li class="heading">Documentations</li> + + <li class="sidenavli "><a href="/docs/latest/" data-permalink="/docs/jmx-metric-monitoring.html" id="">Latest version (v0.5.0)</a></li> + + <li class="divider"></li> + + <li class="heading">Download</li> + + <li class="sidenavli "><a href="/docs/download-latest.html" data-permalink="/docs/jmx-metric-monitoring.html" id="">Latest version (v0.5.0)</a></li> + + <li class="sidenavli "><a href="/docs/download.html" data-permalink="/docs/jmx-metric-monitoring.html" id="">Archived</a></li> + + <li class="divider"></li> + + <li class="heading">Supplement</li> + + <li class="sidenavli "><a href="/docs/security.html" data-permalink="/docs/jmx-metric-monitoring.html" id="">Security</a></li> + + <li class="divider"></li> + + <li class="sidenavli"> + <a href="mailto:d...@eagle.apache.org"; target="_blank">Need Help?</a> + </li> + </ul> + </div> + <div class="col-xs-6 col-sm-9 page-main-content" style="margin-left: -15px" id="loadcontent"> + <h1 class="page-header" style="margin-top: 0px">JMX Metric Monitoring</h1> + <p>JMX metric for Apache Hadoop namenode url <a href="http://127.0.0.1:50070/jmx";>http://127.0.0.1:50070/jmx</a> can be monitored using Apache Eagle (called Eagle in the following). Follow below steps to enable this feature in Eagle.</p> + +<ol> + <li>Install Python script (To populate JMX metric values to Kafka<sup id="fnref:KAFKA"><a href="#fn:KAFKA" class="footnote">1</a></sup> topic periodically).</li> + <li>Deploy âhadoopjmxâ Storm<sup id="fnref:STORM"><a href="#fn:STORM" class="footnote">2</a></sup> topology.</li> + <li>Create new site and policy in UI.</li> + <li>Validate policy alert.</li> +</ol> + +<p><br /></p> + +<h3 id="prerequisite"><strong>Prerequisite</strong></h3> +<ul> + <li>Complete the setup from <a href="/docs/deployment-env.html">Setup Environment</a></li> +</ul> + +<p><br /></p> + +<h3 id="setup"><strong>Setup</strong></h3> +<p>From Hortonworks sandbox just run below setup script to Install Pyton JMX script, Create Kafka topic, update Apache Hbase tables and deploy âhadoopjmxâ Storm topology.</p> + +<pre><code>$ /usr/hdp/current/eagle/examples/hadoop-metric-sandbox-starter.sh +$ /usr/hdp/current/eagle/examples/hadoop-metric-policy-create.sh +</code></pre> + +<p><br /></p> + +<h3 id="application-setup-in-ui"><strong>Application Setup in UI</strong></h3> +<ol> + <li>Login to Eagle UI <a href="http://localhost:9099/eagle-service/";>http://localhost:9099/eagle-service/</a> using username and password as âadminâ and âsecretâ</li> + <li>Click on âAdminâ from top right and click âManagementâ button.</li> + <li>On Admin management page add âNew Siteâ name âhadoopJmxMetricDataSourceâ, by clicking on âNew Siteâ link. +<img src="/images/docs/new-jmx-site.png" alt="add superuser" /></li> + <li>Save the changes.</li> + <li>On eagle home page you should see new tab called âMETRICâ, beside âDAMâ.</li> + <li>Click on âJmxMetricMonitorâ under âMETRICâ.</li> +</ol> + +<p>You should see policy with name âsafeModePolicyâ.</p> + +<p><br /></p> + +<h3 id="demo"><strong>Demo</strong></h3> + +<ul> + <li> + <p>First make sure that Kafka topic ânn_jmx_metric_sandboxâ is populated with JMX metric data periodically.(To make sure that python script is running)</p> + + <pre><code> $ /usr/hdp/2.2.4.2-2/kafka/bin/kafka-console-consumer.sh --zookeeper sandbox.hortonworks.com:2181 --topic nn_jmx_metric_sandbox +</code></pre> + </li> + <li> + <p>Genrate Alert by producing alert triggering message into Kafka topic.</p> + + <pre><code> $ /usr/hdp/2.2.4.2-2/kafka/bin/kafka-console-producer.sh --broker-list sandbox.hortonworks.com:6667 --topic nn_jmx_metric_sandbox + $ {"host": "localhost", "timestamp": 1457033916718, "metric": "hadoop.namenode.fsnamesystemstate.fsstate", "component": "namenode", "site": "sandbox", "value": 1.0} +</code></pre> + </li> +</ul> + +<hr /> + +<h4 id="footnotes"><em>Footnotes</em></h4> + +<div class="footnotes"> + <ol> + <li id="fn:KAFKA"> + <p><em>All mentions of âkafkaâ on this page represent Apache Kafka.</em> <a href="#fnref:KAFKA" class="reversefootnote">↩</a></p> + </li> + <li id="fn:STORM"> + <p><em>All mentions of âstormâ on this page represent Apache Storm.</em> <a href="#fnref:STORM" class="reversefootnote">↩</a></p> + </li> + </ol> +</div> + + </div><!--end of loadcontent--> + </div> + <!--end of centered content--> + </div> +</div> +<!--end of container--> + +<!-- footer start --> +<div class="footerwrapper"> + <div class="container"> + <div class="row"> + <div class="col-md-12"><div style="margin-left:auto; margin-right:auto; text-align:center;font-size: 12px"> +<div> +</div> +<div> +<a href="http://www.apache.org";> +<img id="asf-logo" alt="Apache Software Foundation" src="/images/apache-logo-small.gif"> +</a> +</div> +<div> +Copyright © 2015 <a href="http://www.apache.org";>The Apache Software Foundation</a>, Licensed under the <a href="http://www.apache.org/licenses/LICENSE-2.0";>Apache License, Version 2.0</a>. +</div> +<div> +Apache Eagle, Eagle, Apache Hadoop, Hadoop, Apache HBase, HBase, Apache Hive, Hive, Apache Ambari, Ambari, Apache Spark, Spark, Apache Kafka, Kafka, Apache Storm, Storm, Apache Maven, Maven, Apache Tomcat, Tomcat, Apache Derby, Derby, Apache Cassandra, Cassandra, Apache ZooKeeper, ZooKeeper, Apache, the Apache feather logo, and the Apache project logo are trademarks of The Apache Software Foundation. +</div> + </div></div> + </div> + </div> +</div> +<!-- footer end --> + +<!-- JavaScripts --> +<script src="/js/jquery-1.11.1.min.js"></script> +<script src="/js/jquery.singlePageNav.js"></script> +<script src="/js/jquery.flexslider.js"></script> +<script src="/js/modernizr.min.js"></script> +<script src="/js/svg.js"></script> +<script> + /************** FlexSlider *********************/ + $('.flexslider').flexslider({ + animation: "fade", + directionNav: false + }); +</script> + +<script> + /************** FlexSlider *********************/ + $('.flexslider').flexslider({ + animation: "fade", + directionNav: false + }); +</script> +</body> +</html>
